[Doc] cookbook: Add Jais 3 deployment recipes (#42766)
Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
co-authored by
Xiaoyu Zhang
parent
86f4ede4fb
commit
8f853b0041
@@ -0,0 +1,237 @@
|
||||
---
|
||||
title: Jais 3
|
||||
description: "Deploy the IFM Jais 3 family with SGLang — BF16 recipes for six dense and Mixture-of-Experts checkpoints on NVIDIA H100 GPUs."
|
||||
tag: NEW
|
||||
---
|
||||
|
||||
## Deployment
|
||||
|
||||
<a id="install" />
|
||||
|
||||
<Accordion title="Install SGLang">
|
||||
|
||||
Jais 3 is supported from SGLang 0.5.20:
|
||||
|
||||
```bash Command
|
||||
pip install --upgrade pip
|
||||
pip install "sglang>=0.5.20"
|
||||
```
|
||||
|
||||
For other install methods, see [Install](/docs/get-started/install).
|
||||
|
||||
</Accordion>
|
||||
|
||||
Choose a model size to generate its H100 launch command. Five checkpoints run on a single node; 375B-A23B needs two. The recipes serve each repository's `main` branch.
|
||||
|
||||
import { Deployment } from "/src/snippets/_deployment.jsx";
|
||||
import { config } from "/src/snippets/configs/IFM/jais-3.jsx";
|
||||
import { benchmarks } from "/src/snippets/configs/IFM/jais-3-benchmarks.jsx";
|
||||
|
||||
<Deployment config={config} benchmarks={benchmarks} />
|
||||
|
||||
Each accuracy value in the benchmark cards is the arithmetic mean of two full evaluation runs. Speed values come from a single run.
|
||||
|
||||
## Playground
|
||||
|
||||
The base recipes enable the `k2_horizon` reasoning parser. Use the Playground to configure the `k2_horizon` tool-call parser, TP and MoE expert-parallel overrides, NGRAM speculative decoding, PD disaggregation, and HiCache. A separate validation pass functionally exercised the tool-call parser on all six checkpoints; every other Playground override remains unverified.
|
||||
|
||||
import { Playground } from "/src/snippets/_playground.jsx";
|
||||
|
||||
<Playground config={config} />
|
||||
|
||||
## 1. Model introduction
|
||||
|
||||
**Jais 3** is an IFM model family spanning six checkpoints from **0.9B** to **375B** parameters. The family includes dense models, Mixture-of-Experts models, and the MoVA-36B-A4B checkpoint's routed Mixture-of-Value-Attention path. This page provides native SGLang serving recipes for the family.
|
||||
|
||||
<table style={{width: "100%", borderCollapse: "collapse", tableLayout: "fixed"}}>
|
||||
<colgroup>
|
||||
<col style={{width: "22%"}} />
|
||||
<col style={{width: "25%"}} />
|
||||
<col style={{width: "23%"}} />
|
||||
<col style={{width: "30%"}} />
|
||||
</colgroup>
|
||||
<thead>
|
||||
<tr style={{borderBottom: "2px solid #d55816"}}>
|
||||
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700}}>Checkpoint</th>
|
||||
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700}}>Architecture</th>
|
||||
<th style={{textAlign: "right", padding: "10px 12px", fontWeight: 700}}>Context length</th>
|
||||
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700}}>H100 recipe</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td style={{padding: "9px 12px"}}><strong><a href="https://huggingface.co/IFM/Jais-3-0.9B">0.9B</a></strong></td>
|
||||
<td style={{padding: "9px 12px", background: "rgba(255,255,255,0.02)"}}>Dense</td>
|
||||
<td style={{padding: "9px 12px", textAlign: "right"}}>131,072</td>
|
||||
<td style={{padding: "9px 12px", background: "rgba(255,255,255,0.02)"}}>BF16 · TP1</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style={{padding: "9px 12px"}}><strong><a href="https://huggingface.co/IFM/Jais-3-3.7B">3.7B</a></strong></td>
|
||||
<td style={{padding: "9px 12px", background: "rgba(255,255,255,0.02)"}}>Dense</td>
|
||||
<td style={{padding: "9px 12px", textAlign: "right"}}>524,288</td>
|
||||
<td style={{padding: "9px 12px", background: "rgba(255,255,255,0.02)"}}>BF16 · TP1</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style={{padding: "9px 12px"}}><strong><a href="https://huggingface.co/IFM/Jais-3-7B">7B</a></strong></td>
|
||||
<td style={{padding: "9px 12px", background: "rgba(255,255,255,0.02)"}}>Dense</td>
|
||||
<td style={{padding: "9px 12px", textAlign: "right"}}>524,288</td>
|
||||
<td style={{padding: "9px 12px", background: "rgba(255,255,255,0.02)"}}>BF16 · TP1</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style={{padding: "9px 12px"}}><strong><a href="https://huggingface.co/IFM/Jais-3-32B">32B</a></strong></td>
|
||||
<td style={{padding: "9px 12px", background: "rgba(255,255,255,0.02)"}}>Dense</td>
|
||||
<td style={{padding: "9px 12px", textAlign: "right"}}>524,288</td>
|
||||
<td style={{padding: "9px 12px", background: "rgba(255,255,255,0.02)"}}>BF16 · TP2</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style={{padding: "9px 12px"}}><strong><a href="https://huggingface.co/IFM/Jais-3-MoVA-36B-A4B">MoVA-36B-A4B</a></strong></td>
|
||||
<td style={{padding: "9px 12px", background: "rgba(255,255,255,0.02)"}}>MoE + MoVA</td>
|
||||
<td style={{padding: "9px 12px", textAlign: "right"}}>524,288</td>
|
||||
<td style={{padding: "9px 12px", background: "rgba(255,255,255,0.02)"}}>BF16 · TP2 · EP2</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style={{padding: "9px 12px"}}><strong><a href="https://huggingface.co/IFM/Jais-3-375B-A23B">375B-A23B</a></strong></td>
|
||||
<td style={{padding: "9px 12px", background: "rgba(255,255,255,0.02)"}}>MoE</td>
|
||||
<td style={{padding: "9px 12px", textAlign: "right"}}>524,288</td>
|
||||
<td style={{padding: "9px 12px", background: "rgba(255,255,255,0.02)"}}>BF16 · TP16 · EP16 · 2 nodes</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
**Resources:** [Jais 3 collection](https://huggingface.co/collections/IFM/jais-3) · [Jais 3 375B-A23B model card](https://huggingface.co/IFM/Jais-3-375B-A23B).
|
||||
|
||||
## 2. Configuration tips
|
||||
|
||||
- **Hardware and precision:** The current matrix covers NVIDIA H100 GPUs and BF16 checkpoints only.
|
||||
- **Parallelism:** Use TP1 for 0.9B, 3.7B, and 7B; TP2 for 32B; TP2 with expert parallelism (`--ep 2`) for MoVA-36B-A4B; and TP16 with `--ep 16` across two eight-GPU nodes for 375B-A23B. Every other recipe stays within one node.
|
||||
- **Attention:** The verified base recipes select FlashAttention-3 with `--attention-backend fa3`.
|
||||
- **Parsers:** The base recipes enable reasoning with `--reasoning-parser k2_horizon`. Enable `--tool-call-parser k2_horizon` from the **Parsers** card in the [Playground](#playground) when you need tool calling.
|
||||
- **MoVA-36B-A4B router provenance:** The MoVA-36B-A4B checkpoint's source xLLM router used two GEMM partitions. `--json-model-override-args '{"xllm_source_router_gemm_partitions":2}'` preserves its BF16 partial-GEMM rounding and FP32 reduction order; SGLang does not infer this source topology from runtime TP.
|
||||
- **375B-A23B memory:** The BF16 weights take about 760 GB. One node of eight H100 80 GB GPUs holds 640 GB, so the checkpoint does not fit on a single node; the recipe uses two nodes (16 GPUs, 1,280 GB). Run the generated command on both nodes, with node rank 0 on the head node and 1 on the other.
|
||||
- **375B-A23B loading:** The 375B-A23B recipe disables multithreaded weight loading with `--model-loader-extra-config '{"enable_multithread_load":false}'`.
|
||||
- **Playground features:** TP and EP settings other than the base recipes', NGRAM, PD disaggregation, and HiCache are optional, unverified command overrides. The tool-call parser was functionally exercised separately.
|
||||
- **Deliberate omissions:** Context parallelism is unsupported or unvalidated for xLLM. DP-attention and alternate MoE backend selectors are not exposed until they are validated.
|
||||
- **Validation:** All six recipes completed two full GSM8K evaluation runs on H100; see [Benchmark results](#3-benchmark-results).
|
||||
|
||||
## 3. Benchmark results
|
||||
|
||||
Speed was measured with native `sglang.benchmark.serving` using fixed random token IDs: 8,192 input tokens, 1,024 output tokens, 64 warmup requests, a cache flush, and seed `20260901`. The latency run uses 32 prompts at concurrency 1. The throughput run uses 256 prompts at concurrency 64 for the single-GPU checkpoints, 32 for the two-GPU checkpoints, and 8 for the two-node 375B-A23B. Each speed value comes from a single run, with the load generator on a separate client node; TTFT and TPOT are medians. Throughput per GPU is `(total input tokens + total output tokens) / duration / TP`. Checkpoints and workloads not yet measured show a dash in the benchmark cards.
|
||||
|
||||
GSM8K used the full 1,319-example test split with `sgl-eval`, 32 client threads, a 32,768-token output limit, temperature 0, top-p 0.95, seed 0, and high reasoning effort. Each server ran on H100 from the repository's `main` branch, with no pinned revision. The evaluation is zero-shot: each question is sent as a single user message with the `math` prompt, which asks for the final answer inside `\boxed{}`, and no worked examples. Scoring is rule-based accuracy with no LLM judge: `sgl-eval` extracts the last `\boxed{}` from the final answer and checks it against the reference by symbolic equivalence, so `\$57{,}500` matches `57500`. A response with no boxed answer counts as incorrect, so these scores can be lower than LLM-as-a-judge results on the same outputs.
|
||||
|
||||
Each checkpoint was evaluated twice, and the table shows both runs and their mean. The 0.9B and 3.7B runs were issued one after the other; from 7B onward both were issued together with `--n-repeats 2` against one running server. All requests completed without errors. Responses that reached the output limit stay in the score and count as incorrect; the Truncated column gives the share of such responses in run 1 and run 2.
|
||||
|
||||
| Checkpoint | Run 1 | Run 2 | Mean | Truncated |
|
||||
| --- | ---: | ---: | ---: | ---: |
|
||||
| 0.9B | 83.62% | 83.78% | 83.70% | 0.68% / 0.61% |
|
||||
| 3.7B | 96.36% | 96.13% | 96.25% | 0.00% / 0.15% |
|
||||
| 7B | 95.91% | 95.83% | 95.87% | 0.00% / 0.08% |
|
||||
| 32B | 96.51% | 96.74% | 96.63% | 0.00% / 0.00% |
|
||||
| MoVA-36B-A4B | 95.98% | 96.44% | 96.21% | 0.00% / 0.00% |
|
||||
| 375B-A23B | 96.44% | 96.21% | 96.32% | 0.00% / 0.00% |
|
||||
|
||||
## 4. Advanced usage
|
||||
|
||||
### 4.1 Reasoning
|
||||
|
||||
The base recipes include `--reasoning-parser k2_horizon`. The parser returns thinking in `reasoning_content` and the final answer in `content`.
|
||||
|
||||
The example output below was captured with the MoVA-36B-A4B checkpoint and both parser flags enabled.
|
||||
|
||||
<Accordion title="Reasoning example (Python)">
|
||||
|
||||
```python Example
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI(base_url="http://localhost:30000/v1", api_key="EMPTY")
|
||||
response = client.chat.completions.create(
|
||||
model="IFM/Jais-3-MoVA-36B-A4B",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
)
|
||||
|
||||
message = response.choices[0].message
|
||||
reasoning = getattr(message, "reasoning_content", None)
|
||||
if reasoning is None:
|
||||
reasoning = (message.model_extra or {}).get("reasoning_content")
|
||||
print("Reasoning:", reasoning)
|
||||
print("Answer:", message.content)
|
||||
```
|
||||
|
||||
</Accordion>
|
||||
|
||||
<Accordion title="Example Output">
|
||||
|
||||
```text Output
|
||||
Reasoning: The user says "Hello". As a helpful AI, we should greet them back, maybe ask how we can assist.
|
||||
|
||||
We need to respond with a friendly greeting. Probably something like "Hello! How can I help you today?"
|
||||
|
||||
That's straightforward.
|
||||
|
||||
We should consider any policies for greeting. No issues.
|
||||
|
||||
Let's proceed.
|
||||
Answer: Hello! How can I help you today?
|
||||
```
|
||||
|
||||
</Accordion>
|
||||
|
||||
### 4.2 Tool calling
|
||||
|
||||
Enable **Tool Call Parser** in the **Parsers** card of the [Playground](#playground) to add `--tool-call-parser k2_horizon`. The parser exposes K2 tool output as OpenAI-compatible `message.tool_calls`.
|
||||
|
||||
The example output below was captured with the MoVA-36B-A4B checkpoint and both parser flags enabled.
|
||||
|
||||
<Accordion title="Tool-calling example (Python)">
|
||||
|
||||
```python Example
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI(base_url="http://localhost:30000/v1", api_key="EMPTY")
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Return weather for a city.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model="IFM/Jais-3-MoVA-36B-A4B",
|
||||
messages=[{"role": "user", "content": "Call get_weather for Paris."}],
|
||||
tools=tools,
|
||||
tool_choice={"type": "function", "function": {"name": "get_weather"}},
|
||||
max_tokens=1024,
|
||||
)
|
||||
|
||||
message = response.choices[0].message
|
||||
reasoning = getattr(message, "reasoning_content", None)
|
||||
if reasoning is None:
|
||||
reasoning = (message.model_extra or {}).get("reasoning_content")
|
||||
print("Reasoning:", reasoning)
|
||||
print("Content:", message.content or "")
|
||||
for tool_call in message.tool_calls or []:
|
||||
print(f"Tool call: {tool_call.function.name}({tool_call.function.arguments})")
|
||||
```
|
||||
|
||||
</Accordion>
|
||||
|
||||
<Accordion title="Example Output">
|
||||
|
||||
```text Output
|
||||
Reasoning: The user wants the weather for Paris, using the get_weather function. We'll call the function.
|
||||
Content: [{"name":"get_weather", "parameters": {"city": "Paris"}}]
|
||||
```
|
||||
|
||||
</Accordion>
|
||||
|
||||
### 4.3 HiCache
|
||||
|
||||
For large checkpoints or repeated long-context prefixes, enable **HiCache** in the [Playground](#playground), then choose a storage backend and write policy. Treat the generated override as unverified until it is exercised on your storage stack.
|
||||
@@ -1229,6 +1229,7 @@
|
||||
{
|
||||
"group": "IFM",
|
||||
"pages": [
|
||||
"cookbook/autoregressive/IFM/Jais-3",
|
||||
"cookbook/autoregressive/IFM/K2-Horizon"
|
||||
]
|
||||
},
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
// Jais 3 GSM8K and speed on NVIDIA H100, served from each repository's main
|
||||
// branch. Speed values come from a single run; null means not measured yet.
|
||||
// A value stays null until both of its full evaluation runs are in; each
|
||||
// filled value is the mean of the two. Per-run scores are in the MDX.
|
||||
|
||||
export const benchmarks = [
|
||||
{
|
||||
match: { hw: "h100", variant: "0.9b", quant: "bf16", strategy: "balanced", nodes: "single" },
|
||||
speed: [
|
||||
{
|
||||
workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1, num_prompts: 32 },
|
||||
ttft_ms: 52.35,
|
||||
tpot_ms: 1.87,
|
||||
tokens_per_sec_per_gpu: 4673.93,
|
||||
},
|
||||
{
|
||||
workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64, num_prompts: 256 },
|
||||
ttft_ms: 1433.89,
|
||||
tpot_ms: 13.92,
|
||||
tokens_per_sec_per_gpu: 37488.99,
|
||||
},
|
||||
],
|
||||
accuracy: { gsm8k_pct: 83.70 },
|
||||
notes: "Speed is from a single run. GSM8K is the mean of two full evaluation runs (83.62% and 83.78%). GSM8K truncation was 0.68% and 0.61%.",
|
||||
},
|
||||
{
|
||||
match: { hw: "h100", variant: "3.7b", quant: "bf16", strategy: "balanced", nodes: "single" },
|
||||
speed: [
|
||||
{
|
||||
workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1, num_prompts: 32 },
|
||||
ttft_ms: 167.43,
|
||||
tpot_ms: 6.11,
|
||||
tokens_per_sec_per_gpu: 1436.60,
|
||||
},
|
||||
{
|
||||
workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64, num_prompts: 256 },
|
||||
ttft_ms: 6754.27,
|
||||
tpot_ms: 28.09,
|
||||
tokens_per_sec_per_gpu: 12192.66,
|
||||
},
|
||||
],
|
||||
accuracy: { gsm8k_pct: 96.25 },
|
||||
notes: "Speed is from a single run. GSM8K is the mean of two full evaluation runs (96.36% and 96.13%). GSM8K truncation was 0.00% and 0.15%.",
|
||||
},
|
||||
{
|
||||
match: { hw: "h100", variant: "7b", quant: "bf16", strategy: "balanced", nodes: "single" },
|
||||
speed: [
|
||||
{
|
||||
workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1, num_prompts: 32 },
|
||||
ttft_ms: 254.89,
|
||||
tpot_ms: 8.54,
|
||||
tokens_per_sec_per_gpu: 1024.76,
|
||||
},
|
||||
{
|
||||
workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64, num_prompts: 256 },
|
||||
ttft_ms: 36797.25,
|
||||
tpot_ms: 30.64,
|
||||
tokens_per_sec_per_gpu: 9692.60,
|
||||
},
|
||||
],
|
||||
accuracy: { gsm8k_pct: 95.87 },
|
||||
notes: "Speed is from a single run. GSM8K is the mean of two full evaluation runs (95.91% and 95.83%). GSM8K truncation was 0.00% and 0.08%.",
|
||||
},
|
||||
{
|
||||
match: { hw: "h100", variant: "32b", quant: "bf16", strategy: "balanced", nodes: "single" },
|
||||
speed: [
|
||||
{
|
||||
workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1, num_prompts: 32 },
|
||||
ttft_ms: 608.07,
|
||||
tpot_ms: 17.93,
|
||||
tokens_per_sec_per_gpu: 243.09,
|
||||
},
|
||||
{
|
||||
workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 32, num_prompts: 256 },
|
||||
ttft_ms: 11442.35,
|
||||
tpot_ms: 38.27,
|
||||
tokens_per_sec_per_gpu: 2711.51,
|
||||
},
|
||||
],
|
||||
accuracy: { gsm8k_pct: 96.63 },
|
||||
notes: "Speed is from a single run. GSM8K is the mean of two full evaluation runs (96.51% and 96.74%). GSM8K truncation was 0.00% in both.",
|
||||
},
|
||||
{
|
||||
match: { hw: "h100", variant: "36b", quant: "bf16", strategy: "balanced", nodes: "single" },
|
||||
speed: [
|
||||
{
|
||||
workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1, num_prompts: 32 },
|
||||
ttft_ms: 252.09,
|
||||
tpot_ms: 12.23,
|
||||
tokens_per_sec_per_gpu: 360.99,
|
||||
},
|
||||
{
|
||||
workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 32, num_prompts: 256 },
|
||||
ttft_ms: 4265.27,
|
||||
tpot_ms: 35.24,
|
||||
tokens_per_sec_per_gpu: 3657.62,
|
||||
},
|
||||
],
|
||||
accuracy: { gsm8k_pct: 96.21 },
|
||||
notes: "Speed is from a single run. GSM8K is the mean of two full evaluation runs (95.98% and 96.44%). GSM8K truncation was 0.00% in both.",
|
||||
},
|
||||
{
|
||||
match: { hw: "h100", variant: "375b", quant: "bf16", strategy: "balanced", nodes: "multi-2" },
|
||||
speed: [
|
||||
{
|
||||
workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1, num_prompts: 32 },
|
||||
ttft_ms: 322.76,
|
||||
tpot_ms: 13.19,
|
||||
tokens_per_sec_per_gpu: 41.67,
|
||||
},
|
||||
{
|
||||
workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 8, num_prompts: 256 },
|
||||
ttft_ms: 1395.83,
|
||||
tpot_ms: 19.29,
|
||||
tokens_per_sec_per_gpu: 216.73,
|
||||
},
|
||||
],
|
||||
accuracy: { gsm8k_pct: 96.32 },
|
||||
notes: "Served on two nodes (16 GPUs). Speed is from a single run. GSM8K is the mean of two full evaluation runs (96.44% and 96.21%). GSM8K truncation was 0.00% in both.",
|
||||
},
|
||||
];
|
||||
@@ -0,0 +1,320 @@
|
||||
// Single `export const config` literal — Mintlify re-evaluates this module at
|
||||
// hydration time, so keep the cookbook data self-contained.
|
||||
//
|
||||
// The base recipes below serve each repository's main branch (no pinned
|
||||
// revision) on NVIDIA H100 GPUs, using SGLang's native K2 Horizon support
|
||||
// (released in 0.5.20). Playground overrides remain separate from them.
|
||||
|
||||
export const config = {
|
||||
modelName: "Jais 3",
|
||||
|
||||
latencyPercentile: "P50",
|
||||
|
||||
supportedHardware: ["h100"],
|
||||
runModes: ["python"],
|
||||
|
||||
variants: [
|
||||
{ id: "0.9b", label: "0.9B", subtitle: "Dense" },
|
||||
{ id: "3.7b", label: "3.7B", subtitle: "Dense" },
|
||||
{ id: "7b", label: "7B", subtitle: "Dense" },
|
||||
{ id: "32b", label: "32B", subtitle: "Dense" },
|
||||
{ id: "36b", label: "MoVA-36B-A4B", subtitle: "MoE + MoVA" },
|
||||
{ id: "375b", label: "375B-A23B", subtitle: "MoE" },
|
||||
],
|
||||
quantizations: [
|
||||
{ id: "bf16", label: "BF16" },
|
||||
],
|
||||
strategies: [
|
||||
{ id: "balanced", label: "Balanced" },
|
||||
],
|
||||
nodesOptions: [
|
||||
{ id: "single", label: "Single Node" },
|
||||
{ id: "multi-2", label: "2 Nodes" },
|
||||
],
|
||||
|
||||
modelNames: {
|
||||
"0.9b|bf16": "IFM/Jais-3-0.9B",
|
||||
"3.7b|bf16": "IFM/Jais-3-3.7B",
|
||||
"7b|bf16": "IFM/Jais-3-7B",
|
||||
"32b|bf16": "IFM/Jais-3-32B",
|
||||
"36b|bf16": "IFM/Jais-3-MoVA-36B-A4B",
|
||||
"375b|bf16": "IFM/Jais-3-375B-A23B",
|
||||
},
|
||||
|
||||
placeholders: {
|
||||
HOST_IP: { target: "command", label: "Bind host", default: "0.0.0.0" },
|
||||
PORT: { target: "command", label: "Bind port", default: "30000" },
|
||||
NODE0_IP: { target: "command", label: "Head node IP", default: "<node0-ip>" },
|
||||
NODE_RANK: { target: "command", label: "This node rank", default: "<node-rank>" },
|
||||
CURL_HOST: { target: "curl", label: "Server host", default: "localhost" },
|
||||
CURL_PORT: { target: "curl", label: "Server port", default: "30000" },
|
||||
},
|
||||
|
||||
curl: `curl http://{{CURL_HOST}}:{{CURL_PORT}}/v1/chat/completions \\
|
||||
-H 'Content-Type: application/json' \\
|
||||
-d '{ "model": "{{MODEL_NAME}}", "messages": [{"role":"user","content":"Hello"}] }'`,
|
||||
|
||||
benchmarkCommands: {
|
||||
speed:
|
||||
`python3 -m sglang.benchmark.serving \\
|
||||
--backend sglang \\
|
||||
--base-url http://{{CURL_HOST}}:{{CURL_PORT}} \\
|
||||
--model {{MODEL_NAME}} --served-model-name {{MODEL_NAME}} \\
|
||||
--tokenizer {{MODEL_NAME}} \\
|
||||
--dataset-name {{DATASET}} --tokenize-prompt \\
|
||||
--random-input-len {{ISL}} --random-output-len {{OSL}} \\
|
||||
--random-range-ratio 1.0 \\
|
||||
--num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}} \\
|
||||
--request-rate inf --seed 20260901 \\
|
||||
--temperature 0.0 --top-p 1.0 \\
|
||||
--warmup-requests 64 --flush-cache \\
|
||||
--output-file benchmark.raw.jsonl --output-details`,
|
||||
accuracy: {
|
||||
gsm8k_pct:
|
||||
`pip install sgl-eval
|
||||
sgl-eval run gsm8k \\
|
||||
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
|
||||
--model {{MODEL_NAME}} \\
|
||||
--num-examples 1319 --num-threads 32 --n-repeats 1 \\
|
||||
--max-tokens 32768 --temperature 0.0 --top-p 0.95 \\
|
||||
--seed 0 --reasoning-effort high --prompt math`,
|
||||
},
|
||||
},
|
||||
|
||||
accuracyLabels: [
|
||||
["gsm8k_pct", "GSM8K", "%"],
|
||||
],
|
||||
|
||||
// The page covers a family, so use the largest checkpoint as the canonical
|
||||
// issue-form model while each recipe still resolves its exact HF repository.
|
||||
github: {
|
||||
cookbookModel: "IFM/Jais-3-375B-A23B",
|
||||
},
|
||||
|
||||
playgroundFeatures: {
|
||||
// CP is unsupported/unvalidated for the xLLM path. DP-attention and
|
||||
// alternate MoE backends are also intentionally omitted until validated.
|
||||
attention: {
|
||||
knobs: [
|
||||
{
|
||||
id: "tp",
|
||||
label: "TP",
|
||||
values: [
|
||||
null,
|
||||
{ value: 1, disable: { variant: ["375b"] },
|
||||
disableReason: "375B-A23B BF16 does not fit on one H100 at TP=1." },
|
||||
{ value: 2, disable: { variant: ["375b"] },
|
||||
disableReason: "375B-A23B BF16 does not fit on two H100 GPUs." },
|
||||
{ value: 4, disable: { variant: ["375b"] },
|
||||
disableReason: "375B-A23B BF16 does not fit on four H100 GPUs." },
|
||||
{ value: 8, disable: { variant: ["375b"] },
|
||||
disableReason: "375B-A23B BF16 does not fit on eight H100 GPUs; it needs TP=16 across two nodes." },
|
||||
{ value: 16, disable: { variant: ["0.9b", "3.7b", "7b", "32b", "36b"] },
|
||||
disableReason: "TP=16 spans two eight-GPU nodes and applies only to 375B-A23B." },
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
|
||||
// Jais 3 MoVA-36B-A4B and 375B-A23B contain sparse MoE feed-forward layers. Keep
|
||||
// expert parallelism disabled on the dense variants.
|
||||
// The MoVA recipe runs with EP=2. The 375B recipe is two-node TP=16 with EP=16.
|
||||
moe: {
|
||||
ep: {
|
||||
label: "EP",
|
||||
values: [
|
||||
null,
|
||||
{ value: 1,
|
||||
disable: { variant: ["0.9b", "3.7b", "7b", "32b"] },
|
||||
disableReason: "Expert parallelism applies only to the sparse MoVA-36B-A4B and 375B-A23B variants." },
|
||||
{ value: 2,
|
||||
disable: [
|
||||
{ when: { variant: ["375b"] },
|
||||
reason: "The two-node 375B-A23B recipe runs at TP=16, where SGLang only accepts EP=16." },
|
||||
{ when: { variant: ["0.9b", "3.7b", "7b", "32b"] },
|
||||
reason: "Expert parallelism applies only to the sparse MoVA-36B-A4B and 375B-A23B variants." },
|
||||
{ when: { effTp: [1] },
|
||||
reason: "EP=2 requires an effective TP degree of at least 2." },
|
||||
] },
|
||||
{ value: 4,
|
||||
disable: [
|
||||
{ when: { variant: ["375b"] },
|
||||
reason: "The two-node 375B-A23B recipe runs at TP=16, where SGLang only accepts EP=16." },
|
||||
{ when: { variant: ["0.9b", "3.7b", "7b", "32b"] },
|
||||
reason: "Expert parallelism applies only to the sparse MoVA-36B-A4B and 375B-A23B variants." },
|
||||
{ when: { effTp: [1, 2] },
|
||||
reason: "EP=4 requires an effective TP degree of at least 4." },
|
||||
] },
|
||||
{ value: 8,
|
||||
disable: [
|
||||
{ when: { variant: ["375b"] },
|
||||
reason: "The two-node 375B-A23B recipe runs at TP=16, where SGLang only accepts EP=16." },
|
||||
{ when: { variant: ["0.9b", "3.7b", "7b", "32b"] },
|
||||
reason: "Expert parallelism applies only to the sparse MoVA-36B-A4B and 375B-A23B variants." },
|
||||
{ when: { effTp: [1, 2, 4] },
|
||||
reason: "EP=8 requires an effective TP degree of at least 8." },
|
||||
{ when: { variant: ["36b"] },
|
||||
reason: "MoVA-36B-A4B has 100 routed experts, which is not divisible by EP=8." },
|
||||
] },
|
||||
{ value: 16,
|
||||
disable: [
|
||||
{ when: { variant: ["0.9b", "3.7b", "7b", "32b", "36b"] },
|
||||
reason: "EP=16 applies only to the two-node 375B-A23B recipe at TP=16." },
|
||||
] },
|
||||
],
|
||||
},
|
||||
},
|
||||
|
||||
parsers: {
|
||||
items: [
|
||||
{ id: "reasoning", label: "Reasoning Parser", flag: "--reasoning-parser k2_horizon" },
|
||||
{ id: "toolCall", label: "Tool Call Parser", flag: "--tool-call-parser k2_horizon" },
|
||||
],
|
||||
},
|
||||
|
||||
speculative: {
|
||||
options: [
|
||||
{ id: "current", label: "Inherited from base" },
|
||||
{ id: "off", label: "Off" },
|
||||
{ id: "ngram", label: "NGRAM",
|
||||
flags: ["--speculative-algorithm NGRAM",
|
||||
"--speculative-num-draft-tokens 16",
|
||||
"--speculative-ngram-max-bfs-breadth 10"] },
|
||||
],
|
||||
},
|
||||
|
||||
pdDisagg: {
|
||||
modes: [
|
||||
{ id: "off", label: "Off" },
|
||||
{ id: "prefill", label: "Prefill role" },
|
||||
{ id: "decode", label: "Decode role" },
|
||||
],
|
||||
transferBackends: [
|
||||
{ id: "mooncake", label: "Mooncake" },
|
||||
{ id: "nixl", label: "NiXL" },
|
||||
],
|
||||
ibDevices: [{ id: "auto", label: "Auto" }, "mlx5_0", "mlx5_7"],
|
||||
router: {
|
||||
port: 8000,
|
||||
command:
|
||||
`python3 -m sglang_router.launch_router \\
|
||||
--pd-disaggregation \\
|
||||
--prefill http://<prefill-host>:{{PREFILL_PORT}} \\
|
||||
--decode http://<decode-host>:{{DECODE_PORT}} \\
|
||||
--host 0.0.0.0 --port {{ROUTER_PORT}} \\
|
||||
--disable-circuit-breaker \\
|
||||
--health-check-interval-secs 999999`,
|
||||
},
|
||||
},
|
||||
|
||||
hicache: {
|
||||
backends: [
|
||||
{ id: null, label: "Auto" },
|
||||
{ id: "file", label: "File" },
|
||||
{ id: "mooncake", label: "Mooncake" },
|
||||
{ id: "hf3fs", label: "HF3FS" },
|
||||
{ id: "nixl", label: "NiXL" },
|
||||
],
|
||||
writePolicies: [
|
||||
{ id: "auto", label: "Auto" },
|
||||
{ id: "write_through", label: "Write-through" },
|
||||
{ id: "write_back", label: "Write-back" },
|
||||
{ id: "write_through_selective", label: "Write-through (selective)" },
|
||||
],
|
||||
},
|
||||
},
|
||||
|
||||
cells: [
|
||||
{
|
||||
match: { hw: "h100", variant: "0.9b", quant: "bf16", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [],
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--tp 1",
|
||||
"--dtype bfloat16",
|
||||
"--attention-backend fa3",
|
||||
"--reasoning-parser k2_horizon",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "h100", variant: "3.7b", quant: "bf16", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [],
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--tp 1",
|
||||
"--dtype bfloat16",
|
||||
"--attention-backend fa3",
|
||||
"--reasoning-parser k2_horizon",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "h100", variant: "7b", quant: "bf16", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [],
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--tp 1",
|
||||
"--dtype bfloat16",
|
||||
"--attention-backend fa3",
|
||||
"--reasoning-parser k2_horizon",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "h100", variant: "32b", quant: "bf16", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [],
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--tp 2",
|
||||
"--dtype bfloat16",
|
||||
"--attention-backend fa3",
|
||||
"--reasoning-parser k2_horizon",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "h100", variant: "36b", quant: "bf16", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [],
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--tp 2",
|
||||
"--ep 2",
|
||||
"--dtype bfloat16",
|
||||
"--attention-backend fa3",
|
||||
"--json-model-override-args '{\"xllm_source_router_gemm_partitions\":2}'",
|
||||
"--reasoning-parser k2_horizon",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
// About 760 GB of BF16 weights: more than one eight-H100 node (640 GB)
|
||||
// holds, so this recipe spans two nodes. The benchmarks were measured at
|
||||
// TP=16 without EP; the --ep 16 variant below has not been launched yet.
|
||||
match: { hw: "h100", variant: "375b", quant: "bf16", strategy: "balanced", nodes: "multi-2" },
|
||||
verified: false,
|
||||
env: [],
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--tp 16",
|
||||
"--ep 16",
|
||||
"--dtype bfloat16",
|
||||
"--attention-backend fa3",
|
||||
"--model-loader-extra-config '{\"enable_multithread_load\":false}'",
|
||||
"--reasoning-parser k2_horizon",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
Reference in New Issue
Block a user