From 8f853b0041c11a641dfc126e09ea6836e725ec53 Mon Sep 17 00:00:00 2001 From: GeorgeSherif <65810674+GeorgeSherif@users.noreply.github.com> Date: Thu, 15 Oct 2026 12:09:44 +0400 Subject: [PATCH] [Doc] cookbook: Add Jais 3 deployment recipes (#42766) Co-authored-by: Xiaoyu Zhang <1182563586@qq.com> --- docs/cookbook/autoregressive/IFM/Jais-3.mdx | 237 +++++++++++++ docs/docs.json | 1 + .../configs/IFM/jais-3-benchmarks.jsx | 121 +++++++ docs/src/snippets/configs/IFM/jais-3.jsx | 320 ++++++++++++++++++ 4 files changed, 679 insertions(+) create mode 100644 docs/cookbook/autoregressive/IFM/Jais-3.mdx create mode 100644 docs/src/snippets/configs/IFM/jais-3-benchmarks.jsx create mode 100644 docs/src/snippets/configs/IFM/jais-3.jsx diff --git a/docs/cookbook/autoregressive/IFM/Jais-3.mdx b/docs/cookbook/autoregressive/IFM/Jais-3.mdx new file mode 100644 index 0000000000..4878d72b51 --- /dev/null +++ b/docs/cookbook/autoregressive/IFM/Jais-3.mdx @@ -0,0 +1,237 @@ +--- +title: Jais 3 +description: "Deploy the IFM Jais 3 family with SGLang — BF16 recipes for six dense and Mixture-of-Experts checkpoints on NVIDIA H100 GPUs." +tag: NEW +--- + +## Deployment + + + + + +Jais 3 is supported from SGLang 0.5.20: + +```bash Command +pip install --upgrade pip +pip install "sglang>=0.5.20" +``` + +For other install methods, see [Install](/docs/get-started/install). + + + +Choose a model size to generate its H100 launch command. Five checkpoints run on a single node; 375B-A23B needs two. The recipes serve each repository's `main` branch. + +import { Deployment } from "/src/snippets/_deployment.jsx"; +import { config } from "/src/snippets/configs/IFM/jais-3.jsx"; +import { benchmarks } from "/src/snippets/configs/IFM/jais-3-benchmarks.jsx"; + + + +Each accuracy value in the benchmark cards is the arithmetic mean of two full evaluation runs. Speed values come from a single run. + +## Playground + +The base recipes enable the `k2_horizon` reasoning parser. Use the Playground to configure the `k2_horizon` tool-call parser, TP and MoE expert-parallel overrides, NGRAM speculative decoding, PD disaggregation, and HiCache. A separate validation pass functionally exercised the tool-call parser on all six checkpoints; every other Playground override remains unverified. + +import { Playground } from "/src/snippets/_playground.jsx"; + + + +## 1. Model introduction + +**Jais 3** is an IFM model family spanning six checkpoints from **0.9B** to **375B** parameters. The family includes dense models, Mixture-of-Experts models, and the MoVA-36B-A4B checkpoint's routed Mixture-of-Value-Attention path. This page provides native SGLang serving recipes for the family. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
CheckpointArchitectureContext lengthH100 recipe
0.9BDense131,072BF16 · TP1
3.7BDense524,288BF16 · TP1
7BDense524,288BF16 · TP1
32BDense524,288BF16 · TP2
MoVA-36B-A4BMoE + MoVA524,288BF16 · TP2 · EP2
375B-A23BMoE524,288BF16 · TP16 · EP16 · 2 nodes
+ +**Resources:** [Jais 3 collection](https://huggingface.co/collections/IFM/jais-3) · [Jais 3 375B-A23B model card](https://huggingface.co/IFM/Jais-3-375B-A23B). + +## 2. Configuration tips + +- **Hardware and precision:** The current matrix covers NVIDIA H100 GPUs and BF16 checkpoints only. +- **Parallelism:** Use TP1 for 0.9B, 3.7B, and 7B; TP2 for 32B; TP2 with expert parallelism (`--ep 2`) for MoVA-36B-A4B; and TP16 with `--ep 16` across two eight-GPU nodes for 375B-A23B. Every other recipe stays within one node. +- **Attention:** The verified base recipes select FlashAttention-3 with `--attention-backend fa3`. +- **Parsers:** The base recipes enable reasoning with `--reasoning-parser k2_horizon`. Enable `--tool-call-parser k2_horizon` from the **Parsers** card in the [Playground](#playground) when you need tool calling. +- **MoVA-36B-A4B router provenance:** The MoVA-36B-A4B checkpoint's source xLLM router used two GEMM partitions. `--json-model-override-args '{"xllm_source_router_gemm_partitions":2}'` preserves its BF16 partial-GEMM rounding and FP32 reduction order; SGLang does not infer this source topology from runtime TP. +- **375B-A23B memory:** The BF16 weights take about 760 GB. One node of eight H100 80 GB GPUs holds 640 GB, so the checkpoint does not fit on a single node; the recipe uses two nodes (16 GPUs, 1,280 GB). Run the generated command on both nodes, with node rank 0 on the head node and 1 on the other. +- **375B-A23B loading:** The 375B-A23B recipe disables multithreaded weight loading with `--model-loader-extra-config '{"enable_multithread_load":false}'`. +- **Playground features:** TP and EP settings other than the base recipes', NGRAM, PD disaggregation, and HiCache are optional, unverified command overrides. The tool-call parser was functionally exercised separately. +- **Deliberate omissions:** Context parallelism is unsupported or unvalidated for xLLM. DP-attention and alternate MoE backend selectors are not exposed until they are validated. +- **Validation:** All six recipes completed two full GSM8K evaluation runs on H100; see [Benchmark results](#3-benchmark-results). + +## 3. Benchmark results + +Speed was measured with native `sglang.benchmark.serving` using fixed random token IDs: 8,192 input tokens, 1,024 output tokens, 64 warmup requests, a cache flush, and seed `20260901`. The latency run uses 32 prompts at concurrency 1. The throughput run uses 256 prompts at concurrency 64 for the single-GPU checkpoints, 32 for the two-GPU checkpoints, and 8 for the two-node 375B-A23B. Each speed value comes from a single run, with the load generator on a separate client node; TTFT and TPOT are medians. Throughput per GPU is `(total input tokens + total output tokens) / duration / TP`. Checkpoints and workloads not yet measured show a dash in the benchmark cards. + +GSM8K used the full 1,319-example test split with `sgl-eval`, 32 client threads, a 32,768-token output limit, temperature 0, top-p 0.95, seed 0, and high reasoning effort. Each server ran on H100 from the repository's `main` branch, with no pinned revision. The evaluation is zero-shot: each question is sent as a single user message with the `math` prompt, which asks for the final answer inside `\boxed{}`, and no worked examples. Scoring is rule-based accuracy with no LLM judge: `sgl-eval` extracts the last `\boxed{}` from the final answer and checks it against the reference by symbolic equivalence, so `\$57{,}500` matches `57500`. A response with no boxed answer counts as incorrect, so these scores can be lower than LLM-as-a-judge results on the same outputs. + +Each checkpoint was evaluated twice, and the table shows both runs and their mean. The 0.9B and 3.7B runs were issued one after the other; from 7B onward both were issued together with `--n-repeats 2` against one running server. All requests completed without errors. Responses that reached the output limit stay in the score and count as incorrect; the Truncated column gives the share of such responses in run 1 and run 2. + +| Checkpoint | Run 1 | Run 2 | Mean | Truncated | +| --- | ---: | ---: | ---: | ---: | +| 0.9B | 83.62% | 83.78% | 83.70% | 0.68% / 0.61% | +| 3.7B | 96.36% | 96.13% | 96.25% | 0.00% / 0.15% | +| 7B | 95.91% | 95.83% | 95.87% | 0.00% / 0.08% | +| 32B | 96.51% | 96.74% | 96.63% | 0.00% / 0.00% | +| MoVA-36B-A4B | 95.98% | 96.44% | 96.21% | 0.00% / 0.00% | +| 375B-A23B | 96.44% | 96.21% | 96.32% | 0.00% / 0.00% | + +## 4. Advanced usage + +### 4.1 Reasoning + +The base recipes include `--reasoning-parser k2_horizon`. The parser returns thinking in `reasoning_content` and the final answer in `content`. + +The example output below was captured with the MoVA-36B-A4B checkpoint and both parser flags enabled. + + + +```python Example +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:30000/v1", api_key="EMPTY") +response = client.chat.completions.create( + model="IFM/Jais-3-MoVA-36B-A4B", + messages=[{"role": "user", "content": "Hello"}], +) + +message = response.choices[0].message +reasoning = getattr(message, "reasoning_content", None) +if reasoning is None: + reasoning = (message.model_extra or {}).get("reasoning_content") +print("Reasoning:", reasoning) +print("Answer:", message.content) +``` + + + + + +```text Output +Reasoning: The user says "Hello". As a helpful AI, we should greet them back, maybe ask how we can assist. + +We need to respond with a friendly greeting. Probably something like "Hello! How can I help you today?" + +That's straightforward. + +We should consider any policies for greeting. No issues. + +Let's proceed. +Answer: Hello! How can I help you today? +``` + + + +### 4.2 Tool calling + +Enable **Tool Call Parser** in the **Parsers** card of the [Playground](#playground) to add `--tool-call-parser k2_horizon`. The parser exposes K2 tool output as OpenAI-compatible `message.tool_calls`. + +The example output below was captured with the MoVA-36B-A4B checkpoint and both parser flags enabled. + + + +```python Example +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:30000/v1", api_key="EMPTY") +tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Return weather for a city.", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } +] + +response = client.chat.completions.create( + model="IFM/Jais-3-MoVA-36B-A4B", + messages=[{"role": "user", "content": "Call get_weather for Paris."}], + tools=tools, + tool_choice={"type": "function", "function": {"name": "get_weather"}}, + max_tokens=1024, +) + +message = response.choices[0].message +reasoning = getattr(message, "reasoning_content", None) +if reasoning is None: + reasoning = (message.model_extra or {}).get("reasoning_content") +print("Reasoning:", reasoning) +print("Content:", message.content or "") +for tool_call in message.tool_calls or []: + print(f"Tool call: {tool_call.function.name}({tool_call.function.arguments})") +``` + + + + + +```text Output +Reasoning: The user wants the weather for Paris, using the get_weather function. We'll call the function. +Content: [{"name":"get_weather", "parameters": {"city": "Paris"}}] +``` + + + +### 4.3 HiCache + +For large checkpoints or repeated long-context prefixes, enable **HiCache** in the [Playground](#playground), then choose a storage backend and write policy. Treat the generated override as unverified until it is exercised on your storage stack. diff --git a/docs/docs.json b/docs/docs.json index c714807936..36b3d86299 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -1229,6 +1229,7 @@ { "group": "IFM", "pages": [ + "cookbook/autoregressive/IFM/Jais-3", "cookbook/autoregressive/IFM/K2-Horizon" ] }, diff --git a/docs/src/snippets/configs/IFM/jais-3-benchmarks.jsx b/docs/src/snippets/configs/IFM/jais-3-benchmarks.jsx new file mode 100644 index 0000000000..130d36a8d7 --- /dev/null +++ b/docs/src/snippets/configs/IFM/jais-3-benchmarks.jsx @@ -0,0 +1,121 @@ +// Jais 3 GSM8K and speed on NVIDIA H100, served from each repository's main +// branch. Speed values come from a single run; null means not measured yet. +// A value stays null until both of its full evaluation runs are in; each +// filled value is the mean of the two. Per-run scores are in the MDX. + +export const benchmarks = [ + { + match: { hw: "h100", variant: "0.9b", quant: "bf16", strategy: "balanced", nodes: "single" }, + speed: [ + { + workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1, num_prompts: 32 }, + ttft_ms: 52.35, + tpot_ms: 1.87, + tokens_per_sec_per_gpu: 4673.93, + }, + { + workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64, num_prompts: 256 }, + ttft_ms: 1433.89, + tpot_ms: 13.92, + tokens_per_sec_per_gpu: 37488.99, + }, + ], + accuracy: { gsm8k_pct: 83.70 }, + notes: "Speed is from a single run. GSM8K is the mean of two full evaluation runs (83.62% and 83.78%). GSM8K truncation was 0.68% and 0.61%.", + }, + { + match: { hw: "h100", variant: "3.7b", quant: "bf16", strategy: "balanced", nodes: "single" }, + speed: [ + { + workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1, num_prompts: 32 }, + ttft_ms: 167.43, + tpot_ms: 6.11, + tokens_per_sec_per_gpu: 1436.60, + }, + { + workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64, num_prompts: 256 }, + ttft_ms: 6754.27, + tpot_ms: 28.09, + tokens_per_sec_per_gpu: 12192.66, + }, + ], + accuracy: { gsm8k_pct: 96.25 }, + notes: "Speed is from a single run. GSM8K is the mean of two full evaluation runs (96.36% and 96.13%). GSM8K truncation was 0.00% and 0.15%.", + }, + { + match: { hw: "h100", variant: "7b", quant: "bf16", strategy: "balanced", nodes: "single" }, + speed: [ + { + workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1, num_prompts: 32 }, + ttft_ms: 254.89, + tpot_ms: 8.54, + tokens_per_sec_per_gpu: 1024.76, + }, + { + workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64, num_prompts: 256 }, + ttft_ms: 36797.25, + tpot_ms: 30.64, + tokens_per_sec_per_gpu: 9692.60, + }, + ], + accuracy: { gsm8k_pct: 95.87 }, + notes: "Speed is from a single run. GSM8K is the mean of two full evaluation runs (95.91% and 95.83%). GSM8K truncation was 0.00% and 0.08%.", + }, + { + match: { hw: "h100", variant: "32b", quant: "bf16", strategy: "balanced", nodes: "single" }, + speed: [ + { + workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1, num_prompts: 32 }, + ttft_ms: 608.07, + tpot_ms: 17.93, + tokens_per_sec_per_gpu: 243.09, + }, + { + workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 32, num_prompts: 256 }, + ttft_ms: 11442.35, + tpot_ms: 38.27, + tokens_per_sec_per_gpu: 2711.51, + }, + ], + accuracy: { gsm8k_pct: 96.63 }, + notes: "Speed is from a single run. GSM8K is the mean of two full evaluation runs (96.51% and 96.74%). GSM8K truncation was 0.00% in both.", + }, + { + match: { hw: "h100", variant: "36b", quant: "bf16", strategy: "balanced", nodes: "single" }, + speed: [ + { + workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1, num_prompts: 32 }, + ttft_ms: 252.09, + tpot_ms: 12.23, + tokens_per_sec_per_gpu: 360.99, + }, + { + workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 32, num_prompts: 256 }, + ttft_ms: 4265.27, + tpot_ms: 35.24, + tokens_per_sec_per_gpu: 3657.62, + }, + ], + accuracy: { gsm8k_pct: 96.21 }, + notes: "Speed is from a single run. GSM8K is the mean of two full evaluation runs (95.98% and 96.44%). GSM8K truncation was 0.00% in both.", + }, + { + match: { hw: "h100", variant: "375b", quant: "bf16", strategy: "balanced", nodes: "multi-2" }, + speed: [ + { + workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1, num_prompts: 32 }, + ttft_ms: 322.76, + tpot_ms: 13.19, + tokens_per_sec_per_gpu: 41.67, + }, + { + workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 8, num_prompts: 256 }, + ttft_ms: 1395.83, + tpot_ms: 19.29, + tokens_per_sec_per_gpu: 216.73, + }, + ], + accuracy: { gsm8k_pct: 96.32 }, + notes: "Served on two nodes (16 GPUs). Speed is from a single run. GSM8K is the mean of two full evaluation runs (96.44% and 96.21%). GSM8K truncation was 0.00% in both.", + }, +]; diff --git a/docs/src/snippets/configs/IFM/jais-3.jsx b/docs/src/snippets/configs/IFM/jais-3.jsx new file mode 100644 index 0000000000..1a8f4dcb98 --- /dev/null +++ b/docs/src/snippets/configs/IFM/jais-3.jsx @@ -0,0 +1,320 @@ +// Single `export const config` literal — Mintlify re-evaluates this module at +// hydration time, so keep the cookbook data self-contained. +// +// The base recipes below serve each repository's main branch (no pinned +// revision) on NVIDIA H100 GPUs, using SGLang's native K2 Horizon support +// (released in 0.5.20). Playground overrides remain separate from them. + +export const config = { + modelName: "Jais 3", + + latencyPercentile: "P50", + + supportedHardware: ["h100"], + runModes: ["python"], + + variants: [ + { id: "0.9b", label: "0.9B", subtitle: "Dense" }, + { id: "3.7b", label: "3.7B", subtitle: "Dense" }, + { id: "7b", label: "7B", subtitle: "Dense" }, + { id: "32b", label: "32B", subtitle: "Dense" }, + { id: "36b", label: "MoVA-36B-A4B", subtitle: "MoE + MoVA" }, + { id: "375b", label: "375B-A23B", subtitle: "MoE" }, + ], + quantizations: [ + { id: "bf16", label: "BF16" }, + ], + strategies: [ + { id: "balanced", label: "Balanced" }, + ], + nodesOptions: [ + { id: "single", label: "Single Node" }, + { id: "multi-2", label: "2 Nodes" }, + ], + + modelNames: { + "0.9b|bf16": "IFM/Jais-3-0.9B", + "3.7b|bf16": "IFM/Jais-3-3.7B", + "7b|bf16": "IFM/Jais-3-7B", + "32b|bf16": "IFM/Jais-3-32B", + "36b|bf16": "IFM/Jais-3-MoVA-36B-A4B", + "375b|bf16": "IFM/Jais-3-375B-A23B", + }, + + placeholders: { + HOST_IP: { target: "command", label: "Bind host", default: "0.0.0.0" }, + PORT: { target: "command", label: "Bind port", default: "30000" }, + NODE0_IP: { target: "command", label: "Head node IP", default: "" }, + NODE_RANK: { target: "command", label: "This node rank", default: "" }, + CURL_HOST: { target: "curl", label: "Server host", default: "localhost" }, + CURL_PORT: { target: "curl", label: "Server port", default: "30000" }, + }, + + curl: `curl http://{{CURL_HOST}}:{{CURL_PORT}}/v1/chat/completions \\ +-H 'Content-Type: application/json' \\ +-d '{ "model": "{{MODEL_NAME}}", "messages": [{"role":"user","content":"Hello"}] }'`, + + benchmarkCommands: { + speed: +`python3 -m sglang.benchmark.serving \\ + --backend sglang \\ + --base-url http://{{CURL_HOST}}:{{CURL_PORT}} \\ + --model {{MODEL_NAME}} --served-model-name {{MODEL_NAME}} \\ + --tokenizer {{MODEL_NAME}} \\ + --dataset-name {{DATASET}} --tokenize-prompt \\ + --random-input-len {{ISL}} --random-output-len {{OSL}} \\ + --random-range-ratio 1.0 \\ + --num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}} \\ + --request-rate inf --seed 20260901 \\ + --temperature 0.0 --top-p 1.0 \\ + --warmup-requests 64 --flush-cache \\ + --output-file benchmark.raw.jsonl --output-details`, + accuracy: { + gsm8k_pct: +`pip install sgl-eval +sgl-eval run gsm8k \\ + --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ + --model {{MODEL_NAME}} \\ + --num-examples 1319 --num-threads 32 --n-repeats 1 \\ + --max-tokens 32768 --temperature 0.0 --top-p 0.95 \\ + --seed 0 --reasoning-effort high --prompt math`, + }, + }, + + accuracyLabels: [ + ["gsm8k_pct", "GSM8K", "%"], + ], + + // The page covers a family, so use the largest checkpoint as the canonical + // issue-form model while each recipe still resolves its exact HF repository. + github: { + cookbookModel: "IFM/Jais-3-375B-A23B", + }, + + playgroundFeatures: { + // CP is unsupported/unvalidated for the xLLM path. DP-attention and + // alternate MoE backends are also intentionally omitted until validated. + attention: { + knobs: [ + { + id: "tp", + label: "TP", + values: [ + null, + { value: 1, disable: { variant: ["375b"] }, + disableReason: "375B-A23B BF16 does not fit on one H100 at TP=1." }, + { value: 2, disable: { variant: ["375b"] }, + disableReason: "375B-A23B BF16 does not fit on two H100 GPUs." }, + { value: 4, disable: { variant: ["375b"] }, + disableReason: "375B-A23B BF16 does not fit on four H100 GPUs." }, + { value: 8, disable: { variant: ["375b"] }, + disableReason: "375B-A23B BF16 does not fit on eight H100 GPUs; it needs TP=16 across two nodes." }, + { value: 16, disable: { variant: ["0.9b", "3.7b", "7b", "32b", "36b"] }, + disableReason: "TP=16 spans two eight-GPU nodes and applies only to 375B-A23B." }, + ], + }, + ], + }, + + // Jais 3 MoVA-36B-A4B and 375B-A23B contain sparse MoE feed-forward layers. Keep + // expert parallelism disabled on the dense variants. + // The MoVA recipe runs with EP=2. The 375B recipe is two-node TP=16 with EP=16. + moe: { + ep: { + label: "EP", + values: [ + null, + { value: 1, + disable: { variant: ["0.9b", "3.7b", "7b", "32b"] }, + disableReason: "Expert parallelism applies only to the sparse MoVA-36B-A4B and 375B-A23B variants." }, + { value: 2, + disable: [ + { when: { variant: ["375b"] }, + reason: "The two-node 375B-A23B recipe runs at TP=16, where SGLang only accepts EP=16." }, + { when: { variant: ["0.9b", "3.7b", "7b", "32b"] }, + reason: "Expert parallelism applies only to the sparse MoVA-36B-A4B and 375B-A23B variants." }, + { when: { effTp: [1] }, + reason: "EP=2 requires an effective TP degree of at least 2." }, + ] }, + { value: 4, + disable: [ + { when: { variant: ["375b"] }, + reason: "The two-node 375B-A23B recipe runs at TP=16, where SGLang only accepts EP=16." }, + { when: { variant: ["0.9b", "3.7b", "7b", "32b"] }, + reason: "Expert parallelism applies only to the sparse MoVA-36B-A4B and 375B-A23B variants." }, + { when: { effTp: [1, 2] }, + reason: "EP=4 requires an effective TP degree of at least 4." }, + ] }, + { value: 8, + disable: [ + { when: { variant: ["375b"] }, + reason: "The two-node 375B-A23B recipe runs at TP=16, where SGLang only accepts EP=16." }, + { when: { variant: ["0.9b", "3.7b", "7b", "32b"] }, + reason: "Expert parallelism applies only to the sparse MoVA-36B-A4B and 375B-A23B variants." }, + { when: { effTp: [1, 2, 4] }, + reason: "EP=8 requires an effective TP degree of at least 8." }, + { when: { variant: ["36b"] }, + reason: "MoVA-36B-A4B has 100 routed experts, which is not divisible by EP=8." }, + ] }, + { value: 16, + disable: [ + { when: { variant: ["0.9b", "3.7b", "7b", "32b", "36b"] }, + reason: "EP=16 applies only to the two-node 375B-A23B recipe at TP=16." }, + ] }, + ], + }, + }, + + parsers: { + items: [ + { id: "reasoning", label: "Reasoning Parser", flag: "--reasoning-parser k2_horizon" }, + { id: "toolCall", label: "Tool Call Parser", flag: "--tool-call-parser k2_horizon" }, + ], + }, + + speculative: { + options: [ + { id: "current", label: "Inherited from base" }, + { id: "off", label: "Off" }, + { id: "ngram", label: "NGRAM", + flags: ["--speculative-algorithm NGRAM", + "--speculative-num-draft-tokens 16", + "--speculative-ngram-max-bfs-breadth 10"] }, + ], + }, + + pdDisagg: { + modes: [ + { id: "off", label: "Off" }, + { id: "prefill", label: "Prefill role" }, + { id: "decode", label: "Decode role" }, + ], + transferBackends: [ + { id: "mooncake", label: "Mooncake" }, + { id: "nixl", label: "NiXL" }, + ], + ibDevices: [{ id: "auto", label: "Auto" }, "mlx5_0", "mlx5_7"], + router: { + port: 8000, + command: +`python3 -m sglang_router.launch_router \\ + --pd-disaggregation \\ + --prefill http://:{{PREFILL_PORT}} \\ + --decode http://:{{DECODE_PORT}} \\ + --host 0.0.0.0 --port {{ROUTER_PORT}} \\ + --disable-circuit-breaker \\ + --health-check-interval-secs 999999`, + }, + }, + + hicache: { + backends: [ + { id: null, label: "Auto" }, + { id: "file", label: "File" }, + { id: "mooncake", label: "Mooncake" }, + { id: "hf3fs", label: "HF3FS" }, + { id: "nixl", label: "NiXL" }, + ], + writePolicies: [ + { id: "auto", label: "Auto" }, + { id: "write_through", label: "Write-through" }, + { id: "write_back", label: "Write-back" }, + { id: "write_through_selective", label: "Write-through (selective)" }, + ], + }, + }, + + cells: [ + { + match: { hw: "h100", variant: "0.9b", quant: "bf16", strategy: "balanced", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--tp 1", + "--dtype bfloat16", + "--attention-backend fa3", + "--reasoning-parser k2_horizon", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "h100", variant: "3.7b", quant: "bf16", strategy: "balanced", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--tp 1", + "--dtype bfloat16", + "--attention-backend fa3", + "--reasoning-parser k2_horizon", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "h100", variant: "7b", quant: "bf16", strategy: "balanced", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--tp 1", + "--dtype bfloat16", + "--attention-backend fa3", + "--reasoning-parser k2_horizon", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "h100", variant: "32b", quant: "bf16", strategy: "balanced", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--tp 2", + "--dtype bfloat16", + "--attention-backend fa3", + "--reasoning-parser k2_horizon", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "h100", variant: "36b", quant: "bf16", strategy: "balanced", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--tp 2", + "--ep 2", + "--dtype bfloat16", + "--attention-backend fa3", + "--json-model-override-args '{\"xllm_source_router_gemm_partitions\":2}'", + "--reasoning-parser k2_horizon", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // About 760 GB of BF16 weights: more than one eight-H100 node (640 GB) + // holds, so this recipe spans two nodes. The benchmarks were measured at + // TP=16 without EP; the --ep 16 variant below has not been launched yet. + match: { hw: "h100", variant: "375b", quant: "bf16", strategy: "balanced", nodes: "multi-2" }, + verified: false, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--tp 16", + "--ep 16", + "--dtype bfloat16", + "--attention-backend fa3", + "--model-loader-extra-config '{\"enable_multithread_load\":false}'", + "--reasoning-parser k2_horizon", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + ], +};