diff --git a/.claude/skills/cookbook-add-model/references/authoring-reference.md b/.claude/skills/cookbook-add-model/references/authoring-reference.md index e7dd9f9b9..2c5cf3751 100644 --- a/.claude/skills/cookbook-add-model/references/authoring-reference.md +++ b/.claude/skills/cookbook-add-model/references/authoring-reference.md @@ -164,7 +164,7 @@ schemas (full reference in the `_playground.jsx` header): | `moe` | Backend select (incl. MegaMoE) + EP knob; picking the MegaMoE backend reveals a Quantization sub-select (W4A8/W4A4) | Model is MoE and supports multiple `--moe-*-backend` choices. For Blackwell MoE kernel-fusion, give the `megamoe` backend option a `requiresHw` (and optional `excludesStrategy`) gate, then add a sibling `megamoeQuant` block (`{stripEnv, options}`): W4A8 = `NUM_MAX` only, W4A4 adds the FP4-activations env vars; both strip the DeepEP dispatch env. | | `parsers` | Multi-toggle | Model has reasoning / tool-call parsers. | | `speculative` | Single-select chip group | Model has spec-decoding presets you want to expose. | -| `pdDisagg` | Mode + transfer backend (+ optional per-backend env via `envWhen` hw-gate) + IB device + optional `router{port, command}` | Model supports prefill/decode disaggregation. When a PD role is active and `router` is set, the playground shows the router (SGLang Model Gateway) launch command as a separate companion block and retargets the cURL modal to `router.port` (clients hit the router, not the role servers). | +| `pdDisagg` | Mode + transfer backend (+ optional per-backend env via `envWhen` hw-gate) + IB device + optional `router{port, command}` | Model supports prefill/decode disaggregation. A `modes[]` entry may carry `flags` / `env` that only that role needs (`prefill`'s `--load-balance-method`, `decode`'s `--disaggregation-decode-polling-interval`, ...); they are emitted only while that role is selected, and a flag whose head the base cell already sets is replaced rather than duplicated. Put role-specific settings here, NOT in the cells — a cell carries one recipe, and the role is a Playground overlay. In `router.command` the ports MUST be `{{PREFILL_PORT}}` / `{{DECODE_PORT}}` / `{{ROUTER_PORT}}`: the engine substitutes them from its own `PD_PORTS` (prefill 30000, decode **30100**), so a literal port silently points the router at something the generated decode command never binds. When a PD role is active and `router` is set, the playground shows the router (SGLang Model Gateway) launch command as a companion block and retargets the cURL modal to `router.port`. | | `hicache` | Enable + storage + write policy | Model is large enough that hierarchical KV cache matters. | | `hisparse` | Enable + host-ratio select; whole card gated on the live PD-Disagg mode being `decode` | DSA-style model (DeepSeek-V3.2 / V4, GLM-5) that supports decode-side hierarchical sparse attention. | | `flagSelects` | A config-declared **list** of single-selects, each `{ id, title, stripPrefixes, options }` (option = `{ id, label, flags?, hide?, disable?, disableReason? }`); a flagless option is the "none"/accuracy-safe choice | A titled single-select that picks one value of a flag family the other axes don't model — e.g. KV-cache dtype (`--kv-cache-dtype`), mamba scheduler strategy (`--mamba-scheduler-strategy`). Generic: no engine change to add another. | diff --git a/.claude/skills/cookbook-add-model/templates/config.jsx.tmpl b/.claude/skills/cookbook-add-model/templates/config.jsx.tmpl index 25d584f57..da644495f 100644 --- a/.claude/skills/cookbook-add-model/templates/config.jsx.tmpl +++ b/.claude/skills/cookbook-add-model/templates/config.jsx.tmpl @@ -277,14 +277,18 @@ sgl-eval run gsm8k \\ // `auto` is a sentinel (emits no --disaggregation-ib-device flag). ibDevices: [{ id: "auto", label: "Auto" }, "mlx5_0", "mlx5_7"], // Router fronting prefill + decode; substitute /. + // ALWAYS use {{PREFILL_PORT}} / {{DECODE_PORT}} / {{ROUTER_PORT}} — the + // engine fills them from its own PD_PORTS (prefill serves on 30000, decode + // on 30100). A literal port here will not track the engine and sends the + // router at a port the generated decode command does not listen on. router: { port: 8000, command: `python3 -m sglang_router.launch_router \\ --pd-disaggregation \\ - --prefill http://:30000 \\ - --decode http://:30001 \\ - --host 0.0.0.0 --port 8000 \\ + --prefill http://:{{PREFILL_PORT}} \\ + --decode http://:{{DECODE_PORT}} \\ + --host 0.0.0.0 --port {{ROUTER_PORT}} \\ --disable-circuit-breaker \\ --health-check-interval-secs 999999`, }, diff --git a/docs/cookbook/autoregressive/Qwen/Qwen3.6.mdx b/docs/cookbook/autoregressive/Qwen/Qwen3.6.mdx index a4ae06377..dfe58c543 100644 --- a/docs/cookbook/autoregressive/Qwen/Qwen3.6.mdx +++ b/docs/cookbook/autoregressive/Qwen/Qwen3.6.mdx @@ -2,7 +2,6 @@ title: Qwen3.6 metatags: description: "Deploy Qwen3.6 with SGLang - open-weight multimodal series with a 35B MoE (3B active) variant and a 27B dense variant, hybrid reasoning, tool calling, MTP, and long-context support." -tag: NEW --- import { Qwen36Deployment } from '/src/snippets/autoregressive/qwen36-deployment.jsx'; diff --git a/docs/cookbook/autoregressive/Qwen/Qwen3.8.mdx b/docs/cookbook/autoregressive/Qwen/Qwen3.8.mdx new file mode 100644 index 000000000..b9e80e23f --- /dev/null +++ b/docs/cookbook/autoregressive/Qwen/Qwen3.8.mdx @@ -0,0 +1,315 @@ +--- +title: Qwen3.8 +description: "Deploy Qwen3.8 with SGLang — day-0 recipes for Qwen's 2.4T-parameter (95B active) hybrid GDN/GQA Mixture-of-Experts model on NVIDIA and AMD." +tag: NEW +--- + +## Deployment + + + + + +For all methods and hardware platforms, see the [official SGLang installation guide](../../../docs/get-started/install). The two paths below match the **Python / Docker** toggle in the command panel. + + + + + +```bash Command +pip install --upgrade pip +pip install uv +uv pip install sglang +``` + +Then run the **Python** output of the command panel below in that environment. + + + + + +**NVIDIA GPUs** (H200 / B200 / B300 / GB300) — the launch image, since this is a day-0 model with no release cut yet: + +```bash Command +docker pull lmsysorg/sglang:qwen38 +``` + +**AMD GPUs** — pinned `v0.5.17` builds. The two are **not** interchangeable: they target different GPU architectures *and* different ROCm versions, so pick the one matching your hardware or AITER's kernels won't load. + +MI350X / MI355X (CDNA4, gfx950 — ROCm 7.20): + +```bash Command +docker pull lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260812 +``` + +MI300X (CDNA3, gfx942 — ROCm 7.00): + +```bash Command +docker pull lmsysorg/sglang-rocm:v0.5.17-rocm700-mi30x-20260812 +``` + +For how to launch either image, see [Install → Method 3: Using Docker](../../../docs/get-started/install#method-3-using-docker). Substitute the inner `sglang serve ...` with whatever the command generator below produces. + + + + + + + +Pick your hardware + quantization to generate the launch command. Three of the strategies are operating points on the throughput/latency curve; the fourth swaps the speculative decoder: + +- **Low Latency** — fastest reply for a single user. Pick for chat. +- **Balanced** — good speed with several users at once. Use for typical multi-user serving. +- **High Throughput** — most tokens per second across many users. Best for batch jobs. +- **DSpark** — the trained [DSpark draft model](#3-3-dspark-and-replayssm-speculative-decoding) instead of the checkpoint's built-in MTP head. Only offered where DSpark's constraints allow it (see [3.3](#3-3-dspark-and-replayssm-speculative-decoding)). + +import { Deployment } from "/src/snippets/_deployment.jsx"; +import { config } from "/src/snippets/configs/Qwen/qwen3.8.jsx"; +import { benchmarks } from "/src/snippets/configs/Qwen/qwen3.8-benchmarks.jsx"; + + + +## Playground + +The Playground is where you experiment with **SGLang features beyond the verified matrix**. The Deploy panel above only emits combinations the SGLang team has signed off on; the Playground lets you turn on additional knobs on top of whichever cell the Deploy panel is currently showing. + +The knobs come in two flavors: + +- **Built-in SGLang features** — TP / DP-Attention, MoE backend + EP (including WideEP), reasoning / tool-call parsers, speculative-decoding presets, prefill/decode disaggregation, and HiCache tiers. +- **Qwen3.8 specific features** — the **DSpark** speculative-decoding preset and its **ReplaySSM** opt-out, plus the GDN radix-cache-strategy and KV-cache-precision knobs (see [Configuration Tips](#2-configuration-tips) and [3.3 DSpark and ReplaySSM](#3-3-dspark-and-replayssm-speculative-decoding) below). + +import { Playground } from "/src/snippets/_playground.jsx"; + + + +## 1. Model Introduction + +**Qwen3.8** (`Qwen3.8-2.4T-A95B`) is Qwen's largest open-weight model to date: **2.4T total parameters, 95B active per token**. It continues the hybrid-attention design of the Qwen3.5 / Qwen3.6 series, scaled up to 92 layers. + +- **Hybrid Attention** — 23 repeats of `3 × (Gated DeltaNet → MoE) → 1 × (Gated Attention → MoE)`, so 69 linear-attention layers to 23 full-attention ones. Gated Attention runs 64 query heads over 4 KV heads at head dimension 256. This balances linear computational complexity against long-context modeling quality. +- **GDN (Gated Delta Network)** — the linear-attention layers pair a State Space Model with causal convolution (CausalConv1d), 128 V heads and 16 QK heads at head dimension 128. A fixed-size recurrent state replaces the growing KV cache, so memory is `O(1)` per layer while compute stays `O(N)`. +- **Sparse Mixture-of-Experts** — 512 experts, 10 routed plus 1 shared active per token, expert intermediate dimension 2048. Hidden dimension 8192, vocabulary 248,320. +- **MTP** — the checkpoint ships multi-token-prediction weights trained with multiple steps. That is what the NEXTN speculative recipes on this page decode against. + +**License:** [Qwen3.8-2.4T-A95B](https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B/blob/main/LICENSE). **Context length:** 262,144 native, extensible to 1,010,000 tokens. + +**Recommended generation:** `temperature=1.0`, `top_p=0.95`, `top_k=20`, `min_p=0.0`, `presence_penalty=0.0`, `repetition_penalty=1.0`. Raising `presence_penalty` toward 2 curbs runaway repetition at some risk of language mixing. For agentic work Qwen suggests allowing 262,144 tokens of reasoning and 131,072 for the final response. + +**Resources:** Each precision is its own repo — [BF16](https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B) · [FP8](https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B-FP8) · [NVFP4, NVIDIA Blackwell (RadixArk)](https://huggingface.co/RadixArk/Qwen3.8-2.4T-A95B-NVFP4) · [MXFP4, AMD CDNA4 (Qwen)](https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B-FP8-MXFP4). Speculative-decoding draft model: [`RadixArk/Qwen3.8-Max-DSpark`](https://huggingface.co/RadixArk/Qwen3.8-Max-DSpark) (see [3.3](#3-3-dspark-and-replayssm-speculative-decoding)). + +## 2. Configuration Tips + +Four cells are marked **Not Verified** — **GB300 BF16** and the three GB300 **DSpark** strategies. They have launch recipes but no completed validation run; every other cell on the page has been run, including B300 NVFP4 DSpark. + +**Weight size decides the topology.** At 2.4T parameters BF16 is ≈4.8TB, FP8 ≈2.4TB, NVFP4 ≈1.2TB. FP8 fits no single node here — not even B300, whose 8 × 288GB = 2.30TB misses by a hair — so every FP8 recipe is multi-node. Single-node means FP4: NVFP4 on B300, MXFP4 on MI355X/MI350X. B200 NVFP4 would fit one node but pipelines two, because ~25GB per GPU after weights is too little to serve against. BF16 does not fit 16 GPUs either, so its one recipe is TP32 across 8 GB300 nodes — the only platform where a flat TP32 stays on rack-scale NVLink. + +**Two distinct FP4 checkpoints.** [NVFP4](https://huggingface.co/RadixArk/Qwen3.8-2.4T-A95B-NVFP4) is Blackwell-only; [MXFP4](https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B-FP8-MXFP4) is MI350X/MI355X-only and hybrid (MXFP4 experts, FP8 attention/dense). MI300X is CDNA3 with no hardware MX matmul, so it serves FP8. Leave `--moe-runner-backend` unset on both and the runner resolves from the checkpoint's own `quant_method` — except the NVFP4 wide-EP tier, which pairs `flashinfer_trtllm_routed` with `--moe-a2a-backend flashinfer` by hand because auto cannot resolve that combination. + +### GDN state is the scarce resource, not KV + +Two thirds of the layers are Gated DeltaNet, and their recurrent state lives in its own pool. That pool, not KV, is usually what caps concurrency — and a request's cost depends on the caching strategy: + + + + + + + + + + + + + + + + + + + + + + +
StrategyState slots per request
--disable-radix-cache1
no_buffer3
extra_buffer (this model's auto)5, or 4 where PP disables the overlap scheduler
+ +So `--max-mamba-cache-size` has to match the ratio in force, or it silently clamps `max_running_requests` to a fraction of the target — every cell leaves the pool to `--mamba-full-memory-ratio` except GB300 FP8 Balanced and Low Latency, whose pins are part of tuned capacity sets — Low Latency's `--max-mamba-cache-size 80` is exactly its 16 concurrent requests × 5 slots. And `extra_buffer` needs radix caching on: `mamba_extra_buffer_of()` requires `disable_radix_cache` false, so adding `--disable-radix-cache` makes the strategy inert and drops the budget to one slot. + +### NEXTN caps concurrency at 48 + +MTP weights ship inside the checkpoint, so NEXTN needs no draft model and the 3/1/4 preset fills in automatically. But a speculative cell with no `--max-running-requests` gets **48** from the speculative hook rather than a memory-derived ceiling — pin it explicitly to serve more. `pp_size > 1` rules speculative decoding out entirely in aggregated serving, which is why the H200, B200/B300 FP8, B200 NVFP4 and MI300X recipes carry no MTP. + +### Linear-attention backends differ by GPU generation + +`--mamba-ssm-dtype bfloat16` is load-bearing on SM100: the flashinfer GDN decode default is gated on it, and without it decode silently falls back to Triton. On SM90 the GDN default is Triton for *both* halves, which is why H200 is the one cell pinning `--linear-attn-decode-backend flashinfer` too. The flashinfer GDN prefill default only covers chunk sizes up to 8192, so any cell with a larger `--chunked-prefill-size` must state `--linear-attn-prefill-backend flashinfer` itself. + +`--attention-backend trtllm_mha` is SM100-only. On Blackwell cells that leave it unset, the model hook picks it together with `--page-size 64` — and returns early when the backend *is* named, so an explicit backend also drops that paired page size and nothing then depends on `--speculative-eagle-topk` to keep the backend off Triton. + +### GB300 tiers + +The FP8 ladder spans the whole curve on 4 nodes × 4 GPUs: + + + + + + + + + + + + + + + + + + + + + + + + + + +
TierShapeSpec
Low LatencyTP16 narrow EPNEXTN 3+1
BalancedDP4×TP4 + EP16NEXTN 3+1
High ThroughputDP4×TP4 + EP16off
+ +Balanced and High Throughput share one shape and differ only in capacity. Low Latency is the odd one out: narrow EP wins at low concurrency, where spending ranks on expert parallelism costs more than it returns. MTP is off at saturation because draft-plus-verify overhead outweighs the speedup. NVFP4 has only two tiers — its recipes span different GPU counts (8 vs 16) and the wide-EP one holds capacity fixed across its whole concurrency list, so no third operating point exists. + +Extra build requirements: FP8 Balanced and High Throughput need the **DeepEP v2 wheel** (`2.1.0+01dc3aa`), since their `--moe-a2a-backend deepep_v2` flags do not exist upstream. NVFP4 High Throughput needs `nvfp4_agg_wideep_dep16_flashinfer_setup.sh` run first, and its `SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK=8192` is not optional — unset it falls back to 1024 and startup raises once `1024 × ep_size` no longer covers the largest CuteDSL MoE forward. + +### AllReduce fusion + +The four cells running one flat TP group — GB300 FP8 Low Latency, GB300 NVFP4 Low Latency, GB300 BF16, B300 NVFP4 — set `SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION`, the Qwen3.5 CuteDSL path whose single workspace fuses AllReduce + Residual + RMSNorm with the MoE finalize. Worth 6–13% over the legacy path. Don't pass `--flashinfer-allreduce-fusion-backend` alongside it — the env suppresses the flag with a warning. + +Nothing else can use it: the fusion needs DP-attention off and the built-in TP MoE, so the wide-EP tiers are out, and the pipelined cells put their cross-node traffic on IB rather than NVLink. The three GB300 cells also carry `NCCL_NVLS_ENABLE=1`, because SGLang forces NVLS collectives off when that variable is unset. + +### AMD + +Recommended: **MI355X + MXFP4, single node, TP8**. MI350X emits the identical command (same gfx950, same 288GB, same `mi35x` image); MI300X is CDNA3, takes the `mi30x` image, and needs two nodes for the FP8 weights. + +- **`--mem-fraction-static` looks aggressive on purpose.** With the aiter backend above 8192 context SGLang multiplies it by **0.85** before allocating, so MI355X's `0.9` lands at ≈0.765 and MI300X's `1.0` at ≈0.85. Don't "fix" these downward. MI300X must stay at 1.0 or the weights stop fitting. +- **`--disable-custom-all-reduce` belongs on every MI300X rank** — SGLang resolves it per process, so setting it on one node would leave the two pipeline stages reducing through different code paths. +- MI300X runs `--kv-cache-dtype fp8_e4m3` with `--page-size 16`: at 8 × 192GB per node the shape is memory-bound. + +### ReplaySSM + +A GDN layer's recurrent state overwrites itself every token, so speculative verify has to be rewindable. Snapshotting the whole K×V state per draft step costs 64 KiB per request, layer and head at K=V=128, times γ+1 steps — scratch taken out of the same budget as the persistent state pool. + +[ReplaySSM](https://tridao.me/blog/2026/replayssm/) stores each draft step's raw inputs `Sᵢ = (vᵢ, kᵢ, gᵢ, βᵢ)` instead, a few hundred bytes written by the verify kernel on its way through. Once the sampler fixes the accepted length, one fold kernel replays the accepted prefix from the committed checkpoint and advances it in place. The fold is a verbatim clone of the verify recurrence, so the rebuilt state is bit-identical to the recurrent baseline; draft-step scratch shrinks by roughly two orders of magnitude and is never allocated. + +Folding on every commit is what lets it compose with radix prefix caching over the mutable GDN state: every `--mamba-track-interval` tokens the state is handed to the radix tree, and under `extra_buffer` it goes to a second slot so the running request keeps mutating its own. It is off by default — see [3.3](#3-3-dspark-and-replayssm-speculative-decoding). + +## 3. Advanced Usage + + +The `model` argument in the examples below is the BF16 repo id. Every precision is a **separate repo**, so `model` has to be the checkpoint the server was actually launched with — `…-A95B-FP8`, `…-A95B-NVFP4`, or `…-A95B-FP8-MXFP4`. The Deploy panel's cURL snippet always shows the right id for the cell you have selected. + + +### 3.1 Reasoning + +Qwen3.8 **always** reasons — thinking cannot be turned off, and every response opens with a `…` block. The `qwen3` reasoning parser (toggle **Reasoning Parser** in the **Parsers** card of the [Playground above](#playground)) splits that block into `reasoning_content`, leaving `content` as the answer alone. + +Depth is tunable per request with `reasoning_effort` — `xhigh` (the default), `medium`, or `low`. `preserve_thinking` carries reasoning from earlier turns into context and is on by default. + + + +```python Example +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:30000/v1", api_key="EMPTY") +resp = client.chat.completions.create( + model="Qwen/Qwen3.8-2.4T-A95B", + messages=[{"role": "user", "content": "What is 15% of 240?"}], + reasoning_effort="xhigh", # xhigh (default) | medium | low +) +msg = resp.choices[0].message +print("Reasoning:", getattr(msg, "reasoning_content", None)) +print("Answer:", msg.content) +``` + + + + + +```text Output +Pending update — a sample transcript will be added here. +``` + + + +### 3.2 Tool Calling + +Enable the `qwen3_coder` tool-call parser (toggle **Tool Call Parser** in the **Parsers** card of the [Playground above](#playground)) to surface structured tool calls via `message.tool_calls`. + + + +```python Example +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:30000/v1", api_key="EMPTY") + +tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a location", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string", "description": "The city name"}, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + }, + } +] + +resp = client.chat.completions.create( + model="Qwen/Qwen3.8-2.4T-A95B", + messages=[{"role": "user", "content": "What's the weather in Beijing?"}], + tools=tools, +) +msg = resp.choices[0].message +print("Reasoning:", getattr(msg, "reasoning_content", None)) +print("Content:", msg.content) +print("Tool calls:", msg.tool_calls) +``` + + + + + +```text Output +Pending update — a sample transcript will be added here. +``` + + + +### 3.3 DSpark and ReplaySSM (Speculative Decoding) + +We trained a **DSpark** draft model for Qwen3.8 with SpecForge. Turn it on with the **DSpark** chip in the **Speculative Decoding** card of the [Playground above](#playground) — it emits: + +```bash Command +--speculative-algorithm DSPARK \ +--speculative-draft-model-path RadixArk/Qwen3.8-Max-DSpark +``` + +**ReplaySSM is a separate opt-in.** `--enable-linear-replayssm-spec` defaults off and DSpark does not turn it on, so add it with the **ReplaySSM (spec)** row in the flag-select list. With it on (see [Configuration Tips](#2-configuration-tips) above for how it works), the verify kernel stores each draft step's raw inputs instead of snapshotting the full K×V GDN state, and a single fold kernel replays the accepted prefix from the last committed checkpoint. It's a pure side channel behind a ring buffer — the verify output is bitwise unchanged — so there's no accuracy tradeoff, only a memory one. + +**DSpark does not compose with every cell.** `_handle_dspark` rejects the run outright rather than degrading, so check these before turning the chip on: + +- **`--pp-size` must be 1.** The H200, B200, B300 (FP8) and MI300X cells are all pipelined, so DSpark is unavailable on them. +- **With DP-Attention it additionally requires `--enable-dp-lm-head`**, the built-in TP MoE (`--moe-a2a-backend none`), and no context parallel. That rules out the GB300 wide-EP tiers, which run DP attention over DeepEP v2 or FlashInfer A2A. +- `--speculative-num-steps` is forced to 1, and an omitted `--speculative-draft-model-path` only works if the target checkpoint bundles the draft weights. + +The Playground greys the DSpark chip out on the combinations above. + +The **DSpark** strategy chip in the Deploy panel emits this substitution on the four hw × quantization combinations that clear those constraints: GB300 FP8 (on the low-latency shape — the balanced tier's DeepEP v2 a2a rules DSpark out), GB300 NVFP4, GB300 BF16 and B300 NVFP4. Everything else on the page is either pipelined or wide-EP. + +Because the draft model needs its own weights and KV, those cells run tighter than their NEXTN counterparts — the validated B300 recipe drops `--mem-fraction-static` to 0.80 and trims `--context-length` to 200000 to buy the room back. + +ReplaySSM composes with radix prefix caching, overlap scheduling, and PD decode, so DSpark speculative decoding runs alongside the rest of the stack (WideEP, PD disaggregation, HiCache) rather than requiring any of them to be turned off. diff --git a/docs/cookbook/autoregressive/intro.mdx b/docs/cookbook/autoregressive/intro.mdx index 43e8e81ed..e3f94335c 100644 --- a/docs/cookbook/autoregressive/intro.mdx +++ b/docs/cookbook/autoregressive/intro.mdx @@ -28,7 +28,7 @@ metatags: { if (value.ibDevice && value.ibDevice !== "auto") { adds.push(`--disaggregation-ib-device ${value.ibDevice}`); } + // A `modes[]` entry may declare `flags` / `env` that only the PD role + // it names needs (the prefill worker's balance policy, the decode + // worker's polling interval, ...). Strip the same heads first so the + // role's value wins over a base cell that sets one for its own + // reasons, instead of emitting the flag twice. This runs only inside + // the role branch: applyAllDeltas re-seeds from the base cell on every + // render, so a base flag is never left over from an earlier selection + // and must not be stripped when the role is Off. + const modeMeta = (fc.modes || []).find((m) => m.id === mode); + if (modeMeta && modeMeta.flags && modeMeta.flags.length) { + flags = h.stripFlagsByFirstToken( + flags, modeMeta.flags.map((f) => f.split(/[\s=]/)[0])); + adds.push(...modeMeta.flags); + } // Single-host needs no --dist-init-addr: prefill/decode derive their // ZMQ/dist ports from the role-specific --port (spaced 100 apart, see // PD_PORTS), so the ranges don't overlap. Multi-node still gets a @@ -950,6 +964,10 @@ export const Playground = ({ config }) => { (k) => (gate[k] || []).includes(sel[k])); if (ok) env = [...env, ...meta.env.filter((e) => !env.includes(e))]; } + // Same for env declared on the selected role. + if (modeMeta && modeMeta.env && modeMeta.env.length) { + env = [...env, ...modeMeta.env.filter((e) => !env.includes(e))]; + } } return { flags, env }; }, diff --git a/docs/src/snippets/configs/Qwen/qwen3.8-benchmarks.jsx b/docs/src/snippets/configs/Qwen/qwen3.8-benchmarks.jsx new file mode 100644 index 000000000..4b72daa89 --- /dev/null +++ b/docs/src/snippets/configs/Qwen/qwen3.8-benchmarks.jsx @@ -0,0 +1,23 @@ +// One entry per cell `match` tuple (same 5 keys as config cells). Every entry is +// a bare match with no numbers, so the card shows "pending". + +export const benchmarks = [ + { match: { hw: "h200", variant: "default", quant: "fp8", strategy: "balanced", nodes: "multi-4" } }, + { match: { hw: "b200", variant: "default", quant: "fp8", strategy: "balanced", nodes: "multi-2" } }, + { match: { hw: "b200", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "multi-2" } }, + { match: { hw: "b300", variant: "default", quant: "fp8", strategy: "balanced", nodes: "multi-2" } }, + { match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" } }, + { match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "dspark", nodes: "single" } }, + { match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "multi-4" } }, + { match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "balanced", nodes: "multi-4" } }, + { match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "multi-4" } }, + { match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "dspark", nodes: "multi-4" } }, + { match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "multi-2" } }, + { match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "multi-4" } }, + { match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "dspark", nodes: "multi-2" } }, + { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "balanced", nodes: "multi-8" } }, + { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "dspark", nodes: "multi-8" } }, + { match: { hw: "mi300x", variant: "default", quant: "fp8", strategy: "balanced", nodes: "multi-2" } }, + { match: { hw: "mi350x", variant: "default", quant: "mxfp4", strategy: "balanced", nodes: "single" } }, + { match: { hw: "mi355x", variant: "default", quant: "mxfp4", strategy: "balanced", nodes: "single" } }, +]; diff --git a/docs/src/snippets/configs/Qwen/qwen3.8.jsx b/docs/src/snippets/configs/Qwen/qwen3.8.jsx new file mode 100644 index 000000000..c6454adfc --- /dev/null +++ b/docs/src/snippets/configs/Qwen/qwen3.8.jsx @@ -0,0 +1,939 @@ +// Single `export const config` literal — no spreads/calls/IIFE (Mintlify re-evals at hydration). +// Cells are denormalized: no `--nnodes`/`--node-rank`/`--dist-init-addr`/`--host`/`--port` literals — engine injects them. +// +// Qwen3.8-2.4T-A95B: 92 layers as 23 repeats of (3 x Gated DeltaNet -> MoE, then +// 1 x Gated Attention -> MoE), so 69 linear-attention layers to 23 full-attention +// ones; MoE with 512 experts, 10 routed + 1 shared active; 2.4T total / 95B +// active params. Text-only, and reasoning cannot be disabled. +// +// A hardware x quantization x strategy combination with no launch recipe has no +// cell, and the engine greys it out. + +export const config = { + modelName: "Qwen3.8", + + supportedHardware: ["h200", "b200", "b300", "gb300", "mi300x", "mi350x", "mi355x"], + + variants: [ + { id: "default", label: "Default" }, + ], + // Checkpoint precisions. NVFP4 is NVIDIA's FP4 format (Blackwell only); MXFP4 + // is the OCP format AMD CDNA4 supports natively (mi350x/mi355x only). Not to be + // confused with the Playground's "FlashInfer (MXFP4)" MoE runner chip, an + // unrelated NVIDIA kernel that shares the name. + quantizations: [ + { id: "bf16", label: "BF16" }, + { id: "fp8", label: "FP8" }, + { id: "nvfp4", label: "NVFP4" }, + { id: "mxfp4", label: "MXFP4" }, + ], + // Three operating points on the throughput/latency curve, plus `dspark`, which + // swaps NEXTN for the trained DSpark draft model. Only GB300/FP8 carries the + // full ladder; single-recipe hardware parks under `balanced`. + strategies: [ + { id: "low-latency", label: "Low Latency" }, + { id: "balanced", label: "Balanced" }, + { id: "high-throughput", label: "High Throughput" }, + { id: "dspark", label: "DSpark" }, + ], + // Node counts a recipe spans. GB300 hosts are 4 GPUs, so its TP16 shapes take + // 4 nodes; everything else is an 8-GPU host. + nodesOptions: [ + { id: "single", label: "Single Node" }, + { id: "multi-2", label: "2 Nodes" }, + { id: "multi-4", label: "4 Nodes" }, + { id: "multi-8", label: "8 Nodes" }, + ], + + modelNames: { + "default|bf16": "Qwen/Qwen3.8-2.4T-A95B", + // Separate repo, not a revision of the BF16 one. + "default|fp8": "Qwen/Qwen3.8-2.4T-A95B-FP8", + "default|nvfp4": "RadixArk/Qwen3.8-2.4T-A95B-NVFP4", + "default|mxfp4": "Qwen/Qwen3.8-2.4T-A95B-FP8-MXFP4", + }, + + placeholders: { + HOST_IP: { target: "command", label: "Bind host", default: "0.0.0.0" }, + PORT: { target: "command", label: "Bind port", default: "30000" }, + NODE0_IP: { target: "command", label: "Head node IP", default: "" }, + NODE_RANK: { target: "command", label: "This node rank", default: "" }, + HF_TOKEN: { target: "command", label: "HF token (Docker)", default: "" }, + CURL_HOST: { target: "curl", label: "Server host", default: "localhost" }, + CURL_PORT: { target: "curl", label: "Server port", default: "30000" }, + }, + + curl: `curl http://{{CURL_HOST}}:{{CURL_PORT}}/v1/chat/completions \\ +-H 'Content-Type: application/json' \\ +-d '{ "model": "{{MODEL_NAME}}", "messages": [{"role":"user","content":"Hello"}] }'`, + + latencyPercentile: "Mean", + + // The "⚡ Reproduce" modal's benchmark command. --random-range-ratio 1 pins ISL + // exactly rather than drawing a range, so runs stay comparable. + benchmarkCommands: { + speed: +`python3 -m sglang.bench_serving \\ + --backend sglang-oai \\ + --host {{CURL_HOST}} --port {{CURL_PORT}} \\ + --model {{MODEL_NAME}} \\ + --dataset-name {{DATASET}} \\ + --random-input-len {{ISL}} --random-output-len {{OSL}} --random-range-ratio 1 \\ + --num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}} \\ + --request-rate inf \\ + --flush-cache`, + numPromptsByConc: { 1: 8, 16: 32, 64: 128, 256: 512, 1024: 2048, 4096: 4096 }, + }, + + // Per-hardware image for Docker mode. The two ROCm images are not + // interchangeable: MI300X (gfx942) takes the mi30x build on ROCm 7.00, + // MI350X/MI355X (gfx950) the mi35x build on ROCm 7.20. + dockerImages: { + h200: "lmsysorg/sglang:qwen38", + b200: "lmsysorg/sglang:qwen38", + b300: "lmsysorg/sglang:qwen38", + gb300: "lmsysorg/sglang:qwen38", + mi300x: "lmsysorg/sglang-rocm:v0.5.17-rocm700-mi30x-20260812", + mi350x: "lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260812", + mi355x: "lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260812", + }, + + // Per-hardware notes shown above a multi-node command. Every multi-node recipe + // here crosses IB (only GB300 has rack-scale NVLink and needs no manual NIC + // config); without a pinned socket interface and HCA list, RCCL/GLOO can pick a + // non-routable NIC on a multi-homed host and the rendezvous stalls. The device + // names below are examples — substitute your own (`ip -br addr`, + // `ibv_devinfo`). + multiNodeHints: { + h200: [ + "TP8 x PP4 over IB NDR 400. Pin the rendezvous NIC and list your HCAs on every node:", + " export GLOO_SOCKET_IFNAME=bond0", + " export NCCL_SOCKET_IFNAME=bond0", + " export NCCL_IB_HCA=mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_6,mlx5_7", + " export NCCL_IB_DISABLE=0", + " export SGLANG_HOST_IP=", + ], + b200: [ + "TP8 x PP2 over IB. Pin the rendezvous NIC and list your HCAs on BOTH nodes:", + " export GLOO_SOCKET_IFNAME=", + " export NCCL_SOCKET_IFNAME=", + " export NCCL_IB_HCA=mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_6,mlx5_7", + " export NCCL_IB_DISABLE=0", + " export SGLANG_HOST_IP=", + ], + b300: [ + "TP8 x PP2 over IB. Pin the rendezvous NIC and list your HCAs on BOTH nodes:", + " export GLOO_SOCKET_IFNAME=", + " export NCCL_SOCKET_IFNAME=", + " export NCCL_IB_HCA=mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_6,mlx5_7", + " export NCCL_IB_DISABLE=0", + " export SGLANG_HOST_IP=", + ], + mi300x: [ + "Pin the rendezvous NIC on BOTH nodes (replace eno8303 with your interface):", + " export GLOO_SOCKET_IFNAME=eno8303", + " export NCCL_SOCKET_IFNAME=eno8303", + " export RCCL_SOCKET_IFNAME=eno8303", + ], + }, + + github: { + cookbookModel: "Qwen/Qwen3.8-2.4T-A95B", + }, + + playgroundFeatures: { + + // ----- Card: "Attention Parallelism" ----- + // 23 GQA full-attention layers expose the usual TP/DP-Attention knobs; the + // range is widened past the template default given the model's scale. + attention: { + knobs: [ + { id: "tp", label: "TP", values: [null, 1, 2, 4, 8, 16, 32, 64] }, + { id: "dpAttn", label: "DP-Attention", + values: [null, false, 1, 2, 4, 8, 16, 32, 64], + labels: { "auto": "Auto", "false": "Off" } }, + ], + }, + + // ----- Card: "MoE Parallelism" ----- + // 512 routed experts + 1 shared, top-10 routing. + moe: { + backend: { + options: [ + { id: null, label: "Inherited" }, + { id: "deepep", label: "DeepEP", flags: ["--moe-a2a-backend deepep"] }, + { id: "megamoe", label: "MegaMoE", flags: ["--moe-a2a-backend megamoe"], + requiresHw: ["b200", "b300", "gb300"] }, + // FlashInfer is CUDA-only — this is the NVIDIA MoE runner kernel, NOT + // the AMD "mxfp4" checkpoint quantization above (unrelated despite + // the shared name). Gate it off AMD so it can't be picked there. + { id: "flashinfer_mxfp4", label: "FlashInfer (MXFP4)", flags: ["--moe-runner-backend flashinfer_mxfp4"], + requiresHw: ["h200", "b200", "b300", "gb300"] }, + { id: "marlin", label: "Marlin (W4A16)", flags: ["--moe-runner-backend marlin"] }, + ], + }, + megamoeQuant: { + stripEnv: ["SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK"], + options: [ + { id: "w4a8", label: "W4A8", + env: ["SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320"] }, + { id: "w4a4", label: "W4A4", + env: [ + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320", + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS=1", + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND=1", + ] }, + ], + }, + // WideEP — the launch post's large-scale-EP claim — so the range goes past + // the template default's 16. + ep: { label: "EP", values: [null, 1, 2, 4, 8, 16, 32, 64] }, + }, + + // ----- Card: "Parsers" ----- + parsers: { + items: [ + { id: "reasoning", label: "Reasoning Parser", flag: "--reasoning-parser qwen3" }, + { id: "toolCall", label: "Tool Call Parser", flag: "--tool-call-parser qwen3_coder" }, + ], + }, + + // ----- Card: "Speculative Decoding" ----- + // DSpark is the trained draft model from the launch post. ReplaySSM is a + // separate opt-in (see the flagSelects row below); nothing turns it on + // implicitly, and --enable-linear-replayssm-spec defaults off. + speculative: { + options: [ + { id: "current", label: "Inherited from base" }, + { id: "off", label: "Off (greedy)" }, + { id: "mtp", label: "EAGLE / MTP", + flags: ["--speculative-algorithm EAGLE", "--speculative-num-steps 3", + "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 4"] }, + // _handle_dspark (arg_groups/speculative_hook.py) raises rather than + // degrading: pp_size must be 1, and under DP-Attention it also needs + // --enable-dp-lm-head, moe_a2a_backend none, and no context parallel. + // Gate the chip on both so the panel can't emit a command that aborts. + { id: "dspark", label: "DSpark", + flags: ["--speculative-algorithm DSPARK", + "--speculative-draft-model-path RadixArk/Qwen3.8-Max-DSpark"], + disable: [ + { when: { dpAttnOn: [true] }, + reason: "DSpark with DP-Attention additionally requires --enable-dp-lm-head, the built-in TP MoE (--moe-a2a-backend none) and no context parallel. Turn DP-Attention off in the Attention card above, or pick a cell that doesn't use it." }, + { when: { hw: ["h200", "mi300x"] }, + reason: "DSpark requires pp_size == 1 and this recipe is pipelined (TP x PP across nodes)." }, + { when: { hw: ["b200", "b300"], quant: ["fp8"] }, + reason: "DSpark requires pp_size == 1 and the B200/B300 FP8 recipes are TP8 x PP2." }, + ] }, + { id: "ngram", label: "NGRAM", + flags: ["--speculative-algorithm NGRAM", + "--speculative-num-draft-tokens 16", + "--speculative-ngram-max-bfs-breadth 10"], + disable: { dpAttnOn: [true] }, + disableReason: "NGRAM is incompatible with DP-Attention. Turn DP-Attention off in the Attention card above to use NGRAM." }, + ], + }, + + // ----- Card: "PD Disaggregation" ----- + // Role flags follow the P/D bundle's own prefill and decode workers. Two + // flags those recipes carry are deliberately not emitted: + // --prefill-round-robin-balance is a DeprecatedAction on current SGLang and + // does nothing, and --mamba-track-interval is context-dependent (the source + // recipes set it equal to their context cap) so a fixed value here would be + // wrong for cells serving the native window. + pdDisagg: { + modes: [ + { id: "off", label: "Off" }, + { id: "prefill", label: "Prefill role", + flags: ["--load-balance-method round_robin", + "--enable-symm-mem", + "--scheduler-recv-interval 1"] }, + { id: "decode", label: "Decode role", + flags: ["--enable-symm-mem", + "--scheduler-recv-interval 1", + "--disaggregation-decode-polling-interval 1", + "--skip-server-warmup"], + env: ["SGLANG_DECODE_BOOTSTRAP_TIMEOUT=1000", + "SGLANG_DISAGG_STAGING_POOL_SIZE_MB=4096"] }, + ], + transferBackends: [ + // Shared by the P/D bundle's prefill and decode workers. MC_FORCE_MNNVL=1 + // is in those recipes too but is MNNVL-fabric only, so it is left to the + // operator rather than emitted on hardware that has no such fabric. + { id: "mooncake", label: "Mooncake", + env: ["SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE=100000", + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=100000", + "SGLANG_DISAGGREGATION_WAITING_TIMEOUT=100000", + "SGLANG_USE_MESSAGE_QUEUE_BROADCASTER=0", + "SGLANG_UNBALANCED_MODEL_LOADING_TIMEOUT_S=3600", + "SGLANG_MOONCAKE_CUSTOM_MEM_POOL=True", + "SGLANG_DISAGG_STAGING_BUFFER=1"] }, + { id: "nixl", label: "NiXL" }, + ], + ibDevices: [{ id: "auto", label: "Auto" }, "mlx5_0", "mlx5_7"], + // Ports come from the engine's PD_PORTS, not literals — the decode role + // serves on 30100, so a hardcoded target would not reach it. + router: { + port: 8000, + command: +`python3 -m sglang_router.launch_router \\ + --pd-disaggregation \\ + --prefill http://:{{PREFILL_PORT}} \\ + --decode http://:{{DECODE_PORT}} \\ + --host 0.0.0.0 --port {{ROUTER_PORT}} \\ + --worker-startup-timeout-secs 7200 \\ + --request-timeout-secs 6900 \\ + --pool-idle-timeout-secs 4 \\ + --disable-circuit-breaker`, + }, + }, + + // ----- Card: "Hierarchical KV Cache" ----- + hicache: { + backends: [ + { id: null, label: "Auto" }, + { id: "file", label: "File" }, + { id: "mooncake", label: "Mooncake" }, + { id: "hf3fs", label: "HF3FS" }, + { id: "nixl", label: "NiXL" }, + ], + writePolicies: [ + { id: "auto", label: "Auto" }, + { id: "write_through", label: "Write-through" }, + { id: "write_back", label: "Write-back" }, + { id: "write_through_selective", label: "Write-through (selective)" }, + ], + }, + + // ----- Axis: Flag Selects (GDN state knobs) ----- + flagSelects: [ + { + // Opt-in row for ReplaySSM on a speculative run. Bit-identical to the + // recurrent baseline per the launch post, so there's no accuracy + // tradeoff — only a memory one. + id: "replaySsm", title: "ReplaySSM (spec)", + showWhen: (b) => b.spec === "dspark", + stripPrefixes: ["--enable-linear-replayssm-spec"], + options: [ + { id: "off", label: "Off" }, + { + id: "on", label: "On", + disable: { pdMode: ["prefill"] }, + disableReason: "A PD prefill server never runs speculative verify, so --enable-linear-replayssm-spec is rejected at startup.", + flags: ["--enable-linear-replayssm-spec"], + }, + ], + }, + { + // Radix prefix caching over the GDN state — see "ReplaySSM and Overlap + // for the GDN State" above for what extra_buffer buys. + id: "mambaRadix", title: "GDN Radix Cache Strategy", + stripPrefixes: ["--mamba-radix-cache-strategy"], + options: [ + { id: "auto", label: "Auto (extra_buffer)" }, + { id: "lazy", label: "extra_buffer_lazy", flags: ["--mamba-radix-cache-strategy extra_buffer_lazy"] }, + { id: "nobuf", label: "no_buffer", flags: ["--mamba-radix-cache-strategy no_buffer"] }, + ], + }, + { + id: "kvCacheDtype", title: "KV Cache Precision", + stripPrefixes: ["--kv-cache-dtype"], + options: [ + { id: "auto", label: "Auto (BF16)" }, + { id: "fp8", label: "FP8 (E4M3) — halves KV memory", flags: ["--kv-cache-dtype fp8_e4m3"] }, + ], + }, + ], + }, + + // Ordering: the first cell seeds the Deploy panel's default selection. + cells: [ + { + // GB300 / FP8, balanced — DP4 attention with per-DP TP4, MoE EP16 over + // DeepEP v2 hybrid, NEXTN 3+1 with ReplaySSM. The capacity set + // (max-total-tokens / max-running-requests / mamba pool / decode graph + // ladder) is tuned as a unit; retune the values together. + // + // The SGLANG_DEEPEP_V2_*_PER_RANK envs size the a2a buffers against + // --chunked-prefill-size 32768 — the default 128 cap refuses to start. + match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "balanced", nodes: "multi-4" }, + verified: true, + env: [ + "SGLANG_DEEPEP_V2_EXPAND_PREFILL=1", + "SGLANG_DEEPEP_V2_NUM_MAX_DISPATCH_TOKENS_PER_RANK=2048", + "SGLANG_DEEPEP_V2_MASKED_NUM_MAX_DISPATCH_TOKENS_PER_RANK=384", + "EP_DISABLE_GIN=1", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp-size 16", + "--dp-size 4", + "--ep-size 16", + "--enable-dp-attention", + "--enable-dp-lm-head", + "--enable-dp-attention-local-control-broadcast", + "--moe-dense-tp-size 1", + "--kv-cache-dtype fp8_e4m3", + "--linear-attn-prefill-backend flashinfer", + "--moe-a2a-backend deepep_v2", + "--deepep-v2-mode hybrid", + "--enable-eplb", + "--mamba-ssm-dtype bfloat16", + "--mamba-radix-cache-strategy extra_buffer", + "--speculative-algorithm NEXTN", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--enable-linear-replayssm-spec", + "--mem-fraction-static 0.93", + "--max-total-tokens 360448", + "--max-running-requests 128", + "--max-mamba-cache-size 132", + "--cuda-graph-max-bs-decode 32", + "--cuda-graph-bs-decode 1 2 4 8 16 32", + "--chunked-prefill-size 32768", + "--max-prefill-tokens 8192", + "--reasoning-parser qwen3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // GB300 / FP8, low-latency — TP16 across 4 nodes, narrow EP, NEXTN with + // ReplaySSM, CuteDSL AllReduce fusion. --max-mamba-cache-size 80 is + // 16 concurrent requests x the 5 GDN state slots extra_buffer budgets each; + // retune the two together. + match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "multi-4" }, + verified: true, + env: [ + "SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION=1", + "NCCL_NVLS_ENABLE=1", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp-size 16", + "--kv-cache-dtype fp8_e4m3", + "--attention-backend trtllm_mha", + "--moe-runner-backend flashinfer_trtllm", + "--mamba-ssm-dtype bfloat16", + "--mamba-radix-cache-strategy extra_buffer", + "--speculative-algorithm NEXTN", + "--enable-linear-replayssm-spec", + "--mem-fraction-static 0.95", + "--max-running-requests 16", + "--max-mamba-cache-size 80", + "--max-total-tokens 262144", + "--cuda-graph-max-bs-decode 16", + "--cuda-graph-bs-decode 1 2 4 8 16", + "--chunked-prefill-size 8192", + "--max-prefill-tokens 8192", + "--reasoning-parser qwen3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // GB300 / FP8, high-throughput — same wide-EP shape as balanced, MTP off. + // MASKED=384 caps the decode slab, which would otherwise default to the + // full 2048 and cost GiBs at graph capture. --max-total-tokens 2800000 is + // the accuracy-oriented value; throughput runs use 2000000. + match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "multi-4" }, + verified: true, + env: [ + "SGLANG_DEEPEP_V2_EXPAND_PREFILL=1", + "SGLANG_DEEPEP_V2_NUM_MAX_DISPATCH_TOKENS_PER_RANK=2048", + "SGLANG_DEEPEP_V2_MASKED_NUM_MAX_DISPATCH_TOKENS_PER_RANK=384", + "EP_DISABLE_GIN=1", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp-size 16", + "--dp-size 4", + "--ep-size 16", + "--enable-dp-attention", + "--enable-dp-lm-head", + "--enable-dp-attention-local-control-broadcast", + "--moe-dense-tp-size 1", + "--kv-cache-dtype fp8_e4m3", + "--linear-attn-prefill-backend flashinfer", + "--moe-a2a-backend deepep_v2", + "--deepep-v2-mode hybrid", + "--enable-eplb", + "--mamba-radix-cache-strategy extra_buffer", + "--mamba-ssm-dtype bfloat16", + "--mem-fraction-static 0.93", + "--max-total-tokens 2800000", + "--cuda-graph-max-bs-decode 128", + "--cuda-graph-bs-decode 1 2 4 8 16 32 64 96 128", + "--chunked-prefill-size 32768", + "--max-prefill-tokens 8192", + "--reasoning-parser qwen3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // GB300 / NVFP4, low-latency — TP8 narrow EP across 2 nodes, NEXTN 3+1 with + // ReplaySSM, CuteDSL AllReduce fusion. --fp4-gemm-backend and + // --moe-runner-backend are real overrides: auto picks the CuTe DSL FP4 + // kernels on SM100 and never enables the TRT-LLM fused NVFP4 MoE path. + match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "multi-2" }, + verified: true, + env: [ + "SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION=1", + "NCCL_MNNVL_ENABLE=1", + "NCCL_CUMEM_ENABLE=1", + "NCCL_NVLS_ENABLE=1", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp-size 8", + "--quantization modelopt_fp4", + "--fp4-gemm-backend flashinfer_cutlass", + "--kv-cache-dtype fp8_e4m3", + "--attention-backend trtllm_mha", + "--linear-attn-prefill-backend flashinfer", + "--moe-runner-backend flashinfer_trtllm", + "--mamba-radix-cache-strategy extra_buffer", + "--mamba-ssm-dtype bfloat16", + "--speculative-algorithm NEXTN", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--enable-linear-replayssm-spec", + "--mem-fraction-static 0.90", + "--chunked-prefill-size 8192", + "--max-prefill-tokens 8192", + "--cuda-graph-backend-prefill breakable", + "--cuda-graph-backend-decode full", + "--reasoning-parser qwen3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // GB300 / NVFP4, high-throughput — full DP16 attention with EP16 MoE over + // FlashInfer one-sided A2A, MTP off. flashinfer_trtllm_routed must stay: + // with a2a=flashinfer the auto runner resolution aborts at startup. + // SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK is load-bearing — + // unset it falls back to 1024 and startup raises once 1024 x ep_size no + // longer covers the largest CuteDSL MoE forward. + // --disable-prefill-cuda-graph is required at chunk 131072, where graph + // capture would OOM; --skip-server-warmup goes with it. + match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "multi-4" }, + verified: true, + env: [ + "PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True", + "SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK=8192", + "NCCL_MNNVL_ENABLE=1", + "NCCL_CUMEM_ENABLE=1", + "NCCL_NET_GDR_C2C=1", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp-size 16", + "--dp-size 16", + "--enable-dp-attention", + "--enable-dp-lm-head", + "--enable-dp-attention-local-control-broadcast", + "--moe-dense-tp-size 1", + "--quantization modelopt_fp4", + "--fp4-gemm-backend flashinfer_cutlass", + "--kv-cache-dtype fp8_e4m3", + "--linear-attn-prefill-backend flashinfer", + "--moe-runner-backend flashinfer_trtllm_routed", + "--moe-a2a-backend flashinfer", + "--ep-dispatch-algorithm static", + "--mamba-radix-cache-strategy extra_buffer", + "--mamba-ssm-dtype bfloat16", + "--mem-fraction-static 0.95", + "--chunked-prefill-size 131072", + "--max-prefill-tokens 8192", + "--weight-loader-drop-cache-after-load", + "--model-loader-extra-config '{\"enable_multithread_load\": false}'", + "--disable-prefill-cuda-graph", + "--cuda-graph-backend-decode full", + "--skip-server-warmup", + "--reasoning-parser qwen3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // GB300 / BF16 — full-precision weights, TP32 across 8 nodes x 4 GPUs. + // 4.8TB does not fit 16 GPUs, and GB300 is the one platform where a flat + // TP32 stays on rack-scale NVLink instead of crossing IB. KV stays at model + // precision (no --kv-cache-dtype), the highest-fidelity configuration here. + // The sizing is derived, not measured. + match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "balanced", nodes: "multi-8" }, + env: [ + "SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION=1", + "NCCL_NVLS_ENABLE=1", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp-size 32", + "--mamba-ssm-dtype bfloat16", + "--speculative-algorithm NEXTN", + "--speculative-eagle-topk 1", + "--enable-linear-replayssm-spec", + "--mem-fraction-static 0.95", + "--chunked-prefill-size 8192", + "--max-prefill-tokens 8192", + "--reasoning-parser qwen3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // H200 / FP8 — 4 nodes x 8 GPUs, TP8 x PP4 over IB. The one Hopper recipe, + // and the only cell where the flashinfer linear-attention backends are real + // overrides: SM90 defaults to triton for both GDN halves. --page-size 64 is + // likewise non-default on Hopper. mem-fraction is left to the auto + // heuristic, which prices the graph set into its reserve. + match: { hw: "h200", variant: "default", quant: "fp8", strategy: "balanced", nodes: "multi-4" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp-size 8", + "--pp-size 4", + "--dist-timeout 1800", + "--linear-attn-prefill-backend flashinfer", + "--linear-attn-decode-backend flashinfer", + "--mamba-full-memory-ratio 0.95", + "--mamba-ssm-dtype bfloat16", + "--max-prefill-tokens 8192", + "--page-size 64", + "--reasoning-parser qwen3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // B300 / FP8 — 2 nodes x 8 GPUs, TP8 x PP2 over IB (~2.4TB of weights do not + // fit one 2.30TB node). pp > 1 forbids speculative decoding and + // auto-disables the overlap scheduler, so extra_buffer budgets 4 GDN slots + // per request rather than 5. --context-length 262144 is the model's native + // window written out, not a cap. + match: { hw: "b300", variant: "default", quant: "fp8", strategy: "balanced", nodes: "multi-2" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp-size 8", + "--pp-size 2", + "--dist-timeout 1800", + "--context-length 262144", + "--attention-backend trtllm_mha", + "--linear-attn-prefill-backend flashinfer", + "--linear-attn-decode-backend flashinfer", + "--moe-runner-backend flashinfer_trtllm", + "--moe-a2a-backend none", + "--mamba-radix-cache-strategy extra_buffer", + "--mamba-full-memory-ratio 0.95", + "--mamba-ssm-dtype bfloat16", + "--mem-fraction-static 0.95", + "--max-running-requests 512", + "--chunked-prefill-size 8192", + "--max-prefill-tokens 8192", + "--page-size 64", + "--cuda-graph-backend-decode full", + "--cuda-graph-max-bs-decode 128", + "--cuda-graph-bs-decode 1 2 4 8 16 32 64 128", + "--reasoning-parser qwen3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // B300 / NVFP4 — single node, TP8, NEXTN with ReplaySSM and the CuteDSL + // AllReduce fusion. BF16 KV: the only NVFP4 cell serving KV at model + // precision. --mamba-ssm-dtype bfloat16 is load-bearing — the SM100 + // flashinfer GDN decode default is gated on it, and without it decode falls + // back to Triton. + match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" }, + verified: true, + env: [ + "SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION=1", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp-size 8", + "--moe-runner-backend flashinfer_trtllm", + "--mamba-radix-cache-strategy extra_buffer", + "--mamba-ssm-dtype bfloat16", + "--speculative-algorithm NEXTN", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--enable-linear-replayssm-spec", + "--mem-fraction-static 0.90", + "--chunked-prefill-size 8192", + "--max-prefill-tokens 8192", + "--reasoning-parser qwen3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // B200 / FP8 — 2 nodes x 8 GPUs, TP8 x PP2, same shape as the B300 FP8 cell + // but leaner: ~0.6TB free after weights instead of ~2.1TB, so the + // concurrency ceiling and mem-fraction are both left derived. + match: { hw: "b200", variant: "default", quant: "fp8", strategy: "balanced", nodes: "multi-2" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp-size 8", + "--pp-size 2", + "--dist-timeout 1800", + "--mamba-full-memory-ratio 0.95", + "--mamba-ssm-dtype bfloat16", + "--chunked-prefill-size 8192", + "--reasoning-parser qwen3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // B200 / NVFP4 — 2 nodes x 8 GPUs, TP8 x PP2. Not the single-node shape the + // weights would allow: at TP8 on one node ~153.6GB of weights leaves only + // ~25GB per GPU for the pools, so this recipe pipelines two nodes and cuts + // the per-GPU weight share to ~77GB. pp > 1 forbids speculative decoding. + match: { hw: "b200", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "multi-2" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp-size 8", + "--pp-size 2", + "--dist-timeout 1800", + "--attention-backend trtllm_mha", + "--mamba-ssm-dtype bfloat16", + "--mamba-radix-cache-strategy extra_buffer", + "--moe-runner-backend flashinfer_trtllm", + "--mem-fraction-static 0.88", + "--chunked-prefill-size 8192", + "--max-prefill-tokens 8192", + "--reasoning-parser qwen3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // MI355X / MXFP4 — single node, TP8. SGLANG_USE_AITER gates the AITER + // MXFP4-MoE / GEMM / norm / rope kernels; the ROCm image sets it, a + // bare-pip host does not. mem-fraction 0.9 is pre-scaling — aiter + // multiplies it by 0.85 above 8K context, so ~0.765 effective. + match: { hw: "mi355x", variant: "default", quant: "mxfp4", strategy: "balanced", nodes: "single" }, + verified: true, + env: [ + "SGLANG_USE_AITER=1", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp-size 8", + "--mem-fraction-static 0.9", + "--reasoning-parser qwen3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // MI350X — identical command to MI355X (same CDNA4 gfx950, same 288GB, same + // mi35x ROCm image). + match: { hw: "mi350x", variant: "default", quant: "mxfp4", strategy: "balanced", nodes: "single" }, + verified: true, + env: [ + "SGLANG_USE_AITER=1", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp-size 8", + "--mem-fraction-static 0.9", + "--reasoning-parser qwen3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // MI300X / FP8 — 2 nodes x 8 GPUs, TP8 x PP2 (gfx942 has no MXFP4 hardware, + // and ~2.4TB of FP8 does not fit 1.5TB per node). --disable-custom-all-reduce + // puts the intra-node all-reduce on RCCL and must be on every rank; + // mem-fraction 1.0 is ~0.85 effective after the aiter scale. + match: { hw: "mi300x", variant: "default", quant: "fp8", strategy: "balanced", nodes: "multi-2" }, + verified: true, + env: [ + "SGLANG_USE_AITER=1", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp-size 8", + "--pp-size 2", + "--disable-custom-all-reduce", + "--dist-timeout 3600", + "--page-size 16", + "--kv-cache-dtype fp8_e4m3", + "--chunked-prefill-size 8192", + "--mem-fraction-static 1.0", + "--reasoning-parser qwen3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // GB300 / FP8, DSpark — the low-latency TP16 narrow-EP shape with the DSpark + // draft model in place of NEXTN. The balanced tier is not a candidate: its + // DeepEP v2 a2a rules DSpark out under DP-attention. + match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "dspark", nodes: "multi-4" }, + env: [ + "SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION=1", + "NCCL_NVLS_ENABLE=1", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp-size 16", + "--kv-cache-dtype fp8_e4m3", + "--mamba-ssm-dtype bfloat16", + "--speculative-algorithm DSPARK", + "--speculative-draft-model-path RadixArk/Qwen3.8-Max-DSpark", + "--mem-fraction-static 0.95", + "--chunked-prefill-size 8192", + "--max-prefill-tokens 8192", + "--reasoning-parser qwen3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // GB300 / NVFP4, DSpark — the low-latency TP8 shape with the DSpark draft + // model in place of NEXTN. + match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "dspark", nodes: "multi-2" }, + env: [ + "SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION=1", + "NCCL_MNNVL_ENABLE=1", + "NCCL_CUMEM_ENABLE=1", + "NCCL_NVLS_ENABLE=1", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp-size 8", + "--quantization modelopt_fp4", + "--fp4-gemm-backend flashinfer_cutlass", + "--kv-cache-dtype fp8_e4m3", + "--attention-backend trtllm_mha", + "--linear-attn-prefill-backend flashinfer", + "--moe-runner-backend flashinfer_trtllm", + "--mamba-radix-cache-strategy extra_buffer", + "--mamba-ssm-dtype bfloat16", + "--speculative-algorithm DSPARK", + "--speculative-draft-model-path RadixArk/Qwen3.8-Max-DSpark", + "--mem-fraction-static 0.90", + "--chunked-prefill-size 8192", + "--max-prefill-tokens 8192", + "--cuda-graph-backend-prefill breakable", + "--cuda-graph-backend-decode full", + "--reasoning-parser qwen3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // GB300 / BF16, DSpark — the TP32 shape with the DSpark draft model in place + // of NEXTN. + match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "dspark", nodes: "multi-8" }, + env: [ + "SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION=1", + "NCCL_NVLS_ENABLE=1", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp-size 32", + "--mamba-ssm-dtype bfloat16", + "--speculative-algorithm DSPARK", + "--speculative-draft-model-path RadixArk/Qwen3.8-Max-DSpark", + "--mem-fraction-static 0.95", + "--chunked-prefill-size 8192", + "--max-prefill-tokens 8192", + "--reasoning-parser qwen3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // B300 / NVFP4, DSpark — single-node TP8 with the DSpark draft model. The + // draft needs its own weights and KV, so mem-fraction drops to 0.80 and + // --context-length trims the native window to buy that room back. + // SGLANG_ENABLE_MOE_DEFERRED_FINALIZE defers the MoE finalize so it fuses + // into the CuteDSL AllReduce workspace. + match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "dspark", nodes: "single" }, + verified: true, + env: [ + "SGLANG_ENABLE_MOE_DEFERRED_FINALIZE=1", + "SGLANG_FLASHINFER_MNNVL_CUTEDSL_AR_FUSION=1", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp-size 8", + "--context-length 200000", + "--preferred-sampling-params '{\"top_k\": 20}'", + "--attention-backend trtllm_mha", + "--page-size 64", + "--linear-attn-prefill-backend flashinfer", + "--linear-attn-decode-backend flashinfer", + "--moe-runner-backend flashinfer_trtllm", + "--mamba-radix-cache-strategy extra_buffer", + "--mamba-ssm-dtype bfloat16", + "--speculative-algorithm DSPARK", + "--speculative-draft-model-path RadixArk/Qwen3.8-Max-DSpark", + "--mem-fraction-static 0.80", + "--max-running-requests 128", + "--chunked-prefill-size 8192", + "--max-prefill-tokens 8192", + "--cuda-graph-backend-prefill breakable", + "--cuda-graph-max-bs-prefill 8192", + "--cuda-graph-backend-decode full", + "--cuda-graph-max-bs-decode 128", + "--reasoning-parser qwen3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + ], +};