diff --git a/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx b/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx
index d54773f58..89b7307c8 100644
--- a/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx
+++ b/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx
@@ -56,6 +56,14 @@ import { Qwen38MambaRatioCalculator } from "/src/snippets/_qwen38_mamba_ratio_ca
+
+ The RTX 5090 and RTX PRO 6000 cells above — including every Speculative
+ Decoding / Serving Strategy / SSM dtype combination — were validated at
+ ISL 8192 / OSL 1024, concurrency 1. The other platforms' recipes carry their
+ original validation, which covers the default overlay picks (plus MTP on
+ GB300); non-default overlay picks there are valid but unmeasured.
+
+
### Mamba ratio calculator
@@ -188,7 +196,19 @@ checkpoint's calibration scales automatically.
- **DSpark**: the trained draft model is a separate checkpoint — add
`--speculative-algorithm DSPARK --speculative-draft-model-path
RadixArk/Qwen3.8-27B-DSpark` (the Playground's Speculative Decoding card
- emits this pair).
+ emits this pair). DSpark does **not** take
+ `--speculative-num-draft-tokens`: its verify window is
+ `--speculative-dspark-block-size` (gamma) **+ 1**, and gamma is auto-inferred
+ from the draft checkpoint when the flag is omitted (7 for this checkpoint, so
+ D = 8). That `D` is a term in the balanced ratio —
+ `r = (S + D) x token_equiv / L`, where `token_equiv` is the state slot
+ expressed in KV tokens, `state_bytes / kv_bytes_per_token` (4698 at fp32
+ state / 2394 at bf16, over fp8 KV) — so DSpark needs a materially higher
+ `--mamba-full-memory-ratio` than no-spec at the same `S`, and pinning a
+ different gamma changes the ratio with it. MTP is the opposite case: with
+ `--enable-linear-replayssm-spec` its draft intermediates move onto a fixed
+ ring, so `D = 0` and the ratio returns to the no-spec value. The
+ [calculator](#mamba-ratio-calculator) applies both rules.
- **Hardware fit**: FP8 weights ~28.5GB (not serviceable beyond bs≤2 on
32GB cards); NVFP4 weights ~16.5GB (recommended for RTX 5090-class GPUs).
- `--mamba-radix-cache-strategy extra_buffer_lazy` lowers the state cost per
@@ -197,6 +217,24 @@ checkpoint's calibration scales automatically.
`S` (lazy strategy, or `--disable-radix-cache` for S=1); the
[calculator](#mamba-ratio-calculator) re-derives the ratio for the new `S`.
The balanced ratio itself is VRAM-independent.
+- `--mamba-ssm-dtype`: the GDN state slot is **153.9 MB at `float32`** (the
+ checkpoint's declared precision) and **78.4 MB at `bfloat16`**, so bf16 roughly
+ halves the state pool and hands the difference to KV — measured on an RTX 5090
+ with no speculation, 97,280 KV tokens at bf16 against 68,588 at fp32. On 32GB
+ cards it also decides whether a config fits at all: EAGLE needs
+ `--mem-fraction-static 0.94` at fp32 but 0.92 at bf16. Speed is **not** a
+ one-way trade — with speculative decoding fp32 sometimes wins (NVFP4 + EAGLE:
+ 152.9 vs 144.5 tok/s/user) and sometimes loses (FP8 + EAGLE: 106.3 vs 116.1);
+ measure both for your quantization. Treat
+ `bfloat16` as an accuracy gate and validate it for your workload. On SM120
+ both precisions run the Triton linear-attn prefill path — the FlashInfer GDN
+ prefill fast path gates on SM100, where its validated domain is in fact a
+ bf16 state pool — so no dtype forces an extra flag here. One interaction to
+ know: `--enable-linear-replayssm-spec` auto-selects fp32 state when
+ `--mamba-ssm-dtype` is unset, and an explicit non-fp32 value logs a
+ state-drift warning at boot. The SSM dtype row always emits the flag
+ explicitly, so the bf16 + EAGLE cells run with that warning — accounted for
+ in their validation.
- `--chunked-prefill-size 2048`: decode steps stall behind each prefill chunk
on hybrid GDN models, and 8192-token chunks stall them ~600ms at a time.
2048 keeps decode inter-token latency smooth under mixed load and also
diff --git a/docs/src/snippets/configs/Qwen/qwen3.8-27b-benchmarks.jsx b/docs/src/snippets/configs/Qwen/qwen3.8-27b-benchmarks.jsx
index a5d3ff290..0a1d9dbf8 100644
--- a/docs/src/snippets/configs/Qwen/qwen3.8-27b-benchmarks.jsx
+++ b/docs/src/snippets/configs/Qwen/qwen3.8-27b-benchmarks.jsx
@@ -1,5 +1,15 @@
-// Qwen3.8-27B per-cell benchmark numbers, keyed by the same `match` tuple as
-// qwen3.8-27b.jsx cells. See _deployment.jsx for the speed/accuracy schema.
+// Qwen3.8-27B per-cell benchmark numbers. See _deployment.jsx for the
+// speed/accuracy schema.
+//
+// STRUCTURALLY UNMATCHED since the strategy->overlay migration: every entry
+// below keys on the removed `strategy` match dim, and the three gb300
+// `high-throughput` cells they were measured against were deleted (MTP is now
+// the Speculative Decoding OVERLAY row, which never participates in cell
+// matching). No current cell tuple matches these rows, so the UI shows no
+// benchmark card. The numbers are kept as measurement provenance ONLY — do
+// not re-key them onto the new cells without re-measuring: the overlay-era
+// commands differ (explicit --kv-cache-dtype fp8_e4m3, strategy flag emitted
+// by the tier row) and the accept lengths are protocol-specific.
//
// All six rows are ONE-BATCH measurements (sglang.bench_serving --flush-cache,
// random dataset, ISL=1024 / OSL=1024, --random-range-ratio 1, request-rate inf,
diff --git a/docs/src/snippets/configs/Qwen/qwen3.8-27b.jsx b/docs/src/snippets/configs/Qwen/qwen3.8-27b.jsx
index 33f75733f..7b9eb1c1a 100644
--- a/docs/src/snippets/configs/Qwen/qwen3.8-27b.jsx
+++ b/docs/src/snippets/configs/Qwen/qwen3.8-27b.jsx
@@ -1,38 +1,16 @@
// Single `export const config` literal — no spreads/calls/IIFE (Mintlify re-evals at hydration).
// Cells are denormalized: no `--nnodes`/`--node-rank`/`--dist-init-addr`/`--host`/`--port` literals — engine injects them.
//
-// Qwen3.8-27B: DENSE hybrid Gated Delta Networks VISION-LANGUAGE model — a 27B
-// causal LM plus a vision encoder, served through SGLang's Qwen3-VL path
-// (Qwen3_5ForConditionalGeneration extends Qwen3VLForConditionalGeneration and
-// is registered in the multimodal arch lists). 64 layers as 16 repeats of
-// 3 x (Gated DeltaNet -> FFN) then 1 x (Gated Attention -> FFN): 48
-// linear-attention layers to 16 full-attention. GDN runs 48 value heads and 16
-// QK heads at head_dim 128; attention is GQA 24/4 at head_dim 256. An MTP head
-// trained with multiple steps ships in-checkpoint. Context 262,144 native,
-// extensible to 1,000,000. Dense, so there is no MoE axis.
+// Qwen3.8-27B: dense hybrid Gated Delta Networks vision-language model (48
+// linear-attention + 16 full-attention layers, in-checkpoint MTP head, no MoE
+// axis). Single-GPU on every supported card, hence one node and no
+// parallelism flags in any cell.
//
-// Single-GPU on every supported card — H200 (SM90 datacenter), the SM120
-// workstation pair (RTX PRO 6000 Blackwell / RTX 5090), and DGX Spark (GB10,
-// SM121, 128GB unified memory) — hence one node and no parallelism flags in
-// any cell.
-//
-// PROVENANCE — every flag and value below is transcribed from the
-// pre-migration prose page (one unconditional launch command + its Configuration
-// Tips). Model ids are the exception: BF16/FP8 point at the official Qwen
-// checkpoints, NVFP4 at the RadixArk W4A4 build. That page pinned
-// no sglang version for its measurements, so under the
-// migration skill's reproducible-anchor rule NO measured numbers were carried
-// over: there is no sibling `-benchmarks.jsx`. Cells are nevertheless marked
-// `verified: true` at the maintainers' direction — the badge there reflects
-// their own unpublished validation, not measured data carried by this page. The
-// DGX Spark cells are the exception and stay unverified: that recipe is
-// unvalidated on SM121 / aarch64, as both §2 and the cell comment below say.
-// `benchmarkCommands` below records the page's measurement protocol so the
-// numbers can be re-measured against a pinned build and then land as a
-// benchmarks file.
-//
-// A hardware x quantization combination with no launch recipe has no cell, and
-// the engine greys it out.
+// The RTX 5090 / RTX PRO 6000 cells and both overlay rows carry measured
+// values (8192/1024, concurrency 1); the other platforms' cells are the
+// source page's recipes. Verification per cell is documented above cells[].
+// A hardware x quantization combination with no launch recipe has no cell,
+// and the engine greys it out.
export const config = {
modelName: "Qwen3.8-27B",
@@ -49,32 +27,141 @@ export const config = {
{ id: "rtx5090", label: "RTX 5090", vram: "32GB", vendor: "blackwell" },
],
- variants: [
- { id: "default", label: "Default" },
- ],
- // BF16/FP8 are the official Qwen checkpoints; NVFP4 is the RadixArk
- // W4A4 build. NVFP4 is W4A4 with FP8 projections and declares
- // `kv_cache_quant_algo: FP8`, so under the default `--kv-cache-dtype auto` its
- // KV pool runs fp8_e4m3 off the checkpoint's own calibration scales — no
- // `--kv-cache-dtype` flag in the recipe, and nothing accuracy-degrading added
- // by the cell.
- quantizations: [
- { id: "bf16", label: "BF16" },
- { id: "fp8", label: "FP8" },
- { id: "nvfp4", label: "NVFP4" },
- ],
- // The source page documents ONE operating point: a single general-purpose
- // launch command with no latency/throughput toggle. MTP is described as an
- // opt-in in the tips, not as a second named recipe, so it rides the
- // Playground's speculative axis instead of splitting the strategy dimension.
- strategies: [
- { id: "balanced", label: "Balanced" },
- { id: "high-throughput", label: "High-Throughput" },
- ],
- nodesOptions: [
- { id: "single", label: "Single Node" },
+ // Every cell pins `--kv-cache-dtype fp8_e4m3` at the maintainers' direction
+ // (sign-off recorded in the PR description). NVFP4: a no-op made visible
+ // (the checkpoint's `kv_cache_quant_algo: FP8` already resolved `auto` to
+ // fp8_e4m3). BF16/FP8: a real quality/capacity trade — halves
+ // kv_bytes_per_token but those checkpoints carry no fp8 KV calibration.
+ //
+ // Speculative decoding and GDN state precision are orthogonal knobs, so
+ // they are overlay rows, not match dims (3 x 2 would turn 12 cells into
+ // 72). Options are inline, not `optionsKey` —
+ // docs/scripts/check_cookbook_configs.mjs only reads `dim.options`.
+ matchDims: [
+ { id: "variant", title: "Model Variant", options: [
+ { id: "default", label: "Default" },
+ ] },
+ { id: "quant", title: "Quantization", options: [
+ { id: "bf16", label: "BF16" },
+ { id: "fp8", label: "FP8" },
+ { id: "nvfp4", label: "NVFP4" },
+ ] },
+ { id: "nodes", title: "Nodes", options: [
+ { id: "single", label: "Single Node" },
+ ] },
],
+ overlayDims: [
+ {
+ id: "spec",
+ title: "Speculative Decoding",
+ default: "none",
+ options: [
+ { id: "none", label: "None" },
+ {
+ id: "eagle", label: "EAGLE",
+ // In-checkpoint MTP head; the only availability constraint is the
+ // 32GB RTX 5090, where it needs the NVFP4 weights to leave room.
+ disabled: (sel) => sel.hw === "rtx5090" && sel.quant !== "nvfp4",
+ disableReason:
+ "On the 32GB RTX 5090 the MTP head only fits on top of the NVFP4 weights",
+ // EAGLE and DSPARK need opposite mem-fraction corrections on the
+ // 5090 (EAGLE starves the state pool at boot and wants it UP;
+ // DSpark starves runtime activations and wants it DOWN), so each
+ // option strips the cell's value and re-pins its own.
+ stripPrefixes: (sel) =>
+ sel.hw === "rtx5090" ? ["--mem-fraction-static"] : [],
+ flags: (sel) => [
+ "--speculative-algorithm EAGLE",
+ "--speculative-num-steps 3",
+ "--speculative-eagle-topk 1",
+ "--speculative-num-draft-tokens 4",
+ // ReplaySSM spec-verify only on SM120/SM121, where it was
+ // exercised; it moves the D intermediate SSM states onto a fixed
+ // ring (ratio computed with D=0) and is what makes MTP fit 32GB.
+ // h200 (SM90) and gb300 (SM103) keep the plain MTP recipe.
+ ...(["rtx5090", "rtx6000", "dgx-spark"].includes(sel.hw)
+ ? ["--enable-linear-replayssm-spec"]
+ : []),
+ // Measured on the 5090: bf16 state serves at 0.92, fp32 needs
+ // 0.94 (an fp32 slot is 146.81 MiB vs bf16's 74.81).
+ ...(sel.hw === "rtx5090"
+ ? [sel.ssmDtype === "float32"
+ ? "--mem-fraction-static 0.94"
+ : "--mem-fraction-static 0.92"]
+ : []),
+ ],
+ },
+ {
+ id: "dspark", label: "DSPARK",
+ // Separately-published trained draft model; same 5090 constraint as
+ // EAGLE. No --min-free-slots-delay: at --max-running-requests 1 it
+ // is a strict no-op, and its real semantic (disable the delayer)
+ // would silently bite anyone raising concurrency to 8+.
+ disabled: (sel) => sel.hw === "rtx5090" && sel.quant !== "nvfp4",
+ disableReason:
+ "On the 32GB RTX 5090 the DSpark draft model only fits on top of the NVFP4 weights",
+ stripPrefixes: (sel) =>
+ sel.hw === "rtx5090" ? ["--mem-fraction-static"] : [],
+ flags: (sel) => [
+ "--speculative-algorithm DSPARK",
+ "--speculative-draft-model-path RadixArk/Qwen3.8-27B-DSpark",
+ "--speculative-draft-attention-backend flashinfer",
+ // Measured on the 5090: bf16 state serves at 0.90, fp32 needs
+ // 0.92 — the opposite correction to EAGLE's (see above).
+ ...(sel.hw === "rtx5090"
+ ? [sel.ssmDtype === "float32"
+ ? "--mem-fraction-static 0.92"
+ : "--mem-fraction-static 0.90"]
+ : []),
+ ],
+ },
+ ],
+ },
+ {
+ // Serving tier via the GDN radix-cache strategy, the knob that sets S
+ // (state slots per running request): extra_buffer S=5 (latency tier),
+ // extra_buffer_lazy S=4 (throughput tier — fewer slots, more requests
+ // per pool). The calculator reads this row for the balanced ratio.
+ // The default equals the ENGINE default (extra_buffer), so an untouched
+ // selection matches each platform's original recipe; both options were
+ // characterized on RTX 5090 / RTX PRO 6000 only.
+ id: "tier",
+ title: "Serving Strategy",
+ default: "low-latency",
+ // Owns the flag outright: strip whatever a cell pinned, then re-emit.
+ stripPrefixes: ["--mamba-radix-cache-strategy"],
+ options: [
+ { id: "low-latency", label: "Low-Latency",
+ flags: ["--mamba-radix-cache-strategy extra_buffer"] },
+ { id: "high-throughput", label: "High-Throughput",
+ flags: ["--mamba-radix-cache-strategy extra_buffer_lazy"] },
+ ],
+ },
+ {
+ // One GDN state slot is 146.81 MiB at fp32 and 74.81 MiB at bf16 — the
+ // biggest lever on the state pool that bounds concurrency on small-VRAM
+ // cards. The default equals the ENGINE default (float32, the
+ // checkpoint's declared precision); both precisions were characterized
+ // on RTX 5090 / RTX PRO 6000 only.
+ id: "ssmDtype",
+ title: "Mamba SSM Dtype",
+ default: "float32",
+ options: [
+ // Open on every platform, including with DSPARK on the 5090 (serves
+ // at mem-fraction 0.92 with the engine-default 2048 prefill chunk).
+ { id: "float32", label: "float32", flags: ["--mamba-ssm-dtype float32"] },
+ {
+ id: "bfloat16", label: "bfloat16",
+ disabled: (sel) => sel.hw === "rtx5090" && sel.quant !== "nvfp4",
+ disableReason:
+ "On the 32GB RTX 5090 the bf16 GDN state pool is only a live choice for NVFP4; " +
+ "the BF16 and FP8 checkpoints have no serviceable cell on this card",
+ flags: ["--mamba-ssm-dtype bfloat16"],
+ },
+ ],
+ },
+ ],
modelNames: {
"default|bf16": "Qwen/Qwen3.8-27B",
"default|fp8": "Qwen/Qwen3.8-27B-FP8",
@@ -93,12 +180,9 @@ export const config = {
-H 'Content-Type: application/json' \\
-d '{ "model": "{{MODEL_NAME}}", "messages": [{"role":"user","content":"Hello"}] }'`,
- // The measurement protocol the source page described, kept so its numbers can
- // be reproduced against a pinned build. --random-range-ratio 1 pins ISL
- // exactly rather than drawing a range; --flush-cache measures cache-cold
- // (bench_serving's `random` prompts are deterministic, so a warm rerun would
- // hit the radix cache and inflate throughput) — the page's own "prefix caching
- // disabled" protocol.
+ // The source page's measurement protocol, kept reproducible:
+ // --random-range-ratio 1 pins ISL exactly; --flush-cache measures
+ // cache-cold (bench_serving's `random` prompts are deterministic).
benchmarkCommands: {
speed:
`python3 -m sglang.bench_serving \\
@@ -128,8 +212,8 @@ export const config = {
h200: "lmsysorg/sglang:qwen38-27b",
rtx6000: "lmsysorg/sglang:qwen38-27b",
rtx5090: "lmsysorg/sglang:qwen38-27b",
- // TODO: verify an arm64 build of this tag for DGX Spark (GB10 is aarch64);
- // the x86-only tag will not pull there.
+ // Multi-arch: this tag ships both linux/amd64 and linux/arm64, so it pulls
+ // natively on DGX Spark (GB10 is aarch64).
"dgx-spark": "lmsysorg/sglang:qwen38-27b",
gb300: "lmsysorg/sglang:dev",
},
@@ -140,13 +224,9 @@ export const config = {
playgroundFeatures: {
- // No "Attention Parallelism" card. The source page is single-GPU
- // throughout and no cell carries a parallelism flag, so there is nothing to
- // override: DP-Attention targets MLA models, prefill-CP has no model-side
- // integration for this architecture, and a TP knob would desync the ratio
- // calculator below (its geometry is TP1-only, so it would stop emitting and
- // the command would silently fall back to the 0.9 default this page warns
- // about). Re-add it together with TP-aware geometry in the calculator.
+ // No "Attention Parallelism" card: the page is single-GPU throughout,
+ // and a TP knob would desync the ratio calculator (TP1-only geometry).
+ // Re-add it together with TP-aware geometry in the calculator.
// ----- Card: "Parsers" -----
// Same parser pair the Qwen3.8 flagship page ships, and baked into every
@@ -175,8 +255,11 @@ export const config = {
flags: ["--speculative-algorithm EAGLE", "--speculative-num-steps 3",
"--speculative-eagle-topk 1", "--speculative-num-draft-tokens 4"] },
{ id: "dspark", label: "DSpark",
+ // Same three flags as the Deploy panel's DSPARK option, so the two
+ // paths compose identical commands.
flags: ["--speculative-algorithm DSPARK",
- "--speculative-draft-model-path RadixArk/Qwen3.8-27B-DSpark"] },
+ "--speculative-draft-model-path RadixArk/Qwen3.8-27B-DSpark",
+ "--speculative-draft-attention-backend flashinfer"] },
],
},
@@ -201,9 +284,9 @@ export const config = {
},
{
// Halving kv_bytes_per_token (65.5 KB bf16 -> 32.8 KB fp8) doubles the
- // KV pool at a fixed --mamba-full-memory-ratio. Accuracy-degrading over
- // a bf16-KV checkpoint, so it stays an opt-in and is never in a cell —
- // the NVFP4 checkpoint gets fp8 KV on its own via kv_cache_quant_algo.
+ // KV pool at a fixed --mamba-full-memory-ratio. Every deployment cell
+ // now pins fp8_e4m3, so this row is the opt-OUT: pick BFloat16 to undo
+ // it on the BF16/FP8 checkpoints, which carry no fp8 KV calibration.
id: "kvCacheDtype", title: "KV Cache Precision",
stripPrefixes: ["--kv-cache-dtype"],
options: [
@@ -252,21 +335,16 @@ export const config = {
],
},
- // Every cell is the source page's single launch command with only the model id
- // varying. The H200 and SM120 cells are `verified: true` at the maintainers'
- // direction; the DGX Spark cells are not, matching the unvalidated-on-SM121
- // note on those cells. The page carries no measured data of its own (no
- // `-benchmarks.jsx`), so a badge rests on validation held outside this page —
- // re-check it against the per-platform notes in §2 before trusting a cell.
+ // Verification: RTX 5090 / RTX PRO 6000 cells were measured across their
+ // whole overlay envelope; the h200/gb300 badges carry the source page's
+ // validation, which covers the overlay defaults (plus plain MTP on gb300) —
+ // non-default overlay picks there are valid but unmeasured. DGX Spark stays
+ // unverified (SM121 / aarch64 unvalidated).
//
- // Cells carry NO --mamba-full-memory-ratio. The source page's worked 4.6 held
- // only for the NVFP4 recipe at 4096-in/1024-out; the ratio is a function of
- // the workload, of S (radix-cache strategy / prefix cache), of D (spec) and of
- // kv_bytes_per_token, all of which the Playground can change. So the page's
- // ratio calculator computes it live from the effective config and broadcasts
- // it, and the engines pin it into the rendered command — which they only do
- // while the cell itself stays ratio-free (_deployment.jsx `cellWithRatio`).
- // Adding the flag back here would silently freeze the value again.
+ // Cells carry NO --mamba-full-memory-ratio: the ratio depends on workload,
+ // S, D and kv_bytes_per_token, so the page's calculator computes it live
+ // and the engine pins it into the rendered command — which it only does
+ // while the cell stays ratio-free (_deployment.jsx `cellWithRatio`).
cells: [
{
// H200 141GB, FP8 blockwise (~28.5GB of weights). The 32768-token chunk
@@ -276,12 +354,13 @@ export const config = {
// No NVFP4 cell on this card: SM90 has no FP4 tensor cores, so the W4A4
// checkpoint's MLP would fall back to the Marlin W4A16 weight-only path —
// runnable, but not a recipe this page ships.
- match: { hw: "h200", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" },
+ match: { hw: "h200", variant: "default", quant: "fp8", nodes: "single" },
verified: true,
env: [],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
+ "--kv-cache-dtype fp8_e4m3",
"--mem-fraction-static 0.85",
"--attention-backend flashinfer",
"--chunked-prefill-size 32768",
@@ -294,12 +373,13 @@ export const config = {
},
{
// H200, BF16 reference checkpoint (~54GB of weights).
- match: { hw: "h200", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" },
+ match: { hw: "h200", variant: "default", quant: "bf16", nodes: "single" },
verified: true,
env: [],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
+ "--kv-cache-dtype fp8_e4m3",
"--mem-fraction-static 0.85",
"--attention-backend flashinfer",
"--chunked-prefill-size 32768",
@@ -313,12 +393,13 @@ export const config = {
{
// The page's headline recipe: NVFP4 W4A4 on the 96GB workstation card,
// ~16.5GB of weights, fp8 KV auto-enabled by the checkpoint.
- match: { hw: "rtx6000", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" },
+ match: { hw: "rtx6000", variant: "default", quant: "nvfp4", nodes: "single" },
verified: true,
env: [],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
+ "--kv-cache-dtype fp8_e4m3",
"--mem-fraction-static 0.85",
"--attention-backend flashinfer",
"--chunked-prefill-size 2048",
@@ -330,12 +411,13 @@ export const config = {
},
{
// FP8 blockwise, ~28.5GB of weights — comfortable on 96GB.
- match: { hw: "rtx6000", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" },
+ match: { hw: "rtx6000", variant: "default", quant: "fp8", nodes: "single" },
verified: true,
env: [],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
+ "--kv-cache-dtype fp8_e4m3",
"--mem-fraction-static 0.85",
"--attention-backend flashinfer",
"--chunked-prefill-size 2048",
@@ -347,12 +429,13 @@ export const config = {
},
{
// BF16, the reference checkpoint.
- match: { hw: "rtx6000", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" },
+ match: { hw: "rtx6000", variant: "default", quant: "bf16", nodes: "single" },
verified: true,
env: [],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
+ "--kv-cache-dtype fp8_e4m3",
"--mem-fraction-static 0.85",
"--attention-backend flashinfer",
"--chunked-prefill-size 2048",
@@ -363,20 +446,32 @@ export const config = {
],
},
{
- // RTX 5090 32GB. NVFP4 is the only checkpoint that fits with room to
- // serve (~16.5GB); FP8 at ~28.5GB is not serviceable past bs<=2 and BF16
- // does not fit, so neither has a cell. On this card the GDN state pool —
- // not KV — bounds concurrency: lower S with the Playground's radix-cache
- // strategy (or turn the prefix cache off for S=1) and recompute the ratio.
- match: { hw: "rtx5090", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" },
+ // RTX 5090 32GB. NVFP4 is the only checkpoint that fits (FP8 does not
+ // boot — total_rest_memory negative at every mem-fraction, measured —
+ // and BF16 does not fit). Published operating point is ONE request in
+ // flight; --cuda-graph-max-bs 1 also protects the token pool (default
+ // capture set costs 39,247 -> 37,347 and K 8 -> 7). The `warn` below
+ // carries the user-facing guidance for raising concurrency.
+ match: { hw: "rtx5090", variant: "default", quant: "nvfp4", nodes: "single" },
verified: true,
+ // Rendered with the cell so nobody ships the bs=1 pins into a
+ // multi-user deployment unaware.
+ warn:
+ "This recipe serves ONE request at a time: --max-running-requests 1 " +
+ "and --cuda-graph-max-bs 1 pin it to the validated single-stream " +
+ "envelope. To handle more concurrent requests, raise both flags " +
+ "together and re-derive --mamba-full-memory-ratio (and mem-fraction) " +
+ "with the [Mamba ratio calculator](#mamba-ratio-calculator) — on this " +
+ "32GB card the GDN state pool, not KV, is what runs out first.",
env: [],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
- "--mem-fraction-static 0.85",
+ "--kv-cache-dtype fp8_e4m3",
+ "--mem-fraction-static 0.9",
"--attention-backend flashinfer",
- "--chunked-prefill-size 2048",
+ "--max-running-requests 1",
+ "--cuda-graph-max-bs 1",
"--reasoning-parser qwen3",
"--tool-call-parser qwen3_coder",
"--host {{HOST_IP}}",
@@ -390,11 +485,12 @@ export const config = {
// fraction, and prefill CUDA graphs disabled. Unvalidated on SM121 /
// aarch64.
{
- match: { hw: "dgx-spark", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" },
+ match: { hw: "dgx-spark", variant: "default", quant: "nvfp4", nodes: "single" },
env: [],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
+ "--kv-cache-dtype fp8_e4m3",
"--mem-fraction-static 0.95",
"--attention-backend flashinfer",
"--chunked-prefill-size 8192",
@@ -406,11 +502,12 @@ export const config = {
],
},
{
- match: { hw: "dgx-spark", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" },
+ match: { hw: "dgx-spark", variant: "default", quant: "fp8", nodes: "single" },
env: [],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
+ "--kv-cache-dtype fp8_e4m3",
"--mem-fraction-static 0.95",
"--attention-backend flashinfer",
"--chunked-prefill-size 8192",
@@ -422,11 +519,12 @@ export const config = {
],
},
{
- match: { hw: "dgx-spark", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" },
+ match: { hw: "dgx-spark", variant: "default", quant: "bf16", nodes: "single" },
env: [],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
+ "--kv-cache-dtype fp8_e4m3",
"--mem-fraction-static 0.95",
"--attention-backend flashinfer",
"--chunked-prefill-size 8192",
@@ -437,20 +535,18 @@ export const config = {
"--port {{PORT}}",
],
},
- // GB300 (SM103), single 288GB GPU. All six cells measured on a 4xGB300
- // devbox on 2026-08-14 against lmsysorg/sglang:dev @ c4271c3fe1262fc2adbd162c33b25de5255251c5.
- // With no --attention-backend pin, :dev on GB300 resolves attention to
- // triton (the newer c7c03ec resolves trtllm_mha); the cells keep engine-default
- // resolution so the benchmark card and the cell see the same kernel. The
- // `high-throughput` strategy adds the in-checkpoint MTP head
- // (EAGLE / NEXTN semantics, num-steps 3, topk 1, draft-tokens 4).
+ // GB300 (SM103), single 288GB GPU. Base and plain-MTP arms measured
+ // 2026-08-14 on lmsysorg/sglang:dev @ c4271c3fe (attention resolves to
+ // engine default — no pin, so cell and measurement see the same kernel).
+ // Verified envelope: spec none|eagle at engine-default tier/state dtype.
{
- match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" },
+ match: { hw: "gb300", variant: "default", quant: "nvfp4", nodes: "single" },
verified: true,
env: [],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
+ "--kv-cache-dtype fp8_e4m3",
"--mem-fraction-static 0.85",
"--chunked-prefill-size 2048",
"--reasoning-parser qwen3",
@@ -460,31 +556,13 @@ export const config = {
],
},
{
- match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" },
- verified: true,
- env: [],
- flags: [
- "--trust-remote-code",
- "--model-path {{MODEL_NAME}}",
- "--mem-fraction-static 0.85",
- "--chunked-prefill-size 2048",
- "--reasoning-parser qwen3",
- "--tool-call-parser qwen3_coder",
- "--speculative-algorithm EAGLE",
- "--speculative-num-steps 3",
- "--speculative-eagle-topk 1",
- "--speculative-num-draft-tokens 4",
- "--host {{HOST_IP}}",
- "--port {{PORT}}",
- ],
- },
- {
- match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" },
+ match: { hw: "gb300", variant: "default", quant: "fp8", nodes: "single" },
verified: true,
env: [],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
+ "--kv-cache-dtype fp8_e4m3",
"--mem-fraction-static 0.85",
"--chunked-prefill-size 2048",
"--reasoning-parser qwen3",
@@ -494,54 +572,17 @@ export const config = {
],
},
{
- match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" },
+ match: { hw: "gb300", variant: "default", quant: "bf16", nodes: "single" },
verified: true,
env: [],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
+ "--kv-cache-dtype fp8_e4m3",
"--mem-fraction-static 0.85",
"--chunked-prefill-size 2048",
"--reasoning-parser qwen3",
"--tool-call-parser qwen3_coder",
- "--speculative-algorithm EAGLE",
- "--speculative-num-steps 3",
- "--speculative-eagle-topk 1",
- "--speculative-num-draft-tokens 4",
- "--host {{HOST_IP}}",
- "--port {{PORT}}",
- ],
- },
- {
- match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" },
- verified: true,
- env: [],
- flags: [
- "--trust-remote-code",
- "--model-path {{MODEL_NAME}}",
- "--mem-fraction-static 0.85",
- "--chunked-prefill-size 2048",
- "--reasoning-parser qwen3",
- "--tool-call-parser qwen3_coder",
- "--host {{HOST_IP}}",
- "--port {{PORT}}",
- ],
- },
- {
- match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" },
- verified: true,
- env: [],
- flags: [
- "--trust-remote-code",
- "--model-path {{MODEL_NAME}}",
- "--mem-fraction-static 0.85",
- "--chunked-prefill-size 2048",
- "--reasoning-parser qwen3",
- "--tool-call-parser qwen3_coder",
- "--speculative-algorithm EAGLE",
- "--speculative-num-steps 3",
- "--speculative-eagle-topk 1",
- "--speculative-num-draft-tokens 4",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],