From 3efa0574496b53984bad2d485573a9400e013bdc Mon Sep 17 00:00:00 2001 From: Jimmy Shong <69131491+Jiminator@users.noreply.github.com> Date: Thu, 20 Aug 2026 21:54:51 -0700 Subject: [PATCH] [docs] Retune the Qwen3.8-27B RTX 5090 DFLASH2 cells against 1cf2b8c (#35786) --- .../autoregressive/Qwen/Qwen3.8-27B.mdx | 56 +++++++++++-------- .../src/snippets/configs/Qwen/qwen3.8-27b.jsx | 52 ++++++++++++----- 2 files changed, 71 insertions(+), 37 deletions(-) diff --git a/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx b/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx index 66da53822..f098ba7e5 100644 --- a/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx +++ b/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx @@ -22,7 +22,11 @@ pip install uv uv pip install sglang # For the DFLASH2 cells only — DFlash2 selector support is newer than the -# latest release, so install from source (Method 2) instead of the line above. +# latest release, so build from the commit those cells were validated on +# instead of the line above: +# git clone https://github.com/sgl-project/sglang.git && cd sglang +# git checkout 1cf2b8c54d81802abc15dcf23a29b9cc687bc01e # PR #35496 +# uv pip install -e "python[all]" ``` Then run the **Python** output of the command panel below in that environment. @@ -34,9 +38,11 @@ Then run the **Python** output of the command panel below in that environment. ```bash Command docker pull lmsysorg/sglang:qwen38-27b -# For the DFLASH2 cells only — that tag predates DFlash2 selector support, -# so pull a nightly built from main instead: -# docker pull lmsysorg/sglang:dev +# For the DFLASH2 cells only — that tag predates DFlash2 selector support. +# Build the image from the commit those cells were validated on instead: +# git clone https://github.com/sgl-project/sglang.git && cd sglang +# git checkout 1cf2b8c54d81802abc15dcf23a29b9cc687bc01e # PR #35496 +# docker build -t sglang:dflash2 -f docker/Dockerfile . ``` For how to launch the image, see [Install → Method 3: Using Docker](../../../docs/get-started/install#method-3-using-docker). Substitute the inner `sglang serve ...` with what the command generator below produces. @@ -46,14 +52,16 @@ For how to launch the image, see [Install → Method 3: Using Docker](../../../d -**DFLASH2 needs a build that tracks main.** DFlash2 landed in -[#35371](https://github.com/sgl-project/sglang/pull/35371), and DFlash2 + NVFP4 -— the quantized `lm_head` path — in -[#35496](https://github.com/sgl-project/sglang/pull/35496). Both are newer than -the pinned `lmsysorg/sglang:qwen38-27b` tag and than the latest PyPI release; a -build without #35496 fails on the NVFP4 cells at boot with `requires a dense -FP16/BF16/FP32 target lm_head`. Every other recipe on this page — no -speculation, MTP, DSpark — runs on the pinned tag as written. +**DFLASH2 needs a build from `1cf2b8c` (PR +[#35496](https://github.com/sgl-project/sglang/pull/35496)) or newer.** DFlash2 +landed in [#35371](https://github.com/sgl-project/sglang/pull/35371) and its +quantized-`lm_head` path — what the NVFP4 cells need — in #35496, both newer +than the pinned `lmsysorg/sglang:qwen38-27b` tag and than the latest PyPI +release. A build without #35496 fails on the NVFP4 cells at boot with `requires +a dense FP16/BF16/FP32 target lm_head`. The DFLASH2 pins on this page were +measured on `1cf2b8c` exactly, which is why both install paths above check that +commit out rather than tracking a moving branch. Every other recipe on this +page — no speculation, MTP, DSpark — runs on the pinned tag as written. @@ -269,18 +277,18 @@ checkpoint's calibration scales automatically. RTX PRO 6000 BF16/FP8 cells boot and serve; on H200, DGX Spark and GB300 those cells carry the **Final Verification In Progress** badge. The - RTX PRO 6000 recipe needs no changes. On the 32GB RTX 5090 prefer - `--mamba-ssm-dtype bfloat16` at `--mem-fraction-static 0.90`: measured - strictly better than float32 for this draft (6.1 vs 8.3 ms TPOT, accept - 3.30 vs 3.09) — the opposite of the EAGLE trade, so measure before assuming. - float32 still fits, but only at `--mem-fraction-static 0.945` with - `--mamba-full-memory-ratio 10` pinned in place of the balanced value: the - L = 9216 ratio leaves the fp32 state pool one slot short at every - serviceable mem-fraction (0.94 allocates four of Low-Latency's five slots; - 0.95 OOMs at runtime), and the re-weighted split leaves the Low-Latency KV - pool a single-request envelope (~9.4k tokens) — no headroom for longer - requests or radix reuse. The panel's DFLASH2 option applies these re-pins - automatically. + RTX PRO 6000 recipe needs no changes. On the 32GB RTX 5090 the pins differ per + state dtype, and the panel applies them automatically. bfloat16 serves at + `--mem-fraction-static 0.88` on the balanced ratio (0.90, DSpark's pin, OOMs + on the first request). float32 reaches only the High-Throughput tier, at + `--mem-fraction-static 0.895` with `--mamba-full-memory-ratio 10` overriding + the balanced value — these cells pin `--max-running-requests 1`, so the + balanced ratio provisions KV for concurrency the recipe never uses and + starves the state pool of the slots fp32 needs. The Low-Latency tier is + greyed out for float32: it needs five fp32 slots plus a full request's KV, + and no mem-fraction holds both — buying the fifth slot cuts KV to 7,752 + tokens against the 9,216 one 8192/1024 request needs, while every + mem-fraction with a large enough pool dies in prefill graph capture. - **Hardware fit**: FP8 weights ~28.5GB (not serviceable beyond bs≤2 on 32GB cards); NVFP4 weights ~16.5GB (recommended for RTX 5090-class GPUs). - `--mamba-radix-cache-strategy extra_buffer_lazy` lowers the state cost per diff --git a/docs/src/snippets/configs/Qwen/qwen3.8-27b.jsx b/docs/src/snippets/configs/Qwen/qwen3.8-27b.jsx index 482de8a17..2826ce1b1 100644 --- a/docs/src/snippets/configs/Qwen/qwen3.8-27b.jsx +++ b/docs/src/snippets/configs/Qwen/qwen3.8-27b.jsx @@ -128,10 +128,8 @@ export const config = { disabled: (sel) => sel.hw === "rtx5090" && sel.quant !== "nvfp4", disableReason: "On the 32GB RTX 5090 the DFlash2 draft model only fits on top of the NVFP4 weights", - // 5090: mem-fraction re-pins like DSPARK's, and fp32 additionally - // re-pins the ratio — the balanced L=9216 value leaves the state - // pool one slot short at every serviceable mem-fraction (see the - // DFlash2 bullet in Configuration Tips). + // fp32 is the one case that needs the balanced ratio overridden, so + // that family is stripped too and re-emitted below. stripPrefixes: (sel) => sel.hw === "rtx5090" ? sel.ssmDtype === "float32" @@ -142,15 +140,19 @@ export const config = { "--speculative-algorithm DFLASH", "--speculative-draft-model-path incoai/Qwen3.8-27B-DFlash2", "--speculative-num-draft-tokens 8", - // Measured on the 5090: bf16 state serves at 0.90 (DSPARK's - // pin); fp32 fits only at 0.945 + ratio 10 (0.94 is one state - // slot short, 0.95 OOMs at runtime) and leaves the Low-Latency - // KV pool a single-request envelope. + // Measured on the 5090 at commit 1cf2b8c, the build the Install + // accordion pins for this pick. bf16 serves at 0.88 on the balanced + // ratio (0.90, DSPARK's pin, OOMs on the first request). fp32 needs + // 0.895 AND the balanced ratio overridden to 10: these cells pin + // --max-running-requests 1, so the balanced value provisions KV for + // concurrency this recipe never uses, starving the state pool of the + // slots fp32 needs. Only High-Throughput reaches fp32 (S=4); the SSM + // dtype row greys fp32 out for Low-Latency (S=5). ...(sel.hw === "rtx5090" ? sel.ssmDtype === "float32" - ? ["--mem-fraction-static 0.945", + ? ["--mem-fraction-static 0.895", "--mamba-full-memory-ratio 10"] - : ["--mem-fraction-static 0.90"] + : ["--mem-fraction-static 0.88"] : []), ], }, @@ -186,9 +188,33 @@ export const config = { title: "Mamba SSM Dtype", default: "float32", options: [ - // Open on every platform, including with DSPARK on the 5090 (serves - // at mem-fraction 0.92 with the engine-default 2048 prefill chunk). - { id: "float32", label: "float32", flags: ["--mamba-ssm-dtype float32"] }, + // Open on every platform except the 32GB RTX 5090 under DFLASH2: + // there the fp32 state pool and the prefill CUDA-graph capture cannot + // both fit, at any mem-fraction. Measured on main (2026-08-21, ratio + // pinned at 10 so slots are not the binding term): 0.945 and 0.92 OOM + // inside `Capture target prefill CUDA graph`, 0.90 OOMs on the first + // request, and 0.88 / 0.86 / 0.84 size the state pool below the 5 (LL) + // / 4 (HT) slots one request needs. bf16 state halves the pool and + // serves, and is the faster cell there anyway. + { + id: "float32", label: "float32", + // Only the Low-Latency tier is out of reach: it needs S=5 fp32 slots + // (735MB) plus >=9216 KV tokens for one request, and no mem-fraction + // holds both -- at 0.8975/r14 the pool buys the 5th slot but KV falls + // to 7752 tokens and generation stops after one token, while every + // mem-fraction with a big enough pool (>=0.90) dies in graph capture. + // High-Throughput needs one slot fewer and does fit; see the DFLASH2 + // option's pins. + disabled: (sel) => + sel.hw === "rtx5090" && + sel.spec === "dflash" && + sel.tier === "low-latency", + disableReason: + "On the 32GB RTX 5090 the Low-Latency tier cannot hold five fp32 state " + + "slots and a full request's KV at once — use bfloat16, or the " + + "High-Throughput tier which fits fp32", + flags: ["--mamba-ssm-dtype float32"], + }, { id: "bfloat16", label: "bfloat16", disabled: (sel) => sel.hw === "rtx5090" && sel.quant !== "nvfp4",