From d9f6861359cebf4d9e8c494f7992aa0c07eff863 Mon Sep 17 00:00:00 2001 From: Jimmy Shong <69131491+Jiminator@users.noreply.github.com> Date: Thu, 20 Aug 2026 13:26:55 -0700 Subject: [PATCH] [docs] Add DFlash2 speculative cells to the Qwen3.8-27B cookbook (#35663) --- .../autoregressive/Qwen/Qwen3.8-27B.mdx | 34 +++++++++++-- .../src/snippets/configs/Qwen/qwen3.8-27b.jsx | 48 +++++++++++++++++++ 2 files changed, 79 insertions(+), 3 deletions(-) diff --git a/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx b/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx index 7b7078562..4d2a705bc 100644 --- a/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx +++ b/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx @@ -59,8 +59,12 @@ import { Qwen38MambaRatioCalculator } from "/src/snippets/_qwen38_mamba_ratio_ca The RTX 5090 and RTX PRO 6000 cells above — including every Speculative Decoding / Serving Strategy / SSM dtype combination — were validated at - ISL 8192 / OSL 1024, concurrency 1. The DGX Spark cells cover that same full - combination set, but to a weaker standard: each was confirmed to **boot and + ISL 8192 / OSL 1024, concurrency 1 — for DFLASH2, to that full standard on + NVFP4, and to boot-and-serve on the RTX PRO 6000 BF16/FP8 cells. On the + remaining platforms the DFLASH2 pick is offered but not yet exercised, and + the composed command carries a `# DFLASH2 on this platform: final + verification in progress` hint line while it is selected. The DGX Spark cells cover that same combination set minus + DFLASH2, but to a weaker standard: each was confirmed to **boot and serve** at ISL 8192 / OSL 1024, concurrency 1, with no throughput or acceptance-length numbers taken. The remaining platforms' recipes carry their original validation, which covers the default overlay picks (plus MTP on @@ -89,7 +93,8 @@ ratio = (S + D) x state_bytes / (L x kv_bytes_per_token) slot, and `extra_buffer` frees one more with the overlap scheduler off; the calculator reads both knobs. - `D` — verify intermediate states under speculative decoding: - `--speculative-num-draft-tokens` for EAGLE/MTP (4 at the recommended 3/1/4); + `--speculative-num-draft-tokens` for EAGLE/MTP (4 at the recommended 3/1/4) + and for DFLASH (8, DFlash2's block size); `--speculative-dspark-block-size + 1` for DSPARK, where the block size falls back to the draft checkpoint's `block_size` when the flag is omitted (7 for `RadixArk/Qwen3.8-27B-DSpark`, so `D = 8`); 0 with speculation off or with @@ -220,6 +225,29 @@ checkpoint's calibration scales automatically. `--enable-linear-replayssm-spec` its draft intermediates move onto a fixed ring, so `D = 0` and the ratio returns to the no-spec value. The [calculator](#mamba-ratio-calculator) applies both rules. +- **DFlash2**: a trained block-diffusion draft in a separate checkpoint — add + `--speculative-algorithm DFLASH --speculative-draft-model-path + incoai/Qwen3.8-27B-DFlash2 --speculative-num-draft-tokens 8` (8 is the + draft's block size, and it is the `D` term in the ratio, same value as + DSpark's). The selector projects candidates through the target `lm_head`, + including quantized heads, so it runs on the NVFP4 checkpoint (whose head + is NVFP4-packed; the BF16 and FP8 checkpoints keep a dense head). + Validation: NVFP4 measured end-to-end on RTX PRO 6000 and RTX 5090; the + RTX PRO 6000 BF16/FP8 cells boot and serve; on H200, DGX Spark and GB300 + the pick is offered with a final-verification-in-progress hint line + above the command. The + RTX PRO 6000 recipe needs no changes. On the 32GB RTX 5090 prefer + `--mamba-ssm-dtype bfloat16` at `--mem-fraction-static 0.90`: measured + strictly better than float32 for this draft (6.1 vs 8.3 ms TPOT, accept + 3.30 vs 3.09) — the opposite of the EAGLE trade, so measure before assuming. + float32 still fits, but only at `--mem-fraction-static 0.945` with + `--mamba-full-memory-ratio 10` pinned in place of the balanced value: the + L = 9216 ratio leaves the fp32 state pool one slot short at every + serviceable mem-fraction (0.94 allocates four of Low-Latency's five slots; + 0.95 OOMs at runtime), and the re-weighted split leaves the Low-Latency KV + pool a single-request envelope (~9.4k tokens) — no headroom for longer + requests or radix reuse. The panel's DFLASH2 option applies these re-pins + automatically. - **Hardware fit**: FP8 weights ~28.5GB (not serviceable beyond bs≤2 on 32GB cards); NVFP4 weights ~16.5GB (recommended for RTX 5090-class GPUs). - `--mamba-radix-cache-strategy extra_buffer_lazy` lowers the state cost per diff --git a/docs/src/snippets/configs/Qwen/qwen3.8-27b.jsx b/docs/src/snippets/configs/Qwen/qwen3.8-27b.jsx index 4af730961..c681499db 100644 --- a/docs/src/snippets/configs/Qwen/qwen3.8-27b.jsx +++ b/docs/src/snippets/configs/Qwen/qwen3.8-27b.jsx @@ -116,6 +116,49 @@ export const config = { : []), ], }, + { + id: "dflash", label: "DFLASH2", + // Trained block-diffusion draft, a separate checkpoint. The + // selector projects through the target lm_head — including + // quantized heads — so it runs on the NVFP4 checkpoint too. + // Validated on the SM120 pair (NVFP4 measured end to end; the + // RTX PRO 6000 BF16/FP8 cells boot-and-serve). On the other + // platforms the pick is offered with the in-progress hint below + // — the recipe composes from validated cells but has not been + // exercised there yet. + disabled: (sel) => sel.hw === "rtx5090" && sel.quant !== "nvfp4", + disableReason: + "On the 32GB RTX 5090 the DFlash2 draft model only fits on top of the NVFP4 weights", + hints: (sel) => + ["rtx6000", "rtx5090"].includes(sel.hw) + ? [] + : ["DFLASH2 on this platform: final verification in progress"], + // 5090: mem-fraction re-pins like DSPARK's, and fp32 additionally + // re-pins the ratio — the balanced L=9216 value leaves the state + // pool one slot short at every serviceable mem-fraction (see the + // DFlash2 bullet in Configuration Tips). + stripPrefixes: (sel) => + sel.hw === "rtx5090" + ? sel.ssmDtype === "float32" + ? ["--mem-fraction-static", "--mamba-full-memory-ratio"] + : ["--mem-fraction-static"] + : [], + flags: (sel) => [ + "--speculative-algorithm DFLASH", + "--speculative-draft-model-path incoai/Qwen3.8-27B-DFlash2", + "--speculative-num-draft-tokens 8", + // Measured on the 5090: bf16 state serves at 0.90 (DSPARK's + // pin); fp32 fits only at 0.945 + ratio 10 (0.94 is one state + // slot short, 0.95 OOMs at runtime) and leaves the Low-Latency + // KV pool a single-request envelope. + ...(sel.hw === "rtx5090" + ? sel.ssmDtype === "float32" + ? ["--mem-fraction-static 0.945", + "--mamba-full-memory-ratio 10"] + : ["--mem-fraction-static 0.90"] + : []), + ], + }, ], }, { @@ -260,6 +303,11 @@ export const config = { flags: ["--speculative-algorithm DSPARK", "--speculative-draft-model-path RadixArk/Qwen3.8-27B-DSpark", "--speculative-draft-attention-backend flashinfer"] }, + { id: "dflash", label: "DFlash2", + // Same trio as the Deploy panel's DFLASH2 option. + flags: ["--speculative-algorithm DFLASH", + "--speculative-draft-model-path incoai/Qwen3.8-27B-DFlash2", + "--speculative-num-draft-tokens 8"] }, ], },