minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[Cookbook] Add NVFP4 options for DeepSeek-V4 Flash Official (0731) and Pro Official (0813) (#37351)

Browse Source 
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
zijiexia
2026-09-01 08:03:39 +00:00
committed by GitHub
co-authored by Claude Fable 5
parent 03b33cbe5d
commit 379e33d87e
3 changed files with 189 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx
+9 -1
View File
@@ -311,7 +311,10 @@ sglang serve \
The [`nvidia/DeepSeek-V4-Pro-NVFP4`](https://huggingface.co/nvidia/DeepSeek-V4-Pro-NVFP4) and
[`nvidia/DeepSeek-V4-Flash-NVFP4`](https://huggingface.co/nvidia/DeepSeek-V4-Flash-NVFP4) checkpoints
quantize MoE experts to **NVFP4** while keeping attention and dense layers in
**FP8**. It requires `--moe-runner-backend flashinfer_trtllm_routed` which will be automatically selected if not provided.
**FP8**. The official releases have matching NVFP4 checkpoints at
[`nvidia/DeepSeek-V4-Flash-0731-NVFP4`](https://huggingface.co/nvidia/DeepSeek-V4-Flash-0731-NVFP4) and
[`nvidia/DeepSeek-V4-Pro-0813-NVFP4`](https://huggingface.co/nvidia/DeepSeek-V4-Pro-0813-NVFP4).
All of them require `--moe-runner-backend flashinfer_trtllm_routed` which will be automatically selected if not provided.
```bash Command
sglang serve \
@@ -331,6 +334,11 @@ Requires Blackwell (SM100+). The MTP layer in this checkpoint stays
MXFP4-packed and is routed through the `Mxfp4FlashinferTrtllmMoEMethod` path
automatically.
The official (0731 / 0813) NVFP4 checkpoints preserve the bundled DSpark draft
head, so their low-latency recipes use `--speculative-algorithm DSPARK` instead
of the EAGLE/MTP shape flags — same as the corresponding original-precision
official checkpoints.
<a id="hopper-note" />
**Hopper (H100 / H200) note**
docs/src/snippets/configs/deepseek-ai/deepseek-v4-benchmarks.jsx
+10
View File
@@ -119,6 +119,16 @@ export const benchmarks = [
ttft_ms: 509, tpot_ms: 11.13, tokens_per_sec_per_gpu: 1210 },
],
},
{
match: { hw: "b200", variant: "flash-official", quant: "nvfp4", strategy: "low-latency", nodes: "single" },
sglang_version: "dev@07c8f7294",
accuracy: { gsm8k_pct: 96.82, aime25_pct: 98.96 },
},
{
match: { hw: "b200", variant: "pro-official", quant: "nvfp4", strategy: "low-latency", nodes: "single" },
sglang_version: "dev@07c8f7294",
accuracy: { gsm8k_pct: 96.44, aime25_pct: 98.33 },
},
// ====================================================================
// B300 + FP4
// ====================================================================
docs/src/snippets/configs/deepseek-ai/deepseek-v4.jsx
+170 -1
View File
@@ -50,11 +50,13 @@ export const config = {
"flash|fp8": "deepseek-ai/DeepSeek-V4-Flash",
"flash|nvfp4": "nvidia/DeepSeek-V4-Flash-NVFP4",
"flash-official|fp4": "deepseek-ai/DeepSeek-V4-Flash-0731",
"flash-official|nvfp4": "nvidia/DeepSeek-V4-Flash-0731-NVFP4",
"flash-vision|fp4": "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp",
"pro|fp4": "deepseek-ai/DeepSeek-V4-Pro",
"pro|fp8": "deepseek-ai/DeepSeek-V4-Pro",
"pro|nvfp4": "nvidia/DeepSeek-V4-Pro-NVFP4",
"pro-official|fp4": "deepseek-ai/DeepSeek-V4-Pro-0813",
"pro-official|nvfp4": "nvidia/DeepSeek-V4-Pro-0813-NVFP4",
// H200 FP8 needs the sgl-project repackaging (Hopper can't run FP4-mixed Instruct).
"h200|flash|fp8": "sgl-project/DeepSeek-V4-Flash-FP8",
"h200|pro|fp8": "sgl-project/DeepSeek-V4-Pro-FP8",
@@ -190,6 +192,13 @@ sgl-eval run mmmu_pro \\
// (sgl-project/sglang#37253) — until it does, the variant needs this
// preview build on every hardware.
"flash-vision|fp4": "lmsysorg/sglang:dev-dsv4-flash-vision",
// NVFP4 checkpoints crash at weight load on v0.5.18 (the MXFP4-packed MTP
// layer's FP8 delegate needs the #36275 guard, merged 2026-08-26) — route
// every NVFP4 cell to the nightly until a release contains that fix.
"b200|nvfp4": "lmsysorg/sglang:dev",
"b300|nvfp4": "lmsysorg/sglang:dev",
"gb200|nvfp4": "lmsysorg/sglang:dev",
"gb300|nvfp4": "lmsysorg/sglang:dev",
h100: "lmsysorg/sglang:latest",
h200: "lmsysorg/sglang:latest",
b200: "lmsysorg/sglang:latest",
@@ -850,6 +859,46 @@ sgl-eval run mmmu_pro \\
],
},
// ====================================================================
// B200 + NVFP4 — Official (0731 / 0813)
// Mirrors the Flash/Pro NVFP4 cells; the official checkpoints bundle a
// DSpark draft head, so low-latency uses `--speculative-algorithm DSPARK`
// instead of the EAGLE shape flags. Verified on 8xB200 (GSM8K + AIME25,
// sgl-eval; see the benchmarks entries).
// ====================================================================
{
match: { hw: "b200", variant: "flash-official", quant: "nvfp4", strategy: "low-latency", nodes: "single" },
verified: true,
env: [],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
"--tp 4",
"--moe-runner-backend flashinfer_trtllm_routed",
"--speculative-algorithm DSPARK",
"--disable-flashinfer-autotune",
"--swa-full-tokens-ratio 0.1",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
},
{
match: { hw: "b200", variant: "pro-official", quant: "nvfp4", strategy: "low-latency", nodes: "single" },
verified: true,
env: [],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
"--tp 8",
"--moe-runner-backend flashinfer_trtllm_routed",
"--speculative-algorithm DSPARK",
"--chunked-prefill-size 8192",
"--disable-flashinfer-autotune",
"--swa-full-tokens-ratio 0.1",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
},
// ====================================================================
// B300 + NVFP4
// ====================================================================
{
@@ -892,6 +941,46 @@ sgl-eval run mmmu_pro \\
"--port {{PORT}}",
],
},
// ====================================================================
// B300 + NVFP4 — Official (0731 / 0813)
// Mirrors the Flash/Pro NVFP4 cells; the official checkpoints bundle a
// DSpark draft head, so low-latency uses `--speculative-algorithm DSPARK`
// instead of the EAGLE shape flags. NOT yet run end-to-end on this hardware.
// ====================================================================
{
match: { hw: "b300", variant: "flash-official", quant: "nvfp4", strategy: "low-latency", nodes: "single" },
verificationStatus: "in-progress",
env: [],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
"--tp 4",
"--moe-runner-backend flashinfer_trtllm_routed",
"--speculative-algorithm DSPARK",
"--disable-flashinfer-autotune",
"--swa-full-tokens-ratio 0.1",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
},
{
match: { hw: "b300", variant: "pro-official", quant: "nvfp4", strategy: "low-latency", nodes: "single" },
verificationStatus: "in-progress",
env: [],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
"--tp 8",
"--moe-runner-backend flashinfer_trtllm_routed",
"--speculative-algorithm DSPARK",
"--chunked-prefill-size 8192",
"--disable-flashinfer-autotune",
"--swa-full-tokens-ratio 0.1",
"--mem-fraction-static 0.90",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
},
// ====================================================================
// GB200 + FP4
@@ -1118,6 +1207,46 @@ sgl-eval run mmmu_pro \\
"--port {{PORT}}",
],
},
// ====================================================================
// GB200 + NVFP4 — Official (0731 / 0813)
// Mirrors the Flash/Pro NVFP4 cells; the official checkpoints bundle a
// DSpark draft head, so low-latency uses `--speculative-algorithm DSPARK`
// instead of the EAGLE shape flags. NOT yet run end-to-end on this hardware.
// ====================================================================
{
match: { hw: "gb200", variant: "flash-official", quant: "nvfp4", strategy: "low-latency", nodes: "single" },
verificationStatus: "in-progress",
env: [],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
"--tp 4",
"--moe-runner-backend flashinfer_trtllm_routed",
"--speculative-algorithm DSPARK",
"--disable-flashinfer-autotune",
"--swa-full-tokens-ratio 0.1",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
},
{
match: { hw: "gb200", variant: "pro-official", quant: "nvfp4", strategy: "low-latency", nodes: "multi-2" },
verificationStatus: "in-progress",
env: [],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
"--tp 8",
"--moe-runner-backend flashinfer_trtllm_routed",
"--speculative-algorithm DSPARK",
"--chunked-prefill-size 8192",
"--disable-flashinfer-autotune",
"--swa-full-tokens-ratio 0.1",
"--mem-fraction-static 0.90",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
},
// ====================================================================
// GB300 + FP4
@@ -1722,7 +1851,7 @@ sgl-eval run mmmu_pro \\
},
// ====================================================================
// GB200 + NVFP4
// GB300 + NVFP4
// ====================================================================
{
match: { hw: "gb300", variant: "flash", quant: "nvfp4", strategy: "low-latency", nodes: "single" },
@@ -1764,6 +1893,46 @@ sgl-eval run mmmu_pro \\
"--port {{PORT}}",
],
},
// ====================================================================
// GB300 + NVFP4 — Official (0731 / 0813)
// Mirrors the Flash/Pro NVFP4 cells; the official checkpoints bundle a
// DSpark draft head, so low-latency uses `--speculative-algorithm DSPARK`
// instead of the EAGLE shape flags. NOT yet run end-to-end on this hardware.
// ====================================================================
{
match: { hw: "gb300", variant: "flash-official", quant: "nvfp4", strategy: "low-latency", nodes: "single" },
verificationStatus: "in-progress",
env: [],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
"--tp 4",
"--moe-runner-backend flashinfer_trtllm_routed",
"--speculative-algorithm DSPARK",
"--disable-flashinfer-autotune",
"--swa-full-tokens-ratio 0.1",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
},
{
match: { hw: "gb300", variant: "pro-official", quant: "nvfp4", strategy: "low-latency", nodes: "single" },
verificationStatus: "in-progress",
env: [],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
"--tp 4",
"--moe-runner-backend flashinfer_trtllm_routed",
"--speculative-algorithm DSPARK",
"--chunked-prefill-size 8192",
"--disable-flashinfer-autotune",
"--swa-full-tokens-ratio 0.1",
"--mem-fraction-static 0.90",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
},
// ====================================================================
// H200 + FP8 (deepep, no Marlin)