diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx index 3ada73b40..bafc0b5b1 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx @@ -311,7 +311,10 @@ sglang serve \ The [`nvidia/DeepSeek-V4-Pro-NVFP4`](https://huggingface.co/nvidia/DeepSeek-V4-Pro-NVFP4) and [`nvidia/DeepSeek-V4-Flash-NVFP4`](https://huggingface.co/nvidia/DeepSeek-V4-Flash-NVFP4) checkpoints quantize MoE experts to **NVFP4** while keeping attention and dense layers in -**FP8**. It requires `--moe-runner-backend flashinfer_trtllm_routed` which will be automatically selected if not provided. +**FP8**. The official releases have matching NVFP4 checkpoints at +[`nvidia/DeepSeek-V4-Flash-0731-NVFP4`](https://huggingface.co/nvidia/DeepSeek-V4-Flash-0731-NVFP4) and +[`nvidia/DeepSeek-V4-Pro-0813-NVFP4`](https://huggingface.co/nvidia/DeepSeek-V4-Pro-0813-NVFP4). +All of them require `--moe-runner-backend flashinfer_trtllm_routed` which will be automatically selected if not provided. ```bash Command sglang serve \ @@ -331,6 +334,11 @@ Requires Blackwell (SM100+). The MTP layer in this checkpoint stays MXFP4-packed and is routed through the `Mxfp4FlashinferTrtllmMoEMethod` path automatically. +The official (0731 / 0813) NVFP4 checkpoints preserve the bundled DSpark draft +head, so their low-latency recipes use `--speculative-algorithm DSPARK` instead +of the EAGLE/MTP shape flags — same as the corresponding original-precision +official checkpoints. + **Hopper (H100 / H200) note** diff --git a/docs/src/snippets/configs/deepseek-ai/deepseek-v4-benchmarks.jsx b/docs/src/snippets/configs/deepseek-ai/deepseek-v4-benchmarks.jsx index cd702dda8..db4604f95 100644 --- a/docs/src/snippets/configs/deepseek-ai/deepseek-v4-benchmarks.jsx +++ b/docs/src/snippets/configs/deepseek-ai/deepseek-v4-benchmarks.jsx @@ -119,6 +119,16 @@ export const benchmarks = [ ttft_ms: 509, tpot_ms: 11.13, tokens_per_sec_per_gpu: 1210 }, ], }, + { + match: { hw: "b200", variant: "flash-official", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + sglang_version: "dev@07c8f7294", + accuracy: { gsm8k_pct: 96.82, aime25_pct: 98.96 }, + }, + { + match: { hw: "b200", variant: "pro-official", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + sglang_version: "dev@07c8f7294", + accuracy: { gsm8k_pct: 96.44, aime25_pct: 98.33 }, + }, // ==================================================================== // B300 + FP4 // ==================================================================== diff --git a/docs/src/snippets/configs/deepseek-ai/deepseek-v4.jsx b/docs/src/snippets/configs/deepseek-ai/deepseek-v4.jsx index f00dd8f2a..8111c2e63 100644 --- a/docs/src/snippets/configs/deepseek-ai/deepseek-v4.jsx +++ b/docs/src/snippets/configs/deepseek-ai/deepseek-v4.jsx @@ -50,11 +50,13 @@ export const config = { "flash|fp8": "deepseek-ai/DeepSeek-V4-Flash", "flash|nvfp4": "nvidia/DeepSeek-V4-Flash-NVFP4", "flash-official|fp4": "deepseek-ai/DeepSeek-V4-Flash-0731", + "flash-official|nvfp4": "nvidia/DeepSeek-V4-Flash-0731-NVFP4", "flash-vision|fp4": "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp", "pro|fp4": "deepseek-ai/DeepSeek-V4-Pro", "pro|fp8": "deepseek-ai/DeepSeek-V4-Pro", "pro|nvfp4": "nvidia/DeepSeek-V4-Pro-NVFP4", "pro-official|fp4": "deepseek-ai/DeepSeek-V4-Pro-0813", + "pro-official|nvfp4": "nvidia/DeepSeek-V4-Pro-0813-NVFP4", // H200 FP8 needs the sgl-project repackaging (Hopper can't run FP4-mixed Instruct). "h200|flash|fp8": "sgl-project/DeepSeek-V4-Flash-FP8", "h200|pro|fp8": "sgl-project/DeepSeek-V4-Pro-FP8", @@ -190,6 +192,13 @@ sgl-eval run mmmu_pro \\ // (sgl-project/sglang#37253) — until it does, the variant needs this // preview build on every hardware. "flash-vision|fp4": "lmsysorg/sglang:dev-dsv4-flash-vision", + // NVFP4 checkpoints crash at weight load on v0.5.18 (the MXFP4-packed MTP + // layer's FP8 delegate needs the #36275 guard, merged 2026-08-26) — route + // every NVFP4 cell to the nightly until a release contains that fix. + "b200|nvfp4": "lmsysorg/sglang:dev", + "b300|nvfp4": "lmsysorg/sglang:dev", + "gb200|nvfp4": "lmsysorg/sglang:dev", + "gb300|nvfp4": "lmsysorg/sglang:dev", h100: "lmsysorg/sglang:latest", h200: "lmsysorg/sglang:latest", b200: "lmsysorg/sglang:latest", @@ -850,6 +859,46 @@ sgl-eval run mmmu_pro \\ ], }, // ==================================================================== + // B200 + NVFP4 — Official (0731 / 0813) + // Mirrors the Flash/Pro NVFP4 cells; the official checkpoints bundle a + // DSpark draft head, so low-latency uses `--speculative-algorithm DSPARK` + // instead of the EAGLE shape flags. Verified on 8xB200 (GSM8K + AIME25, + // sgl-eval; see the benchmarks entries). + // ==================================================================== + { + match: { hw: "b200", variant: "flash-official", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 4", + "--moe-runner-backend flashinfer_trtllm_routed", + "--speculative-algorithm DSPARK", + "--disable-flashinfer-autotune", + "--swa-full-tokens-ratio 0.1", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b200", variant: "pro-official", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--moe-runner-backend flashinfer_trtllm_routed", + "--speculative-algorithm DSPARK", + "--chunked-prefill-size 8192", + "--disable-flashinfer-autotune", + "--swa-full-tokens-ratio 0.1", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + // ==================================================================== // B300 + NVFP4 // ==================================================================== { @@ -892,6 +941,46 @@ sgl-eval run mmmu_pro \\ "--port {{PORT}}", ], }, + // ==================================================================== + // B300 + NVFP4 — Official (0731 / 0813) + // Mirrors the Flash/Pro NVFP4 cells; the official checkpoints bundle a + // DSpark draft head, so low-latency uses `--speculative-algorithm DSPARK` + // instead of the EAGLE shape flags. NOT yet run end-to-end on this hardware. + // ==================================================================== + { + match: { hw: "b300", variant: "flash-official", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + verificationStatus: "in-progress", + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 4", + "--moe-runner-backend flashinfer_trtllm_routed", + "--speculative-algorithm DSPARK", + "--disable-flashinfer-autotune", + "--swa-full-tokens-ratio 0.1", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b300", variant: "pro-official", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + verificationStatus: "in-progress", + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--moe-runner-backend flashinfer_trtllm_routed", + "--speculative-algorithm DSPARK", + "--chunked-prefill-size 8192", + "--disable-flashinfer-autotune", + "--swa-full-tokens-ratio 0.1", + "--mem-fraction-static 0.90", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, // ==================================================================== // GB200 + FP4 @@ -1118,6 +1207,46 @@ sgl-eval run mmmu_pro \\ "--port {{PORT}}", ], }, + // ==================================================================== + // GB200 + NVFP4 — Official (0731 / 0813) + // Mirrors the Flash/Pro NVFP4 cells; the official checkpoints bundle a + // DSpark draft head, so low-latency uses `--speculative-algorithm DSPARK` + // instead of the EAGLE shape flags. NOT yet run end-to-end on this hardware. + // ==================================================================== + { + match: { hw: "gb200", variant: "flash-official", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + verificationStatus: "in-progress", + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 4", + "--moe-runner-backend flashinfer_trtllm_routed", + "--speculative-algorithm DSPARK", + "--disable-flashinfer-autotune", + "--swa-full-tokens-ratio 0.1", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "gb200", variant: "pro-official", quant: "nvfp4", strategy: "low-latency", nodes: "multi-2" }, + verificationStatus: "in-progress", + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--moe-runner-backend flashinfer_trtllm_routed", + "--speculative-algorithm DSPARK", + "--chunked-prefill-size 8192", + "--disable-flashinfer-autotune", + "--swa-full-tokens-ratio 0.1", + "--mem-fraction-static 0.90", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, // ==================================================================== // GB300 + FP4 @@ -1722,7 +1851,7 @@ sgl-eval run mmmu_pro \\ }, // ==================================================================== - // GB200 + NVFP4 + // GB300 + NVFP4 // ==================================================================== { match: { hw: "gb300", variant: "flash", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, @@ -1764,6 +1893,46 @@ sgl-eval run mmmu_pro \\ "--port {{PORT}}", ], }, + // ==================================================================== + // GB300 + NVFP4 — Official (0731 / 0813) + // Mirrors the Flash/Pro NVFP4 cells; the official checkpoints bundle a + // DSpark draft head, so low-latency uses `--speculative-algorithm DSPARK` + // instead of the EAGLE shape flags. NOT yet run end-to-end on this hardware. + // ==================================================================== + { + match: { hw: "gb300", variant: "flash-official", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + verificationStatus: "in-progress", + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 4", + "--moe-runner-backend flashinfer_trtllm_routed", + "--speculative-algorithm DSPARK", + "--disable-flashinfer-autotune", + "--swa-full-tokens-ratio 0.1", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "gb300", variant: "pro-official", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + verificationStatus: "in-progress", + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 4", + "--moe-runner-backend flashinfer_trtllm_routed", + "--speculative-algorithm DSPARK", + "--chunked-prefill-size 8192", + "--disable-flashinfer-autotune", + "--swa-full-tokens-ratio 0.1", + "--mem-fraction-static 0.90", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, // ==================================================================== // H200 + FP8 (deepep, no Marlin)