minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

[NVIDIA] Support NVFP4 MoE for DeepSeek-V4 (#25820)

Browse Source
This commit is contained in:
Trevor Morris
2026-06-21 19:35:14 -07:00
committed by GitHub
parent 5deca2d39f
commit c0bb04b67f
10 changed files with 385 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx
+25
View File
@@ -247,6 +247,31 @@ sglang serve \
--enable-deepseek-v4-fp4-indexer
```
**NVFP4 Hybrid Checkpoints**
The [`nvidia/DeepSeek-V4-Pro-NVFP4`](https://huggingface.co/nvidia/DeepSeek-V4-Pro-NVFP4) and
[`nvidia/DeepSeek-V4-Flash-NVFP4`](https://huggingface.co/nvidia/DeepSeek-V4-Flash-NVFP4) checkpoints
quantize MoE experts to **NVFP4** while keeping attention and dense layers in
**FP8**. It requires `--moe-runner-backend flashinfer_trtllm_routed` which will be automatically selected if not provided.
```bash Command
sglang serve \
--model-path nvidia/DeepSeek-V4-Pro-NVFP4 \
--tp 8
```
or
```bash Command
sglang serve \
--model-path nvidia/DeepSeek-V4-Flash-NVFP4 \
--tp 8
```
Requires Blackwell (SM100+). The MTP layer in this checkpoint stays
MXFP4-packed and is routed through the `Mxfp4FlashinferTrtllmMoEMethod` path
automatically.
<a id="hopper-note" />
**Hopper (H100 / H200) note**
docs_new/src/snippets/configs/deepseek-ai/deepseek-v4-benchmarks.jsx
+50
View File
@@ -136,6 +136,31 @@ export const benchmarks = [
match: { hw: "gb200", variant: "pro", quant: "fp4", strategy: "high-throughput", nodes: "multi-2" },
},
// ====================================================================
// GB200 + NVFP4
// ====================================================================
{
match: { hw: "gb200", variant: "flash", quant: "nvfp4", strategy: "low-latency", nodes: "single" },
sglang_version: "PR #25820",
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
ttft_ms: 323.85, tpot_ms: 3.62, tokens_per_sec_per_gpu: 496 },
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
ttft_ms: 397.31, tpot_ms: 8.11, tokens_per_sec_per_gpu: 3663 },
],
accuracy: { gsm8k_pct: 96.66 },
},
{
match: { hw: "gb200", variant: "pro", quant: "nvfp4", strategy: "low-latency", nodes: "multi-2" },
sglang_version: "PR #25820",
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
ttft_ms: 338.20, tpot_ms: 6.25, tokens_per_sec_per_gpu: 161 },
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
ttft_ms: 524.85, tpot_ms: 14.45, tokens_per_sec_per_gpu: 1015 },
],
accuracy: { gsm8k_pct: 95.98 },
},
// ====================================================================
// GB300 + FP4
// ====================================================================
{
@@ -178,6 +203,31 @@ export const benchmarks = [
match: { hw: "gb300", variant: "pro", quant: "fp4", strategy: "high-throughput", nodes: "single" },
},
// ====================================================================
// GB300 + NVFP4
// ====================================================================
{
match: { hw: "gb300", variant: "flash", quant: "nvfp4", strategy: "low-latency", nodes: "single" },
sglang_version: "PR #25820",
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
ttft_ms: 361.72 , tpot_ms: 3.62, tokens_per_sec_per_gpu: 480 },
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
ttft_ms: 422.96, tpot_ms: 8.19, tokens_per_sec_per_gpu: 3733 },
],
accuracy: { gsm8k_pct: 96.44 },
},
{
match: { hw: "gb300", variant: "pro", quant: "nvfp4", strategy: "low-latency", nodes: "single" },
sglang_version: "PR #25820",
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
ttft_ms: 364.33, tpot_ms: 5.33, tokens_per_sec_per_gpu: 381 },
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
ttft_ms: 694.58, tpot_ms: 16.03, tokens_per_sec_per_gpu: 1654 },
],
accuracy: { gsm8k_pct: 96.36 },
},
// ====================================================================
// H200 + FP8
// ====================================================================
{
docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx
+181
View File
@@ -25,6 +25,7 @@ export const config = {
quantizations: [
{ id: "fp8", label: "FP8" },
{ id: "fp4", label: "FP4" },
{ id: "nvfp4", label: "NVFP4" },
],
strategies: [
{ id: "low-latency", label: "Low-Latency" },
@@ -40,8 +41,10 @@ export const config = {
modelNames: {
"flash|fp4": "deepseek-ai/DeepSeek-V4-Flash",
"flash|fp8": "deepseek-ai/DeepSeek-V4-Flash",
"flash|nvfp4": "nvidia/DeepSeek-V4-Flash-NVFP4",
"pro|fp4": "deepseek-ai/DeepSeek-V4-Pro",
"pro|fp8": "deepseek-ai/DeepSeek-V4-Pro",
"pro|nvfp4": "nvidia/DeepSeek-V4-Pro-NVFP4",
// H200 FP8 needs the sgl-project repackaging (Hopper can't run FP4-mixed Instruct).
"h200|flash|fp8": "sgl-project/DeepSeek-V4-Flash-FP8",
"h200|pro|fp8": "sgl-project/DeepSeek-V4-Pro-FP8",
@@ -592,6 +595,95 @@ sgl-eval run aime25 \\
],
},
// ====================================================================
// B200 + NVFP4
// ====================================================================
{
match: { hw: "b200", variant: "pro", quant: "nvfp4", strategy: "low-latency", nodes: "single" },
verified: true,
env: [],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
"--tp 8",
"--moe-runner-backend flashinfer_trtllm_routed",
"--speculative-algorithm EAGLE",
"--speculative-num-steps 3",
"--speculative-eagle-topk 1",
"--speculative-num-draft-tokens 4",
"--chunked-prefill-size 8192",
"--disable-flashinfer-autotune",
"--swa-full-tokens-ratio 0.1",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
},
{
match: { hw: "b200", variant: "flash", quant: "nvfp4", strategy: "low-latency", nodes: "single" },
verified: true,
env: [],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
"--tp 4",
"--moe-runner-backend flashinfer_trtllm_routed",
"--speculative-algorithm EAGLE",
"--speculative-num-steps 3",
"--speculative-eagle-topk 1",
"--speculative-num-draft-tokens 4",
"--chunked-prefill-size 4096",
"--disable-flashinfer-autotune",
"--swa-full-tokens-ratio 0.1",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
},
// ====================================================================
// B300 + NVFP4
// ====================================================================
{
match: { hw: "b300", variant: "flash", quant: "nvfp4", strategy: "low-latency", nodes: "single" },
verified: true,
env: [],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
"--tp 4",
"--moe-runner-backend flashinfer_trtllm_routed",
"--speculative-algorithm EAGLE",
"--speculative-num-steps 3",
"--speculative-eagle-topk 1",
"--speculative-num-draft-tokens 4",
"--chunked-prefill-size 4096",
"--disable-flashinfer-autotune",
"--swa-full-tokens-ratio 0.1",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
},
{
match: { hw: "b300", variant: "pro", quant: "nvfp4", strategy: "low-latency", nodes: "single" },
verified: true,
env: [],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
"--tp 8",
"--moe-runner-backend flashinfer_trtllm_routed",
"--speculative-algorithm EAGLE",
"--speculative-num-steps 3",
"--speculative-eagle-topk 1",
"--speculative-num-draft-tokens 4",
"--chunked-prefill-size 8192",
"--disable-flashinfer-autotune",
"--swa-full-tokens-ratio 0.1",
"--mem-fraction-static 0.90",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
},
// ====================================================================
// GB200 + FP4
// ====================================================================
@@ -726,6 +818,51 @@ sgl-eval run aime25 \\
],
},
// ====================================================================
// GB200 + NVFP4
// ====================================================================
{
match: { hw: "gb200", variant: "flash", quant: "nvfp4", strategy: "low-latency", nodes: "single" },
verified: true,
env: [],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
"--tp 4",
"--moe-runner-backend flashinfer_trtllm_routed",
"--speculative-algorithm EAGLE",
"--speculative-num-steps 3",
"--speculative-eagle-topk 1",
"--speculative-num-draft-tokens 4",
"--chunked-prefill-size 4096",
"--disable-flashinfer-autotune",
"--swa-full-tokens-ratio 0.1",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
},
{
match: { hw: "gb200", variant: "pro", quant: "nvfp4", strategy: "low-latency", nodes: "multi-2" },
verified: true,
env: [],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
"--tp 8",
"--moe-runner-backend flashinfer_trtllm_routed",
"--speculative-algorithm EAGLE",
"--speculative-num-steps 3",
"--speculative-eagle-topk 1",
"--speculative-num-draft-tokens 4",
"--chunked-prefill-size 8192",
"--disable-flashinfer-autotune",
"--swa-full-tokens-ratio 0.1",
"--mem-fraction-static 0.90",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
},
// ====================================================================
// GB300 + FP4
// ====================================================================
@@ -849,6 +986,50 @@ sgl-eval run aime25 \\
"--port {{PORT}}",
],
},
// ====================================================================
// GB200 + NVFP4
// ====================================================================
{
match: { hw: "gb300", variant: "flash", quant: "nvfp4", strategy: "low-latency", nodes: "single" },
verified: true,
env: [],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
"--tp 4",
"--moe-runner-backend flashinfer_trtllm_routed",
"--speculative-algorithm EAGLE",
"--speculative-num-steps 3",
"--speculative-eagle-topk 1",
"--speculative-num-draft-tokens 4",
"--chunked-prefill-size 4096",
"--disable-flashinfer-autotune",
"--swa-full-tokens-ratio 0.1",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
},
{
match: { hw: "gb300", variant: "pro", quant: "nvfp4", strategy: "low-latency", nodes: "single" },
verified: true,
env: [],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
"--tp 4",
"--moe-runner-backend flashinfer_trtllm_routed",
"--speculative-algorithm EAGLE",
"--speculative-num-steps 3",
"--speculative-eagle-topk 1",
"--speculative-num-draft-tokens 4",
"--chunked-prefill-size 8192",
"--disable-flashinfer-autotune",
"--swa-full-tokens-ratio 0.1",
"--mem-fraction-static 0.90",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
},
// ====================================================================
// H200 + FP8 (deepep, no Marlin)