From ae848116662ece923501131ce773a4602223237e Mon Sep 17 00:00:00 2001 From: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com> Date: Sat, 1 Aug 2026 16:05:42 +0800 Subject: [PATCH] [Docs] Add verified H200 and B200 DeepSeek-V4 Flash Official results (#33109) --- .../autoregressive/DeepSeek/DeepSeek-V4.mdx | 2 +- .../deepseek-ai/deepseek-v4-benchmarks.jsx | 60 +++++++++++++++++++ .../configs/deepseek-ai/deepseek-v4.jsx | 24 ++++---- 3 files changed, 74 insertions(+), 12 deletions(-) diff --git a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx index 88fb58b8e..2b785d8e3 100644 --- a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx +++ b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx @@ -155,7 +155,7 @@ import { Playground } from "/src/snippets/_playground.jsx"; DeepSeek-V4-Flash-0731 304 13B - Flash Official (0731), with a bundled DSpark draft head; verified for low-latency serving on 4×GB300 + Flash Official (0731), with a bundled DSpark draft head; verified on 8×B200, 4×GB300, and 4×H200 DeepSeek-V4-Pro diff --git a/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4-benchmarks.jsx b/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4-benchmarks.jsx index 511d5bf4e..2a1f7264e 100644 --- a/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4-benchmarks.jsx +++ b/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4-benchmarks.jsx @@ -6,6 +6,36 @@ export const benchmarks = [ // ==================================================================== // B200 + FP4 // ==================================================================== + { + match: { hw: "b200", variant: "flash-official", quant: "fp4", strategy: "low-latency", nodes: "single" }, + sglang_version: "0.5.16", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 218.90, tpot_ms: 1.28, tokens_per_sec_per_gpu: 481 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 189.89, tpot_ms: 3.38, tokens_per_sec_per_gpu: 3383 }, + ], + }, + { + match: { hw: "b200", variant: "flash-official", quant: "fp4", strategy: "balanced", nodes: "single" }, + sglang_version: "0.5.16", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 }, + ttft_ms: 1629.45, tpot_ms: 34.13, tokens_per_sec_per_gpu: 1595 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 }, + ttft_ms: 2568.55, tpot_ms: 57.15, tokens_per_sec_per_gpu: 4326 }, + ], + }, + { + match: { hw: "b200", variant: "flash-official", quant: "fp4", strategy: "high-throughput", nodes: "single" }, + sglang_version: "0.5.16", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 }, + ttft_ms: 74853.51, tpot_ms: 51.72, tokens_per_sec_per_gpu: 5464 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 4096 }, + ttft_ms: 216241.01, tpot_ms: 51.80, tokens_per_sec_per_gpu: 5301 }, + ], + }, { match: { hw: "b200", variant: "flash", quant: "fp4", strategy: "low-latency", nodes: "single" }, sglang_version: "0.5.15", @@ -395,6 +425,36 @@ export const benchmarks = [ // ==================================================================== // H200 + FP4 // ==================================================================== + { + match: { hw: "h200", variant: "flash-official", quant: "fp4", strategy: "low-latency", nodes: "single" }, + sglang_version: "0.5.16", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 308.29, tpot_ms: 1.72, tokens_per_sec_per_gpu: 606 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 661.70, tpot_ms: 8.39, tokens_per_sec_per_gpu: 2538 }, + ], + }, + { + match: { hw: "h200", variant: "flash-official", quant: "fp4", strategy: "balanced", nodes: "single" }, + sglang_version: "0.5.16", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 }, + ttft_ms: 1617.91, tpot_ms: 38.05, tokens_per_sec_per_gpu: 2994 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 }, + ttft_ms: 1931.94, tpot_ms: 104.53, tokens_per_sec_per_gpu: 4872 }, + ], + }, + { + match: { hw: "h200", variant: "flash-official", quant: "fp4", strategy: "high-throughput", nodes: "single" }, + sglang_version: "0.5.16", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 }, + ttft_ms: 195108.42, tpot_ms: 123.81, tokens_per_sec_per_gpu: 4573 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 4096 }, + ttft_ms: 505508.73, tpot_ms: 123.97, tokens_per_sec_per_gpu: 4542 }, + ], + }, { match: { hw: "h200", variant: "flash", quant: "fp4", strategy: "low-latency", nodes: "single" }, sglang_version: "0.5.15.post1", diff --git a/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx b/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx index b4d619a81..790e7fb8d 100644 --- a/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx +++ b/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx @@ -442,17 +442,18 @@ sgl-eval run aime25 \\ // ==================================================================== { match: { hw: "b200", variant: "flash-official", quant: "fp4", strategy: "low-latency", nodes: "single" }, - verified: false, + verified: true, env: [], flags: [ "--trust-remote-code", "--model-path {{MODEL_NAME}}", - "--tp 4", + "--tp 8", "--moe-runner-backend flashinfer_mxfp4", "--speculative-algorithm DSPARK", "--chunked-prefill-size 4096", "--disable-flashinfer-autotune", "--swa-full-tokens-ratio 0.1", + "--mem-fraction-static 0.90", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -479,13 +480,13 @@ sgl-eval run aime25 \\ }, { match: { hw: "b200", variant: "flash-official", quant: "fp4", strategy: "balanced", nodes: "single" }, - verified: false, + verified: true, env: ["SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=1024"], flags: [ "--trust-remote-code", "--model-path {{MODEL_NAME}}", - "--tp 4", - "--dp 4", + "--tp 8", + "--dp 8", "--enable-dp-attention", "--moe-a2a-backend deepep", "--deepep-config '{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}'", @@ -515,15 +516,15 @@ sgl-eval run aime25 \\ }, { match: { hw: "b200", variant: "flash-official", quant: "fp4", strategy: "high-throughput", nodes: "single" }, - verified: false, + verified: true, env: [ "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320", ], flags: [ "--trust-remote-code", "--model-path {{MODEL_NAME}}", - "--tp 4", - "--dp 4", + "--tp 8", + "--dp 8", "--enable-dp-attention", "--moe-a2a-backend megamoe", "--host {{HOST_IP}}", @@ -1469,7 +1470,7 @@ sgl-eval run aime25 \\ { match: { hw: "h200", variant: "flash-official", quant: "fp4", strategy: "low-latency", nodes: "single" }, - verified: false, + verified: true, env: [], flags: [ "--trust-remote-code", @@ -1500,7 +1501,7 @@ sgl-eval run aime25 \\ }, { match: { hw: "h200", variant: "flash-official", quant: "fp4", strategy: "balanced", nodes: "single" }, - verified: false, + verified: true, env: [], flags: [ "--trust-remote-code", @@ -1508,6 +1509,7 @@ sgl-eval run aime25 \\ "--tp 4", "--moe-runner-backend flashinfer_mxfp4", "--speculative-algorithm DSPARK", + "--mem-fraction-static 0.88", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -1531,7 +1533,7 @@ sgl-eval run aime25 \\ }, { match: { hw: "h200", variant: "flash-official", quant: "fp4", strategy: "high-throughput", nodes: "single" }, - verified: false, + verified: true, env: [], flags: [ "--trust-remote-code",