From 17dadebd4e53f78f849841895b91a976a25f6761 Mon Sep 17 00:00:00 2001 From: zijiexia <37504505+zijiexia@users.noreply.github.com> Date: Thu, 21 May 2026 13:51:40 -0700 Subject: [PATCH] [Docs] DeepSeek-V4: switch H200 FP4 Pro to flashinfer_mxfp4, Flash Balanced too (#25923) Co-authored-by: Claude Opus 4.7 (1M context) --- .../autoregressive/DeepSeek/DeepSeek-V4.mdx | 2 +- .../autoregressive/deepseek-v4-deployment.jsx | 19 +++++++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx index 392282aaa..8546b4d08 100644 --- a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx +++ b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx @@ -120,7 +120,7 @@ The generator currently picks values on the **conservative** side (mirroring an **Hopper (H200) note** We provide two different options for running DeepSeek-V4 models on Hopper devices (H200) -- Original FP4 checkpoints: To run original FP4 checkpoints, we provide two different options for w4a16 MoE kernels: Marlin (`--moe-runner-backend marlin`) and Flashinfer (`--moe-runner-backend flashinfer_mxfp4). For this variant we only support Tensor Parallelism. Complete Pro model can be run on a single H200 node with this option. +- Original FP4 checkpoints: To run original FP4 checkpoints, we provide two different options for w4a16 MoE kernels: Marlin (`--moe-runner-backend marlin`) and Flashinfer (`--moe-runner-backend flashinfer_mxfp4`). For this variant we only support Tensor Parallelism. Complete Pro model can be run on a single H200 node with this option. - Converted FP8 checkpoints: We also provide pre-converted FP8 checkpoints (`sgl-project/DeepSeek-V4-Flash-FP8`, `sgl-project/DeepSeek-V4-Pro-FP8`), which support more parallelism and features. PD-Disagg recipes on H200 may require `docker run --privileged --ulimit memlock=-1` diff --git a/docs_new/src/snippets/autoregressive/deepseek-v4-deployment.jsx b/docs_new/src/snippets/autoregressive/deepseek-v4-deployment.jsx index 50772047b..64a94f86e 100644 --- a/docs_new/src/snippets/autoregressive/deepseek-v4-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/deepseek-v4-deployment.jsx @@ -360,22 +360,33 @@ export const DeepSeekV4Deployment = () => { return buildPDDisaggCommand(hardware, modelSize); } - // H200 (FP4) Marlin path: dedicated branch — Hopper runs the FP4-mixed - // Instruct repos through the Marlin MoE runner, so it doesn't share envs - // or flags with either the FP8 H200 path or the Blackwell paths. + // H200 (FP4) path: dedicated branch — Hopper runs the FP4-mixed Instruct + // repos through one of two w4a16 MoE runners (Marlin or Flashinfer mxfp4), + // so it doesn't share envs or flags with either the FP8 H200 path or the + // Blackwell paths. // Flash: TP=4, single node Pro: TP=8, single node // low-latency: MTP 3 / 1 / 4 (steps / topk / draft-tokens) // balanced: MTP 1 / 1 / 2 // max-throughput: MTP disabled + // + // MoE runner selection (verified on 2026-05-20): + // - Pro: flashinfer_mxfp4 for all recipes + // - Flash Balanced: flashinfer_mxfp4 (~1.5x faster output throughput vs + // Marlin in the balanced throughput benchmark). + // - Flash Low-Latency / Max-Throughput: Marlin (faster than + // flashinfer_mxfp4 in those benchmarks). if (hardware === "h200-fp4") { const verifyKey = `${hardware}|${modelSize}|${recipe}`; if (TBD_RECIPES.has(verifyKey)) return TBD_PLACEHOLDER; + const useFlashinferMxfp4 = isBig || recipe === "balanced"; const fp4Flags = [ " --trust-remote-code", ` --model-path ${slug}`, ` --tp ${tp}`, - " --moe-runner-backend marlin", + useFlashinferMxfp4 + ? " --moe-runner-backend flashinfer_mxfp4" + : " --moe-runner-backend marlin", ]; if (recipe === "low-latency") { fp4Flags.push(" --speculative-algo EAGLE");