From 049f1bf6fb42309d95889262c3f61b3679b267ad Mon Sep 17 00:00:00 2001 From: Yuhao Yang <47235274+yhyang201@users.noreply.github.com> Date: Sun, 26 Apr 2026 11:54:55 +0800 Subject: [PATCH] docs(DeepSeek-V4): add GB200 platform to cookbook recipe (#23725) --- .../autoregressive/DeepSeek/DeepSeek-V4.mdx | 8 ++- .../autoregressive/deepseek-v4-deployment.jsx | 58 +++++++++++++++++-- 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx index e11b3a75a..8a26dd402 100644 --- a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx +++ b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx @@ -29,13 +29,13 @@ tag: NEW DeepSeek-V4-Flash 284B 13B - single-node serving: B200 / GB300 / H200 on 4 GPUs + single-node serving: B200 / GB200 / GB300 / H200 on 4 GPUs DeepSeek-V4-Pro 1.6T 49B - high-capacity: B200 8 GPU / GB300 4 GPU / H200 16 GPU (2 nodes) + high-capacity: B200 8 GPU / GB200 8 GPU (2 nodes) / GB300 4 GPU / H200 16 GPU (2 nodes) @@ -88,6 +88,10 @@ Please refer to the [official SGLang installation guide](../../../docs/get-start NVIDIA B200 lmsysorg/sglang:deepseek-v4-blackwell + + NVIDIA GB200 + lmsysorg/sglang:deepseek-v4-grace-blackwell + NVIDIA GB300 lmsysorg/sglang:deepseek-v4-grace-blackwell diff --git a/docs_new/src/snippets/autoregressive/deepseek-v4-deployment.jsx b/docs_new/src/snippets/autoregressive/deepseek-v4-deployment.jsx index 621c98091..a011cbf0c 100644 --- a/docs_new/src/snippets/autoregressive/deepseek-v4-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/deepseek-v4-deployment.jsx @@ -4,6 +4,7 @@ export const DeepSeekV4Deployment = () => { // // Hardware (quantization determined by GPU generation): // B200 → FP4 weights, Flash TP=4 / Pro TP=8 single-node + // GB200 → FP4 weights, Flash TP=4 / Pro TP=8 2-node // GB300 → FP4 weights, Flash TP=4 / Pro TP=4 single-node // H200 → FP8 weights, Flash TP=4 / Pro TP=16 2-node // Model variant → HF slug: @@ -27,6 +28,7 @@ export const DeepSeekV4Deployment = () => { items: [ { id: "b200", label: "B200 (FP4)", default: true }, { id: "b300", label: "B300 (FP4)", default: false }, + { id: "gb200", label: "GB200 (FP4)", default: false }, { id: "gb300", label: "GB300 (FP4)", default: false }, { id: "h200", label: "H200 (FP8)", default: false }, ], @@ -138,6 +140,8 @@ export const DeepSeekV4Deployment = () => { "b200|big": { slug: "deepseek-ai/DeepSeek-V4-Pro", tp: 8, multinode: false }, "gb300|small": { slug: "deepseek-ai/DeepSeek-V4-Flash", tp: 4, multinode: false }, "gb300|big": { slug: "deepseek-ai/DeepSeek-V4-Pro", tp: 4, multinode: false }, + "gb200|small": { slug: "deepseek-ai/DeepSeek-V4-Flash", tp: 4, multinode: false }, + "gb200|big": { slug: "deepseek-ai/DeepSeek-V4-Pro", tp: 8, multinode: true, nnodes: 2 }, // H200 needs an FP8-only Instruct ckpt (deepseek-ai's Flash/Pro repos ship // FP4-mixed weights that Hopper can't run). sgl-project publishes FP8 // repackagings for both variants. @@ -150,6 +154,8 @@ export const DeepSeekV4Deployment = () => { "b200|big": { tp: 8, multinode: false }, "gb300|small": { tp: 4, multinode: false }, "gb300|big": { tp: 4, multinode: false }, + "gb200|small": { tp: 4, multinode: false }, + "gb200|big": { tp: 8, multinode: true, nnodes: 2 }, "h200|small": { tp: 4, multinode: false }, "h200|big": { tp: 16, multinode: true, nnodes: 2 }, }; @@ -183,12 +189,21 @@ export const DeepSeekV4Deployment = () => { "gb300|big|cp", "gb300|small|pd-disagg", "gb300|big|pd-disagg", + "gb200|small|low-latency", + "gb200|small|balanced", + "gb200|small|max-throughput", + "gb200|small|cp", + "gb200|big|low-latency", + "gb200|big|balanced", + "gb200|big|max-throughput", ]); // Recipes whose command is intentionally not yet provided (e.g. blocked by an // upstream limitation). Showing a minimal placeholder is friendlier to users // than emitting a commented-out invalid command. const TBD_RECIPES = new Set([ "h200|big|cp", + "gb200|small|pd-disagg", + "gb200|big|pd-disagg", ]); const TBD_PLACEHOLDER = "# to be provided"; const BEING_VERIFIED_NOTE = @@ -242,14 +257,18 @@ export const DeepSeekV4Deployment = () => { h200: ["SGLANG_DSV4_FP4_EXPERTS=0"], // allinone _ENV_H200 b200: [], // _ENV_B200 minus NVSHMEM gb300: [], // _ENV_GB300 + // GB200 multinode needs NCCL MNNVL for cross-node NVLink communication. + gb200: multinode ? ["NCCL_MNNVL_ENABLE=1", "NCCL_CUMEM_ENABLE=1"] : [], }[hardware]; // Recipe-specific env (matches allinone exactly, taking size into account). const recipeEnv = []; if (recipe === "low-latency") { - // H200 big low-latency has extra dispatch-token cap (allinone line 233). + // Big low-latency dispatch-token cap. if (hardware === "h200" && isBig) { recipeEnv.push("SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128"); + } else if (hardware === "gb200" && isBig) { + recipeEnv.push("SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=256"); } } else if (recipe === "balanced") { if (hardware === "h200") { @@ -293,6 +312,7 @@ export const DeepSeekV4Deployment = () => { // allinone: // H200 small: pure TP + MTP_314 // H200 big: DP-attn + DeepEP + MTP_314 + cg=32 max-run=64 + multi-node + mem-frac 0.82 + // GB200 big: pure TP + multinode + flashinfer_mxfp4 + MTP_314 + mem-frac 0.82 (no DP-attn/DeepEP) // Blackwell: TP + flashinfer_mxfp4 + MTP_314 + chunked-prefill-size 4096 + autotune-fix // Big Blackwell additionally: mem-frac 0.82 flags.push(` --tp ${tp}`); @@ -335,7 +355,11 @@ export const DeepSeekV4Deployment = () => { flags.push(" --speculative-num-steps 1"); flags.push(" --speculative-eagle-topk 1"); flags.push(" --speculative-num-draft-tokens 2"); - if (isBig) flags.push(" --mem-fraction-static 0.82"); + if (isBig && hardware === "gb200") { + flags.push(" --mem-fraction-static 0.78"); + } else if (isBig) { + flags.push(" --mem-fraction-static 0.82"); + } if (hardware === "h200") { flags.push(" --cuda-graph-max-bs 128"); flags.push(" --max-running-requests 128"); @@ -345,6 +369,9 @@ export const DeepSeekV4Deployment = () => { } else if (isBig && hardware === "gb300") { flags.push(" --cuda-graph-max-bs 128"); flags.push(" --max-running-requests 256"); + } else if (isBig && hardware === "gb200") { + flags.push(" --cuda-graph-max-bs 64"); + flags.push(" --max-running-requests 128"); } // allinone H200 gates DEEPEP_LARGE_SMS_FLAG on !multinode — only H200 big // is multi-node; all Blackwell cells get the flag unconditionally. @@ -359,7 +386,11 @@ export const DeepSeekV4Deployment = () => { flags.push(" --enable-dp-attention"); if (multinode) flags.push(...multiNodeFlags(nnodes)); flags.push(" --moe-a2a-backend deepep"); - if (isBig) flags.push(" --mem-fraction-static 0.82"); + if (isBig && hardware === "gb200") { + flags.push(" --mem-fraction-static 0.78"); + } else if (isBig) { + flags.push(" --mem-fraction-static 0.82"); + } if (hardware === "h200") { flags.push(" --cuda-graph-max-bs 128"); flags.push(" --max-running-requests 256"); @@ -369,6 +400,9 @@ export const DeepSeekV4Deployment = () => { } else if (isBig && hardware === "gb300") { flags.push(" --cuda-graph-max-bs 128"); flags.push(" --max-running-requests 256"); + } else if (isBig && hardware === "gb200") { + flags.push(" --cuda-graph-max-bs 64"); + flags.push(" --max-running-requests 256"); } if (!multinode) flags.push(DEEPEP_LARGE_SMS_FLAG); } else if (recipe === "cp") { @@ -416,7 +450,18 @@ export const DeepSeekV4Deployment = () => { const envAll = [...HW_ENV, ...recipeEnv, ...COMMON_ENV]; const envBlock = envAll.length ? envAll.join(" \\\n") + " \\\n" : ""; const base = `${envBlock}sglang serve \\\n${flags.join(" \\\n")}`; - const withMultinode = multinode ? prependMultiNodeNote(base, nnodes) : base; + // GB200 multinode may need machine-specific NVSHMEM / Gloo env vars; + // emit them as commented hints above the env block so users know to check. + let cmd = base; + if (hardware === "gb200" && multinode) { + cmd = + `# The following env vars may be needed depending on your cluster:\n` + + `# GLOO_SOCKET_IFNAME=\n` + + `# NVSHMEM_ENABLE_NIC_PE_MAPPING=1\n` + + `# NVSHMEM_HCA_LIST=\n` + + cmd; + } + const withMultinode = multinode ? prependMultiNodeNote(cmd, nnodes) : cmd; const verifyKey = `${hardware}|${modelSize}|${recipe}`; if (TBD_RECIPES.has(verifyKey)) return TBD_PLACEHOLDER; return VERIFIED_RECIPES.has(verifyKey) @@ -447,14 +492,15 @@ export const DeepSeekV4Deployment = () => { const specKey = `${hardware}|${modelSize}`; const { tp: pdTp, multinode, nnodes } = PD_TP_SPEC[specKey]; const slug = HW_SIZE_SPEC[specKey].slug; - const ibDevice = { h200: "mlx5_0", b200: "mlx5_7", gb300: "" }[hardware]; + const ibDevice = { h200: "mlx5_0", b200: "mlx5_7", gb300: "", gb200: "" }[hardware]; const isGB300 = hardware === "gb300"; - const isBlackwell = hardware === "b200" || isGB300; + const isBlackwell = hardware === "b200" || hardware === "gb200" || isGB300; const HW_ENV = { h200: ["SGLANG_DSV4_FP4_EXPERTS=0"], b200: [], gb300: [], + gb200: [], }[hardware]; // Whitelist #5: only SGLANG_MOONCAKE_CUSTOM_MEM_POOL kept; MC_FORCE_MNNVL / // NCCL_MNNVL_ENABLE / NCCL_CUMEM_ENABLE may also be needed depending on the