diff --git a/docs_new/src/snippets/autoregressive/deepseek-v4-deployment.jsx b/docs_new/src/snippets/autoregressive/deepseek-v4-deployment.jsx index 66488474b..0f71bf95f 100644 --- a/docs_new/src/snippets/autoregressive/deepseek-v4-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/deepseek-v4-deployment.jsx @@ -351,13 +351,41 @@ export const DeepSeekV4Deployment = () => { } else if (hardware === "gb200" && isBig) { recipeEnv.push("SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=256"); } + // B200/B300 Pro accuracy-verified env vars. + if (isBig && hardware === "b200") { + recipeEnv.push( + "SGLANG_JIT_DEEPGEMM_PRECOMPILE=0", + "SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT=1", + "SGLANG_OPT_USE_JIT_NORM=1", + "SGLANG_OPT_USE_JIT_INDEXER_METADATA=1", + "SGLANG_OPT_USE_TOPK_V2=1", + "SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2=1", + ); + } } else if (recipe === "balanced") { if (hardware === "h200") { recipeEnv.push(isBig ? "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128" : "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=256"); + } else if (isBig && hardware === "b200") { + // B200/B300 Pro accuracy-verified env vars. + recipeEnv.push( + "SGLANG_JIT_DEEPGEMM_PRECOMPILE=0", + "SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT=1", + "SGLANG_OPT_USE_JIT_NORM=1", + "SGLANG_OPT_USE_JIT_INDEXER_METADATA=1", + "SGLANG_OPT_USE_TOPK_V2=1", + "SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2=1", + "SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN=1", + "SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE=0", + "SGLANG_OPT_FIX_HASH_MEGA_MOE=0", + "SGLANG_OPT_USE_FAST_MASK_EP=1", + "SGLANG_OPT_FIX_MEGA_MOE_MEMORY=1", + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=4096", + "SGLANG_OPT_FIX_NEXTN_MEGA_MOE=1", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=0", + ); } else { - // Blackwell: small=1024, big=256 (allinone ternary). recipeEnv.push(isBig ? "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=256" : "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=1024"); @@ -367,6 +395,26 @@ export const DeepSeekV4Deployment = () => { recipeEnv.push(isBig ? "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128" : "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=256"); + } else if (isBig && hardware === "b200") { + // B200/B300 Pro accuracy-verified env vars. + recipeEnv.push( + "SGLANG_JIT_DEEPGEMM_PRECOMPILE=0", + "SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT=1", + "SGLANG_OPT_USE_JIT_NORM=1", + "SGLANG_OPT_USE_JIT_INDEXER_METADATA=1", + "SGLANG_OPT_USE_TOPK_V2=1", + "SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2=1", + "SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN=1", + "SGLANG_OPT_USE_FAST_MASK_EP=1", + "SGLANG_OPT_FIX_MEGA_MOE_MEMORY=1", + "SGLANG_OPT_FIX_NEXTN_MEGA_MOE=1", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=0", + "NVSHMEM_DISABLE_IB=1", + "SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW=1", + "SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE=1", + "SGLANG_OPT_FIX_HASH_MEGA_MOE=1", + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320", + ); } else { recipeEnv.push(isBig ? "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=256" @@ -422,10 +470,17 @@ export const DeepSeekV4Deployment = () => { flags.push(" --speculative-eagle-topk 1"); flags.push(" --speculative-num-draft-tokens 4"); if (hardware !== "h200") { - flags.push(" --chunked-prefill-size 4096"); + // B200/B300 Pro accuracy-verified: chunked-prefill-size 8192 + flags.push(isBig ? " --chunked-prefill-size 8192" : " --chunked-prefill-size 4096"); flags.push(" --disable-flashinfer-autotune"); + flags.push(" --swa-full-tokens-ratio 0.1"); + } + // B200/B300 Pro accuracy-verified: mem-fraction-static 0.90 + if (isBig && hardware !== "h200") { + flags.push(" --mem-fraction-static 0.90"); + } else if (isBig) { + flags.push(" --mem-fraction-static 0.88"); } - if (isBig) flags.push(" --mem-fraction-static 0.88"); } else if (recipe === "balanced") { // allinone balanced: TP + DP + DP-attn + DeepEP + MTP_112. // H200 small: cg=128 max-run=128 | H200 big: cg=128 max-run=128 (same) @@ -435,7 +490,15 @@ export const DeepSeekV4Deployment = () => { flags.push(` --dp ${tp}`); flags.push(" --enable-dp-attention"); if (multinode) flags.push(...multiNodeFlags(nnodes)); - flags.push(" --moe-a2a-backend deepep"); + // B200/B300 Pro accuracy-verified: flashinfer_mxfp4 (not deepep) for balanced. + if (isBig && hardware === "b200") { + flags.push(" --moe-runner-backend flashinfer_mxfp4"); + flags.push(" --disable-flashinfer-autotune"); + flags.push(" --chunked-prefill-size 32768"); + flags.push(" --swa-full-tokens-ratio 0.1"); + } else { + flags.push(" --moe-a2a-backend deepep"); + } flags.push(" --speculative-algo EAGLE"); flags.push(" --speculative-num-steps 1"); flags.push(" --speculative-eagle-topk 1"); @@ -447,7 +510,7 @@ export const DeepSeekV4Deployment = () => { } else if (isBig && hardware === "gb200") { flags.push(" --mem-fraction-static 0.78"); } else if (isBig) { - flags.push(" --mem-fraction-static 0.82"); + flags.push(" --mem-fraction-static 0.92"); } if (hardware === "h200" && isBig) { flags.push(" --cuda-graph-max-bs 8"); @@ -456,8 +519,7 @@ export const DeepSeekV4Deployment = () => { flags.push(" --cuda-graph-max-bs 128"); flags.push(" --max-running-requests 128"); } else if (isBig && hardware === "b200") { - flags.push(" --cuda-graph-max-bs 64"); - flags.push(" --max-running-requests 128"); + flags.push(" --cuda-graph-max-bs 256"); } else if (isBig && hardware === "gb300") { flags.push(" --cuda-graph-max-bs 128"); flags.push(" --max-running-requests 256"); @@ -485,14 +547,18 @@ export const DeepSeekV4Deployment = () => { } else if (isBig && hardware === "gb200") { flags.push(" --mem-fraction-static 0.78"); } else if (isBig) { - flags.push(" --mem-fraction-static 0.82"); + flags.push(" --mem-fraction-static 0.835"); } if (hardware === "h200") { flags.push(" --cuda-graph-max-bs 128"); flags.push(" --max-running-requests 256"); } else if (isBig && hardware === "b200") { - flags.push(" --cuda-graph-max-bs 64"); - flags.push(" --max-running-requests 256"); + // B200/B300 Pro accuracy-verified max-throughput config. + flags.push(" --cuda-graph-max-bs 544"); + flags.push(" --swa-full-tokens-ratio 0.075"); + flags.push(" --chunked-prefill-size 65536"); + flags.push(" --tokenizer-worker-num 8"); + flags.push(" --enable-prefill-delayer"); } else if (isBig && hardware === "gb300") { flags.push(" --cuda-graph-max-bs 128"); flags.push(" --max-running-requests 256"); @@ -545,7 +611,12 @@ export const DeepSeekV4Deployment = () => { // Assemble: [HW env] [recipe env] \ sglang serve \ flags... const envAll = [...HW_ENV, ...recipeEnv]; const envBlock = envAll.length ? envAll.join(" \\\n") + " \\\n" : ""; - const base = `${envBlock}sglang serve \\\n${flags.join(" \\\n")}`; + // B200/B300 Pro recipes carry many accuracy-verified env vars that will be + // consolidated; prepend a shell comment so users know these are temporary. + const simplifyNote = (isBig && hardware === "b200" && recipeEnv.length > 2) + ? "# flags will be simplified\n" + : ""; + const base = `${simplifyNote}${envBlock}sglang serve \\\n${flags.join(" \\\n")}`; // GB200 multinode may need machine-specific NVSHMEM / Gloo env vars; // emit them as commented hints above the env block so users know to check. let cmd = base; @@ -841,6 +912,32 @@ python3 -m sglang_router.launch_router \\
{generateCommand()}
+ + MegaMoE fuses expert dispatch + GEMM into a single kernel for higher throughput on MoE layers. + It is currently verified on B200/B300 Pro (balanced & max-throughput recipes above). + We have not yet tested the full hardware/recipe matrix, but it should work on other platforms (GB200, GB300, Flash). + To enable it, add the flag and env vars: +
+{
+`# Add this flag to the sglang serve command:
+--moe-a2a-backend deepep
+
+# And set these env vars:
+SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE=1
+SGLANG_OPT_FIX_HASH_MEGA_MOE=1
+SGLANG_OPT_FIX_MEGA_MOE_MEMORY=1
+SGLANG_OPT_FIX_NEXTN_MEGA_MOE=1
+SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320
+SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=0`
+ }
+
+ Adjust SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK based on your chunked prefill size (e.g. 4096 for balanced, 8320 for max-throughput).
+ SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=0 — if your config mentions DeepEP dispatch buffer constraints, they do not apply when this is set to 0.
+ These flags are expected to be simplified in a future release.
+