minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

docs(DeepSeek-V4): mark gb300|{small,big}|{cp,pd-disagg} verified + GB300-specific fixes (#23691)

Browse Source
This commit is contained in:
fzyzcjy
2026-04-25 12:21:57 +08:00
committed by GitHub
parent d2c61acf25
commit 8a395994ed
2 changed files with 56 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx
+7
View File
@@ -158,6 +158,13 @@ TCP, which can lead to garbled KV transfer on large checkpoints.
In order to use base models, please enable `SGLANG_FIX_DSV4_BASE_MODEL_LOAD=1` and use latest code, before the next round of testing matrix is finished.
**GB300 PD-Disagg cross-pod MNNVL**
On some GB300 clusters with cross-pod KV transfer over NVLink, mooncake may
fail with `nvlink_transport.cpp:497 Requested address ... not found!`. If
this happens, prepend `MC_FORCE_MNNVL=1 NCCL_MNNVL_ENABLE=1 NCCL_CUMEM_ENABLE=1`
to both prefill and decode `sglang serve` commands.
## 4. Model Invocation
### 4.1 Basic Usage
docs_new/src/snippets/autoregressive/deepseek-v4-deployment.jsx
+49 -5
View File
@@ -176,6 +176,10 @@ export const DeepSeekV4Deployment = () => {
"h200|small|pd-disagg",
// h200|big|pd-disagg: pending verification (needs 4-node H200 cluster with
// shared IB fabric: 2-node prefill + 2-node decode).
"gb300|small|cp",
"gb300|big|cp",
"gb300|small|pd-disagg",
"gb300|big|pd-disagg",
]);
// Recipes whose command is intentionally not yet provided (e.g. blocked by an
// upstream limitation). Showing a minimal placeholder is friendlier to users
@@ -372,7 +376,17 @@ export const DeepSeekV4Deployment = () => {
flags.push(" --enable-nsa-prefill-context-parallel");
flags.push(" --nsa-prefill-cp-mode round-robin-split");
flags.push(" --chunked-prefill-size 16384");
flags.push(" --mem-fraction-static 0.78");
// GB300 big CP needs higher mem-fraction-static: Pro 1.6T weights at
// tp=4 are ~224 GB/card on a 273 GB GB300, so 0.78 leaves a negative
// KV pool (init_memory_pool fails: "Not enough memory ... weights
// 224 GB > static target 213 GB"). 0.88 gives weights 224 + KV 16 +
// runtime 33. Other Blackwell tp=8 paths fit fine at 0.78.
// Verified on 2026-04-25 (journal 2026-04-25-001 Cell B, Δ4).
if (hardware === "gb300" && isBig) {
flags.push(" --mem-fraction-static 0.88");
} else {
flags.push(" --mem-fraction-static 0.78");
}
// allinone _CP_FLAGS has --max-running-requests 1024; Blackwell big cp overrides
// to 256. Human directed (2026-04-24) to emit only one value — keep 256 override
// for big Blackwell, else the default 1024.
@@ -436,7 +450,8 @@ export const DeepSeekV4Deployment = () => {
gb300: [],
}[hardware];
// Whitelist #5: only SGLANG_MOONCAKE_CUSTOM_MEM_POOL kept; MC_FORCE_MNNVL /
// NCCL_MNNVL_ENABLE / NCCL_CUMEM_ENABLE stripped (personal-cluster topology).
// NCCL_MNNVL_ENABLE / NCCL_CUMEM_ENABLE may also be needed depending on the
// GB300 cluster's NVLink/IB topology — see §3.2 "Configuration Tips" note.
const MNNVL_ENV = isGB300 ? ["SGLANG_MOONCAKE_CUSTOM_MEM_POOL=True"] : [];
const COMMON_ENV = ["SGLANG_JIT_DEEPGEMM_PRECOMPILE=0"];
@@ -445,6 +460,18 @@ export const DeepSeekV4Deployment = () => {
if (hardware === "b200" && mode === "decode") {
roleEnv.push("SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=1024");
}
// GB300 PD needs DeepEP dispatch buffer cap on BOTH prefill + decode;
// without it, the first forward fails `deep_ep.cpp:1233` assertion
// `x.size(0) <= num_max_dispatch_tokens_per_rank`. The cap also
// co-moves with --max-running-requests below: 256 for big (which
// uses --max-running-requests 128, per-rank=32 ≤ 256), 1024 for
// small (--max-running-requests 256, per-rank=64 ≤ 1024).
// Verified on 2026-04-25 (journal 2026-04-25-001 §C/§D).
if (isGB300) {
roleEnv.push(modelSize === "big"
? "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=256"
: "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=1024");
}
const envAll = [...HW_ENV, ...roleEnv, ...MNNVL_ENV, ...COMMON_ENV];
const envBlock = envAll.length ? envAll.join(" \\\n") + " \\\n" : "";
@@ -460,7 +487,21 @@ export const DeepSeekV4Deployment = () => {
flags.push(" --disaggregation-transfer-backend mooncake");
if (ibDevice) flags.push(` --disaggregation-ib-device ${ibDevice}`);
if (!isGB300) flags.push(` --dist-init-addr 127.0.0.1:${distPort}`);
if (mode === "decode") flags.push(" --max-running-requests 256");
if (mode === "decode") {
// GB300 big PD decode is the most memory-pressured PD role: Pro 1.6T
// weights at tp=4 take ~224 GB/card on a 273 GB GB300; runtime needs
// headroom for DeepEP buffer + mooncake KV recv + CG private pool.
// Cookbook defaults (mem-frac 0.874, cg_max_bs 512, max-running 256)
// OOM during CG capture. Verified working on 2026-04-25 (journal
// 2026-04-25-001 Cell D, Δ10).
if (isGB300 && modelSize === "big") {
flags.push(" --max-running-requests 128");
flags.push(" --mem-fraction-static 0.83");
flags.push(" --cuda-graph-max-bs 128");
} else {
flags.push(" --max-running-requests 256");
}
}
flags.push(" --host 0.0.0.0");
flags.push(` --port ${port}`);
@@ -476,11 +517,14 @@ export const DeepSeekV4Deployment = () => {
const prefill = `${prefillHeader}\n${buildRole("prefill", 30000, 30335)}`;
const decode = `${decodeHeader}\n${buildRole("decode", 30001, 30435)}`;
// Router addresses prefill / decode by their reachable hostnames / IPs.
// Substitute <prefill-host> / <decode-host> with the actual hosts before
// running. On a same-host deployment, both can be 127.0.0.1.
const router = `# --- Router (port 8000) ---
python3 -m sglang_router.launch_router \\
--pd-disaggregation \\
--prefill http://127.0.0.1:30000 \\
--decode http://127.0.0.1:30001 \\
--prefill http://<prefill-host>:30000 \\
--decode http://<decode-host>:30001 \\
--host 0.0.0.0 --port 8000 \\
--disable-circuit-breaker \\
--health-check-interval-secs 999999`;