minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[docs] DeepSeek-V4: MI355X PD disaggregation recipes for all three strategies (#39396)

Browse Source
This commit is contained in:
Theresa Shan
2026-09-14 02:11:51 -07:00
committed by GitHub
parent 5200508b0f
commit 95140a7b0c
4 changed files with 544 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/src/snippets/_deployment.jsx
+17 -4
View File
@@ -109,6 +109,11 @@ export const Deployment = ({ config, benchmarks }) => {
// ==== 1. Hardware catalog (shared across cookbooks) ====
// VRAM is per-GPU on-chip memory, not per-module.
const AMD_RDMA_DOCKER_FLAGS = [
"--device /dev/infiniband", "--cap-add IPC_LOCK",
"--ulimit memlock=-1", "--ulimit stack=67108864",
"--ulimit nofile=1048576:1048576",
];
const HARDWARE_CATALOG = {
blackwell: [
{ id: "b300", label: "B300", vram: "288GB" },
@@ -128,11 +133,19 @@ export const Deployment = ({ config, benchmarks }) => {
{ id: "h20-3e", label: "H20-3e", vram: "141GB" },
{ id: "h800", label: "H800", vram: "80GB" },
],
// ROCm multi-node runs the RDMA NICs straight through: /dev/infiniband
// covers rdma_cm plus the per-NIC uverbsN nodes, IPC_LOCK + an unlimited
// memlock let the transport pin its registered buffers, and the stack /
// nofile raises are for the per-QP file descriptors a full 8-NIC mesh opens.
amd: [
{ id: "mi300x", label: "MI300X", vram: "192GB" },
{ id: "mi325x", label: "MI325X", vram: "256GB" },
{ id: "mi350x", label: "MI350X", vram: "288GB" },
{ id: "mi355x", label: "MI355X", vram: "288GB" },
{ id: "mi300x", label: "MI300X", vram: "192GB",
multiNodeDockerFlags: [...AMD_RDMA_DOCKER_FLAGS] },
{ id: "mi325x", label: "MI325X", vram: "256GB",
multiNodeDockerFlags: [...AMD_RDMA_DOCKER_FLAGS] },
{ id: "mi350x", label: "MI350X", vram: "288GB",
multiNodeDockerFlags: [...AMD_RDMA_DOCKER_FLAGS] },
{ id: "mi355x", label: "MI355X", vram: "288GB",
multiNodeDockerFlags: [...AMD_RDMA_DOCKER_FLAGS] },
],
// Atlas 800I A3 (910C): 1 card = 2 dies, so --tp-size is 2× the card
// count (32 cards -> --tp-size 64).
docs/src/snippets/_playground.jsx
+229 -27
View File
@@ -225,6 +225,17 @@ export const Playground = ({ config }) => {
Array.isArray(vs) && vs.includes(base[k]));
};
// The PD router is shared by every cell, so a platform whose prefill/decode
// pair wants a different routing policy declares `routerOverrides: [{when,
// port?, command}]` instead of rewriting the default out from under the rest.
// First match wins; fields the override omits fall back to the default.
const resolveRouter = (fc, sel) => {
if (!fc || !fc.router) return null;
const hit = (fc.routerOverrides || []).find(
(r) => r && matchConstraint(sel, r.when));
return hit ? { ...fc.router, ...hit } : fc.router;
};
// Normalize a chip entry into `{value, label?, hidden, disabled,
// disableReason, ...rest}`. `value` resolves to `entry.id` (rich form) or
// `entry.value` (wrapper form), or the entry itself for bare values.
@@ -379,6 +390,16 @@ export const Playground = ({ config }) => {
ANCHOR_NEAR_DPATTN, ANCHOR_NEAR_MOE,
};
// -------- HiCache flag family --------
// Shared because two axes own it: `hicache` emits it, and `umbp` strips the
// whole family when it takes the tier over (the two are mutually exclusive).
const HICACHE_HEADS = [
"--enable-hierarchical-cache", "--hicache-ratio", "--hicache-size",
"--hicache-write-policy", "--hicache-mem-layout", "--hicache-io-backend",
"--hicache-storage-backend", "--hicache-storage-prefetch-policy",
"--hicache-storage-backend-extra-config",
];
// -------- Prefill-CP flag family (shared by the attention axis) --------
// Every flag head that toggles/parameterizes prefill context parallelism:
// the canonical pair plus all per-family legacy spellings.
@@ -531,16 +552,44 @@ export const Playground = ({ config }) => {
]);
}
}
if (value.dpAttn !== null && value.dpAttn !== undefined
// `forceOff: [{when, stripEnv, reason}]` on a knob — the selection
// dictates the value, so neither the pick nor the base cell gets a say.
// Used where a PD role runs a different topology than the aggregated
// cell it derives from: the role is TP-only, but the cell is DP.
const dpForced = (knobEntry("dpAttn").forceOff || []).find(
(r) => r && h.matchConstraint(factsNow(), r.when));
if (dpForced) {
flags = h.stripFlagsByFirstToken(flags, [
...h.DP_HEADS, "--enable-dp-attention",
"--enable-dp-attention-local-control-broadcast",
]);
// The cell's DP-only env would otherwise outlive the flags it tunes.
const stripEnv = dpForced.stripEnv || [];
if (stripEnv.length) {
env = env.filter((e) => !stripEnv.includes(e.split("=")[0]));
}
} else if (value.dpAttn !== null && value.dpAttn !== undefined
&& !blocked("dpAttn", value.dpAttn)) {
// Capture the spelling before stripping — the TP/EP handlers do the
// same, and a lookup on the stripped array always hits the fallback.
const dpHead = h.flagSpelling(flags, h.DP_HEADS, "--dp-size");
flags = h.stripFlagsByFirstToken(flags, [...h.DP_HEADS, "--enable-dp-attention"]);
// The local-control-broadcast companion only means anything with DP
// attention on, so it has to go down with it — stripping just
// `--enable-dp-attention` would leave it orphaned in the command.
// apply re-seeds from the base cell, so this restores it when the
// cell had it and the user is only re-sizing DP rather than disabling.
const hadLocalBroadcast =
h.hasFlag(flags, "--enable-dp-attention-local-control-broadcast");
flags = h.stripFlagsByFirstToken(flags, [
...h.DP_HEADS, "--enable-dp-attention",
"--enable-dp-attention-local-control-broadcast",
]);
if (typeof value.dpAttn === "number" && value.dpAttn > 0) {
flags = h.insertAfter(flags, h.ANCHOR_NEAR_TP, [
`${dpHead} ${value.dpAttn}`,
"--enable-dp-attention",
...(hadLocalBroadcast
? ["--enable-dp-attention-local-control-broadcast"] : []),
]);
}
}
@@ -602,14 +651,18 @@ export const Playground = ({ config }) => {
{knobs.map((knob) => {
const kc = h.evaluateChip(knob, base);
if (kc.hidden) return null;
// Mirror apply's forceOff: show the value the command actually
// gets, grayed, instead of a live-looking pick apply discards.
const forced = (knob.forceOff || []).find(
(r) => r && h.matchConstraint(base, r.when));
return (
<span key={knob.id} style={s.field}>
<span style={s.fieldLabel}>{knob.label || knob.id.toUpperCase()}</span>
{renderSelect(knobDisplay(knob), entriesFor(knob),
{renderSelect(forced ? false : knobDisplay(knob), entriesFor(knob),
(nv) => setKnob(knob.id, nv), base, labelFor(knob),
{ hideValues: hideNullFor(knob),
disabled: kc.disabled,
disabledReason: kc.disableReason })}
disabled: kc.disabled || !!forced,
disabledReason: forced ? forced.reason : kc.disableReason })}
</span>
);
})}
@@ -985,10 +1038,25 @@ export const Playground = ({ config }) => {
// render, so a base flag is never left over from an earlier selection
// and must not be stripped when the role is Off.
const modeMeta = (fc.modes || []).find((m) => m.id === mode);
if (modeMeta && modeMeta.flags && modeMeta.flags.length) {
// A role may scope its flags/env to a subset of hardware via `when`
// (same shape as a backend's `envWhen`), so a ROCm-only prefill recipe
// doesn't leak onto CUDA. Absent `when`, the role applies everywhere.
//
// `roleOverrides: [{when, mode, flags, env}]` then re-specifies a role
// for a narrower selection — a second operating point (low-latency vs
// high-throughput sizing) is a different flag set for the SAME role,
// so it belongs here rather than as a duplicate entry in the Mode
// select. First match wins and replaces the role's own flags/env.
const roleOverride = (fc.roleOverrides || []).find((r) =>
r && r.mode === mode && r.when && h.matchConstraint(sel, r.when));
const roleSpec = roleOverride || modeMeta;
const modeGate = roleOverride ? null : (modeMeta && modeMeta.when);
const modeOk = !modeGate || Object.keys(modeGate).every(
(k) => (modeGate[k] || []).includes(sel[k]));
if (modeOk && roleSpec && roleSpec.flags && roleSpec.flags.length) {
flags = h.stripFlagsByFirstToken(
flags, modeMeta.flags.map((f) => f.split(/[\s=]/)[0]));
adds.push(...modeMeta.flags);
flags, roleSpec.flags.map((f) => f.split(/[\s=]/)[0]));
adds.push(...roleSpec.flags);
}
// Single-host needs no --dist-init-addr: prefill/decode derive their
// ZMQ/dist ports from the role-specific --port (spaced 100 apart, see
@@ -1015,8 +1083,8 @@ export const Playground = ({ config }) => {
if (ok) env = [...env, ...meta.env.filter((e) => !env.includes(e))];
}
// Same for env declared on the selected role.
if (modeMeta && modeMeta.env && modeMeta.env.length) {
env = [...env, ...modeMeta.env.filter((e) => !env.includes(e))];
if (modeOk && roleSpec && roleSpec.env && roleSpec.env.length) {
env = [...env, ...roleSpec.env.filter((e) => !env.includes(e))];
}
}
return { flags, env };
@@ -1034,12 +1102,19 @@ export const Playground = ({ config }) => {
return null;
},
render: ({ axisId, value, setValue, fc, base, s, renderSelect }) => {
render: ({ axisId, value, setValue, fc, base, s, h, renderSelect }) => {
const setSlot = (k, v) => setValue({ ...value, [k]: v });
const showModes = (fc.modes || []).length > 0;
const showBackends = (fc.transferBackends || []).length > 0;
const showIb = (fc.ibDevices || []).length > 0;
if (!showModes && !showBackends && !showIb) return null;
// `notes: [{when, mode, text}]` — a prerequisite the role needs from
// another card, surfaced only while the selection actually trips it.
// `when` matches against constraintBase, so it can key off cross-axis
// facts like dpAttnOn, not just the five cell dimensions.
const note = (fc.notes || []).find(
(n) => n && (!n.mode || [].concat(n.mode).includes(value.mode))
&& h.matchConstraint(base, n.when));
return (
<div key={axisId} style={s.card}>
<div style={s.compactRow}>
@@ -1066,6 +1141,7 @@ export const Playground = ({ config }) => {
</span>
)}
</div>
{note && <div style={s.axisNote}>{note.text}</div>}
</div>
);
},
@@ -1168,10 +1244,7 @@ export const Playground = ({ config }) => {
const backendOptions = fc.backends || [];
const ownedHeads = [
"--enable-hierarchical-cache", "--hicache-ratio", "--hicache-size",
"--hicache-write-policy", "--hicache-mem-layout", "--hicache-io-backend",
"--hicache-storage-backend", "--hicache-storage-prefetch-policy",
"--hicache-storage-backend-extra-config",
...HICACHE_HEADS,
...((fc.requiredFlags || []).map((f) => f.split(/\s/)[0])),
...backendOptions.flatMap((o) => (o.flags || []).map((f) => f.split(/\s/)[0])),
];
@@ -1281,6 +1354,91 @@ export const Playground = ({ config }) => {
},
},
// ---- Axis: UMBP (unified cache external linker) -------------------------
// A SIBLING of HiCache, not a tier inside it: the unified radix tree loads
// and offloads straight against an external store with no host cache tier,
// and sglang refuses the pair outright (arg_groups/hicache_hook.py raises on
// --enable-hierarchical-cache or --hicache-storage-backend alongside it).
// So enabling this strips the whole HiCache family instead of layering on
// it, and it is declared after hicache so that strip runs last.
umbp: {
initState: () => ({ enable: null, backend: null }),
deriveFromBase: (cell, fc, h) => {
const flags = (cell && cell.flags) || [];
return {
enable: h.hasFlag(flags, "--enable-unified-cache-external-linker"),
backend: h.findFlagArg(flags, "--unified-cache-external-linker-backend"),
};
},
apply: ({ flags, env, value, fc, sel, h, derived }) => {
const ownedHeads = [
"--enable-unified-cache-external-linker",
"--unified-cache-external-linker-backend",
...((fc.requiredFlags || []).map((f) => f.split(/\s/)[0])),
];
flags = h.stripFlagsByFirstToken(flags, ownedHeads);
if (fc.requiredEnv && fc.requiredEnv.length) {
env = h.stripEnvByPrefix(env, fc.requiredEnv.map((e) => e.split("=")[0]));
}
const enabled = value.enable !== null
? value.enable : !!(derived && derived.enable);
if (!enabled) return { flags, env };
if (fc.onlyHw && sel && !fc.onlyHw.includes(sel.hw)) return { flags, env };
// The linker keys by DP rank; under pure TP each rank opens its own
// keyspace and the store holds TP copies of the same tokens, so the
// recipe is only meaningful with DP attention on. Read it off the live
// flags rather than the Deploy dims — the attention axis runs first.
if (fc.requiresDpAttention
&& !flags.some((f) => f.split(/[\s=]/)[0] === "--enable-dp-attention")) {
return { flags, env };
}
flags = h.stripFlagsByFirstToken(flags, HICACHE_HEADS);
const backend = value.backend
|| (derived && derived.backend) || fc.defaultBackend || "mori";
flags = h.insertBeforeTail(flags, [
"--enable-unified-cache-external-linker",
`--unified-cache-external-linker-backend ${backend}`,
...(fc.requiredFlags || []),
]);
env = [...env, ...(fc.requiredEnv || []).filter((e) => !env.includes(e))];
return { flags, env };
},
render: ({ axisId, value, setValue, fc, base, s, renderChip, renderSelect, derived }) => {
if (fc.onlyHw && !fc.onlyHw.includes(base.hw)) return null;
const setSlot = (k, v) => setValue({ ...value, [k]: v });
const enabled = value.enable !== null
? value.enable : !!(derived && derived.enable);
const needsDp = !!fc.requiresDpAttention && !base.dpAttnOn;
const backend = value.backend !== null
? value.backend : ((derived && derived.backend) || fc.defaultBackend || "mori");
return (
<div key={axisId} style={s.card}>
<div style={s.compactRow}>
<span style={s.axisTitle}>UMBP</span>
<span style={s.field}>
{renderChip("Enable", enabled, true,
() => setSlot("enable", !enabled),
{ disabled: needsDp,
disabledReason: needsDp
? "Needs DP Attention — the linker keyspace is per DP rank, so under pure TP the store holds one copy per TP rank."
: "" })}
</span>
{(fc.backends || []).length > 0 && (
<span style={s.field}>
<span style={s.fieldLabel}>Store</span>
{renderSelect(backend, fc.backends,
(v) => setSlot("backend", v), base)}
</span>
)}
</div>
</div>
);
},
},
// ---- Axis: Flag Selects (generic, config-declared) ----------------------
// A LIST of single-selects, each declared entirely in config:
// { id, title, stripPrefixes: [...], stripEnv?: [...],
@@ -1463,6 +1621,15 @@ export const Playground = ({ config }) => {
let flags = [...baseFlags];
let env = [...(baseEnv || [])];
let pdMode = null;
// The selected PD role, resolved up front rather than from the loop's
// getRenderHints: axes that run BEFORE pdDisagg still need it, since a role
// can constrain what another axis is allowed to emit (a TP-only role has to
// force DP-Attention off in the attention axis, which composes earlier).
const pdFc = pgFeatures.pdDisagg;
const pdDelta = allDeltas.pdDisagg;
const pdRoleSel = (pdFc && (pdFc.modes || []).length && pdDelta)
? pdDelta.mode
: ((sel && sel.pdMode) || "off");
for (const [axisId, handler] of Object.entries(AXIS_HANDLERS)) {
const fc = pgFeatures[axisId];
if (!fc) continue;
@@ -1471,7 +1638,7 @@ export const Playground = ({ config }) => {
const derived = derivedMap ? derivedMap[axisId] : null;
const specAlgorithm = (findFlagArg(
flags, "--speculative-algorithm") || "").toUpperCase() || null;
const liveSel = { ...sel, specAlgorithm };
const liveSel = { ...sel, specAlgorithm, pdMode: pdRoleSel };
const out = handler.apply({ flags, env, value, fc, sel: liveSel, h: helpers, derived });
flags = out.flags;
env = out.env;
@@ -1541,17 +1708,41 @@ export const Playground = ({ config }) => {
&& config.dockerHostNetworkWhen(sel, { flags: f, env: cellEnv }));
// Mirrors `multiNodeDockerFlags` on the _deployment.jsx HARDWARE_CATALOG
// (Mintlify strips module state, so the engines cannot share it).
const AMD_RDMA_DOCKER_FLAGS = [
"--device /dev/infiniband", "--cap-add IPC_LOCK",
"--ulimit memlock=-1", "--ulimit stack=67108864",
"--ulimit nofile=1048576:1048576",
];
const HW_MULTINODE_DOCKER_FLAGS = {
"dgx-spark": [
"--ulimit memlock=-1:-1", "--cap-add IPC_LOCK", "--device /dev/infiniband",
],
mi300x: AMD_RDMA_DOCKER_FLAGS,
mi325x: AMD_RDMA_DOCKER_FLAGS,
mi350x: AMD_RDMA_DOCKER_FLAGS,
mi355x: AMD_RDMA_DOCKER_FLAGS,
};
const fabricFlags = HW_MULTINODE_DOCKER_FLAGS[sel.hw] || [];
// Mirrors the vendor branch in _deployment.jsx: ROCm reaches its GPUs
// through /dev/kfd + /dev/dri and the video group, not --gpus all.
const isAmdHw = /^mi\d/.test(sel.hw || "");
const dockerLines = [
"docker run --gpus all",
" --shm-size 32g",
...(isAmdHw
? [
"docker run",
" --device=/dev/kfd --device=/dev/dri",
" --group-add video",
" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined",
" --shm-size 32g",
]
: [
"docker run --gpus all",
" --shm-size 32g",
]),
hostNetwork ? " --network host" : ` -p ${servePort}:${servePort}`,
...(multinode ? fabricFlags.map((x) => " " + x) : []),
// A PD pair is cross-host even when each role is a single-node cell, so
// the RDMA fabric flags are needed for `pdMode` too, not just multinode.
...((multinode || pdMode) ? fabricFlags.map((x) => " " + x) : []),
" -v ~/.cache/huggingface:/root/.cache/huggingface",
...(config.dockerMounts || []).map((mount) => ` -v ${mount}`),
` --env "HF_TOKEN={{HF_TOKEN}}"`,
@@ -1582,9 +1773,8 @@ export const Playground = ({ config }) => {
}
if (pdMode === "prefill" || pdMode === "decode") {
const sibling = pdMode === "prefill" ? "decode" : "prefill";
const routerCfg = config.playgroundFeatures
&& config.playgroundFeatures.pdDisagg
&& config.playgroundFeatures.pdDisagg.router;
const routerCfg = resolveRouter(config.playgroundFeatures
&& config.playgroundFeatures.pdDisagg, sel);
const routerPort = (routerCfg && routerCfg.port) || 8000;
const routerLine = routerCfg
? `# then front BOTH with the Router shown below.\n`
@@ -2265,9 +2455,18 @@ export const Playground = ({ config }) => {
? matchedCell : null;
const pgSpecAlgoFlag = pgFlagsLatest.find(
(f) => f.split(/[\s=]/)[0] === "--speculative-algorithm");
// Off PD, the hint is a "you forgot to set it" warning and disappears once
// the ceiling is explicit. A PD role always ships one, so that form would
// never fire there — but the role is exactly where the sizing rule bites,
// since the ceiling has to be set on both roles and the decode graphs sized
// to what it leaves per rank. Under a role the hint switches to that form and
// shows whether or not the flag is present.
const pgPdRole = pdMode === "prefill" || pdMode === "decode";
const pgSpecHint =
!!pgSpecAlgoFlag &&
!pgFlagsLatest.some((f) => f.split(/[\s=]/)[0] === "--max-running-requests");
(pgPdRole
|| !pgFlagsLatest.some(
(f) => f.split(/[\s=]/)[0] === "--max-running-requests"));
const specAlgoLabels = {
EAGLE: "MTP", EAGLE3: "MTP", FROZEN_KV_MTP: "MTP",
DSPARK: "DSpark", DFLASH: "DFlash", NGRAM: "N-gram",
@@ -2305,9 +2504,8 @@ export const Playground = ({ config }) => {
// PD-Disagg router, if configured. When a PD role is active, cURL retargets
// to the router port and a companion router block renders below the command.
const pdRouter = (pdMode !== "off"
&& config.playgroundFeatures
&& config.playgroundFeatures.pdDisagg
&& config.playgroundFeatures.pdDisagg.router) || null;
&& resolveRouter(config.playgroundFeatures
&& config.playgroundFeatures.pdDisagg, base)) || null;
const curlEnv = (pdRouter && pdRouter.port != null)
? { ...env, CURL_PORT: String(pdRouter.port) }
: env;
@@ -2552,7 +2750,11 @@ export const Playground = ({ config }) => {
</pre>
{pgSpecHint && (
<div style={s.mtpWarn}>
⚠️ Speculative decoding ({pgSpecAlgoName}) is on — SGLang resets <code>--max-running-requests</code> to <strong>48</strong> when it isn't set. Add <code>--max-running-requests &lt;N&gt;</code> sized for your target concurrency.
{pgPdRole ? (
<>⚠️ Speculative decoding ({pgSpecAlgoName}) is on — for a target concurrency of N, set <code>--max-running-requests &lt;N*2&gt;</code> on <strong>both</strong> the prefill and decode roles. That ceiling is server-wide and floor-divided by <code>attn_dp_size</code>, so size the decode graphs to the per-rank batch it leaves: <code>--cuda-graph-bs-decode</code> up to <code>N*2 / dp_size</code>.</>
) : (
<>⚠️ Speculative decoding ({pgSpecAlgoName}) is on — SGLang resets <code>--max-running-requests</code> to <strong>48</strong> when it isn't set. Add <code>--max-running-requests &lt;N&gt;</code> sized for your target concurrency.</>
)}
</div>
)}
{pgCpDpHint && (
docs/src/snippets/configs/deepseek-ai/deepseek-v4.jsx
+221 -6
View File
@@ -258,6 +258,22 @@ sgl-eval run mmmu_pro \\
reason: "Prefill Context Parallel is single-machine only (SGLang asserts tp_size <= 8; cross-machine CP has precision issues)." },
] },
{ id: "dpAttn", label: "DP-Attention",
// The low-latency and balanced PD roles run TP-only. That is what lets
// their decode ladders run to the full ceiling (8 and 96): the ceiling
// is server-wide and floor-divided by attn_dp_size, so only at dp_size
// 1 is it also the per-rank batch. Forced rather than left to the
// reader, because switching DP on would cut the slots per rank without
// changing either flag in the command — the ladder would still read 96
// while the role could only ever fill 12. High-throughput is the DP
// point and is deliberately absent here.
forceOff: [
{ when: { hw: ["mi355x"], strategy: ["low-latency", "balanced"],
pdMode: ["prefill", "decode"] },
stripEnv: ["SGLANG_DP_SHARED_EXPERT_LOCAL",
"SGLANG_DP_USE_GATHERV",
"SGLANG_DP_USE_REDUCE_SCATTER"],
reason: "The low-latency and balanced PD roles are TP-only, which is what makes --cuda-graph-bs-decode equal the full ceiling. --max-running-requests is server-wide and floor-divided by attn_dp_size, so DP would cut the per-rank batch below the captured graphs." },
],
values: [
null,
false,
@@ -324,10 +340,15 @@ sgl-eval run mmmu_pro \\
options: [
{ id: "current", label: "Inherited from base" },
{ id: "off", label: "Off (greedy)" },
// Shown on pro-official as well as the originals: the 0813 checkpoint
// bundles a DSpark head but keeps its MTP head, and DSpark cannot run
// under PD disaggregation (§3.8), so this shape is the speculative
// option a PD role actually has. The 1-1-2 shape stays hidden there —
// off PD, DSpark is the better pick, so only one fallback is offered.
{ id: "mtp-314", label: "EAGLE / MTP 3-1-4",
flags: ["--speculative-algorithm EAGLE", "--speculative-num-steps 3",
"--speculative-eagle-topk 1", "--speculative-num-draft-tokens 4"],
hide: { variant: ["flash-official", "flash-vision", "pro-official"] } },
hide: { variant: ["flash-official", "flash-vision"] } },
{ id: "mtp-112", label: "EAGLE / MTP 1-1-2",
flags: ["--speculative-algorithm EAGLE", "--speculative-num-steps 1",
"--speculative-eagle-topk 1", "--speculative-num-draft-tokens 2"],
@@ -357,8 +378,42 @@ sgl-eval run mmmu_pro \\
incompatibleSpeculativeAlgorithms: ["DSPARK"],
modes: [
{ id: "off", label: "Off" },
{ id: "prefill", label: "Prefill role" },
{ id: "decode", label: "Decode role" },
// The AMD role flags are the MI355X 1P x 1D agentic recipe. Both roles
// are gated by `when` because the sizing is ROCm-specific, and they
// differ in two places: the prefill worker runs eager (the dsv4 indexer's
// prefill path is not graph-captured) and dispatches whole chunked-prefill
// batches over MORI, while the decode worker captures graphs for its
// small batch ladder and dispatches at most a step's worth of tokens.
{ id: "prefill", label: "Prefill role",
when: { hw: ["mi355x"], strategy: ["low-latency"] },
env: ["SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16384"],
flags: [
"--load-balance-method round_robin",
"--tokenizer-worker-num 8",
"--stream-interval 20",
"--mem-fraction-static 0.86",
"--max-running-requests 8",
"--swa-full-tokens-ratio 0.1",
"--disable-cuda-graph",
"--context-length 1048576",
"--watchdog-timeout 3600",
"--enable-metrics",
] },
{ id: "decode", label: "Decode role",
when: { hw: ["mi355x"], strategy: ["low-latency"] },
env: ["SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128"],
flags: [
"--load-balance-method round_robin",
"--tokenizer-worker-num 8",
"--stream-interval 20",
"--mem-fraction-static 0.86",
"--max-running-requests 8",
"--swa-full-tokens-ratio 0.1",
"--cuda-graph-bs-decode 1 2 3 4 5 6 7 8",
"--context-length 1048576",
"--watchdog-timeout 3600",
"--enable-metrics",
] },
],
transferBackends: [
{ id: "mooncake", label: "Mooncake",
@@ -372,10 +427,29 @@ sgl-eval run mmmu_pro \\
{ id: "nixl", label: "NiXL" },
// MORI-IO transport is AMD-only — hidden on every non-ROCm platform.
{ id: "mori", label: "MORI",
hide: { hw: ["h100", "h200", "b200", "b300", "gb200", "gb300", "rtx6000"] } },
hide: { hw: ["h100", "h200", "b200", "b300", "gb200", "gb300", "rtx6000"] },
// Transport-wide only. The per-rank dispatch budget is sized per role
// (see `modes` above), so it lives on the role, not here.
env: [
"SGLANG_MORI_COMBINE_DTYPE=auto",
"MORI_IO_SQ_BACKOFF_TIMEOUT_US=500000",
"MORI_IO_QP_MAX_SEND_WR=32767",
],
envWhen: { hw: ["mi300x", "mi355x"] } },
],
// `auto` is a sentinel (emits no --disaggregation-ib-device flag).
ibDevices: [{ id: "auto", label: "Auto" }, "mlx5_0", "mlx5_7"],
// The mlx5 names are ConnectX; ROCm nodes enumerate their NICs as rdmaN,
// so the two families are mutually hidden. The AMD entry is the full
// 8-NIC list in one value because --disaggregation-ib-device takes a
// comma list, and the order is the MI355X NUMA-local pairing.
ibDevices: [
{ id: "auto", label: "Auto" },
{ id: "mlx5_0", label: "mlx5_0", hide: { hw: ["mi300x", "mi355x"] } },
{ id: "mlx5_7", label: "mlx5_7", hide: { hw: ["mi300x", "mi355x"] } },
{ id: "rdma3,rdma0,rdma2,rdma1,rdma7,rdma4,rdma6,rdma5",
label: "rdma0-7 (all NICs)",
hide: { hw: ["h100", "h200", "b200", "b300", "gb200", "gb300", "rtx6000", "rtx5090", "dgx-spark"] } },
],
// Router fronting the prefill + decode roles; substitute <prefill-host>/<decode-host>.
router: {
port: 8000,
@@ -388,6 +462,98 @@ sgl-eval run mmmu_pro \\
--disable-circuit-breaker \\
--health-check-interval-secs 999999`,
},
// The MI355X roles above size for low latency: TP-only, a running-request
// ceiling in the single digits, and a MORI dispatch budget per role. The
// high-throughput point is the same two roles re-sized against the DP
// base cell — TP8/DP8 and the wider batch come from that cell, so these
// only carry what the operating point itself changes. The decode graph
// ladder grows to 32 to cover the larger steady-state batch, and
// --enable-cache-report surfaces the prefix hit rate that decides whether
// the offload tier is paying for itself at this concurrency.
// The balanced point sits between the two: TP-only like low-latency, but
// with a 96-request ceiling and a HiCache tier under the prefill role
// (see the hicache roleOverride below) instead of low-latency's bare KV
// pool or high-throughput's UMBP. It re-sizes more of the base cell than
// the other two because the balanced cell is a DP recipe for aggregated
// serving — its 0.90 / 0.15 / 65536 sizing does not carry over.
roleOverrides: [
{ mode: "prefill",
when: { hw: ["mi355x"], strategy: ["balanced"] },
env: ["SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16384"],
flags: [
"--load-balance-method round_robin",
"--mem-fraction-static 0.86",
"--max-running-requests 96",
"--swa-full-tokens-ratio 0.1",
"--chunked-prefill-size 16384",
"--disable-cuda-graph",
"--context-length 1048576",
"--watchdog-timeout 3600",
"--enable-metrics",
] },
// TP-only, so the server-wide ceiling is also the per-rank batch and
// the graph ladder runs all the way to 96.
{ mode: "decode",
when: { hw: ["mi355x"], strategy: ["balanced"] },
env: ["SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128"],
flags: [
"--load-balance-method round_robin",
"--mem-fraction-static 0.86",
"--max-running-requests 96",
"--swa-full-tokens-ratio 0.1",
"--cuda-graph-bs-decode 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96",
"--context-length 1048576",
"--watchdog-timeout 3600",
"--enable-metrics",
] },
{ mode: "prefill",
when: { hw: ["mi355x"], strategy: ["high-throughput"] },
env: ["SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16384"],
flags: [
"--load-balance-method round_robin",
"--mem-fraction-static 0.92",
"--max-running-requests 256",
"--disable-cuda-graph",
"--context-length 1048576",
"--watchdog-timeout 3600",
"--enable-metrics",
"--enable-cache-report",
] },
{ mode: "decode",
when: { hw: ["mi355x"], strategy: ["high-throughput"] },
env: ["SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128"],
flags: [
"--load-balance-method round_robin",
"--mem-fraction-static 0.92",
"--max-running-requests 256",
"--cuda-graph-bs-decode 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32",
"--context-length 1048576",
"--watchdog-timeout 3600",
"--enable-metrics",
] },
],
// MI355X fronts the 1P x 1D agentic pair with a cache-aware router rather
// than the default round-robin: consistent_hashing keeps a conversation on
// the worker that already holds its prefix, and the tight balance
// thresholds stop that affinity from starving the peer at the small
// running-request ceiling the roles above use. Health checking stays
// enabled here (unlike the default's 999999s interval) because a long
// agentic run should notice a wedged worker, but it is slack enough that
// a multi-minute prefill is not mistaken for a failure.
routerOverrides: [
{ when: { hw: ["mi355x"] },
command:
`python3 -m sglang_router.launch_router \\
--pd-disaggregation \\
--prefill http://<prefill-host>:{{PREFILL_PORT}} \\
--decode http://<decode-host>:{{DECODE_PORT}} \\
--host 0.0.0.0 --port {{ROUTER_PORT}} \\
--policy consistent_hashing --dp-aware \\
--cache-threshold 0.3 \\
--balance-abs-threshold 2 --balance-rel-threshold 1.1 \\
--disable-circuit-breaker --health-failure-threshold 100 \\
--health-check-timeout-secs 600 --health-check-interval-secs 30` },
],
},
// ----- Card 6: "Hierarchical KV Cache" -----
@@ -409,6 +575,23 @@ sgl-eval run mmmu_pro \\
writePolicy: "write_through",
prefetchPolicy: "best_effort",
},
// Balanced on Pro Official: the same shape at a smaller ratio. 2.5 is
// what the 96-request ceiling leaves room for once mem-fraction-static
// drops to 0.86 — the host tier competes with the KV pool for the
// headroom the prefill role gives up.
{
when: {
hw: ["mi355x"], variant: ["pro-official"], quant: ["fp4"],
strategy: ["balanced"], nodes: ["single"],
},
mode: "prefill",
transferBackend: "mori",
memLayout: "page_first",
ioBackend: "direct",
ratio: 2.5,
writePolicy: "write_through",
prefetchPolicy: "best_effort",
},
],
notices: [
{
@@ -420,6 +603,15 @@ sgl-eval run mmmu_pro \\
transferBackend: "mori",
text: "HiCache is not recommended on the decode role with MORI.",
},
{
when: {
hw: ["mi355x"], variant: ["pro-official"], quant: ["fp4"],
strategy: ["balanced"], nodes: ["single"],
},
mode: "decode",
transferBackend: "mori",
text: "HiCache is not recommended on the decode role with MORI.",
},
],
amdStorageFileOnly: true,
backends: [
@@ -440,7 +632,30 @@ sgl-eval run mmmu_pro \\
],
},
// ----- Card 7: "HiSparse" -----
// ----- Card 7: "UMBP" (unified cache external linker) -----
// Sits beside HiCache rather than inside it. HiCache is a tiered cache
// (GPU -> pinned host -> optional storage); UMBP links the unified radix
// tree DIRECTLY to an external store with no host tier at all, so the two
// are alternatives and sglang rejects them together. Enabling this card
// therefore strips the HiCache family from the command.
//
// ROCm-only in practice: the store is MORI's buffer pool, the same
// transport the PD roles use, and there is no CUDA recipe for it yet.
umbp: {
onlyHw: ["mi300x", "mi355x"],
// Under pure TP the linker keys by rank, so an 8-rank prefill worker
// opens eight keyspaces and the pool holds eight copies of the same MLA
// KV — a tier an eighth the size its byte budget suggests. DP attention
// collapses the keys onto one shared keyspace.
requiresDpAttention: true,
defaultBackend: "mori",
backends: [
{ id: "mori", label: "MORI (UMBP)" },
{ id: "mooncake", label: "Mooncake" },
],
},
// ----- Card 8: "HiSparse" -----
// Decode-only: shown/emitted only when the live PD-Disagg mode is `decode`.
hisparse: {
requiredFlags: [