[docs] DeepSeek-V4: MI355X PD disaggregation recipes for all three strategies (#39396)
This commit is contained in:
Theresa Shan
@@ -109,6 +109,11 @@ export const Deployment = ({ config, benchmarks }) => {
|
||||
|
||||
// ==== 1. Hardware catalog (shared across cookbooks) ====
|
||||
// VRAM is per-GPU on-chip memory, not per-module.
|
||||
const AMD_RDMA_DOCKER_FLAGS = [
|
||||
"--device /dev/infiniband", "--cap-add IPC_LOCK",
|
||||
"--ulimit memlock=-1", "--ulimit stack=67108864",
|
||||
"--ulimit nofile=1048576:1048576",
|
||||
];
|
||||
const HARDWARE_CATALOG = {
|
||||
blackwell: [
|
||||
{ id: "b300", label: "B300", vram: "288GB" },
|
||||
@@ -128,11 +133,19 @@ export const Deployment = ({ config, benchmarks }) => {
|
||||
{ id: "h20-3e", label: "H20-3e", vram: "141GB" },
|
||||
{ id: "h800", label: "H800", vram: "80GB" },
|
||||
],
|
||||
// ROCm multi-node runs the RDMA NICs straight through: /dev/infiniband
|
||||
// covers rdma_cm plus the per-NIC uverbsN nodes, IPC_LOCK + an unlimited
|
||||
// memlock let the transport pin its registered buffers, and the stack /
|
||||
// nofile raises are for the per-QP file descriptors a full 8-NIC mesh opens.
|
||||
amd: [
|
||||
{ id: "mi300x", label: "MI300X", vram: "192GB" },
|
||||
{ id: "mi325x", label: "MI325X", vram: "256GB" },
|
||||
{ id: "mi350x", label: "MI350X", vram: "288GB" },
|
||||
{ id: "mi355x", label: "MI355X", vram: "288GB" },
|
||||
{ id: "mi300x", label: "MI300X", vram: "192GB",
|
||||
multiNodeDockerFlags: [...AMD_RDMA_DOCKER_FLAGS] },
|
||||
{ id: "mi325x", label: "MI325X", vram: "256GB",
|
||||
multiNodeDockerFlags: [...AMD_RDMA_DOCKER_FLAGS] },
|
||||
{ id: "mi350x", label: "MI350X", vram: "288GB",
|
||||
multiNodeDockerFlags: [...AMD_RDMA_DOCKER_FLAGS] },
|
||||
{ id: "mi355x", label: "MI355X", vram: "288GB",
|
||||
multiNodeDockerFlags: [...AMD_RDMA_DOCKER_FLAGS] },
|
||||
],
|
||||
// Atlas 800I A3 (910C): 1 card = 2 dies, so --tp-size is 2× the card
|
||||
// count (32 cards -> --tp-size 64).
|
||||
|
||||
@@ -225,6 +225,17 @@ export const Playground = ({ config }) => {
|
||||
Array.isArray(vs) && vs.includes(base[k]));
|
||||
};
|
||||
|
||||
// The PD router is shared by every cell, so a platform whose prefill/decode
|
||||
// pair wants a different routing policy declares `routerOverrides: [{when,
|
||||
// port?, command}]` instead of rewriting the default out from under the rest.
|
||||
// First match wins; fields the override omits fall back to the default.
|
||||
const resolveRouter = (fc, sel) => {
|
||||
if (!fc || !fc.router) return null;
|
||||
const hit = (fc.routerOverrides || []).find(
|
||||
(r) => r && matchConstraint(sel, r.when));
|
||||
return hit ? { ...fc.router, ...hit } : fc.router;
|
||||
};
|
||||
|
||||
// Normalize a chip entry into `{value, label?, hidden, disabled,
|
||||
// disableReason, ...rest}`. `value` resolves to `entry.id` (rich form) or
|
||||
// `entry.value` (wrapper form), or the entry itself for bare values.
|
||||
@@ -379,6 +390,16 @@ export const Playground = ({ config }) => {
|
||||
ANCHOR_NEAR_DPATTN, ANCHOR_NEAR_MOE,
|
||||
};
|
||||
|
||||
// -------- HiCache flag family --------
|
||||
// Shared because two axes own it: `hicache` emits it, and `umbp` strips the
|
||||
// whole family when it takes the tier over (the two are mutually exclusive).
|
||||
const HICACHE_HEADS = [
|
||||
"--enable-hierarchical-cache", "--hicache-ratio", "--hicache-size",
|
||||
"--hicache-write-policy", "--hicache-mem-layout", "--hicache-io-backend",
|
||||
"--hicache-storage-backend", "--hicache-storage-prefetch-policy",
|
||||
"--hicache-storage-backend-extra-config",
|
||||
];
|
||||
|
||||
// -------- Prefill-CP flag family (shared by the attention axis) --------
|
||||
// Every flag head that toggles/parameterizes prefill context parallelism:
|
||||
// the canonical pair plus all per-family legacy spellings.
|
||||
@@ -531,16 +552,44 @@ export const Playground = ({ config }) => {
|
||||
]);
|
||||
}
|
||||
}
|
||||
if (value.dpAttn !== null && value.dpAttn !== undefined
|
||||
// `forceOff: [{when, stripEnv, reason}]` on a knob — the selection
|
||||
// dictates the value, so neither the pick nor the base cell gets a say.
|
||||
// Used where a PD role runs a different topology than the aggregated
|
||||
// cell it derives from: the role is TP-only, but the cell is DP.
|
||||
const dpForced = (knobEntry("dpAttn").forceOff || []).find(
|
||||
(r) => r && h.matchConstraint(factsNow(), r.when));
|
||||
if (dpForced) {
|
||||
flags = h.stripFlagsByFirstToken(flags, [
|
||||
...h.DP_HEADS, "--enable-dp-attention",
|
||||
"--enable-dp-attention-local-control-broadcast",
|
||||
]);
|
||||
// The cell's DP-only env would otherwise outlive the flags it tunes.
|
||||
const stripEnv = dpForced.stripEnv || [];
|
||||
if (stripEnv.length) {
|
||||
env = env.filter((e) => !stripEnv.includes(e.split("=")[0]));
|
||||
}
|
||||
} else if (value.dpAttn !== null && value.dpAttn !== undefined
|
||||
&& !blocked("dpAttn", value.dpAttn)) {
|
||||
// Capture the spelling before stripping — the TP/EP handlers do the
|
||||
// same, and a lookup on the stripped array always hits the fallback.
|
||||
const dpHead = h.flagSpelling(flags, h.DP_HEADS, "--dp-size");
|
||||
flags = h.stripFlagsByFirstToken(flags, [...h.DP_HEADS, "--enable-dp-attention"]);
|
||||
// The local-control-broadcast companion only means anything with DP
|
||||
// attention on, so it has to go down with it — stripping just
|
||||
// `--enable-dp-attention` would leave it orphaned in the command.
|
||||
// apply re-seeds from the base cell, so this restores it when the
|
||||
// cell had it and the user is only re-sizing DP rather than disabling.
|
||||
const hadLocalBroadcast =
|
||||
h.hasFlag(flags, "--enable-dp-attention-local-control-broadcast");
|
||||
flags = h.stripFlagsByFirstToken(flags, [
|
||||
...h.DP_HEADS, "--enable-dp-attention",
|
||||
"--enable-dp-attention-local-control-broadcast",
|
||||
]);
|
||||
if (typeof value.dpAttn === "number" && value.dpAttn > 0) {
|
||||
flags = h.insertAfter(flags, h.ANCHOR_NEAR_TP, [
|
||||
`${dpHead} ${value.dpAttn}`,
|
||||
"--enable-dp-attention",
|
||||
...(hadLocalBroadcast
|
||||
? ["--enable-dp-attention-local-control-broadcast"] : []),
|
||||
]);
|
||||
}
|
||||
}
|
||||
@@ -602,14 +651,18 @@ export const Playground = ({ config }) => {
|
||||
{knobs.map((knob) => {
|
||||
const kc = h.evaluateChip(knob, base);
|
||||
if (kc.hidden) return null;
|
||||
// Mirror apply's forceOff: show the value the command actually
|
||||
// gets, grayed, instead of a live-looking pick apply discards.
|
||||
const forced = (knob.forceOff || []).find(
|
||||
(r) => r && h.matchConstraint(base, r.when));
|
||||
return (
|
||||
<span key={knob.id} style={s.field}>
|
||||
<span style={s.fieldLabel}>{knob.label || knob.id.toUpperCase()}</span>
|
||||
{renderSelect(knobDisplay(knob), entriesFor(knob),
|
||||
{renderSelect(forced ? false : knobDisplay(knob), entriesFor(knob),
|
||||
(nv) => setKnob(knob.id, nv), base, labelFor(knob),
|
||||
{ hideValues: hideNullFor(knob),
|
||||
disabled: kc.disabled,
|
||||
disabledReason: kc.disableReason })}
|
||||
disabled: kc.disabled || !!forced,
|
||||
disabledReason: forced ? forced.reason : kc.disableReason })}
|
||||
</span>
|
||||
);
|
||||
})}
|
||||
@@ -985,10 +1038,25 @@ export const Playground = ({ config }) => {
|
||||
// render, so a base flag is never left over from an earlier selection
|
||||
// and must not be stripped when the role is Off.
|
||||
const modeMeta = (fc.modes || []).find((m) => m.id === mode);
|
||||
if (modeMeta && modeMeta.flags && modeMeta.flags.length) {
|
||||
// A role may scope its flags/env to a subset of hardware via `when`
|
||||
// (same shape as a backend's `envWhen`), so a ROCm-only prefill recipe
|
||||
// doesn't leak onto CUDA. Absent `when`, the role applies everywhere.
|
||||
//
|
||||
// `roleOverrides: [{when, mode, flags, env}]` then re-specifies a role
|
||||
// for a narrower selection — a second operating point (low-latency vs
|
||||
// high-throughput sizing) is a different flag set for the SAME role,
|
||||
// so it belongs here rather than as a duplicate entry in the Mode
|
||||
// select. First match wins and replaces the role's own flags/env.
|
||||
const roleOverride = (fc.roleOverrides || []).find((r) =>
|
||||
r && r.mode === mode && r.when && h.matchConstraint(sel, r.when));
|
||||
const roleSpec = roleOverride || modeMeta;
|
||||
const modeGate = roleOverride ? null : (modeMeta && modeMeta.when);
|
||||
const modeOk = !modeGate || Object.keys(modeGate).every(
|
||||
(k) => (modeGate[k] || []).includes(sel[k]));
|
||||
if (modeOk && roleSpec && roleSpec.flags && roleSpec.flags.length) {
|
||||
flags = h.stripFlagsByFirstToken(
|
||||
flags, modeMeta.flags.map((f) => f.split(/[\s=]/)[0]));
|
||||
adds.push(...modeMeta.flags);
|
||||
flags, roleSpec.flags.map((f) => f.split(/[\s=]/)[0]));
|
||||
adds.push(...roleSpec.flags);
|
||||
}
|
||||
// Single-host needs no --dist-init-addr: prefill/decode derive their
|
||||
// ZMQ/dist ports from the role-specific --port (spaced 100 apart, see
|
||||
@@ -1015,8 +1083,8 @@ export const Playground = ({ config }) => {
|
||||
if (ok) env = [...env, ...meta.env.filter((e) => !env.includes(e))];
|
||||
}
|
||||
// Same for env declared on the selected role.
|
||||
if (modeMeta && modeMeta.env && modeMeta.env.length) {
|
||||
env = [...env, ...modeMeta.env.filter((e) => !env.includes(e))];
|
||||
if (modeOk && roleSpec && roleSpec.env && roleSpec.env.length) {
|
||||
env = [...env, ...roleSpec.env.filter((e) => !env.includes(e))];
|
||||
}
|
||||
}
|
||||
return { flags, env };
|
||||
@@ -1034,12 +1102,19 @@ export const Playground = ({ config }) => {
|
||||
return null;
|
||||
},
|
||||
|
||||
render: ({ axisId, value, setValue, fc, base, s, renderSelect }) => {
|
||||
render: ({ axisId, value, setValue, fc, base, s, h, renderSelect }) => {
|
||||
const setSlot = (k, v) => setValue({ ...value, [k]: v });
|
||||
const showModes = (fc.modes || []).length > 0;
|
||||
const showBackends = (fc.transferBackends || []).length > 0;
|
||||
const showIb = (fc.ibDevices || []).length > 0;
|
||||
if (!showModes && !showBackends && !showIb) return null;
|
||||
// `notes: [{when, mode, text}]` — a prerequisite the role needs from
|
||||
// another card, surfaced only while the selection actually trips it.
|
||||
// `when` matches against constraintBase, so it can key off cross-axis
|
||||
// facts like dpAttnOn, not just the five cell dimensions.
|
||||
const note = (fc.notes || []).find(
|
||||
(n) => n && (!n.mode || [].concat(n.mode).includes(value.mode))
|
||||
&& h.matchConstraint(base, n.when));
|
||||
return (
|
||||
<div key={axisId} style={s.card}>
|
||||
<div style={s.compactRow}>
|
||||
@@ -1066,6 +1141,7 @@ export const Playground = ({ config }) => {
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
{note && <div style={s.axisNote}>{note.text}</div>}
|
||||
</div>
|
||||
);
|
||||
},
|
||||
@@ -1168,10 +1244,7 @@ export const Playground = ({ config }) => {
|
||||
|
||||
const backendOptions = fc.backends || [];
|
||||
const ownedHeads = [
|
||||
"--enable-hierarchical-cache", "--hicache-ratio", "--hicache-size",
|
||||
"--hicache-write-policy", "--hicache-mem-layout", "--hicache-io-backend",
|
||||
"--hicache-storage-backend", "--hicache-storage-prefetch-policy",
|
||||
"--hicache-storage-backend-extra-config",
|
||||
...HICACHE_HEADS,
|
||||
...((fc.requiredFlags || []).map((f) => f.split(/\s/)[0])),
|
||||
...backendOptions.flatMap((o) => (o.flags || []).map((f) => f.split(/\s/)[0])),
|
||||
];
|
||||
@@ -1281,6 +1354,91 @@ export const Playground = ({ config }) => {
|
||||
},
|
||||
},
|
||||
|
||||
// ---- Axis: UMBP (unified cache external linker) -------------------------
|
||||
// A SIBLING of HiCache, not a tier inside it: the unified radix tree loads
|
||||
// and offloads straight against an external store with no host cache tier,
|
||||
// and sglang refuses the pair outright (arg_groups/hicache_hook.py raises on
|
||||
// --enable-hierarchical-cache or --hicache-storage-backend alongside it).
|
||||
// So enabling this strips the whole HiCache family instead of layering on
|
||||
// it, and it is declared after hicache so that strip runs last.
|
||||
umbp: {
|
||||
initState: () => ({ enable: null, backend: null }),
|
||||
|
||||
deriveFromBase: (cell, fc, h) => {
|
||||
const flags = (cell && cell.flags) || [];
|
||||
return {
|
||||
enable: h.hasFlag(flags, "--enable-unified-cache-external-linker"),
|
||||
backend: h.findFlagArg(flags, "--unified-cache-external-linker-backend"),
|
||||
};
|
||||
},
|
||||
|
||||
apply: ({ flags, env, value, fc, sel, h, derived }) => {
|
||||
const ownedHeads = [
|
||||
"--enable-unified-cache-external-linker",
|
||||
"--unified-cache-external-linker-backend",
|
||||
...((fc.requiredFlags || []).map((f) => f.split(/\s/)[0])),
|
||||
];
|
||||
flags = h.stripFlagsByFirstToken(flags, ownedHeads);
|
||||
if (fc.requiredEnv && fc.requiredEnv.length) {
|
||||
env = h.stripEnvByPrefix(env, fc.requiredEnv.map((e) => e.split("=")[0]));
|
||||
}
|
||||
const enabled = value.enable !== null
|
||||
? value.enable : !!(derived && derived.enable);
|
||||
if (!enabled) return { flags, env };
|
||||
if (fc.onlyHw && sel && !fc.onlyHw.includes(sel.hw)) return { flags, env };
|
||||
// The linker keys by DP rank; under pure TP each rank opens its own
|
||||
// keyspace and the store holds TP copies of the same tokens, so the
|
||||
// recipe is only meaningful with DP attention on. Read it off the live
|
||||
// flags rather than the Deploy dims — the attention axis runs first.
|
||||
if (fc.requiresDpAttention
|
||||
&& !flags.some((f) => f.split(/[\s=]/)[0] === "--enable-dp-attention")) {
|
||||
return { flags, env };
|
||||
}
|
||||
flags = h.stripFlagsByFirstToken(flags, HICACHE_HEADS);
|
||||
const backend = value.backend
|
||||
|| (derived && derived.backend) || fc.defaultBackend || "mori";
|
||||
flags = h.insertBeforeTail(flags, [
|
||||
"--enable-unified-cache-external-linker",
|
||||
`--unified-cache-external-linker-backend ${backend}`,
|
||||
...(fc.requiredFlags || []),
|
||||
]);
|
||||
env = [...env, ...(fc.requiredEnv || []).filter((e) => !env.includes(e))];
|
||||
return { flags, env };
|
||||
},
|
||||
|
||||
render: ({ axisId, value, setValue, fc, base, s, renderChip, renderSelect, derived }) => {
|
||||
if (fc.onlyHw && !fc.onlyHw.includes(base.hw)) return null;
|
||||
const setSlot = (k, v) => setValue({ ...value, [k]: v });
|
||||
const enabled = value.enable !== null
|
||||
? value.enable : !!(derived && derived.enable);
|
||||
const needsDp = !!fc.requiresDpAttention && !base.dpAttnOn;
|
||||
const backend = value.backend !== null
|
||||
? value.backend : ((derived && derived.backend) || fc.defaultBackend || "mori");
|
||||
return (
|
||||
<div key={axisId} style={s.card}>
|
||||
<div style={s.compactRow}>
|
||||
<span style={s.axisTitle}>UMBP</span>
|
||||
<span style={s.field}>
|
||||
{renderChip("Enable", enabled, true,
|
||||
() => setSlot("enable", !enabled),
|
||||
{ disabled: needsDp,
|
||||
disabledReason: needsDp
|
||||
? "Needs DP Attention — the linker keyspace is per DP rank, so under pure TP the store holds one copy per TP rank."
|
||||
: "" })}
|
||||
</span>
|
||||
{(fc.backends || []).length > 0 && (
|
||||
<span style={s.field}>
|
||||
<span style={s.fieldLabel}>Store</span>
|
||||
{renderSelect(backend, fc.backends,
|
||||
(v) => setSlot("backend", v), base)}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
},
|
||||
},
|
||||
|
||||
// ---- Axis: Flag Selects (generic, config-declared) ----------------------
|
||||
// A LIST of single-selects, each declared entirely in config:
|
||||
// { id, title, stripPrefixes: [...], stripEnv?: [...],
|
||||
@@ -1463,6 +1621,15 @@ export const Playground = ({ config }) => {
|
||||
let flags = [...baseFlags];
|
||||
let env = [...(baseEnv || [])];
|
||||
let pdMode = null;
|
||||
// The selected PD role, resolved up front rather than from the loop's
|
||||
// getRenderHints: axes that run BEFORE pdDisagg still need it, since a role
|
||||
// can constrain what another axis is allowed to emit (a TP-only role has to
|
||||
// force DP-Attention off in the attention axis, which composes earlier).
|
||||
const pdFc = pgFeatures.pdDisagg;
|
||||
const pdDelta = allDeltas.pdDisagg;
|
||||
const pdRoleSel = (pdFc && (pdFc.modes || []).length && pdDelta)
|
||||
? pdDelta.mode
|
||||
: ((sel && sel.pdMode) || "off");
|
||||
for (const [axisId, handler] of Object.entries(AXIS_HANDLERS)) {
|
||||
const fc = pgFeatures[axisId];
|
||||
if (!fc) continue;
|
||||
@@ -1471,7 +1638,7 @@ export const Playground = ({ config }) => {
|
||||
const derived = derivedMap ? derivedMap[axisId] : null;
|
||||
const specAlgorithm = (findFlagArg(
|
||||
flags, "--speculative-algorithm") || "").toUpperCase() || null;
|
||||
const liveSel = { ...sel, specAlgorithm };
|
||||
const liveSel = { ...sel, specAlgorithm, pdMode: pdRoleSel };
|
||||
const out = handler.apply({ flags, env, value, fc, sel: liveSel, h: helpers, derived });
|
||||
flags = out.flags;
|
||||
env = out.env;
|
||||
@@ -1541,17 +1708,41 @@ export const Playground = ({ config }) => {
|
||||
&& config.dockerHostNetworkWhen(sel, { flags: f, env: cellEnv }));
|
||||
// Mirrors `multiNodeDockerFlags` on the _deployment.jsx HARDWARE_CATALOG
|
||||
// (Mintlify strips module state, so the engines cannot share it).
|
||||
const AMD_RDMA_DOCKER_FLAGS = [
|
||||
"--device /dev/infiniband", "--cap-add IPC_LOCK",
|
||||
"--ulimit memlock=-1", "--ulimit stack=67108864",
|
||||
"--ulimit nofile=1048576:1048576",
|
||||
];
|
||||
const HW_MULTINODE_DOCKER_FLAGS = {
|
||||
"dgx-spark": [
|
||||
"--ulimit memlock=-1:-1", "--cap-add IPC_LOCK", "--device /dev/infiniband",
|
||||
],
|
||||
mi300x: AMD_RDMA_DOCKER_FLAGS,
|
||||
mi325x: AMD_RDMA_DOCKER_FLAGS,
|
||||
mi350x: AMD_RDMA_DOCKER_FLAGS,
|
||||
mi355x: AMD_RDMA_DOCKER_FLAGS,
|
||||
};
|
||||
const fabricFlags = HW_MULTINODE_DOCKER_FLAGS[sel.hw] || [];
|
||||
// Mirrors the vendor branch in _deployment.jsx: ROCm reaches its GPUs
|
||||
// through /dev/kfd + /dev/dri and the video group, not --gpus all.
|
||||
const isAmdHw = /^mi\d/.test(sel.hw || "");
|
||||
const dockerLines = [
|
||||
"docker run --gpus all",
|
||||
" --shm-size 32g",
|
||||
...(isAmdHw
|
||||
? [
|
||||
"docker run",
|
||||
" --device=/dev/kfd --device=/dev/dri",
|
||||
" --group-add video",
|
||||
" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined",
|
||||
" --shm-size 32g",
|
||||
]
|
||||
: [
|
||||
"docker run --gpus all",
|
||||
" --shm-size 32g",
|
||||
]),
|
||||
hostNetwork ? " --network host" : ` -p ${servePort}:${servePort}`,
|
||||
...(multinode ? fabricFlags.map((x) => " " + x) : []),
|
||||
// A PD pair is cross-host even when each role is a single-node cell, so
|
||||
// the RDMA fabric flags are needed for `pdMode` too, not just multinode.
|
||||
...((multinode || pdMode) ? fabricFlags.map((x) => " " + x) : []),
|
||||
" -v ~/.cache/huggingface:/root/.cache/huggingface",
|
||||
...(config.dockerMounts || []).map((mount) => ` -v ${mount}`),
|
||||
` --env "HF_TOKEN={{HF_TOKEN}}"`,
|
||||
@@ -1582,9 +1773,8 @@ export const Playground = ({ config }) => {
|
||||
}
|
||||
if (pdMode === "prefill" || pdMode === "decode") {
|
||||
const sibling = pdMode === "prefill" ? "decode" : "prefill";
|
||||
const routerCfg = config.playgroundFeatures
|
||||
&& config.playgroundFeatures.pdDisagg
|
||||
&& config.playgroundFeatures.pdDisagg.router;
|
||||
const routerCfg = resolveRouter(config.playgroundFeatures
|
||||
&& config.playgroundFeatures.pdDisagg, sel);
|
||||
const routerPort = (routerCfg && routerCfg.port) || 8000;
|
||||
const routerLine = routerCfg
|
||||
? `# then front BOTH with the Router shown below.\n`
|
||||
@@ -2265,9 +2455,18 @@ export const Playground = ({ config }) => {
|
||||
? matchedCell : null;
|
||||
const pgSpecAlgoFlag = pgFlagsLatest.find(
|
||||
(f) => f.split(/[\s=]/)[0] === "--speculative-algorithm");
|
||||
// Off PD, the hint is a "you forgot to set it" warning and disappears once
|
||||
// the ceiling is explicit. A PD role always ships one, so that form would
|
||||
// never fire there — but the role is exactly where the sizing rule bites,
|
||||
// since the ceiling has to be set on both roles and the decode graphs sized
|
||||
// to what it leaves per rank. Under a role the hint switches to that form and
|
||||
// shows whether or not the flag is present.
|
||||
const pgPdRole = pdMode === "prefill" || pdMode === "decode";
|
||||
const pgSpecHint =
|
||||
!!pgSpecAlgoFlag &&
|
||||
!pgFlagsLatest.some((f) => f.split(/[\s=]/)[0] === "--max-running-requests");
|
||||
(pgPdRole
|
||||
|| !pgFlagsLatest.some(
|
||||
(f) => f.split(/[\s=]/)[0] === "--max-running-requests"));
|
||||
const specAlgoLabels = {
|
||||
EAGLE: "MTP", EAGLE3: "MTP", FROZEN_KV_MTP: "MTP",
|
||||
DSPARK: "DSpark", DFLASH: "DFlash", NGRAM: "N-gram",
|
||||
@@ -2305,9 +2504,8 @@ export const Playground = ({ config }) => {
|
||||
// PD-Disagg router, if configured. When a PD role is active, cURL retargets
|
||||
// to the router port and a companion router block renders below the command.
|
||||
const pdRouter = (pdMode !== "off"
|
||||
&& config.playgroundFeatures
|
||||
&& config.playgroundFeatures.pdDisagg
|
||||
&& config.playgroundFeatures.pdDisagg.router) || null;
|
||||
&& resolveRouter(config.playgroundFeatures
|
||||
&& config.playgroundFeatures.pdDisagg, base)) || null;
|
||||
const curlEnv = (pdRouter && pdRouter.port != null)
|
||||
? { ...env, CURL_PORT: String(pdRouter.port) }
|
||||
: env;
|
||||
@@ -2552,7 +2750,11 @@ export const Playground = ({ config }) => {
|
||||
</pre>
|
||||
{pgSpecHint && (
|
||||
<div style={s.mtpWarn}>
|
||||
⚠️ Speculative decoding ({pgSpecAlgoName}) is on — SGLang resets <code>--max-running-requests</code> to <strong>48</strong> when it isn't set. Add <code>--max-running-requests <N></code> sized for your target concurrency.
|
||||
{pgPdRole ? (
|
||||
<>⚠️ Speculative decoding ({pgSpecAlgoName}) is on — for a target concurrency of N, set <code>--max-running-requests <N*2></code> on <strong>both</strong> the prefill and decode roles. That ceiling is server-wide and floor-divided by <code>attn_dp_size</code>, so size the decode graphs to the per-rank batch it leaves: <code>--cuda-graph-bs-decode</code> up to <code>N*2 / dp_size</code>.</>
|
||||
) : (
|
||||
<>⚠️ Speculative decoding ({pgSpecAlgoName}) is on — SGLang resets <code>--max-running-requests</code> to <strong>48</strong> when it isn't set. Add <code>--max-running-requests <N></code> sized for your target concurrency.</>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
{pgCpDpHint && (
|
||||
|
||||
@@ -258,6 +258,22 @@ sgl-eval run mmmu_pro \\
|
||||
reason: "Prefill Context Parallel is single-machine only (SGLang asserts tp_size <= 8; cross-machine CP has precision issues)." },
|
||||
] },
|
||||
{ id: "dpAttn", label: "DP-Attention",
|
||||
// The low-latency and balanced PD roles run TP-only. That is what lets
|
||||
// their decode ladders run to the full ceiling (8 and 96): the ceiling
|
||||
// is server-wide and floor-divided by attn_dp_size, so only at dp_size
|
||||
// 1 is it also the per-rank batch. Forced rather than left to the
|
||||
// reader, because switching DP on would cut the slots per rank without
|
||||
// changing either flag in the command — the ladder would still read 96
|
||||
// while the role could only ever fill 12. High-throughput is the DP
|
||||
// point and is deliberately absent here.
|
||||
forceOff: [
|
||||
{ when: { hw: ["mi355x"], strategy: ["low-latency", "balanced"],
|
||||
pdMode: ["prefill", "decode"] },
|
||||
stripEnv: ["SGLANG_DP_SHARED_EXPERT_LOCAL",
|
||||
"SGLANG_DP_USE_GATHERV",
|
||||
"SGLANG_DP_USE_REDUCE_SCATTER"],
|
||||
reason: "The low-latency and balanced PD roles are TP-only, which is what makes --cuda-graph-bs-decode equal the full ceiling. --max-running-requests is server-wide and floor-divided by attn_dp_size, so DP would cut the per-rank batch below the captured graphs." },
|
||||
],
|
||||
values: [
|
||||
null,
|
||||
false,
|
||||
@@ -324,10 +340,15 @@ sgl-eval run mmmu_pro \\
|
||||
options: [
|
||||
{ id: "current", label: "Inherited from base" },
|
||||
{ id: "off", label: "Off (greedy)" },
|
||||
// Shown on pro-official as well as the originals: the 0813 checkpoint
|
||||
// bundles a DSpark head but keeps its MTP head, and DSpark cannot run
|
||||
// under PD disaggregation (§3.8), so this shape is the speculative
|
||||
// option a PD role actually has. The 1-1-2 shape stays hidden there —
|
||||
// off PD, DSpark is the better pick, so only one fallback is offered.
|
||||
{ id: "mtp-314", label: "EAGLE / MTP 3-1-4",
|
||||
flags: ["--speculative-algorithm EAGLE", "--speculative-num-steps 3",
|
||||
"--speculative-eagle-topk 1", "--speculative-num-draft-tokens 4"],
|
||||
hide: { variant: ["flash-official", "flash-vision", "pro-official"] } },
|
||||
hide: { variant: ["flash-official", "flash-vision"] } },
|
||||
{ id: "mtp-112", label: "EAGLE / MTP 1-1-2",
|
||||
flags: ["--speculative-algorithm EAGLE", "--speculative-num-steps 1",
|
||||
"--speculative-eagle-topk 1", "--speculative-num-draft-tokens 2"],
|
||||
@@ -357,8 +378,42 @@ sgl-eval run mmmu_pro \\
|
||||
incompatibleSpeculativeAlgorithms: ["DSPARK"],
|
||||
modes: [
|
||||
{ id: "off", label: "Off" },
|
||||
{ id: "prefill", label: "Prefill role" },
|
||||
{ id: "decode", label: "Decode role" },
|
||||
// The AMD role flags are the MI355X 1P x 1D agentic recipe. Both roles
|
||||
// are gated by `when` because the sizing is ROCm-specific, and they
|
||||
// differ in two places: the prefill worker runs eager (the dsv4 indexer's
|
||||
// prefill path is not graph-captured) and dispatches whole chunked-prefill
|
||||
// batches over MORI, while the decode worker captures graphs for its
|
||||
// small batch ladder and dispatches at most a step's worth of tokens.
|
||||
{ id: "prefill", label: "Prefill role",
|
||||
when: { hw: ["mi355x"], strategy: ["low-latency"] },
|
||||
env: ["SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16384"],
|
||||
flags: [
|
||||
"--load-balance-method round_robin",
|
||||
"--tokenizer-worker-num 8",
|
||||
"--stream-interval 20",
|
||||
"--mem-fraction-static 0.86",
|
||||
"--max-running-requests 8",
|
||||
"--swa-full-tokens-ratio 0.1",
|
||||
"--disable-cuda-graph",
|
||||
"--context-length 1048576",
|
||||
"--watchdog-timeout 3600",
|
||||
"--enable-metrics",
|
||||
] },
|
||||
{ id: "decode", label: "Decode role",
|
||||
when: { hw: ["mi355x"], strategy: ["low-latency"] },
|
||||
env: ["SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128"],
|
||||
flags: [
|
||||
"--load-balance-method round_robin",
|
||||
"--tokenizer-worker-num 8",
|
||||
"--stream-interval 20",
|
||||
"--mem-fraction-static 0.86",
|
||||
"--max-running-requests 8",
|
||||
"--swa-full-tokens-ratio 0.1",
|
||||
"--cuda-graph-bs-decode 1 2 3 4 5 6 7 8",
|
||||
"--context-length 1048576",
|
||||
"--watchdog-timeout 3600",
|
||||
"--enable-metrics",
|
||||
] },
|
||||
],
|
||||
transferBackends: [
|
||||
{ id: "mooncake", label: "Mooncake",
|
||||
@@ -372,10 +427,29 @@ sgl-eval run mmmu_pro \\
|
||||
{ id: "nixl", label: "NiXL" },
|
||||
// MORI-IO transport is AMD-only — hidden on every non-ROCm platform.
|
||||
{ id: "mori", label: "MORI",
|
||||
hide: { hw: ["h100", "h200", "b200", "b300", "gb200", "gb300", "rtx6000"] } },
|
||||
hide: { hw: ["h100", "h200", "b200", "b300", "gb200", "gb300", "rtx6000"] },
|
||||
// Transport-wide only. The per-rank dispatch budget is sized per role
|
||||
// (see `modes` above), so it lives on the role, not here.
|
||||
env: [
|
||||
"SGLANG_MORI_COMBINE_DTYPE=auto",
|
||||
"MORI_IO_SQ_BACKOFF_TIMEOUT_US=500000",
|
||||
"MORI_IO_QP_MAX_SEND_WR=32767",
|
||||
],
|
||||
envWhen: { hw: ["mi300x", "mi355x"] } },
|
||||
],
|
||||
// `auto` is a sentinel (emits no --disaggregation-ib-device flag).
|
||||
ibDevices: [{ id: "auto", label: "Auto" }, "mlx5_0", "mlx5_7"],
|
||||
// The mlx5 names are ConnectX; ROCm nodes enumerate their NICs as rdmaN,
|
||||
// so the two families are mutually hidden. The AMD entry is the full
|
||||
// 8-NIC list in one value because --disaggregation-ib-device takes a
|
||||
// comma list, and the order is the MI355X NUMA-local pairing.
|
||||
ibDevices: [
|
||||
{ id: "auto", label: "Auto" },
|
||||
{ id: "mlx5_0", label: "mlx5_0", hide: { hw: ["mi300x", "mi355x"] } },
|
||||
{ id: "mlx5_7", label: "mlx5_7", hide: { hw: ["mi300x", "mi355x"] } },
|
||||
{ id: "rdma3,rdma0,rdma2,rdma1,rdma7,rdma4,rdma6,rdma5",
|
||||
label: "rdma0-7 (all NICs)",
|
||||
hide: { hw: ["h100", "h200", "b200", "b300", "gb200", "gb300", "rtx6000", "rtx5090", "dgx-spark"] } },
|
||||
],
|
||||
// Router fronting the prefill + decode roles; substitute <prefill-host>/<decode-host>.
|
||||
router: {
|
||||
port: 8000,
|
||||
@@ -388,6 +462,98 @@ sgl-eval run mmmu_pro \\
|
||||
--disable-circuit-breaker \\
|
||||
--health-check-interval-secs 999999`,
|
||||
},
|
||||
// The MI355X roles above size for low latency: TP-only, a running-request
|
||||
// ceiling in the single digits, and a MORI dispatch budget per role. The
|
||||
// high-throughput point is the same two roles re-sized against the DP
|
||||
// base cell — TP8/DP8 and the wider batch come from that cell, so these
|
||||
// only carry what the operating point itself changes. The decode graph
|
||||
// ladder grows to 32 to cover the larger steady-state batch, and
|
||||
// --enable-cache-report surfaces the prefix hit rate that decides whether
|
||||
// the offload tier is paying for itself at this concurrency.
|
||||
// The balanced point sits between the two: TP-only like low-latency, but
|
||||
// with a 96-request ceiling and a HiCache tier under the prefill role
|
||||
// (see the hicache roleOverride below) instead of low-latency's bare KV
|
||||
// pool or high-throughput's UMBP. It re-sizes more of the base cell than
|
||||
// the other two because the balanced cell is a DP recipe for aggregated
|
||||
// serving — its 0.90 / 0.15 / 65536 sizing does not carry over.
|
||||
roleOverrides: [
|
||||
{ mode: "prefill",
|
||||
when: { hw: ["mi355x"], strategy: ["balanced"] },
|
||||
env: ["SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16384"],
|
||||
flags: [
|
||||
"--load-balance-method round_robin",
|
||||
"--mem-fraction-static 0.86",
|
||||
"--max-running-requests 96",
|
||||
"--swa-full-tokens-ratio 0.1",
|
||||
"--chunked-prefill-size 16384",
|
||||
"--disable-cuda-graph",
|
||||
"--context-length 1048576",
|
||||
"--watchdog-timeout 3600",
|
||||
"--enable-metrics",
|
||||
] },
|
||||
// TP-only, so the server-wide ceiling is also the per-rank batch and
|
||||
// the graph ladder runs all the way to 96.
|
||||
{ mode: "decode",
|
||||
when: { hw: ["mi355x"], strategy: ["balanced"] },
|
||||
env: ["SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128"],
|
||||
flags: [
|
||||
"--load-balance-method round_robin",
|
||||
"--mem-fraction-static 0.86",
|
||||
"--max-running-requests 96",
|
||||
"--swa-full-tokens-ratio 0.1",
|
||||
"--cuda-graph-bs-decode 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96",
|
||||
"--context-length 1048576",
|
||||
"--watchdog-timeout 3600",
|
||||
"--enable-metrics",
|
||||
] },
|
||||
{ mode: "prefill",
|
||||
when: { hw: ["mi355x"], strategy: ["high-throughput"] },
|
||||
env: ["SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16384"],
|
||||
flags: [
|
||||
"--load-balance-method round_robin",
|
||||
"--mem-fraction-static 0.92",
|
||||
"--max-running-requests 256",
|
||||
"--disable-cuda-graph",
|
||||
"--context-length 1048576",
|
||||
"--watchdog-timeout 3600",
|
||||
"--enable-metrics",
|
||||
"--enable-cache-report",
|
||||
] },
|
||||
{ mode: "decode",
|
||||
when: { hw: ["mi355x"], strategy: ["high-throughput"] },
|
||||
env: ["SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128"],
|
||||
flags: [
|
||||
"--load-balance-method round_robin",
|
||||
"--mem-fraction-static 0.92",
|
||||
"--max-running-requests 256",
|
||||
"--cuda-graph-bs-decode 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32",
|
||||
"--context-length 1048576",
|
||||
"--watchdog-timeout 3600",
|
||||
"--enable-metrics",
|
||||
] },
|
||||
],
|
||||
// MI355X fronts the 1P x 1D agentic pair with a cache-aware router rather
|
||||
// than the default round-robin: consistent_hashing keeps a conversation on
|
||||
// the worker that already holds its prefix, and the tight balance
|
||||
// thresholds stop that affinity from starving the peer at the small
|
||||
// running-request ceiling the roles above use. Health checking stays
|
||||
// enabled here (unlike the default's 999999s interval) because a long
|
||||
// agentic run should notice a wedged worker, but it is slack enough that
|
||||
// a multi-minute prefill is not mistaken for a failure.
|
||||
routerOverrides: [
|
||||
{ when: { hw: ["mi355x"] },
|
||||
command:
|
||||
`python3 -m sglang_router.launch_router \\
|
||||
--pd-disaggregation \\
|
||||
--prefill http://<prefill-host>:{{PREFILL_PORT}} \\
|
||||
--decode http://<decode-host>:{{DECODE_PORT}} \\
|
||||
--host 0.0.0.0 --port {{ROUTER_PORT}} \\
|
||||
--policy consistent_hashing --dp-aware \\
|
||||
--cache-threshold 0.3 \\
|
||||
--balance-abs-threshold 2 --balance-rel-threshold 1.1 \\
|
||||
--disable-circuit-breaker --health-failure-threshold 100 \\
|
||||
--health-check-timeout-secs 600 --health-check-interval-secs 30` },
|
||||
],
|
||||
},
|
||||
|
||||
// ----- Card 6: "Hierarchical KV Cache" -----
|
||||
@@ -409,6 +575,23 @@ sgl-eval run mmmu_pro \\
|
||||
writePolicy: "write_through",
|
||||
prefetchPolicy: "best_effort",
|
||||
},
|
||||
// Balanced on Pro Official: the same shape at a smaller ratio. 2.5 is
|
||||
// what the 96-request ceiling leaves room for once mem-fraction-static
|
||||
// drops to 0.86 — the host tier competes with the KV pool for the
|
||||
// headroom the prefill role gives up.
|
||||
{
|
||||
when: {
|
||||
hw: ["mi355x"], variant: ["pro-official"], quant: ["fp4"],
|
||||
strategy: ["balanced"], nodes: ["single"],
|
||||
},
|
||||
mode: "prefill",
|
||||
transferBackend: "mori",
|
||||
memLayout: "page_first",
|
||||
ioBackend: "direct",
|
||||
ratio: 2.5,
|
||||
writePolicy: "write_through",
|
||||
prefetchPolicy: "best_effort",
|
||||
},
|
||||
],
|
||||
notices: [
|
||||
{
|
||||
@@ -420,6 +603,15 @@ sgl-eval run mmmu_pro \\
|
||||
transferBackend: "mori",
|
||||
text: "HiCache is not recommended on the decode role with MORI.",
|
||||
},
|
||||
{
|
||||
when: {
|
||||
hw: ["mi355x"], variant: ["pro-official"], quant: ["fp4"],
|
||||
strategy: ["balanced"], nodes: ["single"],
|
||||
},
|
||||
mode: "decode",
|
||||
transferBackend: "mori",
|
||||
text: "HiCache is not recommended on the decode role with MORI.",
|
||||
},
|
||||
],
|
||||
amdStorageFileOnly: true,
|
||||
backends: [
|
||||
@@ -440,7 +632,30 @@ sgl-eval run mmmu_pro \\
|
||||
],
|
||||
},
|
||||
|
||||
// ----- Card 7: "HiSparse" -----
|
||||
// ----- Card 7: "UMBP" (unified cache external linker) -----
|
||||
// Sits beside HiCache rather than inside it. HiCache is a tiered cache
|
||||
// (GPU -> pinned host -> optional storage); UMBP links the unified radix
|
||||
// tree DIRECTLY to an external store with no host tier at all, so the two
|
||||
// are alternatives and sglang rejects them together. Enabling this card
|
||||
// therefore strips the HiCache family from the command.
|
||||
//
|
||||
// ROCm-only in practice: the store is MORI's buffer pool, the same
|
||||
// transport the PD roles use, and there is no CUDA recipe for it yet.
|
||||
umbp: {
|
||||
onlyHw: ["mi300x", "mi355x"],
|
||||
// Under pure TP the linker keys by rank, so an 8-rank prefill worker
|
||||
// opens eight keyspaces and the pool holds eight copies of the same MLA
|
||||
// KV — a tier an eighth the size its byte budget suggests. DP attention
|
||||
// collapses the keys onto one shared keyspace.
|
||||
requiresDpAttention: true,
|
||||
defaultBackend: "mori",
|
||||
backends: [
|
||||
{ id: "mori", label: "MORI (UMBP)" },
|
||||
{ id: "mooncake", label: "Mooncake" },
|
||||
],
|
||||
},
|
||||
|
||||
// ----- Card 8: "HiSparse" -----
|
||||
// Decode-only: shown/emitted only when the live PD-Disagg mode is `decode`.
|
||||
hisparse: {
|
||||
requiredFlags: [
|
||||
|
||||
Reference in New Issue
Block a user