[Cookbook] Kimi-K3: keep DCP under HiCache L1+L2 with DSPARK (#39190)

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Khoa Pham
2026-09-11 23:14:24 -07:00
committed by GitHub
co-authored by Claude Fable 5.1
parent 55e5e21c88
commit 0d08668821
2 changed files with 11 additions and 25 deletions
@@ -283,7 +283,8 @@ Pending update...
K3's hybrid HiCache tiers the paged MLA KV **and** the KDA/mamba state across L1 (GPU) / L2 (host) / L3 (Mooncake) — enable it from the **HiCache** card in the [Playground above](#playground) for long multi-turn workloads. K3's hybrid HiCache tiers the paged MLA KV **and** the KDA/mamba state across L1 (GPU) / L2 (host) / L3 (Mooncake) — enable it from the **HiCache** card in the [Playground above](#playground) for long multi-turn workloads.
- On the DCP recipes (Blackwell Balanced / High-Throughput, in both the `Unified` and `Decode` roles), the host tiers are not fully DCP-aware yet: **L3 always, and L1+L2 with Spec Decode on, drop the DCP flags** (the command hints call it out — per-request KV capacity shrinks accordingly). L1+L2 with Spec Decode off keeps DCP. Only DCP goes: the MLA KV reverts to TP-replicated, but the cell's other parallelism stays, so B300/GB300/GB200 land on plain TP while B200 Unified keeps its `--pp-size 2` / `--ep-size`. - On the DCP recipes (Blackwell Balanced / High-Throughput, `Unified` and `Decode` roles), **L1+L2 keeps DCP**, with Spec Decode off or on DSPARK. On B200 the DSPARK pipeline collapse scales it too (PP2 × DCPEP8 → DCPEP16).
- **L3 drops the DCP flags** on those recipes (storage keys are not dcp_rank-aware yet; the command hints call it out, and per-request KV capacity shrinks). Only DCP goes: the MLA KV reverts to TP-replicated while the cell's other parallelism stays, so B300/GB300/GB200 land on plain TP and B200 Unified keeps its `--pp-size 2` / `--ep-size`.
- Low-Latency and the Hopper recipes take all tiers unchanged. - Low-Latency and the Hopper recipes take all tiers unchanged.
<a id="pd-disaggregation" /> <a id="pd-disaggregation" />
@@ -81,7 +81,7 @@ export const config = {
const [hw, pdMode] = k.split("|"); const [hw, pdMode] = k.split("|");
return { return {
when: { hw: [hw], pdMode: [pdMode], strategy: [...strategies], when: { hw: [hw], pdMode: [pdMode], strategy: [...strategies],
hicache: ["l2"], spec: ["none"] }, hicache: ["l2"], spec: ["none", "dspark"] },
reason: "This recipe runs DCP, and a storage backend (L3) under DCP is rejected at startup. Switch HiCache to L3 in the Deploy panel (that drops DCP), or stay on L1+L2.", reason: "This recipe runs DCP, and a storage backend (L3) under DCP is rejected at startup. Switch HiCache to L3 in the Deploy panel (that drops DCP), or stay on L1+L2.",
}; };
}); });
@@ -115,10 +115,8 @@ export const config = {
const cell = config.cellFor(s); const cell = config.cellFor(s);
const pp = config.sizeOf(cell, "--pp-size"); const pp = config.sizeOf(cell, "--pp-size");
const out = [`--tp-size ${config.sizeOf(cell, "--tp-size") * pp}`]; const out = [`--tp-size ${config.sizeOf(cell, "--tp-size") * pp}`];
// HiCache under DCP rejects speculative decoding, so when a host tier is on // L3 rejects DCP at startup, so under L3 the DCP half is dropped, not scaled.
// the DCP half is dropped instead of scaled (the HiCache options' own if (config.flagOf(cell, "--dcp-size") && s.hicache !== "l3") {
// stripPrefixes reach cell flags only, never these overlay flags).
if (config.flagOf(cell, "--dcp-size") && s.hicache !== "l2" && s.hicache !== "l3") {
out.push(`--dcp-size ${config.sizeOf(cell, "--dcp-size") * pp}`); out.push(`--dcp-size ${config.sizeOf(cell, "--dcp-size") * pp}`);
} }
if (config.flagOf(cell, "--ep-size")) { if (config.flagOf(cell, "--ep-size")) {
@@ -355,26 +353,17 @@ export const config = {
flags: [ flags: [
"--enable-hierarchical-cache", "--enable-hierarchical-cache",
], ],
// L1/L2 host tiering IS supported under DCP, so DCP stays — except with
// speculative decoding, which HiCache under DCP rejects at startup (the
// draft host pool has no DCP index translation). There the DCP operating
// point is dropped instead of blocking the option, same as L3 does. The
// ratio calculator reads --dcp-size off this command, so dropping it also
// re-solves --mamba-full-memory-ratio for the plain-TP shape.
//
// Which cells carry DCP is read off the cells themselves, so adding or
// reshaping a DCP recipe needs no edit here.
stripPrefixes: (s) => stripPrefixes: (s) =>
s.spec !== "none" && config.hasDcp(s) !["none", "dspark"].includes(s.spec) && config.hasDcp(s)
? ["--dcp-size", "--dcp-comm-backend"] ? ["--dcp-size", "--dcp-comm-backend"]
: [], : [],
hints: (s) => hints: (s) =>
s.spec !== "none" && config.hasDcp(s) !["none", "dspark"].includes(s.spec) && config.hasDcp(s)
? [ ? [
"HiCache under DCP rejects speculative decoding, so this recipe drops DCP", "HiCache under DCP only accepts DSPARK speculative decoding, so this",
"and serves the MLA KV TP-replicated (any PP/EP in the cell stays).", "recipe drops DCP and serves the MLA KV TP-replicated (any PP/EP in the",
"Per-request context is far shorter than the DCP", "cell stays). Per-request context is far shorter than the DCP version —",
"version — DCP is what buys KV capacity. Run the cell NOSPEC to keep DCP.", "DCP is what buys KV capacity. Run the cell NOSPEC or DSPARK to keep DCP.",
] ]
: [], : [],
}, },
@@ -977,10 +966,6 @@ export const config = {
when: { hicache: ["l3"] }, when: { hicache: ["l3"] },
reason: "L3 storage keys are not dcp_rank-aware, so DCP and L3 cannot run together. Use Peak Throughput, or switch HiCache to L1+L2.", reason: "L3 storage keys are not dcp_rank-aware, so DCP and L3 cannot run together. Use Peak Throughput, or switch HiCache to L1+L2.",
}, },
{
when: { hicache: ["l2"], spec: ["dspark"] },
reason: "HiCache under DCP rejects speculative decoding, so this preset cannot add DCP here. Use Peak Throughput, or run the cell NOSPEC.",
},
], ],
env: ["SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=20480"], env: ["SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=20480"],
flags: (v, b) => { flags: (v, b) => {