[Spec] Support DFLASH for Kimi K3 (#40794)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
James Liu
2026-09-23 15:58:39 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 3f697899a3
commit 208f6f7501
3 changed files with 34 additions and 11 deletions
@@ -70,7 +70,7 @@ Pick your hardware, then the deployment shape and operating point. Node count fo
`Long-Context` appears only under the `Prefill` PD mode; for long-context unified serving on B200, start from High-Throughput and raise `--context-length`.
**Spec Decode** — layers onto the strategy without changing it, on every platform except B200. DSPARK proposes 7 draft tokens per step (tune in the Playground) and requires `pp_size == 1`, so on B200 it also drops the pipeline and re-lays the same 16 GPUs flat: PP2 × TP8 → TP16, PP2 × DCPEP8 → DCPEP16. DFLASH has no published draft checkpoint. The win is largest on short interactive traffic and fades as the prompt grows.
**Spec Decode** — layers onto the strategy without changing it, on every platform except B200. DSPARK proposes 7 draft tokens per step (tune in the Playground) and requires `pp_size == 1`, so on B200 it also drops the pipeline and re-lays the same 16 GPUs flat: PP2 × TP8 → TP16, PP2 × DCPEP8 → DCPEP16. DFLASH uses the [Kimi-K3-DFlash](https://huggingface.co/modal-labs/Kimi-K3-DFlash) draft at block size 8; it doesn't support pipeline parallelism or DP attention yet, so it is unavailable on the pipelined B200 Unified recipes. It has only been validated on Blackwell, so the Hopper and AMD recipes don't offer it yet. The win is largest on short interactive traffic and fades as the prompt grows.
<Note>
`--mamba-full-memory-ratio` is the one sizing flag, computed live: set your average request length in the [Mamba ratio calculator](#mamba-ratio-calculator); everything else follows the panels, and the result is pinned into the command. (The Ascend NPU recipes don't use the calculator and set no ratio at all — the Ascend path sizes both pools itself: the A3 Series pins `--max-mamba-cache-size` instead, and the A5 recipe serves with the radix cache off.)
@@ -334,16 +334,27 @@ export const config = {
{
id: "dflash",
label: "DFLASH",
// Listed so the axis is complete, but not selectable: no K3 DFLASH draft
// checkpoint has been published, so there is nothing to point
// --speculative-draft-model-path at. DFLASH is also CUDA-only, rejects DP
// attention, and requires pp_size == 1.
disabled: true,
disableReason:
"No K3 DFLASH draft checkpoint published yet — DSPARK is the available speculative path.",
flags: [
// DFLASH doesn't support pipeline parallelism yet and rejects DP attention
// off NPU, so pipelined and DP-attention recipes are unavailable. The NPU recipes ship DSPARK only. K3 DFLASH
// has only been validated on Blackwell, so Hopper and AMD stay off.
disabled: (s) =>
config.isNpuHw(s) ||
["h100", "h200", "mi350x", "mi355x"].includes(s.hw) ||
config.isPipelined(s) ||
!!config.flagOf(config.cellFor(s), "--enable-dp-attention"),
disableReason: (s) =>
config.isNpuHw(s)
? "Only DSPARK is supported on this recipe."
: ["h100", "h200", "mi350x", "mi355x"].includes(s.hw)
? "K3 DFLASH has not been validated on this hardware yet; use DSPARK."
: "DFLASH doesn't support pipeline parallelism or DP attention yet. Pick a recipe that runs a single pipeline stage without DP attention, or use DSPARK.",
flags: (s) => [
"--speculative-algorithm DFLASH",
"--speculative-draft-model-path <dflash-draft>",
"--speculative-draft-model-path modal-labs/Kimi-K3-DFlash",
// 8 is the recommended default block size for this draft.
"--speculative-dflash-block-size 8",
// Same ReplaySSM rule as DSPARK: the PD prefill role rejects the flag.
...(s.pdMode !== "prefill" ? ["--enable-linear-replayssm-spec"] : []),
],
},
],
@@ -785,7 +796,7 @@ export const config = {
// DSPARK --speculative-dspark-block-size N (gamma, == proposed)
// DFLASH --speculative-dflash-block-size N+1 (verify window)
// EAGLE --speculative-num-steps N (chain; topk>1 is a tree)
// Only DSPARK is selectable today, so only its form is emitted.
// Only DSPARK's form is emitted; the DFLASH option pins its block size.
id: "proposedDraftTokens", title: "Proposed Draft Tokens",
// The NPU recipes pin the shipped block size (7).
showWhen: (b) => b.spec === "dspark" && !config.isNpuHw(b),
+12
View File
@@ -3151,6 +3151,11 @@ class KimiK3LinearForCausalLM(nn.Module):
self.capture_aux_hidden_states = True
self.model.dspark_layers_to_capture = list(layer_ids)
def set_dflash_layers_to_capture(self, layer_ids: list[int]) -> None:
# DFLASH target_layer_ids name layer outputs, which is what the DSPARK
# taps already capture here, so reuse them without the usual +1 shift.
self.set_dspark_layers_to_capture(layer_ids)
@torch.no_grad()
def forward(
self,
@@ -3607,6 +3612,13 @@ class KimiK3ForConditionalGeneration(nn.Module):
return 0
return self.language_model.get_pp_proxy_dspark_hidden_size()
def set_dflash_layers_to_capture(self, layer_ids: list[int]) -> None:
if self.language_model is None:
raise AttributeError(
"DFLASH layer capture is not available in encoder-only mode"
)
self.language_model.set_dflash_layers_to_capture(layer_ids)
def set_dspark_layers_to_capture(self, layer_ids: list[int]) -> None:
if self.language_model is None:
raise AttributeError(