diff --git a/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx b/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx index 6025fa67f8..b6dd00264a 100644 --- a/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx +++ b/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx @@ -70,7 +70,7 @@ Pick your hardware, then the deployment shape and operating point. Node count fo `Long-Context` appears only under the `Prefill` PD mode; for long-context unified serving on B200, start from High-Throughput and raise `--context-length`. -**Spec Decode** — layers onto the strategy without changing it, on every platform except B200. DSPARK proposes 7 draft tokens per step (tune in the Playground) and requires `pp_size == 1`, so on B200 it also drops the pipeline and re-lays the same 16 GPUs flat: PP2 × TP8 → TP16, PP2 × DCPEP8 → DCPEP16. DFLASH has no published draft checkpoint. The win is largest on short interactive traffic and fades as the prompt grows. +**Spec Decode** — layers onto the strategy without changing it, on every platform except B200. DSPARK proposes 7 draft tokens per step (tune in the Playground) and requires `pp_size == 1`, so on B200 it also drops the pipeline and re-lays the same 16 GPUs flat: PP2 × TP8 → TP16, PP2 × DCPEP8 → DCPEP16. DFLASH uses the [Kimi-K3-DFlash](https://huggingface.co/modal-labs/Kimi-K3-DFlash) draft at block size 8; it doesn't support pipeline parallelism or DP attention yet, so it is unavailable on the pipelined B200 Unified recipes. It has only been validated on Blackwell, so the Hopper and AMD recipes don't offer it yet. The win is largest on short interactive traffic and fades as the prompt grows. `--mamba-full-memory-ratio` is the one sizing flag, computed live: set your average request length in the [Mamba ratio calculator](#mamba-ratio-calculator); everything else follows the panels, and the result is pinned into the command. (The Ascend NPU recipes don't use the calculator and set no ratio at all — the Ascend path sizes both pools itself: the A3 Series pins `--max-mamba-cache-size` instead, and the A5 recipe serves with the radix cache off.) diff --git a/docs/src/snippets/configs/moonshotai/kimi-k3.jsx b/docs/src/snippets/configs/moonshotai/kimi-k3.jsx index d962ef3f94..15f632f8ba 100644 --- a/docs/src/snippets/configs/moonshotai/kimi-k3.jsx +++ b/docs/src/snippets/configs/moonshotai/kimi-k3.jsx @@ -334,16 +334,27 @@ export const config = { { id: "dflash", label: "DFLASH", - // Listed so the axis is complete, but not selectable: no K3 DFLASH draft - // checkpoint has been published, so there is nothing to point - // --speculative-draft-model-path at. DFLASH is also CUDA-only, rejects DP - // attention, and requires pp_size == 1. - disabled: true, - disableReason: - "No K3 DFLASH draft checkpoint published yet — DSPARK is the available speculative path.", - flags: [ + // DFLASH doesn't support pipeline parallelism yet and rejects DP attention + // off NPU, so pipelined and DP-attention recipes are unavailable. The NPU recipes ship DSPARK only. K3 DFLASH + // has only been validated on Blackwell, so Hopper and AMD stay off. + disabled: (s) => + config.isNpuHw(s) || + ["h100", "h200", "mi350x", "mi355x"].includes(s.hw) || + config.isPipelined(s) || + !!config.flagOf(config.cellFor(s), "--enable-dp-attention"), + disableReason: (s) => + config.isNpuHw(s) + ? "Only DSPARK is supported on this recipe." + : ["h100", "h200", "mi350x", "mi355x"].includes(s.hw) + ? "K3 DFLASH has not been validated on this hardware yet; use DSPARK." + : "DFLASH doesn't support pipeline parallelism or DP attention yet. Pick a recipe that runs a single pipeline stage without DP attention, or use DSPARK.", + flags: (s) => [ "--speculative-algorithm DFLASH", - "--speculative-draft-model-path ", + "--speculative-draft-model-path modal-labs/Kimi-K3-DFlash", + // 8 is the recommended default block size for this draft. + "--speculative-dflash-block-size 8", + // Same ReplaySSM rule as DSPARK: the PD prefill role rejects the flag. + ...(s.pdMode !== "prefill" ? ["--enable-linear-replayssm-spec"] : []), ], }, ], @@ -785,7 +796,7 @@ export const config = { // DSPARK --speculative-dspark-block-size N (gamma, == proposed) // DFLASH --speculative-dflash-block-size N+1 (verify window) // EAGLE --speculative-num-steps N (chain; topk>1 is a tree) - // Only DSPARK is selectable today, so only its form is emitted. + // Only DSPARK's form is emitted; the DFLASH option pins its block size. id: "proposedDraftTokens", title: "Proposed Draft Tokens", // The NPU recipes pin the shipped block size (7). showWhen: (b) => b.spec === "dspark" && !config.isNpuHw(b), diff --git a/python/sglang/srt/models/kimi_k3.py b/python/sglang/srt/models/kimi_k3.py index bbb8cab8fa..b4a8291aa9 100644 --- a/python/sglang/srt/models/kimi_k3.py +++ b/python/sglang/srt/models/kimi_k3.py @@ -3151,6 +3151,11 @@ class KimiK3LinearForCausalLM(nn.Module): self.capture_aux_hidden_states = True self.model.dspark_layers_to_capture = list(layer_ids) + def set_dflash_layers_to_capture(self, layer_ids: list[int]) -> None: + # DFLASH target_layer_ids name layer outputs, which is what the DSPARK + # taps already capture here, so reuse them without the usual +1 shift. + self.set_dspark_layers_to_capture(layer_ids) + @torch.no_grad() def forward( self, @@ -3607,6 +3612,13 @@ class KimiK3ForConditionalGeneration(nn.Module): return 0 return self.language_model.get_pp_proxy_dspark_hidden_size() + def set_dflash_layers_to_capture(self, layer_ids: list[int]) -> None: + if self.language_model is None: + raise AttributeError( + "DFLASH layer capture is not available in encoder-only mode" + ) + self.language_model.set_dflash_layers_to_capture(layer_ids) + def set_dspark_layers_to_capture(self, layer_ids: list[int]) -> None: if self.language_model is None: raise AttributeError(