[Spec] Reuse K3 auxiliary outputs across decode CUDA graph sizes (#40159)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: hnyls2002 <lsyincs@gmail.com>
This commit is contained in:
James Liu
2026-09-29 14:24:51 -07:00
committed by GitHub
co-authored by Claude Opus 5.5 hnyls2002
parent 46beacf13d
commit 84523d6785
12 changed files with 173 additions and 9 deletions
@@ -841,6 +841,8 @@ class TboForwardBatchPreparer:
token_ids_logprobs=None,
extend_input_logprob_token_ids_gpu=None,
next_token_logits_buffer=None,
# The aux packer runs in the parent model forward, not per child.
aux_hidden_states_buffer=None,
return_hidden_states_before_norm=False,
# TBO children start unplanned — planned by the TBO-aware init
# flow; a stale parent "ready" would wrongly skip that.
+27 -8
View File
@@ -1,9 +1,14 @@
"""Aux hidden states captured for Eagle3/DFlash draft models."""
from typing import List, Optional, Union
from __future__ import annotations
from typing import TYPE_CHECKING, List, Optional, Union
import torch
if TYPE_CHECKING:
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
# Two representations coexist: models migrated to AuxHiddenStatePacker pass one
# packed [tokens, K * hidden] tensor, the rest still pass a list of K tensors.
AuxHiddenStates = Union[torch.Tensor, List[torch.Tensor]]
@@ -27,21 +32,35 @@ class AuxHiddenStatePacker:
Each ``.append()`` writes into one preallocated ``[tokens, K * hidden]``
buffer, avoiding the list path's transient ~2x HBM at ``torch.cat``.
Assumes all captures share leading shape and feature size.
``for_batch`` writes into a decode graph runner's buffer when the batch
carries one, so every graph size aliases it instead of pinning its own.
"""
def __init__(self, num_captures: int) -> None:
def __init__(self, num_captures: int, out: Optional[torch.Tensor] = None) -> None:
self._num_captures = int(num_captures)
self._buffer: Optional[torch.Tensor] = None
self._buffer = out
self._feature_size: Optional[int] = None
self._idx = 0
@classmethod
def for_batch(
cls, forward_batch: ForwardBatch, num_captures: int
) -> AuxHiddenStatePacker:
return cls(num_captures, out=forward_batch.aux_hidden_states_buffer)
def append(self, hidden: torch.Tensor) -> None:
feature_size = int(hidden.shape[-1])
if self._buffer is None:
if self._feature_size is None:
self._feature_size = feature_size
self._buffer = hidden.new_empty(
(*hidden.shape[:-1], feature_size * self._num_captures)
)
shape = (*hidden.shape[:-1], feature_size * self._num_captures)
if self._buffer is None:
self._buffer = hidden.new_empty(shape)
elif self._buffer.shape != shape or self._buffer.dtype != hidden.dtype:
raise ValueError(
f"aux hidden output buffer is {tuple(self._buffer.shape)} "
f"{self._buffer.dtype}, captures need {shape} {hidden.dtype}"
)
start = self._idx * self._feature_size
self._buffer[..., start : start + self._feature_size].copy_(hidden)
self._idx += 1
@@ -55,7 +74,7 @@ class AuxHiddenStatePacker:
def finalize(self) -> torch.Tensor:
"""Return the packed buffer; callers guard the empty case on ``len()``."""
if self._buffer is None or self._idx != self._num_captures:
if self._feature_size is None or self._idx != self._num_captures:
raise RuntimeError(
f"captured {self._idx} of {self._num_captures} aux hidden states"
)
@@ -681,6 +681,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
# Preallocated piecewise-graph attention output, set by RadixAttention.
_attn_output: Optional[torch.Tensor] = None
# Decode-graph-owned destination for AuxHiddenStatePacker; None when eager.
aux_hidden_states_buffer: Optional[torch.Tensor] = None
# Prefill body-CUDA-graph context limit. Attention backends that allocate
# context-shaped metadata use this fixed maximum instead of deriving a
# shape from the live batch. None preserves eager/default graph behavior.
@@ -781,6 +781,13 @@ class ModelRunner:
pp_rank=self.pp_rank,
)
def get_aux_hidden_states_width(self) -> int:
return misc_utils.resolve_aux_hidden_states_width(
model=self.model,
spec_algorithm=self.spec_algorithm,
is_draft_worker=self.is_draft_worker,
)
def get_pp_proxy_topk_size(self) -> Optional[int]:
return misc_utils.resolve_pp_proxy_topk_size(
model_config=self.model_config,
@@ -112,3 +112,20 @@ def resolve_pp_proxy_dspark_hidden_size(
if isinstance(model, _SupportsDSparkPPProxy):
return model.get_pp_proxy_dspark_hidden_size()
return 0
@runtime_checkable
class _SupportsPackedAuxHiddenStates(Protocol):
def get_aux_hidden_states_width(self) -> int: ...
def resolve_aux_hidden_states_width(
*, model: Any, spec_algorithm: Any, is_draft_worker: bool
) -> int:
# Sharing needs the spec worker to consume target hidden states before
# the next target forward overwrites them; the DFlash family does.
if not spec_algorithm.is_dflash_family() or is_draft_worker:
return 0
if isinstance(model, _SupportsPackedAuxHiddenStates):
return model.get_aux_hidden_states_width()
return 0
@@ -439,6 +439,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
pp_proxy_dspark_hidden_size=(
self.model_runner.get_pp_proxy_dspark_hidden_size()
),
aux_hidden_states_width=self.model_runner.get_aux_hidden_states_width(),
)
self.buffers.share_buffers()
# FB-shared slot registry adopting DecodeInputBuffers storage (same
@@ -906,6 +907,11 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
)
mrope_positions = _slot("mrope_positions")
next_token_logits_buffer = buffers.next_token_logits_buffer[:num_tokens]
aux_hidden_states_buffer = (
buffers.aux_hidden_states[:num_tokens]
if buffers.aux_hidden_states is not None
else None
)
rids_int = buffers.rids_int[:bs] if buffers.rids_int is not None else None
bootstrap_room_ids_int = (
buffers.bootstrap_room_ids_int[:bs]
@@ -981,6 +987,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
seq_lens=seq_lens,
seq_lens_cpu=seq_lens_cpu,
next_token_logits_buffer=next_token_logits_buffer,
aux_hidden_states_buffer=aux_hidden_states_buffer,
orig_seq_lens=seq_lens,
out_cache_loc=out_cache_loc,
seq_lens_sum=seq_lens.sum().item(),
@@ -114,6 +114,8 @@ class DecodeInputBuffers(ForwardInputBuffers):
num_token_non_padded: Optional[torch.Tensor]
custom_mask: torch.Tensor
next_token_logits_buffer: torch.Tensor
# Packed aux hidden-state output shared by every captured graph size.
aux_hidden_states: Optional[torch.Tensor]
mamba_track_indices: Optional[torch.Tensor]
mamba_track_mask: Optional[torch.Tensor]
global_num_tokens_gpu: torch.Tensor
@@ -148,6 +150,7 @@ class DecodeInputBuffers(ForwardInputBuffers):
pp_proxy_topk_size: Optional[int] = None,
pp_proxy_residual_num_blocks: Optional[int] = None,
pp_proxy_dspark_hidden_size: int = 0,
aux_hidden_states_width: int = 0,
) -> DecodeInputBuffers:
with torch.device(device):
input_ids = torch.zeros((max_num_token,), dtype=torch.int64)
@@ -175,6 +178,11 @@ class DecodeInputBuffers(ForwardInputBuffers):
mamba_track_mask = (
torch.zeros((max_bs,), dtype=torch.bool) if enable_mamba_track else None
)
aux_hidden_states = (
torch.zeros((max_num_token, aux_hidden_states_width), dtype=dtype)
if aux_hidden_states_width
else None
)
pp_proxy_tensors = (
_allocate_pp_proxy_tensors(
@@ -246,6 +254,7 @@ class DecodeInputBuffers(ForwardInputBuffers):
num_token_non_padded=num_token_non_padded,
custom_mask=custom_mask,
next_token_logits_buffer=next_token_logits_buffer,
aux_hidden_states=aux_hidden_states,
mamba_track_indices=mamba_track_indices,
mamba_track_mask=mamba_track_mask,
encoder_lens=encoder_lens,
+31 -1
View File
@@ -33,6 +33,10 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
from sglang.srt.layers import zero_copy_context
from sglang.srt.layers.activation import SiluAndMul, SituAndMul
from sglang.srt.layers.attn_residual import AttnResidual, aggregate_stream, get_cw
from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
AuxHiddenStatePacker,
)
from sglang.srt.layers.communication import k3_ar_fusion, k3_sp_collective
from sglang.srt.layers.dcp.planner import prepare_decode_context_parallel_metadata
from sglang.srt.layers.dp_attention import (
@@ -2958,7 +2962,14 @@ class KimiK3LinearModel(nn.Module):
and k3_sp_collective.enabled()
)
sp_sharded = False
aux_hidden_states = []
packs_aux = self.packs_aux_hidden_states
aux_hidden_states: AuxHiddenStateAccumulator = (
AuxHiddenStatePacker.for_batch(
forward_batch, len(self.dspark_layers_to_capture)
)
if packs_aux
else []
)
if (
self.dspark_layers_to_capture is not None
and not self.pp_group.is_first_rank
@@ -3049,10 +3060,19 @@ class KimiK3LinearModel(nn.Module):
else:
hidden_states, _ = self.norm(hidden_states, residual)
if packs_aux:
return hidden_states, aux_hidden_states.finalize()
if self.dspark_layers_to_capture is not None:
return hidden_states, aux_hidden_states
return hidden_states
@property
def packs_aux_hidden_states(self) -> bool:
# PP stages keep the list: inherited captures arrive pre-concatenated.
return (
self.dspark_layers_to_capture is not None and self.pp_group.world_size == 1
)
def _dspark_capture_stream(
self,
layer_idx: int,
@@ -3145,6 +3165,11 @@ class KimiK3LinearForCausalLM(nn.Module):
layer < self.model.start_layer - 1 for layer in layers
)
def get_aux_hidden_states_width(self) -> int:
if not self.model.packs_aux_hidden_states:
return 0
return len(self.model.dspark_layers_to_capture) * self.config.hidden_size
def set_dspark_layers_to_capture(self, layer_ids: list[int]) -> None:
if layer_ids is None:
raise ValueError(
@@ -3628,6 +3653,11 @@ class KimiK3ForConditionalGeneration(nn.Module):
)
self.language_model.set_dspark_layers_to_capture(layer_ids)
def get_aux_hidden_states_width(self) -> int:
if self.language_model is None:
return 0
return self.language_model.get_aux_hidden_states_width()
def preprocess_mm_for_encoder(
self,
mm_data,
@@ -2868,6 +2868,8 @@ class DFlashWorkerV2(BaseSpecWorker):
)
hidden = hidden.view(bs, int(self.block_size), -1)
# Consume in this step: every decode graph size shares one aux output,
# which the next target forward overwrites (resolve_aux_hidden_states_width).
self._append_target_hidden_to_draft_kv_by_loc(
target_hidden=hidden.reshape(-1, hidden.shape[-1]),
cache_loc=verify_out_cache_loc,
@@ -929,6 +929,8 @@ class DSparkWorkerV2(BaseSpecWorker):
)
folded_commit = folded_accept and epilogue.folds_commit
# Consume in this step: every decode graph size shares one aux output,
# which the next target forward overwrites (resolve_aux_hidden_states_width).
if not folded_commit:
self._verify_executor.commit_hidden(
batch=batch,