diff --git a/python/sglang/srt/batch_overlap/two_batch_overlap.py b/python/sglang/srt/batch_overlap/two_batch_overlap.py index 86c863307c..dc9854b098 100644 --- a/python/sglang/srt/batch_overlap/two_batch_overlap.py +++ b/python/sglang/srt/batch_overlap/two_batch_overlap.py @@ -841,6 +841,8 @@ class TboForwardBatchPreparer: token_ids_logprobs=None, extend_input_logprob_token_ids_gpu=None, next_token_logits_buffer=None, + # The aux packer runs in the parent model forward, not per child. + aux_hidden_states_buffer=None, return_hidden_states_before_norm=False, # TBO children start unplanned — planned by the TBO-aware init # flow; a stale parent "ready" would wrongly skip that. diff --git a/python/sglang/srt/layers/aux_hidden_states.py b/python/sglang/srt/layers/aux_hidden_states.py index 92c987f5db..f429f100fb 100644 --- a/python/sglang/srt/layers/aux_hidden_states.py +++ b/python/sglang/srt/layers/aux_hidden_states.py @@ -1,9 +1,14 @@ """Aux hidden states captured for Eagle3/DFlash draft models.""" -from typing import List, Optional, Union +from __future__ import annotations + +from typing import TYPE_CHECKING, List, Optional, Union import torch +if TYPE_CHECKING: + from sglang.srt.model_executor.forward_batch_info import ForwardBatch + # Two representations coexist: models migrated to AuxHiddenStatePacker pass one # packed [tokens, K * hidden] tensor, the rest still pass a list of K tensors. AuxHiddenStates = Union[torch.Tensor, List[torch.Tensor]] @@ -27,21 +32,35 @@ class AuxHiddenStatePacker: Each ``.append()`` writes into one preallocated ``[tokens, K * hidden]`` buffer, avoiding the list path's transient ~2x HBM at ``torch.cat``. Assumes all captures share leading shape and feature size. + + ``for_batch`` writes into a decode graph runner's buffer when the batch + carries one, so every graph size aliases it instead of pinning its own. """ - def __init__(self, num_captures: int) -> None: + def __init__(self, num_captures: int, out: Optional[torch.Tensor] = None) -> None: self._num_captures = int(num_captures) - self._buffer: Optional[torch.Tensor] = None + self._buffer = out self._feature_size: Optional[int] = None self._idx = 0 + @classmethod + def for_batch( + cls, forward_batch: ForwardBatch, num_captures: int + ) -> AuxHiddenStatePacker: + return cls(num_captures, out=forward_batch.aux_hidden_states_buffer) + def append(self, hidden: torch.Tensor) -> None: feature_size = int(hidden.shape[-1]) - if self._buffer is None: + if self._feature_size is None: self._feature_size = feature_size - self._buffer = hidden.new_empty( - (*hidden.shape[:-1], feature_size * self._num_captures) - ) + shape = (*hidden.shape[:-1], feature_size * self._num_captures) + if self._buffer is None: + self._buffer = hidden.new_empty(shape) + elif self._buffer.shape != shape or self._buffer.dtype != hidden.dtype: + raise ValueError( + f"aux hidden output buffer is {tuple(self._buffer.shape)} " + f"{self._buffer.dtype}, captures need {shape} {hidden.dtype}" + ) start = self._idx * self._feature_size self._buffer[..., start : start + self._feature_size].copy_(hidden) self._idx += 1 @@ -55,7 +74,7 @@ class AuxHiddenStatePacker: def finalize(self) -> torch.Tensor: """Return the packed buffer; callers guard the empty case on ``len()``.""" - if self._buffer is None or self._idx != self._num_captures: + if self._feature_size is None or self._idx != self._num_captures: raise RuntimeError( f"captured {self._idx} of {self._num_captures} aux hidden states" ) diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index ee2e5d2bf0..0349758734 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -681,6 +681,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): # Preallocated piecewise-graph attention output, set by RadixAttention. _attn_output: Optional[torch.Tensor] = None + # Decode-graph-owned destination for AuxHiddenStatePacker; None when eager. + aux_hidden_states_buffer: Optional[torch.Tensor] = None + # Prefill body-CUDA-graph context limit. Attention backends that allocate # context-shaped metadata use this fixed maximum instead of deriving a # shape from the live batch. None preserves eager/default graph behavior. diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index 604a72f4ed..bc1d618af4 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -781,6 +781,13 @@ class ModelRunner: pp_rank=self.pp_rank, ) + def get_aux_hidden_states_width(self) -> int: + return misc_utils.resolve_aux_hidden_states_width( + model=self.model, + spec_algorithm=self.spec_algorithm, + is_draft_worker=self.is_draft_worker, + ) + def get_pp_proxy_topk_size(self) -> Optional[int]: return misc_utils.resolve_pp_proxy_topk_size( model_config=self.model_config, diff --git a/python/sglang/srt/model_executor/model_runner_components/misc_utils.py b/python/sglang/srt/model_executor/model_runner_components/misc_utils.py index f510693e54..ddc4fdee76 100644 --- a/python/sglang/srt/model_executor/model_runner_components/misc_utils.py +++ b/python/sglang/srt/model_executor/model_runner_components/misc_utils.py @@ -112,3 +112,20 @@ def resolve_pp_proxy_dspark_hidden_size( if isinstance(model, _SupportsDSparkPPProxy): return model.get_pp_proxy_dspark_hidden_size() return 0 + + +@runtime_checkable +class _SupportsPackedAuxHiddenStates(Protocol): + def get_aux_hidden_states_width(self) -> int: ... + + +def resolve_aux_hidden_states_width( + *, model: Any, spec_algorithm: Any, is_draft_worker: bool +) -> int: + # Sharing needs the spec worker to consume target hidden states before + # the next target forward overwrites them; the DFlash family does. + if not spec_algorithm.is_dflash_family() or is_draft_worker: + return 0 + if isinstance(model, _SupportsPackedAuxHiddenStates): + return model.get_aux_hidden_states_width() + return 0 diff --git a/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py index a7b7375411..547ce91771 100644 --- a/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py @@ -439,6 +439,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): pp_proxy_dspark_hidden_size=( self.model_runner.get_pp_proxy_dspark_hidden_size() ), + aux_hidden_states_width=self.model_runner.get_aux_hidden_states_width(), ) self.buffers.share_buffers() # FB-shared slot registry adopting DecodeInputBuffers storage (same @@ -906,6 +907,11 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): ) mrope_positions = _slot("mrope_positions") next_token_logits_buffer = buffers.next_token_logits_buffer[:num_tokens] + aux_hidden_states_buffer = ( + buffers.aux_hidden_states[:num_tokens] + if buffers.aux_hidden_states is not None + else None + ) rids_int = buffers.rids_int[:bs] if buffers.rids_int is not None else None bootstrap_room_ids_int = ( buffers.bootstrap_room_ids_int[:bs] @@ -981,6 +987,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): seq_lens=seq_lens, seq_lens_cpu=seq_lens_cpu, next_token_logits_buffer=next_token_logits_buffer, + aux_hidden_states_buffer=aux_hidden_states_buffer, orig_seq_lens=seq_lens, out_cache_loc=out_cache_loc, seq_lens_sum=seq_lens.sum().item(), diff --git a/python/sglang/srt/model_executor/runner_utils/buffers.py b/python/sglang/srt/model_executor/runner_utils/buffers.py index 30fadd8993..ac7b53a3ef 100644 --- a/python/sglang/srt/model_executor/runner_utils/buffers.py +++ b/python/sglang/srt/model_executor/runner_utils/buffers.py @@ -114,6 +114,8 @@ class DecodeInputBuffers(ForwardInputBuffers): num_token_non_padded: Optional[torch.Tensor] custom_mask: torch.Tensor next_token_logits_buffer: torch.Tensor + # Packed aux hidden-state output shared by every captured graph size. + aux_hidden_states: Optional[torch.Tensor] mamba_track_indices: Optional[torch.Tensor] mamba_track_mask: Optional[torch.Tensor] global_num_tokens_gpu: torch.Tensor @@ -148,6 +150,7 @@ class DecodeInputBuffers(ForwardInputBuffers): pp_proxy_topk_size: Optional[int] = None, pp_proxy_residual_num_blocks: Optional[int] = None, pp_proxy_dspark_hidden_size: int = 0, + aux_hidden_states_width: int = 0, ) -> DecodeInputBuffers: with torch.device(device): input_ids = torch.zeros((max_num_token,), dtype=torch.int64) @@ -175,6 +178,11 @@ class DecodeInputBuffers(ForwardInputBuffers): mamba_track_mask = ( torch.zeros((max_bs,), dtype=torch.bool) if enable_mamba_track else None ) + aux_hidden_states = ( + torch.zeros((max_num_token, aux_hidden_states_width), dtype=dtype) + if aux_hidden_states_width + else None + ) pp_proxy_tensors = ( _allocate_pp_proxy_tensors( @@ -246,6 +254,7 @@ class DecodeInputBuffers(ForwardInputBuffers): num_token_non_padded=num_token_non_padded, custom_mask=custom_mask, next_token_logits_buffer=next_token_logits_buffer, + aux_hidden_states=aux_hidden_states, mamba_track_indices=mamba_track_indices, mamba_track_mask=mamba_track_mask, encoder_lens=encoder_lens, diff --git a/python/sglang/srt/models/kimi_k3.py b/python/sglang/srt/models/kimi_k3.py index 9f3f44b003..6de017b268 100644 --- a/python/sglang/srt/models/kimi_k3.py +++ b/python/sglang/srt/models/kimi_k3.py @@ -33,6 +33,10 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r from sglang.srt.layers import zero_copy_context from sglang.srt.layers.activation import SiluAndMul, SituAndMul from sglang.srt.layers.attn_residual import AttnResidual, aggregate_stream, get_cw +from sglang.srt.layers.aux_hidden_states import ( + AuxHiddenStateAccumulator, + AuxHiddenStatePacker, +) from sglang.srt.layers.communication import k3_ar_fusion, k3_sp_collective from sglang.srt.layers.dcp.planner import prepare_decode_context_parallel_metadata from sglang.srt.layers.dp_attention import ( @@ -2958,7 +2962,14 @@ class KimiK3LinearModel(nn.Module): and k3_sp_collective.enabled() ) sp_sharded = False - aux_hidden_states = [] + packs_aux = self.packs_aux_hidden_states + aux_hidden_states: AuxHiddenStateAccumulator = ( + AuxHiddenStatePacker.for_batch( + forward_batch, len(self.dspark_layers_to_capture) + ) + if packs_aux + else [] + ) if ( self.dspark_layers_to_capture is not None and not self.pp_group.is_first_rank @@ -3049,10 +3060,19 @@ class KimiK3LinearModel(nn.Module): else: hidden_states, _ = self.norm(hidden_states, residual) + if packs_aux: + return hidden_states, aux_hidden_states.finalize() if self.dspark_layers_to_capture is not None: return hidden_states, aux_hidden_states return hidden_states + @property + def packs_aux_hidden_states(self) -> bool: + # PP stages keep the list: inherited captures arrive pre-concatenated. + return ( + self.dspark_layers_to_capture is not None and self.pp_group.world_size == 1 + ) + def _dspark_capture_stream( self, layer_idx: int, @@ -3145,6 +3165,11 @@ class KimiK3LinearForCausalLM(nn.Module): layer < self.model.start_layer - 1 for layer in layers ) + def get_aux_hidden_states_width(self) -> int: + if not self.model.packs_aux_hidden_states: + return 0 + return len(self.model.dspark_layers_to_capture) * self.config.hidden_size + def set_dspark_layers_to_capture(self, layer_ids: list[int]) -> None: if layer_ids is None: raise ValueError( @@ -3628,6 +3653,11 @@ class KimiK3ForConditionalGeneration(nn.Module): ) self.language_model.set_dspark_layers_to_capture(layer_ids) + def get_aux_hidden_states_width(self) -> int: + if self.language_model is None: + return 0 + return self.language_model.get_aux_hidden_states_width() + def preprocess_mm_for_encoder( self, mm_data, diff --git a/python/sglang/srt/speculative/dflash_worker_v2.py b/python/sglang/srt/speculative/dflash_worker_v2.py index 35361a58a5..75e698ee80 100644 --- a/python/sglang/srt/speculative/dflash_worker_v2.py +++ b/python/sglang/srt/speculative/dflash_worker_v2.py @@ -2868,6 +2868,8 @@ class DFlashWorkerV2(BaseSpecWorker): ) hidden = hidden.view(bs, int(self.block_size), -1) + # Consume in this step: every decode graph size shares one aux output, + # which the next target forward overwrites (resolve_aux_hidden_states_width). self._append_target_hidden_to_draft_kv_by_loc( target_hidden=hidden.reshape(-1, hidden.shape[-1]), cache_loc=verify_out_cache_loc, diff --git a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py index e40b4afaf1..a63e3198b7 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py @@ -929,6 +929,8 @@ class DSparkWorkerV2(BaseSpecWorker): ) folded_commit = folded_accept and epilogue.folds_commit + # Consume in this step: every decode graph size shares one aux output, + # which the next target forward overwrites (resolve_aux_hidden_states_width). if not folded_commit: self._verify_executor.commit_hidden( batch=batch, diff --git a/test/registered/unit/batch_overlap/test_tbo_filter_batch_marker.py b/test/registered/unit/batch_overlap/test_tbo_filter_batch_marker.py index 5d1b63805c..ca1417e387 100644 --- a/test/registered/unit/batch_overlap/test_tbo_filter_batch_marker.py +++ b/test/registered/unit/batch_overlap/test_tbo_filter_batch_marker.py @@ -93,6 +93,14 @@ class TestTboFilterBatchMarker(CustomTestCase): child = _filter(parent, lo=0, hi=4) self.assertEqual(child.defer_logits_to_eager, deferred) + def test_filter_batch_drops_aux_hidden_states_buffer(self): + """Decode graph capture sets the shared aux output on the parent before + the TBO split; children must not inherit it.""" + parent = _make_target_verify_batch(8) + parent.aux_hidden_states_buffer = torch.empty(8, 12) + child = _filter(parent, lo=0, hi=4) + self.assertIsNone(child.aux_hidden_states_buffer) + def _make_valued_batch(bs: int) -> ForwardBatch: # Distinct per-position values so a filtered slice is unambiguous. diff --git a/test/registered/unit/model_executor/model_runner_components/test_spec_aux_hidden_state.py b/test/registered/unit/model_executor/model_runner_components/test_spec_aux_hidden_state.py index a48604188f..3a57b6ef58 100644 --- a/test/registered/unit/model_executor/model_runner_components/test_spec_aux_hidden_state.py +++ b/test/registered/unit/model_executor/model_runner_components/test_spec_aux_hidden_state.py @@ -2,7 +2,15 @@ import sys from types import SimpleNamespace import pytest +import torch +from sglang.srt.layers.aux_hidden_states import ( + AuxHiddenStatePacker, + pack_aux_hidden_states, +) +from sglang.srt.model_executor.model_runner_components.misc_utils import ( + resolve_aux_hidden_states_width, +) from sglang.srt.model_executor.model_runner_components.spec_aux_hidden_state import ( _map_muse_target_layer_ids, ) @@ -34,5 +42,55 @@ def test_muse_target_layer_id_mapping(target_model_type, draft_architecture, exp ) +def test_packer_writes_into_graph_buffer_like_list_path(): + """A graph runner hands every graph size a row slice of one buffer; the + packed output must equal the list path's concat and live in that storage, + and a buffer that does not fit the captures must be rejected.""" + captures = [torch.randn(5, 4) for _ in range(3)] + shared = torch.full((8, 12), float("nan")) + packer = AuxHiddenStatePacker.for_batch( + SimpleNamespace(aux_hidden_states_buffer=shared[:5]), 3 + ) + for hidden in captures: + packer.append(hidden) + packed = packer.finalize() + + torch.testing.assert_close(packed, pack_aux_hidden_states(list(captures))) + assert packed.data_ptr() == shared.data_ptr() + assert shared[5:].isnan().all() + + too_small = SimpleNamespace(aux_hidden_states_buffer=shared[:4]) + with pytest.raises(ValueError): + AuxHiddenStatePacker.for_batch(too_small, 3).append(captures[0]) + + +class _SharedAuxModel: + def get_aux_hidden_states_width(self) -> int: + return 12 + + +@pytest.mark.parametrize( + ("model", "dflash", "is_draft_worker", "expected"), + [ + (_SharedAuxModel(), True, False, 12), + # Only the DFlash family is known to consume them within the step. + (_SharedAuxModel(), False, False, 0), + (_SharedAuxModel(), True, True, 0), + (object(), True, False, 0), + ], +) +def test_only_dflash_target_models_share_aux_output( + model, dflash, is_draft_worker, expected +): + assert ( + resolve_aux_hidden_states_width( + model=model, + spec_algorithm=SimpleNamespace(is_dflash_family=lambda: dflash), + is_draft_worker=is_draft_worker, + ) + == expected + ) + + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v"]))