[Spec] Reuse K3 auxiliary outputs across decode CUDA graph sizes (#40159)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: hnyls2002 <lsyincs@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
hnyls2002
parent
46beacf13d
commit
84523d6785
@@ -841,6 +841,8 @@ class TboForwardBatchPreparer:
|
||||
token_ids_logprobs=None,
|
||||
extend_input_logprob_token_ids_gpu=None,
|
||||
next_token_logits_buffer=None,
|
||||
# The aux packer runs in the parent model forward, not per child.
|
||||
aux_hidden_states_buffer=None,
|
||||
return_hidden_states_before_norm=False,
|
||||
# TBO children start unplanned — planned by the TBO-aware init
|
||||
# flow; a stale parent "ready" would wrongly skip that.
|
||||
|
||||
@@ -1,9 +1,14 @@
|
||||
"""Aux hidden states captured for Eagle3/DFlash draft models."""
|
||||
|
||||
from typing import List, Optional, Union
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, List, Optional, Union
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
|
||||
# Two representations coexist: models migrated to AuxHiddenStatePacker pass one
|
||||
# packed [tokens, K * hidden] tensor, the rest still pass a list of K tensors.
|
||||
AuxHiddenStates = Union[torch.Tensor, List[torch.Tensor]]
|
||||
@@ -27,21 +32,35 @@ class AuxHiddenStatePacker:
|
||||
Each ``.append()`` writes into one preallocated ``[tokens, K * hidden]``
|
||||
buffer, avoiding the list path's transient ~2x HBM at ``torch.cat``.
|
||||
Assumes all captures share leading shape and feature size.
|
||||
|
||||
``for_batch`` writes into a decode graph runner's buffer when the batch
|
||||
carries one, so every graph size aliases it instead of pinning its own.
|
||||
"""
|
||||
|
||||
def __init__(self, num_captures: int) -> None:
|
||||
def __init__(self, num_captures: int, out: Optional[torch.Tensor] = None) -> None:
|
||||
self._num_captures = int(num_captures)
|
||||
self._buffer: Optional[torch.Tensor] = None
|
||||
self._buffer = out
|
||||
self._feature_size: Optional[int] = None
|
||||
self._idx = 0
|
||||
|
||||
@classmethod
|
||||
def for_batch(
|
||||
cls, forward_batch: ForwardBatch, num_captures: int
|
||||
) -> AuxHiddenStatePacker:
|
||||
return cls(num_captures, out=forward_batch.aux_hidden_states_buffer)
|
||||
|
||||
def append(self, hidden: torch.Tensor) -> None:
|
||||
feature_size = int(hidden.shape[-1])
|
||||
if self._buffer is None:
|
||||
if self._feature_size is None:
|
||||
self._feature_size = feature_size
|
||||
self._buffer = hidden.new_empty(
|
||||
(*hidden.shape[:-1], feature_size * self._num_captures)
|
||||
)
|
||||
shape = (*hidden.shape[:-1], feature_size * self._num_captures)
|
||||
if self._buffer is None:
|
||||
self._buffer = hidden.new_empty(shape)
|
||||
elif self._buffer.shape != shape or self._buffer.dtype != hidden.dtype:
|
||||
raise ValueError(
|
||||
f"aux hidden output buffer is {tuple(self._buffer.shape)} "
|
||||
f"{self._buffer.dtype}, captures need {shape} {hidden.dtype}"
|
||||
)
|
||||
start = self._idx * self._feature_size
|
||||
self._buffer[..., start : start + self._feature_size].copy_(hidden)
|
||||
self._idx += 1
|
||||
@@ -55,7 +74,7 @@ class AuxHiddenStatePacker:
|
||||
|
||||
def finalize(self) -> torch.Tensor:
|
||||
"""Return the packed buffer; callers guard the empty case on ``len()``."""
|
||||
if self._buffer is None or self._idx != self._num_captures:
|
||||
if self._feature_size is None or self._idx != self._num_captures:
|
||||
raise RuntimeError(
|
||||
f"captured {self._idx} of {self._num_captures} aux hidden states"
|
||||
)
|
||||
|
||||
@@ -681,6 +681,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
# Preallocated piecewise-graph attention output, set by RadixAttention.
|
||||
_attn_output: Optional[torch.Tensor] = None
|
||||
|
||||
# Decode-graph-owned destination for AuxHiddenStatePacker; None when eager.
|
||||
aux_hidden_states_buffer: Optional[torch.Tensor] = None
|
||||
|
||||
# Prefill body-CUDA-graph context limit. Attention backends that allocate
|
||||
# context-shaped metadata use this fixed maximum instead of deriving a
|
||||
# shape from the live batch. None preserves eager/default graph behavior.
|
||||
|
||||
@@ -781,6 +781,13 @@ class ModelRunner:
|
||||
pp_rank=self.pp_rank,
|
||||
)
|
||||
|
||||
def get_aux_hidden_states_width(self) -> int:
|
||||
return misc_utils.resolve_aux_hidden_states_width(
|
||||
model=self.model,
|
||||
spec_algorithm=self.spec_algorithm,
|
||||
is_draft_worker=self.is_draft_worker,
|
||||
)
|
||||
|
||||
def get_pp_proxy_topk_size(self) -> Optional[int]:
|
||||
return misc_utils.resolve_pp_proxy_topk_size(
|
||||
model_config=self.model_config,
|
||||
|
||||
@@ -112,3 +112,20 @@ def resolve_pp_proxy_dspark_hidden_size(
|
||||
if isinstance(model, _SupportsDSparkPPProxy):
|
||||
return model.get_pp_proxy_dspark_hidden_size()
|
||||
return 0
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class _SupportsPackedAuxHiddenStates(Protocol):
|
||||
def get_aux_hidden_states_width(self) -> int: ...
|
||||
|
||||
|
||||
def resolve_aux_hidden_states_width(
|
||||
*, model: Any, spec_algorithm: Any, is_draft_worker: bool
|
||||
) -> int:
|
||||
# Sharing needs the spec worker to consume target hidden states before
|
||||
# the next target forward overwrites them; the DFlash family does.
|
||||
if not spec_algorithm.is_dflash_family() or is_draft_worker:
|
||||
return 0
|
||||
if isinstance(model, _SupportsPackedAuxHiddenStates):
|
||||
return model.get_aux_hidden_states_width()
|
||||
return 0
|
||||
|
||||
@@ -439,6 +439,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
pp_proxy_dspark_hidden_size=(
|
||||
self.model_runner.get_pp_proxy_dspark_hidden_size()
|
||||
),
|
||||
aux_hidden_states_width=self.model_runner.get_aux_hidden_states_width(),
|
||||
)
|
||||
self.buffers.share_buffers()
|
||||
# FB-shared slot registry adopting DecodeInputBuffers storage (same
|
||||
@@ -906,6 +907,11 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
)
|
||||
mrope_positions = _slot("mrope_positions")
|
||||
next_token_logits_buffer = buffers.next_token_logits_buffer[:num_tokens]
|
||||
aux_hidden_states_buffer = (
|
||||
buffers.aux_hidden_states[:num_tokens]
|
||||
if buffers.aux_hidden_states is not None
|
||||
else None
|
||||
)
|
||||
rids_int = buffers.rids_int[:bs] if buffers.rids_int is not None else None
|
||||
bootstrap_room_ids_int = (
|
||||
buffers.bootstrap_room_ids_int[:bs]
|
||||
@@ -981,6 +987,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu=seq_lens_cpu,
|
||||
next_token_logits_buffer=next_token_logits_buffer,
|
||||
aux_hidden_states_buffer=aux_hidden_states_buffer,
|
||||
orig_seq_lens=seq_lens,
|
||||
out_cache_loc=out_cache_loc,
|
||||
seq_lens_sum=seq_lens.sum().item(),
|
||||
|
||||
@@ -114,6 +114,8 @@ class DecodeInputBuffers(ForwardInputBuffers):
|
||||
num_token_non_padded: Optional[torch.Tensor]
|
||||
custom_mask: torch.Tensor
|
||||
next_token_logits_buffer: torch.Tensor
|
||||
# Packed aux hidden-state output shared by every captured graph size.
|
||||
aux_hidden_states: Optional[torch.Tensor]
|
||||
mamba_track_indices: Optional[torch.Tensor]
|
||||
mamba_track_mask: Optional[torch.Tensor]
|
||||
global_num_tokens_gpu: torch.Tensor
|
||||
@@ -148,6 +150,7 @@ class DecodeInputBuffers(ForwardInputBuffers):
|
||||
pp_proxy_topk_size: Optional[int] = None,
|
||||
pp_proxy_residual_num_blocks: Optional[int] = None,
|
||||
pp_proxy_dspark_hidden_size: int = 0,
|
||||
aux_hidden_states_width: int = 0,
|
||||
) -> DecodeInputBuffers:
|
||||
with torch.device(device):
|
||||
input_ids = torch.zeros((max_num_token,), dtype=torch.int64)
|
||||
@@ -175,6 +178,11 @@ class DecodeInputBuffers(ForwardInputBuffers):
|
||||
mamba_track_mask = (
|
||||
torch.zeros((max_bs,), dtype=torch.bool) if enable_mamba_track else None
|
||||
)
|
||||
aux_hidden_states = (
|
||||
torch.zeros((max_num_token, aux_hidden_states_width), dtype=dtype)
|
||||
if aux_hidden_states_width
|
||||
else None
|
||||
)
|
||||
|
||||
pp_proxy_tensors = (
|
||||
_allocate_pp_proxy_tensors(
|
||||
@@ -246,6 +254,7 @@ class DecodeInputBuffers(ForwardInputBuffers):
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
custom_mask=custom_mask,
|
||||
next_token_logits_buffer=next_token_logits_buffer,
|
||||
aux_hidden_states=aux_hidden_states,
|
||||
mamba_track_indices=mamba_track_indices,
|
||||
mamba_track_mask=mamba_track_mask,
|
||||
encoder_lens=encoder_lens,
|
||||
|
||||
@@ -33,6 +33,10 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r
|
||||
from sglang.srt.layers import zero_copy_context
|
||||
from sglang.srt.layers.activation import SiluAndMul, SituAndMul
|
||||
from sglang.srt.layers.attn_residual import AttnResidual, aggregate_stream, get_cw
|
||||
from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
AuxHiddenStatePacker,
|
||||
)
|
||||
from sglang.srt.layers.communication import k3_ar_fusion, k3_sp_collective
|
||||
from sglang.srt.layers.dcp.planner import prepare_decode_context_parallel_metadata
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -2958,7 +2962,14 @@ class KimiK3LinearModel(nn.Module):
|
||||
and k3_sp_collective.enabled()
|
||||
)
|
||||
sp_sharded = False
|
||||
aux_hidden_states = []
|
||||
packs_aux = self.packs_aux_hidden_states
|
||||
aux_hidden_states: AuxHiddenStateAccumulator = (
|
||||
AuxHiddenStatePacker.for_batch(
|
||||
forward_batch, len(self.dspark_layers_to_capture)
|
||||
)
|
||||
if packs_aux
|
||||
else []
|
||||
)
|
||||
if (
|
||||
self.dspark_layers_to_capture is not None
|
||||
and not self.pp_group.is_first_rank
|
||||
@@ -3049,10 +3060,19 @@ class KimiK3LinearModel(nn.Module):
|
||||
else:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
|
||||
if packs_aux:
|
||||
return hidden_states, aux_hidden_states.finalize()
|
||||
if self.dspark_layers_to_capture is not None:
|
||||
return hidden_states, aux_hidden_states
|
||||
return hidden_states
|
||||
|
||||
@property
|
||||
def packs_aux_hidden_states(self) -> bool:
|
||||
# PP stages keep the list: inherited captures arrive pre-concatenated.
|
||||
return (
|
||||
self.dspark_layers_to_capture is not None and self.pp_group.world_size == 1
|
||||
)
|
||||
|
||||
def _dspark_capture_stream(
|
||||
self,
|
||||
layer_idx: int,
|
||||
@@ -3145,6 +3165,11 @@ class KimiK3LinearForCausalLM(nn.Module):
|
||||
layer < self.model.start_layer - 1 for layer in layers
|
||||
)
|
||||
|
||||
def get_aux_hidden_states_width(self) -> int:
|
||||
if not self.model.packs_aux_hidden_states:
|
||||
return 0
|
||||
return len(self.model.dspark_layers_to_capture) * self.config.hidden_size
|
||||
|
||||
def set_dspark_layers_to_capture(self, layer_ids: list[int]) -> None:
|
||||
if layer_ids is None:
|
||||
raise ValueError(
|
||||
@@ -3628,6 +3653,11 @@ class KimiK3ForConditionalGeneration(nn.Module):
|
||||
)
|
||||
self.language_model.set_dspark_layers_to_capture(layer_ids)
|
||||
|
||||
def get_aux_hidden_states_width(self) -> int:
|
||||
if self.language_model is None:
|
||||
return 0
|
||||
return self.language_model.get_aux_hidden_states_width()
|
||||
|
||||
def preprocess_mm_for_encoder(
|
||||
self,
|
||||
mm_data,
|
||||
|
||||
@@ -2868,6 +2868,8 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
)
|
||||
hidden = hidden.view(bs, int(self.block_size), -1)
|
||||
|
||||
# Consume in this step: every decode graph size shares one aux output,
|
||||
# which the next target forward overwrites (resolve_aux_hidden_states_width).
|
||||
self._append_target_hidden_to_draft_kv_by_loc(
|
||||
target_hidden=hidden.reshape(-1, hidden.shape[-1]),
|
||||
cache_loc=verify_out_cache_loc,
|
||||
|
||||
@@ -929,6 +929,8 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
)
|
||||
|
||||
folded_commit = folded_accept and epilogue.folds_commit
|
||||
# Consume in this step: every decode graph size shares one aux output,
|
||||
# which the next target forward overwrites (resolve_aux_hidden_states_width).
|
||||
if not folded_commit:
|
||||
self._verify_executor.commit_hidden(
|
||||
batch=batch,
|
||||
|
||||
Reference in New Issue
Block a user