diff --git a/python/sglang/srt/layers/aux_hidden_states.py b/python/sglang/srt/layers/aux_hidden_states.py index 2843abf098..92c987f5db 100644 --- a/python/sglang/srt/layers/aux_hidden_states.py +++ b/python/sglang/srt/layers/aux_hidden_states.py @@ -9,6 +9,18 @@ import torch AuxHiddenStates = Union[torch.Tensor, List[torch.Tensor]] +class AuxHiddenStateList(list): + """Retain aux snapshots independently of storage reused by later layers.""" + + def capture(self, hidden: torch.Tensor, *, owned: bool = False) -> None: + """Copy borrowed storage, or adopt a value whose ownership is transferred. + + Views and reusable communication buffers are borrowed even when they + are different tensor objects from the source. + """ + self.append(hidden if owned else hidden.clone()) + + class AuxHiddenStatePacker: """Drop-in for the ``[]`` a model collects Eagle3/DFlash captures into. @@ -17,9 +29,6 @@ class AuxHiddenStatePacker: Assumes all captures share leading shape and feature size. """ - # ``append`` copies, so producers need not clone a tensor they later mutate. - copies_on_append = True - def __init__(self, num_captures: int) -> None: self._num_captures = int(num_captures) self._buffer: Optional[torch.Tensor] = None @@ -37,6 +46,10 @@ class AuxHiddenStatePacker: self._buffer[..., start : start + self._feature_size].copy_(hidden) self._idx += 1 + def capture(self, hidden: torch.Tensor, *, owned: bool = False) -> None: + """Write directly to the final packed buffer, without an intermediate copy.""" + self.append(hidden) + def __len__(self) -> int: return self._idx @@ -49,8 +62,8 @@ class AuxHiddenStatePacker: return self._buffer -# What a model hands down the capture path: a plain list, or a packer writing in place. -AuxHiddenStateAccumulator = Union[List[torch.Tensor], AuxHiddenStatePacker] +# Both collectors accept borrowed values through capture; storage belongs to aux. +AuxHiddenStateAccumulator = Union[AuxHiddenStateList, AuxHiddenStatePacker] def pack_aux_hidden_states(aux_hidden_states: AuxHiddenStates) -> torch.Tensor: diff --git a/python/sglang/srt/layers/communicator/boundary.py b/python/sglang/srt/layers/communicator/boundary.py index 3fc3710ff9..6567240804 100644 --- a/python/sglang/srt/layers/communicator/boundary.py +++ b/python/sglang/srt/layers/communicator/boundary.py @@ -47,6 +47,7 @@ from sglang.srt.layers.communicator.ops import ( _mlp_input_slice, _mlp_input_without_dp, _read_input, + move_rows, tp_reduce_scatter, ) from sglang.srt.layers.communicator.residual import ( @@ -479,8 +480,6 @@ class FusedMlpInput(msgspec.Struct, frozen=True): # The group whose sum it completes. completes: SumGroup run: Callable[..., Optional[Tuple[torch.Tensor, torch.Tensor]]] - # It may hand back a new residual and leave the one it took unchanged. - may_return_new_residual: bool def tbo_split_moves(layer_input_rows: Layout) -> Tuple[Callable, Callable]: @@ -548,6 +547,10 @@ class StageEntry(msgspec.Struct, frozen=True): handoff: Optional[Callable] = None # The fused kernels prepare tries first. fused: Tuple["FusedMlpInput", ...] = () + # Return the updated residual to the producer's rows for aux capture. + capture_move: Optional[Callable] = None + capture_move_allocates: bool = False + capture_preserves_residual: Optional[Callable] = None class BoundarySteps(msgspec.Struct, frozen=True): @@ -624,6 +627,19 @@ class Boundary(msgspec.Struct, frozen=True): need.layout.sharded | (self.edge.residual_to.sharded & need.gathers_itself) ) + @property + def capture_move_allocates(self) -> bool: + """Returning through a gather allocates; returning through a cut aliases.""" + return bool(self.edge.residual_to.sharded - self.edge.produced.layout.sharded) + + @property + def capture_move(self) -> Optional[Callable]: + if self.edge.residual_to == self.edge.produced.layout: + return None + return partial( + move_rows, rows=self.edge.residual_to, to=self.edge.produced.layout + ) + def make_boundary( edge: EdgeDecl, @@ -808,6 +824,8 @@ def _select_boundary_steps( input_rows=into_attention.input_rows, input_move=into_attention.input_move, handoff=attention_handoff, + capture_move=into_attention.capture_move, + capture_move_allocates=into_attention.capture_move_allocates, ), ffn=StageEntry( prepare=into_ffn.prepare, diff --git a/python/sglang/srt/layers/communicator/fusions/cutedsl.py b/python/sglang/srt/layers/communicator/fusions/cutedsl.py index c473f73d00..75d7798de5 100644 --- a/python/sglang/srt/layers/communicator/fusions/cutedsl.py +++ b/python/sglang/srt/layers/communicator/fusions/cutedsl.py @@ -250,7 +250,6 @@ class CuteDSLFusion: self._mlp_input_reduce_output_and_update_and_read_residual, layer, ), - may_return_new_residual=True, ), ) return () diff --git a/python/sglang/srt/layers/communicator/layer.py b/python/sglang/srt/layers/communicator/layer.py index 5101bea133..179128e5df 100644 --- a/python/sglang/srt/layers/communicator/layer.py +++ b/python/sglang/srt/layers/communicator/layer.py @@ -59,9 +59,11 @@ from sglang.srt.layers.communicator.layout import ( token_axis_sizes, ) from sglang.srt.layers.communicator.ops import ( + CommunicateSimpleFn, _all_reduce_then_to_local_tokens, _hand_qkv_hook_its_input, _hand_scattered_input_to_attention, + _mlp_input_without_dp, _redistribute_output, _reduce_and_redistribute_output_step, _to_local_tokens, @@ -79,6 +81,7 @@ from sglang.srt.layers.communicator.residual.access import ( finish_layer_stack, fold, from_pp, + norm_output, snapshot, written, ) @@ -94,6 +97,7 @@ from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, is_enable_moe_cp_allgather, ) +from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm from sglang.srt.layers.moe import ( can_merge_post_experts_all_reduce, get_moe_a2a_backend, @@ -244,6 +248,7 @@ class LayerCommunicator: add_to_output = staticmethod(add_to_output) buffer = staticmethod(buffer) fold = staticmethod(fold) + norm_output = staticmethod(norm_output) written = staticmethod(written) finish_layer_stack = staticmethod(finish_layer_stack) @@ -361,6 +366,46 @@ class LayerCommunicator: else None ) + # Bind aux ownership to the selected input implementation once. + for name in ("_steps", "_cp_steps", "_input_scattered_steps", "_sp_steps"): + steps = self.__dict__[name] + if steps is None or steps.ffn is None: + continue + prepare = steps.ffn.prepare + if isinstance(prepare, partial): + prepare = prepare.keywords.get("step", prepare) + norm = self.post_attention_layernorm + known_norm = isinstance(norm, (RMSNorm, GemmaRMSNorm)) and type( + norm + ).forward_with_allreduce_fusion in ( + RMSNorm.forward_with_allreduce_fusion, + GemmaRMSNorm.forward_with_allreduce_fusion, + ) + if ( + known_norm + and isinstance(prepare, partial) + and prepare.func is _mlp_input_without_dp + and not prepare.keywords["gathers_residual"] + and steps.ffn.fused + and steps.ffn.fused[0].run + == self._mlp_input_reduce_output_and_update_and_read_residual + ): + self.__dict__[name] = msgspec.structs.replace( + steps, + attention=msgspec.structs.replace( + steps.attention, + capture_preserves_residual=self._capture_keeps_residual, + ), + ) + + def _capture_keeps_residual(self, value, forward_batch): + # The CUDA wrapper allocates a fresh residual even on backend fallback. + return ( + not _use_aiter + and get_parallel().attn_tp_size > 1 + and apply_flashinfer_allreduce_fusion(value.shape[0]) + ) + def _init_stage(self, stage: "LayerStage") -> None: """A layer that is one stage: every batch runs the two boundaries its declarations give.""" @@ -387,6 +432,7 @@ class LayerCommunicator: input_move=into.input_move, handoff=None if is_ffn else _hand_qkv_hook_its_input, fused=into.fused, + capture_move=into.capture_move, ) self._steps = BoundarySteps( attention=None if is_ffn else entry, @@ -643,7 +689,38 @@ class LayerCommunicator: captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None, post_residual_addition: Optional[torch.Tensor] = None, quant_format: str = "", + capture_output: Optional[Callable] = None, ): + # Aux consumers need a materialized output before the input norm. + # Complete communication here, preserving the existing add+norm kernel + # and its FP32 accumulation. Its residual result also supplies capture. + capture_before_read = capture_output is not None and ( + residual is None + or ( + isinstance(residual, ResidualStream) + and residual.residual is None + and residual.pending is None + ) + or post_residual_addition is not None + ) + if capture_output is not None: + hidden_states = ( + residual.complete(hidden_states) + if isinstance(residual, ResidualStream) + else reduce_output(hidden_states) + ) + if capture_before_read: + # Embeddings precede enter; HF deepstack capture precedes the + # extra addition. Neither is the residual returned by the read. + value, previous = ( + residual.finish(hidden_states) + if isinstance(residual, ResidualStream) + else (hidden_states, residual) + ) + if previous is None: + capture_output(value) + else: + capture_output(value + previous, owned=True) hidden_states, residual = self.prepare_attn( hidden_states, residual, @@ -651,6 +728,17 @@ class LayerCommunicator: quant_format=quant_format, post_residual_addition=post_residual_addition, ) + if capture_output is not None and not capture_before_read: + value = residual.residual + move = self.attn.entry(forward_batch).capture_move + if move is not None: + value = move(value, forward_batch=forward_batch) + keeps = self.attn.entry(forward_batch).capture_preserves_residual + capture_output( + value, + owned=self.attn.entry(forward_batch).capture_move_allocates + or (move is None and keeps is not None and keeps(value, forward_batch)), + ) if captured_last_layer_outputs is not None: residual_value = residual.residual move = self.attn.entry(forward_batch).input_move @@ -663,36 +751,18 @@ class LayerCommunicator: context=self._context, ) ) - if ( - gathered_last_layer_output is residual_value - # An accumulator that copies on append already holds a snapshot. - and not getattr(captured_last_layer_outputs, "copies_on_append", False) - and not self._post_attn_residual_is_read_only( - residual_value, forward_batch - ) - ): - gathered_last_layer_output = residual_value.clone() - captured_last_layer_outputs.append(gathered_last_layer_output) + keeps = self.attn.entry(forward_batch).capture_preserves_residual + captured_last_layer_outputs.capture( + gathered_last_layer_output, + owned=move is CommunicateSimpleFn._scattered_to_tp_attn_full + or ( + move is None + and keeps is not None + and keeps(residual_value, forward_batch) + ), + ) return hidden_states, residual - def _post_attn_residual_is_read_only( - self, residual: torch.Tensor, forward_batch: ForwardBatch - ) -> bool: - """True if ``prepare_mlp``'s post-attention RMSNorm leaves ``residual`` - untouched, so Eagle3 aux capture can keep its reference and skip the clone. - - Of the base fused entry's kernels only flashinfer's writes a fresh - ``residual_out`` (see ``flashinfer_allreduce_residual_rmsnorm``); the aiter - kernel and every plain norm fold into ``residual`` in place. It runs when - ``prepare_mlp`` selected a fused entry that may return a new residual and - the batch is not input-scattered. - """ - return ( - any(f.may_return_new_residual for f in self.ffn.entry(forward_batch).fused) - and not get_attn_tp_context().input_scattered - and apply_flashinfer_allreduce_fusion(residual.shape[0]) - ) - def publish_attn_lora_layout(self) -> None: """Attention consumes the DP-local token batch.""" if self._publish_lora_layout: @@ -829,7 +899,6 @@ class LayerCommunicator: FusedMlpInput( completes=SumGroup.ATTN_TP, run=self._mlp_input_reduce_output_and_update_and_read_residual, - may_return_new_residual=True, ), ) diff --git a/python/sglang/srt/layers/communicator/residual/access.py b/python/sglang/srt/layers/communicator/residual/access.py index 6e2408557d..a3846c085d 100644 --- a/python/sglang/srt/layers/communicator/residual/access.py +++ b/python/sglang/srt/layers/communicator/residual/access.py @@ -100,6 +100,23 @@ def finish_layer_stack( return reduce_output(hidden_states), residual +def norm_output(hidden_states, residual, norm, capture_output=None): + """The final norm and an optional capture of the same updated residual. + + Keep add and norm together: normalizing a separately rounded residual is + not numerically equivalent to the fused kernel's FP32 accumulation. The + capture callback owns retention of the borrowed residual storage. + """ + if residual is None: + if capture_output is not None: + capture_output(hidden_states) + return norm(hidden_states) + hidden_states, residual = norm(hidden_states, residual) + if capture_output is not None: + capture_output(residual) + return hidden_states + + def from_pp( tensors: PPProxyTensors, *, diff --git a/python/sglang/srt/model_executor/runner_backend/breakable_cuda_graph_backend.py b/python/sglang/srt/model_executor/runner_backend/breakable_cuda_graph_backend.py index 32aa6a9b9d..b36c6373d2 100644 --- a/python/sglang/srt/model_executor/runner_backend/breakable_cuda_graph_backend.py +++ b/python/sglang/srt/model_executor/runner_backend/breakable_cuda_graph_backend.py @@ -235,8 +235,8 @@ class BreakableCudaGraphBackend(DedupedCudaGraphMixin, BaseCudaGraphBackend): tensor, output_buffer.tensors[key], num_tokens ) return - if isinstance(output, (list, tuple)) and isinstance( - output_buffer, type(output) + if (isinstance(output, list) and isinstance(output_buffer, list)) or ( + isinstance(output, tuple) and isinstance(output_buffer, tuple) ): if len(output) != len(output_buffer): raise ValueError( diff --git a/python/sglang/srt/models/bailing_moe.py b/python/sglang/srt/models/bailing_moe.py index aa940b6f0c..beb608a05b 100644 --- a/python/sglang/srt/models/bailing_moe.py +++ b/python/sglang/srt/models/bailing_moe.py @@ -31,6 +31,10 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.aux_hidden_states import ( + AuxHiddenStateAccumulator, + AuxHiddenStateList, +) from sglang.srt.layers.communicator import ( LayerCommunicator, LayerFacts, @@ -722,7 +726,8 @@ class BailingMoEBlock(nn.Module): hidden_states: torch.Tensor, forward_batch: ForwardBatch, residual: Optional[torch.Tensor], - captured_last_layer_outputs: Optional[List[torch.Tensor]] = None, + captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None, + capture_output=None, ) -> torch.Tensor: hidden_states, residual = ( self.layer_communicator.prepare_attn_and_capture_last_layer_outputs( @@ -730,6 +735,7 @@ class BailingMoEBlock(nn.Module): residual, forward_batch, captured_last_layer_outputs=captured_last_layer_outputs, + capture_output=capture_output, ) ) @@ -822,16 +828,9 @@ class BailingMoEModel(nn.Module): self.start_layer ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) - aux_hidden_states = [] + aux_hidden_states = AuxHiddenStateList() for i in range(self.start_layer, self.end_layer): with get_global_expert_distribution_recorder().with_current_layer(i): - if i in self.layers_to_capture: - hidden_states, snapshot = self.layers[ - i - ].layer_communicator.capture_output( - hidden_states, residual, at_input=True - ) - aux_hidden_states.append(snapshot) layer = self.layers[i] hidden_states, residual = layer( positions, @@ -843,6 +842,9 @@ class BailingMoEModel(nn.Module): if getattr(layer, "_is_layer_to_capture", False) else None ), + capture_output=aux_hidden_states.capture + if i in self.layers_to_capture + else None, ) last_layer = self.layers[self.end_layer - 1] hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( diff --git a/python/sglang/srt/models/bailing_moe_v3.py b/python/sglang/srt/models/bailing_moe_v3.py index 7abf968a22..d8b8835a08 100644 --- a/python/sglang/srt/models/bailing_moe_v3.py +++ b/python/sglang/srt/models/bailing_moe_v3.py @@ -28,6 +28,7 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers import deep_gemm_wrapper from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList from sglang.srt.layers.communicator import ( LayerCommunicator, LayerFacts, @@ -1178,10 +1179,16 @@ class BailingMoELinearDecoderLayer(nn.Module): forward_batch: ForwardBatch, residual: Optional[torch.Tensor], zero_allocator: BumpAllocator, + capture_output=None, **kwargs, ) -> tuple[torch.Tensor, torch.Tensor]: - hidden_states, residual = self.layer_communicator.prepare_attn( - hidden_states, residual, forward_batch + hidden_states, residual = ( + self.layer_communicator.prepare_attn_and_capture_last_layer_outputs( + hidden_states, + residual, + forward_batch, + capture_output=capture_output, + ) ) if not forward_batch.forward_mode.is_idle(): @@ -1370,7 +1377,11 @@ class BailingMoELinearModel(nn.Module): and capture_mode.need_capture() ) if capture_aux: - dspark_aux_hidden_states: List[torch.Tensor] = [] + dspark_aux_hidden_states = AuxHiddenStateList() + + def capture_output(value, *, owned=False): + if value.shape[0] != 0: + dspark_aux_hidden_states.capture(value, owned=owned) for i in range(self.start_layer, self.end_layer): with get_global_expert_distribution_recorder().with_current_layer(i): @@ -1381,13 +1392,12 @@ class BailingMoELinearModel(nn.Module): forward_batch=forward_batch, residual=residual, zero_allocator=zero_allocator, + capture_output=capture_output + if capture_aux + and i > self.start_layer + and i - 1 in self.layers_to_capture + else None, ) - if capture_aux and i in self.layers_to_capture: - hidden_states, snapshot = layer.layer_communicator.capture_output( - hidden_states, residual, skip_empty=True - ) - if snapshot is not None: - dspark_aux_hidden_states.append(snapshot) last_layer = self.layers[self.end_layer - 1] hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( @@ -1399,10 +1409,16 @@ class BailingMoELinearModel(nn.Module): ) else: if not forward_batch.forward_mode.is_idle(): - if residual is None: - hidden_states = self.norm(hidden_states) - else: - hidden_states, _ = self.norm(hidden_states, residual) + hidden_states = last_layer.layer_communicator.norm_output( + hidden_states, + residual, + self.norm, + capture_output=( + capture_output + if capture_aux and self.end_layer - 1 in self.layers_to_capture + else None + ), + ) if capture_aux and len(dspark_aux_hidden_states) > 0: return hidden_states, dspark_aux_hidden_states return hidden_states diff --git a/python/sglang/srt/models/glm4_moe.py b/python/sglang/srt/models/glm4_moe.py index 1551531c93..218447e353 100644 --- a/python/sglang/srt/models/glm4_moe.py +++ b/python/sglang/srt/models/glm4_moe.py @@ -38,6 +38,7 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList from sglang.srt.layers.communicator import ( LayerCommunicator, LayerFacts, @@ -936,13 +937,17 @@ class Glm4MoeDecoderLayer(nn.Module): hidden_states: torch.Tensor, forward_batch: ForwardBatch, residual: Optional[torch.Tensor], + capture_output=None, ) -> torch.Tensor: - hidden_states, residual = self.layer_communicator.prepare_attn( - hidden_states, - residual, - forward_batch, - quant_format=self.attn_quant_format, + hidden_states, residual = ( + self.layer_communicator.prepare_attn_and_capture_last_layer_outputs( + hidden_states, + residual, + forward_batch, + quant_format=self.attn_quant_format, + capture_output=capture_output, + ) ) hidden_states = self.self_attn( @@ -1103,22 +1108,18 @@ class Glm4MoeModel(nn.Module): elif self.first_k_dense_replace < normal_start_layer: normal_end_layer = normal_start_layer = 0 - aux_hidden_states = [] + aux_hidden_states = AuxHiddenStateList() for i in range(normal_start_layer, normal_end_layer): with get_global_expert_distribution_recorder().with_current_layer(i): - if i in self.layers_to_capture: - hidden_states, snapshot = self.layers[ - i - ].layer_communicator.capture_output( - hidden_states, residual, at_input=True - ) - aux_hidden_states.append(snapshot) layer = self.layers[i] hidden_states, residual = layer( positions, hidden_states, forward_batch, residual, + capture_output=aux_hidden_states.capture + if i in self.layers_to_capture + else None, ) if normal_end_layer != self.end_layer: diff --git a/python/sglang/srt/models/glm4_moe_lite.py b/python/sglang/srt/models/glm4_moe_lite.py index 379a5db7f0..c25925d36c 100644 --- a/python/sglang/srt/models/glm4_moe_lite.py +++ b/python/sglang/srt/models/glm4_moe_lite.py @@ -32,6 +32,7 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList from sglang.srt.layers.communicator import ( LayerCommunicator, LayerFacts, @@ -625,12 +626,16 @@ class Glm4MoeLiteDecoderLayer(nn.Module): forward_batch: ForwardBatch, residual: Optional[torch.Tensor], zero_allocator: BumpAllocator, + capture_output=None, ) -> torch.Tensor: - hidden_states, residual = self.layer_communicator.prepare_attn( - hidden_states, - residual, - forward_batch, - getattr(self, "_gfx95_quant_format", ""), + hidden_states, residual = ( + self.layer_communicator.prepare_attn_and_capture_last_layer_outputs( + hidden_states, + residual, + forward_batch, + quant_format=getattr(self, "_gfx95_quant_format", ""), + capture_output=capture_output, + ) ) hidden_states = self.self_attn( @@ -800,16 +805,9 @@ class Glm4MoeLiteModel(nn.Module): normal_end_layer = self.first_k_dense_replace elif self.first_k_dense_replace < normal_start_layer: normal_end_layer = normal_start_layer = 0 - aux_hidden_states = [] + aux_hidden_states = AuxHiddenStateList() for i in range(normal_start_layer, normal_end_layer): with get_global_expert_distribution_recorder().with_current_layer(i): - if i in self.layers_to_capture: - hidden_states, snapshot = self.layers[ - i - ].layer_communicator.capture_output( - hidden_states, residual, at_input=True - ) - aux_hidden_states.append(snapshot) layer = self.layers[i] hidden_states, residual = layer( positions, @@ -817,6 +815,9 @@ class Glm4MoeLiteModel(nn.Module): forward_batch, residual, zero_allocator, + capture_output=aux_hidden_states.capture + if i in self.layers_to_capture + else None, ) if normal_end_layer != self.end_layer: diff --git a/python/sglang/srt/models/glm5_next.py b/python/sglang/srt/models/glm5_next.py index 10b9ea6d92..525809e395 100644 --- a/python/sglang/srt/models/glm5_next.py +++ b/python/sglang/srt/models/glm5_next.py @@ -25,6 +25,7 @@ from sglang.srt.eplb.expert_distribution import ( from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers.attention import vision_utils from sglang.srt.layers.attention.vision import VisionAttention +from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList from sglang.srt.layers.communicator import ( LayerCommunicator, LayerFacts, @@ -918,13 +919,17 @@ class Glm5NextDecoderLayer(nn.Module): zero_allocator: Optional[BumpAllocator] = None, gemm_output_zero_allocator: BumpAllocator = None, prev_topk_indices: Optional[torch.Tensor] = None, + capture_output=None, ): hidden_states_orig = self.layer_communicator.buffer(hidden_states) - hidden_states, residual = self.layer_communicator.prepare_attn( - hidden_states, - residual, - forward_batch, + hidden_states, residual = ( + self.layer_communicator.prepare_attn_and_capture_last_layer_outputs( + hidden_states, + residual, + forward_batch, + capture_output=capture_output, + ) ) hidden_states = self.self_attn( @@ -1139,7 +1144,7 @@ class Glm5NextModel(nn.Module): normal_end_layer = self.first_k_dense_replace elif self.first_k_dense_replace < normal_start_layer: normal_end_layer = normal_start_layer = 0 - aux_hidden_states = [] + aux_hidden_states = AuxHiddenStateList() topk_indices = None for i in range(normal_start_layer, normal_end_layer): # NOTE: torch dynamo does not support graph break in context manager @@ -1149,22 +1154,16 @@ class Glm5NextModel(nn.Module): else get_global_expert_distribution_recorder().with_current_layer(i) ) with ctx: - if i in self.layers_to_capture: - if self.dflash_capture and self.config.mhc: - # hc_post has completed the streams; contracting them - # creates the retained value without copying the wide input. - aux_hidden_state = self._prepare_aux_hidden_state(hidden_states) - else: - hidden_states, aux_hidden_state = self.layers[ - i - ].layer_communicator.capture_output( - hidden_states, residual, at_input=True - ) + + def capture_output(aux_hidden_state, *, owned=False): + aux_hidden_state = self._prepare_aux_hidden_state(aux_hidden_state) + owned = owned or (self.dflash_capture and self.config.mhc) if self.enable_a2a_moe and i > self.first_k_dense_replace: - aux_hidden_state = get_parallel().attn_tp_group.all_gather( - aux_hidden_state, dim=0 - ) - aux_hidden_states.append(aux_hidden_state) + group = get_parallel().attn_tp_group + aux_hidden_state = group.all_gather(aux_hidden_state, dim=0) + owned = owned or group.world_size > 1 + aux_hidden_states.capture(aux_hidden_state, owned=owned) + layer = self.layers[i] hidden_states, residual, topk_indices = layer( positions, @@ -1174,6 +1173,9 @@ class Glm5NextModel(nn.Module): zero_allocator, gemm_output_zero_allocator, prev_topk_indices=topk_indices, + capture_output=capture_output + if i in self.layers_to_capture + else None, ) if normal_end_layer != self.end_layer: diff --git a/python/sglang/srt/models/gpt_oss.py b/python/sglang/srt/models/gpt_oss.py index 110bdcf360..669ab0566b 100644 --- a/python/sglang/srt/models/gpt_oss.py +++ b/python/sglang/srt/models/gpt_oss.py @@ -29,6 +29,7 @@ from transformers import PretrainedConfig from sglang.kernels.jit.utils import is_arch_support_pdl from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation +from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList from sglang.srt.layers.communicator import ( LayerCommunicator, LayerFacts, @@ -618,9 +619,15 @@ class GptOssDecoderLayer(nn.Module): hidden_states: torch.Tensor, forward_batch: ForwardBatch, residual: Optional[torch.Tensor], + capture_output=None, ) -> Tuple[torch.Tensor, torch.Tensor]: - hidden_states, residual = self.layer_communicator.prepare_attn( - hidden_states, residual, forward_batch + hidden_states, residual = ( + self.layer_communicator.prepare_attn_and_capture_last_layer_outputs( + hidden_states, + residual, + forward_batch, + capture_output=capture_output, + ) ) if hidden_states.shape[0] != 0: @@ -710,24 +717,19 @@ class GptOssModel(nn.Module): # Capture hidden-state boundaries: boundary 0 is the embedding output, # and boundary i + 1 is the output after transformer block i. - aux_hidden_states = [] - if self.start_layer in self.layers_to_capture: - aux_hidden_states.append( - self.layers[self.start_layer].layer_communicator.snapshot( - hidden_states, residual, at_input=True - ) - ) + aux_hidden_states = AuxHiddenStateList() for i in range(self.start_layer, self.end_layer): with get_global_expert_distribution_recorder().with_current_layer(i): layer = self.layers[i] hidden_states, residual = layer( - positions, hidden_states, forward_batch, residual + positions, + hidden_states, + forward_batch, + residual, + capture_output=aux_hidden_states.capture + if i in self.layers_to_capture + else None, ) - if i + 1 in self.layers_to_capture: - hidden_states, snapshot = layer.layer_communicator.capture_output( - hidden_states, residual - ) - aux_hidden_states.append(snapshot) last_layer = self.layers[self.end_layer - 1] hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( hidden_states, residual, forward_batch @@ -741,10 +743,20 @@ class GptOssModel(nn.Module): ) else: if hidden_states.shape[0] != 0: - if residual is None: - hidden_states = self.norm(hidden_states) - else: - hidden_states, _ = self.norm(hidden_states, residual) + hidden_states = last_layer.layer_communicator.norm_output( + hidden_states, + residual, + self.norm, + capture_output=( + aux_hidden_states.append + if self.end_layer in self.layers_to_capture + else None + ), + ) + elif self.end_layer in self.layers_to_capture: + aux_hidden_states.append( + last_layer.layer_communicator.snapshot(hidden_states, residual) + ) if len(aux_hidden_states) == 0: return hidden_states diff --git a/python/sglang/srt/models/interns2_mobius.py b/python/sglang/srt/models/interns2_mobius.py index 6b49b2e81a..4c24814f90 100644 --- a/python/sglang/srt/models/interns2_mobius.py +++ b/python/sglang/srt/models/interns2_mobius.py @@ -10,6 +10,10 @@ from sglang.srt.configs.interns2_mobius import ( InternS2MobiusConfig, InternS2MobiusTextConfig, ) +from sglang.srt.layers.aux_hidden_states import ( + AuxHiddenStateAccumulator, + AuxHiddenStateList, +) from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import is_dp_attention_enabled @@ -644,7 +648,7 @@ class InternS2MobiusAttentionDecoderLayer( residual: torch.Tensor | None, forward_batch: ForwardBatch, meta_mlp: nn.ModuleList, - captured_last_layer_outputs: list[torch.Tensor] | None = None, + captured_last_layer_outputs: AuxHiddenStateAccumulator | None = None, **kwargs, ): del kwargs @@ -764,7 +768,7 @@ class InternS2MobiusForCausalLM(Qwen3_5ForCausalLM): self.embed_tokens(input_ids) if input_embeds is None else input_embeds ) residual = residual_batch.start(forward_batch) - aux_hidden_states = [] + aux_hidden_states = AuxHiddenStateList() for layer_idx, layer in enumerate(self.layers): hidden_states, residual = layer( positions=positions, diff --git a/python/sglang/srt/models/laguna.py b/python/sglang/srt/models/laguna.py index 9f464268c9..1264eded30 100644 --- a/python/sglang/srt/models/laguna.py +++ b/python/sglang/srt/models/laguna.py @@ -19,6 +19,7 @@ from torch import nn from sglang.srt.configs.laguna import LagunaConfig, normalize_gating from sglang.srt.environ import envs from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList from sglang.srt.layers.communicator import ( LayerCommunicator, LayerFacts, @@ -475,9 +476,15 @@ class LagunaDecoderLayer(nn.Module): hidden_states: torch.Tensor, forward_batch: ForwardBatch, residual: Optional[torch.Tensor], + capture_output=None, ) -> Tuple[torch.Tensor, torch.Tensor]: - hidden_states, residual = self.layer_communicator.prepare_attn( - hidden_states, residual, forward_batch + hidden_states, residual = ( + self.layer_communicator.prepare_attn_and_capture_last_layer_outputs( + hidden_states, + residual, + forward_batch, + capture_output=capture_output, + ) ) if hidden_states.shape[0] != 0: hidden_states = self.self_attn( @@ -589,18 +596,17 @@ class LagunaModel(nn.Module): self.start_layer ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) - aux_hidden_states = [] + aux_hidden_states = AuxHiddenStateList() for i in range(self.start_layer, self.end_layer): - if i in self.layers_to_capture: - hidden_states, snapshot = self.layers[ - i - ].layer_communicator.capture_output( - hidden_states, residual, at_input=True - ) - aux_hidden_states.append(snapshot) layer = self.layers[i] hidden_states, residual = layer( - positions, hidden_states, forward_batch, residual + positions, + hidden_states, + forward_batch, + residual, + capture_output=aux_hidden_states.capture + if i in self.layers_to_capture + else None, ) last_layer = self.layers[self.end_layer - 1] @@ -613,14 +619,16 @@ class LagunaModel(nn.Module): ) if hidden_states.shape[0] != 0: - if self.end_layer in self.layers_to_capture: - aux_hidden_states.append( - last_layer.layer_communicator.snapshot(hidden_states, residual) - ) - if residual is None: - hidden_states = self.norm(hidden_states) - else: - hidden_states, _ = self.norm(hidden_states, residual) + hidden_states = last_layer.layer_communicator.norm_output( + hidden_states, + residual, + self.norm, + capture_output=( + aux_hidden_states.append + if self.end_layer in self.layers_to_capture + else None + ), + ) if len(aux_hidden_states) == 0: return hidden_states return hidden_states, aux_hidden_states diff --git a/python/sglang/srt/models/minimax_m2.py b/python/sglang/srt/models/minimax_m2.py index 2164d98c43..e1150472e2 100644 --- a/python/sglang/srt/models/minimax_m2.py +++ b/python/sglang/srt/models/minimax_m2.py @@ -18,7 +18,7 @@ import logging from contextlib import nullcontext from functools import lru_cache -from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union +from typing import Any, Dict, Iterable, Optional, Set, Tuple, Union import torch import triton @@ -30,6 +30,10 @@ from sglang.kernels.kernel_api_logging import debug_kernel_api from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo +from sglang.srt.layers.aux_hidden_states import ( + AuxHiddenStateAccumulator, + AuxHiddenStateList, +) from sglang.srt.layers.communicator import ( LayerCommunicator, LayerFacts, @@ -997,7 +1001,7 @@ class MiniMaxM2DecoderLayer(nn.Module): hidden_states: torch.Tensor, forward_batch: ForwardBatch, residual: Optional[torch.Tensor], - captured_last_layer_outputs: Optional[List[torch.Tensor]] = None, + captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None, ) -> torch.Tensor: # Self Attention hidden_states, residual = ( @@ -1149,7 +1153,7 @@ class MiniMaxM2Model(nn.Module): self.start_layer ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) - aux_hidden_states = [] + aux_hidden_states = AuxHiddenStateList() if forward_batch.can_run_tbo: hidden_states, residual = model_forward_maybe_tbo( layers=self.layers, diff --git a/python/sglang/srt/models/minimax_m3.py b/python/sglang/srt/models/minimax_m3.py index 15edb41322..b8cab5d3c4 100644 --- a/python/sglang/srt/models/minimax_m3.py +++ b/python/sglang/srt/models/minimax_m3.py @@ -35,6 +35,10 @@ from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.aux_hidden_states import ( + AuxHiddenStateAccumulator, + AuxHiddenStateList, +) from sglang.srt.layers.communicator import ( LayerCommunicator, LayerFacts, @@ -1369,7 +1373,7 @@ class MiniMaxM3DecoderLayer(nn.Module): hidden_states: torch.Tensor, forward_batch: ForwardBatch, residual: Optional[torch.Tensor], - captured_last_layer_outputs: Optional[List[torch.Tensor]] = None, + captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None, **kwargs, ) -> torch.Tensor: hidden_states, residual = ( @@ -1484,7 +1488,7 @@ class MiniMaxM3Model(nn.Module): self.start_layer ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) - aux_hidden_states = [] + aux_hidden_states = AuxHiddenStateList() if forward_batch.can_run_tbo: hidden_states, residual = model_forward_maybe_tbo( layers=self.layers, diff --git a/python/sglang/srt/models/qwen2_moe.py b/python/sglang/srt/models/qwen2_moe.py index 59f8089b24..a8f2ecc5ba 100644 --- a/python/sglang/srt/models/qwen2_moe.py +++ b/python/sglang/srt/models/qwen2_moe.py @@ -40,6 +40,10 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.aux_hidden_states import ( + AuxHiddenStateAccumulator, + AuxHiddenStateList, +) from sglang.srt.layers.communicator import ( LayerCommunicator, LayerFacts, @@ -1070,7 +1074,7 @@ class Qwen2MoeDecoderLayer(nn.Module): hidden_states: torch.Tensor, forward_batch: ForwardBatch, residual: Optional[torch.Tensor], - captured_last_layer_outputs: Optional[List[torch.Tensor]] = None, + captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None, **kwargs, ) -> Tuple[torch.Tensor, torch.Tensor]: @@ -1183,7 +1187,7 @@ class Qwen2MoeModel(nn.Module): self.start_layer ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) - aux_hidden_states = [] + aux_hidden_states = AuxHiddenStateList() if forward_batch.can_run_tbo: hidden_states, residual = model_forward_maybe_tbo( layers=self.layers, diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py index f2a7c29cd2..1658edd2b5 100644 --- a/python/sglang/srt/models/qwen3_5.py +++ b/python/sglang/srt/models/qwen3_5.py @@ -23,7 +23,6 @@ import torch import torch.nn as nn import triton -# Layers - Attention from sglang.kernels.ops.attention.fla.layernorm_gated import RMSNorm as RMSNormGated from sglang.kernels.ops.attention.triton_gdn_fused_proj import ( fused_qkvzba_split_reshape_cat_contiguous, @@ -41,6 +40,12 @@ from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers.attention.mamba.mamba import mamba_v2_sharded_weight_loader + +# Layers - Attention +from sglang.srt.layers.aux_hidden_states import ( + AuxHiddenStateAccumulator, + AuxHiddenStateList, +) from sglang.srt.layers.communicator import ( LayerCommunicator, LayerFacts, @@ -1540,7 +1545,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module): hidden_states: torch.Tensor, residual: Optional[torch.Tensor], forward_batch: ForwardBatch, - captured_last_layer_outputs: Optional[list[torch.Tensor]] = None, + captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None, **kwargs, ): hidden_states, residual = ( @@ -1825,7 +1830,7 @@ class Qwen3_5ForCausalLM(nn.Module): self.start_layer ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) - aux_hidden_states = [] + aux_hidden_states = AuxHiddenStateList() # Pass through decoder layers for layer_idx in range(self.start_layer, self.end_layer): layer = self.layers[layer_idx] diff --git a/python/sglang/srt/models/qwen3_moe.py b/python/sglang/srt/models/qwen3_moe.py index ba1c21aa31..f3ee5b1e74 100644 --- a/python/sglang/srt/models/qwen3_moe.py +++ b/python/sglang/srt/models/qwen3_moe.py @@ -29,6 +29,7 @@ from transformers import PretrainedConfig from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo +from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm @@ -882,7 +883,8 @@ class Qwen3MoeDecoderLayer(nn.Module): hidden_states: torch.Tensor, forward_batch: ForwardBatch, residual: Optional[torch.Tensor], - captured_last_layer_outputs: Optional[List[torch.Tensor]] = None, + captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None, + capture_output=None, **kwargs, ) -> Tuple[torch.Tensor, torch.Tensor]: @@ -892,6 +894,7 @@ class Qwen3MoeDecoderLayer(nn.Module): residual, forward_batch, captured_last_layer_outputs=captured_last_layer_outputs, + capture_output=capture_output, **kwargs, ) ) diff --git a/python/sglang/srt/models/qwen3_next.py b/python/sglang/srt/models/qwen3_next.py index 28522c9ecd..73f16b58ce 100644 --- a/python/sglang/srt/models/qwen3_next.py +++ b/python/sglang/srt/models/qwen3_next.py @@ -15,6 +15,10 @@ from sglang.srt.configs.qwen3_next import Qwen3NextConfig from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers.attention.mamba.mamba import mamba_v2_sharded_weight_loader +from sglang.srt.layers.aux_hidden_states import ( + AuxHiddenStateAccumulator, + AuxHiddenStateList, +) from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( @@ -550,7 +554,7 @@ class Qwen3HybridLinearDecoderLayer(nn.Module): self, hidden_states: torch.Tensor, residual: Optional[torch.Tensor], - captured_last_layer_outputs: Optional[list[torch.Tensor]] = None, + captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None, **kwargs, ): forward_batch = kwargs.get("forward_batch", None) @@ -826,7 +830,7 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module): hidden_states: torch.Tensor, residual: Optional[torch.Tensor], forward_batch: ForwardBatch, - captured_last_layer_outputs: Optional[list[torch.Tensor]] = None, + captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None, **kwargs: Any, ): hidden_states, residual = ( @@ -931,7 +935,7 @@ class Qwen3NextModel(nn.Module): hidden_states = self.embed_tokens(input_ids) residual = residual_batch.start(forward_batch) - aux_hidden_states = [] + aux_hidden_states = AuxHiddenStateList() for i in range(len(self.layers)): layer = self.layers[i] with get_global_expert_distribution_recorder().with_current_layer(i): diff --git a/python/sglang/srt/models/qwen3_vl_moe.py b/python/sglang/srt/models/qwen3_vl_moe.py index 2b6f941ae7..22035e14e6 100644 --- a/python/sglang/srt/models/qwen3_vl_moe.py +++ b/python/sglang/srt/models/qwen3_vl_moe.py @@ -24,6 +24,7 @@ import torch.nn as nn from sglang.srt.configs.qwen3_vl import Qwen3VLMoeConfig, Qwen3VLMoeTextConfig from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation +from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.layers.quantization.base_config import QuantizationConfig @@ -102,16 +103,11 @@ class Qwen3MoeLLMModel(Qwen3MoeModel): self.start_layer ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) - aux_hidden_states = [] + aux_hidden_states = AuxHiddenStateList() for layer_idx, layer in enumerate( self.layers[self.start_layer : self.end_layer] ): layer_idx += self.start_layer - if layer_idx in self.layers_to_capture: - hidden_states, snapshot = layer.layer_communicator.capture_output( - hidden_states, residual, at_input=True - ) - aux_hidden_states.append(snapshot) if self.use_hf_deepstack_order: # HF-order path (RL on-policy / FSDP). SGLang applies residual at the START of the @@ -126,6 +122,9 @@ class Qwen3MoeLLMModel(Qwen3MoeModel): forward_batch, residual, post_residual_addition=deepstack_embeds, + capture_output=aux_hidden_states.capture + if layer_idx in self.layers_to_capture + else None, ) else: # Inference path: add deepstack directly to hidden_states at the end of the layer @@ -135,6 +134,9 @@ class Qwen3MoeLLMModel(Qwen3MoeModel): hidden_states, forward_batch, residual, + capture_output=aux_hidden_states.capture + if layer_idx in self.layers_to_capture + else None, ) if ( input_deepstack_embeds is not None diff --git a/python/sglang/srt/models/qwen4_exp.py b/python/sglang/srt/models/qwen4_exp.py index 68c46cb2d3..18623a1cc0 100644 --- a/python/sglang/srt/models/qwen4_exp.py +++ b/python/sglang/srt/models/qwen4_exp.py @@ -21,6 +21,7 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import ( from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation +from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList from sglang.srt.layers.communicator import get_attn_tp_context from sglang.srt.layers.dp_attention import ( attn_tp_all_gather, @@ -1748,7 +1749,7 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM): if self.has_ple else None ) - aux_hidden_states = [] + aux_hidden_states = AuxHiddenStateList() for i in range(self.start_layer, self.end_layer): layer = self.layers[i] if i + 1 < self.end_layer: diff --git a/test/registered/debug_utils/test_engine_dumper_comparator_e2e.py b/test/registered/debug_utils/test_engine_dumper_comparator_e2e.py index f6208d1f1e..80bce38c24 100644 --- a/test/registered/debug_utils/test_engine_dumper_comparator_e2e.py +++ b/test/registered/debug_utils/test_engine_dumper_comparator_e2e.py @@ -74,6 +74,7 @@ patches: residual, forward_batch, captured_last_layer_outputs=captured_last_layer_outputs, + capture_output=capture_output, **kwargs, ) ) @@ -131,6 +132,7 @@ patches: residual, forward_batch, captured_last_layer_outputs=captured_last_layer_outputs, + capture_output=capture_output, **kwargs, ) ) diff --git a/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py b/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py index b99fc9df96..c6d15f9605 100644 --- a/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py +++ b/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py @@ -172,7 +172,6 @@ def test_cutedsl_entries_come_before_the_base_fused_kernel(): assert fusions[1].run == base # The workspace reduces over the TP group, which is the attention-TP group here. assert [f.completes for f in fusions] == [SumGroup.ATTN_TP] * 2 - assert all(f.may_return_new_residual for f in fusions) # A residual on each rank's slice is gathered first, which the # workspace does not do. comm._declared = SimpleNamespace( diff --git a/test/registered/unit/layers/test_aux_hidden_states.py b/test/registered/unit/layers/test_aux_hidden_states.py new file mode 100644 index 0000000000..7d76daffe5 --- /dev/null +++ b/test/registered/unit/layers/test_aux_hidden_states.py @@ -0,0 +1,204 @@ +"""Aux owns retained storage, including captures from reusable gather buffers.""" + +import unittest +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import torch + +from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList, AuxHiddenStatePacker +from sglang.srt.layers.communicator import LayerCommunicator as BoundaryPlan +from sglang.srt.layers.communicator.residual.access import norm_output +from sglang.srt.layers.communicator.residual.stream import ResidualStream +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +class TestAuxStorage(CustomTestCase): + def test_constructed_layer_binds_read_only_residual_capture(self): + import test_declared_decoder_boundary as fixture + + from sglang.srt.layers import communicator as comm + from sglang.srt.layers import layernorm + from sglang.test.communicator_patch import patch_communicator + + parallel = fixture.parallel_of(attn_dp=1, attn_tp=2) + with ( + fixture.planning(parallel), + patch_communicator("_use_aiter", False), + patch_communicator("apply_flashinfer_allreduce_fusion", return_value=True), + ): + layer = comm.LayerCommunicator( + layer_facts=fixture.layer_facts(1, 3), + input_layernorm=layernorm.RMSNorm(4), + post_attention_layernorm=layernorm.RMSNorm(4), + ) + keeps = layer._steps.attention.capture_preserves_residual + self.assertIsNotNone(keeps) + self.assertTrue(keeps(torch.ones(2, 4), SimpleNamespace())) + + def test_breakable_graph_copies_aux_collector_into_plain_list(self): + from sglang.srt.model_executor.runner_backend.breakable_cuda_graph_backend import ( + BreakableCudaGraphBackend, + ) + + backend = BreakableCudaGraphBackend.__new__(BreakableCudaGraphBackend) + source = AuxHiddenStateList([torch.full((4, 3), 7.0)]) + buffers = [torch.zeros(4, 3)] + backend._copy_output_to_buffer( + (torch.ones(4, 3), source), (torch.zeros(4, 3), buffers), 2 + ) + torch.testing.assert_close(buffers[0][:2], torch.full((2, 3), 7.0)) + self.assertEqual(buffers[0][2:].count_nonzero(), 0) + with self.assertRaises(TypeError): + backend._copy_output_to_buffer(source, tuple(buffers), 2) + + def test_capture_move_owns_gather_but_not_slice(self): + from sglang.srt.layers.communicator.boundary import Boundary + from sglang.srt.layers.communicator.layout import Layout, TokenAxis + from sglang.test.communicator_patch import patch_communicator + + full = Layout(frozenset()) + sharded = Layout(frozenset({TokenAxis.ATTN_TP_SCATTER})) + source = torch.ones(2, 3) + for rows, target, owns in ((sharded, full, True), (full, sharded, False)): + edge = SimpleNamespace( + residual_to=rows, produced=SimpleNamespace(layout=target) + ) + boundary = Boundary(edge=edge, prepare=None) + self.assertEqual(boundary.capture_move_allocates, owns) + with ( + patch_communicator( + "_redistribute_from_attn_tp_shards", + side_effect=lambda x: torch.cat((x, x)), + ), + patch_communicator( + "get_parallel", + return_value=SimpleNamespace(attn_tp_size=2, attn_tp_rank=0), + ), + ): + value = boundary.capture_move(source, forward_batch=None) + outputs = AuxHiddenStateList() + if owns: + with patch.object( + torch.Tensor, "clone", side_effect=AssertionError("extra clone") + ): + outputs.capture(value, owned=boundary.capture_move_allocates) + self.assertIs(outputs[0], value) + else: + outputs.capture(value, owned=boundary.capture_move_allocates) + self.assertIsNot(outputs[0], value) + + def test_list_snapshots_borrowed_views_and_reused_buffers(self): + outputs = AuxHiddenStateList() + buffer = torch.full((4, 3), 2.0) + outputs.capture(buffer[:2]) + buffer.fill_(5) + outputs.capture(buffer[:2]) + buffer.zero_() + torch.testing.assert_close(outputs[0], torch.full((2, 3), 2.0)) + torch.testing.assert_close(outputs[1], torch.full((2, 3), 5.0)) + + def test_list_adopts_owned_output_without_copying(self): + outputs = AuxHiddenStateList() + value = torch.ones(2, 3) + with patch.object( + torch.Tensor, "clone", side_effect=AssertionError("extra clone") + ): + outputs.capture(value, owned=True) + self.assertIs(outputs[0], value) + + def boundary(self, stream, *, move=None): + boundary = BoundaryPlan.__new__(BoundaryPlan) + boundary.input_layernorm = None + boundary._context = None + boundary.prepare_attn = Mock(return_value=(stream.residual, stream)) + boundary.attn = SimpleNamespace( + entry=lambda _: SimpleNamespace( + input_move=move, + capture_move=None, + capture_move_allocates=False, + capture_preserves_residual=None, + ) + ) + return boundary + + def test_packer_receives_borrowed_boundary_value_without_intermediate_clone(self): + for callback in (False, True): + with self.subTest(callback=callback): + value = torch.full((2, 3), 4.0) + stream = ResidualStream(value) + stage = self.boundary(stream) + outputs = AuxHiddenStatePacker(1) + kwargs = ( + {"capture_output": outputs.capture} + if callback + else {"captured_last_layer_outputs": outputs} + ) + with patch.object( + torch.Tensor, "clone", side_effect=AssertionError("extra clone") + ): + stage.prepare_attn_and_capture_last_layer_outputs( + value, stream, None, **kwargs + ) + value.zero_() + torch.testing.assert_close(outputs.finalize(), torch.full((2, 3), 4.0)) + + def test_gather_storage_is_borrowed_even_if_it_is_a_different_tensor(self): + source = torch.ones(2, 3) + gathered = torch.full((4, 3), 7.0) + stream = ResidualStream(source) + move = Mock(return_value=gathered) + stage = self.boundary(stream, move=move) + outputs = AuxHiddenStateList() + stage.prepare_attn_and_capture_last_layer_outputs(source, stream, None, outputs) + gathered.zero_() + source.zero_() + torch.testing.assert_close(outputs[0], torch.full((4, 3), 7.0)) + move.assert_called_once() + + def test_embedding_and_final_norm_capture_write_directly_to_packer(self): + for residual in (None, torch.ones(2, 3)): + with self.subTest(residual=residual): + hidden = torch.full((2, 3), 4.0) + outputs = AuxHiddenStatePacker(1) + updated = torch.full((2, 3), 5.0) + norm = Mock( + return_value=hidden if residual is None else (hidden, updated) + ) + with patch.object( + torch.Tensor, "clone", side_effect=AssertionError("extra clone") + ): + norm_output(hidden, residual, norm, outputs.capture) + hidden.zero_() + updated.zero_() + torch.testing.assert_close( + outputs.finalize(), + torch.full((2, 3), 4.0 if residual is None else 5.0), + ) + + def test_initial_capture_precedes_in_place_prepare_without_intermediate_clone(self): + hidden = torch.full((2, 3), 4.0) + stream = ResidualStream() + outputs = AuxHiddenStatePacker(1) + stage = self.boundary(stream) + + def prepare(value, stream, batch, **kwargs): + value.zero_() + stream.write(value) + return value, stream + + stage.prepare_attn = prepare + with patch.object( + torch.Tensor, "clone", side_effect=AssertionError("extra clone") + ): + stage.prepare_attn_and_capture_last_layer_outputs( + hidden, stream, None, capture_output=outputs.capture + ) + torch.testing.assert_close(outputs.finalize(), torch.full((2, 3), 4.0)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layers/test_boundary_edges.py b/test/registered/unit/layers/test_boundary_edges.py index 9d44762531..e0c4867f54 100644 --- a/test/registered/unit/layers/test_boundary_edges.py +++ b/test/registered/unit/layers/test_boundary_edges.py @@ -261,11 +261,9 @@ class TestNonAlternatingEdges(CustomTestCase): residual=full, residual_to=full, ) - over_tp = comm.FusedMlpInput( - completes=SumGroup.TP, run=MagicMock(), may_return_new_residual=False - ) + over_tp = comm.FusedMlpInput(completes=SumGroup.TP, run=MagicMock()) over_attention_tp = comm.FusedMlpInput( - completes=SumGroup.ATTN_TP, run=MagicMock(), may_return_new_residual=False + completes=SumGroup.ATTN_TP, run=MagicMock() ) boundary = make_boundary(edge, fusions=(over_tp, over_attention_tp)) self.assertEqual(boundary.fused, (over_tp,)) @@ -370,9 +368,7 @@ class TestTheProducersUpdateChoosesTheOrder(CustomTestCase): def test_fused_kernels_take_only_a_plain_add(self): full = rows(sizes(tp=2)) - fused = comm.FusedMlpInput( - completes=SumGroup.ATTN_TP, run=MagicMock(), may_return_new_residual=False - ) + fused = comm.FusedMlpInput(completes=SumGroup.ATTN_TP, run=MagicMock()) carried = MagicMock() for update, offered in ((comm.ADD, True), (_WrittenIn(), False)): with self.subTest(adds_plainly=offered): @@ -569,9 +565,7 @@ class TestTheConsumerRunsItsDeclaredRead(CustomTestCase): torch.testing.assert_close(out, expected + 100) def test_a_fused_kernel_runs_only_for_a_read_that_leaves_the_residual(self): - fused = FusedMlpInput( - completes=SumGroup.ATTN_TP, run=MagicMock(), may_return_new_residual=True - ) + fused = FusedMlpInput(completes=SumGroup.ATTN_TP, run=MagicMock()) for norms_plainly in (True, False): with self.subTest(norms_plainly=norms_plainly): step, tried = self._ffn_input( diff --git a/test/registered/unit/layers/test_declared_decoder_boundary.py b/test/registered/unit/layers/test_declared_decoder_boundary.py index 4f2cbf203c..e7b5ba5e2a 100644 --- a/test/registered/unit/layers/test_declared_decoder_boundary.py +++ b/test/registered/unit/layers/test_declared_decoder_boundary.py @@ -276,9 +276,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase): (entry,) if fuses else (), ) self.assertIs( - any( - f.may_return_new_residual for f in communicator._steps.ffn.fused - ), + bool(communicator._steps.ffn.fused), fuses, ) self.assertFalse(communicator._steps.returns_over_dp) @@ -1095,11 +1093,10 @@ class TestFusedKernelsTakeOnlyTheStepsTheyComplete(CustomTestCase): norm only where those are the steps, and only if it completes the sum the attention output owes.""" - def fused(self, completes, may_return_new_residual): + def fused(self, completes): return comm.FusedMlpInput( completes=completes, run=lambda h, r, fb: None, - may_return_new_residual=may_return_new_residual, ) def select(self, *, attn_dp, fusions): @@ -1125,45 +1122,18 @@ class TestFusedKernelsTakeOnlyTheStepsTheyComplete(CustomTestCase): return steps, fused def test_a_kernel_over_another_group_is_not_chosen(self): - over_tp = self.fused(SumGroup.TP, True) - over_attention_tp = self.fused(SumGroup.ATTN_TP, False) + over_tp = self.fused(SumGroup.TP) + over_attention_tp = self.fused(SumGroup.ATTN_TP) steps, chosen = self.select(attn_dp=1, fusions=(over_tp, over_attention_tp)) self.assertIs(steps.func, comm_ops._mlp_input_without_dp) self.assertEqual(steps.keywords["fusions"], (over_attention_tp.run,)) self.assertEqual(chosen, (over_attention_tp,)) def test_no_kernel_is_chosen_where_rows_are_gathered(self): - steps, chosen = self.select( - attn_dp=2, fusions=(self.fused(SumGroup.ATTN_TP, True),) - ) + steps, chosen = self.select(attn_dp=2, fusions=(self.fused(SumGroup.ATTN_TP),)) self.assertIs(steps.func, comm_ops._mlp_input_dp_partial) self.assertEqual(chosen, ()) - def test_aux_capture_follows_the_chosen_kernels(self): - for kernels, expected in ( - ((self.fused(SumGroup.ATTN_TP, False),), False), - ((self.fused(SumGroup.ATTN_TP, True),), True), - ((self.fused(SumGroup.TP, True),), False), - ): - with self.subTest(kernels=kernels): - - class Declaring(LayerCommunicator): - def _select_mlp_input_fusions(self): - return kernels - - with planning(parallel_of(attn_dp=1, attn_tp=2)): - communicator = Declaring( - layer_facts=layer_facts(1, 3), - input_layernorm=Norm(), - post_attention_layernorm=Norm(), - ) - self.assertIs( - any( - f.may_return_new_residual for f in communicator._steps.ffn.fused - ), - expected, - ) - class TestTheSequenceParallelRegion(CustomTestCase): """While a LayerNorm SP region is active, the layer runs the steps its @@ -1492,7 +1462,6 @@ class TestOneRepresentation(CustomTestCase): "_ffn_output", "_postprocess_scatters_to_local_tokens", "_ffn_sum_is_movable", - "_mlp_input_may_return_new_residual", ) def test_the_layer_holds_steps_only(self): diff --git a/test/registered/unit/layers/test_residual_stream.py b/test/registered/unit/layers/test_residual_stream.py index b8e67139c1..0da8eb5905 100644 --- a/test/registered/unit/layers/test_residual_stream.py +++ b/test/registered/unit/layers/test_residual_stream.py @@ -4,9 +4,11 @@ from unittest.mock import Mock import torch +from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList from sglang.srt.layers.communicator import ( BoundarySteps, EdgeDecl, + LayerCommunicator, Layout, StageEntry, StageInput, @@ -162,6 +164,107 @@ class TestResidualStream(CustomTestCase): with self.assertRaises(RuntimeError): self.stream.input(hidden.clone()) + def test_capture_and_compute_share_the_fused_read(self): + boundary = LayerCommunicator.__new__(LayerCommunicator) + events = [] + outputs = AuxHiddenStateList() + + def prepare(hidden, stream, batch, **kwargs): + events.append("add_norm") + self.assertIsNone(stream.pending.owed) + # A fused read returns both the norm output and its updated residual. + updated = hidden + stream.residual + stream.write(updated) + return updated * 3, stream + + def capture(value, *, owned=False): + events.append("capture") + outputs.capture(value, owned=owned) + + boundary.prepare_attn = Mock(side_effect=prepare) + boundary.attn = SimpleNamespace( + entry=lambda batch: SimpleNamespace( + capture_move=None, + capture_move_allocates=False, + capture_preserves_residual=None, + ) + ) + output, stream = boundary.prepare_attn_and_capture_last_layer_outputs( + self.hidden, self.stream, None, capture_output=capture + ) + self.assertEqual(events, ["add_norm", "capture"]) + self.group.all_reduce.assert_called_once() + boundary.prepare_attn.assert_called_once() + torch.testing.assert_close(output, torch.full((2, 4), 15.0)) + torch.testing.assert_close(outputs[0], torch.full((2, 4), 5.0)) + stream.residual.zero_() + torch.testing.assert_close(outputs[0], torch.full((2, 4), 5.0)) + + def test_deepstack_capture_precedes_the_extra_addition(self): + boundary = LayerCommunicator.__new__(LayerCommunicator) + extra = torch.full_like(self.partial, 7.0) + outputs = AuxHiddenStateList() + + def prepare(hidden, stream, batch, **kwargs): + self.assertEqual(len(outputs), 1) + self.assertIs(kwargs["post_residual_addition"], extra) + updated = hidden + stream.residual + extra + stream.write(updated) + return updated * 3, stream + + boundary.prepare_attn = Mock(side_effect=prepare) + boundary.attn = SimpleNamespace( + entry=lambda batch: SimpleNamespace( + capture_move=None, + capture_move_allocates=False, + capture_preserves_residual=None, + ) + ) + output, _ = boundary.prepare_attn_and_capture_last_layer_outputs( + self.hidden, + self.stream, + None, + post_residual_addition=extra, + capture_output=outputs.capture, + ) + torch.testing.assert_close(outputs[0], torch.full((2, 4), 5.0)) + torch.testing.assert_close(output, torch.full((2, 4), 36.0)) + self.group.all_reduce.assert_called_once() + + def test_capture_restores_the_producers_rows_after_a_scattered_read(self): + from unittest.mock import patch + + from sglang.srt.layers.communicator import TokenAxis + + full = Layout(frozenset()) + local = Layout(frozenset({TokenAxis.ATTN_TP_SCATTER})) + boundary = make_boundary( + EdgeDecl(StageOutput(full), StageInput(local), full, local) + ) + shard = torch.full((2, 4), 3.0) + expected = torch.cat([shard, torch.full_like(shard, 7.0)]) + with patch( + "sglang.srt.layers.communicator.ops._redistribute_from_attn_tp_shards", + return_value=expected, + ) as gather: + captured = boundary.capture_move(shard, forward_batch=None) + gather.assert_called_once_with(shard) + torch.testing.assert_close(captured, expected) + torch.testing.assert_close(shard, torch.full((2, 4), 3.0)) + + def test_final_capture_uses_the_norms_residual_result(self): + from sglang.srt.layers.communicator.residual.access import norm_output + + outputs = AuxHiddenStateList() + updated = torch.full_like(self.partial, 5.0) + normalized = torch.full_like(self.partial, 13.0) + norm = Mock(return_value=(normalized, updated)) + result = norm_output(self.partial, self.residual, norm, outputs.capture) + self.assertIs(result, normalized) + norm.assert_called_once_with(self.partial, self.residual) + updated.zero_() + torch.testing.assert_close(outputs[0], torch.full((2, 4), 5.0)) + def test_deepstack_adds_once_after_completing_the_sum(self): hidden, stream = add_to_output( self.hidden, self.stream, torch.full((2, 4), 7.0) diff --git a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py index ad08e8a454..9829acf395 100644 --- a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py +++ b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py @@ -118,6 +118,9 @@ class DeferringLayer(nn.Module): # later norms mutate the residual; captured snapshots must stay intact residual.add_(hidden_states) stream.write(residual) + capture = kwargs.get("capture_output") + if capture is not None: + capture(residual.clone()) with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit: partial = torch.full_like(hidden_states, 0.5) hidden_states, residual = ffn_exit.finish(partial, stream) @@ -130,7 +133,7 @@ class SumNorm(nn.Module): def forward(self, hidden_states, residual=None, **kwargs): if residual is None: return hidden_states - return hidden_states + residual, residual + return hidden_states + residual, hidden_states + residual def build_model(model_cls, *, defer, capture): @@ -211,6 +214,29 @@ class TestAuxCaptureDeferredAllreduce(CustomTestCase): reduce.call_count, NUM_LAYERS - 1 if defer else 0 ) + def test_terminal_capture_uses_the_final_norm_boundary(self): + inputs = torch.ones(2, 4) + batch = SimpleNamespace( + can_run_tbo=False, + forward_mode=ForwardMode.DECODE, + capture_hidden_mode=SimpleNamespace(need_capture=lambda: True), + ) + for cls in (GptOssModel, LagunaModel, BailingMoELinearModel): + with self.subTest(model=cls.__name__): + model = build_model(cls, defer=True, capture=True) + model.layers_to_capture = [ + NUM_LAYERS - 1 if cls is BailingMoELinearModel else NUM_LAYERS + ] + result, snapshots = model( + input_ids=None, + positions=None, + forward_batch=batch, + input_embeds=inputs.clone(), + ) + self.assertEqual(len(snapshots), 1) + torch.testing.assert_close(snapshots[0], inputs + NUM_LAYERS) + torch.testing.assert_close(result, inputs + NUM_LAYERS) + class TestPipelineResidualReception(CustomTestCase): def test_models_keep_the_received_residual_contribution(self): @@ -238,6 +264,29 @@ class TestPipelineResidualReception(CustomTestCase): ) torch.testing.assert_close(result, initial_residual + NUM_LAYERS) + def test_bailing_capture_does_not_include_a_previous_pipeline_ranks_layer(self): + model = build_model(BailingMoELinearModel, defer=True, capture=True) + model.pp_group.is_first_rank = False + model.start_layer = 2 + model.layers_to_capture = [model.start_layer - 1] + inputs = torch.zeros(2, 4) + residual = torch.full_like(inputs, 0.25) + batch = SimpleNamespace( + can_run_tbo=False, + forward_mode=ForwardMode.DECODE, + capture_hidden_mode=SimpleNamespace(need_capture=lambda: True), + ) + result = model( + None, + None, + batch, + pp_proxy_tensors=PPProxyTensors( + {"hidden_states": inputs, "residual": residual} + ), + ) + self.assertIsInstance(result, torch.Tensor) + torch.testing.assert_close(result, torch.full_like(inputs, 2.25)) + def test_written_streams_do_not_read_a_separate_residual(self): comm_instance = comm.LayerCommunicator.__new__(comm.LayerCommunicator) comm_instance._residual = SimpleNamespace(