From 6f72cc3b642c65a09ec0e19ab3542da8ece3da16 Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Mon, 28 Sep 2026 20:14:04 -0700 Subject: [PATCH] [Refactor] Centralize decoder output access (#41548) --- .../srt/layers/communicator/__init__.py | 2 - .../srt/layers/communicator/boundary.py | 147 ++++++++++-------- .../sglang/srt/layers/communicator/layer.py | 73 ++++++--- python/sglang/srt/layers/communicator/ops.py | 9 -- .../layers/communicator/residual/access.py | 104 +++++++++++++ python/sglang/srt/models/bailing_moe.py | 14 +- .../sglang/srt/models/bailing_moe_linear.py | 5 +- python/sglang/srt/models/bailing_moe_v3.py | 23 ++- python/sglang/srt/models/deepseek_v2.py | 8 +- .../srt/models/dots3_common/modeling.py | 5 +- python/sglang/srt/models/falcon_h1.py | 4 + python/sglang/srt/models/gigachat35.py | 9 +- python/sglang/srt/models/gigachat35_mtp.py | 8 +- python/sglang/srt/models/glm4_moe.py | 14 +- python/sglang/srt/models/glm4_moe_lite.py | 14 +- python/sglang/srt/models/glm5_next.py | 33 ++-- python/sglang/srt/models/gpt_oss.py | 18 +-- python/sglang/srt/models/interns2_mobius.py | 4 + python/sglang/srt/models/laguna.py | 16 +- python/sglang/srt/models/llada2.py | 9 +- python/sglang/srt/models/llama4.py | 10 +- python/sglang/srt/models/longcat_flash.py | 11 +- .../sglang/srt/models/longcat_flash_nextn.py | 4 + python/sglang/srt/models/mimo_v2.py | 11 +- python/sglang/srt/models/mimo_v2_nextn.py | 8 +- python/sglang/srt/models/minimax_m2.py | 5 +- python/sglang/srt/models/minimax_m3.py | 5 +- python/sglang/srt/models/moss_vl.py | 11 +- python/sglang/srt/models/nemotron_h.py | 43 ++--- python/sglang/srt/models/nemotron_h_mtp.py | 12 +- python/sglang/srt/models/qwen2.py | 16 +- python/sglang/srt/models/qwen2_moe.py | 10 +- python/sglang/srt/models/qwen3.py | 5 + python/sglang/srt/models/qwen3_5.py | 13 +- python/sglang/srt/models/qwen3_moe.py | 5 + python/sglang/srt/models/qwen3_vl.py | 13 +- python/sglang/srt/models/qwen3_vl_moe.py | 19 +-- python/sglang/srt/models/sarvam_moe.py | 15 +- python/sglang/srt/models/sdar.py | 9 +- python/sglang/srt/models/sdar_moe.py | 5 +- python/sglang/srt/models/step3_vl.py | 4 + python/sglang/srt/models/step3p5.py | 9 +- python/sglang/srt/models/step3p5_mtp.py | 4 +- python/sglang/srt/models/xllm.py | 9 +- python/sglang/test/communicator_patch.py | 3 +- .../unit/layers/test_boundary_edges.py | 6 +- .../layers/test_declared_decoder_boundary.py | 26 ++-- .../layers/test_dense_mlp_under_prefill_cp.py | 4 +- .../test_aux_capture_deferred_allreduce.py | 74 ++++++++- .../test_deepstack_deferred_allreduce.py | 50 ++++++ .../models/test_glm5_next_dflash_capture.py | 5 +- .../unit/models/test_layer_stack_exit.py | 105 ++++++++++++- 52 files changed, 750 insertions(+), 298 deletions(-) create mode 100644 python/sglang/srt/layers/communicator/residual/access.py diff --git a/python/sglang/srt/layers/communicator/__init__.py b/python/sglang/srt/layers/communicator/__init__.py index 97713e78cc..cfae0136d1 100644 --- a/python/sglang/srt/layers/communicator/__init__.py +++ b/python/sglang/srt/layers/communicator/__init__.py @@ -59,7 +59,6 @@ from sglang.srt.layers.communicator.layout import ( from sglang.srt.layers.communicator.ops import ( CommunicateSimpleFn, CommunicateSummableTensorPairFn, - layer_input_buffer, move_rows, tp_reduce_scatter, ) @@ -119,7 +118,6 @@ __all__ = [ "enable_moe_dense_fully_dp", "get_attn_tp_context", "input_scattered_layer_sides", - "layer_input_buffer", "make_boundary", "make_output_boundary", "moe_cp_gathers_sparse_moe_input", diff --git a/python/sglang/srt/layers/communicator/boundary.py b/python/sglang/srt/layers/communicator/boundary.py index dfcbef2ca8..3d75f47663 100644 --- a/python/sglang/srt/layers/communicator/boundary.py +++ b/python/sglang/srt/layers/communicator/boundary.py @@ -100,6 +100,14 @@ class StageOutput(msgspec.Struct, frozen=True): update: StageUpdate = ADD +class StageDecl(msgspec.Struct, frozen=True): + """A computing stage's two sides: the rows its input must be on, and what + its output is.""" + + input: StageInput + output: StageOutput + + class DecoderLayerSides(msgspec.Struct, frozen=True): """What a decoder layer declares about its attention, its FFN and the rows between them; ``decoder_layer_edges`` turns it into the layer's @@ -107,10 +115,8 @@ class DecoderLayerSides(msgspec.Struct, frozen=True): # The rows the layer's input and residual arrive on. input_rows: Layout - attention: StageInput - attention_output: StageOutput - ffn: StageInput - ffn_output: StageOutput + attention: StageDecl + ffn: StageDecl # The rows the residual is on while the FFN runs. ffn_residual_rows: Layout # The rows the layer hands on. @@ -122,14 +128,6 @@ class DecoderLayerSides(msgspec.Struct, frozen=True): residual_joins_attention_sum: bool = False -class StageDecl(msgspec.Struct, frozen=True): - """A computing stage's two sides: the rows its input must be on, and what - its output is.""" - - input: StageInput - output: StageOutput - - class EdgeDecl(msgspec.Struct, frozen=True): """One boundary between two stages, as the layer that runs one side of it sees it: what arrives from the producer, what the consumer needs, and the @@ -167,7 +165,7 @@ def decoder_layer_edges(sides: DecoderLayerSides) -> DecoderLayerEdges: sides.input_rows, group=sides.input_owes, always_leaves=owes, - update=sides.ffn_output.update, + update=sides.ffn.output.update, ) # Completing what the input owes leaves it and the residual on each # attention-TP rank's slice. @@ -179,19 +177,19 @@ def decoder_layer_edges(sides: DecoderLayerSides) -> DecoderLayerEdges: return DecoderLayerEdges( into_attention=EdgeDecl( produced=arrived, - need=sides.attention, + need=sides.attention.input, residual=sides.input_rows, residual_to=attention_rows, ), into_ffn=EdgeDecl( - produced=sides.attention_output, - need=sides.ffn, + produced=sides.attention.output, + need=sides.ffn.input, residual=attention_rows, residual_to=sides.ffn_residual_rows, residual_joins_sum=sides.residual_joins_attention_sum, ), out_of_ffn=EdgeDecl( - produced=sides.ffn_output, + produced=sides.ffn.output, need=StageInput(sides.output_rows), residual=sides.ffn_residual_rows, residual_to=sides.output_rows, @@ -207,12 +205,14 @@ def with_residual( replace = msgspec.structs.replace return replace( sides, - attention=replace(sides.attention, read=residual.attention_read), - attention_output=replace( - sides.attention_output, update=residual.attention_update + attention=StageDecl( + input=replace(sides.attention.input, read=residual.attention_read), + output=replace(sides.attention.output, update=residual.attention_update), + ), + ffn=StageDecl( + input=replace(sides.ffn.input, read=residual.ffn_read), + output=replace(sides.ffn.output, update=residual.ffn_update), ), - ffn=replace(sides.ffn, read=residual.ffn_read), - ffn_output=replace(sides.ffn_output, update=residual.ffn_update), ) @@ -270,14 +270,17 @@ def sequence_parallel_layer_sides( gathers = frozenset({TokenAxis.ATTN_TP_SCATTER}) return DecoderLayerSides( input_rows=local, - # qkv and gate_up gather the slices into their GEMM. - attention=StageInput(attention, gathers_itself=gathers, read=NORM_QUANT_READ), - # o_proj and down reduce-scatter out of theirs, whether fused or not. - attention_output=StageOutput(local), - ffn=StageInput( - Layout.sharded_over(axis_sizes=axis_sizes), gathers_itself=gathers + # qkv and gate_up gather into GEMM; o_proj and down reduce-scatter out. + attention=StageDecl( + input=StageInput(attention, gathers_itself=gathers, read=NORM_QUANT_READ), + output=StageOutput(local), + ), + ffn=StageDecl( + input=StageInput( + Layout.sharded_over(axis_sizes=axis_sizes), gathers_itself=gathers + ), + output=StageOutput(local), ), - ffn_output=StageOutput(local), ffn_residual_rows=local, output_rows=local, ) @@ -311,19 +314,21 @@ def scattered_residual_layer_sides( ) return DecoderLayerSides( input_rows=attention if is_first_layer else local, - attention=StageInput( - attention, - gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}), - read=NORM_QUANT_READ, + attention=StageDecl( + input=StageInput( + attention, + gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}), + read=NORM_QUANT_READ, + ), + output=StageOutput(attention, group=SumGroup.ATTN_TP, always_leaves=True), ), - attention_output=StageOutput( - attention, group=SumGroup.ATTN_TP, always_leaves=True - ), - ffn=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)), - ffn_output=StageOutput( - attention, - group=ffn_group, - leaves_for_reduce_scatter=leaves_for_reduce_scatter, + ffn=StageDecl( + input=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)), + output=StageOutput( + attention, + group=ffn_group, + leaves_for_reduce_scatter=leaves_for_reduce_scatter, + ), ), ffn_residual_rows=local, output_rows=attention if is_last_layer else local, @@ -347,18 +352,20 @@ def input_scattered_layer_sides( ) return DecoderLayerSides( input_rows=attention, - attention=StageInput( - attention, - gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}), - read=NORM_QUANT_READ, + attention=StageDecl( + input=StageInput( + attention, + gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}), + read=NORM_QUANT_READ, + ), + output=StageOutput(attention, group=SumGroup.ATTN_TP, always_leaves=True), ), - attention_output=StageOutput( - attention, group=SumGroup.ATTN_TP, always_leaves=True - ), - ffn=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)), - # The next layer's input completes the sum; the last layer's FFN does. - ffn_output=StageOutput( - attention, group=ffn_group, leaves_for_reduce_scatter=hands_on_partial + ffn=StageDecl( + input=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)), + # The next input completes the sum; the last FFN completes its own. + output=StageOutput( + attention, group=ffn_group, leaves_for_reduce_scatter=hands_on_partial + ), ), ffn_residual_rows=attention, output_rows=attention, @@ -412,24 +419,26 @@ def decoder_layer_sides( attention_tp = axis_sizes[TokenAxis.ATTN_TP_SCATTER] > 1 return DecoderLayerSides( input_rows=local if previous_on_local_rows else attention, - attention=StageInput( - attention, - gathers_itself=( - frozenset({TokenAxis.ATTN_TP_SCATTER}) - if attention_gathers_local_rows - else frozenset() + attention=StageDecl( + input=StageInput( + attention, + gathers_itself=( + frozenset({TokenAxis.ATTN_TP_SCATTER}) + if attention_gathers_local_rows + else frozenset() + ), + read=NORM_QUANT_READ, + ), + # The output projection leaves the attention-TP sum to prepare_mlp. + output=StageOutput( + attention, + group=SumGroup.ATTN_TP if attention_tp else None, + always_leaves=attention_tp, ), - read=NORM_QUANT_READ, ), - # The output projection leaves the attention-TP sum to prepare_mlp. - attention_output=StageOutput( - attention, - group=SumGroup.ATTN_TP if attention_tp else None, - always_leaves=attention_tp, - ), - ffn=StageInput(ffn), - ffn_output=( - StageOutput(ffn) + ffn=StageDecl( + input=StageInput(ffn), + output=StageOutput(ffn) if ffn_on_local_rows else StageOutput( ffn, @@ -437,7 +446,7 @@ def decoder_layer_sides( leaves_for_next_layer=leaves_for_next_layer, leaves_for_reduce_scatter=leaves_for_reduce_scatter, leaves_for_reduce_scatterv=leaves_for_reduce_scatterv, - ) + ), ), ffn_residual_rows=local if ffn_on_local_rows else attention, output_rows=( diff --git a/python/sglang/srt/layers/communicator/layer.py b/python/sglang/srt/layers/communicator/layer.py index e2722e360f..2620d20f1f 100644 --- a/python/sglang/srt/layers/communicator/layer.py +++ b/python/sglang/srt/layers/communicator/layer.py @@ -68,11 +68,18 @@ from sglang.srt.layers.communicator.ops import ( move_rows, ) from sglang.srt.layers.communicator.output import ( - HandoffOutput, UnreducedOutput, reduce_output, ) from sglang.srt.layers.communicator.residual import LayerResidual +from sglang.srt.layers.communicator.residual.access import ( + add_to_output, + buffer, + finish_layer_stack, + fold, + from_pp, + snapshot, +) from sglang.srt.layers.communicator.residual.add_norm import ( PLAIN_RESIDUAL, aiter_all_reduce_fusion_enabled_for, @@ -221,6 +228,11 @@ class LayerFusions(Protocol): class LayerCommunicator: + add_to_output = staticmethod(add_to_output) + buffer = staticmethod(buffer) + fold = staticmethod(fold) + finish_layer_stack = staticmethod(finish_layer_stack) + # Communicators built without __init__ (e.g. test doubles) publish no LoRA # layout and try no fused kernel at the FFN exit. _publish_lora_layout: bool = False @@ -229,6 +241,7 @@ class LayerCommunicator: _residual: LayerResidual = PLAIN_RESIDUAL # The fused kernels a backend gives the layer, if any. fusions: Optional[LayerFusions] = None + stage_edges = None def __init__( self, @@ -534,14 +547,14 @@ class LayerCommunicator: if self._residual.attention_update.adds_plainly: scattered = input_scattered_layer_sides( axis_sizes=token_axis_sizes(), - ffn_group=sides.ffn_output.group, + ffn_group=sides.ffn.output.group, hands_on_partial=self.allow_reduce_scatter and not self.is_last_layer, ) handoff = _hand_qkv_hook_its_input else: scattered = scattered_residual_layer_sides( axis_sizes=token_axis_sizes(), - ffn_group=sides.ffn_output.group, + ffn_group=sides.ffn.output.group, is_first_layer=self.layer_facts.is_first_layer, is_last_layer=self.is_last_layer, leaves_for_reduce_scatter=self.allow_reduce_scatter, @@ -566,6 +579,39 @@ class LayerCommunicator: **kwargs, ) + def snapshot(self, hidden_states, residual, *, at_input: bool = False): + group = None + if residual is not None and self.stage_edges is not None: + edge = self.stage_edges[0 if at_input else 1] + if edge.produced.always_leaves: + group = _sum_group(edge.produced.group) + return snapshot(hidden_states, residual, group=group) + + def capture_output( + self, + hidden_states, + residual, + *, + at_input: bool = False, + skip_empty: bool = False, + ): + """Complete the carried output on the main path, then snapshot it. + Keep this explicit completion distinct from a read-only snapshot: the + next prepare must not select a fusion for a sum already completed here.""" + if skip_empty: + storage = buffer(hidden_states) + if storage is not None and storage.shape[0] == 0: + return hidden_states, None + hidden_states = reduce_output(hidden_states) + return hidden_states, self.snapshot(hidden_states, residual, at_input=at_input) + + def from_pp(self, tensors, *, allow_missing_residual: bool = False): + return from_pp( + tensors, + residual_in_hidden=self._residual.ffn_update.at_producer, + allow_missing_residual=allow_missing_residual, + ) + def prepare_attn_and_capture_last_layer_outputs( self, hidden_states: torch.Tensor, @@ -977,7 +1023,7 @@ class LayerCommunicator: if self._declared is None: raise NotImplementedError("a branch on a layer that is one stage") sides = self._declared - return sides.ffn.layout, sides.ffn_residual_rows, sides.output_rows + return sides.ffn.input.layout, sides.ffn_residual_rows, sides.output_rows def branch_input( self, @@ -1022,23 +1068,6 @@ class LayerCommunicator: residual = move_rows(residual, rows, to, forward_batch) return contribution + hidden_states, residual - def finish_layer_stack( - self, - hidden_states: Union[torch.Tensor, UnreducedOutput, HandoffOutput], - residual: Optional[torch.Tensor], - forward_batch: ForwardBatch, - *, - final_norm_takes_handoff: bool = False, - ) -> Tuple[Union[torch.Tensor, HandoffOutput], Optional[torch.Tensor]]: - """Complete what this layer left for a next layer. Call it on the last - layer of this rank before its output reaches the final norm, the next - pipeline rank, or any other consumer outside the layers. A final norm - that does a producer's handoff together with its own work - (``final_norm_takes_handoff``) receives it as it is.""" - if final_norm_takes_handoff and isinstance(hidden_states, HandoffOutput): - return hidden_states, residual - return reduce_output(hidden_states), residual - def _select_ffn_exit_fusions( self, ) -> Tuple[Callable[[ForwardBatch], Optional["FfnExitFusion"]], ...]: @@ -1304,7 +1333,7 @@ class MHCLayerCommunicator(LayerCommunicator): # MHC has not been run with an FFN input gathered over attention CP. if ( TokenAxis.ATTN_CP - in sides.attention_output.layout.sharded - sides.ffn.layout.sharded + in sides.attention.output.layout.sharded - sides.ffn.input.layout.sharded ): raise NotImplementedError( f"MHCLayerCommunicator with a gather over attention CP: {sides=}" diff --git a/python/sglang/srt/layers/communicator/ops.py b/python/sglang/srt/layers/communicator/ops.py index 438197d287..38a5958b24 100644 --- a/python/sglang/srt/layers/communicator/ops.py +++ b/python/sglang/srt/layers/communicator/ops.py @@ -102,15 +102,6 @@ def tp_reduce_scatter( return output, residual -def layer_input_buffer( - hidden_states: Union[torch.Tensor, UnreducedOutput], -) -> torch.Tensor: - """The tensor holding a layer's input, for reusing its memory without reading it.""" - if isinstance(hidden_states, UnreducedOutput): - return hidden_states.partial - return hidden_states - - class CommunicateSimpleFn: @staticmethod def _trivial( diff --git a/python/sglang/srt/layers/communicator/residual/access.py b/python/sglang/srt/layers/communicator/residual/access.py new file mode 100644 index 0000000000..70b38ee456 --- /dev/null +++ b/python/sglang/srt/layers/communicator/residual/access.py @@ -0,0 +1,104 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Access to a layer output outside the next stage's prepare call.""" + +from typing import Optional, Tuple, Union + +import torch + +from sglang.srt.distributed import GroupCoordinator +from sglang.srt.layers.communicator.output import ( + HandoffOutput, + UnreducedOutput, + reduce_output, +) +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors + + +def buffer( + hidden_states: Union[torch.Tensor, UnreducedOutput, HandoffOutput], +) -> Optional[torch.Tensor]: + """Storage that can be reused after prepare consumes the input. A finalize + handoff has no reusable layer-output tensor. This does not complete or read + the value held in the storage.""" + if isinstance(hidden_states, UnreducedOutput): + return hidden_states.partial + if isinstance(hidden_states, HandoffOutput): + return None + return hidden_states + + +def add_to_output(hidden_states, residual, extra: torch.Tensor) -> Tuple: + """Complete the output before adding an extra contribution exactly once. + Leave its residual update for the next prepare call.""" + hidden_states = reduce_output(hidden_states) + hidden_states.add_(extra) + return hidden_states, residual + + +def fold(hidden_states, residual) -> Tuple[torch.Tensor, None]: + """Finish a plain layer output and fold its residual into a complete value. + The following stage receives it with no outstanding residual addition.""" + hidden_states = reduce_output(hidden_states) + if residual is not None: + hidden_states = hidden_states + residual + return hidden_states, None + + +def finish_layer_stack( + hidden_states: Union[torch.Tensor, UnreducedOutput, HandoffOutput], + residual: Optional[torch.Tensor], + forward_batch: ForwardBatch, + *, + final_norm_takes_handoff: bool = False, +) -> Tuple[Union[torch.Tensor, HandoffOutput], Optional[torch.Tensor]]: + """Complete what this layer left for a next layer. Call it on the last + layer of this rank before its output reaches the final norm, the next + pipeline rank, or any other consumer outside the layers. A final norm + that does a producer's handoff together with its own work + (``final_norm_takes_handoff``) receives it as it is.""" + if final_norm_takes_handoff and isinstance(hidden_states, HandoffOutput): + return hidden_states, residual + return reduce_output(hidden_states), residual + + +def from_pp( + tensors: PPProxyTensors, + *, + residual_in_hidden: bool = False, + allow_missing_residual: bool = False, +) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + """Receive the layer-stack state without reducing a declared partial sum. + MHC carries its written streams in hidden_states; ordinary layers receive + a separate residual tensor.""" + if residual_in_hidden: + residual = None + elif allow_missing_residual: + residual = tensors.tensors.get("residual") + else: + residual = tensors["residual"] + return tensors["hidden_states"], residual + + +def snapshot( + hidden_states: torch.Tensor, + residual: Optional[torch.Tensor], + *, + group: Optional[GroupCoordinator] = None, +) -> torch.Tensor: + """Copy a complete output with its plain residual. A statically declared + sum is reduced on a copy; the main output and residual remain unchanged.""" + if group is not None: + hidden_states = group.all_reduce(hidden_states.clone()) + return hidden_states.clone() if residual is None else hidden_states + residual diff --git a/python/sglang/srt/models/bailing_moe.py b/python/sglang/srt/models/bailing_moe.py index f9187514c9..a8e4645419 100644 --- a/python/sglang/srt/models/bailing_moe.py +++ b/python/sglang/srt/models/bailing_moe.py @@ -35,7 +35,6 @@ from sglang.srt.layers.communicator import ( LayerCommunicator, LayerFacts, enable_moe_dense_fully_dp, - reduce_output, ) from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, @@ -818,17 +817,20 @@ class BailingMoEModel(nn.Module): residual = None else: assert pp_proxy_tensors is not None - hidden_states = pp_proxy_tensors["hidden_states"] - residual = pp_proxy_tensors["residual"] + hidden_states, residual = self.layers[ + self.start_layer + ].layer_communicator.from_pp(pp_proxy_tensors) aux_hidden_states = [] for i in range(self.start_layer, self.end_layer): with get_global_expert_distribution_recorder().with_current_layer(i): if i in self.layers_to_capture: - hidden_states = reduce_output(hidden_states) - aux_hidden_states.append( - hidden_states if residual is None else hidden_states + residual + hidden_states, snapshot = self.layers[ + i + ].layer_communicator.capture_output( + hidden_states, residual, at_input=True ) + aux_hidden_states.append(snapshot) layer = self.layers[i] hidden_states, residual = layer( positions, diff --git a/python/sglang/srt/models/bailing_moe_linear.py b/python/sglang/srt/models/bailing_moe_linear.py index 5bcf27a284..d5ebb4c895 100644 --- a/python/sglang/srt/models/bailing_moe_linear.py +++ b/python/sglang/srt/models/bailing_moe_linear.py @@ -983,8 +983,9 @@ class BailingMoELinearModel(nn.Module): residual = None else: assert pp_proxy_tensors is not None - hidden_states = pp_proxy_tensors["hidden_states"] - residual = pp_proxy_tensors["residual"] + hidden_states, residual = self.layers[ + self.start_layer + ].layer_communicator.from_pp(pp_proxy_tensors) total_num_layers = self.end_layer - self.start_layer device = inputs_embeds.device if inputs_embeds is not None else input_ids.device diff --git a/python/sglang/srt/models/bailing_moe_v3.py b/python/sglang/srt/models/bailing_moe_v3.py index 23a0729a50..9e3dc8b56b 100644 --- a/python/sglang/srt/models/bailing_moe_v3.py +++ b/python/sglang/srt/models/bailing_moe_v3.py @@ -32,8 +32,6 @@ from sglang.srt.layers.communicator import ( LayerCommunicator, LayerFacts, enable_moe_dense_fully_dp, - layer_input_buffer, - reduce_output, ) from sglang.srt.layers.dp_attention import is_dp_attention_enabled from sglang.srt.layers.layernorm import RMSNorm @@ -1344,8 +1342,9 @@ class BailingMoELinearModel(nn.Module): residual = None else: assert pp_proxy_tensors is not None - hidden_states = pp_proxy_tensors["hidden_states"] - residual = pp_proxy_tensors["residual"] + hidden_states, residual = self.layers[ + self.start_layer + ].layer_communicator.from_pp(pp_proxy_tensors) total_num_layers = self.end_layer - self.start_layer device = hidden_states.device @@ -1382,16 +1381,12 @@ class BailingMoELinearModel(nn.Module): residual=residual, zero_allocator=zero_allocator, ) - if ( - capture_aux - and i in self.layers_to_capture - and layer_input_buffer(hidden_states).shape[0] != 0 - ): - hidden_states = reduce_output(hidden_states) - if residual is None: - dspark_aux_hidden_states.append(hidden_states) - else: - dspark_aux_hidden_states.append(hidden_states + residual) + if capture_aux and i in self.layers_to_capture: + hidden_states, snapshot = layer.layer_communicator.capture_output( + hidden_states, residual, skip_empty=True + ) + if snapshot is not None: + dspark_aux_hidden_states.append(snapshot) last_layer = self.layers[self.end_layer - 1] hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index 487ee74c6e..f6aaec7548 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -72,7 +72,6 @@ from sglang.srt.layers.communicator import ( LayerFacts, enable_moe_dense_fully_dp, get_attn_tp_context, - layer_input_buffer, ) from sglang.srt.layers.communicator.adapters.context_parallel import ( maybe_prefetch_next_full_attention_kv, @@ -2706,7 +2705,7 @@ class DeepseekV2DecoderLayer(nn.Module): captured_last_layer_outputs: Optional[AuxHiddenStateAccumulator] = None, next_full_attention_layer_id: Optional[int] = None, ) -> torch.Tensor: - hidden_states_orig = layer_input_buffer(hidden_states) + hidden_states_orig = self.layer_communicator.buffer(hidden_states) hidden_states, residual = ( self.layer_communicator.prepare_attn_and_capture_last_layer_outputs( hidden_states, @@ -3032,8 +3031,9 @@ class DeepseekV2Model(nn.Module): initial_topk_indices = None else: assert pp_proxy_tensors is not None - hidden_states = pp_proxy_tensors["hidden_states"] - residual = pp_proxy_tensors["residual"] + hidden_states, residual = self.layers[ + self.start_layer + ].layer_communicator.from_pp(pp_proxy_tensors) initial_topk_indices = pp_proxy_tensors.tensors.get("topk_indices") index_topk_share = IndexTopKShareState(forward_batch, initial_topk_indices) if not self.pp_group.is_first_rank: diff --git a/python/sglang/srt/models/dots3_common/modeling.py b/python/sglang/srt/models/dots3_common/modeling.py index 4ca0c649a5..241edfcdb7 100644 --- a/python/sglang/srt/models/dots3_common/modeling.py +++ b/python/sglang/srt/models/dots3_common/modeling.py @@ -1768,8 +1768,9 @@ class Dots3Model(nn.Module): residual = None else: assert pp_proxy_tensors is not None - hidden_states = pp_proxy_tensors["hidden_states"] - residual = pp_proxy_tensors["residual"] + hidden_states, residual = self.layers[ + self.start_layer + ].layer_communicator.from_pp(pp_proxy_tensors) normal_start_layer = self.start_layer normal_end_layer = self.end_layer diff --git a/python/sglang/srt/models/falcon_h1.py b/python/sglang/srt/models/falcon_h1.py index 243fb83070..2a8e298bd4 100644 --- a/python/sglang/srt/models/falcon_h1.py +++ b/python/sglang/srt/models/falcon_h1.py @@ -432,6 +432,10 @@ class FalconH1Model(nn.Module): forward_batch=forward_batch, ) + hidden_states, residual = self.layers[-1].layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) + if not forward_batch.forward_mode.is_idle(): if residual is None: hidden_states = self.final_layernorm(hidden_states) diff --git a/python/sglang/srt/models/gigachat35.py b/python/sglang/srt/models/gigachat35.py index 9bb4af479e..90116925b4 100644 --- a/python/sglang/srt/models/gigachat35.py +++ b/python/sglang/srt/models/gigachat35.py @@ -561,8 +561,9 @@ class GigaChat35Model(nn.Module): residual = None else: assert pp_proxy_tensors is not None - hidden_states = pp_proxy_tensors["hidden_states"] - residual = pp_proxy_tensors["residual"] + hidden_states, residual = self.layers[ + self.start_layer + ].layer_communicator.from_pp(pp_proxy_tensors) total_num_layers = self.end_layer - self.start_layer zero_allocator = BumpAllocator( @@ -580,6 +581,10 @@ class GigaChat35Model(nn.Module): zero_allocator=zero_allocator, ) + hidden_states, residual = self.layers[ + self.end_layer - 1 + ].layer_communicator.finish_layer_stack(hidden_states, residual, forward_batch) + if not self.pp_group.is_last_rank: return PPProxyTensors( {"hidden_states": hidden_states, "residual": residual} diff --git a/python/sglang/srt/models/gigachat35_mtp.py b/python/sglang/srt/models/gigachat35_mtp.py index aa37034149..04ae77c264 100644 --- a/python/sglang/srt/models/gigachat35_mtp.py +++ b/python/sglang/srt/models/gigachat35_mtp.py @@ -115,10 +115,14 @@ class GigaChat35ModelNextN(nn.Module): zero_allocator=zero_allocator, ) + hidden_states, residual = self.decoder.layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) + hidden_states_before_norm = None if not forward_batch.forward_mode.is_idle(): - hidden_states_before_norm = ( - hidden_states if residual is None else hidden_states + residual + hidden_states_before_norm = self.decoder.layer_communicator.snapshot( + hidden_states, residual ) if residual is not None: hidden_states, _ = self.shared_head.norm(hidden_states, residual) diff --git a/python/sglang/srt/models/glm4_moe.py b/python/sglang/srt/models/glm4_moe.py index a165b4b07c..de40514159 100644 --- a/python/sglang/srt/models/glm4_moe.py +++ b/python/sglang/srt/models/glm4_moe.py @@ -42,7 +42,6 @@ from sglang.srt.layers.communicator import ( LayerCommunicator, LayerFacts, enable_moe_dense_fully_dp, - reduce_output, ) from sglang.srt.layers.dp_attention import ( is_allocation_symmetric, @@ -1088,8 +1087,9 @@ class Glm4MoeModel(nn.Module): residual = None else: assert pp_proxy_tensors is not None - hidden_states = pp_proxy_tensors["hidden_states"] - residual = pp_proxy_tensors["residual"] + hidden_states, residual = self.layers[ + self.start_layer + ].layer_communicator.from_pp(pp_proxy_tensors) normal_start_layer = self.start_layer normal_end_layer = self.end_layer @@ -1106,8 +1106,12 @@ class Glm4MoeModel(nn.Module): for i in range(normal_start_layer, normal_end_layer): with get_global_expert_distribution_recorder().with_current_layer(i): if i in self.layers_to_capture: - hidden_states = reduce_output(hidden_states) - aux_hidden_states.append(hidden_states + residual) + hidden_states, snapshot = self.layers[ + i + ].layer_communicator.capture_output( + hidden_states, residual, at_input=True + ) + aux_hidden_states.append(snapshot) layer = self.layers[i] hidden_states, residual = layer( positions, diff --git a/python/sglang/srt/models/glm4_moe_lite.py b/python/sglang/srt/models/glm4_moe_lite.py index fb9e290ed7..ca97c0343e 100644 --- a/python/sglang/srt/models/glm4_moe_lite.py +++ b/python/sglang/srt/models/glm4_moe_lite.py @@ -37,7 +37,6 @@ from sglang.srt.layers.communicator import ( LayerFacts, enable_moe_dense_fully_dp, get_attn_tp_context, - reduce_output, ) from sglang.srt.layers.dp_attention import ( is_allocation_symmetric, @@ -780,8 +779,9 @@ class Glm4MoeLiteModel(nn.Module): residual = None else: assert pp_proxy_tensors is not None - hidden_states = pp_proxy_tensors["hidden_states"] - residual = pp_proxy_tensors["residual"] + hidden_states, residual = self.layers[ + self.start_layer + ].layer_communicator.from_pp(pp_proxy_tensors) device = hidden_states.device zero_allocator = BumpAllocator( buffer_size=total_num_layers * 2 * (2 if forward_batch.can_run_tbo else 1), @@ -803,8 +803,12 @@ class Glm4MoeLiteModel(nn.Module): for i in range(normal_start_layer, normal_end_layer): with get_global_expert_distribution_recorder().with_current_layer(i): if i in self.layers_to_capture: - hidden_states = reduce_output(hidden_states) - aux_hidden_states.append(hidden_states + residual) + hidden_states, snapshot = self.layers[ + i + ].layer_communicator.capture_output( + hidden_states, residual, at_input=True + ) + aux_hidden_states.append(snapshot) layer = self.layers[i] hidden_states, residual = layer( positions, diff --git a/python/sglang/srt/models/glm5_next.py b/python/sglang/srt/models/glm5_next.py index 455392dadc..6dfd8a69e4 100644 --- a/python/sglang/srt/models/glm5_next.py +++ b/python/sglang/srt/models/glm5_next.py @@ -31,8 +31,6 @@ from sglang.srt.layers.communicator import ( MHCLayerCommunicator, enable_moe_dense_fully_dp, get_attn_tp_context, - layer_input_buffer, - reduce_output, ) from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( @@ -920,7 +918,7 @@ class Glm5NextDecoderLayer(nn.Module): gemm_output_zero_allocator: BumpAllocator = None, prev_topk_indices: Optional[torch.Tensor] = None, ): - hidden_states_orig = layer_input_buffer(hidden_states) + hidden_states_orig = self.layer_communicator.buffer(hidden_states) hidden_states, residual = self.layer_communicator.prepare_attn( hidden_states, @@ -1083,14 +1081,7 @@ class Glm5NextModel(nn.Module): def get_input_embeddings(self) -> torch.Tensor: return self.embed_tokens - def _prepare_aux_hidden_state( - self, hidden_states: torch.Tensor, residual: Optional[torch.Tensor] - ) -> torch.Tensor: - # mHC folds the residual into widened hidden state, so residual remains None - # until hc_contract merges it; only plain residual streams are added here. - aux_hidden_state = ( - hidden_states if residual is None else hidden_states + residual - ) + def _prepare_aux_hidden_state(self, aux_hidden_state: torch.Tensor) -> torch.Tensor: if self.dflash_capture and self.config.mhc: aux_hidden_state = hc_contract(aux_hidden_state, self.config.hc_mult) return aux_hidden_state @@ -1112,9 +1103,9 @@ class Glm5NextModel(nn.Module): residual = None else: assert pp_proxy_tensors is not None - hidden_states = pp_proxy_tensors["hidden_states"] - # mHC carries its residual streams in hidden_states across PP stages. - residual = None if self.config.mhc else pp_proxy_tensors["residual"] + hidden_states, residual = self.layers[ + self.start_layer + ].layer_communicator.from_pp(pp_proxy_tensors) device = hidden_states.device zero_allocator = BumpAllocator( buffer_size=total_num_layers * 2 * (2 if forward_batch.can_run_tbo else 1), @@ -1158,10 +1149,16 @@ class Glm5NextModel(nn.Module): ) with ctx: if i in self.layers_to_capture: - hidden_states = reduce_output(hidden_states) - aux_hidden_state = self._prepare_aux_hidden_state( - hidden_states, residual - ) + if self.dflash_capture and self.config.mhc: + # hc_post has completed the streams; contracting them + # creates the retained value without copying the wide input. + aux_hidden_state = self._prepare_aux_hidden_state(hidden_states) + else: + hidden_states, aux_hidden_state = self.layers[ + i + ].layer_communicator.capture_output( + hidden_states, residual, at_input=True + ) if self.enable_a2a_moe and i > self.first_k_dense_replace: aux_hidden_state = get_parallel().attn_tp_group.all_gather( aux_hidden_state, dim=0 diff --git a/python/sglang/srt/models/gpt_oss.py b/python/sglang/srt/models/gpt_oss.py index b06a2b7f8a..e1574b589b 100644 --- a/python/sglang/srt/models/gpt_oss.py +++ b/python/sglang/srt/models/gpt_oss.py @@ -32,7 +32,6 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers.communicator import ( LayerCommunicator, LayerFacts, - reduce_output, ) from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, @@ -704,15 +703,18 @@ class GptOssModel(nn.Module): residual = None else: assert pp_proxy_tensors is not None - hidden_states = pp_proxy_tensors["hidden_states"] - residual = pp_proxy_tensors["residual"] + hidden_states, residual = self.layers[ + self.start_layer + ].layer_communicator.from_pp(pp_proxy_tensors) # Capture hidden-state boundaries: boundary 0 is the embedding output, # and boundary i + 1 is the output after transformer block i. aux_hidden_states = [] if self.start_layer in self.layers_to_capture: aux_hidden_states.append( - hidden_states + residual if residual is not None else hidden_states + self.layers[self.start_layer].layer_communicator.snapshot( + hidden_states, residual, at_input=True + ) ) for i in range(self.start_layer, self.end_layer): with get_global_expert_distribution_recorder().with_current_layer(i): @@ -721,12 +723,10 @@ class GptOssModel(nn.Module): positions, hidden_states, forward_batch, residual ) if i + 1 in self.layers_to_capture: - hidden_states = reduce_output(hidden_states) - aux_hidden_states.append( - hidden_states + residual - if residual is not None - else hidden_states + hidden_states, snapshot = layer.layer_communicator.capture_output( + hidden_states, residual ) + aux_hidden_states.append(snapshot) last_layer = self.layers[self.end_layer - 1] hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( hidden_states, residual, forward_batch diff --git a/python/sglang/srt/models/interns2_mobius.py b/python/sglang/srt/models/interns2_mobius.py index ff477e564c..f0514b5323 100644 --- a/python/sglang/srt/models/interns2_mobius.py +++ b/python/sglang/srt/models/interns2_mobius.py @@ -787,6 +787,10 @@ class InternS2MobiusForCausalLM(Qwen3_5ForCausalLM): input_deepstack_embeds[:, start : start + self.hidden_size] ) + hidden_states, residual = self.layers[-1].layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) + if hidden_states.shape[0] != 0: if residual is None: hidden_states = self.norm(hidden_states) diff --git a/python/sglang/srt/models/laguna.py b/python/sglang/srt/models/laguna.py index a84fb9ed56..7c419ed4d5 100644 --- a/python/sglang/srt/models/laguna.py +++ b/python/sglang/srt/models/laguna.py @@ -22,7 +22,6 @@ from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import ( LayerCommunicator, LayerFacts, - reduce_output, ) from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, @@ -585,16 +584,19 @@ class LagunaModel(nn.Module): residual = None else: assert pp_proxy_tensors is not None - hidden_states = pp_proxy_tensors["hidden_states"] - residual = pp_proxy_tensors["residual"] + hidden_states, residual = self.layers[ + self.start_layer + ].layer_communicator.from_pp(pp_proxy_tensors) aux_hidden_states = [] for i in range(self.start_layer, self.end_layer): if i in self.layers_to_capture: - hidden_states = reduce_output(hidden_states) - aux_hidden_states.append( - hidden_states + residual if residual is not None else hidden_states + hidden_states, snapshot = self.layers[ + i + ].layer_communicator.capture_output( + hidden_states, residual, at_input=True ) + aux_hidden_states.append(snapshot) layer = self.layers[i] hidden_states, residual = layer( positions, hidden_states, forward_batch, residual @@ -612,7 +614,7 @@ class LagunaModel(nn.Module): if hidden_states.shape[0] != 0: if self.end_layer in self.layers_to_capture: aux_hidden_states.append( - hidden_states + residual if residual is not None else hidden_states + last_layer.layer_communicator.snapshot(hidden_states, residual) ) if residual is None: hidden_states = self.norm(hidden_states) diff --git a/python/sglang/srt/models/llada2.py b/python/sglang/srt/models/llada2.py index d41b22b735..6aec5f5ded 100644 --- a/python/sglang/srt/models/llada2.py +++ b/python/sglang/srt/models/llada2.py @@ -749,8 +749,9 @@ class LLaDA2MoeModel(nn.Module): residual = None else: assert pp_proxy_tensors is not None - hidden_states = pp_proxy_tensors["hidden_states"] - residual = pp_proxy_tensors["residual"] + hidden_states, residual = self.layers[ + self.start_layer + ].layer_communicator.from_pp(pp_proxy_tensors) for i in range(self.start_layer, self.end_layer): with get_global_expert_distribution_recorder().with_current_layer(i): @@ -761,6 +762,10 @@ class LLaDA2MoeModel(nn.Module): forward_batch, residual, ) + + hidden_states, residual = self.layers[ + self.end_layer - 1 + ].layer_communicator.finish_layer_stack(hidden_states, residual, forward_batch) if not self.pp_group.is_last_rank: return PPProxyTensors( { diff --git a/python/sglang/srt/models/llama4.py b/python/sglang/srt/models/llama4.py index 5acf60cf9a..47736bda68 100644 --- a/python/sglang/srt/models/llama4.py +++ b/python/sglang/srt/models/llama4.py @@ -522,7 +522,11 @@ class Llama4Model(nn.Module): aux_hidden_states = [] for i in range(len(self.layers)): if i in self.layers_to_capture: - aux_hidden_states.append(hidden_states + residual) + aux_hidden_states.append( + self.layers[i].layer_communicator.snapshot( + hidden_states, residual, at_input=True + ) + ) layer = self.layers[i] hidden_states, residual = layer( positions, @@ -530,6 +534,10 @@ class Llama4Model(nn.Module): forward_batch, residual, ) + + hidden_states, residual = self.layers[-1].layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) if not forward_batch.forward_mode.is_idle(): hidden_states, _ = self.norm(hidden_states, residual) diff --git a/python/sglang/srt/models/longcat_flash.py b/python/sglang/srt/models/longcat_flash.py index 0d1c4aeeb7..4693507d05 100644 --- a/python/sglang/srt/models/longcat_flash.py +++ b/python/sglang/srt/models/longcat_flash.py @@ -630,7 +630,11 @@ class LongcatFlashModel(nn.Module): topk_indices = None for i in range(total_num_layers): if i in self.layers_to_capture: - aux_hidden_states.append(hidden_states + residual) + aux_hidden_states.append( + self.layers[i].moe_layer_communicator.snapshot( + hidden_states, residual, at_input=True + ) + ) with get_global_expert_distribution_recorder().with_current_layer(i): layer = self.layers[i] hidden_states, residual, topk_indices = layer( @@ -642,6 +646,11 @@ class LongcatFlashModel(nn.Module): topk_indices, ) + last_layer = self.layers[-1] + hidden_states, residual = last_layer.moe_layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) + if hidden_states.shape[0] != 0: if residual is None: hidden_states = self.norm(hidden_states) diff --git a/python/sglang/srt/models/longcat_flash_nextn.py b/python/sglang/srt/models/longcat_flash_nextn.py index 363c7e3e77..314712f366 100644 --- a/python/sglang/srt/models/longcat_flash_nextn.py +++ b/python/sglang/srt/models/longcat_flash_nextn.py @@ -275,6 +275,10 @@ class LongcatFlashModelNextN(nn.Module): positions, hidden_states, forward_batch, residual, zero_allocator ) + hidden_states, residual = self.decoder.layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) + if not forward_batch.forward_mode.is_idle(): if residual is not None: hidden_states, _ = self.final_layernorm(hidden_states, residual) diff --git a/python/sglang/srt/models/mimo_v2.py b/python/sglang/srt/models/mimo_v2.py index 07ff88fe60..86b41e048e 100644 --- a/python/sglang/srt/models/mimo_v2.py +++ b/python/sglang/srt/models/mimo_v2.py @@ -1040,8 +1040,9 @@ class MiMoV2Model(nn.Module): residual = None else: assert pp_proxy_tensors is not None - hidden_states = pp_proxy_tensors["hidden_states"] - residual = pp_proxy_tensors["residual"] + hidden_states, residual = self.layers[ + self.start_layer + ].layer_communicator.from_pp(pp_proxy_tensors) aux_hidden_states = AuxHiddenStatePacker(len(self.layers_to_capture)) if forward_batch.can_run_tbo and not self.layers_to_capture: @@ -1090,7 +1091,7 @@ class MiMoV2Model(nn.Module): and self.config.num_hidden_layers in self.layers_to_capture ): aux_hidden_states.append( - hidden_states if residual is None else hidden_states + residual + last_layer.layer_communicator.snapshot(hidden_states, residual) ) hidden_states_before_norm = None @@ -1104,8 +1105,8 @@ class MiMoV2Model(nn.Module): else: if hidden_states.shape[0] > 0: if forward_batch.return_hidden_states_before_norm: - hidden_states_before_norm = ( - hidden_states if residual is None else hidden_states + residual + hidden_states_before_norm = last_layer.layer_communicator.snapshot( + hidden_states, residual ) if residual is None: hidden_states = self.norm(hidden_states) diff --git a/python/sglang/srt/models/mimo_v2_nextn.py b/python/sglang/srt/models/mimo_v2_nextn.py index f317b6b610..c0c7676966 100644 --- a/python/sglang/srt/models/mimo_v2_nextn.py +++ b/python/sglang/srt/models/mimo_v2_nextn.py @@ -225,11 +225,15 @@ class MiMoV2ModelNextN(nn.Module): forward_batch=forward_batch, residual=None, ) + + hidden_states, residual = self.mtp_block.layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) hidden_states_before_norm = None if not forward_batch.forward_mode.is_idle(): if forward_batch.return_hidden_states_before_norm: - hidden_states_before_norm = ( - hidden_states if residual is None else hidden_states + residual + hidden_states_before_norm = self.mtp_block.layer_communicator.snapshot( + hidden_states, residual ) if residual is not None: hidden_states, _ = self.final_layernorm(hidden_states, residual) diff --git a/python/sglang/srt/models/minimax_m2.py b/python/sglang/srt/models/minimax_m2.py index faa1a06f45..e95b5122ad 100644 --- a/python/sglang/srt/models/minimax_m2.py +++ b/python/sglang/srt/models/minimax_m2.py @@ -1144,8 +1144,9 @@ class MiniMaxM2Model(nn.Module): residual = None else: assert pp_proxy_tensors is not None - hidden_states = pp_proxy_tensors["hidden_states"] - residual = pp_proxy_tensors["residual"] + hidden_states, residual = self.layers[ + self.start_layer + ].layer_communicator.from_pp(pp_proxy_tensors) aux_hidden_states = [] if forward_batch.can_run_tbo: diff --git a/python/sglang/srt/models/minimax_m3.py b/python/sglang/srt/models/minimax_m3.py index d1d22bd532..e59169e29e 100644 --- a/python/sglang/srt/models/minimax_m3.py +++ b/python/sglang/srt/models/minimax_m3.py @@ -1479,8 +1479,9 @@ class MiniMaxM3Model(nn.Module): residual = None else: assert pp_proxy_tensors is not None - hidden_states = pp_proxy_tensors["hidden_states"] - residual = pp_proxy_tensors["residual"] + hidden_states, residual = self.layers[ + self.start_layer + ].layer_communicator.from_pp(pp_proxy_tensors) aux_hidden_states = [] if forward_batch.can_run_tbo: diff --git a/python/sglang/srt/models/moss_vl.py b/python/sglang/srt/models/moss_vl.py index 1b9968a53b..0607b5dcb4 100644 --- a/python/sglang/srt/models/moss_vl.py +++ b/python/sglang/srt/models/moss_vl.py @@ -1114,10 +1114,9 @@ class MossVLTextModel(nn.Module): for decoder_layer in self.layers: if isinstance(decoder_layer, MossVLCrossAttentionDecoderLayer): if not skip_cross_attention: - # Fuse residual before cross-attention - if residual is not None: - hidden_states = hidden_states + residual - residual = None + hidden_states, residual = LayerCommunicator.fold( + hidden_states, residual + ) hidden_states = decoder_layer( hidden_states=hidden_states, cross_attention_states=cross_attention_states, @@ -1137,6 +1136,10 @@ class MossVLTextModel(nn.Module): else: raise ValueError(f"Unknown decoder layer type {type(decoder_layer)}") + hidden_states, residual = LayerCommunicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) + if residual is not None: hidden_states, _ = self.norm(hidden_states, residual) else: diff --git a/python/sglang/srt/models/nemotron_h.py b/python/sglang/srt/models/nemotron_h.py index ae6b2ae6ab..d0e5e30746 100644 --- a/python/sglang/srt/models/nemotron_h.py +++ b/python/sglang/srt/models/nemotron_h.py @@ -31,9 +31,7 @@ from sglang.srt.layers.attention.hybrid_linear_attn_backend import ( Mamba2AttnBackend, ) from sglang.srt.layers.attention.mamba.mamba import MambaMixer2 -from sglang.srt.layers.communicator import UnreducedOutput, reduce_output from sglang.srt.layers.dp_attention import ( - attn_tp_all_reduce, is_dp_attention_enabled, ) from sglang.srt.layers.layernorm import RMSNorm @@ -384,10 +382,10 @@ class NemotronHMLPLikeDecoderLayer(nn.Module): def forward( self, *, - hidden_states: torch.Tensor | UnreducedOutput, + hidden_states, residual: torch.Tensor | None, forward_batch: ForwardBatch, - ) -> tuple[torch.Tensor | UnreducedOutput, torch.Tensor]: + ): hidden_states, residual = self.layer_communicator.prepare_mlp( hidden_states, residual, forward_batch ) @@ -466,10 +464,10 @@ class NemotronHAttnLikeDecoderLayer(nn.Module): def forward( self, *, - hidden_states: torch.Tensor | UnreducedOutput, + hidden_states, residual: torch.Tensor | None, forward_batch: ForwardBatch, - ) -> tuple[torch.Tensor | UnreducedOutput, torch.Tensor]: + ): hidden_states, residual = self.layer_communicator.prepare_attn( hidden_states, residual, forward_batch ) @@ -721,24 +719,6 @@ class NemotronHModel(nn.Module): self.norm_f = PPMissingLayer(return_tuple=True) self.layers_to_capture: set[int] = set() - def _capture_hidden_states(self, hidden_states, residual, boundary_idx): - if residual is not None and self._owes_attention_sum_at(boundary_idx): - # Reduce a copy so the FFN stage still receives its partial sum. - hidden_states = attn_tp_all_reduce(hidden_states.clone()) - # Later norms update the input residual in place. - return hidden_states.clone() if residual is None else hidden_states + residual - - def _owes_attention_sum_at(self, boundary_idx: int) -> bool: - """Whether the value at this boundary is a mixer's partial sum that the - FFN stage after it completes: what the layer after the boundary - declares it takes, or at this rank's end what the last layer declares - it hands on.""" - if boundary_idx < self.end_layer: - edge = self.layers[boundary_idx].layer_communicator.stage_edges[0] - else: - edge = self.layers[boundary_idx - 1].layer_communicator.stage_edges[1] - return edge.produced.always_leaves - def forward( self, input_ids: torch.Tensor, @@ -755,16 +735,19 @@ class NemotronHModel(nn.Module): residual = None else: assert pp_proxy_tensors is not None - hidden_states = pp_proxy_tensors["hidden_states"] - residual = pp_proxy_tensors["residual"] + hidden_states, residual = self.layers[ + self.start_layer + ].layer_communicator.from_pp(pp_proxy_tensors) aux_hidden_states = [] for i in range(self.start_layer, self.end_layer): if i in self.layers_to_capture: - hidden_states = reduce_output(hidden_states) - aux_hidden_states.append( - self._capture_hidden_states(hidden_states, residual, i) + hidden_states, snapshot = self.layers[ + i + ].layer_communicator.capture_output( + hidden_states, residual, at_input=True ) + aux_hidden_states.append(snapshot) layer = self.layers[i] if not isinstance(layer, Layers): raise ValueError(f"Unknown layer type: {type(layer)}") @@ -784,7 +767,7 @@ class NemotronHModel(nn.Module): ) if self.end_layer in self.layers_to_capture: aux_hidden_states.append( - self._capture_hidden_states(hidden_states, residual, self.end_layer) + last_layer.layer_communicator.snapshot(hidden_states, residual) ) if residual is None: hidden_states = self.norm_f(hidden_states) diff --git a/python/sglang/srt/models/nemotron_h_mtp.py b/python/sglang/srt/models/nemotron_h_mtp.py index 09bc6e7809..3e88bd2581 100644 --- a/python/sglang/srt/models/nemotron_h_mtp.py +++ b/python/sglang/srt/models/nemotron_h_mtp.py @@ -114,9 +114,9 @@ class NemotronHMTPAttentionDecoderLayer(NemotronHAttentionDecoderLayer): hidden_states, residual = self.layer_communicator.finish_layer_stack( hidden_states, residual, forward_batch ) - if residual is not None: - hidden_states = hidden_states + residual - residual = None + hidden_states, residual = self.layer_communicator.fold( + hidden_states, residual + ) hidden_states = self.final_layernorm(hidden_states) @@ -198,9 +198,9 @@ class NemotronHMTPMoEDecoderLayer(NemotronHMoEDecoderLayer): hidden_states, residual = self.layer_communicator.finish_layer_stack( hidden_states, residual, forward_batch ) - if residual is not None: - hidden_states = hidden_states + residual - residual = None + hidden_states, residual = self.layer_communicator.fold( + hidden_states, residual + ) hidden_states = self.final_layernorm(hidden_states) diff --git a/python/sglang/srt/models/qwen2.py b/python/sglang/srt/models/qwen2.py index 9181b275cf..042621490e 100644 --- a/python/sglang/srt/models/qwen2.py +++ b/python/sglang/srt/models/qwen2.py @@ -26,6 +26,11 @@ from sglang.srt.distributed import ( get_pp_indices, ) from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.communicator.residual.access import ( + finish_layer_stack, + from_pp, + snapshot, +) from sglang.srt.layers.dp_attention import is_dp_attention_enabled from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( @@ -408,15 +413,12 @@ class Qwen2Model(nn.Module): residual = None else: assert pp_proxy_tensors is not None - hidden_states = pp_proxy_tensors["hidden_states"] - residual = pp_proxy_tensors["residual"] + hidden_states, residual = from_pp(pp_proxy_tensors) aux_hidden_states = [] for i in range(self.start_layer, self.end_layer): if i in self.layers_to_capture: - aux_hidden_states.append( - hidden_states + residual if residual is not None else hidden_states - ) + aux_hidden_states.append(snapshot(hidden_states, residual)) layer = self.layers[i] hidden_states, residual = layer( positions, @@ -424,6 +426,10 @@ class Qwen2Model(nn.Module): forward_batch, residual, ) + + hidden_states, residual = finish_layer_stack( + hidden_states, residual, forward_batch + ) if not self.pp_group.is_last_rank: return PPProxyTensors( { diff --git a/python/sglang/srt/models/qwen2_moe.py b/python/sglang/srt/models/qwen2_moe.py index 9b33dfd043..d1e96bf135 100644 --- a/python/sglang/srt/models/qwen2_moe.py +++ b/python/sglang/srt/models/qwen2_moe.py @@ -1178,8 +1178,9 @@ class Qwen2MoeModel(nn.Module): residual = None else: assert pp_proxy_tensors is not None - hidden_states = pp_proxy_tensors["hidden_states"] - residual = pp_proxy_tensors["residual"] + hidden_states, residual = self.layers[ + self.start_layer + ].layer_communicator.from_pp(pp_proxy_tensors) aux_hidden_states = [] if forward_batch.can_run_tbo: @@ -1322,6 +1323,11 @@ class Qwen2MoeForCausalLM(nn.Module): ) if end == self.model.config.num_hidden_layers: + forward_batch.hidden_states, forward_batch.residual = self.model.layers[ + end - 1 + ].layer_communicator.finish_layer_stack( + forward_batch.hidden_states, forward_batch.residual, forward_batch + ) # norm hidden_states, _ = self.model.norm( forward_batch.hidden_states, forward_batch.residual diff --git a/python/sglang/srt/models/qwen3.py b/python/sglang/srt/models/qwen3.py index 88371e9138..cdee95cd2c 100644 --- a/python/sglang/srt/models/qwen3.py +++ b/python/sglang/srt/models/qwen3.py @@ -569,6 +569,11 @@ class Qwen3ForCausalLM(nn.Module): ) if end == self.model.config.num_hidden_layers: + forward_batch.hidden_states, forward_batch.residual = self.model.layers[ + end - 1 + ].layer_communicator.finish_layer_stack( + forward_batch.hidden_states, forward_batch.residual, forward_batch + ) # norm hidden_states, _ = self.model.norm( forward_batch.hidden_states, forward_batch.residual diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py index 9609348326..5df575e712 100644 --- a/python/sglang/srt/models/qwen3_5.py +++ b/python/sglang/srt/models/qwen3_5.py @@ -46,7 +46,6 @@ from sglang.srt.layers.attention.mamba.mamba import mamba_v2_sharded_weight_load from sglang.srt.layers.communicator import ( LayerCommunicator, LayerFacts, - reduce_output, ) from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, @@ -1821,8 +1820,9 @@ class Qwen3_5ForCausalLM(nn.Module): residual = None else: assert pp_proxy_tensors is not None - hidden_states = pp_proxy_tensors["hidden_states"] - residual = pp_proxy_tensors["residual"] + hidden_states, residual = self.layers[ + self.start_layer + ].layer_communicator.from_pp(pp_proxy_tensors) aux_hidden_states = [] # Pass through decoder layers @@ -1850,9 +1850,10 @@ class Qwen3_5ForCausalLM(nn.Module): and layer_idx < 3 ): sep = self.hidden_size * layer_idx - hidden_states = reduce_output(hidden_states) - hidden_states.add_( - input_deepstack_embeds[:, sep : sep + self.hidden_size] + hidden_states, residual = layer.layer_communicator.add_to_output( + hidden_states, + residual, + input_deepstack_embeds[:, sep : sep + self.hidden_size], ) last_layer = self.layers[self.end_layer - 1] diff --git a/python/sglang/srt/models/qwen3_moe.py b/python/sglang/srt/models/qwen3_moe.py index f8d268b353..2c9363de64 100644 --- a/python/sglang/srt/models/qwen3_moe.py +++ b/python/sglang/srt/models/qwen3_moe.py @@ -1101,6 +1101,11 @@ class Qwen3MoeForCausalLM(nn.Module): ) if end == self.model.config.num_hidden_layers: + forward_batch.hidden_states, forward_batch.residual = self.model.layers[ + end - 1 + ].layer_communicator.finish_layer_stack( + forward_batch.hidden_states, forward_batch.residual, forward_batch + ) # norm hidden_states, _ = self.model.norm( forward_batch.hidden_states, forward_batch.residual diff --git a/python/sglang/srt/models/qwen3_vl.py b/python/sglang/srt/models/qwen3_vl.py index 9e673ab2af..2701e7dfd9 100644 --- a/python/sglang/srt/models/qwen3_vl.py +++ b/python/sglang/srt/models/qwen3_vl.py @@ -1207,8 +1207,9 @@ class Qwen3LLMModel(Qwen3Model): residual = None else: assert pp_proxy_tensors is not None - hidden_states = pp_proxy_tensors["hidden_states"] - residual = pp_proxy_tensors["residual"] + hidden_states, residual = self.layers[ + self.start_layer + ].layer_communicator.from_pp(pp_proxy_tensors) aux_hidden_states = [] for layer_idx, layer in enumerate( @@ -1217,7 +1218,9 @@ class Qwen3LLMModel(Qwen3Model): layer_idx = layer_idx + self.start_layer if layer_idx in self.layers_to_capture: aux_hidden_states.append( - hidden_states + residual if residual is not None else hidden_states + layer.layer_communicator.snapshot( + hidden_states, residual, at_input=True + ) ) if self.use_hf_deepstack_order: @@ -1252,6 +1255,10 @@ class Qwen3LLMModel(Qwen3Model): input_deepstack_embeds[:, sep : sep + self.hidden_size] ) + hidden_states, residual = self.layers[ + self.end_layer - 1 + ].layer_communicator.finish_layer_stack(hidden_states, residual, forward_batch) + # Handle deepstack for the last processed layer (HF-order path only). last_deepstack = ( self.get_deepstack_embeds(self.end_layer - 1, input_deepstack_embeds) diff --git a/python/sglang/srt/models/qwen3_vl_moe.py b/python/sglang/srt/models/qwen3_vl_moe.py index 929dc5cbb0..eba3915927 100644 --- a/python/sglang/srt/models/qwen3_vl_moe.py +++ b/python/sglang/srt/models/qwen3_vl_moe.py @@ -24,7 +24,6 @@ import torch.nn as nn from sglang.srt.configs.qwen3_vl import Qwen3VLMoeConfig, Qwen3VLMoeTextConfig from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation -from sglang.srt.layers.communicator import reduce_output from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.utils import get_layer_id @@ -98,8 +97,9 @@ class Qwen3MoeLLMModel(Qwen3MoeModel): residual = None else: assert pp_proxy_tensors is not None - hidden_states = pp_proxy_tensors["hidden_states"] - residual = pp_proxy_tensors["residual"] + hidden_states, residual = self.layers[ + self.start_layer + ].layer_communicator.from_pp(pp_proxy_tensors) aux_hidden_states = [] for layer_idx, layer in enumerate( @@ -107,10 +107,10 @@ class Qwen3MoeLLMModel(Qwen3MoeModel): ): layer_idx += self.start_layer if layer_idx in self.layers_to_capture: - hidden_states = reduce_output(hidden_states) - aux_hidden_states.append( - hidden_states + residual if residual is not None else hidden_states + hidden_states, snapshot = layer.layer_communicator.capture_output( + hidden_states, residual, at_input=True ) + aux_hidden_states.append(snapshot) if self.use_hf_deepstack_order: # HF-order path (RL on-policy / FSDP). SGLang applies residual at the START of the @@ -140,9 +140,10 @@ class Qwen3MoeLLMModel(Qwen3MoeModel): and layer_idx in self.deepstack_embed_to_decoder_layer ): sep = self.hidden_size * layer_idx - hidden_states = reduce_output(hidden_states) - hidden_states.add_( - input_deepstack_embeds[:, sep : sep + self.hidden_size] + hidden_states, residual = layer.layer_communicator.add_to_output( + hidden_states, + residual, + input_deepstack_embeds[:, sep : sep + self.hidden_size], ) # Handle deepstack for the last processed layer (HF-order path only). diff --git a/python/sglang/srt/models/sarvam_moe.py b/python/sglang/srt/models/sarvam_moe.py index d5b9fbc81a..b3cfa3cf77 100644 --- a/python/sglang/srt/models/sarvam_moe.py +++ b/python/sglang/srt/models/sarvam_moe.py @@ -1154,8 +1154,9 @@ class SarvamMLAModel(nn.Module): residual = None else: assert pp_proxy_tensors is not None - hidden_states = pp_proxy_tensors["hidden_states"] - residual = pp_proxy_tensors["residual"] + hidden_states, residual = self.layers[ + self.start_layer + ].layer_communicator.from_pp(pp_proxy_tensors) for i in range(self.start_layer, self.end_layer): layer = self.layers[i] @@ -1279,6 +1280,11 @@ class SarvamMLAForCausalLM(nn.Module): ) if end == self.model.config.num_hidden_layers: + forward_batch.hidden_states, forward_batch.residual = self.model.layers[ + end - 1 + ].layer_communicator.finish_layer_stack( + forward_batch.hidden_states, forward_batch.residual, forward_batch + ) if forward_batch.residual is None: hidden_states = self.model.norm(forward_batch.hidden_states) else: @@ -1462,6 +1468,11 @@ class SarvamMoEForCausalLM(BailingMoEForCausalLM): ) if end == self.model.config.num_hidden_layers: + forward_batch.hidden_states, forward_batch.residual = self.model.layers[ + end - 1 + ].layer_communicator.finish_layer_stack( + forward_batch.hidden_states, forward_batch.residual, forward_batch + ) if forward_batch.residual is None: hidden_states = self.model.norm(forward_batch.hidden_states) else: diff --git a/python/sglang/srt/models/sdar.py b/python/sglang/srt/models/sdar.py index 9513062360..98d71a637c 100644 --- a/python/sglang/srt/models/sdar.py +++ b/python/sglang/srt/models/sdar.py @@ -406,8 +406,9 @@ class SDARModel(nn.Module): residual = None else: assert pp_proxy_tensors is not None - hidden_states = pp_proxy_tensors["hidden_states"] - residual = pp_proxy_tensors.get("residual", None) + hidden_states, residual = self.layers[ + self.start_layer + ].layer_communicator.from_pp(pp_proxy_tensors, allow_missing_residual=True) for i in range(self.start_layer, self.end_layer): layer = self.layers[i] @@ -415,6 +416,10 @@ class SDARModel(nn.Module): positions, hidden_states, forward_batch, residual ) + hidden_states, residual = self.layers[ + self.end_layer - 1 + ].layer_communicator.finish_layer_stack(hidden_states, residual, forward_batch) + if not self.pp_group.is_last_rank: return PPProxyTensors( {"hidden_states": hidden_states, "residual": residual} diff --git a/python/sglang/srt/models/sdar_moe.py b/python/sglang/srt/models/sdar_moe.py index aab0af6905..baefae74c6 100644 --- a/python/sglang/srt/models/sdar_moe.py +++ b/python/sglang/srt/models/sdar_moe.py @@ -472,8 +472,9 @@ class SDARMoeModel(nn.Module): residual = None else: assert pp_proxy_tensors is not None - hidden_states = pp_proxy_tensors["hidden_states"] - residual = pp_proxy_tensors.get("residual", None) + hidden_states, residual = self.layers[ + self.start_layer + ].layer_communicator.from_pp(pp_proxy_tensors) for i in range(self.start_layer, self.end_layer): layer = self.layers[i] diff --git a/python/sglang/srt/models/step3_vl.py b/python/sglang/srt/models/step3_vl.py index d0645e47c7..18087becc3 100644 --- a/python/sglang/srt/models/step3_vl.py +++ b/python/sglang/srt/models/step3_vl.py @@ -478,6 +478,10 @@ class Step3TextModel(nn.Module): positions, hidden_states, forward_batch, residual ) + hidden_states, residual = self.layers[-1].layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) + if hidden_states.shape[0] != 0: if residual is None: hidden_states = self.norm(hidden_states) diff --git a/python/sglang/srt/models/step3p5.py b/python/sglang/srt/models/step3p5.py index 272d418bb6..69769dfc2f 100644 --- a/python/sglang/srt/models/step3p5.py +++ b/python/sglang/srt/models/step3p5.py @@ -700,8 +700,9 @@ class Step3p5Model(nn.Module): residual = None else: assert pp_proxy_tensors is not None - hidden_states = pp_proxy_tensors["hidden_states"] - residual = pp_proxy_tensors["residual"] + hidden_states, residual = self.layers[ + self.start_layer + ].layer_communicator.from_pp(pp_proxy_tensors) for i in range(self.start_layer, self.end_layer): layer = self.layers[i] @@ -735,8 +736,8 @@ class Step3p5Model(nn.Module): else: if hidden_states.shape[0] > 0: # if forward_batch.return_hidden_states_before_norm: - hidden_states_before_norm = ( - hidden_states if residual is None else hidden_states + residual + hidden_states_before_norm = last_layer.layer_communicator.snapshot( + hidden_states, residual ) if residual is None: hidden_states = self.norm(hidden_states) diff --git a/python/sglang/srt/models/step3p5_mtp.py b/python/sglang/srt/models/step3p5_mtp.py index c6c5a56fa7..ef882ed478 100644 --- a/python/sglang/srt/models/step3p5_mtp.py +++ b/python/sglang/srt/models/step3p5_mtp.py @@ -121,8 +121,8 @@ class Step3p5AMultiTokenPredictor(nn.Module): hidden_states_before_norm = None if not forward_batch.forward_mode.is_idle(): # if forward_batch.return_hidden_states_before_norm: - hidden_states_before_norm = ( - hidden_states if residual is None else hidden_states + residual + hidden_states_before_norm = self.mtp_block.layer_communicator.snapshot( + hidden_states, residual ) if residual is not None: hidden_states, _ = self.shared_head.norm(hidden_states, residual) diff --git a/python/sglang/srt/models/xllm.py b/python/sglang/srt/models/xllm.py index 44975120c3..ea53dee826 100644 --- a/python/sglang/srt/models/xllm.py +++ b/python/sglang/srt/models/xllm.py @@ -1640,8 +1640,9 @@ class XllmModel(nn.Module): residual = None else: assert pp_proxy_tensors is not None - hidden_states = pp_proxy_tensors["hidden_states"] - residual = pp_proxy_tensors["residual"] + hidden_states, residual = self.layers[ + self.start_layer + ].layer_communicator.from_pp(pp_proxy_tensors) for i in range(self.start_layer, self.end_layer): ctx = ( @@ -1657,6 +1658,10 @@ class XllmModel(nn.Module): forward_batch, residual, ) + + hidden_states, residual = self.layers[ + self.end_layer - 1 + ].layer_communicator.finish_layer_stack(hidden_states, residual, forward_batch) if not self.pp_group.is_last_rank: return PPProxyTensors( { diff --git a/python/sglang/test/communicator_patch.py b/python/sglang/test/communicator_patch.py index 5db7f4f5c5..d082da4abb 100644 --- a/python/sglang/test/communicator_patch.py +++ b/python/sglang/test/communicator_patch.py @@ -12,7 +12,7 @@ from sglang.srt.layers import communicator from sglang.srt.layers.communicator import boundary, layer, layout, ops, output from sglang.srt.layers.communicator import residual as residual_contract from sglang.srt.layers.communicator.adapters import attention -from sglang.srt.layers.communicator.residual import add_norm, mhc +from sglang.srt.layers.communicator.residual import access, add_norm, mhc COMMUNICATOR_MODULES = ( communicator, @@ -20,6 +20,7 @@ COMMUNICATOR_MODULES = ( output, attention, residual_contract, + access, add_norm, mhc, ops, diff --git a/test/registered/unit/layers/test_boundary_edges.py b/test/registered/unit/layers/test_boundary_edges.py index d46a73f519..9ff78ef6bd 100644 --- a/test/registered/unit/layers/test_boundary_edges.py +++ b/test/registered/unit/layers/test_boundary_edges.py @@ -138,9 +138,9 @@ class TestAMoeOnEachCpShard(CustomTestCase): ffn_shards_over_cp=True, ) cp_rows = rows(axis_sizes, TokenAxis.ATTN_CP) - self.assertEqual(sides.ffn.layout, cp_rows) - self.assertEqual(sides.ffn_output.layout, cp_rows) - self.assertIs(sides.ffn_output.group, SumGroup.MOE_OUTPUT) + self.assertEqual(sides.ffn.input.layout, cp_rows) + self.assertEqual(sides.ffn.output.layout, cp_rows) + self.assertIs(sides.ffn.output.group, SumGroup.MOE_OUTPUT) edges = decoder_layer_edges(sides) self.assertEqual(edges.into_ffn.need.layout, edges.into_ffn.produced.layout) diff --git a/test/registered/unit/layers/test_declared_decoder_boundary.py b/test/registered/unit/layers/test_declared_decoder_boundary.py index 65daed5c8b..ca1cdf16a5 100644 --- a/test/registered/unit/layers/test_declared_decoder_boundary.py +++ b/test/registered/unit/layers/test_declared_decoder_boundary.py @@ -931,7 +931,7 @@ class TestTwoBatchOverlap(CustomTestCase): ) steps = communicator._steps self.assertEqual( - steps.ffn.input_rows, communicator._declared.ffn.layout + steps.ffn.input_rows, communicator._declared.ffn.input.layout ) published = {} with patch_communicator( @@ -969,11 +969,11 @@ class TestTheAttentionOutputDecidesItsSum(CustomTestCase): ) owed = attn_tp > 1 self.assertIs( - sides.attention_output.group, SumGroup.ATTN_TP if owed else None + sides.attention.output.group, SumGroup.ATTN_TP if owed else None ) - self.assertIs(sides.attention_output.always_leaves, owed) - self.assertIs(sides.ffn_output.group, SumGroup.TP) - self.assertFalse(sides.ffn_output.always_leaves) + self.assertIs(sides.attention.output.always_leaves, owed) + self.assertIs(sides.ffn.output.group, SumGroup.TP) + self.assertFalse(sides.ffn.output.always_leaves) def run_steps(self, produced, *, force=False): sizes = { @@ -986,7 +986,7 @@ class TestTheAttentionOutputDecidesItsSum(CustomTestCase): produced, residual=sides.input_rows, residual_to=sides.ffn_residual_rows, - need=sides.ffn, + need=sides.ffn.input, update=comm.ADD, fusions=(), force_layernorm_before_gather=force, @@ -1108,10 +1108,10 @@ class TestFusedKernelsTakeOnlyTheStepsTheyComplete(CustomTestCase): } ) steps, fused, _ = comm_boundary._select_input_steps( - sides.attention_output, + sides.attention.output, residual=sides.input_rows, residual_to=sides.ffn_residual_rows, - need=sides.ffn, + need=sides.ffn.input, update=comm.ADD, fusions=fusions, force_layernorm_before_gather=False, @@ -1179,7 +1179,7 @@ class TestTheSequenceParallelRegion(CustomTestCase): prepare=comm_ops._read_input, input_rows=comm.Layout(frozenset({TokenAxis.ATTN_TP_SCATTER})), ), - ffn_output=sequence_parallel_layer_sides(axis_sizes=SIZES).ffn_output, + ffn_output=sequence_parallel_layer_sides(axis_sizes=SIZES).ffn.output, ffn_output_move=comm.CommunicateSummableTensorPairFn._trivial, ffn_sum_is_movable=False, ) @@ -1212,10 +1212,10 @@ class TestTheSequenceParallelRegion(CustomTestCase): ) local = frozenset({TokenAxis.ATTN_TP_SCATTER}) self.assertEqual(sides.input_rows.sharded, local) - self.assertEqual(sides.attention.gathers_itself, local) - self.assertEqual(sides.ffn.gathers_itself, local) - self.assertIsNone(sides.attention_output.group) - self.assertIsNone(sides.ffn_output.group) + self.assertEqual(sides.attention.input.gathers_itself, local) + self.assertEqual(sides.ffn.input.gathers_itself, local) + self.assertIsNone(sides.attention.output.group) + self.assertIsNone(sides.ffn.output.group) steps = self.unbound(comm_boundary._select_boundary_steps(sides)) self.assertEqual( ( diff --git a/test/registered/unit/layers/test_dense_mlp_under_prefill_cp.py b/test/registered/unit/layers/test_dense_mlp_under_prefill_cp.py index e5a7d452da..8fd8a596a2 100644 --- a/test/registered/unit/layers/test_dense_mlp_under_prefill_cp.py +++ b/test/registered/unit/layers/test_dense_mlp_under_prefill_cp.py @@ -39,8 +39,8 @@ class TestDenseMlpUnderPrefillCP(CustomTestCase): leaves_for_reduce_scatterv=False, ) # The TP group spans every CP rank, so the MLP takes every row. - self.assertIn(TokenAxis.ATTN_CP, sides.attention.layout.sharded) - self.assertNotIn(TokenAxis.ATTN_CP, sides.ffn.layout.sharded) + self.assertIn(TokenAxis.ATTN_CP, sides.attention.input.layout.sharded) + self.assertNotIn(TokenAxis.ATTN_CP, sides.ffn.input.layout.sharded) moves = CpMoves( gather=comm_ops._mlp_input_gather_moe_cp, take_back=comm.CommunicateSummableTensorPairFn._scatter_hidden_states_moe, diff --git a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py index 290d3fa4b9..2dcf8d765f 100644 --- a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py +++ b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py @@ -8,7 +8,7 @@ from torch import nn from sglang.srt.layers import communicator as comm from sglang.srt.layers.communicator import ops as comm_ops -from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.srt.model_executor.forward_batch_info import ForwardMode, PPProxyTensors from sglang.srt.models.bailing_moe import BailingMoEModel from sglang.srt.models.bailing_moe_v3 import BailingMoELinearModel from sglang.srt.models.glm4_moe import Glm4MoeModel @@ -156,6 +156,23 @@ def build_model(model_cls, *, defer, capture): class TestAuxCaptureDeferredAllreduce(CustomTestCase): + def test_skipping_an_empty_capture_does_not_complete_its_output(self): + boundary = comm.LayerCommunicator.__new__(comm.LayerCommunicator) + partial = torch.empty(0, 4) + owed = comm.UnreducedOutput(partial, group=GROUP) + with patch.object(GROUP, "all_reduce") as reduce: + hidden, captured = boundary.capture_output(owed, partial, skip_empty=True) + self.assertIs(hidden, owed) + self.assertIsNone(captured) + reduce.assert_not_called() + + def test_snapshot_without_a_residual_does_not_alias_the_main_output(self): + boundary = comm.LayerCommunicator.__new__(comm.LayerCommunicator) + hidden = torch.ones(2, 4) + captured = boundary.snapshot(hidden, None) + hidden.zero_() + torch.testing.assert_close(captured, torch.ones_like(hidden)) + def test_capture_matches_eager_reduction(self): inputs = torch.tensor([[0.25, -0.5, 0.75, 1.0], [1.5, 2.0, -1.0, 0.0]]) batch = SimpleNamespace( @@ -191,5 +208,60 @@ class TestAuxCaptureDeferredAllreduce(CustomTestCase): ) +class TestPipelineResidualReception(CustomTestCase): + def test_models_keep_the_received_residual_contribution(self): + inputs = torch.zeros(2, 4) + initial_residual = torch.full_like(inputs, 0.25) + batch = SimpleNamespace( + can_run_tbo=False, + forward_mode=ForwardMode.DECODE, + capture_hidden_mode=SimpleNamespace(need_capture=lambda: False), + ) + for model_cls in MODELS: + with self.subTest(model=model_cls.__name__): + model = build_model(model_cls, defer=True, capture=False) + model.pp_group.is_first_rank = False + result = model( + input_ids=None, + positions=None, + forward_batch=batch, + pp_proxy_tensors=PPProxyTensors( + { + "hidden_states": inputs.clone(), + "residual": initial_residual.clone(), + } + ), + ) + torch.testing.assert_close(result, initial_residual + NUM_LAYERS) + + def test_written_streams_do_not_read_a_separate_residual(self): + comm_instance = comm.LayerCommunicator.__new__(comm.LayerCommunicator) + comm_instance._residual = SimpleNamespace( + ffn_update=SimpleNamespace(at_producer=True) + ) + streams = torch.randn(2, 4, 3) + hidden, residual = comm_instance.from_pp( + PPProxyTensors({"hidden_states": streams}) + ) + self.assertIs(hidden, streams) + self.assertIsNone(residual) + + def test_optional_residual_and_declared_partial_keep_the_wire_values(self): + comm_instance = comm.LayerCommunicator.__new__(comm.LayerCommunicator) + partial = torch.randn(2, 4) + prior = torch.randn_like(partial) + hidden, residual = comm_instance.from_pp( + PPProxyTensors({"hidden_states": partial, "residual": prior}) + ) + self.assertIs(hidden, partial) + self.assertIs(residual, prior) + missing = PPProxyTensors({"hidden_states": partial}) + with self.assertRaises(KeyError): + comm_instance.from_pp(missing) + hidden, residual = comm_instance.from_pp(missing, allow_missing_residual=True) + self.assertIs(hidden, partial) + self.assertIsNone(residual) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/models/test_deepstack_deferred_allreduce.py b/test/registered/unit/models/test_deepstack_deferred_allreduce.py index 02af8943ce..f27cba6825 100644 --- a/test/registered/unit/models/test_deepstack_deferred_allreduce.py +++ b/test/registered/unit/models/test_deepstack_deferred_allreduce.py @@ -135,5 +135,55 @@ class TestDeepstackOnDeferredReduction(CustomTestCase): ) +class TestSplitPrefillCompletion(CustomTestCase): + def test_only_the_final_segment_completes_the_carried_output(self): + from sglang.srt.models.qwen2_moe import Qwen2MoeForCausalLM + from sglang.srt.models.qwen3 import Qwen3ForCausalLM + from sglang.srt.models.qwen3_moe import Qwen3MoeForCausalLM + from sglang.srt.models.sarvam_moe import ( + SarvamMLAForCausalLM, + SarvamMoEForCausalLM, + ) + + for model_cls in ( + Qwen2MoeForCausalLM, + Qwen3ForCausalLM, + Qwen3MoeForCausalLM, + SarvamMLAForCausalLM, + SarvamMoEForCausalLM, + ): + for tokens in (0, TOKENS): + with self.subTest(model=model_cls.__name__, tokens=tokens): + # Also leave the final output: the stack exit must handle it + # independently of the last layer's fusion decision. + wrapper = SimpleNamespace( + model=SimpleNamespace( + layers=[DeferringLayer(False) for _ in range(NUM_LAYERS)], + config=SimpleNamespace(num_hidden_layers=NUM_LAYERS), + norm=SumNorm(), + ), + lm_head=None, + logits_processor=lambda ids, hidden, head, batch: hidden, + ) + batch = SimpleNamespace(residual=None) + first = model_cls.forward_split_prefill( + wrapper, + None, + None, + batch, + (0, 2), + input_embeds=torch.zeros(tokens, HIDDEN), + ) + self.assertIsNone(first) + self.assertIsInstance(batch.hidden_states, UnreducedOutput) + result = model_cls.forward_split_prefill( + wrapper, None, None, batch, (2, NUM_LAYERS) + ) + self.assertIsInstance(result, torch.Tensor) + torch.testing.assert_close( + result, torch.full((tokens, HIDDEN), float(NUM_LAYERS)) + ) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/models/test_glm5_next_dflash_capture.py b/test/registered/unit/models/test_glm5_next_dflash_capture.py index 4d896b4e9b..df4f27ff34 100644 --- a/test/registered/unit/models/test_glm5_next_dflash_capture.py +++ b/test/registered/unit/models/test_glm5_next_dflash_capture.py @@ -13,6 +13,7 @@ from types import SimpleNamespace import torch from torch import nn +from sglang.srt.layers.communicator import LayerCommunicator from sglang.srt.models.glm5_next import Glm5NextModel from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -29,7 +30,9 @@ class TestGlm5NextDflashCapture(CustomTestCase): hidden_states = torch.arange(24, dtype=torch.float32).reshape(2, 12) - actual = model._prepare_aux_hidden_state(hidden_states, None) + boundary = LayerCommunicator.__new__(LayerCommunicator) + _, captured = boundary.capture_output(hidden_states, None) + actual = model._prepare_aux_hidden_state(captured) expected = hidden_states.unflatten(-1, (4, -1)).mean(dim=-2) torch.testing.assert_close(actual, expected) diff --git a/test/registered/unit/models/test_layer_stack_exit.py b/test/registered/unit/models/test_layer_stack_exit.py index 7540cfedee..437650d4fd 100644 --- a/test/registered/unit/models/test_layer_stack_exit.py +++ b/test/registered/unit/models/test_layer_stack_exit.py @@ -83,7 +83,8 @@ def defers(node): class Census: - def __init__(self): + def __init__(self, include_complete=False): + self.include_complete = include_complete self.trees = {} self.classes = {} for path in sorted(MODELS_DIR.rglob("*.py")): @@ -110,7 +111,17 @@ class Census: name for name, defs in self.classes.items() for path, node in defs - if defers(node) or any(any(calls(node, h)) for h in helpers[path]) + if defers(node) + or any(any(calls(node, h)) for h in helpers[path]) + or ( + self.include_complete + and any( + isinstance(call, ast.Call) + and isinstance(call.func, ast.Name) + and call.func.id in {"LayerCommunicator", "MHCLayerCommunicator"} + for call in ast.walk(node) + ) + ) } # A subclass defers through its base unless it replaces forward # without delegating to it. @@ -121,7 +132,7 @@ class Census: if name not in deferring for _, node in defs if set(base_names(node)) & deferring - and not declares_no_deferral(node) + and (self.include_complete or not declares_no_deferral(node)) and ( method(node, "forward") is None or delegates_to_super(method(node, "forward")) @@ -273,6 +284,7 @@ class TestLayerStackExit(CustomTestCase): def setUpClass(cls): cls.census = Census() cls.subjects = cls.census.subjects() + cls.complete_census = Census(include_complete=True) def test_derivation_finds_each_kind_of_stack(self): for name in ( @@ -287,12 +299,14 @@ class TestLayerStackExit(CustomTestCase): def test_each_stack_ends_at_the_exit(self): problems = [] - for name, (forward, layers) in sorted(self.subjects.items()): + for name, (forward, layers) in sorted(self.complete_census.subjects().items()): exits = list(calls(forward, EXIT)) if not exits: if delegates_to_super(forward): continue - if layers and all(self.census.layer_calls_exit(n) for n in layers): + if layers and all( + self.complete_census.layer_calls_exit(n) for n in layers + ): continue # the last layer ends the stack itself problems.append(f"{name}: no {EXIT} call") continue @@ -309,6 +323,87 @@ class TestLayerStackExit(CustomTestCase): ) self.assertEqual(problems, []) + def test_pipeline_residual_is_received_through_the_boundary(self): + problems = [] + for name, (forward, _) in self.complete_census.subjects().items(): + for node in ast.walk(forward): + if ( + isinstance(node, ast.Subscript) + and isinstance(node.value, ast.Name) + and node.value.id == "pp_proxy_tensors" + and isinstance(node.slice, ast.Constant) + and node.slice.value == "residual" + ): + problems.append(f"{name}:{node.lineno}") + self.assertEqual(problems, []) + + def test_models_use_boundary_access_for_output_values(self): + problems = [] + forbidden = { + "UnreducedOutput", + "HandoffOutput", + "reduce_output", + "layer_input_buffer", + } + for path, tree in self.complete_census.trees.items(): + for node in ast.walk(tree): + if isinstance(node, ast.Name) and node.id in forbidden: + problems.append(f"{path.name}:{node.lineno}: {node.id}") + for name, (forward, _) in self.complete_census.subjects().items(): + for node in ast.walk(forward): + if ( + isinstance(node, ast.BinOp) + and isinstance(node.op, ast.Add) + and isinstance(node.left, ast.Name) + and isinstance(node.right, ast.Name) + and {node.left.id, node.right.id} == {"hidden_states", "residual"} + ): + problems.append(f"{name}:{node.lineno}: residual add") + self.assertEqual(problems, []) + + def test_split_prefill_completes_only_at_the_final_segment(self): + census = self.complete_census + containers = {name for _, name in census.stacks} + while True: + grown = { + name + for name, definitions in census.classes.items() + for path, node in definitions + if set(base_names(node)) & containers + or census.built_layers(path, node, containers) + } - containers + if not grown: + break + containers |= grown + checked = 0 + for name in sorted(containers): + for path, node in census.classes[name]: + forward = method(node, "forward_split_prefill") + if forward is None: + continue + if any(calls(forward, "forward_split_prefill")): + continue # delegates to the inherited implementation + with self.subTest(model=name, file=path.name): + exits = list(calls(forward, EXIT)) + self.assertEqual(len(exits), 1) + exit_call = exits[0] + self.assertFalse(inside_loop(forward, exit_call)) + final_segments = [ + item + for item in ast.walk(forward) + if isinstance(item, ast.If) + and isinstance(item.test, ast.Compare) + and isinstance(item.test.left, ast.Name) + and item.test.left.id == "end" + and any(part is exit_call for part in ast.walk(item)) + ] + self.assertEqual(len(final_segments), 1) + for call in ast.walk(forward): + if isinstance(call, ast.Call) and is_final_norm(call): + self.assertLess(exit_call.lineno, call.lineno) + checked += 1 + self.assertGreater(checked, 0) + def test_in_place_writes_between_layers_see_the_reduced_sum(self): problems = [] for name, (forward, _) in sorted(self.subjects.items()):