From 5df1667077ab88adb6fd0cfd4ec1c88b6fc6e69b Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Thu, 24 Sep 2026 12:15:07 -0700 Subject: [PATCH] [Refactor] Pass each layer stack's output through a communicator exit (#41081) --- python/sglang/srt/layers/communicator.py | 11 + python/sglang/srt/models/bailing_moe.py | 5 +- .../sglang/srt/models/bailing_moe_linear.py | 6 +- python/sglang/srt/models/bailing_moe_nextn.py | 3 + python/sglang/srt/models/bailing_moe_v3.py | 5 +- python/sglang/srt/models/deepseek_nextn.py | 5 + python/sglang/srt/models/deepseek_v2.py | 6 +- .../srt/models/dots3_common/modeling.py | 6 +- .../sglang/srt/models/dots3_common/nextn.py | 3 + python/sglang/srt/models/glm4_moe.py | 5 +- python/sglang/srt/models/glm4_moe_lite.py | 5 +- .../sglang/srt/models/glm4_moe_lite_nextn.py | 3 + python/sglang/srt/models/glm4_moe_nextn.py | 3 + python/sglang/srt/models/glm5_next.py | 5 +- python/sglang/srt/models/gpt_oss.py | 5 +- python/sglang/srt/models/laguna.py | 5 +- python/sglang/srt/models/mimo_v2.py | 7 +- python/sglang/srt/models/minimax_m2.py | 6 +- python/sglang/srt/models/minimax_m3.py | 6 +- python/sglang/srt/models/nemotron_h.py | 11 +- python/sglang/srt/models/nemotron_h_mtp.py | 6 + python/sglang/srt/models/qwen2_moe.py | 6 +- python/sglang/srt/models/qwen3_5.py | 6 +- python/sglang/srt/models/qwen3_next.py | 4 + python/sglang/srt/models/qwen3_vl_moe.py | 5 +- python/sglang/srt/models/sarvam_moe.py | 6 +- python/sglang/srt/models/sdar_moe.py | 6 +- python/sglang/srt/models/step3p5.py | 6 +- python/sglang/srt/models/step3p5_mtp.py | 3 + .../unit/layers/test_communicator_ffn_exit.py | 18 + .../test_deepstack_deferred_allreduce.py | 3 + .../unit/models/test_layer_stack_exit.py | 326 ++++++++++++++++++ ...st_pp_send_completes_deferred_allreduce.py | 174 ---------- 33 files changed, 470 insertions(+), 210 deletions(-) create mode 100644 test/registered/unit/models/test_layer_stack_exit.py delete mode 100644 test/registered/unit/models/test_pp_send_completes_deferred_allreduce.py diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py index 648f70e571..9b42e19e18 100644 --- a/python/sglang/srt/layers/communicator.py +++ b/python/sglang/srt/layers/communicator.py @@ -943,6 +943,17 @@ class LayerCommunicator: result as a context manager around the FFN call, then call ``finish``.""" return FfnExit(self, forward_batch) + def finish_layer_stack( + self, + hidden_states: torch.Tensor, + residual: Optional[torch.Tensor], + forward_batch: ForwardBatch, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + """Complete what this layer left for a next layer. Call it on the last + layer of this rank before its output reaches the final norm, the next + pipeline rank, or any other consumer outside the layers.""" + return complete_deferred_allreduce(hidden_states), residual + def should_use_reduce_scatter(self, forward_batch: ForwardBatch): if not self.allow_reduce_scatter: return False diff --git a/python/sglang/srt/models/bailing_moe.py b/python/sglang/srt/models/bailing_moe.py index 2057a33ceb..c5c9a6c325 100644 --- a/python/sglang/srt/models/bailing_moe.py +++ b/python/sglang/srt/models/bailing_moe.py @@ -849,8 +849,11 @@ class BailingMoEModel(nn.Module): else None ), ) + last_layer = self.layers[self.end_layer - 1] + hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) if not self.pp_group.is_last_rank: - hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( { "hidden_states": hidden_states, diff --git a/python/sglang/srt/models/bailing_moe_linear.py b/python/sglang/srt/models/bailing_moe_linear.py index 90935af46f..7b6d93bd98 100644 --- a/python/sglang/srt/models/bailing_moe_linear.py +++ b/python/sglang/srt/models/bailing_moe_linear.py @@ -21,7 +21,6 @@ from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, - complete_deferred_allreduce, ) from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, @@ -1011,8 +1010,11 @@ class BailingMoELinearModel(nn.Module): residual=residual, zero_allocator=zero_allocator, ) + last_layer = self.layers[self.end_layer - 1] + hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) if not self.pp_group.is_last_rank: - hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( {"hidden_states": hidden_states, "residual": residual} ) diff --git a/python/sglang/srt/models/bailing_moe_nextn.py b/python/sglang/srt/models/bailing_moe_nextn.py index 8abb0a1bb6..90bc8a57c6 100644 --- a/python/sglang/srt/models/bailing_moe_nextn.py +++ b/python/sglang/srt/models/bailing_moe_nextn.py @@ -183,6 +183,9 @@ class BailingMoEModelNextN(nn.Module): positions, hidden_states, forward_batch, residual ) + hidden_states, residual = self.decoder.layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) if not forward_batch.forward_mode.is_idle(): if residual is not None: hidden_states, _ = self.final_layernorm(hidden_states, residual) diff --git a/python/sglang/srt/models/bailing_moe_v3.py b/python/sglang/srt/models/bailing_moe_v3.py index d5c5b81d52..87a1520127 100644 --- a/python/sglang/srt/models/bailing_moe_v3.py +++ b/python/sglang/srt/models/bailing_moe_v3.py @@ -1393,8 +1393,11 @@ class BailingMoELinearModel(nn.Module): else: dspark_aux_hidden_states.append(hidden_states + residual) + last_layer = self.layers[self.end_layer - 1] + hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) if not self.pp_group.is_last_rank: - hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( {"hidden_states": hidden_states, "residual": residual} ) diff --git a/python/sglang/srt/models/deepseek_nextn.py b/python/sglang/srt/models/deepseek_nextn.py index f4dc6bd83a..adc7374bfa 100644 --- a/python/sglang/srt/models/deepseek_nextn.py +++ b/python/sglang/srt/models/deepseek_nextn.py @@ -226,6 +226,11 @@ class DeepseekModelNextN(nn.Module): zero_allocator, prev_topk_indices=index_topk_share.topk_indices, ) + hidden_states, residual = ( + self.decoder.layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) + ) if not forward_batch.forward_mode.is_idle(): if residual is not None: hidden_states, _ = self.shared_head.norm(hidden_states, residual) diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index ece4d7c994..fdb989c57b 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -70,7 +70,6 @@ from sglang.srt.layers.aux_hidden_states import ( from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, - complete_deferred_allreduce, enable_moe_dense_fully_dp, get_attn_tp_context, ) @@ -3109,8 +3108,11 @@ class DeepseekV2Model(nn.Module): zero_allocator=zero_allocator, ) + last_layer = self.layers[self.end_layer - 1] + hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) if not self.pp_group.is_last_rank: - hidden_states = complete_deferred_allreduce(hidden_states) proxy_tensors = { "hidden_states": hidden_states, "residual": residual, diff --git a/python/sglang/srt/models/dots3_common/modeling.py b/python/sglang/srt/models/dots3_common/modeling.py index 2b4d18cc25..3c8ef85808 100644 --- a/python/sglang/srt/models/dots3_common/modeling.py +++ b/python/sglang/srt/models/dots3_common/modeling.py @@ -60,7 +60,6 @@ from sglang.srt.layers.attention.dsa.dsa_indexer import Indexer from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, - complete_deferred_allreduce, enable_moe_dense_fully_dp, ) from sglang.srt.layers.dp_attention import is_dp_attention_enabled @@ -1823,8 +1822,11 @@ class Dots3Model(nn.Module): zero_allocator=zero_allocator, ) + last_layer = self.layers[self.end_layer - 1] + hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) if not self.pp_group.is_last_rank: - hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( { "hidden_states": hidden_states, diff --git a/python/sglang/srt/models/dots3_common/nextn.py b/python/sglang/srt/models/dots3_common/nextn.py index d97853dbe3..c5f431d2ad 100644 --- a/python/sglang/srt/models/dots3_common/nextn.py +++ b/python/sglang/srt/models/dots3_common/nextn.py @@ -118,6 +118,9 @@ class Dot3NoteModelNextN(nn.Module): hidden_states, residual = head.decoder( positions, hidden_states, forward_batch, residual, zero_allocator ) + hidden_states, residual = head.decoder.layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) if not forward_batch.forward_mode.is_idle(): if residual is None: diff --git a/python/sglang/srt/models/glm4_moe.py b/python/sglang/srt/models/glm4_moe.py index 99efde41bb..a55485a436 100644 --- a/python/sglang/srt/models/glm4_moe.py +++ b/python/sglang/srt/models/glm4_moe.py @@ -1139,8 +1139,11 @@ class Glm4MoeModel(nn.Module): ].layer_scatter_modes.layer_output_mode, ) + last_layer = self.layers[self.end_layer - 1] + hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) if not self.pp_group.is_last_rank: - hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( { "hidden_states": hidden_states, diff --git a/python/sglang/srt/models/glm4_moe_lite.py b/python/sglang/srt/models/glm4_moe_lite.py index 7eb44eb1e5..e086f53203 100644 --- a/python/sglang/srt/models/glm4_moe_lite.py +++ b/python/sglang/srt/models/glm4_moe_lite.py @@ -838,8 +838,11 @@ class Glm4MoeLiteModel(nn.Module): zero_allocator=zero_allocator, ) + last_layer = self.layers[self.end_layer - 1] + hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) if not self.pp_group.is_last_rank: - hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( { "hidden_states": hidden_states, diff --git a/python/sglang/srt/models/glm4_moe_lite_nextn.py b/python/sglang/srt/models/glm4_moe_lite_nextn.py index 7efe650cf3..c93d607cc7 100644 --- a/python/sglang/srt/models/glm4_moe_lite_nextn.py +++ b/python/sglang/srt/models/glm4_moe_lite_nextn.py @@ -120,6 +120,9 @@ class Glm4MoeLiteModelNextN(nn.Module): positions, hidden_states, forward_batch, residual, zero_allocator ) + hidden_states, residual = self.decoder.layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) if not forward_batch.forward_mode.is_idle(): if residual is not None: hidden_states, _ = self.shared_head.norm(hidden_states, residual) diff --git a/python/sglang/srt/models/glm4_moe_nextn.py b/python/sglang/srt/models/glm4_moe_nextn.py index f29928efb3..f742e4b732 100644 --- a/python/sglang/srt/models/glm4_moe_nextn.py +++ b/python/sglang/srt/models/glm4_moe_nextn.py @@ -106,6 +106,9 @@ class Glm4MoeModelNextN(nn.Module): positions, hidden_states, forward_batch, residual ) + hidden_states, residual = self.decoder.layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) if not forward_batch.forward_mode.is_idle(): if residual is not None: hidden_states, _ = self.shared_head.norm(hidden_states, residual) diff --git a/python/sglang/srt/models/glm5_next.py b/python/sglang/srt/models/glm5_next.py index 3f1879cc29..e1e29654e3 100644 --- a/python/sglang/srt/models/glm5_next.py +++ b/python/sglang/srt/models/glm5_next.py @@ -1216,8 +1216,11 @@ class Glm5NextModel(nn.Module): zero_allocator=zero_allocator, ) + last_layer = self.layers[self.end_layer - 1] + hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) if not self.pp_group.is_last_rank: - hidden_states = complete_deferred_allreduce(hidden_states) if self.config.mhc: return PPProxyTensors({"hidden_states": hidden_states}) return PPProxyTensors( diff --git a/python/sglang/srt/models/gpt_oss.py b/python/sglang/srt/models/gpt_oss.py index b876a1fe0d..8fef5939d3 100644 --- a/python/sglang/srt/models/gpt_oss.py +++ b/python/sglang/srt/models/gpt_oss.py @@ -737,8 +737,11 @@ class GptOssModel(nn.Module): if residual is not None else hidden_states ) + last_layer = self.layers[self.end_layer - 1] + hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) if not self.pp_group.is_last_rank: - hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( { "hidden_states": hidden_states, diff --git a/python/sglang/srt/models/laguna.py b/python/sglang/srt/models/laguna.py index 3d45cf0557..beecee42e2 100644 --- a/python/sglang/srt/models/laguna.py +++ b/python/sglang/srt/models/laguna.py @@ -606,8 +606,11 @@ class LagunaModel(nn.Module): positions, hidden_states, forward_batch, residual ) + last_layer = self.layers[self.end_layer - 1] + hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) if not self.pp_group.is_last_rank: - hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( {"hidden_states": hidden_states, "residual": residual} ) diff --git a/python/sglang/srt/models/mimo_v2.py b/python/sglang/srt/models/mimo_v2.py index 7a4afc3d3d..d0acdf362e 100644 --- a/python/sglang/srt/models/mimo_v2.py +++ b/python/sglang/srt/models/mimo_v2.py @@ -39,7 +39,6 @@ from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, ScatterMode, - complete_deferred_allreduce, enable_moe_dense_fully_dp, ) from sglang.srt.layers.dp_attention import ( @@ -1093,6 +1092,11 @@ class MiMoV2Model(nn.Module): ), ) + last_layer = self.layers[self.end_layer - 1] + hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) + # A draft targeting the final layer ("after layer # num_hidden_layers-1") maps to capture index num_hidden_layers, # past the layer loop; capture the pre-norm output here instead. @@ -1106,7 +1110,6 @@ class MiMoV2Model(nn.Module): hidden_states_before_norm = None if not self.pp_group.is_last_rank: - hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( { "hidden_states": hidden_states, diff --git a/python/sglang/srt/models/minimax_m2.py b/python/sglang/srt/models/minimax_m2.py index b00735cc21..dd4fd222a7 100644 --- a/python/sglang/srt/models/minimax_m2.py +++ b/python/sglang/srt/models/minimax_m2.py @@ -37,7 +37,6 @@ from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, ScatterMode, - complete_deferred_allreduce, ) from sglang.srt.layers.dp_attention import ( attn_tp_all_reduce, @@ -1186,8 +1185,11 @@ class MiniMaxM2Model(nn.Module): ), ) + last_layer = self.layers[self.end_layer - 1] + hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) if not self.pp_group.is_last_rank: - hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( {"hidden_states": hidden_states, "residual": residual} ) diff --git a/python/sglang/srt/models/minimax_m3.py b/python/sglang/srt/models/minimax_m3.py index 79a4160d71..8bbbe5af88 100644 --- a/python/sglang/srt/models/minimax_m3.py +++ b/python/sglang/srt/models/minimax_m3.py @@ -39,7 +39,6 @@ from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, ScatterMode, - complete_deferred_allreduce, enable_moe_dense_fully_dp, ) from sglang.srt.layers.dp_attention import ( @@ -1539,8 +1538,11 @@ class MiniMaxM3Model(nn.Module): ), ) + last_layer = self.layers[self.end_layer - 1] + hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) if not self.pp_group.is_last_rank: - hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( {"hidden_states": hidden_states, "residual": residual} ) diff --git a/python/sglang/srt/models/nemotron_h.py b/python/sglang/srt/models/nemotron_h.py index 44be8131d2..81b4f60aa1 100644 --- a/python/sglang/srt/models/nemotron_h.py +++ b/python/sglang/srt/models/nemotron_h.py @@ -34,7 +34,6 @@ from sglang.srt.layers.attention.hybrid_linear_attn_backend import ( Mamba2AttnBackend, ) from sglang.srt.layers.attention.mamba.mamba import MambaMixer2 -from sglang.srt.layers.communicator import complete_deferred_allreduce from sglang.srt.layers.dp_attention import ( attn_tp_all_reduce, is_dp_attention_enabled, @@ -831,17 +830,15 @@ class NemotronHModel(nn.Module): forward_batch=forward_batch, ) + last_layer = self.layers[self.end_layer - 1] + hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) if not self.pp_group.is_last_rank: - hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( {"hidden_states": hidden_states, "residual": residual} ) if self.end_layer in self.layers_to_capture: - if residual is not None and getattr( - hidden_states, "_sglang_needs_allreduce_fusion", False - ): - hidden_states = tensor_model_parallel_all_reduce(hidden_states) - hidden_states._sglang_needs_allreduce_fusion = False aux_hidden_states.append( self._capture_hidden_states(hidden_states, residual, self.end_layer) ) diff --git a/python/sglang/srt/models/nemotron_h_mtp.py b/python/sglang/srt/models/nemotron_h_mtp.py index 5f77563863..6522fc06c6 100644 --- a/python/sglang/srt/models/nemotron_h_mtp.py +++ b/python/sglang/srt/models/nemotron_h_mtp.py @@ -111,6 +111,9 @@ class NemotronHMTPAttentionDecoderLayer(NemotronHAttentionDecoderLayer): ) if self.has_end_norm: + hidden_states, residual = self.layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) if residual is not None: hidden_states = hidden_states + residual residual = None @@ -193,6 +196,9 @@ class NemotronHMTPMoEDecoderLayer(NemotronHMoEDecoderLayer): ) if self.has_end_norm: + hidden_states, residual = self.layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) if residual is not None: hidden_states = hidden_states + residual residual = None diff --git a/python/sglang/srt/models/qwen2_moe.py b/python/sglang/srt/models/qwen2_moe.py index 54de047bbc..3995ff04ab 100644 --- a/python/sglang/srt/models/qwen2_moe.py +++ b/python/sglang/srt/models/qwen2_moe.py @@ -45,7 +45,6 @@ from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, ScatterMode, - complete_deferred_allreduce, ) from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, @@ -1228,8 +1227,11 @@ class Qwen2MoeModel(nn.Module): ), ) + last_layer = self.layers[self.end_layer - 1] + hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) if not self.pp_group.is_last_rank: - hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( { "hidden_states": hidden_states, diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py index 4b3ca94981..8a0f8b1288 100644 --- a/python/sglang/srt/models/qwen3_5.py +++ b/python/sglang/srt/models/qwen3_5.py @@ -1943,9 +1943,13 @@ class Qwen3_5ForCausalLM(nn.Module): input_deepstack_embeds[:, sep : sep + self.hidden_size] ) + last_layer = self.layers[self.end_layer - 1] + hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) + # Return intermediate tensors for pipeline parallelism if not self.pp_group.is_last_rank: - hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( { "hidden_states": hidden_states, diff --git a/python/sglang/srt/models/qwen3_next.py b/python/sglang/srt/models/qwen3_next.py index 83cc0ae4d5..cce93f3f86 100644 --- a/python/sglang/srt/models/qwen3_next.py +++ b/python/sglang/srt/models/qwen3_next.py @@ -949,6 +949,10 @@ class Qwen3NextModel(nn.Module): ), ) + last_layer = self.layers[-1] + hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) if not forward_batch.forward_mode.is_idle(): if residual is None: hidden_states = self.norm(hidden_states) diff --git a/python/sglang/srt/models/qwen3_vl_moe.py b/python/sglang/srt/models/qwen3_vl_moe.py index e319d8b57c..67278454ff 100644 --- a/python/sglang/srt/models/qwen3_vl_moe.py +++ b/python/sglang/srt/models/qwen3_vl_moe.py @@ -152,8 +152,11 @@ class Qwen3MoeLLMModel(Qwen3MoeModel): else None ) + last_layer = self.layers[self.end_layer - 1] + hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) if not self.pp_group.is_last_rank: - hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( { "hidden_states": hidden_states, diff --git a/python/sglang/srt/models/sarvam_moe.py b/python/sglang/srt/models/sarvam_moe.py index e34deda704..34355089f6 100644 --- a/python/sglang/srt/models/sarvam_moe.py +++ b/python/sglang/srt/models/sarvam_moe.py @@ -22,7 +22,6 @@ from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, - complete_deferred_allreduce, enable_moe_dense_fully_dp, ) from sglang.srt.layers.dp_attention import ( @@ -1171,8 +1170,11 @@ class SarvamMLAModel(nn.Module): positions, hidden_states, forward_batch, residual ) + last_layer = self.layers[self.end_layer - 1] + hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) if not self.pp_group.is_last_rank: - hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( {"hidden_states": hidden_states, "residual": residual} ) diff --git a/python/sglang/srt/models/sdar_moe.py b/python/sglang/srt/models/sdar_moe.py index f5fee0fc2a..f63132413f 100644 --- a/python/sglang/srt/models/sdar_moe.py +++ b/python/sglang/srt/models/sdar_moe.py @@ -19,7 +19,6 @@ from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, - complete_deferred_allreduce, ) from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, @@ -490,8 +489,11 @@ class SDARMoeModel(nn.Module): positions, hidden_states, forward_batch, residual ) + last_layer = self.layers[self.end_layer - 1] + hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) if not self.pp_group.is_last_rank: - hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( {"hidden_states": hidden_states, "residual": residual} ) diff --git a/python/sglang/srt/models/step3p5.py b/python/sglang/srt/models/step3p5.py index f0cdc38009..e4479db47d 100644 --- a/python/sglang/srt/models/step3p5.py +++ b/python/sglang/srt/models/step3p5.py @@ -14,7 +14,6 @@ from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, - complete_deferred_allreduce, ) from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, @@ -738,8 +737,11 @@ class Step3p5Model(nn.Module): residual, ) # break + last_layer = self.layers[self.end_layer - 1] + hidden_states, residual = last_layer.layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) if not self.pp_group.is_last_rank: - hidden_states = complete_deferred_allreduce(hidden_states) return PPProxyTensors( { "hidden_states": hidden_states, diff --git a/python/sglang/srt/models/step3p5_mtp.py b/python/sglang/srt/models/step3p5_mtp.py index 491eea15d2..c6c5a56fa7 100644 --- a/python/sglang/srt/models/step3p5_mtp.py +++ b/python/sglang/srt/models/step3p5_mtp.py @@ -115,6 +115,9 @@ class Step3p5AMultiTokenPredictor(nn.Module): forward_batch=forward_batch, residual=None, ) + hidden_states, residual = self.mtp_block.layer_communicator.finish_layer_stack( + hidden_states, residual, forward_batch + ) hidden_states_before_norm = None if not forward_batch.forward_mode.is_idle(): # if forward_batch.return_hidden_states_before_norm: diff --git a/test/registered/unit/layers/test_communicator_ffn_exit.py b/test/registered/unit/layers/test_communicator_ffn_exit.py index dd9228edf9..16b7d17baa 100644 --- a/test/registered/unit/layers/test_communicator_ffn_exit.py +++ b/test/registered/unit/layers/test_communicator_ffn_exit.py @@ -189,6 +189,24 @@ class TestCompleteDeferredAllreduce(CustomTestCase): self.assertIsNone(complete_deferred_allreduce(None)) self.all_reduce.assert_not_called() + def test_finish_layer_stack_completes_the_last_layer(self): + for fuse in (True, False): + with self.subTest(fuse=fuse): + self.all_reduce.reset_mock() + communicator = make_communicator(fuse=fuse, reduce_scatter=False) + residual = torch.zeros(3, 4) + with communicator.ffn_exit(object()) as ffn_exit: + hidden_states = torch.ones(3, 4) + hidden_states, _ = ffn_exit.finish(hidden_states, residual) + + hidden_states, residual_out = communicator.finish_layer_stack( + hidden_states, residual, object() + ) + self.assertEqual(self.all_reduce.call_count, int(fuse)) + expected = 3.0 if fuse else 2.0 # all-reduce stub / postprocess stub + torch.testing.assert_close(hidden_states, torch.full((3, 4), expected)) + self.assertIs(residual_out, residual) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/models/test_deepstack_deferred_allreduce.py b/test/registered/unit/models/test_deepstack_deferred_allreduce.py index 4172e17bdb..a5ee8eccb6 100644 --- a/test/registered/unit/models/test_deepstack_deferred_allreduce.py +++ b/test/registered/unit/models/test_deepstack_deferred_allreduce.py @@ -9,6 +9,7 @@ from unittest.mock import patch import torch from torch import nn +from sglang.srt.layers.communicator import LayerCommunicator from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -34,6 +35,8 @@ class DeferringLayer(nn.Module): def __init__(self, is_last_layer): super().__init__() self.is_last_layer = is_last_layer + # The model ends its layers at this communicator's finish_layer_stack. + self.layer_communicator = LayerCommunicator.__new__(LayerCommunicator) def forward( self, positions=None, hidden_states=None, forward_batch=None, residual=None, **_ diff --git a/test/registered/unit/models/test_layer_stack_exit.py b/test/registered/unit/models/test_layer_stack_exit.py new file mode 100644 index 0000000000..acaeec9ae9 --- /dev/null +++ b/test/registered/unit/models/test_layer_stack_exit.py @@ -0,0 +1,326 @@ +"""A decoder layer may leave its FFN all-reduce to the next layer's input norm. +The last layer on a rank has no next layer there, so a model built from such +layers must pass their output through ``finish_layer_stack`` before the final +norm or the send to the next pipeline rank. Between layers, an in-place write to +a layer's output needs the completed sum as well. Every set below is derived +from the model sources.""" + +import ast +import unittest +from pathlib import Path + +import sglang +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=10, suite="base-a-test-cpu") + +MODELS_DIR = Path(sglang.__file__).resolve().parent / "srt" / "models" +MARKER = "_sglang_needs_allreduce_fusion" +EXIT = "finish_layer_stack" +COMPLETE = "complete_deferred_allreduce" +FINAL_NORMS = {"norm", "norm_f", "final_layernorm"} +IN_PLACE = {"add_", "sub_", "mul_", "copy_"} + + +def base_names(node): + return [ + base.id if isinstance(base, ast.Name) else base.attr + for base in node.bases + if isinstance(base, (ast.Name, ast.Attribute)) + ] + + +def method(node, name): + for item in node.body: + if isinstance(item, ast.FunctionDef) and item.name == name: + return item + return None + + +def called_name(call): + func = call.func + return func.id if isinstance(func, ast.Name) else getattr(func, "attr", None) + + +def calls(node, name): + for sub in ast.walk(node): + if isinstance(sub, ast.Call) and called_name(sub) == name: + yield sub + + +def delegates_to_super(forward): + return any( + isinstance(call.func, ast.Attribute) + and isinstance(call.func.value, ast.Call) + and getattr(call.func.value.func, "id", None) == "super" + for call in calls(forward, "forward") + ) + + +def defers(node): + """Asks whether to leave the reduction to the next layer (directly or + through ffn_exit), or sets the marker itself.""" + if any(calls(node, "ffn_exit")) or any( + calls(node, "should_fuse_mlp_allreduce_with_next_layer") + ): + return True + return any( + isinstance(sub, ast.Assign) + and isinstance(sub.value, ast.Constant) + and sub.value.value is True + and any( + isinstance(target, ast.Attribute) and target.attr == MARKER + for target in sub.targets + ) + for sub in ast.walk(node) + ) + + +class Census: + def __init__(self): + self.trees = {} + self.classes = {} + for path in sorted(MODELS_DIR.rglob("*.py")): + tree = ast.parse(path.read_text()) + self.trees[path] = tree + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef): + self.classes.setdefault(node.name, []).append((path, node)) + self.deferring = self._deferring_layers() + self.family = self._layer_family() + self.stacks = self._layer_stacks() + + def _deferring_layers(self): + # Module-level helpers that defer on a layer's behalf. + helpers = { + path: { + st.name + for st in tree.body + if isinstance(st, ast.FunctionDef) and defers(st) + } + for path, tree in self.trees.items() + } + deferring = { + name + for name, defs in self.classes.items() + for path, node in defs + if defers(node) or any(any(calls(node, h)) for h in helpers[path]) + } + # A subclass defers through its base unless it replaces forward + # without delegating to it. + while True: + inherited = { + name + for name, defs in self.classes.items() + if name not in deferring + for _, node in defs + if set(base_names(node)) & deferring + and ( + method(node, "forward") is None + or delegates_to_super(method(node, "forward")) + ) + } + if not inherited: + return deferring + deferring |= inherited + + def _layer_family(self): + """Classes derived from a deferring layer, whether or not they still + defer (a subclass that replaces forward may not).""" + family = set(self.deferring) + while True: + derived = { + name + for name, defs in self.classes.items() + if name not in family + for _, node in defs + if set(base_names(node)) & family + } + if not derived: + return family + family |= derived + + def registries(self, path, names): + """Module-level names bound to a value that references ``names``, such + as a table of decoder layer types.""" + out = {} + for st in self.trees[path].body: + if isinstance(st, (ast.Assign, ast.AnnAssign)) and st.value is not None: + members = { + sub.id + for sub in ast.walk(st.value) + if isinstance(sub, ast.Name) and sub.id in names + } + targets = st.targets if isinstance(st, ast.Assign) else [st.target] + for target in targets: + if members and isinstance(target, ast.Name): + out[target.id] = members + return out + + def built_layers(self, path, node, names): + """The classes in ``names`` this class references other than as a base + or an isinstance argument, looking through module-level tables.""" + skip = {id(base) for base in node.bases} + for call in ast.walk(node): + if ( + isinstance(call, ast.Call) + and called_name(call) in ("isinstance", "issubclass") + and len(call.args) > 1 + ): + skip |= {id(sub) for sub in ast.walk(call.args[1])} + tables = self.registries(path, names) + found = set() + for sub in ast.walk(node): + if id(sub) in skip: + continue + name = sub.id if isinstance(sub, ast.Name) else getattr(sub, "attr", None) + if name in names: + found.add(name) + elif name in tables: + found |= tables[name] + return found + + def _layer_stacks(self): + """Classes whose instances run deferring layers: they build them, hold a + forward-less container that does, or subclass such a class.""" + stacks = {} + runners = set(self.deferring) + while True: + grown = False + for name, defs in self.classes.items(): + if name in self.deferring: + continue + for path, node in defs: + key = (path, node.name) + if key in stacks: + continue + layers = self.built_layers(path, node, runners) + parents = set(base_names(node)) & {n for _, n in stacks} + if not layers and not parents: + continue + if not layers and self.built_layers( + path, node, self.family - self.deferring + ): + continue # a subclass that builds layers which never defer + stacks[key] = (node, layers & self.deferring) + if self.used_forward(path, node) is None: + runners.add(name) + grown = True + if not grown: + return stacks + + def used_forward(self, path, node, seen=()): + forward = method(node, "forward") + if forward is not None: + return path, node, forward + for base in base_names(node): + for base_path, base_node in self.classes.get(base, ()): + if (base_path, base) not in seen: + found = self.used_forward( + base_path, base_node, seen + ((base_path, base),) + ) + if found: + return found + return None + + def subjects(self): + """forward methods the layer stacks run, with the deferring layers + each stack builds.""" + out = {} + for (path, _), (node, layers) in self.stacks.items(): + found = self.used_forward(path, node) + if found is None: + continue # a container, run by the class that holds it + fpath, fnode, forward = found + key = f"{fpath.relative_to(MODELS_DIR)}:{fnode.name}" + entry = out.setdefault(key, (forward, set())) + entry[1].update(layers) + return out + + def layer_calls_exit(self, name): + return any( + method(node, "forward") is not None + and any(calls(method(node, "forward"), EXIT)) + for _, node in self.classes.get(name, ()) + ) + + +def inside_loop(tree, target): + for node in ast.walk(tree): + if isinstance(node, (ast.For, ast.While)) and node is not tree: + if any(sub is target for sub in ast.walk(node)): + return True + return False + + +def is_final_norm(call): + func, chain = call.func, [] + while isinstance(func, ast.Attribute): + chain.append(func.attr) + func = func.value + return bool(FINAL_NORMS & set(chain)) + + +class TestLayerStackExit(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.census = Census() + cls.subjects = cls.census.subjects() + + def test_derivation_finds_each_kind_of_stack(self): + for name in ( + "glm4_moe_lite.py:Glm4MoeLiteModel", # builds its layers + "qwen2_moe.py:Qwen2MoeModel", # run by Qwen3-MoE, which passes them in + "qwen3_next.py:Qwen3NextModel", # layers defer through a helper + "nemotron_h.py:NemotronHModel", # layers come from a type table + "dots3_common/nextn.py:Dot3NoteModelNextN", # holds an MTP head + "mistral_large_3_eagle.py:MistralLarge3EagleModel", # subclass + ): + self.assertIn(name, self.subjects) + + def test_each_stack_ends_at_the_exit(self): + problems = [] + for name, (forward, layers) in sorted(self.subjects.items()): + exits = list(calls(forward, EXIT)) + if not exits: + if delegates_to_super(forward): + continue + if layers and all(self.census.layer_calls_exit(n) for n in layers): + continue # the last layer ends the stack itself + problems.append(f"{name}: no {EXIT} call") + continue + if len(exits) > 1 or inside_loop(forward, exits[0]): + problems.append(f"{name}: {EXIT} must be called once, after the loop") + continue + line = exits[0].lineno + for call in ast.walk(forward): + if not isinstance(call, ast.Call) or call.lineno >= line: + continue + if called_name(call) == "PPProxyTensors" or is_final_norm(call): + problems.append( + f"{name} line {call.lineno}: {called_name(call)} before {EXIT}" + ) + self.assertEqual(problems, []) + + def test_in_place_writes_between_layers_see_the_reduced_sum(self): + problems = [] + for name, (forward, _) in sorted(self.subjects.items()): + completes = [call.lineno for call in calls(forward, COMPLETE)] + for call in ast.walk(forward): + if ( + isinstance(call, ast.Call) + and isinstance(call.func, ast.Attribute) + and call.func.attr in IN_PLACE + and isinstance(call.func.value, ast.Name) + and call.func.value.id == "hidden_states" + and not any(line < call.lineno for line in completes) + ): + problems.append(f"{name} line {call.lineno}") + self.assertEqual( + problems, [], f"call {COMPLETE}(hidden_states) before writing in place" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/models/test_pp_send_completes_deferred_allreduce.py b/test/registered/unit/models/test_pp_send_completes_deferred_allreduce.py deleted file mode 100644 index 1e1544766b..0000000000 --- a/test/registered/unit/models/test_pp_send_completes_deferred_allreduce.py +++ /dev/null @@ -1,174 +0,0 @@ -"""A decoder layer may leave its FFN all-reduce to the next layer's input norm. -The last layer on a non-last pipeline rank has no next layer there, so every -model forward that sends hidden states produced by such layers must complete -the reduction first. Both sets are derived from the model sources.""" - -import ast -import unittest -from pathlib import Path - -import sglang -from sglang.test.ci.ci_register import register_cpu_ci -from sglang.test.test_utils import CustomTestCase - -register_cpu_ci(est_time=10, suite="base-a-test-cpu") - -MODELS_DIR = Path(sglang.__file__).resolve().parent / "srt" / "models" -MARKER = "_sglang_needs_allreduce_fusion" -COMPLETE = "complete_deferred_allreduce" - - -def base_names(node): - return [ - base.id if isinstance(base, ast.Name) else base.attr - for base in node.bases - if isinstance(base, (ast.Name, ast.Attribute)) - ] - - -def method(node, name): - for item in node.body: - if isinstance(item, ast.FunctionDef) and item.name == name: - return item - return None - - -def calls(node, name): - for sub in ast.walk(node): - if isinstance(sub, ast.Call): - func = sub.func - called = ( - func.id if isinstance(func, ast.Name) else getattr(func, "attr", None) - ) - if called == name: - yield sub - - -def defers_in_own_body(node): - """Asks whether to leave the reduction to the next layer (directly or - through ffn_exit), or sets the marker itself.""" - if any(calls(node, "ffn_exit")) or any( - calls(node, "should_fuse_mlp_allreduce_with_next_layer") - ): - return True - for sub in ast.walk(node): - if isinstance(sub, ast.Assign) and isinstance(sub.value, ast.Constant): - for target in sub.targets: - if ( - isinstance(target, ast.Attribute) - and target.attr == MARKER - and sub.value.value is True - ): - return True - return False - - -def load_classes(): - classes = {} - for path in sorted(MODELS_DIR.rglob("*.py")): - tree = ast.parse(path.read_text()) - for node in ast.walk(tree): - if isinstance(node, ast.ClassDef): - classes.setdefault(node.name, []).append((path, tree, node)) - return classes - - -def deferring_layer_classes(classes): - deferring = { - name - for name, defs in classes.items() - if any(defers_in_own_body(node) for _, _, node in defs) - } - # A subclass defers through its base unless it replaces forward without - # delegating to it. - while True: - inherited = set() - for name, defs in classes.items(): - if name in deferring: - continue - for _, _, node in defs: - forward = method(node, "forward") - delegates = forward is None or any( - isinstance(call.func, ast.Attribute) - and isinstance(call.func.value, ast.Call) - and getattr(call.func.value.func, "id", None) == "super" - for call in calls(forward, "forward") - ) - if delegates and set(base_names(node)) & deferring: - inherited.add(name) - if not inherited: - return deferring - deferring |= inherited - - -def uses_deferring_layer(tree, deferring): - """References a deferring layer class other than to subclass it.""" - in_bases = { - id(base) - for node in ast.walk(tree) - if isinstance(node, ast.ClassDef) - for base in node.bases - } - for node in ast.walk(tree): - if id(node) in in_bases: - continue - if isinstance(node, ast.Name) and node.id in deferring: - return True - if isinstance(node, ast.Attribute) and node.attr in deferring: - return True - return False - - -def pp_senders(classes): - """forward methods, of models built from deferring layers, that construct - PPProxyTensors -- including ones inherited from a base model class.""" - deferring = deferring_layer_classes(classes) - senders = {} - pending = [] - for defs in classes.values(): - for path, tree, node in defs: - if uses_deferring_layer(tree, deferring): - pending.append((path, node)) - seen = set() - while pending: - path, node = pending.pop() - key = (path, node.name) - if key in seen: - continue - seen.add(key) - forward = method(node, "forward") - if forward is not None and any(calls(forward, "PPProxyTensors")): - senders[f"{path.relative_to(MODELS_DIR)}:{node.name}"] = forward - for base in base_names(node): - for base_path, _, base_node in classes.get(base, ()): - pending.append((base_path, base_node)) - return senders - - -class TestPPSendCompletesDeferredAllreduce(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.classes = load_classes() - cls.senders = pp_senders(cls.classes) - - def test_derivation_finds_the_known_paths(self): - # One model whose own layers defer, one that inherits its forward. - self.assertIn("glm4_moe_lite.py:Glm4MoeLiteModel", self.senders) - self.assertIn("qwen2_moe.py:Qwen2MoeModel", self.senders) - - def test_each_pp_send_completes_the_deferred_reduction_first(self): - missing = [] - for name, forward in sorted(self.senders.items()): - completes = [call.lineno for call in calls(forward, COMPLETE)] - for send in calls(forward, "PPProxyTensors"): - if not any(line < send.lineno for line in completes): - missing.append(f"{name} (line {send.lineno})") - self.assertEqual( - missing, - [], - f"call {COMPLETE}(hidden_states) before sending to the next pipeline rank", - ) - - -if __name__ == "__main__": - unittest.main()