From d3fcf8997cf2fc7ea51cdd7a852dc57306cb0a5a Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Sat, 26 Sep 2026 02:40:34 -0700 Subject: [PATCH] [Refactor] Run the LayerNorm SP region's boundary steps in the communicator itself (#41255) --- python/sglang/srt/layers/communicator.py | 92 ++++++--------- .../unit/layers/test_communicator_ffn_exit.py | 8 +- .../test_layer_communicator_fusion_gate.py | 2 +- .../unit/layers/test_layernorm_sp.py | 108 ++++++++++++++++++ .../test_postprocess_reduce_scatterv.py | 2 +- .../unit/layers/test_prepare_attn_steps.py | 2 +- .../lora/test_triton_dp_attention_unit.py | 2 +- 7 files changed, 150 insertions(+), 66 deletions(-) diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py index 443b5c090d..62c4395091 100644 --- a/python/sglang/srt/layers/communicator.py +++ b/python/sglang/srt/layers/communicator.py @@ -444,6 +444,8 @@ class LayerScatterModes: middle_residual_mode: ScatterMode layer_output_mode: ScatterMode is_layer_sparse: bool = False + # The model's first layer: its input is the embedding, not a layer output. + is_first_layer: bool = False # The model's last layer: its output goes to the final norm, not a next layer. is_last_layer: bool = False @@ -457,6 +459,7 @@ class LayerScatterModes: middle_residual_mode=cls._compute_middle_residual_mode(context), layer_output_mode=cls._compute_layer_output_mode(context), is_layer_sparse=context.is_layer_sparse, + is_first_layer=context.layer_id == 0, is_last_layer=context.layer_id == context.num_layers - 1, ) @@ -719,7 +722,6 @@ class LayerCommunicator: fused_ar_quant_keep_bf16: bool = False, # False for a layer whose FFN always completes its own reduction. allow_deferred_ffn_reduction: bool = True, - _is_sp_variant: bool = False, ): self.layer_scatter_modes = layer_scatter_modes self.input_layernorm = input_layernorm @@ -754,30 +756,9 @@ class LayerCommunicator: get_lora().enable_lora ) - # Under LayerNorm SP the norm/residual run on the sequence shard with no - # collectives, so delegate to an all-SCATTERED sibling while the region is - # active. _is_sp_variant stops the sibling from building its own. - self._sp_variant: Optional[LayerCommunicator] = None - if not _is_sp_variant and layernorm_sp.layernorm_sp_enabled(): - self._sp_variant = LayerCommunicator( - layer_scatter_modes=LayerScatterModes( - layer_input_mode=ScatterMode.SCATTERED, - attn_mode=ScatterMode.SCATTERED, - mlp_mode=ScatterMode.SCATTERED, - middle_residual_mode=ScatterMode.SCATTERED, - layer_output_mode=ScatterMode.SCATTERED, - is_last_layer=layer_scatter_modes.is_last_layer, - ), - input_layernorm=input_layernorm, - post_attention_layernorm=post_attention_layernorm, - allow_reduce_scatter=allow_reduce_scatter, - qkv_latent_func=qkv_latent_func, - force_layernorm_before_dp_gather=force_layernorm_before_dp_gather, - enable_fused_ar_quant=enable_fused_ar_quant, - fused_ar_quant_keep_bf16=fused_ar_quant_keep_bf16, - allow_deferred_ffn_reduction=allow_deferred_ffn_reduction, - _is_sp_variant=True, - ) + # Under LayerNorm SP the residual and norms run on this rank's sequence + # shard with no collectives while the region is active. + self._sp_region = layernorm_sp.layernorm_sp_enabled() def _post_init_communicate(self): self._communicate_simple_fn = CommunicateSimpleFn.get_fn( @@ -891,24 +872,14 @@ class LayerCommunicator: forward_batch=forward_batch, ) hidden_states = owed.partial - # residual is None marks the first decoder layer, where the SP region - # opens: re-evaluated per forward so a crash mid-loop cannot leak into - # the next one. - if self._sp_variant is not None: - if residual is None: - get_forward().set( - "sp_active", forward_batch.forward_mode == ForwardMode.EXTEND - ) - if get_forward().sp_active: - hidden_states = layernorm_sp.sp_entry_scatter(hidden_states) + # The SP region opens at the first layer, re-evaluated per forward so a + # crash mid-loop cannot leak into the next one. + if self._sp_region and self.layer_scatter_modes.is_first_layer: + get_forward().set( + "sp_active", layernorm_sp.runs_sp(forward_batch.forward_mode) + ) if get_forward().sp_active: - return self._sp_variant.prepare_attn( - hidden_states, - residual, - forward_batch, - quant_format, - post_residual_addition, - ) + hidden_states = layernorm_sp.sp_entry_scatter(hidden_states) if get_attn_tp_context().input_scattered: hidden_states, residual = self._tp_reduce_scatter( hidden_states, @@ -932,13 +903,18 @@ class LayerCommunicator: forward_batch=forward_batch, ) + def _in_sp_region(self) -> bool: + return self._sp_region and get_forward().sp_active + def _finish_prepare_attn(self, hidden_states, residual, forward_batch): - """Tail every prepare_attn path must run, or ``attn_inputs`` is unset.""" - hidden_states = self._communicate_simple_fn( - hidden_states=hidden_states, - forward_batch=forward_batch, - context=self._context, - ) + """Tail every prepare_attn path must run, or ``attn_inputs`` is unset. In + the SP region the attention input stays on this rank's sequence shard.""" + if not self._in_sp_region(): + hidden_states = self._communicate_simple_fn( + hidden_states=hidden_states, + forward_batch=forward_batch, + context=self._context, + ) if self.qkv_latent_func is not None: attn_inputs = AttentionInputs( hidden_states, forward_batch, self.qkv_latent_func @@ -1082,9 +1058,13 @@ class LayerCommunicator: cache=None, ): self.publish_mlp_lora_layout() - if self._sp_variant is not None and get_forward().sp_active: - return self._sp_variant.prepare_mlp( - hidden_states, residual, forward_batch, cache + if self._in_sp_region(): + return _mlp_input_norm( + hidden_states, + residual, + forward_batch, + self.post_attention_layernorm, + self._context, ) if cache is not None: self._context.cache = cache @@ -1110,10 +1090,8 @@ class LayerCommunicator: residual: torch.Tensor, forward_batch: ForwardBatch, ): - if self._sp_variant is not None and get_forward().sp_active: - return self._sp_variant.postprocess_layer( - hidden_states, residual, forward_batch - ) + if self._in_sp_region(): + return hidden_states, residual return self._communicate_summable_tensor_pair_fn( hidden_states=hidden_states, residual=residual, @@ -1127,9 +1105,7 @@ class LayerCommunicator: """Whether the next layer's input can run this layer's move of its FFN output back to this rank's tokens: the base postprocess scatter, outside an active LayerNorm SP region.""" - return self._postprocess_scatters_to_local_tokens and not ( - self._sp_variant is not None and get_forward().sp_active - ) + return self._postprocess_scatters_to_local_tokens and not self._in_sp_region() def _postprocess_dp_step( self, forward_batch: ForwardBatch diff --git a/test/registered/unit/layers/test_communicator_ffn_exit.py b/test/registered/unit/layers/test_communicator_ffn_exit.py index 05c51d251a..47d7371c6b 100644 --- a/test/registered/unit/layers/test_communicator_ffn_exit.py +++ b/test/registered/unit/layers/test_communicator_ffn_exit.py @@ -48,7 +48,7 @@ def make_communicator( return_value=reduce_scatter ) communicator.is_last_layer = False - communicator._sp_variant = None + communicator._sp_region = False communicator._postprocess_scatters_to_local_tokens = scatters_to_local_tokens communicator._postprocess_dp_step = MagicMock(return_value=reduce_scatter_step) communicator.ffn_reduction_group = MagicMock(return_value=group or make_group()) @@ -277,12 +277,12 @@ class TestSelectFfnCompletion(CustomTestCase): the FFN runs.""" def communicator( - self, *, fuse=False, is_last_layer=False, scatters=True, sp_variant=None + self, *, fuse=False, is_last_layer=False, scatters=True, sp_region=False ): communicator = LayerCommunicator.__new__(LayerCommunicator) communicator.allow_deferred_ffn_reduction = True communicator.is_last_layer = is_last_layer - communicator._sp_variant = sp_variant + communicator._sp_region = sp_region communicator._postprocess_scatters_to_local_tokens = scatters communicator.allow_reduce_scatter = True communicator.layer_scatter_modes = types.SimpleNamespace(is_layer_sparse=True) @@ -326,7 +326,7 @@ class TestSelectFfnCompletion(CustomTestCase): self.assertIsNone(self.left(communicator, step)) def test_postprocess_keeps_an_active_layernorm_sp_region(self): - communicator = self.communicator(sp_variant=object()) + communicator = self.communicator(sp_region=True) with get_forward().scoped(sp_active=True): self.assertIsNone( self.left(communicator, comm._reduce_and_redistribute_output_varlen) diff --git a/test/registered/unit/layers/test_layer_communicator_fusion_gate.py b/test/registered/unit/layers/test_layer_communicator_fusion_gate.py index 02417a6580..461be20005 100644 --- a/test/registered/unit/layers/test_layer_communicator_fusion_gate.py +++ b/test/registered/unit/layers/test_layer_communicator_fusion_gate.py @@ -323,7 +323,7 @@ class TestDeferFfnReduction(CustomTestCase): communicator = _fake_communicator() communicator.is_last_layer = is_last_layer communicator._postprocess_scatters_to_local_tokens = scatters_to_local_tokens - communicator._sp_variant = object() if sp_active else None + communicator._sp_region = sp_active communicator.allow_reduce_scatter = True communicator.layer_scatter_modes.is_layer_sparse = True forward_batch = types.SimpleNamespace( diff --git a/test/registered/unit/layers/test_layernorm_sp.py b/test/registered/unit/layers/test_layernorm_sp.py index 3c49e6d729..121a08d195 100644 --- a/test/registered/unit/layers/test_layernorm_sp.py +++ b/test/registered/unit/layers/test_layernorm_sp.py @@ -7,8 +7,12 @@ fused matmul fast-paths need a real TP group and are covered by the e2e test. import unittest from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import torch from sglang.srt.arg_groups.layernorm_sp_hook import validate_layernorm_sp +from sglang.srt.layers import communicator as comm from sglang.srt.layers import layernorm_sp from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.runtime_context import ( @@ -129,5 +133,109 @@ class TestLayerNormSPActiveFlag(CustomTestCase): self.assertFalse(get_forward().sp_active) +class _Norm: + def __call__(self, x, residual=None, post_residual_addition=None): + if residual is None: + return x * 2 + s = x + residual + return s * 2, s + + +class TestSpRegionSteps(CustomTestCase): + """The communicator opens the SP region at the model's first layer on a + prefill forward and, inside it, keeps every boundary on this rank's sequence + shard: add + norm, no move, no collective.""" + + def communicator(self, *, first_layer): + c = comm.LayerCommunicator.__new__(comm.LayerCommunicator) + c._sp_region = True + c.layer_scatter_modes = SimpleNamespace(is_first_layer=first_layer) + c._attn_input_fusions = () + c.input_layernorm = _Norm() + c.post_attention_layernorm = _Norm() + c.qkv_latent_func = None + c._context = SimpleNamespace(cache=None) + c._communicate_simple_fn = MagicMock(side_effect=AssertionError("moved")) + c._communicate_summable_tensor_pair_fn = MagicMock( + side_effect=AssertionError("postprocess ran") + ) + return c + + def run_prepare_attn(self, communicator, mode, hidden, residual): + batch = SimpleNamespace(forward_mode=mode) + with ( + get_flags().sp.override(enabled=True), + get_forward().scoped(sp_active=False), + patch.object( + comm, + "get_attn_tp_context", + return_value=SimpleNamespace(input_scattered=False), + ), + patch.object( + layernorm_sp, "sp_entry_scatter", side_effect=lambda h: h[:1] + ) as scatter, + ): + out = communicator.prepare_attn(hidden, residual, batch) + return out, get_forward().sp_active, scatter + + def test_the_first_layer_opens_the_region_on_prefill(self): + hidden = torch.ones(2, 4) + (h, r), active, scatter = self.run_prepare_attn( + self.communicator(first_layer=True), ForwardMode.EXTEND, hidden, None + ) + self.assertTrue(active) + scatter.assert_called_once() + torch.testing.assert_close(h, torch.full((1, 4), 2.0)) + torch.testing.assert_close(r, torch.ones(1, 4)) + + def test_decode_leaves_the_region_closed(self): + communicator = self.communicator(first_layer=True) + communicator._communicate_simple_fn = MagicMock( + side_effect=lambda **k: k["hidden_states"] + ) + (h, _), active, scatter = self.run_prepare_attn( + communicator, ForwardMode.DECODE, torch.ones(2, 4), None + ) + self.assertFalse(active) + scatter.assert_not_called() + communicator._communicate_simple_fn.assert_called_once() + + def test_a_later_layer_does_not_reopen_the_region(self): + communicator = self.communicator(first_layer=False) + communicator._communicate_simple_fn = MagicMock( + side_effect=lambda **k: k["hidden_states"] + ) + _, active, scatter = self.run_prepare_attn( + communicator, ForwardMode.EXTEND, torch.ones(2, 4), torch.ones(2, 4) + ) + self.assertFalse(active) + scatter.assert_not_called() + + def test_inside_the_region_each_boundary_stays_on_the_shard(self): + communicator = self.communicator(first_layer=False) + hidden, residual = torch.ones(1, 4), torch.full((1, 4), 3.0) + with ( + get_flags().sp.override(enabled=True), + get_forward().scoped(sp_active=True), + ): + with patch.object( + comm, + "get_attn_tp_context", + return_value=SimpleNamespace(input_scattered=False), + ): + h, r = communicator.prepare_attn( + hidden, + residual.clone(), + SimpleNamespace(forward_mode=ForwardMode.EXTEND), + ) + torch.testing.assert_close(h, torch.full((1, 4), 8.0)) + h, r = communicator.prepare_mlp(hidden, residual.clone(), object()) + torch.testing.assert_close(h, torch.full((1, 4), 8.0)) + torch.testing.assert_close(r, torch.full((1, 4), 4.0)) + out = communicator.postprocess_layer(hidden, residual, object()) + self.assertIs(out[0], hidden) + self.assertIs(out[1], residual) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/layers/test_postprocess_reduce_scatterv.py b/test/registered/unit/layers/test_postprocess_reduce_scatterv.py index f0d57837c5..eb398b025b 100644 --- a/test/registered/unit/layers/test_postprocess_reduce_scatterv.py +++ b/test/registered/unit/layers/test_postprocess_reduce_scatterv.py @@ -140,7 +140,7 @@ class TestPostprocessReduceScatterv(CustomTestCase): def test_postprocess_passes_the_layer_sparsity(self): seen = {} communicator = LayerCommunicator.__new__(LayerCommunicator) - communicator._sp_variant = None + communicator._sp_region = False communicator._context = None communicator.allow_reduce_scatter = False communicator.layer_scatter_modes = types.SimpleNamespace(is_layer_sparse=True) diff --git a/test/registered/unit/layers/test_prepare_attn_steps.py b/test/registered/unit/layers/test_prepare_attn_steps.py index 21fbf4f7d7..5fa3b00323 100644 --- a/test/registered/unit/layers/test_prepare_attn_steps.py +++ b/test/registered/unit/layers/test_prepare_attn_steps.py @@ -42,7 +42,7 @@ class Norm: def communicator(norm): c = comm.LayerCommunicator.__new__(comm.LayerCommunicator) c.input_layernorm = norm - c._sp_variant = None + c._sp_region = False c.qkv_latent_func = None c._context = None c.enable_fused_ar_quant = False diff --git a/test/registered/unit/lora/test_triton_dp_attention_unit.py b/test/registered/unit/lora/test_triton_dp_attention_unit.py index ac8ea4f9d9..586bad4450 100644 --- a/test/registered/unit/lora/test_triton_dp_attention_unit.py +++ b/test/registered/unit/lora/test_triton_dp_attention_unit.py @@ -222,7 +222,7 @@ def test_communicator_publishes_layout_at_each_transition( communicator._publish_lora_layout = publish_lora_layout communicator.layer_scatter_modes = SimpleNamespace(mlp_mode=mlp_mode) communicator._context = SimpleNamespace() - communicator._sp_variant = None + communicator._sp_region = False communicator.post_attention_layernorm = None communicator.input_layernorm = lambda x: x communicator.qkv_latent_func = None