[Refactor] Run the LayerNorm SP region's boundary steps in the communicator itself (#41255)

This commit is contained in:
Cheng Wan
2026-09-26 02:40:34 -07:00
committed by GitHub
parent 35b369ebba
commit d3fcf8997c
7 changed files with 150 additions and 66 deletions
+34 -58
View File
@@ -444,6 +444,8 @@ class LayerScatterModes:
middle_residual_mode: ScatterMode
layer_output_mode: ScatterMode
is_layer_sparse: bool = False
# The model's first layer: its input is the embedding, not a layer output.
is_first_layer: bool = False
# The model's last layer: its output goes to the final norm, not a next layer.
is_last_layer: bool = False
@@ -457,6 +459,7 @@ class LayerScatterModes:
middle_residual_mode=cls._compute_middle_residual_mode(context),
layer_output_mode=cls._compute_layer_output_mode(context),
is_layer_sparse=context.is_layer_sparse,
is_first_layer=context.layer_id == 0,
is_last_layer=context.layer_id == context.num_layers - 1,
)
@@ -719,7 +722,6 @@ class LayerCommunicator:
fused_ar_quant_keep_bf16: bool = False,
# False for a layer whose FFN always completes its own reduction.
allow_deferred_ffn_reduction: bool = True,
_is_sp_variant: bool = False,
):
self.layer_scatter_modes = layer_scatter_modes
self.input_layernorm = input_layernorm
@@ -754,30 +756,9 @@ class LayerCommunicator:
get_lora().enable_lora
)
# Under LayerNorm SP the norm/residual run on the sequence shard with no
# collectives, so delegate to an all-SCATTERED sibling while the region is
# active. _is_sp_variant stops the sibling from building its own.
self._sp_variant: Optional[LayerCommunicator] = None
if not _is_sp_variant and layernorm_sp.layernorm_sp_enabled():
self._sp_variant = LayerCommunicator(
layer_scatter_modes=LayerScatterModes(
layer_input_mode=ScatterMode.SCATTERED,
attn_mode=ScatterMode.SCATTERED,
mlp_mode=ScatterMode.SCATTERED,
middle_residual_mode=ScatterMode.SCATTERED,
layer_output_mode=ScatterMode.SCATTERED,
is_last_layer=layer_scatter_modes.is_last_layer,
),
input_layernorm=input_layernorm,
post_attention_layernorm=post_attention_layernorm,
allow_reduce_scatter=allow_reduce_scatter,
qkv_latent_func=qkv_latent_func,
force_layernorm_before_dp_gather=force_layernorm_before_dp_gather,
enable_fused_ar_quant=enable_fused_ar_quant,
fused_ar_quant_keep_bf16=fused_ar_quant_keep_bf16,
allow_deferred_ffn_reduction=allow_deferred_ffn_reduction,
_is_sp_variant=True,
)
# Under LayerNorm SP the residual and norms run on this rank's sequence
# shard with no collectives while the region is active.
self._sp_region = layernorm_sp.layernorm_sp_enabled()
def _post_init_communicate(self):
self._communicate_simple_fn = CommunicateSimpleFn.get_fn(
@@ -891,24 +872,14 @@ class LayerCommunicator:
forward_batch=forward_batch,
)
hidden_states = owed.partial
# residual is None marks the first decoder layer, where the SP region
# opens: re-evaluated per forward so a crash mid-loop cannot leak into
# the next one.
if self._sp_variant is not None:
if residual is None:
get_forward().set(
"sp_active", forward_batch.forward_mode == ForwardMode.EXTEND
)
if get_forward().sp_active:
hidden_states = layernorm_sp.sp_entry_scatter(hidden_states)
# The SP region opens at the first layer, re-evaluated per forward so a
# crash mid-loop cannot leak into the next one.
if self._sp_region and self.layer_scatter_modes.is_first_layer:
get_forward().set(
"sp_active", layernorm_sp.runs_sp(forward_batch.forward_mode)
)
if get_forward().sp_active:
return self._sp_variant.prepare_attn(
hidden_states,
residual,
forward_batch,
quant_format,
post_residual_addition,
)
hidden_states = layernorm_sp.sp_entry_scatter(hidden_states)
if get_attn_tp_context().input_scattered:
hidden_states, residual = self._tp_reduce_scatter(
hidden_states,
@@ -932,13 +903,18 @@ class LayerCommunicator:
forward_batch=forward_batch,
)
def _in_sp_region(self) -> bool:
return self._sp_region and get_forward().sp_active
def _finish_prepare_attn(self, hidden_states, residual, forward_batch):
"""Tail every prepare_attn path must run, or ``attn_inputs`` is unset."""
hidden_states = self._communicate_simple_fn(
hidden_states=hidden_states,
forward_batch=forward_batch,
context=self._context,
)
"""Tail every prepare_attn path must run, or ``attn_inputs`` is unset. In
the SP region the attention input stays on this rank's sequence shard."""
if not self._in_sp_region():
hidden_states = self._communicate_simple_fn(
hidden_states=hidden_states,
forward_batch=forward_batch,
context=self._context,
)
if self.qkv_latent_func is not None:
attn_inputs = AttentionInputs(
hidden_states, forward_batch, self.qkv_latent_func
@@ -1082,9 +1058,13 @@ class LayerCommunicator:
cache=None,
):
self.publish_mlp_lora_layout()
if self._sp_variant is not None and get_forward().sp_active:
return self._sp_variant.prepare_mlp(
hidden_states, residual, forward_batch, cache
if self._in_sp_region():
return _mlp_input_norm(
hidden_states,
residual,
forward_batch,
self.post_attention_layernorm,
self._context,
)
if cache is not None:
self._context.cache = cache
@@ -1110,10 +1090,8 @@ class LayerCommunicator:
residual: torch.Tensor,
forward_batch: ForwardBatch,
):
if self._sp_variant is not None and get_forward().sp_active:
return self._sp_variant.postprocess_layer(
hidden_states, residual, forward_batch
)
if self._in_sp_region():
return hidden_states, residual
return self._communicate_summable_tensor_pair_fn(
hidden_states=hidden_states,
residual=residual,
@@ -1127,9 +1105,7 @@ class LayerCommunicator:
"""Whether the next layer's input can run this layer's move of its FFN
output back to this rank's tokens: the base postprocess scatter, outside
an active LayerNorm SP region."""
return self._postprocess_scatters_to_local_tokens and not (
self._sp_variant is not None and get_forward().sp_active
)
return self._postprocess_scatters_to_local_tokens and not self._in_sp_region()
def _postprocess_dp_step(
self, forward_batch: ForwardBatch
@@ -48,7 +48,7 @@ def make_communicator(
return_value=reduce_scatter
)
communicator.is_last_layer = False
communicator._sp_variant = None
communicator._sp_region = False
communicator._postprocess_scatters_to_local_tokens = scatters_to_local_tokens
communicator._postprocess_dp_step = MagicMock(return_value=reduce_scatter_step)
communicator.ffn_reduction_group = MagicMock(return_value=group or make_group())
@@ -277,12 +277,12 @@ class TestSelectFfnCompletion(CustomTestCase):
the FFN runs."""
def communicator(
self, *, fuse=False, is_last_layer=False, scatters=True, sp_variant=None
self, *, fuse=False, is_last_layer=False, scatters=True, sp_region=False
):
communicator = LayerCommunicator.__new__(LayerCommunicator)
communicator.allow_deferred_ffn_reduction = True
communicator.is_last_layer = is_last_layer
communicator._sp_variant = sp_variant
communicator._sp_region = sp_region
communicator._postprocess_scatters_to_local_tokens = scatters
communicator.allow_reduce_scatter = True
communicator.layer_scatter_modes = types.SimpleNamespace(is_layer_sparse=True)
@@ -326,7 +326,7 @@ class TestSelectFfnCompletion(CustomTestCase):
self.assertIsNone(self.left(communicator, step))
def test_postprocess_keeps_an_active_layernorm_sp_region(self):
communicator = self.communicator(sp_variant=object())
communicator = self.communicator(sp_region=True)
with get_forward().scoped(sp_active=True):
self.assertIsNone(
self.left(communicator, comm._reduce_and_redistribute_output_varlen)
@@ -323,7 +323,7 @@ class TestDeferFfnReduction(CustomTestCase):
communicator = _fake_communicator()
communicator.is_last_layer = is_last_layer
communicator._postprocess_scatters_to_local_tokens = scatters_to_local_tokens
communicator._sp_variant = object() if sp_active else None
communicator._sp_region = sp_active
communicator.allow_reduce_scatter = True
communicator.layer_scatter_modes.is_layer_sparse = True
forward_batch = types.SimpleNamespace(
@@ -7,8 +7,12 @@ fused matmul fast-paths need a real TP group and are covered by the e2e test.
import unittest
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import torch
from sglang.srt.arg_groups.layernorm_sp_hook import validate_layernorm_sp
from sglang.srt.layers import communicator as comm
from sglang.srt.layers import layernorm_sp
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.runtime_context import (
@@ -129,5 +133,109 @@ class TestLayerNormSPActiveFlag(CustomTestCase):
self.assertFalse(get_forward().sp_active)
class _Norm:
def __call__(self, x, residual=None, post_residual_addition=None):
if residual is None:
return x * 2
s = x + residual
return s * 2, s
class TestSpRegionSteps(CustomTestCase):
"""The communicator opens the SP region at the model's first layer on a
prefill forward and, inside it, keeps every boundary on this rank's sequence
shard: add + norm, no move, no collective."""
def communicator(self, *, first_layer):
c = comm.LayerCommunicator.__new__(comm.LayerCommunicator)
c._sp_region = True
c.layer_scatter_modes = SimpleNamespace(is_first_layer=first_layer)
c._attn_input_fusions = ()
c.input_layernorm = _Norm()
c.post_attention_layernorm = _Norm()
c.qkv_latent_func = None
c._context = SimpleNamespace(cache=None)
c._communicate_simple_fn = MagicMock(side_effect=AssertionError("moved"))
c._communicate_summable_tensor_pair_fn = MagicMock(
side_effect=AssertionError("postprocess ran")
)
return c
def run_prepare_attn(self, communicator, mode, hidden, residual):
batch = SimpleNamespace(forward_mode=mode)
with (
get_flags().sp.override(enabled=True),
get_forward().scoped(sp_active=False),
patch.object(
comm,
"get_attn_tp_context",
return_value=SimpleNamespace(input_scattered=False),
),
patch.object(
layernorm_sp, "sp_entry_scatter", side_effect=lambda h: h[:1]
) as scatter,
):
out = communicator.prepare_attn(hidden, residual, batch)
return out, get_forward().sp_active, scatter
def test_the_first_layer_opens_the_region_on_prefill(self):
hidden = torch.ones(2, 4)
(h, r), active, scatter = self.run_prepare_attn(
self.communicator(first_layer=True), ForwardMode.EXTEND, hidden, None
)
self.assertTrue(active)
scatter.assert_called_once()
torch.testing.assert_close(h, torch.full((1, 4), 2.0))
torch.testing.assert_close(r, torch.ones(1, 4))
def test_decode_leaves_the_region_closed(self):
communicator = self.communicator(first_layer=True)
communicator._communicate_simple_fn = MagicMock(
side_effect=lambda **k: k["hidden_states"]
)
(h, _), active, scatter = self.run_prepare_attn(
communicator, ForwardMode.DECODE, torch.ones(2, 4), None
)
self.assertFalse(active)
scatter.assert_not_called()
communicator._communicate_simple_fn.assert_called_once()
def test_a_later_layer_does_not_reopen_the_region(self):
communicator = self.communicator(first_layer=False)
communicator._communicate_simple_fn = MagicMock(
side_effect=lambda **k: k["hidden_states"]
)
_, active, scatter = self.run_prepare_attn(
communicator, ForwardMode.EXTEND, torch.ones(2, 4), torch.ones(2, 4)
)
self.assertFalse(active)
scatter.assert_not_called()
def test_inside_the_region_each_boundary_stays_on_the_shard(self):
communicator = self.communicator(first_layer=False)
hidden, residual = torch.ones(1, 4), torch.full((1, 4), 3.0)
with (
get_flags().sp.override(enabled=True),
get_forward().scoped(sp_active=True),
):
with patch.object(
comm,
"get_attn_tp_context",
return_value=SimpleNamespace(input_scattered=False),
):
h, r = communicator.prepare_attn(
hidden,
residual.clone(),
SimpleNamespace(forward_mode=ForwardMode.EXTEND),
)
torch.testing.assert_close(h, torch.full((1, 4), 8.0))
h, r = communicator.prepare_mlp(hidden, residual.clone(), object())
torch.testing.assert_close(h, torch.full((1, 4), 8.0))
torch.testing.assert_close(r, torch.full((1, 4), 4.0))
out = communicator.postprocess_layer(hidden, residual, object())
self.assertIs(out[0], hidden)
self.assertIs(out[1], residual)
if __name__ == "__main__":
unittest.main()
@@ -140,7 +140,7 @@ class TestPostprocessReduceScatterv(CustomTestCase):
def test_postprocess_passes_the_layer_sparsity(self):
seen = {}
communicator = LayerCommunicator.__new__(LayerCommunicator)
communicator._sp_variant = None
communicator._sp_region = False
communicator._context = None
communicator.allow_reduce_scatter = False
communicator.layer_scatter_modes = types.SimpleNamespace(is_layer_sparse=True)
@@ -42,7 +42,7 @@ class Norm:
def communicator(norm):
c = comm.LayerCommunicator.__new__(comm.LayerCommunicator)
c.input_layernorm = norm
c._sp_variant = None
c._sp_region = False
c.qkv_latent_func = None
c._context = None
c.enable_fused_ar_quant = False
@@ -222,7 +222,7 @@ def test_communicator_publishes_layout_at_each_transition(
communicator._publish_lora_layout = publish_lora_layout
communicator.layer_scatter_modes = SimpleNamespace(mlp_mode=mlp_mode)
communicator._context = SimpleNamespace()
communicator._sp_variant = None
communicator._sp_region = False
communicator.post_attention_layernorm = None
communicator.input_layernorm = lambda x: x
communicator.qkv_latent_func = None