[Refactor] Run the LayerNorm SP region's boundary steps in the communicator itself (#41255)
This commit is contained in:
@@ -444,6 +444,8 @@ class LayerScatterModes:
|
||||
middle_residual_mode: ScatterMode
|
||||
layer_output_mode: ScatterMode
|
||||
is_layer_sparse: bool = False
|
||||
# The model's first layer: its input is the embedding, not a layer output.
|
||||
is_first_layer: bool = False
|
||||
# The model's last layer: its output goes to the final norm, not a next layer.
|
||||
is_last_layer: bool = False
|
||||
|
||||
@@ -457,6 +459,7 @@ class LayerScatterModes:
|
||||
middle_residual_mode=cls._compute_middle_residual_mode(context),
|
||||
layer_output_mode=cls._compute_layer_output_mode(context),
|
||||
is_layer_sparse=context.is_layer_sparse,
|
||||
is_first_layer=context.layer_id == 0,
|
||||
is_last_layer=context.layer_id == context.num_layers - 1,
|
||||
)
|
||||
|
||||
@@ -719,7 +722,6 @@ class LayerCommunicator:
|
||||
fused_ar_quant_keep_bf16: bool = False,
|
||||
# False for a layer whose FFN always completes its own reduction.
|
||||
allow_deferred_ffn_reduction: bool = True,
|
||||
_is_sp_variant: bool = False,
|
||||
):
|
||||
self.layer_scatter_modes = layer_scatter_modes
|
||||
self.input_layernorm = input_layernorm
|
||||
@@ -754,30 +756,9 @@ class LayerCommunicator:
|
||||
get_lora().enable_lora
|
||||
)
|
||||
|
||||
# Under LayerNorm SP the norm/residual run on the sequence shard with no
|
||||
# collectives, so delegate to an all-SCATTERED sibling while the region is
|
||||
# active. _is_sp_variant stops the sibling from building its own.
|
||||
self._sp_variant: Optional[LayerCommunicator] = None
|
||||
if not _is_sp_variant and layernorm_sp.layernorm_sp_enabled():
|
||||
self._sp_variant = LayerCommunicator(
|
||||
layer_scatter_modes=LayerScatterModes(
|
||||
layer_input_mode=ScatterMode.SCATTERED,
|
||||
attn_mode=ScatterMode.SCATTERED,
|
||||
mlp_mode=ScatterMode.SCATTERED,
|
||||
middle_residual_mode=ScatterMode.SCATTERED,
|
||||
layer_output_mode=ScatterMode.SCATTERED,
|
||||
is_last_layer=layer_scatter_modes.is_last_layer,
|
||||
),
|
||||
input_layernorm=input_layernorm,
|
||||
post_attention_layernorm=post_attention_layernorm,
|
||||
allow_reduce_scatter=allow_reduce_scatter,
|
||||
qkv_latent_func=qkv_latent_func,
|
||||
force_layernorm_before_dp_gather=force_layernorm_before_dp_gather,
|
||||
enable_fused_ar_quant=enable_fused_ar_quant,
|
||||
fused_ar_quant_keep_bf16=fused_ar_quant_keep_bf16,
|
||||
allow_deferred_ffn_reduction=allow_deferred_ffn_reduction,
|
||||
_is_sp_variant=True,
|
||||
)
|
||||
# Under LayerNorm SP the residual and norms run on this rank's sequence
|
||||
# shard with no collectives while the region is active.
|
||||
self._sp_region = layernorm_sp.layernorm_sp_enabled()
|
||||
|
||||
def _post_init_communicate(self):
|
||||
self._communicate_simple_fn = CommunicateSimpleFn.get_fn(
|
||||
@@ -891,24 +872,14 @@ class LayerCommunicator:
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
hidden_states = owed.partial
|
||||
# residual is None marks the first decoder layer, where the SP region
|
||||
# opens: re-evaluated per forward so a crash mid-loop cannot leak into
|
||||
# the next one.
|
||||
if self._sp_variant is not None:
|
||||
if residual is None:
|
||||
get_forward().set(
|
||||
"sp_active", forward_batch.forward_mode == ForwardMode.EXTEND
|
||||
)
|
||||
if get_forward().sp_active:
|
||||
hidden_states = layernorm_sp.sp_entry_scatter(hidden_states)
|
||||
# The SP region opens at the first layer, re-evaluated per forward so a
|
||||
# crash mid-loop cannot leak into the next one.
|
||||
if self._sp_region and self.layer_scatter_modes.is_first_layer:
|
||||
get_forward().set(
|
||||
"sp_active", layernorm_sp.runs_sp(forward_batch.forward_mode)
|
||||
)
|
||||
if get_forward().sp_active:
|
||||
return self._sp_variant.prepare_attn(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
quant_format,
|
||||
post_residual_addition,
|
||||
)
|
||||
hidden_states = layernorm_sp.sp_entry_scatter(hidden_states)
|
||||
if get_attn_tp_context().input_scattered:
|
||||
hidden_states, residual = self._tp_reduce_scatter(
|
||||
hidden_states,
|
||||
@@ -932,13 +903,18 @@ class LayerCommunicator:
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
def _in_sp_region(self) -> bool:
|
||||
return self._sp_region and get_forward().sp_active
|
||||
|
||||
def _finish_prepare_attn(self, hidden_states, residual, forward_batch):
|
||||
"""Tail every prepare_attn path must run, or ``attn_inputs`` is unset."""
|
||||
hidden_states = self._communicate_simple_fn(
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
context=self._context,
|
||||
)
|
||||
"""Tail every prepare_attn path must run, or ``attn_inputs`` is unset. In
|
||||
the SP region the attention input stays on this rank's sequence shard."""
|
||||
if not self._in_sp_region():
|
||||
hidden_states = self._communicate_simple_fn(
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
context=self._context,
|
||||
)
|
||||
if self.qkv_latent_func is not None:
|
||||
attn_inputs = AttentionInputs(
|
||||
hidden_states, forward_batch, self.qkv_latent_func
|
||||
@@ -1082,9 +1058,13 @@ class LayerCommunicator:
|
||||
cache=None,
|
||||
):
|
||||
self.publish_mlp_lora_layout()
|
||||
if self._sp_variant is not None and get_forward().sp_active:
|
||||
return self._sp_variant.prepare_mlp(
|
||||
hidden_states, residual, forward_batch, cache
|
||||
if self._in_sp_region():
|
||||
return _mlp_input_norm(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
self.post_attention_layernorm,
|
||||
self._context,
|
||||
)
|
||||
if cache is not None:
|
||||
self._context.cache = cache
|
||||
@@ -1110,10 +1090,8 @@ class LayerCommunicator:
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
):
|
||||
if self._sp_variant is not None and get_forward().sp_active:
|
||||
return self._sp_variant.postprocess_layer(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
if self._in_sp_region():
|
||||
return hidden_states, residual
|
||||
return self._communicate_summable_tensor_pair_fn(
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
@@ -1127,9 +1105,7 @@ class LayerCommunicator:
|
||||
"""Whether the next layer's input can run this layer's move of its FFN
|
||||
output back to this rank's tokens: the base postprocess scatter, outside
|
||||
an active LayerNorm SP region."""
|
||||
return self._postprocess_scatters_to_local_tokens and not (
|
||||
self._sp_variant is not None and get_forward().sp_active
|
||||
)
|
||||
return self._postprocess_scatters_to_local_tokens and not self._in_sp_region()
|
||||
|
||||
def _postprocess_dp_step(
|
||||
self, forward_batch: ForwardBatch
|
||||
|
||||
@@ -48,7 +48,7 @@ def make_communicator(
|
||||
return_value=reduce_scatter
|
||||
)
|
||||
communicator.is_last_layer = False
|
||||
communicator._sp_variant = None
|
||||
communicator._sp_region = False
|
||||
communicator._postprocess_scatters_to_local_tokens = scatters_to_local_tokens
|
||||
communicator._postprocess_dp_step = MagicMock(return_value=reduce_scatter_step)
|
||||
communicator.ffn_reduction_group = MagicMock(return_value=group or make_group())
|
||||
@@ -277,12 +277,12 @@ class TestSelectFfnCompletion(CustomTestCase):
|
||||
the FFN runs."""
|
||||
|
||||
def communicator(
|
||||
self, *, fuse=False, is_last_layer=False, scatters=True, sp_variant=None
|
||||
self, *, fuse=False, is_last_layer=False, scatters=True, sp_region=False
|
||||
):
|
||||
communicator = LayerCommunicator.__new__(LayerCommunicator)
|
||||
communicator.allow_deferred_ffn_reduction = True
|
||||
communicator.is_last_layer = is_last_layer
|
||||
communicator._sp_variant = sp_variant
|
||||
communicator._sp_region = sp_region
|
||||
communicator._postprocess_scatters_to_local_tokens = scatters
|
||||
communicator.allow_reduce_scatter = True
|
||||
communicator.layer_scatter_modes = types.SimpleNamespace(is_layer_sparse=True)
|
||||
@@ -326,7 +326,7 @@ class TestSelectFfnCompletion(CustomTestCase):
|
||||
self.assertIsNone(self.left(communicator, step))
|
||||
|
||||
def test_postprocess_keeps_an_active_layernorm_sp_region(self):
|
||||
communicator = self.communicator(sp_variant=object())
|
||||
communicator = self.communicator(sp_region=True)
|
||||
with get_forward().scoped(sp_active=True):
|
||||
self.assertIsNone(
|
||||
self.left(communicator, comm._reduce_and_redistribute_output_varlen)
|
||||
|
||||
@@ -323,7 +323,7 @@ class TestDeferFfnReduction(CustomTestCase):
|
||||
communicator = _fake_communicator()
|
||||
communicator.is_last_layer = is_last_layer
|
||||
communicator._postprocess_scatters_to_local_tokens = scatters_to_local_tokens
|
||||
communicator._sp_variant = object() if sp_active else None
|
||||
communicator._sp_region = sp_active
|
||||
communicator.allow_reduce_scatter = True
|
||||
communicator.layer_scatter_modes.is_layer_sparse = True
|
||||
forward_batch = types.SimpleNamespace(
|
||||
|
||||
@@ -7,8 +7,12 @@ fused matmul fast-paths need a real TP group and are covered by the e2e test.
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.arg_groups.layernorm_sp_hook import validate_layernorm_sp
|
||||
from sglang.srt.layers import communicator as comm
|
||||
from sglang.srt.layers import layernorm_sp
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.runtime_context import (
|
||||
@@ -129,5 +133,109 @@ class TestLayerNormSPActiveFlag(CustomTestCase):
|
||||
self.assertFalse(get_forward().sp_active)
|
||||
|
||||
|
||||
class _Norm:
|
||||
def __call__(self, x, residual=None, post_residual_addition=None):
|
||||
if residual is None:
|
||||
return x * 2
|
||||
s = x + residual
|
||||
return s * 2, s
|
||||
|
||||
|
||||
class TestSpRegionSteps(CustomTestCase):
|
||||
"""The communicator opens the SP region at the model's first layer on a
|
||||
prefill forward and, inside it, keeps every boundary on this rank's sequence
|
||||
shard: add + norm, no move, no collective."""
|
||||
|
||||
def communicator(self, *, first_layer):
|
||||
c = comm.LayerCommunicator.__new__(comm.LayerCommunicator)
|
||||
c._sp_region = True
|
||||
c.layer_scatter_modes = SimpleNamespace(is_first_layer=first_layer)
|
||||
c._attn_input_fusions = ()
|
||||
c.input_layernorm = _Norm()
|
||||
c.post_attention_layernorm = _Norm()
|
||||
c.qkv_latent_func = None
|
||||
c._context = SimpleNamespace(cache=None)
|
||||
c._communicate_simple_fn = MagicMock(side_effect=AssertionError("moved"))
|
||||
c._communicate_summable_tensor_pair_fn = MagicMock(
|
||||
side_effect=AssertionError("postprocess ran")
|
||||
)
|
||||
return c
|
||||
|
||||
def run_prepare_attn(self, communicator, mode, hidden, residual):
|
||||
batch = SimpleNamespace(forward_mode=mode)
|
||||
with (
|
||||
get_flags().sp.override(enabled=True),
|
||||
get_forward().scoped(sp_active=False),
|
||||
patch.object(
|
||||
comm,
|
||||
"get_attn_tp_context",
|
||||
return_value=SimpleNamespace(input_scattered=False),
|
||||
),
|
||||
patch.object(
|
||||
layernorm_sp, "sp_entry_scatter", side_effect=lambda h: h[:1]
|
||||
) as scatter,
|
||||
):
|
||||
out = communicator.prepare_attn(hidden, residual, batch)
|
||||
return out, get_forward().sp_active, scatter
|
||||
|
||||
def test_the_first_layer_opens_the_region_on_prefill(self):
|
||||
hidden = torch.ones(2, 4)
|
||||
(h, r), active, scatter = self.run_prepare_attn(
|
||||
self.communicator(first_layer=True), ForwardMode.EXTEND, hidden, None
|
||||
)
|
||||
self.assertTrue(active)
|
||||
scatter.assert_called_once()
|
||||
torch.testing.assert_close(h, torch.full((1, 4), 2.0))
|
||||
torch.testing.assert_close(r, torch.ones(1, 4))
|
||||
|
||||
def test_decode_leaves_the_region_closed(self):
|
||||
communicator = self.communicator(first_layer=True)
|
||||
communicator._communicate_simple_fn = MagicMock(
|
||||
side_effect=lambda **k: k["hidden_states"]
|
||||
)
|
||||
(h, _), active, scatter = self.run_prepare_attn(
|
||||
communicator, ForwardMode.DECODE, torch.ones(2, 4), None
|
||||
)
|
||||
self.assertFalse(active)
|
||||
scatter.assert_not_called()
|
||||
communicator._communicate_simple_fn.assert_called_once()
|
||||
|
||||
def test_a_later_layer_does_not_reopen_the_region(self):
|
||||
communicator = self.communicator(first_layer=False)
|
||||
communicator._communicate_simple_fn = MagicMock(
|
||||
side_effect=lambda **k: k["hidden_states"]
|
||||
)
|
||||
_, active, scatter = self.run_prepare_attn(
|
||||
communicator, ForwardMode.EXTEND, torch.ones(2, 4), torch.ones(2, 4)
|
||||
)
|
||||
self.assertFalse(active)
|
||||
scatter.assert_not_called()
|
||||
|
||||
def test_inside_the_region_each_boundary_stays_on_the_shard(self):
|
||||
communicator = self.communicator(first_layer=False)
|
||||
hidden, residual = torch.ones(1, 4), torch.full((1, 4), 3.0)
|
||||
with (
|
||||
get_flags().sp.override(enabled=True),
|
||||
get_forward().scoped(sp_active=True),
|
||||
):
|
||||
with patch.object(
|
||||
comm,
|
||||
"get_attn_tp_context",
|
||||
return_value=SimpleNamespace(input_scattered=False),
|
||||
):
|
||||
h, r = communicator.prepare_attn(
|
||||
hidden,
|
||||
residual.clone(),
|
||||
SimpleNamespace(forward_mode=ForwardMode.EXTEND),
|
||||
)
|
||||
torch.testing.assert_close(h, torch.full((1, 4), 8.0))
|
||||
h, r = communicator.prepare_mlp(hidden, residual.clone(), object())
|
||||
torch.testing.assert_close(h, torch.full((1, 4), 8.0))
|
||||
torch.testing.assert_close(r, torch.full((1, 4), 4.0))
|
||||
out = communicator.postprocess_layer(hidden, residual, object())
|
||||
self.assertIs(out[0], hidden)
|
||||
self.assertIs(out[1], residual)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -140,7 +140,7 @@ class TestPostprocessReduceScatterv(CustomTestCase):
|
||||
def test_postprocess_passes_the_layer_sparsity(self):
|
||||
seen = {}
|
||||
communicator = LayerCommunicator.__new__(LayerCommunicator)
|
||||
communicator._sp_variant = None
|
||||
communicator._sp_region = False
|
||||
communicator._context = None
|
||||
communicator.allow_reduce_scatter = False
|
||||
communicator.layer_scatter_modes = types.SimpleNamespace(is_layer_sparse=True)
|
||||
|
||||
@@ -42,7 +42,7 @@ class Norm:
|
||||
def communicator(norm):
|
||||
c = comm.LayerCommunicator.__new__(comm.LayerCommunicator)
|
||||
c.input_layernorm = norm
|
||||
c._sp_variant = None
|
||||
c._sp_region = False
|
||||
c.qkv_latent_func = None
|
||||
c._context = None
|
||||
c.enable_fused_ar_quant = False
|
||||
|
||||
@@ -222,7 +222,7 @@ def test_communicator_publishes_layout_at_each_transition(
|
||||
communicator._publish_lora_layout = publish_lora_layout
|
||||
communicator.layer_scatter_modes = SimpleNamespace(mlp_mode=mlp_mode)
|
||||
communicator._context = SimpleNamespace()
|
||||
communicator._sp_variant = None
|
||||
communicator._sp_region = False
|
||||
communicator.post_attention_layernorm = None
|
||||
communicator.input_layernorm = lambda x: x
|
||||
communicator.qkv_latent_func = None
|
||||
|
||||
Reference in New Issue
Block a user