[Refactor] Choose the LayerNorm SP region's and input-scattered batches' steps from declarations (#41419)

This commit is contained in:
Cheng Wan
2026-09-27 16:47:32 -07:00
committed by GitHub
parent 846181f1c4
commit 9f87d28db6
11 changed files with 514 additions and 70 deletions
@@ -100,6 +100,76 @@ class DecoderLayerSides(msgspec.Struct, frozen=True):
ffn_residual_rows: Layout
# The rows the layer hands on.
output_rows: Layout
# The group the layer's input arrives as a partial sum over, if any.
input_owes: Optional[SumGroup] = None
# Whether the residual is added into one rank's share of the attention
# output's sum before that sum completes, instead of after it.
residual_joins_attention_sum: bool = False
def sequence_parallel_layer_sides(
*, axis_sizes: Mapping[TokenAxis, int]
) -> DecoderLayerSides:
"""A decoder layer while a LayerNorm SP region is active. Its linears
all-gather their input and reduce-scatter their output themselves, so the
layer's rows, its residual and both outputs stay on each TP rank's slice."""
attention = Layout.sharded_over(
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
)
local = Layout.sharded_over(
TokenAxis.ATTN_DP,
TokenAxis.ATTN_CP,
TokenAxis.ATTN_TP_SCATTER,
axis_sizes=axis_sizes,
)
gathers = frozenset({TokenAxis.ATTN_TP_SCATTER})
return DecoderLayerSides(
input_rows=local,
# qkv and gate_up gather the slices into their GEMM.
attention=StageInput(attention, gathers_itself=gathers),
# o_proj and down reduce-scatter out of theirs, whether fused or not.
attention_output=StageOutput(local),
ffn=StageInput(
Layout.sharded_over(axis_sizes=axis_sizes), gathers_itself=gathers
),
ffn_output=StageOutput(local),
ffn_residual_rows=local,
output_rows=local,
)
def input_scattered_layer_sides(
*,
axis_sizes: Mapping[TokenAxis, int],
ffn_group: SumGroup,
hands_on_partial: bool,
) -> DecoderLayerSides:
"""A decoder layer on a batch whose attention input is scattered over
attention TP. Each layer's input is a partial sum over TP on the full rows:
the embedding and every FFN before it leave that sum, and a reduce-scatter
completes it onto each rank's slice, which the attention gathers itself.
The residual comes back to the full rows inside the attention output's sum."""
attention = Layout.sharded_over(
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
)
return DecoderLayerSides(
input_rows=attention,
attention=StageInput(
attention, gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER})
),
attention_output=StageOutput(
attention, group=SumGroup.ATTN_TP, always_leaves=True
),
ffn=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)),
# The next layer's input completes the sum; the last layer's FFN does.
ffn_output=StageOutput(
attention, group=ffn_group, leaves_for_reduce_scatter=hands_on_partial
),
ffn_residual_rows=attention,
output_rows=attention,
input_owes=SumGroup.TP,
residual_joins_attention_sum=True,
)
def decoder_layer_sides(
+172 -45
View File
@@ -45,6 +45,8 @@ from sglang.srt.layers.boundary_layout import (
SumGroup,
TokenAxis,
decoder_layer_sides,
input_scattered_layer_sides,
sequence_parallel_layer_sides,
)
from sglang.srt.layers.cp.utils import (
is_mla_cp_active,
@@ -759,6 +761,20 @@ class LayerCommunicator:
self._select_declared_boundaries(sides)
else:
self._select_boundaries_from_scatter_modes()
# The steps a batch with input-scattered attention runs; None where it
# cannot run or the steps come from the scatter modes.
self._input_scattered_steps = (
_select_boundary_steps(
input_scattered_layer_sides(
axis_sizes=_token_axis_sizes(),
ffn_group=sides.ffn_output.group,
hands_on_partial=allow_reduce_scatter
and not layer_scatter_modes.is_last_layer,
)
)
if sides is not None and self._input_can_be_scattered()
else None
)
self._attn_input_fusions = self._select_attn_input_fusions()
self._speculative_algo = SpeculativeAlgorithm.from_string(
get_spec().speculative_algorithm
@@ -768,15 +784,36 @@ class LayerCommunicator:
get_lora().enable_lora
)
# Under LayerNorm SP the residual and norms run on this rank's sequence
# shard with no collectives while the region is active.
self._sp_region = layernorm_sp.layernorm_sp_enabled()
# Under LayerNorm SP, the steps the layer runs while the region is
# active; None without SP. The two are exclusive: SP runs a model without
# q_lora, which input-scattered attention needs.
self._sp_steps = (
_select_boundary_steps(
sequence_parallel_layer_sides(axis_sizes=_token_axis_sizes())
)
if layernorm_sp.layernorm_sp_enabled()
else None
)
def _input_can_be_scattered(self) -> bool:
"""Whether a batch may run this layer with input-scattered attention:
configured, on pure TP without an a2a backend. The rest of what
``AttnTpContext.init_context`` requires is only known once the model is
built."""
parallel = get_parallel()
return (
parallel.enable_attn_tp_input_scattered
and parallel.tp_size > 1
and parallel.attn_dp_size == 1
and parallel.attn_cp_size == 1
and get_moe_a2a_backend().is_none()
)
def _declared_sides(self) -> Optional[DecoderLayerSides]:
"""The declarations this layer's boundaries are chosen from: an
attention and an FFN, without CP, LayerNorm SP or input-scattered
attention, in a layer whose previous layer is known. None when the
steps come from the scatter modes."""
attention and an FFN, without CP, in a layer whose previous layer is
known. None when the steps come from the scatter modes. An active
LayerNorm SP region and input-scattered attention have their own."""
modes = self.layer_scatter_modes
parallel = get_parallel()
# A MoE dispatched per DP shard computes on this rank's local rows and
@@ -785,18 +822,12 @@ class LayerCommunicator:
if not (
self._takes_declared_boundaries
and parallel.attn_cp_size == 1
and not layernorm_sp.layernorm_sp_enabled()
and not parallel.enable_attn_tp_input_scattered
and not enable_moe_dense_fully_dp()
and (modes.is_first_layer or modes.is_previous_layer_sparse is not None)
):
return None
return decoder_layer_sides(
axis_sizes={
TokenAxis.ATTN_DP: parallel.attn_dp_size,
TokenAxis.ATTN_CP: parallel.attn_cp_size,
TokenAxis.ATTN_TP_SCATTER: parallel.attn_tp_size,
},
axis_sizes=_token_axis_sizes(),
ffn_on_local_rows=modes.is_layer_sparse and moe_on_local_rows,
previous_on_local_rows=(
not modes.is_first_layer
@@ -991,13 +1022,18 @@ class LayerCommunicator:
hidden_states = owed.partial
# The SP region opens at the first layer, re-evaluated per forward so a
# crash mid-loop cannot leak into the next one.
if self._sp_region and self.layer_scatter_modes.is_first_layer:
if self._sp_steps is not None and self.layer_scatter_modes.is_first_layer:
get_forward().set(
"sp_active", layernorm_sp.runs_sp(forward_batch.forward_mode)
)
if get_forward().sp_active:
hidden_states = layernorm_sp.sp_entry_scatter(hidden_states)
if get_attn_tp_context().input_scattered:
steps = self._batch_steps()
if steps is not None and steps.layer_input is not None:
hidden_states, residual = steps.layer_input(
hidden_states, residual, self._context
)
elif get_attn_tp_context().input_scattered:
hidden_states, residual = self._tp_reduce_scatter(
hidden_states,
residual,
@@ -1021,17 +1057,29 @@ class LayerCommunicator:
)
def _in_sp_region(self) -> bool:
return self._sp_region and get_forward().sp_active
return self._sp_steps is not None and get_forward().sp_active
def _batch_steps(self) -> Optional["BoundarySteps"]:
"""The steps this batch runs in place of the layer's ordinary ones: an
active LayerNorm SP region's, or input-scattered attention's."""
if self._in_sp_region():
return self._sp_steps
if (
self._input_scattered_steps is not None
and get_attn_tp_context().input_scattered
):
return self._input_scattered_steps
return None
def _finish_prepare_attn(self, hidden_states, residual, forward_batch):
"""Tail every prepare_attn path must run, or ``attn_inputs`` is unset. In
the SP region the attention input stays on this rank's sequence shard."""
if not self._in_sp_region():
hidden_states = self._communicate_simple_fn(
hidden_states=hidden_states,
forward_batch=forward_batch,
context=self._context,
)
"""Tail every prepare_attn path must run, or ``attn_inputs`` is unset."""
steps = self._batch_steps()
move = self._communicate_simple_fn if steps is None else steps.attention_input
hidden_states = move(
hidden_states=hidden_states,
forward_batch=forward_batch,
context=self._context,
)
if self.qkv_latent_func is not None:
attn_inputs = AttentionInputs(
hidden_states, forward_batch, self.qkv_latent_func
@@ -1175,7 +1223,7 @@ class LayerCommunicator:
):
self.publish_mlp_lora_layout()
if self._in_sp_region():
return _mlp_input_norm(
return self._sp_steps.ffn_input(
hidden_states,
residual,
forward_batch,
@@ -1185,7 +1233,9 @@ class LayerCommunicator:
if cache is not None:
self._context.cache = cache
return self._mlp_input(
steps = self._batch_steps()
mlp_input = self._mlp_input if steps is None else steps.ffn_input
return mlp_input(
hidden_states,
residual,
forward_batch,
@@ -1206,8 +1256,14 @@ class LayerCommunicator:
residual: torch.Tensor,
forward_batch: ForwardBatch,
):
if self._in_sp_region():
return hidden_states, residual
steps = self._batch_steps()
if steps is not None:
return steps.ffn_output_move(
hidden_states=hidden_states,
residual=residual,
forward_batch=forward_batch,
context=self._context,
)
if self._postprocess_scatters_to_local_tokens:
step = self._postprocess_dp_step(forward_batch) or _redistribute_output
return _to_local_tokens(step, forward_batch, hidden_states), residual
@@ -1246,6 +1302,9 @@ class LayerCommunicator:
"""Whether the FFN leaves its sum out because a reduce-scatter completes
it: the attention-DP one ``dp_step`` names, or the CP / input-scattered
one."""
steps = self._batch_steps()
if steps is not None:
return steps.ffn_output.leaves_for_reduce_scatter
if not self._ffn_output.leaves_for_reduce_scatter:
return False
if dp_step is not None:
@@ -1274,7 +1333,9 @@ class LayerCommunicator:
forward_batch=forward_batch,
dp_step=dp_step,
)
if not self._ffn_output.leaves_for_next_layer:
steps = self._batch_steps()
ffn_output = self._ffn_output if steps is None else steps.ffn_output
if not ffn_output.leaves_for_next_layer:
return FfnCompletion(
defer_moe_finalize=False,
fuse_mlp_allreduce=False,
@@ -1843,6 +1904,70 @@ def _sum_group(group: SumGroup) -> GroupCoordinator:
return post_experts_reduction_group()
def _token_axis_sizes() -> Dict[TokenAxis, int]:
parallel = get_parallel()
return {
TokenAxis.ATTN_DP: parallel.attn_dp_size,
TokenAxis.ATTN_CP: parallel.attn_cp_size,
TokenAxis.ATTN_TP_SCATTER: parallel.attn_tp_size,
}
class BoundarySteps(msgspec.Struct, frozen=True):
"""The steps one set of declarations chooses for a layer's boundaries:
into the attention, from the attention output to the FFN input, and the
FFN output on to the rows the layer hands on."""
attention_input: Callable
ffn_input: Callable
ffn_output_move: Callable
# What the FFN exit reads in place of the layer's own FFN output declaration.
ffn_output: StageOutput
# Completes what the layer's input owes before the input norm; None when it
# owes nothing.
layer_input: Optional[Callable] = None
def _select_boundary_steps(sides: DecoderLayerSides) -> BoundarySteps:
"""The steps for a batch's own declarations: nothing moves back over
attention DP and no fused kernel runs, so the steps do not wait on a
per-batch decision."""
returns_over_dp, ffn_output_move = _select_ffn_output_move(
sides.ffn_output, residual=sides.ffn_residual_rows, to=sides.output_rows
)
if returns_over_dp or sides.ffn_output.leaves_for_next_layer:
raise NotImplementedError(f"{sides=}")
rows = sides.input_rows
layer_input = None
if sides.input_owes is not None:
# A reduce-scatter completes the TP sum onto each rank's slice, which the
# attention takes: its group is the TP group without attention DP or CP.
if (
sides.input_owes is not SumGroup.TP
or rows.sharded
or TokenAxis.ATTN_TP_SCATTER not in sides.attention.gathers_itself
):
raise NotImplementedError(f"{sides=}")
layer_input = tp_reduce_scatter
rows = Layout(frozenset({TokenAxis.ATTN_TP_SCATTER}))
ffn_input, _ = _select_ffn_input(
sides.attention_output,
residual=rows,
residual_to=sides.ffn_residual_rows,
need=sides.ffn,
force_layernorm_before_gather=False,
fusions=(),
residual_joins_sum=sides.residual_joins_attention_sum,
)
return BoundarySteps(
attention_input=_select_attention_input_move(rows, sides.attention),
ffn_input=ffn_input,
ffn_output_move=ffn_output_move,
ffn_output=sides.ffn_output,
layer_input=layer_input,
)
def _select_attention_input_move(rows: Layout, need: StageInput) -> Callable:
"""How the rows a layer takes become its attention's input: as they are,
or gathered over attention TP from each rank's slice, unless the attention
@@ -1866,6 +1991,7 @@ def _select_ffn_input(
need: StageInput,
force_layernorm_before_gather: bool,
fusions: Tuple[FusedMlpInput, ...],
residual_joins_sum: bool = False,
) -> Tuple[Callable, Tuple[FusedMlpInput, ...]]:
"""The steps from the attention output to the FFN input, and the fused
kernels they try first: complete the attention-TP sum, move the residual to
@@ -1881,7 +2007,7 @@ def _select_ffn_input(
SumGroup.ATTN_TP,
):
raise NotImplementedError(f"{produced=}")
gathered = produced.layout.sharded - need.layout.sharded
gathered = produced.layout.sharded - need.layout.sharded - need.gathers_itself
sliced = need.layout.sharded - produced.layout.sharded
if sliced:
# Each attention-TP rank takes its own slice: the reduce-scatter
@@ -1910,6 +2036,10 @@ def _select_ffn_input(
if not gathered:
if not owes_attention_tp:
return _mlp_input_norm, ()
if gathers_residual and residual_joins_sum:
# Each rank adds its slice of the residual into its share of the
# sum, so the all-reduce also brings the residual back to every row.
return _mlp_input_residual_into_sum, ()
fused = tuple(f for f in fusions if f.completes is produced.group)
return (
partial(
@@ -1985,21 +2115,6 @@ def mlp_input_kind(
and context.attn_tp_size == 1
):
return MlpInputKind.NORM
if (
hidden_in == ScatterMode.SCATTERED
and residual_in == ScatterMode.SCATTERED
and hidden_out == ScatterMode.SCATTERED
and residual_out == ScatterMode.SCATTERED
):
# Megatron LayerNorm sequence parallelism (layers/layernorm_sp.py):
# activations stay sequence-sharded across the attn->mlp boundary, so
# there is nothing to gather or scatter here -- just the residual add
# plus LayerNorm on the local shard. The row-parallel o_proj already
# issued the reduce-scatter (g-bar) that the all-reduce would have
# done, and the g all-gather is fused into the next column-parallel
# linear. Distinct from the branch above because under pure TP
# attn_tp_size == tp_size > 1, so that gate does not fire.
return MlpInputKind.NORM
if hidden_in == ScatterMode.TP_ATTN_FULL and residual_in in (
ScatterMode.SCATTERED,
ScatterMode.TP_ATTN_FULL,
@@ -2080,6 +2195,18 @@ def _mlp_input_scatter(
return hidden_states, residual
def _mlp_input_residual_into_sum(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
layernorm: torch.nn.Module,
context: CommunicateContext,
):
return _tp_all_reduce_with_scattered_residual(
hidden_states, residual, layernorm, context
)
def _tp_all_reduce_with_scattered_residual(
hidden_states: torch.Tensor,
residual: torch.Tensor,
@@ -8,7 +8,13 @@ import torch
import sglang
from sglang.srt.layers import communicator as comm
from sglang.srt.layers.boundary_layout import Layout, StageOutput, SumGroup
from sglang.srt.layers.boundary_layout import (
Layout,
StageOutput,
SumGroup,
TokenAxis,
sequence_parallel_layer_sides,
)
from sglang.srt.layers.communicator import (
LayerCommunicator,
UnreducedOutput,
@@ -21,6 +27,19 @@ from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
def sp_region_steps():
"""The steps a layer runs while a LayerNorm SP region is active."""
return comm._select_boundary_steps(
sequence_parallel_layer_sides(
axis_sizes={
TokenAxis.ATTN_DP: 1,
TokenAxis.ATTN_CP: 1,
TokenAxis.ATTN_TP_SCATTER: 2,
}
)
)
def make_group(scale=3):
"""The group a deferred sum is owed over, with a stubbed all-reduce."""
return types.SimpleNamespace(all_reduce=MagicMock(side_effect=lambda h: h * scale))
@@ -51,7 +70,8 @@ def make_communicator(
return_value=reduce_scatter
)
communicator.is_last_layer = False
communicator._sp_region = False
communicator._sp_steps = None
communicator._input_scattered_steps = None
communicator._postprocess_scatters_to_local_tokens = scatters_to_local_tokens
communicator._postprocess_dp_step = MagicMock(return_value=reduce_scatter_step)
communicator.ffn_reduction_group = MagicMock(return_value=group or make_group())
@@ -284,7 +304,8 @@ class TestSelectFfnCompletion(CustomTestCase):
):
communicator = LayerCommunicator.__new__(LayerCommunicator)
communicator.is_last_layer = is_last_layer
communicator._sp_region = sp_region
communicator._sp_steps = sp_region_steps() if sp_region else None
communicator._input_scattered_steps = None
communicator._postprocess_scatters_to_local_tokens = scatters
communicator._ffn_output = StageOutput(
Layout(frozenset()),
@@ -123,14 +123,16 @@ class TestBoundarySelection(CustomTestCase):
make_context(),
Kind.NORM,
),
((SCATTERED, SCATTERED, SCATTERED, SCATTERED), tp, Kind.NORM),
):
with self.subTest(layout=layout):
self.assertIs(comm.mlp_input_kind(modes(*layout), context), kind)
with self.assertRaises(NotImplementedError):
comm.mlp_input_kind(
modes(SCATTERED, SCATTERED, TP_ATTN_FULL, TP_ATTN_FULL), tp
)
for layout in (
(SCATTERED, SCATTERED, TP_ATTN_FULL, TP_ATTN_FULL),
# The LayerNorm SP region chooses its steps from its declarations.
(SCATTERED, SCATTERED, SCATTERED, SCATTERED),
):
with self.subTest(layout=layout), self.assertRaises(NotImplementedError):
comm.mlp_input_kind(modes(*layout), tp)
def test_postprocess(self):
self.assertIs(
@@ -24,6 +24,8 @@ from sglang.srt.layers.boundary_layout import (
SumGroup,
TokenAxis,
decoder_layer_sides,
input_scattered_layer_sides,
sequence_parallel_layer_sides,
)
from sglang.srt.layers.communicator import (
LayerCommunicator,
@@ -410,10 +412,6 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
def test_layers_that_keep_the_scatter_modes(self):
dp = parallel_of(attn_dp=2, attn_tp=2)
cases = {
"input-scattered attention": (
layer_facts(1, 3),
parallel_of(attn_dp=1, attn_tp=4, enable_attn_tp_input_scattered=True),
),
"attention CP": (
layer_facts(1, 3),
parallel_of(attn_dp=2, attn_tp=1, attn_cp=2),
@@ -431,8 +429,8 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
communicator.allow_deferred_ffn_reduction = True
communicator.allow_reduce_scatter = False
self.assertIsNone(communicator._declared_sides())
# Modes given directly (Nemotron-H's stages, the SP sibling) do not say
# which rows the layer takes.
# Modes given directly (Nemotron-H's stages) do not say which rows the
# layer takes.
direct = LayerScatterModes(
layer_input_mode=ScatterMode.TP_ATTN_FULL,
attn_mode=ScatterMode.TP_ATTN_FULL,
@@ -441,10 +439,6 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
layer_output_mode=ScatterMode.TP_ATTN_FULL,
)
self.assertFalse(self.declared(build(direct, dp)))
with planning(dp, sp=True):
communicator = LayerCommunicator.__new__(LayerCommunicator)
communicator.layer_scatter_modes = layer_facts(1, 3)
self.assertIsNone(communicator._declared_sides())
def test_subclasses_that_pick_their_own_steps(self):
for cls in (
@@ -620,6 +614,192 @@ class TestFusedKernelsTakeOnlyTheStepsTheyComplete(CustomTestCase):
self.assertIs(communicator._mlp_input_may_return_new_residual, expected)
class TestTheSequenceParallelRegion(CustomTestCase):
"""While a LayerNorm SP region is active, the layer runs the steps its
region declarations choose: the linears gather and reduce-scatter
themselves, so every boundary stays on this rank's slice. Other batches
take the layer's ordinary declared steps."""
SIZES = {TokenAxis.ATTN_DP: 1, TokenAxis.ATTN_CP: 1, TokenAxis.ATTN_TP_SCATTER: 2}
SP_STEPS = comm.BoundarySteps(
attention_input=comm.CommunicateSimpleFn._trivial,
ffn_input=comm._mlp_input_norm,
ffn_output_move=comm.CommunicateSummableTensorPairFn._trivial,
ffn_output=sequence_parallel_layer_sides(axis_sizes=SIZES).ffn_output,
)
def test_the_region_declarations_choose_local_steps(self):
for attn_tp in (2, 4):
with self.subTest(attn_tp=attn_tp):
sides = sequence_parallel_layer_sides(
axis_sizes={
TokenAxis.ATTN_DP: 1,
TokenAxis.ATTN_CP: 1,
TokenAxis.ATTN_TP_SCATTER: attn_tp,
}
)
local = frozenset({TokenAxis.ATTN_TP_SCATTER})
self.assertEqual(sides.input_rows.sharded, local)
self.assertEqual(sides.attention.gathers_itself, local)
self.assertEqual(sides.ffn.gathers_itself, local)
self.assertIsNone(sides.attention_output.group)
self.assertIsNone(sides.ffn_output.group)
steps = comm._select_boundary_steps(sides)
self.assertEqual(
(steps.attention_input, steps.ffn_input, steps.ffn_output_move),
(
self.SP_STEPS.attention_input,
self.SP_STEPS.ffn_input,
self.SP_STEPS.ffn_output_move,
),
)
self.assertIsNone(steps.layer_input)
def test_a_layer_under_sp_takes_both_sets_of_steps(self):
parallel = parallel_of(attn_dp=1, attn_tp=2)
for layer_id in range(3):
with self.subTest(layer_id=layer_id):
with planning(parallel, sp=True):
communicator = LayerCommunicator(
layer_scatter_modes=layer_facts(layer_id, 3),
input_layernorm=Norm(),
post_attention_layernorm=Norm(),
)
self.assertEqual(communicator._sp_steps, self.SP_STEPS)
self.assertIs(communicator._mlp_input.func, comm._mlp_input_without_dp)
def test_without_sp_there_is_no_region(self):
with planning(parallel_of(attn_dp=1, attn_tp=2)):
communicator = LayerCommunicator(
layer_scatter_modes=layer_facts(1, 3),
input_layernorm=Norm(),
post_attention_layernorm=Norm(),
)
self.assertIsNone(communicator._sp_steps)
def test_declarations_left_to_the_ffn_exit_are_not_batch_steps(self):
# A sum the FFN exit may hand to the next layer, or a move back over
# attention DP, waits on a per-batch decision.
for sizes, leaves_next in (
(self.SIZES, True),
({**self.SIZES, TokenAxis.ATTN_DP: 2}, False),
):
with self.subTest(sizes=sizes), self.assertRaises(NotImplementedError):
comm._select_boundary_steps(sides_of(sizes, leaves_next=leaves_next))
class TestInputScatteredAttention(CustomTestCase):
"""On a batch whose attention input is scattered over attention TP, a layer
runs the steps that batch's declarations choose: its input arrives as a TP
partial that a reduce-scatter completes onto each rank's slice, and the
residual comes back to every row inside the attention output's sum."""
SIZES = {TokenAxis.ATTN_DP: 1, TokenAxis.ATTN_CP: 1, TokenAxis.ATTN_TP_SCATTER: 2}
def test_the_declarations_choose_the_steps(self):
for hands_on in (False, True):
with self.subTest(hands_on_partial=hands_on):
sides = input_scattered_layer_sides(
axis_sizes=self.SIZES,
ffn_group=SumGroup.TP,
hands_on_partial=hands_on,
)
steps = comm._select_boundary_steps(sides)
self.assertIs(steps.layer_input, comm.tp_reduce_scatter)
self.assertIs(steps.attention_input, comm.CommunicateSimpleFn._trivial)
self.assertIs(steps.ffn_input, comm._mlp_input_residual_into_sum)
self.assertIs(
steps.ffn_output_move,
comm.CommunicateSummableTensorPairFn._trivial,
)
self.assertIs(steps.ffn_output.leaves_for_reduce_scatter, hands_on)
self.assertFalse(steps.ffn_output.leaves_for_next_layer)
def test_the_order_of_a_residual_on_the_slice_is_declared(self):
# The same layouts take two orders: with a scattered input the residual
# joins the attention output's sum; after a MoE on local rows it is
# gathered first.
sizes = self.SIZES
attention = comm.Layout.sharded_over(axis_sizes=sizes)
local = comm.Layout(frozenset({TokenAxis.ATTN_TP_SCATTER}))
owed = comm.StageOutput(attention, group=SumGroup.ATTN_TP, always_leaves=True)
for joins, step in (
(True, comm._mlp_input_residual_into_sum),
(False, comm._mlp_input_without_dp),
):
with self.subTest(residual_joins_sum=joins):
steps, _ = comm._select_ffn_input(
owed,
residual=local,
residual_to=attention,
need=comm.StageInput(attention),
force_layernorm_before_gather=False,
fusions=(),
residual_joins_sum=joins,
)
self.assertIs(getattr(steps, "func", steps), step)
def test_which_layers_can_scatter_their_input(self):
configured = dict(enable_attn_tp_input_scattered=True)
for name, parallel, a2a, expected in (
("pure TP", parallel_of(attn_dp=1, attn_tp=2, **configured), False, True),
("not configured", parallel_of(attn_dp=1, attn_tp=2), False, False),
(
"attention DP",
parallel_of(attn_dp=2, attn_tp=2, **configured),
False,
False,
),
("a2a", parallel_of(attn_dp=1, attn_tp=2, **configured), True, False),
):
with self.subTest(name):
communicator = build(
layer_facts(1, 3, sparse=a2a, previous_sparse=a2a),
parallel,
a2a=a2a,
allow_reduce_scatter=True,
)
self.assertIs(communicator._input_scattered_steps is not None, expected)
def test_a_batch_runs_them_only_while_its_input_is_scattered(self):
parallel = parallel_of(
attn_dp=1, attn_tp=2, enable_attn_tp_input_scattered=True
)
communicator = build(layer_facts(1, 3), parallel, allow_reduce_scatter=True)
for scattered in (False, True):
with (
self.subTest(input_scattered=scattered),
patch.object(
comm,
"get_attn_tp_context",
lambda: SimpleNamespace(input_scattered=scattered),
),
patch.object(
comm, "get_forward", lambda: SimpleNamespace(sp_active=False)
),
):
self.assertIs(
communicator._batch_steps(),
communicator._input_scattered_steps if scattered else None,
)
def test_the_last_layer_completes_its_own_sum(self):
parallel = parallel_of(
attn_dp=1, attn_tp=2, enable_attn_tp_input_scattered=True
)
for layer_id, allow, hands_on in (
(1, True, True),
(2, True, False),
(1, False, False),
):
with self.subTest(layer_id=layer_id, allow_reduce_scatter=allow):
communicator = build(
layer_facts(layer_id, 3), parallel, allow_reduce_scatter=allow
)
steps = communicator._input_scattered_steps
self.assertIs(steps.ffn_output.leaves_for_reduce_scatter, hands_on)
# ---------------------------------------------------------------------------
# Two consecutive layers, rank by rank.
@@ -6,7 +6,13 @@ from unittest.mock import MagicMock, patch
import torch
from sglang.srt.layers import communicator as comm
from sglang.srt.layers.boundary_layout import Layout, StageOutput, SumGroup
from sglang.srt.layers.boundary_layout import (
Layout,
StageOutput,
SumGroup,
TokenAxis,
sequence_parallel_layer_sides,
)
from sglang.srt.layers.communicator import LayerCommunicator, ScatterMode
from sglang.srt.layers.moe import (
can_merge_post_experts_all_reduce,
@@ -21,6 +27,19 @@ from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=11, suite="base-a-test-cpu")
def sp_region_steps():
"""The steps a layer runs while a LayerNorm SP region is active."""
return comm._select_boundary_steps(
sequence_parallel_layer_sides(
axis_sizes={
TokenAxis.ATTN_DP: 1,
TokenAxis.ATTN_CP: 1,
TokenAxis.ATTN_TP_SCATTER: 2,
}
)
)
def _fake_communicator(mlp_mode=ScatterMode.TP_ATTN_FULL):
communicator = LayerCommunicator.__new__(LayerCommunicator)
communicator._speculative_algo = None
@@ -329,7 +348,8 @@ class TestDeferFfnReduction(CustomTestCase):
communicator = _fake_communicator()
communicator.is_last_layer = is_last_layer
communicator._postprocess_scatters_to_local_tokens = scatters_to_local_tokens
communicator._sp_region = sp_active
communicator._sp_steps = sp_region_steps() if sp_active else None
communicator._input_scattered_steps = None
communicator._ffn_output = StageOutput(
Layout(frozenset()),
group=SumGroup.MOE_OUTPUT,
@@ -14,6 +14,10 @@ import torch
from sglang.srt.arg_groups.layernorm_sp_hook import validate_layernorm_sp
from sglang.srt.layers import communicator as comm
from sglang.srt.layers import layernorm_sp
from sglang.srt.layers.boundary_layout import (
TokenAxis,
sequence_parallel_layer_sides,
)
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.runtime_context import (
get_flags,
@@ -28,6 +32,19 @@ from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=9, suite="base-a-test-cpu")
def sp_region_steps():
"""The steps a layer runs while a LayerNorm SP region is active."""
return comm._select_boundary_steps(
sequence_parallel_layer_sides(
axis_sizes={
TokenAxis.ATTN_DP: 1,
TokenAxis.ATTN_CP: 1,
TokenAxis.ATTN_TP_SCATTER: 2,
}
)
)
def _initialize(*, enable=True, arch="Qwen3ForCausalLM"):
publish(
ServerArgs(model_path="dummy", enable_layernorm_sp=enable), role="tokenizer"
@@ -148,7 +165,8 @@ class TestSpRegionSteps(CustomTestCase):
def communicator(self, *, first_layer):
c = comm.LayerCommunicator.__new__(comm.LayerCommunicator)
c._sp_region = True
c._sp_steps = sp_region_steps()
c._input_scattered_steps = None
c.layer_scatter_modes = SimpleNamespace(is_first_layer=first_layer)
c._attn_input_fusions = ()
c.input_layernorm = _Norm()
@@ -140,7 +140,8 @@ class TestPostprocessReduceScatterv(CustomTestCase):
def test_postprocess_passes_the_layer_sparsity(self):
seen = {}
communicator = LayerCommunicator.__new__(LayerCommunicator)
communicator._sp_region = False
communicator._sp_steps = None
communicator._input_scattered_steps = None
communicator._context = None
communicator._postprocess_scatters_to_local_tokens = False
communicator.allow_reduce_scatter = False
@@ -157,7 +158,8 @@ class TestPostprocessReduceScatterv(CustomTestCase):
for step in (object(), None):
with self.subTest(reduce_scatter=step is not None):
communicator = LayerCommunicator.__new__(LayerCommunicator)
communicator._sp_region = False
communicator._sp_steps = None
communicator._input_scattered_steps = None
communicator._postprocess_scatters_to_local_tokens = True
communicator._postprocess_dp_step = lambda forward_batch: step
with patch.object(
@@ -42,7 +42,8 @@ class Norm:
def communicator(norm):
c = comm.LayerCommunicator.__new__(comm.LayerCommunicator)
c.input_layernorm = norm
c._sp_region = False
c._sp_steps = None
c._input_scattered_steps = None
c.qkv_latent_func = None
c._context = None
c.enable_fused_ar_quant = False
@@ -222,7 +222,8 @@ def test_communicator_publishes_layout_at_each_transition(
communicator._publish_lora_layout = publish_lora_layout
communicator.layer_scatter_modes = SimpleNamespace(mlp_mode=mlp_mode)
communicator._context = SimpleNamespace()
communicator._sp_region = False
communicator._sp_steps = None
communicator._input_scattered_steps = None
communicator.post_attention_layernorm = None
communicator.input_layernorm = lambda x: x
communicator.qkv_latent_func = None
@@ -57,6 +57,8 @@ class DeferringLayer(nn.Module):
self.layer_communicator._ffn_sum_moves_to_next_layer = lambda batch, **_: defer
self.layer_communicator.is_last_layer = False
self.layer_communicator._postprocess_scatters_to_local_tokens = False
self.layer_communicator._sp_steps = None
self.layer_communicator._input_scattered_steps = None
self.layer_communicator.ffn_reduction_group = lambda: GROUP
self.layer_communicator._ffn_leaves_sum_to_reduce_scatter = (
lambda batch, dp_step: False