diff --git a/python/sglang/srt/layers/boundary_layout.py b/python/sglang/srt/layers/boundary_layout.py index 6afdf94d0a..f1779824f8 100644 --- a/python/sglang/srt/layers/boundary_layout.py +++ b/python/sglang/srt/layers/boundary_layout.py @@ -100,6 +100,76 @@ class DecoderLayerSides(msgspec.Struct, frozen=True): ffn_residual_rows: Layout # The rows the layer hands on. output_rows: Layout + # The group the layer's input arrives as a partial sum over, if any. + input_owes: Optional[SumGroup] = None + # Whether the residual is added into one rank's share of the attention + # output's sum before that sum completes, instead of after it. + residual_joins_attention_sum: bool = False + + +def sequence_parallel_layer_sides( + *, axis_sizes: Mapping[TokenAxis, int] +) -> DecoderLayerSides: + """A decoder layer while a LayerNorm SP region is active. Its linears + all-gather their input and reduce-scatter their output themselves, so the + layer's rows, its residual and both outputs stay on each TP rank's slice.""" + attention = Layout.sharded_over( + TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes + ) + local = Layout.sharded_over( + TokenAxis.ATTN_DP, + TokenAxis.ATTN_CP, + TokenAxis.ATTN_TP_SCATTER, + axis_sizes=axis_sizes, + ) + gathers = frozenset({TokenAxis.ATTN_TP_SCATTER}) + return DecoderLayerSides( + input_rows=local, + # qkv and gate_up gather the slices into their GEMM. + attention=StageInput(attention, gathers_itself=gathers), + # o_proj and down reduce-scatter out of theirs, whether fused or not. + attention_output=StageOutput(local), + ffn=StageInput( + Layout.sharded_over(axis_sizes=axis_sizes), gathers_itself=gathers + ), + ffn_output=StageOutput(local), + ffn_residual_rows=local, + output_rows=local, + ) + + +def input_scattered_layer_sides( + *, + axis_sizes: Mapping[TokenAxis, int], + ffn_group: SumGroup, + hands_on_partial: bool, +) -> DecoderLayerSides: + """A decoder layer on a batch whose attention input is scattered over + attention TP. Each layer's input is a partial sum over TP on the full rows: + the embedding and every FFN before it leave that sum, and a reduce-scatter + completes it onto each rank's slice, which the attention gathers itself. + The residual comes back to the full rows inside the attention output's sum.""" + attention = Layout.sharded_over( + TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes + ) + return DecoderLayerSides( + input_rows=attention, + attention=StageInput( + attention, gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER}) + ), + attention_output=StageOutput( + attention, group=SumGroup.ATTN_TP, always_leaves=True + ), + ffn=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)), + # The next layer's input completes the sum; the last layer's FFN does. + ffn_output=StageOutput( + attention, group=ffn_group, leaves_for_reduce_scatter=hands_on_partial + ), + ffn_residual_rows=attention, + output_rows=attention, + input_owes=SumGroup.TP, + residual_joins_attention_sum=True, + ) def decoder_layer_sides( diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py index 833124de6e..95adeb1b83 100644 --- a/python/sglang/srt/layers/communicator.py +++ b/python/sglang/srt/layers/communicator.py @@ -45,6 +45,8 @@ from sglang.srt.layers.boundary_layout import ( SumGroup, TokenAxis, decoder_layer_sides, + input_scattered_layer_sides, + sequence_parallel_layer_sides, ) from sglang.srt.layers.cp.utils import ( is_mla_cp_active, @@ -759,6 +761,20 @@ class LayerCommunicator: self._select_declared_boundaries(sides) else: self._select_boundaries_from_scatter_modes() + # The steps a batch with input-scattered attention runs; None where it + # cannot run or the steps come from the scatter modes. + self._input_scattered_steps = ( + _select_boundary_steps( + input_scattered_layer_sides( + axis_sizes=_token_axis_sizes(), + ffn_group=sides.ffn_output.group, + hands_on_partial=allow_reduce_scatter + and not layer_scatter_modes.is_last_layer, + ) + ) + if sides is not None and self._input_can_be_scattered() + else None + ) self._attn_input_fusions = self._select_attn_input_fusions() self._speculative_algo = SpeculativeAlgorithm.from_string( get_spec().speculative_algorithm @@ -768,15 +784,36 @@ class LayerCommunicator: get_lora().enable_lora ) - # Under LayerNorm SP the residual and norms run on this rank's sequence - # shard with no collectives while the region is active. - self._sp_region = layernorm_sp.layernorm_sp_enabled() + # Under LayerNorm SP, the steps the layer runs while the region is + # active; None without SP. The two are exclusive: SP runs a model without + # q_lora, which input-scattered attention needs. + self._sp_steps = ( + _select_boundary_steps( + sequence_parallel_layer_sides(axis_sizes=_token_axis_sizes()) + ) + if layernorm_sp.layernorm_sp_enabled() + else None + ) + + def _input_can_be_scattered(self) -> bool: + """Whether a batch may run this layer with input-scattered attention: + configured, on pure TP without an a2a backend. The rest of what + ``AttnTpContext.init_context`` requires is only known once the model is + built.""" + parallel = get_parallel() + return ( + parallel.enable_attn_tp_input_scattered + and parallel.tp_size > 1 + and parallel.attn_dp_size == 1 + and parallel.attn_cp_size == 1 + and get_moe_a2a_backend().is_none() + ) def _declared_sides(self) -> Optional[DecoderLayerSides]: """The declarations this layer's boundaries are chosen from: an - attention and an FFN, without CP, LayerNorm SP or input-scattered - attention, in a layer whose previous layer is known. None when the - steps come from the scatter modes.""" + attention and an FFN, without CP, in a layer whose previous layer is + known. None when the steps come from the scatter modes. An active + LayerNorm SP region and input-scattered attention have their own.""" modes = self.layer_scatter_modes parallel = get_parallel() # A MoE dispatched per DP shard computes on this rank's local rows and @@ -785,18 +822,12 @@ class LayerCommunicator: if not ( self._takes_declared_boundaries and parallel.attn_cp_size == 1 - and not layernorm_sp.layernorm_sp_enabled() - and not parallel.enable_attn_tp_input_scattered and not enable_moe_dense_fully_dp() and (modes.is_first_layer or modes.is_previous_layer_sparse is not None) ): return None return decoder_layer_sides( - axis_sizes={ - TokenAxis.ATTN_DP: parallel.attn_dp_size, - TokenAxis.ATTN_CP: parallel.attn_cp_size, - TokenAxis.ATTN_TP_SCATTER: parallel.attn_tp_size, - }, + axis_sizes=_token_axis_sizes(), ffn_on_local_rows=modes.is_layer_sparse and moe_on_local_rows, previous_on_local_rows=( not modes.is_first_layer @@ -991,13 +1022,18 @@ class LayerCommunicator: hidden_states = owed.partial # The SP region opens at the first layer, re-evaluated per forward so a # crash mid-loop cannot leak into the next one. - if self._sp_region and self.layer_scatter_modes.is_first_layer: + if self._sp_steps is not None and self.layer_scatter_modes.is_first_layer: get_forward().set( "sp_active", layernorm_sp.runs_sp(forward_batch.forward_mode) ) if get_forward().sp_active: hidden_states = layernorm_sp.sp_entry_scatter(hidden_states) - if get_attn_tp_context().input_scattered: + steps = self._batch_steps() + if steps is not None and steps.layer_input is not None: + hidden_states, residual = steps.layer_input( + hidden_states, residual, self._context + ) + elif get_attn_tp_context().input_scattered: hidden_states, residual = self._tp_reduce_scatter( hidden_states, residual, @@ -1021,17 +1057,29 @@ class LayerCommunicator: ) def _in_sp_region(self) -> bool: - return self._sp_region and get_forward().sp_active + return self._sp_steps is not None and get_forward().sp_active + + def _batch_steps(self) -> Optional["BoundarySteps"]: + """The steps this batch runs in place of the layer's ordinary ones: an + active LayerNorm SP region's, or input-scattered attention's.""" + if self._in_sp_region(): + return self._sp_steps + if ( + self._input_scattered_steps is not None + and get_attn_tp_context().input_scattered + ): + return self._input_scattered_steps + return None def _finish_prepare_attn(self, hidden_states, residual, forward_batch): - """Tail every prepare_attn path must run, or ``attn_inputs`` is unset. In - the SP region the attention input stays on this rank's sequence shard.""" - if not self._in_sp_region(): - hidden_states = self._communicate_simple_fn( - hidden_states=hidden_states, - forward_batch=forward_batch, - context=self._context, - ) + """Tail every prepare_attn path must run, or ``attn_inputs`` is unset.""" + steps = self._batch_steps() + move = self._communicate_simple_fn if steps is None else steps.attention_input + hidden_states = move( + hidden_states=hidden_states, + forward_batch=forward_batch, + context=self._context, + ) if self.qkv_latent_func is not None: attn_inputs = AttentionInputs( hidden_states, forward_batch, self.qkv_latent_func @@ -1175,7 +1223,7 @@ class LayerCommunicator: ): self.publish_mlp_lora_layout() if self._in_sp_region(): - return _mlp_input_norm( + return self._sp_steps.ffn_input( hidden_states, residual, forward_batch, @@ -1185,7 +1233,9 @@ class LayerCommunicator: if cache is not None: self._context.cache = cache - return self._mlp_input( + steps = self._batch_steps() + mlp_input = self._mlp_input if steps is None else steps.ffn_input + return mlp_input( hidden_states, residual, forward_batch, @@ -1206,8 +1256,14 @@ class LayerCommunicator: residual: torch.Tensor, forward_batch: ForwardBatch, ): - if self._in_sp_region(): - return hidden_states, residual + steps = self._batch_steps() + if steps is not None: + return steps.ffn_output_move( + hidden_states=hidden_states, + residual=residual, + forward_batch=forward_batch, + context=self._context, + ) if self._postprocess_scatters_to_local_tokens: step = self._postprocess_dp_step(forward_batch) or _redistribute_output return _to_local_tokens(step, forward_batch, hidden_states), residual @@ -1246,6 +1302,9 @@ class LayerCommunicator: """Whether the FFN leaves its sum out because a reduce-scatter completes it: the attention-DP one ``dp_step`` names, or the CP / input-scattered one.""" + steps = self._batch_steps() + if steps is not None: + return steps.ffn_output.leaves_for_reduce_scatter if not self._ffn_output.leaves_for_reduce_scatter: return False if dp_step is not None: @@ -1274,7 +1333,9 @@ class LayerCommunicator: forward_batch=forward_batch, dp_step=dp_step, ) - if not self._ffn_output.leaves_for_next_layer: + steps = self._batch_steps() + ffn_output = self._ffn_output if steps is None else steps.ffn_output + if not ffn_output.leaves_for_next_layer: return FfnCompletion( defer_moe_finalize=False, fuse_mlp_allreduce=False, @@ -1843,6 +1904,70 @@ def _sum_group(group: SumGroup) -> GroupCoordinator: return post_experts_reduction_group() +def _token_axis_sizes() -> Dict[TokenAxis, int]: + parallel = get_parallel() + return { + TokenAxis.ATTN_DP: parallel.attn_dp_size, + TokenAxis.ATTN_CP: parallel.attn_cp_size, + TokenAxis.ATTN_TP_SCATTER: parallel.attn_tp_size, + } + + +class BoundarySteps(msgspec.Struct, frozen=True): + """The steps one set of declarations chooses for a layer's boundaries: + into the attention, from the attention output to the FFN input, and the + FFN output on to the rows the layer hands on.""" + + attention_input: Callable + ffn_input: Callable + ffn_output_move: Callable + # What the FFN exit reads in place of the layer's own FFN output declaration. + ffn_output: StageOutput + # Completes what the layer's input owes before the input norm; None when it + # owes nothing. + layer_input: Optional[Callable] = None + + +def _select_boundary_steps(sides: DecoderLayerSides) -> BoundarySteps: + """The steps for a batch's own declarations: nothing moves back over + attention DP and no fused kernel runs, so the steps do not wait on a + per-batch decision.""" + returns_over_dp, ffn_output_move = _select_ffn_output_move( + sides.ffn_output, residual=sides.ffn_residual_rows, to=sides.output_rows + ) + if returns_over_dp or sides.ffn_output.leaves_for_next_layer: + raise NotImplementedError(f"{sides=}") + rows = sides.input_rows + layer_input = None + if sides.input_owes is not None: + # A reduce-scatter completes the TP sum onto each rank's slice, which the + # attention takes: its group is the TP group without attention DP or CP. + if ( + sides.input_owes is not SumGroup.TP + or rows.sharded + or TokenAxis.ATTN_TP_SCATTER not in sides.attention.gathers_itself + ): + raise NotImplementedError(f"{sides=}") + layer_input = tp_reduce_scatter + rows = Layout(frozenset({TokenAxis.ATTN_TP_SCATTER})) + ffn_input, _ = _select_ffn_input( + sides.attention_output, + residual=rows, + residual_to=sides.ffn_residual_rows, + need=sides.ffn, + force_layernorm_before_gather=False, + fusions=(), + residual_joins_sum=sides.residual_joins_attention_sum, + ) + return BoundarySteps( + attention_input=_select_attention_input_move(rows, sides.attention), + ffn_input=ffn_input, + ffn_output_move=ffn_output_move, + ffn_output=sides.ffn_output, + layer_input=layer_input, + ) + + def _select_attention_input_move(rows: Layout, need: StageInput) -> Callable: """How the rows a layer takes become its attention's input: as they are, or gathered over attention TP from each rank's slice, unless the attention @@ -1866,6 +1991,7 @@ def _select_ffn_input( need: StageInput, force_layernorm_before_gather: bool, fusions: Tuple[FusedMlpInput, ...], + residual_joins_sum: bool = False, ) -> Tuple[Callable, Tuple[FusedMlpInput, ...]]: """The steps from the attention output to the FFN input, and the fused kernels they try first: complete the attention-TP sum, move the residual to @@ -1881,7 +2007,7 @@ def _select_ffn_input( SumGroup.ATTN_TP, ): raise NotImplementedError(f"{produced=}") - gathered = produced.layout.sharded - need.layout.sharded + gathered = produced.layout.sharded - need.layout.sharded - need.gathers_itself sliced = need.layout.sharded - produced.layout.sharded if sliced: # Each attention-TP rank takes its own slice: the reduce-scatter @@ -1910,6 +2036,10 @@ def _select_ffn_input( if not gathered: if not owes_attention_tp: return _mlp_input_norm, () + if gathers_residual and residual_joins_sum: + # Each rank adds its slice of the residual into its share of the + # sum, so the all-reduce also brings the residual back to every row. + return _mlp_input_residual_into_sum, () fused = tuple(f for f in fusions if f.completes is produced.group) return ( partial( @@ -1985,21 +2115,6 @@ def mlp_input_kind( and context.attn_tp_size == 1 ): return MlpInputKind.NORM - if ( - hidden_in == ScatterMode.SCATTERED - and residual_in == ScatterMode.SCATTERED - and hidden_out == ScatterMode.SCATTERED - and residual_out == ScatterMode.SCATTERED - ): - # Megatron LayerNorm sequence parallelism (layers/layernorm_sp.py): - # activations stay sequence-sharded across the attn->mlp boundary, so - # there is nothing to gather or scatter here -- just the residual add - # plus LayerNorm on the local shard. The row-parallel o_proj already - # issued the reduce-scatter (g-bar) that the all-reduce would have - # done, and the g all-gather is fused into the next column-parallel - # linear. Distinct from the branch above because under pure TP - # attn_tp_size == tp_size > 1, so that gate does not fire. - return MlpInputKind.NORM if hidden_in == ScatterMode.TP_ATTN_FULL and residual_in in ( ScatterMode.SCATTERED, ScatterMode.TP_ATTN_FULL, @@ -2080,6 +2195,18 @@ def _mlp_input_scatter( return hidden_states, residual +def _mlp_input_residual_into_sum( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + layernorm: torch.nn.Module, + context: CommunicateContext, +): + return _tp_all_reduce_with_scattered_residual( + hidden_states, residual, layernorm, context + ) + + def _tp_all_reduce_with_scattered_residual( hidden_states: torch.Tensor, residual: torch.Tensor, diff --git a/test/registered/unit/layers/test_communicator_ffn_exit.py b/test/registered/unit/layers/test_communicator_ffn_exit.py index 92f114d620..06ce59cf88 100644 --- a/test/registered/unit/layers/test_communicator_ffn_exit.py +++ b/test/registered/unit/layers/test_communicator_ffn_exit.py @@ -8,7 +8,13 @@ import torch import sglang from sglang.srt.layers import communicator as comm -from sglang.srt.layers.boundary_layout import Layout, StageOutput, SumGroup +from sglang.srt.layers.boundary_layout import ( + Layout, + StageOutput, + SumGroup, + TokenAxis, + sequence_parallel_layer_sides, +) from sglang.srt.layers.communicator import ( LayerCommunicator, UnreducedOutput, @@ -21,6 +27,19 @@ from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=10, suite="base-a-test-cpu") +def sp_region_steps(): + """The steps a layer runs while a LayerNorm SP region is active.""" + return comm._select_boundary_steps( + sequence_parallel_layer_sides( + axis_sizes={ + TokenAxis.ATTN_DP: 1, + TokenAxis.ATTN_CP: 1, + TokenAxis.ATTN_TP_SCATTER: 2, + } + ) + ) + + def make_group(scale=3): """The group a deferred sum is owed over, with a stubbed all-reduce.""" return types.SimpleNamespace(all_reduce=MagicMock(side_effect=lambda h: h * scale)) @@ -51,7 +70,8 @@ def make_communicator( return_value=reduce_scatter ) communicator.is_last_layer = False - communicator._sp_region = False + communicator._sp_steps = None + communicator._input_scattered_steps = None communicator._postprocess_scatters_to_local_tokens = scatters_to_local_tokens communicator._postprocess_dp_step = MagicMock(return_value=reduce_scatter_step) communicator.ffn_reduction_group = MagicMock(return_value=group or make_group()) @@ -284,7 +304,8 @@ class TestSelectFfnCompletion(CustomTestCase): ): communicator = LayerCommunicator.__new__(LayerCommunicator) communicator.is_last_layer = is_last_layer - communicator._sp_region = sp_region + communicator._sp_steps = sp_region_steps() if sp_region else None + communicator._input_scattered_steps = None communicator._postprocess_scatters_to_local_tokens = scatters communicator._ffn_output = StageOutput( Layout(frozenset()), diff --git a/test/registered/unit/layers/test_communicator_layout.py b/test/registered/unit/layers/test_communicator_layout.py index 781a42518a..5edb1ac3ea 100644 --- a/test/registered/unit/layers/test_communicator_layout.py +++ b/test/registered/unit/layers/test_communicator_layout.py @@ -123,14 +123,16 @@ class TestBoundarySelection(CustomTestCase): make_context(), Kind.NORM, ), - ((SCATTERED, SCATTERED, SCATTERED, SCATTERED), tp, Kind.NORM), ): with self.subTest(layout=layout): self.assertIs(comm.mlp_input_kind(modes(*layout), context), kind) - with self.assertRaises(NotImplementedError): - comm.mlp_input_kind( - modes(SCATTERED, SCATTERED, TP_ATTN_FULL, TP_ATTN_FULL), tp - ) + for layout in ( + (SCATTERED, SCATTERED, TP_ATTN_FULL, TP_ATTN_FULL), + # The LayerNorm SP region chooses its steps from its declarations. + (SCATTERED, SCATTERED, SCATTERED, SCATTERED), + ): + with self.subTest(layout=layout), self.assertRaises(NotImplementedError): + comm.mlp_input_kind(modes(*layout), tp) def test_postprocess(self): self.assertIs( diff --git a/test/registered/unit/layers/test_declared_decoder_boundary.py b/test/registered/unit/layers/test_declared_decoder_boundary.py index fe54a4597f..87e74214e4 100644 --- a/test/registered/unit/layers/test_declared_decoder_boundary.py +++ b/test/registered/unit/layers/test_declared_decoder_boundary.py @@ -24,6 +24,8 @@ from sglang.srt.layers.boundary_layout import ( SumGroup, TokenAxis, decoder_layer_sides, + input_scattered_layer_sides, + sequence_parallel_layer_sides, ) from sglang.srt.layers.communicator import ( LayerCommunicator, @@ -410,10 +412,6 @@ class TestWhichLayersUseDeclarations(CustomTestCase): def test_layers_that_keep_the_scatter_modes(self): dp = parallel_of(attn_dp=2, attn_tp=2) cases = { - "input-scattered attention": ( - layer_facts(1, 3), - parallel_of(attn_dp=1, attn_tp=4, enable_attn_tp_input_scattered=True), - ), "attention CP": ( layer_facts(1, 3), parallel_of(attn_dp=2, attn_tp=1, attn_cp=2), @@ -431,8 +429,8 @@ class TestWhichLayersUseDeclarations(CustomTestCase): communicator.allow_deferred_ffn_reduction = True communicator.allow_reduce_scatter = False self.assertIsNone(communicator._declared_sides()) - # Modes given directly (Nemotron-H's stages, the SP sibling) do not say - # which rows the layer takes. + # Modes given directly (Nemotron-H's stages) do not say which rows the + # layer takes. direct = LayerScatterModes( layer_input_mode=ScatterMode.TP_ATTN_FULL, attn_mode=ScatterMode.TP_ATTN_FULL, @@ -441,10 +439,6 @@ class TestWhichLayersUseDeclarations(CustomTestCase): layer_output_mode=ScatterMode.TP_ATTN_FULL, ) self.assertFalse(self.declared(build(direct, dp))) - with planning(dp, sp=True): - communicator = LayerCommunicator.__new__(LayerCommunicator) - communicator.layer_scatter_modes = layer_facts(1, 3) - self.assertIsNone(communicator._declared_sides()) def test_subclasses_that_pick_their_own_steps(self): for cls in ( @@ -620,6 +614,192 @@ class TestFusedKernelsTakeOnlyTheStepsTheyComplete(CustomTestCase): self.assertIs(communicator._mlp_input_may_return_new_residual, expected) +class TestTheSequenceParallelRegion(CustomTestCase): + """While a LayerNorm SP region is active, the layer runs the steps its + region declarations choose: the linears gather and reduce-scatter + themselves, so every boundary stays on this rank's slice. Other batches + take the layer's ordinary declared steps.""" + + SIZES = {TokenAxis.ATTN_DP: 1, TokenAxis.ATTN_CP: 1, TokenAxis.ATTN_TP_SCATTER: 2} + SP_STEPS = comm.BoundarySteps( + attention_input=comm.CommunicateSimpleFn._trivial, + ffn_input=comm._mlp_input_norm, + ffn_output_move=comm.CommunicateSummableTensorPairFn._trivial, + ffn_output=sequence_parallel_layer_sides(axis_sizes=SIZES).ffn_output, + ) + + def test_the_region_declarations_choose_local_steps(self): + for attn_tp in (2, 4): + with self.subTest(attn_tp=attn_tp): + sides = sequence_parallel_layer_sides( + axis_sizes={ + TokenAxis.ATTN_DP: 1, + TokenAxis.ATTN_CP: 1, + TokenAxis.ATTN_TP_SCATTER: attn_tp, + } + ) + local = frozenset({TokenAxis.ATTN_TP_SCATTER}) + self.assertEqual(sides.input_rows.sharded, local) + self.assertEqual(sides.attention.gathers_itself, local) + self.assertEqual(sides.ffn.gathers_itself, local) + self.assertIsNone(sides.attention_output.group) + self.assertIsNone(sides.ffn_output.group) + steps = comm._select_boundary_steps(sides) + self.assertEqual( + (steps.attention_input, steps.ffn_input, steps.ffn_output_move), + ( + self.SP_STEPS.attention_input, + self.SP_STEPS.ffn_input, + self.SP_STEPS.ffn_output_move, + ), + ) + self.assertIsNone(steps.layer_input) + + def test_a_layer_under_sp_takes_both_sets_of_steps(self): + parallel = parallel_of(attn_dp=1, attn_tp=2) + for layer_id in range(3): + with self.subTest(layer_id=layer_id): + with planning(parallel, sp=True): + communicator = LayerCommunicator( + layer_scatter_modes=layer_facts(layer_id, 3), + input_layernorm=Norm(), + post_attention_layernorm=Norm(), + ) + self.assertEqual(communicator._sp_steps, self.SP_STEPS) + self.assertIs(communicator._mlp_input.func, comm._mlp_input_without_dp) + + def test_without_sp_there_is_no_region(self): + with planning(parallel_of(attn_dp=1, attn_tp=2)): + communicator = LayerCommunicator( + layer_scatter_modes=layer_facts(1, 3), + input_layernorm=Norm(), + post_attention_layernorm=Norm(), + ) + self.assertIsNone(communicator._sp_steps) + + def test_declarations_left_to_the_ffn_exit_are_not_batch_steps(self): + # A sum the FFN exit may hand to the next layer, or a move back over + # attention DP, waits on a per-batch decision. + for sizes, leaves_next in ( + (self.SIZES, True), + ({**self.SIZES, TokenAxis.ATTN_DP: 2}, False), + ): + with self.subTest(sizes=sizes), self.assertRaises(NotImplementedError): + comm._select_boundary_steps(sides_of(sizes, leaves_next=leaves_next)) + + +class TestInputScatteredAttention(CustomTestCase): + """On a batch whose attention input is scattered over attention TP, a layer + runs the steps that batch's declarations choose: its input arrives as a TP + partial that a reduce-scatter completes onto each rank's slice, and the + residual comes back to every row inside the attention output's sum.""" + + SIZES = {TokenAxis.ATTN_DP: 1, TokenAxis.ATTN_CP: 1, TokenAxis.ATTN_TP_SCATTER: 2} + + def test_the_declarations_choose_the_steps(self): + for hands_on in (False, True): + with self.subTest(hands_on_partial=hands_on): + sides = input_scattered_layer_sides( + axis_sizes=self.SIZES, + ffn_group=SumGroup.TP, + hands_on_partial=hands_on, + ) + steps = comm._select_boundary_steps(sides) + self.assertIs(steps.layer_input, comm.tp_reduce_scatter) + self.assertIs(steps.attention_input, comm.CommunicateSimpleFn._trivial) + self.assertIs(steps.ffn_input, comm._mlp_input_residual_into_sum) + self.assertIs( + steps.ffn_output_move, + comm.CommunicateSummableTensorPairFn._trivial, + ) + self.assertIs(steps.ffn_output.leaves_for_reduce_scatter, hands_on) + self.assertFalse(steps.ffn_output.leaves_for_next_layer) + + def test_the_order_of_a_residual_on_the_slice_is_declared(self): + # The same layouts take two orders: with a scattered input the residual + # joins the attention output's sum; after a MoE on local rows it is + # gathered first. + sizes = self.SIZES + attention = comm.Layout.sharded_over(axis_sizes=sizes) + local = comm.Layout(frozenset({TokenAxis.ATTN_TP_SCATTER})) + owed = comm.StageOutput(attention, group=SumGroup.ATTN_TP, always_leaves=True) + for joins, step in ( + (True, comm._mlp_input_residual_into_sum), + (False, comm._mlp_input_without_dp), + ): + with self.subTest(residual_joins_sum=joins): + steps, _ = comm._select_ffn_input( + owed, + residual=local, + residual_to=attention, + need=comm.StageInput(attention), + force_layernorm_before_gather=False, + fusions=(), + residual_joins_sum=joins, + ) + self.assertIs(getattr(steps, "func", steps), step) + + def test_which_layers_can_scatter_their_input(self): + configured = dict(enable_attn_tp_input_scattered=True) + for name, parallel, a2a, expected in ( + ("pure TP", parallel_of(attn_dp=1, attn_tp=2, **configured), False, True), + ("not configured", parallel_of(attn_dp=1, attn_tp=2), False, False), + ( + "attention DP", + parallel_of(attn_dp=2, attn_tp=2, **configured), + False, + False, + ), + ("a2a", parallel_of(attn_dp=1, attn_tp=2, **configured), True, False), + ): + with self.subTest(name): + communicator = build( + layer_facts(1, 3, sparse=a2a, previous_sparse=a2a), + parallel, + a2a=a2a, + allow_reduce_scatter=True, + ) + self.assertIs(communicator._input_scattered_steps is not None, expected) + + def test_a_batch_runs_them_only_while_its_input_is_scattered(self): + parallel = parallel_of( + attn_dp=1, attn_tp=2, enable_attn_tp_input_scattered=True + ) + communicator = build(layer_facts(1, 3), parallel, allow_reduce_scatter=True) + for scattered in (False, True): + with ( + self.subTest(input_scattered=scattered), + patch.object( + comm, + "get_attn_tp_context", + lambda: SimpleNamespace(input_scattered=scattered), + ), + patch.object( + comm, "get_forward", lambda: SimpleNamespace(sp_active=False) + ), + ): + self.assertIs( + communicator._batch_steps(), + communicator._input_scattered_steps if scattered else None, + ) + + def test_the_last_layer_completes_its_own_sum(self): + parallel = parallel_of( + attn_dp=1, attn_tp=2, enable_attn_tp_input_scattered=True + ) + for layer_id, allow, hands_on in ( + (1, True, True), + (2, True, False), + (1, False, False), + ): + with self.subTest(layer_id=layer_id, allow_reduce_scatter=allow): + communicator = build( + layer_facts(layer_id, 3), parallel, allow_reduce_scatter=allow + ) + steps = communicator._input_scattered_steps + self.assertIs(steps.ffn_output.leaves_for_reduce_scatter, hands_on) + + # --------------------------------------------------------------------------- # Two consecutive layers, rank by rank. diff --git a/test/registered/unit/layers/test_layer_communicator_fusion_gate.py b/test/registered/unit/layers/test_layer_communicator_fusion_gate.py index 6020292ce4..ef9ec60cde 100644 --- a/test/registered/unit/layers/test_layer_communicator_fusion_gate.py +++ b/test/registered/unit/layers/test_layer_communicator_fusion_gate.py @@ -6,7 +6,13 @@ from unittest.mock import MagicMock, patch import torch from sglang.srt.layers import communicator as comm -from sglang.srt.layers.boundary_layout import Layout, StageOutput, SumGroup +from sglang.srt.layers.boundary_layout import ( + Layout, + StageOutput, + SumGroup, + TokenAxis, + sequence_parallel_layer_sides, +) from sglang.srt.layers.communicator import LayerCommunicator, ScatterMode from sglang.srt.layers.moe import ( can_merge_post_experts_all_reduce, @@ -21,6 +27,19 @@ from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=11, suite="base-a-test-cpu") +def sp_region_steps(): + """The steps a layer runs while a LayerNorm SP region is active.""" + return comm._select_boundary_steps( + sequence_parallel_layer_sides( + axis_sizes={ + TokenAxis.ATTN_DP: 1, + TokenAxis.ATTN_CP: 1, + TokenAxis.ATTN_TP_SCATTER: 2, + } + ) + ) + + def _fake_communicator(mlp_mode=ScatterMode.TP_ATTN_FULL): communicator = LayerCommunicator.__new__(LayerCommunicator) communicator._speculative_algo = None @@ -329,7 +348,8 @@ class TestDeferFfnReduction(CustomTestCase): communicator = _fake_communicator() communicator.is_last_layer = is_last_layer communicator._postprocess_scatters_to_local_tokens = scatters_to_local_tokens - communicator._sp_region = sp_active + communicator._sp_steps = sp_region_steps() if sp_active else None + communicator._input_scattered_steps = None communicator._ffn_output = StageOutput( Layout(frozenset()), group=SumGroup.MOE_OUTPUT, diff --git a/test/registered/unit/layers/test_layernorm_sp.py b/test/registered/unit/layers/test_layernorm_sp.py index 121a08d195..0df066fa00 100644 --- a/test/registered/unit/layers/test_layernorm_sp.py +++ b/test/registered/unit/layers/test_layernorm_sp.py @@ -14,6 +14,10 @@ import torch from sglang.srt.arg_groups.layernorm_sp_hook import validate_layernorm_sp from sglang.srt.layers import communicator as comm from sglang.srt.layers import layernorm_sp +from sglang.srt.layers.boundary_layout import ( + TokenAxis, + sequence_parallel_layer_sides, +) from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.runtime_context import ( get_flags, @@ -28,6 +32,19 @@ from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=9, suite="base-a-test-cpu") +def sp_region_steps(): + """The steps a layer runs while a LayerNorm SP region is active.""" + return comm._select_boundary_steps( + sequence_parallel_layer_sides( + axis_sizes={ + TokenAxis.ATTN_DP: 1, + TokenAxis.ATTN_CP: 1, + TokenAxis.ATTN_TP_SCATTER: 2, + } + ) + ) + + def _initialize(*, enable=True, arch="Qwen3ForCausalLM"): publish( ServerArgs(model_path="dummy", enable_layernorm_sp=enable), role="tokenizer" @@ -148,7 +165,8 @@ class TestSpRegionSteps(CustomTestCase): def communicator(self, *, first_layer): c = comm.LayerCommunicator.__new__(comm.LayerCommunicator) - c._sp_region = True + c._sp_steps = sp_region_steps() + c._input_scattered_steps = None c.layer_scatter_modes = SimpleNamespace(is_first_layer=first_layer) c._attn_input_fusions = () c.input_layernorm = _Norm() diff --git a/test/registered/unit/layers/test_postprocess_reduce_scatterv.py b/test/registered/unit/layers/test_postprocess_reduce_scatterv.py index 2769a055e3..ecb6245b21 100644 --- a/test/registered/unit/layers/test_postprocess_reduce_scatterv.py +++ b/test/registered/unit/layers/test_postprocess_reduce_scatterv.py @@ -140,7 +140,8 @@ class TestPostprocessReduceScatterv(CustomTestCase): def test_postprocess_passes_the_layer_sparsity(self): seen = {} communicator = LayerCommunicator.__new__(LayerCommunicator) - communicator._sp_region = False + communicator._sp_steps = None + communicator._input_scattered_steps = None communicator._context = None communicator._postprocess_scatters_to_local_tokens = False communicator.allow_reduce_scatter = False @@ -157,7 +158,8 @@ class TestPostprocessReduceScatterv(CustomTestCase): for step in (object(), None): with self.subTest(reduce_scatter=step is not None): communicator = LayerCommunicator.__new__(LayerCommunicator) - communicator._sp_region = False + communicator._sp_steps = None + communicator._input_scattered_steps = None communicator._postprocess_scatters_to_local_tokens = True communicator._postprocess_dp_step = lambda forward_batch: step with patch.object( diff --git a/test/registered/unit/layers/test_prepare_attn_steps.py b/test/registered/unit/layers/test_prepare_attn_steps.py index 5fa3b00323..4a5fe39d3e 100644 --- a/test/registered/unit/layers/test_prepare_attn_steps.py +++ b/test/registered/unit/layers/test_prepare_attn_steps.py @@ -42,7 +42,8 @@ class Norm: def communicator(norm): c = comm.LayerCommunicator.__new__(comm.LayerCommunicator) c.input_layernorm = norm - c._sp_region = False + c._sp_steps = None + c._input_scattered_steps = None c.qkv_latent_func = None c._context = None c.enable_fused_ar_quant = False diff --git a/test/registered/unit/lora/test_triton_dp_attention_unit.py b/test/registered/unit/lora/test_triton_dp_attention_unit.py index 586bad4450..d26320e4ed 100644 --- a/test/registered/unit/lora/test_triton_dp_attention_unit.py +++ b/test/registered/unit/lora/test_triton_dp_attention_unit.py @@ -222,7 +222,8 @@ def test_communicator_publishes_layout_at_each_transition( communicator._publish_lora_layout = publish_lora_layout communicator.layer_scatter_modes = SimpleNamespace(mlp_mode=mlp_mode) communicator._context = SimpleNamespace() - communicator._sp_region = False + communicator._sp_steps = None + communicator._input_scattered_steps = None communicator.post_attention_layernorm = None communicator.input_layernorm = lambda x: x communicator.qkv_latent_func = None diff --git a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py index f52516e334..19620976f2 100644 --- a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py +++ b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py @@ -57,6 +57,8 @@ class DeferringLayer(nn.Module): self.layer_communicator._ffn_sum_moves_to_next_layer = lambda batch, **_: defer self.layer_communicator.is_last_layer = False self.layer_communicator._postprocess_scatters_to_local_tokens = False + self.layer_communicator._sp_steps = None + self.layer_communicator._input_scattered_steps = None self.layer_communicator.ffn_reduction_group = lambda: GROUP self.layer_communicator._ffn_leaves_sum_to_reduce_scatter = ( lambda batch, dp_step: False