From 446d181e23d05554cfd5534469bbdfbe56d4f673 Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Sun, 27 Sep 2026 16:46:50 -0700 Subject: [PATCH] [Refactor] Choose the boundaries of plain-TP dense layers and MoE layers from declarations (#41417) --- python/sglang/srt/layers/boundary_layout.py | 65 ++- python/sglang/srt/layers/communicator.py | 159 +++++-- .../srt/layers/moe/cutedsl_ar_fusion.py | 6 +- .../unit/layers/moe/test_cutedsl_ar_fusion.py | 11 +- ...y.py => test_declared_decoder_boundary.py} | 406 ++++++++++++++---- 5 files changed, 490 insertions(+), 157 deletions(-) rename test/registered/unit/layers/{test_declared_dense_boundary.py => test_declared_decoder_boundary.py} (65%) diff --git a/python/sglang/srt/layers/boundary_layout.py b/python/sglang/srt/layers/boundary_layout.py index 61bc0306ab..6afdf94d0a 100644 --- a/python/sglang/srt/layers/boundary_layout.py +++ b/python/sglang/srt/layers/boundary_layout.py @@ -56,6 +56,9 @@ class StageInput(msgspec.Struct, frozen=True): compute group does not span.""" layout: Layout + # Token axes the consumer gathers over itself when its input arrives + # sharded over them. + gathers_itself: FrozenSet[TokenAxis] = frozenset() class StageOutput(msgspec.Struct, frozen=True): @@ -87,32 +90,58 @@ class DecoderLayerSides(msgspec.Struct, frozen=True): """The declarations both sides of a decoder layer's boundaries are chosen from.""" - # The rows the layer takes and hands on; the residual stays on them. - layer_rows: Layout + # The rows the layer's input and residual arrive on. + input_rows: Layout attention: StageInput attention_output: StageOutput ffn: StageInput ffn_output: StageOutput + # The rows the residual is on while the FFN runs. + ffn_residual_rows: Layout + # The rows the layer hands on. + output_rows: Layout -def dense_decoder_layer_sides( +def decoder_layer_sides( *, axis_sizes: Mapping[TokenAxis, int], + ffn_on_local_rows: bool, + previous_on_local_rows: bool, + is_last_layer: bool, + attention_gathers_local_rows: bool, + ffn_group: SumGroup, leaves_for_next_layer: bool, leaves_for_reduce_scatter: bool, + leaves_for_reduce_scatterv: bool, ) -> DecoderLayerSides: - """An attention followed by a dense MLP on the TP group, derived from the - groups each computes over.""" + """An attention followed by an FFN, derived from the groups each computes + over. The FFN runs either on the TP group (a dense MLP, or a MoE not + dispatched per DP shard) or on this rank's local rows (a MoE dispatched per + DP shard, which completes its own combine).""" # Attention computes over the attention-TP ranks of one DP (and CP) shard. attention = Layout.sharded_over( TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes ) - # The TP group spans every token axis, so its MLP needs every row. - ffn = Layout.sharded_over(axis_sizes=axis_sizes) + # Each attention-TP rank's own slice of those rows. + local = Layout.sharded_over( + TokenAxis.ATTN_DP, + TokenAxis.ATTN_CP, + TokenAxis.ATTN_TP_SCATTER, + axis_sizes=axis_sizes, + ) + # The TP group spans every token axis, so an FFN on it needs every row. + ffn = local if ffn_on_local_rows else Layout.sharded_over(axis_sizes=axis_sizes) attention_tp = axis_sizes[TokenAxis.ATTN_TP_SCATTER] > 1 return DecoderLayerSides( - layer_rows=attention, - attention=StageInput(attention), + input_rows=local if previous_on_local_rows else attention, + attention=StageInput( + attention, + gathers_itself=( + frozenset({TokenAxis.ATTN_TP_SCATTER}) + if attention_gathers_local_rows + else frozenset() + ), + ), # The output projection leaves the attention-TP sum to prepare_mlp. attention_output=StageOutput( attention, @@ -120,11 +149,17 @@ def dense_decoder_layer_sides( always_leaves=attention_tp, ), ffn=StageInput(ffn), - ffn_output=StageOutput( - ffn, - group=SumGroup.TP, - leaves_for_next_layer=leaves_for_next_layer, - leaves_for_reduce_scatter=leaves_for_reduce_scatter, - leaves_for_reduce_scatterv=leaves_for_reduce_scatter, + ffn_output=( + StageOutput(ffn) + if ffn_on_local_rows + else StageOutput( + ffn, + group=ffn_group, + leaves_for_next_layer=leaves_for_next_layer, + leaves_for_reduce_scatter=leaves_for_reduce_scatter, + leaves_for_reduce_scatterv=leaves_for_reduce_scatterv, + ) ), + ffn_residual_rows=local if ffn_on_local_rows else attention, + output_rows=local if ffn_on_local_rows and not is_last_layer else attention, ) diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py index 2dbda21fc6..5980c008c4 100644 --- a/python/sglang/srt/layers/communicator.py +++ b/python/sglang/srt/layers/communicator.py @@ -44,7 +44,7 @@ from sglang.srt.layers.boundary_layout import ( StageOutput, SumGroup, TokenAxis, - dense_decoder_layer_sides, + decoder_layer_sides, ) from sglang.srt.layers.cp.utils import ( is_mla_cp_active, @@ -75,6 +75,7 @@ from sglang.srt.layers.flashinfer_comm_fusion import ( from sglang.srt.layers.moe import ( can_merge_post_experts_all_reduce, get_moe_a2a_backend, + is_moe_input_scattered_across_dp_ranks, post_experts_output_is_complete, post_experts_reduction_group, should_use_dp_reduce_scatterv, @@ -773,50 +774,75 @@ class LayerCommunicator: def _declared_sides(self) -> Optional[DecoderLayerSides]: """The declarations this layer's boundaries are chosen from: an - attention and a dense MLP on the TP group under attention DP, without CP - or LayerNorm SP, in a layer that takes the rows a dense layer or the - embedding hands on. None when the steps come from the scatter modes.""" + attention and an FFN, without CP, LayerNorm SP or input-scattered + attention, in a layer whose previous layer is known. None when the + steps come from the scatter modes.""" modes = self.layer_scatter_modes parallel = get_parallel() + # A MoE dispatched per DP shard computes on this rank's local rows and + # hands its layer's output on there. + moe_on_local_rows = is_moe_input_scattered_across_dp_ranks() if not ( self._takes_declared_boundaries - and parallel.attn_dp_size > 1 and parallel.attn_cp_size == 1 and not layernorm_sp.layernorm_sp_enabled() - and not modes.is_layer_sparse + and not parallel.enable_attn_tp_input_scattered and not enable_moe_dense_fully_dp() - and (modes.is_first_layer or modes.is_previous_layer_sparse is False) + and (modes.is_first_layer or modes.is_previous_layer_sparse is not None) ): return None - return dense_decoder_layer_sides( + return decoder_layer_sides( axis_sizes={ TokenAxis.ATTN_DP: parallel.attn_dp_size, TokenAxis.ATTN_CP: parallel.attn_cp_size, TokenAxis.ATTN_TP_SCATTER: parallel.attn_tp_size, }, + ffn_on_local_rows=modes.is_layer_sparse and moe_on_local_rows, + previous_on_local_rows=( + not modes.is_first_layer + and modes.is_previous_layer_sparse + and moe_on_local_rows + ), + is_last_layer=modes.is_last_layer, + attention_gathers_local_rows=_use_ag_after_qlora, + ffn_group=SumGroup.MOE_OUTPUT if modes.is_layer_sparse else SumGroup.TP, leaves_for_next_layer=self.allow_deferred_ffn_reduction, leaves_for_reduce_scatter=self.allow_reduce_scatter, + # A MoE block leaves its sum to reduce_scatterv whenever that combine + # applies (should_skip_post_experts_all_reduce). + leaves_for_reduce_scatterv=( + self.allow_reduce_scatter or modes.is_layer_sparse + ), ) def _select_declared_boundaries(self, sides: DecoderLayerSides) -> None: """Choose the three boundary sides this layer owns from the declarations: the attention input, the attention -> FFN steps, and the - FFN output's way back to the layer's rows.""" + FFN output's way on to the rows the layer hands on.""" self._communicate_simple_fn = _select_attention_input_move( - sides.layer_rows, sides.attention + sides.input_rows, sides.attention ) self._mlp_input = _select_ffn_input( sides.attention_output, - residual=sides.layer_rows, + residual=sides.input_rows, + residual_to=sides.ffn_residual_rows, need=sides.ffn, force_layernorm_before_gather=self.force_layernorm_before_dp_gather, + fusions=self._select_mlp_input_fusions(), + ) + self._mlp_input_may_fuse = ( + self._mlp_input_reduce_output_and_update_and_read_residual + in getattr(self._mlp_input, "keywords", {}).get("fusions", ()) ) - # Neither fused kernel runs under attention DP. - self._mlp_input_may_fuse = False self._ffn_output = sides.ffn_output - self._postprocess_scatters_to_local_tokens = _ffn_output_returns_over_dp( - sides.ffn_output, sides.layer_rows + ( + self._postprocess_scatters_to_local_tokens, + postprocess, + ) = _select_ffn_output_move( + sides.ffn_output, residual=sides.ffn_residual_rows, to=sides.output_rows ) + if postprocess is not None: + self._communicate_summable_tensor_pair_fn = postprocess self._ffn_sum_is_movable = sides.ffn_output.group is not None def _select_boundaries_from_scatter_modes(self) -> None: @@ -1097,9 +1123,7 @@ class LayerCommunicator: ) # Neither fused kernel runs under attention DP. fusions = ( - self._select_mlp_input_fusions(residual_input_mode) - if self._context.attn_dp_size == 1 - else () + self._select_mlp_input_fusions() if self._context.attn_dp_size == 1 else () ) steps = partial( _mlp_input_gather, @@ -1112,9 +1136,7 @@ class LayerCommunicator: self._mlp_input_reduce_output_and_update_and_read_residual in fusions, ) - def _select_mlp_input_fusions( - self, residual_input_mode: ScatterMode - ) -> Tuple[Callable, ...]: + def _select_mlp_input_fusions(self) -> Tuple[Callable, ...]: """The fused kernels that complete the attention output's all-reduce together with the residual update and the post-attention norm, in the order they are tried. Each takes (hidden_states, residual, forward_batch) @@ -1803,29 +1825,34 @@ def _sum_group(group: SumGroup) -> GroupCoordinator: def _select_attention_input_move(rows: Layout, need: StageInput) -> Callable: - """How the rows a layer takes become its attention's input.""" - if rows == need.layout: - return CommunicateSimpleFn._trivial - raise NotImplementedError(f"{rows=} {need=}") + """How the rows a layer takes become its attention's input: as they are, + or gathered over attention TP from each rank's slice, unless the attention + gathers them itself.""" + gathered = rows.sharded - need.layout.sharded - need.gathers_itself + if not need.layout.sharded <= rows.sharded or gathered not in ( + frozenset(), + {TokenAxis.ATTN_TP_SCATTER}, + ): + raise NotImplementedError(f"{rows=} {need=}") + if gathered: + return CommunicateSimpleFn._scattered_to_tp_attn_full + return CommunicateSimpleFn._trivial def _select_ffn_input( produced: StageOutput, *, residual: Layout, + residual_to: Layout, need: StageInput, force_layernorm_before_gather: bool, + fusions: Tuple[Callable, ...], ) -> Callable: """The steps from the attention output to the FFN input: complete the - attention-TP sum, add the residual and normalize, and gather the rows the - FFN's group needs, with the residual left on the attention's rows.""" - gathered = produced.layout.sharded - need.layout.sharded - if ( - residual != produced.layout - or not need.layout.sharded <= produced.layout.sharded - or gathered != {TokenAxis.ATTN_DP} - ): - raise NotImplementedError(f"{produced=} {residual=} {need=}") + attention-TP sum, move the residual to the rows it has while the FFN runs, + add it and normalize, and bring the rows to what the FFN's group needs: a + gather over attention DP, or each rank's own slice. The fused kernels in + ``fusions`` are tried first when nothing is gathered or sliced.""" # What the attention output owes decides the steps: the attention-TP sum, # always left by the output projection, or nothing. owes_attention_tp = produced.group is SumGroup.ATTN_TP @@ -1834,28 +1861,70 @@ def _select_ffn_input( SumGroup.ATTN_TP, ): raise NotImplementedError(f"{produced=}") + gathered = produced.layout.sharded - need.layout.sharded + sliced = need.layout.sharded - produced.layout.sharded + if sliced: + # Each attention-TP rank takes its own slice: the reduce-scatter + # completes the attention-TP sum and slices in one collective. + if ( + sliced != {TokenAxis.ATTN_TP_SCATTER} + or gathered + or not owes_attention_tp + or residual_to != need.layout + or residual not in (produced.layout, need.layout) + ): + raise NotImplementedError(f"{produced=} {residual=} {need=}") + return partial(_mlp_input_scatter, scatters_residual=residual != residual_to) + if ( + residual_to != produced.layout + or gathered not in (frozenset(), {TokenAxis.ATTN_DP}) + or residual.sharded - residual_to.sharded + not in (frozenset(), {TokenAxis.ATTN_TP_SCATTER}) + ): + raise NotImplementedError(f"{produced=} {residual=} {need=}") + # A residual arriving on each rank's slice is gathered back first. + gathers_residual = residual != residual_to + if not gathered: + if not owes_attention_tp: + return _mlp_input_norm + return partial( + _mlp_input_without_dp, gathers_residual=gathers_residual, fusions=fusions + ) # The partial order adds the residual on attention-TP rank 0 before the DP # gather's collective completes that sum, which only a plain residual add # allows. if owes_attention_tp and not force_layernorm_before_gather: - return partial(_mlp_input_dp_partial, gathers_residual=False) + return partial(_mlp_input_dp_partial, gathers_residual=gathers_residual) return partial( _mlp_input_dp_replicate, - gathers_residual=False, + gathers_residual=gathers_residual, reduces_attention_tp=owes_attention_tp, ) -def _ffn_output_returns_over_dp(produced: StageOutput, rows: Layout) -> bool: - """Whether the FFN output goes back to the layer's rows by undoing the - attention-DP gather.""" - returned = rows.sharded - produced.layout.sharded - if not produced.layout.sharded <= rows.sharded or returned not in ( - frozenset(), - {TokenAxis.ATTN_DP}, +def _select_ffn_output_move( + produced: StageOutput, *, residual: Layout, to: Layout +) -> Tuple[bool, Optional[Callable]]: + """How the FFN output reaches the rows the layer hands on: whether it goes + back by undoing the attention-DP gather (the FFN exit and postprocess run + that step), or else the postprocess that moves it, None when there is none + to choose here.""" + if produced.layout == residual: + if to == residual: + return False, CommunicateSummableTensorPairFn._trivial + if to.sharded == residual.sharded - {TokenAxis.ATTN_TP_SCATTER}: + # Each rank's slice back to the attention's rows: fold the residual + # into the output, then gather over attention TP. + return False, CommunicateSummableTensorPairFn._gather + raise NotImplementedError(f"{produced=} {residual=} {to=}") + returned = residual.sharded - produced.layout.sharded + if ( + to != residual + or not produced.layout.sharded <= residual.sharded + or returned != {TokenAxis.ATTN_DP} ): - raise NotImplementedError(f"{produced=} {rows=}") - return bool(returned) + raise NotImplementedError(f"{produced=} {residual=} {to=}") + return True, None class MlpInputKind(Enum): diff --git a/python/sglang/srt/layers/moe/cutedsl_ar_fusion.py b/python/sglang/srt/layers/moe/cutedsl_ar_fusion.py index 1cf9b5f57a..5eb3055004 100644 --- a/python/sglang/srt/layers/moe/cutedsl_ar_fusion.py +++ b/python/sglang/srt/layers/moe/cutedsl_ar_fusion.py @@ -239,11 +239,11 @@ class CuteDSLFusionLayerCommunicator(LayerCommunicator): gamma=_fused_norm_gamma(self.input_layernorm), ) - def _select_mlp_input_fusions(self, residual_input_mode): - fusions = super()._select_mlp_input_fusions(residual_input_mode) + def _select_mlp_input_fusions(self): + fusions = super()._select_mlp_input_fusions() parallel = get_parallel() if ( - residual_input_mode is ScatterMode.TP_ATTN_FULL + self.layer_scatter_modes.layer_input_mode is ScatterMode.TP_ATTN_FULL and parallel.attn_tp_size == parallel.tp_size and _fused_norm_gamma(self.post_attention_layernorm) is not None ): diff --git a/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py b/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py index 9dbd1fcae2..e6769dbc51 100644 --- a/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py +++ b/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py @@ -119,12 +119,15 @@ def test_cutedsl_entries_come_before_the_base_fused_kernel(): f"{_MODULE}.get_parallel", return_value=SimpleNamespace(attn_tp_size=2, tp_size=2), ): - assert comm._select_mlp_input_fusions(ScatterMode.TP_ATTN_FULL) == ( - cutedsl, - base, + comm.layer_scatter_modes = SimpleNamespace( + layer_input_mode=ScatterMode.TP_ATTN_FULL ) + assert comm._select_mlp_input_fusions() == (cutedsl, base) # A scattered residual is gathered first, which the workspace does not do. - assert comm._select_mlp_input_fusions(ScatterMode.SCATTERED) == (base,) + comm.layer_scatter_modes = SimpleNamespace( + layer_input_mode=ScatterMode.SCATTERED + ) + assert comm._select_mlp_input_fusions() == (base,) def test_a_replicated_output_producer_keeps_its_own_all_reduce(eligible): diff --git a/test/registered/unit/layers/test_declared_dense_boundary.py b/test/registered/unit/layers/test_declared_decoder_boundary.py similarity index 65% rename from test/registered/unit/layers/test_declared_dense_boundary.py rename to test/registered/unit/layers/test_declared_decoder_boundary.py index 9ba2111169..16eff6ad24 100644 --- a/test/registered/unit/layers/test_declared_dense_boundary.py +++ b/test/registered/unit/layers/test_declared_decoder_boundary.py @@ -1,5 +1,7 @@ -"""The boundaries of an attention and a dense MLP on the TP group under attention -DP, chosen from both sides' declarations. +"""The boundaries of an attention and an FFN, chosen from both sides' +declarations. The FFN runs on the TP group (a dense MLP, or a MoE not +dispatched per DP shard) or on each attention-TP rank's slice of its DP shard's +rows (a MoE dispatched per DP shard). The numeric checks run every rank of a DP x attention-TP world as a thread over fake collectives that wait for all members of their group, build the layers' @@ -21,7 +23,7 @@ from sglang.srt.layers import communicator as comm from sglang.srt.layers.boundary_layout import ( SumGroup, TokenAxis, - dense_decoder_layer_sides, + decoder_layer_sides, ) from sglang.srt.layers.communicator import ( LayerCommunicator, @@ -57,6 +59,7 @@ def parallel_of(*, attn_dp, attn_tp, attn_cp=1, **overrides): moe_dp_size=1, dwdp_size=1, enable_dp_attention=attn_dp > 1, + enable_attn_tp_input_scattered=False, tp_group=SimpleNamespace(name="tp"), attn_tp_group=SimpleNamespace(name="attn_tp"), ) @@ -65,7 +68,7 @@ def parallel_of(*, attn_dp, attn_tp, attn_cp=1, **overrides): @contextmanager -def planning(parallel, *, sp=False): +def planning(parallel, *, sp=False, a2a=False): """What layer planning and communicator construction read, without the process-wide parallel state. ``parallel`` may be a callable, for a per-thread parallel state.""" @@ -80,8 +83,11 @@ def planning(parallel, *, sp=False): comm, "get_spec", lambda: SimpleNamespace(speculative_algorithm=None) ), patch.object( - comm, "get_moe_a2a_backend", lambda: SimpleNamespace(is_none=lambda: True) + comm, + "get_moe_a2a_backend", + lambda: SimpleNamespace(is_none=lambda: not a2a), ), + patch.object(comm, "is_moe_input_scattered_across_dp_ranks", lambda: a2a), patch.object( comm, "should_use_flashinfer_cutlass_moe_fp4_allgather", lambda: False ), @@ -102,6 +108,13 @@ class Norm: return s * 2, s +class FusableNorm(Norm): + """A norm with the fused all-reduce + add + norm method.""" + + def forward_with_allreduce_fusion(self, *args, **kwargs): + raise AssertionError("not run here") + + class FactsOnly: """Scatter modes whose mode fields fail when read: only the layer facts are there.""" @@ -113,17 +126,40 @@ class FactsOnly: raise AssertionError(f"read scatter mode {name!r}") -def dense_layer_facts(layer_id, num_layers, *, previous_sparse=False): +def layer_facts(layer_id, num_layers, *, sparse=False, previous_sparse=False): return FactsOnly( - is_layer_sparse=False, + is_layer_sparse=sparse, is_first_layer=layer_id == 0, is_last_layer=layer_id == num_layers - 1, is_previous_layer_sparse=None if layer_id == 0 else previous_sparse, ) -def build(modes, parallel, *, cls=LayerCommunicator, sp=False, **kwargs): - with planning(parallel, sp=sp): +def sides_of( + axis_sizes, + *, + sparse=False, + a2a=False, + previous_a2a=False, + last=False, + leaves_next=False, + leaves_rs=False, +): + return decoder_layer_sides( + axis_sizes=axis_sizes, + ffn_on_local_rows=sparse and a2a, + previous_on_local_rows=previous_a2a, + is_last_layer=last, + attention_gathers_local_rows=False, + ffn_group=SumGroup.MOE_OUTPUT if sparse else SumGroup.TP, + leaves_for_next_layer=leaves_next, + leaves_for_reduce_scatter=leaves_rs, + leaves_for_reduce_scatterv=leaves_rs or sparse, + ) + + +def build(modes, parallel, *, cls=LayerCommunicator, sp=False, a2a=False, **kwargs): + with planning(parallel, sp=sp, a2a=a2a): return cls( layer_scatter_modes=modes, input_layernorm=Norm(), @@ -132,8 +168,10 @@ def build(modes, parallel, *, cls=LayerCommunicator, sp=False, **kwargs): ) -def planned_modes(layer_id, num_layers, *, sparse, previous_sparse, parallel): - with planning(parallel): +def planned_modes( + layer_id, num_layers, *, sparse, previous_sparse, parallel, a2a=False +): + with planning(parallel, a2a=a2a): return LayerScatterModes.init_new( layer_id=layer_id, num_layers=num_layers, @@ -148,13 +186,37 @@ class TestDeclarationsMatchScatterModes(CustomTestCase): layouts of the modes layer planning gives the same layers.""" def test_every_attention_dp_and_tp(self): - for attn_dp, attn_tp, layer_id in itertools.product( - (2, 4, 8), (1, 2, 4), (0, 1, 3) + for ( + attn_dp, + attn_tp, + layer_id, + sparse, + previous_sparse, + a2a, + ) in itertools.product( + (1, 2, 4, 8), + (1, 2, 4), + (0, 1, 3), + (False, True), + (False, True), + (False, True), ): - with self.subTest(attn_dp=attn_dp, attn_tp=attn_tp, layer_id=layer_id): + with self.subTest( + attn_dp=attn_dp, + attn_tp=attn_tp, + layer_id=layer_id, + sparse=sparse, + previous_sparse=previous_sparse, + a2a=a2a, + ): parallel = parallel_of(attn_dp=attn_dp, attn_tp=attn_tp) modes = planned_modes( - layer_id, 4, sparse=False, previous_sparse=False, parallel=parallel + layer_id, + 4, + sparse=sparse, + previous_sparse=previous_sparse, + parallel=parallel, + a2a=a2a, ) sizes = { TokenAxis.ATTN_DP: attn_dp, @@ -164,29 +226,37 @@ class TestDeclarationsMatchScatterModes(CustomTestCase): layouts = scatter_mode_layouts( attn_dp_size=attn_dp, attn_cp_size=1, attn_tp_size=attn_tp ) - sides = dense_decoder_layer_sides( - axis_sizes=sizes, - leaves_for_next_layer=True, - leaves_for_reduce_scatter=True, + sides = sides_of( + sizes, + sparse=sparse, + a2a=a2a, + previous_a2a=layer_id > 0 and previous_sparse and a2a, + last=layer_id == 3, + leaves_next=True, + leaves_rs=True, ) - self.assertEqual(sides.layer_rows, layouts[modes.layer_input_mode]) - self.assertEqual(sides.layer_rows, layouts[modes.middle_residual_mode]) - self.assertEqual(sides.layer_rows, layouts[modes.layer_output_mode]) + self.assertEqual(sides.input_rows, layouts[modes.layer_input_mode]) + self.assertEqual( + sides.ffn_residual_rows, layouts[modes.middle_residual_mode] + ) + self.assertEqual(sides.output_rows, layouts[modes.layer_output_mode]) self.assertEqual(sides.attention.layout, layouts[modes.attn_mode]) self.assertEqual(sides.ffn.layout, layouts[modes.mlp_mode]) - self.assertEqual(sides.attention_output.layout, sides.layer_rows) + self.assertEqual( + sides.attention_output.layout, layouts[modes.attn_mode] + ) self.assertEqual(sides.ffn_output.layout, sides.ffn.layout) + # A MoE dispatched per DP shard hands on a complete output. + self.assertIs(sides.ffn_output.group is None, sparse and a2a) def test_the_attention_output_owes_the_attention_tp_sum(self): for attn_tp in (1, 2): - sides = dense_decoder_layer_sides( - axis_sizes={ + sides = sides_of( + { TokenAxis.ATTN_DP: 2, TokenAxis.ATTN_CP: 1, TokenAxis.ATTN_TP_SCATTER: attn_tp, - }, - leaves_for_next_layer=False, - leaves_for_reduce_scatter=False, + } ) owed = attn_tp > 1 self.assertIs( @@ -202,16 +272,18 @@ class TestWhichLayersUseDeclarations(CustomTestCase): declarations, or none do.""" def declared(self, communicator): - return communicator._mlp_input.func in ( + return getattr(communicator._mlp_input, "func", None) in ( comm._mlp_input_dp_partial, comm._mlp_input_dp_replicate, + comm._mlp_input_without_dp, + comm._mlp_input_scatter, ) def test_a_dense_model(self): parallel = parallel_of(attn_dp=2, attn_tp=2) for layer_id in range(3): with self.subTest(layer_id=layer_id): - communicator = build(dense_layer_facts(layer_id, 3), parallel) + communicator = build(layer_facts(layer_id, 3), parallel) self.assertTrue(self.declared(communicator)) self.assertIs( communicator._communicate_simple_fn, @@ -219,6 +291,36 @@ class TestWhichLayersUseDeclarations(CustomTestCase): ) self.assertTrue(communicator._postprocess_scatters_to_local_tokens) + def test_plain_tp(self): + for norm, fuses in ((Norm, False), (FusableNorm, True)): + with self.subTest(norm=norm.__name__): + with planning(parallel_of(attn_dp=1, attn_tp=2)): + communicator = LayerCommunicator( + layer_scatter_modes=layer_facts(1, 3), + input_layernorm=norm(), + post_attention_layernorm=norm(), + ) + self.assertIs(communicator._mlp_input.func, comm._mlp_input_without_dp) + entry = ( + communicator._mlp_input_reduce_output_and_update_and_read_residual + ) + self.assertEqual( + communicator._mlp_input.keywords["fusions"], + (entry,) if fuses else (), + ) + self.assertIs(communicator._mlp_input_may_fuse, fuses) + self.assertFalse(communicator._postprocess_scatters_to_local_tokens) + self.assertIs( + communicator._communicate_summable_tensor_pair_fn, + comm.CommunicateSummableTensorPairFn._trivial, + ) + # One rank: the attention output is complete, only the norm is left. + one_rank = parallel_of(attn_dp=1, attn_tp=1) + single = build(layer_facts(1, 3), one_rank) + with planning(one_rank): + self.assertIsNotNone(single._declared_sides()) + self.assertIs(single._mlp_input, comm._mlp_input_norm) + def test_the_order_follows_the_attention_output(self): for attn_tp, force, order in ( (2, False, comm._mlp_input_dp_partial), @@ -227,45 +329,99 @@ class TestWhichLayersUseDeclarations(CustomTestCase): ): with self.subTest(attn_tp=attn_tp, force=force): communicator = build( - dense_layer_facts(1, 3), + layer_facts(1, 3), parallel_of(attn_dp=2, attn_tp=attn_tp), force_layernorm_before_dp_gather=force, ) self.assertIs(communicator._mlp_input.func, order) def test_a_dense_first_layer_before_sparse_ones(self): - # DeepSeek-V2-Lite: layer 0 is dense, the rest sparse. parallel = parallel_of(attn_dp=2, attn_tp=2) - first = build( - planned_modes(0, 3, sparse=False, previous_sparse=False, parallel=parallel), + for a2a in (False, True): + with self.subTest(a2a=a2a): + layers = [ + build( + planned_modes( + i, + 4, + sparse=sparse, + previous_sparse=previous, + parallel=parallel, + a2a=a2a, + ), + parallel, + a2a=a2a, + allow_reduce_scatter=True, + ) + for i, (sparse, previous) in enumerate( + ((False, False), (True, False), (True, True), (False, True)) + ) + ] + self.assertEqual([self.declared(layer) for layer in layers], [True] * 4) + if not a2a: + moe = layers[1]._ffn_output + self.assertIs(moe.group, SumGroup.MOE_OUTPUT) + self.assertTrue(moe.leaves_for_reduce_scatterv) + continue + # The first a2a layer slices the residual it takes from a + # dense layer; the next one takes it already sliced. + self.assertEqual( + [layers[i]._mlp_input.func for i in (1, 2)], + [comm._mlp_input_scatter] * 2, + ) + self.assertEqual( + [ + layers[i]._mlp_input.keywords["scatters_residual"] + for i in (1, 2) + ], + [True, False], + ) + self.assertIsNone(layers[1]._ffn_output.group) + self.assertIs( + layers[2]._communicate_simple_fn, + comm.CommunicateSimpleFn._scattered_to_tp_attn_full, + ) + self.assertIs(layers[3]._mlp_input.func, comm._mlp_input_dp_partial) + self.assertTrue(layers[3]._mlp_input.keywords["gathers_residual"]) + + def test_the_last_a2a_layer_folds_the_residual_back(self): + parallel = parallel_of(attn_dp=2, attn_tp=2) + last = build( + planned_modes( + 3, 4, sparse=True, previous_sparse=True, parallel=parallel, a2a=True + ), parallel, - allow_reduce_scatter=True, + a2a=True, ) - second = build( - planned_modes(1, 3, sparse=True, previous_sparse=False, parallel=parallel), - parallel, - allow_reduce_scatter=True, + self.assertIs( + last._communicate_summable_tensor_pair_fn, + comm.CommunicateSummableTensorPairFn._gather, ) - self.assertTrue(self.declared(first)) - self.assertFalse(self.declared(second)) - self.assertIs(second._mlp_input.func, comm._mlp_input_gather) + + def test_an_attention_that_gathers_its_slice_itself(self): + parallel = parallel_of(attn_dp=2, attn_tp=2) + modes = planned_modes( + 2, 4, sparse=True, previous_sparse=True, parallel=parallel, a2a=True + ) + with patch.object(comm, "_use_ag_after_qlora", True): + layer = build(modes, parallel, a2a=True) + self.assertIs(layer._communicate_simple_fn, comm.CommunicateSimpleFn._trivial) def test_layers_that_keep_the_scatter_modes(self): dp = parallel_of(attn_dp=2, attn_tp=2) cases = { - "no attention DP": ( - dense_layer_facts(1, 3), - parallel_of(attn_dp=1, attn_tp=4), + "input-scattered attention": ( + layer_facts(1, 3), + parallel_of(attn_dp=1, attn_tp=4, enable_attn_tp_input_scattered=True), ), "attention CP": ( - dense_layer_facts(1, 3), + layer_facts(1, 3), parallel_of(attn_dp=2, attn_tp=1, attn_cp=2), ), "dense MLP fully DP": ( - dense_layer_facts(1, 3), + layer_facts(1, 3), parallel_of(attn_dp=2, attn_tp=2, moe_dense_tp_size=1), ), - "after a sparse layer": (dense_layer_facts(1, 3, previous_sparse=True), dp), } for name, (facts, parallel) in cases.items(): with self.subTest(name): @@ -287,7 +443,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase): self.assertFalse(self.declared(build(direct, dp))) with planning(dp, sp=True): communicator = LayerCommunicator.__new__(LayerCommunicator) - communicator.layer_scatter_modes = dense_layer_facts(1, 3) + communicator.layer_scatter_modes = layer_facts(1, 3) self.assertIsNone(communicator._declared_sides()) def test_subclasses_that_pick_their_own_steps(self): @@ -311,16 +467,14 @@ class TestTheAttentionOutputDecidesItsSum(CustomTestCase): TokenAxis.ATTN_CP: 1, TokenAxis.ATTN_TP_SCATTER: 2, } - sides = dense_decoder_layer_sides( - axis_sizes=sizes, - leaves_for_next_layer=False, - leaves_for_reduce_scatter=False, - ) + sides = sides_of(sizes) steps = comm._select_ffn_input( produced, - residual=sides.layer_rows, + residual=sides.input_rows, + residual_to=sides.ffn_residual_rows, need=sides.ffn, force_layernorm_before_gather=force, + fusions=(), ) reduced = [] context = SimpleNamespace(attn_tp_size=2, attn_tp_rank=0) @@ -349,15 +503,13 @@ class TestTheAttentionOutputDecidesItsSum(CustomTestCase): return len(reduced), hidden, residual def test_a_complete_output_is_not_summed_again(self): - layout = dense_decoder_layer_sides( - axis_sizes={ + layout = sides_of( + { TokenAxis.ATTN_DP: 2, TokenAxis.ATTN_CP: 1, TokenAxis.ATTN_TP_SCATTER: 2, - }, - leaves_for_next_layer=False, - leaves_for_reduce_scatter=False, - ).layer_rows + } + ).input_rows complete = comm.StageOutput(layout) for force in (False, True): with self.subTest(force=force): @@ -367,15 +519,13 @@ class TestTheAttentionOutputDecidesItsSum(CustomTestCase): torch.testing.assert_close(hidden, torch.full((1, HIDDEN), 20.0)) def test_an_owed_output_is_summed_once(self): - layout = dense_decoder_layer_sides( - axis_sizes={ + layout = sides_of( + { TokenAxis.ATTN_DP: 2, TokenAxis.ATTN_CP: 1, TokenAxis.ATTN_TP_SCATTER: 2, - }, - leaves_for_next_layer=False, - leaves_for_reduce_scatter=False, - ).layer_rows + } + ).input_rows owed = comm.StageOutput(layout, group=SumGroup.ATTN_TP, always_leaves=True) reductions, hidden, residual = self.run_steps(owed, force=True) self.assertEqual(reductions, 1) @@ -383,15 +533,13 @@ class TestTheAttentionOutputDecidesItsSum(CustomTestCase): torch.testing.assert_close(residual, torch.full((1, HIDDEN), 17.0)) def test_declarations_the_steps_cannot_run_are_rejected(self): - layout = dense_decoder_layer_sides( - axis_sizes={ + layout = sides_of( + { TokenAxis.ATTN_DP: 2, TokenAxis.ATTN_CP: 1, TokenAxis.ATTN_TP_SCATTER: 2, - }, - leaves_for_next_layer=False, - leaves_for_reduce_scatter=False, - ).layer_rows + } + ).input_rows for produced in ( # Leaves the attention-TP sum only when asked, like a mixer exit. comm.StageOutput(layout, group=SumGroup.ATTN_TP), @@ -408,7 +556,7 @@ class TestTheAttentionOutputDecidesItsSum(CustomTestCase): # --------------------------------------------------------------------------- -# Two consecutive dense layers, rank by rank. +# Two consecutive layers, rank by rank. class World: @@ -535,7 +683,7 @@ def state(): @contextmanager -def running(*, reduce_scatterv): +def running(*, reduce_scatterv, a2a=False): replaced = { "get_forward": lambda: state().flags, "attention_tensor_model_parallel_all_reduce": lambda x: ( @@ -551,10 +699,16 @@ def running(*, reduce_scatterv): "dp_gather_partial": partial(fake_dp_gather, is_partial=True), "dp_scatter": fake_dp_scatter, "dp_reduce_scatter_tensor": fake_dp_reduce_scatter_tensor, + "attn_tp_reduce_scatter_tensor": lambda out, x: ( + state().parallel.attn_tp_group.reduce_scatter_tensor(out, x) + ), + "attn_tp_all_gather_into_tensor": lambda out, x: ( + state().parallel.attn_tp_group.all_gather_into_tensor(out, x) + ), "get_dp_global_num_tokens": lambda: state().dp_rows, "should_use_dp_reduce_scatterv": lambda: reduce_scatterv, "can_use_dp_reduce_scatter": lambda: True, - "is_dp_attention_enabled": lambda: True, + "is_dp_attention_enabled": lambda: state().parallel.attn_dp_size > 1, "apply_flashinfer_allreduce_fusion": lambda n: False, "post_experts_output_is_complete": lambda **kw: False, "post_experts_reduction_group": lambda: state().parallel.tp_group, @@ -569,7 +723,7 @@ def running(*, reduce_scatterv): "is_allocation_symmetric": lambda: False, } with ExitStack() as stack: - stack.enter_context(planning(lambda: state().parallel)) + stack.enter_context(planning(lambda: state().parallel, a2a=a2a)) for name, value in replaced.items(): stack.enter_context(patch.object(comm, name, value)) yield @@ -593,8 +747,22 @@ def dense_mlp(x, state): return state.parallel.tp_group.all_reduce(partial_sum) +def moe(x, state): + """A MoE block not dispatched per DP shard: 5 * x, reduced over the MoE + output's group unless a published flag asks it to leave the sum, or the + attention-DP reduce_scatterv takes it, as the MoE blocks do.""" + partial_sum = 5 * x * WEIGHTS[state.parallel.tp_size][state.parallel.tp_rank] + if ( + state.flags.fuse_mlp_allreduce + or state.flags.mlp_reduce_scatter + or comm.should_use_dp_reduce_scatterv() + ): + return partial_sum + return comm.post_experts_reduction_group().all_reduce(partial_sum) + + def reference(x): - """(hidden, residual) after two layers of Norm / attention / dense MLP.""" + """(hidden, residual) after two layers of Norm / attention / FFN.""" residual = x for _ in range(2): hidden = 2 * residual @@ -605,8 +773,19 @@ def reference(x): return ffn, residual - ffn -class TestTwoDenseLayers(CustomTestCase): - def run_world(self, *, attn_dp, attn_tp, rows, padding, reduce_scatterv, leaves): +class TestTwoLayers(CustomTestCase): + def run_world( + self, + *, + attn_dp, + attn_tp, + rows, + padding, + reduce_scatterv, + leaves, + sparse=(False, False), + a2a=False, + ): tp = attn_dp * attn_tp world = World(tp) WORLD[0] = world @@ -638,6 +817,8 @@ class TestTwoDenseLayers(CustomTestCase): ), dp_padding_mode=SimpleNamespace(is_max_len=lambda: max_len), global_dp_buffer_len=global_rows, + # Without attention DP the FFN exit counts this rank's tokens. + input_ids=torch.zeros(rows[0]), ) states = [] for rank in range(tp): @@ -668,7 +849,12 @@ class TestTwoDenseLayers(CustomTestCase): s = states[rank] layers = [ LayerCommunicator( - layer_scatter_modes=dense_layer_facts(i, 2), + layer_scatter_modes=layer_facts( + i, + 2, + sparse=sparse[i], + previous_sparse=i > 0 and sparse[i - 1], + ), input_layernorm=Norm(), post_attention_layernorm=Norm(), allow_reduce_scatter=leaves, @@ -680,46 +866,67 @@ class TestTwoDenseLayers(CustomTestCase): hidden[: s.rows] = embeddings[s.dp] residual = None handed_on = [] - for layer in layers: + for layer_index, layer in enumerate(layers): hidden, residual = layer.prepare_attn(hidden, residual, forward_batch) hidden = attention(hidden, s) hidden, residual = layer.prepare_mlp(hidden, residual, forward_batch) with layer.ffn_exit(forward_batch) as ffn_exit: - hidden = dense_mlp(hidden, s) + if not sparse[layer_index]: + hidden = dense_mlp(hidden, s) + elif a2a: + # On this rank's slice; the combine completes the sum. + hidden = 5 * hidden + else: + hidden = moe(hidden, s) hidden, residual = ffn_exit.finish(hidden, residual) handed_on.append(type(hidden)) hidden, residual = layers[-1].finish_layer_stack( hidden, residual, forward_batch ) - return hidden[: s.rows], residual[: s.rows], handed_on + # The last a2a layer folds the residual into its output. + if residual is not None: + residual = residual[: s.rows] + return hidden[: s.rows], residual, handed_on - with running(reduce_scatterv=reduce_scatterv): + with running(reduce_scatterv=reduce_scatterv, a2a=a2a): results, errors = world.run(states, forward) for rank, error in enumerate(errors): if error is not None: raise AssertionError(f"rank {rank} raised") from error for rank, (hidden, residual, handed_on) in enumerate(results): want_hidden, want_residual = reference(embeddings[states[rank].dp]) + if residual is None: + want_hidden, want_residual = want_hidden + want_residual, None torch.testing.assert_close(hidden, want_hidden, rtol=0, atol=0) - torch.testing.assert_close(residual, want_residual, rtol=0, atol=0) + self.assertIs(residual is None, want_residual is None) + if residual is not None: + torch.testing.assert_close(residual, want_residual, rtol=0, atol=0) # Every rank ran the same collectives. self.assertEqual(len({tuple(s.calls) for s in states if s.dp == 0}), 1) return results def test_rows_come_back_to_each_rank(self): - for (attn_dp, attn_tp), rows, padding, leaves in itertools.product( - ((2, 2), (4, 1), (2, 1)), + for (attn_dp, attn_tp), rows, padding, leaves, sparse in itertools.product( + ((2, 2), (4, 1), (2, 1), (1, 2), (1, 4)), ("ragged", "idle"), ("sum_len", "max_len"), (False, True), + ((False, False), (False, True), (True, True)), ): token_rows = { + ("ragged", 1): [3], + ("idle", 1): [0], ("ragged", 2): [3, 1], ("idle", 2): [2, 0], ("ragged", 4): [2, 3, 1, 1], ("idle", 4): [2, 0, 3, 1], }[rows, attn_dp] - for reduce_scatterv in (False, True) if attn_tp == 1 else (False,): + for reduce_scatterv, a2a in itertools.product( + (False, True) if attn_tp == 1 and attn_dp > 1 else (False,), + (False, True) if any(sparse) else (False,), + ): + if reduce_scatterv and a2a: + continue with self.subTest( attn_dp=attn_dp, attn_tp=attn_tp, @@ -727,6 +934,8 @@ class TestTwoDenseLayers(CustomTestCase): padding=padding, leaves=leaves, reduce_scatterv=reduce_scatterv, + sparse=sparse, + a2a=a2a, ): results = self.run_world( attn_dp=attn_dp, @@ -735,13 +944,30 @@ class TestTwoDenseLayers(CustomTestCase): padding=padding, reduce_scatterv=reduce_scatterv, leaves=leaves, + sparse=sparse, + a2a=a2a, ) first_layer_hands_on = results[0][2][0] # A producer that may leave its sum leaves it for the next # layer's input; one that may not hands on a complete value. + # Without attention DP an empty batch completes its own sum. + # A MoE dispatched per DP shard hands on a complete value. + # Otherwise the reduce-scatter back to this rank's tokens + # can be left to the next layer; the all-reduce only + # without an a2a backend. + has_tokens = attn_dp > 1 or sum(token_rows) > 0 + reduce_scatter_left = attn_dp > 1 and ( + padding == "max_len" or reduce_scatterv + ) + owes = ( + leaves + and has_tokens + and not (sparse[0] and a2a) + and (reduce_scatter_left or not a2a) + ) self.assertIs( first_layer_hands_on, - UnreducedOutput if leaves else torch.Tensor, + UnreducedOutput if owes else torch.Tensor, )