From 4fac02fe5dad6002453620743be523c4835242d3 Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Sun, 27 Sep 2026 16:53:15 -0700 Subject: [PATCH] [Refactor] Choose MoE layers' boundaries from declarations when moe_dp_size equals attn_cp_size (#41435) --- python/sglang/srt/layers/boundary_layout.py | 12 +++- python/sglang/srt/layers/communicator.py | 30 ++++++++-- .../unit/layers/test_boundary_edges.py | 27 +++++++++ .../layers/test_declared_decoder_boundary.py | 56 +++++++++++++++++++ 4 files changed, 119 insertions(+), 6 deletions(-) diff --git a/python/sglang/srt/layers/boundary_layout.py b/python/sglang/srt/layers/boundary_layout.py index ff7df0f1fa..b85b36d8aa 100644 --- a/python/sglang/srt/layers/boundary_layout.py +++ b/python/sglang/srt/layers/boundary_layout.py @@ -335,11 +335,14 @@ def decoder_layer_sides( leaves_for_reduce_scatter: bool, leaves_for_reduce_scatterv: bool, hands_on_attention_rows: bool = False, + ffn_shards_over_cp: bool = False, ) -> DecoderLayerSides: """An attention followed by an FFN, derived from the groups each computes over. The FFN runs either on the TP group (a dense MLP, or a MoE not dispatched per DP shard) or on this rank's local rows (a MoE dispatched per DP shard, which completes its own combine, or a dense MLP on every rank). + A MoE whose data-parallel groups are the CP ranks (``ffn_shards_over_cp``) + computes each CP shard on its own ranks: its rows stay sharded over CP. A layer on local rows hands on the attention's rows when it is the last, or with ``hands_on_attention_rows``.""" # Attention computes over the attention-TP ranks of one DP (and CP) shard. @@ -354,7 +357,14 @@ def decoder_layer_sides( axis_sizes=axis_sizes, ) # The TP group spans every token axis, so an FFN on it needs every row. - ffn = local if ffn_on_local_rows else Layout.sharded_over(axis_sizes=axis_sizes) + ffn = ( + local + if ffn_on_local_rows + else Layout.sharded_over( + *((TokenAxis.ATTN_CP,) if ffn_shards_over_cp else ()), + axis_sizes=axis_sizes, + ) + ) attention_tp = axis_sizes[TokenAxis.ATTN_TP_SCATTER] > 1 return DecoderLayerSides( input_rows=local if previous_on_local_rows else attention, diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py index fda86c76d3..a987c2ff88 100644 --- a/python/sglang/srt/layers/communicator.py +++ b/python/sglang/srt/layers/communicator.py @@ -1004,6 +1004,15 @@ class LayerCommunicator: moe_on_local_rows = is_moe_input_scattered_across_dp_ranks() dense_on_local_rows = enable_moe_dense_fully_dp() + # A MoE whose data-parallel groups are the CP ranks computes each CP + # shard of a GQA prefill CP on its own ranks. + ffn_on_cp_shards = ( + modes.is_layer_sparse + and parallel.attn_cp_size > 1 + and parallel.moe_dp_size == parallel.attn_cp_size + and not _gathers_over_attention_cp() + ) + def on_local_rows(sparse: bool) -> bool: return moe_on_local_rows if sparse else dense_on_local_rows @@ -1039,11 +1048,15 @@ class LayerCommunicator: may_leave = not cp_active may_leave_to_reduce_scatter = True else: - # Otherwise under CP the FFN completes its own sum. - may_leave = may_leave_to_reduce_scatter = parallel.attn_cp_size == 1 + # Otherwise under CP an FFN over every CP shard completes its own sum; + # one on its own CP shard hands on its rows as without CP. + may_leave = may_leave_to_reduce_scatter = ( + parallel.attn_cp_size == 1 or ffn_on_cp_shards + ) return decoder_layer_sides( axis_sizes=token_axis_sizes(cp_active=cp_active), ffn_on_local_rows=on_local_rows(modes.is_layer_sparse), + ffn_shards_over_cp=ffn_on_cp_shards, previous_on_local_rows=( not modes.is_first_layer and on_local_rows(modes.is_previous_layer_sparse) @@ -2304,10 +2317,17 @@ def tbo_split_moves(layer_input_rows: Layout) -> Tuple[Callable, Callable]: def _cp_on_declarations() -> bool: """Whether attention CP is one the declarations cover: a prefill CP that - shards tokens, with DSA or MLA attention, or with the FFN input gathered - over a MoE-CP group that is the whole CP group.""" + shards tokens, with DSA or MLA attention, with the FFN input gathered over a + MoE-CP group that is the whole CP group, or, without attention DP, with the + MoE's data-parallel groups the CP ranks (a MoE then computes each CP shard + on its own ranks).""" + parallel = get_parallel() return _generic_prefill_cp_shards_tokens() and ( - _gathers_over_attention_cp() or get_parallel().moe_dp_size == 1 + _gathers_over_attention_cp() + or parallel.moe_dp_size == 1 + or ( + parallel.moe_dp_size == parallel.attn_cp_size and parallel.attn_dp_size == 1 + ) ) diff --git a/test/registered/unit/layers/test_boundary_edges.py b/test/registered/unit/layers/test_boundary_edges.py index b13b38a852..f1cd675128 100644 --- a/test/registered/unit/layers/test_boundary_edges.py +++ b/test/registered/unit/layers/test_boundary_edges.py @@ -85,6 +85,33 @@ class TestDecoderLayerEdges(CustomTestCase): self.assertIs(boundary.prepare.keywords["layer_input"], comm.tp_reduce_scatter) +class TestAMoeOnEachCpShard(CustomTestCase): + """A MoE whose data-parallel groups are the CP ranks computes each CP shard + on its own ranks: its input stays sharded over CP, and the attention output + reaches it without a gather.""" + + def test_the_ffn_rows_stay_sharded_over_cp(self): + axis_sizes = sizes(cp=2, tp=2) + sides = decoder_layer_sides( + axis_sizes=axis_sizes, + ffn_on_local_rows=False, + previous_on_local_rows=False, + is_last_layer=False, + attention_gathers_local_rows=False, + ffn_group=SumGroup.MOE_OUTPUT, + leaves_for_next_layer=False, + leaves_for_reduce_scatter=False, + leaves_for_reduce_scatterv=False, + ffn_shards_over_cp=True, + ) + cp_rows = rows(axis_sizes, TokenAxis.ATTN_CP) + self.assertEqual(sides.ffn.layout, cp_rows) + self.assertEqual(sides.ffn_output.layout, cp_rows) + self.assertIs(sides.ffn_output.group, SumGroup.MOE_OUTPUT) + edges = decoder_layer_edges(sides) + self.assertEqual(edges.into_ffn.need.layout, edges.into_ffn.produced.layout) + + class TestTheConsumerHalfReadsOnlyItsOwnSide(CustomTestCase): """Two layers build the two halves of one boundary without sharing it; the consumer's half never depends on what the producer may leave for a batch, diff --git a/test/registered/unit/layers/test_declared_decoder_boundary.py b/test/registered/unit/layers/test_declared_decoder_boundary.py index 4be051d84f..ff1f6b0646 100644 --- a/test/registered/unit/layers/test_declared_decoder_boundary.py +++ b/test/registered/unit/layers/test_declared_decoder_boundary.py @@ -1715,6 +1715,34 @@ class TestPrefillCP(CustomTestCase): comm.CommunicateSummableTensorPairFn._trivial, ) + def test_a_moe_on_its_own_cp_shard_hands_on_its_sum(self): + # MoE DP equal to CP: each CP rank's MoE runs on its own TP ranks. + parallel = self.cp_parallel(moe_dp_size=2, moe_tp_size=2) + communicator = build( + layer_facts(1, 3, sparse=True, previous_sparse=True), + parallel, + allow_deferred_ffn_reduction=True, + ) + for steps in (communicator._cp_steps, communicator._steps): + # The attention's rows are the MoE's: nothing to gather or take back. + self.assertIs(steps.ffn_input.func, comm._mlp_input_without_dp) + self.assertIs( + steps.ffn_output_move, comm.CommunicateSummableTensorPairFn._trivial + ) + self.assertTrue(steps.ffn_output.leaves_for_next_layer) + dense = build(layer_facts(1, 3), parallel, allow_deferred_ffn_reduction=True) + self.assertIs(dense._cp_steps.ffn_input.func, comm._mlp_input_gather_moe_cp) + self.assertFalse(dense._cp_steps.ffn_output.leaves_for_next_layer) + # DSA and MLA CP gather the MoE's input over attention CP whatever its DP. + dsa = self.dsa_parallel(moe_dp_size=2, moe_tp_size=1) + modes = planned_modes( + 1, 3, sparse=True, previous_sparse=False, parallel=dsa, dsa_cp=True + ) + communicator = build(modes, dsa, dsa_cp=True, allow_reduce_scatter=True) + self.assertIs( + communicator._cp_steps.ffn_input.func, comm._mlp_input_gather_attention_cp + ) + def test_the_fused_kernels_run_on_each_chunk(self): with planning(self.cp_parallel()): communicator = LayerCommunicator( @@ -1762,6 +1790,34 @@ class TestPrefillCP(CustomTestCase): False, True, ), + ( + "a MoE whose data-parallel groups are the CP ranks", + parallel_of( + attn_dp=1, + attn_tp=2, + attn_cp=2, + enable_prefill_cp=True, + moe_dp_size=2, + ), + True, + True, + False, + False, + ), + ( + "a MoE whose data-parallel groups are the CP ranks, under attention DP", + parallel_of( + attn_dp=2, + attn_tp=1, + attn_cp=2, + enable_prefill_cp=True, + moe_dp_size=2, + ), + True, + False, + False, + False, + ), ( "a MoE-CP group narrower than CP", parallel_of(