From 31329c222cb0e4e172fcc4c9a31b12c906b43654 Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Sun, 27 Sep 2026 16:52:10 -0700 Subject: [PATCH] =?UTF-8?q?[Fix]=20Run=20MoE=20layers=20under=20attention?= =?UTF-8?q?=20DP=20and=20GQA=20prefill=20CP=20on=20the=20declared=20DP=20?= =?UTF-8?q?=C3=97=20CP=20gather=20(#41432)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- python/sglang/srt/layers/communicator.py | 6 ++++-- .../unit/layers/test_declared_decoder_boundary.py | 10 +++++++++- .../unit/layers/test_dp_attention_cp_gather.py | 15 +++++++++++---- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py index c018f9b377..fda86c76d3 100644 --- a/python/sglang/srt/layers/communicator.py +++ b/python/sglang/srt/layers/communicator.py @@ -1020,12 +1020,14 @@ class LayerCommunicator: if not ( (parallel.attn_cp_size == 1 or _cp_on_declarations()) - # MoE layers under attention DP and GQA prefill CP keep the - # scatter-mode steps. + # A MoE dispatched per DP shard under attention DP and GQA prefill + # CP keeps the scatter-mode steps: it would dispatch each rank's CP + # shard of the tokens, which is not covered. and not ( parallel.attn_cp_size > 1 and parallel.attn_dp_size > 1 and modes.is_layer_sparse + and moe_on_local_rows and not _gathers_over_attention_cp() ) and (modes.is_first_layer or modes.is_previous_layer_sparse is not None) diff --git a/test/registered/unit/layers/test_declared_decoder_boundary.py b/test/registered/unit/layers/test_declared_decoder_boundary.py index 49885106ee..4be051d84f 100644 --- a/test/registered/unit/layers/test_declared_decoder_boundary.py +++ b/test/registered/unit/layers/test_declared_decoder_boundary.py @@ -1753,7 +1753,15 @@ class TestPrefillCP(CustomTestCase): False, ), ("CP under attention DP", dp_cp, False, True, False, False), - ("a MoE under attention DP and GQA CP", dp_cp, True, False, False, False), + ("a MoE under attention DP and GQA CP", dp_cp, True, True, False, False), + ( + "a MoE dispatched per DP shard under attention DP and GQA CP", + dp_cp, + True, + False, + False, + True, + ), ( "a MoE-CP group narrower than CP", parallel_of( diff --git a/test/registered/unit/layers/test_dp_attention_cp_gather.py b/test/registered/unit/layers/test_dp_attention_cp_gather.py index d41b5f8537..317af1e7c7 100644 --- a/test/registered/unit/layers/test_dp_attention_cp_gather.py +++ b/test/registered/unit/layers/test_dp_attention_cp_gather.py @@ -134,6 +134,7 @@ class TestDpCpGather(CustomTestCase): norm_rows=lambda rows: rows, force_layernorm_before_dp_gather=False, ffn_input=None, + sparse=False, ): """Gather on every rank, check the FFN input and residual, then take back an FFN output of three times the input and check each rank's rows. @@ -237,14 +238,12 @@ class TestDpCpGather(CustomTestCase): ), ]: stack.enter_context(patch.object(*target, value)) - # A dense layer in the middle of the model: its steps come from - # the declarations, the modes only give the layer facts. communicator = comm.LayerCommunicator( layer_scatter_modes=SimpleNamespace( is_first_layer=False, is_last_layer=False, - is_layer_sparse=False, - is_previous_layer_sparse=False, + is_layer_sparse=sparse, + is_previous_layer_sparse=sparse, is_next_layer_sparse=False, ), input_layernorm=norm, @@ -331,6 +330,14 @@ class TestDpCpGather(CustomTestCase): ) self.run_ranks(groups, [ceil_align(12, 2), ceil_align(9, 2)], attn_tp_size=2) + def test_a_moe_layer_takes_the_same_gather(self): + """A MoE on the TP group gathers and takes back its rows like a dense + MLP: only its sum group differs, and under CP it completes that sum.""" + groups = [cp_active([7, 5], seed=0), cp_active([9], seed=1)] + self.run_ranks( + groups, [ceil_align(12, 2), ceil_align(9, 2)], attn_tp_size=2, sparse=True + ) + def test_prefill_beside_an_idle_dp_group(self): groups = [ cp_active([7, 5], seed=0),