diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py index aa845ef28f..c507fe72a5 100644 --- a/python/sglang/srt/layers/communicator.py +++ b/python/sglang/srt/layers/communicator.py @@ -438,6 +438,7 @@ class LayerScatterModes: mlp_mode: ScatterMode middle_residual_mode: ScatterMode layer_output_mode: ScatterMode + is_layer_sparse: bool = False @classmethod def init_new(cls, **kwargs): @@ -448,6 +449,7 @@ class LayerScatterModes: mlp_mode=cls._compute_mlp_mode(context), middle_residual_mode=cls._compute_middle_residual_mode(context), layer_output_mode=cls._compute_layer_output_mode(context), + is_layer_sparse=context.is_layer_sparse, ) @classmethod @@ -917,6 +919,7 @@ class LayerCommunicator: forward_batch=forward_batch, context=self._context, allow_reduce_scatter=self.allow_reduce_scatter, + is_layer_sparse=self.layer_scatter_modes.is_layer_sparse, ) def ffn_exit(self, forward_batch: ForwardBatch) -> "FfnExit": @@ -1617,6 +1620,7 @@ class CommunicateSummableTensorPairFn: forward_batch: ForwardBatch, context: CommunicateContext, allow_reduce_scatter: bool = False, + is_layer_sparse: bool = False, ): if get_parallel().tp_size == get_parallel().attn_dp_size: group = get_parallel().tp_group @@ -1626,7 +1630,12 @@ class CommunicateSummableTensorPairFn: get_local_dp_buffer(group), hidden_states, ) - if should_use_dp_reduce_scatterv(): + # A MoE block leaves its sum to reduce_scatterv whenever it applies + # (should_skip_post_experts_all_reduce); a dense MLP does only under the + # published mlp_reduce_scatter, which needs allow_reduce_scatter. + if should_use_dp_reduce_scatterv() and ( + allow_reduce_scatter or is_layer_sparse + ): get_parallel().tp_group.reduce_scatterv( global_hidden_states, output=hidden_states, diff --git a/python/sglang/srt/layers/communicator_dsa_cp.py b/python/sglang/srt/layers/communicator_dsa_cp.py index b4696e4992..c64c012734 100644 --- a/python/sglang/srt/layers/communicator_dsa_cp.py +++ b/python/sglang/srt/layers/communicator_dsa_cp.py @@ -231,6 +231,7 @@ class DSACPCommunicateSummableTensorPairFn(CommunicateSummableTensorPairFn): forward_batch: ForwardBatch, context: CommunicateContext, allow_reduce_scatter: bool = False, + **kwargs, ): # for prefill: full -> attn tp scattered # for decode: full -> attn tp full diff --git a/python/sglang/srt/layers/moe/utils.py b/python/sglang/srt/layers/moe/utils.py index 8482c3764a..2ef21189b1 100644 --- a/python/sglang/srt/layers/moe/utils.py +++ b/python/sglang/srt/layers/moe/utils.py @@ -779,6 +779,20 @@ def reduce_moe_output(hidden_states: torch.Tensor) -> torch.Tensor: return hidden_states +def should_add_replicated_moe_output() -> bool: + """Whether this rank adds an output every TP rank holds in full, such as a + shared expert replicated with tp_size=1, to its MoE output. + + Call it after the MoE block's own reduction. When a later step still sums + the output over TP, only TP rank 0 adds it, so the sum counts it once. + """ + parallel = get_parallel() + summed_later = should_skip_post_experts_all_reduce( + is_tp_path=True + ) and not post_experts_output_is_complete(is_tp_path=True) + return not (parallel.tp_size > 1 and summed_later and parallel.tp_rank != 0) + + def can_merge_post_experts_all_reduce() -> bool: """Whether the EP and MoE-TP reductions can collapse into one _TP all-reduce. diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index fdb989c57b..6b91e05853 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -115,6 +115,7 @@ from sglang.srt.layers.moe.utils import ( is_sbo_enabled, is_shared_experts_fusion_disabled, is_tbo_enabled, + should_add_replicated_moe_output, ) from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.quantization.fp8 import Fp8Config @@ -1222,7 +1223,7 @@ class DeepseekV2MoE(nn.Module): final_hidden_states = post_experts_all_reduce(final_hidden_states) # TP1 shared experts are replicated, so add them after all-reduce to # avoid summing the same shared output once per TP rank. - if self._shared_expert_tp1: + if self._shared_expert_tp1 and should_add_replicated_moe_output(): final_hidden_states += shared_output return final_hidden_states @@ -1378,7 +1379,11 @@ class DeepseekV2MoE(nn.Module): final_hidden_states = post_experts_all_reduce(final_hidden_states) # TP1 shared experts are replicated, so add them after all-reduce to # avoid summing the same shared output once per TP rank. - if shared_output is not None and self._shared_expert_tp1: + if ( + shared_output is not None + and self._shared_expert_tp1 + and should_add_replicated_moe_output() + ): final_hidden_states += shared_output return final_hidden_states diff --git a/python/sglang/srt/models/laguna.py b/python/sglang/srt/models/laguna.py index 546b2ea3a3..84d06fc7f3 100644 --- a/python/sglang/srt/models/laguna.py +++ b/python/sglang/srt/models/laguna.py @@ -39,6 +39,7 @@ from sglang.srt.layers.moe import reduce_moe_output from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.layers.moe.topk import TopK +from sglang.srt.layers.moe.utils import should_add_replicated_moe_output from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.radix_attention import RadixAttention from sglang.srt.layers.rotary_embedding import get_rope @@ -224,7 +225,7 @@ class LagunaMoE(nn.Module): final = routed_out + shared_out final = reduce_moe_output(final) - if self._shared_expert_tp1: + if self._shared_expert_tp1 and should_add_replicated_moe_output(): final = final + shared_out return final diff --git a/python/sglang/srt/models/longcat_flash_nextn.py b/python/sglang/srt/models/longcat_flash_nextn.py index 40c1bc9096..dd48b333a8 100644 --- a/python/sglang/srt/models/longcat_flash_nextn.py +++ b/python/sglang/srt/models/longcat_flash_nextn.py @@ -149,6 +149,7 @@ class LongcatFlashDenseDecoderLayer(nn.Module): intermediate_size=config.intermediate_size, hidden_act=config.hidden_act, quant_config=quant_config, + reduce_results=True, prefix=add_prefix(f"mlps", prefix), ) self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) diff --git a/python/sglang/srt/models/nemotron_h_utils.py b/python/sglang/srt/models/nemotron_h_utils.py index ecace8e27a..c2222127d9 100644 --- a/python/sglang/srt/models/nemotron_h_utils.py +++ b/python/sglang/srt/models/nemotron_h_utils.py @@ -35,6 +35,7 @@ def _build_layer_scatter_modes(is_sparse: bool = False) -> LayerScatterModes: mlp_mode=mlp_mode, middle_residual_mode=middle_residual_mode, layer_output_mode=ScatterMode.TP_ATTN_FULL, + is_layer_sparse=is_sparse, ) diff --git a/test/registered/unit/layers/moe/test_moe_output_reduction.py b/test/registered/unit/layers/moe/test_moe_output_reduction.py index ff3abcc549..1c384f7638 100644 --- a/test/registered/unit/layers/moe/test_moe_output_reduction.py +++ b/test/registered/unit/layers/moe/test_moe_output_reduction.py @@ -14,6 +14,7 @@ from sglang.srt.layers.moe import utils as moe_utils from sglang.srt.layers.moe.utils import ( post_experts_output_is_complete, reduce_moe_output, + should_add_replicated_moe_output, ) from sglang.srt.runtime_context import get_forward from sglang.test.ci.ci_register import register_cpu_ci @@ -34,13 +35,21 @@ def a2a(name=None): @contextlib.contextmanager def moe_config( - *, tp_size=2, dwdp_size=1, backend=None, fp4_allgather=False, reduce_scatterv=False + *, + tp_size=2, + tp_rank=0, + dwdp_size=1, + backend=None, + fp4_allgather=False, + reduce_scatterv=False, ): with ( patch.object( moe_utils, "get_parallel", - return_value=types.SimpleNamespace(tp_size=tp_size, dwdp_size=dwdp_size), + return_value=types.SimpleNamespace( + tp_size=tp_size, tp_rank=tp_rank, dwdp_size=dwdp_size + ), ), patch.object(moe_utils, "get_moe_a2a_backend", return_value=a2a(backend)), patch.object( @@ -106,6 +115,83 @@ class TestReduceMoeOutput(CustomTestCase): self.assertFalse(post_experts_output_is_complete(is_tp_path=False)) +class TestReplicatedMoeOutput(CustomTestCase): + """A shared expert replicated with tp_size=1 holds its full output on every + rank; whatever sums the MoE output over TP must count it once.""" + + def summed_over_two_ranks(self, *, flags=None, **config): + routed = [torch.full((2, 3), 1.0), torch.full((2, 3), 2.0)] + shared = torch.full((2, 3), 10.0) + outputs = [] + for rank in (0, 1): + with ( + moe_config(tp_rank=rank, **config), + get_forward().scoped(**(flags or {})), + ): + output = routed[rank] + if should_add_replicated_moe_output(): + output = output + shared + outputs.append(output) + return outputs + + def test_a_later_sum_counts_it_once(self): + for flags, config in ( + ({"fuse_mlp_allreduce": True}, {}), + ({"mlp_reduce_scatter": True}, {}), + ({}, {"reduce_scatterv": True}), + ): + with self.subTest(flags=flags, config=config): + outputs = self.summed_over_two_ranks(flags=flags, **config) + torch.testing.assert_close(sum(outputs), torch.full((2, 3), 13.0)) + + def test_every_rank_adds_it_to_a_reduced_or_complete_output(self): + for config in ({}, {"backend": "flashinfer"}, {"fp4_allgather": True}): + with self.subTest(**config): + outputs = self.summed_over_two_ranks(**config) + torch.testing.assert_close(outputs[1], torch.full((2, 3), 12.0)) + + def test_a_single_rank_adds_it(self): + with ( + moe_config(tp_size=1, tp_rank=0), + get_forward().scoped(fuse_mlp_allreduce=True), + ): + self.assertTrue(should_add_replicated_moe_output()) + + +def reads_the_tp1_shared_expert_switch(tree): + return any( + isinstance(node, ast.Attribute) and node.attr == "SGLANG_SHARED_EXPERT_TP1" + for node in ast.walk(tree) + ) + + +def calls(tree, name): + return any( + isinstance(node, ast.Call) + and name in (getattr(node.func, "id", None), getattr(node.func, "attr", None)) + for node in ast.walk(tree) + ) + + +class TestReplicatedSharedExpertsCountOnce(CustomTestCase): + def test_models_that_replicate_the_shared_expert_add_it_once(self): + # SGLANG_SHARED_EXPERT_TP1 replicates the shared expert without an a2a + # backend, so the MoE output can still owe a later TP sum. + subjects = {} + for path in sorted(MODELS_DIR.rglob("*.py")): + tree = ast.parse(path.read_text()) + if reads_the_tp1_shared_expert_switch(tree): + subjects[str(path.relative_to(MODELS_DIR))] = calls( + tree, "should_add_replicated_moe_output" + ) + self.assertTrue(subjects, "no model reads SGLANG_SHARED_EXPERT_TP1") + self.assertEqual( + [name for name, ok in subjects.items() if not ok], + [], + "add the replicated shared output under should_add_replicated_moe_output", + ) + + def open_coded_reductions(tree): """``if`` statements that test should_skip_post_experts_all_reduce for the TP path and call tensor_model_parallel_all_reduce themselves.""" diff --git a/test/registered/unit/layers/test_postprocess_reduce_scatterv.py b/test/registered/unit/layers/test_postprocess_reduce_scatterv.py new file mode 100644 index 0000000000..5cdaa5ecd7 --- /dev/null +++ b/test/registered/unit/layers/test_postprocess_reduce_scatterv.py @@ -0,0 +1,194 @@ +"""Under DP attention with reduce_scatterv, postprocess sums a FULL-mode layer's +output while scattering it back to the local tokens only when the FFN left its +all-reduce out.""" + +import contextlib +import itertools +import types +import unittest +from unittest.mock import patch + +import torch + +from sglang.srt.layers import communicator as comm +from sglang.srt.layers.communicator import ( + CommunicateSummableTensorPairFn, + LayerCommunicator, + LayerScatterModes, + ScatterMode, +) +from sglang.srt.layers.moe import utils as moe_utils +from sglang.srt.layers.moe.utils import ( + should_skip_mlp_all_reduce, + should_skip_post_experts_all_reduce, +) +from sglang.srt.runtime_context import get_forward +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +@contextlib.contextmanager +def reduce_scatterv_applies(): + with ( + patch.object(comm, "should_use_dp_reduce_scatterv", return_value=True), + patch.object(moe_utils, "should_use_dp_reduce_scatterv", return_value=True), + ): + yield + + +def postprocess_sums(*, allow_reduce_scatter, is_layer_sparse): + calls = [] + parallel = types.SimpleNamespace( + tp_size=2, + attn_dp_size=2, + tp_group=types.SimpleNamespace( + reduce_scatterv=lambda *args, **kwargs: calls.append("reduce_scatterv") + ), + ) + forward_batch = types.SimpleNamespace( + dp_padding_mode=types.SimpleNamespace(is_max_len=lambda: False) + ) + with ( + reduce_scatterv_applies(), + patch.object(comm, "get_parallel", return_value=parallel), + patch.object(comm, "get_local_dp_buffer", return_value=torch.empty(1, 4)), + patch.object(comm, "get_dp_global_num_tokens", return_value=[1, 1]), + patch.object( + comm, "dp_scatter", side_effect=lambda *args: calls.append("dp_scatter") + ), + ): + CommunicateSummableTensorPairFn._scatter_hidden_states( + torch.zeros(2, 4), + None, + forward_batch, + context=None, + allow_reduce_scatter=allow_reduce_scatter, + is_layer_sparse=is_layer_sparse, + ) + assert len(calls) == 1, calls + return calls[0] == "reduce_scatterv" + + +def ffn_leaves_the_sum_out(*, allow_reduce_scatter, is_layer_sparse): + # A layer that allows reduce-scatter publishes mlp_reduce_scatter while + # reduce_scatterv applies; a dense MLP reads only that flag. + with ( + reduce_scatterv_applies(), + patch.object( + moe_utils, + "get_parallel", + return_value=types.SimpleNamespace(tp_size=2, dwdp_size=1), + ), + patch.object( + moe_utils, + "get_moe_a2a_backend", + return_value=types.SimpleNamespace( + is_flashinfer=lambda: False, + is_pplx=lambda: False, + is_flashinfer_megamoe=lambda: False, + ), + ), + patch.object( + moe_utils, + "should_use_flashinfer_cutlass_moe_fp4_allgather", + return_value=False, + ), + get_forward().scoped(mlp_reduce_scatter=allow_reduce_scatter), + ): + if is_layer_sparse: + return should_skip_post_experts_all_reduce(is_tp_path=True) + return should_skip_mlp_all_reduce() + + +class TestPostprocessReduceScatterv(CustomTestCase): + def test_sums_exactly_when_the_ffn_left_the_sum_out(self): + for allow, sparse in itertools.product((False, True), repeat=2): + case = dict(allow_reduce_scatter=allow, is_layer_sparse=sparse) + with self.subTest(**case): + self.assertEqual( + postprocess_sums(**case), ffn_leaves_the_sum_out(**case) + ) + + def test_a_dense_layer_without_reduce_scatter_is_not_summed_again(self): + self.assertFalse( + postprocess_sums(allow_reduce_scatter=False, is_layer_sparse=False) + ) + + def test_layer_modes_record_whether_the_layer_is_sparse(self): + for sparse in (False, True): + with ( + self.subTest(sparse=sparse), + patch.object( + comm, "sparse_mlp_scatter_mode", return_value=ScatterMode.FULL + ), + patch.object(comm, "enable_moe_dense_fully_dp", return_value=False), + patch.object( + comm, "_generic_prefill_cp_shards_tokens", return_value=False + ), + ): + modes = LayerScatterModes.init_new( + layer_id=1, + num_layers=4, + is_layer_sparse=sparse, + is_previous_layer_sparse=sparse, + is_next_layer_sparse=sparse, + ) + self.assertEqual(modes.is_layer_sparse, sparse) + + def test_postprocess_passes_the_layer_sparsity(self): + seen = {} + communicator = LayerCommunicator.__new__(LayerCommunicator) + communicator._sp_variant = None + communicator._context = None + communicator.allow_reduce_scatter = False + communicator.layer_scatter_modes = types.SimpleNamespace(is_layer_sparse=True) + communicator._communicate_summable_tensor_pair_fn = lambda **kwargs: ( + seen.update(kwargs) or (None, None) + ) + communicator.postprocess_layer(None, None, None) + self.assertIs(seen["is_layer_sparse"], True) + + +class TestLongcatNextnReducesItsMlp(CustomTestCase): + """Its dense layer allows no reduce-scatter, so postprocess never sums the + MLP output; the MLP must all-reduce it itself.""" + + def test_the_dense_mlp_reduces_its_output(self): + from sglang.srt.models import longcat_flash_nextn as nextn + + mlp_kwargs = {} + config = types.SimpleNamespace( + hidden_size=8, + intermediate_size=16, + hidden_act="silu", + num_attention_heads=1, + qk_nope_head_dim=1, + qk_rope_head_dim=1, + v_head_dim=1, + q_lora_rank=None, + kv_lora_rank=1, + rope_theta=1.0, + max_position_embeddings=8, + rms_norm_eps=1e-6, + num_hidden_layers=1, + ) + with ( + patch.object(nextn, "DeepseekV2AttentionMLA"), + patch.object( + nextn, + "LongcatFlashMLP", + side_effect=lambda **kwargs: mlp_kwargs.update(kwargs), + ), + patch.object(nextn, "RMSNorm"), + patch.object(nextn, "get_parallel"), + patch.object(nextn, "LayerScatterModes"), + patch.object(nextn, "LayerCommunicator"), + ): + nextn.LongcatFlashDenseDecoderLayer(config, layer_id=0) + self.assertIs(mlp_kwargs.get("reduce_results"), True) + + +if __name__ == "__main__": + unittest.main()