From 62ae032c4601e6f890a00bacad4cdafddc03ac24 Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Thu, 24 Sep 2026 12:15:33 -0700 Subject: [PATCH] [Refactor] Share the MoE output all-reduce between models (#41097) --- python/sglang/srt/layers/moe/__init__.py | 2 + python/sglang/srt/layers/moe/utils.py | 62 +++++--- python/sglang/srt/models/bailing_moe.py | 10 +- .../sglang/srt/models/bailing_moe_linear.py | 10 +- python/sglang/srt/models/exaone_moe.py | 10 +- python/sglang/srt/models/glm4_moe.py | 13 +- python/sglang/srt/models/glm4_moe_lite.py | 15 +- python/sglang/srt/models/gpt_oss.py | 10 +- python/sglang/srt/models/interns2_mobius.py | 14 +- python/sglang/srt/models/laguna.py | 10 +- python/sglang/srt/models/llada2.py | 10 +- python/sglang/srt/models/llama4.py | 10 +- python/sglang/srt/models/mimo_v2.py | 10 +- python/sglang/srt/models/minimax_m2.py | 10 +- python/sglang/srt/models/nemotron_h.py | 7 +- python/sglang/srt/models/qwen2_moe.py | 12 +- python/sglang/srt/models/sarvam_moe.py | 12 +- python/sglang/srt/models/sdar_moe.py | 10 +- python/sglang/srt/models/step3p5.py | 7 +- python/sglang/srt/models/xllm.py | 12 +- .../layers/moe/test_moe_output_reduction.py | 145 ++++++++++++++++++ 21 files changed, 228 insertions(+), 173 deletions(-) create mode 100644 test/registered/unit/layers/moe/test_moe_output_reduction.py diff --git a/python/sglang/srt/layers/moe/__init__.py b/python/sglang/srt/layers/moe/__init__.py index f3b05fa1ee..52ea18338e 100644 --- a/python/sglang/srt/layers/moe/__init__.py +++ b/python/sglang/srt/layers/moe/__init__.py @@ -14,6 +14,7 @@ from sglang.srt.layers.moe.utils import ( is_moe_input_scattered_across_dp_ranks, is_tbo_enabled, post_experts_all_reduce, + reduce_moe_output, should_skip_mlp_all_reduce, should_skip_post_experts_all_reduce, should_use_dp_reduce_scatterv, @@ -34,6 +35,7 @@ __all__ = [ "can_merge_post_experts_all_reduce", "deferred_post_experts_all_reduce", "post_experts_all_reduce", + "reduce_moe_output", "should_skip_post_experts_all_reduce", "should_use_dp_reduce_scatterv", "should_use_flashinfer_cutlass_moe_fp4_allgather", diff --git a/python/sglang/srt/layers/moe/utils.py b/python/sglang/srt/layers/moe/utils.py index 9840e671f8..8482c3764a 100644 --- a/python/sglang/srt/layers/moe/utils.py +++ b/python/sglang/srt/layers/moe/utils.py @@ -732,31 +732,51 @@ def should_skip_mlp_all_reduce() -> bool: return f.fuse_mlp_allreduce or f.mlp_reduce_scatter +def post_experts_output_is_complete(*, is_tp_path: bool) -> bool: + """Whether the experts' output owes no sum over the MoE-TP group + (``is_tp_path=True``) or the EP group: the combine already summed it, or each + rank computed its own tokens in full. + + This is a property of the MoE configuration. Whether the MoE block or a later + step runs a sum that is still owed is decided separately. + """ + if get_parallel().dwdp_size > 1: + return True + if is_tp_path and should_use_flashinfer_cutlass_moe_fp4_allgather(): + # The combine reduce-scatters back to the local tokens. + return True + a2a = get_moe_a2a_backend() + # The flashinfer and pplx combines, and the megamoe kernel's internal + # combine, sum each token's expert outputs back to its source rank. + return a2a.is_flashinfer() or a2a.is_pplx() or a2a.is_flashinfer_megamoe() + + def should_skip_post_experts_all_reduce(*, is_tp_path: bool) -> bool: - """Whether a downstream component will fuse, replace, or absorb the post-experts all-reduce. + """Whether the MoE block should leave out its post-experts all-reduce: a later + step runs it (fused into the next norm, or as the reduce-scatter back to the + local tokens), or there is nothing to sum. Pass ``is_tp_path=True`` for the TP all-reduce, ``False`` for the EP one. """ - if should_skip_mlp_all_reduce(): - return True - if get_parallel().dwdp_size > 1: - return True - if should_use_dp_reduce_scatterv(): - return True - if is_tp_path and should_use_flashinfer_cutlass_moe_fp4_allgather(): - return True - if get_moe_a2a_backend().is_flashinfer(): - return True - if get_moe_a2a_backend().is_pplx(): - # pplx's AllToAll.combine already sums each token's expert outputs back - # to the source rank - return True - if get_moe_a2a_backend().is_flashinfer_megamoe(): - # The mega kernel does its EP all-to-all + combine internally and - # returns per-rank outputs, so any further EP/TP all-reduce would - # double-count. Same opt-in as the flashinfer a2a dispatcher. - return True - return False + return ( + should_skip_mlp_all_reduce() + or should_use_dp_reduce_scatterv() + or post_experts_output_is_complete(is_tp_path=is_tp_path) + ) + + +def reduce_moe_output(hidden_states: torch.Tensor) -> torch.Tensor: + """All-reduce a MoE block's output (routed plus shared experts) over TP, + unless a later step does it or there is nothing to sum.""" + from sglang.srt.distributed.communication_op import ( + tensor_model_parallel_all_reduce, + ) + + if get_parallel().tp_size > 1 and not should_skip_post_experts_all_reduce( + is_tp_path=True + ): + return tensor_model_parallel_all_reduce(hidden_states) + return hidden_states def can_merge_post_experts_all_reduce() -> bool: diff --git a/python/sglang/srt/models/bailing_moe.py b/python/sglang/srt/models/bailing_moe.py index c5c9a6c325..4c7c94f612 100644 --- a/python/sglang/srt/models/bailing_moe.py +++ b/python/sglang/srt/models/bailing_moe.py @@ -27,9 +27,6 @@ import torch.nn.functional as F from torch import nn from transformers import PretrainedConfig -from sglang.srt.distributed import ( - tensor_model_parallel_all_reduce, -) from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo @@ -53,7 +50,7 @@ from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe import ( get_deepep_mode, get_moe_a2a_backend, - should_skip_post_experts_all_reduce, + reduce_moe_output, ) from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE @@ -455,10 +452,7 @@ class BailingMoESparseMoeBlock(nn.Module): if self.num_shared_experts > 0: final_hidden_states = final_hidden_states + shared_output - if self.tp_size > 1 and not should_skip_post_experts_all_reduce( - is_tp_path=True, - ): - final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states) + final_hidden_states = reduce_moe_output(final_hidden_states) return final_hidden_states.view(num_tokens, hidden_size) def forward_deepep( diff --git a/python/sglang/srt/models/bailing_moe_linear.py b/python/sglang/srt/models/bailing_moe_linear.py index 7b6d93bd98..c293384473 100644 --- a/python/sglang/srt/models/bailing_moe_linear.py +++ b/python/sglang/srt/models/bailing_moe_linear.py @@ -12,9 +12,6 @@ from transformers import PretrainedConfig from sglang.kernels.ops.attention.fla.layernorm_gated import RMSNorm as RMSNormGated from sglang.kernels.ops.attention.fla.layernorm_gated import layernorm_fn from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz -from sglang.srt.distributed import ( - tensor_model_parallel_all_reduce, -) from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.layers import deep_gemm_wrapper from sglang.srt.layers.activation import SiluAndMul @@ -33,7 +30,7 @@ from sglang.srt.layers.linear import ( RowParallelLinear, ) from sglang.srt.layers.logits_processor import LogitsProcessor -from sglang.srt.layers.moe import should_skip_post_experts_all_reduce +from sglang.srt.layers.moe import reduce_moe_output from sglang.srt.layers.moe.ep_moe.layer import DeepEPMoE, get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.layers.moe.topk import TopK @@ -366,10 +363,7 @@ class BailingMoE(nn.Module): if self.num_shared_experts > 0: final_hidden_states = final_hidden_states + shared_output - if self.tp_size > 1 and not should_skip_post_experts_all_reduce( - is_tp_path=True, - ): - final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states) + final_hidden_states = reduce_moe_output(final_hidden_states) return final_hidden_states diff --git a/python/sglang/srt/models/exaone_moe.py b/python/sglang/srt/models/exaone_moe.py index c87f0fd77c..211a12de39 100755 --- a/python/sglang/srt/models/exaone_moe.py +++ b/python/sglang/srt/models/exaone_moe.py @@ -24,9 +24,6 @@ import torch from torch import nn from transformers import PretrainedConfig -from sglang.srt.distributed import ( - tensor_model_parallel_all_reduce, -) from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo @@ -44,7 +41,7 @@ from sglang.srt.layers.linear import ( from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput from sglang.srt.layers.moe import ( get_moe_a2a_backend, - should_skip_post_experts_all_reduce, + reduce_moe_output, ) from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton import FusedMoE @@ -290,10 +287,7 @@ class ExaoneMoESparseMoEBlock(nn.Module): if shared_output is not None: final_hidden_states = final_hidden_states + shared_output - if self.tp_size > 1 and not should_skip_post_experts_all_reduce( - is_tp_path=True, - ): - final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states) + final_hidden_states = reduce_moe_output(final_hidden_states) return final_hidden_states.view(num_tokens, hidden_dim) diff --git a/python/sglang/srt/models/glm4_moe.py b/python/sglang/srt/models/glm4_moe.py index a55485a436..1bb562fe62 100644 --- a/python/sglang/srt/models/glm4_moe.py +++ b/python/sglang/srt/models/glm4_moe.py @@ -29,7 +29,6 @@ from sglang.srt.batch_overlap.single_batch_overlap import SboFlags from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo from sglang.srt.distributed import ( get_pp_indices, - tensor_model_parallel_all_reduce, ) from sglang.srt.distributed.device_communicators.pynccl_allocator import ( use_symmetric_memory, @@ -58,7 +57,7 @@ from sglang.srt.layers.linear import ( from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe import ( get_moe_a2a_backend, - should_skip_post_experts_all_reduce, + reduce_moe_output, should_use_flashinfer_cutlass_moe_fp4_allgather, ) from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class @@ -621,10 +620,7 @@ class Glm4MoeSparseMoeBlock(nn.Module): current_stream.wait_stream(self.alt_stream) final_hidden_states += shared_output - if self.tp_size > 1 and not should_skip_post_experts_all_reduce( - is_tp_path=True, - ): - final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states) + final_hidden_states = reduce_moe_output(final_hidden_states) return final_hidden_states def forward_normal( @@ -650,10 +646,7 @@ class Glm4MoeSparseMoeBlock(nn.Module): final_hidden_states_out = torch.empty_like(final_hidden_states) torch.add(final_hidden_states, shared_output, out=final_hidden_states_out) final_hidden_states = final_hidden_states_out - if self.tp_size > 1 and not should_skip_post_experts_all_reduce( - is_tp_path=True, - ): - final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states) + final_hidden_states = reduce_moe_output(final_hidden_states) return final_hidden_states def forward_deepep( diff --git a/python/sglang/srt/models/glm4_moe_lite.py b/python/sglang/srt/models/glm4_moe_lite.py index e086f53203..f8423f568e 100644 --- a/python/sglang/srt/models/glm4_moe_lite.py +++ b/python/sglang/srt/models/glm4_moe_lite.py @@ -25,9 +25,6 @@ from transformers import PretrainedConfig from sglang.srt.batch_overlap.single_batch_overlap import SboFlags from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo -from sglang.srt.distributed import ( - tensor_model_parallel_all_reduce, -) from sglang.srt.distributed.device_communicators.pynccl_allocator import ( use_symmetric_memory, ) @@ -51,7 +48,7 @@ from sglang.srt.layers.linear import MergedColumnParallelLinear, RowParallelLine from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe import ( get_moe_a2a_backend, - should_skip_post_experts_all_reduce, + reduce_moe_output, should_use_flashinfer_cutlass_moe_fp4_allgather, ) from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class @@ -340,10 +337,7 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module): current_stream.wait_stream(self.alt_stream) final_hidden_states += shared_output - if self.tp_size > 1 and not should_skip_post_experts_all_reduce( - is_tp_path=True, - ): - final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states) + final_hidden_states = reduce_moe_output(final_hidden_states) return final_hidden_states def forward_normal( @@ -369,10 +363,7 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module): final_hidden_states_out = torch.empty_like(final_hidden_states) torch.add(final_hidden_states, shared_output, out=final_hidden_states_out) final_hidden_states = final_hidden_states_out - if self.tp_size > 1 and not should_skip_post_experts_all_reduce( - is_tp_path=True, - ): - final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states) + final_hidden_states = reduce_moe_output(final_hidden_states) return final_hidden_states def forward_deepep( diff --git a/python/sglang/srt/models/gpt_oss.py b/python/sglang/srt/models/gpt_oss.py index 8fef5939d3..4f668ddb5b 100644 --- a/python/sglang/srt/models/gpt_oss.py +++ b/python/sglang/srt/models/gpt_oss.py @@ -27,9 +27,6 @@ from torch import nn from transformers import PretrainedConfig from sglang.kernels.jit.utils import is_arch_support_pdl -from sglang.srt.distributed import ( - tensor_model_parallel_all_reduce, -) from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers.communicator import ( @@ -49,7 +46,7 @@ from sglang.srt.layers.linear import ( from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe import ( get_moe_a2a_backend, - should_skip_post_experts_all_reduce, + reduce_moe_output, ) from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE @@ -337,10 +334,7 @@ class GptOssSparseMoeBlock(nn.Module): topk_output = self.topk(router_input, router_logits) final_hidden_states = self.experts(hidden_states, topk_output) - if self.tp_size > 1 and not should_skip_post_experts_all_reduce( - is_tp_path=True, - ): - final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states) + final_hidden_states = reduce_moe_output(final_hidden_states) # When input was pre-padded, FusedMoE.forward_impl captured the # padded width as `origin_hidden_states_dim` and skipped its own diff --git a/python/sglang/srt/models/interns2_mobius.py b/python/sglang/srt/models/interns2_mobius.py index 5705c7cc02..b4bedd6de5 100644 --- a/python/sglang/srt/models/interns2_mobius.py +++ b/python/sglang/srt/models/interns2_mobius.py @@ -10,7 +10,6 @@ from sglang.srt.configs.interns2_mobius import ( InternS2MobiusConfig, InternS2MobiusTextConfig, ) -from sglang.srt.distributed import tensor_model_parallel_all_reduce from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes from sglang.srt.layers.dp_attention import is_dp_attention_enabled from sglang.srt.layers.layernorm import GemmaRMSNorm @@ -20,7 +19,7 @@ from sglang.srt.layers.linear import ( RowParallelLinear, ) from sglang.srt.layers.moe import ( - should_skip_post_experts_all_reduce, + reduce_moe_output, ) from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.topk import TopK @@ -372,15 +371,6 @@ class InternS2MobiusRoutedExpertBank(nn.Module): return output.reshape(original_shape) -def _mobius_reduce_combined_output(combined: torch.Tensor) -> torch.Tensor: - """Apply the one ordinary TP reduction unless the scoped runtime owns it.""" - if get_parallel().tp_size > 1 and not should_skip_post_experts_all_reduce( - is_tp_path=True - ): - return tensor_model_parallel_all_reduce(combined) - return combined - - def _get_mobius_routed_bank(meta_mlp: nn.ModuleList, layer_id: int) -> nn.Module: if not meta_mlp: raise ValueError( @@ -421,7 +411,7 @@ class _InternS2MobiusDecoderMixin: routed = _get_mobius_routed_bank(meta_mlp, self.layer_id).forward_routed( hidden_states, forward_batch ) - return _mobius_reduce_combined_output(routed + shared) + return reduce_moe_output(routed + shared) def _forward_after_attention( self, diff --git a/python/sglang/srt/models/laguna.py b/python/sglang/srt/models/laguna.py index beecee42e2..546b2ea3a3 100644 --- a/python/sglang/srt/models/laguna.py +++ b/python/sglang/srt/models/laguna.py @@ -17,9 +17,6 @@ import torch.nn.functional as F from torch import nn from sglang.srt.configs.laguna import LagunaConfig, normalize_gating -from sglang.srt.distributed import ( - tensor_model_parallel_all_reduce, -) from sglang.srt.environ import envs from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import ( @@ -38,7 +35,7 @@ from sglang.srt.layers.linear import ( RowParallelLinear, ) from sglang.srt.layers.logits_processor import LogitsProcessor -from sglang.srt.layers.moe import should_skip_post_experts_all_reduce +from sglang.srt.layers.moe import reduce_moe_output from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.layers.moe.topk import TopK @@ -226,10 +223,7 @@ class LagunaMoE(nn.Module): else: final = routed_out + shared_out - if self.tp_size > 1 and not should_skip_post_experts_all_reduce( - is_tp_path=True, - ): - final = tensor_model_parallel_all_reduce(final) + final = reduce_moe_output(final) if self._shared_expert_tp1: final = final + shared_out return final diff --git a/python/sglang/srt/models/llada2.py b/python/sglang/srt/models/llada2.py index 7a8dd4b03b..4f4774c354 100644 --- a/python/sglang/srt/models/llada2.py +++ b/python/sglang/srt/models/llada2.py @@ -27,9 +27,6 @@ import torch.nn.functional as F from torch import nn from transformers import PretrainedConfig -from sglang.srt.distributed import ( - tensor_model_parallel_all_reduce, -) from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo @@ -52,7 +49,7 @@ from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe import ( get_deepep_mode, get_moe_a2a_backend, - should_skip_post_experts_all_reduce, + reduce_moe_output, ) from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE @@ -384,10 +381,7 @@ class LLaDA2MoeSparseMoeBlock(nn.Module): if self.num_shared_experts > 0: final_hidden_states = final_hidden_states + shared_output - if self.tp_size > 1 and not should_skip_post_experts_all_reduce( - is_tp_path=True, - ): - final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states) + final_hidden_states = reduce_moe_output(final_hidden_states) return final_hidden_states.view(num_tokens, hidden_size) def forward_deepep( diff --git a/python/sglang/srt/models/llama4.py b/python/sglang/srt/models/llama4.py index d7d8a26e96..af4f259e1c 100644 --- a/python/sglang/srt/models/llama4.py +++ b/python/sglang/srt/models/llama4.py @@ -25,9 +25,6 @@ import torch from torch import nn from transformers import Llama4TextConfig -from sglang.srt.distributed import ( - tensor_model_parallel_all_reduce, -) from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, @@ -38,7 +35,7 @@ from sglang.srt.layers.linear import ( ReplicatedLinear, RowParallelLinear, ) -from sglang.srt.layers.moe import should_skip_post_experts_all_reduce +from sglang.srt.layers.moe import reduce_moe_output from sglang.srt.layers.moe.fused_moe_triton import FusedMoE from sglang.srt.layers.moe.topk import TopK from sglang.srt.layers.quantization.base_config import QuantizationConfig @@ -145,10 +142,7 @@ class Llama4MoE(nn.Module): out_aD = routed_out + shared_out - if self.tp_size > 1 and not should_skip_post_experts_all_reduce( - is_tp_path=True, - ): - out_aD = tensor_model_parallel_all_reduce(out_aD) + out_aD = reduce_moe_output(out_aD) return out_aD diff --git a/python/sglang/srt/models/mimo_v2.py b/python/sglang/srt/models/mimo_v2.py index d0acdf362e..e1f27f560d 100644 --- a/python/sglang/srt/models/mimo_v2.py +++ b/python/sglang/srt/models/mimo_v2.py @@ -24,9 +24,6 @@ from torch import nn from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo from sglang.srt.configs.model_config import get_mimo_v2_fused_qkv_expected_tp_size -from sglang.srt.distributed import ( - tensor_model_parallel_all_reduce, -) from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo @@ -54,7 +51,7 @@ from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe import ( get_moe_a2a_backend, get_moe_runner_backend, - should_skip_post_experts_all_reduce, + reduce_moe_output, ) from sglang.srt.layers.moe.ep_moe.layer import DeepEPMoE, get_moe_impl_class from sglang.srt.layers.moe.topk import TopK, TopKOutputFormat @@ -500,10 +497,7 @@ class MiMoV2MoE(nn.Module): final_hidden_states = self.experts(hidden_states, topk_output) - if self.tp_size > 1 and not should_skip_post_experts_all_reduce( - is_tp_path=True, - ): - final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states) + final_hidden_states = reduce_moe_output(final_hidden_states) return final_hidden_states diff --git a/python/sglang/srt/models/minimax_m2.py b/python/sglang/srt/models/minimax_m2.py index dd4fd222a7..b0c968ba13 100644 --- a/python/sglang/srt/models/minimax_m2.py +++ b/python/sglang/srt/models/minimax_m2.py @@ -28,9 +28,6 @@ from transformers import PretrainedConfig from sglang.kernels.kernel_api_logging import debug_kernel_api from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo -from sglang.srt.distributed import ( - tensor_model_parallel_all_reduce, -) from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.communicator import ( @@ -51,7 +48,7 @@ from sglang.srt.layers.linear import ( from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe import ( get_moe_a2a_backend, - should_skip_post_experts_all_reduce, + reduce_moe_output, ) from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE @@ -601,10 +598,7 @@ class MiniMaxM2MoE(nn.Module): topk_output = self.topk.empty_topk_output(hidden_states.device) final_hidden_states = self.experts(hidden_states, topk_output) - if self.tp_size > 1 and not should_skip_post_experts_all_reduce( - is_tp_path=True - ): - final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states) + final_hidden_states = reduce_moe_output(final_hidden_states) return final_hidden_states.view(num_tokens, hidden_dim) diff --git a/python/sglang/srt/models/nemotron_h.py b/python/sglang/srt/models/nemotron_h.py index 81b4f60aa1..d20b290fb4 100644 --- a/python/sglang/srt/models/nemotron_h.py +++ b/python/sglang/srt/models/nemotron_h.py @@ -52,7 +52,7 @@ from sglang.srt.layers.moe.topk import TopK from sglang.srt.layers.moe.utils import ( RoutingMethodType, get_moe_a2a_backend, - should_skip_post_experts_all_reduce, + reduce_moe_output, ) from sglang.srt.layers.quantization import QuantizationConfig from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod @@ -372,10 +372,7 @@ class NemotronHMoE(nn.Module): elif shared_output is not None: final_hidden_states += shared_output - if self.tp_size > 1 and not should_skip_post_experts_all_reduce( - is_tp_path=True, - ): - final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states) + final_hidden_states = reduce_moe_output(final_hidden_states) return final_hidden_states.view(num_tokens, hidden_dim) diff --git a/python/sglang/srt/models/qwen2_moe.py b/python/sglang/srt/models/qwen2_moe.py index 3995ff04ab..732084936f 100644 --- a/python/sglang/srt/models/qwen2_moe.py +++ b/python/sglang/srt/models/qwen2_moe.py @@ -34,7 +34,6 @@ from sglang.kernels.ops.elementwise.elementwise import ( from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo from sglang.srt.distributed import ( get_pp_indices, - tensor_model_parallel_all_reduce, ) from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder @@ -60,7 +59,7 @@ from sglang.srt.layers.linear import ( from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe import ( get_moe_a2a_backend, - should_skip_post_experts_all_reduce, + reduce_moe_output, ) from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton import FusedMoE @@ -880,14 +879,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module): ) else: final_hidden_states += shared_output - if ( - self.tp_size > 1 - and not should_skip_post_experts_all_reduce( - is_tp_path=True, - ) - and not get_moe_a2a_backend().is_flashinfer() - ): - final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states) + final_hidden_states = reduce_moe_output(final_hidden_states) # Debug removed - was causing issues during CUDA graph capture diff --git a/python/sglang/srt/models/sarvam_moe.py b/python/sglang/srt/models/sarvam_moe.py index 34355089f6..9af712f12c 100644 --- a/python/sglang/srt/models/sarvam_moe.py +++ b/python/sglang/srt/models/sarvam_moe.py @@ -35,7 +35,7 @@ from sglang.srt.layers.linear import ( RowParallelLinear, ) from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput -from sglang.srt.layers.moe import should_skip_post_experts_all_reduce +from sglang.srt.layers.moe import reduce_moe_output from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.layers.moe.topk import TopK @@ -368,10 +368,7 @@ class SarvamMoESparseMoeBlock(nn.Module): final_hidden_states = final_hidden_states * self.routed_scaling_factor current_stream.wait_stream(self.alt_stream) final_hidden_states = final_hidden_states + shared_out - if self.tp_size > 1 and not should_skip_post_experts_all_reduce( - is_tp_path=True, - ): - final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states) + final_hidden_states = reduce_moe_output(final_hidden_states) return final_hidden_states.view(num_tokens, hidden_dim) def forward_normal( @@ -406,10 +403,7 @@ class SarvamMoESparseMoeBlock(nn.Module): elif self.routed_scaling_factor != 1.0: final_hidden_states = final_hidden_states * self.routed_scaling_factor - if self.tp_size > 1 and not should_skip_post_experts_all_reduce( - is_tp_path=True, - ): - final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states) + final_hidden_states = reduce_moe_output(final_hidden_states) return final_hidden_states.view(num_tokens, hidden_dim) diff --git a/python/sglang/srt/models/sdar_moe.py b/python/sglang/srt/models/sdar_moe.py index f63132413f..ec5eeb8bfb 100644 --- a/python/sglang/srt/models/sdar_moe.py +++ b/python/sglang/srt/models/sdar_moe.py @@ -10,9 +10,6 @@ import torch from torch import nn from transformers import PretrainedConfig -from sglang.srt.distributed import ( - tensor_model_parallel_all_reduce, -) from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo @@ -32,7 +29,7 @@ from sglang.srt.layers.linear import ( from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe import ( get_moe_a2a_backend, - should_skip_post_experts_all_reduce, + reduce_moe_output, ) from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE @@ -149,10 +146,7 @@ class SDARMoeSparseMoeBlock(nn.Module): topk_output = self.topk(hidden_states, router_logits) out = self.experts(hidden_states, topk_output) # (T, H) - if self.tp_size > 1 and not should_skip_post_experts_all_reduce( - is_tp_path=True, - ): - out = tensor_model_parallel_all_reduce(out) + out = reduce_moe_output(out) return out.view(num_tokens, hidden_dim) diff --git a/python/sglang/srt/models/step3p5.py b/python/sglang/srt/models/step3p5.py index e4479db47d..604cbfe1dc 100644 --- a/python/sglang/srt/models/step3p5.py +++ b/python/sglang/srt/models/step3p5.py @@ -29,7 +29,7 @@ from sglang.srt.layers.linear import ( from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe import ( get_moe_a2a_backend, - should_skip_post_experts_all_reduce, + reduce_moe_output, ) from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE @@ -225,10 +225,7 @@ class Step3p5MoEMLP(nn.Module): router_logits=topk_output.router_logits, ) final_hidden_states = self.experts(hidden_states, topk_output) - if self.tp_size > 1 and not should_skip_post_experts_all_reduce( - is_tp_path=True, - ): - final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states) + final_hidden_states = reduce_moe_output(final_hidden_states) return final_hidden_states.view(num_tokens, hidden_dim) diff --git a/python/sglang/srt/models/xllm.py b/python/sglang/srt/models/xllm.py index e09afdcb3d..33faedf085 100644 --- a/python/sglang/srt/models/xllm.py +++ b/python/sglang/srt/models/xllm.py @@ -35,7 +35,6 @@ import torch.nn.functional as F from torch import nn from transformers import PretrainedConfig -from sglang.srt.distributed import tensor_model_parallel_all_reduce from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo @@ -57,7 +56,7 @@ from sglang.srt.layers.linear import ( from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe import ( get_moe_a2a_backend, - should_skip_post_experts_all_reduce, + reduce_moe_output, ) from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton import FusedMoE @@ -1061,13 +1060,8 @@ class XllmSparseMoeBlock(nn.Module): if shared_output is not None: final_hidden_states += shared_output - if ( - self.tp_size > 1 - and not use_reduce_scatter - and not should_skip_post_experts_all_reduce(is_tp_path=True) - and not get_moe_a2a_backend().is_flashinfer() - ): - final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states) + if not use_reduce_scatter: + final_hidden_states = reduce_moe_output(final_hidden_states) return final_hidden_states.view(num_tokens, hidden_dim) diff --git a/test/registered/unit/layers/moe/test_moe_output_reduction.py b/test/registered/unit/layers/moe/test_moe_output_reduction.py new file mode 100644 index 0000000000..ff3abcc549 --- /dev/null +++ b/test/registered/unit/layers/moe/test_moe_output_reduction.py @@ -0,0 +1,145 @@ +"""When a MoE block all-reduces its own output, and why it does not.""" + +import ast +import contextlib +import types +import unittest +from pathlib import Path +from unittest.mock import patch + +import torch + +import sglang +from sglang.srt.layers.moe import utils as moe_utils +from sglang.srt.layers.moe.utils import ( + post_experts_output_is_complete, + reduce_moe_output, +) +from sglang.srt.runtime_context import get_forward +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=10, suite="base-a-test-cpu") + +MODELS_DIR = Path(sglang.__file__).resolve().parent / "srt" / "models" + + +def a2a(name=None): + names = ("flashinfer", "pplx", "flashinfer_megamoe") + return types.SimpleNamespace( + **{f"is_{n}": (lambda n=n: n == name) for n in names}, + is_none=lambda: name is None, + ) + + +@contextlib.contextmanager +def moe_config( + *, tp_size=2, dwdp_size=1, backend=None, fp4_allgather=False, reduce_scatterv=False +): + with ( + patch.object( + moe_utils, + "get_parallel", + return_value=types.SimpleNamespace(tp_size=tp_size, dwdp_size=dwdp_size), + ), + patch.object(moe_utils, "get_moe_a2a_backend", return_value=a2a(backend)), + patch.object( + moe_utils, + "should_use_flashinfer_cutlass_moe_fp4_allgather", + return_value=fp4_allgather, + ), + patch.object( + moe_utils, "should_use_dp_reduce_scatterv", return_value=reduce_scatterv + ), + ): + yield + + +class TestReduceMoeOutput(CustomTestCase): + def all_reduces(self, **flags): + calls = [] + with ( + patch( + "sglang.srt.distributed.communication_op.tensor_model_parallel_all_reduce", + side_effect=lambda x: calls.append(x) or x * 2, + ), + get_forward().scoped(**flags), + ): + output = reduce_moe_output(torch.ones(2, 3)) + return len(calls), output + + def test_partial_output_is_reduced_once(self): + with moe_config(): + count, output = self.all_reduces() + self.assertEqual(count, 1) + torch.testing.assert_close(output, torch.full((2, 3), 2.0)) + + def test_single_rank_has_nothing_to_sum(self): + with moe_config(tp_size=1): + self.assertEqual(self.all_reduces()[0], 0) + + def test_a_later_step_owns_the_sum(self): + for flags, config in ( + ({"fuse_mlp_allreduce": True}, {}), + ({"mlp_reduce_scatter": True}, {}), + ({}, {"reduce_scatterv": True}), + ): + with self.subTest(flags=flags, config=config), moe_config(**config): + self.assertEqual(self.all_reduces(**flags)[0], 0) + # Who runs the sum does not change whether one is owed. + self.assertFalse(post_experts_output_is_complete(is_tp_path=True)) + + def test_output_is_already_complete(self): + for config in ( + {"backend": "flashinfer"}, + {"backend": "pplx"}, + {"backend": "flashinfer_megamoe"}, + {"dwdp_size": 2}, + {"fp4_allgather": True}, + ): + with self.subTest(**config), moe_config(**config): + self.assertEqual(self.all_reduces()[0], 0) + self.assertTrue(post_experts_output_is_complete(is_tp_path=True)) + + def test_fp4_allgather_only_completes_the_tp_sum(self): + with moe_config(fp4_allgather=True): + self.assertFalse(post_experts_output_is_complete(is_tp_path=False)) + + +def open_coded_reductions(tree): + """``if`` statements that test should_skip_post_experts_all_reduce for the TP + path and call tensor_model_parallel_all_reduce themselves.""" + for node in ast.walk(tree): + if not isinstance(node, ast.If): + continue + tests_skip = any( + isinstance(sub, ast.Call) + and getattr(sub.func, "id", None) == "should_skip_post_experts_all_reduce" + and any( + kw.arg == "is_tp_path" and getattr(kw.value, "value", None) is True + for kw in sub.keywords + ) + for sub in ast.walk(node.test) + ) + reduces = any( + isinstance(sub, ast.Call) + and getattr(sub.func, "id", None) == "tensor_model_parallel_all_reduce" + for stmt in node.body + for sub in ast.walk(stmt) + ) + if tests_skip and reduces: + yield node.lineno + + +class TestModelsUseTheSharedReduction(CustomTestCase): + def test_no_model_open_codes_the_tp_reduction(self): + found = [ + f"{path.relative_to(MODELS_DIR)}:{line}" + for path in sorted(MODELS_DIR.rglob("*.py")) + for line in open_coded_reductions(ast.parse(path.read_text())) + ] + self.assertEqual(found, [], "use reduce_moe_output") + + +if __name__ == "__main__": + unittest.main()