[Refactor] Share the MoE output all-reduce between models (#41097)

This commit is contained in:
Cheng Wan
2026-09-24 12:15:33 -07:00
committed by GitHub
parent 5df1667077
commit 62ae032c46
21 changed files with 228 additions and 173 deletions
+2
View File
@@ -14,6 +14,7 @@ from sglang.srt.layers.moe.utils import (
is_moe_input_scattered_across_dp_ranks,
is_tbo_enabled,
post_experts_all_reduce,
reduce_moe_output,
should_skip_mlp_all_reduce,
should_skip_post_experts_all_reduce,
should_use_dp_reduce_scatterv,
@@ -34,6 +35,7 @@ __all__ = [
"can_merge_post_experts_all_reduce",
"deferred_post_experts_all_reduce",
"post_experts_all_reduce",
"reduce_moe_output",
"should_skip_post_experts_all_reduce",
"should_use_dp_reduce_scatterv",
"should_use_flashinfer_cutlass_moe_fp4_allgather",
+41 -21
View File
@@ -732,31 +732,51 @@ def should_skip_mlp_all_reduce() -> bool:
return f.fuse_mlp_allreduce or f.mlp_reduce_scatter
def post_experts_output_is_complete(*, is_tp_path: bool) -> bool:
"""Whether the experts' output owes no sum over the MoE-TP group
(``is_tp_path=True``) or the EP group: the combine already summed it, or each
rank computed its own tokens in full.
This is a property of the MoE configuration. Whether the MoE block or a later
step runs a sum that is still owed is decided separately.
"""
if get_parallel().dwdp_size > 1:
return True
if is_tp_path and should_use_flashinfer_cutlass_moe_fp4_allgather():
# The combine reduce-scatters back to the local tokens.
return True
a2a = get_moe_a2a_backend()
# The flashinfer and pplx combines, and the megamoe kernel's internal
# combine, sum each token's expert outputs back to its source rank.
return a2a.is_flashinfer() or a2a.is_pplx() or a2a.is_flashinfer_megamoe()
def should_skip_post_experts_all_reduce(*, is_tp_path: bool) -> bool:
"""Whether a downstream component will fuse, replace, or absorb the post-experts all-reduce.
"""Whether the MoE block should leave out its post-experts all-reduce: a later
step runs it (fused into the next norm, or as the reduce-scatter back to the
local tokens), or there is nothing to sum.
Pass ``is_tp_path=True`` for the TP all-reduce, ``False`` for the EP one.
"""
if should_skip_mlp_all_reduce():
return True
if get_parallel().dwdp_size > 1:
return True
if should_use_dp_reduce_scatterv():
return True
if is_tp_path and should_use_flashinfer_cutlass_moe_fp4_allgather():
return True
if get_moe_a2a_backend().is_flashinfer():
return True
if get_moe_a2a_backend().is_pplx():
# pplx's AllToAll.combine already sums each token's expert outputs back
# to the source rank
return True
if get_moe_a2a_backend().is_flashinfer_megamoe():
# The mega kernel does its EP all-to-all + combine internally and
# returns per-rank outputs, so any further EP/TP all-reduce would
# double-count. Same opt-in as the flashinfer a2a dispatcher.
return True
return False
return (
should_skip_mlp_all_reduce()
or should_use_dp_reduce_scatterv()
or post_experts_output_is_complete(is_tp_path=is_tp_path)
)
def reduce_moe_output(hidden_states: torch.Tensor) -> torch.Tensor:
"""All-reduce a MoE block's output (routed plus shared experts) over TP,
unless a later step does it or there is nothing to sum."""
from sglang.srt.distributed.communication_op import (
tensor_model_parallel_all_reduce,
)
if get_parallel().tp_size > 1 and not should_skip_post_experts_all_reduce(
is_tp_path=True
):
return tensor_model_parallel_all_reduce(hidden_states)
return hidden_states
def can_merge_post_experts_all_reduce() -> bool:
+2 -8
View File
@@ -27,9 +27,6 @@ import torch.nn.functional as F
from torch import nn
from transformers import PretrainedConfig
from sglang.srt.distributed import (
tensor_model_parallel_all_reduce,
)
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
@@ -53,7 +50,7 @@ from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_deepep_mode,
get_moe_a2a_backend,
should_skip_post_experts_all_reduce,
reduce_moe_output,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
@@ -455,10 +452,7 @@ class BailingMoESparseMoeBlock(nn.Module):
if self.num_shared_experts > 0:
final_hidden_states = final_hidden_states + shared_output
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
is_tp_path=True,
):
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states.view(num_tokens, hidden_size)
def forward_deepep(
@@ -12,9 +12,6 @@ from transformers import PretrainedConfig
from sglang.kernels.ops.attention.fla.layernorm_gated import RMSNorm as RMSNormGated
from sglang.kernels.ops.attention.fla.layernorm_gated import layernorm_fn
from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz
from sglang.srt.distributed import (
tensor_model_parallel_all_reduce,
)
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.activation import SiluAndMul
@@ -33,7 +30,7 @@ from sglang.srt.layers.linear import (
RowParallelLinear,
)
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import should_skip_post_experts_all_reduce
from sglang.srt.layers.moe import reduce_moe_output
from sglang.srt.layers.moe.ep_moe.layer import DeepEPMoE, get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.moe.topk import TopK
@@ -366,10 +363,7 @@ class BailingMoE(nn.Module):
if self.num_shared_experts > 0:
final_hidden_states = final_hidden_states + shared_output
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
is_tp_path=True,
):
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states
+2 -8
View File
@@ -24,9 +24,6 @@ import torch
from torch import nn
from transformers import PretrainedConfig
from sglang.srt.distributed import (
tensor_model_parallel_all_reduce,
)
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
@@ -44,7 +41,7 @@ from sglang.srt.layers.linear import (
from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
should_skip_post_experts_all_reduce,
reduce_moe_output,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
@@ -290,10 +287,7 @@ class ExaoneMoESparseMoEBlock(nn.Module):
if shared_output is not None:
final_hidden_states = final_hidden_states + shared_output
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
is_tp_path=True,
):
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states.view(num_tokens, hidden_dim)
+3 -10
View File
@@ -29,7 +29,6 @@ from sglang.srt.batch_overlap.single_batch_overlap import SboFlags
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo
from sglang.srt.distributed import (
get_pp_indices,
tensor_model_parallel_all_reduce,
)
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
use_symmetric_memory,
@@ -58,7 +57,7 @@ from sglang.srt.layers.linear import (
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
should_skip_post_experts_all_reduce,
reduce_moe_output,
should_use_flashinfer_cutlass_moe_fp4_allgather,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
@@ -621,10 +620,7 @@ class Glm4MoeSparseMoeBlock(nn.Module):
current_stream.wait_stream(self.alt_stream)
final_hidden_states += shared_output
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
is_tp_path=True,
):
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states
def forward_normal(
@@ -650,10 +646,7 @@ class Glm4MoeSparseMoeBlock(nn.Module):
final_hidden_states_out = torch.empty_like(final_hidden_states)
torch.add(final_hidden_states, shared_output, out=final_hidden_states_out)
final_hidden_states = final_hidden_states_out
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
is_tp_path=True,
):
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states
def forward_deepep(
+3 -12
View File
@@ -25,9 +25,6 @@ from transformers import PretrainedConfig
from sglang.srt.batch_overlap.single_batch_overlap import SboFlags
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo
from sglang.srt.distributed import (
tensor_model_parallel_all_reduce,
)
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
use_symmetric_memory,
)
@@ -51,7 +48,7 @@ from sglang.srt.layers.linear import MergedColumnParallelLinear, RowParallelLine
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
should_skip_post_experts_all_reduce,
reduce_moe_output,
should_use_flashinfer_cutlass_moe_fp4_allgather,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
@@ -340,10 +337,7 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module):
current_stream.wait_stream(self.alt_stream)
final_hidden_states += shared_output
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
is_tp_path=True,
):
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states
def forward_normal(
@@ -369,10 +363,7 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module):
final_hidden_states_out = torch.empty_like(final_hidden_states)
torch.add(final_hidden_states, shared_output, out=final_hidden_states_out)
final_hidden_states = final_hidden_states_out
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
is_tp_path=True,
):
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states
def forward_deepep(
+2 -8
View File
@@ -27,9 +27,6 @@ from torch import nn
from transformers import PretrainedConfig
from sglang.kernels.jit.utils import is_arch_support_pdl
from sglang.srt.distributed import (
tensor_model_parallel_all_reduce,
)
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.communicator import (
@@ -49,7 +46,7 @@ from sglang.srt.layers.linear import (
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
should_skip_post_experts_all_reduce,
reduce_moe_output,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
@@ -337,10 +334,7 @@ class GptOssSparseMoeBlock(nn.Module):
topk_output = self.topk(router_input, router_logits)
final_hidden_states = self.experts(hidden_states, topk_output)
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
is_tp_path=True,
):
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
final_hidden_states = reduce_moe_output(final_hidden_states)
# When input was pre-padded, FusedMoE.forward_impl captured the
# padded width as `origin_hidden_states_dim` and skipped its own
+2 -12
View File
@@ -10,7 +10,6 @@ from sglang.srt.configs.interns2_mobius import (
InternS2MobiusConfig,
InternS2MobiusTextConfig,
)
from sglang.srt.distributed import tensor_model_parallel_all_reduce
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layernorm import GemmaRMSNorm
@@ -20,7 +19,7 @@ from sglang.srt.layers.linear import (
RowParallelLinear,
)
from sglang.srt.layers.moe import (
should_skip_post_experts_all_reduce,
reduce_moe_output,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.topk import TopK
@@ -372,15 +371,6 @@ class InternS2MobiusRoutedExpertBank(nn.Module):
return output.reshape(original_shape)
def _mobius_reduce_combined_output(combined: torch.Tensor) -> torch.Tensor:
"""Apply the one ordinary TP reduction unless the scoped runtime owns it."""
if get_parallel().tp_size > 1 and not should_skip_post_experts_all_reduce(
is_tp_path=True
):
return tensor_model_parallel_all_reduce(combined)
return combined
def _get_mobius_routed_bank(meta_mlp: nn.ModuleList, layer_id: int) -> nn.Module:
if not meta_mlp:
raise ValueError(
@@ -421,7 +411,7 @@ class _InternS2MobiusDecoderMixin:
routed = _get_mobius_routed_bank(meta_mlp, self.layer_id).forward_routed(
hidden_states, forward_batch
)
return _mobius_reduce_combined_output(routed + shared)
return reduce_moe_output(routed + shared)
def _forward_after_attention(
self,
+2 -8
View File
@@ -17,9 +17,6 @@ import torch.nn.functional as F
from torch import nn
from sglang.srt.configs.laguna import LagunaConfig, normalize_gating
from sglang.srt.distributed import (
tensor_model_parallel_all_reduce,
)
from sglang.srt.environ import envs
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.communicator import (
@@ -38,7 +35,7 @@ from sglang.srt.layers.linear import (
RowParallelLinear,
)
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import should_skip_post_experts_all_reduce
from sglang.srt.layers.moe import reduce_moe_output
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.moe.topk import TopK
@@ -226,10 +223,7 @@ class LagunaMoE(nn.Module):
else:
final = routed_out + shared_out
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
is_tp_path=True,
):
final = tensor_model_parallel_all_reduce(final)
final = reduce_moe_output(final)
if self._shared_expert_tp1:
final = final + shared_out
return final
+2 -8
View File
@@ -27,9 +27,6 @@ import torch.nn.functional as F
from torch import nn
from transformers import PretrainedConfig
from sglang.srt.distributed import (
tensor_model_parallel_all_reduce,
)
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
@@ -52,7 +49,7 @@ from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_deepep_mode,
get_moe_a2a_backend,
should_skip_post_experts_all_reduce,
reduce_moe_output,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
@@ -384,10 +381,7 @@ class LLaDA2MoeSparseMoeBlock(nn.Module):
if self.num_shared_experts > 0:
final_hidden_states = final_hidden_states + shared_output
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
is_tp_path=True,
):
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states.view(num_tokens, hidden_size)
def forward_deepep(
+2 -8
View File
@@ -25,9 +25,6 @@ import torch
from torch import nn
from transformers import Llama4TextConfig
from sglang.srt.distributed import (
tensor_model_parallel_all_reduce,
)
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -38,7 +35,7 @@ from sglang.srt.layers.linear import (
ReplicatedLinear,
RowParallelLinear,
)
from sglang.srt.layers.moe import should_skip_post_experts_all_reduce
from sglang.srt.layers.moe import reduce_moe_output
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
from sglang.srt.layers.moe.topk import TopK
from sglang.srt.layers.quantization.base_config import QuantizationConfig
@@ -145,10 +142,7 @@ class Llama4MoE(nn.Module):
out_aD = routed_out + shared_out
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
is_tp_path=True,
):
out_aD = tensor_model_parallel_all_reduce(out_aD)
out_aD = reduce_moe_output(out_aD)
return out_aD
+2 -8
View File
@@ -24,9 +24,6 @@ from torch import nn
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo
from sglang.srt.configs.model_config import get_mimo_v2_fused_qkv_expected_tp_size
from sglang.srt.distributed import (
tensor_model_parallel_all_reduce,
)
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
@@ -54,7 +51,7 @@ from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
get_moe_runner_backend,
should_skip_post_experts_all_reduce,
reduce_moe_output,
)
from sglang.srt.layers.moe.ep_moe.layer import DeepEPMoE, get_moe_impl_class
from sglang.srt.layers.moe.topk import TopK, TopKOutputFormat
@@ -500,10 +497,7 @@ class MiMoV2MoE(nn.Module):
final_hidden_states = self.experts(hidden_states, topk_output)
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
is_tp_path=True,
):
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states
+2 -8
View File
@@ -28,9 +28,6 @@ from transformers import PretrainedConfig
from sglang.kernels.kernel_api_logging import debug_kernel_api
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo
from sglang.srt.distributed import (
tensor_model_parallel_all_reduce,
)
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.communicator import (
@@ -51,7 +48,7 @@ from sglang.srt.layers.linear import (
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
should_skip_post_experts_all_reduce,
reduce_moe_output,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
@@ -601,10 +598,7 @@ class MiniMaxM2MoE(nn.Module):
topk_output = self.topk.empty_topk_output(hidden_states.device)
final_hidden_states = self.experts(hidden_states, topk_output)
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
is_tp_path=True
):
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states.view(num_tokens, hidden_dim)
+2 -5
View File
@@ -52,7 +52,7 @@ from sglang.srt.layers.moe.topk import TopK
from sglang.srt.layers.moe.utils import (
RoutingMethodType,
get_moe_a2a_backend,
should_skip_post_experts_all_reduce,
reduce_moe_output,
)
from sglang.srt.layers.quantization import QuantizationConfig
from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod
@@ -372,10 +372,7 @@ class NemotronHMoE(nn.Module):
elif shared_output is not None:
final_hidden_states += shared_output
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
is_tp_path=True,
):
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states.view(num_tokens, hidden_dim)
+2 -10
View File
@@ -34,7 +34,6 @@ from sglang.kernels.ops.elementwise.elementwise import (
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo
from sglang.srt.distributed import (
get_pp_indices,
tensor_model_parallel_all_reduce,
)
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
@@ -60,7 +59,7 @@ from sglang.srt.layers.linear import (
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
should_skip_post_experts_all_reduce,
reduce_moe_output,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
@@ -880,14 +879,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
)
else:
final_hidden_states += shared_output
if (
self.tp_size > 1
and not should_skip_post_experts_all_reduce(
is_tp_path=True,
)
and not get_moe_a2a_backend().is_flashinfer()
):
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
final_hidden_states = reduce_moe_output(final_hidden_states)
# Debug removed - was causing issues during CUDA graph capture
+3 -9
View File
@@ -35,7 +35,7 @@ from sglang.srt.layers.linear import (
RowParallelLinear,
)
from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput
from sglang.srt.layers.moe import should_skip_post_experts_all_reduce
from sglang.srt.layers.moe import reduce_moe_output
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.moe.topk import TopK
@@ -368,10 +368,7 @@ class SarvamMoESparseMoeBlock(nn.Module):
final_hidden_states = final_hidden_states * self.routed_scaling_factor
current_stream.wait_stream(self.alt_stream)
final_hidden_states = final_hidden_states + shared_out
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
is_tp_path=True,
):
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states.view(num_tokens, hidden_dim)
def forward_normal(
@@ -406,10 +403,7 @@ class SarvamMoESparseMoeBlock(nn.Module):
elif self.routed_scaling_factor != 1.0:
final_hidden_states = final_hidden_states * self.routed_scaling_factor
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
is_tp_path=True,
):
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states.view(num_tokens, hidden_dim)
+2 -8
View File
@@ -10,9 +10,6 @@ import torch
from torch import nn
from transformers import PretrainedConfig
from sglang.srt.distributed import (
tensor_model_parallel_all_reduce,
)
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
@@ -32,7 +29,7 @@ from sglang.srt.layers.linear import (
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
should_skip_post_experts_all_reduce,
reduce_moe_output,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
@@ -149,10 +146,7 @@ class SDARMoeSparseMoeBlock(nn.Module):
topk_output = self.topk(hidden_states, router_logits)
out = self.experts(hidden_states, topk_output) # (T, H)
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
is_tp_path=True,
):
out = tensor_model_parallel_all_reduce(out)
out = reduce_moe_output(out)
return out.view(num_tokens, hidden_dim)
+2 -5
View File
@@ -29,7 +29,7 @@ from sglang.srt.layers.linear import (
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
should_skip_post_experts_all_reduce,
reduce_moe_output,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
@@ -225,10 +225,7 @@ class Step3p5MoEMLP(nn.Module):
router_logits=topk_output.router_logits,
)
final_hidden_states = self.experts(hidden_states, topk_output)
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
is_tp_path=True,
):
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states.view(num_tokens, hidden_dim)
+3 -9
View File
@@ -35,7 +35,6 @@ import torch.nn.functional as F
from torch import nn
from transformers import PretrainedConfig
from sglang.srt.distributed import tensor_model_parallel_all_reduce
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
@@ -57,7 +56,7 @@ from sglang.srt.layers.linear import (
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
should_skip_post_experts_all_reduce,
reduce_moe_output,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
@@ -1061,13 +1060,8 @@ class XllmSparseMoeBlock(nn.Module):
if shared_output is not None:
final_hidden_states += shared_output
if (
self.tp_size > 1
and not use_reduce_scatter
and not should_skip_post_experts_all_reduce(is_tp_path=True)
and not get_moe_a2a_backend().is_flashinfer()
):
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
if not use_reduce_scatter:
final_hidden_states = reduce_moe_output(final_hidden_states)
return final_hidden_states.view(num_tokens, hidden_dim)