[Refactor] Share the MoE output all-reduce between models (#41097)
This commit is contained in:
@@ -14,6 +14,7 @@ from sglang.srt.layers.moe.utils import (
|
||||
is_moe_input_scattered_across_dp_ranks,
|
||||
is_tbo_enabled,
|
||||
post_experts_all_reduce,
|
||||
reduce_moe_output,
|
||||
should_skip_mlp_all_reduce,
|
||||
should_skip_post_experts_all_reduce,
|
||||
should_use_dp_reduce_scatterv,
|
||||
@@ -34,6 +35,7 @@ __all__ = [
|
||||
"can_merge_post_experts_all_reduce",
|
||||
"deferred_post_experts_all_reduce",
|
||||
"post_experts_all_reduce",
|
||||
"reduce_moe_output",
|
||||
"should_skip_post_experts_all_reduce",
|
||||
"should_use_dp_reduce_scatterv",
|
||||
"should_use_flashinfer_cutlass_moe_fp4_allgather",
|
||||
|
||||
@@ -732,31 +732,51 @@ def should_skip_mlp_all_reduce() -> bool:
|
||||
return f.fuse_mlp_allreduce or f.mlp_reduce_scatter
|
||||
|
||||
|
||||
def post_experts_output_is_complete(*, is_tp_path: bool) -> bool:
|
||||
"""Whether the experts' output owes no sum over the MoE-TP group
|
||||
(``is_tp_path=True``) or the EP group: the combine already summed it, or each
|
||||
rank computed its own tokens in full.
|
||||
|
||||
This is a property of the MoE configuration. Whether the MoE block or a later
|
||||
step runs a sum that is still owed is decided separately.
|
||||
"""
|
||||
if get_parallel().dwdp_size > 1:
|
||||
return True
|
||||
if is_tp_path and should_use_flashinfer_cutlass_moe_fp4_allgather():
|
||||
# The combine reduce-scatters back to the local tokens.
|
||||
return True
|
||||
a2a = get_moe_a2a_backend()
|
||||
# The flashinfer and pplx combines, and the megamoe kernel's internal
|
||||
# combine, sum each token's expert outputs back to its source rank.
|
||||
return a2a.is_flashinfer() or a2a.is_pplx() or a2a.is_flashinfer_megamoe()
|
||||
|
||||
|
||||
def should_skip_post_experts_all_reduce(*, is_tp_path: bool) -> bool:
|
||||
"""Whether a downstream component will fuse, replace, or absorb the post-experts all-reduce.
|
||||
"""Whether the MoE block should leave out its post-experts all-reduce: a later
|
||||
step runs it (fused into the next norm, or as the reduce-scatter back to the
|
||||
local tokens), or there is nothing to sum.
|
||||
|
||||
Pass ``is_tp_path=True`` for the TP all-reduce, ``False`` for the EP one.
|
||||
"""
|
||||
if should_skip_mlp_all_reduce():
|
||||
return True
|
||||
if get_parallel().dwdp_size > 1:
|
||||
return True
|
||||
if should_use_dp_reduce_scatterv():
|
||||
return True
|
||||
if is_tp_path and should_use_flashinfer_cutlass_moe_fp4_allgather():
|
||||
return True
|
||||
if get_moe_a2a_backend().is_flashinfer():
|
||||
return True
|
||||
if get_moe_a2a_backend().is_pplx():
|
||||
# pplx's AllToAll.combine already sums each token's expert outputs back
|
||||
# to the source rank
|
||||
return True
|
||||
if get_moe_a2a_backend().is_flashinfer_megamoe():
|
||||
# The mega kernel does its EP all-to-all + combine internally and
|
||||
# returns per-rank outputs, so any further EP/TP all-reduce would
|
||||
# double-count. Same opt-in as the flashinfer a2a dispatcher.
|
||||
return True
|
||||
return False
|
||||
return (
|
||||
should_skip_mlp_all_reduce()
|
||||
or should_use_dp_reduce_scatterv()
|
||||
or post_experts_output_is_complete(is_tp_path=is_tp_path)
|
||||
)
|
||||
|
||||
|
||||
def reduce_moe_output(hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""All-reduce a MoE block's output (routed plus shared experts) over TP,
|
||||
unless a later step does it or there is nothing to sum."""
|
||||
from sglang.srt.distributed.communication_op import (
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
|
||||
if get_parallel().tp_size > 1 and not should_skip_post_experts_all_reduce(
|
||||
is_tp_path=True
|
||||
):
|
||||
return tensor_model_parallel_all_reduce(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
|
||||
def can_merge_post_experts_all_reduce() -> bool:
|
||||
|
||||
@@ -27,9 +27,6 @@ import torch.nn.functional as F
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.distributed import (
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
@@ -53,7 +50,7 @@ from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe import (
|
||||
get_deepep_mode,
|
||||
get_moe_a2a_backend,
|
||||
should_skip_post_experts_all_reduce,
|
||||
reduce_moe_output,
|
||||
)
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
@@ -455,10 +452,7 @@ class BailingMoESparseMoeBlock(nn.Module):
|
||||
if self.num_shared_experts > 0:
|
||||
final_hidden_states = final_hidden_states + shared_output
|
||||
|
||||
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
|
||||
is_tp_path=True,
|
||||
):
|
||||
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
return final_hidden_states.view(num_tokens, hidden_size)
|
||||
|
||||
def forward_deepep(
|
||||
|
||||
@@ -12,9 +12,6 @@ from transformers import PretrainedConfig
|
||||
from sglang.kernels.ops.attention.fla.layernorm_gated import RMSNorm as RMSNormGated
|
||||
from sglang.kernels.ops.attention.fla.layernorm_gated import layernorm_fn
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz
|
||||
from sglang.srt.distributed import (
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.layers import deep_gemm_wrapper
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
@@ -33,7 +30,7 @@ from sglang.srt.layers.linear import (
|
||||
RowParallelLinear,
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe import should_skip_post_experts_all_reduce
|
||||
from sglang.srt.layers.moe import reduce_moe_output
|
||||
from sglang.srt.layers.moe.ep_moe.layer import DeepEPMoE, get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.layers.moe.topk import TopK
|
||||
@@ -366,10 +363,7 @@ class BailingMoE(nn.Module):
|
||||
if self.num_shared_experts > 0:
|
||||
final_hidden_states = final_hidden_states + shared_output
|
||||
|
||||
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
|
||||
is_tp_path=True,
|
||||
):
|
||||
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
return final_hidden_states
|
||||
|
||||
|
||||
|
||||
@@ -24,9 +24,6 @@ import torch
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.distributed import (
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
@@ -44,7 +41,7 @@ from sglang.srt.layers.linear import (
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput
|
||||
from sglang.srt.layers.moe import (
|
||||
get_moe_a2a_backend,
|
||||
should_skip_post_experts_all_reduce,
|
||||
reduce_moe_output,
|
||||
)
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
|
||||
@@ -290,10 +287,7 @@ class ExaoneMoESparseMoEBlock(nn.Module):
|
||||
|
||||
if shared_output is not None:
|
||||
final_hidden_states = final_hidden_states + shared_output
|
||||
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
|
||||
is_tp_path=True,
|
||||
):
|
||||
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
|
||||
return final_hidden_states.view(num_tokens, hidden_dim)
|
||||
|
||||
|
||||
@@ -29,7 +29,6 @@ from sglang.srt.batch_overlap.single_batch_overlap import SboFlags
|
||||
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo
|
||||
from sglang.srt.distributed import (
|
||||
get_pp_indices,
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
use_symmetric_memory,
|
||||
@@ -58,7 +57,7 @@ from sglang.srt.layers.linear import (
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe import (
|
||||
get_moe_a2a_backend,
|
||||
should_skip_post_experts_all_reduce,
|
||||
reduce_moe_output,
|
||||
should_use_flashinfer_cutlass_moe_fp4_allgather,
|
||||
)
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
@@ -621,10 +620,7 @@ class Glm4MoeSparseMoeBlock(nn.Module):
|
||||
|
||||
current_stream.wait_stream(self.alt_stream)
|
||||
final_hidden_states += shared_output
|
||||
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
|
||||
is_tp_path=True,
|
||||
):
|
||||
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
return final_hidden_states
|
||||
|
||||
def forward_normal(
|
||||
@@ -650,10 +646,7 @@ class Glm4MoeSparseMoeBlock(nn.Module):
|
||||
final_hidden_states_out = torch.empty_like(final_hidden_states)
|
||||
torch.add(final_hidden_states, shared_output, out=final_hidden_states_out)
|
||||
final_hidden_states = final_hidden_states_out
|
||||
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
|
||||
is_tp_path=True,
|
||||
):
|
||||
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
return final_hidden_states
|
||||
|
||||
def forward_deepep(
|
||||
|
||||
@@ -25,9 +25,6 @@ from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.batch_overlap.single_batch_overlap import SboFlags
|
||||
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo
|
||||
from sglang.srt.distributed import (
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
use_symmetric_memory,
|
||||
)
|
||||
@@ -51,7 +48,7 @@ from sglang.srt.layers.linear import MergedColumnParallelLinear, RowParallelLine
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe import (
|
||||
get_moe_a2a_backend,
|
||||
should_skip_post_experts_all_reduce,
|
||||
reduce_moe_output,
|
||||
should_use_flashinfer_cutlass_moe_fp4_allgather,
|
||||
)
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
@@ -340,10 +337,7 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module):
|
||||
|
||||
current_stream.wait_stream(self.alt_stream)
|
||||
final_hidden_states += shared_output
|
||||
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
|
||||
is_tp_path=True,
|
||||
):
|
||||
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
return final_hidden_states
|
||||
|
||||
def forward_normal(
|
||||
@@ -369,10 +363,7 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module):
|
||||
final_hidden_states_out = torch.empty_like(final_hidden_states)
|
||||
torch.add(final_hidden_states, shared_output, out=final_hidden_states_out)
|
||||
final_hidden_states = final_hidden_states_out
|
||||
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
|
||||
is_tp_path=True,
|
||||
):
|
||||
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
return final_hidden_states
|
||||
|
||||
def forward_deepep(
|
||||
|
||||
@@ -27,9 +27,6 @@ from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.kernels.jit.utils import is_arch_support_pdl
|
||||
from sglang.srt.distributed import (
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.communicator import (
|
||||
@@ -49,7 +46,7 @@ from sglang.srt.layers.linear import (
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe import (
|
||||
get_moe_a2a_backend,
|
||||
should_skip_post_experts_all_reduce,
|
||||
reduce_moe_output,
|
||||
)
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
@@ -337,10 +334,7 @@ class GptOssSparseMoeBlock(nn.Module):
|
||||
topk_output = self.topk(router_input, router_logits)
|
||||
final_hidden_states = self.experts(hidden_states, topk_output)
|
||||
|
||||
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
|
||||
is_tp_path=True,
|
||||
):
|
||||
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
|
||||
# When input was pre-padded, FusedMoE.forward_impl captured the
|
||||
# padded width as `origin_hidden_states_dim` and skipped its own
|
||||
|
||||
@@ -10,7 +10,6 @@ from sglang.srt.configs.interns2_mobius import (
|
||||
InternS2MobiusConfig,
|
||||
InternS2MobiusTextConfig,
|
||||
)
|
||||
from sglang.srt.distributed import tensor_model_parallel_all_reduce
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layernorm import GemmaRMSNorm
|
||||
@@ -20,7 +19,7 @@ from sglang.srt.layers.linear import (
|
||||
RowParallelLinear,
|
||||
)
|
||||
from sglang.srt.layers.moe import (
|
||||
should_skip_post_experts_all_reduce,
|
||||
reduce_moe_output,
|
||||
)
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.topk import TopK
|
||||
@@ -372,15 +371,6 @@ class InternS2MobiusRoutedExpertBank(nn.Module):
|
||||
return output.reshape(original_shape)
|
||||
|
||||
|
||||
def _mobius_reduce_combined_output(combined: torch.Tensor) -> torch.Tensor:
|
||||
"""Apply the one ordinary TP reduction unless the scoped runtime owns it."""
|
||||
if get_parallel().tp_size > 1 and not should_skip_post_experts_all_reduce(
|
||||
is_tp_path=True
|
||||
):
|
||||
return tensor_model_parallel_all_reduce(combined)
|
||||
return combined
|
||||
|
||||
|
||||
def _get_mobius_routed_bank(meta_mlp: nn.ModuleList, layer_id: int) -> nn.Module:
|
||||
if not meta_mlp:
|
||||
raise ValueError(
|
||||
@@ -421,7 +411,7 @@ class _InternS2MobiusDecoderMixin:
|
||||
routed = _get_mobius_routed_bank(meta_mlp, self.layer_id).forward_routed(
|
||||
hidden_states, forward_batch
|
||||
)
|
||||
return _mobius_reduce_combined_output(routed + shared)
|
||||
return reduce_moe_output(routed + shared)
|
||||
|
||||
def _forward_after_attention(
|
||||
self,
|
||||
|
||||
@@ -17,9 +17,6 @@ import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.configs.laguna import LagunaConfig, normalize_gating
|
||||
from sglang.srt.distributed import (
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.communicator import (
|
||||
@@ -38,7 +35,7 @@ from sglang.srt.layers.linear import (
|
||||
RowParallelLinear,
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe import should_skip_post_experts_all_reduce
|
||||
from sglang.srt.layers.moe import reduce_moe_output
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.layers.moe.topk import TopK
|
||||
@@ -226,10 +223,7 @@ class LagunaMoE(nn.Module):
|
||||
else:
|
||||
final = routed_out + shared_out
|
||||
|
||||
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
|
||||
is_tp_path=True,
|
||||
):
|
||||
final = tensor_model_parallel_all_reduce(final)
|
||||
final = reduce_moe_output(final)
|
||||
if self._shared_expert_tp1:
|
||||
final = final + shared_out
|
||||
return final
|
||||
|
||||
@@ -27,9 +27,6 @@ import torch.nn.functional as F
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.distributed import (
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
@@ -52,7 +49,7 @@ from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe import (
|
||||
get_deepep_mode,
|
||||
get_moe_a2a_backend,
|
||||
should_skip_post_experts_all_reduce,
|
||||
reduce_moe_output,
|
||||
)
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
@@ -384,10 +381,7 @@ class LLaDA2MoeSparseMoeBlock(nn.Module):
|
||||
if self.num_shared_experts > 0:
|
||||
final_hidden_states = final_hidden_states + shared_output
|
||||
|
||||
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
|
||||
is_tp_path=True,
|
||||
):
|
||||
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
return final_hidden_states.view(num_tokens, hidden_size)
|
||||
|
||||
def forward_deepep(
|
||||
|
||||
@@ -25,9 +25,6 @@ import torch
|
||||
from torch import nn
|
||||
from transformers import Llama4TextConfig
|
||||
|
||||
from sglang.srt.distributed import (
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -38,7 +35,7 @@ from sglang.srt.layers.linear import (
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from sglang.srt.layers.moe import should_skip_post_experts_all_reduce
|
||||
from sglang.srt.layers.moe import reduce_moe_output
|
||||
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
|
||||
from sglang.srt.layers.moe.topk import TopK
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
@@ -145,10 +142,7 @@ class Llama4MoE(nn.Module):
|
||||
|
||||
out_aD = routed_out + shared_out
|
||||
|
||||
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
|
||||
is_tp_path=True,
|
||||
):
|
||||
out_aD = tensor_model_parallel_all_reduce(out_aD)
|
||||
out_aD = reduce_moe_output(out_aD)
|
||||
|
||||
return out_aD
|
||||
|
||||
|
||||
@@ -24,9 +24,6 @@ from torch import nn
|
||||
|
||||
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo
|
||||
from sglang.srt.configs.model_config import get_mimo_v2_fused_qkv_expected_tp_size
|
||||
from sglang.srt.distributed import (
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
@@ -54,7 +51,7 @@ from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe import (
|
||||
get_moe_a2a_backend,
|
||||
get_moe_runner_backend,
|
||||
should_skip_post_experts_all_reduce,
|
||||
reduce_moe_output,
|
||||
)
|
||||
from sglang.srt.layers.moe.ep_moe.layer import DeepEPMoE, get_moe_impl_class
|
||||
from sglang.srt.layers.moe.topk import TopK, TopKOutputFormat
|
||||
@@ -500,10 +497,7 @@ class MiMoV2MoE(nn.Module):
|
||||
|
||||
final_hidden_states = self.experts(hidden_states, topk_output)
|
||||
|
||||
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
|
||||
is_tp_path=True,
|
||||
):
|
||||
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
|
||||
return final_hidden_states
|
||||
|
||||
|
||||
@@ -28,9 +28,6 @@ from transformers import PretrainedConfig
|
||||
|
||||
from sglang.kernels.kernel_api_logging import debug_kernel_api
|
||||
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo
|
||||
from sglang.srt.distributed import (
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.communicator import (
|
||||
@@ -51,7 +48,7 @@ from sglang.srt.layers.linear import (
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe import (
|
||||
get_moe_a2a_backend,
|
||||
should_skip_post_experts_all_reduce,
|
||||
reduce_moe_output,
|
||||
)
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
@@ -601,10 +598,7 @@ class MiniMaxM2MoE(nn.Module):
|
||||
topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
|
||||
final_hidden_states = self.experts(hidden_states, topk_output)
|
||||
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
|
||||
is_tp_path=True
|
||||
):
|
||||
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
|
||||
return final_hidden_states.view(num_tokens, hidden_dim)
|
||||
|
||||
|
||||
@@ -52,7 +52,7 @@ from sglang.srt.layers.moe.topk import TopK
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
RoutingMethodType,
|
||||
get_moe_a2a_backend,
|
||||
should_skip_post_experts_all_reduce,
|
||||
reduce_moe_output,
|
||||
)
|
||||
from sglang.srt.layers.quantization import QuantizationConfig
|
||||
from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod
|
||||
@@ -372,10 +372,7 @@ class NemotronHMoE(nn.Module):
|
||||
elif shared_output is not None:
|
||||
final_hidden_states += shared_output
|
||||
|
||||
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
|
||||
is_tp_path=True,
|
||||
):
|
||||
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
|
||||
return final_hidden_states.view(num_tokens, hidden_dim)
|
||||
|
||||
|
||||
@@ -34,7 +34,6 @@ from sglang.kernels.ops.elementwise.elementwise import (
|
||||
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo
|
||||
from sglang.srt.distributed import (
|
||||
get_pp_indices,
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
@@ -60,7 +59,7 @@ from sglang.srt.layers.linear import (
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe import (
|
||||
get_moe_a2a_backend,
|
||||
should_skip_post_experts_all_reduce,
|
||||
reduce_moe_output,
|
||||
)
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
|
||||
@@ -880,14 +879,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
|
||||
)
|
||||
else:
|
||||
final_hidden_states += shared_output
|
||||
if (
|
||||
self.tp_size > 1
|
||||
and not should_skip_post_experts_all_reduce(
|
||||
is_tp_path=True,
|
||||
)
|
||||
and not get_moe_a2a_backend().is_flashinfer()
|
||||
):
|
||||
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
|
||||
# Debug removed - was causing issues during CUDA graph capture
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ from sglang.srt.layers.linear import (
|
||||
RowParallelLinear,
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput
|
||||
from sglang.srt.layers.moe import should_skip_post_experts_all_reduce
|
||||
from sglang.srt.layers.moe import reduce_moe_output
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.layers.moe.topk import TopK
|
||||
@@ -368,10 +368,7 @@ class SarvamMoESparseMoeBlock(nn.Module):
|
||||
final_hidden_states = final_hidden_states * self.routed_scaling_factor
|
||||
current_stream.wait_stream(self.alt_stream)
|
||||
final_hidden_states = final_hidden_states + shared_out
|
||||
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
|
||||
is_tp_path=True,
|
||||
):
|
||||
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
return final_hidden_states.view(num_tokens, hidden_dim)
|
||||
|
||||
def forward_normal(
|
||||
@@ -406,10 +403,7 @@ class SarvamMoESparseMoeBlock(nn.Module):
|
||||
elif self.routed_scaling_factor != 1.0:
|
||||
final_hidden_states = final_hidden_states * self.routed_scaling_factor
|
||||
|
||||
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
|
||||
is_tp_path=True,
|
||||
):
|
||||
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
|
||||
return final_hidden_states.view(num_tokens, hidden_dim)
|
||||
|
||||
|
||||
@@ -10,9 +10,6 @@ import torch
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.distributed import (
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
@@ -32,7 +29,7 @@ from sglang.srt.layers.linear import (
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe import (
|
||||
get_moe_a2a_backend,
|
||||
should_skip_post_experts_all_reduce,
|
||||
reduce_moe_output,
|
||||
)
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
@@ -149,10 +146,7 @@ class SDARMoeSparseMoeBlock(nn.Module):
|
||||
topk_output = self.topk(hidden_states, router_logits)
|
||||
out = self.experts(hidden_states, topk_output) # (T, H)
|
||||
|
||||
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
|
||||
is_tp_path=True,
|
||||
):
|
||||
out = tensor_model_parallel_all_reduce(out)
|
||||
out = reduce_moe_output(out)
|
||||
|
||||
return out.view(num_tokens, hidden_dim)
|
||||
|
||||
|
||||
@@ -29,7 +29,7 @@ from sglang.srt.layers.linear import (
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe import (
|
||||
get_moe_a2a_backend,
|
||||
should_skip_post_experts_all_reduce,
|
||||
reduce_moe_output,
|
||||
)
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
@@ -225,10 +225,7 @@ class Step3p5MoEMLP(nn.Module):
|
||||
router_logits=topk_output.router_logits,
|
||||
)
|
||||
final_hidden_states = self.experts(hidden_states, topk_output)
|
||||
if self.tp_size > 1 and not should_skip_post_experts_all_reduce(
|
||||
is_tp_path=True,
|
||||
):
|
||||
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
|
||||
return final_hidden_states.view(num_tokens, hidden_dim)
|
||||
|
||||
|
||||
@@ -35,7 +35,6 @@ import torch.nn.functional as F
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.distributed import tensor_model_parallel_all_reduce
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
@@ -57,7 +56,7 @@ from sglang.srt.layers.linear import (
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe import (
|
||||
get_moe_a2a_backend,
|
||||
should_skip_post_experts_all_reduce,
|
||||
reduce_moe_output,
|
||||
)
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
|
||||
@@ -1061,13 +1060,8 @@ class XllmSparseMoeBlock(nn.Module):
|
||||
|
||||
if shared_output is not None:
|
||||
final_hidden_states += shared_output
|
||||
if (
|
||||
self.tp_size > 1
|
||||
and not use_reduce_scatter
|
||||
and not should_skip_post_experts_all_reduce(is_tp_path=True)
|
||||
and not get_moe_a2a_backend().is_flashinfer()
|
||||
):
|
||||
final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states)
|
||||
if not use_reduce_scatter:
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
|
||||
return final_hidden_states.view(num_tokens, hidden_dim)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user