[Fix] Step-3.5: build the shared expert without TP under all-to-all MoE backends (#41754)
This commit is contained in:
@@ -4,9 +4,6 @@ import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.distributed import (
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
@@ -32,6 +29,7 @@ from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe import (
|
||||
get_moe_a2a_backend,
|
||||
reduce_moe_output,
|
||||
should_use_flashinfer_cutlass_moe_fp4_allgather,
|
||||
)
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
@@ -50,7 +48,7 @@ from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.runtime_context import get_exec, get_forward, get_parallel, get_stream
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel, get_stream
|
||||
from sglang.srt.utils import add_prefix, is_cuda, is_non_idle_and_non_empty, make_layers
|
||||
|
||||
Step3p5Config = None
|
||||
@@ -183,15 +181,16 @@ class Step3p5MoEMLP(nn.Module):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: Optional[ForwardBatch] = None,
|
||||
shared_output: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
|
||||
if (
|
||||
not get_moe_a2a_backend().is_deepep()
|
||||
and not get_moe_a2a_backend().is_ascend_fuseep()
|
||||
):
|
||||
return self.forward_normal(hidden_states)
|
||||
return self.forward_normal(hidden_states, shared_output)
|
||||
else:
|
||||
return self.forward_deepep(hidden_states, forward_batch)
|
||||
return self.forward_deepep(hidden_states, forward_batch, shared_output)
|
||||
|
||||
def get_moe_weights(self):
|
||||
return [
|
||||
@@ -206,6 +205,7 @@ class Step3p5MoEMLP(nn.Module):
|
||||
def forward_normal(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
shared_output: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
num_tokens, hidden_dim = hidden_states.shape
|
||||
hidden_states = hidden_states.view(-1, hidden_dim)
|
||||
@@ -227,12 +227,17 @@ class Step3p5MoEMLP(nn.Module):
|
||||
router_logits=topk_output.router_logits,
|
||||
)
|
||||
final_hidden_states = self.experts(hidden_states, topk_output)
|
||||
if shared_output is not None:
|
||||
final_hidden_states += shared_output
|
||||
final_hidden_states = reduce_moe_output(final_hidden_states)
|
||||
|
||||
return final_hidden_states.view(num_tokens, hidden_dim)
|
||||
|
||||
def forward_deepep(
|
||||
self, hidden_states: torch.Tensor, forward_batch: ForwardBatch
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
shared_output: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
if hidden_states.shape[0] > 0:
|
||||
# router_logits: (num_tokens, n_experts)
|
||||
@@ -251,6 +256,9 @@ class Step3p5MoEMLP(nn.Module):
|
||||
hidden_states=hidden_states,
|
||||
topk_output=topk_output,
|
||||
)
|
||||
if shared_output is not None:
|
||||
shared_output.add_(final_hidden_states)
|
||||
final_hidden_states = shared_output
|
||||
return final_hidden_states
|
||||
|
||||
def op_gate(self, state):
|
||||
@@ -538,9 +546,9 @@ class Step3p5DecoderLayer(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
)
|
||||
# reduce_results=False: share_expert output stays unreduced and is
|
||||
# combined with the (also unreduced) MoE output, then a single
|
||||
# all-reduce covers both — saving one full-TP all-reduce per layer.
|
||||
# The MoE block adds the shared expert's output to the routed output
|
||||
# before its one reduction. Where the routed output arrives complete
|
||||
# on each rank's own tokens, the shared expert is not TP-sharded.
|
||||
self.share_expert = Step3p5MLP(
|
||||
hidden_size=self.hidden_size,
|
||||
intermediate_size=config.share_expert_dim,
|
||||
@@ -548,6 +556,18 @@ class Step3p5DecoderLayer(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("share_expert", prefix),
|
||||
reduce_results=False,
|
||||
**(
|
||||
dict(tp_rank=0, tp_size=1)
|
||||
if get_moe_a2a_backend().is_deepep()
|
||||
or get_moe_a2a_backend().is_mooncake()
|
||||
or get_moe_a2a_backend().is_nixl()
|
||||
or get_moe_a2a_backend().is_mori()
|
||||
or get_moe_a2a_backend().is_ascend_fuseep()
|
||||
or get_moe_a2a_backend().is_flashinfer()
|
||||
or get_moe_a2a_backend().is_flashinfer_megamoe()
|
||||
or should_use_flashinfer_cutlass_moe_fp4_allgather()
|
||||
else {}
|
||||
),
|
||||
)
|
||||
self.use_moe = True
|
||||
else:
|
||||
@@ -610,16 +630,10 @@ class Step3p5DecoderLayer(nn.Module):
|
||||
|
||||
if self.use_moe:
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
# Both share_expert and MoE return unreduced (TP-partial) outputs.
|
||||
# Combine them first, then do a single all-reduce — saving one
|
||||
# full-TP all-reduce per layer.
|
||||
# Force fuse_mlp_allreduce=True so MoE skips its internal AR.
|
||||
share_output = self.share_expert(hidden_states)
|
||||
with get_forward().scoped(fuse_mlp_allreduce=True):
|
||||
moe_output = self.moe(hidden_states, forward_batch)
|
||||
hidden_states = moe_output + share_output
|
||||
if not ffn_exit.fuse_mlp_allreduce and not ffn_exit.mlp_reduce_scatter:
|
||||
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
|
||||
hidden_states = self.moe(
|
||||
hidden_states, forward_batch, shared_output=share_output
|
||||
)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
|
||||
Reference in New Issue
Block a user