[Fix] Step-3.5 DeepEP routed scaling and Sarvam shared expert under dense TP1 (#42302)
This commit is contained in:
@@ -34,6 +34,7 @@ from sglang.srt.layers.linear import (
|
||||
RowParallelLinear,
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput
|
||||
from sglang.srt.layers.moe import post_experts_output_is_complete
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.layers.moe.topk import TopK
|
||||
@@ -297,10 +298,6 @@ class SarvamMoESparseMoeBlock(nn.Module):
|
||||
and config.num_shared_experts > 0
|
||||
):
|
||||
intermediate_size = config.moe_intermediate_size * config.num_shared_experts
|
||||
if is_dense_ffn_fully_dp():
|
||||
shared_tp_rank, shared_tp_size = 0, 1
|
||||
else:
|
||||
shared_tp_rank, shared_tp_size = None, None
|
||||
self.shared_experts = SarvamMoEMLP(
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=intermediate_size,
|
||||
@@ -308,8 +305,13 @@ class SarvamMoESparseMoeBlock(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("shared_experts", prefix),
|
||||
reduce_results=False,
|
||||
tp_rank=shared_tp_rank,
|
||||
tp_size=shared_tp_size,
|
||||
# The shared output joins the routed output's TP sum; where that
|
||||
# output is already complete on each rank, it is not TP-sharded.
|
||||
**(
|
||||
dict(tp_rank=0, tp_size=1)
|
||||
if post_experts_output_is_complete(is_tp_path=True)
|
||||
else {}
|
||||
),
|
||||
)
|
||||
else:
|
||||
self.shared_experts = None
|
||||
|
||||
@@ -236,7 +236,13 @@ class Step3p5MoEMLP(nn.Module):
|
||||
) -> torch.Tensor:
|
||||
if hidden_states.shape[0] > 0:
|
||||
# router_logits: (num_tokens, n_experts)
|
||||
router_logits, _ = self.gate(hidden_states)
|
||||
if self.need_fp32_gate:
|
||||
router_logits = torch.matmul(
|
||||
hidden_states.to(torch.float32),
|
||||
self.gate.weight.t().to(torch.float32),
|
||||
)
|
||||
else:
|
||||
router_logits, _ = self.gate(hidden_states)
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
@@ -252,8 +258,10 @@ class Step3p5MoEMLP(nn.Module):
|
||||
topk_output=topk_output,
|
||||
)
|
||||
if shared_output is not None:
|
||||
shared_output.add_(final_hidden_states)
|
||||
shared_output.add_(final_hidden_states, alpha=self.routed_scaling_factor)
|
||||
final_hidden_states = shared_output
|
||||
else:
|
||||
final_hidden_states *= self.routed_scaling_factor
|
||||
return final_hidden_states
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user