diff --git a/python/sglang/srt/models/sarvam_moe.py b/python/sglang/srt/models/sarvam_moe.py index cabd911796..5bbfbcf45f 100644 --- a/python/sglang/srt/models/sarvam_moe.py +++ b/python/sglang/srt/models/sarvam_moe.py @@ -34,6 +34,7 @@ from sglang.srt.layers.linear import ( RowParallelLinear, ) from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput +from sglang.srt.layers.moe import post_experts_output_is_complete from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.layers.moe.topk import TopK @@ -297,10 +298,6 @@ class SarvamMoESparseMoeBlock(nn.Module): and config.num_shared_experts > 0 ): intermediate_size = config.moe_intermediate_size * config.num_shared_experts - if is_dense_ffn_fully_dp(): - shared_tp_rank, shared_tp_size = 0, 1 - else: - shared_tp_rank, shared_tp_size = None, None self.shared_experts = SarvamMoEMLP( hidden_size=config.hidden_size, intermediate_size=intermediate_size, @@ -308,8 +305,13 @@ class SarvamMoESparseMoeBlock(nn.Module): quant_config=quant_config, prefix=add_prefix("shared_experts", prefix), reduce_results=False, - tp_rank=shared_tp_rank, - tp_size=shared_tp_size, + # The shared output joins the routed output's TP sum; where that + # output is already complete on each rank, it is not TP-sharded. + **( + dict(tp_rank=0, tp_size=1) + if post_experts_output_is_complete(is_tp_path=True) + else {} + ), ) else: self.shared_experts = None diff --git a/python/sglang/srt/models/step3p5.py b/python/sglang/srt/models/step3p5.py index 7514ed13be..505194bbd1 100644 --- a/python/sglang/srt/models/step3p5.py +++ b/python/sglang/srt/models/step3p5.py @@ -236,7 +236,13 @@ class Step3p5MoEMLP(nn.Module): ) -> torch.Tensor: if hidden_states.shape[0] > 0: # router_logits: (num_tokens, n_experts) - router_logits, _ = self.gate(hidden_states) + if self.need_fp32_gate: + router_logits = torch.matmul( + hidden_states.to(torch.float32), + self.gate.weight.t().to(torch.float32), + ) + else: + router_logits, _ = self.gate(hidden_states) topk_output = self.topk( hidden_states, router_logits, @@ -252,8 +258,10 @@ class Step3p5MoEMLP(nn.Module): topk_output=topk_output, ) if shared_output is not None: - shared_output.add_(final_hidden_states) + shared_output.add_(final_hidden_states, alpha=self.routed_scaling_factor) final_hidden_states = shared_output + else: + final_hidden_states *= self.routed_scaling_factor return final_hidden_states