From e2f4fedf04fe95ef38dcea925168022d0fd3970e Mon Sep 17 00:00:00 2001 From: "jacky.cheng" Date: Thu, 24 Sep 2026 11:21:33 +0800 Subject: [PATCH] [AMD] Critical fix enabling Qwen3.8 FP8: restore dropped fused shared-expert weights (#40754) --- python/sglang/srt/models/qwen3_5.py | 104 +++++++++++++++++++++++++++- 1 file changed, 101 insertions(+), 3 deletions(-) diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py index ef20cf9454..798305ee70 100644 --- a/python/sglang/srt/models/qwen3_5.py +++ b/python/sglang/srt/models/qwen3_5.py @@ -2121,6 +2121,23 @@ class Qwen3_5MoeForCausalLM(Qwen3_5ForCausalLM): prefix: str = "", ) -> None: super().__init__(config=config, quant_config=quant_config, prefix=prefix) + # When aiter shared-expert fusion is on, the shared expert is served as an + # extra fused MoE slot (index == num_experts). load_weights must then remap + # the checkpoint's mlp.shared_expert.* onto that slot; otherwise the shared + # expert weights are silently dropped and accuracy collapses. + self.num_fused_shared_experts = 0 + if _use_aiter and not _disable_shared_experts_fusion(): + self.num_fused_shared_experts = self._get_num_fused_shared_experts() + self.enable_shared_expert_fusion = self.num_fused_shared_experts > 0 + + def _get_num_fused_shared_experts(self) -> int: + # This is a backbone-style module (holds self.layers directly), and under + # PP the non-local slots are PPMissingLayer (no .mlp), so scan for the + # first real MoE layer instead of assuming layers[0]. + for layer in self.layers: + if hasattr(layer, "mlp") and hasattr(layer.mlp, "num_fused_shared_experts"): + return layer.mlp.num_fused_shared_experts + return 0 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): weights = QWEN3_5_KV_SCALE_MAPPER.apply(weights) @@ -2138,13 +2155,19 @@ class Qwen3_5MoeForCausalLM(Qwen3_5ForCausalLM): ("in_proj_ba.", "in_proj_a.", 1), ] + num_experts = self.config.num_experts + # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) expert_params_mapping = FusedMoE.make_expert_params_mapping( ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", ckpt_up_proj_name="up_proj", - num_experts=self.config.num_experts, + num_experts=( + num_experts + if not self.enable_shared_expert_fusion + else num_experts + self.num_fused_shared_experts + ), ) # Skip loading extra parameters for GPTQ/modelopt models. @@ -2167,7 +2190,37 @@ class Qwen3_5MoeForCausalLM(Qwen3_5ForCausalLM): ("experts.w2_weight", "experts.down_proj", 0, "w2"), ] - num_experts = self.config.num_experts + if self.enable_shared_expert_fusion: + # When shared experts are fused, map them to the extra routed slot: + # mlp.shared_expert.gate_up_proj -> experts.{num_experts}.gate_up_proj -> w13, expert_id=num_experts + # mlp.shared_expert.down_proj -> experts.{num_experts}.down_proj -> w2, expert_id=num_experts + fused_expert_params_mapping += [ + ( + "experts.w13_", + f"experts.{num_experts}.gate_up_proj.", + num_experts, + "w1", + ), + ( + "experts.w2_", + f"experts.{num_experts}.down_proj.", + num_experts, + "w2", + ), + ## shared experts may contain gate_proj and up_proj instead of gate_up_proj + ( + "experts.w13_", + f"experts.{num_experts}.gate_proj.", + num_experts, + "w1", + ), + ( + "experts.w13_", + f"experts.{num_experts}.up_proj.", + num_experts, + "w3", + ), + ] def load_fused_expert_weights( name: str, @@ -2215,6 +2268,13 @@ class Qwen3_5MoeForCausalLM(Qwen3_5ForCausalLM): ): continue + if self.enable_shared_expert_fusion and "mlp.shared_expert." in name: + # Firstly map mlp.shared_expert.xx_proj to mlp.experts.{num_experts}.xx_proj + name = name.replace( + "mlp.shared_expert.", + f"mlp.experts.{num_experts}.", + ) + for param_name, weight_name, shard_id in stacked_params_mapping: if "experts.gate_up_proj" in name or "experts.down_proj" in name: is_fused_expert = True @@ -2257,7 +2317,9 @@ class Qwen3_5MoeForCausalLM(Qwen3_5ForCausalLM): is_expert_weight = True name_mapped = name.replace(weight_name, param_name) if is_fused_expert: + # is_fused_expert is True, the checkpoint contains gate_up_proj and down_proj for each expert if "experts.gate_up_proj" in name: + # experts.gate_up_proj contains all routed experts, excluding shared experts loaded_weight = loaded_weight.chunk(2, dim=-2) load_fused_expert_weights( name_mapped, @@ -2273,7 +2335,8 @@ class Qwen3_5MoeForCausalLM(Qwen3_5ForCausalLM): "w3", num_experts, ) - else: + elif "experts.down_proj" in name: + # experts.down_proj contains all routed experts, excluding shared experts load_fused_expert_weights( name_mapped, params_dict, @@ -2281,6 +2344,41 @@ class Qwen3_5MoeForCausalLM(Qwen3_5ForCausalLM): shard_id, num_experts, ) + elif self.enable_shared_expert_fusion: + # shared experts should be loaded to experts.w13_weight and experts.w2_weight + param = params_dict[name_mapped] + weight_loader = getattr( + param, "weight_loader", default_weight_loader + ) + if f"{num_experts}.gate_up_proj" in name: + # split into w1 and w3 + loaded_weight = loaded_weight.chunk(2, dim=-2) + # load to experts.w13_weight, shard_id = w1, expert_id = num_experts + weight_loader( + param, + loaded_weight[0], + name_mapped, + "w1", + expert_id, + ) + # load to experts.w13_weight, shard_id = w3, expert_id = num_experts + weight_loader( + param, + loaded_weight[1], + name_mapped, + "w3", + expert_id, + ) + else: + # load down_proj to experts.w2_weight, shard_id = w2, expert_id = num_experts + # Or load gate_proj and up_proj to experts.w13_weight, shard_id = w1/w3, expert_id = num_experts + weight_loader( + param, + loaded_weight, + name_mapped, + shard_id, + expert_id, + ) else: # Skip loading extra parameters for GPTQ/modelopt models. if (