[AMD] Critical fix enabling Qwen3.8 FP8: restore dropped fused shared-expert weights (#40754)

This commit is contained in:
jacky.cheng
2026-09-23 20:21:33 -07:00
committed by GitHub
parent 77983865d8
commit e2f4fedf04
+101 -3
View File
@@ -2121,6 +2121,23 @@ class Qwen3_5MoeForCausalLM(Qwen3_5ForCausalLM):
prefix: str = "",
) -> None:
super().__init__(config=config, quant_config=quant_config, prefix=prefix)
# When aiter shared-expert fusion is on, the shared expert is served as an
# extra fused MoE slot (index == num_experts). load_weights must then remap
# the checkpoint's mlp.shared_expert.* onto that slot; otherwise the shared
# expert weights are silently dropped and accuracy collapses.
self.num_fused_shared_experts = 0
if _use_aiter and not _disable_shared_experts_fusion():
self.num_fused_shared_experts = self._get_num_fused_shared_experts()
self.enable_shared_expert_fusion = self.num_fused_shared_experts > 0
def _get_num_fused_shared_experts(self) -> int:
# This is a backbone-style module (holds self.layers directly), and under
# PP the non-local slots are PPMissingLayer (no .mlp), so scan for the
# first real MoE layer instead of assuming layers[0].
for layer in self.layers:
if hasattr(layer, "mlp") and hasattr(layer.mlp, "num_fused_shared_experts"):
return layer.mlp.num_fused_shared_experts
return 0
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
weights = QWEN3_5_KV_SCALE_MAPPER.apply(weights)
@@ -2138,13 +2155,19 @@ class Qwen3_5MoeForCausalLM(Qwen3_5ForCausalLM):
("in_proj_ba.", "in_proj_a.", 1),
]
num_experts = self.config.num_experts
# Params for weights, fp8 weight scales, fp8 activation scales
# (param_name, weight_name, expert_id, shard_id)
expert_params_mapping = FusedMoE.make_expert_params_mapping(
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.num_experts,
num_experts=(
num_experts
if not self.enable_shared_expert_fusion
else num_experts + self.num_fused_shared_experts
),
)
# Skip loading extra parameters for GPTQ/modelopt models.
@@ -2167,7 +2190,37 @@ class Qwen3_5MoeForCausalLM(Qwen3_5ForCausalLM):
("experts.w2_weight", "experts.down_proj", 0, "w2"),
]
num_experts = self.config.num_experts
if self.enable_shared_expert_fusion:
# When shared experts are fused, map them to the extra routed slot:
# mlp.shared_expert.gate_up_proj -> experts.{num_experts}.gate_up_proj -> w13, expert_id=num_experts
# mlp.shared_expert.down_proj -> experts.{num_experts}.down_proj -> w2, expert_id=num_experts
fused_expert_params_mapping += [
(
"experts.w13_",
f"experts.{num_experts}.gate_up_proj.",
num_experts,
"w1",
),
(
"experts.w2_",
f"experts.{num_experts}.down_proj.",
num_experts,
"w2",
),
## shared experts may contain gate_proj and up_proj instead of gate_up_proj
(
"experts.w13_",
f"experts.{num_experts}.gate_proj.",
num_experts,
"w1",
),
(
"experts.w13_",
f"experts.{num_experts}.up_proj.",
num_experts,
"w3",
),
]
def load_fused_expert_weights(
name: str,
@@ -2215,6 +2268,13 @@ class Qwen3_5MoeForCausalLM(Qwen3_5ForCausalLM):
):
continue
if self.enable_shared_expert_fusion and "mlp.shared_expert." in name:
# Firstly map mlp.shared_expert.xx_proj to mlp.experts.{num_experts}.xx_proj
name = name.replace(
"mlp.shared_expert.",
f"mlp.experts.{num_experts}.",
)
for param_name, weight_name, shard_id in stacked_params_mapping:
if "experts.gate_up_proj" in name or "experts.down_proj" in name:
is_fused_expert = True
@@ -2257,7 +2317,9 @@ class Qwen3_5MoeForCausalLM(Qwen3_5ForCausalLM):
is_expert_weight = True
name_mapped = name.replace(weight_name, param_name)
if is_fused_expert:
# is_fused_expert is True, the checkpoint contains gate_up_proj and down_proj for each expert
if "experts.gate_up_proj" in name:
# experts.gate_up_proj contains all routed experts, excluding shared experts
loaded_weight = loaded_weight.chunk(2, dim=-2)
load_fused_expert_weights(
name_mapped,
@@ -2273,7 +2335,8 @@ class Qwen3_5MoeForCausalLM(Qwen3_5ForCausalLM):
"w3",
num_experts,
)
else:
elif "experts.down_proj" in name:
# experts.down_proj contains all routed experts, excluding shared experts
load_fused_expert_weights(
name_mapped,
params_dict,
@@ -2281,6 +2344,41 @@ class Qwen3_5MoeForCausalLM(Qwen3_5ForCausalLM):
shard_id,
num_experts,
)
elif self.enable_shared_expert_fusion:
# shared experts should be loaded to experts.w13_weight and experts.w2_weight
param = params_dict[name_mapped]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
if f"{num_experts}.gate_up_proj" in name:
# split into w1 and w3
loaded_weight = loaded_weight.chunk(2, dim=-2)
# load to experts.w13_weight, shard_id = w1, expert_id = num_experts
weight_loader(
param,
loaded_weight[0],
name_mapped,
"w1",
expert_id,
)
# load to experts.w13_weight, shard_id = w3, expert_id = num_experts
weight_loader(
param,
loaded_weight[1],
name_mapped,
"w3",
expert_id,
)
else:
# load down_proj to experts.w2_weight, shard_id = w2, expert_id = num_experts
# Or load gate_proj and up_proj to experts.w13_weight, shard_id = w1/w3, expert_id = num_experts
weight_loader(
param,
loaded_weight,
name_mapped,
shard_id,
expert_id,
)
else:
# Skip loading extra parameters for GPTQ/modelopt models.
if (