[AMD] Critical fix enabling Qwen3.8 FP8: restore dropped fused shared-expert weights (#40754)
This commit is contained in:
@@ -2121,6 +2121,23 @@ class Qwen3_5MoeForCausalLM(Qwen3_5ForCausalLM):
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__(config=config, quant_config=quant_config, prefix=prefix)
|
||||
# When aiter shared-expert fusion is on, the shared expert is served as an
|
||||
# extra fused MoE slot (index == num_experts). load_weights must then remap
|
||||
# the checkpoint's mlp.shared_expert.* onto that slot; otherwise the shared
|
||||
# expert weights are silently dropped and accuracy collapses.
|
||||
self.num_fused_shared_experts = 0
|
||||
if _use_aiter and not _disable_shared_experts_fusion():
|
||||
self.num_fused_shared_experts = self._get_num_fused_shared_experts()
|
||||
self.enable_shared_expert_fusion = self.num_fused_shared_experts > 0
|
||||
|
||||
def _get_num_fused_shared_experts(self) -> int:
|
||||
# This is a backbone-style module (holds self.layers directly), and under
|
||||
# PP the non-local slots are PPMissingLayer (no .mlp), so scan for the
|
||||
# first real MoE layer instead of assuming layers[0].
|
||||
for layer in self.layers:
|
||||
if hasattr(layer, "mlp") and hasattr(layer.mlp, "num_fused_shared_experts"):
|
||||
return layer.mlp.num_fused_shared_experts
|
||||
return 0
|
||||
|
||||
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
|
||||
weights = QWEN3_5_KV_SCALE_MAPPER.apply(weights)
|
||||
@@ -2138,13 +2155,19 @@ class Qwen3_5MoeForCausalLM(Qwen3_5ForCausalLM):
|
||||
("in_proj_ba.", "in_proj_a.", 1),
|
||||
]
|
||||
|
||||
num_experts = self.config.num_experts
|
||||
|
||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
expert_params_mapping = FusedMoE.make_expert_params_mapping(
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=self.config.num_experts,
|
||||
num_experts=(
|
||||
num_experts
|
||||
if not self.enable_shared_expert_fusion
|
||||
else num_experts + self.num_fused_shared_experts
|
||||
),
|
||||
)
|
||||
|
||||
# Skip loading extra parameters for GPTQ/modelopt models.
|
||||
@@ -2167,7 +2190,37 @@ class Qwen3_5MoeForCausalLM(Qwen3_5ForCausalLM):
|
||||
("experts.w2_weight", "experts.down_proj", 0, "w2"),
|
||||
]
|
||||
|
||||
num_experts = self.config.num_experts
|
||||
if self.enable_shared_expert_fusion:
|
||||
# When shared experts are fused, map them to the extra routed slot:
|
||||
# mlp.shared_expert.gate_up_proj -> experts.{num_experts}.gate_up_proj -> w13, expert_id=num_experts
|
||||
# mlp.shared_expert.down_proj -> experts.{num_experts}.down_proj -> w2, expert_id=num_experts
|
||||
fused_expert_params_mapping += [
|
||||
(
|
||||
"experts.w13_",
|
||||
f"experts.{num_experts}.gate_up_proj.",
|
||||
num_experts,
|
||||
"w1",
|
||||
),
|
||||
(
|
||||
"experts.w2_",
|
||||
f"experts.{num_experts}.down_proj.",
|
||||
num_experts,
|
||||
"w2",
|
||||
),
|
||||
## shared experts may contain gate_proj and up_proj instead of gate_up_proj
|
||||
(
|
||||
"experts.w13_",
|
||||
f"experts.{num_experts}.gate_proj.",
|
||||
num_experts,
|
||||
"w1",
|
||||
),
|
||||
(
|
||||
"experts.w13_",
|
||||
f"experts.{num_experts}.up_proj.",
|
||||
num_experts,
|
||||
"w3",
|
||||
),
|
||||
]
|
||||
|
||||
def load_fused_expert_weights(
|
||||
name: str,
|
||||
@@ -2215,6 +2268,13 @@ class Qwen3_5MoeForCausalLM(Qwen3_5ForCausalLM):
|
||||
):
|
||||
continue
|
||||
|
||||
if self.enable_shared_expert_fusion and "mlp.shared_expert." in name:
|
||||
# Firstly map mlp.shared_expert.xx_proj to mlp.experts.{num_experts}.xx_proj
|
||||
name = name.replace(
|
||||
"mlp.shared_expert.",
|
||||
f"mlp.experts.{num_experts}.",
|
||||
)
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if "experts.gate_up_proj" in name or "experts.down_proj" in name:
|
||||
is_fused_expert = True
|
||||
@@ -2257,7 +2317,9 @@ class Qwen3_5MoeForCausalLM(Qwen3_5ForCausalLM):
|
||||
is_expert_weight = True
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
if is_fused_expert:
|
||||
# is_fused_expert is True, the checkpoint contains gate_up_proj and down_proj for each expert
|
||||
if "experts.gate_up_proj" in name:
|
||||
# experts.gate_up_proj contains all routed experts, excluding shared experts
|
||||
loaded_weight = loaded_weight.chunk(2, dim=-2)
|
||||
load_fused_expert_weights(
|
||||
name_mapped,
|
||||
@@ -2273,7 +2335,8 @@ class Qwen3_5MoeForCausalLM(Qwen3_5ForCausalLM):
|
||||
"w3",
|
||||
num_experts,
|
||||
)
|
||||
else:
|
||||
elif "experts.down_proj" in name:
|
||||
# experts.down_proj contains all routed experts, excluding shared experts
|
||||
load_fused_expert_weights(
|
||||
name_mapped,
|
||||
params_dict,
|
||||
@@ -2281,6 +2344,41 @@ class Qwen3_5MoeForCausalLM(Qwen3_5ForCausalLM):
|
||||
shard_id,
|
||||
num_experts,
|
||||
)
|
||||
elif self.enable_shared_expert_fusion:
|
||||
# shared experts should be loaded to experts.w13_weight and experts.w2_weight
|
||||
param = params_dict[name_mapped]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
if f"{num_experts}.gate_up_proj" in name:
|
||||
# split into w1 and w3
|
||||
loaded_weight = loaded_weight.chunk(2, dim=-2)
|
||||
# load to experts.w13_weight, shard_id = w1, expert_id = num_experts
|
||||
weight_loader(
|
||||
param,
|
||||
loaded_weight[0],
|
||||
name_mapped,
|
||||
"w1",
|
||||
expert_id,
|
||||
)
|
||||
# load to experts.w13_weight, shard_id = w3, expert_id = num_experts
|
||||
weight_loader(
|
||||
param,
|
||||
loaded_weight[1],
|
||||
name_mapped,
|
||||
"w3",
|
||||
expert_id,
|
||||
)
|
||||
else:
|
||||
# load down_proj to experts.w2_weight, shard_id = w2, expert_id = num_experts
|
||||
# Or load gate_proj and up_proj to experts.w13_weight, shard_id = w1/w3, expert_id = num_experts
|
||||
weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name_mapped,
|
||||
shard_id,
|
||||
expert_id,
|
||||
)
|
||||
else:
|
||||
# Skip loading extra parameters for GPTQ/modelopt models.
|
||||
if (
|
||||
|
||||
Reference in New Issue
Block a user