[Fix] megamoe: Support EPLB rebalance on SM100 (#44847)

This commit is contained in:
Liangsheng Yin
2026-10-16 22:22:40 -07:00
committed by GitHub
parent 3fc08b939a
commit 9259553f09
2 changed files with 19 additions and 3 deletions
+10
View File
@@ -579,6 +579,16 @@ def build_mega_moe_experts_weights(experts) -> None:
experts.mega_l1_weights = (experts.w13_weight.data, w13_sf_utccp)
experts.mega_l2_weights = (experts.w2_weight.data, w2_sf_utccp)
# EPLB moves each expert's slice of every parameter, so the UTCCP copies
# move with the scales they are transposed from.
experts.register_parameter(
"mega_w13_weight_scale_utccp",
torch.nn.Parameter(w13_sf_utccp, requires_grad=False),
)
experts.register_parameter(
"mega_w2_weight_scale_utccp",
torch.nn.Parameter(w2_sf_utccp, requires_grad=False),
)
experts._mega_moe_weights_built = True
@@ -2635,9 +2635,15 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
layer.mega_l1_weights = l1_pair
layer.mega_l2_weights = l2_pair
layer.mega_l1_alphas = l1_alphas
layer.mega_l2_alphas = l2_alphas
layer.mega_l2_act_scales = l2_act_scales
# Parameters, so EPLB moves each expert's values with its weights.
for name, value in (
("mega_l1_alphas", l1_alphas),
("mega_l2_alphas", l2_alphas),
("mega_l2_act_scales", l2_act_scales),
):
layer.register_parameter(
name, torch.nn.Parameter(value, requires_grad=False)
)
# Free the checkpoint layout.
layer.w13_weight.data = l1_pair[0]
layer.w13_weight_scale.data = l1_pair[1]