[Fix] megamoe: Support EPLB rebalance on SM100 (#44847)
This commit is contained in:
@@ -579,6 +579,16 @@ def build_mega_moe_experts_weights(experts) -> None:
|
||||
|
||||
experts.mega_l1_weights = (experts.w13_weight.data, w13_sf_utccp)
|
||||
experts.mega_l2_weights = (experts.w2_weight.data, w2_sf_utccp)
|
||||
# EPLB moves each expert's slice of every parameter, so the UTCCP copies
|
||||
# move with the scales they are transposed from.
|
||||
experts.register_parameter(
|
||||
"mega_w13_weight_scale_utccp",
|
||||
torch.nn.Parameter(w13_sf_utccp, requires_grad=False),
|
||||
)
|
||||
experts.register_parameter(
|
||||
"mega_w2_weight_scale_utccp",
|
||||
torch.nn.Parameter(w2_sf_utccp, requires_grad=False),
|
||||
)
|
||||
|
||||
experts._mega_moe_weights_built = True
|
||||
|
||||
|
||||
@@ -2635,9 +2635,15 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
|
||||
|
||||
layer.mega_l1_weights = l1_pair
|
||||
layer.mega_l2_weights = l2_pair
|
||||
layer.mega_l1_alphas = l1_alphas
|
||||
layer.mega_l2_alphas = l2_alphas
|
||||
layer.mega_l2_act_scales = l2_act_scales
|
||||
# Parameters, so EPLB moves each expert's values with its weights.
|
||||
for name, value in (
|
||||
("mega_l1_alphas", l1_alphas),
|
||||
("mega_l2_alphas", l2_alphas),
|
||||
("mega_l2_act_scales", l2_act_scales),
|
||||
):
|
||||
layer.register_parameter(
|
||||
name, torch.nn.Parameter(value, requires_grad=False)
|
||||
)
|
||||
# Free the checkpoint layout.
|
||||
layer.w13_weight.data = l1_pair[0]
|
||||
layer.w13_weight_scale.data = l1_pair[1]
|
||||
|
||||
Reference in New Issue
Block a user