diff --git a/python/sglang/srt/layers/moe/mega_moe.py b/python/sglang/srt/layers/moe/mega_moe.py index 20d03d8cc7..02a7f91606 100644 --- a/python/sglang/srt/layers/moe/mega_moe.py +++ b/python/sglang/srt/layers/moe/mega_moe.py @@ -579,6 +579,16 @@ def build_mega_moe_experts_weights(experts) -> None: experts.mega_l1_weights = (experts.w13_weight.data, w13_sf_utccp) experts.mega_l2_weights = (experts.w2_weight.data, w2_sf_utccp) + # EPLB moves each expert's slice of every parameter, so the UTCCP copies + # move with the scales they are transposed from. + experts.register_parameter( + "mega_w13_weight_scale_utccp", + torch.nn.Parameter(w13_sf_utccp, requires_grad=False), + ) + experts.register_parameter( + "mega_w2_weight_scale_utccp", + torch.nn.Parameter(w2_sf_utccp, requires_grad=False), + ) experts._mega_moe_weights_built = True diff --git a/python/sglang/srt/layers/quantization/modelopt_quant.py b/python/sglang/srt/layers/quantization/modelopt_quant.py index af7f8bb039..ce72eed7f7 100755 --- a/python/sglang/srt/layers/quantization/modelopt_quant.py +++ b/python/sglang/srt/layers/quantization/modelopt_quant.py @@ -2635,9 +2635,15 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase): layer.mega_l1_weights = l1_pair layer.mega_l2_weights = l2_pair - layer.mega_l1_alphas = l1_alphas - layer.mega_l2_alphas = l2_alphas - layer.mega_l2_act_scales = l2_act_scales + # Parameters, so EPLB moves each expert's values with its weights. + for name, value in ( + ("mega_l1_alphas", l1_alphas), + ("mega_l2_alphas", l2_alphas), + ("mega_l2_act_scales", l2_act_scales), + ): + layer.register_parameter( + name, torch.nn.Parameter(value, requires_grad=False) + ) # Free the checkpoint layout. layer.w13_weight.data = l1_pair[0] layer.w13_weight_scale.data = l1_pair[1]