[MegaMoE] Preserve W13 layout for ModelOpt NVFP4 experts (#39388)

Co-authored-by: Brayden Zhong <b8zhong@uwaterloo.ca>
This commit is contained in:
shyeh25
2026-10-01 16:12:34 -07:00
committed by GitHub
co-authored by Brayden Zhong
parent f17f7705a5
commit 5423a4d885
2 changed files with 16 additions and 2 deletions
@@ -1210,8 +1210,14 @@ class FusedMoE(torch.nn.Module):
if shard_id not in ("w1", "w2", "w3"):
raise ValueError(f"shard_id must be ['w1','w2','w3'] but got {shard_id}.")
use_flashinfer_trtllm_weight_layout = (
method.use_flashinfer_trtllm_weight_layout
if isinstance(method, ModelOptNvFp4FusedMoEMethod)
else self.use_flashinfer_trtllm_moe
)
# Flashinfer assumes w31 format for w13_weight. Same for the scales.
if self.use_flashinfer_trtllm_moe and (
if use_flashinfer_trtllm_weight_layout and (
isinstance(method, ModelOptNvFp4FusedMoEMethod)
or isinstance(method, Fp8MoEMethod)
or isinstance(method, UnquantizedFusedMoEMethod)
@@ -2356,6 +2356,11 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
get_moe_runner_backend().is_flashinfer_trtllm()
or get_moe_runner_backend().is_flashinfer_trtllm_routed()
)
# MegaMoE consumes canonical W13 directly, regardless of the nominal
# runner.
self.use_flashinfer_trtllm_weight_layout = (
self.enable_flashinfer_trtllm_moe and not get_moe_a2a_backend().is_megamoe()
)
self._cache_permute_indices = {}
@property
@@ -2576,6 +2581,9 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
layer.register_parameter("w2_input_scale", w2_input_scale)
def _build_mega_moe_weights(self, layer: torch.nn.Module) -> None:
assert not self.use_flashinfer_trtllm_weight_layout, (
"MegaMoE NVFP4 weights must use canonical W13 layout"
)
# Activations are quantized per token at dispatch, so w13_input_scale
# is not used.
import deep_gemm
@@ -2633,7 +2641,7 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
if getattr(layer, "inference_moe_w13_interleaved", False) and not getattr(
layer, "_w13_deinterleaved", False
):
up_first = self.enable_flashinfer_trtllm_moe
up_first = self.use_flashinfer_trtllm_weight_layout
layer.w13_weight.data = deinterleave_w13(
layer.w13_weight.data, up_first=up_first
)