[MegaMoE] Preserve W13 layout for ModelOpt NVFP4 experts (#39388)
Co-authored-by: Brayden Zhong <b8zhong@uwaterloo.ca>
This commit is contained in:
@@ -1210,8 +1210,14 @@ class FusedMoE(torch.nn.Module):
|
||||
if shard_id not in ("w1", "w2", "w3"):
|
||||
raise ValueError(f"shard_id must be ['w1','w2','w3'] but got {shard_id}.")
|
||||
|
||||
use_flashinfer_trtllm_weight_layout = (
|
||||
method.use_flashinfer_trtllm_weight_layout
|
||||
if isinstance(method, ModelOptNvFp4FusedMoEMethod)
|
||||
else self.use_flashinfer_trtllm_moe
|
||||
)
|
||||
|
||||
# Flashinfer assumes w31 format for w13_weight. Same for the scales.
|
||||
if self.use_flashinfer_trtllm_moe and (
|
||||
if use_flashinfer_trtllm_weight_layout and (
|
||||
isinstance(method, ModelOptNvFp4FusedMoEMethod)
|
||||
or isinstance(method, Fp8MoEMethod)
|
||||
or isinstance(method, UnquantizedFusedMoEMethod)
|
||||
|
||||
@@ -2356,6 +2356,11 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
|
||||
get_moe_runner_backend().is_flashinfer_trtllm()
|
||||
or get_moe_runner_backend().is_flashinfer_trtllm_routed()
|
||||
)
|
||||
# MegaMoE consumes canonical W13 directly, regardless of the nominal
|
||||
# runner.
|
||||
self.use_flashinfer_trtllm_weight_layout = (
|
||||
self.enable_flashinfer_trtllm_moe and not get_moe_a2a_backend().is_megamoe()
|
||||
)
|
||||
self._cache_permute_indices = {}
|
||||
|
||||
@property
|
||||
@@ -2576,6 +2581,9 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
|
||||
layer.register_parameter("w2_input_scale", w2_input_scale)
|
||||
|
||||
def _build_mega_moe_weights(self, layer: torch.nn.Module) -> None:
|
||||
assert not self.use_flashinfer_trtllm_weight_layout, (
|
||||
"MegaMoE NVFP4 weights must use canonical W13 layout"
|
||||
)
|
||||
# Activations are quantized per token at dispatch, so w13_input_scale
|
||||
# is not used.
|
||||
import deep_gemm
|
||||
@@ -2633,7 +2641,7 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
|
||||
if getattr(layer, "inference_moe_w13_interleaved", False) and not getattr(
|
||||
layer, "_w13_deinterleaved", False
|
||||
):
|
||||
up_first = self.enable_flashinfer_trtllm_moe
|
||||
up_first = self.use_flashinfer_trtllm_weight_layout
|
||||
layer.w13_weight.data = deinterleave_w13(
|
||||
layer.w13_weight.data, up_first=up_first
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user