[Fix] moe: gated NVFP4 MoE intermediate alignment for FlashInfer TRTLLM (#44138)

This commit is contained in:
Qiaolin Yu
2026-10-19 12:52:36 -07:00
committed by GitHub
parent 9ab74b8517
commit 5385866108
3 changed files with 66 additions and 18 deletions
@@ -425,16 +425,6 @@ class FusedMoE(torch.nn.Module):
)
self.use_deep_gemm = get_moe_runner_backend().is_deep_gemm()
# flashinfer_trtllm kernel requires intermediate_size to be a multiple of 128
# Pad the intermediate_size_per_partition if necessary
if (
self.use_flashinfer_trtllm_moe
and self.intermediate_size_per_partition % 128 != 0
):
self.intermediate_size_per_partition = round_up(
self.intermediate_size_per_partition, 128
)
self.quant_config = quant_config
self.use_flashinfer_mxfp4_moe = get_moe_runner_backend().is_flashinfer_mxfp4()
# TODO maybe we should remove this `if`, since `Mxfp4MoEMethod` does another round-up logic
@@ -504,6 +494,20 @@ class FusedMoE(torch.nn.Module):
self.quant_method = make_for_layer(
layer=self, base_method=self.quant_method
)
# flashinfer_trtllm pads intermediate_size to a multiple of 128 here, except
# NVFP4 which aligns it in create_weights
if (
self.use_flashinfer_trtllm_moe
and not isinstance(self.quant_method, ModelOptNvFp4FusedMoEMethod)
and self.intermediate_size_per_partition % 128 != 0
):
self.intermediate_size_per_partition = round_up(
self.intermediate_size_per_partition, 128
)
self.moe_runner_config.intermediate_size_per_partition = (
self.intermediate_size_per_partition
)
_validate_hpc_ops_quant_method(self.quant_method)
_validate_deepep_v2_quant_method(self.quant_method)
if (
@@ -661,7 +665,7 @@ class FusedMoE(torch.nn.Module):
# This handles the case where the loaded weights are smaller than the padded expert_data
# Use narrow_padded_param_and_loaded_weight for:
# 1. CPU (always)
# 2. GPU with flashinfer_trtllm padding (when intermediate_size is padded to 128)
# 2. GPU with flashinfer_trtllm padding (intermediate_size padded to 128, or 64 for gated NVFP4)
# 3. GPU with Aiter padding
aiter_padded = (
_use_aiter
@@ -2447,6 +2447,19 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
params_dtype: torch.dtype,
**extra_weight_attrs,
):
if self.enable_flashinfer_trtllm_moe:
alignment = (
64
if layer.moe_runner_config.is_gated
and self.use_flashinfer_trtllm_weight_layout
else 128
)
intermediate_size_per_partition = round_up(
intermediate_size_per_partition, alignment
)
layer.moe_runner_config.intermediate_size_per_partition = (
intermediate_size_per_partition
)
# TODO(ch-wan): check if this is needed
layer.intermediate_size_per_partition = intermediate_size_per_partition
layer.params_dtype = params_dtype