[Fix] moe: gated NVFP4 MoE intermediate alignment for FlashInfer TRTLLM (#44138)
This commit is contained in:
@@ -425,16 +425,6 @@ class FusedMoE(torch.nn.Module):
|
||||
)
|
||||
self.use_deep_gemm = get_moe_runner_backend().is_deep_gemm()
|
||||
|
||||
# flashinfer_trtllm kernel requires intermediate_size to be a multiple of 128
|
||||
# Pad the intermediate_size_per_partition if necessary
|
||||
if (
|
||||
self.use_flashinfer_trtllm_moe
|
||||
and self.intermediate_size_per_partition % 128 != 0
|
||||
):
|
||||
self.intermediate_size_per_partition = round_up(
|
||||
self.intermediate_size_per_partition, 128
|
||||
)
|
||||
|
||||
self.quant_config = quant_config
|
||||
self.use_flashinfer_mxfp4_moe = get_moe_runner_backend().is_flashinfer_mxfp4()
|
||||
# TODO maybe we should remove this `if`, since `Mxfp4MoEMethod` does another round-up logic
|
||||
@@ -504,6 +494,20 @@ class FusedMoE(torch.nn.Module):
|
||||
self.quant_method = make_for_layer(
|
||||
layer=self, base_method=self.quant_method
|
||||
)
|
||||
# flashinfer_trtllm pads intermediate_size to a multiple of 128 here, except
|
||||
# NVFP4 which aligns it in create_weights
|
||||
if (
|
||||
self.use_flashinfer_trtllm_moe
|
||||
and not isinstance(self.quant_method, ModelOptNvFp4FusedMoEMethod)
|
||||
and self.intermediate_size_per_partition % 128 != 0
|
||||
):
|
||||
self.intermediate_size_per_partition = round_up(
|
||||
self.intermediate_size_per_partition, 128
|
||||
)
|
||||
self.moe_runner_config.intermediate_size_per_partition = (
|
||||
self.intermediate_size_per_partition
|
||||
)
|
||||
|
||||
_validate_hpc_ops_quant_method(self.quant_method)
|
||||
_validate_deepep_v2_quant_method(self.quant_method)
|
||||
if (
|
||||
@@ -661,7 +665,7 @@ class FusedMoE(torch.nn.Module):
|
||||
# This handles the case where the loaded weights are smaller than the padded expert_data
|
||||
# Use narrow_padded_param_and_loaded_weight for:
|
||||
# 1. CPU (always)
|
||||
# 2. GPU with flashinfer_trtllm padding (when intermediate_size is padded to 128)
|
||||
# 2. GPU with flashinfer_trtllm padding (intermediate_size padded to 128, or 64 for gated NVFP4)
|
||||
# 3. GPU with Aiter padding
|
||||
aiter_padded = (
|
||||
_use_aiter
|
||||
|
||||
@@ -2447,6 +2447,19 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
if self.enable_flashinfer_trtllm_moe:
|
||||
alignment = (
|
||||
64
|
||||
if layer.moe_runner_config.is_gated
|
||||
and self.use_flashinfer_trtllm_weight_layout
|
||||
else 128
|
||||
)
|
||||
intermediate_size_per_partition = round_up(
|
||||
intermediate_size_per_partition, alignment
|
||||
)
|
||||
layer.moe_runner_config.intermediate_size_per_partition = (
|
||||
intermediate_size_per_partition
|
||||
)
|
||||
# TODO(ch-wan): check if this is needed
|
||||
layer.intermediate_size_per_partition = intermediate_size_per_partition
|
||||
layer.params_dtype = params_dtype
|
||||
|
||||
Reference in New Issue
Block a user