fix(minimax): use routed TRT-LLM for NVFP4 MoE auto on SM100 (#32229)
Co-authored-by: Mohammad Miadh Angkad <176301910+mmangkad@users.noreply.github.com>
This commit is contained in:
co-authored by
Mohammad Miadh Angkad
parent
410088c91e
commit
5d85f25f75
@@ -731,10 +731,21 @@ def _mimo_v2_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
|
||||
@_register_for("MiniMaxM2ForCausalLM")
|
||||
def _minimax_m2_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
overrides = {"enable_tf32_matmul": True}
|
||||
logger.info(
|
||||
"Enable TF32 matmul for MiniMaxM2ForCausalLM model to improve gate gemm performance."
|
||||
)
|
||||
return {"enable_tf32_matmul": True}
|
||||
if (
|
||||
is_sm100_supported()
|
||||
and server_args.moe_runner_backend == "auto"
|
||||
and server_args.get_model_config().quantization == "modelopt_fp4"
|
||||
):
|
||||
overrides["moe_runner_backend"] = "flashinfer_trtllm_routed"
|
||||
logger.info(
|
||||
"Use flashinfer_trtllm_routed as MoE runner backend on SM10X "
|
||||
"for MiniMaxM2ForCausalLM with modelopt_fp4."
|
||||
)
|
||||
return overrides
|
||||
|
||||
|
||||
@_register_for("MiniMaxM3SparseForCausalLM", "MiniMaxM3SparseForConditionalGeneration")
|
||||
|
||||
Reference in New Issue
Block a user