[Fix] Select the Marlin MoE runner for MiMo-V2 packed MXFP4 experts on SM90 (#42172)

This commit is contained in:
Khoa Pham
2026-10-01 20:01:03 -07:00
committed by GitHub
parent ef867fa40d
commit 6be84e1b72
@@ -47,4 +47,13 @@ def _mimo_v2_overrides(server_args: Any, hf_config: Any) -> dict:
"MiMoV2 on SM100: moe_runner_backend=%s.",
overrides["moe_runner_backend"],
)
elif (
get_platform().is_sm90
and cfg.moe_runner_backend == "auto"
and get_quantization_config(hf_config) == "fp8"
and model_config_of(server_args).is_fp4_experts
):
# The auto runner resolves to Triton FP8, which cannot read packed MXFP4.
overrides["moe_runner_backend"] = "marlin"
logger.info("MiMoV2 on SM90: moe_runner_backend=marlin.")
return overrides