From 6be84e1b72df81aa7edf40640de0d8e529d96c59 Mon Sep 17 00:00:00 2001 From: Khoa Pham Date: Thu, 1 Oct 2026 20:01:03 -0700 Subject: [PATCH] [Fix] Select the Marlin MoE runner for MiMo-V2 packed MXFP4 experts on SM90 (#42172) --- python/sglang/srt/arg_groups/model_overrides/mimo_v2.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/python/sglang/srt/arg_groups/model_overrides/mimo_v2.py b/python/sglang/srt/arg_groups/model_overrides/mimo_v2.py index 98ec3b370f..8ab773b07a 100644 --- a/python/sglang/srt/arg_groups/model_overrides/mimo_v2.py +++ b/python/sglang/srt/arg_groups/model_overrides/mimo_v2.py @@ -47,4 +47,13 @@ def _mimo_v2_overrides(server_args: Any, hf_config: Any) -> dict: "MiMoV2 on SM100: moe_runner_backend=%s.", overrides["moe_runner_backend"], ) + elif ( + get_platform().is_sm90 + and cfg.moe_runner_backend == "auto" + and get_quantization_config(hf_config) == "fp8" + and model_config_of(server_args).is_fp4_experts + ): + # The auto runner resolves to Triton FP8, which cannot read packed MXFP4. + overrides["moe_runner_backend"] = "marlin" + logger.info("MiMoV2 on SM90: moe_runner_backend=marlin.") return overrides