From 80bb3fb6511ac421ed3b4309067681392203f3bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Bernl=C3=B6hr?= Date: Fri, 2 Oct 2026 00:33:30 +0200 Subject: [PATCH] [Fix] Select the MXFP4 MoE runner for MiMo-V2 packed experts on SM100 (#41668) Co-authored-by: jbernloehr --- .../srt/arg_groups/model_overrides/mimo_v2.py | 19 +++++++++--- test/registered/unit/test_model_overrides.py | 30 +++++++++++++++++++ 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/python/sglang/srt/arg_groups/model_overrides/mimo_v2.py b/python/sglang/srt/arg_groups/model_overrides/mimo_v2.py index 68cdfc58dd..98ec3b370f 100644 --- a/python/sglang/srt/arg_groups/model_overrides/mimo_v2.py +++ b/python/sglang/srt/arg_groups/model_overrides/mimo_v2.py @@ -6,6 +6,7 @@ from typing import Any, Dict from sglang.srt.arg_groups.model_override_base import ( _register_for, is_attention_backend_not_set, + model_config_of, resolving_view, ) from sglang.srt.runtime_context import get_platform @@ -27,13 +28,23 @@ def _mimo_v2_overrides(server_args: Any, hf_config: Any) -> dict: overrides["attention_backend"] = "fa4" logger.info("MiMoV2 on SM100: attention_backend=fa4.") - # On Blackwell "auto" falls through to the triton fused-MoE runner, ~12% - # slower at bs=1 decode. FP4 checkpoints use flashinfer_mxfp4 instead. + # Mixed checkpoints also advertise quant_method=fp8; select the runner + # using the routed-expert layout already resolved by ModelConfig. if ( get_platform().is_sm100 and cfg.moe_runner_backend == "auto" and get_quantization_config(hf_config) == "fp8" ): - overrides["moe_runner_backend"] = "flashinfer_trtllm" - logger.info("MiMoV2 FP8 on SM100: moe_runner_backend=flashinfer_trtllm.") + if model_config_of(server_args).is_fp4_experts: + # Let the all-to-all backend choose its compatible runner. + if cfg.moe_a2a_backend == "none": + overrides["moe_runner_backend"] = "flashinfer_mxfp4" + else: + # Avoid the slower Triton default for ordinary FP8 checkpoints. + overrides["moe_runner_backend"] = "flashinfer_trtllm" + if "moe_runner_backend" in overrides: + logger.info( + "MiMoV2 on SM100: moe_runner_backend=%s.", + overrides["moe_runner_backend"], + ) return overrides diff --git a/test/registered/unit/test_model_overrides.py b/test/registered/unit/test_model_overrides.py index 3d7f003a4e..64ab395b63 100644 --- a/test/registered/unit/test_model_overrides.py +++ b/test/registered/unit/test_model_overrides.py @@ -1169,6 +1169,7 @@ class TestGoldenModelOverrides(_IsolatedPublish): attention_backend=None, prefill_attention_backend=None, decode_attention_backend=None, + _model_config=SimpleNamespace(is_fp4_experts=False), ) defaults.update(kw) return SimpleNamespace(**defaults) @@ -1202,6 +1203,35 @@ class TestGoldenModelOverrides(_IsolatedPublish): {}, ) + def test_mimo_v2_sm100_mixed_mxfp4_selects_native_runner(self): + for architecture in ("MiMoV2ForCausalLM", "MiMoV2FlashForCausalLM"): + for a2a_backend in ("none", "deepep"): + for runner in ("auto", "deep_gemm", "flashinfer_mxfp4"): + with ( + self.subTest( + architecture=architecture, a2a=a2a_backend, runner=runner + ), + override_platform(is_sm100=True), + ): + args = SimpleNamespace( + speculative_algorithm=None, + moe_runner_backend=runner, + moe_a2a_backend=a2a_backend, + _model_config=SimpleNamespace(is_fp4_experts=True), + attention_backend=None, + prefill_attention_backend=None, + decode_attention_backend=None, + ) + expected = {"attention_backend": "fa4"} + if runner == "auto" and a2a_backend == "none": + expected["moe_runner_backend"] = "flashinfer_mxfp4" + self.assertEqual( + collect_model_override_declarations( + architecture, args, _hf("fp8") + ), + [("_mimo_v2_overrides", expected)], + ) + def test_mimo_v2_family_is_registered(self): with override_platform(is_sm100=False): self.assertEqual(