diff --git a/python/sglang/srt/arg_groups/model_overrides/mimo_v2.py b/python/sglang/srt/arg_groups/model_overrides/mimo_v2.py index c0ad3d3738..68cdfc58dd 100644 --- a/python/sglang/srt/arg_groups/model_overrides/mimo_v2.py +++ b/python/sglang/srt/arg_groups/model_overrides/mimo_v2.py @@ -5,6 +5,7 @@ from typing import Any, Dict from sglang.srt.arg_groups.model_override_base import ( _register_for, + is_attention_backend_not_set, resolving_view, ) from sglang.srt.runtime_context import get_platform @@ -22,6 +23,10 @@ def _mimo_v2_overrides(server_args: Any, hf_config: Any) -> dict: logger.info("Enable multi-layer EAGLE speculative decoding for MiMoV2 model.") overrides["enable_multi_layer_eagle"] = True + if get_platform().is_sm100 and is_attention_backend_not_set(cfg): + overrides["attention_backend"] = "fa4" + logger.info("MiMoV2 on SM100: attention_backend=fa4.") + # On Blackwell "auto" falls through to the triton fused-MoE runner, ~12% # slower at bs=1 decode. FP4 checkpoints use flashinfer_mxfp4 instead. if ( diff --git a/test/registered/unit/test_model_overrides.py b/test/registered/unit/test_model_overrides.py index e77d007ddc..3d7f003a4e 100644 --- a/test/registered/unit/test_model_overrides.py +++ b/test/registered/unit/test_model_overrides.py @@ -1159,27 +1159,48 @@ class TestGoldenModelOverrides(_IsolatedPublish): ) self.assertEqual(_mimo_v2_overrides(_args(), None), {}) - def test_mimo_v2_sm100_fp8_pins_flashinfer_trtllm_moe(self): - """Blackwell FP8 must not be left on the triton fused-MoE runner.""" + def test_mimo_v2_sm100_defaults(self): from sglang.srt.arg_groups.model_overrides.mimo_v2 import _mimo_v2_overrides def _args(**kw): - defaults = dict(speculative_algorithm=None, moe_runner_backend="auto") + defaults = dict( + speculative_algorithm=None, + moe_runner_backend="auto", + attention_backend=None, + prefill_attention_backend=None, + decode_attention_backend=None, + ) defaults.update(kw) return SimpleNamespace(**defaults) with override_platform(is_sm100=True): self.assertEqual( _mimo_v2_overrides(_args(), _hf("fp8")), - {"moe_runner_backend": "flashinfer_trtllm"}, + {"attention_backend": "fa4", "moe_runner_backend": "flashinfer_trtllm"}, ) # An explicit user choice is never overwritten. self.assertEqual( - _mimo_v2_overrides(_args(moe_runner_backend="triton"), _hf("fp8")), {} + _mimo_v2_overrides(_args(moe_runner_backend="triton"), _hf("fp8")), + {"attention_backend": "fa4"}, ) # FP4 checkpoints run through flashinfer_mxfp4, so they must not be # pinned to flashinfer_trtllm. - self.assertEqual(_mimo_v2_overrides(_args(), _hf("mxfp4")), {}) + self.assertEqual( + _mimo_v2_overrides(_args(), _hf("mxfp4")), + {"attention_backend": "fa4"}, + ) + for field in ( + "attention_backend", + "prefill_attention_backend", + "decode_attention_backend", + ): + self.assertEqual( + _mimo_v2_overrides( + _args(moe_runner_backend="triton", **{field: "triton"}), + _hf("fp8"), + ), + {}, + ) def test_mimo_v2_family_is_registered(self): with override_platform(is_sm100=False):