[Perf] Use FA4 by default for MiMo on SM100 (#41886)

Co-authored-by: lianghougong <lianghouhong@xiaohongshu.com>
This commit is contained in:
leonardHONG
2026-10-01 09:05:12 +08:00
committed by GitHub
co-authored by lianghougong
parent b37e6f79fc
commit a702970ecb
2 changed files with 32 additions and 6 deletions
@@ -5,6 +5,7 @@ from typing import Any, Dict
from sglang.srt.arg_groups.model_override_base import (
_register_for,
is_attention_backend_not_set,
resolving_view,
)
from sglang.srt.runtime_context import get_platform
@@ -22,6 +23,10 @@ def _mimo_v2_overrides(server_args: Any, hf_config: Any) -> dict:
logger.info("Enable multi-layer EAGLE speculative decoding for MiMoV2 model.")
overrides["enable_multi_layer_eagle"] = True
if get_platform().is_sm100 and is_attention_backend_not_set(cfg):
overrides["attention_backend"] = "fa4"
logger.info("MiMoV2 on SM100: attention_backend=fa4.")
# On Blackwell "auto" falls through to the triton fused-MoE runner, ~12%
# slower at bs=1 decode. FP4 checkpoints use flashinfer_mxfp4 instead.
if (
+27 -6
View File
@@ -1159,27 +1159,48 @@ class TestGoldenModelOverrides(_IsolatedPublish):
)
self.assertEqual(_mimo_v2_overrides(_args(), None), {})
def test_mimo_v2_sm100_fp8_pins_flashinfer_trtllm_moe(self):
"""Blackwell FP8 must not be left on the triton fused-MoE runner."""
def test_mimo_v2_sm100_defaults(self):
from sglang.srt.arg_groups.model_overrides.mimo_v2 import _mimo_v2_overrides
def _args(**kw):
defaults = dict(speculative_algorithm=None, moe_runner_backend="auto")
defaults = dict(
speculative_algorithm=None,
moe_runner_backend="auto",
attention_backend=None,
prefill_attention_backend=None,
decode_attention_backend=None,
)
defaults.update(kw)
return SimpleNamespace(**defaults)
with override_platform(is_sm100=True):
self.assertEqual(
_mimo_v2_overrides(_args(), _hf("fp8")),
{"moe_runner_backend": "flashinfer_trtllm"},
{"attention_backend": "fa4", "moe_runner_backend": "flashinfer_trtllm"},
)
# An explicit user choice is never overwritten.
self.assertEqual(
_mimo_v2_overrides(_args(moe_runner_backend="triton"), _hf("fp8")), {}
_mimo_v2_overrides(_args(moe_runner_backend="triton"), _hf("fp8")),
{"attention_backend": "fa4"},
)
# FP4 checkpoints run through flashinfer_mxfp4, so they must not be
# pinned to flashinfer_trtllm.
self.assertEqual(_mimo_v2_overrides(_args(), _hf("mxfp4")), {})
self.assertEqual(
_mimo_v2_overrides(_args(), _hf("mxfp4")),
{"attention_backend": "fa4"},
)
for field in (
"attention_backend",
"prefill_attention_backend",
"decode_attention_backend",
):
self.assertEqual(
_mimo_v2_overrides(
_args(moe_runner_backend="triton", **{field: "triton"}),
_hf("fp8"),
),
{},
)
def test_mimo_v2_family_is_registered(self):
with override_platform(is_sm100=False):