[Perf] Use FA4 by default for MiMo on SM100 (#41886)
Co-authored-by: lianghougong <lianghouhong@xiaohongshu.com>
This commit is contained in:
co-authored by
lianghougong
parent
b37e6f79fc
commit
a702970ecb
@@ -5,6 +5,7 @@ from typing import Any, Dict
|
||||
|
||||
from sglang.srt.arg_groups.model_override_base import (
|
||||
_register_for,
|
||||
is_attention_backend_not_set,
|
||||
resolving_view,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_platform
|
||||
@@ -22,6 +23,10 @@ def _mimo_v2_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
logger.info("Enable multi-layer EAGLE speculative decoding for MiMoV2 model.")
|
||||
overrides["enable_multi_layer_eagle"] = True
|
||||
|
||||
if get_platform().is_sm100 and is_attention_backend_not_set(cfg):
|
||||
overrides["attention_backend"] = "fa4"
|
||||
logger.info("MiMoV2 on SM100: attention_backend=fa4.")
|
||||
|
||||
# On Blackwell "auto" falls through to the triton fused-MoE runner, ~12%
|
||||
# slower at bs=1 decode. FP4 checkpoints use flashinfer_mxfp4 instead.
|
||||
if (
|
||||
|
||||
@@ -1159,27 +1159,48 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
)
|
||||
self.assertEqual(_mimo_v2_overrides(_args(), None), {})
|
||||
|
||||
def test_mimo_v2_sm100_fp8_pins_flashinfer_trtllm_moe(self):
|
||||
"""Blackwell FP8 must not be left on the triton fused-MoE runner."""
|
||||
def test_mimo_v2_sm100_defaults(self):
|
||||
from sglang.srt.arg_groups.model_overrides.mimo_v2 import _mimo_v2_overrides
|
||||
|
||||
def _args(**kw):
|
||||
defaults = dict(speculative_algorithm=None, moe_runner_backend="auto")
|
||||
defaults = dict(
|
||||
speculative_algorithm=None,
|
||||
moe_runner_backend="auto",
|
||||
attention_backend=None,
|
||||
prefill_attention_backend=None,
|
||||
decode_attention_backend=None,
|
||||
)
|
||||
defaults.update(kw)
|
||||
return SimpleNamespace(**defaults)
|
||||
|
||||
with override_platform(is_sm100=True):
|
||||
self.assertEqual(
|
||||
_mimo_v2_overrides(_args(), _hf("fp8")),
|
||||
{"moe_runner_backend": "flashinfer_trtllm"},
|
||||
{"attention_backend": "fa4", "moe_runner_backend": "flashinfer_trtllm"},
|
||||
)
|
||||
# An explicit user choice is never overwritten.
|
||||
self.assertEqual(
|
||||
_mimo_v2_overrides(_args(moe_runner_backend="triton"), _hf("fp8")), {}
|
||||
_mimo_v2_overrides(_args(moe_runner_backend="triton"), _hf("fp8")),
|
||||
{"attention_backend": "fa4"},
|
||||
)
|
||||
# FP4 checkpoints run through flashinfer_mxfp4, so they must not be
|
||||
# pinned to flashinfer_trtllm.
|
||||
self.assertEqual(_mimo_v2_overrides(_args(), _hf("mxfp4")), {})
|
||||
self.assertEqual(
|
||||
_mimo_v2_overrides(_args(), _hf("mxfp4")),
|
||||
{"attention_backend": "fa4"},
|
||||
)
|
||||
for field in (
|
||||
"attention_backend",
|
||||
"prefill_attention_backend",
|
||||
"decode_attention_backend",
|
||||
):
|
||||
self.assertEqual(
|
||||
_mimo_v2_overrides(
|
||||
_args(moe_runner_backend="triton", **{field: "triton"}),
|
||||
_hf("fp8"),
|
||||
),
|
||||
{},
|
||||
)
|
||||
|
||||
def test_mimo_v2_family_is_registered(self):
|
||||
with override_platform(is_sm100=False):
|
||||
|
||||
Reference in New Issue
Block a user