[Fix] Select the MXFP4 MoE runner for MiMo-V2 packed experts on SM100 (#41668)

Co-authored-by: jbernloehr <janbernloehr@users.noreply.github.com>
This commit is contained in:
Jan Bernlöhr
2026-10-01 15:33:30 -07:00
committed by GitHub
co-authored by jbernloehr
parent a21dcbd320
commit 80bb3fb651
2 changed files with 45 additions and 4 deletions
@@ -6,6 +6,7 @@ from typing import Any, Dict
from sglang.srt.arg_groups.model_override_base import (
_register_for,
is_attention_backend_not_set,
model_config_of,
resolving_view,
)
from sglang.srt.runtime_context import get_platform
@@ -27,13 +28,23 @@ def _mimo_v2_overrides(server_args: Any, hf_config: Any) -> dict:
overrides["attention_backend"] = "fa4"
logger.info("MiMoV2 on SM100: attention_backend=fa4.")
# On Blackwell "auto" falls through to the triton fused-MoE runner, ~12%
# slower at bs=1 decode. FP4 checkpoints use flashinfer_mxfp4 instead.
# Mixed checkpoints also advertise quant_method=fp8; select the runner
# using the routed-expert layout already resolved by ModelConfig.
if (
get_platform().is_sm100
and cfg.moe_runner_backend == "auto"
and get_quantization_config(hf_config) == "fp8"
):
overrides["moe_runner_backend"] = "flashinfer_trtllm"
logger.info("MiMoV2 FP8 on SM100: moe_runner_backend=flashinfer_trtllm.")
if model_config_of(server_args).is_fp4_experts:
# Let the all-to-all backend choose its compatible runner.
if cfg.moe_a2a_backend == "none":
overrides["moe_runner_backend"] = "flashinfer_mxfp4"
else:
# Avoid the slower Triton default for ordinary FP8 checkpoints.
overrides["moe_runner_backend"] = "flashinfer_trtllm"
if "moe_runner_backend" in overrides:
logger.info(
"MiMoV2 on SM100: moe_runner_backend=%s.",
overrides["moe_runner_backend"],
)
return overrides