[Fix] Select the MXFP4 MoE runner for MiMo-V2 packed experts on SM100 (#41668)
Co-authored-by: jbernloehr <janbernloehr@users.noreply.github.com>
This commit is contained in:
co-authored by
jbernloehr
parent
a21dcbd320
commit
80bb3fb651
@@ -6,6 +6,7 @@ from typing import Any, Dict
|
||||
from sglang.srt.arg_groups.model_override_base import (
|
||||
_register_for,
|
||||
is_attention_backend_not_set,
|
||||
model_config_of,
|
||||
resolving_view,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_platform
|
||||
@@ -27,13 +28,23 @@ def _mimo_v2_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
overrides["attention_backend"] = "fa4"
|
||||
logger.info("MiMoV2 on SM100: attention_backend=fa4.")
|
||||
|
||||
# On Blackwell "auto" falls through to the triton fused-MoE runner, ~12%
|
||||
# slower at bs=1 decode. FP4 checkpoints use flashinfer_mxfp4 instead.
|
||||
# Mixed checkpoints also advertise quant_method=fp8; select the runner
|
||||
# using the routed-expert layout already resolved by ModelConfig.
|
||||
if (
|
||||
get_platform().is_sm100
|
||||
and cfg.moe_runner_backend == "auto"
|
||||
and get_quantization_config(hf_config) == "fp8"
|
||||
):
|
||||
overrides["moe_runner_backend"] = "flashinfer_trtllm"
|
||||
logger.info("MiMoV2 FP8 on SM100: moe_runner_backend=flashinfer_trtllm.")
|
||||
if model_config_of(server_args).is_fp4_experts:
|
||||
# Let the all-to-all backend choose its compatible runner.
|
||||
if cfg.moe_a2a_backend == "none":
|
||||
overrides["moe_runner_backend"] = "flashinfer_mxfp4"
|
||||
else:
|
||||
# Avoid the slower Triton default for ordinary FP8 checkpoints.
|
||||
overrides["moe_runner_backend"] = "flashinfer_trtllm"
|
||||
if "moe_runner_backend" in overrides:
|
||||
logger.info(
|
||||
"MiMoV2 on SM100: moe_runner_backend=%s.",
|
||||
overrides["moe_runner_backend"],
|
||||
)
|
||||
return overrides
|
||||
|
||||
Reference in New Issue
Block a user