minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[Kimi K3] Select FlashInfer MXFP4 for SM107 auto MoE (#35554)

Browse Source
This commit is contained in:
Lee Nau
2026-08-20 14:09:31 -07:00
committed by GitHub
parent d9f6861359
commit ad367d72b0
1 changed files with 5 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/arg_groups/overrides.py
+5 -5
View File
@@ -574,17 +574,17 @@ def _is_mxfp4_pack_quantized(hf_config: Any) -> bool:
def _kimi_k3_moe_runner_overrides(server_args: Any, hf_config: Any) -> dict: def _kimi_k3_moe_runner_overrides(server_args: Any, hf_config: Any) -> dict:
# MoE runner default, independent of the attention-backend gate above. # MoE runner default, independent of the attention-backend gate above.
# trtllm-gen fused MoE (flashinfer_mxfp4) beats marlin on both the decode # trtllm-gen fused MoE (flashinfer_mxfp4) beats marlin on both the decode
# (M=bs) and the target-verify (M=bs*(gamma+1)) regimes on SM100/SM103; # (M=bs) and the target-verify (M=bs*(gamma+1)) regimes on SM100/SM103.
# FlashInfer 0.6.17+ ships the required SiTU kernels and is a pinned # SM107 uses the same packed-MXFP4 runner; leaving auto unresolved falls
# project dependency. # back to BF16 weight materialization during model loading.
if server_args.moe_runner_backend != "auto": if server_args.moe_runner_backend != "auto":
return {} return {}
if not (is_sm100_supported() and get_device_sm() in (100, 103)): if not (is_sm100_supported() and get_device_sm() in (100, 103, 107)):
return {} return {}
if not _is_mxfp4_pack_quantized(hf_config): if not _is_mxfp4_pack_quantized(hf_config):
return {} return {}
logger.info( logger.info(
"Kimi-K3 on SM100/SM103: moe_runner_backend=flashinfer_mxfp4 " "Kimi-K3 on SM100/SM103/SM107: moe_runner_backend=flashinfer_mxfp4 "
"(FlashInfer SiTU kernels)." "(FlashInfer SiTU kernels)."
) )
return {"moe_runner_backend": "flashinfer_mxfp4"} return {"moe_runner_backend": "flashinfer_mxfp4"}