From 09ca4fc96b3c1aad9d774985efc06c55d7ff6798 Mon Sep 17 00:00:00 2001 From: Lianmin Zheng Date: Fri, 26 Jun 2026 18:15:08 -0700 Subject: [PATCH] Skip FlashInfer FP8 autotune for MXFP8 quantized models (#29462) --- python/sglang/srt/model_executor/runner/base_runner.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/model_executor/runner/base_runner.py b/python/sglang/srt/model_executor/runner/base_runner.py index 25b1a7706..1be6c95b7 100644 --- a/python/sglang/srt/model_executor/runner/base_runner.py +++ b/python/sglang/srt/model_executor/runner/base_runner.py @@ -302,7 +302,13 @@ class BaseRunner(ABC): "modelopt_fp8", "modelopt_mixed", ) - fp8_gemm_needs_autotune = ( + # Online MXFP8 (microscaling) linears dispatch to flashinfer's + # ``mm_mxfp8``, which the flashinfer fp8 autotune dummy run does not + # exercise correctly -- it triggers an illegal memory access inside the + # mxfp8 cutlass cubin. The mxfp8 gemm is fixed-config and needs no + # tuning, so skip autotune for these models. + model_uses_mxfp8 = "mxfp8" in (mr.model_config.quantization or "") + fp8_gemm_needs_autotune = not model_uses_mxfp8 and ( get_fp8_gemm_runner_backend().is_flashinfer_cutlass() or (model_uses_modelopt_fp8 and is_sm100_supported()) )