From 0eae9423d86ddf0c820dd47c97b97fb779467b91 Mon Sep 17 00:00:00 2001 From: Brayden Zhong Date: Tue, 21 Jul 2026 14:09:01 -0700 Subject: [PATCH] Change the FP8 per-tensor GEMM backend on SM120 to cuBLAS (#31961) Co-authored-by: Brayden Zhong --- python/sglang/srt/layers/quantization/fp8_utils.py | 2 +- python/sglang/srt/layers/quantization/modelopt_quant.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/python/sglang/srt/layers/quantization/fp8_utils.py b/python/sglang/srt/layers/quantization/fp8_utils.py index f6c6b2fbb..5093f8602 100755 --- a/python/sglang/srt/layers/quantization/fp8_utils.py +++ b/python/sglang/srt/layers/quantization/fp8_utils.py @@ -1710,7 +1710,7 @@ def apply_fp8_linear_bmm_flashinfer( input_scale: torch.Tensor, bias: Optional[torch.Tensor] = None, ) -> torch.Tensor: - """Per-tensor static fp8 linear via flashinfer bmm_fp8 (SM10X only).""" + """Per-tensor static fp8 linear via flashinfer bmm_fp8 (SM100/SM120 Blackwell).""" output_shape = [*input.shape[:-1], weight.shape[1]] input_2d = input.view(-1, input.shape[-1]) qinput, x_scale = static_quant_fp8(input_2d, input_scale, repeat_scale=False) diff --git a/python/sglang/srt/layers/quantization/modelopt_quant.py b/python/sglang/srt/layers/quantization/modelopt_quant.py index 9675c604e..cee4efc39 100755 --- a/python/sglang/srt/layers/quantization/modelopt_quant.py +++ b/python/sglang/srt/layers/quantization/modelopt_quant.py @@ -499,7 +499,9 @@ class ModelOptFp8LinearMethod(LinearMethodBase): super().__init__() self.quant_config = quant_config self.cutlass_fp8_supported = cutlass_fp8_supported() - self.enable_flashinfer_bmm = is_sm100_supported() and is_flashinfer_available() + self.enable_flashinfer_bmm = ( + is_sm100_supported() or is_sm120_supported() + ) and is_flashinfer_available() def create_weights( self,