From 3c07ff2654359ac443d66e9b4ff5bf106864e22c Mon Sep 17 00:00:00 2001 From: Cao E Date: Tue, 20 Oct 2026 11:24:16 +0800 Subject: [PATCH] [Perf] [XPU] fp8: Optimize FP8 scale layouts (#40052) Co-authored-by: Ma Mingfei --- .../schemes/compressed_tensors_w8a8_fp8.py | 13 +- python/sglang/srt/layers/quantization/fp8.py | 51 ++++--- .../srt/layers/quantization/fp8_utils.py | 50 ++++++- .../registered/e2e/xpu/test_xpu_fp8_linear.py | 138 ++++++++++++++++++ 4 files changed, 226 insertions(+), 26 deletions(-) diff --git a/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py b/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py index a829de606a..7d5104cbc2 100644 --- a/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py +++ b/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py @@ -24,15 +24,17 @@ from sglang.srt.layers.quantization.fp8_utils import ( deepgemm_w8a8_block_fp8_linear_with_fallback, dispatch_w8a8_block_fp8_linear, normalize_e4m3fn_to_e4m3fnuz, + prepare_xpu_block_scale_for_scaled_mm, requant_block_scale_ue8m0_for_deepgemm, validate_fp8_block_shape, ) from sglang.srt.layers.quantization.utils import requantize_with_max_scale -from sglang.srt.utils import get_bool_env_var, is_hip +from sglang.srt.utils import get_bool_env_var, is_hip, is_xpu __all__ = ["CompressedTensorsW8A8Fp8"] _is_hip = is_hip() +_is_xpu = is_xpu() _use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip if _use_aiter: from aiter.ops.shuffle import shuffle_weight @@ -215,6 +217,15 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsLinearScheme): output_dtype=getattr(layer, "orig_dtype", None), weight_shape=layer.weight.shape, ) + if _is_xpu: + scale_reordered = prepare_xpu_block_scale_for_scaled_mm( + layer.weight_scale.data, + self.weight_block_size, + self.w8a8_block_fp8_linear, + ) + if scale_reordered is not layer.weight_scale.data: + with torch.no_grad(): + layer.weight_scale.set_(scale_reordered) else: raise ValueError(f"Unknown quantization strategy {self.strategy}") diff --git a/python/sglang/srt/layers/quantization/fp8.py b/python/sglang/srt/layers/quantization/fp8.py index 360e62e3b6..27096041a3 100644 --- a/python/sglang/srt/layers/quantization/fp8.py +++ b/python/sglang/srt/layers/quantization/fp8.py @@ -69,10 +69,10 @@ from sglang.srt.layers.quantization.fp8_utils import ( input_to_float8, mxfp8_group_quantize, normalize_e4m3fn_to_e4m3fnuz, + prepare_xpu_block_scale_for_scaled_mm, requant_block_scale_ue8m0_for_deepgemm, resolve_block_fp8_mxfp8_backend, resolve_mxfp8_dense_gemm_backend, - torch_w8a8_block_fp8_linear, unshuffle_aiter_fp8_weight, use_aiter_bpreshuffle_gemm, ) @@ -128,6 +128,7 @@ _is_musa = is_musa() _is_npu = is_npu() _is_cpu_amx_available = cpu_has_amx_support() _is_cpu = is_cpu() +_is_xpu = is_xpu() _is_fp8_fnuz = is_fp8_fnuz() _is_gfx95_supported = is_gfx95_supported() @@ -907,26 +908,13 @@ class Fp8LinearMethod(LinearMethodBase): layer.weight.is_shuffled = True self._maybe_prepare_dense_mxscale(layer) - if ( - is_xpu() - and self.w8a8_block_fp8_linear is torch_w8a8_block_fp8_linear - and self.weight_block_size in ([1, 128], [128, 128]) - and layer.weight_scale_inv.ndim == 2 - ): - # Keep the checkpoint's logical [N-blocks, K-blocks] shape, but use - # transpose-contiguous storage. For [1, 128], scaled_mm transposes - # scale_b internally; for [128, 128], the wrapper passes scale_b.t(). - # This avoids a per-forward contiguous/copy in either path. - scale = layer.weight_scale_inv.data - scale_b_is_contiguous = scale.t().is_contiguous() - if not scale_b_is_contiguous: - scale_reordered = torch.empty_strided( - scale.shape, - (1, scale.shape[0]), - dtype=scale.dtype, - device=scale.device, - ) - scale_reordered.copy_(scale) + if _is_xpu: + scale_reordered = prepare_xpu_block_scale_for_scaled_mm( + layer.weight_scale_inv.data, + self.weight_block_size, + self.w8a8_block_fp8_linear, + ) + if scale_reordered is not layer.weight_scale_inv.data: with torch.no_grad(): layer.weight_scale_inv.set_(scale_reordered) @@ -1112,6 +1100,7 @@ class Fp8LinearMethod(LinearMethodBase): if not self.is_checkpoint_fp8_serialized: if ( self.cutlass_fp8_supported + or _is_xpu or self.use_marlin or (_use_aiter and self.use_aiter_fp8_per_token) ): @@ -1165,6 +1154,7 @@ class Fp8LinearMethod(LinearMethodBase): # cutlass sgl-kernel and marlin only support per-channel scale; aiter supports per-channel scale if ( (self.cutlass_fp8_supported and not use_sm120_fp8_pertensor) + or _is_xpu or self.use_marlin or (_use_aiter and self.use_aiter_fp8_per_token) ): @@ -1229,6 +1219,23 @@ class Fp8LinearMethod(LinearMethodBase): layer.input_scale.max(), requires_grad=False ) + if _is_xpu: + # A scale is row-wise iff it carries exactly one value per + # output channel, regardless of shape ([N], [1, N], [N, 1], ...). + num_channels = sum(layer.logical_widths) + is_rowwise = layer.weight_scale.numel() == num_channels + if is_rowwise and ( + layer.weight_scale.shape != (num_channels, 1) + or not layer.weight_scale.t().is_contiguous() + ): + with torch.no_grad(): + layer.weight_scale.set_( + layer.weight_scale.data.reshape( + num_channels, 1 + ).contiguous() + ) + self.use_per_token_if_dynamic = is_rowwise + if _is_cpu: assert _is_cpu_amx_available, ( "Fp8LinearMethod on CPU requires that CPU has AMX support" @@ -3042,7 +3049,7 @@ class Fp8MoEMethod(FusedMoEMethodBase): if quant_info is not None: return self.runner.run(dispatch_output, quant_info) - if is_xpu() and not get_moe_runner_backend().is_triton(): + if _is_xpu and not get_moe_runner_backend().is_triton(): # sgl-kernel-xpu path from sgl_kernel import fused_experts diff --git a/python/sglang/srt/layers/quantization/fp8_utils.py b/python/sglang/srt/layers/quantization/fp8_utils.py index 097a75066f..4f4f352cf8 100755 --- a/python/sglang/srt/layers/quantization/fp8_utils.py +++ b/python/sglang/srt/layers/quantization/fp8_utils.py @@ -384,6 +384,9 @@ def use_rowwise_torch_scaled_mm(): # The condition is determined once as the operations # are time consuming. return get_device_capability() >= (9, 4) and torch_release >= (2, 7) + if _is_xpu: + # XPU's torch._scaled_mm backend supports rowwise scaling. + return True return False @@ -789,6 +792,35 @@ def torch_w8a8_block_fp8_linear( return output.view(*input.shape[:-1], weight.shape[0]) +def prepare_xpu_block_scale_for_scaled_mm( + weight_scale: torch.Tensor, + block_size: List[int], + linear_fn: Callable, +) -> torch.Tensor: + """Materialize the XPU block scale layout consumed by scaled_mm once.""" + # Keep logical [N-blocks, K-blocks] shape, but use + # transpose-contiguous storage. For [1, 128], scaled_mm transposes + # scale_b internally; for [128, 128], the wrapper passes scale_b.t(). + # This avoids a per-forward contiguous/copy in either path. + if ( + linear_fn is not torch_w8a8_block_fp8_linear + or not isinstance(block_size, (list, tuple)) + or tuple(block_size) not in ((1, 128), (128, 128)) + or weight_scale.ndim != 2 + or weight_scale.t().is_contiguous() + ): + return weight_scale + + reordered = torch.empty_strided( + weight_scale.shape, + (1, weight_scale.shape[0]), + dtype=weight_scale.dtype, + device=weight_scale.device, + ) + reordered.copy_(weight_scale) + return reordered + + def resolve_mxfp8_dense_gemm_backend() -> Mxfp8DenseGemmBackend: """Pick the MXFP8 dense linear backend, honoring `--fp8-gemm-backend` only when it names a backend that owns an MXFP8 dense kernel.""" @@ -2334,7 +2366,9 @@ def _apply_fallback_scaled_mm( ) output = _process_scaled_mm_output(output, input_2d_shape, output_shape) - x_scale = torch.narrow(x_scale, 0, 0, input_2d_shape[0]) + # A single scale has no row dimension to trim, regardless of its shape. + if x_scale.numel() > 1: + x_scale = torch.narrow(x_scale, 0, 0, input_2d_shape[0]) output = output * x_scale * weight_scale.t() if bias is not None: @@ -2558,12 +2592,22 @@ def apply_fp8_linear( # When the number of token is 1, # per-token scale has shape (1, 1), per-tensor scale has shape (1) or (). per_tensor_activations = (x_scale.numel() == 1) and x_scale.dim() < 2 - + rowwise_scale_layout_supported = not _is_xpu or ( + x_scale.ndim == 2 + and x_scale.shape[1] == 1 + and x_scale.is_contiguous() + and weight_scale.ndim == 2 + and weight_scale.shape[1] == 1 + and weight_scale.t().is_contiguous() + ) if ( use_per_token_if_dynamic and not per_tensor_weights and not per_tensor_activations - and (USE_ROWWISE_TORCH_SCALED_MM or _use_aiter) + and ( + (USE_ROWWISE_TORCH_SCALED_MM and rowwise_scale_layout_supported) + or _use_aiter + ) ): # into this sector means use dynamic per-token-per-channel quant # per-token scale quant for input matrix, every row(one token) have one scale factor diff --git a/test/registered/e2e/xpu/test_xpu_fp8_linear.py b/test/registered/e2e/xpu/test_xpu_fp8_linear.py index f0bb30a8d1..f13727b6be 100644 --- a/test/registered/e2e/xpu/test_xpu_fp8_linear.py +++ b/test/registered/e2e/xpu/test_xpu_fp8_linear.py @@ -8,12 +8,16 @@ Usage: import unittest from typing import List +from unittest.mock import patch import torch +from sglang.srt.layers.quantization.fp8 import Fp8Config, Fp8LinearMethod from sglang.srt.layers.quantization.fp8_utils import ( + apply_fp8_linear, dispatch_w8a8_block_fp8_linear, per_token_group_quant_fp8, + scaled_fp8_quant, torch_w8a8_block_fp8_linear, ) from sglang.test.ci.ci_register import register_xpu_ci @@ -331,6 +335,140 @@ class TestXPUFP8Linear(CustomTestCase): ).to(torch.bfloat16) torch.testing.assert_close(out, ref, rtol=0.05, atol=0.1) + def test_channelwise_xpu_rowwise_scaled_mm_gate(self): + """Use fused rowwise scaled_mm without changing accuracy.""" + method = Fp8LinearMethod( + Fp8Config(is_checkpoint_fp8_serialized=False, weight_block_size=None) + ) + K, N = 256, 384 + weight_bf16 = torch.randn(N, K, dtype=torch.bfloat16, device=self.device) / 3 + + layer = torch.nn.Module() + layer.logical_widths = [N] + layer.weight = torch.nn.Parameter(weight_bf16, requires_grad=False) + layer.orig_dtype = torch.bfloat16 + layer.input_scale = None + self.assertFalse(method.cutlass_fp8_supported) + method.process_weights_after_loading(layer) + self.assertTrue(method.use_per_token_if_dynamic) + weight = layer.weight + weight_scale = layer.weight_scale + self.assertEqual(tuple(weight_scale.shape), (N, 1)) + self.assertTrue(weight_scale.t().is_contiguous()) + original_scaled_mm = torch._scaled_mm + + for m in (1, 8): + with self.subTest(m=m): + x = torch.randn(m, K, dtype=torch.bfloat16, device=self.device) / 3 + q_input, input_scale = scaled_fp8_quant( + x, use_per_token_if_dynamic=True + ) + reference = (q_input.float() * input_scale.float()) @ ( + weight.float() * weight_scale.t() + ) + calls = [] + + def traced_scaled_mm(*args, **kwargs): + calls.append( + ( + tuple(kwargs["scale_a"].shape), + tuple(kwargs["scale_b"].shape), + ) + ) + return original_scaled_mm(*args, **kwargs) + + with patch.object(torch, "_scaled_mm", side_effect=traced_scaled_mm): + output = apply_fp8_linear( + input=x, + weight=weight, + weight_scale=weight_scale, + use_per_token_if_dynamic=method.use_per_token_if_dynamic, + compressed_tensor_quant=True, + ) + + torch.testing.assert_close( + output, reference.to(torch.bfloat16), rtol=0.01, atol=0.1 + ) + expected_scale_shapes = ((m, 1), (1, N)) + self.assertEqual(calls, [expected_scale_shapes]) + + def test_static_input_scale_with_channelwise_weight_scale(self): + """Static activation_scheme + per-channel weight_scale must not crash. + + process_weights_after_loading forces XPU through the channelwise + weight_scale branch unconditionally, but a static checkpoint gives a + 0-dim per-tensor input_scale (not a per-token one). That combination + used to hit `_apply_fallback_scaled_mm`'s `torch.narrow` on a 0-dim + tensor. + """ + method = Fp8LinearMethod( + Fp8Config(is_checkpoint_fp8_serialized=True, weight_block_size=None) + ) + method.quant_config.activation_scheme = "static" + N, K = 384, 256 + weight_bf16 = torch.randn(N, K, dtype=torch.bfloat16, device=self.device) / 3 + weight_scale = ( + weight_bf16.float().abs().amax(dim=1, keepdim=True).clamp_min(1e-12) + / torch.finfo(torch.float8_e4m3fn).max + ) + + layer = torch.nn.Module() + layer.logical_widths = [N] + layer.weight = torch.nn.Parameter( + weight_bf16.to(torch.float8_e4m3fn), requires_grad=False + ) + layer.weight_scale = torch.nn.Parameter(weight_scale, requires_grad=False) + layer.orig_dtype = torch.bfloat16 + layer.input_scale = torch.nn.Parameter( + torch.tensor(0.05, device=self.device), requires_grad=False + ) + method.process_weights_after_loading(layer) + self.assertEqual(layer.input_scale.shape, torch.Size([])) + + x = torch.randn(4, K, dtype=torch.bfloat16, device=self.device) / 3 + output = method.apply(layer, x) + + q_input = ( + (x.float() / layer.input_scale.float()) + .clamp(-448, 448) + .to(torch.float8_e4m3fn) + ) + # layer.weight is stored transposed as (K, N) after loading. + reference = (q_input.float() * layer.input_scale.float()) @ ( + layer.weight.float() * layer.weight_scale.float().t() + ) + torch.testing.assert_close( + output, reference.to(torch.bfloat16), rtol=0.05, atol=0.2 + ) + + def test_prequantized_vector_scale_with_channelwise_weight_scale(self): + """A one-element pre-quantized input scale is treated as scalar.""" + M, K, N = 4, 256, 384 + q_input = torch.randn(M, K, dtype=torch.bfloat16, device=self.device).to( + torch.float8_e4m3fn + ) + weight = torch.randn(K, N, dtype=torch.bfloat16, device=self.device).to( + torch.float8_e4m3fn + ) + weight_scale = torch.full((N, 1), 0.5, device=self.device) + input_scale = torch.tensor([0.25], device=self.device) + + output = apply_fp8_linear( + input=q_input, + weight=weight, + weight_scale=weight_scale, + input_scale=input_scale, + use_per_token_if_dynamic=True, + compressed_tensor_quant=True, + pre_quant_output_dtype=torch.bfloat16, + ) + reference = (q_input.float() * input_scale.float()) @ ( + weight.float() * weight_scale.t() + ) + torch.testing.assert_close( + output, reference.to(torch.bfloat16), rtol=0.01, atol=0.1 + ) + if __name__ == "__main__": unittest.main()