From 9ae43b0eee1ba5db389cf3dec1baa9258ef85ad7 Mon Sep 17 00:00:00 2001 From: Rohit Kumar Singh <9626333+SKRohit@users.noreply.github.com> Date: Tue, 29 Sep 2026 10:46:03 +0530 Subject: [PATCH] [XPU] Support compressed-tensors W4A16 by reusing the torch int4pack path (#40828) Co-authored-by: github-actions[bot] Co-authored-by: Singh Co-authored-by: Singh Co-authored-by: Ma Mingfei --- .../compressed_tensors_wna16_kernels.py | 73 +++++++++++ .../xpu/quantization/int4pack_utils.py | 12 ++ .../compressed_tensors/compressed_tensors.py | 10 +- .../compressed_tensors/schemes/__init__.py | 7 +- .../schemes/compressed_tensors_wNa16.py | 51 +++++++- test/registered/xpu/test_xpu_int4_dense.py | 117 +++++++++++++++++- 6 files changed, 265 insertions(+), 5 deletions(-) create mode 100644 python/sglang/srt/hardware_backend/xpu/quantization/compressed_tensors_wna16_kernels.py diff --git a/python/sglang/srt/hardware_backend/xpu/quantization/compressed_tensors_wna16_kernels.py b/python/sglang/srt/hardware_backend/xpu/quantization/compressed_tensors_wna16_kernels.py new file mode 100644 index 0000000000..1893bd087c --- /dev/null +++ b/python/sglang/srt/hardware_backend/xpu/quantization/compressed_tensors_wna16_kernels.py @@ -0,0 +1,73 @@ +# SPDX-License-Identifier: Apache-2.0 +"""compressed-tensors WNA16 int4 dense linear for Intel XPU.""" + +from __future__ import annotations + +from typing import Optional + +import torch + +from sglang.srt.hardware_backend.xpu.quantization.int4pack_utils import ( + pack_int4_to_uint8, + unpack_compressed_tensors_qweight, + xpu_int4pack_mm, +) +from sglang.srt.layers.quantization.utils import replace_parameter + +# Symmetric WNA16 stores each int4 code biased by 8 (``uint4b8``). Marlin folds +# that bias into its scales; the native XPU op takes an explicit zero-point. +_UINT4B8_ZERO_POINT = 8 + + +class CompressedTensorsWNA16XPULinearKernel: + # 4-bit symmetric, group-quantized, no act-order; XPUCompressedTensorsWNA16 + # rejects everything else before this kernel is constructed. + def __init__(self, group_size: int): + self.group_size = group_size + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + weight_packed = layer.weight_packed.data # [N, K // 8] int32 + scales = layer.weight_scale.data # [N, K // gs] + + codes = unpack_compressed_tensors_qweight(weight_packed) # [N, K] + qweight_uint8 = pack_int4_to_uint8(codes) # [N, K // 2] + qweight_packed = torch.ops.aten._convert_weight_to_int4pack(qweight_uint8, 8) + + # The native op wants group-major scales and zero-points: [K // gs, N]. + xpu_scales = scales.t().contiguous() + zero_points = torch.full( + xpu_scales.shape, + _UINT4B8_ZERO_POINT, + dtype=torch.int8, + device=xpu_scales.device, + ) + + replace_parameter(layer, "weight_packed", qweight_packed) + layer.register_parameter( + "xpu_scales", + torch.nn.Parameter(xpu_scales, requires_grad=False), + ) + layer.register_parameter( + "xpu_zero_points", + torch.nn.Parameter(zero_points, requires_grad=False), + ) + del layer.weight_scale + + layer.xpu_out_features = codes.shape[0] + layer.xpu_group_size = self.group_size + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + return xpu_int4pack_mm( + x=x, + qweight_packed=layer.weight_packed, + group_size=layer.xpu_group_size, + scales=layer.xpu_scales, + zero_points=layer.xpu_zero_points, + out_features=layer.xpu_out_features, + bias=bias, + ) diff --git a/python/sglang/srt/hardware_backend/xpu/quantization/int4pack_utils.py b/python/sglang/srt/hardware_backend/xpu/quantization/int4pack_utils.py index eac5b84d47..686dd9e65d 100644 --- a/python/sglang/srt/hardware_backend/xpu/quantization/int4pack_utils.py +++ b/python/sglang/srt/hardware_backend/xpu/quantization/int4pack_utils.py @@ -51,6 +51,18 @@ def unpack_gptq_qweight(qweight: torch.Tensor) -> torch.Tensor: return codes.reshape(-1, n) # [K, N] +def unpack_compressed_tensors_qweight(weight_packed: torch.Tensor) -> torch.Tensor: + """``[N, K // 8]`` int32 packed along K -> ``[N, K]`` codes in ``[0, 15]``. + + ``pack_quantized`` uses natural nibble order, unlike AWQ's interleave. + """ + n = weight_packed.shape[0] + shifts = _nibble_shifts(weight_packed.device) # [8] + # [N, K // 8, 8]; sub-index i selects k = col * 8 + i + codes = (weight_packed.unsqueeze(-1) >> shifts.view(1, 1, 8)) & 0xF + return codes.reshape(n, -1) # [N, K] + + def unpack_gptq_qzeros(qzeros: torch.Tensor) -> torch.Tensor: """``[K // gs, N // 8]`` int32 packed along N -> ``[K // gs, N]`` codes.""" rows = qzeros.shape[0] diff --git a/python/sglang/srt/layers/quantization/compressed_tensors/compressed_tensors.py b/python/sglang/srt/layers/quantization/compressed_tensors/compressed_tensors.py index 3051ecdfd1..e1214b466e 100644 --- a/python/sglang/srt/layers/quantization/compressed_tensors/compressed_tensors.py +++ b/python/sglang/srt/layers/quantization/compressed_tensors/compressed_tensors.py @@ -57,6 +57,7 @@ from sglang.srt.layers.quantization.compressed_tensors.schemes import ( NPUCompressedTensorsW4A16Int4DynamicMoE, NPUCompressedTensorsW8A8Int8, NPUCompressedTensorsW8A8Int8DynamicMoE, + XPUCompressedTensorsWNA16, ) from sglang.srt.layers.quantization.compressed_tensors.utils import ( check_equal_or_regex_match, @@ -699,7 +700,10 @@ class CompressedTensorsConfig(QuantizationConfig): quant_format == CompressionFormat.pack_quantized.value and weight_quant.num_bits in WNA16_SUPPORTED_BITS ): - return CompressedTensorsWNA16( + scheme_cls = ( + XPUCompressedTensorsWNA16 if _is_xpu else CompressedTensorsWNA16 + ) + return scheme_cls( num_bits=weight_quant.num_bits, strategy=weight_quant.strategy, group_size=weight_quant.group_size, @@ -988,7 +992,9 @@ class CompressedTensorsConfig(QuantizationConfig): # (e.g. fp8 needs ada lovelace) # Note: NPU devices do not support min_capability function if _is_xpu: - if not isinstance(scheme, CompressedTensorsW8A8Fp8): + if not isinstance( + scheme, (CompressedTensorsW8A8Fp8, XPUCompressedTensorsWNA16) + ): raise RuntimeError( f"{scheme.__class__.__name__} is not supported on XPU " "(no XPU kernel implementation)." diff --git a/python/sglang/srt/layers/quantization/compressed_tensors/schemes/__init__.py b/python/sglang/srt/layers/quantization/compressed_tensors/schemes/__init__.py index 7c02099429..75f7aa7c23 100644 --- a/python/sglang/srt/layers/quantization/compressed_tensors/schemes/__init__.py +++ b/python/sglang/srt/layers/quantization/compressed_tensors/schemes/__init__.py @@ -18,7 +18,11 @@ from .compressed_tensors_w8a8_int8 import ( ) from .compressed_tensors_w8a8_int8_moe import NPUCompressedTensorsW8A8Int8DynamicMoE from .compressed_tensors_w8a16_fp8 import CompressedTensorsW8A16Fp8 -from .compressed_tensors_wNa16 import WNA16_SUPPORTED_BITS, CompressedTensorsWNA16 +from .compressed_tensors_wNa16 import ( + WNA16_SUPPORTED_BITS, + CompressedTensorsWNA16, + XPUCompressedTensorsWNA16, +) from .compressed_tensors_wNa16_moe import ( CompressedTensorsWNA16MoE, CompressedTensorsWNA16TritonMoE, @@ -35,6 +39,7 @@ __all__ = [ "NPUCompressedTensorsW8A8Int8", "NPUCompressedTensorsW8A8Int8DynamicMoE", "CompressedTensorsWNA16", + "XPUCompressedTensorsWNA16", "CompressedTensorsWNA16MoE", "CompressedTensorsWNA16TritonMoE", "NPUCompressedTensorsW4A16Int4DynamicMoE", diff --git a/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py b/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py index ff3c7e16ee..9afe881070 100644 --- a/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py +++ b/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py @@ -51,7 +51,7 @@ ScalarType, scalar_types = get_scalar_types() logger = logging.getLogger(__name__) -__all__ = ["CompressedTensorsWNA16"] +__all__ = ["CompressedTensorsWNA16", "XPUCompressedTensorsWNA16"] WNA16_SUPPORTED_TYPES_MAP = { 4: scalar_types.uint4b8, 8: scalar_types.uint8b128 @@ -338,3 +338,52 @@ class CompressedTensorsWNA16(CompressedTensorsLinearScheme): is_k_full=self.is_k_full, bias=bias, ) + + +class XPUCompressedTensorsWNA16(CompressedTensorsWNA16): + """WNA16 on Intel XPU: Using torch's int4pack op.""" + + def __init__(self, + strategy: str, + num_bits: int, + group_size: Optional[int] = None, + symmetric: Optional[bool] = True, + actorder: Optional[ActivationOrdering] = None): + super().__init__(strategy, num_bits, group_size, symmetric, actorder) + + from sglang.srt.hardware_backend.xpu.quantization.compressed_tensors_wna16_kernels import ( + CompressedTensorsWNA16XPULinearKernel, + ) + from sglang.srt.hardware_backend.xpu.quantization.int4pack_utils import ( + SUPPORTED_GROUP_SIZES, + ) + + # _weight_int4pack_mm_with_scales_and_zeros cannot express any of these, + # so reject at construction rather than after the weights are loaded. + if num_bits != 4: + raise NotImplementedError( + f"compressed-tensors WNA16 on XPU supports 4-bit weights only, " + f"got num_bits={num_bits}.") + if self.group_size not in SUPPORTED_GROUP_SIZES: + raise NotImplementedError( + f"compressed-tensors WNA16 on XPU requires group_size in " + f"{SUPPORTED_GROUP_SIZES}, got {self.group_size} " + "(channelwise/-1 is out of scope).") + if not self.symmetric: + raise NotImplementedError( + "compressed-tensors WNA16 on XPU only supports symmetric weight " + "quantization; this checkpoint carries a weight zero-point.") + if self.has_g_idx: + raise NotImplementedError( + "compressed-tensors WNA16 on XPU does not support activation " + "reordering (actorder=group).") + + self.kernel = CompressedTensorsWNA16XPULinearKernel( + group_size=self.group_size) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + self.kernel.process_weights_after_loading(layer) + + def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor, + bias: Optional[torch.Tensor]) -> torch.Tensor: + return self.kernel.apply(layer, x, bias) diff --git a/test/registered/xpu/test_xpu_int4_dense.py b/test/registered/xpu/test_xpu_int4_dense.py index 8f0902e161..2ea8b665e3 100644 --- a/test/registered/xpu/test_xpu_int4_dense.py +++ b/test/registered/xpu/test_xpu_int4_dense.py @@ -9,7 +9,7 @@ from sglang.srt.utils import is_xpu from sglang.test.ci.ci_register import register_xpu_ci from sglang.test.test_utils import CustomTestCase -register_xpu_ci(est_time=20, suite="stage-b-test-1-gpu-xpu") +register_xpu_ci(est_time=45, suite="stage-b-test-1-gpu-xpu") DEV = "xpu" @@ -58,6 +58,20 @@ def _gptq_pack_qzeros(zc: torch.Tensor) -> torch.Tensor: return packed +def _ct_pack_qweight(codes: torch.Tensor) -> torch.Tensor: + """``[N, K]`` codes (0..15) -> ``[N, K // 8]`` int32 packed along K. + + compressed-tensors uses natural nibble order; validated against the library's + own ``pack_to_int32`` in ``test_unpack_matches_compressed_tensors_packer``. + """ + n, k = codes.shape + codes = codes.reshape(n, k // 8, 8) + packed = torch.zeros(n, k // 8, dtype=torch.int32, device=codes.device) + for i in range(8): + packed |= codes[:, :, i].to(torch.int32) << (4 * i) + return packed + + def _make_layer(): """A bare ``LinearBase`` with only ``nn.Module`` machinery initialised.""" from sglang.srt.layers.linear import LinearBase @@ -239,5 +253,106 @@ class TestXPUInt4DenseKernel(CustomTestCase): self._run_gptq(8, k, n, gs, dtype, desc_act, "", tp_size=2) +@unittest.skipIf(not is_xpu(), "XPU int4 dense UT requires an Intel XPU") +class TestXPUCompressedTensorsWNA16Kernel(CustomTestCase): + """compressed-tensors WNA16 int4pack kernel numerics vs a pure-torch dequant.""" + + def test_unpack_matches_compressed_tensors_packer(self): + # Pin the format against the library that writes it, so the unpacker and + # this file's packer cannot be self-consistently wrong together. + try: + from compressed_tensors.compressors.pack_quantized.helpers import ( + pack_to_int32, + ) + except ImportError as exc: # pragma: no cover - depends on installed version + self.skipTest(f"compressed_tensors pack_to_int32 unavailable: {exc}") + + from sglang.srt.hardware_backend.xpu.quantization.int4pack_utils import ( + unpack_compressed_tensors_qweight, + ) + + torch.manual_seed(0) + n, k = 64, 256 + w_int = torch.randint(-8, 8, (n, k), dtype=torch.int8) + ref_packed = pack_to_int32(w_int, num_bits=4, packed_dim=1) + self.assertEqual(tuple(ref_packed.shape), (n, k // 8)) + + # uint4b8 stores every signed code biased by +8. + biased = w_int.to(torch.int32) + 8 + codes = unpack_compressed_tensors_qweight(ref_packed.to(DEV)) + self.assertTrue(torch.equal(codes.cpu(), biased)) + self.assertTrue( + torch.equal(_ct_pack_qweight(biased), ref_packed.to(torch.int32)) + ) + + def test_wna16_numeric(self): + for dtype in (torch.float16, torch.bfloat16): + for m in M_VALUES: + for k, n, gs in SHAPES: + with self.subTest(dtype=dtype, M=m, K=k, N=n, gs=gs): + self._run_wna16(m, k, n, gs, dtype) + + def test_wna16_bias(self): + # xpu_int4pack_mm adds bias outside the GEMM; keep that path covered. + for dtype in (torch.float16, torch.bfloat16): + for k, n, gs in SHAPES: + with self.subTest(dtype=dtype, K=k, N=n, gs=gs): + self._run_wna16(8, k, n, gs, dtype, with_bias=True) + + def _run_wna16(self, m, k, n, gs, dtype, with_bias=False): + from sglang.srt.hardware_backend.xpu.quantization.compressed_tensors_wna16_kernels import ( + CompressedTensorsWNA16XPULinearKernel, + ) + + torch.manual_seed(0) + codes = torch.randint(0, 16, (n, k), device=DEV) # [N, K] uint4b8 codes + scales = torch.rand(n, k // gs, device=DEV, dtype=dtype) * 0.05 + 0.005 + + w_ref = (codes.to(dtype) - 8) * scales.repeat_interleave(gs, dim=1) + x = torch.randn(m, k, device=DEV, dtype=dtype) + bias = torch.randn(n, device=DEV, dtype=dtype) if with_bias else None + ref = x @ w_ref.t() + if bias is not None: + ref = ref + bias + + layer = _make_layer() + layer.weight_packed = torch.nn.Parameter( + _ct_pack_qweight(codes), requires_grad=False + ) + layer.weight_scale = torch.nn.Parameter(scales, requires_grad=False) + + kernel = CompressedTensorsWNA16XPULinearKernel(group_size=gs) + kernel.process_weights_after_loading(layer) + out = kernel.apply(layer, x, bias) + + self.assertEqual(tuple(out.shape), (m, n)) + self.assertTrue(torch.isfinite(out).all()) + rel = (out - ref).abs().max().item() / ref.abs().max().item() + self.assertLess(rel, REL_TOL[dtype], f"rel={rel:.2e}") + + def test_wna16_rejects_unsupported_configs(self): + """Configs the native XPU op cannot express must fail at scheme + construction, not after the weights are loaded -- 8-bit WNA16 in + particular, which would otherwise be packed as int4.""" + from compressed_tensors.quantization import ActivationOrdering + + from sglang.srt.layers.quantization.compressed_tensors.schemes import ( + XPUCompressedTensorsWNA16, + ) + + base = dict(strategy="group", num_bits=4, group_size=32, symmetric=True) + cases = ( + ({**base, "num_bits": 8, "group_size": 128}, r"4-bit"), + ({**base, "group_size": 48}, r"group_size"), + ({**base, "strategy": "channel", "group_size": -1}, r"group_size"), + ({**base, "symmetric": False}, r"symmetric"), + ({**base, "actorder": ActivationOrdering.GROUP}, r"activation reordering"), + ) + for kwargs, pattern in cases: + with self.subTest(**kwargs): + with self.assertRaisesRegex(NotImplementedError, pattern): + XPUCompressedTensorsWNA16(**kwargs) + + if __name__ == "__main__": unittest.main()