[XPU] Support compressed-tensors W4A16 by reusing the torch int4pack path (#40828)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: Singh <rohitsi2@iil-gnrap01.iind.intel.com>
Co-authored-by: Singh <rohitsi2@iil-login.iind.intel.com>
Co-authored-by: Ma Mingfei <mingfei.ma@intel.com>
This commit is contained in:
Rohit Kumar Singh
2026-09-29 13:16:03 +08:00
committed by GitHub
co-authored by github-actions[bot] Singh Singh Ma Mingfei
parent 5781206833
commit 9ae43b0eee
6 changed files with 265 additions and 5 deletions
@@ -0,0 +1,73 @@
# SPDX-License-Identifier: Apache-2.0
"""compressed-tensors WNA16 int4 dense linear for Intel XPU."""
from __future__ import annotations
from typing import Optional
import torch
from sglang.srt.hardware_backend.xpu.quantization.int4pack_utils import (
pack_int4_to_uint8,
unpack_compressed_tensors_qweight,
xpu_int4pack_mm,
)
from sglang.srt.layers.quantization.utils import replace_parameter
# Symmetric WNA16 stores each int4 code biased by 8 (``uint4b8``). Marlin folds
# that bias into its scales; the native XPU op takes an explicit zero-point.
_UINT4B8_ZERO_POINT = 8
class CompressedTensorsWNA16XPULinearKernel:
# 4-bit symmetric, group-quantized, no act-order; XPUCompressedTensorsWNA16
# rejects everything else before this kernel is constructed.
def __init__(self, group_size: int):
self.group_size = group_size
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
weight_packed = layer.weight_packed.data # [N, K // 8] int32
scales = layer.weight_scale.data # [N, K // gs]
codes = unpack_compressed_tensors_qweight(weight_packed) # [N, K]
qweight_uint8 = pack_int4_to_uint8(codes) # [N, K // 2]
qweight_packed = torch.ops.aten._convert_weight_to_int4pack(qweight_uint8, 8)
# The native op wants group-major scales and zero-points: [K // gs, N].
xpu_scales = scales.t().contiguous()
zero_points = torch.full(
xpu_scales.shape,
_UINT4B8_ZERO_POINT,
dtype=torch.int8,
device=xpu_scales.device,
)
replace_parameter(layer, "weight_packed", qweight_packed)
layer.register_parameter(
"xpu_scales",
torch.nn.Parameter(xpu_scales, requires_grad=False),
)
layer.register_parameter(
"xpu_zero_points",
torch.nn.Parameter(zero_points, requires_grad=False),
)
del layer.weight_scale
layer.xpu_out_features = codes.shape[0]
layer.xpu_group_size = self.group_size
def apply(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: Optional[torch.Tensor] = None,
) -> torch.Tensor:
return xpu_int4pack_mm(
x=x,
qweight_packed=layer.weight_packed,
group_size=layer.xpu_group_size,
scales=layer.xpu_scales,
zero_points=layer.xpu_zero_points,
out_features=layer.xpu_out_features,
bias=bias,
)
@@ -51,6 +51,18 @@ def unpack_gptq_qweight(qweight: torch.Tensor) -> torch.Tensor:
return codes.reshape(-1, n) # [K, N]
def unpack_compressed_tensors_qweight(weight_packed: torch.Tensor) -> torch.Tensor:
"""``[N, K // 8]`` int32 packed along K -> ``[N, K]`` codes in ``[0, 15]``.
``pack_quantized`` uses natural nibble order, unlike AWQ's interleave.
"""
n = weight_packed.shape[0]
shifts = _nibble_shifts(weight_packed.device) # [8]
# [N, K // 8, 8]; sub-index i selects k = col * 8 + i
codes = (weight_packed.unsqueeze(-1) >> shifts.view(1, 1, 8)) & 0xF
return codes.reshape(n, -1) # [N, K]
def unpack_gptq_qzeros(qzeros: torch.Tensor) -> torch.Tensor:
"""``[K // gs, N // 8]`` int32 packed along N -> ``[K // gs, N]`` codes."""
rows = qzeros.shape[0]
@@ -57,6 +57,7 @@ from sglang.srt.layers.quantization.compressed_tensors.schemes import (
NPUCompressedTensorsW4A16Int4DynamicMoE,
NPUCompressedTensorsW8A8Int8,
NPUCompressedTensorsW8A8Int8DynamicMoE,
XPUCompressedTensorsWNA16,
)
from sglang.srt.layers.quantization.compressed_tensors.utils import (
check_equal_or_regex_match,
@@ -699,7 +700,10 @@ class CompressedTensorsConfig(QuantizationConfig):
quant_format == CompressionFormat.pack_quantized.value
and weight_quant.num_bits in WNA16_SUPPORTED_BITS
):
return CompressedTensorsWNA16(
scheme_cls = (
XPUCompressedTensorsWNA16 if _is_xpu else CompressedTensorsWNA16
)
return scheme_cls(
num_bits=weight_quant.num_bits,
strategy=weight_quant.strategy,
group_size=weight_quant.group_size,
@@ -988,7 +992,9 @@ class CompressedTensorsConfig(QuantizationConfig):
# (e.g. fp8 needs ada lovelace)
# Note: NPU devices do not support min_capability function
if _is_xpu:
if not isinstance(scheme, CompressedTensorsW8A8Fp8):
if not isinstance(
scheme, (CompressedTensorsW8A8Fp8, XPUCompressedTensorsWNA16)
):
raise RuntimeError(
f"{scheme.__class__.__name__} is not supported on XPU "
"(no XPU kernel implementation)."
@@ -18,7 +18,11 @@ from .compressed_tensors_w8a8_int8 import (
)
from .compressed_tensors_w8a8_int8_moe import NPUCompressedTensorsW8A8Int8DynamicMoE
from .compressed_tensors_w8a16_fp8 import CompressedTensorsW8A16Fp8
from .compressed_tensors_wNa16 import WNA16_SUPPORTED_BITS, CompressedTensorsWNA16
from .compressed_tensors_wNa16 import (
WNA16_SUPPORTED_BITS,
CompressedTensorsWNA16,
XPUCompressedTensorsWNA16,
)
from .compressed_tensors_wNa16_moe import (
CompressedTensorsWNA16MoE,
CompressedTensorsWNA16TritonMoE,
@@ -35,6 +39,7 @@ __all__ = [
"NPUCompressedTensorsW8A8Int8",
"NPUCompressedTensorsW8A8Int8DynamicMoE",
"CompressedTensorsWNA16",
"XPUCompressedTensorsWNA16",
"CompressedTensorsWNA16MoE",
"CompressedTensorsWNA16TritonMoE",
"NPUCompressedTensorsW4A16Int4DynamicMoE",
@@ -51,7 +51,7 @@ ScalarType, scalar_types = get_scalar_types()
logger = logging.getLogger(__name__)
__all__ = ["CompressedTensorsWNA16"]
__all__ = ["CompressedTensorsWNA16", "XPUCompressedTensorsWNA16"]
WNA16_SUPPORTED_TYPES_MAP = {
4: scalar_types.uint4b8,
8: scalar_types.uint8b128
@@ -338,3 +338,52 @@ class CompressedTensorsWNA16(CompressedTensorsLinearScheme):
is_k_full=self.is_k_full,
bias=bias,
)
class XPUCompressedTensorsWNA16(CompressedTensorsWNA16):
"""WNA16 on Intel XPU: Using torch's int4pack op."""
def __init__(self,
strategy: str,
num_bits: int,
group_size: Optional[int] = None,
symmetric: Optional[bool] = True,
actorder: Optional[ActivationOrdering] = None):
super().__init__(strategy, num_bits, group_size, symmetric, actorder)
from sglang.srt.hardware_backend.xpu.quantization.compressed_tensors_wna16_kernels import (
CompressedTensorsWNA16XPULinearKernel,
)
from sglang.srt.hardware_backend.xpu.quantization.int4pack_utils import (
SUPPORTED_GROUP_SIZES,
)
# _weight_int4pack_mm_with_scales_and_zeros cannot express any of these,
# so reject at construction rather than after the weights are loaded.
if num_bits != 4:
raise NotImplementedError(
f"compressed-tensors WNA16 on XPU supports 4-bit weights only, "
f"got num_bits={num_bits}.")
if self.group_size not in SUPPORTED_GROUP_SIZES:
raise NotImplementedError(
f"compressed-tensors WNA16 on XPU requires group_size in "
f"{SUPPORTED_GROUP_SIZES}, got {self.group_size} "
"(channelwise/-1 is out of scope).")
if not self.symmetric:
raise NotImplementedError(
"compressed-tensors WNA16 on XPU only supports symmetric weight "
"quantization; this checkpoint carries a weight zero-point.")
if self.has_g_idx:
raise NotImplementedError(
"compressed-tensors WNA16 on XPU does not support activation "
"reordering (actorder=group).")
self.kernel = CompressedTensorsWNA16XPULinearKernel(
group_size=self.group_size)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
self.kernel.process_weights_after_loading(layer)
def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor,
bias: Optional[torch.Tensor]) -> torch.Tensor:
return self.kernel.apply(layer, x, bias)