[XPU] Support compressed-tensors W4A16 by reusing the torch int4pack path (#40828)
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Co-authored-by: Singh <rohitsi2@iil-gnrap01.iind.intel.com> Co-authored-by: Singh <rohitsi2@iil-login.iind.intel.com> Co-authored-by: Ma Mingfei <mingfei.ma@intel.com>
This commit is contained in:
co-authored by
github-actions[bot]
Singh
Singh
Ma Mingfei
parent
5781206833
commit
9ae43b0eee
+73
@@ -0,0 +1,73 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""compressed-tensors WNA16 int4 dense linear for Intel XPU."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.hardware_backend.xpu.quantization.int4pack_utils import (
|
||||
pack_int4_to_uint8,
|
||||
unpack_compressed_tensors_qweight,
|
||||
xpu_int4pack_mm,
|
||||
)
|
||||
from sglang.srt.layers.quantization.utils import replace_parameter
|
||||
|
||||
# Symmetric WNA16 stores each int4 code biased by 8 (``uint4b8``). Marlin folds
|
||||
# that bias into its scales; the native XPU op takes an explicit zero-point.
|
||||
_UINT4B8_ZERO_POINT = 8
|
||||
|
||||
|
||||
class CompressedTensorsWNA16XPULinearKernel:
|
||||
# 4-bit symmetric, group-quantized, no act-order; XPUCompressedTensorsWNA16
|
||||
# rejects everything else before this kernel is constructed.
|
||||
def __init__(self, group_size: int):
|
||||
self.group_size = group_size
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
weight_packed = layer.weight_packed.data # [N, K // 8] int32
|
||||
scales = layer.weight_scale.data # [N, K // gs]
|
||||
|
||||
codes = unpack_compressed_tensors_qweight(weight_packed) # [N, K]
|
||||
qweight_uint8 = pack_int4_to_uint8(codes) # [N, K // 2]
|
||||
qweight_packed = torch.ops.aten._convert_weight_to_int4pack(qweight_uint8, 8)
|
||||
|
||||
# The native op wants group-major scales and zero-points: [K // gs, N].
|
||||
xpu_scales = scales.t().contiguous()
|
||||
zero_points = torch.full(
|
||||
xpu_scales.shape,
|
||||
_UINT4B8_ZERO_POINT,
|
||||
dtype=torch.int8,
|
||||
device=xpu_scales.device,
|
||||
)
|
||||
|
||||
replace_parameter(layer, "weight_packed", qweight_packed)
|
||||
layer.register_parameter(
|
||||
"xpu_scales",
|
||||
torch.nn.Parameter(xpu_scales, requires_grad=False),
|
||||
)
|
||||
layer.register_parameter(
|
||||
"xpu_zero_points",
|
||||
torch.nn.Parameter(zero_points, requires_grad=False),
|
||||
)
|
||||
del layer.weight_scale
|
||||
|
||||
layer.xpu_out_features = codes.shape[0]
|
||||
layer.xpu_group_size = self.group_size
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
return xpu_int4pack_mm(
|
||||
x=x,
|
||||
qweight_packed=layer.weight_packed,
|
||||
group_size=layer.xpu_group_size,
|
||||
scales=layer.xpu_scales,
|
||||
zero_points=layer.xpu_zero_points,
|
||||
out_features=layer.xpu_out_features,
|
||||
bias=bias,
|
||||
)
|
||||
@@ -51,6 +51,18 @@ def unpack_gptq_qweight(qweight: torch.Tensor) -> torch.Tensor:
|
||||
return codes.reshape(-1, n) # [K, N]
|
||||
|
||||
|
||||
def unpack_compressed_tensors_qweight(weight_packed: torch.Tensor) -> torch.Tensor:
|
||||
"""``[N, K // 8]`` int32 packed along K -> ``[N, K]`` codes in ``[0, 15]``.
|
||||
|
||||
``pack_quantized`` uses natural nibble order, unlike AWQ's interleave.
|
||||
"""
|
||||
n = weight_packed.shape[0]
|
||||
shifts = _nibble_shifts(weight_packed.device) # [8]
|
||||
# [N, K // 8, 8]; sub-index i selects k = col * 8 + i
|
||||
codes = (weight_packed.unsqueeze(-1) >> shifts.view(1, 1, 8)) & 0xF
|
||||
return codes.reshape(n, -1) # [N, K]
|
||||
|
||||
|
||||
def unpack_gptq_qzeros(qzeros: torch.Tensor) -> torch.Tensor:
|
||||
"""``[K // gs, N // 8]`` int32 packed along N -> ``[K // gs, N]`` codes."""
|
||||
rows = qzeros.shape[0]
|
||||
|
||||
@@ -57,6 +57,7 @@ from sglang.srt.layers.quantization.compressed_tensors.schemes import (
|
||||
NPUCompressedTensorsW4A16Int4DynamicMoE,
|
||||
NPUCompressedTensorsW8A8Int8,
|
||||
NPUCompressedTensorsW8A8Int8DynamicMoE,
|
||||
XPUCompressedTensorsWNA16,
|
||||
)
|
||||
from sglang.srt.layers.quantization.compressed_tensors.utils import (
|
||||
check_equal_or_regex_match,
|
||||
@@ -699,7 +700,10 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
quant_format == CompressionFormat.pack_quantized.value
|
||||
and weight_quant.num_bits in WNA16_SUPPORTED_BITS
|
||||
):
|
||||
return CompressedTensorsWNA16(
|
||||
scheme_cls = (
|
||||
XPUCompressedTensorsWNA16 if _is_xpu else CompressedTensorsWNA16
|
||||
)
|
||||
return scheme_cls(
|
||||
num_bits=weight_quant.num_bits,
|
||||
strategy=weight_quant.strategy,
|
||||
group_size=weight_quant.group_size,
|
||||
@@ -988,7 +992,9 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
# (e.g. fp8 needs ada lovelace)
|
||||
# Note: NPU devices do not support min_capability function
|
||||
if _is_xpu:
|
||||
if not isinstance(scheme, CompressedTensorsW8A8Fp8):
|
||||
if not isinstance(
|
||||
scheme, (CompressedTensorsW8A8Fp8, XPUCompressedTensorsWNA16)
|
||||
):
|
||||
raise RuntimeError(
|
||||
f"{scheme.__class__.__name__} is not supported on XPU "
|
||||
"(no XPU kernel implementation)."
|
||||
|
||||
@@ -18,7 +18,11 @@ from .compressed_tensors_w8a8_int8 import (
|
||||
)
|
||||
from .compressed_tensors_w8a8_int8_moe import NPUCompressedTensorsW8A8Int8DynamicMoE
|
||||
from .compressed_tensors_w8a16_fp8 import CompressedTensorsW8A16Fp8
|
||||
from .compressed_tensors_wNa16 import WNA16_SUPPORTED_BITS, CompressedTensorsWNA16
|
||||
from .compressed_tensors_wNa16 import (
|
||||
WNA16_SUPPORTED_BITS,
|
||||
CompressedTensorsWNA16,
|
||||
XPUCompressedTensorsWNA16,
|
||||
)
|
||||
from .compressed_tensors_wNa16_moe import (
|
||||
CompressedTensorsWNA16MoE,
|
||||
CompressedTensorsWNA16TritonMoE,
|
||||
@@ -35,6 +39,7 @@ __all__ = [
|
||||
"NPUCompressedTensorsW8A8Int8",
|
||||
"NPUCompressedTensorsW8A8Int8DynamicMoE",
|
||||
"CompressedTensorsWNA16",
|
||||
"XPUCompressedTensorsWNA16",
|
||||
"CompressedTensorsWNA16MoE",
|
||||
"CompressedTensorsWNA16TritonMoE",
|
||||
"NPUCompressedTensorsW4A16Int4DynamicMoE",
|
||||
|
||||
+50
-1
@@ -51,7 +51,7 @@ ScalarType, scalar_types = get_scalar_types()
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
__all__ = ["CompressedTensorsWNA16"]
|
||||
__all__ = ["CompressedTensorsWNA16", "XPUCompressedTensorsWNA16"]
|
||||
WNA16_SUPPORTED_TYPES_MAP = {
|
||||
4: scalar_types.uint4b8,
|
||||
8: scalar_types.uint8b128
|
||||
@@ -338,3 +338,52 @@ class CompressedTensorsWNA16(CompressedTensorsLinearScheme):
|
||||
is_k_full=self.is_k_full,
|
||||
bias=bias,
|
||||
)
|
||||
|
||||
|
||||
class XPUCompressedTensorsWNA16(CompressedTensorsWNA16):
|
||||
"""WNA16 on Intel XPU: Using torch's int4pack op."""
|
||||
|
||||
def __init__(self,
|
||||
strategy: str,
|
||||
num_bits: int,
|
||||
group_size: Optional[int] = None,
|
||||
symmetric: Optional[bool] = True,
|
||||
actorder: Optional[ActivationOrdering] = None):
|
||||
super().__init__(strategy, num_bits, group_size, symmetric, actorder)
|
||||
|
||||
from sglang.srt.hardware_backend.xpu.quantization.compressed_tensors_wna16_kernels import (
|
||||
CompressedTensorsWNA16XPULinearKernel,
|
||||
)
|
||||
from sglang.srt.hardware_backend.xpu.quantization.int4pack_utils import (
|
||||
SUPPORTED_GROUP_SIZES,
|
||||
)
|
||||
|
||||
# _weight_int4pack_mm_with_scales_and_zeros cannot express any of these,
|
||||
# so reject at construction rather than after the weights are loaded.
|
||||
if num_bits != 4:
|
||||
raise NotImplementedError(
|
||||
f"compressed-tensors WNA16 on XPU supports 4-bit weights only, "
|
||||
f"got num_bits={num_bits}.")
|
||||
if self.group_size not in SUPPORTED_GROUP_SIZES:
|
||||
raise NotImplementedError(
|
||||
f"compressed-tensors WNA16 on XPU requires group_size in "
|
||||
f"{SUPPORTED_GROUP_SIZES}, got {self.group_size} "
|
||||
"(channelwise/-1 is out of scope).")
|
||||
if not self.symmetric:
|
||||
raise NotImplementedError(
|
||||
"compressed-tensors WNA16 on XPU only supports symmetric weight "
|
||||
"quantization; this checkpoint carries a weight zero-point.")
|
||||
if self.has_g_idx:
|
||||
raise NotImplementedError(
|
||||
"compressed-tensors WNA16 on XPU does not support activation "
|
||||
"reordering (actorder=group).")
|
||||
|
||||
self.kernel = CompressedTensorsWNA16XPULinearKernel(
|
||||
group_size=self.group_size)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
self.kernel.process_weights_after_loading(layer)
|
||||
|
||||
def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor,
|
||||
bias: Optional[torch.Tensor]) -> torch.Tensor:
|
||||
return self.kernel.apply(layer, x, bias)
|
||||
|
||||
Reference in New Issue
Block a user