[Perf] [XPU] fp8: Optimize FP8 scale layouts (#40052)
Co-authored-by: Ma Mingfei <mingfei.ma@intel.com>
This commit is contained in:
+12
-1
@@ -24,15 +24,17 @@ from sglang.srt.layers.quantization.fp8_utils import (
|
||||
deepgemm_w8a8_block_fp8_linear_with_fallback,
|
||||
dispatch_w8a8_block_fp8_linear,
|
||||
normalize_e4m3fn_to_e4m3fnuz,
|
||||
prepare_xpu_block_scale_for_scaled_mm,
|
||||
requant_block_scale_ue8m0_for_deepgemm,
|
||||
validate_fp8_block_shape,
|
||||
)
|
||||
from sglang.srt.layers.quantization.utils import requantize_with_max_scale
|
||||
from sglang.srt.utils import get_bool_env_var, is_hip
|
||||
from sglang.srt.utils import get_bool_env_var, is_hip, is_xpu
|
||||
|
||||
__all__ = ["CompressedTensorsW8A8Fp8"]
|
||||
|
||||
_is_hip = is_hip()
|
||||
_is_xpu = is_xpu()
|
||||
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip
|
||||
if _use_aiter:
|
||||
from aiter.ops.shuffle import shuffle_weight
|
||||
@@ -215,6 +217,15 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsLinearScheme):
|
||||
output_dtype=getattr(layer, "orig_dtype", None),
|
||||
weight_shape=layer.weight.shape,
|
||||
)
|
||||
if _is_xpu:
|
||||
scale_reordered = prepare_xpu_block_scale_for_scaled_mm(
|
||||
layer.weight_scale.data,
|
||||
self.weight_block_size,
|
||||
self.w8a8_block_fp8_linear,
|
||||
)
|
||||
if scale_reordered is not layer.weight_scale.data:
|
||||
with torch.no_grad():
|
||||
layer.weight_scale.set_(scale_reordered)
|
||||
|
||||
else:
|
||||
raise ValueError(f"Unknown quantization strategy {self.strategy}")
|
||||
|
||||
@@ -69,10 +69,10 @@ from sglang.srt.layers.quantization.fp8_utils import (
|
||||
input_to_float8,
|
||||
mxfp8_group_quantize,
|
||||
normalize_e4m3fn_to_e4m3fnuz,
|
||||
prepare_xpu_block_scale_for_scaled_mm,
|
||||
requant_block_scale_ue8m0_for_deepgemm,
|
||||
resolve_block_fp8_mxfp8_backend,
|
||||
resolve_mxfp8_dense_gemm_backend,
|
||||
torch_w8a8_block_fp8_linear,
|
||||
unshuffle_aiter_fp8_weight,
|
||||
use_aiter_bpreshuffle_gemm,
|
||||
)
|
||||
@@ -128,6 +128,7 @@ _is_musa = is_musa()
|
||||
_is_npu = is_npu()
|
||||
_is_cpu_amx_available = cpu_has_amx_support()
|
||||
_is_cpu = is_cpu()
|
||||
_is_xpu = is_xpu()
|
||||
_is_fp8_fnuz = is_fp8_fnuz()
|
||||
_is_gfx95_supported = is_gfx95_supported()
|
||||
|
||||
@@ -907,26 +908,13 @@ class Fp8LinearMethod(LinearMethodBase):
|
||||
layer.weight.is_shuffled = True
|
||||
self._maybe_prepare_dense_mxscale(layer)
|
||||
|
||||
if (
|
||||
is_xpu()
|
||||
and self.w8a8_block_fp8_linear is torch_w8a8_block_fp8_linear
|
||||
and self.weight_block_size in ([1, 128], [128, 128])
|
||||
and layer.weight_scale_inv.ndim == 2
|
||||
):
|
||||
# Keep the checkpoint's logical [N-blocks, K-blocks] shape, but use
|
||||
# transpose-contiguous storage. For [1, 128], scaled_mm transposes
|
||||
# scale_b internally; for [128, 128], the wrapper passes scale_b.t().
|
||||
# This avoids a per-forward contiguous/copy in either path.
|
||||
scale = layer.weight_scale_inv.data
|
||||
scale_b_is_contiguous = scale.t().is_contiguous()
|
||||
if not scale_b_is_contiguous:
|
||||
scale_reordered = torch.empty_strided(
|
||||
scale.shape,
|
||||
(1, scale.shape[0]),
|
||||
dtype=scale.dtype,
|
||||
device=scale.device,
|
||||
)
|
||||
scale_reordered.copy_(scale)
|
||||
if _is_xpu:
|
||||
scale_reordered = prepare_xpu_block_scale_for_scaled_mm(
|
||||
layer.weight_scale_inv.data,
|
||||
self.weight_block_size,
|
||||
self.w8a8_block_fp8_linear,
|
||||
)
|
||||
if scale_reordered is not layer.weight_scale_inv.data:
|
||||
with torch.no_grad():
|
||||
layer.weight_scale_inv.set_(scale_reordered)
|
||||
|
||||
@@ -1112,6 +1100,7 @@ class Fp8LinearMethod(LinearMethodBase):
|
||||
if not self.is_checkpoint_fp8_serialized:
|
||||
if (
|
||||
self.cutlass_fp8_supported
|
||||
or _is_xpu
|
||||
or self.use_marlin
|
||||
or (_use_aiter and self.use_aiter_fp8_per_token)
|
||||
):
|
||||
@@ -1165,6 +1154,7 @@ class Fp8LinearMethod(LinearMethodBase):
|
||||
# cutlass sgl-kernel and marlin only support per-channel scale; aiter supports per-channel scale
|
||||
if (
|
||||
(self.cutlass_fp8_supported and not use_sm120_fp8_pertensor)
|
||||
or _is_xpu
|
||||
or self.use_marlin
|
||||
or (_use_aiter and self.use_aiter_fp8_per_token)
|
||||
):
|
||||
@@ -1229,6 +1219,23 @@ class Fp8LinearMethod(LinearMethodBase):
|
||||
layer.input_scale.max(), requires_grad=False
|
||||
)
|
||||
|
||||
if _is_xpu:
|
||||
# A scale is row-wise iff it carries exactly one value per
|
||||
# output channel, regardless of shape ([N], [1, N], [N, 1], ...).
|
||||
num_channels = sum(layer.logical_widths)
|
||||
is_rowwise = layer.weight_scale.numel() == num_channels
|
||||
if is_rowwise and (
|
||||
layer.weight_scale.shape != (num_channels, 1)
|
||||
or not layer.weight_scale.t().is_contiguous()
|
||||
):
|
||||
with torch.no_grad():
|
||||
layer.weight_scale.set_(
|
||||
layer.weight_scale.data.reshape(
|
||||
num_channels, 1
|
||||
).contiguous()
|
||||
)
|
||||
self.use_per_token_if_dynamic = is_rowwise
|
||||
|
||||
if _is_cpu:
|
||||
assert _is_cpu_amx_available, (
|
||||
"Fp8LinearMethod on CPU requires that CPU has AMX support"
|
||||
@@ -3042,7 +3049,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
if quant_info is not None:
|
||||
return self.runner.run(dispatch_output, quant_info)
|
||||
|
||||
if is_xpu() and not get_moe_runner_backend().is_triton():
|
||||
if _is_xpu and not get_moe_runner_backend().is_triton():
|
||||
# sgl-kernel-xpu path
|
||||
from sgl_kernel import fused_experts
|
||||
|
||||
|
||||
@@ -384,6 +384,9 @@ def use_rowwise_torch_scaled_mm():
|
||||
# The condition is determined once as the operations
|
||||
# are time consuming.
|
||||
return get_device_capability() >= (9, 4) and torch_release >= (2, 7)
|
||||
if _is_xpu:
|
||||
# XPU's torch._scaled_mm backend supports rowwise scaling.
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
@@ -789,6 +792,35 @@ def torch_w8a8_block_fp8_linear(
|
||||
return output.view(*input.shape[:-1], weight.shape[0])
|
||||
|
||||
|
||||
def prepare_xpu_block_scale_for_scaled_mm(
|
||||
weight_scale: torch.Tensor,
|
||||
block_size: List[int],
|
||||
linear_fn: Callable,
|
||||
) -> torch.Tensor:
|
||||
"""Materialize the XPU block scale layout consumed by scaled_mm once."""
|
||||
# Keep logical [N-blocks, K-blocks] shape, but use
|
||||
# transpose-contiguous storage. For [1, 128], scaled_mm transposes
|
||||
# scale_b internally; for [128, 128], the wrapper passes scale_b.t().
|
||||
# This avoids a per-forward contiguous/copy in either path.
|
||||
if (
|
||||
linear_fn is not torch_w8a8_block_fp8_linear
|
||||
or not isinstance(block_size, (list, tuple))
|
||||
or tuple(block_size) not in ((1, 128), (128, 128))
|
||||
or weight_scale.ndim != 2
|
||||
or weight_scale.t().is_contiguous()
|
||||
):
|
||||
return weight_scale
|
||||
|
||||
reordered = torch.empty_strided(
|
||||
weight_scale.shape,
|
||||
(1, weight_scale.shape[0]),
|
||||
dtype=weight_scale.dtype,
|
||||
device=weight_scale.device,
|
||||
)
|
||||
reordered.copy_(weight_scale)
|
||||
return reordered
|
||||
|
||||
|
||||
def resolve_mxfp8_dense_gemm_backend() -> Mxfp8DenseGemmBackend:
|
||||
"""Pick the MXFP8 dense linear backend, honoring `--fp8-gemm-backend` only when it
|
||||
names a backend that owns an MXFP8 dense kernel."""
|
||||
@@ -2334,7 +2366,9 @@ def _apply_fallback_scaled_mm(
|
||||
)
|
||||
|
||||
output = _process_scaled_mm_output(output, input_2d_shape, output_shape)
|
||||
x_scale = torch.narrow(x_scale, 0, 0, input_2d_shape[0])
|
||||
# A single scale has no row dimension to trim, regardless of its shape.
|
||||
if x_scale.numel() > 1:
|
||||
x_scale = torch.narrow(x_scale, 0, 0, input_2d_shape[0])
|
||||
|
||||
output = output * x_scale * weight_scale.t()
|
||||
if bias is not None:
|
||||
@@ -2558,12 +2592,22 @@ def apply_fp8_linear(
|
||||
# When the number of token is 1,
|
||||
# per-token scale has shape (1, 1), per-tensor scale has shape (1) or ().
|
||||
per_tensor_activations = (x_scale.numel() == 1) and x_scale.dim() < 2
|
||||
|
||||
rowwise_scale_layout_supported = not _is_xpu or (
|
||||
x_scale.ndim == 2
|
||||
and x_scale.shape[1] == 1
|
||||
and x_scale.is_contiguous()
|
||||
and weight_scale.ndim == 2
|
||||
and weight_scale.shape[1] == 1
|
||||
and weight_scale.t().is_contiguous()
|
||||
)
|
||||
if (
|
||||
use_per_token_if_dynamic
|
||||
and not per_tensor_weights
|
||||
and not per_tensor_activations
|
||||
and (USE_ROWWISE_TORCH_SCALED_MM or _use_aiter)
|
||||
and (
|
||||
(USE_ROWWISE_TORCH_SCALED_MM and rowwise_scale_layout_supported)
|
||||
or _use_aiter
|
||||
)
|
||||
):
|
||||
# into this sector means use dynamic per-token-per-channel quant
|
||||
# per-token scale quant for input matrix, every row(one token) have one scale factor
|
||||
|
||||
Reference in New Issue
Block a user