[Perf] [XPU] fp8: Optimize FP8 scale layouts (#40052)

Co-authored-by: Ma Mingfei <mingfei.ma@intel.com>
This commit is contained in:
Cao E
2026-10-20 11:24:16 +08:00
committed by GitHub
co-authored by Ma Mingfei
parent 3eb4bd598f
commit 3c07ff2654
4 changed files with 226 additions and 26 deletions
@@ -24,15 +24,17 @@ from sglang.srt.layers.quantization.fp8_utils import (
deepgemm_w8a8_block_fp8_linear_with_fallback,
dispatch_w8a8_block_fp8_linear,
normalize_e4m3fn_to_e4m3fnuz,
prepare_xpu_block_scale_for_scaled_mm,
requant_block_scale_ue8m0_for_deepgemm,
validate_fp8_block_shape,
)
from sglang.srt.layers.quantization.utils import requantize_with_max_scale
from sglang.srt.utils import get_bool_env_var, is_hip
from sglang.srt.utils import get_bool_env_var, is_hip, is_xpu
__all__ = ["CompressedTensorsW8A8Fp8"]
_is_hip = is_hip()
_is_xpu = is_xpu()
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip
if _use_aiter:
from aiter.ops.shuffle import shuffle_weight
@@ -215,6 +217,15 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsLinearScheme):
output_dtype=getattr(layer, "orig_dtype", None),
weight_shape=layer.weight.shape,
)
if _is_xpu:
scale_reordered = prepare_xpu_block_scale_for_scaled_mm(
layer.weight_scale.data,
self.weight_block_size,
self.w8a8_block_fp8_linear,
)
if scale_reordered is not layer.weight_scale.data:
with torch.no_grad():
layer.weight_scale.set_(scale_reordered)
else:
raise ValueError(f"Unknown quantization strategy {self.strategy}")
+29 -22
View File
@@ -69,10 +69,10 @@ from sglang.srt.layers.quantization.fp8_utils import (
input_to_float8,
mxfp8_group_quantize,
normalize_e4m3fn_to_e4m3fnuz,
prepare_xpu_block_scale_for_scaled_mm,
requant_block_scale_ue8m0_for_deepgemm,
resolve_block_fp8_mxfp8_backend,
resolve_mxfp8_dense_gemm_backend,
torch_w8a8_block_fp8_linear,
unshuffle_aiter_fp8_weight,
use_aiter_bpreshuffle_gemm,
)
@@ -128,6 +128,7 @@ _is_musa = is_musa()
_is_npu = is_npu()
_is_cpu_amx_available = cpu_has_amx_support()
_is_cpu = is_cpu()
_is_xpu = is_xpu()
_is_fp8_fnuz = is_fp8_fnuz()
_is_gfx95_supported = is_gfx95_supported()
@@ -907,26 +908,13 @@ class Fp8LinearMethod(LinearMethodBase):
layer.weight.is_shuffled = True
self._maybe_prepare_dense_mxscale(layer)
if (
is_xpu()
and self.w8a8_block_fp8_linear is torch_w8a8_block_fp8_linear
and self.weight_block_size in ([1, 128], [128, 128])
and layer.weight_scale_inv.ndim == 2
):
# Keep the checkpoint's logical [N-blocks, K-blocks] shape, but use
# transpose-contiguous storage. For [1, 128], scaled_mm transposes
# scale_b internally; for [128, 128], the wrapper passes scale_b.t().
# This avoids a per-forward contiguous/copy in either path.
scale = layer.weight_scale_inv.data
scale_b_is_contiguous = scale.t().is_contiguous()
if not scale_b_is_contiguous:
scale_reordered = torch.empty_strided(
scale.shape,
(1, scale.shape[0]),
dtype=scale.dtype,
device=scale.device,
)
scale_reordered.copy_(scale)
if _is_xpu:
scale_reordered = prepare_xpu_block_scale_for_scaled_mm(
layer.weight_scale_inv.data,
self.weight_block_size,
self.w8a8_block_fp8_linear,
)
if scale_reordered is not layer.weight_scale_inv.data:
with torch.no_grad():
layer.weight_scale_inv.set_(scale_reordered)
@@ -1112,6 +1100,7 @@ class Fp8LinearMethod(LinearMethodBase):
if not self.is_checkpoint_fp8_serialized:
if (
self.cutlass_fp8_supported
or _is_xpu
or self.use_marlin
or (_use_aiter and self.use_aiter_fp8_per_token)
):
@@ -1165,6 +1154,7 @@ class Fp8LinearMethod(LinearMethodBase):
# cutlass sgl-kernel and marlin only support per-channel scale; aiter supports per-channel scale
if (
(self.cutlass_fp8_supported and not use_sm120_fp8_pertensor)
or _is_xpu
or self.use_marlin
or (_use_aiter and self.use_aiter_fp8_per_token)
):
@@ -1229,6 +1219,23 @@ class Fp8LinearMethod(LinearMethodBase):
layer.input_scale.max(), requires_grad=False
)
if _is_xpu:
# A scale is row-wise iff it carries exactly one value per
# output channel, regardless of shape ([N], [1, N], [N, 1], ...).
num_channels = sum(layer.logical_widths)
is_rowwise = layer.weight_scale.numel() == num_channels
if is_rowwise and (
layer.weight_scale.shape != (num_channels, 1)
or not layer.weight_scale.t().is_contiguous()
):
with torch.no_grad():
layer.weight_scale.set_(
layer.weight_scale.data.reshape(
num_channels, 1
).contiguous()
)
self.use_per_token_if_dynamic = is_rowwise
if _is_cpu:
assert _is_cpu_amx_available, (
"Fp8LinearMethod on CPU requires that CPU has AMX support"
@@ -3042,7 +3049,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
if quant_info is not None:
return self.runner.run(dispatch_output, quant_info)
if is_xpu() and not get_moe_runner_backend().is_triton():
if _is_xpu and not get_moe_runner_backend().is_triton():
# sgl-kernel-xpu path
from sgl_kernel import fused_experts
@@ -384,6 +384,9 @@ def use_rowwise_torch_scaled_mm():
# The condition is determined once as the operations
# are time consuming.
return get_device_capability() >= (9, 4) and torch_release >= (2, 7)
if _is_xpu:
# XPU's torch._scaled_mm backend supports rowwise scaling.
return True
return False
@@ -789,6 +792,35 @@ def torch_w8a8_block_fp8_linear(
return output.view(*input.shape[:-1], weight.shape[0])
def prepare_xpu_block_scale_for_scaled_mm(
weight_scale: torch.Tensor,
block_size: List[int],
linear_fn: Callable,
) -> torch.Tensor:
"""Materialize the XPU block scale layout consumed by scaled_mm once."""
# Keep logical [N-blocks, K-blocks] shape, but use
# transpose-contiguous storage. For [1, 128], scaled_mm transposes
# scale_b internally; for [128, 128], the wrapper passes scale_b.t().
# This avoids a per-forward contiguous/copy in either path.
if (
linear_fn is not torch_w8a8_block_fp8_linear
or not isinstance(block_size, (list, tuple))
or tuple(block_size) not in ((1, 128), (128, 128))
or weight_scale.ndim != 2
or weight_scale.t().is_contiguous()
):
return weight_scale
reordered = torch.empty_strided(
weight_scale.shape,
(1, weight_scale.shape[0]),
dtype=weight_scale.dtype,
device=weight_scale.device,
)
reordered.copy_(weight_scale)
return reordered
def resolve_mxfp8_dense_gemm_backend() -> Mxfp8DenseGemmBackend:
"""Pick the MXFP8 dense linear backend, honoring `--fp8-gemm-backend` only when it
names a backend that owns an MXFP8 dense kernel."""
@@ -2334,7 +2366,9 @@ def _apply_fallback_scaled_mm(
)
output = _process_scaled_mm_output(output, input_2d_shape, output_shape)
x_scale = torch.narrow(x_scale, 0, 0, input_2d_shape[0])
# A single scale has no row dimension to trim, regardless of its shape.
if x_scale.numel() > 1:
x_scale = torch.narrow(x_scale, 0, 0, input_2d_shape[0])
output = output * x_scale * weight_scale.t()
if bias is not None:
@@ -2558,12 +2592,22 @@ def apply_fp8_linear(
# When the number of token is 1,
# per-token scale has shape (1, 1), per-tensor scale has shape (1) or ().
per_tensor_activations = (x_scale.numel() == 1) and x_scale.dim() < 2
rowwise_scale_layout_supported = not _is_xpu or (
x_scale.ndim == 2
and x_scale.shape[1] == 1
and x_scale.is_contiguous()
and weight_scale.ndim == 2
and weight_scale.shape[1] == 1
and weight_scale.t().is_contiguous()
)
if (
use_per_token_if_dynamic
and not per_tensor_weights
and not per_tensor_activations
and (USE_ROWWISE_TORCH_SCALED_MM or _use_aiter)
and (
(USE_ROWWISE_TORCH_SCALED_MM and rowwise_scale_layout_supported)
or _use_aiter
)
):
# into this sector means use dynamic per-token-per-channel quant
# per-token scale quant for input matrix, every row(one token) have one scale factor