From 6e98c84c601a6a18405403c12ce047f167a054b4 Mon Sep 17 00:00:00 2001 From: vorapolsiloai <115975949+vorapolsiloai@users.noreply.github.com> Date: Wed, 21 Oct 2026 19:37:47 +0300 Subject: [PATCH] [Perf] [AMD] quark: Add opt-in AITER ASM FP4 GEMM for Quark dense linear (#39598) --- python/sglang/srt/environ.py | 3 + .../quark/schemes/quark_w4a4_mxfp4.py | 204 ++++++++++++++++++ .../quantization/test_quark_mxfp4_asm.py | 121 +++++++++++ 3 files changed, 328 insertions(+) create mode 100644 test/registered/unit/layers/quantization/test_quark_mxfp4_asm.py diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 29f788db03..66fcd0fa79 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -979,6 +979,9 @@ class Envs: SGLANG_ROCM_SMALLM_ROUTER = EnvBool(True) SGLANG_ROCM_SMALLM_MOE = EnvBool(True) SGLANG_ROCM_SMALLM_FP8_PROJ = EnvBool(True) + # Quark MXFP4 dense linear on gfx95: use AITER's preshuffled ASM/CK FP4 GEMM + # (shared a4w4 tuned CSVs) instead of the Triton gemm_afp4wfp4 path. + SGLANG_ROCM_USE_AITER_FP4_ASM_GEMM = EnvBool(False) # Fold the KDA [f_a|b] tail into the wide [q,k,v,g] projection so the whole # in-proj is one GEMM. Decode is bandwidth bound there, so the 144 extra # output columns ride along nearly free. diff --git a/python/sglang/srt/layers/quantization/quark/schemes/quark_w4a4_mxfp4.py b/python/sglang/srt/layers/quantization/quark/schemes/quark_w4a4_mxfp4.py index f860a3fbe9..91689ca7a7 100644 --- a/python/sglang/srt/layers/quantization/quark/schemes/quark_w4a4_mxfp4.py +++ b/python/sglang/srt/layers/quantization/quark/schemes/quark_w4a4_mxfp4.py @@ -6,6 +6,7 @@ from typing import Any, Callable, Optional import torch +from sglang.srt.environ import envs from sglang.srt.layers.parameter import ( GroupQuantScaleParameter, ModelWeightParameter, @@ -29,6 +30,82 @@ NVFP4_BLOCK_SIZE = 16 _is_hip = is_hip() +_ASM_FP4_SCALE_ROW_MULTIPLE = 32 +_ASM_FP4_SCALE_COL_MULTIPLE = 8 +# Workaround: gemm_a4w4's untuned ASM heuristic returns wrong rows 1..M-1 for +# N <= 128 at 2 <= M <= 64 (measured on ROCm/aiter@e7d2453f2, MI355X). +# TODO(vorapolsiloai): link the AITER issue; drop once the heuristic is fixed. +_ASM_FP4_UNSAFE_MAX_N = 128 + +# Keep this path opt-in while the AITER ASM integration is validated across the +# Quark model matrix. Unlike the Triton fallback, it consumes AITER's +# a4w4_blockscale_tuned_gemm CSVs. +_use_aiter_asm_fp4_gemm = _is_hip and envs.SGLANG_ROCM_USE_AITER_FP4_ASM_GEMM.get() + + +def _asm_fp4_scale_swizzle_supported(weight_scale: torch.Tensor) -> bool: + if weight_scale.ndim != 2: + return False + rows, cols = weight_scale.shape + return ( + rows % _ASM_FP4_SCALE_ROW_MULTIPLE == 0 + and cols % _ASM_FP4_SCALE_COL_MULTIPLE == 0 + ) + + +def _swizzle_asm_fp4_scale(scale: torch.Tensor) -> torch.Tensor: + """Convert a (rows, K/32) E8M0 scale tensor to AITER ASM tile order.""" + rows, cols = scale.shape + return ( + scale.view(rows // 32, 2, 16, cols // 8, 2, 4, 1) + .permute(0, 3, 5, 2, 4, 1, 6) + .contiguous() + .view(rows, cols) + ) + + +# Padding of AITER's per_1x32_f4_quant(shuffle=True) activation scales. +_ASM_FP4_ACT_SCALE_ROW_MULTIPLE = 256 +_E8M0_ONE = 0x7F + + +def _shuffle_asm_fp4_act_scale(x_scales: torch.Tensor) -> torch.Tensor: + """Pad and swizzle row-major (M, K/32) E8M0 activation scales for gemm_a4w4.""" + rows, cols = x_scales.shape + row_tile, col_tile = _ASM_FP4_ACT_SCALE_ROW_MULTIPLE, _ASM_FP4_SCALE_COL_MULTIPLE + # Pad with E8M0 1.0, not garbage: the kernel reads whole tiles, and 0xFF is NaN. + padded = torch.full( + ( + (rows + row_tile - 1) // row_tile * row_tile, + (cols + col_tile - 1) // col_tile * col_tile, + ), + _E8M0_ONE, + dtype=torch.uint8, + device=x_scales.device, + ) + padded[:rows, :cols] = x_scales.view(torch.uint8) + return _swizzle_asm_fp4_scale(padded).view(torch.float8_e8m0fnu) + + +def _asm_fp4_prequantized_input( + x: tuple[torch.Tensor, ...], +) -> tuple[torch.Tensor, torch.Tensor]: + """Adapt an already-quantized ``(x_fp4, x_scales)`` activation for gemm_a4w4.""" + # The 3- and 5-tuples request Triton-only fused epilogues (atomic pre-quant, + # split + cat); weights are already preshuffled, so there is no fallback. + if len(x) != 2: + raise NotImplementedError( + "AITER ASM FP4 GEMM only supports the (x_fp4, x_scales) tuple input; " + f"got a {len(x)}-tuple. Disable SGLANG_ROCM_USE_AITER_FP4_ASM_GEMM " + "for models that use this projection." + ) + x_q, x_scales = x + # Only B is preshuffled (bpreshuffle=True), so the packed FP4 activation is + # consumed as-is. The scales come from the Triton fused-quant kernels in + # row-major (M, K/32) order, which gemm_a4w4 cannot read directly. + return x_q.view(torch.float4_e2m1fn_x2), _shuffle_asm_fp4_act_scale(x_scales) + + # On GPUs that lack the fp4-activation WMMA scale instruction # (V_WMMA_SCALE_F32_32X16X128_F4, e.g. gfx1250) the a4w4 (fp4 x fp4) linear GEMM # cannot run. The MoE path is switched to a8w4 via AITER_FORCE_A8W4=1 (handled @@ -83,6 +160,9 @@ def _dequant_mxfp4_to_bf16( if _is_hip: + from aiter import gemm_a4w4 as _gemm_a4w4_orig + from aiter import per_1x32_f4_quant_hip as _per_1x32_f4_quant_hip_orig + from aiter.ops.shuffle import shuffle_weight from aiter.ops.triton.gemm.fused.fused_gemm_afp4wfp4_split_cat import ( fused_gemm_afp4wfp4_split_cat as _fused_gemm_afp4wfp4_split_cat_orig, ) @@ -171,6 +251,73 @@ if _is_hip: def dynamic_mxfp4_quant(x): return torch.ops.sglang.aiter_dynamic_mxfp4_quant(x) + def _aiter_per_1x32_f4_quant( + x: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + return _per_1x32_f4_quant_hip_orig(x, shuffle=True) + + def _aiter_per_1x32_f4_quant_fake( + x: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + rows, cols = x.shape + x_fp4 = torch.empty((rows, cols // 2), dtype=torch.uint8, device=x.device) + scales = torch.empty( + (rows, (cols + 31) // 32), dtype=torch.uint8, device=x.device + ) + return x_fp4, scales + + direct_register_custom_op( + op_name="aiter_per_1x32_f4_quant", + op_func=_aiter_per_1x32_f4_quant, + mutates_args=[], + fake_impl=_aiter_per_1x32_f4_quant_fake, + ) + + def per_1x32_f4_quant(x): + return torch.ops.sglang.aiter_per_1x32_f4_quant(x) + + def _aiter_gemm_a4w4( + x: torch.Tensor, + w: torch.Tensor, + x_scales: torch.Tensor, + w_scales: torch.Tensor, + output_dtype_ref: torch.Tensor, + ) -> torch.Tensor: + return _gemm_a4w4_orig( + x, + w.view(x.dtype), + x_scales, + w_scales.view(x_scales.dtype), + dtype=output_dtype_ref.dtype, + bpreshuffle=True, + ) + + def _aiter_gemm_a4w4_fake( + x: torch.Tensor, + w: torch.Tensor, + x_scales: torch.Tensor, + w_scales: torch.Tensor, + output_dtype_ref: torch.Tensor, + ) -> torch.Tensor: + del x_scales, w_scales + return torch.empty( + (x.shape[0], w.shape[0]), + dtype=output_dtype_ref.dtype, + device=x.device, + ) + + direct_register_custom_op( + op_name="aiter_gemm_a4w4", + op_func=_aiter_gemm_a4w4, + mutates_args=[], + fake_impl=_aiter_gemm_a4w4_fake, + ) + + def gemm_a4w4(x, w, x_scales, w_scales, output_dtype_ref): + return torch.ops.sglang.aiter_gemm_a4w4( + x, w, x_scales, w_scales, output_dtype_ref + ) + def _aiter_fused_gemm_split_cat( x: torch.Tensor, w: torch.Tensor, @@ -270,6 +417,41 @@ class QuarkW4A4MXFP4(QuarkLinearScheme): # FP4 block scales are folded into the bf16 weight; drop them. layer.weight_scale = None layer.dequantized_bf16 = True + return + + layer.use_aiter_asm_fp4_gemm = False + if not _use_aiter_asm_fp4_gemm or not is_gfx95_supported(): + return + + if not _asm_fp4_scale_swizzle_supported(layer.weight_scale.data): + logger.warning_once( + "AITER ASM FP4 GEMM requires weight-scale dimensions divisible " + "by (%d, %d), but this layer has shape %s. Falling back to the " + "AITER Triton FP4 GEMM for this layer.", + _ASM_FP4_SCALE_ROW_MULTIPLE, + _ASM_FP4_SCALE_COL_MULTIPLE, + tuple(layer.weight_scale.shape), + ) + return + + if layer.weight.shape[0] <= _ASM_FP4_UNSAFE_MAX_N: + logger.warning_once( + "AITER ASM FP4 GEMM is unreliable for N <= %d; keeping the AITER " + "Triton FP4 GEMM for a layer with N=%d.", + _ASM_FP4_UNSAFE_MAX_N, + layer.weight.shape[0], + ) + return + + layer.weight_scale = torch.nn.Parameter( + _swizzle_asm_fp4_scale(layer.weight_scale.data), + requires_grad=False, + ) + layer.weight = torch.nn.Parameter( + shuffle_weight(layer.weight.data, layout=(16, 16)), + requires_grad=False, + ) + layer.use_aiter_asm_fp4_gemm = True def create_weights( self, @@ -689,6 +871,28 @@ class QuarkW4A4MXFP4(QuarkLinearScheme): x = x[0] return torch.nn.functional.linear(x, layer.weight, bias) + if getattr(layer, "use_aiter_asm_fp4_gemm", False): + output_shape = None + if isinstance(x, tuple): + x_q, x_scales = _asm_fp4_prequantized_input(x) + else: + if x.dim() == 3: + output_shape = [*x.shape[:-1], layer.weight.shape[0]] + x = x.view(-1, x.shape[-1]) + x_q, x_scales = per_1x32_f4_quant(x) + + output_dtype_ref = torch.empty(0, dtype=self.out_dtype, device=x_q.device) + y = gemm_a4w4( + x_q, + layer.weight, + x_scales, + layer.weight_scale, + output_dtype_ref, + ) + if bias is not None: + y = y + bias + return y.view(*output_shape) if output_shape is not None else y + # Bias will be added after the GEMM if provided three_d = False fused_gemm_split_cat = False diff --git a/test/registered/unit/layers/quantization/test_quark_mxfp4_asm.py b/test/registered/unit/layers/quantization/test_quark_mxfp4_asm.py new file mode 100644 index 0000000000..dea930ecd0 --- /dev/null +++ b/test/registered/unit/layers/quantization/test_quark_mxfp4_asm.py @@ -0,0 +1,121 @@ +"""CPU-only tests for Quark MXFP4 AITER ASM layout helpers.""" + +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +import unittest +from unittest import mock + +import torch + +from sglang.srt.layers.quantization.quark.schemes import quark_w4a4_mxfp4 +from sglang.srt.layers.quantization.quark.schemes.quark_w4a4_mxfp4 import ( + _asm_fp4_prequantized_input, + _asm_fp4_scale_swizzle_supported, + _shuffle_asm_fp4_act_scale, + _swizzle_asm_fp4_scale, +) +from sglang.test.test_utils import CustomTestCase + + +class TestQuarkMxfp4AsmScaleLayout(CustomTestCase): + def test_supported_shape(self): + scale = torch.empty((32, 8), dtype=torch.uint8) + self.assertTrue(_asm_fp4_scale_swizzle_supported(scale)) + + def test_rejects_unsupported_shapes(self): + self.assertFalse( + _asm_fp4_scale_swizzle_supported(torch.empty((16, 8), dtype=torch.uint8)) + ) + self.assertFalse( + _asm_fp4_scale_swizzle_supported(torch.empty((32, 4), dtype=torch.uint8)) + ) + self.assertFalse( + _asm_fp4_scale_swizzle_supported(torch.empty((1, 32, 8), dtype=torch.uint8)) + ) + + def test_swizzle_matches_aiter_tile_permutation(self): + scale = torch.arange(32 * 8, dtype=torch.int32).view(32, 8) + expected = ( + scale.view(1, 2, 16, 1, 2, 4, 1) + .permute(0, 3, 5, 2, 4, 1, 6) + .contiguous() + .view(32, 8) + ) + actual = _swizzle_asm_fp4_scale(scale) + torch.testing.assert_close(actual, expected) + self.assertEqual(actual.shape, scale.shape) + self.assertTrue(actual.is_contiguous()) + + +class TestQuarkMxfp4AsmTupleInput(CustomTestCase): + def test_act_scale_shuffle_matches_aiter_padded_layout(self): + """Row-major (M, K/32) activation scales from the Triton fused-quant + kernels must land in the (pad256(M), pad8(K/32)) tile order that + per_1x32_f4_quant(shuffle=True) emits, with E8M0 1.0 in the padding.""" + rows, cols = 3, 6 + scale = torch.randint(0, 0x7F, (rows, cols), dtype=torch.uint8) + + shuffled = _shuffle_asm_fp4_act_scale(scale) + + self.assertEqual(shuffled.dtype, torch.float8_e8m0fnu) + self.assertEqual(tuple(shuffled.shape), (256, 8)) + # Invert the AITER tile permutation (0, 3, 5, 2, 4, 1, 6). + unshuffled = ( + shuffled.view(torch.uint8) + .view(256 // 32, 8 // 8, 4, 16, 2, 2, 1) + .permute(0, 5, 3, 1, 4, 2, 6) + .reshape(256, 8) + ) + expected = torch.full((256, 8), 0x7F, dtype=torch.uint8) + expected[:rows, :cols] = scale + torch.testing.assert_close(unshuffled, expected, rtol=0, atol=0) + + def test_only_quantized_pair_is_accepted(self): + rows, k = 4, 64 + x_q = torch.zeros((rows, k // 2), dtype=torch.uint8) + x_scales = torch.full((rows, k // 32), 0x7F, dtype=torch.uint8) + + a, a_scales = _asm_fp4_prequantized_input((x_q, x_scales)) + self.assertEqual(a.dtype, torch.float4_e2m1fn_x2) + self.assertEqual(a.data_ptr(), x_q.data_ptr()) + self.assertEqual(tuple(a_scales.shape), (256, 8)) + + # The 3- and 5-tuples ask for Triton-only fused epilogues; they must not + # be silently unpacked as (x, x_scales). + for fused in ((x_q, x_scales, torch.empty(0)), (x_q,) * 5): + with self.assertRaises(NotImplementedError): + _asm_fp4_prequantized_input(fused) + + +class TestQuarkMxfp4AsmNarrowLayer(CustomTestCase): + def test_narrow_layer_stays_on_triton(self): + """With the ASM flag on, a layer with N <= 128 (Qwen3.8 GDN in_proj_ba, + N=96) must keep the Triton GEMM: AITER's untuned ASM heuristic returns + wrong rows 1..M-1 for it at small batch sizes.""" + n, k = 96, 5120 + layer = torch.nn.Module() + weight = torch.nn.Parameter( + torch.zeros((n, k // 2), dtype=torch.uint8), requires_grad=False + ) + layer.weight = weight + layer.weight_scale = torch.nn.Parameter( + torch.zeros((n, k // 32), dtype=torch.uint8), requires_grad=False + ) + scheme = quark_w4a4_mxfp4.QuarkW4A4MXFP4({}, {}) + + with ( + mock.patch.object(quark_w4a4_mxfp4, "_use_aiter_asm_fp4_gemm", True), + mock.patch.object( + quark_w4a4_mxfp4, "is_gfx95_supported", return_value=True + ), + ): + scheme.process_weights_after_loading(layer) + + self.assertFalse(layer.use_aiter_asm_fp4_gemm) + self.assertIs(layer.weight, weight) + + +if __name__ == "__main__": + unittest.main()