From 533020b500ba781bfafd1f5cbadabaf03d8986f7 Mon Sep 17 00:00:00 2001 From: kk <43161300+kkHuang-amd@users.noreply.github.com> Date: Wed, 21 Oct 2026 07:19:32 +0800 Subject: [PATCH] [Perf] [AMD] dsv41: run gfx950 MXFP8 dense GEMMs on aiter's preshuffled FlyDSL kernel (#45123) Co-authored-by: wunhuang --- .../kernels/ops/gemm/mxfp8_aiter_gfx95.py | 30 +++++++++++++++++-- .../speculative/dspark/dspark_draft_model.py | 4 +++ python/sglang/srt/environ.py | 3 ++ .../sglang/srt/layers/quantization/fp8_hip.py | 16 ++++++++++ .../kernels/ops/gemm/test_mxfp8_amd_gfx95.py | 27 +++++++++++++++-- 5 files changed, 76 insertions(+), 4 deletions(-) diff --git a/python/sglang/kernels/ops/gemm/mxfp8_aiter_gfx95.py b/python/sglang/kernels/ops/gemm/mxfp8_aiter_gfx95.py index 0474036caa..d3dac69fbf 100644 --- a/python/sglang/kernels/ops/gemm/mxfp8_aiter_gfx95.py +++ b/python/sglang/kernels/ops/gemm/mxfp8_aiter_gfx95.py @@ -5,12 +5,31 @@ ue8m0 [M, K/32]; aiter picks the kernel config from its tuned MXFP8 tables.""" from __future__ import annotations +import importlib.util +from functools import cache from typing import Optional import torch from aiter import gemm_a8w8_blockscale from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import mxfp8_operand +from sglang.srt.environ import envs + + +@cache +def _aiter_has_preshuffled_mxfp8() -> bool: + # gemm_a8w8_blockscale forwards preshuffled group32 operands to FlyDSL since ROCm/aiter#5896 + return importlib.util.find_spec("aiter.ops.flydsl.batched_gemm_a8w8") is not None + + +def aiter_preshuffle_supported(n: int, k: int) -> bool: + """Whether an [N, K] weight of the aiter route is kept (16, 16)-preshuffled.""" + return ( + envs.SGLANG_ROCM_MXFP8_AITER_PRESHUFFLE.get() + and n % 16 == 0 + and k % 32 == 0 + and _aiter_has_preshuffled_mxfp8() + ) def aiter_mxfp8_blockscaled_linear( @@ -19,9 +38,11 @@ def aiter_mxfp8_blockscaled_linear( weight_scale_ue8m0: torch.Tensor, input_scale: Optional[torch.Tensor] = None, bias: Optional[torch.Tensor] = None, + weight_preshuffled: bool = False, ) -> torch.Tensor: """Dense linear of the aiter route. input is bf16 (plain or on the fp8 grid, where the per-32 - requant is exact), or fp8 e4m3 with input_scale ue8m0 [M, K/32].""" + requant is exact), or fp8 e4m3 with input_scale ue8m0 [M, K/32]. weight_preshuffled: weight is + aiter's (16, 16) shuffle of the [N, K] weight (aiter_preshuffle_supported).""" input_2d = input.view(-1, input.shape[-1]) n = weight.shape[0] if input_2d.shape[0] == 0: @@ -29,7 +50,12 @@ def aiter_mxfp8_blockscaled_linear( else: xq, xs = mxfp8_operand(input_2d, input_scale) out = gemm_a8w8_blockscale( - xq, weight, xs, weight_scale_ue8m0, dtype=torch.bfloat16 + xq, + weight, + xs, + weight_scale_ue8m0, + dtype=torch.bfloat16, + isBpreshuffled=weight_preshuffled, ) if bias is not None: out = out + bias diff --git a/python/sglang/kernels/ops/speculative/dspark/dspark_draft_model.py b/python/sglang/kernels/ops/speculative/dspark/dspark_draft_model.py index 2890366ac2..820be33b83 100644 --- a/python/sglang/kernels/ops/speculative/dspark/dspark_draft_model.py +++ b/python/sglang/kernels/ops/speculative/dspark/dspark_draft_model.py @@ -253,6 +253,10 @@ def commit_kv_proj_fused( weight_scale_ue8m0=stacked.mxfp8_scale, input_scale=None, bias=None, + # whole 16-row tiles: the row-stacked preshuffled weights stay preshuffled + weight_preshuffled=getattr( + wkv_linears[0], "mxfp8_aiter_preshuffled", False + ), ) else: kv_all = quant_method.w8a8_mxfp8_linear( diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 4184054347..dfbef7c03f 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1747,6 +1747,9 @@ class Envs: # down_proj, the wo_a GEMM for wo_b, and the FFN norm for the shared expert's gate_up. SGLANG_HIP_SHARED_ACT_MXFP8 = EnvBool(_default_hip) SGLANG_HIP_WO_A_MXFP8 = EnvBool(_default_hip) + # gfx950 aiter MXFP8 dense route: keep the weight (16, 16)-preshuffled and run aiter's FlyDSL MXFP8 + # GEMM (needs aiter with ROCm/aiter#5896) instead of the Triton group32 GEMM on the plain weight. + SGLANG_ROCM_MXFP8_AITER_PRESHUFFLE = EnvBool(False) SGLANG_HIP_FFN_NORM_MXFP8 = EnvBool(_default_hip) # V4.1 low-ratio decode / target-verify indexer: score with aiter's schedule-free row-group # FP4 MQA logits kernel, a request's verify rows sharing each key load. diff --git a/python/sglang/srt/layers/quantization/fp8_hip.py b/python/sglang/srt/layers/quantization/fp8_hip.py index c4c8a10a0e..168a4a77db 100644 --- a/python/sglang/srt/layers/quantization/fp8_hip.py +++ b/python/sglang/srt/layers/quantization/fp8_hip.py @@ -53,6 +53,21 @@ def process_dense_weights(method, layer: torch.nn.Module, scale_u8) -> None: ue8m0_weight_scale(layer.weight_scale_inv.data), ) _prepare_flydsl_mxfp8(layer) + from aiter.ops.shuffle import shuffle_weight + + from sglang.kernels.ops.gemm.mxfp8_aiter_gfx95 import ( + aiter_preshuffle_supported, + ) + + n, k = layer.weight.shape + layer.mxfp8_aiter_preshuffled = aiter_preshuffle_supported(n, k) + if layer.mxfp8_aiter_preshuffled: + # same bytes in aiter's (16, 16) tile order; the scales stay compact row-major + copy_or_rebind_param( + layer, "weight", shuffle_weight(layer.weight.data, (16, 16)) + ) + # mxfp8_flydsl_gfx95 reads the plain [N, K] weight + layer.weight_scale_mx_flydsl = None return assert backend.is_gfx95_mxfp8_native() n, k = layer.weight.shape @@ -116,6 +131,7 @@ def apply_dense( weight_scale_ue8m0=layer.weight_scale_mx_e8m0, input_scale=input_scale, bias=bias, + weight_preshuffled=getattr(layer, "mxfp8_aiter_preshuffled", False), ) if mxfp8_ready and input_scale is None: return method.w8a8_mxfp8_linear( diff --git a/test/registered/kernels/ops/gemm/test_mxfp8_amd_gfx95.py b/test/registered/kernels/ops/gemm/test_mxfp8_amd_gfx95.py index d358a0bcf7..9f90250935 100644 --- a/test/registered/kernels/ops/gemm/test_mxfp8_amd_gfx95.py +++ b/test/registered/kernels/ops/gemm/test_mxfp8_amd_gfx95.py @@ -18,6 +18,7 @@ from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import ( fake_quant_fp8_activation, fp8_grid_quantize, ) +from sglang.srt.environ import envs from sglang.srt.layers.quantization import fp8_hip from sglang.srt.layers.quantization.fp8_utils import ( Mxfp8DenseGemmBackend, @@ -158,6 +159,14 @@ def _aiter_mxfp8_gemm_available() -> bool: return True +def _aiter_has_preshuffled_mxfp8() -> bool: + if not _aiter_mxfp8_gemm_available(): + return False + from sglang.kernels.ops.gemm import mxfp8_aiter_gfx95 + + return mxfp8_aiter_gfx95._aiter_has_preshuffled_mxfp8() + + # decode / verify rows and the dot_scaled buckets the native test covers AITER_MS = (1, 6, 33, 1025) @@ -180,16 +189,30 @@ class TestMxfp8AiterRouteGfx95(CustomTestCase): w8a8_mxfp8_linear=dispatch_block_fp8_mxfp8_linear(backend), ) fp8_hip.process_dense_weights(method, layer, None) - return lambda x: fp8_hip.apply_dense(method, layer, x, None) + route = lambda x: fp8_hip.apply_dense(method, layer, x, None) + route.layer = layer + return route def test_within_one_bf16_ulp_of_the_bf16_route(self): + self._check_route(AITER_MS) + + @unittest.skipUnless( + _aiter_has_preshuffled_mxfp8(), "aiter FlyDSL MXFP8 GEMM (ROCm/aiter#5896)" + ) + def test_preshuffled_weight_within_one_bf16_ulp_of_the_bf16_route(self): + # 2049: a prefill row count, which the preshuffled weight also serves + with envs.SGLANG_ROCM_MXFP8_AITER_PRESHUFFLE.override(True): + self._check_route(AITER_MS + (2049,), preshuffled=True) + + def _check_route(self, ms, preshuffled=False): for n, k in ROUTE_SHAPES: torch.manual_seed(0) w = torch.randn(n, k, device="cuda", dtype=torch.bfloat16) wq, ws = _quant_weight_block32(w) w_bf16 = dequant_block_fp8_weight_to_bf16(wq, ws, [32, 32]) route = self._route(wq, ws) - for m in AITER_MS: + self.assertEqual(route.layer.mxfp8_aiter_preshuffled, preshuffled, (n, k)) + for m in ms: x = torch.randn(m, k, device="cuda", dtype=torch.bfloat16) ref = bf16_dequant_blockscaled_linear(x, w_bf16) out = route(x)