[Perf] [AMD] dsv41: run gfx950 MXFP8 dense GEMMs on aiter's preshuffled FlyDSL kernel (#45123)

Co-authored-by: wunhuang <wunhuang@amd.com>
This commit is contained in:
kk
2026-10-20 16:19:32 -07:00
committed by GitHub
co-authored by wunhuang
parent 4f8f06faf3
commit 533020b500
5 changed files with 76 additions and 4 deletions
@@ -5,12 +5,31 @@ ue8m0 [M, K/32]; aiter picks the kernel config from its tuned MXFP8 tables."""
from __future__ import annotations
import importlib.util
from functools import cache
from typing import Optional
import torch
from aiter import gemm_a8w8_blockscale
from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import mxfp8_operand
from sglang.srt.environ import envs
@cache
def _aiter_has_preshuffled_mxfp8() -> bool:
# gemm_a8w8_blockscale forwards preshuffled group32 operands to FlyDSL since ROCm/aiter#5896
return importlib.util.find_spec("aiter.ops.flydsl.batched_gemm_a8w8") is not None
def aiter_preshuffle_supported(n: int, k: int) -> bool:
"""Whether an [N, K] weight of the aiter route is kept (16, 16)-preshuffled."""
return (
envs.SGLANG_ROCM_MXFP8_AITER_PRESHUFFLE.get()
and n % 16 == 0
and k % 32 == 0
and _aiter_has_preshuffled_mxfp8()
)
def aiter_mxfp8_blockscaled_linear(
@@ -19,9 +38,11 @@ def aiter_mxfp8_blockscaled_linear(
weight_scale_ue8m0: torch.Tensor,
input_scale: Optional[torch.Tensor] = None,
bias: Optional[torch.Tensor] = None,
weight_preshuffled: bool = False,
) -> torch.Tensor:
"""Dense linear of the aiter route. input is bf16 (plain or on the fp8 grid, where the per-32
requant is exact), or fp8 e4m3 with input_scale ue8m0 [M, K/32]."""
requant is exact), or fp8 e4m3 with input_scale ue8m0 [M, K/32]. weight_preshuffled: weight is
aiter's (16, 16) shuffle of the [N, K] weight (aiter_preshuffle_supported)."""
input_2d = input.view(-1, input.shape[-1])
n = weight.shape[0]
if input_2d.shape[0] == 0:
@@ -29,7 +50,12 @@ def aiter_mxfp8_blockscaled_linear(
else:
xq, xs = mxfp8_operand(input_2d, input_scale)
out = gemm_a8w8_blockscale(
xq, weight, xs, weight_scale_ue8m0, dtype=torch.bfloat16
xq,
weight,
xs,
weight_scale_ue8m0,
dtype=torch.bfloat16,
isBpreshuffled=weight_preshuffled,
)
if bias is not None:
out = out + bias
@@ -253,6 +253,10 @@ def commit_kv_proj_fused(
weight_scale_ue8m0=stacked.mxfp8_scale,
input_scale=None,
bias=None,
# whole 16-row tiles: the row-stacked preshuffled weights stay preshuffled
weight_preshuffled=getattr(
wkv_linears[0], "mxfp8_aiter_preshuffled", False
),
)
else:
kv_all = quant_method.w8a8_mxfp8_linear(
+3
View File
@@ -1747,6 +1747,9 @@ class Envs:
# down_proj, the wo_a GEMM for wo_b, and the FFN norm for the shared expert's gate_up.
SGLANG_HIP_SHARED_ACT_MXFP8 = EnvBool(_default_hip)
SGLANG_HIP_WO_A_MXFP8 = EnvBool(_default_hip)
# gfx950 aiter MXFP8 dense route: keep the weight (16, 16)-preshuffled and run aiter's FlyDSL MXFP8
# GEMM (needs aiter with ROCm/aiter#5896) instead of the Triton group32 GEMM on the plain weight.
SGLANG_ROCM_MXFP8_AITER_PRESHUFFLE = EnvBool(False)
SGLANG_HIP_FFN_NORM_MXFP8 = EnvBool(_default_hip)
# V4.1 low-ratio decode / target-verify indexer: score with aiter's schedule-free row-group
# FP4 MQA logits kernel, a request's verify rows sharing each key load.
@@ -53,6 +53,21 @@ def process_dense_weights(method, layer: torch.nn.Module, scale_u8) -> None:
ue8m0_weight_scale(layer.weight_scale_inv.data),
)
_prepare_flydsl_mxfp8(layer)
from aiter.ops.shuffle import shuffle_weight
from sglang.kernels.ops.gemm.mxfp8_aiter_gfx95 import (
aiter_preshuffle_supported,
)
n, k = layer.weight.shape
layer.mxfp8_aiter_preshuffled = aiter_preshuffle_supported(n, k)
if layer.mxfp8_aiter_preshuffled:
# same bytes in aiter's (16, 16) tile order; the scales stay compact row-major
copy_or_rebind_param(
layer, "weight", shuffle_weight(layer.weight.data, (16, 16))
)
# mxfp8_flydsl_gfx95 reads the plain [N, K] weight
layer.weight_scale_mx_flydsl = None
return
assert backend.is_gfx95_mxfp8_native()
n, k = layer.weight.shape
@@ -116,6 +131,7 @@ def apply_dense(
weight_scale_ue8m0=layer.weight_scale_mx_e8m0,
input_scale=input_scale,
bias=bias,
weight_preshuffled=getattr(layer, "mxfp8_aiter_preshuffled", False),
)
if mxfp8_ready and input_scale is None:
return method.w8a8_mxfp8_linear(