[Perf] [AMD] dsv41: run gfx950 MXFP8 dense GEMMs on aiter's preshuffled FlyDSL kernel (#45123)
Co-authored-by: wunhuang <wunhuang@amd.com>
This commit is contained in:
@@ -5,12 +5,31 @@ ue8m0 [M, K/32]; aiter picks the kernel config from its tuned MXFP8 tables."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
from functools import cache
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
from aiter import gemm_a8w8_blockscale
|
||||
|
||||
from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import mxfp8_operand
|
||||
from sglang.srt.environ import envs
|
||||
|
||||
|
||||
@cache
|
||||
def _aiter_has_preshuffled_mxfp8() -> bool:
|
||||
# gemm_a8w8_blockscale forwards preshuffled group32 operands to FlyDSL since ROCm/aiter#5896
|
||||
return importlib.util.find_spec("aiter.ops.flydsl.batched_gemm_a8w8") is not None
|
||||
|
||||
|
||||
def aiter_preshuffle_supported(n: int, k: int) -> bool:
|
||||
"""Whether an [N, K] weight of the aiter route is kept (16, 16)-preshuffled."""
|
||||
return (
|
||||
envs.SGLANG_ROCM_MXFP8_AITER_PRESHUFFLE.get()
|
||||
and n % 16 == 0
|
||||
and k % 32 == 0
|
||||
and _aiter_has_preshuffled_mxfp8()
|
||||
)
|
||||
|
||||
|
||||
def aiter_mxfp8_blockscaled_linear(
|
||||
@@ -19,9 +38,11 @@ def aiter_mxfp8_blockscaled_linear(
|
||||
weight_scale_ue8m0: torch.Tensor,
|
||||
input_scale: Optional[torch.Tensor] = None,
|
||||
bias: Optional[torch.Tensor] = None,
|
||||
weight_preshuffled: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""Dense linear of the aiter route. input is bf16 (plain or on the fp8 grid, where the per-32
|
||||
requant is exact), or fp8 e4m3 with input_scale ue8m0 [M, K/32]."""
|
||||
requant is exact), or fp8 e4m3 with input_scale ue8m0 [M, K/32]. weight_preshuffled: weight is
|
||||
aiter's (16, 16) shuffle of the [N, K] weight (aiter_preshuffle_supported)."""
|
||||
input_2d = input.view(-1, input.shape[-1])
|
||||
n = weight.shape[0]
|
||||
if input_2d.shape[0] == 0:
|
||||
@@ -29,7 +50,12 @@ def aiter_mxfp8_blockscaled_linear(
|
||||
else:
|
||||
xq, xs = mxfp8_operand(input_2d, input_scale)
|
||||
out = gemm_a8w8_blockscale(
|
||||
xq, weight, xs, weight_scale_ue8m0, dtype=torch.bfloat16
|
||||
xq,
|
||||
weight,
|
||||
xs,
|
||||
weight_scale_ue8m0,
|
||||
dtype=torch.bfloat16,
|
||||
isBpreshuffled=weight_preshuffled,
|
||||
)
|
||||
if bias is not None:
|
||||
out = out + bias
|
||||
|
||||
@@ -253,6 +253,10 @@ def commit_kv_proj_fused(
|
||||
weight_scale_ue8m0=stacked.mxfp8_scale,
|
||||
input_scale=None,
|
||||
bias=None,
|
||||
# whole 16-row tiles: the row-stacked preshuffled weights stay preshuffled
|
||||
weight_preshuffled=getattr(
|
||||
wkv_linears[0], "mxfp8_aiter_preshuffled", False
|
||||
),
|
||||
)
|
||||
else:
|
||||
kv_all = quant_method.w8a8_mxfp8_linear(
|
||||
|
||||
@@ -1747,6 +1747,9 @@ class Envs:
|
||||
# down_proj, the wo_a GEMM for wo_b, and the FFN norm for the shared expert's gate_up.
|
||||
SGLANG_HIP_SHARED_ACT_MXFP8 = EnvBool(_default_hip)
|
||||
SGLANG_HIP_WO_A_MXFP8 = EnvBool(_default_hip)
|
||||
# gfx950 aiter MXFP8 dense route: keep the weight (16, 16)-preshuffled and run aiter's FlyDSL MXFP8
|
||||
# GEMM (needs aiter with ROCm/aiter#5896) instead of the Triton group32 GEMM on the plain weight.
|
||||
SGLANG_ROCM_MXFP8_AITER_PRESHUFFLE = EnvBool(False)
|
||||
SGLANG_HIP_FFN_NORM_MXFP8 = EnvBool(_default_hip)
|
||||
# V4.1 low-ratio decode / target-verify indexer: score with aiter's schedule-free row-group
|
||||
# FP4 MQA logits kernel, a request's verify rows sharing each key load.
|
||||
|
||||
@@ -53,6 +53,21 @@ def process_dense_weights(method, layer: torch.nn.Module, scale_u8) -> None:
|
||||
ue8m0_weight_scale(layer.weight_scale_inv.data),
|
||||
)
|
||||
_prepare_flydsl_mxfp8(layer)
|
||||
from aiter.ops.shuffle import shuffle_weight
|
||||
|
||||
from sglang.kernels.ops.gemm.mxfp8_aiter_gfx95 import (
|
||||
aiter_preshuffle_supported,
|
||||
)
|
||||
|
||||
n, k = layer.weight.shape
|
||||
layer.mxfp8_aiter_preshuffled = aiter_preshuffle_supported(n, k)
|
||||
if layer.mxfp8_aiter_preshuffled:
|
||||
# same bytes in aiter's (16, 16) tile order; the scales stay compact row-major
|
||||
copy_or_rebind_param(
|
||||
layer, "weight", shuffle_weight(layer.weight.data, (16, 16))
|
||||
)
|
||||
# mxfp8_flydsl_gfx95 reads the plain [N, K] weight
|
||||
layer.weight_scale_mx_flydsl = None
|
||||
return
|
||||
assert backend.is_gfx95_mxfp8_native()
|
||||
n, k = layer.weight.shape
|
||||
@@ -116,6 +131,7 @@ def apply_dense(
|
||||
weight_scale_ue8m0=layer.weight_scale_mx_e8m0,
|
||||
input_scale=input_scale,
|
||||
bias=bias,
|
||||
weight_preshuffled=getattr(layer, "mxfp8_aiter_preshuffled", False),
|
||||
)
|
||||
if mxfp8_ready and input_scale is None:
|
||||
return method.w8a8_mxfp8_linear(
|
||||
|
||||
Reference in New Issue
Block a user