[AMD][V4.1][*/N] Fuse MXFP8 activation quant into producer kernels on gfx950 (#42055)

Co-authored-by: wunhuang <wunhuang@amd.com>
This commit is contained in:
kk
2026-10-02 08:34:44 -07:00
committed by GitHub
co-authored by wunhuang
parent e667ab10d5
commit a74f259f38
9 changed files with 185 additions and 33 deletions
@@ -68,9 +68,12 @@ def _silu_and_mul_clamp_kernel(
)
def silu_and_mul_clamp_fp8_grid_supported(intermediate: int) -> bool:
"""The fp8-grid epilogue needs whole 32-wide groups inside one program."""
return intermediate % 32 == 0 and intermediate <= 1024
def silu_and_mul_clamp_fp8_grid_supported(
intermediate: int, multi_block: bool = False
) -> bool:
"""The fp8-grid epilogue needs whole 32-wide groups inside one program; 1024-wide programs
keep that for any multiple of 32 (multi_block)."""
return intermediate % 32 == 0 and (multi_block or intermediate <= 1024)
def silu_and_mul_clamp_triton(
@@ -87,8 +90,8 @@ def silu_and_mul_clamp_triton(
inter_size = N // 2
fp8_grid = fp8_grid or emit_fp8
if fp8_grid:
assert silu_and_mul_clamp_fp8_grid_supported(inter_size), (
f"inter_size {inter_size}: the fp8-grid epilogue needs a multiple of 32 up to 1024"
assert silu_and_mul_clamp_fp8_grid_supported(inter_size, multi_block=True), (
f"inter_size {inter_size}: the fp8-grid epilogue needs a multiple of 32"
)
out = torch.empty(
(M, inter_size),
@@ -11,7 +11,10 @@ import torch
import triton
import triton.language as tl
from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import fp8_grid_round
from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import (
fp8_grid_quant,
fp8_grid_round,
)
# 16 x 32 x 512, 2 warps: the fastest N tile holding whole 32-groups; nonkdim 16 is aiter's MFMA
_BLOCK_M, _BLOCK_N, _BLOCK_K = 16, 32, 512
@@ -50,11 +53,15 @@ def _batched_gemm_bf16_fp8_grid_kernel(
stride_cb,
stride_cm,
stride_cn,
scale_ptr,
stride_sb,
stride_sm,
BLOCK_SIZE_M: tl.constexpr,
BLOCK_SIZE_N: tl.constexpr,
BLOCK_SIZE_K: tl.constexpr,
EVEN_K: tl.constexpr,
FP8_GRID: tl.constexpr,
EMIT_FP8: tl.constexpr,
cache_modifier: tl.constexpr,
num_warps: tl.constexpr,
num_stages: tl.constexpr,
@@ -117,12 +124,21 @@ def _batched_gemm_bf16_fp8_grid_kernel(
a_ptrs += BLOCK_SIZE_K * stride_ak
b_ptrs += BLOCK_SIZE_K * stride_bk
c = accumulator.to(c_ptr.type.element_ty)
if FP8_GRID:
# the consumer's fake-quant on the bf16-rounded output: one ue8m0 group per 32 N elements
xg = tl.reshape(c.to(tl.float32), (BLOCK_SIZE_M * (BLOCK_SIZE_N // 32), 32))
c = tl.reshape(fp8_grid_round(xg), (BLOCK_SIZE_M, BLOCK_SIZE_N))
c = c.to(c_ptr.type.element_ty)
if EMIT_FP8:
# the consumer's own quant of the bf16-rounded output: fp8 codes + one ue8m0 per 32 N
xg = tl.reshape(
accumulator.to(tl.bfloat16).to(tl.float32),
(BLOCK_SIZE_M * (BLOCK_SIZE_N // 32), 32),
)
q8, e8 = fp8_grid_quant(xg)
c = tl.reshape(q8, (BLOCK_SIZE_M, BLOCK_SIZE_N))
else:
c = accumulator.to(c_ptr.type.element_ty)
if FP8_GRID:
# the consumer's fake-quant on the bf16-rounded output: one ue8m0 group per 32 N elements
xg = tl.reshape(c.to(tl.float32), (BLOCK_SIZE_M * (BLOCK_SIZE_N // 32), 32))
c = tl.reshape(fp8_grid_round(xg), (BLOCK_SIZE_M, BLOCK_SIZE_N))
c = c.to(c_ptr.type.element_ty)
offs_cm = tl.cast(pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M), tl.int64)
offs_cn = tl.cast(pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N), tl.int64)
@@ -134,6 +150,16 @@ def _batched_gemm_bf16_fp8_grid_kernel(
)
c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N)
tl.store(c_ptrs, c, mask=c_mask)
if EMIT_FP8:
offs_sn = pid_n * (BLOCK_SIZE_N // 32) + tl.arange(0, BLOCK_SIZE_N // 32)
tl.store(
scale_ptr
+ stride_sb * batch_id
+ stride_sm * offs_cm[:, None]
+ offs_sn[None, :],
tl.reshape(e8, (BLOCK_SIZE_M, BLOCK_SIZE_N // 32)).to(tl.uint8),
mask=(offs_cm[:, None] < M) & (offs_sn[None, :] < N // 32),
)
@triton.jit
@@ -208,10 +234,14 @@ def _batched_gemm_split_k_reduce_kernel(
N,
stride_cb,
stride_cm,
scale_ptr,
stride_sb,
stride_sm,
BLOCK_SIZE_M: tl.constexpr,
BLOCK_SIZE_N: tl.constexpr,
SPLIT_K: tl.constexpr,
FP8_GRID: tl.constexpr,
EMIT_FP8: tl.constexpr,
):
"""Grid (G, row tiles x N tiles): sums the partials of one output tile in split order,
rounds to bf16 and, with FP8_GRID, onto the consumer's fp8 grid."""
@@ -231,11 +261,19 @@ def _batched_gemm_split_k_reduce_kernel(
+ (tile + tl.arange(0, BLOCK_SIZE_M))[:, None] * N
+ offs_n[None, :]
)
c = acc.to(c_ptr.type.element_ty)
if FP8_GRID:
xg = tl.reshape(c.to(tl.float32), (BLOCK_SIZE_M * (BLOCK_SIZE_N // 32), 32))
c = tl.reshape(fp8_grid_round(xg), (BLOCK_SIZE_M, BLOCK_SIZE_N))
c = c.to(c_ptr.type.element_ty)
if EMIT_FP8:
xg = tl.reshape(
acc.to(tl.bfloat16).to(tl.float32),
(BLOCK_SIZE_M * (BLOCK_SIZE_N // 32), 32),
)
q8, e8 = fp8_grid_quant(xg)
c = tl.reshape(q8, (BLOCK_SIZE_M, BLOCK_SIZE_N))
else:
c = acc.to(c_ptr.type.element_ty)
if FP8_GRID:
xg = tl.reshape(c.to(tl.float32), (BLOCK_SIZE_M * (BLOCK_SIZE_N // 32), 32))
c = tl.reshape(fp8_grid_round(xg), (BLOCK_SIZE_M, BLOCK_SIZE_N))
c = c.to(c_ptr.type.element_ty)
c_ptrs = (
c_ptr
+ stride_cb * batch_id
@@ -243,6 +281,16 @@ def _batched_gemm_split_k_reduce_kernel(
+ offs_n[None, :]
)
tl.store(c_ptrs, c, mask=(offs_m[:, None] < M) & (offs_n[None, :] < N))
if EMIT_FP8:
offs_sn = pid_n * (BLOCK_SIZE_N // 32) + tl.arange(0, BLOCK_SIZE_N // 32)
tl.store(
scale_ptr
+ stride_sb * batch_id
+ stride_sm * tl.cast(offs_m, tl.int64)[:, None]
+ offs_sn[None, :],
tl.reshape(e8, (BLOCK_SIZE_M, BLOCK_SIZE_N // 32)).to(tl.uint8),
mask=(offs_m[:, None] < M) & (offs_sn[None, :] < N // 32),
)
def _split_k_applies(T: int, D: int, R: int) -> bool:
@@ -255,7 +303,11 @@ def _split_k_applies(T: int, D: int, R: int) -> bool:
def _batched_gemm_split_k(
x: torch.Tensor, w: torch.Tensor, out: torch.Tensor, fp8_grid: bool
x: torch.Tensor,
w: torch.Tensor,
out: torch.Tensor,
fp8_grid: bool,
scale: Optional[torch.Tensor] = None,
) -> None:
T, G, D = x.shape
R = w.shape[1]
@@ -293,10 +345,14 @@ def _batched_gemm_split_k(
R,
R,
G * R,
scale if scale is not None else out,
R // 32,
G * R // 32,
BLOCK_SIZE_M=_BLOCK_M,
BLOCK_SIZE_N=_BLOCK_N,
SPLIT_K=_SPLIT_K,
FP8_GRID=fp8_grid,
EMIT_FP8=scale is not None,
num_warps=_NUM_WARPS,
)
@@ -306,26 +362,38 @@ def batched_gemm_bf16_fp8_grid(
w: torch.Tensor,
fp8_grid: bool = True,
split_k: Optional[bool] = None,
) -> torch.Tensor:
emit_fp8: bool = False,
):
"""x [T, G, D] bf16, w [G, R, D] bf16 -> [T, G * R] bf16 with out[t, g*R:(g+1)*R] =
x[t, g] @ w[g]^T, on the fp8 grid when fp8_grid; split_k=None picks the regime by shape,
False keeps the single launch at every T (deterministic inference)."""
False keeps the single launch at every T (deterministic inference).
emit_fp8 returns (fp8 e4m3 [T, G * R], ue8m0 uint8 [T, G * R // 32]) instead."""
assert x.dim() == 3 and w.dim() == 3, (x.shape, w.shape)
T, G, D = x.shape
assert w.shape[0] == G and w.shape[2] == D, (x.shape, w.shape)
R = w.shape[1]
assert x.dtype == torch.bfloat16 and w.dtype == torch.bfloat16
assert x.stride(2) == 1 and w.is_contiguous()
assert not fp8_grid or R % 32 == 0, R
out = torch.empty((T, G * R), dtype=torch.bfloat16, device=x.device)
assert not (fp8_grid or emit_fp8) or R % 32 == 0, R
out = torch.empty(
(T, G * R),
dtype=torch.float8_e4m3fn if emit_fp8 else torch.bfloat16,
device=x.device,
)
scale = (
torch.empty((T, G * R // 32), dtype=torch.uint8, device=x.device)
if emit_fp8
else None
)
result = (out, scale) if emit_fp8 else out
if T == 0:
return out
return result
if split_k is None:
split_k = _split_k_applies(T, D, R)
if split_k:
assert _split_k_applies(T, D, R), (T, D, R)
_batched_gemm_split_k(x, w, out, fp8_grid)
return out
_batched_gemm_split_k(x, w, out, fp8_grid, scale)
return result
block_m, block_n, block_k, num_warps = _tile_config(T)
grid = (G, triton.cdiv(T, block_m) * triton.cdiv(R, block_n))
_batched_gemm_bf16_fp8_grid_kernel[grid](
@@ -344,15 +412,19 @@ def batched_gemm_bf16_fp8_grid(
R,
G * R,
1,
scale if scale is not None else out,
R // 32,
G * R // 32,
BLOCK_SIZE_M=block_m,
BLOCK_SIZE_N=block_n,
BLOCK_SIZE_K=block_k,
EVEN_K=(D % block_k == 0),
FP8_GRID=fp8_grid,
EMIT_FP8=emit_fp8,
cache_modifier=_CACHE_MODIFIER,
num_warps=num_warps,
num_stages=_NUM_STAGES,
waves_per_eu=_WAVES_PER_EU,
matrix_instr_nonkdim=_MFMA_NONKDIM,
)
return out
return result
+6
View File
@@ -1592,6 +1592,12 @@ class Envs:
# DSpark draft block on the HIP radix backend: build the attention metadata inside the
# draft CUDA graph from the raw inputs instead of eagerly before every replay.
SGLANG_HIP_DSPARK_DRAFT_RAW_METADATA = EnvBool(_default_hip)
# gfx950 MXFP8 dense routes (aiter group32 / native): the producer emits fp8 + ue8m0 for its
# consumer instead of bf16 plus a separate quant launch -- the shared expert's SwiGLU for
# down_proj, the wo_a GEMM for wo_b, and the FFN norm for the shared expert's gate_up.
SGLANG_HIP_SHARED_ACT_MXFP8 = EnvBool(_default_hip)
SGLANG_HIP_WO_A_MXFP8 = EnvBool(_default_hip)
SGLANG_HIP_FFN_NORM_MXFP8 = EnvBool(_default_hip)
# cache, GEMM, and distributed
SGLANG_OPT_FP8_WO_A_GEMM = EnvBool(True)
@@ -10,6 +10,7 @@ from sglang.kernels.ops.activation.silu_and_mul_clamp_hip import (
silu_and_mul_clamp_fp8_grid_supported,
silu_and_mul_clamp_triton,
)
from sglang.srt.environ import envs
from sglang.srt.layers.quantization.fp8 import Fp8LinearMethod
@@ -33,7 +34,9 @@ def resolve_fused_clamp_route(mlp, half_width: int) -> None:
quant_method.mxfp8_dense_backend.is_gfx95_mxfp8_native()
or quant_method.mxfp8_dense_backend.is_gfx95_mxfp8_aiter()
)
and silu_and_mul_clamp_fp8_grid_supported(half_width)
and silu_and_mul_clamp_fp8_grid_supported(
half_width, multi_block=envs.SGLANG_HIP_SHARED_ACT_MXFP8.get()
)
)
# the native and aiter MXFP8 routes take fp8 + ue8m0 straight from the epilogue
mlp._hip_act_native_consumer = bool(
@@ -165,12 +165,36 @@ def post_attention_norm(layer, x: torch.Tensor, coefficients=None) -> torch.Tens
if coefficients is not None:
coefficients.materialize()
return norm(x)
if _ffn_norm_emits_mxfp8(layer, x.shape[0]):
x_quant, out = rmsnorm_with_sinkhorn(
x, norm.weight.data, norm.variance_epsilon, coefficients, emit_fp8=True
)
# the shared expert's gate_up takes it in place of its own quant (_forward_shared_experts)
out._hip_mxfp8_operand = x_quant
return out
_, out = rmsnorm_with_sinkhorn(
x, norm.weight.data, norm.variance_epsilon, coefficients, fake_quant=False
)
return out
_FFN_NORM_EMIT_MXFP8 = envs.SGLANG_HIP_FFN_NORM_MXFP8.get()
def _ffn_norm_emits_mxfp8(layer, num_tokens: int) -> bool:
"""SGLANG_HIP_FFN_NORM_MXFP8: whether the FFN norm launch also emits fp8 + ue8m0 for the
shared expert's gate_up (its route consumes them at this token count)."""
if not _FFN_NORM_EMIT_MXFP8:
return False
consumer = getattr(layer, "_ffn_shared_mxfp8_consumer", None)
if consumer is None:
shared = getattr(getattr(layer, "mlp", None), "shared_experts", None)
consumer = layer._ffn_shared_mxfp8_consumer = _mxfp8_consumer(
getattr(shared, "gate_up_proj", None)
)
return consumer
def live_rows(activation, num_tokens: int):
"""The first num_tokens rows of a break input; the gfx950 fused q_norm
hands the indexer its Fp8GridActivation wrapper, whose rows live in .x."""
@@ -196,15 +220,37 @@ def wo_b_takes_fp8_grid(attn) -> bool:
return attn._wo_b_fp8_grid_operand
def wo_a_fp8_grid_matmul(o: torch.Tensor, wo_a: torch.Tensor, fp8_grid: bool):
_WO_A_EMIT_MXFP8 = envs.SGLANG_HIP_WO_A_MXFP8.get()
def wo_b_emits_mxfp8(attn, num_tokens: int) -> bool:
"""SGLANG_HIP_WO_A_MXFP8: whether the wo_a GEMM hands wo_b fp8 + ue8m0 from its epilogue
(wo_b's route consumes them at this token count) instead of bf16 plus a separate quant."""
if not _WO_A_EMIT_MXFP8 or _wo_a_fp8_grid_gemm is None:
return False
consumer = getattr(attn, "_wo_b_mxfp8_consumer", None)
if consumer is None:
consumer = attn._wo_b_mxfp8_consumer = _mxfp8_consumer(attn.wo_b)
return consumer
def wo_a_fp8_grid_matmul(
o: torch.Tensor, wo_a: torch.Tensor, fp8_grid: bool, emit_fp8: bool = False
):
"""o [T, G, D] @ wo_a [G, R, D]^T through the gfx950 fork of the aiter batched
GEMM: bf16 [T, G, R], or with fp8_grid rounded onto wo_b's fp8 grid as
an Fp8GridActivation [T, G * R]; None when the fork did not import."""
an Fp8GridActivation [T, G * R], or with emit_fp8 an Mxfp8Activation [T, G * R];
None when the fork did not import."""
if _wo_a_fp8_grid_gemm is None:
return None
# the split-K regime ends at 64 rows, so a request's verify and decode rows would
# take different reduction orders; deterministic inference keeps the single chain
split_k = False if get_exec().deterministic.enable_deterministic_inference else None
if emit_fp8:
q, scale = _wo_a_fp8_grid_gemm(
o, wo_a, fp8_grid=False, split_k=split_k, emit_fp8=True
)
return Mxfp8Activation(q, scale)
y = _wo_a_fp8_grid_gemm(o, wo_a, fp8_grid=fp8_grid, split_k=split_k)
if fp8_grid:
return Fp8GridActivation(y)
@@ -28,6 +28,7 @@ from sglang.srt.utils import is_gfx95_supported
live_rows = gfx95_dense.live_rows
wo_a_fp8_grid_matmul = gfx95_dense.wo_a_fp8_grid_matmul
wo_b_emits_mxfp8 = gfx95_dense.wo_b_emits_mxfp8
_is_gfx95_supported = is_gfx95_supported()
+14 -1
View File
@@ -259,6 +259,7 @@ _moe_quant_once_logged = False
_enable_pcg_dsv2_dual_stream = (
_is_cuda and envs.SGLANG_ENABLE_PCG_DSV2_DUAL_STREAM.get()
)
_hip_shared_act_mxfp8 = _is_hip and envs.SGLANG_HIP_SHARED_ACT_MXFP8.get()
class DeepseekV2MLP(nn.Module):
@@ -411,7 +412,16 @@ class DeepseekV2MLP(nn.Module):
if self.use_fused_clamp_act_mul and not self._fused_clamp_fp8_checked:
_hip_act.resolve_fused_clamp_route(self, gate_up.shape[-1] // 2)
if self.use_fused_clamp_act_mul and self.swiglu_limit is not None:
# SGLANG_HIP_SHARED_ACT_MXFP8: an MXFP8 down_proj takes fp8 + ue8m0 straight from the Triton
# epilogue instead of aiter's bf16 output plus a separate quant launch.
if (
self.use_fused_clamp_act_mul
and self.swiglu_limit is not None
and _hip_shared_act_mxfp8
and self._hip_act_fp8_grid
):
x = _hip_act.silu_and_mul_clamp(self, gate_up)
elif self.use_fused_clamp_act_mul and self.swiglu_limit is not None:
from aiter.ops.triton.fusions.fused_clamp_act_mul import (
fused_clamp_act_mul,
)
@@ -1733,6 +1743,9 @@ class DeepseekV2MoE(nn.Module):
pre_quant_input: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
):
if (hidden_states.shape[0] > 0) and (self.num_fused_shared_experts == 0):
if pre_quant_input is None and _is_hip:
# SGLANG_HIP_FFN_NORM_MXFP8: the FFN norm launch's fp8 + ue8m0 of these rows
pre_quant_input = getattr(hidden_states, "_hip_mxfp8_operand", None)
if pre_quant_input is not None:
# SGLANG_OPT_MOE_QUANT_ONCE: (q, s) rows may be padded to a
# multiple of 4; the padded rows flow through the MLP (all ops
+9 -2
View File
@@ -462,7 +462,8 @@ def _apply_wo_a_bf16_matmul(
is_prefill: bool = False,
fast_path: bool = False,
fp8_grid: bool = False,
) -> torch.Tensor | Mxfp8SwizzledInput:
emit_fp8: bool = False,
) -> torch.Tensor | Mxfp8SwizzledInput | Fp8GridActivation | Mxfp8Activation:
# o [T, G, D] @ wo_a [G, R, D] -> [T, G, R]; the fast paths below are gated
# on the exact validated TP4 shapes and write token-major output directly.
global _wo_a_aiter_batched_gemm_disabled
@@ -528,7 +529,7 @@ def _apply_wo_a_bf16_matmul(
return result
# the 16-row decode kernels also serve V4.1's target-verify rows
if (is_decode or is_target_verify) and hip_fast_path:
y = _hip.wo_a_fp8_grid_matmul(o, wo_a, fp8_grid)
y = _hip.wo_a_fp8_grid_matmul(o, wo_a, fp8_grid, emit_fp8=emit_fp8)
if y is not None:
return y
if (
@@ -669,6 +670,10 @@ def _apply_gguf_grouped_wo_a(
if TYPE_CHECKING:
from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import (
Fp8GridActivation,
Mxfp8Activation,
)
from sglang.srt.layers.attention.deepseek_v4_backend import (
DeepseekV4AttnBackend,
LateLayerTail,
@@ -2579,6 +2584,8 @@ class MQALayer(MqaAttentionBase):
fast_path=self.is_dsv41,
fuse_mxfp8_quant=fuse_mxfp8_quant,
fp8_grid=_is_hip and _hip.wo_a_emits_fp8_grid(self),
emit_fp8=_is_hip
and _hip.wo_b_emits_mxfp8(self, o.shape[0]),
)
else:
o = _apply_gguf_grouped_wo_a(