[AMD] Speed up Wan2.2 DiT FP8 attention per-tensor quantization (#34695)

This commit is contained in:
jacky.cheng
2026-09-23 19:49:26 -07:00
committed by GitHub
parent 8b5d77c268
commit e98b2f7538
@@ -8,6 +8,8 @@ import os
import aiter
import torch
import triton
import triton.language as tl
from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import (
AttentionBackend,
@@ -29,11 +31,141 @@ _fp8_dtype = torch.float8_e4m3fn
# fmha_fwd_hd128_fp8_gfx950 ASM kernel. Support full MHA with q/k/v head_dim == 128 -- e.g., Wan 2.2 self- and cross-attention.
_FMHA_FP8_HEAD_DIM = 128
_FP8_MAX = torch.finfo(_fp8_dtype).max
if _use_fp8_attn:
logger.info("DiT FP8 attention enabled via SGLANG_DIFFUSION_AITER_FP8_ATTN=1")
@triton.jit
def _per_tensor_quant_fp8_kernel(
x_ptr,
y_ptr,
scale_ptr,
scratch_ptr, # int32[3]: [0] = barrier counter, [1]/[2] = amax bits (ping-pong)
n_elements,
gen, # launch generation, monotonically increasing
INV_FP8_MAX: tl.constexpr,
BLOCK: tl.constexpr,
NUM_PROGRAMS: tl.constexpr,
MAX_SPIN: tl.constexpr,
):
"""Grid-wide amax reduction plus rescale-and-cast, in a single launch.
The grid is persistent (one program per CU) and the two phases are separated
by a grid-wide barrier, so all ``NUM_PROGRAMS`` programs must be co-resident.
No host-side scratch reset is needed: the amax slot ping-pongs between two
generations, and each launch clears the slot the next one reduces into.
"""
pid = tl.program_id(0)
cur = 1 + (gen % 2)
nxt = 1 + ((gen + 1) % 2)
stride = NUM_PROGRAMS * BLOCK
# phase 1: per-program amax, folded into one global atomic_max
local_amax = 0.0
start = pid * BLOCK
while start < n_elements:
off = start + tl.arange(0, BLOCK)
mask = off < n_elements
x = tl.load(x_ptr + off, mask=mask, other=0.0).to(tl.float32)
local_amax = tl.maximum(local_amax, tl.max(tl.abs(x)))
start += stride
# amax is non-negative, so int ordering of the bit pattern matches float order
tl.atomic_max(scratch_ptr + cur, local_amax.to(tl.int32, bitcast=True))
# Grid-wide barrier. The spin has to be a *bounded* loop: with the trip count
# driven only by the atomic, the compiler hoists the load out of the loop and
# the kernel hangs on any grid large enough that a program actually spins.
tl.debug_barrier()
tl.atomic_add(scratch_ptr, 1)
target = (gen + 1) * NUM_PROGRAMS
i = 0
while i < MAX_SPIN:
if tl.atomic_add(scratch_ptr, 0) >= target:
i = MAX_SPIN
else:
i += 1
amax = tl.atomic_add(scratch_ptr + cur, 0).to(tl.float32, bitcast=True)
# Multiply by the reciprocal rather than divide: torch lowers ``amax / 448.0``
# that way, and the two differ by 1 ULP -- enough to move the elements that
# saturate and so to break bit-exactness with aiter.
scale = amax * INV_FP8_MAX
if pid == 0:
tl.store(scale_ptr, scale)
tl.store(scratch_ptr + nxt, 0)
# phase 2: rescale + cast
start = pid * BLOCK
while start < n_elements:
off = start + tl.arange(0, BLOCK)
mask = off < n_elements
x = tl.load(x_ptr + off, mask=mask, other=0.0).to(tl.float32)
y = tl.fdiv(x, scale, ieee_rounding=True)
tl.store(y_ptr + off, y.to(y_ptr.dtype.element_ty), mask=mask)
start += stride
_QUANT_MAX_PROGRAMS = 256 # gfx950 CU count
_QUANT_BLOCK = 16384 # with num_warps=4, swept best on the Wan2.2 q/k/v shapes
_QUANT_NUM_WARPS = 4
# Per device: the persistent grid size and the int32[3] barrier/amax scratch.
_quant_grid: dict[torch.device, int] = {}
_quant_scratch: dict[torch.device, torch.Tensor] = {}
_quant_gen: dict[torch.device, int] = {}
def _per_tensor_quant_fp8(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
"""Bit-exact drop-in for ``aiter.per_tensor_quant(x, quant_dtype=fp8)``, 6.6x
faster on [1, 90000, 40, 128] bf16 (3.36 ms -> 0.50 ms on gfx950).
aiter's reference upcasts to fp32 and runs abs/max/div/cast as four more eager
kernels; its fused HIP/Triton paths decompose over rows of ``size(-1)``, which
at head_dim 128 is one workgroup per 128 elements contending on one atomic.
The reduction has to be decomposed over elements instead, which is what the
persistent kernel above does -- in one launch, at the ~2.3 GB traffic floor.
"""
xc = x.contiguous()
y = torch.empty_like(xc, dtype=_fp8_dtype)
scale = torch.empty(1, dtype=torch.float32, device=xc.device)
device = xc.device
grid = _quant_grid.get(device)
if grid is None:
# The barrier requires a co-resident grid, so never ask for more
# programs than the device has CUs (partitioned GPUs report fewer).
grid = min(
_QUANT_MAX_PROGRAMS,
torch.cuda.get_device_properties(device).multi_processor_count,
)
_quant_grid[device] = grid
_quant_scratch[device] = torch.zeros(3, dtype=torch.int32, device=device)
_quant_gen[device] = 0
gen = _quant_gen[device]
_quant_gen[device] = gen + 1
_per_tensor_quant_fp8_kernel[(grid,)](
xc,
y,
scale,
_quant_scratch[device],
xc.numel(),
gen,
INV_FP8_MAX=1.0 / _FP8_MAX,
BLOCK=_QUANT_BLOCK,
NUM_PROGRAMS=grid,
MAX_SPIN=100_000_000,
num_warps=_QUANT_NUM_WARPS,
num_stages=1,
)
return y.view(x.shape), scale
def _can_use_fmha_fp8_prefill(
q_head_dim: int,
k_head_dim: int,
@@ -165,9 +297,9 @@ class AITerImpl(AttentionImpl):
"""
if _use_fp8_attn:
if query.dtype != _fp8_dtype:
q_fp8, q_scale = aiter.per_tensor_quant(query, quant_dtype=_fp8_dtype)
k_fp8, k_scale = aiter.per_tensor_quant(key, quant_dtype=_fp8_dtype)
v_fp8, v_scale = aiter.per_tensor_quant(value, quant_dtype=_fp8_dtype)
q_fp8, q_scale = _per_tensor_quant_fp8(query)
k_fp8, k_scale = _per_tensor_quant_fp8(key)
v_fp8, v_scale = _per_tensor_quant_fp8(value)
else:
q_fp8, k_fp8, v_fp8 = query, key, value
one = torch.tensor(1.0, dtype=torch.float32, device=query.device)