diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/aiter.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/aiter.py index 9606c48497..095bdfd069 100755 --- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/aiter.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/aiter.py @@ -8,6 +8,8 @@ import os import aiter import torch +import triton +import triton.language as tl from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import ( AttentionBackend, @@ -29,11 +31,141 @@ _fp8_dtype = torch.float8_e4m3fn # fmha_fwd_hd128_fp8_gfx950 ASM kernel. Support full MHA with q/k/v head_dim == 128 -- e.g., Wan 2.2 self- and cross-attention. _FMHA_FP8_HEAD_DIM = 128 +_FP8_MAX = torch.finfo(_fp8_dtype).max + if _use_fp8_attn: logger.info("DiT FP8 attention enabled via SGLANG_DIFFUSION_AITER_FP8_ATTN=1") +@triton.jit +def _per_tensor_quant_fp8_kernel( + x_ptr, + y_ptr, + scale_ptr, + scratch_ptr, # int32[3]: [0] = barrier counter, [1]/[2] = amax bits (ping-pong) + n_elements, + gen, # launch generation, monotonically increasing + INV_FP8_MAX: tl.constexpr, + BLOCK: tl.constexpr, + NUM_PROGRAMS: tl.constexpr, + MAX_SPIN: tl.constexpr, +): + """Grid-wide amax reduction plus rescale-and-cast, in a single launch. + + The grid is persistent (one program per CU) and the two phases are separated + by a grid-wide barrier, so all ``NUM_PROGRAMS`` programs must be co-resident. + No host-side scratch reset is needed: the amax slot ping-pongs between two + generations, and each launch clears the slot the next one reduces into. + """ + pid = tl.program_id(0) + cur = 1 + (gen % 2) + nxt = 1 + ((gen + 1) % 2) + stride = NUM_PROGRAMS * BLOCK + + # phase 1: per-program amax, folded into one global atomic_max + local_amax = 0.0 + start = pid * BLOCK + while start < n_elements: + off = start + tl.arange(0, BLOCK) + mask = off < n_elements + x = tl.load(x_ptr + off, mask=mask, other=0.0).to(tl.float32) + local_amax = tl.maximum(local_amax, tl.max(tl.abs(x))) + start += stride + + # amax is non-negative, so int ordering of the bit pattern matches float order + tl.atomic_max(scratch_ptr + cur, local_amax.to(tl.int32, bitcast=True)) + + # Grid-wide barrier. The spin has to be a *bounded* loop: with the trip count + # driven only by the atomic, the compiler hoists the load out of the loop and + # the kernel hangs on any grid large enough that a program actually spins. + tl.debug_barrier() + tl.atomic_add(scratch_ptr, 1) + target = (gen + 1) * NUM_PROGRAMS + i = 0 + while i < MAX_SPIN: + if tl.atomic_add(scratch_ptr, 0) >= target: + i = MAX_SPIN + else: + i += 1 + + amax = tl.atomic_add(scratch_ptr + cur, 0).to(tl.float32, bitcast=True) + # Multiply by the reciprocal rather than divide: torch lowers ``amax / 448.0`` + # that way, and the two differ by 1 ULP -- enough to move the elements that + # saturate and so to break bit-exactness with aiter. + scale = amax * INV_FP8_MAX + + if pid == 0: + tl.store(scale_ptr, scale) + tl.store(scratch_ptr + nxt, 0) + + # phase 2: rescale + cast + start = pid * BLOCK + while start < n_elements: + off = start + tl.arange(0, BLOCK) + mask = off < n_elements + x = tl.load(x_ptr + off, mask=mask, other=0.0).to(tl.float32) + y = tl.fdiv(x, scale, ieee_rounding=True) + tl.store(y_ptr + off, y.to(y_ptr.dtype.element_ty), mask=mask) + start += stride + + +_QUANT_MAX_PROGRAMS = 256 # gfx950 CU count +_QUANT_BLOCK = 16384 # with num_warps=4, swept best on the Wan2.2 q/k/v shapes +_QUANT_NUM_WARPS = 4 + +# Per device: the persistent grid size and the int32[3] barrier/amax scratch. +_quant_grid: dict[torch.device, int] = {} +_quant_scratch: dict[torch.device, torch.Tensor] = {} +_quant_gen: dict[torch.device, int] = {} + + +def _per_tensor_quant_fp8(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """Bit-exact drop-in for ``aiter.per_tensor_quant(x, quant_dtype=fp8)``, 6.6x + faster on [1, 90000, 40, 128] bf16 (3.36 ms -> 0.50 ms on gfx950). + + aiter's reference upcasts to fp32 and runs abs/max/div/cast as four more eager + kernels; its fused HIP/Triton paths decompose over rows of ``size(-1)``, which + at head_dim 128 is one workgroup per 128 elements contending on one atomic. + The reduction has to be decomposed over elements instead, which is what the + persistent kernel above does -- in one launch, at the ~2.3 GB traffic floor. + """ + xc = x.contiguous() + y = torch.empty_like(xc, dtype=_fp8_dtype) + scale = torch.empty(1, dtype=torch.float32, device=xc.device) + + device = xc.device + grid = _quant_grid.get(device) + if grid is None: + # The barrier requires a co-resident grid, so never ask for more + # programs than the device has CUs (partitioned GPUs report fewer). + grid = min( + _QUANT_MAX_PROGRAMS, + torch.cuda.get_device_properties(device).multi_processor_count, + ) + _quant_grid[device] = grid + _quant_scratch[device] = torch.zeros(3, dtype=torch.int32, device=device) + _quant_gen[device] = 0 + gen = _quant_gen[device] + _quant_gen[device] = gen + 1 + + _per_tensor_quant_fp8_kernel[(grid,)]( + xc, + y, + scale, + _quant_scratch[device], + xc.numel(), + gen, + INV_FP8_MAX=1.0 / _FP8_MAX, + BLOCK=_QUANT_BLOCK, + NUM_PROGRAMS=grid, + MAX_SPIN=100_000_000, + num_warps=_QUANT_NUM_WARPS, + num_stages=1, + ) + return y.view(x.shape), scale + + def _can_use_fmha_fp8_prefill( q_head_dim: int, k_head_dim: int, @@ -165,9 +297,9 @@ class AITerImpl(AttentionImpl): """ if _use_fp8_attn: if query.dtype != _fp8_dtype: - q_fp8, q_scale = aiter.per_tensor_quant(query, quant_dtype=_fp8_dtype) - k_fp8, k_scale = aiter.per_tensor_quant(key, quant_dtype=_fp8_dtype) - v_fp8, v_scale = aiter.per_tensor_quant(value, quant_dtype=_fp8_dtype) + q_fp8, q_scale = _per_tensor_quant_fp8(query) + k_fp8, k_scale = _per_tensor_quant_fp8(key) + v_fp8, v_scale = _per_tensor_quant_fp8(value) else: q_fp8, k_fp8, v_fp8 = query, key, value one = torch.tensor(1.0, dtype=torch.float32, device=query.device)