From 9d58189c12e4e14a7eea20f24f9aa7b17e221778 Mon Sep 17 00:00:00 2001 From: Emre Albayrak <70805503+emre570@users.noreply.github.com> Date: Tue, 22 Sep 2026 12:44:37 +0300 Subject: [PATCH] [diffusion] attention: add fp8_fa_sm120 FP8 backend for SM120 GPUs (#40175) Co-authored-by: Xiaoyu Zhang <1182563586@qq.com> --- .../sglang-diffusion/attention_backends.mdx | 15 + .../ops/attention/fp8_fa_sm120/__init__.py | 12 + .../ops/attention/fp8_fa_sm120/fused_prep.py | 232 +++++++++ .../ops/attention/fp8_fa_sm120/kernel.py | 475 ++++++++++++++++++ .../ops/attention/fp8_fa_sm120/plan.py | 150 ++++++ .../attention/backends/fp8_fa_sm120_attn.py | 191 +++++++ .../multimodal_gen/runtime/platforms/cuda.py | 31 ++ .../runtime/platforms/interface.py | 1 + .../test/unit/test_fp8_fa_sm120_attn.py | 222 ++++++++ 9 files changed, 1329 insertions(+) create mode 100644 python/sglang/kernels/ops/attention/fp8_fa_sm120/__init__.py create mode 100644 python/sglang/kernels/ops/attention/fp8_fa_sm120/fused_prep.py create mode 100644 python/sglang/kernels/ops/attention/fp8_fa_sm120/kernel.py create mode 100644 python/sglang/kernels/ops/attention/fp8_fa_sm120/plan.py create mode 100644 python/sglang/multimodal_gen/runtime/layers/attention/backends/fp8_fa_sm120_attn.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_fp8_fa_sm120_attn.py diff --git a/docs/docs/sglang-diffusion/attention_backends.mdx b/docs/docs/sglang-diffusion/attention_backends.mdx index d9c6e33d69..5ba2e5cb1f 100644 --- a/docs/docs/sglang-diffusion/attention_backends.mdx +++ b/docs/docs/sglang-diffusion/attention_backends.mdx @@ -139,6 +139,11 @@ For SGLang-native pipelines, the CLI accepts the lowercase names of `AttentionBa RAIN_FUSION_ATTN Requires attentions which can be installed with sgl_kernel_npu; available only for NPU. + + fp8_fa_sm120 + FP8_FA_SM120 + FP8 (E4M3) dense attention for SM120 GPUs (GeForce RTX 50, RTX PRO Blackwell), written in CuTe-DSL. Q/K/V are quantized per head on the fly; the output stays BF16. Non-causal, batch 1, head dim 128, BF16 inputs; other calls use torch_cudnn_sdpa. Opt-in only: outputs differ from BF16 attention by about 5% relative RMS per call. Each new sequence length compiles once (about 10 s). + @@ -790,6 +795,16 @@ Hybrid window attention constraints: ✅ NPU-only. Requires attentions from sgl_kernel_npu Configuration via --attention-backend-config. + + fp8_fa_sm120 + Yes + ❌ + ❌ + ❌ + ❌ + ❌ + CUDA SM120 only; falls back to torch_cudnn_sdpa on other devices and for unsupported calls. + diff --git a/python/sglang/kernels/ops/attention/fp8_fa_sm120/__init__.py b/python/sglang/kernels/ops/attention/fp8_fa_sm120/__init__.py new file mode 100644 index 0000000000..45df385e6f --- /dev/null +++ b/python/sglang/kernels/ops/attention/fp8_fa_sm120/__init__.py @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: Apache-2.0 +"""SM120 FP8 attention: CuTe-DSL kernel, fused Triton prep and the per-call entry point.""" + +from sglang.kernels.ops.attention.fp8_fa_sm120.plan import ( + HEAD_DIM, + Workspace, + fp8_attention, + kernel_key, + validate_inputs, +) + +__all__ = ["HEAD_DIM", "Workspace", "fp8_attention", "kernel_key", "validate_inputs"] diff --git a/python/sglang/kernels/ops/attention/fp8_fa_sm120/fused_prep.py b/python/sglang/kernels/ops/attention/fp8_fa_sm120/fused_prep.py new file mode 100644 index 0000000000..19d96269f9 --- /dev/null +++ b/python/sglang/kernels/ops/attention/fp8_fa_sm120/fused_prep.py @@ -0,0 +1,232 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Fused FP8 preparation for the SM120 kernel: two Triton passes into the workspace. + +Replaces a Torch chain (FP32 copy, amax, divide, cast, +padded copy, V transpose copy) with one per-head amax pass and one pack pass that +writes the kernel's own buffers directly: + + Q/K [H, padded_S, 128] same token/column order as the input + V [H, 128, padded_S] transposed here, so the kernel stays unchanged + +Scales stay per head and the quantization recipe is the same (amax/448, div.rn, +clamp, E4M3), so the kernel's numerics do not move. The pack pass writes every +position of the padded buffers, padding included, because the workspace comes +from torch.empty. +""" + +import torch +import triton +import triton.language as tl + +# Power of two >= 16: the V^T key permutation works in groups of 16 tokens. +BLOCK_TOKENS = 64 + + +# Triton's `/` lowers to div.full.f32 (approximate); Torch divides with div.rn. +# The FP8 bytes must match the Torch prepare() path, so the exact division is kept. +@triton.jit +def _divide_rn(numerator, denominator): + return tl.inline_asm_elementwise( + "div.rn.f32 $0, $1, $2;", + "=f,f,f", + [numerator, denominator], + dtype=tl.float32, + is_pure=True, + pack=1, + ) + + +@triton.jit +def _load_tile( + base, + head, + tokens, + columns, + valid, + head_stride: tl.constexpr, + token_stride: tl.constexpr, +): + offsets = head * head_stride + tokens[:, None] * token_stride + columns[None, :] + return tl.load(base + offsets, mask=valid, other=0.0).to(tl.float32) + + +@triton.jit +def _quantize(values, scale): + scaled = _divide_rn(values, scale) + scaled = tl.maximum(tl.minimum(scaled, 448.0), -448.0) + return scaled.to(tl.float8e4nv) + + +# Pass 1: one program per (head, token block); each reduces its Q/K/V tile to +# three scalars and folds them into maximums[operand, head]. +@triton.jit +def _head_amax_qkv( + q, + k, + v, + maximums, + sequence, + head_stride: tl.constexpr, + token_stride: tl.constexpr, + heads: tl.constexpr, + block_tokens: tl.constexpr, +): + head = tl.program_id(0) + tokens = tl.program_id(1) * block_tokens + tl.arange(0, block_tokens) + columns = tl.arange(0, 128) + valid = tokens[:, None] < sequence + + q_tile = _load_tile(q, head, tokens, columns, valid, head_stride, token_stride) + tl.atomic_max(maximums + 0 * heads + head, tl.max(tl.abs(q_tile))) + + k_tile = _load_tile(k, head, tokens, columns, valid, head_stride, token_stride) + tl.atomic_max(maximums + 1 * heads + head, tl.max(tl.abs(k_tile))) + + v_tile = _load_tile(v, head, tokens, columns, valid, head_stride, token_stride) + tl.atomic_max(maximums + 2 * heads + head, tl.max(tl.abs(v_tile))) + + +# Torch evaluates `tensor / 448.0` as `tensor * (1/448)` with the reciprocal in +# FP32 (CPU-scalar divisor fast path). The same product keeps the scales bit-equal. +INVERSE_FP8_MAX = (torch.ones((), dtype=torch.float32) / 448.0).item() + + +# scales[operand, head] = amax * (1/448), or 1 for an all-zero head. +@triton.jit +def _head_scales( + maximums, + scales, + inverse_fp8_max, + count: tl.constexpr, + block: tl.constexpr, +): + offsets = tl.arange(0, block) + valid = offsets < count + maximum = tl.load(maximums + offsets, mask=valid, other=0.0) + scale = tl.where(maximum > 0.0, maximum * inverse_fp8_max, 1.0) + tl.store(scales + offsets, scale, mask=valid) + + +# Pass 2: one program per (head, token block) up to padded_queries. Q/K tiles go +# out in [token, column] order; the V tile is transposed in registers/shared and +# goes out in [column, token]. Loads are masked on the input token, stores on the +# buffer position, so padding positions receive quantized zeros. +@triton.jit +def _pack_qkv( + q, + k, + v, + q_fp8, + k_fp8, + v_fp8, + scales, + sequence, + head_stride: tl.constexpr, + token_stride: tl.constexpr, + padded_queries: tl.constexpr, + padded_keys: tl.constexpr, + heads: tl.constexpr, + block_tokens: tl.constexpr, +): + head = tl.program_id(0) + tokens = tl.program_id(1) * block_tokens + tl.arange(0, block_tokens) + columns = tl.arange(0, 128) + valid = tokens[:, None] < sequence + + q_scale = tl.load(scales + 0 * heads + head) + k_scale = tl.load(scales + 1 * heads + head) + v_scale = tl.load(scales + 2 * heads + head) + + # Q is padded to the 128-row query tile, K/V to the 32-key tile. + hsd_offsets = head * padded_queries * 128 + tokens[:, None] * 128 + columns[None, :] + k_offsets = head * padded_keys * 128 + tokens[:, None] * 128 + columns[None, :] + in_queries = tokens[:, None] < padded_queries + in_keys = tokens[:, None] < padded_keys + + q_tile = _load_tile(q, head, tokens, columns, valid, head_stride, token_stride) + tl.store(q_fp8 + hsd_offsets, _quantize(q_tile, q_scale), mask=in_queries) + + k_tile = _load_tile(k, head, tokens, columns, valid, head_stride, token_stride) + tl.store(k_fp8 + k_offsets, _quantize(k_tile, k_scale), mask=in_keys) + + # The kernel packs P straight from the score fragment; V^T's stored key + # order must follow it: stored position p holds key + # p//16*16 + 8*((p%4)//2) + 2*((p%16)//4) + p%2 (see key_order_for_positions). + # The permutation is applied on the LOAD side as a row gather: every gathered + # row is still 256 contiguous bytes, and the transposed store stays contiguous. + # Permuting the store instead scatters bytes inside 16 B windows and doubled + # the prepare time. + source_tokens = ( + tokens // 16 * 16 + + 8 * ((tokens % 4) // 2) + + 2 * ((tokens % 16) // 4) + + tokens % 2 + ) + valid_source = source_tokens[:, None] < sequence + + hds_offsets = ( + head * 128 * padded_keys + columns[:, None] * padded_keys + tokens[None, :] + ) + in_keys_transposed = tokens[None, :] < padded_keys + + v_tile = _load_tile( + v, head, source_tokens, columns, valid_source, head_stride, token_stride + ) + tl.store( + v_fp8 + hds_offsets, + tl.trans(_quantize(v_tile, v_scale)), + mask=in_keys_transposed, + ) + + +def fused_prepare(q, k, v, workspace): + """Three launches: per-head amax, scale finalize, direct pack into the workspace. + + q, k, v are [H, S, 128] views with one shared stride set; the workspace buffers are + sized for this S and H. + """ + heads = q.shape[0] + sequence = q.shape[1] + padded_queries = workspace.q_fp8.shape[1] + padded_keys = workspace.k_fp8.shape[1] + head_stride = q.stride(0) + token_stride = q.stride(1) + + with torch.cuda.device(q.device): + _head_amax_qkv[(heads, triton.cdiv(sequence, BLOCK_TOKENS))]( + q, + k, + v, + workspace.maximums, + sequence, + head_stride=head_stride, + token_stride=token_stride, + heads=heads, + block_tokens=BLOCK_TOKENS, + num_warps=8, + ) + _head_scales[(1,)]( + workspace.maximums, + workspace.scales, + INVERSE_FP8_MAX, + count=3 * heads, + block=triton.next_power_of_2(3 * heads), + num_warps=1, + ) + _pack_qkv[(heads, triton.cdiv(padded_queries, BLOCK_TOKENS))]( + q, + k, + v, + workspace.q_fp8, + workspace.k_fp8, + workspace.v_fp8, + workspace.scales, + sequence, + head_stride=head_stride, + token_stride=token_stride, + padded_queries=padded_queries, + padded_keys=padded_keys, + heads=heads, + block_tokens=BLOCK_TOKENS, + num_warps=8, + ) diff --git a/python/sglang/kernels/ops/attention/fp8_fa_sm120/kernel.py b/python/sglang/kernels/ops/attention/fp8_fa_sm120/kernel.py new file mode 100644 index 0000000000..b318f723ba --- /dev/null +++ b/python/sglang/kernels/ops/attention/fp8_fa_sm120/kernel.py @@ -0,0 +1,475 @@ +# SPDX-License-Identifier: Apache-2.0 +"""FP8 flash-attention forward for SM120 in CuTe-DSL: E4M3 QK and PV, FP32 softmax. + + Q, K [H, padded_S, 128] E4M3, queries padded to 128 rows, keys to 32 + V^T [H, 128, padded_S] E4M3, keys stored in key_order_for_positions() order + scales [3, H] FP32 per-head descale of Q, K, V + O [H, S, 128] BF16 + LSE [H, S] FP32 + +One CTA covers 128 queries x 32 keys on 128 threads: 4 warps, 32 query rows per warp. +Every thread keeps the online-softmax state (running max m, running sum l) of its four +query rows in registers; the max is reduced across the quad with two butterfly shuffles +per key tile, the sum once after the key loop. + +Per key tile: K and V^T arrive in one of two shared-memory stages and the next tile is +requested right after the barrier, so the copy overlaps the math. QK runs as four E4M3 +mma.sync blocks into FP32 score registers. The probabilities are scaled by 256, cast to +E4M3 and packed in registers from the score fragment into the A fragment of PV; they +never go through shared memory. The A fragment holds the keys of every 16-key group in +a fixed permuted order, which is why V^T is stored in that order. PV accumulates into +FP32 O registers that are rescaled to the new max on every tile and normalized by l at +the end. + +Non-causal only. No dropout, no autograd. Inputs must be finite. +""" + +import math + +import cutlass +import cutlass.cute as cute +import cutlass.utils as utils + + +def _make_gmem_tiled_copy(atom_copy, dtype, copy_bits, minor_size, num_threads): + copy_elems = copy_bits // dtype.width + shape_dim_1 = minor_size // copy_elems + thread_layout = cute.make_layout( + (num_threads // shape_dim_1, shape_dim_1), stride=(shape_dim_1, 1) + ) + value_layout = cute.make_layout((1, copy_elems)) + return cute.make_tiled_copy_tv(atom_copy, thread_layout, value_layout) + + +def _make_smem_layout_fp8(dtype, copy_bits, smem_tiler): + major_size = smem_tiler[1] + row_bytes = major_size * dtype.width // 8 + chunk_bytes = copy_bits // 8 + swizzle_bits = min(int(math.log2(row_bytes // chunk_bytes)), 3) + base_bits = int(math.log2(chunk_bytes)) + shift_bits = int(math.log2(128 // chunk_bytes)) + swizzle = cute.make_swizzle(swizzle_bits, base_bits, shift_bits) + atom = cute.make_layout((8, major_size), stride=(major_size, 1)) + layout = cute.tile_to_shape(atom, smem_tiler, (0, 1, 2)) + return layout, swizzle + + +@cute.kernel +def _fp8_attention_sm120( + softmax_scale: cutlass.Float32, + mQ: cute.Tensor, + mK: cute.Tensor, + mV: cute.Tensor, + mO: cute.Tensor, + mLSE: cute.Tensor, + mScales: cute.Tensor, + sQ_layout: cute.Layout, + sK_layout: cute.Layout, + sV_layout: cute.Layout, + sQ_swizzle: cute.Swizzle, + sK_swizzle: cute.Swizzle, + sV_swizzle: cute.Swizzle, + tiled_copy_Q: cute.TiledCopy, + tiled_copy_K: cute.TiledCopy, + tiled_copy_V: cute.TiledCopy, + tiled_mma: cute.TiledMma, + cta_tiler: cutlass.Constexpr = (128, 32, 128), +): + tidx, _, _ = cute.arch.thread_idx() + bidx, _, bidz = cute.arch.block_idx() + + qk_scale = mScales[0, bidz] * mScales[1, bidz] * softmax_scale + pv_scale = mScales[2, bidz] / 256.0 + sequence = mLSE.shape[0] + + m = cute.make_rmem_tensor((4,), cutlass.Float32) + m.fill(-cutlass.Float32.inf) + + l = cute.make_rmem_tensor((4,), cutlass.Float32) + l.fill(0.0) + + gQ = cute.local_tile( + mQ[None, None, bidz], cta_tiler, (bidx, None, 0), proj=(1, None, 1) + ) + gK = cute.local_tile( + mK[None, None, bidz], cta_tiler, (None, None, 0), proj=(None, 1, 1) + ) + gV = cute.local_tile(mV[None, None, bidz], (128, 32), (0, None)) + gO = cute.local_tile(mO[None, None, bidz], (128, 128), (bidx, 0)) + + @cute.struct + class SharedStorageQKV: + q: cute.struct.Align[ + cute.struct.MemRange[mQ.element_type, cute.cosize(sQ_layout)], 16 + ] + k: cute.struct.Align[ + cute.struct.MemRange[mK.element_type, cute.cosize(sK_layout)], 16 + ] + v: cute.struct.Align[ + cute.struct.MemRange[mV.element_type, cute.cosize(sV_layout)], 16 + ] + + smem = utils.SmemAllocator() + storage = smem.allocate(SharedStorageQKV.size_in_bytes(), byte_alignment=16) + sQ = SharedStorageQKV(storage).q.get_tensor(sQ_layout, swizzle=sQ_swizzle) + sK = SharedStorageQKV(storage).k.get_tensor(sK_layout, swizzle=sK_swizzle) + sV = SharedStorageQKV(storage).v.get_tensor(sV_layout, swizzle=sV_swizzle) + + thr_copy_Q = tiled_copy_Q.get_slice(tidx) + thr_copy_K = tiled_copy_K.get_slice(tidx) + thr_copy_V = tiled_copy_V.get_slice(tidx) + + tQgQ = thr_copy_Q.partition_S(gQ) + tKgK = thr_copy_K.partition_S(gK) + tVgV = thr_copy_V.partition_S(gV) + + tQsQ = thr_copy_Q.partition_D(sQ) + tKsK = thr_copy_K.partition_D(sK) + tVsV = thr_copy_V.partition_D(sV) + + k_tile_count = cute.size(tKgK, mode=[3]) + + thr_mma = tiled_mma.get_slice(tidx) + tCgO = thr_mma.partition_C(gO) + + tCsQ = thr_mma.partition_A(sQ) + tCsK = thr_mma.partition_B(sK) + tCsV = thr_mma.partition_B(sV) + + tCrQ = tiled_mma.make_fragment_A(tCsQ[None, None, None, 0]) + tCrK = tiled_mma.make_fragment_B(tCsK[None, None, None, 0]) + tCrV = tiled_mma.make_fragment_B(tCsV[None, None, None, 0]) + + acc_shape = thr_mma.partition_shape_C((128, 32)) + tCrC = cute.make_rmem_tensor(acc_shape, cutlass.Float32) + + acc_shape_O = thr_mma.partition_shape_C((128, 128)) + tCrO = cute.make_rmem_tensor(acc_shape_O, cutlass.Float32) + tCrO.fill(0.0) + + tCrP = cute.make_rmem_tensor( + cute.make_layout(((4, 2, 2), 2, 1), stride=((1, 4, 8), 16, 0)), + mQ.element_type, + ) + num_k_block_PV = cute.size(tCrP, mode=[2]) + + pack_shape = ((2, 2), 2, (2, 2)) + tCrC_as_pack = cute.make_tensor( + tCrC.iterator, + cute.make_layout(pack_shape, stride=((1, 2), 4, (8, 16))), + ) + tCrP_as_pack = cute.make_tensor( + tCrP.iterator, + cute.make_layout(pack_shape, stride=((1, 4), 16, (2, 8))), + ) + + atom_copy_s2r_Q = cute.make_copy_atom( + cute.nvgpu.warp.LdMatrix8x16x8bOp(transpose=False, num_matrices=4), + mQ.element_type, + ) + atom_copy_s2r_K = cute.make_copy_atom( + cute.nvgpu.warp.LdMatrix8x16x8bOp(transpose=False, num_matrices=4), + mK.element_type, + ) + atom_copy_s2r_V = cute.make_copy_atom( + cute.nvgpu.warp.LdMatrix8x16x8bOp(transpose=False, num_matrices=4), + mV.element_type, + ) + + tiled_copy_s2r_Q = cute.make_tiled_copy_A(atom_copy_s2r_Q, tiled_mma) + tiled_copy_s2r_K = cute.make_tiled_copy_B(atom_copy_s2r_K, tiled_mma) + tiled_copy_s2r_V = cute.make_tiled_copy_B(atom_copy_s2r_V, tiled_mma) + + ldmatrix_Q = tiled_copy_s2r_Q.get_slice(tidx) + ldmatrix_K = tiled_copy_s2r_K.get_slice(tidx) + ldmatrix_V = tiled_copy_s2r_V.get_slice(tidx) + + tCsQ_copy_view = ldmatrix_Q.partition_S(sQ) + tCrQ_copy_view = ldmatrix_Q.retile(tCrQ) + tCsK_copy_view = ldmatrix_K.partition_S(sK) + tCrK_copy_view = ldmatrix_K.retile(tCrK) + tCsV_copy_view = ldmatrix_V.partition_S(sV) + tCrV_copy_view = ldmatrix_V.retile(tCrV) + + num_k_block = cute.size(tCrQ, mode=[2]) + + tCsQ_p = tCsQ_copy_view[None, None, None, 0] + + cute.copy(tiled_copy_Q, tQgQ[None, None, None], tQsQ[None, None, None, 0]) + cute.copy(tiled_copy_K, tKgK[None, None, None, 0], tKsK[None, None, None, 0]) + cute.copy(tiled_copy_V, tVgV[None, None, None, 0], tVsV[None, None, None, 0]) + cute.arch.cp_async_commit_group() + + for k_tile in range(k_tile_count): + stage = k_tile % 2 + next_stage = 1 - stage + + cute.arch.cp_async_wait_group(0) + cute.arch.sync_threads() + + if k_tile + 1 < k_tile_count: + cute.copy( + tiled_copy_K, + tKgK[None, None, None, k_tile + 1], + tKsK[None, None, None, next_stage], + ) + cute.copy( + tiled_copy_V, + tVgV[None, None, None, k_tile + 1], + tVsV[None, None, None, next_stage], + ) + cute.arch.cp_async_commit_group() + + tCsK_p = tCsK_copy_view[None, None, None, stage] + tCsV_p = tCsV_copy_view[None, None, None, stage] + + tCrC.fill(0.0) + + for k_block in cutlass.range(num_k_block, unroll_full=True): + cute.copy( + tiled_copy_s2r_Q, + tCsQ_p[None, None, k_block], + tCrQ_copy_view[None, None, k_block], + ) + cute.copy( + tiled_copy_s2r_K, + tCsK_p[None, None, k_block], + tCrK_copy_view[None, None, k_block], + ) + cute.gemm( + tiled_mma, + tCrC, + tCrQ[None, None, k_block], + tCrK[None, None, k_block], + tCrC, + ) + + for m_tile in cutlass.range_constexpr(2): + for row in cutlass.range_constexpr(2): + state = m_tile * 2 + row + tCrC_row = tCrC[(None, row), m_tile, None] + row_scores = tCrC_row.load() * qk_scale + + if cutlass.const_expr(mK.shape[0] != sequence): + tCrC_row.store(row_scores) + for column_group in cutlass.range_constexpr(4): + for column_pair in cutlass.range_constexpr(2): + key_row = ( + k_tile * 32 + + (tidx % 4) * 2 + + column_group * 8 + + column_pair + ) + if key_row >= sequence: + tCrC_row[ + column_pair, column_group + ] = -cutlass.Float32.inf + row_scores = tCrC_row.load() + + local_max = row_scores.reduce( + cute.ReductionOp.MAX, + -cutlass.Float32.inf, + 0, + ) + + neighbor_max = cute.arch.shuffle_sync_bfly(local_max, offset=1) + pair_max = cute.arch.fmax(local_max, neighbor_max) + + neighbor_max = cute.arch.shuffle_sync_bfly(pair_max, offset=2) + tile_max = cute.arch.fmax(pair_max, neighbor_max) + + m_new = cute.arch.fmax(m[state], tile_max) + + alpha = cute.math.exp2( + (m[state] - m_new) * math.log2(math.e), + fastmath=True, + ) + p = cute.math.exp2( + (row_scores - m_new) * math.log2(math.e), + fastmath=True, + ) + + p_partial_sums = cute.make_rmem_tensor((4,), cutlass.Float32) + for column_group in cutlass.range_constexpr(4): + p_partial_sums[column_group] = ( + p[0, column_group] + p[1, column_group] + ) + + for level in cutlass.range_constexpr(2): + for sum_index in cutlass.range_constexpr(2 >> level): + p_partial_sums[sum_index] = ( + p_partial_sums[2 * sum_index] + + p_partial_sums[2 * sum_index + 1] + ) + tile_sum = p_partial_sums[0] + + l[state] = alpha * l[state] + tile_sum + m[state] = m_new + + tCrC_row.store(p) + + tCrO_row = tCrO[(None, row), m_tile, None] + tCrO_row.store(tCrO_row.load() * alpha) + + tCrP_as_pack.store((tCrC_as_pack.load() * 256.0).to(mQ.element_type)) + + for k_block in cutlass.range(num_k_block_PV, unroll_full=True): + cute.copy( + tiled_copy_s2r_V, + tCsV_p[None, None, k_block], + tCrV_copy_view[None, None, k_block], + ) + cute.gemm( + tiled_mma, + tCrO, + tCrP[None, None, k_block], + tCrV[None, None, k_block], + tCrO, + ) + + warp_id = tidx // 32 + lane_id = tidx % 32 + quad_id = lane_id // 4 + + for m_tile in cutlass.range_constexpr(2): + for row in cutlass.range_constexpr(2): + state = m_tile * 2 + row + row_sum = l[state] + + neighbor_sum = cute.arch.shuffle_sync_bfly(row_sum, offset=1) + row_sum += neighbor_sum + + neighbor_sum = cute.arch.shuffle_sync_bfly(row_sum, offset=2) + row_sum += neighbor_sum + + l[state] = row_sum + lse = m[state] + cute.math.log(row_sum) + + query_row = bidx * 128 + m_tile * 64 + warp_id * 16 + quad_id + row * 8 + + valid_query = True + if cutlass.const_expr(sequence % 128 != 0): + valid_query = query_row < sequence + + if valid_query and lane_id % 4 == 0: + mLSE[query_row, bidz] = lse + + tCrO_row = tCrO[(None, row), m_tile, None] + inverse_row_sum = pv_scale / row_sum + tCrO_row.store(tCrO_row.load() * inverse_row_sum) + + if valid_query: + tCrO_row_out = cute.make_fragment_like(tCrO_row, mO.element_type) + tCrO_row_out.store(tCrO_row.load().to(mO.element_type)) + cute.autovec_copy(tCrO_row_out, tCgO[(None, row), m_tile, None]) + + +@cute.jit +def fp8_attention_host( + mQ: cute.Tensor, + mK: cute.Tensor, + mV: cute.Tensor, + mO: cute.Tensor, + mLSE: cute.Tensor, + mScales: cute.Tensor, + softmax_scale: cutlass.Float32, + stream, +): + mQ = cute.make_tensor(mQ.iterator, cute.select(mQ.layout, mode=[1, 2, 0])) + mK = cute.make_tensor(mK.iterator, cute.select(mK.layout, mode=[1, 2, 0])) + mV = cute.make_tensor(mV.iterator, cute.select(mV.layout, mode=[1, 2, 0])) + mO = cute.make_tensor(mO.iterator, cute.select(mO.layout, mode=[1, 2, 0])) + mLSE = cute.make_tensor(mLSE.iterator, cute.select(mLSE.layout, mode=[1, 0])) + + mma_op = cute.nvgpu.warp.MmaFP8Op( + mQ.element_type, + cutlass.Float32, + (16, 8, 32), + ) + tiled_mma = cute.make_tiled_mma( + mma_op, + (4, 1, 1), + permutation_mnk=(128, 16, 32), + ) + + copy_bits = 128 + num_threads = 128 + + sQ_layout, sQ_swizzle = _make_smem_layout_fp8( + mQ.element_type, + copy_bits, + (128, 128, 1), + ) + sK_layout, sK_swizzle = _make_smem_layout_fp8( + mK.element_type, + copy_bits, + (32, 128, 2), + ) + sV_layout, sV_swizzle = _make_smem_layout_fp8( + mV.element_type, + copy_bits, + (128, 32, 2), + ) + + atom_copy_g2s = cute.make_copy_atom( + cute.nvgpu.cpasync.CopyG2SOp(cache_mode=cute.nvgpu.LoadCacheMode.GLOBAL), + mQ.element_type, + num_bits_per_copy=copy_bits, + ) + tiled_copy_Q = _make_gmem_tiled_copy( + atom_copy_g2s, + mQ.element_type, + copy_bits, + 128, + num_threads, + ) + tiled_copy_K = _make_gmem_tiled_copy( + atom_copy_g2s, + mK.element_type, + copy_bits, + 128, + num_threads, + ) + tiled_copy_V = _make_gmem_tiled_copy( + atom_copy_g2s, + mV.element_type, + copy_bits, + 32, + num_threads, + ) + + _fp8_attention_sm120( + softmax_scale, + mQ, + mK, + mV, + mO, + mLSE, + mScales, + sQ_layout, + sK_layout, + sV_layout, + sQ_swizzle, + sK_swizzle, + sV_swizzle, + tiled_copy_Q, + tiled_copy_K, + tiled_copy_V, + tiled_mma, + ).launch( + grid=(cute.ceil_div(mQ.shape[0], 128), 1, mQ.shape[2]), + block=(num_threads, 1, 1), + stream=stream, + ) + + +def key_order_for_positions(positions): + """Key held at each stored V^T position. + + Inside every 16-key group, position 4t + i holds key 8*(i//2) + 2t + i%2: + the k order of the PV A fragment packed straight from the score fragment. + """ + group = positions // 16 * 16 + quad = (positions % 16) // 4 + element = positions % 4 + return group + 8 * (element // 2) + 2 * quad + element % 2 diff --git a/python/sglang/kernels/ops/attention/fp8_fa_sm120/plan.py b/python/sglang/kernels/ops/attention/fp8_fa_sm120/plan.py new file mode 100644 index 0000000000..b263da3b5c --- /dev/null +++ b/python/sglang/kernels/ops/attention/fp8_fa_sm120/plan.py @@ -0,0 +1,150 @@ +# SPDX-License-Identifier: Apache-2.0 +"""FP8 attention on SGLang's [S, H, 128] strided BF16 views. + +fp8_attention() quantizes Q/K/V per head with the fused Triton passes, runs the CuTe-DSL +kernel and returns (output [S, H, 128] BF16, lse [H, S] FP32). The compiled kernel is +cached per (sequence, heads, device); the E4M3, scale, output and LSE buffers come from +the caching allocator on every call and belong to the caller, nothing is retained +between calls. +""" + +import logging +import math + +import cuda.bindings.driver as cuda +import cutlass +import cutlass.cute as cute +import msgspec +import torch +from cutlass.cute.runtime import from_dlpack + +from sglang.kernels.ops.attention.fp8_fa_sm120.fused_prep import fused_prepare +from sglang.kernels.ops.attention.fp8_fa_sm120.kernel import fp8_attention_host + +logger = logging.getLogger(__name__) + +HEAD_DIM = 128 +# Q and K/V are padded separately; the kernel compiles its key mask only when S % 32 != 0. +QUERY_TILE = 128 +KEY_TILE = 32 + +# Compiled kernels only, KBs of host state each and about 10 s to build; the buffers +# they run on are allocated per call. +_KERNEL_CACHE: dict[tuple, object] = {} + + +def validate_inputs(q, k, v, softmax_scale): + """Strided [S, H, 128] BF16 views on one CUDA device; returns the scale as float.""" + if ( + q.ndim != 3 + or q.shape[-1] != HEAD_DIM + or q.shape != k.shape + or q.shape != v.shape + ): + raise ValueError("Q/K/V must have equal shape [sequence, heads, 128]") + if min(q.shape[:2]) < 1: + raise ValueError("Sequence and heads must be positive") + for tensor in (q, k, v): + if ( + not tensor.is_cuda + or tensor.device != q.device + or tensor.dtype != torch.bfloat16 + ): + raise ValueError("Q/K/V must share a CUDA device and use BF16") + if tensor.stride(-1) != 1 or tensor.stride(-2) != HEAD_DIM: + raise ValueError( + "Feature and head dimensions must be contiguous within each token" + ) + if tensor.data_ptr() % 16 or tensor.stride(0) % 8: + raise ValueError("Token rows must be aligned to 16 bytes") + if tensor.stride() != q.stride(): + raise ValueError("Q/K/V must share strides") + scale = HEAD_DIM**-0.5 if softmax_scale is None else float(softmax_scale) + if not math.isfinite(scale): + raise ValueError("Softmax scale must be finite") + return scale + + +def kernel_key(q): + """Cache key: the kernel bakes the buffer shapes, which follow from (S, H).""" + return (q.shape[0], q.shape[1], q.device.index) + + +class Workspace(msgspec.Struct, frozen=True, kw_only=True): + """Kernel-side buffers for one call; the kernel works in [H, S, 128].""" + + q_fp8: torch.Tensor + k_fp8: torch.Tensor + v_fp8: torch.Tensor + scales: torch.Tensor + maximums: torch.Tensor + output: torch.Tensor + lse: torch.Tensor + + +def _allocate_workspace(q): + sequence = q.shape[0] + heads = q.shape[1] + padded_queries = (sequence + QUERY_TILE - 1) // QUERY_TILE * QUERY_TILE + padded_keys = (sequence + KEY_TILE - 1) // KEY_TILE * KEY_TILE + device = q.device + return Workspace( + q_fp8=torch.empty( + (heads, padded_queries, HEAD_DIM), device=device, dtype=torch.float8_e4m3fn + ), + k_fp8=torch.empty( + (heads, padded_keys, HEAD_DIM), device=device, dtype=torch.float8_e4m3fn + ), + v_fp8=torch.empty( + (heads, HEAD_DIM, padded_keys), device=device, dtype=torch.float8_e4m3fn + ), + scales=torch.empty((3, heads), device=device, dtype=torch.float32), + maximums=torch.zeros((3, heads), device=device, dtype=torch.float32), + output=torch.empty(q.shape, device=device, dtype=torch.bfloat16), + lse=torch.empty((heads, sequence), device=device, dtype=torch.float32), + ) + + +def _kernel_views(workspace): + # Byte DLPack views work across Torch versions without FP8 DLPack support. + mQ = from_dlpack(workspace.q_fp8.view(torch.uint8), assumed_align=16) + mK = from_dlpack(workspace.k_fp8.view(torch.uint8), assumed_align=16) + mV = from_dlpack(workspace.v_fp8.view(torch.uint8), assumed_align=16) + mQ.element_type = cutlass.Float8E4M3FN + mK.element_type = cutlass.Float8E4M3FN + mV.element_type = cutlass.Float8E4M3FN + mO = from_dlpack(workspace.output.permute(1, 0, 2), assumed_align=16) + mLSE = from_dlpack(workspace.lse, assumed_align=16) + mScales = from_dlpack(workspace.scales, assumed_align=16) + return (mQ, mK, mV, mO, mLSE, mScales) + + +def fp8_attention(q, k, v, softmax_scale=None): + """Attention over strided [S, H, 128] BF16 views; returns (output [S, H, 128], lse [H, S]).""" + scale = validate_inputs(q, k, v, softmax_scale) + kernel_scale = cutlass.Float32(scale) + key = kernel_key(q) + + with torch.cuda.device(q.device): + workspace = _allocate_workspace(q) + views = _kernel_views(workspace) + stream = cuda.CUstream(torch.cuda.current_stream(q.device).cuda_stream) + + compiled = _KERNEL_CACHE.get(key) + if compiled is None: + logger.info( + "fp8_fa_sm120 attention: compiling for S=%d H=%d (once per shape)", + q.shape[0], + q.shape[1], + ) + compiled = cute.compile(fp8_attention_host, *views, kernel_scale, stream) + _KERNEL_CACHE[key] = compiled + + fused_prepare( + q=q.permute(1, 0, 2), + k=k.permute(1, 0, 2), + v=v.permute(1, 0, 2), + workspace=workspace, + ) + compiled(*views, kernel_scale, stream) + return workspace.output, workspace.lse diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/fp8_fa_sm120_attn.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/fp8_fa_sm120_attn.py new file mode 100644 index 0000000000..b8faa26f75 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/fp8_fa_sm120_attn.py @@ -0,0 +1,191 @@ +# SPDX-License-Identifier: Apache-2.0 +"""FP8 attention backend for SM120 GPUs (GeForce RTX 50, RTX PRO Blackwell). + +Opt in with ``--attention-backend fp8_fa_sm120``. Q, K and V are quantized per head to +E4M3 on the fly (two fused Triton passes), attention runs in FP8 with FP32 +accumulation, and the output is BF16. Expect about 5% relative RMS against cuDNN BF16 +per call on normally distributed inputs. + +Scope: dense, non-causal, batch 1, head_dim 128, BF16 inputs on an SM120 device. +Everything else goes to cuDNN SDPA. Each distinct sequence length compiles once +(about 10 s); later calls with the same shape reuse the compiled kernel. The FP8 and +output buffers are allocated per call and belong to the caller. +""" + +import torch + +from sglang.kernels.ops.attention.fp8_fa_sm120 import HEAD_DIM, fp8_attention +from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import ( + AttentionBackend, + AttentionImpl, + AttentionMetadata, + trailing_padding_used_len, +) +from sglang.multimodal_gen.runtime.layers.attention.backends.sdpa import CudnnSDPAImpl +from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +class FP8FlashAttentionSM120Backend(AttentionBackend): + accept_output_buffer: bool = False + + @staticmethod + def get_supported_head_sizes() -> list[int]: + return [HEAD_DIM] + + @staticmethod + def get_enum() -> AttentionBackendEnum: + return AttentionBackendEnum.FP8_FA_SM120 + + @staticmethod + def get_impl_cls() -> type["FP8FlashAttentionSM120Impl"]: + return FP8FlashAttentionSM120Impl + + +class FP8FlashAttentionSM120Impl(AttentionImpl): + def __init__( + self, + num_heads: int, + head_size: int, + causal: bool, + softmax_scale: float, + num_kv_heads: int | None = None, + prefix: str = "", + **extra_impl_args, + ) -> None: + self.num_heads = num_heads + self.head_size = head_size + self.causal = causal + self.softmax_scale = softmax_scale + self.fallback = CudnnSDPAImpl( + num_heads=num_heads, + head_size=head_size, + causal=causal, + softmax_scale=softmax_scale, + num_kv_heads=num_kv_heads, + prefix=prefix, + **extra_impl_args, + ) + self._reported_fallbacks: set[str] = set() + + # --- dispatch ------------------------------------------------------------- + + def _fallback_reason(self, query, key, value) -> str | None: + """None when the kernel applies to these [S, H, D] tensors, else why not.""" + if self.causal: + return "causal attention" + if query.shape != key.shape or key.shape != value.shape: + return "Q/K/V shapes differ" + if query.ndim != 3 or query.shape[-1] != HEAD_DIM: + return f"head_dim {query.shape[-1]} (kernel is head_dim {HEAD_DIM})" + if query.dtype != torch.bfloat16: + return f"dtype {query.dtype} (kernel takes BF16)" + if not query.is_cuda: + return "non-CUDA tensors" + if torch.cuda.get_device_capability(query.device) != (12, 0): + return "device is not SM120" + return None + + def _report_fallback(self, reason: str) -> None: + if reason in self._reported_fallbacks: + return + self._reported_fallbacks.add(reason) + logger.warning( + "fp8_fa_sm120 attention: %s; using cuDNN SDPA for these calls.", reason + ) + + # --- kernel path ---------------------------------------------------------- + + def _run(self, query, key, value): + """[S, H, 128] strided BF16 in, a new contiguous [S, H, 128] BF16 out.""" + output, _ = fp8_attention(query, key, value, softmax_scale=self.softmax_scale) + return output + + # --- AttentionImpl -------------------------------------------------------- + + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + # [B, S, H, D]; the kernel handles one dense sequence at a time. + if query.shape[0] != 1: + self._report_fallback(f"batch {query.shape[0]}") + return self.fallback.forward(query, key, value, attn_metadata) + reason = self._fallback_reason(query[0], key[0], value[0]) + if reason is not None: + self._report_fallback(reason) + return self.fallback.forward(query, key, value, attn_metadata) + try: + output = self._run(query[0], key[0], value[0]) + except ValueError as error: + self._report_fallback(str(error)) + return self.fallback.forward(query, key, value, attn_metadata) + return output.unsqueeze(0) + + def forward_varlen( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + *, + cu_seqlens: torch.Tensor, + max_seqlen: int, + cu_seqlens_host: tuple[int, ...] | None = None, + ) -> torch.Tensor: + bounds = ( + cu_seqlens_host + if cu_seqlens_host is not None + else tuple(int(item) for item in cu_seqlens.tolist()) + ) + reason = self._fallback_reason(query, key, value) + if reason is not None: + self._report_fallback(reason) + return self.fallback.forward_varlen( + query, + key, + value, + cu_seqlens=cu_seqlens, + max_seqlen=max_seqlen, + cu_seqlens_host=bounds, + ) + + # MiniMax-H3 packs one live document as (0, used, total); the tail is + # 64-aligned padding that downstream masks, so it stays zero. + used = trailing_padding_used_len( + total_tokens=query.shape[0], + max_seqlen=max_seqlen, + bounds=bounds, + ) + try: + if used is not None: + live_output = self._run(query[:used], key[:used], value[:used]) + if used == query.shape[0]: + return live_output + output = torch.zeros_like(query) + output[:used].copy_(live_output) + return output + + output = torch.empty_like(query) + for start, stop in zip(bounds[:-1], bounds[1:]): + if start == stop: + continue + segment_output = self._run( + query[start:stop], key[start:stop], value[start:stop] + ) + output[start:stop].copy_(segment_output) + return output + except ValueError as error: + self._report_fallback(str(error)) + return self.fallback.forward_varlen( + query, + key, + value, + cu_seqlens=cu_seqlens, + max_seqlen=max_seqlen, + cu_seqlens_host=bounds, + ) diff --git a/python/sglang/multimodal_gen/runtime/platforms/cuda.py b/python/sglang/multimodal_gen/runtime/platforms/cuda.py index f1878f8f0f..19388b1b9f 100644 --- a/python/sglang/multimodal_gen/runtime/platforms/cuda.py +++ b/python/sglang/multimodal_gen/runtime/platforms/cuda.py @@ -516,6 +516,36 @@ class _FlashAttentionBackendResolver(_CudaAttentionBackendResolver): return AttentionBackendEnum.FA +class _FP8FlashAttentionSM120BackendResolver(_CudaAttentionBackendResolver): + backend = AttentionBackendEnum.FP8_FA_SM120 + + # CuTe-DSL mma.sync FP8 kernel written for SM120 (GeForce RTX 50 / RTX PRO + # Blackwell). Dense non-causal head_dim 128 only; the backend itself falls + # back to cuDNN SDPA for other calls. + @classmethod + def resolve(cls, platform) -> str | AttentionBackendEnum: + if platform.get_device_capability() != (12, 0): + logger.warning( + "fp8_fa_sm120 attention needs an SM120 device; falling back to cuDNN SDPA." + ) + return AttentionBackendEnum.TORCH_CUDNN_SDPA + try: + import cutlass.cute # noqa: F401 + import triton # noqa: F401 + + from sglang.multimodal_gen.runtime.layers.attention.backends.fp8_fa_sm120_attn import ( # noqa: F401 + FP8FlashAttentionSM120Backend, + ) + + return "sglang.multimodal_gen.runtime.layers.attention.backends.fp8_fa_sm120_attn.FP8FlashAttentionSM120Backend" + except ImportError as error: + logger.warning( + "fp8_fa_sm120 attention backend failed to import (%s); falling back to cuDNN SDPA.", + error, + ) + return AttentionBackendEnum.TORCH_CUDNN_SDPA + + _CUDA_ATTENTION_BACKEND_RESOLVERS = { resolver.backend: resolver for resolver in ( @@ -539,6 +569,7 @@ _CUDA_ATTENTION_BACKEND_RESOLVERS = { _SubBlockSparseAttentionBackendResolver, _FlashAttention2BackendResolver, _FlashAttentionBackendResolver, + _FP8FlashAttentionSM120BackendResolver, ) } diff --git a/python/sglang/multimodal_gen/runtime/platforms/interface.py b/python/sglang/multimodal_gen/runtime/platforms/interface.py index 7d971204fa..2a2a68be4b 100644 --- a/python/sglang/multimodal_gen/runtime/platforms/interface.py +++ b/python/sglang/multimodal_gen/runtime/platforms/interface.py @@ -50,6 +50,7 @@ class AttentionBackendEnum(enum.Enum): SOL_ATTN = enum.auto() SUBBLOCK_SPARSE_ATTN = enum.auto() CUBE_SPARSE_ATTN = enum.auto() + FP8_FA_SM120 = enum.auto() NO_ATTENTION = enum.auto() def __str__(self): diff --git a/python/sglang/multimodal_gen/test/unit/test_fp8_fa_sm120_attn.py b/python/sglang/multimodal_gen/test/unit/test_fp8_fa_sm120_attn.py new file mode 100644 index 0000000000..c240c8f1b7 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_fp8_fa_sm120_attn.py @@ -0,0 +1,222 @@ +# SPDX-License-Identifier: Apache-2.0 +"""fp8_fa_sm120 attention backend against cuDNN SDPA on MiniMax-H3 style inputs. + +Needs an SM120 GPU; skipped elsewhere. Shapes are kept small (8 heads) so the +CuTe-DSL compile and the runs stay in seconds. +""" + +import pytest +import torch + +pytestmark = pytest.mark.skipif( + not torch.cuda.is_available() or torch.cuda.get_device_capability() != (12, 0), + reason="fp8_fa_sm120 attention needs an SM120 GPU", +) + +HEADS = 8 +HEAD_DIM = 128 +SOFTMAX_SCALE = HEAD_DIM**-0.5 + + +def _make_impls(causal=False): + from sglang.multimodal_gen.runtime.layers.attention.backends.fp8_fa_sm120_attn import ( + FP8FlashAttentionSM120Impl, + ) + from sglang.multimodal_gen.runtime.layers.attention.backends.sdpa import ( + CudnnSDPAImpl, + ) + + ours = FP8FlashAttentionSM120Impl( + num_heads=HEADS, head_size=HEAD_DIM, causal=causal, softmax_scale=SOFTMAX_SCALE + ) + reference = CudnnSDPAImpl( + num_heads=HEADS, head_size=HEAD_DIM, causal=causal, softmax_scale=SOFTMAX_SCALE + ) + return ours, reference + + +def _fused_qkv_views(sequence, seed=0): + """Q/K/V as views into one [S, 3*H*D] projection output, like the DiT hands them over.""" + generator = torch.Generator(device="cuda").manual_seed(seed) + qkv = torch.randn( + (sequence, 3 * HEADS * HEAD_DIM), + device="cuda", + dtype=torch.bfloat16, + generator=generator, + ) + q = qkv[:, 0 : HEADS * HEAD_DIM].view(sequence, HEADS, HEAD_DIM) + k = qkv[:, HEADS * HEAD_DIM : 2 * HEADS * HEAD_DIM].view(sequence, HEADS, HEAD_DIM) + v = qkv[:, 2 * HEADS * HEAD_DIM :].view(sequence, HEADS, HEAD_DIM) + return q, k, v + + +def _error_metrics(candidate, reference): + candidate = candidate.float() + reference = reference.float() + relative_rms = (candidate - reference).pow(2).mean().sqrt() / reference.pow( + 2 + ).mean().sqrt() + cosine = torch.nn.functional.cosine_similarity( + candidate.flatten(), reference.flatten(), dim=0 + ) + return float(relative_rms), float(cosine) + + +@pytest.mark.parametrize("sequence", [4096, 5980]) +def test_forward_matches_cudnn(sequence): + ours, reference = _make_impls() + q, k, v = _fused_qkv_views(sequence) + + output = ours.forward(q.unsqueeze(0), k.unsqueeze(0), v.unsqueeze(0), None) + expected = reference.forward(q.unsqueeze(0), k.unsqueeze(0), v.unsqueeze(0), None) + + assert output.shape == expected.shape + assert output.dtype == torch.bfloat16 + assert torch.isfinite(output.float()).all() + relative_rms, cosine = _error_metrics(output, expected) + # Per-head E4M3 quantization of Q/K/V: ~5% relative RMS on normal inputs. + assert relative_rms < 0.08, relative_rms + assert cosine > 0.995, cosine + + +def _compiled_kernels(): + from sglang.kernels.ops.attention.fp8_fa_sm120.plan import _KERNEL_CACHE + + return len(_KERNEL_CACHE) + + +def test_kernel_is_reused_across_calls(): + ours, reference = _make_impls() + q, k, v = _fused_qkv_views(4096, seed=1) + first = ours.forward(q.unsqueeze(0), k.unsqueeze(0), v.unsqueeze(0), None) + kernels_after_first = _compiled_kernels() + + q2, k2, v2 = _fused_qkv_views(4096, seed=2) + second = ours.forward(q2.unsqueeze(0), k2.unsqueeze(0), v2.unsqueeze(0), None) + assert _compiled_kernels() == kernels_after_first + + # A second impl (another DiT layer) shares the same compiled kernel. + other, _ = _make_impls() + other.forward(q2.unsqueeze(0), k2.unsqueeze(0), v2.unsqueeze(0), None) + assert _compiled_kernels() == kernels_after_first + + expected = reference.forward( + q2.unsqueeze(0), k2.unsqueeze(0), v2.unsqueeze(0), None + ) + relative_rms, _ = _error_metrics(second, expected) + assert relative_rms < 0.08, relative_rms + assert not torch.equal(first, second) + + +def test_varlen_trailing_padding_keeps_tail_zero(): + ours, _ = _make_impls() + used = 5980 + total = 6016 # 64-aligned tail padding, as MiniMax-H3 packs it + q, k, v = _fused_qkv_views(total) + cu_seqlens = torch.tensor([0, used, total], device="cuda", dtype=torch.int32) + + output = ours.forward_varlen( + q, + k, + v, + cu_seqlens=cu_seqlens, + max_seqlen=used, + cu_seqlens_host=(0, used, total), + ) + live = ours.forward( + q[:used].unsqueeze(0), k[:used].unsqueeze(0), v[:used].unsqueeze(0), None + )[0] + + assert output.shape == q.shape + assert torch.equal(output[:used], live) + assert torch.count_nonzero(output[used:]) == 0 + + +def test_varlen_multiple_segments(): + ours, reference = _make_impls() + bounds = (0, 1024, 3072, 4096) + q, k, v = _fused_qkv_views(bounds[-1]) + cu_seqlens = torch.tensor(bounds, device="cuda", dtype=torch.int32) + + output = ours.forward_varlen( + q, k, v, cu_seqlens=cu_seqlens, max_seqlen=2048, cu_seqlens_host=bounds + ) + expected = reference.forward_varlen( + q, k, v, cu_seqlens=cu_seqlens, max_seqlen=2048, cu_seqlens_host=bounds + ) + + for start, stop in zip(bounds[:-1], bounds[1:]): + relative_rms, _ = _error_metrics(output[start:stop], expected[start:stop]) + assert relative_rms < 0.08, (start, stop, relative_rms) + + +def test_distinct_shapes_release_memory(): + """A call must not retain GPU buffers keyed by shape: each distinct sequence length + used to keep its FP8/output buffers and the last Q/K/V alive, ~2.2 GiB at H3 size.""" + ours, _ = _make_impls() + shapes = [_fused_qkv_views(1052), _fused_qkv_views(1088)] + q, k, v = shapes[0] + ours.forward(q.unsqueeze(0), k.unsqueeze(0), v.unsqueeze(0), None) + torch.cuda.synchronize() + baseline = torch.cuda.memory_allocated() + + for q, k, v in shapes: + output = ours.forward(q.unsqueeze(0), k.unsqueeze(0), v.unsqueeze(0), None) + assert torch.isfinite(output.float()).all() + del output + + torch.cuda.synchronize() + assert torch.cuda.memory_allocated() == baseline + + +def test_prep_defines_every_padded_byte(): + """The FP8 buffers come from torch.empty, so the prep must write every padding + position; an E4M3 NaN byte left in the V^T tail makes masked keys poison the row.""" + from sglang.kernels.ops.attention.fp8_fa_sm120.fused_prep import fused_prepare + from sglang.kernels.ops.attention.fp8_fa_sm120.plan import _allocate_workspace + + q, k, v = _fused_qkv_views(1052) + workspace = _allocate_workspace(q) + for buffer in (workspace.q_fp8, workspace.k_fp8, workspace.v_fp8): + buffer.view(torch.uint8).fill_(0xFF) + + fused_prepare( + q=q.permute(1, 0, 2), + k=k.permute(1, 0, 2), + v=v.permute(1, 0, 2), + workspace=workspace, + ) + + for buffer in (workspace.q_fp8, workspace.k_fp8, workspace.v_fp8): + raw = buffer.view(torch.uint8) + assert not ((raw == 0xFF) | (raw == 0x7F)).any() + + +def test_causal_falls_back_to_cudnn(): + ours, reference = _make_impls(causal=True) + q, k, v = _fused_qkv_views(1024) + kernels_before = _compiled_kernels() + + output = ours.forward(q.unsqueeze(0), k.unsqueeze(0), v.unsqueeze(0), None) + expected = reference.forward(q.unsqueeze(0), k.unsqueeze(0), v.unsqueeze(0), None) + + assert torch.equal(output, expected) + assert _compiled_kernels() == kernels_before + + +def test_backend_resolves_by_name(): + from sglang.multimodal_gen.runtime.platforms import ( + AttentionBackendEnum, + current_platform, + ) + + class_path = current_platform.get_attn_backend_cls_str( + AttentionBackendEnum.FP8_FA_SM120, HEAD_DIM, torch.bfloat16 + ) + assert class_path.endswith("fp8_fa_sm120_attn.FP8FlashAttentionSM120Backend") + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main([__file__, "-v"]))