[diffusion] attention: add fp8_fa_sm120 FP8 backend for SM120 GPUs (#40175)

Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
Emre Albayrak
2026-09-22 17:44:37 +08:00
committed by GitHub
co-authored by Xiaoyu Zhang
parent debbb5cde9
commit 9d58189c12
9 changed files with 1329 additions and 0 deletions
@@ -0,0 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
"""SM120 FP8 attention: CuTe-DSL kernel, fused Triton prep and the per-call entry point."""
from sglang.kernels.ops.attention.fp8_fa_sm120.plan import (
HEAD_DIM,
Workspace,
fp8_attention,
kernel_key,
validate_inputs,
)
__all__ = ["HEAD_DIM", "Workspace", "fp8_attention", "kernel_key", "validate_inputs"]
@@ -0,0 +1,232 @@
# SPDX-License-Identifier: Apache-2.0
"""Fused FP8 preparation for the SM120 kernel: two Triton passes into the workspace.
Replaces a Torch chain (FP32 copy, amax, divide, cast,
padded copy, V transpose copy) with one per-head amax pass and one pack pass that
writes the kernel's own buffers directly:
Q/K [H, padded_S, 128] same token/column order as the input
V [H, 128, padded_S] transposed here, so the kernel stays unchanged
Scales stay per head and the quantization recipe is the same (amax/448, div.rn,
clamp, E4M3), so the kernel's numerics do not move. The pack pass writes every
position of the padded buffers, padding included, because the workspace comes
from torch.empty.
"""
import torch
import triton
import triton.language as tl
# Power of two >= 16: the V^T key permutation works in groups of 16 tokens.
BLOCK_TOKENS = 64
# Triton's `/` lowers to div.full.f32 (approximate); Torch divides with div.rn.
# The FP8 bytes must match the Torch prepare() path, so the exact division is kept.
@triton.jit
def _divide_rn(numerator, denominator):
return tl.inline_asm_elementwise(
"div.rn.f32 $0, $1, $2;",
"=f,f,f",
[numerator, denominator],
dtype=tl.float32,
is_pure=True,
pack=1,
)
@triton.jit
def _load_tile(
base,
head,
tokens,
columns,
valid,
head_stride: tl.constexpr,
token_stride: tl.constexpr,
):
offsets = head * head_stride + tokens[:, None] * token_stride + columns[None, :]
return tl.load(base + offsets, mask=valid, other=0.0).to(tl.float32)
@triton.jit
def _quantize(values, scale):
scaled = _divide_rn(values, scale)
scaled = tl.maximum(tl.minimum(scaled, 448.0), -448.0)
return scaled.to(tl.float8e4nv)
# Pass 1: one program per (head, token block); each reduces its Q/K/V tile to
# three scalars and folds them into maximums[operand, head].
@triton.jit
def _head_amax_qkv(
q,
k,
v,
maximums,
sequence,
head_stride: tl.constexpr,
token_stride: tl.constexpr,
heads: tl.constexpr,
block_tokens: tl.constexpr,
):
head = tl.program_id(0)
tokens = tl.program_id(1) * block_tokens + tl.arange(0, block_tokens)
columns = tl.arange(0, 128)
valid = tokens[:, None] < sequence
q_tile = _load_tile(q, head, tokens, columns, valid, head_stride, token_stride)
tl.atomic_max(maximums + 0 * heads + head, tl.max(tl.abs(q_tile)))
k_tile = _load_tile(k, head, tokens, columns, valid, head_stride, token_stride)
tl.atomic_max(maximums + 1 * heads + head, tl.max(tl.abs(k_tile)))
v_tile = _load_tile(v, head, tokens, columns, valid, head_stride, token_stride)
tl.atomic_max(maximums + 2 * heads + head, tl.max(tl.abs(v_tile)))
# Torch evaluates `tensor / 448.0` as `tensor * (1/448)` with the reciprocal in
# FP32 (CPU-scalar divisor fast path). The same product keeps the scales bit-equal.
INVERSE_FP8_MAX = (torch.ones((), dtype=torch.float32) / 448.0).item()
# scales[operand, head] = amax * (1/448), or 1 for an all-zero head.
@triton.jit
def _head_scales(
maximums,
scales,
inverse_fp8_max,
count: tl.constexpr,
block: tl.constexpr,
):
offsets = tl.arange(0, block)
valid = offsets < count
maximum = tl.load(maximums + offsets, mask=valid, other=0.0)
scale = tl.where(maximum > 0.0, maximum * inverse_fp8_max, 1.0)
tl.store(scales + offsets, scale, mask=valid)
# Pass 2: one program per (head, token block) up to padded_queries. Q/K tiles go
# out in [token, column] order; the V tile is transposed in registers/shared and
# goes out in [column, token]. Loads are masked on the input token, stores on the
# buffer position, so padding positions receive quantized zeros.
@triton.jit
def _pack_qkv(
q,
k,
v,
q_fp8,
k_fp8,
v_fp8,
scales,
sequence,
head_stride: tl.constexpr,
token_stride: tl.constexpr,
padded_queries: tl.constexpr,
padded_keys: tl.constexpr,
heads: tl.constexpr,
block_tokens: tl.constexpr,
):
head = tl.program_id(0)
tokens = tl.program_id(1) * block_tokens + tl.arange(0, block_tokens)
columns = tl.arange(0, 128)
valid = tokens[:, None] < sequence
q_scale = tl.load(scales + 0 * heads + head)
k_scale = tl.load(scales + 1 * heads + head)
v_scale = tl.load(scales + 2 * heads + head)
# Q is padded to the 128-row query tile, K/V to the 32-key tile.
hsd_offsets = head * padded_queries * 128 + tokens[:, None] * 128 + columns[None, :]
k_offsets = head * padded_keys * 128 + tokens[:, None] * 128 + columns[None, :]
in_queries = tokens[:, None] < padded_queries
in_keys = tokens[:, None] < padded_keys
q_tile = _load_tile(q, head, tokens, columns, valid, head_stride, token_stride)
tl.store(q_fp8 + hsd_offsets, _quantize(q_tile, q_scale), mask=in_queries)
k_tile = _load_tile(k, head, tokens, columns, valid, head_stride, token_stride)
tl.store(k_fp8 + k_offsets, _quantize(k_tile, k_scale), mask=in_keys)
# The kernel packs P straight from the score fragment; V^T's stored key
# order must follow it: stored position p holds key
# p//16*16 + 8*((p%4)//2) + 2*((p%16)//4) + p%2 (see key_order_for_positions).
# The permutation is applied on the LOAD side as a row gather: every gathered
# row is still 256 contiguous bytes, and the transposed store stays contiguous.
# Permuting the store instead scatters bytes inside 16 B windows and doubled
# the prepare time.
source_tokens = (
tokens // 16 * 16
+ 8 * ((tokens % 4) // 2)
+ 2 * ((tokens % 16) // 4)
+ tokens % 2
)
valid_source = source_tokens[:, None] < sequence
hds_offsets = (
head * 128 * padded_keys + columns[:, None] * padded_keys + tokens[None, :]
)
in_keys_transposed = tokens[None, :] < padded_keys
v_tile = _load_tile(
v, head, source_tokens, columns, valid_source, head_stride, token_stride
)
tl.store(
v_fp8 + hds_offsets,
tl.trans(_quantize(v_tile, v_scale)),
mask=in_keys_transposed,
)
def fused_prepare(q, k, v, workspace):
"""Three launches: per-head amax, scale finalize, direct pack into the workspace.
q, k, v are [H, S, 128] views with one shared stride set; the workspace buffers are
sized for this S and H.
"""
heads = q.shape[0]
sequence = q.shape[1]
padded_queries = workspace.q_fp8.shape[1]
padded_keys = workspace.k_fp8.shape[1]
head_stride = q.stride(0)
token_stride = q.stride(1)
with torch.cuda.device(q.device):
_head_amax_qkv[(heads, triton.cdiv(sequence, BLOCK_TOKENS))](
q,
k,
v,
workspace.maximums,
sequence,
head_stride=head_stride,
token_stride=token_stride,
heads=heads,
block_tokens=BLOCK_TOKENS,
num_warps=8,
)
_head_scales[(1,)](
workspace.maximums,
workspace.scales,
INVERSE_FP8_MAX,
count=3 * heads,
block=triton.next_power_of_2(3 * heads),
num_warps=1,
)
_pack_qkv[(heads, triton.cdiv(padded_queries, BLOCK_TOKENS))](
q,
k,
v,
workspace.q_fp8,
workspace.k_fp8,
workspace.v_fp8,
workspace.scales,
sequence,
head_stride=head_stride,
token_stride=token_stride,
padded_queries=padded_queries,
padded_keys=padded_keys,
heads=heads,
block_tokens=BLOCK_TOKENS,
num_warps=8,
)
@@ -0,0 +1,475 @@
# SPDX-License-Identifier: Apache-2.0
"""FP8 flash-attention forward for SM120 in CuTe-DSL: E4M3 QK and PV, FP32 softmax.
Q, K [H, padded_S, 128] E4M3, queries padded to 128 rows, keys to 32
V^T [H, 128, padded_S] E4M3, keys stored in key_order_for_positions() order
scales [3, H] FP32 per-head descale of Q, K, V
O [H, S, 128] BF16
LSE [H, S] FP32
One CTA covers 128 queries x 32 keys on 128 threads: 4 warps, 32 query rows per warp.
Every thread keeps the online-softmax state (running max m, running sum l) of its four
query rows in registers; the max is reduced across the quad with two butterfly shuffles
per key tile, the sum once after the key loop.
Per key tile: K and V^T arrive in one of two shared-memory stages and the next tile is
requested right after the barrier, so the copy overlaps the math. QK runs as four E4M3
mma.sync blocks into FP32 score registers. The probabilities are scaled by 256, cast to
E4M3 and packed in registers from the score fragment into the A fragment of PV; they
never go through shared memory. The A fragment holds the keys of every 16-key group in
a fixed permuted order, which is why V^T is stored in that order. PV accumulates into
FP32 O registers that are rescaled to the new max on every tile and normalized by l at
the end.
Non-causal only. No dropout, no autograd. Inputs must be finite.
"""
import math
import cutlass
import cutlass.cute as cute
import cutlass.utils as utils
def _make_gmem_tiled_copy(atom_copy, dtype, copy_bits, minor_size, num_threads):
copy_elems = copy_bits // dtype.width
shape_dim_1 = minor_size // copy_elems
thread_layout = cute.make_layout(
(num_threads // shape_dim_1, shape_dim_1), stride=(shape_dim_1, 1)
)
value_layout = cute.make_layout((1, copy_elems))
return cute.make_tiled_copy_tv(atom_copy, thread_layout, value_layout)
def _make_smem_layout_fp8(dtype, copy_bits, smem_tiler):
major_size = smem_tiler[1]
row_bytes = major_size * dtype.width // 8
chunk_bytes = copy_bits // 8
swizzle_bits = min(int(math.log2(row_bytes // chunk_bytes)), 3)
base_bits = int(math.log2(chunk_bytes))
shift_bits = int(math.log2(128 // chunk_bytes))
swizzle = cute.make_swizzle(swizzle_bits, base_bits, shift_bits)
atom = cute.make_layout((8, major_size), stride=(major_size, 1))
layout = cute.tile_to_shape(atom, smem_tiler, (0, 1, 2))
return layout, swizzle
@cute.kernel
def _fp8_attention_sm120(
softmax_scale: cutlass.Float32,
mQ: cute.Tensor,
mK: cute.Tensor,
mV: cute.Tensor,
mO: cute.Tensor,
mLSE: cute.Tensor,
mScales: cute.Tensor,
sQ_layout: cute.Layout,
sK_layout: cute.Layout,
sV_layout: cute.Layout,
sQ_swizzle: cute.Swizzle,
sK_swizzle: cute.Swizzle,
sV_swizzle: cute.Swizzle,
tiled_copy_Q: cute.TiledCopy,
tiled_copy_K: cute.TiledCopy,
tiled_copy_V: cute.TiledCopy,
tiled_mma: cute.TiledMma,
cta_tiler: cutlass.Constexpr = (128, 32, 128),
):
tidx, _, _ = cute.arch.thread_idx()
bidx, _, bidz = cute.arch.block_idx()
qk_scale = mScales[0, bidz] * mScales[1, bidz] * softmax_scale
pv_scale = mScales[2, bidz] / 256.0
sequence = mLSE.shape[0]
m = cute.make_rmem_tensor((4,), cutlass.Float32)
m.fill(-cutlass.Float32.inf)
l = cute.make_rmem_tensor((4,), cutlass.Float32)
l.fill(0.0)
gQ = cute.local_tile(
mQ[None, None, bidz], cta_tiler, (bidx, None, 0), proj=(1, None, 1)
)
gK = cute.local_tile(
mK[None, None, bidz], cta_tiler, (None, None, 0), proj=(None, 1, 1)
)
gV = cute.local_tile(mV[None, None, bidz], (128, 32), (0, None))
gO = cute.local_tile(mO[None, None, bidz], (128, 128), (bidx, 0))
@cute.struct
class SharedStorageQKV:
q: cute.struct.Align[
cute.struct.MemRange[mQ.element_type, cute.cosize(sQ_layout)], 16
]
k: cute.struct.Align[
cute.struct.MemRange[mK.element_type, cute.cosize(sK_layout)], 16
]
v: cute.struct.Align[
cute.struct.MemRange[mV.element_type, cute.cosize(sV_layout)], 16
]
smem = utils.SmemAllocator()
storage = smem.allocate(SharedStorageQKV.size_in_bytes(), byte_alignment=16)
sQ = SharedStorageQKV(storage).q.get_tensor(sQ_layout, swizzle=sQ_swizzle)
sK = SharedStorageQKV(storage).k.get_tensor(sK_layout, swizzle=sK_swizzle)
sV = SharedStorageQKV(storage).v.get_tensor(sV_layout, swizzle=sV_swizzle)
thr_copy_Q = tiled_copy_Q.get_slice(tidx)
thr_copy_K = tiled_copy_K.get_slice(tidx)
thr_copy_V = tiled_copy_V.get_slice(tidx)
tQgQ = thr_copy_Q.partition_S(gQ)
tKgK = thr_copy_K.partition_S(gK)
tVgV = thr_copy_V.partition_S(gV)
tQsQ = thr_copy_Q.partition_D(sQ)
tKsK = thr_copy_K.partition_D(sK)
tVsV = thr_copy_V.partition_D(sV)
k_tile_count = cute.size(tKgK, mode=[3])
thr_mma = tiled_mma.get_slice(tidx)
tCgO = thr_mma.partition_C(gO)
tCsQ = thr_mma.partition_A(sQ)
tCsK = thr_mma.partition_B(sK)
tCsV = thr_mma.partition_B(sV)
tCrQ = tiled_mma.make_fragment_A(tCsQ[None, None, None, 0])
tCrK = tiled_mma.make_fragment_B(tCsK[None, None, None, 0])
tCrV = tiled_mma.make_fragment_B(tCsV[None, None, None, 0])
acc_shape = thr_mma.partition_shape_C((128, 32))
tCrC = cute.make_rmem_tensor(acc_shape, cutlass.Float32)
acc_shape_O = thr_mma.partition_shape_C((128, 128))
tCrO = cute.make_rmem_tensor(acc_shape_O, cutlass.Float32)
tCrO.fill(0.0)
tCrP = cute.make_rmem_tensor(
cute.make_layout(((4, 2, 2), 2, 1), stride=((1, 4, 8), 16, 0)),
mQ.element_type,
)
num_k_block_PV = cute.size(tCrP, mode=[2])
pack_shape = ((2, 2), 2, (2, 2))
tCrC_as_pack = cute.make_tensor(
tCrC.iterator,
cute.make_layout(pack_shape, stride=((1, 2), 4, (8, 16))),
)
tCrP_as_pack = cute.make_tensor(
tCrP.iterator,
cute.make_layout(pack_shape, stride=((1, 4), 16, (2, 8))),
)
atom_copy_s2r_Q = cute.make_copy_atom(
cute.nvgpu.warp.LdMatrix8x16x8bOp(transpose=False, num_matrices=4),
mQ.element_type,
)
atom_copy_s2r_K = cute.make_copy_atom(
cute.nvgpu.warp.LdMatrix8x16x8bOp(transpose=False, num_matrices=4),
mK.element_type,
)
atom_copy_s2r_V = cute.make_copy_atom(
cute.nvgpu.warp.LdMatrix8x16x8bOp(transpose=False, num_matrices=4),
mV.element_type,
)
tiled_copy_s2r_Q = cute.make_tiled_copy_A(atom_copy_s2r_Q, tiled_mma)
tiled_copy_s2r_K = cute.make_tiled_copy_B(atom_copy_s2r_K, tiled_mma)
tiled_copy_s2r_V = cute.make_tiled_copy_B(atom_copy_s2r_V, tiled_mma)
ldmatrix_Q = tiled_copy_s2r_Q.get_slice(tidx)
ldmatrix_K = tiled_copy_s2r_K.get_slice(tidx)
ldmatrix_V = tiled_copy_s2r_V.get_slice(tidx)
tCsQ_copy_view = ldmatrix_Q.partition_S(sQ)
tCrQ_copy_view = ldmatrix_Q.retile(tCrQ)
tCsK_copy_view = ldmatrix_K.partition_S(sK)
tCrK_copy_view = ldmatrix_K.retile(tCrK)
tCsV_copy_view = ldmatrix_V.partition_S(sV)
tCrV_copy_view = ldmatrix_V.retile(tCrV)
num_k_block = cute.size(tCrQ, mode=[2])
tCsQ_p = tCsQ_copy_view[None, None, None, 0]
cute.copy(tiled_copy_Q, tQgQ[None, None, None], tQsQ[None, None, None, 0])
cute.copy(tiled_copy_K, tKgK[None, None, None, 0], tKsK[None, None, None, 0])
cute.copy(tiled_copy_V, tVgV[None, None, None, 0], tVsV[None, None, None, 0])
cute.arch.cp_async_commit_group()
for k_tile in range(k_tile_count):
stage = k_tile % 2
next_stage = 1 - stage
cute.arch.cp_async_wait_group(0)
cute.arch.sync_threads()
if k_tile + 1 < k_tile_count:
cute.copy(
tiled_copy_K,
tKgK[None, None, None, k_tile + 1],
tKsK[None, None, None, next_stage],
)
cute.copy(
tiled_copy_V,
tVgV[None, None, None, k_tile + 1],
tVsV[None, None, None, next_stage],
)
cute.arch.cp_async_commit_group()
tCsK_p = tCsK_copy_view[None, None, None, stage]
tCsV_p = tCsV_copy_view[None, None, None, stage]
tCrC.fill(0.0)
for k_block in cutlass.range(num_k_block, unroll_full=True):
cute.copy(
tiled_copy_s2r_Q,
tCsQ_p[None, None, k_block],
tCrQ_copy_view[None, None, k_block],
)
cute.copy(
tiled_copy_s2r_K,
tCsK_p[None, None, k_block],
tCrK_copy_view[None, None, k_block],
)
cute.gemm(
tiled_mma,
tCrC,
tCrQ[None, None, k_block],
tCrK[None, None, k_block],
tCrC,
)
for m_tile in cutlass.range_constexpr(2):
for row in cutlass.range_constexpr(2):
state = m_tile * 2 + row
tCrC_row = tCrC[(None, row), m_tile, None]
row_scores = tCrC_row.load() * qk_scale
if cutlass.const_expr(mK.shape[0] != sequence):
tCrC_row.store(row_scores)
for column_group in cutlass.range_constexpr(4):
for column_pair in cutlass.range_constexpr(2):
key_row = (
k_tile * 32
+ (tidx % 4) * 2
+ column_group * 8
+ column_pair
)
if key_row >= sequence:
tCrC_row[
column_pair, column_group
] = -cutlass.Float32.inf
row_scores = tCrC_row.load()
local_max = row_scores.reduce(
cute.ReductionOp.MAX,
-cutlass.Float32.inf,
0,
)
neighbor_max = cute.arch.shuffle_sync_bfly(local_max, offset=1)
pair_max = cute.arch.fmax(local_max, neighbor_max)
neighbor_max = cute.arch.shuffle_sync_bfly(pair_max, offset=2)
tile_max = cute.arch.fmax(pair_max, neighbor_max)
m_new = cute.arch.fmax(m[state], tile_max)
alpha = cute.math.exp2(
(m[state] - m_new) * math.log2(math.e),
fastmath=True,
)
p = cute.math.exp2(
(row_scores - m_new) * math.log2(math.e),
fastmath=True,
)
p_partial_sums = cute.make_rmem_tensor((4,), cutlass.Float32)
for column_group in cutlass.range_constexpr(4):
p_partial_sums[column_group] = (
p[0, column_group] + p[1, column_group]
)
for level in cutlass.range_constexpr(2):
for sum_index in cutlass.range_constexpr(2 >> level):
p_partial_sums[sum_index] = (
p_partial_sums[2 * sum_index]
+ p_partial_sums[2 * sum_index + 1]
)
tile_sum = p_partial_sums[0]
l[state] = alpha * l[state] + tile_sum
m[state] = m_new
tCrC_row.store(p)
tCrO_row = tCrO[(None, row), m_tile, None]
tCrO_row.store(tCrO_row.load() * alpha)
tCrP_as_pack.store((tCrC_as_pack.load() * 256.0).to(mQ.element_type))
for k_block in cutlass.range(num_k_block_PV, unroll_full=True):
cute.copy(
tiled_copy_s2r_V,
tCsV_p[None, None, k_block],
tCrV_copy_view[None, None, k_block],
)
cute.gemm(
tiled_mma,
tCrO,
tCrP[None, None, k_block],
tCrV[None, None, k_block],
tCrO,
)
warp_id = tidx // 32
lane_id = tidx % 32
quad_id = lane_id // 4
for m_tile in cutlass.range_constexpr(2):
for row in cutlass.range_constexpr(2):
state = m_tile * 2 + row
row_sum = l[state]
neighbor_sum = cute.arch.shuffle_sync_bfly(row_sum, offset=1)
row_sum += neighbor_sum
neighbor_sum = cute.arch.shuffle_sync_bfly(row_sum, offset=2)
row_sum += neighbor_sum
l[state] = row_sum
lse = m[state] + cute.math.log(row_sum)
query_row = bidx * 128 + m_tile * 64 + warp_id * 16 + quad_id + row * 8
valid_query = True
if cutlass.const_expr(sequence % 128 != 0):
valid_query = query_row < sequence
if valid_query and lane_id % 4 == 0:
mLSE[query_row, bidz] = lse
tCrO_row = tCrO[(None, row), m_tile, None]
inverse_row_sum = pv_scale / row_sum
tCrO_row.store(tCrO_row.load() * inverse_row_sum)
if valid_query:
tCrO_row_out = cute.make_fragment_like(tCrO_row, mO.element_type)
tCrO_row_out.store(tCrO_row.load().to(mO.element_type))
cute.autovec_copy(tCrO_row_out, tCgO[(None, row), m_tile, None])
@cute.jit
def fp8_attention_host(
mQ: cute.Tensor,
mK: cute.Tensor,
mV: cute.Tensor,
mO: cute.Tensor,
mLSE: cute.Tensor,
mScales: cute.Tensor,
softmax_scale: cutlass.Float32,
stream,
):
mQ = cute.make_tensor(mQ.iterator, cute.select(mQ.layout, mode=[1, 2, 0]))
mK = cute.make_tensor(mK.iterator, cute.select(mK.layout, mode=[1, 2, 0]))
mV = cute.make_tensor(mV.iterator, cute.select(mV.layout, mode=[1, 2, 0]))
mO = cute.make_tensor(mO.iterator, cute.select(mO.layout, mode=[1, 2, 0]))
mLSE = cute.make_tensor(mLSE.iterator, cute.select(mLSE.layout, mode=[1, 0]))
mma_op = cute.nvgpu.warp.MmaFP8Op(
mQ.element_type,
cutlass.Float32,
(16, 8, 32),
)
tiled_mma = cute.make_tiled_mma(
mma_op,
(4, 1, 1),
permutation_mnk=(128, 16, 32),
)
copy_bits = 128
num_threads = 128
sQ_layout, sQ_swizzle = _make_smem_layout_fp8(
mQ.element_type,
copy_bits,
(128, 128, 1),
)
sK_layout, sK_swizzle = _make_smem_layout_fp8(
mK.element_type,
copy_bits,
(32, 128, 2),
)
sV_layout, sV_swizzle = _make_smem_layout_fp8(
mV.element_type,
copy_bits,
(128, 32, 2),
)
atom_copy_g2s = cute.make_copy_atom(
cute.nvgpu.cpasync.CopyG2SOp(cache_mode=cute.nvgpu.LoadCacheMode.GLOBAL),
mQ.element_type,
num_bits_per_copy=copy_bits,
)
tiled_copy_Q = _make_gmem_tiled_copy(
atom_copy_g2s,
mQ.element_type,
copy_bits,
128,
num_threads,
)
tiled_copy_K = _make_gmem_tiled_copy(
atom_copy_g2s,
mK.element_type,
copy_bits,
128,
num_threads,
)
tiled_copy_V = _make_gmem_tiled_copy(
atom_copy_g2s,
mV.element_type,
copy_bits,
32,
num_threads,
)
_fp8_attention_sm120(
softmax_scale,
mQ,
mK,
mV,
mO,
mLSE,
mScales,
sQ_layout,
sK_layout,
sV_layout,
sQ_swizzle,
sK_swizzle,
sV_swizzle,
tiled_copy_Q,
tiled_copy_K,
tiled_copy_V,
tiled_mma,
).launch(
grid=(cute.ceil_div(mQ.shape[0], 128), 1, mQ.shape[2]),
block=(num_threads, 1, 1),
stream=stream,
)
def key_order_for_positions(positions):
"""Key held at each stored V^T position.
Inside every 16-key group, position 4t + i holds key 8*(i//2) + 2t + i%2:
the k order of the PV A fragment packed straight from the score fragment.
"""
group = positions // 16 * 16
quad = (positions % 16) // 4
element = positions % 4
return group + 8 * (element // 2) + 2 * quad + element % 2
@@ -0,0 +1,150 @@
# SPDX-License-Identifier: Apache-2.0
"""FP8 attention on SGLang's [S, H, 128] strided BF16 views.
fp8_attention() quantizes Q/K/V per head with the fused Triton passes, runs the CuTe-DSL
kernel and returns (output [S, H, 128] BF16, lse [H, S] FP32). The compiled kernel is
cached per (sequence, heads, device); the E4M3, scale, output and LSE buffers come from
the caching allocator on every call and belong to the caller, nothing is retained
between calls.
"""
import logging
import math
import cuda.bindings.driver as cuda
import cutlass
import cutlass.cute as cute
import msgspec
import torch
from cutlass.cute.runtime import from_dlpack
from sglang.kernels.ops.attention.fp8_fa_sm120.fused_prep import fused_prepare
from sglang.kernels.ops.attention.fp8_fa_sm120.kernel import fp8_attention_host
logger = logging.getLogger(__name__)
HEAD_DIM = 128
# Q and K/V are padded separately; the kernel compiles its key mask only when S % 32 != 0.
QUERY_TILE = 128
KEY_TILE = 32
# Compiled kernels only, KBs of host state each and about 10 s to build; the buffers
# they run on are allocated per call.
_KERNEL_CACHE: dict[tuple, object] = {}
def validate_inputs(q, k, v, softmax_scale):
"""Strided [S, H, 128] BF16 views on one CUDA device; returns the scale as float."""
if (
q.ndim != 3
or q.shape[-1] != HEAD_DIM
or q.shape != k.shape
or q.shape != v.shape
):
raise ValueError("Q/K/V must have equal shape [sequence, heads, 128]")
if min(q.shape[:2]) < 1:
raise ValueError("Sequence and heads must be positive")
for tensor in (q, k, v):
if (
not tensor.is_cuda
or tensor.device != q.device
or tensor.dtype != torch.bfloat16
):
raise ValueError("Q/K/V must share a CUDA device and use BF16")
if tensor.stride(-1) != 1 or tensor.stride(-2) != HEAD_DIM:
raise ValueError(
"Feature and head dimensions must be contiguous within each token"
)
if tensor.data_ptr() % 16 or tensor.stride(0) % 8:
raise ValueError("Token rows must be aligned to 16 bytes")
if tensor.stride() != q.stride():
raise ValueError("Q/K/V must share strides")
scale = HEAD_DIM**-0.5 if softmax_scale is None else float(softmax_scale)
if not math.isfinite(scale):
raise ValueError("Softmax scale must be finite")
return scale
def kernel_key(q):
"""Cache key: the kernel bakes the buffer shapes, which follow from (S, H)."""
return (q.shape[0], q.shape[1], q.device.index)
class Workspace(msgspec.Struct, frozen=True, kw_only=True):
"""Kernel-side buffers for one call; the kernel works in [H, S, 128]."""
q_fp8: torch.Tensor
k_fp8: torch.Tensor
v_fp8: torch.Tensor
scales: torch.Tensor
maximums: torch.Tensor
output: torch.Tensor
lse: torch.Tensor
def _allocate_workspace(q):
sequence = q.shape[0]
heads = q.shape[1]
padded_queries = (sequence + QUERY_TILE - 1) // QUERY_TILE * QUERY_TILE
padded_keys = (sequence + KEY_TILE - 1) // KEY_TILE * KEY_TILE
device = q.device
return Workspace(
q_fp8=torch.empty(
(heads, padded_queries, HEAD_DIM), device=device, dtype=torch.float8_e4m3fn
),
k_fp8=torch.empty(
(heads, padded_keys, HEAD_DIM), device=device, dtype=torch.float8_e4m3fn
),
v_fp8=torch.empty(
(heads, HEAD_DIM, padded_keys), device=device, dtype=torch.float8_e4m3fn
),
scales=torch.empty((3, heads), device=device, dtype=torch.float32),
maximums=torch.zeros((3, heads), device=device, dtype=torch.float32),
output=torch.empty(q.shape, device=device, dtype=torch.bfloat16),
lse=torch.empty((heads, sequence), device=device, dtype=torch.float32),
)
def _kernel_views(workspace):
# Byte DLPack views work across Torch versions without FP8 DLPack support.
mQ = from_dlpack(workspace.q_fp8.view(torch.uint8), assumed_align=16)
mK = from_dlpack(workspace.k_fp8.view(torch.uint8), assumed_align=16)
mV = from_dlpack(workspace.v_fp8.view(torch.uint8), assumed_align=16)
mQ.element_type = cutlass.Float8E4M3FN
mK.element_type = cutlass.Float8E4M3FN
mV.element_type = cutlass.Float8E4M3FN
mO = from_dlpack(workspace.output.permute(1, 0, 2), assumed_align=16)
mLSE = from_dlpack(workspace.lse, assumed_align=16)
mScales = from_dlpack(workspace.scales, assumed_align=16)
return (mQ, mK, mV, mO, mLSE, mScales)
def fp8_attention(q, k, v, softmax_scale=None):
"""Attention over strided [S, H, 128] BF16 views; returns (output [S, H, 128], lse [H, S])."""
scale = validate_inputs(q, k, v, softmax_scale)
kernel_scale = cutlass.Float32(scale)
key = kernel_key(q)
with torch.cuda.device(q.device):
workspace = _allocate_workspace(q)
views = _kernel_views(workspace)
stream = cuda.CUstream(torch.cuda.current_stream(q.device).cuda_stream)
compiled = _KERNEL_CACHE.get(key)
if compiled is None:
logger.info(
"fp8_fa_sm120 attention: compiling for S=%d H=%d (once per shape)",
q.shape[0],
q.shape[1],
)
compiled = cute.compile(fp8_attention_host, *views, kernel_scale, stream)
_KERNEL_CACHE[key] = compiled
fused_prepare(
q=q.permute(1, 0, 2),
k=k.permute(1, 0, 2),
v=v.permute(1, 0, 2),
workspace=workspace,
)
compiled(*views, kernel_scale, stream)
return workspace.output, workspace.lse
@@ -0,0 +1,191 @@
# SPDX-License-Identifier: Apache-2.0
"""FP8 attention backend for SM120 GPUs (GeForce RTX 50, RTX PRO Blackwell).
Opt in with ``--attention-backend fp8_fa_sm120``. Q, K and V are quantized per head to
E4M3 on the fly (two fused Triton passes), attention runs in FP8 with FP32
accumulation, and the output is BF16. Expect about 5% relative RMS against cuDNN BF16
per call on normally distributed inputs.
Scope: dense, non-causal, batch 1, head_dim 128, BF16 inputs on an SM120 device.
Everything else goes to cuDNN SDPA. Each distinct sequence length compiles once
(about 10 s); later calls with the same shape reuse the compiled kernel. The FP8 and
output buffers are allocated per call and belong to the caller.
"""
import torch
from sglang.kernels.ops.attention.fp8_fa_sm120 import HEAD_DIM, fp8_attention
from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import (
AttentionBackend,
AttentionImpl,
AttentionMetadata,
trailing_padding_used_len,
)
from sglang.multimodal_gen.runtime.layers.attention.backends.sdpa import CudnnSDPAImpl
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
logger = init_logger(__name__)
class FP8FlashAttentionSM120Backend(AttentionBackend):
accept_output_buffer: bool = False
@staticmethod
def get_supported_head_sizes() -> list[int]:
return [HEAD_DIM]
@staticmethod
def get_enum() -> AttentionBackendEnum:
return AttentionBackendEnum.FP8_FA_SM120
@staticmethod
def get_impl_cls() -> type["FP8FlashAttentionSM120Impl"]:
return FP8FlashAttentionSM120Impl
class FP8FlashAttentionSM120Impl(AttentionImpl):
def __init__(
self,
num_heads: int,
head_size: int,
causal: bool,
softmax_scale: float,
num_kv_heads: int | None = None,
prefix: str = "",
**extra_impl_args,
) -> None:
self.num_heads = num_heads
self.head_size = head_size
self.causal = causal
self.softmax_scale = softmax_scale
self.fallback = CudnnSDPAImpl(
num_heads=num_heads,
head_size=head_size,
causal=causal,
softmax_scale=softmax_scale,
num_kv_heads=num_kv_heads,
prefix=prefix,
**extra_impl_args,
)
self._reported_fallbacks: set[str] = set()
# --- dispatch -------------------------------------------------------------
def _fallback_reason(self, query, key, value) -> str | None:
"""None when the kernel applies to these [S, H, D] tensors, else why not."""
if self.causal:
return "causal attention"
if query.shape != key.shape or key.shape != value.shape:
return "Q/K/V shapes differ"
if query.ndim != 3 or query.shape[-1] != HEAD_DIM:
return f"head_dim {query.shape[-1]} (kernel is head_dim {HEAD_DIM})"
if query.dtype != torch.bfloat16:
return f"dtype {query.dtype} (kernel takes BF16)"
if not query.is_cuda:
return "non-CUDA tensors"
if torch.cuda.get_device_capability(query.device) != (12, 0):
return "device is not SM120"
return None
def _report_fallback(self, reason: str) -> None:
if reason in self._reported_fallbacks:
return
self._reported_fallbacks.add(reason)
logger.warning(
"fp8_fa_sm120 attention: %s; using cuDNN SDPA for these calls.", reason
)
# --- kernel path ----------------------------------------------------------
def _run(self, query, key, value):
"""[S, H, 128] strided BF16 in, a new contiguous [S, H, 128] BF16 out."""
output, _ = fp8_attention(query, key, value, softmax_scale=self.softmax_scale)
return output
# --- AttentionImpl --------------------------------------------------------
def forward(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
attn_metadata: AttentionMetadata,
) -> torch.Tensor:
# [B, S, H, D]; the kernel handles one dense sequence at a time.
if query.shape[0] != 1:
self._report_fallback(f"batch {query.shape[0]}")
return self.fallback.forward(query, key, value, attn_metadata)
reason = self._fallback_reason(query[0], key[0], value[0])
if reason is not None:
self._report_fallback(reason)
return self.fallback.forward(query, key, value, attn_metadata)
try:
output = self._run(query[0], key[0], value[0])
except ValueError as error:
self._report_fallback(str(error))
return self.fallback.forward(query, key, value, attn_metadata)
return output.unsqueeze(0)
def forward_varlen(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
*,
cu_seqlens: torch.Tensor,
max_seqlen: int,
cu_seqlens_host: tuple[int, ...] | None = None,
) -> torch.Tensor:
bounds = (
cu_seqlens_host
if cu_seqlens_host is not None
else tuple(int(item) for item in cu_seqlens.tolist())
)
reason = self._fallback_reason(query, key, value)
if reason is not None:
self._report_fallback(reason)
return self.fallback.forward_varlen(
query,
key,
value,
cu_seqlens=cu_seqlens,
max_seqlen=max_seqlen,
cu_seqlens_host=bounds,
)
# MiniMax-H3 packs one live document as (0, used, total); the tail is
# 64-aligned padding that downstream masks, so it stays zero.
used = trailing_padding_used_len(
total_tokens=query.shape[0],
max_seqlen=max_seqlen,
bounds=bounds,
)
try:
if used is not None:
live_output = self._run(query[:used], key[:used], value[:used])
if used == query.shape[0]:
return live_output
output = torch.zeros_like(query)
output[:used].copy_(live_output)
return output
output = torch.empty_like(query)
for start, stop in zip(bounds[:-1], bounds[1:]):
if start == stop:
continue
segment_output = self._run(
query[start:stop], key[start:stop], value[start:stop]
)
output[start:stop].copy_(segment_output)
return output
except ValueError as error:
self._report_fallback(str(error))
return self.fallback.forward_varlen(
query,
key,
value,
cu_seqlens=cu_seqlens,
max_seqlen=max_seqlen,
cu_seqlens_host=bounds,
)
@@ -516,6 +516,36 @@ class _FlashAttentionBackendResolver(_CudaAttentionBackendResolver):
return AttentionBackendEnum.FA
class _FP8FlashAttentionSM120BackendResolver(_CudaAttentionBackendResolver):
backend = AttentionBackendEnum.FP8_FA_SM120
# CuTe-DSL mma.sync FP8 kernel written for SM120 (GeForce RTX 50 / RTX PRO
# Blackwell). Dense non-causal head_dim 128 only; the backend itself falls
# back to cuDNN SDPA for other calls.
@classmethod
def resolve(cls, platform) -> str | AttentionBackendEnum:
if platform.get_device_capability() != (12, 0):
logger.warning(
"fp8_fa_sm120 attention needs an SM120 device; falling back to cuDNN SDPA."
)
return AttentionBackendEnum.TORCH_CUDNN_SDPA
try:
import cutlass.cute # noqa: F401
import triton # noqa: F401
from sglang.multimodal_gen.runtime.layers.attention.backends.fp8_fa_sm120_attn import ( # noqa: F401
FP8FlashAttentionSM120Backend,
)
return "sglang.multimodal_gen.runtime.layers.attention.backends.fp8_fa_sm120_attn.FP8FlashAttentionSM120Backend"
except ImportError as error:
logger.warning(
"fp8_fa_sm120 attention backend failed to import (%s); falling back to cuDNN SDPA.",
error,
)
return AttentionBackendEnum.TORCH_CUDNN_SDPA
_CUDA_ATTENTION_BACKEND_RESOLVERS = {
resolver.backend: resolver
for resolver in (
@@ -539,6 +569,7 @@ _CUDA_ATTENTION_BACKEND_RESOLVERS = {
_SubBlockSparseAttentionBackendResolver,
_FlashAttention2BackendResolver,
_FlashAttentionBackendResolver,
_FP8FlashAttentionSM120BackendResolver,
)
}
@@ -50,6 +50,7 @@ class AttentionBackendEnum(enum.Enum):
SOL_ATTN = enum.auto()
SUBBLOCK_SPARSE_ATTN = enum.auto()
CUBE_SPARSE_ATTN = enum.auto()
FP8_FA_SM120 = enum.auto()
NO_ATTENTION = enum.auto()
def __str__(self):
@@ -0,0 +1,222 @@
# SPDX-License-Identifier: Apache-2.0
"""fp8_fa_sm120 attention backend against cuDNN SDPA on MiniMax-H3 style inputs.
Needs an SM120 GPU; skipped elsewhere. Shapes are kept small (8 heads) so the
CuTe-DSL compile and the runs stay in seconds.
"""
import pytest
import torch
pytestmark = pytest.mark.skipif(
not torch.cuda.is_available() or torch.cuda.get_device_capability() != (12, 0),
reason="fp8_fa_sm120 attention needs an SM120 GPU",
)
HEADS = 8
HEAD_DIM = 128
SOFTMAX_SCALE = HEAD_DIM**-0.5
def _make_impls(causal=False):
from sglang.multimodal_gen.runtime.layers.attention.backends.fp8_fa_sm120_attn import (
FP8FlashAttentionSM120Impl,
)
from sglang.multimodal_gen.runtime.layers.attention.backends.sdpa import (
CudnnSDPAImpl,
)
ours = FP8FlashAttentionSM120Impl(
num_heads=HEADS, head_size=HEAD_DIM, causal=causal, softmax_scale=SOFTMAX_SCALE
)
reference = CudnnSDPAImpl(
num_heads=HEADS, head_size=HEAD_DIM, causal=causal, softmax_scale=SOFTMAX_SCALE
)
return ours, reference
def _fused_qkv_views(sequence, seed=0):
"""Q/K/V as views into one [S, 3*H*D] projection output, like the DiT hands them over."""
generator = torch.Generator(device="cuda").manual_seed(seed)
qkv = torch.randn(
(sequence, 3 * HEADS * HEAD_DIM),
device="cuda",
dtype=torch.bfloat16,
generator=generator,
)
q = qkv[:, 0 : HEADS * HEAD_DIM].view(sequence, HEADS, HEAD_DIM)
k = qkv[:, HEADS * HEAD_DIM : 2 * HEADS * HEAD_DIM].view(sequence, HEADS, HEAD_DIM)
v = qkv[:, 2 * HEADS * HEAD_DIM :].view(sequence, HEADS, HEAD_DIM)
return q, k, v
def _error_metrics(candidate, reference):
candidate = candidate.float()
reference = reference.float()
relative_rms = (candidate - reference).pow(2).mean().sqrt() / reference.pow(
2
).mean().sqrt()
cosine = torch.nn.functional.cosine_similarity(
candidate.flatten(), reference.flatten(), dim=0
)
return float(relative_rms), float(cosine)
@pytest.mark.parametrize("sequence", [4096, 5980])
def test_forward_matches_cudnn(sequence):
ours, reference = _make_impls()
q, k, v = _fused_qkv_views(sequence)
output = ours.forward(q.unsqueeze(0), k.unsqueeze(0), v.unsqueeze(0), None)
expected = reference.forward(q.unsqueeze(0), k.unsqueeze(0), v.unsqueeze(0), None)
assert output.shape == expected.shape
assert output.dtype == torch.bfloat16
assert torch.isfinite(output.float()).all()
relative_rms, cosine = _error_metrics(output, expected)
# Per-head E4M3 quantization of Q/K/V: ~5% relative RMS on normal inputs.
assert relative_rms < 0.08, relative_rms
assert cosine > 0.995, cosine
def _compiled_kernels():
from sglang.kernels.ops.attention.fp8_fa_sm120.plan import _KERNEL_CACHE
return len(_KERNEL_CACHE)
def test_kernel_is_reused_across_calls():
ours, reference = _make_impls()
q, k, v = _fused_qkv_views(4096, seed=1)
first = ours.forward(q.unsqueeze(0), k.unsqueeze(0), v.unsqueeze(0), None)
kernels_after_first = _compiled_kernels()
q2, k2, v2 = _fused_qkv_views(4096, seed=2)
second = ours.forward(q2.unsqueeze(0), k2.unsqueeze(0), v2.unsqueeze(0), None)
assert _compiled_kernels() == kernels_after_first
# A second impl (another DiT layer) shares the same compiled kernel.
other, _ = _make_impls()
other.forward(q2.unsqueeze(0), k2.unsqueeze(0), v2.unsqueeze(0), None)
assert _compiled_kernels() == kernels_after_first
expected = reference.forward(
q2.unsqueeze(0), k2.unsqueeze(0), v2.unsqueeze(0), None
)
relative_rms, _ = _error_metrics(second, expected)
assert relative_rms < 0.08, relative_rms
assert not torch.equal(first, second)
def test_varlen_trailing_padding_keeps_tail_zero():
ours, _ = _make_impls()
used = 5980
total = 6016 # 64-aligned tail padding, as MiniMax-H3 packs it
q, k, v = _fused_qkv_views(total)
cu_seqlens = torch.tensor([0, used, total], device="cuda", dtype=torch.int32)
output = ours.forward_varlen(
q,
k,
v,
cu_seqlens=cu_seqlens,
max_seqlen=used,
cu_seqlens_host=(0, used, total),
)
live = ours.forward(
q[:used].unsqueeze(0), k[:used].unsqueeze(0), v[:used].unsqueeze(0), None
)[0]
assert output.shape == q.shape
assert torch.equal(output[:used], live)
assert torch.count_nonzero(output[used:]) == 0
def test_varlen_multiple_segments():
ours, reference = _make_impls()
bounds = (0, 1024, 3072, 4096)
q, k, v = _fused_qkv_views(bounds[-1])
cu_seqlens = torch.tensor(bounds, device="cuda", dtype=torch.int32)
output = ours.forward_varlen(
q, k, v, cu_seqlens=cu_seqlens, max_seqlen=2048, cu_seqlens_host=bounds
)
expected = reference.forward_varlen(
q, k, v, cu_seqlens=cu_seqlens, max_seqlen=2048, cu_seqlens_host=bounds
)
for start, stop in zip(bounds[:-1], bounds[1:]):
relative_rms, _ = _error_metrics(output[start:stop], expected[start:stop])
assert relative_rms < 0.08, (start, stop, relative_rms)
def test_distinct_shapes_release_memory():
"""A call must not retain GPU buffers keyed by shape: each distinct sequence length
used to keep its FP8/output buffers and the last Q/K/V alive, ~2.2 GiB at H3 size."""
ours, _ = _make_impls()
shapes = [_fused_qkv_views(1052), _fused_qkv_views(1088)]
q, k, v = shapes[0]
ours.forward(q.unsqueeze(0), k.unsqueeze(0), v.unsqueeze(0), None)
torch.cuda.synchronize()
baseline = torch.cuda.memory_allocated()
for q, k, v in shapes:
output = ours.forward(q.unsqueeze(0), k.unsqueeze(0), v.unsqueeze(0), None)
assert torch.isfinite(output.float()).all()
del output
torch.cuda.synchronize()
assert torch.cuda.memory_allocated() == baseline
def test_prep_defines_every_padded_byte():
"""The FP8 buffers come from torch.empty, so the prep must write every padding
position; an E4M3 NaN byte left in the V^T tail makes masked keys poison the row."""
from sglang.kernels.ops.attention.fp8_fa_sm120.fused_prep import fused_prepare
from sglang.kernels.ops.attention.fp8_fa_sm120.plan import _allocate_workspace
q, k, v = _fused_qkv_views(1052)
workspace = _allocate_workspace(q)
for buffer in (workspace.q_fp8, workspace.k_fp8, workspace.v_fp8):
buffer.view(torch.uint8).fill_(0xFF)
fused_prepare(
q=q.permute(1, 0, 2),
k=k.permute(1, 0, 2),
v=v.permute(1, 0, 2),
workspace=workspace,
)
for buffer in (workspace.q_fp8, workspace.k_fp8, workspace.v_fp8):
raw = buffer.view(torch.uint8)
assert not ((raw == 0xFF) | (raw == 0x7F)).any()
def test_causal_falls_back_to_cudnn():
ours, reference = _make_impls(causal=True)
q, k, v = _fused_qkv_views(1024)
kernels_before = _compiled_kernels()
output = ours.forward(q.unsqueeze(0), k.unsqueeze(0), v.unsqueeze(0), None)
expected = reference.forward(q.unsqueeze(0), k.unsqueeze(0), v.unsqueeze(0), None)
assert torch.equal(output, expected)
assert _compiled_kernels() == kernels_before
def test_backend_resolves_by_name():
from sglang.multimodal_gen.runtime.platforms import (
AttentionBackendEnum,
current_platform,
)
class_path = current_platform.get_attn_backend_cls_str(
AttentionBackendEnum.FP8_FA_SM120, HEAD_DIM, torch.bfloat16
)
assert class_path.endswith("fp8_fa_sm120_attn.FP8FlashAttentionSM120Backend")
if __name__ == "__main__":
import sys
sys.exit(pytest.main([__file__, "-v"]))