[Perf] [NVIDIA] fp8_fa_sm120: Use the block-scaled MMA encoding (#43390)
Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
co-authored by
Xiaoyu Zhang
parent
0dc6de2e74
commit
10ec53ac2f
@@ -21,6 +21,11 @@ a fixed permuted order, which is why V^T is stored in that order. PV accumulates
|
||||
FP32 O registers that are rescaled to the new max on every tile and normalized by l at
|
||||
the end.
|
||||
|
||||
QK and PV use the block-scaled MMA encoding (kind::mxf8f6f4) with every UE8M0 scale set
|
||||
to 1.0 when the compile target has it. It computes the same E4M3 x E4M3 product with FP32
|
||||
accumulation as the plain encoding, bit for bit; GeForce SM120 runs the plain encoding at
|
||||
half rate.
|
||||
|
||||
Non-causal only. No dropout, no autograd. Inputs must be finite.
|
||||
"""
|
||||
|
||||
@@ -29,6 +34,8 @@ import math
|
||||
import cutlass
|
||||
import cutlass.cute as cute
|
||||
import cutlass.utils as utils
|
||||
import cutlass.utils.blackwell_helpers as sm120_utils
|
||||
from cutlass.cutlass_dsl import BaseDSL
|
||||
|
||||
|
||||
def _make_gmem_tiled_copy(atom_copy, dtype, copy_bits, minor_size, num_threads):
|
||||
@@ -41,6 +48,37 @@ def _make_gmem_tiled_copy(atom_copy, dtype, copy_bits, minor_size, num_threads):
|
||||
return cute.make_tiled_copy_tv(atom_copy, thread_layout, value_layout)
|
||||
|
||||
|
||||
def _block_scaled_mma_supported():
|
||||
# MmaMXF8Op assembles only for the SM120 / SM121 "a" and "f" targets.
|
||||
arch = BaseDSL._get_dsl().get_arch_enum()
|
||||
return arch in cute.nvgpu.warp.MmaMXF8Op.admissible_archs
|
||||
|
||||
|
||||
# UE8M0 scale byte: 2^(byte - 127), so 0x7F is exactly 1.0.
|
||||
UNIT_SCALE_BYTE = 0x7F
|
||||
|
||||
|
||||
def _make_unit_scale_fragment_A(thr_mma, tidx, tile_mk):
|
||||
# The reference tensor only carries the (M, K) tile shape; stride 0, nothing is read.
|
||||
reference = cute.make_rmem_tensor(
|
||||
cute.make_layout(tile_mk, stride=(0, 0)),
|
||||
cutlass.Float8E8M0FNU,
|
||||
)
|
||||
fragment = sm120_utils.partition_fragment_SFA(reference, thr_mma, tidx)
|
||||
cute.recast_tensor(fragment, cutlass.Uint8).fill(UNIT_SCALE_BYTE)
|
||||
return fragment
|
||||
|
||||
|
||||
def _make_unit_scale_fragment_B(thr_mma, tidx, tile_nk):
|
||||
reference = cute.make_rmem_tensor(
|
||||
cute.make_layout(tile_nk, stride=(0, 0)),
|
||||
cutlass.Float8E8M0FNU,
|
||||
)
|
||||
fragment = sm120_utils.partition_fragment_SFB(reference, thr_mma, tidx)
|
||||
cute.recast_tensor(fragment, cutlass.Uint8).fill(UNIT_SCALE_BYTE)
|
||||
return fragment
|
||||
|
||||
|
||||
def _make_smem_layout_fp8(dtype, copy_bits, smem_tiler):
|
||||
major_size = smem_tiler[1]
|
||||
row_bytes = major_size * dtype.width // 8
|
||||
@@ -74,6 +112,7 @@ def _fp8_attention_sm120(
|
||||
tiled_copy_V: cute.TiledCopy,
|
||||
tiled_mma: cute.TiledMma,
|
||||
cta_tiler: cutlass.Constexpr = (128, 32, 128),
|
||||
block_scaled_mma: cutlass.Constexpr = True,
|
||||
):
|
||||
tidx, _, _ = cute.arch.thread_idx()
|
||||
bidx, _, bidz = cute.arch.block_idx()
|
||||
@@ -140,6 +179,13 @@ def _fp8_attention_sm120(
|
||||
tCrK = tiled_mma.make_fragment_B(tCsK[None, None, None, 0])
|
||||
tCrV = tiled_mma.make_fragment_B(tCsV[None, None, None, 0])
|
||||
|
||||
# Scale fragments: A is (value, m_tile, k_block), B is (value, (n_tile, k_block)).
|
||||
if cutlass.const_expr(block_scaled_mma):
|
||||
tCrSFQ = _make_unit_scale_fragment_A(thr_mma, tidx, (128, 128))
|
||||
tCrSFK = _make_unit_scale_fragment_B(thr_mma, tidx, (32, 128))
|
||||
tCrSFP = _make_unit_scale_fragment_A(thr_mma, tidx, (128, 32))
|
||||
tCrSFV = _make_unit_scale_fragment_B(thr_mma, tidx, (128, 32))
|
||||
|
||||
acc_shape = thr_mma.partition_shape_C((128, 32))
|
||||
tCrC = cute.make_rmem_tensor(acc_shape, cutlass.Float32)
|
||||
|
||||
@@ -236,13 +282,28 @@ def _fp8_attention_sm120(
|
||||
tCsK_p[None, None, k_block],
|
||||
tCrK_copy_view[None, None, k_block],
|
||||
)
|
||||
cute.gemm(
|
||||
tiled_mma,
|
||||
tCrC,
|
||||
tCrQ[None, None, k_block],
|
||||
tCrK[None, None, k_block],
|
||||
tCrC,
|
||||
)
|
||||
if cutlass.const_expr(block_scaled_mma):
|
||||
cute.gemm(
|
||||
tiled_mma,
|
||||
tCrC,
|
||||
[
|
||||
tCrQ[None, None, k_block],
|
||||
tCrSFQ[None, None, k_block],
|
||||
],
|
||||
[
|
||||
tCrK[None, None, k_block],
|
||||
tCrSFK[None, (None, k_block)],
|
||||
],
|
||||
tCrC,
|
||||
)
|
||||
else:
|
||||
cute.gemm(
|
||||
tiled_mma,
|
||||
tCrC,
|
||||
tCrQ[None, None, k_block],
|
||||
tCrK[None, None, k_block],
|
||||
tCrC,
|
||||
)
|
||||
|
||||
for m_tile in cutlass.range_constexpr(2):
|
||||
for row in cutlass.range_constexpr(2):
|
||||
@@ -319,13 +380,28 @@ def _fp8_attention_sm120(
|
||||
tCsV_p[None, None, k_block],
|
||||
tCrV_copy_view[None, None, k_block],
|
||||
)
|
||||
cute.gemm(
|
||||
tiled_mma,
|
||||
tCrO,
|
||||
tCrP[None, None, k_block],
|
||||
tCrV[None, None, k_block],
|
||||
tCrO,
|
||||
)
|
||||
if cutlass.const_expr(block_scaled_mma):
|
||||
cute.gemm(
|
||||
tiled_mma,
|
||||
tCrO,
|
||||
[
|
||||
tCrP[None, None, k_block],
|
||||
tCrSFP[None, None, k_block],
|
||||
],
|
||||
[
|
||||
tCrV[None, None, k_block],
|
||||
tCrSFV[None, (None, k_block)],
|
||||
],
|
||||
tCrO,
|
||||
)
|
||||
else:
|
||||
cute.gemm(
|
||||
tiled_mma,
|
||||
tCrO,
|
||||
tCrP[None, None, k_block],
|
||||
tCrV[None, None, k_block],
|
||||
tCrO,
|
||||
)
|
||||
|
||||
warp_id = tidx // 32
|
||||
lane_id = tidx % 32
|
||||
@@ -374,6 +450,7 @@ def fp8_attention_host(
|
||||
mScales: cute.Tensor,
|
||||
softmax_scale: cutlass.Float32,
|
||||
stream,
|
||||
block_scaled_mma: cutlass.Constexpr = True,
|
||||
):
|
||||
mQ = cute.make_tensor(mQ.iterator, cute.select(mQ.layout, mode=[1, 2, 0]))
|
||||
mK = cute.make_tensor(mK.iterator, cute.select(mK.layout, mode=[1, 2, 0]))
|
||||
@@ -381,11 +458,20 @@ def fp8_attention_host(
|
||||
mO = cute.make_tensor(mO.iterator, cute.select(mO.layout, mode=[1, 2, 0]))
|
||||
mLSE = cute.make_tensor(mLSE.iterator, cute.select(mLSE.layout, mode=[1, 0]))
|
||||
|
||||
mma_op = cute.nvgpu.warp.MmaFP8Op(
|
||||
mQ.element_type,
|
||||
cutlass.Float32,
|
||||
(16, 8, 32),
|
||||
)
|
||||
# block_scaled_mma=False keeps the plain encoding; the unit test compares the two.
|
||||
block_scaled_mma = block_scaled_mma and _block_scaled_mma_supported()
|
||||
if cutlass.const_expr(block_scaled_mma):
|
||||
mma_op = cute.nvgpu.warp.MmaMXF8Op(
|
||||
mQ.element_type,
|
||||
cutlass.Float32,
|
||||
cutlass.Float8E8M0FNU,
|
||||
)
|
||||
else:
|
||||
mma_op = cute.nvgpu.warp.MmaFP8Op(
|
||||
mQ.element_type,
|
||||
cutlass.Float32,
|
||||
(16, 8, 32),
|
||||
)
|
||||
tiled_mma = cute.make_tiled_mma(
|
||||
mma_op,
|
||||
(4, 1, 1),
|
||||
@@ -456,6 +542,7 @@ def fp8_attention_host(
|
||||
tiled_copy_K,
|
||||
tiled_copy_V,
|
||||
tiled_mma,
|
||||
block_scaled_mma=block_scaled_mma,
|
||||
).launch(
|
||||
grid=(cute.ceil_div(mQ.shape[0], 128), 1, mQ.shape[2]),
|
||||
block=(num_threads, 1, 1),
|
||||
|
||||
@@ -121,9 +121,14 @@ def _kernel_views(workspace):
|
||||
|
||||
def fp8_attention(q, k, v, softmax_scale=None):
|
||||
"""Attention over strided [S, H, 128] BF16 views; returns (output [S, H, 128], lse [H, S])."""
|
||||
return _fp8_attention(q, k, v, softmax_scale, block_scaled_mma=True)
|
||||
|
||||
|
||||
def _fp8_attention(q, k, v, softmax_scale, block_scaled_mma):
|
||||
# block_scaled_mma is not a user option: the output is bit-equal either way.
|
||||
scale = validate_inputs(q, k, v, softmax_scale)
|
||||
kernel_scale = cutlass.Float32(scale)
|
||||
key = kernel_key(q)
|
||||
key = (*kernel_key(q), block_scaled_mma)
|
||||
|
||||
with torch.cuda.device(q.device):
|
||||
workspace = _allocate_workspace(q)
|
||||
@@ -137,7 +142,13 @@ def fp8_attention(q, k, v, softmax_scale=None):
|
||||
q.shape[0],
|
||||
q.shape[1],
|
||||
)
|
||||
compiled = cute.compile(fp8_attention_host, *views, kernel_scale, stream)
|
||||
compiled = cute.compile(
|
||||
fp8_attention_host,
|
||||
*views,
|
||||
kernel_scale,
|
||||
stream,
|
||||
block_scaled_mma=block_scaled_mma,
|
||||
)
|
||||
_KERNEL_CACHE[key] = compiled
|
||||
|
||||
fused_prepare(
|
||||
|
||||
@@ -192,6 +192,20 @@ def test_prep_defines_every_padded_byte():
|
||||
assert not ((raw == 0xFF) | (raw == 0x7F)).any()
|
||||
|
||||
|
||||
def test_block_scaled_mma_is_bit_equal():
|
||||
"""Unit UE8M0 scales must not move one output bit; S=1052 also covers the key mask."""
|
||||
from sglang.kernels.ops.attention.fp8_fa_sm120.plan import _fp8_attention
|
||||
|
||||
for sequence in (1052, 4096):
|
||||
q, k, v = _fused_qkv_views(sequence)
|
||||
output, lse = _fp8_attention(q, k, v, SOFTMAX_SCALE, block_scaled_mma=False)
|
||||
scaled_output, scaled_lse = _fp8_attention(
|
||||
q, k, v, SOFTMAX_SCALE, block_scaled_mma=True
|
||||
)
|
||||
assert torch.equal(output.view(torch.int16), scaled_output.view(torch.int16))
|
||||
assert torch.equal(lse.view(torch.int32), scaled_lse.view(torch.int32))
|
||||
|
||||
|
||||
def test_causal_falls_back_to_cudnn():
|
||||
ours, reference = _make_impls(causal=True)
|
||||
q, k, v = _fused_qkv_views(1024)
|
||||
|
||||
Reference in New Issue
Block a user