[AMD] Fuse prefill index-Q RoPE fake-quant and fp4 quantize kernels in DSV4.1 (#42911)
This commit is contained in:
@@ -8,8 +8,10 @@ from typing import TYPE_CHECKING, NamedTuple, Optional, Tuple, Union
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
from triton.language.extra import libdevice
|
||||
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer import (
|
||||
_ceil_ue8m0_exp,
|
||||
quantize_fp4_indexer_row,
|
||||
quantize_fp4_indexer_tensor,
|
||||
)
|
||||
@@ -743,6 +745,120 @@ def _reduce_scale_bf16_kernel(
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fp4_e2m1_code_rne_lean(x):
|
||||
"""_fp4_e2m1_code_rne for every fp32 x (checked over all 2**32): within each spacing of the
|
||||
e2m1 grid, rint's ties-to-even picks the even code at the midpoints."""
|
||||
ax = tl.minimum(tl.abs(x), 6.0)
|
||||
lo = ax < 2.0
|
||||
mid = ax < 4.0
|
||||
rstep = tl.where(lo, 2.0, tl.where(mid, 1.0, 0.5))
|
||||
idx = (libdevice.rint(ax * rstep) + tl.where(lo, 0.0, tl.where(mid, 2.0, 4.0))).to(
|
||||
tl.uint8
|
||||
)
|
||||
idx = tl.where(ax == ax, idx, 0)
|
||||
sign = ((x < 0) & (idx != 0)).to(tl.uint8)
|
||||
return idx | (sign << 3)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _index_q_quant_groups(v, AMAX_FLOOR: tl.constexpr):
|
||||
"""[16, G, 32] fp32 through the fake-quant of rope_tail_fake_quant_fp4_row (rounded to bf16
|
||||
as the standalone kernel stores it) and quantize_fp4_indexer_row (RNE): (packed codes
|
||||
[16, G * 16], ue8m0 exponents [16, G]). The power-of-two divisions run as multiplies by
|
||||
exact reciprocals, which round identically."""
|
||||
amax = tl.maximum(tl.max(tl.abs(v), axis=2), AMAX_FLOOR) * (1.0 / 6.0)
|
||||
bits = amax.to(tl.int32, bitcast=True)
|
||||
expo = ((bits >> 23) & 0xFF) - 127
|
||||
expo = expo + ((bits & 0x7FFFFF) != 0).to(tl.int32)
|
||||
scale = ((expo + 127) << 23).to(tl.float32, bitcast=True)
|
||||
# 1 / scale: 2^-expo for a finite amax (expo <= 126), 0 for inf (scale = inf),
|
||||
# -inf for nan (expo = 129 wraps scale to -0.0)
|
||||
rscale = tl.where(
|
||||
expo <= 126,
|
||||
((127 - expo) << 23).to(tl.float32, bitcast=True),
|
||||
tl.where(expo == 128, 0.0, float("-inf")),
|
||||
)
|
||||
s = tl.minimum(tl.maximum(v * rscale[:, :, None], -6.0), 6.0)
|
||||
mag = tl.abs(s)
|
||||
rstep = tl.where(mag < 2.0, 2.0, tl.where(mag < 4.0, 1.0, 0.5))
|
||||
step = ((254 << 23) - rstep.to(tl.int32, bitcast=True)).to(tl.float32, bitcast=True)
|
||||
# rint is odd-symmetric, so the sign rides along; a zero may come out as -0, which neither
|
||||
# the amax nor the codes below tell apart
|
||||
values = (libdevice.rint(s * rstep) * step * scale[:, :, None]).to(tl.bfloat16)
|
||||
values = values.to(tl.float32)
|
||||
|
||||
exp = _ceil_ue8m0_exp(tl.maximum(tl.max(tl.abs(values), axis=2) / 6.0, 1.0e-4))
|
||||
# 1 / 2^(exp - 127); exp = 254 (an inf / nan group) as 0.5 * 2^-126, where any value
|
||||
# rounded differently is below 2^-126 and codes 0 either way
|
||||
top = exp == 254
|
||||
ra = tl.where(top, 0.5, 1.0)
|
||||
rb = tl.where(top, 2.0**-126, ((254 - exp) << 23).to(tl.float32, bitcast=True))
|
||||
code = _fp4_e2m1_code_rne_lean(values * ra[:, :, None] * rb[:, :, None])
|
||||
G: tl.constexpr = v.shape[1]
|
||||
c0, c1 = tl.split(tl.reshape(code, (16, G * 16, 2)))
|
||||
return (c0 & 0x0F) | ((c1 & 0x0F) << 4), exp
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _index_q_rope_pack_kernel(
|
||||
q_ptr, # [T, H * 128] bf16 (wq_b output)
|
||||
f_ptr, # [max_pos, RD // 2, 2] fp32: the real view of the complex freqs table
|
||||
pos_ptr, # [T] int
|
||||
q_fp4_ptr, # [T, H, 64] int8
|
||||
q_scale_ptr, # [T, 1, 4, 16, 4] uint8
|
||||
stride_qt,
|
||||
num_pos,
|
||||
H: tl.constexpr,
|
||||
D: tl.constexpr,
|
||||
RD: tl.constexpr,
|
||||
AMAX_FLOOR: tl.constexpr,
|
||||
):
|
||||
"""Grid (T, H // 16): 16 heads of one token per program, through _index_q_quant_groups, in
|
||||
the FlyDSL MQA-logits layout."""
|
||||
NOPE: tl.constexpr = D - RD
|
||||
t = tl.program_id(0)
|
||||
h0 = tl.program_id(1) * 16
|
||||
hh = h0 + tl.arange(0, 16)
|
||||
pos = tl.load(pos_ptr + t).to(tl.int64)
|
||||
# the caller owns positions < num_pos; an out-of-range row reads entry 0 instead of past the table
|
||||
pos = tl.where((pos >= 0) & (pos < num_pos), pos, 0)
|
||||
q_row = q_ptr + t.to(tl.int64) * stride_qt + hh[:, None] * D
|
||||
out_row = q_fp4_ptr + (t.to(tl.int64) * H + hh[:, None]) * (D // 2)
|
||||
|
||||
x = tl.load(q_row + tl.arange(0, NOPE)[None, :]).to(tl.float32)
|
||||
packed, exp_nope = _index_q_quant_groups(
|
||||
tl.reshape(x, (16, NOPE // 32, 32)), AMAX_FLOOR
|
||||
)
|
||||
tl.store(out_row + tl.arange(0, NOPE // 2)[None, :], packed)
|
||||
|
||||
# RoPE on the last RD features, adjacent pairs as one complex number
|
||||
x = tl.load(q_row + NOPE + tl.arange(0, RD)[None, :]).to(tl.float32)
|
||||
re, im = tl.split(tl.reshape(x, (16, RD // 2, 2)))
|
||||
j = tl.arange(0, RD // 2)
|
||||
fr = tl.load(f_ptr + pos * RD + 2 * j)[None, :]
|
||||
fi = tl.load(f_ptr + pos * RD + 2 * j + 1)[None, :]
|
||||
re_rot = (re * fr - im * fi).to(tl.bfloat16).to(tl.float32)
|
||||
im_rot = (re * fi + im * fr).to(tl.bfloat16).to(tl.float32)
|
||||
packed, exp_rope = _index_q_quant_groups(
|
||||
tl.reshape(tl.join(re_rot, im_rot), (16, RD // 32, 32)), AMAX_FLOOR
|
||||
)
|
||||
tl.store(out_row + NOPE // 2 + tl.arange(0, RD // 2)[None, :], packed)
|
||||
|
||||
# scale bytes: chunk c of head h at [t, 0, c, h % 16, h // 16]
|
||||
base = q_scale_ptr + t.to(tl.int64) * (4 * 16 * 4)
|
||||
i = tl.arange(0, 16)[:, None]
|
||||
c_nope = tl.arange(0, NOPE // 32)[None, :]
|
||||
c_rope = NOPE // 32 + tl.arange(0, RD // 32)[None, :]
|
||||
tl.store(base + (c_nope * 16 + i) * 4 + h0 // 16, exp_nope.to(tl.uint8))
|
||||
tl.store(base + (c_rope * 16 + i) * 4 + h0 // 16, exp_rope.to(tl.uint8))
|
||||
if h0 == 0:
|
||||
# groups this head count does not have stay zero
|
||||
slot = (tl.arange(0, 4)[None, :] * 16 + i) * 4
|
||||
for g in tl.static_range(H // 16, 4):
|
||||
tl.store(base + slot + g, tl.zeros((16, 4), dtype=tl.uint8))
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _index_q_pack_weights_kernel(
|
||||
q_ptr, # [T, H * 128] bf16 (wq_b output)
|
||||
@@ -817,6 +933,54 @@ def _index_q_pack_weights_kernel(
|
||||
)
|
||||
|
||||
|
||||
def index_q_rope_pack_flydsl(
|
||||
q: torch.Tensor,
|
||||
freqs_cis: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
rope_dim: int,
|
||||
*,
|
||||
num_heads: int,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""The index-Q RoPE and fp4 pack at any row count: (q_fp4 [T, H, 64] int8, q_scale
|
||||
[T, 1, 4, 16, 4] uint8), bitwise pack_fp4_query_flydsl(_rope_fq4(q.view(T, H, 128),
|
||||
freqs_cis[positions], rope_dim)) without the bf16 fake-quant round trip through memory."""
|
||||
T = q.shape[0]
|
||||
H = num_heads
|
||||
assert q.dtype == torch.bfloat16 and q.dim() == 2 and q.shape[1] == H * 128
|
||||
assert q.stride(1) == 1
|
||||
assert H % 16 == 0 and 0 < H <= 64, H
|
||||
assert freqs_cis.dtype == torch.complex64 and freqs_cis.shape[1] == rope_dim // 2
|
||||
# the plain and rotated halves are each whole 32-groups of a power-of-two width
|
||||
assert rope_dim == 64, rope_dim
|
||||
assert positions.shape == (T,)
|
||||
# the kernel reads positions[t] at pos_ptr + t
|
||||
positions = positions.contiguous()
|
||||
f_real = torch.view_as_real(freqs_cis)
|
||||
assert f_real.is_contiguous()
|
||||
q_fp4 = torch.empty((T, H, 64), dtype=torch.int8, device=q.device)
|
||||
q_scale = torch.empty((T, 1, 4, 16, 4), dtype=torch.uint8, device=q.device)
|
||||
if T == 0:
|
||||
return q_fp4, q_scale
|
||||
_index_q_rope_pack_kernel[(T, H // 16)](
|
||||
q,
|
||||
f_real,
|
||||
positions,
|
||||
q_fp4,
|
||||
q_scale,
|
||||
q.stride(0),
|
||||
f_real.shape[0],
|
||||
H=H,
|
||||
D=128,
|
||||
RD=rope_dim,
|
||||
AMAX_FLOOR=FP4_AMAX_FLOOR,
|
||||
num_warps=2,
|
||||
# bitwise parity with the standalone RoPE kernel: a contracted FMA in the rotation flips
|
||||
# the bf16 rounding of a few values in 10^8
|
||||
enable_fp_fusion=False,
|
||||
)
|
||||
return q_fp4, q_scale
|
||||
|
||||
|
||||
def index_q_rope_pack_weights_flydsl(
|
||||
q: torch.Tensor,
|
||||
freqs_cis: torch.Tensor,
|
||||
|
||||
@@ -23,6 +23,7 @@ from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import (
|
||||
FP4DecodeWorkspace,
|
||||
FP4PrefillWorkspace,
|
||||
aiter_fp4_paged_mqa_logits,
|
||||
index_q_rope_pack_flydsl,
|
||||
index_q_rope_pack_weights_flydsl,
|
||||
indexer_head_weights,
|
||||
logits_rows_per_chunk,
|
||||
@@ -91,6 +92,19 @@ def _indexer_inputs(layer, x, q_lora, pos):
|
||||
indexer.head_weight_scale,
|
||||
num_heads=indexer.n_heads,
|
||||
)
|
||||
if (
|
||||
indexer.n_heads % 16 == 0
|
||||
and indexer.n_heads <= 64
|
||||
and indexer.index_head_dim == 128
|
||||
and indexer.rope_head_dim == 64
|
||||
and layer.freqs_cis.dtype == torch.complex64
|
||||
):
|
||||
# rows past the GEMV's cap: wq_b, then one launch for RoPE and fp4 pack
|
||||
q, _ = indexer.wq_b(q_lora)
|
||||
q_fp4, q_scale = index_q_rope_pack_flydsl(
|
||||
q, layer.freqs_cis, pos, indexer.rope_head_dim, num_heads=indexer.n_heads
|
||||
)
|
||||
return q_fp4, q_scale, _indexer_head_weights(indexer, x)
|
||||
# [T, H, 128] fp4 grid; the RoPE launch gathers freqs_cis[pos] itself
|
||||
q = indexer.queries(q_lora, layer.freqs_cis, positions=pos)
|
||||
q_fp4, q_scale = pack_fp4_query_flydsl(q)
|
||||
|
||||
@@ -19,6 +19,8 @@ import sys
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.attention.deepseek_v4_rope import precompute_freqs_cis
|
||||
from sglang.kernels.ops.attention.dsv4 import (
|
||||
@@ -26,14 +28,19 @@ from sglang.kernels.ops.attention.dsv4 import (
|
||||
compress_norm_rope_store,
|
||||
)
|
||||
from sglang.kernels.ops.attention.dsv4.compress import CompressorPrefillPlan
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer import quantize_fp4_indexer_tensor
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer import (
|
||||
_fp4_e2m1_code_rne,
|
||||
quantize_fp4_indexer_tensor,
|
||||
)
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import (
|
||||
FP4KWriteMetadata,
|
||||
_decode_cta_count,
|
||||
_fp4_e2m1_code_rne_lean,
|
||||
_guard_page_table,
|
||||
aiter_fp4_paged_mqa_logits,
|
||||
aiter_k_indexer_fp4_cache_write,
|
||||
aiter_q_indexer_fp4,
|
||||
index_q_rope_pack_flydsl,
|
||||
index_q_rope_pack_weights_flydsl,
|
||||
indexer_head_weights,
|
||||
pack_fp4_query_flydsl,
|
||||
@@ -332,6 +339,130 @@ def test_index_q_pack_weights_matches_standalone() -> None:
|
||||
assert int(ulps.max()) <= 1
|
||||
|
||||
|
||||
def _assert_index_q_rope_pack_matches(
|
||||
q: torch.Tensor, freqs: torch.Tensor, pos: torch.Tensor, num_heads: int
|
||||
) -> None:
|
||||
"""index_q_rope_pack_flydsl is bitwise the standalone RoPE fake-quant and FlyDSL pack."""
|
||||
rope_dim = 64
|
||||
ref_q = rope_tail_fake_quant_fp4(
|
||||
q.view(q.shape[0], num_heads, 128), freqs, rope_dim, positions=pos.contiguous()
|
||||
)
|
||||
ref_fp4, ref_scale = pack_fp4_query_flydsl(ref_q)
|
||||
q_fp4, q_scale = index_q_rope_pack_flydsl(
|
||||
q, freqs, pos, rope_dim, num_heads=num_heads
|
||||
)
|
||||
assert q_fp4.dtype == ref_fp4.dtype and q_scale.dtype == ref_scale.dtype
|
||||
assert torch.equal(q_fp4, ref_fp4)
|
||||
assert torch.equal(q_scale, ref_scale)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_heads", [16, 32, 64])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 333, 16384])
|
||||
def test_index_q_rope_pack_matches_standalone(num_tokens: int, num_heads: int) -> None:
|
||||
"""The prefill index-Q launch (RoPE and two-stage fp4 pack in the FlyDSL layout) is
|
||||
bitwise the two standalone launches it replaces, past the decode path's caps (64 tokens
|
||||
for the split-K GEMV, T * H <= 4096 for the fused head-weight reduce)."""
|
||||
|
||||
torch.manual_seed(0)
|
||||
rope_dim, max_pos = 64, 65536
|
||||
q = (torch.randn(num_tokens, num_heads * 128, device="cuda") * 3).bfloat16()
|
||||
# rows small enough to reach the pack's 1e-4 scale floor; the fake-quant floor
|
||||
# (6 * 2^-126) is covered by test_index_q_rope_pack_amax_floors
|
||||
q[: num_tokens // 10 + 1] *= 1e-6
|
||||
if num_tokens > 4:
|
||||
# groups whose reciprocal scales take the inf / nan / near-overflow branches
|
||||
q[1, 3] = float("inf")
|
||||
q[2, 200] = float("nan")
|
||||
q[3, :128] = 3.0e38
|
||||
q[4, 64:] = float("-inf")
|
||||
freqs = precompute_freqs_cis(rope_dim, max_pos, 0, 10000, 1, 32, 1).to("cuda")
|
||||
pos = torch.randint(0, max_pos, (num_tokens,), device="cuda", dtype=torch.int64)
|
||||
_assert_index_q_rope_pack_matches(q, freqs, pos, num_heads)
|
||||
|
||||
|
||||
def test_index_q_rope_pack_strided_positions() -> None:
|
||||
"""A non-contiguous positions view reads positions[t], not the t-th stored element."""
|
||||
torch.manual_seed(0)
|
||||
num_tokens, num_heads, max_pos = 333, 64, 65536
|
||||
q = (torch.randn(num_tokens, num_heads * 128, device="cuda") * 3).bfloat16()
|
||||
freqs = precompute_freqs_cis(64, max_pos, 0, 10000, 1, 32, 1).to("cuda")
|
||||
pos = torch.randint(0, max_pos, (2 * num_tokens,), device="cuda")[::2]
|
||||
assert not pos.is_contiguous()
|
||||
_assert_index_q_rope_pack_matches(q, freqs, pos, num_heads)
|
||||
|
||||
|
||||
def _bf16_neighbors(x: float) -> list[float]:
|
||||
"""x rounded to bf16 and the bf16 values one ulp below and above it."""
|
||||
bits = int(torch.tensor(x).bfloat16().view(torch.int16))
|
||||
return [
|
||||
float(torch.tensor(b, dtype=torch.int16).view(torch.bfloat16))
|
||||
for b in (bits - 1, bits, bits + 1)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_heads", [16, 64])
|
||||
def test_index_q_rope_pack_amax_floors(num_heads: int) -> None:
|
||||
"""Zero groups, bf16 subnormals and amaxes one ulp around both floors take the
|
||||
reciprocal-scale branches bitwise like the standalone divisions. The fake-quant floor
|
||||
(amax 6 * 2^-126) keeps a zero group off 0 / 0; anything that small then codes 0 under
|
||||
the pack floor (scale 1e-4, which ceils to 2^-13 and so first differs at amax 6 * 2^-14)."""
|
||||
torch.manual_seed(0)
|
||||
num_tokens, max_pos = 256, 65536
|
||||
group_amax = torch.tensor(
|
||||
[
|
||||
0.0,
|
||||
2.0**-133, # smallest bf16 subnormal
|
||||
2.0**-130,
|
||||
127 * 2.0**-133, # largest bf16 subnormal
|
||||
2.0**-126,
|
||||
*_bf16_neighbors(6 * 2.0**-126),
|
||||
2.0**-120,
|
||||
1.0e-30,
|
||||
*_bf16_neighbors(6 * 2.0**-14),
|
||||
*_bf16_neighbors(6.0e-4),
|
||||
1.0,
|
||||
]
|
||||
)
|
||||
# every 32-group gets one of the amaxes above, its largest element at exactly that value
|
||||
shape = (num_tokens, num_heads, 4, 32)
|
||||
u = torch.randn(shape)
|
||||
u = u / u.abs().amax(dim=-1, keepdim=True)
|
||||
m = group_amax[torch.randint(0, group_amax.numel(), shape[:3])]
|
||||
q = (u * m[..., None]).bfloat16()
|
||||
assert torch.equal(q.float().abs().amax(dim=-1), m.bfloat16().float())
|
||||
q[0] = 0
|
||||
q[1, 3] = 0
|
||||
q = q.view(num_tokens, num_heads * 128).cuda()
|
||||
freqs = precompute_freqs_cis(64, max_pos, 0, 10000, 1, 32, 1).to("cuda")
|
||||
pos = torch.randint(0, max_pos, (num_tokens,), device="cuda", dtype=torch.int64)
|
||||
_assert_index_q_rope_pack_matches(q, freqs, pos, num_heads)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _e2m1_codes_kernel(x_ptr, ref_ptr, lean_ptr, n, BLOCK: tl.constexpr):
|
||||
offs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
|
||||
mask = offs < n
|
||||
x = tl.load(x_ptr + offs, mask=mask)
|
||||
tl.store(ref_ptr + offs, _fp4_e2m1_code_rne(x), mask=mask)
|
||||
tl.store(lean_ptr + offs, _fp4_e2m1_code_rne_lean(x), mask=mask)
|
||||
|
||||
|
||||
def test_fp4_e2m1_code_rne_lean_matches() -> None:
|
||||
"""The prefill index-Q kernel's e2m1 code equals _fp4_e2m1_code_rne on every bf16 value
|
||||
(all the kernel feeds it: bf16 values times a power of two) and on random fp32 bits."""
|
||||
every_bf16 = torch.arange(1 << 16, dtype=torch.int32).to(torch.int16)
|
||||
every_bf16 = every_bf16.view(torch.bfloat16).float()
|
||||
scaled = torch.cat([every_bf16 * 2.0**k for k in (-8, -1, 0, 1, 8)])
|
||||
random_bits = torch.randint(-(1 << 31), 1 << 31, (1 << 22,), dtype=torch.int64)
|
||||
x = torch.cat([scaled, random_bits.to(torch.int32).view(torch.float32)]).cuda()
|
||||
ref = torch.empty(x.shape, dtype=torch.uint8, device="cuda")
|
||||
lean = torch.empty_like(ref)
|
||||
_e2m1_codes_kernel[(triton.cdiv(x.numel(), 1024),)](
|
||||
x, ref, lean, x.numel(), BLOCK=1024
|
||||
)
|
||||
assert torch.equal(lean, ref)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 16, 96])
|
||||
def test_fp4_index_cache_store_layout(num_tokens: int) -> None:
|
||||
"""Scattered slots land in the paged layout and touch nothing else."""
|
||||
|
||||
Reference in New Issue
Block a user