[AMD] Fuse prefill index-Q RoPE fake-quant and fp4 quantize kernels in DSV4.1 (#42911)

This commit is contained in:
Xinyi Song
2026-10-12 22:33:53 -07:00
committed by GitHub
parent 42e43cf33f
commit a69e893b1a
3 changed files with 310 additions and 1 deletions
@@ -8,8 +8,10 @@ from typing import TYPE_CHECKING, NamedTuple, Optional, Tuple, Union
import torch
import triton
import triton.language as tl
from triton.language.extra import libdevice
from sglang.kernels.ops.attention.dsv4.fp4_indexer import (
_ceil_ue8m0_exp,
quantize_fp4_indexer_row,
quantize_fp4_indexer_tensor,
)
@@ -743,6 +745,120 @@ def _reduce_scale_bf16_kernel(
)
@triton.jit
def _fp4_e2m1_code_rne_lean(x):
"""_fp4_e2m1_code_rne for every fp32 x (checked over all 2**32): within each spacing of the
e2m1 grid, rint's ties-to-even picks the even code at the midpoints."""
ax = tl.minimum(tl.abs(x), 6.0)
lo = ax < 2.0
mid = ax < 4.0
rstep = tl.where(lo, 2.0, tl.where(mid, 1.0, 0.5))
idx = (libdevice.rint(ax * rstep) + tl.where(lo, 0.0, tl.where(mid, 2.0, 4.0))).to(
tl.uint8
)
idx = tl.where(ax == ax, idx, 0)
sign = ((x < 0) & (idx != 0)).to(tl.uint8)
return idx | (sign << 3)
@triton.jit
def _index_q_quant_groups(v, AMAX_FLOOR: tl.constexpr):
"""[16, G, 32] fp32 through the fake-quant of rope_tail_fake_quant_fp4_row (rounded to bf16
as the standalone kernel stores it) and quantize_fp4_indexer_row (RNE): (packed codes
[16, G * 16], ue8m0 exponents [16, G]). The power-of-two divisions run as multiplies by
exact reciprocals, which round identically."""
amax = tl.maximum(tl.max(tl.abs(v), axis=2), AMAX_FLOOR) * (1.0 / 6.0)
bits = amax.to(tl.int32, bitcast=True)
expo = ((bits >> 23) & 0xFF) - 127
expo = expo + ((bits & 0x7FFFFF) != 0).to(tl.int32)
scale = ((expo + 127) << 23).to(tl.float32, bitcast=True)
# 1 / scale: 2^-expo for a finite amax (expo <= 126), 0 for inf (scale = inf),
# -inf for nan (expo = 129 wraps scale to -0.0)
rscale = tl.where(
expo <= 126,
((127 - expo) << 23).to(tl.float32, bitcast=True),
tl.where(expo == 128, 0.0, float("-inf")),
)
s = tl.minimum(tl.maximum(v * rscale[:, :, None], -6.0), 6.0)
mag = tl.abs(s)
rstep = tl.where(mag < 2.0, 2.0, tl.where(mag < 4.0, 1.0, 0.5))
step = ((254 << 23) - rstep.to(tl.int32, bitcast=True)).to(tl.float32, bitcast=True)
# rint is odd-symmetric, so the sign rides along; a zero may come out as -0, which neither
# the amax nor the codes below tell apart
values = (libdevice.rint(s * rstep) * step * scale[:, :, None]).to(tl.bfloat16)
values = values.to(tl.float32)
exp = _ceil_ue8m0_exp(tl.maximum(tl.max(tl.abs(values), axis=2) / 6.0, 1.0e-4))
# 1 / 2^(exp - 127); exp = 254 (an inf / nan group) as 0.5 * 2^-126, where any value
# rounded differently is below 2^-126 and codes 0 either way
top = exp == 254
ra = tl.where(top, 0.5, 1.0)
rb = tl.where(top, 2.0**-126, ((254 - exp) << 23).to(tl.float32, bitcast=True))
code = _fp4_e2m1_code_rne_lean(values * ra[:, :, None] * rb[:, :, None])
G: tl.constexpr = v.shape[1]
c0, c1 = tl.split(tl.reshape(code, (16, G * 16, 2)))
return (c0 & 0x0F) | ((c1 & 0x0F) << 4), exp
@triton.jit
def _index_q_rope_pack_kernel(
q_ptr, # [T, H * 128] bf16 (wq_b output)
f_ptr, # [max_pos, RD // 2, 2] fp32: the real view of the complex freqs table
pos_ptr, # [T] int
q_fp4_ptr, # [T, H, 64] int8
q_scale_ptr, # [T, 1, 4, 16, 4] uint8
stride_qt,
num_pos,
H: tl.constexpr,
D: tl.constexpr,
RD: tl.constexpr,
AMAX_FLOOR: tl.constexpr,
):
"""Grid (T, H // 16): 16 heads of one token per program, through _index_q_quant_groups, in
the FlyDSL MQA-logits layout."""
NOPE: tl.constexpr = D - RD
t = tl.program_id(0)
h0 = tl.program_id(1) * 16
hh = h0 + tl.arange(0, 16)
pos = tl.load(pos_ptr + t).to(tl.int64)
# the caller owns positions < num_pos; an out-of-range row reads entry 0 instead of past the table
pos = tl.where((pos >= 0) & (pos < num_pos), pos, 0)
q_row = q_ptr + t.to(tl.int64) * stride_qt + hh[:, None] * D
out_row = q_fp4_ptr + (t.to(tl.int64) * H + hh[:, None]) * (D // 2)
x = tl.load(q_row + tl.arange(0, NOPE)[None, :]).to(tl.float32)
packed, exp_nope = _index_q_quant_groups(
tl.reshape(x, (16, NOPE // 32, 32)), AMAX_FLOOR
)
tl.store(out_row + tl.arange(0, NOPE // 2)[None, :], packed)
# RoPE on the last RD features, adjacent pairs as one complex number
x = tl.load(q_row + NOPE + tl.arange(0, RD)[None, :]).to(tl.float32)
re, im = tl.split(tl.reshape(x, (16, RD // 2, 2)))
j = tl.arange(0, RD // 2)
fr = tl.load(f_ptr + pos * RD + 2 * j)[None, :]
fi = tl.load(f_ptr + pos * RD + 2 * j + 1)[None, :]
re_rot = (re * fr - im * fi).to(tl.bfloat16).to(tl.float32)
im_rot = (re * fi + im * fr).to(tl.bfloat16).to(tl.float32)
packed, exp_rope = _index_q_quant_groups(
tl.reshape(tl.join(re_rot, im_rot), (16, RD // 32, 32)), AMAX_FLOOR
)
tl.store(out_row + NOPE // 2 + tl.arange(0, RD // 2)[None, :], packed)
# scale bytes: chunk c of head h at [t, 0, c, h % 16, h // 16]
base = q_scale_ptr + t.to(tl.int64) * (4 * 16 * 4)
i = tl.arange(0, 16)[:, None]
c_nope = tl.arange(0, NOPE // 32)[None, :]
c_rope = NOPE // 32 + tl.arange(0, RD // 32)[None, :]
tl.store(base + (c_nope * 16 + i) * 4 + h0 // 16, exp_nope.to(tl.uint8))
tl.store(base + (c_rope * 16 + i) * 4 + h0 // 16, exp_rope.to(tl.uint8))
if h0 == 0:
# groups this head count does not have stay zero
slot = (tl.arange(0, 4)[None, :] * 16 + i) * 4
for g in tl.static_range(H // 16, 4):
tl.store(base + slot + g, tl.zeros((16, 4), dtype=tl.uint8))
@triton.jit
def _index_q_pack_weights_kernel(
q_ptr, # [T, H * 128] bf16 (wq_b output)
@@ -817,6 +933,54 @@ def _index_q_pack_weights_kernel(
)
def index_q_rope_pack_flydsl(
q: torch.Tensor,
freqs_cis: torch.Tensor,
positions: torch.Tensor,
rope_dim: int,
*,
num_heads: int,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""The index-Q RoPE and fp4 pack at any row count: (q_fp4 [T, H, 64] int8, q_scale
[T, 1, 4, 16, 4] uint8), bitwise pack_fp4_query_flydsl(_rope_fq4(q.view(T, H, 128),
freqs_cis[positions], rope_dim)) without the bf16 fake-quant round trip through memory."""
T = q.shape[0]
H = num_heads
assert q.dtype == torch.bfloat16 and q.dim() == 2 and q.shape[1] == H * 128
assert q.stride(1) == 1
assert H % 16 == 0 and 0 < H <= 64, H
assert freqs_cis.dtype == torch.complex64 and freqs_cis.shape[1] == rope_dim // 2
# the plain and rotated halves are each whole 32-groups of a power-of-two width
assert rope_dim == 64, rope_dim
assert positions.shape == (T,)
# the kernel reads positions[t] at pos_ptr + t
positions = positions.contiguous()
f_real = torch.view_as_real(freqs_cis)
assert f_real.is_contiguous()
q_fp4 = torch.empty((T, H, 64), dtype=torch.int8, device=q.device)
q_scale = torch.empty((T, 1, 4, 16, 4), dtype=torch.uint8, device=q.device)
if T == 0:
return q_fp4, q_scale
_index_q_rope_pack_kernel[(T, H // 16)](
q,
f_real,
positions,
q_fp4,
q_scale,
q.stride(0),
f_real.shape[0],
H=H,
D=128,
RD=rope_dim,
AMAX_FLOOR=FP4_AMAX_FLOOR,
num_warps=2,
# bitwise parity with the standalone RoPE kernel: a contracted FMA in the rotation flips
# the bf16 rounding of a few values in 10^8
enable_fp_fusion=False,
)
return q_fp4, q_scale
def index_q_rope_pack_weights_flydsl(
q: torch.Tensor,
freqs_cis: torch.Tensor,
@@ -23,6 +23,7 @@ from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import (
FP4DecodeWorkspace,
FP4PrefillWorkspace,
aiter_fp4_paged_mqa_logits,
index_q_rope_pack_flydsl,
index_q_rope_pack_weights_flydsl,
indexer_head_weights,
logits_rows_per_chunk,
@@ -91,6 +92,19 @@ def _indexer_inputs(layer, x, q_lora, pos):
indexer.head_weight_scale,
num_heads=indexer.n_heads,
)
if (
indexer.n_heads % 16 == 0
and indexer.n_heads <= 64
and indexer.index_head_dim == 128
and indexer.rope_head_dim == 64
and layer.freqs_cis.dtype == torch.complex64
):
# rows past the GEMV's cap: wq_b, then one launch for RoPE and fp4 pack
q, _ = indexer.wq_b(q_lora)
q_fp4, q_scale = index_q_rope_pack_flydsl(
q, layer.freqs_cis, pos, indexer.rope_head_dim, num_heads=indexer.n_heads
)
return q_fp4, q_scale, _indexer_head_weights(indexer, x)
# [T, H, 128] fp4 grid; the RoPE launch gathers freqs_cis[pos] itself
q = indexer.queries(q_lora, layer.freqs_cis, positions=pos)
q_fp4, q_scale = pack_fp4_query_flydsl(q)
@@ -19,6 +19,8 @@ import sys
import pytest
import torch
import triton
import triton.language as tl
from sglang.kernels.ops.attention.deepseek_v4_rope import precompute_freqs_cis
from sglang.kernels.ops.attention.dsv4 import (
@@ -26,14 +28,19 @@ from sglang.kernels.ops.attention.dsv4 import (
compress_norm_rope_store,
)
from sglang.kernels.ops.attention.dsv4.compress import CompressorPrefillPlan
from sglang.kernels.ops.attention.dsv4.fp4_indexer import quantize_fp4_indexer_tensor
from sglang.kernels.ops.attention.dsv4.fp4_indexer import (
_fp4_e2m1_code_rne,
quantize_fp4_indexer_tensor,
)
from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import (
FP4KWriteMetadata,
_decode_cta_count,
_fp4_e2m1_code_rne_lean,
_guard_page_table,
aiter_fp4_paged_mqa_logits,
aiter_k_indexer_fp4_cache_write,
aiter_q_indexer_fp4,
index_q_rope_pack_flydsl,
index_q_rope_pack_weights_flydsl,
indexer_head_weights,
pack_fp4_query_flydsl,
@@ -332,6 +339,130 @@ def test_index_q_pack_weights_matches_standalone() -> None:
assert int(ulps.max()) <= 1
def _assert_index_q_rope_pack_matches(
q: torch.Tensor, freqs: torch.Tensor, pos: torch.Tensor, num_heads: int
) -> None:
"""index_q_rope_pack_flydsl is bitwise the standalone RoPE fake-quant and FlyDSL pack."""
rope_dim = 64
ref_q = rope_tail_fake_quant_fp4(
q.view(q.shape[0], num_heads, 128), freqs, rope_dim, positions=pos.contiguous()
)
ref_fp4, ref_scale = pack_fp4_query_flydsl(ref_q)
q_fp4, q_scale = index_q_rope_pack_flydsl(
q, freqs, pos, rope_dim, num_heads=num_heads
)
assert q_fp4.dtype == ref_fp4.dtype and q_scale.dtype == ref_scale.dtype
assert torch.equal(q_fp4, ref_fp4)
assert torch.equal(q_scale, ref_scale)
@pytest.mark.parametrize("num_heads", [16, 32, 64])
@pytest.mark.parametrize("num_tokens", [1, 333, 16384])
def test_index_q_rope_pack_matches_standalone(num_tokens: int, num_heads: int) -> None:
"""The prefill index-Q launch (RoPE and two-stage fp4 pack in the FlyDSL layout) is
bitwise the two standalone launches it replaces, past the decode path's caps (64 tokens
for the split-K GEMV, T * H <= 4096 for the fused head-weight reduce)."""
torch.manual_seed(0)
rope_dim, max_pos = 64, 65536
q = (torch.randn(num_tokens, num_heads * 128, device="cuda") * 3).bfloat16()
# rows small enough to reach the pack's 1e-4 scale floor; the fake-quant floor
# (6 * 2^-126) is covered by test_index_q_rope_pack_amax_floors
q[: num_tokens // 10 + 1] *= 1e-6
if num_tokens > 4:
# groups whose reciprocal scales take the inf / nan / near-overflow branches
q[1, 3] = float("inf")
q[2, 200] = float("nan")
q[3, :128] = 3.0e38
q[4, 64:] = float("-inf")
freqs = precompute_freqs_cis(rope_dim, max_pos, 0, 10000, 1, 32, 1).to("cuda")
pos = torch.randint(0, max_pos, (num_tokens,), device="cuda", dtype=torch.int64)
_assert_index_q_rope_pack_matches(q, freqs, pos, num_heads)
def test_index_q_rope_pack_strided_positions() -> None:
"""A non-contiguous positions view reads positions[t], not the t-th stored element."""
torch.manual_seed(0)
num_tokens, num_heads, max_pos = 333, 64, 65536
q = (torch.randn(num_tokens, num_heads * 128, device="cuda") * 3).bfloat16()
freqs = precompute_freqs_cis(64, max_pos, 0, 10000, 1, 32, 1).to("cuda")
pos = torch.randint(0, max_pos, (2 * num_tokens,), device="cuda")[::2]
assert not pos.is_contiguous()
_assert_index_q_rope_pack_matches(q, freqs, pos, num_heads)
def _bf16_neighbors(x: float) -> list[float]:
"""x rounded to bf16 and the bf16 values one ulp below and above it."""
bits = int(torch.tensor(x).bfloat16().view(torch.int16))
return [
float(torch.tensor(b, dtype=torch.int16).view(torch.bfloat16))
for b in (bits - 1, bits, bits + 1)
]
@pytest.mark.parametrize("num_heads", [16, 64])
def test_index_q_rope_pack_amax_floors(num_heads: int) -> None:
"""Zero groups, bf16 subnormals and amaxes one ulp around both floors take the
reciprocal-scale branches bitwise like the standalone divisions. The fake-quant floor
(amax 6 * 2^-126) keeps a zero group off 0 / 0; anything that small then codes 0 under
the pack floor (scale 1e-4, which ceils to 2^-13 and so first differs at amax 6 * 2^-14)."""
torch.manual_seed(0)
num_tokens, max_pos = 256, 65536
group_amax = torch.tensor(
[
0.0,
2.0**-133, # smallest bf16 subnormal
2.0**-130,
127 * 2.0**-133, # largest bf16 subnormal
2.0**-126,
*_bf16_neighbors(6 * 2.0**-126),
2.0**-120,
1.0e-30,
*_bf16_neighbors(6 * 2.0**-14),
*_bf16_neighbors(6.0e-4),
1.0,
]
)
# every 32-group gets one of the amaxes above, its largest element at exactly that value
shape = (num_tokens, num_heads, 4, 32)
u = torch.randn(shape)
u = u / u.abs().amax(dim=-1, keepdim=True)
m = group_amax[torch.randint(0, group_amax.numel(), shape[:3])]
q = (u * m[..., None]).bfloat16()
assert torch.equal(q.float().abs().amax(dim=-1), m.bfloat16().float())
q[0] = 0
q[1, 3] = 0
q = q.view(num_tokens, num_heads * 128).cuda()
freqs = precompute_freqs_cis(64, max_pos, 0, 10000, 1, 32, 1).to("cuda")
pos = torch.randint(0, max_pos, (num_tokens,), device="cuda", dtype=torch.int64)
_assert_index_q_rope_pack_matches(q, freqs, pos, num_heads)
@triton.jit
def _e2m1_codes_kernel(x_ptr, ref_ptr, lean_ptr, n, BLOCK: tl.constexpr):
offs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
mask = offs < n
x = tl.load(x_ptr + offs, mask=mask)
tl.store(ref_ptr + offs, _fp4_e2m1_code_rne(x), mask=mask)
tl.store(lean_ptr + offs, _fp4_e2m1_code_rne_lean(x), mask=mask)
def test_fp4_e2m1_code_rne_lean_matches() -> None:
"""The prefill index-Q kernel's e2m1 code equals _fp4_e2m1_code_rne on every bf16 value
(all the kernel feeds it: bf16 values times a power of two) and on random fp32 bits."""
every_bf16 = torch.arange(1 << 16, dtype=torch.int32).to(torch.int16)
every_bf16 = every_bf16.view(torch.bfloat16).float()
scaled = torch.cat([every_bf16 * 2.0**k for k in (-8, -1, 0, 1, 8)])
random_bits = torch.randint(-(1 << 31), 1 << 31, (1 << 22,), dtype=torch.int64)
x = torch.cat([scaled, random_bits.to(torch.int32).view(torch.float32)]).cuda()
ref = torch.empty(x.shape, dtype=torch.uint8, device="cuda")
lean = torch.empty_like(ref)
_e2m1_codes_kernel[(triton.cdiv(x.numel(), 1024),)](
x, ref, lean, x.numel(), BLOCK=1024
)
assert torch.equal(lean, ref)
@pytest.mark.parametrize("num_tokens", [1, 16, 96])
def test_fp4_index_cache_store_layout(num_tokens: int) -> None:
"""Scattered slots land in the paged layout and touch nothing else."""