[AMD] Fuse prefill index-Q RoPE fake-quant and fp4 quantize kernels in DSV4.1 (#42911)

This commit is contained in:
Xinyi Song
2026-10-12 22:33:53 -07:00
committed by GitHub
parent 42e43cf33f
commit a69e893b1a
3 changed files with 310 additions and 1 deletions
@@ -8,8 +8,10 @@ from typing import TYPE_CHECKING, NamedTuple, Optional, Tuple, Union
import torch
import triton
import triton.language as tl
from triton.language.extra import libdevice
from sglang.kernels.ops.attention.dsv4.fp4_indexer import (
_ceil_ue8m0_exp,
quantize_fp4_indexer_row,
quantize_fp4_indexer_tensor,
)
@@ -743,6 +745,120 @@ def _reduce_scale_bf16_kernel(
)
@triton.jit
def _fp4_e2m1_code_rne_lean(x):
"""_fp4_e2m1_code_rne for every fp32 x (checked over all 2**32): within each spacing of the
e2m1 grid, rint's ties-to-even picks the even code at the midpoints."""
ax = tl.minimum(tl.abs(x), 6.0)
lo = ax < 2.0
mid = ax < 4.0
rstep = tl.where(lo, 2.0, tl.where(mid, 1.0, 0.5))
idx = (libdevice.rint(ax * rstep) + tl.where(lo, 0.0, tl.where(mid, 2.0, 4.0))).to(
tl.uint8
)
idx = tl.where(ax == ax, idx, 0)
sign = ((x < 0) & (idx != 0)).to(tl.uint8)
return idx | (sign << 3)
@triton.jit
def _index_q_quant_groups(v, AMAX_FLOOR: tl.constexpr):
"""[16, G, 32] fp32 through the fake-quant of rope_tail_fake_quant_fp4_row (rounded to bf16
as the standalone kernel stores it) and quantize_fp4_indexer_row (RNE): (packed codes
[16, G * 16], ue8m0 exponents [16, G]). The power-of-two divisions run as multiplies by
exact reciprocals, which round identically."""
amax = tl.maximum(tl.max(tl.abs(v), axis=2), AMAX_FLOOR) * (1.0 / 6.0)
bits = amax.to(tl.int32, bitcast=True)
expo = ((bits >> 23) & 0xFF) - 127
expo = expo + ((bits & 0x7FFFFF) != 0).to(tl.int32)
scale = ((expo + 127) << 23).to(tl.float32, bitcast=True)
# 1 / scale: 2^-expo for a finite amax (expo <= 126), 0 for inf (scale = inf),
# -inf for nan (expo = 129 wraps scale to -0.0)
rscale = tl.where(
expo <= 126,
((127 - expo) << 23).to(tl.float32, bitcast=True),
tl.where(expo == 128, 0.0, float("-inf")),
)
s = tl.minimum(tl.maximum(v * rscale[:, :, None], -6.0), 6.0)
mag = tl.abs(s)
rstep = tl.where(mag < 2.0, 2.0, tl.where(mag < 4.0, 1.0, 0.5))
step = ((254 << 23) - rstep.to(tl.int32, bitcast=True)).to(tl.float32, bitcast=True)
# rint is odd-symmetric, so the sign rides along; a zero may come out as -0, which neither
# the amax nor the codes below tell apart
values = (libdevice.rint(s * rstep) * step * scale[:, :, None]).to(tl.bfloat16)
values = values.to(tl.float32)
exp = _ceil_ue8m0_exp(tl.maximum(tl.max(tl.abs(values), axis=2) / 6.0, 1.0e-4))
# 1 / 2^(exp - 127); exp = 254 (an inf / nan group) as 0.5 * 2^-126, where any value
# rounded differently is below 2^-126 and codes 0 either way
top = exp == 254
ra = tl.where(top, 0.5, 1.0)
rb = tl.where(top, 2.0**-126, ((254 - exp) << 23).to(tl.float32, bitcast=True))
code = _fp4_e2m1_code_rne_lean(values * ra[:, :, None] * rb[:, :, None])
G: tl.constexpr = v.shape[1]
c0, c1 = tl.split(tl.reshape(code, (16, G * 16, 2)))
return (c0 & 0x0F) | ((c1 & 0x0F) << 4), exp
@triton.jit
def _index_q_rope_pack_kernel(
q_ptr, # [T, H * 128] bf16 (wq_b output)
f_ptr, # [max_pos, RD // 2, 2] fp32: the real view of the complex freqs table
pos_ptr, # [T] int
q_fp4_ptr, # [T, H, 64] int8
q_scale_ptr, # [T, 1, 4, 16, 4] uint8
stride_qt,
num_pos,
H: tl.constexpr,
D: tl.constexpr,
RD: tl.constexpr,
AMAX_FLOOR: tl.constexpr,
):
"""Grid (T, H // 16): 16 heads of one token per program, through _index_q_quant_groups, in
the FlyDSL MQA-logits layout."""
NOPE: tl.constexpr = D - RD
t = tl.program_id(0)
h0 = tl.program_id(1) * 16
hh = h0 + tl.arange(0, 16)
pos = tl.load(pos_ptr + t).to(tl.int64)
# the caller owns positions < num_pos; an out-of-range row reads entry 0 instead of past the table
pos = tl.where((pos >= 0) & (pos < num_pos), pos, 0)
q_row = q_ptr + t.to(tl.int64) * stride_qt + hh[:, None] * D
out_row = q_fp4_ptr + (t.to(tl.int64) * H + hh[:, None]) * (D // 2)
x = tl.load(q_row + tl.arange(0, NOPE)[None, :]).to(tl.float32)
packed, exp_nope = _index_q_quant_groups(
tl.reshape(x, (16, NOPE // 32, 32)), AMAX_FLOOR
)
tl.store(out_row + tl.arange(0, NOPE // 2)[None, :], packed)
# RoPE on the last RD features, adjacent pairs as one complex number
x = tl.load(q_row + NOPE + tl.arange(0, RD)[None, :]).to(tl.float32)
re, im = tl.split(tl.reshape(x, (16, RD // 2, 2)))
j = tl.arange(0, RD // 2)
fr = tl.load(f_ptr + pos * RD + 2 * j)[None, :]
fi = tl.load(f_ptr + pos * RD + 2 * j + 1)[None, :]
re_rot = (re * fr - im * fi).to(tl.bfloat16).to(tl.float32)
im_rot = (re * fi + im * fr).to(tl.bfloat16).to(tl.float32)
packed, exp_rope = _index_q_quant_groups(
tl.reshape(tl.join(re_rot, im_rot), (16, RD // 32, 32)), AMAX_FLOOR
)
tl.store(out_row + NOPE // 2 + tl.arange(0, RD // 2)[None, :], packed)
# scale bytes: chunk c of head h at [t, 0, c, h % 16, h // 16]
base = q_scale_ptr + t.to(tl.int64) * (4 * 16 * 4)
i = tl.arange(0, 16)[:, None]
c_nope = tl.arange(0, NOPE // 32)[None, :]
c_rope = NOPE // 32 + tl.arange(0, RD // 32)[None, :]
tl.store(base + (c_nope * 16 + i) * 4 + h0 // 16, exp_nope.to(tl.uint8))
tl.store(base + (c_rope * 16 + i) * 4 + h0 // 16, exp_rope.to(tl.uint8))
if h0 == 0:
# groups this head count does not have stay zero
slot = (tl.arange(0, 4)[None, :] * 16 + i) * 4
for g in tl.static_range(H // 16, 4):
tl.store(base + slot + g, tl.zeros((16, 4), dtype=tl.uint8))
@triton.jit
def _index_q_pack_weights_kernel(
q_ptr, # [T, H * 128] bf16 (wq_b output)
@@ -817,6 +933,54 @@ def _index_q_pack_weights_kernel(
)
def index_q_rope_pack_flydsl(
q: torch.Tensor,
freqs_cis: torch.Tensor,
positions: torch.Tensor,
rope_dim: int,
*,
num_heads: int,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""The index-Q RoPE and fp4 pack at any row count: (q_fp4 [T, H, 64] int8, q_scale
[T, 1, 4, 16, 4] uint8), bitwise pack_fp4_query_flydsl(_rope_fq4(q.view(T, H, 128),
freqs_cis[positions], rope_dim)) without the bf16 fake-quant round trip through memory."""
T = q.shape[0]
H = num_heads
assert q.dtype == torch.bfloat16 and q.dim() == 2 and q.shape[1] == H * 128
assert q.stride(1) == 1
assert H % 16 == 0 and 0 < H <= 64, H
assert freqs_cis.dtype == torch.complex64 and freqs_cis.shape[1] == rope_dim // 2
# the plain and rotated halves are each whole 32-groups of a power-of-two width
assert rope_dim == 64, rope_dim
assert positions.shape == (T,)
# the kernel reads positions[t] at pos_ptr + t
positions = positions.contiguous()
f_real = torch.view_as_real(freqs_cis)
assert f_real.is_contiguous()
q_fp4 = torch.empty((T, H, 64), dtype=torch.int8, device=q.device)
q_scale = torch.empty((T, 1, 4, 16, 4), dtype=torch.uint8, device=q.device)
if T == 0:
return q_fp4, q_scale
_index_q_rope_pack_kernel[(T, H // 16)](
q,
f_real,
positions,
q_fp4,
q_scale,
q.stride(0),
f_real.shape[0],
H=H,
D=128,
RD=rope_dim,
AMAX_FLOOR=FP4_AMAX_FLOOR,
num_warps=2,
# bitwise parity with the standalone RoPE kernel: a contracted FMA in the rotation flips
# the bf16 rounding of a few values in 10^8
enable_fp_fusion=False,
)
return q_fp4, q_scale
def index_q_rope_pack_weights_flydsl(
q: torch.Tensor,
freqs_cis: torch.Tensor,
@@ -23,6 +23,7 @@ from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import (
FP4DecodeWorkspace,
FP4PrefillWorkspace,
aiter_fp4_paged_mqa_logits,
index_q_rope_pack_flydsl,
index_q_rope_pack_weights_flydsl,
indexer_head_weights,
logits_rows_per_chunk,
@@ -91,6 +92,19 @@ def _indexer_inputs(layer, x, q_lora, pos):
indexer.head_weight_scale,
num_heads=indexer.n_heads,
)
if (
indexer.n_heads % 16 == 0
and indexer.n_heads <= 64
and indexer.index_head_dim == 128
and indexer.rope_head_dim == 64
and layer.freqs_cis.dtype == torch.complex64
):
# rows past the GEMV's cap: wq_b, then one launch for RoPE and fp4 pack
q, _ = indexer.wq_b(q_lora)
q_fp4, q_scale = index_q_rope_pack_flydsl(
q, layer.freqs_cis, pos, indexer.rope_head_dim, num_heads=indexer.n_heads
)
return q_fp4, q_scale, _indexer_head_weights(indexer, x)
# [T, H, 128] fp4 grid; the RoPE launch gathers freqs_cis[pos] itself
q = indexer.queries(q_lora, layer.freqs_cis, positions=pos)
q_fp4, q_scale = pack_fp4_query_flydsl(q)