diff --git a/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_hip.py b/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_hip.py index cef8fab212..9a56eec586 100644 --- a/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_hip.py +++ b/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_hip.py @@ -8,8 +8,10 @@ from typing import TYPE_CHECKING, NamedTuple, Optional, Tuple, Union import torch import triton import triton.language as tl +from triton.language.extra import libdevice from sglang.kernels.ops.attention.dsv4.fp4_indexer import ( + _ceil_ue8m0_exp, quantize_fp4_indexer_row, quantize_fp4_indexer_tensor, ) @@ -743,6 +745,120 @@ def _reduce_scale_bf16_kernel( ) +@triton.jit +def _fp4_e2m1_code_rne_lean(x): + """_fp4_e2m1_code_rne for every fp32 x (checked over all 2**32): within each spacing of the + e2m1 grid, rint's ties-to-even picks the even code at the midpoints.""" + ax = tl.minimum(tl.abs(x), 6.0) + lo = ax < 2.0 + mid = ax < 4.0 + rstep = tl.where(lo, 2.0, tl.where(mid, 1.0, 0.5)) + idx = (libdevice.rint(ax * rstep) + tl.where(lo, 0.0, tl.where(mid, 2.0, 4.0))).to( + tl.uint8 + ) + idx = tl.where(ax == ax, idx, 0) + sign = ((x < 0) & (idx != 0)).to(tl.uint8) + return idx | (sign << 3) + + +@triton.jit +def _index_q_quant_groups(v, AMAX_FLOOR: tl.constexpr): + """[16, G, 32] fp32 through the fake-quant of rope_tail_fake_quant_fp4_row (rounded to bf16 + as the standalone kernel stores it) and quantize_fp4_indexer_row (RNE): (packed codes + [16, G * 16], ue8m0 exponents [16, G]). The power-of-two divisions run as multiplies by + exact reciprocals, which round identically.""" + amax = tl.maximum(tl.max(tl.abs(v), axis=2), AMAX_FLOOR) * (1.0 / 6.0) + bits = amax.to(tl.int32, bitcast=True) + expo = ((bits >> 23) & 0xFF) - 127 + expo = expo + ((bits & 0x7FFFFF) != 0).to(tl.int32) + scale = ((expo + 127) << 23).to(tl.float32, bitcast=True) + # 1 / scale: 2^-expo for a finite amax (expo <= 126), 0 for inf (scale = inf), + # -inf for nan (expo = 129 wraps scale to -0.0) + rscale = tl.where( + expo <= 126, + ((127 - expo) << 23).to(tl.float32, bitcast=True), + tl.where(expo == 128, 0.0, float("-inf")), + ) + s = tl.minimum(tl.maximum(v * rscale[:, :, None], -6.0), 6.0) + mag = tl.abs(s) + rstep = tl.where(mag < 2.0, 2.0, tl.where(mag < 4.0, 1.0, 0.5)) + step = ((254 << 23) - rstep.to(tl.int32, bitcast=True)).to(tl.float32, bitcast=True) + # rint is odd-symmetric, so the sign rides along; a zero may come out as -0, which neither + # the amax nor the codes below tell apart + values = (libdevice.rint(s * rstep) * step * scale[:, :, None]).to(tl.bfloat16) + values = values.to(tl.float32) + + exp = _ceil_ue8m0_exp(tl.maximum(tl.max(tl.abs(values), axis=2) / 6.0, 1.0e-4)) + # 1 / 2^(exp - 127); exp = 254 (an inf / nan group) as 0.5 * 2^-126, where any value + # rounded differently is below 2^-126 and codes 0 either way + top = exp == 254 + ra = tl.where(top, 0.5, 1.0) + rb = tl.where(top, 2.0**-126, ((254 - exp) << 23).to(tl.float32, bitcast=True)) + code = _fp4_e2m1_code_rne_lean(values * ra[:, :, None] * rb[:, :, None]) + G: tl.constexpr = v.shape[1] + c0, c1 = tl.split(tl.reshape(code, (16, G * 16, 2))) + return (c0 & 0x0F) | ((c1 & 0x0F) << 4), exp + + +@triton.jit +def _index_q_rope_pack_kernel( + q_ptr, # [T, H * 128] bf16 (wq_b output) + f_ptr, # [max_pos, RD // 2, 2] fp32: the real view of the complex freqs table + pos_ptr, # [T] int + q_fp4_ptr, # [T, H, 64] int8 + q_scale_ptr, # [T, 1, 4, 16, 4] uint8 + stride_qt, + num_pos, + H: tl.constexpr, + D: tl.constexpr, + RD: tl.constexpr, + AMAX_FLOOR: tl.constexpr, +): + """Grid (T, H // 16): 16 heads of one token per program, through _index_q_quant_groups, in + the FlyDSL MQA-logits layout.""" + NOPE: tl.constexpr = D - RD + t = tl.program_id(0) + h0 = tl.program_id(1) * 16 + hh = h0 + tl.arange(0, 16) + pos = tl.load(pos_ptr + t).to(tl.int64) + # the caller owns positions < num_pos; an out-of-range row reads entry 0 instead of past the table + pos = tl.where((pos >= 0) & (pos < num_pos), pos, 0) + q_row = q_ptr + t.to(tl.int64) * stride_qt + hh[:, None] * D + out_row = q_fp4_ptr + (t.to(tl.int64) * H + hh[:, None]) * (D // 2) + + x = tl.load(q_row + tl.arange(0, NOPE)[None, :]).to(tl.float32) + packed, exp_nope = _index_q_quant_groups( + tl.reshape(x, (16, NOPE // 32, 32)), AMAX_FLOOR + ) + tl.store(out_row + tl.arange(0, NOPE // 2)[None, :], packed) + + # RoPE on the last RD features, adjacent pairs as one complex number + x = tl.load(q_row + NOPE + tl.arange(0, RD)[None, :]).to(tl.float32) + re, im = tl.split(tl.reshape(x, (16, RD // 2, 2))) + j = tl.arange(0, RD // 2) + fr = tl.load(f_ptr + pos * RD + 2 * j)[None, :] + fi = tl.load(f_ptr + pos * RD + 2 * j + 1)[None, :] + re_rot = (re * fr - im * fi).to(tl.bfloat16).to(tl.float32) + im_rot = (re * fi + im * fr).to(tl.bfloat16).to(tl.float32) + packed, exp_rope = _index_q_quant_groups( + tl.reshape(tl.join(re_rot, im_rot), (16, RD // 32, 32)), AMAX_FLOOR + ) + tl.store(out_row + NOPE // 2 + tl.arange(0, RD // 2)[None, :], packed) + + # scale bytes: chunk c of head h at [t, 0, c, h % 16, h // 16] + base = q_scale_ptr + t.to(tl.int64) * (4 * 16 * 4) + i = tl.arange(0, 16)[:, None] + c_nope = tl.arange(0, NOPE // 32)[None, :] + c_rope = NOPE // 32 + tl.arange(0, RD // 32)[None, :] + tl.store(base + (c_nope * 16 + i) * 4 + h0 // 16, exp_nope.to(tl.uint8)) + tl.store(base + (c_rope * 16 + i) * 4 + h0 // 16, exp_rope.to(tl.uint8)) + if h0 == 0: + # groups this head count does not have stay zero + slot = (tl.arange(0, 4)[None, :] * 16 + i) * 4 + for g in tl.static_range(H // 16, 4): + tl.store(base + slot + g, tl.zeros((16, 4), dtype=tl.uint8)) + + @triton.jit def _index_q_pack_weights_kernel( q_ptr, # [T, H * 128] bf16 (wq_b output) @@ -817,6 +933,54 @@ def _index_q_pack_weights_kernel( ) +def index_q_rope_pack_flydsl( + q: torch.Tensor, + freqs_cis: torch.Tensor, + positions: torch.Tensor, + rope_dim: int, + *, + num_heads: int, +) -> Tuple[torch.Tensor, torch.Tensor]: + """The index-Q RoPE and fp4 pack at any row count: (q_fp4 [T, H, 64] int8, q_scale + [T, 1, 4, 16, 4] uint8), bitwise pack_fp4_query_flydsl(_rope_fq4(q.view(T, H, 128), + freqs_cis[positions], rope_dim)) without the bf16 fake-quant round trip through memory.""" + T = q.shape[0] + H = num_heads + assert q.dtype == torch.bfloat16 and q.dim() == 2 and q.shape[1] == H * 128 + assert q.stride(1) == 1 + assert H % 16 == 0 and 0 < H <= 64, H + assert freqs_cis.dtype == torch.complex64 and freqs_cis.shape[1] == rope_dim // 2 + # the plain and rotated halves are each whole 32-groups of a power-of-two width + assert rope_dim == 64, rope_dim + assert positions.shape == (T,) + # the kernel reads positions[t] at pos_ptr + t + positions = positions.contiguous() + f_real = torch.view_as_real(freqs_cis) + assert f_real.is_contiguous() + q_fp4 = torch.empty((T, H, 64), dtype=torch.int8, device=q.device) + q_scale = torch.empty((T, 1, 4, 16, 4), dtype=torch.uint8, device=q.device) + if T == 0: + return q_fp4, q_scale + _index_q_rope_pack_kernel[(T, H // 16)]( + q, + f_real, + positions, + q_fp4, + q_scale, + q.stride(0), + f_real.shape[0], + H=H, + D=128, + RD=rope_dim, + AMAX_FLOOR=FP4_AMAX_FLOOR, + num_warps=2, + # bitwise parity with the standalone RoPE kernel: a contracted FMA in the rotation flips + # the bf16 rounding of a few values in 10^8 + enable_fp_fusion=False, + ) + return q_fp4, q_scale + + def index_q_rope_pack_weights_flydsl( q: torch.Tensor, freqs_cis: torch.Tensor, diff --git a/python/sglang/srt/layers/attention/dsv4/low_ratio_backend_hip.py b/python/sglang/srt/layers/attention/dsv4/low_ratio_backend_hip.py index 1dc995cc00..97e619a147 100644 --- a/python/sglang/srt/layers/attention/dsv4/low_ratio_backend_hip.py +++ b/python/sglang/srt/layers/attention/dsv4/low_ratio_backend_hip.py @@ -23,6 +23,7 @@ from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import ( FP4DecodeWorkspace, FP4PrefillWorkspace, aiter_fp4_paged_mqa_logits, + index_q_rope_pack_flydsl, index_q_rope_pack_weights_flydsl, indexer_head_weights, logits_rows_per_chunk, @@ -91,6 +92,19 @@ def _indexer_inputs(layer, x, q_lora, pos): indexer.head_weight_scale, num_heads=indexer.n_heads, ) + if ( + indexer.n_heads % 16 == 0 + and indexer.n_heads <= 64 + and indexer.index_head_dim == 128 + and indexer.rope_head_dim == 64 + and layer.freqs_cis.dtype == torch.complex64 + ): + # rows past the GEMV's cap: wq_b, then one launch for RoPE and fp4 pack + q, _ = indexer.wq_b(q_lora) + q_fp4, q_scale = index_q_rope_pack_flydsl( + q, layer.freqs_cis, pos, indexer.rope_head_dim, num_heads=indexer.n_heads + ) + return q_fp4, q_scale, _indexer_head_weights(indexer, x) # [T, H, 128] fp4 grid; the RoPE launch gathers freqs_cis[pos] itself q = indexer.queries(q_lora, layer.freqs_cis, positions=pos) q_fp4, q_scale = pack_fp4_query_flydsl(q) diff --git a/test/registered/kernels/ops/attention/test_fp4_indexer_hip.py b/test/registered/kernels/ops/attention/test_fp4_indexer_hip.py index 6b185a7db7..e6db184b4e 100644 --- a/test/registered/kernels/ops/attention/test_fp4_indexer_hip.py +++ b/test/registered/kernels/ops/attention/test_fp4_indexer_hip.py @@ -19,6 +19,8 @@ import sys import pytest import torch +import triton +import triton.language as tl from sglang.kernels.ops.attention.deepseek_v4_rope import precompute_freqs_cis from sglang.kernels.ops.attention.dsv4 import ( @@ -26,14 +28,19 @@ from sglang.kernels.ops.attention.dsv4 import ( compress_norm_rope_store, ) from sglang.kernels.ops.attention.dsv4.compress import CompressorPrefillPlan -from sglang.kernels.ops.attention.dsv4.fp4_indexer import quantize_fp4_indexer_tensor +from sglang.kernels.ops.attention.dsv4.fp4_indexer import ( + _fp4_e2m1_code_rne, + quantize_fp4_indexer_tensor, +) from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import ( FP4KWriteMetadata, _decode_cta_count, + _fp4_e2m1_code_rne_lean, _guard_page_table, aiter_fp4_paged_mqa_logits, aiter_k_indexer_fp4_cache_write, aiter_q_indexer_fp4, + index_q_rope_pack_flydsl, index_q_rope_pack_weights_flydsl, indexer_head_weights, pack_fp4_query_flydsl, @@ -332,6 +339,130 @@ def test_index_q_pack_weights_matches_standalone() -> None: assert int(ulps.max()) <= 1 +def _assert_index_q_rope_pack_matches( + q: torch.Tensor, freqs: torch.Tensor, pos: torch.Tensor, num_heads: int +) -> None: + """index_q_rope_pack_flydsl is bitwise the standalone RoPE fake-quant and FlyDSL pack.""" + rope_dim = 64 + ref_q = rope_tail_fake_quant_fp4( + q.view(q.shape[0], num_heads, 128), freqs, rope_dim, positions=pos.contiguous() + ) + ref_fp4, ref_scale = pack_fp4_query_flydsl(ref_q) + q_fp4, q_scale = index_q_rope_pack_flydsl( + q, freqs, pos, rope_dim, num_heads=num_heads + ) + assert q_fp4.dtype == ref_fp4.dtype and q_scale.dtype == ref_scale.dtype + assert torch.equal(q_fp4, ref_fp4) + assert torch.equal(q_scale, ref_scale) + + +@pytest.mark.parametrize("num_heads", [16, 32, 64]) +@pytest.mark.parametrize("num_tokens", [1, 333, 16384]) +def test_index_q_rope_pack_matches_standalone(num_tokens: int, num_heads: int) -> None: + """The prefill index-Q launch (RoPE and two-stage fp4 pack in the FlyDSL layout) is + bitwise the two standalone launches it replaces, past the decode path's caps (64 tokens + for the split-K GEMV, T * H <= 4096 for the fused head-weight reduce).""" + + torch.manual_seed(0) + rope_dim, max_pos = 64, 65536 + q = (torch.randn(num_tokens, num_heads * 128, device="cuda") * 3).bfloat16() + # rows small enough to reach the pack's 1e-4 scale floor; the fake-quant floor + # (6 * 2^-126) is covered by test_index_q_rope_pack_amax_floors + q[: num_tokens // 10 + 1] *= 1e-6 + if num_tokens > 4: + # groups whose reciprocal scales take the inf / nan / near-overflow branches + q[1, 3] = float("inf") + q[2, 200] = float("nan") + q[3, :128] = 3.0e38 + q[4, 64:] = float("-inf") + freqs = precompute_freqs_cis(rope_dim, max_pos, 0, 10000, 1, 32, 1).to("cuda") + pos = torch.randint(0, max_pos, (num_tokens,), device="cuda", dtype=torch.int64) + _assert_index_q_rope_pack_matches(q, freqs, pos, num_heads) + + +def test_index_q_rope_pack_strided_positions() -> None: + """A non-contiguous positions view reads positions[t], not the t-th stored element.""" + torch.manual_seed(0) + num_tokens, num_heads, max_pos = 333, 64, 65536 + q = (torch.randn(num_tokens, num_heads * 128, device="cuda") * 3).bfloat16() + freqs = precompute_freqs_cis(64, max_pos, 0, 10000, 1, 32, 1).to("cuda") + pos = torch.randint(0, max_pos, (2 * num_tokens,), device="cuda")[::2] + assert not pos.is_contiguous() + _assert_index_q_rope_pack_matches(q, freqs, pos, num_heads) + + +def _bf16_neighbors(x: float) -> list[float]: + """x rounded to bf16 and the bf16 values one ulp below and above it.""" + bits = int(torch.tensor(x).bfloat16().view(torch.int16)) + return [ + float(torch.tensor(b, dtype=torch.int16).view(torch.bfloat16)) + for b in (bits - 1, bits, bits + 1) + ] + + +@pytest.mark.parametrize("num_heads", [16, 64]) +def test_index_q_rope_pack_amax_floors(num_heads: int) -> None: + """Zero groups, bf16 subnormals and amaxes one ulp around both floors take the + reciprocal-scale branches bitwise like the standalone divisions. The fake-quant floor + (amax 6 * 2^-126) keeps a zero group off 0 / 0; anything that small then codes 0 under + the pack floor (scale 1e-4, which ceils to 2^-13 and so first differs at amax 6 * 2^-14).""" + torch.manual_seed(0) + num_tokens, max_pos = 256, 65536 + group_amax = torch.tensor( + [ + 0.0, + 2.0**-133, # smallest bf16 subnormal + 2.0**-130, + 127 * 2.0**-133, # largest bf16 subnormal + 2.0**-126, + *_bf16_neighbors(6 * 2.0**-126), + 2.0**-120, + 1.0e-30, + *_bf16_neighbors(6 * 2.0**-14), + *_bf16_neighbors(6.0e-4), + 1.0, + ] + ) + # every 32-group gets one of the amaxes above, its largest element at exactly that value + shape = (num_tokens, num_heads, 4, 32) + u = torch.randn(shape) + u = u / u.abs().amax(dim=-1, keepdim=True) + m = group_amax[torch.randint(0, group_amax.numel(), shape[:3])] + q = (u * m[..., None]).bfloat16() + assert torch.equal(q.float().abs().amax(dim=-1), m.bfloat16().float()) + q[0] = 0 + q[1, 3] = 0 + q = q.view(num_tokens, num_heads * 128).cuda() + freqs = precompute_freqs_cis(64, max_pos, 0, 10000, 1, 32, 1).to("cuda") + pos = torch.randint(0, max_pos, (num_tokens,), device="cuda", dtype=torch.int64) + _assert_index_q_rope_pack_matches(q, freqs, pos, num_heads) + + +@triton.jit +def _e2m1_codes_kernel(x_ptr, ref_ptr, lean_ptr, n, BLOCK: tl.constexpr): + offs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK) + mask = offs < n + x = tl.load(x_ptr + offs, mask=mask) + tl.store(ref_ptr + offs, _fp4_e2m1_code_rne(x), mask=mask) + tl.store(lean_ptr + offs, _fp4_e2m1_code_rne_lean(x), mask=mask) + + +def test_fp4_e2m1_code_rne_lean_matches() -> None: + """The prefill index-Q kernel's e2m1 code equals _fp4_e2m1_code_rne on every bf16 value + (all the kernel feeds it: bf16 values times a power of two) and on random fp32 bits.""" + every_bf16 = torch.arange(1 << 16, dtype=torch.int32).to(torch.int16) + every_bf16 = every_bf16.view(torch.bfloat16).float() + scaled = torch.cat([every_bf16 * 2.0**k for k in (-8, -1, 0, 1, 8)]) + random_bits = torch.randint(-(1 << 31), 1 << 31, (1 << 22,), dtype=torch.int64) + x = torch.cat([scaled, random_bits.to(torch.int32).view(torch.float32)]).cuda() + ref = torch.empty(x.shape, dtype=torch.uint8, device="cuda") + lean = torch.empty_like(ref) + _e2m1_codes_kernel[(triton.cdiv(x.numel(), 1024),)]( + x, ref, lean, x.numel(), BLOCK=1024 + ) + assert torch.equal(lean, ref) + + @pytest.mark.parametrize("num_tokens", [1, 16, 96]) def test_fp4_index_cache_store_layout(num_tokens: int) -> None: """Scattered slots land in the paged layout and touch nothing else."""