From 31b3758c99d074be5f2c1676b5dbabda9530bd9b Mon Sep 17 00:00:00 2001 From: "minke.yu" Date: Mon, 19 Oct 2026 10:33:13 +0800 Subject: [PATCH] PR #42365: fuse Hopper prefill indexer scoring into one Triton kernel Source: https://github.com/sgl-project/sglang/pull/42365 (applied via GitHub .diff, 3way) Conflicts resolved: unioned env flags with #40945 SM90 flags; merged fused scoring loop with #40860 candidate_blocks path (candidate > fused > torch priority). --- .../sglang/kernels/ops/attention/__init__.py | 1 + .../ops/attention/dsv4/fp4_indexer_prefill.py | 112 ++++++++++++ python/sglang/srt/environ.py | 4 + .../attention/dsv4/v41_indexer/scoring.py | 23 ++- .../dsv4/test_fp4_indexer_prefill.py | 163 ++++++++++++++++++ 5 files changed, 298 insertions(+), 5 deletions(-) create mode 100644 python/sglang/kernels/ops/attention/dsv4/fp4_indexer_prefill.py create mode 100644 test/registered/kernels/ops/attention/dsv4/test_fp4_indexer_prefill.py diff --git a/python/sglang/kernels/ops/attention/__init__.py b/python/sglang/kernels/ops/attention/__init__.py index c2e046daa8..b3e86dcc57 100644 --- a/python/sglang/kernels/ops/attention/__init__.py +++ b/python/sglang/kernels/ops/attention/__init__.py @@ -184,6 +184,7 @@ for _mod, _fn in [ ("dsa.cp_split", "dsa_cp_interleave_q_seqs_kernel"), ("dsv4.fp4_indexer", "quantize_fp4_indexer_tensor"), ("dsv4.fp4_indexer", "store_fp4_index_k_cache"), + ("dsv4.fp4_indexer_prefill", "fused_index_scores"), ]: register_kernel( KernelSpec( diff --git a/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_prefill.py b/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_prefill.py new file mode 100644 index 0000000000..0be018689b --- /dev/null +++ b/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_prefill.py @@ -0,0 +1,112 @@ +"""Prefill-time low-ratio indexer scores for Hopper. + +The torch prefill path scores a chunk of query rows against the request's +dequantized index K as `einsum -> relu -> * weights -> sum over heads`, which +materializes a bf16 [rows, heads, lc] tensor and streams it through HBM several +times. This Triton kernel keeps the head reduction in a [BLOCK_M, BLOCK_N] +register tile and writes only the fp32 [rows, lc] result, with the length mask +already applied. + +Numerics follow the torch path and the decode kernel in fp4_indexer.py +(bf16 dot, bf16 relu/weight product, bf16 head reduction), so prefill and decode +select the same positions; only the head summation order differs. +""" + +import torch +import triton +import triton.language as tl + +from sglang.kernels.ops.attention.dsv4.fp4_indexer import INDEX_HEAD_DIM + +# Measured best on H200 among 7 swept tiles. The grid walks query-row blocks +# outermost so a row block's queries (BLOCK_M * heads * d bf16) stay in L2 +# while the index K is streamed past them. +_CONFIG = dict(BLOCK_M=64, BLOCK_N=128, num_warps=4, num_stages=2) + + +@triton.jit +def _index_scores_kernel( + q_ptr, # [M, H, D] bf16 + k_ptr, # [N, D] bf16 + w_ptr, # [M, H] bf16 + lens_ptr, # [M] int, visible positions per row + out_ptr, # [M, N] fp32, -inf at positions >= lens + M, + N, + stride_qm, + stride_qh, + stride_kn, + stride_wm, + stride_om, + H: tl.constexpr, + D: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, +): + pid = tl.program_id(0) + n_blocks = tl.cdiv(N, BLOCK_N) + pid_m = pid // n_blocks + pid_n = pid % n_blocks + offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M) + offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N) + offs_d = tl.arange(0, D) + mask_m = offs_m < M + mask_n = offs_n < N + + k = tl.load( + k_ptr + offs_n[:, None].to(tl.int64) * stride_kn + offs_d[None, :], + mask=mask_n[:, None], + other=0.0, + ) + k_t = tl.trans(k) + q_rows = q_ptr + offs_m[:, None].to(tl.int64) * stride_qm + offs_d[None, :] + w_rows = w_ptr + offs_m.to(tl.int64) * stride_wm + acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32) + for h in range(H): + q = tl.load(q_rows + h * stride_qh, mask=mask_m[:, None], other=0.0) + w = tl.load(w_rows + h, mask=mask_m, other=0.0).to(tl.float32) + # reference rounding points, as in the decode kernel: + # bf16 dot -> relu -> * bf16 weight -> bf16 -> sum -> bf16 + s = tl.dot(q, k_t).to(tl.bfloat16).to(tl.float32) + acc += (tl.maximum(s, 0.0) * w[:, None]).to(tl.bfloat16).to(tl.float32) + acc = acc.to(tl.bfloat16).to(tl.float32) + + lens = tl.load(lens_ptr + offs_m, mask=mask_m, other=0) + acc = tl.where(offs_n[None, :] < lens[:, None], acc, float("-inf")) + tl.store( + out_ptr + offs_m[:, None].to(tl.int64) * stride_om + offs_n[None, :], + acc, + mask=mask_m[:, None] & mask_n[None, :], + ) + + +def fused_index_scores( + q: torch.Tensor, k: torch.Tensor, weights: torch.Tensor, lens: torch.Tensor +) -> torch.Tensor: + """q [t, H, d] bf16, k [n, d] bf16, weights [t, H] bf16, lens [t] int -> + [t, n] fp32: sum over heads of relu(q_h k^T) * w_h, -inf at positions >= lens.""" + t, heads, head_dim = q.shape + n = k.shape[0] + assert q.dtype == k.dtype == torch.bfloat16 and head_dim == INDEX_HEAD_DIM + assert q.stride(2) == 1 and k.stride(1) == 1 and weights.stride(1) == 1 + assert lens.stride(0) == 1 + out = torch.empty(t, n, dtype=torch.float32, device=q.device) + grid = (triton.cdiv(t, _CONFIG["BLOCK_M"]) * triton.cdiv(n, _CONFIG["BLOCK_N"]),) + _index_scores_kernel[grid]( + q, + k, + weights, + lens, + out, + t, + n, + q.stride(0), + q.stride(1), + k.stride(0), + weights.stride(0), + out.stride(0), + H=heads, + D=head_dim, + **_CONFIG, + ) + return out diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index cb8de48ff7..9348812db0 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1586,6 +1586,10 @@ class Envs: SGLANG_OPT_DSV41_SM90_LENGTH_AWARE_INDEXER = EnvBool(False) # Compact candidate lists; requires the length-aware static-verify path. SGLANG_OPT_DSV41_SM90_COMPACT_CANDIDATES = EnvBool(False) + # Score the torch prefill indexer with one Triton kernel instead of the + # einsum/relu/weight/sum chain; False, or SGLANG_DSV41_TORCH_PREFILL_INDEXER, + # restores the torch scoring. + SGLANG_OPT_USE_FUSED_DSV41_INDEXER_SCORES = EnvBool(True) SGLANG_FP8_PAGED_MQA_LOGITS_TORCH = EnvBool(False) SGLANG_OPT_FLASHMLA_SPARSE_PREFILL = EnvBool(True) diff --git a/python/sglang/srt/layers/attention/dsv4/v41_indexer/scoring.py b/python/sglang/srt/layers/attention/dsv4/v41_indexer/scoring.py index 6a309297ca..62009cd5d9 100644 --- a/python/sglang/srt/layers/attention/dsv4/v41_indexer/scoring.py +++ b/python/sglang/srt/layers/attention/dsv4/v41_indexer/scoring.py @@ -16,6 +16,7 @@ from sglang.kernels.ops.attention.dsv4.fp4_indexer import ( fp4_index_logits_decode, fp4_index_logits_paged, ) +from sglang.kernels.ops.attention.dsv4.fp4_indexer_prefill import fused_index_scores from sglang.kernels.ops.attention.dsv4.fp4_indexer_rope import index_q_rope_pack_weights from sglang.kernels.ops.attention.dsv4.index_logits import flat_index_logits_tiles from sglang.kernels.ops.attention.dsv4.topk import ( @@ -23,6 +24,7 @@ from sglang.kernels.ops.attention.dsv4.topk import ( topk_transform_paged_v2, topk_transform_ragged_v2, ) +from sglang.srt.environ import envs from sglang.srt.runtime_context import get_platform from sglang.srt.utils.common import async_h2d @@ -339,8 +341,17 @@ def _score_chunks( block_size: int = 8, ) -> Iterator[ChunkScores]: lc, j = request.lc, request.columns + # The torch prefill indexer flag asks for the torch oracle, not this kernel. + fused = ( + q.is_cuda + and envs.SGLANG_OPT_USE_FUSED_DSV41_INDEXER_SCORES.get() + and not envs.SGLANG_DSV41_TORCH_PREFILL_INDEXER.get() + ) # Chunk rows so the [rows, heads, lc] bf16 scores stay under the budget. - rows_per_chunk = max(1, _TORCH_SCORE_BUDGET_BYTES // (q.shape[1] * lc * 2)) + # The fused kernel holds only the fp32 [rows, lc] scores; at a quarter of + # the budget its [rows, lc] copies downstream stay under the torch peak. + row_bytes = lc * 16 if fused else q.shape[1] * lc * 2 + rows_per_chunk = max(1, _TORCH_SCORE_BUDGET_BYTES // row_bytes) if candidate_blocks is not None: from sglang.kernels.ops.attention.dsv4.candidate_bf16_mqa import ( can_use_candidate_bf16_mqa, @@ -358,13 +369,15 @@ def _score_chunks( rows = slice(start, start + rows_per_chunk) tok_c, lens_c = request.tok[rows], request.lens[rows] blocks = None if candidate_blocks is None else candidate_blocks[tok_c] - if blocks is None: - s = indexer.scores(q[tok_c], index_k, weights[tok_c]) - s = s.masked_fill(j[None, :] >= lens_c[:, None], -torch.inf) - else: + if blocks is not None: s = candidate_bf16_mqa_logits( q[tok_c], index_k, weights[tok_c], blocks, lens_c ) + elif fused: + s = fused_index_scores(q[tok_c], index_k, weights[tok_c], lens_c) + else: + s = indexer.scores(q[tok_c], index_k, weights[tok_c]) + s = s.masked_fill(j[None, :] >= lens_c[:, None], -torch.inf) yield ChunkScores( rows=rows, tok=tok_c, lens=lens_c, scores=s, candidate_blocks=blocks ) diff --git a/test/registered/kernels/ops/attention/dsv4/test_fp4_indexer_prefill.py b/test/registered/kernels/ops/attention/dsv4/test_fp4_indexer_prefill.py new file mode 100644 index 0000000000..92d993837f --- /dev/null +++ b/test/registered/kernels/ops/attention/dsv4/test_fp4_indexer_prefill.py @@ -0,0 +1,163 @@ +"""Fused prefill indexer scores: values and top-k selection vs the torch scoring, +the length mask, and `_score_chunks` with the kernel switched on and off.""" + +import pytest +import torch + +from sglang.kernels.ops.attention.dsv4.fp4_indexer_prefill import fused_index_scores +from sglang.srt.environ import envs +from sglang.srt.layers.attention.dsv4.v41_indexer import scoring +from sglang.srt.layers.attention.dsv4.v41_indexer.scoring import ( + RequestScores, + _score_chunks, +) +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large") +pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA only") + +HEADS = 32 +HEAD_DIM = 128 +TOPK = 512 +MIN_OVERLAP = 0.99 +MIN_EXACT = 0.99 +MAX_REL_ERR = 1e-2 + + +def _torch_scores(q, k, weights): + """`DeepseekV41Indexer.scores`: bf16 up to the head reduction.""" + s = torch.einsum("bhd,nd->bhn", q, k) + return (s.relu() * weights.unsqueeze(-1)).sum(dim=1).float() + + +def _inputs(rows, lc, seed): + g = torch.Generator(device="cuda").manual_seed(seed) + q = torch.randn(rows, HEADS, HEAD_DIM, generator=g, device="cuda").bfloat16() + k = torch.randn(lc, HEAD_DIM, generator=g, device="cuda").bfloat16() + weights = torch.randn(rows, HEADS, generator=g, device="cuda").bfloat16() + # Prefill tail of a prompt: row i sees lc - rows + 1 + i positions. + lens = torch.arange(lc - rows + 1, lc + 1, device="cuda").clamp_min(1) + return q, k, weights, lens + + +def _mask(scores, lens): + j = torch.arange(scores.shape[1], device=scores.device) + return scores.masked_fill(j[None, :] >= lens[:, None], -torch.inf) + + +def _topk_overlap(a, b, k): + ia, ib = a.topk(k, dim=-1).indices, b.topk(k, dim=-1).indices + hits = (ia.unsqueeze(-1) == ib.unsqueeze(-2)).any(-1).sum(-1) + return (hits.float() / k).mean().item() + + +class _Indexer: + def scores(self, q, k, weights): + return _torch_scores(q, k, weights) + + +def _chunk_scores(fused, q, k, weights, lens, budget_bytes=None): + rows = q.shape[0] + lc = k.shape[0] + request = RequestScores( + lc=lc, + k=min(TOPK, lc), + columns=torch.arange(lc, device=q.device), + tok=torch.arange(rows, device=q.device), + lens=lens, + slots=torch.arange(lc, device=q.device), + ) + overrides = [ + envs.SGLANG_OPT_USE_FUSED_DSV41_INDEXER_SCORES.override(fused), + envs.SGLANG_DSV41_TORCH_PREFILL_INDEXER.override(False), + ] + if budget_bytes is not None: + # Force several row chunks so the fused and torch paths both iterate. + from unittest.mock import patch + + overrides.append( + patch.object(scoring, "_TORCH_SCORE_BUDGET_BYTES", budget_bytes) + ) + from contextlib import ExitStack + + with ExitStack() as stack: + for ctx in overrides: + stack.enter_context(ctx) + return list( + _score_chunks( + indexer=_Indexer(), q=q, weights=weights, index_k=k, request=request + ) + ) + + +def _cat_scores(chunks): + return torch.cat([chunk.scores for chunk in chunks], dim=0) + + +@pytest.mark.parametrize("rows,lc", [(1, 700), (77, 1000), (300, 4099), (513, 20011)]) +def test_fused_index_scores_random_inputs_match_torch_scoring(rows, lc): + q, k, weights, lens = _inputs(rows, lc, seed=rows) + out = fused_index_scores(q, k, weights, lens) + ref = _mask(_torch_scores(q, k, weights), lens) + + assert out.dtype == torch.float32 and out.shape == (rows, lc) + assert torch.equal(out.isneginf(), ref.isneginf()) + visible = ref.isfinite() + # Same bf16 rounding points as torch: values differ only by head summation + # order, i.e. at most one bf16 step on a few entries. + exact = (out == ref)[visible].float().mean().item() + assert exact >= MIN_EXACT, f"bit-equal fraction {exact}" + rel_err = (out - ref)[visible].abs().max() / ref[visible].abs().max() + assert rel_err.item() <= MAX_REL_ERR + overlap = _topk_overlap(out, ref, min(TOPK, int(lens.min()))) + assert overlap >= MIN_OVERLAP, f"top-{TOPK} overlap {overlap}" + + +@pytest.mark.parametrize("budget_bytes", [None, 1 << 20]) +def test_score_chunks_fused_selects_like_torch_scoring(budget_bytes): + rows, lc = 600, 3001 + q, k, weights, lens = _inputs(rows, lc, seed=2) + fused_chunks = _chunk_scores(True, q, k, weights, lens, budget_bytes) + torch_chunks = _chunk_scores(False, q, k, weights, lens, budget_bytes) + if budget_bytes is not None: + # lc * 16 bytes per fused row; a 1 MiB budget yields several chunks. + assert len(fused_chunks) > 1 + fused = _cat_scores(fused_chunks) + torch_ref = _cat_scores(torch_chunks) + + assert fused.shape == torch_ref.shape == (rows, lc) + assert torch.equal(fused.isneginf(), torch_ref.isneginf()) + overlap = _topk_overlap(fused, torch_ref, min(TOPK, int(lens.min()))) + assert overlap >= MIN_OVERLAP, f"{budget_bytes=}: overlap {overlap}" + + +def test_score_chunks_oracle_flag_skips_fused_kernel(monkeypatch): + """The torch prefill indexer flag is the oracle for the dense fp4 kernel: it + must score with torch even when the fused kernel is enabled.""" + + def _fail(*args, **kwargs): + raise AssertionError("fused kernel ran under the torch oracle flag") + + monkeypatch.setattr(scoring, "fused_index_scores", _fail) + rows, lc = 64, 1001 + q, k, weights, lens = _inputs(rows, lc, seed=3) + request = RequestScores( + lc=lc, + k=min(TOPK, lc), + columns=torch.arange(lc, device=q.device), + tok=torch.arange(rows, device=q.device), + lens=lens, + slots=torch.arange(lc, device=q.device), + ) + with ( + envs.SGLANG_OPT_USE_FUSED_DSV41_INDEXER_SCORES.override(True), + envs.SGLANG_DSV41_TORCH_PREFILL_INDEXER.override(True), + ): + chunks = list( + _score_chunks( + indexer=_Indexer(), q=q, weights=weights, index_k=k, request=request + ) + ) + scores = _cat_scores(chunks) + assert scores.shape == (rows, lc) + assert bool(scores.isfinite().any())