PR #42365: fuse Hopper prefill indexer scoring into one Triton kernel

Source: https://github.com/sgl-project/sglang/pull/42365 (applied via GitHub .diff, 3way)
Conflicts resolved: unioned env flags with #40945 SM90 flags; merged fused
scoring loop with #40860 candidate_blocks path (candidate > fused > torch priority).
This commit is contained in:
2026-10-19 10:33:13 +08:00
parent 740372bef0
commit 31b3758c99
5 changed files with 298 additions and 5 deletions
@@ -184,6 +184,7 @@ for _mod, _fn in [
("dsa.cp_split", "dsa_cp_interleave_q_seqs_kernel"),
("dsv4.fp4_indexer", "quantize_fp4_indexer_tensor"),
("dsv4.fp4_indexer", "store_fp4_index_k_cache"),
("dsv4.fp4_indexer_prefill", "fused_index_scores"),
]:
register_kernel(
KernelSpec(
@@ -0,0 +1,112 @@
"""Prefill-time low-ratio indexer scores for Hopper.
The torch prefill path scores a chunk of query rows against the request's
dequantized index K as `einsum -> relu -> * weights -> sum over heads`, which
materializes a bf16 [rows, heads, lc] tensor and streams it through HBM several
times. This Triton kernel keeps the head reduction in a [BLOCK_M, BLOCK_N]
register tile and writes only the fp32 [rows, lc] result, with the length mask
already applied.
Numerics follow the torch path and the decode kernel in fp4_indexer.py
(bf16 dot, bf16 relu/weight product, bf16 head reduction), so prefill and decode
select the same positions; only the head summation order differs.
"""
import torch
import triton
import triton.language as tl
from sglang.kernels.ops.attention.dsv4.fp4_indexer import INDEX_HEAD_DIM
# Measured best on H200 among 7 swept tiles. The grid walks query-row blocks
# outermost so a row block's queries (BLOCK_M * heads * d bf16) stay in L2
# while the index K is streamed past them.
_CONFIG = dict(BLOCK_M=64, BLOCK_N=128, num_warps=4, num_stages=2)
@triton.jit
def _index_scores_kernel(
q_ptr, # [M, H, D] bf16
k_ptr, # [N, D] bf16
w_ptr, # [M, H] bf16
lens_ptr, # [M] int, visible positions per row
out_ptr, # [M, N] fp32, -inf at positions >= lens
M,
N,
stride_qm,
stride_qh,
stride_kn,
stride_wm,
stride_om,
H: tl.constexpr,
D: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
):
pid = tl.program_id(0)
n_blocks = tl.cdiv(N, BLOCK_N)
pid_m = pid // n_blocks
pid_n = pid % n_blocks
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
offs_d = tl.arange(0, D)
mask_m = offs_m < M
mask_n = offs_n < N
k = tl.load(
k_ptr + offs_n[:, None].to(tl.int64) * stride_kn + offs_d[None, :],
mask=mask_n[:, None],
other=0.0,
)
k_t = tl.trans(k)
q_rows = q_ptr + offs_m[:, None].to(tl.int64) * stride_qm + offs_d[None, :]
w_rows = w_ptr + offs_m.to(tl.int64) * stride_wm
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
for h in range(H):
q = tl.load(q_rows + h * stride_qh, mask=mask_m[:, None], other=0.0)
w = tl.load(w_rows + h, mask=mask_m, other=0.0).to(tl.float32)
# reference rounding points, as in the decode kernel:
# bf16 dot -> relu -> * bf16 weight -> bf16 -> sum -> bf16
s = tl.dot(q, k_t).to(tl.bfloat16).to(tl.float32)
acc += (tl.maximum(s, 0.0) * w[:, None]).to(tl.bfloat16).to(tl.float32)
acc = acc.to(tl.bfloat16).to(tl.float32)
lens = tl.load(lens_ptr + offs_m, mask=mask_m, other=0)
acc = tl.where(offs_n[None, :] < lens[:, None], acc, float("-inf"))
tl.store(
out_ptr + offs_m[:, None].to(tl.int64) * stride_om + offs_n[None, :],
acc,
mask=mask_m[:, None] & mask_n[None, :],
)
def fused_index_scores(
q: torch.Tensor, k: torch.Tensor, weights: torch.Tensor, lens: torch.Tensor
) -> torch.Tensor:
"""q [t, H, d] bf16, k [n, d] bf16, weights [t, H] bf16, lens [t] int ->
[t, n] fp32: sum over heads of relu(q_h k^T) * w_h, -inf at positions >= lens."""
t, heads, head_dim = q.shape
n = k.shape[0]
assert q.dtype == k.dtype == torch.bfloat16 and head_dim == INDEX_HEAD_DIM
assert q.stride(2) == 1 and k.stride(1) == 1 and weights.stride(1) == 1
assert lens.stride(0) == 1
out = torch.empty(t, n, dtype=torch.float32, device=q.device)
grid = (triton.cdiv(t, _CONFIG["BLOCK_M"]) * triton.cdiv(n, _CONFIG["BLOCK_N"]),)
_index_scores_kernel[grid](
q,
k,
weights,
lens,
out,
t,
n,
q.stride(0),
q.stride(1),
k.stride(0),
weights.stride(0),
out.stride(0),
H=heads,
D=head_dim,
**_CONFIG,
)
return out
+4
View File
@@ -1586,6 +1586,10 @@ class Envs:
SGLANG_OPT_DSV41_SM90_LENGTH_AWARE_INDEXER = EnvBool(False)
# Compact candidate lists; requires the length-aware static-verify path.
SGLANG_OPT_DSV41_SM90_COMPACT_CANDIDATES = EnvBool(False)
# Score the torch prefill indexer with one Triton kernel instead of the
# einsum/relu/weight/sum chain; False, or SGLANG_DSV41_TORCH_PREFILL_INDEXER,
# restores the torch scoring.
SGLANG_OPT_USE_FUSED_DSV41_INDEXER_SCORES = EnvBool(True)
SGLANG_FP8_PAGED_MQA_LOGITS_TORCH = EnvBool(False)
SGLANG_OPT_FLASHMLA_SPARSE_PREFILL = EnvBool(True)
@@ -16,6 +16,7 @@ from sglang.kernels.ops.attention.dsv4.fp4_indexer import (
fp4_index_logits_decode,
fp4_index_logits_paged,
)
from sglang.kernels.ops.attention.dsv4.fp4_indexer_prefill import fused_index_scores
from sglang.kernels.ops.attention.dsv4.fp4_indexer_rope import index_q_rope_pack_weights
from sglang.kernels.ops.attention.dsv4.index_logits import flat_index_logits_tiles
from sglang.kernels.ops.attention.dsv4.topk import (
@@ -23,6 +24,7 @@ from sglang.kernels.ops.attention.dsv4.topk import (
topk_transform_paged_v2,
topk_transform_ragged_v2,
)
from sglang.srt.environ import envs
from sglang.srt.runtime_context import get_platform
from sglang.srt.utils.common import async_h2d
@@ -339,8 +341,17 @@ def _score_chunks(
block_size: int = 8,
) -> Iterator[ChunkScores]:
lc, j = request.lc, request.columns
# The torch prefill indexer flag asks for the torch oracle, not this kernel.
fused = (
q.is_cuda
and envs.SGLANG_OPT_USE_FUSED_DSV41_INDEXER_SCORES.get()
and not envs.SGLANG_DSV41_TORCH_PREFILL_INDEXER.get()
)
# Chunk rows so the [rows, heads, lc] bf16 scores stay under the budget.
rows_per_chunk = max(1, _TORCH_SCORE_BUDGET_BYTES // (q.shape[1] * lc * 2))
# The fused kernel holds only the fp32 [rows, lc] scores; at a quarter of
# the budget its [rows, lc] copies downstream stay under the torch peak.
row_bytes = lc * 16 if fused else q.shape[1] * lc * 2
rows_per_chunk = max(1, _TORCH_SCORE_BUDGET_BYTES // row_bytes)
if candidate_blocks is not None:
from sglang.kernels.ops.attention.dsv4.candidate_bf16_mqa import (
can_use_candidate_bf16_mqa,
@@ -358,13 +369,15 @@ def _score_chunks(
rows = slice(start, start + rows_per_chunk)
tok_c, lens_c = request.tok[rows], request.lens[rows]
blocks = None if candidate_blocks is None else candidate_blocks[tok_c]
if blocks is None:
s = indexer.scores(q[tok_c], index_k, weights[tok_c])
s = s.masked_fill(j[None, :] >= lens_c[:, None], -torch.inf)
else:
if blocks is not None:
s = candidate_bf16_mqa_logits(
q[tok_c], index_k, weights[tok_c], blocks, lens_c
)
elif fused:
s = fused_index_scores(q[tok_c], index_k, weights[tok_c], lens_c)
else:
s = indexer.scores(q[tok_c], index_k, weights[tok_c])
s = s.masked_fill(j[None, :] >= lens_c[:, None], -torch.inf)
yield ChunkScores(
rows=rows, tok=tok_c, lens=lens_c, scores=s, candidate_blocks=blocks
)