PR #40945: SM90 static verification indexing (grouped/length-aware indexer)

Source: https://github.com/sgl-project/sglang/pull/40945 (applied via GitHub .diff, 3way)
Adaptation: PR base predates #41659 which removed the Selection struct;
ported sm90_decode.py and call sites to the inputs.out_page_indices interface
(raw_indices=None on decode paths, reset via inputs.reset_outputs()).
This commit is contained in:
2026-10-19 10:30:41 +08:00
parent 19fe20baec
commit 740372bef0
10 changed files with 1207 additions and 6 deletions
@@ -0,0 +1,170 @@
"""Fused request-to-KV mapping for opt-in SM90 32-head static verification."""
import torch
import triton
import triton.language as tl
INDEX_HEAD_DIM = 128
PAYLOAD_BYTES = tl.constexpr(64)
SCALE_BYTES = tl.constexpr(4)
@triton.jit
def _e2m1_decode(code):
# code: uint 0..15 -> e2m1 value. exp = bits 2..1, mantissa = bit 0, sign = bit 3.
e = (code >> 1) & 3
m = (code & 1).to(tl.float32)
sub = m * 0.5
exp = tl.where(e == 1, 1.0, tl.where(e == 2, 2.0, 4.0))
nor = (1.0 + m * 0.5) * exp
v = tl.where(e == 0, sub, nor)
return tl.where((code >> 3) == 1, -v, v)
@triton.jit
def _fp4_index_logits_kernel(
q_ptr, # [B, H, D] bf16, fq4 queries (already rope'd)
w_ptr, # [B, H] bf16 head weights (softmax scale folded in)
slots_ptr, # [requests, capacity * ratio] int32 request-to-token mapping
lens_ptr, # [B] int64 visible compressed positions per request
table_ptr, # [num_pages, page_size * 64 + page_size * 4] uint8
out_ptr, # [B, L] fp32 logits, -inf beyond lens
L,
page_size,
row_stride,
stride_qb,
stride_qh,
stride_wb,
H: tl.constexpr,
HALF_D: tl.constexpr, # D // 2 == 64 nibble-pairs per row
BLOCK_L: tl.constexpr,
req_ptr,
req_stride: tl.constexpr,
ratio: tl.constexpr,
):
b = tl.program_id(0)
lb = tl.program_id(1)
offs_l = lb * BLOCK_L + tl.arange(0, BLOCK_L)
offs_h = tl.arange(0, H)
offs_i = tl.arange(
0, HALF_D
) # byte index i holds elements 2i (low nibble), 2i+1 (high nibble)
n_vis = tl.load(lens_ptr + b)
# Keep capacity-sized CUDA Graph buffers while skipping invisible tiles,
# including mapping loads, dequantization, dot products and head reduction.
if lb * BLOCK_L >= n_vis:
tl.store(out_ptr + b * L + offs_l, float("-inf"), mask=offs_l < L)
else:
valid = offs_l < tl.minimum(n_vis, L)
request = tl.load(req_ptr + b).to(tl.int64)
slot = (
tl.load(
slots_ptr + request * req_stride + offs_l * ratio,
mask=valid,
other=0,
).to(tl.int64)
// ratio
)
page = slot // page_size
off = slot % page_size
row_base = page * row_stride
# K payload: [BLOCK_L, HALF_D] uint8
pay = tl.load(
table_ptr
+ row_base[:, None]
+ off[:, None] * PAYLOAD_BYTES
+ offs_i[None, :],
mask=valid[:, None],
other=0,
)
low = _e2m1_decode(pay & 0x0F)
high = _e2m1_decode((pay >> 4) & 0x0F)
# e8m0 block scales: each scale covers 16 payload bytes (32 elements).
offs_s = tl.arange(0, SCALE_BYTES)
exps = tl.load(
table_ptr
+ row_base[:, None]
+ page_size * PAYLOAD_BYTES
+ off[:, None] * SCALE_BYTES
+ offs_s[None, :],
mask=valid[:, None],
other=127,
)
scale = tl.exp2(exps.to(tl.float32) - 127.0)
scale = scale[:, :, None]
k_shape: tl.constexpr = (BLOCK_L, SCALE_BYTES, HALF_D // SCALE_BYTES)
k_low = tl.reshape(tl.reshape(low, k_shape) * scale, (BLOCK_L, HALF_D)).to(
tl.bfloat16
)
k_high = tl.reshape(tl.reshape(high, k_shape) * scale, (BLOCK_L, HALF_D)).to(
tl.bfloat16
)
# queries: even / odd elements, [H, HALF_D] bf16
q_even = tl.load(
q_ptr + b * stride_qb + offs_h[:, None] * stride_qh + 2 * offs_i[None, :]
)
q_odd = tl.load(
q_ptr
+ b * stride_qb
+ offs_h[:, None] * stride_qh
+ 2 * offs_i[None, :]
+ 1
)
acc = tl.dot(q_even, tl.trans(k_low)) # [H, BLOCK_L] fp32
acc += tl.dot(q_odd, tl.trans(k_high))
# reference rounding points: bf16 dot -> relu -> * bf16 weight -> bf16 -> sum -> bf16
s = acc.to(tl.bfloat16).to(tl.float32)
s = tl.maximum(s, 0.0)
w = tl.load(w_ptr + b * stride_wb + offs_h).to(tl.float32)
s = (s * w[:, None]).to(tl.bfloat16).to(tl.float32)
logit = tl.sum(s, axis=0).to(tl.bfloat16).to(tl.float32)
logit = tl.where(valid, logit, float("-inf"))
tl.store(out_ptr + b * L + offs_l, logit, mask=offs_l < L)
def fp4_index_logits_mapped_sm90(
q,
weights,
req_to_token,
req,
lens,
table,
page_size,
ratio,
width,
):
"""Score with request mapping fused into the Triton kernel.
The caller validates the common tensor/device contract. No dense slots
tensor or host read of dynamic lengths is needed during graph replay.
"""
rows, heads, _ = q.shape
out = torch.empty((rows, width), dtype=torch.float32, device=q.device)
if rows == 0 or width == 0:
return out
_fp4_index_logits_kernel[(rows, triton.cdiv(width, 64))](
q,
weights,
req_to_token,
lens,
table,
out,
width,
page_size,
table.stride(0),
q.stride(0),
q.stride(1),
weights.stride(0),
H=heads,
HALF_D=64,
BLOCK_L=64,
req_ptr=req,
req_stride=req_to_token.stride(0),
ratio=ratio,
num_warps=4,
)
return out
@@ -0,0 +1,86 @@
"""Sort selected logical positions and map slots without dense intermediates."""
import torch
import triton
import triton.language as tl
@triton.jit
def _sort_map_topk(
selected,
lens,
req,
mapping,
pages,
raw,
selected_row_stride: tl.constexpr,
selected_col_stride: tl.constexpr,
lens_stride: tl.constexpr,
req_stride: tl.constexpr,
mapping_row_stride: tl.constexpr,
pages_row_stride: tl.constexpr,
raw_row_stride: tl.constexpr,
K: tl.constexpr,
RATIO: tl.constexpr,
HAS_RAW: tl.constexpr,
BLOCK: tl.constexpr,
):
row = tl.program_id(0)
col = tl.arange(0, BLOCK)
logical = tl.load(
selected + row * selected_row_stride + col * selected_col_stride,
col < K,
other=0x7FFFFFFFFFFFFFFF,
)
logical = tl.sort(logical, descending=False)
visible = tl.load(lens + row * lens_stride)
request = tl.load(req + row * req_stride).to(tl.int64)
valid = (col < K) & (logical < visible)
# Invalid candidate positions can equal the scan bound; never read their
# mappings. The backend has already filled the unused output tail with -1.
position = tl.where(valid, logical, 0)
slot = tl.load(
mapping + request * mapping_row_stride + position * RATIO,
valid,
other=0,
)
tl.store(
pages + row * pages_row_stride + col,
tl.where(valid, slot // RATIO, -1),
col < K,
)
if HAS_RAW:
tl.store(
raw + row * raw_row_stride + col, tl.where(valid, logical, -1), col < K
)
def sort_map_topk(selected, lens, req, mapping, pages, raw, ratio):
"""Publish sorted TopK values; selection and tie-breaking stay in PyTorch.
The caller supplies int64 selected positions, with invalid candidates
replaced by the scan bound, and pre-fills both output tensors with -1.
Only the opt-in backend uses this small-K epilogue.
"""
rows, k = selected.shape
assert selected.dtype == torch.int64 and 0 < k <= 1024
_sort_map_topk[(rows,)](
selected,
lens,
req,
mapping,
pages,
raw if raw is not None else pages,
selected.stride(0),
selected.stride(1),
lens.stride(0),
req.stride(0),
mapping.stride(0),
pages.stride(0),
raw.stride(0) if raw is not None else 0,
K=k,
RATIO=ratio,
HAS_RAW=raw is not None,
BLOCK=triton.next_power_of_2(k),
num_warps=4,
)
@@ -0,0 +1,524 @@
"""Opt-in 32-head Hopper indexer with device-side prefix lengths.
Score buffers keep their graph-stable capacity, but only the visible prefix
(and its final 64-element tile) is initialized. Every consumer MUST use the
device lengths. TopK v2 is exact by score, with arbitrary cutoff-tie selection;
it does not promise PyTorch's choice of indices with equal scores.
"""
import torch
import triton
import triton.language as tl
from .sm90_fp4_indexer import _e2m1_decode
from .topk import topk_transform_ragged_v2
@triton.jit
def _prepare_lengths(
POS,
CAPACITY,
VISIBLE,
LENGTHS,
ROWS: tl.constexpr,
POS_STRIDE: tl.constexpr,
RATIO: tl.constexpr,
WIDTH: tl.constexpr,
HAS_CANDIDATES: tl.constexpr,
):
row = tl.program_id(0) * 256 + tl.arange(0, 256)
pos = tl.load(POS + row * POS_STRIDE, row < ROWS, other=-1)
n = tl.minimum(tl.maximum((pos + 1) // RATIO, 0), WIDTH).to(tl.int32)
tl.store(VISIBLE + row, n, row < ROWS)
if HAS_CANDIDATES:
capacity = tl.load(CAPACITY + row, row < ROWS, other=0)
tl.store(LENGTHS + row, tl.minimum(n, capacity), row < ROWS)
def prepare_candidate_lengths(pos, ratio, width, candidates=None):
"""Fuse position conversion, clamping, casting and compact score bounds."""
visible = torch.empty(pos.shape[0], dtype=torch.int32, device=pos.device)
lengths = torch.empty_like(visible) if candidates is not None else visible
_prepare_lengths[(triton.cdiv(pos.shape[0], 256),)](
pos,
candidates.lengths if candidates is not None else visible,
visible,
lengths,
pos.shape[0],
pos.stride(0),
ratio,
width,
candidates is not None,
num_warps=4,
)
return visible, lengths
@triton.jit
def _prefix_logits(
Q,
W,
MAPPING,
REQ,
LENS,
TABLE,
MASK,
OUT,
BLOCKS,
IS_PREFIX,
VISIBLE,
Q_STRIDE: tl.constexpr,
W_STRIDE: tl.constexpr,
MAP_STRIDE: tl.constexpr,
TABLE_STRIDE: tl.constexpr,
MASK_STRIDE: tl.constexpr,
OUT_STRIDE: tl.constexpr,
BLOCK_STRIDE: tl.constexpr,
PAGE: tl.constexpr,
RATIO: tl.constexpr,
HAS_MASK: tl.constexpr,
HAS_BLOCKS: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
row = tl.program_id(0)
n = tl.load(LENS + row)
request = tl.load(REQ + row).to(tl.int64)
h = tl.arange(0, 32)
d = tl.arange(0, 64)
# Reuse query/weights across the tiles owned by this CTA.
qe = tl.load(Q + row * Q_STRIDE + h[:, None] * 128 + 2 * d[None, :])
qo = tl.load(Q + row * Q_STRIDE + h[:, None] * 128 + 2 * d[None, :] + 1)
weight = tl.load(W + row * W_STRIDE + h).to(tl.float32)
if HAS_BLOCKS:
visible_n = tl.load(VISIBLE + row)
is_prefix = tl.load(IS_PREFIX + row)
for tile in range(tl.program_id(1), tl.cdiv(n, 64), tl.num_programs(1)):
col = tile * 64 + tl.arange(0, 64)
visible = col < n
valid = visible
position = col
if HAS_BLOCKS:
# If all selected blocks form a prefix, keep the short-context
# scoring path's contiguous accesses and avoid the block gather.
if is_prefix == 0:
block = tl.load(
BLOCKS + row * BLOCK_STRIDE + col // BLOCK_SIZE,
visible,
other=-1,
)
position = block * BLOCK_SIZE + col % BLOCK_SIZE
valid = valid & (position >= 0) & (position < visible_n)
if HAS_MASK:
valid = valid & tl.load(
MASK + row * MASK_STRIDE + col, visible, other=False
)
if tl.sum(valid.to(tl.int32), 0) > 0:
slot = (
tl.load(
MAPPING + request * MAP_STRIDE + position * RATIO, valid, other=0
).to(tl.int64)
// RATIO
)
base = (slot // PAGE) * TABLE_STRIDE
off = slot % PAGE
payload = tl.load(
TABLE + base[:, None] + off[:, None] * 64 + d[None, :],
valid[:, None],
other=0,
)
scales = tl.load(
TABLE
+ base[:, None]
+ PAGE * 64
+ off[:, None] * 4
+ tl.arange(0, 4)[None, :],
valid[:, None],
other=127,
)
scale = tl.exp2(scales.to(tl.float32) - 127.0)[:, :, None]
lo = _e2m1_decode(payload & 15)
hi = _e2m1_decode(payload >> 4)
kl = tl.reshape(tl.reshape(lo, (64, 4, 16)) * scale, (64, 64)).to(
tl.bfloat16
)
kh = tl.reshape(tl.reshape(hi, (64, 4, 16)) * scale, (64, 64)).to(
tl.bfloat16
)
acc = tl.dot(qe, tl.trans(kl))
acc += tl.dot(qo, tl.trans(kh))
score = tl.maximum(acc.to(tl.bfloat16).to(tl.float32), 0.0)
score = (score * weight[:, None]).to(tl.bfloat16).to(tl.float32)
score = tl.sum(score, 0).to(tl.bfloat16).to(tl.float32)
score = tl.where(valid, score, -float("inf"))
else:
score = tl.full((64,), -float("inf"), tl.float32)
# TopK uses 16-byte loads at the prefix boundary. Initialize the whole
# last tile, including the <= 3 padding floats beyond the logical width.
tl.store(OUT + row * OUT_STRIDE + col, score, col < OUT_STRIDE)
def prefix_logits(
q,
weights,
mapping,
req,
lens,
table,
page,
ratio,
width,
mask=None,
*,
candidates=None,
visible=None,
):
"""Return partially initialized scores, bounded by int32 device lengths.
lens bounds the score buffer in [0, width]. With candidates, visible is the
original prefix length and lens bounds the compact candidate positions.
"""
assert candidates is None or (visible is not None and mask is None)
rows = q.shape[0]
stride = triton.cdiv(width, 4) * 4
out = torch.empty((rows, stride), dtype=torch.float32, device=q.device)
sm_count = torch.cuda.get_device_properties(q.device).multi_processor_count
tiles = min(
triton.cdiv(width, 64),
max(8, triton.next_power_of_2(triton.cdiv(8 * sm_count, rows))),
)
_prefix_logits[(rows, tiles)](
q,
weights,
mapping,
req,
lens,
table,
mask if mask is not None else out,
out,
candidates.blocks if candidates is not None else out,
candidates.is_prefix if candidates is not None else lens,
visible if visible is not None else lens,
q.stride(0),
weights.stride(0),
mapping.stride(0),
table.stride(0),
mask.stride(0) if mask is not None else 0,
stride,
candidates.blocks.stride(0) if candidates is not None else 0,
page,
ratio,
mask is not None,
candidates is not None,
candidates.block_size if candidates is not None else 1,
num_warps=4,
)
return out
def select_prefix_topk(scores, lens, k):
"""Exact TopK by score over each visible prefix; -1 pads short rows."""
selected = torch.empty(
(scores.shape[0], k), dtype=torch.int32, device=scores.device
)
# The paged small-batch dispatch can request a 16-CTA cluster, while H20
# supports at most 8. Reuse v2's length-aware non-cluster entry point.
offsets = torch.zeros_like(lens)
topk_transform_ragged_v2(scores, lens, out_offsets=offsets, out_indices=selected)
return selected
@triton.jit
def _block_scores(
SCORES,
LENS,
OUT,
BLOCK_LENS,
SCORE_STRIDE: tl.constexpr,
OUT_STRIDE: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
TILE: tl.constexpr,
):
row = tl.program_id(0)
n = tl.load(LENS + row)
count = tl.cdiv(n, BLOCK_SIZE)
if tl.program_id(1) == 0:
tl.store(BLOCK_LENS + row, count)
for tile in range(tl.program_id(1), tl.cdiv(count, TILE), tl.num_programs(1)):
block = tile * TILE + tl.arange(0, TILE)
pos = block[:, None] * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)[None, :]
v = tl.load(SCORES + row * SCORE_STRIDE + pos, pos < n, other=-float("inf"))
score = tl.max(v, 1)
score = tl.where(block == count - 1, float("inf"), score)
tl.store(OUT + row * OUT_STRIDE + block, score, block < OUT_STRIDE)
@triton.jit
def _publish_blocks(
SELECTED,
SCORES,
MASK,
K: tl.constexpr,
SCORE_STRIDE: tl.constexpr,
WIDTH: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
row = tl.program_id(0)
i = tl.program_id(1) * 256 + tl.arange(0, 256)
block = tl.load(SELECTED + row * K + i // BLOCK_SIZE, i < K * BLOCK_SIZE, other=-1)
valid = (i < K * BLOCK_SIZE) & (block >= 0)
score = tl.load(
SCORES + row * SCORE_STRIDE + tl.maximum(block, 0), valid, other=-float("inf")
)
pos = block * BLOCK_SIZE + i % BLOCK_SIZE
# Keep the whole last block, exactly as the existing candidate mask does;
# visibility is enforced independently by every score consumer.
tl.store(
MASK + row * WIDTH + pos, True, valid & (score > -float("inf")) & (pos < WIDTH)
)
def _candidate_block_topk(scores, lens, width, topk_blocks, block_size):
rows = scores.shape[0]
blocks = triton.cdiv(width, block_size)
stride = triton.cdiv(blocks, 4) * 4
block_scores = torch.empty(
(rows, stride), dtype=torch.float32, device=scores.device
)
block_lens = torch.empty_like(lens)
_block_scores[(rows, min(16, triton.cdiv(blocks, 128)))](
scores,
lens,
block_scores,
block_lens,
scores.stride(0),
stride,
block_size,
128,
num_warps=4,
)
k = min(topk_blocks, blocks)
selected = select_prefix_topk(block_scores, block_lens, k)
return selected, block_scores
def candidate_mask(scores, lens, width, topk_blocks, block_size):
selected, block_scores = _candidate_block_topk(
scores, lens, width, topk_blocks, block_size
)
rows, k = selected.shape
# Preserve CandidateMasks' full-width contract for fallback consumers. Only
# the source layer initializes this bool buffer; consumers read valid tiles.
mask = torch.zeros((rows, width), dtype=torch.bool, device=scores.device)
_publish_blocks[(rows, triton.cdiv(k * block_size, 256))](
selected,
block_scores,
mask,
k,
block_scores.stride(0),
width,
block_size,
num_warps=4,
)
return mask
@triton.jit
def _sort_blocks(
SELECTED,
SCORES,
LENS,
BLOCKS,
LENGTHS,
IS_PREFIX,
K: tl.constexpr,
SCORE_STRIDE: tl.constexpr,
WIDTH: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
BLOCK: tl.constexpr,
):
row = tl.program_id(0)
i = tl.arange(0, BLOCK)
count = tl.cdiv(tl.load(LENS + row), BLOCK_SIZE)
if count <= K:
# Short contexts keep every finite block. Avoid a 2048-element bitonic
# sort when the candidate list is already the contiguous prefix.
block = i
valid = (i < K) & (i < count)
else:
block = tl.load(SELECTED + row * K + i, i < K, other=-1)
valid = (i < K) & (block >= 0)
score = tl.load(
SCORES + row * SCORE_STRIDE + tl.maximum(block, 0),
valid,
other=-float("inf"),
)
valid = valid & (score > -float("inf"))
block = tl.where(valid, block, 0x7FFFFFFF)
# All -inf blocks are excluded even when K covers the prefix. Such holes
# need compaction just like the selected subset of a long context does.
if count > K or tl.sum(valid.to(tl.int32), 0) != count:
block = tl.sort(block, descending=False)
valid = block != 0x7FFFFFFF
count = tl.sum(valid.to(tl.int32), 0)
tl.store(BLOCKS + row * K + i, tl.where(valid, block, -1), i < K)
# Keep entire selected blocks, matching CandidateMasks. Consumer visibility
# independently masks the final partial block and any differing ratio.
tl.store(LENGTHS + row, tl.minimum(count * BLOCK_SIZE, WIDTH))
tl.store(IS_PREFIX + row, tl.sum((valid & (block != i)).to(tl.int32), 0) == 0)
def candidate_blocks(scores, lens, width, topk_blocks, block_size):
"""Publish a compact list using prefix scores and non-cluster TopK."""
from sglang.srt.layers.attention.dsv4.v41_indexer.sm90_decode import CandidateBlocks
selected, block_scores = _candidate_block_topk(
scores, lens, width, topk_blocks, block_size
)
rows, k = selected.shape
blocks = torch.empty_like(selected)
lengths = torch.empty_like(lens)
is_prefix = torch.empty_like(lens)
_sort_blocks[(rows,)](
selected,
block_scores,
lens,
blocks,
lengths,
is_prefix,
k,
block_scores.stride(0),
width,
block_size,
triton.next_power_of_2(k),
num_warps=4,
)
return CandidateBlocks(blocks, lengths, is_prefix, width, block_size)
@triton.jit
def _blocks_to_mask(
BLOCKS,
MASK,
K: tl.constexpr,
WIDTH: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
row = tl.program_id(0)
i = tl.program_id(1) * 256 + tl.arange(0, 256)
block = tl.load(BLOCKS + row * K + i // BLOCK_SIZE, i < K * BLOCK_SIZE, other=-1)
pos = block * BLOCK_SIZE + i % BLOCK_SIZE
tl.store(
MASK + row * WIDTH + pos,
True,
(i < K * BLOCK_SIZE) & (block >= 0) & (pos < WIDTH),
)
def materialize_candidate_mask(candidates):
"""Build the legacy mask only when a later layer takes a fallback path."""
rows, k = candidates.blocks.shape
mask = torch.zeros(
(rows, candidates.width), dtype=torch.bool, device=candidates.blocks.device
)
_blocks_to_mask[(rows, triton.cdiv(k * candidates.block_size, 256))](
candidates.blocks,
mask,
k,
candidates.width,
candidates.block_size,
num_warps=4,
)
return mask
@triton.jit
def _publish_topk(
SELECTED,
SCORES,
LENS,
REQ,
MAPPING,
PAGES,
RAW,
BLOCKS,
K: tl.constexpr,
SCORE_STRIDE: tl.constexpr,
MAP_STRIDE: tl.constexpr,
PAGE_STRIDE: tl.constexpr,
RAW_STRIDE: tl.constexpr,
BLOCK_STRIDE: tl.constexpr,
RATIO: tl.constexpr,
HAS_RAW: tl.constexpr,
FILTER_MASKED: tl.constexpr,
HAS_BLOCKS: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
BLOCK: tl.constexpr,
):
row = tl.program_id(0)
col = tl.arange(0, BLOCK)
logical = tl.load(SELECTED + row * K + col, col < K, other=-1)
n = tl.load(LENS + row)
valid = (logical >= 0) & (logical < n) & (col < K)
if FILTER_MASKED:
score = tl.load(
SCORES + row * SCORE_STRIDE + tl.maximum(logical, 0),
valid,
other=-float("inf"),
)
valid = valid & (score > -float("inf"))
if HAS_BLOCKS:
block = tl.load(
BLOCKS + row * BLOCK_STRIDE + tl.maximum(logical, 0) // BLOCK_SIZE,
valid,
other=-1,
)
logical = block * BLOCK_SIZE + logical % BLOCK_SIZE
logical = tl.sort(tl.where(valid, logical, 0x7FFFFFFF), descending=False)
valid = (logical != 0x7FFFFFFF) & (col < K)
request = tl.load(REQ + row).to(tl.int64)
position = tl.where(valid, logical, 0).to(tl.int64)
slot = tl.load(MAPPING + request * MAP_STRIDE + position * RATIO, valid, other=0)
tl.store(
PAGES + row * PAGE_STRIDE + col, tl.where(valid, slot // RATIO, -1), col < K
)
if HAS_RAW:
tl.store(RAW + row * RAW_STRIDE + col, tl.where(valid, logical, -1), col < K)
def publish_topk(
selected,
scores,
lens,
req,
mapping,
pages,
raw,
ratio,
filter_masked,
*,
candidates=None,
):
rows, k = selected.shape
_publish_topk[(rows,)](
selected,
scores,
lens,
req,
mapping,
pages,
raw if raw is not None else pages,
candidates.blocks if candidates is not None else selected,
k,
scores.stride(0),
mapping.stride(0),
pages.stride(0),
raw.stride(0) if raw is not None else 0,
candidates.blocks.stride(0) if candidates is not None else 0,
ratio,
raw is not None,
filter_masked,
candidates is not None,
candidates.block_size if candidates is not None else 1,
triton.next_power_of_2(k),
num_warps=4,
)
+7
View File
@@ -1579,6 +1579,13 @@ class Envs:
# Run the DeepSeek-V4.1 ratio-1/2 prefill indexer on the torch path instead
# of the DeepGEMM dense fp4 logits kernel (test oracle / fallback).
SGLANG_DSV41_TORCH_PREFILL_INDEXER = EnvBool(False)
# Fused mapping for SM90 32-head, ratio-1/2 static target verification.
SGLANG_OPT_DSV41_SM90_GROUPED_INDEXER = EnvBool(False)
# Persistent prefix scoring and length-aware TopK; requires GROUPED_INDEXER.
# Equal-score cutoff ties can select different positions than PyTorch.
SGLANG_OPT_DSV41_SM90_LENGTH_AWARE_INDEXER = EnvBool(False)
# Compact candidate lists; requires the length-aware static-verify path.
SGLANG_OPT_DSV41_SM90_COMPACT_CANDIDATES = EnvBool(False)
SGLANG_FP8_PAGED_MQA_LOGITS_TORCH = EnvBool(False)
SGLANG_OPT_FLASHMLA_SPARSE_PREFILL = EnvBool(True)
@@ -3131,7 +3131,7 @@ class DeepseekV4AttnBackend(
is_source = is_consumer = False
inputs = self._make_low_ratio_decode_indexer_inputs(
layer, x, q_lora, req, pos, mode
layer, x, q_lora, req, pos, forward_batch
)
if is_source:
published = self.decode_candidates.publish_decode(inputs)
@@ -3179,7 +3179,9 @@ class DeepseekV4AttnBackend(
)
self.full_topk_indexer.topk_prefill_captured(inputs)
def _make_low_ratio_decode_indexer_inputs(self, layer, x, q_lora, req, pos, mode):
def _make_low_ratio_decode_indexer_inputs(
self, layer, x, q_lora, req, pos, forward_batch
):
ratio = layer.compress_ratio
metadata = (
self.forward_metadata.c1_indexer_metadata
@@ -3187,6 +3189,24 @@ class DeepseekV4AttnBackend(
else self.forward_metadata.c2_indexer_metadata
)
assert metadata is not None, f"no decode indexer metadata for {ratio = }"
group_size = 1
if (
envs.SGLANG_OPT_DSV41_SM90_GROUPED_INDEXER.get()
and forward_batch.forward_mode.is_target_verify()
and not self.is_dspark_draft
and read_ragged_verify_mode() is RaggedVerifyMode.STATIC
and self.forward_metadata.late_layer_tail is None
and self.speculative_num_draft_tokens is not None
and self.speculative_num_draft_tokens > 1
and forward_batch.spec_info.draft_token_num
== self.speculative_num_draft_tokens
and req.shape[0]
== forward_batch.batch_size * self.speculative_num_draft_tokens
and x.shape[0] == q_lora.shape[0] == req.shape[0]
):
# Only static verify guarantees fixed request-major groups.
# Do not read GPU request ids or lengths during graph capture.
group_size = self.speculative_num_draft_tokens
return DecodeInputs(
indexer=layer.indexer,
layer_id=layer.layer_id,
@@ -3197,7 +3217,8 @@ class DeepseekV4AttnBackend(
positions=pos,
req_rows=req,
paged_metadata=metadata,
is_verify=mode.is_target_verify(),
is_verify=forward_batch.forward_mode.is_target_verify(),
group_size=group_size,
out_page_indices=self.forward_metadata.core_metadata.sparse_page_indices(
ratio
),
@@ -3,7 +3,7 @@ its best blocks, a consumer takes its top-k among them."""
from __future__ import annotations
from typing import TYPE_CHECKING, List, Optional, Tuple
from typing import TYPE_CHECKING, List, Optional, Tuple, Union
import msgspec
import torch
@@ -27,6 +27,7 @@ from .scoring import (
write_decode,
write_prefill,
)
from .sm90_decode import CandidateBlocks, try_sm90_decode
from .types import (
CandidateMetadata,
DecodeInputs,
@@ -93,6 +94,14 @@ class DenseBlocksBackend:
self._torch_consume_prefill(inputs, published)
def publish_decode(self, inputs: DecodeInputs):
handled, published = try_sm90_decode(
inputs=inputs,
token_to_kv_pool=self.token_to_kv_pool,
req_to_token=self.req_to_token,
is_source=True,
)
if handled:
return published
d = decode_scores(
inputs=inputs,
token_to_kv_pool=self.token_to_kv_pool,
@@ -128,8 +137,17 @@ class DenseBlocksBackend:
def consume_decode(
self,
inputs: DecodeInputs,
published: Optional[BlockIds],
published: Optional[Union[BlockIds, CandidateBlocks]],
) -> None:
handled, _ = try_sm90_decode(
inputs=inputs,
token_to_kv_pool=self.token_to_kv_pool,
req_to_token=self.req_to_token,
is_consumer=True,
published=published,
)
if handled:
return
d = decode_scores(
inputs=inputs,
token_to_kv_pool=self.token_to_kv_pool,
@@ -28,6 +28,7 @@ from .scoring import (
select_decode,
write_prefill,
)
from .sm90_decode import try_sm90_decode
from .types import (
CapturedPrefillInputs,
DecodeInputs,
@@ -134,6 +135,13 @@ class FullTopKIndexer:
)
def _torch_decode(self, inputs: DecodeInputs) -> None:
handled, _ = try_sm90_decode(
inputs=inputs,
token_to_kv_pool=self.token_to_kv_pool,
req_to_token=self.req_to_token,
)
if handled:
return
d = decode_scores(
inputs=inputs,
token_to_kv_pool=self.token_to_kv_pool,
@@ -0,0 +1,256 @@
"""Opt-in SM90 static-verification selection for the V4.1 decode backends."""
from __future__ import annotations
from dataclasses import dataclass
from typing import TYPE_CHECKING, Optional, Tuple
import torch
from sglang.srt.environ import envs
from .types import CandidateMetadata, DecodeInputs
if TYPE_CHECKING:
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
@dataclass
class CandidateBlocks(CandidateMetadata):
"""Sorted block IDs with device lengths refreshed on each graph replay.
Ordinary decode consumers can use ``blocks`` directly. Length-aware
consumers also use the device lengths; never cache a derived full mask.
"""
blocks: torch.Tensor # [rows, topk_blocks] int32, -1 padding
lengths: torch.Tensor # [rows] int32, selected capacity in positions
is_prefix: torch.Tensor # [rows] int32, blocks are consecutive from zero
width: int # source capacity in compressed positions
block_size: int
def _candidate_mask(published, width: int, block_size: int) -> torch.Tensor:
"""Adapt a standard block-ID publish to prefix scoring, without caching."""
if isinstance(published, CandidateBlocks):
from sglang.kernels.ops.attention.dsv4.sm90_length_aware_indexer import (
materialize_candidate_mask,
)
assert published.width >= width and published.block_size == block_size
return materialize_candidate_mask(published)
blocks = published.blocks
num_blocks = (width + block_size - 1) // block_size
keep = torch.zeros(
(*blocks.shape[:-1], num_blocks + 1), dtype=torch.bool, device=blocks.device
)
ids = blocks.to(torch.int64)
ids = ids.masked_fill((ids < 0) | (ids >= num_blocks), num_blocks)
keep.scatter_(-1, ids, True)
return keep[..., :num_blocks].repeat_interleave(block_size, dim=-1)[..., :width]
def try_sm90_decode(
*,
inputs: DecodeInputs,
token_to_kv_pool: DeepSeekV4TokenToKVPool,
req_to_token: torch.Tensor,
is_source: bool = False,
is_consumer: bool = False,
published: Optional[CandidateMetadata] = None,
) -> Tuple[bool, Optional[CandidateMetadata]]:
"""Return whether selection was handled, plus a source's published blocks.
Candidate roles come from the caller: graph variants may bypass filtering
even for a layer whose indexer normally publishes or consumes candidates.
"""
ratio = inputs.compress_ratio
if not (
envs.SGLANG_OPT_DSV41_SM90_GROUPED_INDEXER.get()
and inputs.is_verify
and inputs.group_size > 1
and ratio in (1, 2)
):
return False, None
indexer = inputs.indexer
req, pos = inputs.req_rows, inputs.positions
bs = req.shape[0]
assert pos.shape[0] == bs, (
f"decode expects one token per request, {pos.shape=} {bs=}"
)
# Metadata reserves graph replay capacity; visibility remains on device.
lmax = min(
inputs.paged_metadata.max_compressed_seq_len, req_to_token.shape[1] // ratio
)
if bs == 0 or lmax == 0:
inputs.reset_outputs()
return True, None
q = indexer.queries(inputs.q_lora, inputs.freqs_cis[pos])
weights = indexer.head_weights(inputs.x)
table = token_to_kv_pool.get_index_k_with_scale_buffer(inputs.layer_id)
if not (
q.is_cuda
and torch.version.cuda is not None
and torch.cuda.get_device_capability(q.device)[0] == 9
and q.dtype == weights.dtype == torch.bfloat16
and q.shape[1:] == (32, 128)
and q.is_contiguous()
and weights.is_contiguous()
and req_to_token.dtype == torch.int32
and req_to_token.stride(1) == 1
and table.dtype == torch.uint8
and table.dim() == 2
and table.stride(1) == 1
):
return False, None
from sglang.kernels.ops.attention.dsv4.candidate_blocks import (
select_candidate_block_ids,
topk_among_blocks,
)
# Imported only after dispatch to avoid a module-initialization cycle.
from .dense_blocks import BlockIds
inputs.reset_outputs()
if is_consumer:
assert published is not None and published.blocks.shape[0] == bs
use_length_aware = (
envs.SGLANG_OPT_DSV41_SM90_LENGTH_AWARE_INDEXER.get()
and 0 < indexer.index_topk <= 2048
and (
not is_source
or (
0 < indexer.candidate_topk_blocks <= 2048
and indexer.candidate_block_size in (1, 2, 4, 8, 16, 32, 64, 128)
)
)
)
if use_length_aware:
from sglang.kernels.ops.attention.dsv4.sm90_length_aware_indexer import (
candidate_blocks,
prefix_logits,
prepare_candidate_lengths,
publish_topk,
select_prefix_topk,
)
request = req.to(torch.int64).contiguous()
compact = envs.SGLANG_OPT_DSV41_SM90_COMPACT_CANDIDATES.get() and (
is_source or is_consumer
)
candidates = (
published
if compact and is_consumer and isinstance(published, CandidateBlocks)
else None
)
if compact:
visible, score_lens = prepare_candidate_lengths(
pos, ratio, lmax, candidates
)
else:
visible = ((pos + 1) // ratio).clamp(0, lmax).to(torch.int32).contiguous()
score_lens = visible
consume = None
if is_consumer and candidates is None:
consume = _candidate_mask(published, lmax, indexer.candidate_block_size)
assert consume.shape[0] == bs and consume.shape[1] >= lmax
assert consume.stride(1) == 1
score_width = lmax
if candidates is not None:
assert candidates.width >= lmax
assert candidates.block_size == indexer.candidate_block_size
score_width = min(lmax, candidates.blocks.shape[1] * candidates.block_size)
scores = prefix_logits(
q,
weights,
req_to_token,
request,
score_lens,
table,
table.shape[1] // 68,
ratio,
score_width,
consume,
candidates=candidates,
visible=visible,
)
result = None
if is_source:
result = candidate_blocks(
scores,
visible,
lmax,
indexer.candidate_topk_blocks,
indexer.candidate_block_size,
)
if not compact:
# Main's decode protocol carries block IDs even without the
# compact scoring flag. Drop only the optional device lengths.
result = BlockIds(blocks=result.blocks)
idx = select_prefix_topk(
scores, score_lens, min(indexer.index_topk, score_width)
)
publish_topk(
idx,
scores,
score_lens,
request,
req_to_token,
inputs.out_page_indices,
None,
ratio,
is_consumer,
candidates=candidates,
)
return True, result
from sglang.kernels.ops.attention.dsv4.sm90_fp4_indexer import (
fp4_index_logits_mapped_sm90,
)
lens = (pos + 1) // ratio
scores = fp4_index_logits_mapped_sm90(
q,
weights,
req_to_token,
req.to(torch.int64).contiguous(),
lens.to(torch.int64).contiguous(),
table,
table.shape[1] // 68,
ratio,
lmax,
)
result = None
if is_source:
result = BlockIds(
blocks=select_candidate_block_ids(
scores,
lens[:, None],
indexer.candidate_topk_blocks,
indexer.candidate_block_size,
)
)
k = min(indexer.index_topk, lmax)
if is_consumer:
idx = topk_among_blocks(
scores, lens, published.blocks, k, block_size=indexer.candidate_block_size
)
idx = idx.masked_fill(idx < 0, lmax)
else:
idx = scores.topk(k, dim=-1, sorted=False).indices
if 0 < k <= 1024:
from sglang.kernels.ops.attention.dsv4.sm90_fp4_topk import sort_map_topk
sort_map_topk(
idx, lens, req, req_to_token, inputs.out_page_indices, None, ratio
)
else:
idx = idx.sort(dim=-1).values
reach = idx < lens[:, None]
slots = (
req_to_token[req[:, None], idx.clamp_max(lmax - 1) * ratio].to(torch.int64)
// ratio
)
inputs.out_page_indices[:bs, :k] = torch.where(reach, slots, -1).to(torch.int32)
return True, result
@@ -81,6 +81,8 @@ class DecodeInputs(msgspec.Struct, frozen=True, kw_only=True):
req_rows: torch.Tensor # [rows] req_to_token row of each query row
paged_metadata: PagedIndexerMetadata
is_verify: bool
# Fixed request-major groups, validated by the attention backend for static verify.
group_size: int = 1
# [rows, topk] int32 index-K pool slots of the selection, unordered, -1 padded.
out_page_indices: torch.Tensor