PR #40945: SM90 static verification indexing (grouped/length-aware indexer)
Source: https://github.com/sgl-project/sglang/pull/40945 (applied via GitHub .diff, 3way) Adaptation: PR base predates #41659 which removed the Selection struct; ported sm90_decode.py and call sites to the inputs.out_page_indices interface (raw_indices=None on decode paths, reset via inputs.reset_outputs()).
This commit is contained in:
@@ -0,0 +1,170 @@
|
||||
"""Fused request-to-KV mapping for opt-in SM90 32-head static verification."""
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
INDEX_HEAD_DIM = 128
|
||||
PAYLOAD_BYTES = tl.constexpr(64)
|
||||
SCALE_BYTES = tl.constexpr(4)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _e2m1_decode(code):
|
||||
# code: uint 0..15 -> e2m1 value. exp = bits 2..1, mantissa = bit 0, sign = bit 3.
|
||||
e = (code >> 1) & 3
|
||||
m = (code & 1).to(tl.float32)
|
||||
sub = m * 0.5
|
||||
exp = tl.where(e == 1, 1.0, tl.where(e == 2, 2.0, 4.0))
|
||||
nor = (1.0 + m * 0.5) * exp
|
||||
v = tl.where(e == 0, sub, nor)
|
||||
return tl.where((code >> 3) == 1, -v, v)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fp4_index_logits_kernel(
|
||||
q_ptr, # [B, H, D] bf16, fq4 queries (already rope'd)
|
||||
w_ptr, # [B, H] bf16 head weights (softmax scale folded in)
|
||||
slots_ptr, # [requests, capacity * ratio] int32 request-to-token mapping
|
||||
lens_ptr, # [B] int64 visible compressed positions per request
|
||||
table_ptr, # [num_pages, page_size * 64 + page_size * 4] uint8
|
||||
out_ptr, # [B, L] fp32 logits, -inf beyond lens
|
||||
L,
|
||||
page_size,
|
||||
row_stride,
|
||||
stride_qb,
|
||||
stride_qh,
|
||||
stride_wb,
|
||||
H: tl.constexpr,
|
||||
HALF_D: tl.constexpr, # D // 2 == 64 nibble-pairs per row
|
||||
BLOCK_L: tl.constexpr,
|
||||
req_ptr,
|
||||
req_stride: tl.constexpr,
|
||||
ratio: tl.constexpr,
|
||||
):
|
||||
b = tl.program_id(0)
|
||||
lb = tl.program_id(1)
|
||||
offs_l = lb * BLOCK_L + tl.arange(0, BLOCK_L)
|
||||
offs_h = tl.arange(0, H)
|
||||
offs_i = tl.arange(
|
||||
0, HALF_D
|
||||
) # byte index i holds elements 2i (low nibble), 2i+1 (high nibble)
|
||||
|
||||
n_vis = tl.load(lens_ptr + b)
|
||||
# Keep capacity-sized CUDA Graph buffers while skipping invisible tiles,
|
||||
# including mapping loads, dequantization, dot products and head reduction.
|
||||
if lb * BLOCK_L >= n_vis:
|
||||
tl.store(out_ptr + b * L + offs_l, float("-inf"), mask=offs_l < L)
|
||||
else:
|
||||
valid = offs_l < tl.minimum(n_vis, L)
|
||||
request = tl.load(req_ptr + b).to(tl.int64)
|
||||
slot = (
|
||||
tl.load(
|
||||
slots_ptr + request * req_stride + offs_l * ratio,
|
||||
mask=valid,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
// ratio
|
||||
)
|
||||
page = slot // page_size
|
||||
off = slot % page_size
|
||||
row_base = page * row_stride
|
||||
|
||||
# K payload: [BLOCK_L, HALF_D] uint8
|
||||
pay = tl.load(
|
||||
table_ptr
|
||||
+ row_base[:, None]
|
||||
+ off[:, None] * PAYLOAD_BYTES
|
||||
+ offs_i[None, :],
|
||||
mask=valid[:, None],
|
||||
other=0,
|
||||
)
|
||||
low = _e2m1_decode(pay & 0x0F)
|
||||
high = _e2m1_decode((pay >> 4) & 0x0F)
|
||||
# e8m0 block scales: each scale covers 16 payload bytes (32 elements).
|
||||
offs_s = tl.arange(0, SCALE_BYTES)
|
||||
exps = tl.load(
|
||||
table_ptr
|
||||
+ row_base[:, None]
|
||||
+ page_size * PAYLOAD_BYTES
|
||||
+ off[:, None] * SCALE_BYTES
|
||||
+ offs_s[None, :],
|
||||
mask=valid[:, None],
|
||||
other=127,
|
||||
)
|
||||
scale = tl.exp2(exps.to(tl.float32) - 127.0)
|
||||
scale = scale[:, :, None]
|
||||
k_shape: tl.constexpr = (BLOCK_L, SCALE_BYTES, HALF_D // SCALE_BYTES)
|
||||
k_low = tl.reshape(tl.reshape(low, k_shape) * scale, (BLOCK_L, HALF_D)).to(
|
||||
tl.bfloat16
|
||||
)
|
||||
k_high = tl.reshape(tl.reshape(high, k_shape) * scale, (BLOCK_L, HALF_D)).to(
|
||||
tl.bfloat16
|
||||
)
|
||||
|
||||
# queries: even / odd elements, [H, HALF_D] bf16
|
||||
q_even = tl.load(
|
||||
q_ptr + b * stride_qb + offs_h[:, None] * stride_qh + 2 * offs_i[None, :]
|
||||
)
|
||||
q_odd = tl.load(
|
||||
q_ptr
|
||||
+ b * stride_qb
|
||||
+ offs_h[:, None] * stride_qh
|
||||
+ 2 * offs_i[None, :]
|
||||
+ 1
|
||||
)
|
||||
|
||||
acc = tl.dot(q_even, tl.trans(k_low)) # [H, BLOCK_L] fp32
|
||||
acc += tl.dot(q_odd, tl.trans(k_high))
|
||||
# reference rounding points: bf16 dot -> relu -> * bf16 weight -> bf16 -> sum -> bf16
|
||||
s = acc.to(tl.bfloat16).to(tl.float32)
|
||||
s = tl.maximum(s, 0.0)
|
||||
w = tl.load(w_ptr + b * stride_wb + offs_h).to(tl.float32)
|
||||
s = (s * w[:, None]).to(tl.bfloat16).to(tl.float32)
|
||||
logit = tl.sum(s, axis=0).to(tl.bfloat16).to(tl.float32)
|
||||
logit = tl.where(valid, logit, float("-inf"))
|
||||
tl.store(out_ptr + b * L + offs_l, logit, mask=offs_l < L)
|
||||
|
||||
|
||||
def fp4_index_logits_mapped_sm90(
|
||||
q,
|
||||
weights,
|
||||
req_to_token,
|
||||
req,
|
||||
lens,
|
||||
table,
|
||||
page_size,
|
||||
ratio,
|
||||
width,
|
||||
):
|
||||
"""Score with request mapping fused into the Triton kernel.
|
||||
|
||||
The caller validates the common tensor/device contract. No dense slots
|
||||
tensor or host read of dynamic lengths is needed during graph replay.
|
||||
"""
|
||||
rows, heads, _ = q.shape
|
||||
out = torch.empty((rows, width), dtype=torch.float32, device=q.device)
|
||||
if rows == 0 or width == 0:
|
||||
return out
|
||||
_fp4_index_logits_kernel[(rows, triton.cdiv(width, 64))](
|
||||
q,
|
||||
weights,
|
||||
req_to_token,
|
||||
lens,
|
||||
table,
|
||||
out,
|
||||
width,
|
||||
page_size,
|
||||
table.stride(0),
|
||||
q.stride(0),
|
||||
q.stride(1),
|
||||
weights.stride(0),
|
||||
H=heads,
|
||||
HALF_D=64,
|
||||
BLOCK_L=64,
|
||||
req_ptr=req,
|
||||
req_stride=req_to_token.stride(0),
|
||||
ratio=ratio,
|
||||
num_warps=4,
|
||||
)
|
||||
return out
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Sort selected logical positions and map slots without dense intermediates."""
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _sort_map_topk(
|
||||
selected,
|
||||
lens,
|
||||
req,
|
||||
mapping,
|
||||
pages,
|
||||
raw,
|
||||
selected_row_stride: tl.constexpr,
|
||||
selected_col_stride: tl.constexpr,
|
||||
lens_stride: tl.constexpr,
|
||||
req_stride: tl.constexpr,
|
||||
mapping_row_stride: tl.constexpr,
|
||||
pages_row_stride: tl.constexpr,
|
||||
raw_row_stride: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
RATIO: tl.constexpr,
|
||||
HAS_RAW: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
col = tl.arange(0, BLOCK)
|
||||
logical = tl.load(
|
||||
selected + row * selected_row_stride + col * selected_col_stride,
|
||||
col < K,
|
||||
other=0x7FFFFFFFFFFFFFFF,
|
||||
)
|
||||
logical = tl.sort(logical, descending=False)
|
||||
visible = tl.load(lens + row * lens_stride)
|
||||
request = tl.load(req + row * req_stride).to(tl.int64)
|
||||
valid = (col < K) & (logical < visible)
|
||||
# Invalid candidate positions can equal the scan bound; never read their
|
||||
# mappings. The backend has already filled the unused output tail with -1.
|
||||
position = tl.where(valid, logical, 0)
|
||||
slot = tl.load(
|
||||
mapping + request * mapping_row_stride + position * RATIO,
|
||||
valid,
|
||||
other=0,
|
||||
)
|
||||
tl.store(
|
||||
pages + row * pages_row_stride + col,
|
||||
tl.where(valid, slot // RATIO, -1),
|
||||
col < K,
|
||||
)
|
||||
if HAS_RAW:
|
||||
tl.store(
|
||||
raw + row * raw_row_stride + col, tl.where(valid, logical, -1), col < K
|
||||
)
|
||||
|
||||
|
||||
def sort_map_topk(selected, lens, req, mapping, pages, raw, ratio):
|
||||
"""Publish sorted TopK values; selection and tie-breaking stay in PyTorch.
|
||||
|
||||
The caller supplies int64 selected positions, with invalid candidates
|
||||
replaced by the scan bound, and pre-fills both output tensors with -1.
|
||||
Only the opt-in backend uses this small-K epilogue.
|
||||
"""
|
||||
rows, k = selected.shape
|
||||
assert selected.dtype == torch.int64 and 0 < k <= 1024
|
||||
_sort_map_topk[(rows,)](
|
||||
selected,
|
||||
lens,
|
||||
req,
|
||||
mapping,
|
||||
pages,
|
||||
raw if raw is not None else pages,
|
||||
selected.stride(0),
|
||||
selected.stride(1),
|
||||
lens.stride(0),
|
||||
req.stride(0),
|
||||
mapping.stride(0),
|
||||
pages.stride(0),
|
||||
raw.stride(0) if raw is not None else 0,
|
||||
K=k,
|
||||
RATIO=ratio,
|
||||
HAS_RAW=raw is not None,
|
||||
BLOCK=triton.next_power_of_2(k),
|
||||
num_warps=4,
|
||||
)
|
||||
@@ -0,0 +1,524 @@
|
||||
"""Opt-in 32-head Hopper indexer with device-side prefix lengths.
|
||||
|
||||
Score buffers keep their graph-stable capacity, but only the visible prefix
|
||||
(and its final 64-element tile) is initialized. Every consumer MUST use the
|
||||
device lengths. TopK v2 is exact by score, with arbitrary cutoff-tie selection;
|
||||
it does not promise PyTorch's choice of indices with equal scores.
|
||||
"""
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from .sm90_fp4_indexer import _e2m1_decode
|
||||
from .topk import topk_transform_ragged_v2
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _prepare_lengths(
|
||||
POS,
|
||||
CAPACITY,
|
||||
VISIBLE,
|
||||
LENGTHS,
|
||||
ROWS: tl.constexpr,
|
||||
POS_STRIDE: tl.constexpr,
|
||||
RATIO: tl.constexpr,
|
||||
WIDTH: tl.constexpr,
|
||||
HAS_CANDIDATES: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0) * 256 + tl.arange(0, 256)
|
||||
pos = tl.load(POS + row * POS_STRIDE, row < ROWS, other=-1)
|
||||
n = tl.minimum(tl.maximum((pos + 1) // RATIO, 0), WIDTH).to(tl.int32)
|
||||
tl.store(VISIBLE + row, n, row < ROWS)
|
||||
if HAS_CANDIDATES:
|
||||
capacity = tl.load(CAPACITY + row, row < ROWS, other=0)
|
||||
tl.store(LENGTHS + row, tl.minimum(n, capacity), row < ROWS)
|
||||
|
||||
|
||||
def prepare_candidate_lengths(pos, ratio, width, candidates=None):
|
||||
"""Fuse position conversion, clamping, casting and compact score bounds."""
|
||||
visible = torch.empty(pos.shape[0], dtype=torch.int32, device=pos.device)
|
||||
lengths = torch.empty_like(visible) if candidates is not None else visible
|
||||
_prepare_lengths[(triton.cdiv(pos.shape[0], 256),)](
|
||||
pos,
|
||||
candidates.lengths if candidates is not None else visible,
|
||||
visible,
|
||||
lengths,
|
||||
pos.shape[0],
|
||||
pos.stride(0),
|
||||
ratio,
|
||||
width,
|
||||
candidates is not None,
|
||||
num_warps=4,
|
||||
)
|
||||
return visible, lengths
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _prefix_logits(
|
||||
Q,
|
||||
W,
|
||||
MAPPING,
|
||||
REQ,
|
||||
LENS,
|
||||
TABLE,
|
||||
MASK,
|
||||
OUT,
|
||||
BLOCKS,
|
||||
IS_PREFIX,
|
||||
VISIBLE,
|
||||
Q_STRIDE: tl.constexpr,
|
||||
W_STRIDE: tl.constexpr,
|
||||
MAP_STRIDE: tl.constexpr,
|
||||
TABLE_STRIDE: tl.constexpr,
|
||||
MASK_STRIDE: tl.constexpr,
|
||||
OUT_STRIDE: tl.constexpr,
|
||||
BLOCK_STRIDE: tl.constexpr,
|
||||
PAGE: tl.constexpr,
|
||||
RATIO: tl.constexpr,
|
||||
HAS_MASK: tl.constexpr,
|
||||
HAS_BLOCKS: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
n = tl.load(LENS + row)
|
||||
request = tl.load(REQ + row).to(tl.int64)
|
||||
h = tl.arange(0, 32)
|
||||
d = tl.arange(0, 64)
|
||||
# Reuse query/weights across the tiles owned by this CTA.
|
||||
qe = tl.load(Q + row * Q_STRIDE + h[:, None] * 128 + 2 * d[None, :])
|
||||
qo = tl.load(Q + row * Q_STRIDE + h[:, None] * 128 + 2 * d[None, :] + 1)
|
||||
weight = tl.load(W + row * W_STRIDE + h).to(tl.float32)
|
||||
if HAS_BLOCKS:
|
||||
visible_n = tl.load(VISIBLE + row)
|
||||
is_prefix = tl.load(IS_PREFIX + row)
|
||||
for tile in range(tl.program_id(1), tl.cdiv(n, 64), tl.num_programs(1)):
|
||||
col = tile * 64 + tl.arange(0, 64)
|
||||
visible = col < n
|
||||
valid = visible
|
||||
position = col
|
||||
if HAS_BLOCKS:
|
||||
# If all selected blocks form a prefix, keep the short-context
|
||||
# scoring path's contiguous accesses and avoid the block gather.
|
||||
if is_prefix == 0:
|
||||
block = tl.load(
|
||||
BLOCKS + row * BLOCK_STRIDE + col // BLOCK_SIZE,
|
||||
visible,
|
||||
other=-1,
|
||||
)
|
||||
position = block * BLOCK_SIZE + col % BLOCK_SIZE
|
||||
valid = valid & (position >= 0) & (position < visible_n)
|
||||
if HAS_MASK:
|
||||
valid = valid & tl.load(
|
||||
MASK + row * MASK_STRIDE + col, visible, other=False
|
||||
)
|
||||
if tl.sum(valid.to(tl.int32), 0) > 0:
|
||||
slot = (
|
||||
tl.load(
|
||||
MAPPING + request * MAP_STRIDE + position * RATIO, valid, other=0
|
||||
).to(tl.int64)
|
||||
// RATIO
|
||||
)
|
||||
base = (slot // PAGE) * TABLE_STRIDE
|
||||
off = slot % PAGE
|
||||
payload = tl.load(
|
||||
TABLE + base[:, None] + off[:, None] * 64 + d[None, :],
|
||||
valid[:, None],
|
||||
other=0,
|
||||
)
|
||||
scales = tl.load(
|
||||
TABLE
|
||||
+ base[:, None]
|
||||
+ PAGE * 64
|
||||
+ off[:, None] * 4
|
||||
+ tl.arange(0, 4)[None, :],
|
||||
valid[:, None],
|
||||
other=127,
|
||||
)
|
||||
scale = tl.exp2(scales.to(tl.float32) - 127.0)[:, :, None]
|
||||
lo = _e2m1_decode(payload & 15)
|
||||
hi = _e2m1_decode(payload >> 4)
|
||||
kl = tl.reshape(tl.reshape(lo, (64, 4, 16)) * scale, (64, 64)).to(
|
||||
tl.bfloat16
|
||||
)
|
||||
kh = tl.reshape(tl.reshape(hi, (64, 4, 16)) * scale, (64, 64)).to(
|
||||
tl.bfloat16
|
||||
)
|
||||
acc = tl.dot(qe, tl.trans(kl))
|
||||
acc += tl.dot(qo, tl.trans(kh))
|
||||
score = tl.maximum(acc.to(tl.bfloat16).to(tl.float32), 0.0)
|
||||
score = (score * weight[:, None]).to(tl.bfloat16).to(tl.float32)
|
||||
score = tl.sum(score, 0).to(tl.bfloat16).to(tl.float32)
|
||||
score = tl.where(valid, score, -float("inf"))
|
||||
else:
|
||||
score = tl.full((64,), -float("inf"), tl.float32)
|
||||
# TopK uses 16-byte loads at the prefix boundary. Initialize the whole
|
||||
# last tile, including the <= 3 padding floats beyond the logical width.
|
||||
tl.store(OUT + row * OUT_STRIDE + col, score, col < OUT_STRIDE)
|
||||
|
||||
|
||||
def prefix_logits(
|
||||
q,
|
||||
weights,
|
||||
mapping,
|
||||
req,
|
||||
lens,
|
||||
table,
|
||||
page,
|
||||
ratio,
|
||||
width,
|
||||
mask=None,
|
||||
*,
|
||||
candidates=None,
|
||||
visible=None,
|
||||
):
|
||||
"""Return partially initialized scores, bounded by int32 device lengths.
|
||||
|
||||
lens bounds the score buffer in [0, width]. With candidates, visible is the
|
||||
original prefix length and lens bounds the compact candidate positions.
|
||||
"""
|
||||
assert candidates is None or (visible is not None and mask is None)
|
||||
rows = q.shape[0]
|
||||
stride = triton.cdiv(width, 4) * 4
|
||||
out = torch.empty((rows, stride), dtype=torch.float32, device=q.device)
|
||||
sm_count = torch.cuda.get_device_properties(q.device).multi_processor_count
|
||||
tiles = min(
|
||||
triton.cdiv(width, 64),
|
||||
max(8, triton.next_power_of_2(triton.cdiv(8 * sm_count, rows))),
|
||||
)
|
||||
_prefix_logits[(rows, tiles)](
|
||||
q,
|
||||
weights,
|
||||
mapping,
|
||||
req,
|
||||
lens,
|
||||
table,
|
||||
mask if mask is not None else out,
|
||||
out,
|
||||
candidates.blocks if candidates is not None else out,
|
||||
candidates.is_prefix if candidates is not None else lens,
|
||||
visible if visible is not None else lens,
|
||||
q.stride(0),
|
||||
weights.stride(0),
|
||||
mapping.stride(0),
|
||||
table.stride(0),
|
||||
mask.stride(0) if mask is not None else 0,
|
||||
stride,
|
||||
candidates.blocks.stride(0) if candidates is not None else 0,
|
||||
page,
|
||||
ratio,
|
||||
mask is not None,
|
||||
candidates is not None,
|
||||
candidates.block_size if candidates is not None else 1,
|
||||
num_warps=4,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def select_prefix_topk(scores, lens, k):
|
||||
"""Exact TopK by score over each visible prefix; -1 pads short rows."""
|
||||
selected = torch.empty(
|
||||
(scores.shape[0], k), dtype=torch.int32, device=scores.device
|
||||
)
|
||||
# The paged small-batch dispatch can request a 16-CTA cluster, while H20
|
||||
# supports at most 8. Reuse v2's length-aware non-cluster entry point.
|
||||
offsets = torch.zeros_like(lens)
|
||||
topk_transform_ragged_v2(scores, lens, out_offsets=offsets, out_indices=selected)
|
||||
return selected
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _block_scores(
|
||||
SCORES,
|
||||
LENS,
|
||||
OUT,
|
||||
BLOCK_LENS,
|
||||
SCORE_STRIDE: tl.constexpr,
|
||||
OUT_STRIDE: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
TILE: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
n = tl.load(LENS + row)
|
||||
count = tl.cdiv(n, BLOCK_SIZE)
|
||||
if tl.program_id(1) == 0:
|
||||
tl.store(BLOCK_LENS + row, count)
|
||||
for tile in range(tl.program_id(1), tl.cdiv(count, TILE), tl.num_programs(1)):
|
||||
block = tile * TILE + tl.arange(0, TILE)
|
||||
pos = block[:, None] * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)[None, :]
|
||||
v = tl.load(SCORES + row * SCORE_STRIDE + pos, pos < n, other=-float("inf"))
|
||||
score = tl.max(v, 1)
|
||||
score = tl.where(block == count - 1, float("inf"), score)
|
||||
tl.store(OUT + row * OUT_STRIDE + block, score, block < OUT_STRIDE)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _publish_blocks(
|
||||
SELECTED,
|
||||
SCORES,
|
||||
MASK,
|
||||
K: tl.constexpr,
|
||||
SCORE_STRIDE: tl.constexpr,
|
||||
WIDTH: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
i = tl.program_id(1) * 256 + tl.arange(0, 256)
|
||||
block = tl.load(SELECTED + row * K + i // BLOCK_SIZE, i < K * BLOCK_SIZE, other=-1)
|
||||
valid = (i < K * BLOCK_SIZE) & (block >= 0)
|
||||
score = tl.load(
|
||||
SCORES + row * SCORE_STRIDE + tl.maximum(block, 0), valid, other=-float("inf")
|
||||
)
|
||||
pos = block * BLOCK_SIZE + i % BLOCK_SIZE
|
||||
# Keep the whole last block, exactly as the existing candidate mask does;
|
||||
# visibility is enforced independently by every score consumer.
|
||||
tl.store(
|
||||
MASK + row * WIDTH + pos, True, valid & (score > -float("inf")) & (pos < WIDTH)
|
||||
)
|
||||
|
||||
|
||||
def _candidate_block_topk(scores, lens, width, topk_blocks, block_size):
|
||||
rows = scores.shape[0]
|
||||
blocks = triton.cdiv(width, block_size)
|
||||
stride = triton.cdiv(blocks, 4) * 4
|
||||
block_scores = torch.empty(
|
||||
(rows, stride), dtype=torch.float32, device=scores.device
|
||||
)
|
||||
block_lens = torch.empty_like(lens)
|
||||
_block_scores[(rows, min(16, triton.cdiv(blocks, 128)))](
|
||||
scores,
|
||||
lens,
|
||||
block_scores,
|
||||
block_lens,
|
||||
scores.stride(0),
|
||||
stride,
|
||||
block_size,
|
||||
128,
|
||||
num_warps=4,
|
||||
)
|
||||
k = min(topk_blocks, blocks)
|
||||
selected = select_prefix_topk(block_scores, block_lens, k)
|
||||
return selected, block_scores
|
||||
|
||||
|
||||
def candidate_mask(scores, lens, width, topk_blocks, block_size):
|
||||
selected, block_scores = _candidate_block_topk(
|
||||
scores, lens, width, topk_blocks, block_size
|
||||
)
|
||||
rows, k = selected.shape
|
||||
# Preserve CandidateMasks' full-width contract for fallback consumers. Only
|
||||
# the source layer initializes this bool buffer; consumers read valid tiles.
|
||||
mask = torch.zeros((rows, width), dtype=torch.bool, device=scores.device)
|
||||
_publish_blocks[(rows, triton.cdiv(k * block_size, 256))](
|
||||
selected,
|
||||
block_scores,
|
||||
mask,
|
||||
k,
|
||||
block_scores.stride(0),
|
||||
width,
|
||||
block_size,
|
||||
num_warps=4,
|
||||
)
|
||||
return mask
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _sort_blocks(
|
||||
SELECTED,
|
||||
SCORES,
|
||||
LENS,
|
||||
BLOCKS,
|
||||
LENGTHS,
|
||||
IS_PREFIX,
|
||||
K: tl.constexpr,
|
||||
SCORE_STRIDE: tl.constexpr,
|
||||
WIDTH: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
i = tl.arange(0, BLOCK)
|
||||
count = tl.cdiv(tl.load(LENS + row), BLOCK_SIZE)
|
||||
if count <= K:
|
||||
# Short contexts keep every finite block. Avoid a 2048-element bitonic
|
||||
# sort when the candidate list is already the contiguous prefix.
|
||||
block = i
|
||||
valid = (i < K) & (i < count)
|
||||
else:
|
||||
block = tl.load(SELECTED + row * K + i, i < K, other=-1)
|
||||
valid = (i < K) & (block >= 0)
|
||||
score = tl.load(
|
||||
SCORES + row * SCORE_STRIDE + tl.maximum(block, 0),
|
||||
valid,
|
||||
other=-float("inf"),
|
||||
)
|
||||
valid = valid & (score > -float("inf"))
|
||||
block = tl.where(valid, block, 0x7FFFFFFF)
|
||||
# All -inf blocks are excluded even when K covers the prefix. Such holes
|
||||
# need compaction just like the selected subset of a long context does.
|
||||
if count > K or tl.sum(valid.to(tl.int32), 0) != count:
|
||||
block = tl.sort(block, descending=False)
|
||||
valid = block != 0x7FFFFFFF
|
||||
count = tl.sum(valid.to(tl.int32), 0)
|
||||
tl.store(BLOCKS + row * K + i, tl.where(valid, block, -1), i < K)
|
||||
# Keep entire selected blocks, matching CandidateMasks. Consumer visibility
|
||||
# independently masks the final partial block and any differing ratio.
|
||||
tl.store(LENGTHS + row, tl.minimum(count * BLOCK_SIZE, WIDTH))
|
||||
tl.store(IS_PREFIX + row, tl.sum((valid & (block != i)).to(tl.int32), 0) == 0)
|
||||
|
||||
|
||||
def candidate_blocks(scores, lens, width, topk_blocks, block_size):
|
||||
"""Publish a compact list using prefix scores and non-cluster TopK."""
|
||||
from sglang.srt.layers.attention.dsv4.v41_indexer.sm90_decode import CandidateBlocks
|
||||
|
||||
selected, block_scores = _candidate_block_topk(
|
||||
scores, lens, width, topk_blocks, block_size
|
||||
)
|
||||
rows, k = selected.shape
|
||||
blocks = torch.empty_like(selected)
|
||||
lengths = torch.empty_like(lens)
|
||||
is_prefix = torch.empty_like(lens)
|
||||
_sort_blocks[(rows,)](
|
||||
selected,
|
||||
block_scores,
|
||||
lens,
|
||||
blocks,
|
||||
lengths,
|
||||
is_prefix,
|
||||
k,
|
||||
block_scores.stride(0),
|
||||
width,
|
||||
block_size,
|
||||
triton.next_power_of_2(k),
|
||||
num_warps=4,
|
||||
)
|
||||
return CandidateBlocks(blocks, lengths, is_prefix, width, block_size)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _blocks_to_mask(
|
||||
BLOCKS,
|
||||
MASK,
|
||||
K: tl.constexpr,
|
||||
WIDTH: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
i = tl.program_id(1) * 256 + tl.arange(0, 256)
|
||||
block = tl.load(BLOCKS + row * K + i // BLOCK_SIZE, i < K * BLOCK_SIZE, other=-1)
|
||||
pos = block * BLOCK_SIZE + i % BLOCK_SIZE
|
||||
tl.store(
|
||||
MASK + row * WIDTH + pos,
|
||||
True,
|
||||
(i < K * BLOCK_SIZE) & (block >= 0) & (pos < WIDTH),
|
||||
)
|
||||
|
||||
|
||||
def materialize_candidate_mask(candidates):
|
||||
"""Build the legacy mask only when a later layer takes a fallback path."""
|
||||
rows, k = candidates.blocks.shape
|
||||
mask = torch.zeros(
|
||||
(rows, candidates.width), dtype=torch.bool, device=candidates.blocks.device
|
||||
)
|
||||
_blocks_to_mask[(rows, triton.cdiv(k * candidates.block_size, 256))](
|
||||
candidates.blocks,
|
||||
mask,
|
||||
k,
|
||||
candidates.width,
|
||||
candidates.block_size,
|
||||
num_warps=4,
|
||||
)
|
||||
return mask
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _publish_topk(
|
||||
SELECTED,
|
||||
SCORES,
|
||||
LENS,
|
||||
REQ,
|
||||
MAPPING,
|
||||
PAGES,
|
||||
RAW,
|
||||
BLOCKS,
|
||||
K: tl.constexpr,
|
||||
SCORE_STRIDE: tl.constexpr,
|
||||
MAP_STRIDE: tl.constexpr,
|
||||
PAGE_STRIDE: tl.constexpr,
|
||||
RAW_STRIDE: tl.constexpr,
|
||||
BLOCK_STRIDE: tl.constexpr,
|
||||
RATIO: tl.constexpr,
|
||||
HAS_RAW: tl.constexpr,
|
||||
FILTER_MASKED: tl.constexpr,
|
||||
HAS_BLOCKS: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
col = tl.arange(0, BLOCK)
|
||||
logical = tl.load(SELECTED + row * K + col, col < K, other=-1)
|
||||
n = tl.load(LENS + row)
|
||||
valid = (logical >= 0) & (logical < n) & (col < K)
|
||||
if FILTER_MASKED:
|
||||
score = tl.load(
|
||||
SCORES + row * SCORE_STRIDE + tl.maximum(logical, 0),
|
||||
valid,
|
||||
other=-float("inf"),
|
||||
)
|
||||
valid = valid & (score > -float("inf"))
|
||||
if HAS_BLOCKS:
|
||||
block = tl.load(
|
||||
BLOCKS + row * BLOCK_STRIDE + tl.maximum(logical, 0) // BLOCK_SIZE,
|
||||
valid,
|
||||
other=-1,
|
||||
)
|
||||
logical = block * BLOCK_SIZE + logical % BLOCK_SIZE
|
||||
logical = tl.sort(tl.where(valid, logical, 0x7FFFFFFF), descending=False)
|
||||
valid = (logical != 0x7FFFFFFF) & (col < K)
|
||||
request = tl.load(REQ + row).to(tl.int64)
|
||||
position = tl.where(valid, logical, 0).to(tl.int64)
|
||||
slot = tl.load(MAPPING + request * MAP_STRIDE + position * RATIO, valid, other=0)
|
||||
tl.store(
|
||||
PAGES + row * PAGE_STRIDE + col, tl.where(valid, slot // RATIO, -1), col < K
|
||||
)
|
||||
if HAS_RAW:
|
||||
tl.store(RAW + row * RAW_STRIDE + col, tl.where(valid, logical, -1), col < K)
|
||||
|
||||
|
||||
def publish_topk(
|
||||
selected,
|
||||
scores,
|
||||
lens,
|
||||
req,
|
||||
mapping,
|
||||
pages,
|
||||
raw,
|
||||
ratio,
|
||||
filter_masked,
|
||||
*,
|
||||
candidates=None,
|
||||
):
|
||||
rows, k = selected.shape
|
||||
_publish_topk[(rows,)](
|
||||
selected,
|
||||
scores,
|
||||
lens,
|
||||
req,
|
||||
mapping,
|
||||
pages,
|
||||
raw if raw is not None else pages,
|
||||
candidates.blocks if candidates is not None else selected,
|
||||
k,
|
||||
scores.stride(0),
|
||||
mapping.stride(0),
|
||||
pages.stride(0),
|
||||
raw.stride(0) if raw is not None else 0,
|
||||
candidates.blocks.stride(0) if candidates is not None else 0,
|
||||
ratio,
|
||||
raw is not None,
|
||||
filter_masked,
|
||||
candidates is not None,
|
||||
candidates.block_size if candidates is not None else 1,
|
||||
triton.next_power_of_2(k),
|
||||
num_warps=4,
|
||||
)
|
||||
@@ -1579,6 +1579,13 @@ class Envs:
|
||||
# Run the DeepSeek-V4.1 ratio-1/2 prefill indexer on the torch path instead
|
||||
# of the DeepGEMM dense fp4 logits kernel (test oracle / fallback).
|
||||
SGLANG_DSV41_TORCH_PREFILL_INDEXER = EnvBool(False)
|
||||
# Fused mapping for SM90 32-head, ratio-1/2 static target verification.
|
||||
SGLANG_OPT_DSV41_SM90_GROUPED_INDEXER = EnvBool(False)
|
||||
# Persistent prefix scoring and length-aware TopK; requires GROUPED_INDEXER.
|
||||
# Equal-score cutoff ties can select different positions than PyTorch.
|
||||
SGLANG_OPT_DSV41_SM90_LENGTH_AWARE_INDEXER = EnvBool(False)
|
||||
# Compact candidate lists; requires the length-aware static-verify path.
|
||||
SGLANG_OPT_DSV41_SM90_COMPACT_CANDIDATES = EnvBool(False)
|
||||
SGLANG_FP8_PAGED_MQA_LOGITS_TORCH = EnvBool(False)
|
||||
SGLANG_OPT_FLASHMLA_SPARSE_PREFILL = EnvBool(True)
|
||||
|
||||
|
||||
@@ -3131,7 +3131,7 @@ class DeepseekV4AttnBackend(
|
||||
is_source = is_consumer = False
|
||||
|
||||
inputs = self._make_low_ratio_decode_indexer_inputs(
|
||||
layer, x, q_lora, req, pos, mode
|
||||
layer, x, q_lora, req, pos, forward_batch
|
||||
)
|
||||
if is_source:
|
||||
published = self.decode_candidates.publish_decode(inputs)
|
||||
@@ -3179,7 +3179,9 @@ class DeepseekV4AttnBackend(
|
||||
)
|
||||
self.full_topk_indexer.topk_prefill_captured(inputs)
|
||||
|
||||
def _make_low_ratio_decode_indexer_inputs(self, layer, x, q_lora, req, pos, mode):
|
||||
def _make_low_ratio_decode_indexer_inputs(
|
||||
self, layer, x, q_lora, req, pos, forward_batch
|
||||
):
|
||||
ratio = layer.compress_ratio
|
||||
metadata = (
|
||||
self.forward_metadata.c1_indexer_metadata
|
||||
@@ -3187,6 +3189,24 @@ class DeepseekV4AttnBackend(
|
||||
else self.forward_metadata.c2_indexer_metadata
|
||||
)
|
||||
assert metadata is not None, f"no decode indexer metadata for {ratio = }"
|
||||
group_size = 1
|
||||
if (
|
||||
envs.SGLANG_OPT_DSV41_SM90_GROUPED_INDEXER.get()
|
||||
and forward_batch.forward_mode.is_target_verify()
|
||||
and not self.is_dspark_draft
|
||||
and read_ragged_verify_mode() is RaggedVerifyMode.STATIC
|
||||
and self.forward_metadata.late_layer_tail is None
|
||||
and self.speculative_num_draft_tokens is not None
|
||||
and self.speculative_num_draft_tokens > 1
|
||||
and forward_batch.spec_info.draft_token_num
|
||||
== self.speculative_num_draft_tokens
|
||||
and req.shape[0]
|
||||
== forward_batch.batch_size * self.speculative_num_draft_tokens
|
||||
and x.shape[0] == q_lora.shape[0] == req.shape[0]
|
||||
):
|
||||
# Only static verify guarantees fixed request-major groups.
|
||||
# Do not read GPU request ids or lengths during graph capture.
|
||||
group_size = self.speculative_num_draft_tokens
|
||||
return DecodeInputs(
|
||||
indexer=layer.indexer,
|
||||
layer_id=layer.layer_id,
|
||||
@@ -3197,7 +3217,8 @@ class DeepseekV4AttnBackend(
|
||||
positions=pos,
|
||||
req_rows=req,
|
||||
paged_metadata=metadata,
|
||||
is_verify=mode.is_target_verify(),
|
||||
is_verify=forward_batch.forward_mode.is_target_verify(),
|
||||
group_size=group_size,
|
||||
out_page_indices=self.forward_metadata.core_metadata.sparse_page_indices(
|
||||
ratio
|
||||
),
|
||||
|
||||
@@ -3,7 +3,7 @@ its best blocks, a consumer takes its top-k among them."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, List, Optional, Tuple
|
||||
from typing import TYPE_CHECKING, List, Optional, Tuple, Union
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
@@ -27,6 +27,7 @@ from .scoring import (
|
||||
write_decode,
|
||||
write_prefill,
|
||||
)
|
||||
from .sm90_decode import CandidateBlocks, try_sm90_decode
|
||||
from .types import (
|
||||
CandidateMetadata,
|
||||
DecodeInputs,
|
||||
@@ -93,6 +94,14 @@ class DenseBlocksBackend:
|
||||
self._torch_consume_prefill(inputs, published)
|
||||
|
||||
def publish_decode(self, inputs: DecodeInputs):
|
||||
handled, published = try_sm90_decode(
|
||||
inputs=inputs,
|
||||
token_to_kv_pool=self.token_to_kv_pool,
|
||||
req_to_token=self.req_to_token,
|
||||
is_source=True,
|
||||
)
|
||||
if handled:
|
||||
return published
|
||||
d = decode_scores(
|
||||
inputs=inputs,
|
||||
token_to_kv_pool=self.token_to_kv_pool,
|
||||
@@ -128,8 +137,17 @@ class DenseBlocksBackend:
|
||||
def consume_decode(
|
||||
self,
|
||||
inputs: DecodeInputs,
|
||||
published: Optional[BlockIds],
|
||||
published: Optional[Union[BlockIds, CandidateBlocks]],
|
||||
) -> None:
|
||||
handled, _ = try_sm90_decode(
|
||||
inputs=inputs,
|
||||
token_to_kv_pool=self.token_to_kv_pool,
|
||||
req_to_token=self.req_to_token,
|
||||
is_consumer=True,
|
||||
published=published,
|
||||
)
|
||||
if handled:
|
||||
return
|
||||
d = decode_scores(
|
||||
inputs=inputs,
|
||||
token_to_kv_pool=self.token_to_kv_pool,
|
||||
|
||||
@@ -28,6 +28,7 @@ from .scoring import (
|
||||
select_decode,
|
||||
write_prefill,
|
||||
)
|
||||
from .sm90_decode import try_sm90_decode
|
||||
from .types import (
|
||||
CapturedPrefillInputs,
|
||||
DecodeInputs,
|
||||
@@ -134,6 +135,13 @@ class FullTopKIndexer:
|
||||
)
|
||||
|
||||
def _torch_decode(self, inputs: DecodeInputs) -> None:
|
||||
handled, _ = try_sm90_decode(
|
||||
inputs=inputs,
|
||||
token_to_kv_pool=self.token_to_kv_pool,
|
||||
req_to_token=self.req_to_token,
|
||||
)
|
||||
if handled:
|
||||
return
|
||||
d = decode_scores(
|
||||
inputs=inputs,
|
||||
token_to_kv_pool=self.token_to_kv_pool,
|
||||
|
||||
@@ -0,0 +1,256 @@
|
||||
"""Opt-in SM90 static-verification selection for the V4.1 decode backends."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
|
||||
from .types import CandidateMetadata, DecodeInputs
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
|
||||
|
||||
|
||||
@dataclass
|
||||
class CandidateBlocks(CandidateMetadata):
|
||||
"""Sorted block IDs with device lengths refreshed on each graph replay.
|
||||
|
||||
Ordinary decode consumers can use ``blocks`` directly. Length-aware
|
||||
consumers also use the device lengths; never cache a derived full mask.
|
||||
"""
|
||||
|
||||
blocks: torch.Tensor # [rows, topk_blocks] int32, -1 padding
|
||||
lengths: torch.Tensor # [rows] int32, selected capacity in positions
|
||||
is_prefix: torch.Tensor # [rows] int32, blocks are consecutive from zero
|
||||
width: int # source capacity in compressed positions
|
||||
block_size: int
|
||||
|
||||
|
||||
def _candidate_mask(published, width: int, block_size: int) -> torch.Tensor:
|
||||
"""Adapt a standard block-ID publish to prefix scoring, without caching."""
|
||||
if isinstance(published, CandidateBlocks):
|
||||
from sglang.kernels.ops.attention.dsv4.sm90_length_aware_indexer import (
|
||||
materialize_candidate_mask,
|
||||
)
|
||||
|
||||
assert published.width >= width and published.block_size == block_size
|
||||
return materialize_candidate_mask(published)
|
||||
blocks = published.blocks
|
||||
num_blocks = (width + block_size - 1) // block_size
|
||||
keep = torch.zeros(
|
||||
(*blocks.shape[:-1], num_blocks + 1), dtype=torch.bool, device=blocks.device
|
||||
)
|
||||
ids = blocks.to(torch.int64)
|
||||
ids = ids.masked_fill((ids < 0) | (ids >= num_blocks), num_blocks)
|
||||
keep.scatter_(-1, ids, True)
|
||||
return keep[..., :num_blocks].repeat_interleave(block_size, dim=-1)[..., :width]
|
||||
|
||||
|
||||
def try_sm90_decode(
|
||||
*,
|
||||
inputs: DecodeInputs,
|
||||
token_to_kv_pool: DeepSeekV4TokenToKVPool,
|
||||
req_to_token: torch.Tensor,
|
||||
is_source: bool = False,
|
||||
is_consumer: bool = False,
|
||||
published: Optional[CandidateMetadata] = None,
|
||||
) -> Tuple[bool, Optional[CandidateMetadata]]:
|
||||
"""Return whether selection was handled, plus a source's published blocks.
|
||||
|
||||
Candidate roles come from the caller: graph variants may bypass filtering
|
||||
even for a layer whose indexer normally publishes or consumes candidates.
|
||||
"""
|
||||
ratio = inputs.compress_ratio
|
||||
if not (
|
||||
envs.SGLANG_OPT_DSV41_SM90_GROUPED_INDEXER.get()
|
||||
and inputs.is_verify
|
||||
and inputs.group_size > 1
|
||||
and ratio in (1, 2)
|
||||
):
|
||||
return False, None
|
||||
indexer = inputs.indexer
|
||||
req, pos = inputs.req_rows, inputs.positions
|
||||
bs = req.shape[0]
|
||||
assert pos.shape[0] == bs, (
|
||||
f"decode expects one token per request, {pos.shape=} {bs=}"
|
||||
)
|
||||
# Metadata reserves graph replay capacity; visibility remains on device.
|
||||
lmax = min(
|
||||
inputs.paged_metadata.max_compressed_seq_len, req_to_token.shape[1] // ratio
|
||||
)
|
||||
if bs == 0 or lmax == 0:
|
||||
inputs.reset_outputs()
|
||||
return True, None
|
||||
q = indexer.queries(inputs.q_lora, inputs.freqs_cis[pos])
|
||||
weights = indexer.head_weights(inputs.x)
|
||||
table = token_to_kv_pool.get_index_k_with_scale_buffer(inputs.layer_id)
|
||||
if not (
|
||||
q.is_cuda
|
||||
and torch.version.cuda is not None
|
||||
and torch.cuda.get_device_capability(q.device)[0] == 9
|
||||
and q.dtype == weights.dtype == torch.bfloat16
|
||||
and q.shape[1:] == (32, 128)
|
||||
and q.is_contiguous()
|
||||
and weights.is_contiguous()
|
||||
and req_to_token.dtype == torch.int32
|
||||
and req_to_token.stride(1) == 1
|
||||
and table.dtype == torch.uint8
|
||||
and table.dim() == 2
|
||||
and table.stride(1) == 1
|
||||
):
|
||||
return False, None
|
||||
|
||||
from sglang.kernels.ops.attention.dsv4.candidate_blocks import (
|
||||
select_candidate_block_ids,
|
||||
topk_among_blocks,
|
||||
)
|
||||
|
||||
# Imported only after dispatch to avoid a module-initialization cycle.
|
||||
from .dense_blocks import BlockIds
|
||||
|
||||
inputs.reset_outputs()
|
||||
if is_consumer:
|
||||
assert published is not None and published.blocks.shape[0] == bs
|
||||
use_length_aware = (
|
||||
envs.SGLANG_OPT_DSV41_SM90_LENGTH_AWARE_INDEXER.get()
|
||||
and 0 < indexer.index_topk <= 2048
|
||||
and (
|
||||
not is_source
|
||||
or (
|
||||
0 < indexer.candidate_topk_blocks <= 2048
|
||||
and indexer.candidate_block_size in (1, 2, 4, 8, 16, 32, 64, 128)
|
||||
)
|
||||
)
|
||||
)
|
||||
if use_length_aware:
|
||||
from sglang.kernels.ops.attention.dsv4.sm90_length_aware_indexer import (
|
||||
candidate_blocks,
|
||||
prefix_logits,
|
||||
prepare_candidate_lengths,
|
||||
publish_topk,
|
||||
select_prefix_topk,
|
||||
)
|
||||
|
||||
request = req.to(torch.int64).contiguous()
|
||||
compact = envs.SGLANG_OPT_DSV41_SM90_COMPACT_CANDIDATES.get() and (
|
||||
is_source or is_consumer
|
||||
)
|
||||
candidates = (
|
||||
published
|
||||
if compact and is_consumer and isinstance(published, CandidateBlocks)
|
||||
else None
|
||||
)
|
||||
if compact:
|
||||
visible, score_lens = prepare_candidate_lengths(
|
||||
pos, ratio, lmax, candidates
|
||||
)
|
||||
else:
|
||||
visible = ((pos + 1) // ratio).clamp(0, lmax).to(torch.int32).contiguous()
|
||||
score_lens = visible
|
||||
consume = None
|
||||
if is_consumer and candidates is None:
|
||||
consume = _candidate_mask(published, lmax, indexer.candidate_block_size)
|
||||
assert consume.shape[0] == bs and consume.shape[1] >= lmax
|
||||
assert consume.stride(1) == 1
|
||||
score_width = lmax
|
||||
if candidates is not None:
|
||||
assert candidates.width >= lmax
|
||||
assert candidates.block_size == indexer.candidate_block_size
|
||||
score_width = min(lmax, candidates.blocks.shape[1] * candidates.block_size)
|
||||
scores = prefix_logits(
|
||||
q,
|
||||
weights,
|
||||
req_to_token,
|
||||
request,
|
||||
score_lens,
|
||||
table,
|
||||
table.shape[1] // 68,
|
||||
ratio,
|
||||
score_width,
|
||||
consume,
|
||||
candidates=candidates,
|
||||
visible=visible,
|
||||
)
|
||||
result = None
|
||||
if is_source:
|
||||
result = candidate_blocks(
|
||||
scores,
|
||||
visible,
|
||||
lmax,
|
||||
indexer.candidate_topk_blocks,
|
||||
indexer.candidate_block_size,
|
||||
)
|
||||
if not compact:
|
||||
# Main's decode protocol carries block IDs even without the
|
||||
# compact scoring flag. Drop only the optional device lengths.
|
||||
result = BlockIds(blocks=result.blocks)
|
||||
idx = select_prefix_topk(
|
||||
scores, score_lens, min(indexer.index_topk, score_width)
|
||||
)
|
||||
publish_topk(
|
||||
idx,
|
||||
scores,
|
||||
score_lens,
|
||||
request,
|
||||
req_to_token,
|
||||
inputs.out_page_indices,
|
||||
None,
|
||||
ratio,
|
||||
is_consumer,
|
||||
candidates=candidates,
|
||||
)
|
||||
return True, result
|
||||
|
||||
from sglang.kernels.ops.attention.dsv4.sm90_fp4_indexer import (
|
||||
fp4_index_logits_mapped_sm90,
|
||||
)
|
||||
|
||||
lens = (pos + 1) // ratio
|
||||
scores = fp4_index_logits_mapped_sm90(
|
||||
q,
|
||||
weights,
|
||||
req_to_token,
|
||||
req.to(torch.int64).contiguous(),
|
||||
lens.to(torch.int64).contiguous(),
|
||||
table,
|
||||
table.shape[1] // 68,
|
||||
ratio,
|
||||
lmax,
|
||||
)
|
||||
result = None
|
||||
if is_source:
|
||||
result = BlockIds(
|
||||
blocks=select_candidate_block_ids(
|
||||
scores,
|
||||
lens[:, None],
|
||||
indexer.candidate_topk_blocks,
|
||||
indexer.candidate_block_size,
|
||||
)
|
||||
)
|
||||
k = min(indexer.index_topk, lmax)
|
||||
if is_consumer:
|
||||
idx = topk_among_blocks(
|
||||
scores, lens, published.blocks, k, block_size=indexer.candidate_block_size
|
||||
)
|
||||
idx = idx.masked_fill(idx < 0, lmax)
|
||||
else:
|
||||
idx = scores.topk(k, dim=-1, sorted=False).indices
|
||||
if 0 < k <= 1024:
|
||||
from sglang.kernels.ops.attention.dsv4.sm90_fp4_topk import sort_map_topk
|
||||
|
||||
sort_map_topk(
|
||||
idx, lens, req, req_to_token, inputs.out_page_indices, None, ratio
|
||||
)
|
||||
else:
|
||||
idx = idx.sort(dim=-1).values
|
||||
reach = idx < lens[:, None]
|
||||
slots = (
|
||||
req_to_token[req[:, None], idx.clamp_max(lmax - 1) * ratio].to(torch.int64)
|
||||
// ratio
|
||||
)
|
||||
inputs.out_page_indices[:bs, :k] = torch.where(reach, slots, -1).to(torch.int32)
|
||||
return True, result
|
||||
@@ -81,6 +81,8 @@ class DecodeInputs(msgspec.Struct, frozen=True, kw_only=True):
|
||||
req_rows: torch.Tensor # [rows] req_to_token row of each query row
|
||||
paged_metadata: PagedIndexerMetadata
|
||||
is_verify: bool
|
||||
# Fixed request-major groups, validated by the attention backend for static verify.
|
||||
group_size: int = 1
|
||||
|
||||
# [rows, topk] int32 index-K pool slots of the selection, unordered, -1 padded.
|
||||
out_page_indices: torch.Tensor
|
||||
|
||||
Reference in New Issue
Block a user