[QSA] Bound the decode indexer's work by the context length (#43463)

Co-authored-by: luoyuan.luo <luoyuan.luo@antgroup.com>
This commit is contained in:
Yuan Luo
2026-10-12 22:47:45 +08:00
committed by GitHub
co-authored by luoyuan.luo
parent 1a553927b0
commit 23a1f53fc3
6 changed files with 276 additions and 52 deletions
@@ -33,6 +33,7 @@ def _qsa_fused_kv_prepare(
RATIO: tl.constexpr,
IB: tl.constexpr,
CHAIN: tl.constexpr,
PAGE: tl.constexpr,
):
tiles = tl.cdiv(S, B) + 2
pid = tl.program_id(0)
@@ -48,10 +49,18 @@ def _qsa_fused_kv_prepare(
c = tile * B + tl.arange(0, B)
if EXPAND:
n = tl.minimum((tl.load(LENGTH + row) // RATIO), IB) * RATIO
blocks = tl.load(IDX + row * IB + c // RATIO, c < IB * RATIO, -1)
expanded = blocks * RATIO + c % RATIO
qpos = tl.load(QPOS + row)
tail_start = (qpos + 1) // RATIO * RATIO
if PAGE > 0:
# Same count as the COUNTS tile below; the paged consumer never
# reads past the page that holds it.
tail_count = tl.maximum(
0, tl.minimum(qpos + 1, tl.load(LENGTH + row)) - tail_start
)
if tile * B >= (n + tail_count + PAGE - 1) // PAGE * PAGE:
return
blocks = tl.load(IDX + row * IB + c // RATIO, c < IB * RATIO, -1)
expanded = blocks * RATIO + c % RATIO
tail_off = c - n
tail = tail_start + tail_off
pos = tl.where(
@@ -136,7 +145,12 @@ def fused_kv_prepare(
compress_ratio=1,
query_positions=None,
chain_positions=False,
visible_page_size=0,
):
"""With block indices (compress_ratio > 1) and ``visible_page_size`` set, a
row's packed K/V is written only up to its valid count rounded up to that
page: a paged consumer bounded by ``counts`` never reads further, and the
rest of the row keeps whatever the buffer held."""
rows, input_topk = indices.shape
topk = input_topk * compress_ratio + compress_ratio - 1
if rows == 0:
@@ -174,5 +188,6 @@ def fused_kv_prepare(
compress_ratio,
input_topk,
chain_positions,
visible_page_size if compress_ratio > 1 else 0,
num_warps=4,
)
+93 -47
View File
@@ -1,6 +1,7 @@
"""Weight-free TileLang MQA operators for the simple QSA indexer;
the torch implementations are the fallback and the reference."""
import functools
import math
from typing import Optional
@@ -225,8 +226,8 @@ if HAS_TILELANG:
heads: int,
head_dim: int,
page_size: int = 64,
groups_per_cta: int = 1,
num_stages: int = 3,
ctas_per_row: int = 1,
min_scored_len: int = 0,
threads: int = 128,
dtype: str = "bfloat16",
):
@@ -249,49 +250,54 @@ if HAS_TILELANG:
Logits: T.Tensor([batch, max_model_len], T.float32), # type: ignore
Scale: T.float32,
):
with T.Kernel(
batch,
T.ceildiv(T.ceildiv(max_pages, sub_pages), groups_per_cta),
threads=threads,
) as (bx, group_block):
# A fixed number of CTAs per row strides over that row's groups, so the
# grid no longer scales with the page-table width (the max context).
with T.Kernel(batch, ctas_per_row, threads=threads) as (bx, cta):
q_shared = T.alloc_shared([heads, head_dim], dtype)
k_shared = T.alloc_shared([GROUP, head_dim], dtype)
scores = T.alloc_fragment([GROUP, heads], T.float32)
reduced = T.alloc_fragment([GROUP], T.float32)
T.copy(Q[bx, 0, :, :], q_shared)
context_len = T.alloc_var(T.int32)
context_len = ContextLens[bx]
num_groups = T.ceildiv(context_len, GROUP)
for gi in T.Pipelined(groups_per_cta, num_stages=num_stages):
group = group_block * groups_per_cta + gi
if group * GROUP < context_len:
# TileLang's pipeliner rejects dynamic loops around smem copies;
# unroll at the Python level instead.
for sp in range(sub_pages):
if (group * sub_pages + sp) * page_size < context_len:
T.copy(
KCache[
PageTable[bx, group * sub_pages + sp],
:,
0,
:,
],
k_shared[sp * page_size : (sp + 1) * page_size, :],
)
T.gemm(
k_shared,
q_shared,
scores,
transpose_B=True,
clear_accum=True,
policy=T.GemmWarpPolicy.FullCol,
)
for token, head in T.Parallel(GROUP, heads):
scores[token, head] = T.max(scores[token, head], 0.0)
T.reduce_sum(scores, reduced, dim=1, clear=True)
for token in T.Parallel(GROUP):
position = group * GROUP + token
if position < context_len:
Logits[bx, position] = reduced[token] / Scale
# Rows at or below min_scored_len select every block, so their
# scores are never read.
if context_len > min_scored_len:
T.copy(Q[bx, 0, :, :], q_shared)
for step in T.serial(T.ceildiv(num_groups, ctas_per_row)):
group = step * ctas_per_row + cta
if group < num_groups:
# TileLang's pipeliner rejects dynamic loops around smem
# copies; unroll the sub-pages at the Python level.
for sp in range(sub_pages):
if (group * sub_pages + sp) * page_size < context_len:
T.copy(
KCache[
PageTable[bx, group * sub_pages + sp],
:,
0,
:,
],
k_shared[
sp * page_size : (sp + 1) * page_size, :
],
)
T.gemm(
k_shared,
q_shared,
scores,
transpose_B=True,
clear_accum=True,
policy=T.GemmWarpPolicy.FullCol,
)
for token, head in T.Parallel(GROUP, heads):
scores[token, head] = T.max(scores[token, head], 0.0)
T.reduce_sum(scores, reduced, dim=1, clear=True)
for token in T.Parallel(GROUP):
position = group * GROUP + token
if position < context_len:
Logits[bx, position] = reduced[token] / Scale
return kernel
@@ -351,6 +357,24 @@ def tilelang_qsa_mqa_prefill(
return logits
# CTAs per decode row: few enough levels to bound the TileLang specializations, and
# a grid of about four waves (at least one CTA per row).
_DECODE_CTAS_PER_ROW_LEVELS = (1, 16, 256)
@functools.lru_cache(maxsize=None)
def _num_sms(device_index: int) -> int:
return torch.cuda.get_device_properties(device_index).multi_processor_count
def _decode_ctas_per_row(rows: int, device: torch.device) -> int:
wanted = -(-4 * _num_sms(device.index or 0) // max(rows, 1))
for level in _DECODE_CTAS_PER_ROW_LEVELS:
if level >= wanted:
return level
return _DECODE_CTAS_PER_ROW_LEVELS[-1]
def tilelang_qsa_mqa_decode(
q: torch.Tensor,
k_cache: torch.Tensor,
@@ -358,8 +382,15 @@ def tilelang_qsa_mqa_decode(
context_lens: torch.Tensor,
max_model_len: int,
score_scale: Optional[float] = None,
*,
min_scored_len: int = 0,
) -> torch.Tensor:
"""Validated TileLang paged decode kernel with weights removed."""
"""Validated TileLang paged decode kernel with weights removed.
With ``min_scored_len`` > 0, rows whose context length is at most that are not
scored and the logits past each row's context length are left unset: only for
a top-k that selects every block of a row that fits k and reads nothing past
the length."""
if not HAS_TILELANG:
raise RuntimeError("TileLang is unavailable")
@@ -370,12 +401,17 @@ def tilelang_qsa_mqa_decode(
"TileLang QSA decode requires a compressed page size of "
f"8/16/32/64 (64-row GEMM sub-page packing), got {page_size}"
)
logits = torch.full(
(q.shape[0], max_model_len),
-float("inf"),
dtype=torch.float32,
device=q.device,
)
if min_scored_len > 0:
logits = torch.empty(
(q.shape[0], max_model_len), dtype=torch.float32, device=q.device
)
else:
logits = torch.full(
(q.shape[0], max_model_len),
-float("inf"),
dtype=torch.float32,
device=q.device,
)
if not q.shape[0] or not max_model_len:
return logits
# CUDA MMA accepts an eight-wide N dimension, while ROCm MFMA requires
@@ -400,6 +436,8 @@ def tilelang_qsa_mqa_decode(
heads=kernel_heads,
head_dim=head_dim,
page_size=page_size,
ctas_per_row=_decode_ctas_per_row(q.shape[0], q.device),
min_scored_len=min_scored_len,
dtype=_TILELANG_DTYPES[scoring_dtype],
)(
q_kernel.unsqueeze(1).contiguous(),
@@ -431,10 +469,18 @@ def qsa_mqa_decode(
context_lens: torch.Tensor,
max_model_len: int,
score_scale: Optional[float] = None,
*,
min_scored_len: int = 0,
) -> torch.Tensor:
if q.is_cuda and HAS_TILELANG:
return tilelang_qsa_mqa_decode(
q, k_cache, page_table, context_lens, max_model_len, score_scale
q,
k_cache,
page_table,
context_lens,
max_model_len,
score_scale,
min_scored_len=min_scored_len,
)
return torch_qsa_mqa_decode(
q, k_cache, page_table, context_lens, max_model_len, score_scale
@@ -499,14 +499,18 @@ class QSAIndexer(MultiPlatformOp):
sequence_lengths: torch.Tensor,
defer_expansion: bool = False,
) -> torch.Tensor:
# fast_topk (k = 512) writes 0..length-1 for rows that fit k without reading
# their scores, and never reads past a row's length.
use_fast_topk = q.is_cuda and self.block_topk == 512
logits = qsa_mqa_decode(
q,
compressed_cache,
compressed_page_table,
compressed_lengths,
max_model_len,
min_scored_len=self.block_topk if use_fast_topk else 0,
)
if logits.is_cuda and self.block_topk == 512:
if use_fast_topk:
# Decode rows start at zero, so compressed lengths double as row lengths;
# skip the generic zero-fill + subtract.
from sglang.kernels.ops.attention.fast_topk import fast_topk
@@ -1655,9 +1655,11 @@ class QwenSparseAttnBackend(AttentionBackend):
buffers = self._fa2_scratch.get(key)
if buffers is None or buffers[0].shape[0] < capacity:
shape = (capacity, num_kv_heads, head_dim)
# Zeroed once: a row is packed only up to its last visible page, so
# bytes past it are never written and must not be NaN garbage.
buffers = (
torch.empty(shape, dtype=dtype, device=device),
torch.empty(shape, dtype=dtype, device=device),
torch.zeros(shape, dtype=dtype, device=device),
torch.zeros(shape, dtype=dtype, device=device),
)
self._fa2_scratch[key] = buffers
return buffers[0][:capacity], buffers[1][:capacity]
@@ -1746,6 +1748,8 @@ class QwenSparseAttnBackend(AttentionBackend):
if compress_ratio > 1
else None
),
# trtllm_decode reads each row only up to valid_counts.
visible_page_size=page,
)
else:
qwen_sparse_valid_counts_triton(