From 23a1f53fc3a0e7f5fc94c4a4abd4eb9bcf9c6042 Mon Sep 17 00:00:00 2001 From: Yuan Luo Date: Mon, 12 Oct 2026 22:47:45 +0800 Subject: [PATCH] [QSA] Bound the decode indexer's work by the context length (#43463) Co-authored-by: luoyuan.luo --- .../srt/layers/attention/qsa/fused_kv.py | 19 ++- python/sglang/srt/layers/attention/qsa/mqa.py | 140 ++++++++++++------ .../srt/layers/attention/qsa/qsa_indexer.py | 6 +- .../attention/qwen_sparse_attn_backend.py | 8 +- .../qsa/test_qsa_fused_kv_prepare.py | 71 +++++++++ .../ops/attention/qsa/test_qsa_indexer.py | 84 +++++++++++ 6 files changed, 276 insertions(+), 52 deletions(-) diff --git a/python/sglang/srt/layers/attention/qsa/fused_kv.py b/python/sglang/srt/layers/attention/qsa/fused_kv.py index a4936bfd7c..8da03a2d64 100644 --- a/python/sglang/srt/layers/attention/qsa/fused_kv.py +++ b/python/sglang/srt/layers/attention/qsa/fused_kv.py @@ -33,6 +33,7 @@ def _qsa_fused_kv_prepare( RATIO: tl.constexpr, IB: tl.constexpr, CHAIN: tl.constexpr, + PAGE: tl.constexpr, ): tiles = tl.cdiv(S, B) + 2 pid = tl.program_id(0) @@ -48,10 +49,18 @@ def _qsa_fused_kv_prepare( c = tile * B + tl.arange(0, B) if EXPAND: n = tl.minimum((tl.load(LENGTH + row) // RATIO), IB) * RATIO - blocks = tl.load(IDX + row * IB + c // RATIO, c < IB * RATIO, -1) - expanded = blocks * RATIO + c % RATIO qpos = tl.load(QPOS + row) tail_start = (qpos + 1) // RATIO * RATIO + if PAGE > 0: + # Same count as the COUNTS tile below; the paged consumer never + # reads past the page that holds it. + tail_count = tl.maximum( + 0, tl.minimum(qpos + 1, tl.load(LENGTH + row)) - tail_start + ) + if tile * B >= (n + tail_count + PAGE - 1) // PAGE * PAGE: + return + blocks = tl.load(IDX + row * IB + c // RATIO, c < IB * RATIO, -1) + expanded = blocks * RATIO + c % RATIO tail_off = c - n tail = tail_start + tail_off pos = tl.where( @@ -136,7 +145,12 @@ def fused_kv_prepare( compress_ratio=1, query_positions=None, chain_positions=False, + visible_page_size=0, ): + """With block indices (compress_ratio > 1) and ``visible_page_size`` set, a + row's packed K/V is written only up to its valid count rounded up to that + page: a paged consumer bounded by ``counts`` never reads further, and the + rest of the row keeps whatever the buffer held.""" rows, input_topk = indices.shape topk = input_topk * compress_ratio + compress_ratio - 1 if rows == 0: @@ -174,5 +188,6 @@ def fused_kv_prepare( compress_ratio, input_topk, chain_positions, + visible_page_size if compress_ratio > 1 else 0, num_warps=4, ) diff --git a/python/sglang/srt/layers/attention/qsa/mqa.py b/python/sglang/srt/layers/attention/qsa/mqa.py index 529e302450..7949506945 100644 --- a/python/sglang/srt/layers/attention/qsa/mqa.py +++ b/python/sglang/srt/layers/attention/qsa/mqa.py @@ -1,6 +1,7 @@ """Weight-free TileLang MQA operators for the simple QSA indexer; the torch implementations are the fallback and the reference.""" +import functools import math from typing import Optional @@ -225,8 +226,8 @@ if HAS_TILELANG: heads: int, head_dim: int, page_size: int = 64, - groups_per_cta: int = 1, - num_stages: int = 3, + ctas_per_row: int = 1, + min_scored_len: int = 0, threads: int = 128, dtype: str = "bfloat16", ): @@ -249,49 +250,54 @@ if HAS_TILELANG: Logits: T.Tensor([batch, max_model_len], T.float32), # type: ignore Scale: T.float32, ): - with T.Kernel( - batch, - T.ceildiv(T.ceildiv(max_pages, sub_pages), groups_per_cta), - threads=threads, - ) as (bx, group_block): + # A fixed number of CTAs per row strides over that row's groups, so the + # grid no longer scales with the page-table width (the max context). + with T.Kernel(batch, ctas_per_row, threads=threads) as (bx, cta): q_shared = T.alloc_shared([heads, head_dim], dtype) k_shared = T.alloc_shared([GROUP, head_dim], dtype) scores = T.alloc_fragment([GROUP, heads], T.float32) reduced = T.alloc_fragment([GROUP], T.float32) - T.copy(Q[bx, 0, :, :], q_shared) + context_len = T.alloc_var(T.int32) context_len = ContextLens[bx] + num_groups = T.ceildiv(context_len, GROUP) - for gi in T.Pipelined(groups_per_cta, num_stages=num_stages): - group = group_block * groups_per_cta + gi - if group * GROUP < context_len: - # TileLang's pipeliner rejects dynamic loops around smem copies; - # unroll at the Python level instead. - for sp in range(sub_pages): - if (group * sub_pages + sp) * page_size < context_len: - T.copy( - KCache[ - PageTable[bx, group * sub_pages + sp], - :, - 0, - :, - ], - k_shared[sp * page_size : (sp + 1) * page_size, :], - ) - T.gemm( - k_shared, - q_shared, - scores, - transpose_B=True, - clear_accum=True, - policy=T.GemmWarpPolicy.FullCol, - ) - for token, head in T.Parallel(GROUP, heads): - scores[token, head] = T.max(scores[token, head], 0.0) - T.reduce_sum(scores, reduced, dim=1, clear=True) - for token in T.Parallel(GROUP): - position = group * GROUP + token - if position < context_len: - Logits[bx, position] = reduced[token] / Scale + # Rows at or below min_scored_len select every block, so their + # scores are never read. + if context_len > min_scored_len: + T.copy(Q[bx, 0, :, :], q_shared) + for step in T.serial(T.ceildiv(num_groups, ctas_per_row)): + group = step * ctas_per_row + cta + if group < num_groups: + # TileLang's pipeliner rejects dynamic loops around smem + # copies; unroll the sub-pages at the Python level. + for sp in range(sub_pages): + if (group * sub_pages + sp) * page_size < context_len: + T.copy( + KCache[ + PageTable[bx, group * sub_pages + sp], + :, + 0, + :, + ], + k_shared[ + sp * page_size : (sp + 1) * page_size, : + ], + ) + T.gemm( + k_shared, + q_shared, + scores, + transpose_B=True, + clear_accum=True, + policy=T.GemmWarpPolicy.FullCol, + ) + for token, head in T.Parallel(GROUP, heads): + scores[token, head] = T.max(scores[token, head], 0.0) + T.reduce_sum(scores, reduced, dim=1, clear=True) + for token in T.Parallel(GROUP): + position = group * GROUP + token + if position < context_len: + Logits[bx, position] = reduced[token] / Scale return kernel @@ -351,6 +357,24 @@ def tilelang_qsa_mqa_prefill( return logits +# CTAs per decode row: few enough levels to bound the TileLang specializations, and +# a grid of about four waves (at least one CTA per row). +_DECODE_CTAS_PER_ROW_LEVELS = (1, 16, 256) + + +@functools.lru_cache(maxsize=None) +def _num_sms(device_index: int) -> int: + return torch.cuda.get_device_properties(device_index).multi_processor_count + + +def _decode_ctas_per_row(rows: int, device: torch.device) -> int: + wanted = -(-4 * _num_sms(device.index or 0) // max(rows, 1)) + for level in _DECODE_CTAS_PER_ROW_LEVELS: + if level >= wanted: + return level + return _DECODE_CTAS_PER_ROW_LEVELS[-1] + + def tilelang_qsa_mqa_decode( q: torch.Tensor, k_cache: torch.Tensor, @@ -358,8 +382,15 @@ def tilelang_qsa_mqa_decode( context_lens: torch.Tensor, max_model_len: int, score_scale: Optional[float] = None, + *, + min_scored_len: int = 0, ) -> torch.Tensor: - """Validated TileLang paged decode kernel with weights removed.""" + """Validated TileLang paged decode kernel with weights removed. + + With ``min_scored_len`` > 0, rows whose context length is at most that are not + scored and the logits past each row's context length are left unset: only for + a top-k that selects every block of a row that fits k and reads nothing past + the length.""" if not HAS_TILELANG: raise RuntimeError("TileLang is unavailable") @@ -370,12 +401,17 @@ def tilelang_qsa_mqa_decode( "TileLang QSA decode requires a compressed page size of " f"8/16/32/64 (64-row GEMM sub-page packing), got {page_size}" ) - logits = torch.full( - (q.shape[0], max_model_len), - -float("inf"), - dtype=torch.float32, - device=q.device, - ) + if min_scored_len > 0: + logits = torch.empty( + (q.shape[0], max_model_len), dtype=torch.float32, device=q.device + ) + else: + logits = torch.full( + (q.shape[0], max_model_len), + -float("inf"), + dtype=torch.float32, + device=q.device, + ) if not q.shape[0] or not max_model_len: return logits # CUDA MMA accepts an eight-wide N dimension, while ROCm MFMA requires @@ -400,6 +436,8 @@ def tilelang_qsa_mqa_decode( heads=kernel_heads, head_dim=head_dim, page_size=page_size, + ctas_per_row=_decode_ctas_per_row(q.shape[0], q.device), + min_scored_len=min_scored_len, dtype=_TILELANG_DTYPES[scoring_dtype], )( q_kernel.unsqueeze(1).contiguous(), @@ -431,10 +469,18 @@ def qsa_mqa_decode( context_lens: torch.Tensor, max_model_len: int, score_scale: Optional[float] = None, + *, + min_scored_len: int = 0, ) -> torch.Tensor: if q.is_cuda and HAS_TILELANG: return tilelang_qsa_mqa_decode( - q, k_cache, page_table, context_lens, max_model_len, score_scale + q, + k_cache, + page_table, + context_lens, + max_model_len, + score_scale, + min_scored_len=min_scored_len, ) return torch_qsa_mqa_decode( q, k_cache, page_table, context_lens, max_model_len, score_scale diff --git a/python/sglang/srt/layers/attention/qsa/qsa_indexer.py b/python/sglang/srt/layers/attention/qsa/qsa_indexer.py index 6f39595cb9..883671944c 100644 --- a/python/sglang/srt/layers/attention/qsa/qsa_indexer.py +++ b/python/sglang/srt/layers/attention/qsa/qsa_indexer.py @@ -499,14 +499,18 @@ class QSAIndexer(MultiPlatformOp): sequence_lengths: torch.Tensor, defer_expansion: bool = False, ) -> torch.Tensor: + # fast_topk (k = 512) writes 0..length-1 for rows that fit k without reading + # their scores, and never reads past a row's length. + use_fast_topk = q.is_cuda and self.block_topk == 512 logits = qsa_mqa_decode( q, compressed_cache, compressed_page_table, compressed_lengths, max_model_len, + min_scored_len=self.block_topk if use_fast_topk else 0, ) - if logits.is_cuda and self.block_topk == 512: + if use_fast_topk: # Decode rows start at zero, so compressed lengths double as row lengths; # skip the generic zero-fill + subtract. from sglang.kernels.ops.attention.fast_topk import fast_topk diff --git a/python/sglang/srt/layers/attention/qwen_sparse_attn_backend.py b/python/sglang/srt/layers/attention/qwen_sparse_attn_backend.py index 6eabce2797..31db21423f 100644 --- a/python/sglang/srt/layers/attention/qwen_sparse_attn_backend.py +++ b/python/sglang/srt/layers/attention/qwen_sparse_attn_backend.py @@ -1655,9 +1655,11 @@ class QwenSparseAttnBackend(AttentionBackend): buffers = self._fa2_scratch.get(key) if buffers is None or buffers[0].shape[0] < capacity: shape = (capacity, num_kv_heads, head_dim) + # Zeroed once: a row is packed only up to its last visible page, so + # bytes past it are never written and must not be NaN garbage. buffers = ( - torch.empty(shape, dtype=dtype, device=device), - torch.empty(shape, dtype=dtype, device=device), + torch.zeros(shape, dtype=dtype, device=device), + torch.zeros(shape, dtype=dtype, device=device), ) self._fa2_scratch[key] = buffers return buffers[0][:capacity], buffers[1][:capacity] @@ -1746,6 +1748,8 @@ class QwenSparseAttnBackend(AttentionBackend): if compress_ratio > 1 else None ), + # trtllm_decode reads each row only up to valid_counts. + visible_page_size=page, ) else: qwen_sparse_valid_counts_triton( diff --git a/test/registered/kernels/ops/attention/qsa/test_qsa_fused_kv_prepare.py b/test/registered/kernels/ops/attention/qsa/test_qsa_fused_kv_prepare.py index 6ae2cf3947..bc08dc7595 100644 --- a/test/registered/kernels/ops/attention/qsa/test_qsa_fused_kv_prepare.py +++ b/test/registered/kernels/ops/attention/qsa/test_qsa_fused_kv_prepare.py @@ -236,6 +236,77 @@ def test_fused_block_expansion(requests, width, dtype, padding, chain_positions) torch.testing.assert_close(actual.float(), ref.float(), rtol=0, atol=0) +@pytest.mark.parametrize("width", [1, 4]) +def test_fused_block_expansion_writes_only_visible_pages(width): + """With visible_page_size, a row's packed K/V matches the full write up to its + valid count rounded up to the page (counts at, below and past page edges), + and nothing after that page is written.""" + torch.manual_seed(7) + page, stride = 64, 2112 + bases = [1, 2, 3, 4, 5, 63, 64, 65, 127, 128, 129, 2047, 2048, 2049, 2052, 8192] + requests = len(bases) + rows, length = requests * width, 9216 + mapping = (torch.randperm(requests * length, device="cuda") + 1).view( + requests, length + ) + req = torch.arange(requests, device="cuda").repeat_interleave(width).int() + lens = torch.tensor( + [bases[r // width] + r % width for r in range(rows)], + device="cuda", + dtype=torch.int32, + ) + positions = lens - 1 + loc = mapping[req.long(), positions.long()].contiguous() + blocks = torch.full((rows, 512), -1, device="cuda", dtype=torch.int32) + for row in range(rows): + n = int(lens[row]) // 4 + chosen = torch.randperm(n, device="cuda")[:512] + blocks[row, : chosen.numel()] = chosen.int() + k = torch.randn(requests * length + 1, 1, 256, device="cuda", dtype=torch.bfloat16) + v = torch.randn_like(k) + nk = torch.randn(rows, 1, 256, device="cuda", dtype=torch.bfloat16) + nv = torch.randn_like(nk) + + def run(visible_page_size, fill): + kc, vc = k.clone(), v.clone() + counts = torch.empty(rows, device="cuda", dtype=torch.int32) + ok = torch.full( + (rows * stride, 1, 256), fill, device="cuda", dtype=torch.bfloat16 + ) + ov = torch.full_like(ok, fill) + fused_kv_prepare( + kc, + vc, + nk, + nv, + loc, + mapping, + req, + blocks, + lens, + counts, + ok, + ov, + width, + compress_ratio=4, + query_positions=positions, + chain_positions=True, + visible_page_size=visible_page_size, + ) + return kc, vc, counts, ok.view(rows, stride, -1), ov.view(rows, stride, -1) + + full = run(0, 0.0) + paged = run(page, float("nan")) + for actual, ref in zip(paged[:3], full[:3]): + assert torch.equal(actual, ref) + counts = full[2] + for row in range(rows): + visible = -(-int(counts[row]) // page) * page + for actual, ref in zip(paged[3:], full[3:]): + assert torch.equal(actual[row, :visible], ref[row, :visible]) + assert torch.isnan(actual[row, visible:]).all() + + def test_block_indices_expand_on_attention_fallback(monkeypatch): from types import SimpleNamespace diff --git a/test/registered/kernels/ops/attention/qsa/test_qsa_indexer.py b/test/registered/kernels/ops/attention/qsa/test_qsa_indexer.py index cff24ef01e..8733d00146 100644 --- a/test/registered/kernels/ops/attention/qsa/test_qsa_indexer.py +++ b/test/registered/kernels/ops/attention/qsa/test_qsa_indexer.py @@ -319,6 +319,90 @@ def test_tilelang_mqa_rejects_mixed_operand_dtypes(): tilelang_qsa_mqa_decode(q, cache, page_table, context_lens, 64) +def _decode_operands(rows, max_pages, page_size, context_lens, device): + q = torch.randn(rows, NUM_Q_HEADS, HEAD_DIM, device=device).to(torch.bfloat16) + cache = torch.randn(1024, page_size, 1, HEAD_DIM, device=device).to(torch.bfloat16) + page_table = torch.randint( + 0, 1024, (rows, max_pages), device=device, dtype=torch.int32 + ) + return q, cache, page_table, context_lens.to(device=device, dtype=torch.int32) + + +@pytest.mark.parametrize("rows", [6, 100, 700]) +def test_tilelang_mqa_decode_grid_covers_every_group(rows): + """The decode kernel strides 256, 16 or 1 CTAs per row (by row count) over the + row's 64-token groups. Every group of every row must be scored, and nothing + is written past a row's context length.""" + from sglang.srt.layers.attention.qsa.mqa import ( + HAS_TILELANG, + tilelang_qsa_mqa_decode, + torch_qsa_mqa_decode, + ) + + if not HAS_TILELANG: + pytest.skip("tilelang unavailable") + device = torch.device("cuda") + torch.manual_seed(rows) + max_pages, page_size = 128, 16 + max_model_len = max_pages * page_size + lens = torch.randint(1, max_model_len + 1, (rows,)) + lens[0] = max_model_len + q, cache, page_table, lens = _decode_operands( + rows, max_pages, page_size, lens, device + ) + ref = torch_qsa_mqa_decode(q, cache, page_table, lens, max_model_len) + out = tilelang_qsa_mqa_decode(q, cache, page_table, lens, max_model_len) + finite = torch.isfinite(ref) + assert torch.equal(finite, torch.isfinite(out)) + torch.testing.assert_close(out[finite], ref[finite], rtol=2e-3, atol=2e-3) + + +def test_tilelang_mqa_decode_skips_rows_that_fit_topk(): + """With min_scored_len = k the kernel leaves rows that fit k unscored and + nothing past a row's length is set; fast_topk(k) must still select exactly + what it selects on fully scored logits (every block of a row that fits k).""" + from sglang.kernels.ops.attention.fast_topk import fast_topk + from sglang.srt.layers.attention.qsa.mqa import ( + HAS_TILELANG, + tilelang_qsa_mqa_decode, + torch_qsa_mqa_decode, + ) + + if not HAS_TILELANG: + pytest.skip("tilelang unavailable") + device = torch.device("cuda") + torch.manual_seed(5) + max_pages, page_size, block_topk = 128, 16, 512 + max_model_len = max_pages * page_size + lens = torch.tensor([1, 300, 511, 512, 513, 1000, 2047, 2048] * 4) + rows = lens.numel() + q, cache, page_table, lens = _decode_operands( + rows, max_pages, page_size, lens, device + ) + ref = torch_qsa_mqa_decode(q, cache, page_table, lens, max_model_len) + # The caching allocator hands the NaN block to the unset logits, so a top-k + # that read an unscored row or a column past a length would show it. + poison = torch.full((rows, max_model_len), float("nan"), device=device) + del poison + out = tilelang_qsa_mqa_decode( + q, cache, page_table, lens, max_model_len, min_scored_len=block_topk + ) + sel_ref = fast_topk(ref, lens, topk=block_topk) + sel_out = fast_topk(out, lens, topk=block_topk) + for row in range(rows): + length = int(lens[row]) + if length <= block_topk: + assert torch.equal(sel_out[row], sel_ref[row]) + continue + torch.testing.assert_close( + out[row, :length], ref[row, :length], rtol=2e-3, atol=2e-3 + ) + for a, b in zip( + _selected_score_multisets(ref, sel_ref), _selected_score_multisets(ref, sel_out) + ): + torch.testing.assert_close(a, b, rtol=2e-3, atol=2e-3) + + @pytest.mark.parametrize("dtype", [torch.int32, torch.int64]) def test_expand_block_indices_int_inputs(dtype): device = torch.device("cuda")