From d75d5b33f71a64ef1d20ee19b0706046dbc3737f Mon Sep 17 00:00:00 2001 From: DarkSharpness <76582120+DarkSharpness@users.noreply.github.com> Date: Sat, 3 Oct 2026 18:25:22 +0800 Subject: [PATCH] [DSv4.1] Faster fp4 index-K gather and combine_topk_swa_indices (#41658) Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: BBuf <1182563586@qq.com> --- .../kernels/ops/attention/dsv4/fp4_indexer.py | 61 ++++++++++++ .../attention/dsv4/sparse_prefill_kernels.py | 99 +++++++++++-------- .../attention/dsv4/sparse_prefill_utils.py | 24 +++-- .../srt/mem_cache/deepseek_v4_memory_pool.py | 6 ++ .../test_combine_topk_swa_indices.py | 52 +++++++++- 5 files changed, 191 insertions(+), 51 deletions(-) diff --git a/python/sglang/kernels/ops/attention/dsv4/fp4_indexer.py b/python/sglang/kernels/ops/attention/dsv4/fp4_indexer.py index b304330164..213d97c2b9 100644 --- a/python/sglang/kernels/ops/attention/dsv4/fp4_indexer.py +++ b/python/sglang/kernels/ops/attention/dsv4/fp4_indexer.py @@ -461,6 +461,67 @@ def store_fp4_index_k_cache( ) +@triton.jit +def _gather_fp4_index_k_kernel( + cache, + cache_stride, + slots, + payload, + scales, + n, + page_size: tl.constexpr, + BLOCK_ROWS: tl.constexpr, +): + # int32 view of a page: page_size * 16 payload words, then page_size scale words. + rows = tl.program_id(0) * BLOCK_ROWS + tl.arange(0, BLOCK_ROWS) + valid = rows < n + slot = tl.load(slots + rows, valid, 0).to(tl.int64) + page = slot // page_size + offset = slot - page * page_size + page_base = cache + page * cache_stride + words = tl.arange(0, 16) + payload_words = tl.load( + page_base[:, None] + offset[:, None] * 16 + words[None, :], valid[:, None] + ) + tl.store( + payload + rows[:, None].to(tl.int64) * 16 + words[None, :], + payload_words, + valid[:, None], + ) + tl.store(scales + rows, tl.load(page_base + page_size * 16 + offset, valid), valid) + + +def gather_fp4_index_k( + cache: torch.Tensor, slots: torch.Tensor, *, page_size: int +) -> tuple[torch.Tensor, torch.Tensor]: + """Packed fp4 rows at ``slots`` of the index-K ``cache`` (``[pages, page_size * + 68]`` uint8, ``store_fp4_index_k_cache``'s layout): (payload int8 ``[n, 64]``, + ue8m0 scales packed int32 ``[n]``), the layout ``quantize_fp4_indexer_tensor`` + returns.""" + assert cache.shape[1] == page_size * INDEX_K_SLOT_BYTES and cache.stride(1) == 1 + # The kernel addresses slots as `slots + rows`, so a strided view would be read + # as if its elements were packed. + assert slots.is_contiguous() + words = cache.view(torch.int32) + n = slots.shape[0] + payload = torch.empty((n, 16), device=cache.device, dtype=torch.int32) + scales = torch.empty((n,), device=cache.device, dtype=torch.int32) + if n > 0: + block_rows = 64 + _gather_fp4_index_k_kernel[(triton.cdiv(n, block_rows),)]( + words, + words.stride(0), + slots, + payload, + scales, + n, + page_size, + BLOCK_ROWS=block_rows, + num_warps=4, + ) + return payload.view(torch.int8), scales + + @triton.jit def _index_k_rope_pack_kernel( X, diff --git a/python/sglang/kernels/ops/attention/dsv4/sparse_prefill_kernels.py b/python/sglang/kernels/ops/attention/dsv4/sparse_prefill_kernels.py index bd58b65374..558dc9c901 100644 --- a/python/sglang/kernels/ops/attention/dsv4/sparse_prefill_kernels.py +++ b/python/sglang/kernels/ops/attention/dsv4/sparse_prefill_kernels.py @@ -36,7 +36,7 @@ def _build_swa_token_ids_kernel( tl.store(out_ptr + out_off + i, swa_id) -@triton.jit(do_not_specialize=["top_k"]) +@triton.jit(do_not_specialize=["num_tokens", "num_reqs", "top_k"]) def _combine_topk_swa_indices_kernel( combined_indices_ptr, combined_indices_stride, @@ -49,59 +49,76 @@ def _combine_topk_swa_indices_kernel( gather_lens_ptr, compressed_base_ptr, swa_base_ptr, + num_tokens, + num_reqs, top_k, COMPRESS_RATIO: tl.constexpr, WINDOW_SIZE: tl.constexpr, - PADDED_TOP_K: tl.constexpr, + WIDTH: tl.constexpr, + PADDED_WIDTH: tl.constexpr, + BLOCK_T: tl.constexpr, + SEARCH_STEPS: tl.constexpr, ): - batch_idx = tl.program_id(0) - worker_id = tl.program_id(1) - num_workers = tl.num_programs(1) + # BLOCK_T consecutive tokens per program, so the grid scales with tokens, not requests. + token = tl.program_id(0) * BLOCK_T + tl.arange(0, BLOCK_T) + valid = token < num_tokens # query_start_loc may be a global tensor; rebase to chunk-local offsets # by subtracting the chunk's starting value. base = tl.load(query_start_loc_ptr) - query_start = tl.load(query_start_loc_ptr + batch_idx) - base - query_end = tl.load(query_start_loc_ptr + batch_idx + 1) - base - seq_len = tl.load(seq_lens_ptr + batch_idx) - gather_len = tl.load(gather_lens_ptr + batch_idx) - compressed_base = tl.load(compressed_base_ptr + batch_idx) - swa_base = tl.load(swa_base_ptr + batch_idx) + # Each token's request: the last r with query_start_loc[r] - base <= token. + lo = tl.zeros([BLOCK_T], dtype=tl.int32) + hi = tl.full([BLOCK_T], 0, dtype=tl.int32) + num_reqs + for _ in tl.static_range(SEARCH_STEPS): + mid = (lo + hi + 1) // 2 + start = tl.load(query_start_loc_ptr + mid, mask=lo < hi, other=0) - base + go_right = (lo < hi) & (start <= token) + lo = tl.where(go_right, mid, lo) + hi = tl.where((lo < hi) & ~go_right, mid - 1, hi) + batch_idx = tl.minimum(lo, num_reqs - 1) + # Rows past the last request's queries belong to no request: -1 rows, length 0. + owned = valid & (token < tl.load(query_start_loc_ptr + num_reqs) - base) + + pos = tl.load(query_pos_ptr + token, mask=owned, other=0) + seq_len = tl.load(seq_lens_ptr + batch_idx, mask=owned, other=0) + gather_len = tl.load(gather_lens_ptr + batch_idx, mask=owned, other=0) + compressed_base = tl.load(compressed_base_ptr + batch_idx, mask=owned, other=0) + swa_base = tl.load(swa_base_ptr + batch_idx, mask=owned, other=0) # SWA portion of the gathered buffer starts from position # (seq_len - gather_len), not 0. The +pos-gather_start formula maps a # query's window back into the workspace's SWA region. gather_start = seq_len - gather_len - for token_idx in range(query_start + worker_id, query_end, num_workers): - pos = tl.load(query_pos_ptr + token_idx) - # -1 entries inside the top-k span stay -1 (attention skips them). - # top_k=0 disables the compressed portion for SWA-only layers. - topk_len = tl.minimum((pos + 1) // COMPRESS_RATIO, top_k) - swa_len = tl.minimum(pos + 1, WINDOW_SIZE) + # -1 entries inside the top-k span stay -1 (attention skips them). + # top_k=0 disables the compressed portion for SWA-only layers. + topk_len = tl.where(owned, tl.minimum((pos + 1) // COMPRESS_RATIO, top_k), 0) + swa_len = tl.where(owned, tl.minimum(pos + 1, WINDOW_SIZE), 0) - combined_row = token_idx.to(tl.int64) * combined_indices_stride - topk_row = token_idx.to(tl.int64) * topk_indices_stride + combined_row = token.to(tl.int64)[:, None] * combined_indices_stride + topk_row = token.to(tl.int64)[:, None] * topk_indices_stride - offset = tl.arange(0, PADDED_TOP_K) - mask = offset < topk_len - topk_vals = tl.load( - topk_indices_ptr + topk_row + offset, - mask=mask, - ) - tl.store( - combined_indices_ptr + combined_row + offset, - tl.where(topk_vals >= 0, topk_vals + compressed_base, -1), - mask=mask, - ) + # Whole rows, -1 padded, so the output needs no fill pass: + # [top-k (-1 holes kept) | window | -1 up to WIDTH]. + offset = tl.arange(0, PADDED_WIDTH)[None, :] + in_topk = offset < topk_len[:, None] + topk_vals = tl.load( + topk_indices_ptr + topk_row + offset, mask=valid[:, None] & in_topk, other=-1 + ) + # Workspace SWA index: swa_base[r] + (gather_offset_in_buffer). + # For positions [pos - swa_len + 1, pos], the buffer offsets are + # [pos - swa_len + 1 - gather_start, pos - gather_start]. + swa_start = swa_base + pos - swa_len + 1 - gather_start + swa_vals = swa_start[:, None] + (offset - topk_len[:, None]) + in_swa = offset < (topk_len + swa_len)[:, None] + vals = tl.where( + in_topk, + tl.where(topk_vals >= 0, topk_vals + compressed_base[:, None], -1), + tl.where(in_swa, swa_vals, -1), + ) + tl.store( + combined_indices_ptr + combined_row + offset, + vals, + mask=valid[:, None] & (offset < WIDTH), + ) - offset = tl.arange(0, WINDOW_SIZE) - # Workspace SWA index: swa_base[r] + (gather_offset_in_buffer). - # For positions [pos - swa_len + 1, pos], the buffer offsets are - # [pos - swa_len + 1 - gather_start, pos - gather_start]. - tl.store( - combined_indices_ptr + combined_row + topk_len + offset, - swa_base + offset + pos - swa_len + 1 - gather_start, - mask=offset < swa_len, - ) - - tl.store(combined_lens_ptr + token_idx, topk_len + swa_len) + tl.store(combined_lens_ptr + token, topk_len + swa_len, mask=valid) diff --git a/python/sglang/srt/layers/attention/dsv4/sparse_prefill_utils.py b/python/sglang/srt/layers/attention/dsv4/sparse_prefill_utils.py index 49a8b52ddd..d7eb1c9af4 100644 --- a/python/sglang/srt/layers/attention/dsv4/sparse_prefill_utils.py +++ b/python/sglang/srt/layers/attention/dsv4/sparse_prefill_utils.py @@ -135,8 +135,8 @@ def combine_topk_swa_indices( Returns int32 indices [num_tokens, padded_topk_swa] and per-token scanned-prefix lengths, including -1 entries skipped by attention. Width is padded to 128. - Preallocated out_indices must contain -1 outside the written prefix; - reuse requires chunk-invariant scanned-prefix lengths. out_lens is overwritten. + Rows are written whole and -1 padded, so preallocated out_indices / out_lens + are overwritten entirely; rows past the last request are -1 with length 0. """ assert topk_indices.dtype == torch.int32 assert query_start_loc.dtype == torch.int32 @@ -154,9 +154,9 @@ def combine_topk_swa_indices( num_reqs = seq_lens.shape[0] combined_topk = combined_topk_width(topk, window_size) if out_indices is None: - combined_indices = torch.full( + # The kernel writes whole -1 padded rows. + combined_indices = torch.empty( (num_tokens, combined_topk), - -1, dtype=torch.int32, device=topk_indices.device, ) @@ -165,7 +165,7 @@ def combine_topk_swa_indices( assert out_indices.dtype == torch.int32 combined_indices = out_indices if out_lens is None: - combined_lens = torch.zeros( + combined_lens = torch.empty( num_tokens, dtype=torch.int32, device=topk_indices.device ) else: @@ -173,8 +173,10 @@ def combine_topk_swa_indices( assert out_lens.dtype == torch.int32 combined_lens = out_lens - NUM_WORKERS = 128 - _combine_topk_swa_indices_kernel[(num_reqs, NUM_WORKERS)]( + if num_tokens == 0: + return combined_indices, combined_lens + block_t = 2 # Measured on B200 with num_warps=2, 16384 tokens, bs 1 to 64. + _combine_topk_swa_indices_kernel[(triton.cdiv(num_tokens, block_t),)]( combined_indices, combined_indices.stride(0), combined_lens, @@ -186,10 +188,16 @@ def combine_topk_swa_indices( gather_lens, compressed_base, swa_base, + num_tokens, + num_reqs, top_k=topk, COMPRESS_RATIO=compress_ratio, WINDOW_SIZE=window_size, - PADDED_TOP_K=triton.next_power_of_2(topk_indices.shape[-1]), + WIDTH=combined_topk, + PADDED_WIDTH=triton.next_power_of_2(combined_topk), + BLOCK_T=block_t, + SEARCH_STEPS=max(num_reqs, 1).bit_length(), + num_warps=2, ) return combined_indices, combined_lens diff --git a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py index 7203761b64..cb0a66052b 100644 --- a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py +++ b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py @@ -699,6 +699,12 @@ class DeepSeekV4IndexerPool(KVCache): from the page layout [page_size * 64 payload | page_size * 4 scale].""" assert self.use_fp4_indexer, "packed readback only applies to the fp4 layout" buf = self.index_k_with_scale_buffer[layer_id - self.start_layer] + if buf.is_cuda: + from sglang.kernels.ops.attention.dsv4.fp4_indexer import ( + gather_fp4_index_k, + ) + + return gather_fp4_index_k(buf, slots, page_size=self.page_size) slots = slots.to(torch.int64) p = self.page_size page, off = (slots // p).unsqueeze(-1), slots % p diff --git a/test/registered/kernels/ops/attention/test_combine_topk_swa_indices.py b/test/registered/kernels/ops/attention/test_combine_topk_swa_indices.py index ff7fbdee15..87a743f45e 100644 --- a/test/registered/kernels/ops/attention/test_combine_topk_swa_indices.py +++ b/test/registered/kernels/ops/attention/test_combine_topk_swa_indices.py @@ -10,6 +10,9 @@ Checks the Triton kernel against a per-row torch reference: 3. ``test_non_trailing_query_positions``: absolute ``query_pos`` that are not the trailing extend tokens, with a cross-chunk ``query_start_loc`` offset. 4. ``test_swa_only_layer``: ``topk == 0`` writes only the window. +5. ``test_reused_buffers_are_overwritten_whole``: the kernel is handed + uninitialized and already-written buffers, so it must overwrite every row it + is given rather than rely on a ``-1`` fill pass. """ import pytest @@ -80,12 +83,15 @@ def _check(**kwargs): assert torch.equal(got_idx, ref_idx) -def _trailing_case(seq_lens, extend_lens, topk, compress_ratio, seed=0): - """Rows are the trailing ``extend_lens[r]`` tokens of request ``r``.""" +def _trailing_case(seq_lens, extend_lens, topk, compress_ratio, seed=0, pad_rows=0): + """Rows are the trailing ``extend_lens[r]`` tokens of request ``r``, followed by + ``pad_rows`` rows past the last request.""" gen = torch.Generator(device="cpu").manual_seed(seed) query_pos = [] for seq_len, extend_len in zip(seq_lens, extend_lens): query_pos.extend(range(seq_len - extend_len, seq_len)) + # Past the last request, so the kernel never reads these positions. + query_pos.extend([0] * pad_rows) num_tokens = len(query_pos) topk_indices = torch.randint( 0, 1 << 20, (num_tokens, topk), generator=gen, dtype=torch.int32 @@ -163,6 +169,48 @@ def test_non_trailing_query_positions(): ) +def test_reused_buffers_are_overwritten_whole(): + """Callers hand the kernel uninitialized or already-written buffers, so every + row it is given must be overwritten in full. A row that keeps part of an + earlier call's longer selection, or a row past the last request that keeps + anything at all, is a stale index attention would read as live.""" + width = -(-(64 + WINDOW) // TOPK_ALIGNMENT) * TOPK_ALIGNMENT + # Long selections first: every row fills 64 + WINDOW entries. + wide = _trailing_case([512, 640], [8, 8], topk=64, compress_ratio=4) + num_tokens = wide["topk_indices"].shape[0] + out_indices = torch.empty((num_tokens, width), dtype=torch.int32, device=DEVICE) + out_lens = torch.empty(num_tokens, dtype=torch.int32, device=DEVICE) + + # An uninitialized buffer is poison, not -1. + out_indices.fill_(1 << 30) + out_lens.fill_(1 << 30) + got_idx, got_lens = combine_topk_swa_indices( + **wide, out_indices=out_indices, out_lens=out_lens + ) + assert got_idx.data_ptr() == out_indices.data_ptr() + ref_idx, ref_lens = _reference(**wide) + assert torch.equal(got_lens, ref_lens) + assert torch.equal(got_idx, ref_idx) + assert int(got_lens[0]) == 64 + WINDOW + + # Same buffers, now holding the first call's output: shorter selections, a + # zero-length request, and rows past the last request. + narrow = _trailing_case( + [20, 36, 48], [2, 0, 3], topk=64, compress_ratio=4, pad_rows=num_tokens - 5 + ) + assert narrow["topk_indices"].shape[0] == num_tokens + got_idx, got_lens = combine_topk_swa_indices( + **narrow, out_indices=out_indices, out_lens=out_lens + ) + ref_idx, ref_lens = _reference(**narrow) + assert torch.equal(got_lens, ref_lens), (got_lens.tolist(), ref_lens.tolist()) + assert torch.equal(got_idx, ref_idx) + # The shorter rows and the padding rows must not keep the first call's tails. + assert int(got_lens[0]) < 64 + WINDOW + assert (got_lens[5:] == 0).all() + assert (got_idx[5:] == -1).all() + + def test_swa_only_layer(): case = _trailing_case([200, 50], [2, 2], topk=0, compress_ratio=4) case["topk_indices"] = torch.zeros((4, 1), dtype=torch.int32, device=DEVICE)