[DSv4.1] Faster fp4 index-K gather and combine_topk_swa_indices (#41658)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: BBuf <1182563586@qq.com>
This commit is contained in:
DarkSharpness
2026-10-03 18:25:22 +08:00
committed by GitHub
co-authored by Claude Opus 5.5 BBuf
parent 379ec90fb2
commit d75d5b33f7
5 changed files with 191 additions and 51 deletions
@@ -461,6 +461,67 @@ def store_fp4_index_k_cache(
)
@triton.jit
def _gather_fp4_index_k_kernel(
cache,
cache_stride,
slots,
payload,
scales,
n,
page_size: tl.constexpr,
BLOCK_ROWS: tl.constexpr,
):
# int32 view of a page: page_size * 16 payload words, then page_size scale words.
rows = tl.program_id(0) * BLOCK_ROWS + tl.arange(0, BLOCK_ROWS)
valid = rows < n
slot = tl.load(slots + rows, valid, 0).to(tl.int64)
page = slot // page_size
offset = slot - page * page_size
page_base = cache + page * cache_stride
words = tl.arange(0, 16)
payload_words = tl.load(
page_base[:, None] + offset[:, None] * 16 + words[None, :], valid[:, None]
)
tl.store(
payload + rows[:, None].to(tl.int64) * 16 + words[None, :],
payload_words,
valid[:, None],
)
tl.store(scales + rows, tl.load(page_base + page_size * 16 + offset, valid), valid)
def gather_fp4_index_k(
cache: torch.Tensor, slots: torch.Tensor, *, page_size: int
) -> tuple[torch.Tensor, torch.Tensor]:
"""Packed fp4 rows at ``slots`` of the index-K ``cache`` (``[pages, page_size *
68]`` uint8, ``store_fp4_index_k_cache``'s layout): (payload int8 ``[n, 64]``,
ue8m0 scales packed int32 ``[n]``), the layout ``quantize_fp4_indexer_tensor``
returns."""
assert cache.shape[1] == page_size * INDEX_K_SLOT_BYTES and cache.stride(1) == 1
# The kernel addresses slots as `slots + rows`, so a strided view would be read
# as if its elements were packed.
assert slots.is_contiguous()
words = cache.view(torch.int32)
n = slots.shape[0]
payload = torch.empty((n, 16), device=cache.device, dtype=torch.int32)
scales = torch.empty((n,), device=cache.device, dtype=torch.int32)
if n > 0:
block_rows = 64
_gather_fp4_index_k_kernel[(triton.cdiv(n, block_rows),)](
words,
words.stride(0),
slots,
payload,
scales,
n,
page_size,
BLOCK_ROWS=block_rows,
num_warps=4,
)
return payload.view(torch.int8), scales
@triton.jit
def _index_k_rope_pack_kernel(
X,
@@ -36,7 +36,7 @@ def _build_swa_token_ids_kernel(
tl.store(out_ptr + out_off + i, swa_id)
@triton.jit(do_not_specialize=["top_k"])
@triton.jit(do_not_specialize=["num_tokens", "num_reqs", "top_k"])
def _combine_topk_swa_indices_kernel(
combined_indices_ptr,
combined_indices_stride,
@@ -49,59 +49,76 @@ def _combine_topk_swa_indices_kernel(
gather_lens_ptr,
compressed_base_ptr,
swa_base_ptr,
num_tokens,
num_reqs,
top_k,
COMPRESS_RATIO: tl.constexpr,
WINDOW_SIZE: tl.constexpr,
PADDED_TOP_K: tl.constexpr,
WIDTH: tl.constexpr,
PADDED_WIDTH: tl.constexpr,
BLOCK_T: tl.constexpr,
SEARCH_STEPS: tl.constexpr,
):
batch_idx = tl.program_id(0)
worker_id = tl.program_id(1)
num_workers = tl.num_programs(1)
# BLOCK_T consecutive tokens per program, so the grid scales with tokens, not requests.
token = tl.program_id(0) * BLOCK_T + tl.arange(0, BLOCK_T)
valid = token < num_tokens
# query_start_loc may be a global tensor; rebase to chunk-local offsets
# by subtracting the chunk's starting value.
base = tl.load(query_start_loc_ptr)
query_start = tl.load(query_start_loc_ptr + batch_idx) - base
query_end = tl.load(query_start_loc_ptr + batch_idx + 1) - base
seq_len = tl.load(seq_lens_ptr + batch_idx)
gather_len = tl.load(gather_lens_ptr + batch_idx)
compressed_base = tl.load(compressed_base_ptr + batch_idx)
swa_base = tl.load(swa_base_ptr + batch_idx)
# Each token's request: the last r with query_start_loc[r] - base <= token.
lo = tl.zeros([BLOCK_T], dtype=tl.int32)
hi = tl.full([BLOCK_T], 0, dtype=tl.int32) + num_reqs
for _ in tl.static_range(SEARCH_STEPS):
mid = (lo + hi + 1) // 2
start = tl.load(query_start_loc_ptr + mid, mask=lo < hi, other=0) - base
go_right = (lo < hi) & (start <= token)
lo = tl.where(go_right, mid, lo)
hi = tl.where((lo < hi) & ~go_right, mid - 1, hi)
batch_idx = tl.minimum(lo, num_reqs - 1)
# Rows past the last request's queries belong to no request: -1 rows, length 0.
owned = valid & (token < tl.load(query_start_loc_ptr + num_reqs) - base)
pos = tl.load(query_pos_ptr + token, mask=owned, other=0)
seq_len = tl.load(seq_lens_ptr + batch_idx, mask=owned, other=0)
gather_len = tl.load(gather_lens_ptr + batch_idx, mask=owned, other=0)
compressed_base = tl.load(compressed_base_ptr + batch_idx, mask=owned, other=0)
swa_base = tl.load(swa_base_ptr + batch_idx, mask=owned, other=0)
# SWA portion of the gathered buffer starts from position
# (seq_len - gather_len), not 0. The +pos-gather_start formula maps a
# query's window back into the workspace's SWA region.
gather_start = seq_len - gather_len
for token_idx in range(query_start + worker_id, query_end, num_workers):
pos = tl.load(query_pos_ptr + token_idx)
# -1 entries inside the top-k span stay -1 (attention skips them).
# top_k=0 disables the compressed portion for SWA-only layers.
topk_len = tl.minimum((pos + 1) // COMPRESS_RATIO, top_k)
swa_len = tl.minimum(pos + 1, WINDOW_SIZE)
# -1 entries inside the top-k span stay -1 (attention skips them).
# top_k=0 disables the compressed portion for SWA-only layers.
topk_len = tl.where(owned, tl.minimum((pos + 1) // COMPRESS_RATIO, top_k), 0)
swa_len = tl.where(owned, tl.minimum(pos + 1, WINDOW_SIZE), 0)
combined_row = token_idx.to(tl.int64) * combined_indices_stride
topk_row = token_idx.to(tl.int64) * topk_indices_stride
combined_row = token.to(tl.int64)[:, None] * combined_indices_stride
topk_row = token.to(tl.int64)[:, None] * topk_indices_stride
offset = tl.arange(0, PADDED_TOP_K)
mask = offset < topk_len
topk_vals = tl.load(
topk_indices_ptr + topk_row + offset,
mask=mask,
)
tl.store(
combined_indices_ptr + combined_row + offset,
tl.where(topk_vals >= 0, topk_vals + compressed_base, -1),
mask=mask,
)
# Whole rows, -1 padded, so the output needs no fill pass:
# [top-k (-1 holes kept) | window | -1 up to WIDTH].
offset = tl.arange(0, PADDED_WIDTH)[None, :]
in_topk = offset < topk_len[:, None]
topk_vals = tl.load(
topk_indices_ptr + topk_row + offset, mask=valid[:, None] & in_topk, other=-1
)
# Workspace SWA index: swa_base[r] + (gather_offset_in_buffer).
# For positions [pos - swa_len + 1, pos], the buffer offsets are
# [pos - swa_len + 1 - gather_start, pos - gather_start].
swa_start = swa_base + pos - swa_len + 1 - gather_start
swa_vals = swa_start[:, None] + (offset - topk_len[:, None])
in_swa = offset < (topk_len + swa_len)[:, None]
vals = tl.where(
in_topk,
tl.where(topk_vals >= 0, topk_vals + compressed_base[:, None], -1),
tl.where(in_swa, swa_vals, -1),
)
tl.store(
combined_indices_ptr + combined_row + offset,
vals,
mask=valid[:, None] & (offset < WIDTH),
)
offset = tl.arange(0, WINDOW_SIZE)
# Workspace SWA index: swa_base[r] + (gather_offset_in_buffer).
# For positions [pos - swa_len + 1, pos], the buffer offsets are
# [pos - swa_len + 1 - gather_start, pos - gather_start].
tl.store(
combined_indices_ptr + combined_row + topk_len + offset,
swa_base + offset + pos - swa_len + 1 - gather_start,
mask=offset < swa_len,
)
tl.store(combined_lens_ptr + token_idx, topk_len + swa_len)
tl.store(combined_lens_ptr + token, topk_len + swa_len, mask=valid)
@@ -135,8 +135,8 @@ def combine_topk_swa_indices(
Returns int32 indices [num_tokens, padded_topk_swa] and per-token scanned-prefix
lengths, including -1 entries skipped by attention. Width is padded to 128.
Preallocated out_indices must contain -1 outside the written prefix;
reuse requires chunk-invariant scanned-prefix lengths. out_lens is overwritten.
Rows are written whole and -1 padded, so preallocated out_indices / out_lens
are overwritten entirely; rows past the last request are -1 with length 0.
"""
assert topk_indices.dtype == torch.int32
assert query_start_loc.dtype == torch.int32
@@ -154,9 +154,9 @@ def combine_topk_swa_indices(
num_reqs = seq_lens.shape[0]
combined_topk = combined_topk_width(topk, window_size)
if out_indices is None:
combined_indices = torch.full(
# The kernel writes whole -1 padded rows.
combined_indices = torch.empty(
(num_tokens, combined_topk),
-1,
dtype=torch.int32,
device=topk_indices.device,
)
@@ -165,7 +165,7 @@ def combine_topk_swa_indices(
assert out_indices.dtype == torch.int32
combined_indices = out_indices
if out_lens is None:
combined_lens = torch.zeros(
combined_lens = torch.empty(
num_tokens, dtype=torch.int32, device=topk_indices.device
)
else:
@@ -173,8 +173,10 @@ def combine_topk_swa_indices(
assert out_lens.dtype == torch.int32
combined_lens = out_lens
NUM_WORKERS = 128
_combine_topk_swa_indices_kernel[(num_reqs, NUM_WORKERS)](
if num_tokens == 0:
return combined_indices, combined_lens
block_t = 2 # Measured on B200 with num_warps=2, 16384 tokens, bs 1 to 64.
_combine_topk_swa_indices_kernel[(triton.cdiv(num_tokens, block_t),)](
combined_indices,
combined_indices.stride(0),
combined_lens,
@@ -186,10 +188,16 @@ def combine_topk_swa_indices(
gather_lens,
compressed_base,
swa_base,
num_tokens,
num_reqs,
top_k=topk,
COMPRESS_RATIO=compress_ratio,
WINDOW_SIZE=window_size,
PADDED_TOP_K=triton.next_power_of_2(topk_indices.shape[-1]),
WIDTH=combined_topk,
PADDED_WIDTH=triton.next_power_of_2(combined_topk),
BLOCK_T=block_t,
SEARCH_STEPS=max(num_reqs, 1).bit_length(),
num_warps=2,
)
return combined_indices, combined_lens
@@ -699,6 +699,12 @@ class DeepSeekV4IndexerPool(KVCache):
from the page layout [page_size * 64 payload | page_size * 4 scale]."""
assert self.use_fp4_indexer, "packed readback only applies to the fp4 layout"
buf = self.index_k_with_scale_buffer[layer_id - self.start_layer]
if buf.is_cuda:
from sglang.kernels.ops.attention.dsv4.fp4_indexer import (
gather_fp4_index_k,
)
return gather_fp4_index_k(buf, slots, page_size=self.page_size)
slots = slots.to(torch.int64)
p = self.page_size
page, off = (slots // p).unsqueeze(-1), slots % p