[DSv4.1] Faster fp4 index-K gather and combine_topk_swa_indices (#41658)
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: BBuf <1182563586@qq.com>
(cherry picked from commit d75d5b33f7)
This commit is contained in:
@@ -461,6 +461,67 @@ def store_fp4_index_k_cache(
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _gather_fp4_index_k_kernel(
|
||||
cache,
|
||||
cache_stride,
|
||||
slots,
|
||||
payload,
|
||||
scales,
|
||||
n,
|
||||
page_size: tl.constexpr,
|
||||
BLOCK_ROWS: tl.constexpr,
|
||||
):
|
||||
# int32 view of a page: page_size * 16 payload words, then page_size scale words.
|
||||
rows = tl.program_id(0) * BLOCK_ROWS + tl.arange(0, BLOCK_ROWS)
|
||||
valid = rows < n
|
||||
slot = tl.load(slots + rows, valid, 0).to(tl.int64)
|
||||
page = slot // page_size
|
||||
offset = slot - page * page_size
|
||||
page_base = cache + page * cache_stride
|
||||
words = tl.arange(0, 16)
|
||||
payload_words = tl.load(
|
||||
page_base[:, None] + offset[:, None] * 16 + words[None, :], valid[:, None]
|
||||
)
|
||||
tl.store(
|
||||
payload + rows[:, None].to(tl.int64) * 16 + words[None, :],
|
||||
payload_words,
|
||||
valid[:, None],
|
||||
)
|
||||
tl.store(scales + rows, tl.load(page_base + page_size * 16 + offset, valid), valid)
|
||||
|
||||
|
||||
def gather_fp4_index_k(
|
||||
cache: torch.Tensor, slots: torch.Tensor, *, page_size: int
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Packed fp4 rows at ``slots`` of the index-K ``cache`` (``[pages, page_size *
|
||||
68]`` uint8, ``store_fp4_index_k_cache``'s layout): (payload int8 ``[n, 64]``,
|
||||
ue8m0 scales packed int32 ``[n]``), the layout ``quantize_fp4_indexer_tensor``
|
||||
returns."""
|
||||
assert cache.shape[1] == page_size * INDEX_K_SLOT_BYTES and cache.stride(1) == 1
|
||||
# The kernel addresses slots as `slots + rows`, so a strided view would be read
|
||||
# as if its elements were packed.
|
||||
assert slots.is_contiguous()
|
||||
words = cache.view(torch.int32)
|
||||
n = slots.shape[0]
|
||||
payload = torch.empty((n, 16), device=cache.device, dtype=torch.int32)
|
||||
scales = torch.empty((n,), device=cache.device, dtype=torch.int32)
|
||||
if n > 0:
|
||||
block_rows = 64
|
||||
_gather_fp4_index_k_kernel[(triton.cdiv(n, block_rows),)](
|
||||
words,
|
||||
words.stride(0),
|
||||
slots,
|
||||
payload,
|
||||
scales,
|
||||
n,
|
||||
page_size,
|
||||
BLOCK_ROWS=block_rows,
|
||||
num_warps=4,
|
||||
)
|
||||
return payload.view(torch.int8), scales
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _index_k_rope_pack_kernel(
|
||||
X,
|
||||
|
||||
@@ -36,7 +36,7 @@ def _build_swa_token_ids_kernel(
|
||||
tl.store(out_ptr + out_off + i, swa_id)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["top_k"])
|
||||
@triton.jit(do_not_specialize=["num_tokens", "num_reqs", "top_k"])
|
||||
def _combine_topk_swa_indices_kernel(
|
||||
combined_indices_ptr,
|
||||
combined_indices_stride,
|
||||
@@ -49,59 +49,76 @@ def _combine_topk_swa_indices_kernel(
|
||||
gather_lens_ptr,
|
||||
compressed_base_ptr,
|
||||
swa_base_ptr,
|
||||
num_tokens,
|
||||
num_reqs,
|
||||
top_k,
|
||||
COMPRESS_RATIO: tl.constexpr,
|
||||
WINDOW_SIZE: tl.constexpr,
|
||||
PADDED_TOP_K: tl.constexpr,
|
||||
WIDTH: tl.constexpr,
|
||||
PADDED_WIDTH: tl.constexpr,
|
||||
BLOCK_T: tl.constexpr,
|
||||
SEARCH_STEPS: tl.constexpr,
|
||||
):
|
||||
batch_idx = tl.program_id(0)
|
||||
worker_id = tl.program_id(1)
|
||||
num_workers = tl.num_programs(1)
|
||||
# BLOCK_T consecutive tokens per program, so the grid scales with tokens, not requests.
|
||||
token = tl.program_id(0) * BLOCK_T + tl.arange(0, BLOCK_T)
|
||||
valid = token < num_tokens
|
||||
|
||||
# query_start_loc may be a global tensor; rebase to chunk-local offsets
|
||||
# by subtracting the chunk's starting value.
|
||||
base = tl.load(query_start_loc_ptr)
|
||||
query_start = tl.load(query_start_loc_ptr + batch_idx) - base
|
||||
query_end = tl.load(query_start_loc_ptr + batch_idx + 1) - base
|
||||
seq_len = tl.load(seq_lens_ptr + batch_idx)
|
||||
gather_len = tl.load(gather_lens_ptr + batch_idx)
|
||||
compressed_base = tl.load(compressed_base_ptr + batch_idx)
|
||||
swa_base = tl.load(swa_base_ptr + batch_idx)
|
||||
# Each token's request: the last r with query_start_loc[r] - base <= token.
|
||||
lo = tl.zeros([BLOCK_T], dtype=tl.int32)
|
||||
hi = tl.full([BLOCK_T], 0, dtype=tl.int32) + num_reqs
|
||||
for _ in tl.static_range(SEARCH_STEPS):
|
||||
mid = (lo + hi + 1) // 2
|
||||
start = tl.load(query_start_loc_ptr + mid, mask=lo < hi, other=0) - base
|
||||
go_right = (lo < hi) & (start <= token)
|
||||
lo = tl.where(go_right, mid, lo)
|
||||
hi = tl.where((lo < hi) & ~go_right, mid - 1, hi)
|
||||
batch_idx = tl.minimum(lo, num_reqs - 1)
|
||||
# Rows past the last request's queries belong to no request: -1 rows, length 0.
|
||||
owned = valid & (token < tl.load(query_start_loc_ptr + num_reqs) - base)
|
||||
|
||||
pos = tl.load(query_pos_ptr + token, mask=owned, other=0)
|
||||
seq_len = tl.load(seq_lens_ptr + batch_idx, mask=owned, other=0)
|
||||
gather_len = tl.load(gather_lens_ptr + batch_idx, mask=owned, other=0)
|
||||
compressed_base = tl.load(compressed_base_ptr + batch_idx, mask=owned, other=0)
|
||||
swa_base = tl.load(swa_base_ptr + batch_idx, mask=owned, other=0)
|
||||
# SWA portion of the gathered buffer starts from position
|
||||
# (seq_len - gather_len), not 0. The +pos-gather_start formula maps a
|
||||
# query's window back into the workspace's SWA region.
|
||||
gather_start = seq_len - gather_len
|
||||
|
||||
for token_idx in range(query_start + worker_id, query_end, num_workers):
|
||||
pos = tl.load(query_pos_ptr + token_idx)
|
||||
# -1 entries inside the top-k span stay -1 (attention skips them).
|
||||
# top_k=0 disables the compressed portion for SWA-only layers.
|
||||
topk_len = tl.minimum((pos + 1) // COMPRESS_RATIO, top_k)
|
||||
swa_len = tl.minimum(pos + 1, WINDOW_SIZE)
|
||||
# -1 entries inside the top-k span stay -1 (attention skips them).
|
||||
# top_k=0 disables the compressed portion for SWA-only layers.
|
||||
topk_len = tl.where(owned, tl.minimum((pos + 1) // COMPRESS_RATIO, top_k), 0)
|
||||
swa_len = tl.where(owned, tl.minimum(pos + 1, WINDOW_SIZE), 0)
|
||||
|
||||
combined_row = token_idx.to(tl.int64) * combined_indices_stride
|
||||
topk_row = token_idx.to(tl.int64) * topk_indices_stride
|
||||
combined_row = token.to(tl.int64)[:, None] * combined_indices_stride
|
||||
topk_row = token.to(tl.int64)[:, None] * topk_indices_stride
|
||||
|
||||
offset = tl.arange(0, PADDED_TOP_K)
|
||||
mask = offset < topk_len
|
||||
topk_vals = tl.load(
|
||||
topk_indices_ptr + topk_row + offset,
|
||||
mask=mask,
|
||||
)
|
||||
tl.store(
|
||||
combined_indices_ptr + combined_row + offset,
|
||||
tl.where(topk_vals >= 0, topk_vals + compressed_base, -1),
|
||||
mask=mask,
|
||||
)
|
||||
# Whole rows, -1 padded, so the output needs no fill pass:
|
||||
# [top-k (-1 holes kept) | window | -1 up to WIDTH].
|
||||
offset = tl.arange(0, PADDED_WIDTH)[None, :]
|
||||
in_topk = offset < topk_len[:, None]
|
||||
topk_vals = tl.load(
|
||||
topk_indices_ptr + topk_row + offset, mask=valid[:, None] & in_topk, other=-1
|
||||
)
|
||||
# Workspace SWA index: swa_base[r] + (gather_offset_in_buffer).
|
||||
# For positions [pos - swa_len + 1, pos], the buffer offsets are
|
||||
# [pos - swa_len + 1 - gather_start, pos - gather_start].
|
||||
swa_start = swa_base + pos - swa_len + 1 - gather_start
|
||||
swa_vals = swa_start[:, None] + (offset - topk_len[:, None])
|
||||
in_swa = offset < (topk_len + swa_len)[:, None]
|
||||
vals = tl.where(
|
||||
in_topk,
|
||||
tl.where(topk_vals >= 0, topk_vals + compressed_base[:, None], -1),
|
||||
tl.where(in_swa, swa_vals, -1),
|
||||
)
|
||||
tl.store(
|
||||
combined_indices_ptr + combined_row + offset,
|
||||
vals,
|
||||
mask=valid[:, None] & (offset < WIDTH),
|
||||
)
|
||||
|
||||
offset = tl.arange(0, WINDOW_SIZE)
|
||||
# Workspace SWA index: swa_base[r] + (gather_offset_in_buffer).
|
||||
# For positions [pos - swa_len + 1, pos], the buffer offsets are
|
||||
# [pos - swa_len + 1 - gather_start, pos - gather_start].
|
||||
tl.store(
|
||||
combined_indices_ptr + combined_row + topk_len + offset,
|
||||
swa_base + offset + pos - swa_len + 1 - gather_start,
|
||||
mask=offset < swa_len,
|
||||
)
|
||||
|
||||
tl.store(combined_lens_ptr + token_idx, topk_len + swa_len)
|
||||
tl.store(combined_lens_ptr + token, topk_len + swa_len, mask=valid)
|
||||
|
||||
@@ -135,8 +135,8 @@ def combine_topk_swa_indices(
|
||||
|
||||
Returns int32 indices [num_tokens, padded_topk_swa] and per-token scanned-prefix
|
||||
lengths, including -1 entries skipped by attention. Width is padded to 128.
|
||||
Preallocated out_indices must contain -1 outside the written prefix;
|
||||
reuse requires chunk-invariant scanned-prefix lengths. out_lens is overwritten.
|
||||
Rows are written whole and -1 padded, so preallocated out_indices / out_lens
|
||||
are overwritten entirely; rows past the last request are -1 with length 0.
|
||||
"""
|
||||
assert topk_indices.dtype == torch.int32
|
||||
assert query_start_loc.dtype == torch.int32
|
||||
@@ -154,9 +154,9 @@ def combine_topk_swa_indices(
|
||||
num_reqs = seq_lens.shape[0]
|
||||
combined_topk = combined_topk_width(topk, window_size)
|
||||
if out_indices is None:
|
||||
combined_indices = torch.full(
|
||||
# The kernel writes whole -1 padded rows.
|
||||
combined_indices = torch.empty(
|
||||
(num_tokens, combined_topk),
|
||||
-1,
|
||||
dtype=torch.int32,
|
||||
device=topk_indices.device,
|
||||
)
|
||||
@@ -165,7 +165,7 @@ def combine_topk_swa_indices(
|
||||
assert out_indices.dtype == torch.int32
|
||||
combined_indices = out_indices
|
||||
if out_lens is None:
|
||||
combined_lens = torch.zeros(
|
||||
combined_lens = torch.empty(
|
||||
num_tokens, dtype=torch.int32, device=topk_indices.device
|
||||
)
|
||||
else:
|
||||
@@ -173,8 +173,10 @@ def combine_topk_swa_indices(
|
||||
assert out_lens.dtype == torch.int32
|
||||
combined_lens = out_lens
|
||||
|
||||
NUM_WORKERS = 128
|
||||
_combine_topk_swa_indices_kernel[(num_reqs, NUM_WORKERS)](
|
||||
if num_tokens == 0:
|
||||
return combined_indices, combined_lens
|
||||
block_t = 2 # Measured on B200 with num_warps=2, 16384 tokens, bs 1 to 64.
|
||||
_combine_topk_swa_indices_kernel[(triton.cdiv(num_tokens, block_t),)](
|
||||
combined_indices,
|
||||
combined_indices.stride(0),
|
||||
combined_lens,
|
||||
@@ -186,10 +188,16 @@ def combine_topk_swa_indices(
|
||||
gather_lens,
|
||||
compressed_base,
|
||||
swa_base,
|
||||
num_tokens,
|
||||
num_reqs,
|
||||
top_k=topk,
|
||||
COMPRESS_RATIO=compress_ratio,
|
||||
WINDOW_SIZE=window_size,
|
||||
PADDED_TOP_K=triton.next_power_of_2(topk_indices.shape[-1]),
|
||||
WIDTH=combined_topk,
|
||||
PADDED_WIDTH=triton.next_power_of_2(combined_topk),
|
||||
BLOCK_T=block_t,
|
||||
SEARCH_STEPS=max(num_reqs, 1).bit_length(),
|
||||
num_warps=2,
|
||||
)
|
||||
return combined_indices, combined_lens
|
||||
|
||||
|
||||
@@ -684,6 +684,12 @@ class DeepSeekV4IndexerPool(KVCache):
|
||||
from the page layout [page_size * 64 payload | page_size * 4 scale]."""
|
||||
assert self.use_fp4_indexer, "packed readback only applies to the fp4 layout"
|
||||
buf = self.index_k_with_scale_buffer[layer_id - self.start_layer]
|
||||
if buf.is_cuda:
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer import (
|
||||
gather_fp4_index_k,
|
||||
)
|
||||
|
||||
return gather_fp4_index_k(buf, slots, page_size=self.page_size)
|
||||
slots = slots.to(torch.int64)
|
||||
p = self.page_size
|
||||
page, off = (slots // p).unsqueeze(-1), slots % p
|
||||
|
||||
Reference in New Issue
Block a user