[Perf] dsv4: Map V4.1 prefill index selections to pool slots in one kernel (#45938)

This commit is contained in:
Liangsheng Yin
2026-10-22 19:50:31 -07:00
committed by GitHub
parent 2da6c0c237
commit c4fa35b829
2 changed files with 54 additions and 4 deletions
@@ -184,6 +184,54 @@ def finish_flat_indexer_topk(
)
@triton.jit
def _flat_selection_slots_kernel(
Raw,
Starts,
Slots,
Pages,
K: tl.constexpr,
RAW_STRIDE: tl.constexpr,
PAGE_STRIDE: tl.constexpr,
BLOCK: tl.constexpr,
):
row = tl.program_id(0).to(tl.int64)
col = tl.arange(0, BLOCK)
raw = tl.load(Raw + row * RAW_STRIDE + col, col < K, -1)
chosen = raw >= 0
slot = tl.load(Slots + (raw + tl.load(Starts + row)), chosen, -1)
tl.store(Pages + row * PAGE_STRIDE + col, slot.to(tl.int32), col < K)
def flat_selection_slots(
raw_indices: torch.Tensor,
request_starts: torch.Tensor,
slots: torch.Tensor,
page_indices: torch.Tensor,
) -> None:
"""``page_indices[r, j] = slots[request_starts[r] + raw_indices[r, j]]``, or -1
where the request-local position ``raw_indices[r, j]`` is negative."""
rows, k = raw_indices.shape
if not rows or not k:
return
assert raw_indices.dtype == torch.int32 and raw_indices.stride(1) == 1
assert request_starts.shape == (rows,) and request_starts.is_contiguous()
assert slots.ndim == 1 and slots.is_contiguous()
assert page_indices.shape[0] >= rows and page_indices.shape[1] >= k
assert page_indices.dtype == torch.int32 and page_indices.stride(1) == 1
_flat_selection_slots_kernel[(rows,)](
raw_indices,
request_starts,
slots,
page_indices,
k,
raw_indices.stride(0),
page_indices.stride(0),
triton.next_power_of_2(k),
num_warps=4,
)
@triton.jit
def _index_k_rope_pack_kernel(
X,
@@ -12,6 +12,7 @@ import torch
from sglang.kernels.ops.attention.dsv4.fp4_indexer import (
finish_paged_indexer_topk,
flat_selection_slots,
fp4_index_logits_decode,
fp4_index_logits_paged,
)
@@ -83,10 +84,11 @@ class DeepGEMMPrefillData(msgspec.Struct, frozen=True):
"""The pool slots of the request-local ``inputs.out_raw_indices`` rows."""
if inputs.out_page_indices is None:
return
raw = inputs.out_raw_indices[: self.num_rows]
columns = (raw + self.request_starts[:, None]).clamp_min(0)
inputs.out_page_indices[: self.num_rows, : raw.shape[1]] = torch.where(
raw >= 0, self.k_slots[columns], -1
flat_selection_slots(
inputs.out_raw_indices[: self.num_rows],
self.request_starts,
self.k_slots,
inputs.out_page_indices,
)
def write_selection(self, selected: torch.Tensor, inputs: PrefillInputs) -> None: