[Perf] dsv4: Map V4.1 prefill index selections to pool slots in one kernel (#45938)
This commit is contained in:
@@ -184,6 +184,54 @@ def finish_flat_indexer_topk(
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _flat_selection_slots_kernel(
|
||||
Raw,
|
||||
Starts,
|
||||
Slots,
|
||||
Pages,
|
||||
K: tl.constexpr,
|
||||
RAW_STRIDE: tl.constexpr,
|
||||
PAGE_STRIDE: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
col = tl.arange(0, BLOCK)
|
||||
raw = tl.load(Raw + row * RAW_STRIDE + col, col < K, -1)
|
||||
chosen = raw >= 0
|
||||
slot = tl.load(Slots + (raw + tl.load(Starts + row)), chosen, -1)
|
||||
tl.store(Pages + row * PAGE_STRIDE + col, slot.to(tl.int32), col < K)
|
||||
|
||||
|
||||
def flat_selection_slots(
|
||||
raw_indices: torch.Tensor,
|
||||
request_starts: torch.Tensor,
|
||||
slots: torch.Tensor,
|
||||
page_indices: torch.Tensor,
|
||||
) -> None:
|
||||
"""``page_indices[r, j] = slots[request_starts[r] + raw_indices[r, j]]``, or -1
|
||||
where the request-local position ``raw_indices[r, j]`` is negative."""
|
||||
rows, k = raw_indices.shape
|
||||
if not rows or not k:
|
||||
return
|
||||
assert raw_indices.dtype == torch.int32 and raw_indices.stride(1) == 1
|
||||
assert request_starts.shape == (rows,) and request_starts.is_contiguous()
|
||||
assert slots.ndim == 1 and slots.is_contiguous()
|
||||
assert page_indices.shape[0] >= rows and page_indices.shape[1] >= k
|
||||
assert page_indices.dtype == torch.int32 and page_indices.stride(1) == 1
|
||||
_flat_selection_slots_kernel[(rows,)](
|
||||
raw_indices,
|
||||
request_starts,
|
||||
slots,
|
||||
page_indices,
|
||||
k,
|
||||
raw_indices.stride(0),
|
||||
page_indices.stride(0),
|
||||
triton.next_power_of_2(k),
|
||||
num_warps=4,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _index_k_rope_pack_kernel(
|
||||
X,
|
||||
|
||||
@@ -12,6 +12,7 @@ import torch
|
||||
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer import (
|
||||
finish_paged_indexer_topk,
|
||||
flat_selection_slots,
|
||||
fp4_index_logits_decode,
|
||||
fp4_index_logits_paged,
|
||||
)
|
||||
@@ -83,10 +84,11 @@ class DeepGEMMPrefillData(msgspec.Struct, frozen=True):
|
||||
"""The pool slots of the request-local ``inputs.out_raw_indices`` rows."""
|
||||
if inputs.out_page_indices is None:
|
||||
return
|
||||
raw = inputs.out_raw_indices[: self.num_rows]
|
||||
columns = (raw + self.request_starts[:, None]).clamp_min(0)
|
||||
inputs.out_page_indices[: self.num_rows, : raw.shape[1]] = torch.where(
|
||||
raw >= 0, self.k_slots[columns], -1
|
||||
flat_selection_slots(
|
||||
inputs.out_raw_indices[: self.num_rows],
|
||||
self.request_starts,
|
||||
self.k_slots,
|
||||
inputs.out_page_indices,
|
||||
)
|
||||
|
||||
def write_selection(self, selected: torch.Tensor, inputs: PrefillInputs) -> None:
|
||||
|
||||
Reference in New Issue
Block a user