diff --git a/python/sglang/kernels/ops/attention/dsv4/fp4_indexer.py b/python/sglang/kernels/ops/attention/dsv4/fp4_indexer.py index 5cfe28d295..37357a3a83 100644 --- a/python/sglang/kernels/ops/attention/dsv4/fp4_indexer.py +++ b/python/sglang/kernels/ops/attention/dsv4/fp4_indexer.py @@ -184,6 +184,54 @@ def finish_flat_indexer_topk( ) +@triton.jit +def _flat_selection_slots_kernel( + Raw, + Starts, + Slots, + Pages, + K: tl.constexpr, + RAW_STRIDE: tl.constexpr, + PAGE_STRIDE: tl.constexpr, + BLOCK: tl.constexpr, +): + row = tl.program_id(0).to(tl.int64) + col = tl.arange(0, BLOCK) + raw = tl.load(Raw + row * RAW_STRIDE + col, col < K, -1) + chosen = raw >= 0 + slot = tl.load(Slots + (raw + tl.load(Starts + row)), chosen, -1) + tl.store(Pages + row * PAGE_STRIDE + col, slot.to(tl.int32), col < K) + + +def flat_selection_slots( + raw_indices: torch.Tensor, + request_starts: torch.Tensor, + slots: torch.Tensor, + page_indices: torch.Tensor, +) -> None: + """``page_indices[r, j] = slots[request_starts[r] + raw_indices[r, j]]``, or -1 + where the request-local position ``raw_indices[r, j]`` is negative.""" + rows, k = raw_indices.shape + if not rows or not k: + return + assert raw_indices.dtype == torch.int32 and raw_indices.stride(1) == 1 + assert request_starts.shape == (rows,) and request_starts.is_contiguous() + assert slots.ndim == 1 and slots.is_contiguous() + assert page_indices.shape[0] >= rows and page_indices.shape[1] >= k + assert page_indices.dtype == torch.int32 and page_indices.stride(1) == 1 + _flat_selection_slots_kernel[(rows,)]( + raw_indices, + request_starts, + slots, + page_indices, + k, + raw_indices.stride(0), + page_indices.stride(0), + triton.next_power_of_2(k), + num_warps=4, + ) + + @triton.jit def _index_k_rope_pack_kernel( X, diff --git a/python/sglang/srt/layers/attention/dsv4/v41_indexer/scoring.py b/python/sglang/srt/layers/attention/dsv4/v41_indexer/scoring.py index 76c229fcb6..7d7983d8c2 100644 --- a/python/sglang/srt/layers/attention/dsv4/v41_indexer/scoring.py +++ b/python/sglang/srt/layers/attention/dsv4/v41_indexer/scoring.py @@ -12,6 +12,7 @@ import torch from sglang.kernels.ops.attention.dsv4.fp4_indexer import ( finish_paged_indexer_topk, + flat_selection_slots, fp4_index_logits_decode, fp4_index_logits_paged, ) @@ -83,10 +84,11 @@ class DeepGEMMPrefillData(msgspec.Struct, frozen=True): """The pool slots of the request-local ``inputs.out_raw_indices`` rows.""" if inputs.out_page_indices is None: return - raw = inputs.out_raw_indices[: self.num_rows] - columns = (raw + self.request_starts[:, None]).clamp_min(0) - inputs.out_page_indices[: self.num_rows, : raw.shape[1]] = torch.where( - raw >= 0, self.k_slots[columns], -1 + flat_selection_slots( + inputs.out_raw_indices[: self.num_rows], + self.request_starts, + self.k_slots, + inputs.out_page_indices, ) def write_selection(self, selected: torch.Tensor, inputs: PrefillInputs) -> None: