From c79510cc2a334191ef515b99127420491795b841 Mon Sep 17 00:00:00 2001 From: Yuan Luo Date: Tue, 22 Sep 2026 07:06:06 -0700 Subject: [PATCH] [DSv4.1] Score prefill consumer index layers on candidate blocks with DeepGEMM (#40352) Co-authored-by: luoyuan.luo Co-authored-by: DarkSharpness <76582120+DarkSharpness@users.noreply.github.com> --- .../layers/attention/deepseek_v4_backend.py | 161 ++++++------ .../attention/dsv4/candidate_indexer.py | 133 +++++++++- .../dsv4/candidate_indexer_deep_gemm.py | 182 +++++++++++++- .../attention/dsv4/dense_prefill_indexer.py | 153 +++++++++++- .../test_dsv41_prefill_sparse_indexer.py | 229 ++++++++++++++++++ 5 files changed, 753 insertions(+), 105 deletions(-) create mode 100644 test/registered/kernels/ops/attention/test_dsv41_prefill_sparse_indexer.py diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index dfe5f43c29..d57fb45c30 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -60,7 +60,9 @@ from sglang.srt.layers.attention.dsv4.candidate_indexer import ( CandidateMasks, CandidateMetadata, IndexerInputs, - PrefillCandidateBlocks, + PrefillIndexerInputs, + cut_request_masks, + expand_index_page_table, make_candidate_indexer, mask_topk_scores, published_masks, @@ -71,7 +73,7 @@ from sglang.srt.layers.attention.dsv4.compressor_v2 import ( FusedCompressMetadata, create_paged_compressor_data, ) -from sglang.srt.layers.attention.dsv4.dense_prefill_indexer import dense_prefill_topk +from sglang.srt.layers.attention.dsv4.dense_prefill_indexer import plain_prefill_topk from sglang.srt.layers.attention.dsv4.dsv41_sparse import ( _rope_fq4, token_req_indices, @@ -258,33 +260,6 @@ def _maybe_precompute_flashmla_sched_meta( flashmla_metadata.num_splits = num_splits -def _expand_index_page_table( - page_table: torch.Tensor, - *, - full_page_size: int, - compress_ratio: int, - index_page_size: int, -) -> torch.Tensor: - """Block table of a low-ratio indexer-K pool, which pages at `index_page_size` - slots: [bs, n] -> [bs, n * blocks_per_page] int32. The kernel reads compressed - slot j at page_table[b, j // index_page_size] * index_page_size + j % - index_page_size, which after this expansion is the c1/c2 pool slot of the same - position.""" - slots_per_page = full_page_size // compress_ratio - assert slots_per_page % index_page_size == 0, ( - f"{full_page_size = } / {compress_ratio = } must be a multiple of " - f"{index_page_size = }" - ) - blocks_per_page = slots_per_page // index_page_size - if blocks_per_page == 1: - return page_table - bs, n = page_table.shape - base = page_table.to(torch.int64) * blocks_per_page - offsets = torch.arange(blocks_per_page, device=page_table.device, dtype=torch.int64) - expanded = base.unsqueeze(-1) + offsets # [bs, n, blocks_per_page] - return expanded.reshape(bs, n * blocks_per_page).to(torch.int32) - - # Arbitrary cap on one bf16 [rows, heads, lc] score chunk; transients run ~3x this. _TORCH_INDEXER_SCORE_BUDGET_BYTES = 1 << 30 @@ -1357,7 +1332,7 @@ class DeepseekV4AttnBackend( index_page_size = self.token_to_kv_pool.get_index_k_page_size( compress_ratio ) - page_table = _expand_index_page_table( + page_table = expand_index_page_table( page_table, full_page_size=self.page_size, compress_ratio=compress_ratio, @@ -1544,7 +1519,7 @@ class DeepseekV4AttnBackend( if max_seq_len is not None: num_pages = min(max_seq_len // self.page_size, num_pages) index_page_size = self.token_to_kv_pool.get_index_k_page_size(compress_ratio) - page_table = _expand_index_page_table( + page_table = expand_index_page_table( core.page_table[:, :num_pages], full_page_size=self.page_size, compress_ratio=compress_ratio, @@ -1740,23 +1715,6 @@ class DeepseekV4AttnBackend( get_local_dp_buffer_len(), ) tail = tail_metadata.late_layer_tail - tail_lens_cpu = ( - tail.local_lens_cpu - if tail.cp_metadata is not None - else tail.extend_seq_lens_cpu - ) - # TODO(candidate): goes away once the source publishes its tail rows straight - # onto the tail metadata (publish_prefill); until then cut the full masks. - full_masks = self.forward_metadata.candidate_metadata - if isinstance(full_masks, PrefillCandidateBlocks): - tail_metadata.candidate_metadata = full_masks.tail(tail_lens_cpu) - elif isinstance(full_masks, CandidateMasks) and full_masks.request_masks: - tail_metadata.candidate_metadata = CandidateMasks( - request_masks=[ - mask[mask.shape[0] - t :] - for mask, t in zip(full_masks.request_masks, tail_lens_cpu) - ] - ) # The layers before the switch published top-k into the full metadata's # buffers; carry the tail rows into the tail metadata's (padding stays -1). full_core = saved[0].core_attn_metadata @@ -3243,12 +3201,8 @@ class DeepseekV4AttnBackend( num_tokens = pos.shape[0] if not slot_chunks or num_tokens == 0: if indexer.is_candidate_source: - self.forward_metadata.candidate_metadata = PrefillCandidateBlocks( - request_blocks=[ - torch.empty((length, 0), dtype=torch.int32, device=device) - for length in q_lens_cpu - ] - ) + # no row to publish for; the consumers return here as well + self.forward_metadata.candidate_metadata = None return k_slots = torch.cat(slot_chunks) k_fp4, k_sf = pool.get_low_ratio_index_k_fp4(layer.layer_id, k_slots) @@ -3265,26 +3219,28 @@ class DeepseekV4AttnBackend( q_lens.to(torch.int64), output_size=num_tokens, ) - candidates = None - if indexer.uses_candidates and not indexer.is_candidate_source: - candidates = self.forward_metadata.candidate_metadata - assert isinstance(candidates, PrefillCandidateBlocks) - topk = indexer.index_topk - selected, published = dense_prefill_topk( - q=(q_fp4, q_sf), - kv=(k_fp4, k_sf), - weights=weights, - starts=ks, - lengths=compress_lens, - request_lengths=list(zip(q_lens_cpu, lc_per_req)), - topk=topk, - candidate_topk_blocks=indexer.candidate_topk_blocks, - candidate_block_size=indexer.candidate_block_size, - publish_candidates=indexer.is_candidate_source, - candidates=candidates, + inputs = self._prefill_indexer_inputs( + layer, + (q_fp4, q_sf), + (k_fp4, k_sf), + weights, + compress_lens, + ks, + lc_per_req, + list(q_lens_cpu), ) - if published is not None: - self.forward_metadata.candidate_metadata = published + topk = indexer.index_topk + selected = torch.full((num_tokens, topk), -1, dtype=torch.int32, device=device) + if indexer.is_candidate_source: + self._publish_prefill( + self.candidate_indexer.publish_prefill(inputs, selected) + ) + elif indexer.uses_candidates: + self.candidate_indexer.select_prefill( + self.forward_metadata.candidate_metadata, inputs, selected + ) + else: + plain_prefill_topk(inputs, selected) # ascending positions, padding last: the layout the consumers expect unselected = torch.iinfo(torch.int32).max selected = selected.masked_fill(selected < 0, unselected).sort(dim=-1).values @@ -3297,6 +3253,61 @@ class DeepseekV4AttnBackend( chosen, selected - ks[:, None], -1 ) + def _prefill_indexer_inputs( + self, layer, q, kv, weights, compress_lens, ks, lc_per_req, q_lens_cpu + ) -> PrefillIndexerInputs: + pool = self.token_to_kv_pool + ratio = layer.compress_ratio + num_tokens = q[0].shape[0] + index_page_size = pool.get_index_k_page_size(ratio) + k_cache = pool.get_index_k_with_scale_buffer(layer.layer_id) + assert k_cache.dim() == 2 + return PrefillIndexerInputs( + q_fp4=q[0], + q_sf=q[1], + weights=weights, + compress_lens=compress_lens, + request_starts=ks, + lens_per_request=lc_per_req, + rows_per_request=q_lens_cpu, + kv=kv, + k_cache=k_cache.view(k_cache.shape[0], index_page_size, 1, 68), + page_size=index_page_size, + kv_page_table=self.forward_metadata.core_metadata.page_table[:num_tokens], + kv_page_size=self.page_size, + compress_ratio=ratio, + ) + + def _tail_lens_to_publish(self) -> Optional[List[int]]: + """Rows per request of the late-layer tail, when the layers after the + switch will consume what is published now.""" + tail_metadata = self.tail_forward_metadata + if tail_metadata is None or tail_metadata is self.forward_metadata: + return None + tail = tail_metadata.late_layer_tail + return ( + tail.local_lens_cpu + if tail.cp_metadata is not None + else tail.extend_seq_lens_cpu + ) + + def _publish_prefill(self, published: CandidateMetadata) -> None: + self.forward_metadata.candidate_metadata = published + tail_lens = self._tail_lens_to_publish() + if tail_lens is not None: + self.tail_forward_metadata.candidate_metadata = ( + self.candidate_indexer.prefill_tail(published, tail_lens) + ) + + def _publish_prefill_masks(self, masks: CandidateMasks) -> None: + """The torch prefill path's inline masks (see the TODO on CandidateMasks).""" + self.forward_metadata.candidate_metadata = masks + tail_lens = self._tail_lens_to_publish() + if tail_lens is not None: + self.tail_forward_metadata.candidate_metadata = cut_request_masks( + masks, tail_lens + ) + def _low_ratio_index_topk_prefill_graph(self, layer, pos, q, w) -> None: from sglang.kernels.ops.attention.dsv4.fp4_indexer import ( quantize_fp4_indexer_tensor, @@ -3607,9 +3618,7 @@ class DeepseekV4AttnBackend( if masks is not None: publish.append(torch.cat(masks) if len(masks) > 1 else masks[0]) if publish is not None: - self.forward_metadata.candidate_metadata = CandidateMasks( - request_masks=publish - ) + self._publish_prefill_masks(CandidateMasks(request_masks=publish)) def get_swa_out_cache_loc(self, forward_batch: ForwardBatch) -> torch.Tensor: """Idle always re-translates at store time: its metadata may be stale, and diff --git a/python/sglang/srt/layers/attention/dsv4/candidate_indexer.py b/python/sglang/srt/layers/attention/dsv4/candidate_indexer.py index 81466f8652..f6df650b7d 100644 --- a/python/sglang/srt/layers/attention/dsv4/candidate_indexer.py +++ b/python/sglang/srt/layers/attention/dsv4/candidate_indexer.py @@ -1,19 +1,15 @@ from __future__ import annotations +from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import TYPE_CHECKING, List, Optional, Union +from typing import List, Optional, Tuple, Union import msgspec import torch import torch.nn.functional as F from sglang.srt.layers.attention.dsv4.metadata import PagedIndexerMetadata -from sglang.srt.runtime_context import get_platform - -if TYPE_CHECKING: - from sglang.srt.layers.attention.dsv4.candidate_indexer_deep_gemm import ( - DeepGemmCandidateIndexer, - ) +from sglang.srt.runtime_context import get_parallel, get_platform class CandidateMetadata: @@ -40,11 +36,104 @@ class IndexerInputs: return self.q_fp4.shape[0] +class PrefillIndexerInputs(msgspec.Struct, frozen=True): + """One index layer's operands on the dense fp4 prefill path: the chunk's query + rows, one request's rows consecutive.""" + + q_fp4: torch.Tensor # [rows, heads, 64] int8, packed fp4 + q_sf: torch.Tensor # [rows, heads] int32, packed ue8m0 + weights: torch.Tensor # [rows, heads] fp32 head weights + compress_lens: torch.Tensor # [rows] int32, compressed positions the row sees + # [rows] int32: first flattened-K column of the row's request; selections + # are flattened-K columns, request start + compressed position + request_starts: torch.Tensor + lens_per_request: List[int] # compressed positions at each request's newest token + rows_per_request: List[int] # query rows of each request + # the flattened index K of the batch (int8 [n, 64], int32 [n]), column j of + # a row's dense scores against kv[request_starts[t] + j] + kv: Tuple[torch.Tensor, torch.Tensor] + k_cache: torch.Tensor # [pages, page_size, 1, 68] uint8, the layer's index-K pool + page_size: int # index-K pool page size, in slots + # [rows, kv pages] int32: the row's request in the KV pool, whose pages hold + # kv_page_size tokens; expand_index_page_table gives the index-K pool's + kv_page_table: torch.Tensor + kv_page_size: int + compress_ratio: int + + @property + def num_rows(self) -> int: + return self.q_fp4.shape[0] + + +def expand_index_page_table( + page_table: torch.Tensor, + *, + full_page_size: int, + compress_ratio: int, + index_page_size: int, +) -> torch.Tensor: + """Block table of a low-ratio indexer-K pool, which pages at `index_page_size` + slots: [bs, n] -> [bs, n * blocks_per_page] int32. The kernel reads compressed + slot j at page_table[b, j // index_page_size] * index_page_size + j % + index_page_size, which after this expansion is the c1/c2 pool slot of the same + position.""" + slots_per_page = full_page_size // compress_ratio + assert slots_per_page % index_page_size == 0, ( + f"{full_page_size = } / {compress_ratio = } must be a multiple of " + f"{index_page_size = }" + ) + blocks_per_page = slots_per_page // index_page_size + if blocks_per_page == 1: + return page_table + bs, n = page_table.shape + base = page_table.to(torch.int64) * blocks_per_page + offsets = torch.arange(blocks_per_page, device=page_table.device, dtype=torch.int64) + expanded = base.unsqueeze(-1) + offsets # [bs, n, blocks_per_page] + return expanded.reshape(bs, n * blocks_per_page).to(torch.int32) + + +class CandidateIndexer(ABC): + """The two-level low-ratio indexer. The candidate source publishes what its + consumers need to select among its top candidate blocks, on the forward + metadata; a consumer selects its top-k from that. + + TODO(candidate): Hopper decode and the torch prefill path still publish and + consume masks inline in the backend; their decode side is not behind this + protocol yet.""" + + @abstractmethod + def publish_prefill( + self, inputs: PrefillIndexerInputs, out_positions: torch.Tensor + ) -> CandidateMetadata: + """The source layer's own top-``k`` (``k = out_positions.shape[1]``) as + flattened-K columns, ``-1`` padded, unordered, plus its candidates for + the chunk: both come from one pass over its dense scores.""" + + @abstractmethod + def select_prefill( + self, + published: CandidateMetadata, + inputs: PrefillIndexerInputs, + out_positions: torch.Tensor, + ) -> None: + """A consumer's top-``k`` over its published candidates, in the same + layout as ``publish_prefill`` writes.""" + + @abstractmethod + def prefill_tail( + self, published: CandidateMetadata, tail_lens: List[int] + ) -> CandidateMetadata: + """The candidates of the last ``tail_lens[b]`` rows of each request, for + the late layers that run on the tail only.""" + + def make_candidate_indexer( topk_blocks: int, block_size: int -) -> Optional[DeepGemmCandidateIndexer]: - """The paged fp4 decode path's two-level indexer; None on Hopper, whose decode - indexer selects through masks inline.""" +) -> Optional[CandidateIndexer]: + """DeepGEMM's paged sparse indexer on SM100, with the dense-score + implementation for prefill under context parallelism (a rank's local rows + are not the page table's rows); None on Hopper, whose decode and prefill + indexers select through masks inline.""" if topk_blocks <= 0 or get_platform().device_sm < 100: return None from sglang.srt.layers.deep_gemm_wrapper.configurer import ( @@ -59,18 +148,36 @@ def make_candidate_indexer( from sglang.srt.layers.attention.dsv4.candidate_indexer_deep_gemm import ( DeepGemmCandidateIndexer, ) + from sglang.srt.layers.attention.dsv4.dense_prefill_indexer import ( + DenseCandidateIndexer, + ) - return DeepGemmCandidateIndexer(topk_blocks, block_size) + prefill_dense = None + if get_parallel().attn_cp_size > 1: + prefill_dense = DenseCandidateIndexer(topk_blocks, block_size) + return DeepGemmCandidateIndexer( + topk_blocks, block_size, prefill_dense=prefill_dense + ) -# TODO(candidate): Hopper decode and prefill still select through these masks -# inline in the backend; move them behind the protocol as publish/select_prefill. +# TODO(candidate): Hopper decode and the torch prefill path still select through +# these masks inline in the backend. @dataclass class CandidateMasks(CandidateMetadata): mask: Optional[torch.Tensor] = None # decode: [rows, width] bool request_masks: Optional[List[torch.Tensor]] = None # prefill: [rows_b, lc_b] each +def cut_request_masks(masks: CandidateMasks, tail_lens: List[int]) -> CandidateMasks: + """The last ``tail_lens[b]`` rows of each request's mask.""" + assert masks.request_masks is not None, "prefill masks missing" + return CandidateMasks( + request_masks=[ + mask[mask.shape[0] - t :] for mask, t in zip(masks.request_masks, tail_lens) + ] + ) + + class PrefillCandidateBlocks(CandidateMetadata, msgspec.Struct): request_blocks: List[torch.Tensor] diff --git a/python/sglang/srt/layers/attention/dsv4/candidate_indexer_deep_gemm.py b/python/sglang/srt/layers/attention/dsv4/candidate_indexer_deep_gemm.py index 41d4dd591e..afa2d3d404 100644 --- a/python/sglang/srt/layers/attention/dsv4/candidate_indexer_deep_gemm.py +++ b/python/sglang/srt/layers/attention/dsv4/candidate_indexer_deep_gemm.py @@ -1,7 +1,7 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Optional +from typing import List, Optional import torch @@ -14,10 +14,18 @@ from sglang.kernels.ops.attention.dsv4.topk import ( plan_topk_v2, topk_transform_bf16_small, topk_transform_paged_v2, + topk_transform_ragged_v2, ) from sglang.srt.layers.attention.dsv4.candidate_indexer import ( + CandidateIndexer, CandidateMetadata, IndexerInputs, + PrefillIndexerInputs, + expand_index_page_table, +) +from sglang.srt.layers.attention.dsv4.dense_prefill_indexer import ( + DenseCandidateIndexer, + score_tiles, ) from sglang.srt.layers.attention.dsv4.indexer import ( deep_gemm_fp4_paged_mqa_logits, @@ -134,16 +142,36 @@ def topk_transform_sparse( ) -# TODO(dark): support publish prefill/select prefill +@dataclass +class PrefillSparseBlockTable(SparseBlockTable): + """Published by a prefill chunk's candidate source: one row per query token. + The inputs stay attached so the late-layer tail can rebuild the table for + its rows (the DeepGEMM schedule is per row set, it cannot be sliced).""" + + compress_lens: torch.Tensor # [rows] int32 + page_table: torch.Tensor # [rows, index pages] int32 + request_ids: torch.Tensor # [rows] int32 + q_dtype: torch.dtype + page_size: int # index-K pool page size (slots) + rows_per_request: List[int] # rows of each request, in row order + + # TODO(dark): support fusion of publish + topk of publish layer -class DeepGemmCandidateIndexer: - def __init__(self, topk_blocks: int, block_size: int): +class DeepGemmCandidateIndexer(CandidateIndexer): + def __init__( + self, + topk_blocks: int, + block_size: int, + prefill_dense: Optional[DenseCandidateIndexer] = None, + ): assert block_size == CANDIDATE_BLOCK_SIZE, block_size self.topk_blocks = topk_blocks self.block_size = block_size self.alt_stream = torch.cuda.Stream() self._row_ids: Optional[torch.Tensor] = None self._retired_row_ids: list = [] # captured graphs keep reading the buffers they saw + # set for the forwards whose prefill rows the page table does not describe + self._prefill_dense = prefill_dense def _request_ids( self, request_ids: Optional[torch.Tensor], rows: int, device: torch.device @@ -253,3 +281,149 @@ class DeepGemmCandidateIndexer: logits = self._scores(table, inputs) # decode carries no raw_indices; the kernel writes slots only topk_transform_sparse(logits, table.valid_lens, table, page_indices) + + def publish_prefill( + self, inputs: PrefillIndexerInputs, out_positions: torch.Tensor + ) -> CandidateMetadata: + """The source layer's own top-k and the block table of the chunk from + one pass over its dense scores, tile by tile: per tile the plain top-k + into ``out_positions`` and one read for the block keys.""" + if self._prefill_dense is not None: + return self._prefill_dense.publish_prefill(inputs, out_positions) + rows = inputs.num_rows + device = inputs.q_fp4.device + nblocks, valid_lens = candidate_row_lens(inputs.compress_lens, self.topk_blocks) + blocks = torch.empty(rows, self.topk_blocks, dtype=torch.int32, device=device) + # the block keys read the score rows through 32-byte vectors + for tile, logits in score_tiles(inputs, width_align=8): + lens = inputs.compress_lens[tile] + topk_transform_ragged_v2( + logits, + lens, + out_offsets=inputs.request_starts[tile], + out_indices=out_positions[tile], + ) + # keys past a row's block count stay unset; the top-k reads a row + # up to its block count only (v2 wants the stride a multiple of 4) + width_blocks = ( + logits.shape[1] + CANDIDATE_BLOCK_SIZE - 1 + ) // CANDIDATE_BLOCK_SIZE + keys = logits.new_empty(logits.shape[0], (width_blocks + 3) // 4 * 4) + amax8_varlen(logits, lens, out=keys) + topk_transform_ragged_v2( + keys, + nblocks[tile], + out_offsets=torch.zeros( + logits.shape[0], dtype=torch.int32, device=device + ), + out_indices=blocks[tile], + ) + request_ids = torch.repeat_interleave( + torch.arange( + len(inputs.rows_per_request), dtype=torch.int32, device=device + ), + torch.tensor(inputs.rows_per_request, dtype=torch.int64, device=device), + output_size=rows, + ) + # index-K pool pages of each row's request + page_table = expand_index_page_table( + inputs.kv_page_table, + full_page_size=inputs.kv_page_size, + compress_ratio=inputs.compress_ratio, + index_page_size=inputs.page_size, + ).contiguous() + return self._prefill_table( + blocks, + inputs.compress_lens, + page_table, + inputs.page_size, + request_ids, + inputs.rows_per_request, + inputs.q_fp4.dtype, + valid_lens, + ) + + def _prefill_table( + self, + blocks, + compress_lens, + page_table, + page_size, + request_ids, + rows_per_request, + q_dtype, + valid_lens, + ) -> PrefillSparseBlockTable: + # in place: ascending, INT32_MAX padded, plus the blocks as pool slots / 8 + phys_blocks = sort_candidate_blocks( + blocks, compress_lens, page_table, page_size + ) + schedule = build_sparse_indexer_schedule( + blocks, compress_lens, page_table, page_size, q_dtype, request_ids + ) + ready = torch.cuda.Event() + ready.record() + return PrefillSparseBlockTable( + blocks=blocks, + schedule=schedule, + phys_blocks=phys_blocks, + valid_lens=valid_lens, + ready=ready, + compress_lens=compress_lens, + page_table=page_table, + request_ids=request_ids, + q_dtype=q_dtype, + page_size=page_size, + rows_per_request=rows_per_request, + ) + + def prefill_tail( + self, published: CandidateMetadata, tail_lens: List[int] + ) -> CandidateMetadata: + if self._prefill_dense is not None: + return self._prefill_dense.prefill_tail(published, tail_lens) + table = published + assert isinstance(table, PrefillSparseBlockTable), "prefill block table missing" + rows, start = [], 0 + for n, t in zip(table.rows_per_request, tail_lens): + rows.append(torch.arange(start + n - t, start + n)) + start += n + rows = torch.cat(rows).to(table.blocks.device) + compress_lens = table.compress_lens[rows].contiguous() + _, valid_lens = candidate_row_lens(compress_lens, self.topk_blocks) + return self._prefill_table( + table.blocks[rows].contiguous(), + compress_lens, + table.page_table[rows].contiguous(), + table.page_size, + table.request_ids[rows].contiguous(), + tail_lens, + table.q_dtype, + valid_lens, + ) + + def select_prefill( + self, + published: CandidateMetadata, + inputs: PrefillIndexerInputs, + out_positions: torch.Tensor, + ) -> None: + if self._prefill_dense is not None: + return self._prefill_dense.select_prefill(published, inputs, out_positions) + table = published + assert isinstance(table, PrefillSparseBlockTable), "prefill block table missing" + rows, heads = inputs.q_sf.shape + logits = sparse_logits( + inputs.q_fp4.view(rows, 1, heads, 64), + inputs.q_sf.view(rows, 1, heads), + inputs.k_cache, + inputs.weights.to(torch.bfloat16), + table, + ) + # request-relative compressed positions, -1 padded + topk_transform_bf16_small( + logits, table.valid_lens, table.blocks, out_positions, CANDIDATE_BLOCK_SIZE + ) + out_positions.add_( + torch.where(out_positions >= 0, inputs.request_starts[:, None], 0) + ) diff --git a/python/sglang/srt/layers/attention/dsv4/dense_prefill_indexer.py b/python/sglang/srt/layers/attention/dsv4/dense_prefill_indexer.py index fdc5829786..d4a478a24d 100644 --- a/python/sglang/srt/layers/attention/dsv4/dense_prefill_indexer.py +++ b/python/sglang/srt/layers/attention/dsv4/dense_prefill_indexer.py @@ -1,9 +1,14 @@ from __future__ import annotations +from typing import Iterator, List, Optional + import torch from sglang.srt.layers.attention.dsv4.candidate_indexer import ( + CandidateIndexer, + CandidateMetadata, PrefillCandidateBlocks, + PrefillIndexerInputs, candidate_block_mask, mask_topk_scores, select_candidate_block_ids, @@ -58,18 +63,7 @@ def dense_prefill_topk( width = ceil_align(max((n for _, n in request_lengths), default=0), 4) if row == 0 or width == 0: return selected, published - row_alignment = 128 // q[0].shape[1] - rows_per_chunk = mqa_logits_rows_per_chunk( - num_rows=ceil_align(row, row_alignment), - row_bytes=mqa_logits_row_bytes(width), - budget_bytes=_SCORE_BUDGET_BYTES, - ) - if rows_per_chunk is None: - rows_per_chunk = row - else: - rows_per_chunk = max( - row_alignment, rows_per_chunk // row_alignment * row_alignment - ) + rows_per_chunk = _rows_per_chunk(row, width, heads=q[0].shape[1]) for offset in range(0, row, rows_per_chunk): rows = slice(offset, min(offset + rows_per_chunk, row)) _select_tile( @@ -89,6 +83,141 @@ def dense_prefill_topk( return selected, published +def _rows_per_chunk(rows: int, width: int, *, heads: int) -> int: + """Query rows per logits tile so one fp32 [rows, width] tile fits the + budget; the row count stays a multiple of the kernel's row alignment.""" + row_alignment = 128 // heads + rows_per_chunk = mqa_logits_rows_per_chunk( + num_rows=ceil_align(rows, row_alignment), + row_bytes=mqa_logits_row_bytes(width), + budget_bytes=_SCORE_BUDGET_BYTES, + ) + if rows_per_chunk is None: + return rows + return max(row_alignment, rows_per_chunk // row_alignment * row_alignment) + + +def score_tiles( + inputs: PrefillIndexerInputs, *, width_align: int = 4 +) -> Iterator[tuple[slice, torch.Tensor]]: + """The dense scores of ``inputs`` row tile by row tile under the budget: + ``(rows, logits)`` with fp32 ``logits[i, j]`` the score of query row + ``rows.start + i`` against ``kv[request_starts + j]``, garbage past the row's + ``compress_lens``; the width is the batch's largest context aligned to + ``width_align`` columns.""" + from deep_gemm import fp8_fp4_mqa_logits + + rows = inputs.num_rows + width = ceil_align(max(inputs.lens_per_request, default=0), width_align) + if rows == 0 or width == 0: + return + rows_per_chunk = _rows_per_chunk(rows, width, heads=inputs.q_fp4.shape[1]) + for offset in range(0, rows, rows_per_chunk): + tile = slice(offset, min(offset + rows_per_chunk, rows)) + starts = inputs.request_starts[tile] + yield ( + tile, + fp8_fp4_mqa_logits( + (inputs.q_fp4[tile], inputs.q_sf[tile]), + inputs.kv, + inputs.weights[tile], + starts, + starts + inputs.compress_lens[tile], + False, + width, + ), + ) + + +def _dense_topk( + inputs: PrefillIndexerInputs, + out_positions: torch.Tensor, + *, + topk_blocks: int, + block_size: int, + publish: bool, + candidates: Optional[PrefillCandidateBlocks], +) -> Optional[PrefillCandidateBlocks]: + selected, published = dense_prefill_topk( + q=(inputs.q_fp4, inputs.q_sf), + kv=inputs.kv, + weights=inputs.weights, + starts=inputs.request_starts, + lengths=inputs.compress_lens, + request_lengths=list(zip(inputs.rows_per_request, inputs.lens_per_request)), + topk=out_positions.shape[1], + candidate_topk_blocks=topk_blocks, + candidate_block_size=block_size, + publish_candidates=publish, + candidates=candidates, + ) + out_positions.copy_(selected) + return published + + +def plain_prefill_topk( + inputs: PrefillIndexerInputs, out_positions: torch.Tensor +) -> None: + """The top-k of an index layer outside the candidate scheme, over its dense + scores tile by tile.""" + _dense_topk( + inputs, + out_positions, + topk_blocks=0, + block_size=1, + publish=False, + candidates=None, + ) + + +class DenseCandidateIndexer(CandidateIndexer): + """Candidates as block ids per request (``PrefillCandidateBlocks``): the + source keeps its best blocks from its dense scores, a consumer masks its own + dense scores to -inf outside them and runs the plain top-k, tile by tile. + The prefill implementation for the CP layout; Hopper still runs the same + selection inline in the backend.""" + + def __init__(self, topk_blocks: int, block_size: int): + self.topk_blocks = topk_blocks + self.block_size = block_size + + def publish_prefill( + self, inputs: PrefillIndexerInputs, out_positions: torch.Tensor + ) -> PrefillCandidateBlocks: + published = _dense_topk( + inputs, + out_positions, + topk_blocks=self.topk_blocks, + block_size=self.block_size, + publish=True, + candidates=None, + ) + assert published is not None + return published + + def select_prefill( + self, + published: CandidateMetadata, + inputs: PrefillIndexerInputs, + out_positions: torch.Tensor, + ) -> None: + assert isinstance(published, PrefillCandidateBlocks), "candidate blocks missing" + _dense_topk( + inputs, + out_positions, + topk_blocks=self.topk_blocks, + block_size=self.block_size, + publish=False, + candidates=published, + ) + + def prefill_tail( + self, published: CandidateMetadata, tail_lens: List[int] + ) -> PrefillCandidateBlocks: + assert isinstance(published, PrefillCandidateBlocks), "candidate blocks missing" + return published.tail(tail_lens) + + def _select_tile( *, q: tuple[torch.Tensor, torch.Tensor], diff --git a/test/registered/kernels/ops/attention/test_dsv41_prefill_sparse_indexer.py b/test/registered/kernels/ops/attention/test_dsv41_prefill_sparse_indexer.py new file mode 100644 index 0000000000..96d146a3cf --- /dev/null +++ b/test/registered/kernels/ops/attention/test_dsv41_prefill_sparse_indexer.py @@ -0,0 +1,229 @@ +"""The DeepGEMM two-level indexer on the dense prefill path against the torch +block selection and the dense implementation of the same protocol.""" + +import unittest +from typing import NamedTuple + +import msgspec +import torch + +from sglang.kernels.ops.attention.dsv4.fp4_indexer import quantize_fp4_indexer_tensor +from sglang.srt.layers.attention.dsv4.candidate_indexer import ( + PrefillIndexerInputs, + select_candidate_blocks, +) +from sglang.srt.layers.attention.dsv4.dense_prefill_indexer import ( + DenseCandidateIndexer, +) +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=60, stage="base-b-kernel-unit", runner_config="4-gpu-b200") + +HEADS, DIM = 32, 128 +TOPK_BLOCKS, BLOCK, TOPK = 2048, 8, 512 +PAGE = 128 # index-K pool page size under DeepGEMM's paged sparse logits +CASES = [(256, 40000), (64, 3000)] # (query rows, context): above and below 2048 blocks + +# The sparse kernel scores with bf16 weights and accumulation, the dense one in +# fp32; picks a few bf16 ulps from the selection floor can go either way. +MIN_OVERLAP = 0.95 +FLOOR_TOLERANCE = 2**-5 +# A tail table is a different DeepGEMM row set: its row pairing moves boundary +# scores as well. +MIN_TAIL_OVERLAP = 0.98 + + +class Case(NamedTuple): + dense: torch.Tensor # fp32 [rows, width] scores of every row against its context + lens: torch.Tensor # [rows] int32 causal lengths + inputs: PrefillIndexerInputs + + +def make_case(rows, ctx, seed) -> Case: + """One request: `rows` query tokens whose causal lengths end at `ctx`, its + index K in the pool page layout [pages, PAGE * 64 payload | PAGE * 4 scale].""" + from deep_gemm import fp8_fp4_mqa_logits + + torch.manual_seed(seed) + dev = "cuda" + n_slots = (ctx + PAGE - 1) // PAGE * PAGE + k_fp4, k_sf = quantize_fp4_indexer_tensor( + torch.randn(n_slots, DIM, device=dev, dtype=torch.bfloat16), rne=True + ) + n_pages = n_slots // PAGE + k_cache = torch.cat( + [ + k_fp4.view(torch.uint8).reshape(n_pages, PAGE * 64), + k_sf.view(torch.uint8).reshape(n_pages, PAGE * 4), + ], + 1, + ).view(n_pages, PAGE, 1, 68) + q_fp4, q_sf = quantize_fp4_indexer_tensor( + torch.randn(rows * HEADS, DIM, device=dev, dtype=torch.bfloat16), rne=True + ) + q_fp4, q_sf = q_fp4.view(rows, HEADS, 64), q_sf.view(rows, HEADS) + weights = torch.rand(rows, HEADS, device=dev) + lens = torch.linspace(ctx - rows + 1, ctx, rows, device=dev).to(torch.int32) + lens[0] = 0 # a query that sees no compressed position yet + width = (ctx + 7) // 8 * 8 + starts = torch.zeros(rows, device=dev, dtype=torch.int32) + dense = fp8_fp4_mqa_logits( + (q_fp4, q_sf), + (k_fp4[:width], k_sf[:width]), + weights, + starts, + lens, + False, + width, + ) + inputs = PrefillIndexerInputs( + q_fp4=q_fp4, + q_sf=q_sf, + weights=weights, + compress_lens=lens, + request_starts=starts, + lens_per_request=[ctx], + rows_per_request=[rows], + kv=(k_fp4[:width], k_sf[:width]), + k_cache=k_cache, + page_size=PAGE, + # KV pages of PAGE tokens at ratio 1: the index page table is the KV one + kv_page_table=torch.arange(n_pages, device=dev, dtype=torch.int32) + .expand(rows, -1) + .contiguous(), + kv_page_size=PAGE, + compress_ratio=1, + ) + return Case(dense, lens, inputs) + + +def rows_of(inputs: PrefillIndexerInputs, idx: torch.Tensor) -> PrefillIndexerInputs: + """The inputs of a subset of rows (one request).""" + return msgspec.structs.replace( + inputs, + q_fp4=inputs.q_fp4[idx], + q_sf=inputs.q_sf[idx], + weights=inputs.weights[idx], + compress_lens=inputs.compress_lens[idx], + request_starts=inputs.request_starts[idx], + rows_per_request=[idx.numel()], + kv_page_table=inputs.kv_page_table[idx], + ) + + +def reference_blocks(case: Case) -> torch.Tensor: + """[rows, blocks] bool: the torch block selection of the dense scores.""" + j = torch.arange(case.dense.shape[1], device=case.dense.device) + scores = case.dense.masked_fill(j[None, :] >= case.lens[:, None], -torch.inf) + keep = select_candidate_blocks( + scores, case.lens[:, None], topk_blocks=TOPK_BLOCKS, block_size=BLOCK + ) + return keep.unflatten(1, (-1, BLOCK)).any(-1) + + +def publish(indexer, inputs): + """(published metadata, the source layer's own top-k).""" + own = torch.full((inputs.num_rows, TOPK), -1, dtype=torch.int32, device="cuda") + return indexer.publish_prefill(inputs, own), own + + +def select(indexer, published, inputs): + positions = torch.full( + (inputs.num_rows, TOPK), -1, dtype=torch.int32, device="cuda" + ) + indexer.select_prefill(published, inputs, positions) + return positions + + +def picks(positions: torch.Tensor, row: int) -> set: + return set(positions[row].tolist()) - {-1} + + +@unittest.skipUnless( + torch.cuda.is_available() and torch.cuda.get_device_capability()[0] == 10, + "DeepGEMM's paged sparse MQA logits need SM100", +) +class TestPrefillSparseIndexer(CustomTestCase): + def setUp(self): + from sglang.srt.layers.attention.dsv4.candidate_indexer_deep_gemm import ( + DeepGemmCandidateIndexer, + ) + + self.sparse = DeepGemmCandidateIndexer(TOPK_BLOCKS, BLOCK) + self.dense = DenseCandidateIndexer(TOPK_BLOCKS, BLOCK) + + @torch.inference_mode() + def test_publish_prefill_is_the_torch_block_selection(self): + """The published blocks equal `select_candidate_blocks` block for block + (ragged lengths, an empty row, block counts above and below 2048), and the + source's own top-k from the same tiled pass is the dense one.""" + for rows, ctx in CASES: + with self.subTest(rows=rows, ctx=ctx): + case = make_case(rows, ctx, seed=rows + ctx) + table, own = publish(self.sparse, case.inputs) + expected = reference_blocks(case) + nb = expected.shape[1] + # INT32_MAX padding lands in a spare column instead of a block + got = torch.zeros(rows, nb + 1, dtype=torch.bool, device="cuda") + got.scatter_(1, table.blocks.clamp_max(nb).long(), True) + self.assertTrue(torch.equal(got[:, :nb], expected)) + self.assertFalse(got[0, :nb].any(), "an empty row keeps no block") + _, own_dense = publish(self.dense, case.inputs) + for r in range(rows): + self.assertEqual(picks(own, r), picks(own_dense, r), r) + + @torch.inference_mode() + def test_select_prefill_matches_the_dense_implementation(self): + """A consumer scoring its blocks sparsely picks what the dense + implementation picks from the same inputs, up to bf16 rounding at the + selection floor, and never a position outside its blocks.""" + for rows, ctx in CASES: + with self.subTest(rows=rows, ctx=ctx): + case = make_case(rows, ctx, seed=rows * 3 + ctx) + got = select( + self.sparse, publish(self.sparse, case.inputs)[0], case.inputs + ) + want = select( + self.dense, publish(self.dense, case.inputs)[0], case.inputs + ) + keep = reference_blocks(case).repeat_interleave(BLOCK, dim=1) + self.assertEqual(picks(got, 0), set(), "an empty row selects nothing") + for r in range(1, rows): + g, w = picks(got, r), picks(want, r) + self.assertEqual(len(g), len(w), r) + self.assertTrue( + keep[r, list(g)].all(), f"row {r}: outside its blocks" + ) + self.assertGreaterEqual(len(g & w), MIN_OVERLAP * len(w), r) + floor = case.dense[r, list(w)].min() + self.assertTrue( + ( + case.dense[r, list(g)] + >= floor - floor.abs() * FLOOR_TOLERANCE + ).all(), + r, + ) + + @torch.inference_mode() + def test_prefill_tail_rebuilds_the_last_rows(self): + """The tail table carries the last rows' blocks and lengths and selects + what the full table selects for those rows.""" + rows, tail = 128, 16 + case = make_case(rows, 20000, seed=7) + table, _ = publish(self.sparse, case.inputs) + idx = torch.arange(rows - tail, rows, device="cuda") + sub = self.sparse.prefill_tail(table, [tail]) + self.assertTrue(torch.equal(sub.blocks, table.blocks[idx])) + self.assertTrue(torch.equal(sub.valid_lens, table.valid_lens[idx])) + self.assertTrue(torch.equal(sub.compress_lens, table.compress_lens[idx])) + + full = select(self.sparse, table, case.inputs) + part = select(self.sparse, sub, rows_of(case.inputs, idx)) + for r in range(tail): + a, b = picks(full, rows - tail + r), picks(part, r) + self.assertGreaterEqual(len(a & b), MIN_TAIL_OVERLAP * len(a), r) + + +if __name__ == "__main__": + unittest.main()