From 0967a013c2abe097bc5043511af3d58bf71e9c4d Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Fri, 25 Sep 2026 18:28:04 -0700 Subject: [PATCH] [DSv4.1] Move the ratio-1/2 index top-k ops into kernels/ops/attention/dsv4 (#41291) Co-authored-by: DarkSharpness <2040703891@qq.com> --- .../ops/attention/dsv4/candidate_blocks.py | 164 +++++++++++++++++- .../ops/attention/dsv4/candidate_table.py | 68 ++++---- .../ops/attention/dsv4/index_logits.py | 126 ++++++++++++++ .../sglang/kernels/ops/attention/dsv4/topk.py | 40 +++++ .../layers/attention/deepseek_v4_backend.py | 29 ++-- .../attention/dsv4/candidate_indexer.py | 85 +-------- .../dsv4/candidate_indexer_deep_gemm.py | 130 ++++---------- .../attention/dsv4/dense_prefill_indexer.py | 69 ++------ .../srt/layers/attention/dsv4/indexer.py | 27 --- .../test_dsv41_prefill_sparse_indexer.py | 6 +- .../layers/test_dsv41_candidate_blocks.py | 4 +- 11 files changed, 419 insertions(+), 329 deletions(-) create mode 100644 python/sglang/kernels/ops/attention/dsv4/index_logits.py diff --git a/python/sglang/kernels/ops/attention/dsv4/candidate_blocks.py b/python/sglang/kernels/ops/attention/dsv4/candidate_blocks.py index b70d0f2d2c..b0c378923b 100644 --- a/python/sglang/kernels/ops/attention/dsv4/candidate_blocks.py +++ b/python/sglang/kernels/ops/attention/dsv4/candidate_blocks.py @@ -1,10 +1,24 @@ -"""Per-row candidate block counts and sparse-row lengths for the paged indexer.""" +"""Candidate block selection of the two-level indexer: the per-row block counts +and sparse-row lengths, the block keys and the JIT block top-k, and the torch +block ids and masks.""" + +from typing import Optional, Union import torch +import torch.nn.functional as F import triton import triton.language as tl -from sglang.kernels.jit.utils import is_arch_support_pdl +from sglang.kernels.jit.utils import ( + cache_once, + is_arch_support_pdl, + load_jit, + make_cpp_args, +) + +from .candidate_table import CANDIDATE_BLOCK_SIZE +from .topk import plan_topk_v2, topk_transform_paged_v2 +from .utils import make_name @triton.jit @@ -62,3 +76,149 @@ def candidate_row_lens( **pdl_kwargs, ) return nblocks, valid + + +@cache_once +def _jit_block_amax_module(): + args = make_cpp_args(is_arch_support_pdl()) + return load_jit( + make_name("block_amax"), + *args, + cuda_files=["deepseek_v4/block_amax.cuh"], + cuda_wrappers=[("amax8_varlen", f"BlockAmaxKernel<{args}>::amax8_varlen")], + ) + + +def amax8_varlen( + scores: torch.Tensor, + seq_lens: torch.Tensor, + topk: int = 0, + *, + max_seqlen: int = 0, + out: Optional[torch.Tensor] = None, +) -> torch.Tensor: + """Level-one keys of the two-level indexer: ``out[b, i]`` is the max of + ``scores[b, 8 i : 8 i + 8]`` for ``i < ceil(seq_lens[b] / 8)``, the last of + them ``+inf`` (the newest block is always selected), nothing written past + that count. Rows with at most ``topk`` blocks are skipped (every block is + selected anyway); ``topk=0`` never skips. ``out`` is allocated as + ``[rows, ceil(max_seqlen / 8)]`` when not given, ``max_seqlen`` defaulting to + the width of ``scores``; every ``seq_lens[b]`` must fit in ``8 * out.shape[1]``. + fp32 only for now; ``scores`` rows must be 32-byte aligned (stride a multiple + of 8). Returns ``out``. + """ + if out is None: + num_tokens, max_len = scores.shape + if max_seqlen == 0: + max_seqlen = max_len + out = scores.new_empty(num_tokens, (max_seqlen + 7) // 8) + _jit_block_amax_module().amax8_varlen(scores, seq_lens, out, topk) + return out + + +def amax_topk_blocks( + logits: torch.Tensor, + seq_lens: torch.Tensor, + nblocks: torch.Tensor, + topk_blocks: int, + max_seq_len: Optional[int] = None, +) -> torch.Tensor: + """Per row the ``topk_blocks`` blocks of 8 positions with the largest block + maximum among its first ``seq_lens[b]`` positions, the newest block always + included: block ids in no particular order, ``-1`` past the row's count. + ``nblocks`` is ``ceil(seq_lens / 8)`` as int32.""" + rows = logits.shape[0] + block = CANDIDATE_BLOCK_SIZE + if max_seq_len is None: + max_seq_len = logits.shape[1] + # NOTE: plan cannot be the previous kernel of topk_transform_paged_v2 + plan = plan_topk_v2(nblocks) + # block maxima, the newest block +inf; the top-k reads each row up to nblocks + # only, so nothing past a row's keys is initialised (v2 needs stride % 4 == 0) + keys = logits.new_empty(rows, -(-max_seq_len // (4 * block)) * 4) + amax8_varlen(logits, seq_lens, out=keys) + blocks = torch.empty(rows, topk_blocks, dtype=torch.int32, device=logits.device) + topk_transform_paged_v2(keys, nblocks, None, blocks, 1, plan) + return blocks + + +def mask_topk_scores( + scores: torch.Tensor, + indices: torch.Tensor, + offsets: Optional[torch.Tensor] = None, +) -> torch.Tensor: + """Keep masked indexer scores out of attention even when top-k underfills.""" + columns = indices.to(torch.int64) + if offsets is not None: + columns = columns - offsets[:, None] + selected_scores = scores.gather(1, columns.clamp(0, scores.shape[1] - 1)) + valid = ( + (columns >= 0) & (columns < scores.shape[1]) & (selected_scores > -torch.inf) + ) + return indices.masked_fill(~valid, -1) + + +def _candidate_block_topk( + logits: torch.Tensor, + compress_lens: Union[torch.Tensor, int], + topk_blocks: int, + block_size: int, +) -> torch.return_types.topk: + width = logits.size(-1) + padding = -width % block_size + scores = F.pad(logits, (0, padding), value=-torch.inf) if padding else logits + scores = scores.unflatten(-1, (-1, block_size)).amax(dim=-1) + num_blocks = scores.size(-1) + + last = (compress_lens - 1) // block_size + scores = scores.masked_fill( + torch.arange(num_blocks, device=logits.device) == last, torch.inf + ) + + return scores.topk(min(topk_blocks, num_blocks), dim=-1) + + +def select_candidate_block_ids( + logits: torch.Tensor, + compress_lens: Union[torch.Tensor, int], + topk_blocks: int, + block_size: int, +) -> torch.Tensor: + top = _candidate_block_topk( + logits=logits, + compress_lens=compress_lens, + topk_blocks=topk_blocks, + block_size=block_size, + ) + return top.indices.to(torch.int32).masked_fill_(~(top.values > -torch.inf), -1) + + +def candidate_block_mask( + blocks: torch.Tensor, width: int, block_size: int +) -> torch.Tensor: + num_blocks = (width + block_size - 1) // block_size + keep = torch.zeros( + (*blocks.shape[:-1], num_blocks + 1), dtype=torch.bool, device=blocks.device + ) + keep.scatter_(-1, blocks.to(torch.int64).masked_fill(blocks < 0, num_blocks), True) + return keep[..., :num_blocks].repeat_interleave(block_size, dim=-1)[..., :width] + + +def select_candidate_blocks( + logits: torch.Tensor, + compress_lens: Union[torch.Tensor, int], + topk_blocks: int, + block_size: int, +) -> torch.Tensor: + top = _candidate_block_topk( + logits=logits, + compress_lens=compress_lens, + topk_blocks=topk_blocks, + block_size=block_size, + ) + width = logits.shape[-1] + num_blocks = (width + block_size - 1) // block_size + keep = torch.zeros( + (*logits.shape[:-1], num_blocks), dtype=torch.bool, device=logits.device + ).scatter_(-1, top.indices, top.values > -torch.inf) + return keep.repeat_interleave(block_size, dim=-1)[..., :width] diff --git a/python/sglang/kernels/ops/attention/dsv4/candidate_table.py b/python/sglang/kernels/ops/attention/dsv4/candidate_table.py index e956d2e690..0887165718 100644 --- a/python/sglang/kernels/ops/attention/dsv4/candidate_table.py +++ b/python/sglang/kernels/ops/attention/dsv4/candidate_table.py @@ -1,5 +1,5 @@ -"""Candidate-table kernels of the two-level indexer: level-one block keys and -the sorted, page-transformed block table.""" +"""The sparse table of the two-level indexer from a row's selected blocks: the +sorted, page-transformed block table and DeepGEMM's schedule for it.""" from __future__ import annotations @@ -19,43 +19,7 @@ from .utils import make_name if TYPE_CHECKING: pass - -@cache_once -def _jit_block_amax_module(): - args = make_cpp_args(is_arch_support_pdl()) - return load_jit( - make_name("block_amax"), - *args, - cuda_files=["deepseek_v4/block_amax.cuh"], - cuda_wrappers=[("amax8_varlen", f"BlockAmaxKernel<{args}>::amax8_varlen")], - ) - - -def amax8_varlen( - scores: torch.Tensor, - seq_lens: torch.Tensor, - topk: int = 0, - *, - max_seqlen: int = 0, - out: Optional[torch.Tensor] = None, -) -> torch.Tensor: - """Level-one keys of the two-level indexer: ``out[b, i]`` is the max of - ``scores[b, 8 i : 8 i + 8]`` for ``i < ceil(seq_lens[b] / 8)``, the last of - them ``+inf`` (the newest block is always selected), nothing written past - that count. Rows with at most ``topk`` blocks are skipped (every block is - selected anyway); ``topk=0`` never skips. ``out`` is allocated as - ``[rows, ceil(max_seqlen / 8)]`` when not given, ``max_seqlen`` defaulting to - the width of ``scores``; every ``seq_lens[b]`` must fit in ``8 * out.shape[1]``. - fp32 only for now; ``scores`` rows must be 32-byte aligned (stride a multiple - of 8). Returns ``out``. - """ - if out is None: - num_tokens, max_len = scores.shape - if max_seqlen == 0: - max_seqlen = max_len - out = scores.new_empty(num_tokens, (max_seqlen + 7) // 8) - _jit_block_amax_module().amax8_varlen(scores, seq_lens, out, topk) - return out +CANDIDATE_BLOCK_SIZE = 8 # positions per block; DeepGEMM accepts 8 or 16 @cache_once @@ -91,3 +55,29 @@ def sort_candidate_blocks( blocks, seq_lens, page_table, out_pages, page_size ) return out_pages + + +def build_sparse_indexer_schedule( + blocks: torch.Tensor, + seq_lens: torch.Tensor, + page_table: torch.Tensor, + page_size: int, + q_dtype: torch.dtype, + request_ids: torch.Tensor, +) -> torch.Tensor: + """DeepGEMM's schedule for the published blocks: ``seq_lens`` ``[rows]`` + int32, ``page_table`` ``[rows, pages]`` int32 at the index pool's page size. + ``request_ids`` ``[rows]`` int32 lets DeepGEMM pair two rows of a request on + one KV pass; each row keeps its own block list and output layout, and paired + rows must share their page-table row.""" + import deep_gemm + + return deep_gemm.get_paged_sparse_mqa_logits_metadata( + seq_lens.contiguous(), + page_table, + request_ids, + page_size, + blocks, + q_dtype, + CANDIDATE_BLOCK_SIZE, + ) diff --git a/python/sglang/kernels/ops/attention/dsv4/index_logits.py b/python/sglang/kernels/ops/attention/dsv4/index_logits.py new file mode 100644 index 0000000000..7832d770c5 --- /dev/null +++ b/python/sglang/kernels/ops/attention/dsv4/index_logits.py @@ -0,0 +1,126 @@ +"""The index logits of the DeepSeek V4.1 ratio-1/2 index layers, by the layout of +the index K they read: the flattened K of a prefill chunk (tile by tile under a +memory budget), the paged pool, or the published blocks of a sparse table.""" + +from __future__ import annotations + +from typing import Iterator, Tuple + +import torch + +from sglang.srt.layers.attention.mqa_logits_utils import ( + mqa_logits_row_bytes, + mqa_logits_rows_per_chunk, +) +from sglang.srt.utils.common import ceil_align + +from .candidate_table import CANDIDATE_BLOCK_SIZE + + +def flat_index_logits_rows_per_tile( + rows: int, width: int, *, heads: int, budget_bytes: int +) -> int: + """Rows per fp32 logits tile within ``budget_bytes``, at the kernel's row + alignment.""" + row_alignment = 128 // heads + rows_per_chunk = mqa_logits_rows_per_chunk( + num_rows=ceil_align(rows, row_alignment), + row_bytes=mqa_logits_row_bytes(width), + budget_bytes=budget_bytes, + ) + if rows_per_chunk is None: + return rows + return max(row_alignment, rows_per_chunk // row_alignment * row_alignment) + + +def flat_index_logits_tiles( + *, + q: tuple[torch.Tensor, torch.Tensor], + kv: tuple[torch.Tensor, torch.Tensor], + weights: torch.Tensor, + starts: torch.Tensor, + lengths: torch.Tensor, + context_lengths: list[int], + budget_bytes: int, + width_align: int = 4, +) -> Iterator[tuple[slice, torch.Tensor]]: + """``(rows, logits)`` per row tile, each fp32 tile within ``budget_bytes``: + ``logits[i, j]`` scores query row ``rows.start + i`` against ``kv[starts + j]``, + garbage past the row's ``lengths``; the width is ``max(context_lengths)`` + aligned to ``width_align``.""" + from deep_gemm import fp8_fp4_mqa_logits + + rows = q[0].shape[0] + width = ceil_align(max(context_lengths, default=0), width_align) + if rows == 0 or width == 0: + return + rows_per_chunk = flat_index_logits_rows_per_tile( + rows, width, heads=q[0].shape[1], budget_bytes=budget_bytes + ) + for offset in range(0, rows, rows_per_chunk): + tile = slice(offset, min(offset + rows_per_chunk, rows)) + tile_starts = starts[tile] + yield ( + tile, + fp8_fp4_mqa_logits( + (q[0][tile], q[1][tile]), + kv, + weights[tile], + tile_starts, + tile_starts + lengths[tile], + False, + width, + ), + ) + + +def deep_gemm_fp4_paged_mqa_logits( + q_fp4: Tuple[torch.Tensor, torch.Tensor], + k_cache: torch.Tensor, + weights: torch.Tensor, + seq_lens: torch.Tensor, + page_table: torch.Tensor, + deep_gemm_metadata, + max_seq_len: int, +) -> torch.Tensor: + """DeepGEMM paged fp4 logits; no hadamard, the reference does not apply one.""" + from deep_gemm import fp8_fp4_paged_mqa_logits + + sl = seq_lens.to(torch.int32) + if sl.dim() == 1: + sl = sl.unsqueeze(-1) + return fp8_fp4_paged_mqa_logits( + q_fp4, + k_cache, + weights, + sl, + page_table, + deep_gemm_metadata, + max_seq_len, + False, + ) + + +def sparse_logits( + q_fp4: torch.Tensor, + q_sf: torch.Tensor, + k_cache: torch.Tensor, + weights: torch.Tensor, + schedule: torch.Tensor, + topk_blocks: int, +) -> torch.Tensor: + """bf16 logits ``[rows, topk_blocks * 8]`` of the published blocks: ``q_fp4`` + ``[rows, 1, heads, 64]`` int8 with ``q_sf`` ``[rows, 1, heads]`` int32 (packed + ue8m0), ``k_cache`` ``[pages, page_size, 1, 68]`` uint8 whose page stride is + a multiple of 512 bytes, ``weights`` ``[rows, heads]`` bf16, ``schedule`` the + ``build_sparse_indexer_schedule`` of the ``topk_blocks`` published blocks.""" + import deep_gemm + + return deep_gemm.fp8_fp4_paged_sparse_mqa_logits( + (q_fp4, q_sf), + k_cache, + weights, + schedule, + topk_blocks, + CANDIDATE_BLOCK_SIZE, + ) diff --git a/python/sglang/kernels/ops/attention/dsv4/topk.py b/python/sglang/kernels/ops/attention/dsv4/topk.py index 74128f2b0f..1a57689230 100644 --- a/python/sglang/kernels/ops/attention/dsv4/topk.py +++ b/python/sglang/kernels/ops/attention/dsv4/topk.py @@ -13,6 +13,7 @@ from sglang.kernels.jit.utils import ( ) from sglang.srt.utils import is_xpu +from .candidate_table import CANDIDATE_BLOCK_SIZE from .utils import make_name @@ -123,6 +124,30 @@ def topk_transform_paged( ) +def topk_transform_paged_torch( + scores: torch.Tensor, + seq_lens: torch.Tensor, + page_tables: torch.Tensor, + out_page_indices: torch.Tensor, + page_size: int, + out_raw_indices: Optional[torch.Tensor] = None, +) -> None: + """The torch ``topk_transform_paged``: top-``k`` (``k = out_page_indices.shape[1]``) + of each row within its first ``seq_lens[b]`` columns, ascending, as pool slots + through ``page_tables`` and as positions when given; ``-1`` padded.""" + topk = out_page_indices.shape[1] + columns = torch.arange(scores.shape[1], device=seq_lens.device) + lens_c = seq_lens.unsqueeze(-1) + # Columns past a row's length hold garbage. + s = scores.masked_fill(columns[None, :] >= lens_c, -torch.inf) + idx = s.topk(topk, dim=-1, sorted=False).indices.sort(dim=-1).values + reach = idx < lens_c + slots = page_tables.gather(-1, idx // page_size) * page_size + (idx % page_size) + out_page_indices.copy_(torch.where(reach, slots, -1).to(torch.int32)) + if out_raw_indices is not None: + out_raw_indices.copy_(torch.where(reach, idx, -1).to(torch.int32)) + + # metadata is (batch+1, 2) int32: row 0 = {cluster_threshold, num_cluster_items}; # rows 1..N = {batch_id, seq_len} of items routed to the persistent cluster pool. _PLAN_METADATA_INTS_PER_BATCH = 2 @@ -301,3 +326,18 @@ def topk_transform_packed_v2( page_size, row_to_batch, ) + + +def topk_transform_sparse( + logits: torch.Tensor, + valid_lens: torch.Tensor, + phys_blocks: torch.Tensor, + page_indices: torch.Tensor, +) -> None: + """Top-``k`` (``k = page_indices.shape[1]``) of each row of the bf16 sparse + ``logits`` within its first ``valid_lens[b]`` columns, as pool slots through + ``phys_blocks`` (the published blocks as pool slots / 8); ``-1`` padded, + unordered.""" + topk_transform_bf16_small( + logits, valid_lens, phys_blocks, page_indices, CANDIDATE_BLOCK_SIZE + ) diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index 467110ffa1..4a7255c494 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -19,6 +19,10 @@ import msgspec import torch import torch.nn.functional as F +from sglang.kernels.ops.attention.dsv4.candidate_blocks import ( + mask_topk_scores, + select_candidate_blocks, +) from sglang.kernels.ops.attention.dsv4.decode_attention_sm100 import ( can_use_swapab_attention, ) @@ -30,6 +34,9 @@ from sglang.kernels.ops.attention.dsv4.dequant_k_cache import ( q8kv8_padded_num_heads, ) from sglang.kernels.ops.attention.dsv4.fp4_indexer import fp4_index_logits_decode +from sglang.kernels.ops.attention.dsv4.index_logits import ( + deep_gemm_fp4_paged_mqa_logits, +) from sglang.kernels.ops.attention.dsv4.kv_layout import KVLayout from sglang.kernels.ops.attention.dsv4.metadata_kernel import ( fill_all_compressed_indices, @@ -38,6 +45,7 @@ from sglang.kernels.ops.attention.dsv4.metadata_kernel import ( init_compression_metadata as _init_compression_metadata_triton, ) from sglang.kernels.ops.attention.dsv4.online_c128_mtp import OnlineC128MTPController +from sglang.kernels.ops.attention.dsv4.topk import topk_transform_paged_torch from sglang.kernels.ops.attention.dsv4_attn_metadata_kernels import ( BuildCausalSwaPageIndices, BuildPageTablePositions, @@ -64,9 +72,7 @@ from sglang.srt.layers.attention.dsv4.candidate_indexer import ( cut_request_masks, expand_index_page_table, make_candidate_indexer, - mask_topk_scores, published_masks, - select_candidate_blocks, ) from sglang.srt.layers.attention.dsv4.compressor_v2 import ( CompressorBackendMixin, @@ -80,7 +86,6 @@ from sglang.srt.layers.attention.dsv4.dsv41_sparse import ( ) from sglang.srt.layers.attention.dsv4.indexer import ( C4IndexerBackendMixin, - deep_gemm_fp4_paged_mqa_logits, topk_transform_paged_from_metadata, ) from sglang.srt.layers.attention.dsv4.metadata import ( @@ -3340,7 +3345,6 @@ class DeepseekV4AttnBackend( page_indices = core.sparse_page_indices(ratio) raw_indices = core.sparse_raw_indices(ratio) topk = min(indexer.index_topk, width) - columns = torch.arange(width, device=lens.device) for rows, plan in metadata.row_chunks(): logits = deep_gemm_fp4_paged_mqa_logits( (q_fp4[rows], q_sf[rows]), @@ -3351,17 +3355,14 @@ class DeepseekV4AttnBackend( plan, width, ) - lens_c = lens[rows].unsqueeze(-1) - # Columns past a row's length hold garbage. - s = logits.masked_fill(columns[None, :] >= lens_c, -torch.inf) - idx = s.topk(topk, dim=-1, sorted=False).indices.sort(dim=-1).values - reach = idx < lens_c - slots = page_table[rows].gather(-1, idx // page_size) * page_size + ( - idx % page_size + topk_transform_paged_torch( + logits, + lens[rows], + page_table[rows], + page_indices[rows, :topk], + page_size, + raw_indices[rows, :topk] if raw_indices is not None else None, ) - page_indices[rows, :topk] = torch.where(reach, slots, -1).to(torch.int32) - if raw_indices is not None: - raw_indices[rows, :topk] = torch.where(reach, idx, -1).to(torch.int32) def _low_ratio_index_topk_decode(self, layer, x, q_lora, pos, req=None) -> None: from sglang.srt.model_executor.runner_utils.capture_mode import ( diff --git a/python/sglang/srt/layers/attention/dsv4/candidate_indexer.py b/python/sglang/srt/layers/attention/dsv4/candidate_indexer.py index f6df650b7d..9a78e2618c 100644 --- a/python/sglang/srt/layers/attention/dsv4/candidate_indexer.py +++ b/python/sglang/srt/layers/attention/dsv4/candidate_indexer.py @@ -2,11 +2,10 @@ from __future__ import annotations from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import List, Optional, Tuple, Union +from typing import List, Optional, Tuple import msgspec import torch -import torch.nn.functional as F from sglang.srt.layers.attention.dsv4.metadata import PagedIndexerMetadata from sglang.srt.runtime_context import get_parallel, get_platform @@ -193,85 +192,3 @@ class PrefillCandidateBlocks(CandidateMetadata, msgspec.Struct): def published_masks(candidate) -> CandidateMasks: assert isinstance(candidate, CandidateMasks), "candidate masks missing" return candidate - - -def mask_topk_scores( - scores: torch.Tensor, - indices: torch.Tensor, - offsets: Optional[torch.Tensor] = None, -) -> torch.Tensor: - """Keep masked indexer scores out of attention even when top-k underfills.""" - columns = indices.to(torch.int64) - if offsets is not None: - columns = columns - offsets[:, None] - selected_scores = scores.gather(1, columns.clamp(0, scores.shape[1] - 1)) - valid = ( - (columns >= 0) & (columns < scores.shape[1]) & (selected_scores > -torch.inf) - ) - return indices.masked_fill(~valid, -1) - - -def _candidate_block_topk( - logits: torch.Tensor, - compress_lens: Union[torch.Tensor, int], - topk_blocks: int, - block_size: int, -) -> torch.return_types.topk: - width = logits.size(-1) - padding = -width % block_size - scores = F.pad(logits, (0, padding), value=-torch.inf) if padding else logits - scores = scores.unflatten(-1, (-1, block_size)).amax(dim=-1) - num_blocks = scores.size(-1) - - last = (compress_lens - 1) // block_size - scores = scores.masked_fill( - torch.arange(num_blocks, device=logits.device) == last, torch.inf - ) - - return scores.topk(min(topk_blocks, num_blocks), dim=-1) - - -def select_candidate_block_ids( - logits: torch.Tensor, - compress_lens: Union[torch.Tensor, int], - topk_blocks: int, - block_size: int, -) -> torch.Tensor: - top = _candidate_block_topk( - logits=logits, - compress_lens=compress_lens, - topk_blocks=topk_blocks, - block_size=block_size, - ) - return top.indices.to(torch.int32).masked_fill_(~(top.values > -torch.inf), -1) - - -def candidate_block_mask( - blocks: torch.Tensor, width: int, block_size: int -) -> torch.Tensor: - num_blocks = (width + block_size - 1) // block_size - keep = torch.zeros( - (*blocks.shape[:-1], num_blocks + 1), dtype=torch.bool, device=blocks.device - ) - keep.scatter_(-1, blocks.to(torch.int64).masked_fill(blocks < 0, num_blocks), True) - return keep[..., :num_blocks].repeat_interleave(block_size, dim=-1)[..., :width] - - -def select_candidate_blocks( - logits: torch.Tensor, - compress_lens: Union[torch.Tensor, int], - topk_blocks: int, - block_size: int, -) -> torch.Tensor: - top = _candidate_block_topk( - logits=logits, - compress_lens=compress_lens, - topk_blocks=topk_blocks, - block_size=block_size, - ) - width = logits.shape[-1] - num_blocks = (width + block_size - 1) // block_size - keep = torch.zeros( - (*logits.shape[:-1], num_blocks), dtype=torch.bool, device=logits.device - ).scatter_(-1, top.indices, top.values > -torch.inf) - return keep.repeat_interleave(block_size, dim=-1)[..., :width] diff --git a/python/sglang/srt/layers/attention/dsv4/candidate_indexer_deep_gemm.py b/python/sglang/srt/layers/attention/dsv4/candidate_indexer_deep_gemm.py index 26f3081338..2cb85a5bb9 100644 --- a/python/sglang/srt/layers/attention/dsv4/candidate_indexer_deep_gemm.py +++ b/python/sglang/srt/layers/attention/dsv4/candidate_indexer_deep_gemm.py @@ -5,16 +5,26 @@ from typing import List, Optional import torch -from sglang.kernels.ops.attention.dsv4.candidate_blocks import candidate_row_lens -from sglang.kernels.ops.attention.dsv4.candidate_table import ( +from sglang.kernels.ops.attention.dsv4.candidate_blocks import ( amax8_varlen, + amax_topk_blocks, + candidate_row_lens, +) +from sglang.kernels.ops.attention.dsv4.candidate_table import ( + CANDIDATE_BLOCK_SIZE, + build_sparse_indexer_schedule, sort_candidate_blocks, ) +from sglang.kernels.ops.attention.dsv4.index_logits import ( + deep_gemm_fp4_paged_mqa_logits, + flat_index_logits_tiles, + sparse_logits, +) from sglang.kernels.ops.attention.dsv4.topk import ( - plan_topk_v2, topk_transform_bf16_small, topk_transform_paged_v2, topk_transform_ragged_v2, + topk_transform_sparse, ) from sglang.srt.layers.attention.dsv4.candidate_indexer import ( CandidateIndexer, @@ -24,16 +34,13 @@ from sglang.srt.layers.attention.dsv4.candidate_indexer import ( expand_index_page_table, ) from sglang.srt.layers.attention.dsv4.dense_prefill_indexer import ( + _SCORE_BUDGET_BYTES, DenseCandidateIndexer, - score_tiles, ) from sglang.srt.layers.attention.dsv4.indexer import ( - deep_gemm_fp4_paged_mqa_logits, topk_transform_paged_from_metadata, ) -CANDIDATE_BLOCK_SIZE = 8 # positions per block; DeepGEMM accepts 8 or 16 - @dataclass class SparseBlockTable(CandidateMetadata): @@ -53,96 +60,6 @@ class SparseBlockTable(CandidateMetadata): ready: torch.cuda.Event -def amax_topk_blocks( - logits: torch.Tensor, - seq_lens: torch.Tensor, - nblocks: torch.Tensor, - topk_blocks: int, - max_seq_len: Optional[int] = None, -) -> torch.Tensor: - """Per row the ``topk_blocks`` blocks of 8 positions with the largest block - maximum among its first ``seq_lens[b]`` positions, the newest block always - included: block ids in no particular order, ``-1`` past the row's count. - ``nblocks`` is ``ceil(seq_lens / 8)`` as int32.""" - rows = logits.shape[0] - block = CANDIDATE_BLOCK_SIZE - if max_seq_len is None: - max_seq_len = logits.shape[1] - # NOTE: plan cannot be the previous kernel of topk_transform_paged_v2 - plan = plan_topk_v2(nblocks) - # block maxima, the newest block +inf; the top-k reads each row up to nblocks - # only, so nothing past a row's keys is initialised (v2 needs stride % 4 == 0) - keys = logits.new_empty(rows, -(-max_seq_len // (4 * block)) * 4) - amax8_varlen(logits, seq_lens, out=keys) - blocks = torch.empty(rows, topk_blocks, dtype=torch.int32, device=logits.device) - topk_transform_paged_v2(keys, nblocks, None, blocks, 1, plan) - return blocks - - -def build_sparse_indexer_schedule( - blocks: torch.Tensor, - seq_lens: torch.Tensor, - page_table: torch.Tensor, - page_size: int, - q_dtype: torch.dtype, - request_ids: torch.Tensor, -) -> torch.Tensor: - """DeepGEMM's schedule for the published blocks: ``seq_lens`` ``[rows]`` - int32, ``page_table`` ``[rows, pages]`` int32 at the index pool's page size. - ``request_ids`` ``[rows]`` int32 lets DeepGEMM pair two rows of a request on - one KV pass; each row keeps its own block list and output layout, and paired - rows must share their page-table row.""" - import deep_gemm - - return deep_gemm.get_paged_sparse_mqa_logits_metadata( - seq_lens.contiguous(), - page_table, - request_ids, - page_size, - blocks, - q_dtype, - CANDIDATE_BLOCK_SIZE, - ) - - -def sparse_logits( - q_fp4: torch.Tensor, - q_sf: torch.Tensor, - k_cache: torch.Tensor, - weights: torch.Tensor, - table: SparseBlockTable, -) -> torch.Tensor: - """bf16 logits ``[rows, topk_blocks * 8]`` of the published blocks: ``q_fp4`` - ``[rows, 1, heads, 64]`` int8 with ``q_sf`` ``[rows, 1, heads]`` int32 (packed - ue8m0), ``k_cache`` ``[pages, page_size, 1, 68]`` uint8 whose page stride is - a multiple of 512 bytes, ``weights`` ``[rows, heads]`` bf16.""" - import deep_gemm - - return deep_gemm.fp8_fp4_paged_sparse_mqa_logits( - (q_fp4, q_sf), - k_cache, - weights, - table.schedule, - table.blocks.shape[1], - CANDIDATE_BLOCK_SIZE, - ) - - -def topk_transform_sparse( - logits: torch.Tensor, - valid_lens: torch.Tensor, - table: SparseBlockTable, - page_indices: torch.Tensor, -) -> None: - """Top-``k`` (``k = page_indices.shape[1]``) of every row of the sparse - ``logits`` (bf16 ``[rows, topk_blocks * 8]``) within its first ``valid_lens[b]`` - columns, written as pool slots, ``-1`` where a row has fewer than ``k`` valid - columns, in no particular order.""" - topk_transform_bf16_small( - logits, valid_lens, table.phys_blocks, page_indices, CANDIDATE_BLOCK_SIZE - ) - - @dataclass class PrefillSparseBlockTable(SparseBlockTable): """Published by a prefill chunk's candidate source: one row per query token. @@ -347,7 +264,8 @@ class DeepGemmCandidateIndexer(CandidateIndexer): inputs.q_sf, inputs.k_cache, inputs.weights.to(torch.bfloat16), - table, + table.schedule, + table.blocks.shape[1], ) def select_decode( @@ -362,7 +280,7 @@ class DeepGemmCandidateIndexer(CandidateIndexer): torch.cuda.current_stream().wait_event(table.ready) logits = self._scores(table, inputs) # decode carries no raw_indices; the kernel writes slots only - topk_transform_sparse(logits, table.valid_lens, table, page_indices) + topk_transform_sparse(logits, table.valid_lens, table.phys_blocks, page_indices) def publish_prefill( self, inputs: PrefillIndexerInputs, out_positions: torch.Tensor @@ -377,7 +295,16 @@ class DeepGemmCandidateIndexer(CandidateIndexer): nblocks, valid_lens = candidate_row_lens(inputs.compress_lens, self.topk_blocks) blocks = torch.empty(rows, self.topk_blocks, dtype=torch.int32, device=device) # the block keys read the score rows through 32-byte vectors - for tile, logits in score_tiles(inputs, width_align=8): + for tile, logits in flat_index_logits_tiles( + q=(inputs.q_fp4, inputs.q_sf), + kv=inputs.kv, + weights=inputs.weights, + starts=inputs.request_starts, + lengths=inputs.compress_lens, + context_lengths=inputs.lens_per_request, + budget_bytes=_SCORE_BUDGET_BYTES, + width_align=8, + ): lens = inputs.compress_lens[tile] topk_transform_ragged_v2( logits, @@ -500,7 +427,8 @@ class DeepGemmCandidateIndexer(CandidateIndexer): inputs.q_sf.view(rows, 1, heads), inputs.k_cache, inputs.weights.to(torch.bfloat16), - table, + table.schedule, + table.blocks.shape[1], ) # request-relative compressed positions, -1 padded topk_transform_bf16_small( diff --git a/python/sglang/srt/layers/attention/dsv4/dense_prefill_indexer.py b/python/sglang/srt/layers/attention/dsv4/dense_prefill_indexer.py index d4a478a24d..bbde5ce9a4 100644 --- a/python/sglang/srt/layers/attention/dsv4/dense_prefill_indexer.py +++ b/python/sglang/srt/layers/attention/dsv4/dense_prefill_indexer.py @@ -1,21 +1,22 @@ from __future__ import annotations -from typing import Iterator, List, Optional +from typing import List, Optional import torch +from sglang.kernels.ops.attention.dsv4.candidate_blocks import ( + candidate_block_mask, + mask_topk_scores, + select_candidate_block_ids, +) +from sglang.kernels.ops.attention.dsv4.index_logits import ( + flat_index_logits_rows_per_tile, +) from sglang.srt.layers.attention.dsv4.candidate_indexer import ( CandidateIndexer, CandidateMetadata, PrefillCandidateBlocks, PrefillIndexerInputs, - candidate_block_mask, - mask_topk_scores, - select_candidate_block_ids, -) -from sglang.srt.layers.attention.mqa_logits_utils import ( - mqa_logits_row_bytes, - mqa_logits_rows_per_chunk, ) from sglang.srt.utils.common import ceil_align @@ -63,7 +64,9 @@ def dense_prefill_topk( width = ceil_align(max((n for _, n in request_lengths), default=0), 4) if row == 0 or width == 0: return selected, published - rows_per_chunk = _rows_per_chunk(row, width, heads=q[0].shape[1]) + rows_per_chunk = flat_index_logits_rows_per_tile( + row, width, heads=q[0].shape[1], budget_bytes=_SCORE_BUDGET_BYTES + ) for offset in range(0, row, rows_per_chunk): rows = slice(offset, min(offset + rows_per_chunk, row)) _select_tile( @@ -83,52 +86,6 @@ def dense_prefill_topk( return selected, published -def _rows_per_chunk(rows: int, width: int, *, heads: int) -> int: - """Query rows per logits tile so one fp32 [rows, width] tile fits the - budget; the row count stays a multiple of the kernel's row alignment.""" - row_alignment = 128 // heads - rows_per_chunk = mqa_logits_rows_per_chunk( - num_rows=ceil_align(rows, row_alignment), - row_bytes=mqa_logits_row_bytes(width), - budget_bytes=_SCORE_BUDGET_BYTES, - ) - if rows_per_chunk is None: - return rows - return max(row_alignment, rows_per_chunk // row_alignment * row_alignment) - - -def score_tiles( - inputs: PrefillIndexerInputs, *, width_align: int = 4 -) -> Iterator[tuple[slice, torch.Tensor]]: - """The dense scores of ``inputs`` row tile by row tile under the budget: - ``(rows, logits)`` with fp32 ``logits[i, j]`` the score of query row - ``rows.start + i`` against ``kv[request_starts + j]``, garbage past the row's - ``compress_lens``; the width is the batch's largest context aligned to - ``width_align`` columns.""" - from deep_gemm import fp8_fp4_mqa_logits - - rows = inputs.num_rows - width = ceil_align(max(inputs.lens_per_request, default=0), width_align) - if rows == 0 or width == 0: - return - rows_per_chunk = _rows_per_chunk(rows, width, heads=inputs.q_fp4.shape[1]) - for offset in range(0, rows, rows_per_chunk): - tile = slice(offset, min(offset + rows_per_chunk, rows)) - starts = inputs.request_starts[tile] - yield ( - tile, - fp8_fp4_mqa_logits( - (inputs.q_fp4[tile], inputs.q_sf[tile]), - inputs.kv, - inputs.weights[tile], - starts, - starts + inputs.compress_lens[tile], - False, - width, - ), - ) - - def _dense_topk( inputs: PrefillIndexerInputs, out_positions: torch.Tensor, @@ -235,7 +192,7 @@ def _select_tile( ) -> None: from deep_gemm import fp8_fp4_mqa_logits - from sglang.kernels.ops.attention.dsv4 import topk_transform_ragged_v2 + from sglang.kernels.ops.attention.dsv4.topk import topk_transform_ragged_v2 logits = fp8_fp4_mqa_logits(q, kv, weights, starts, starts + lengths, False, width) if publish is not None or consume is not None: diff --git a/python/sglang/srt/layers/attention/dsv4/indexer.py b/python/sglang/srt/layers/attention/dsv4/indexer.py index 31b1230706..f0bac3d410 100644 --- a/python/sglang/srt/layers/attention/dsv4/indexer.py +++ b/python/sglang/srt/layers/attention/dsv4/indexer.py @@ -443,33 +443,6 @@ def topk_transform_flashinfer_fused( ) -def deep_gemm_fp4_paged_mqa_logits( - q_fp4: Tuple[torch.Tensor, torch.Tensor], - k_cache: torch.Tensor, - weights: torch.Tensor, - seq_lens: torch.Tensor, - page_table: torch.Tensor, - deep_gemm_metadata, - max_seq_len: int, -) -> torch.Tensor: - """DeepGEMM paged fp4 logits; no hadamard, the reference does not apply one.""" - from deep_gemm import fp8_fp4_paged_mqa_logits - - sl = seq_lens.to(torch.int32) - if sl.dim() == 1: - sl = sl.unsqueeze(-1) - return fp8_fp4_paged_mqa_logits( - q_fp4, - k_cache, - weights, - sl, - page_table, - deep_gemm_metadata, - max_seq_len, - False, - ) - - def topk_transform_paged_from_metadata( logits: torch.Tensor, metadata, diff --git a/test/registered/kernels/ops/attention/test_dsv41_prefill_sparse_indexer.py b/test/registered/kernels/ops/attention/test_dsv41_prefill_sparse_indexer.py index 96d146a3cf..e5d62892ec 100644 --- a/test/registered/kernels/ops/attention/test_dsv41_prefill_sparse_indexer.py +++ b/test/registered/kernels/ops/attention/test_dsv41_prefill_sparse_indexer.py @@ -7,11 +7,9 @@ from typing import NamedTuple import msgspec import torch +from sglang.kernels.ops.attention.dsv4.candidate_blocks import select_candidate_blocks from sglang.kernels.ops.attention.dsv4.fp4_indexer import quantize_fp4_indexer_tensor -from sglang.srt.layers.attention.dsv4.candidate_indexer import ( - PrefillIndexerInputs, - select_candidate_blocks, -) +from sglang.srt.layers.attention.dsv4.candidate_indexer import PrefillIndexerInputs from sglang.srt.layers.attention.dsv4.dense_prefill_indexer import ( DenseCandidateIndexer, ) diff --git a/test/registered/unit/layers/test_dsv41_candidate_blocks.py b/test/registered/unit/layers/test_dsv41_candidate_blocks.py index 79915cf5c2..ff33378fe5 100644 --- a/test/registered/unit/layers/test_dsv41_candidate_blocks.py +++ b/test/registered/unit/layers/test_dsv41_candidate_blocks.py @@ -2,12 +2,12 @@ import unittest import torch -from sglang.srt.layers.attention.dsv4.candidate_indexer import ( - PrefillCandidateBlocks, +from sglang.kernels.ops.attention.dsv4.candidate_blocks import ( candidate_block_mask, select_candidate_block_ids, select_candidate_blocks, ) +from sglang.srt.layers.attention.dsv4.candidate_indexer import PrefillCandidateBlocks from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase