[DSv4.1] Move the ratio-1/2 index top-k ops into kernels/ops/attention/dsv4 (#41291)

Co-authored-by: DarkSharpness <2040703891@qq.com>
This commit is contained in:
Liangsheng Yin
2026-09-25 18:28:04 -07:00
committed by GitHub
co-authored by DarkSharpness
parent 85374532f2
commit 0967a013c2
11 changed files with 419 additions and 329 deletions
@@ -1,10 +1,24 @@
"""Per-row candidate block counts and sparse-row lengths for the paged indexer."""
"""Candidate block selection of the two-level indexer: the per-row block counts
and sparse-row lengths, the block keys and the JIT block top-k, and the torch
block ids and masks."""
from typing import Optional, Union
import torch
import torch.nn.functional as F
import triton
import triton.language as tl
from sglang.kernels.jit.utils import is_arch_support_pdl
from sglang.kernels.jit.utils import (
cache_once,
is_arch_support_pdl,
load_jit,
make_cpp_args,
)
from .candidate_table import CANDIDATE_BLOCK_SIZE
from .topk import plan_topk_v2, topk_transform_paged_v2
from .utils import make_name
@triton.jit
@@ -62,3 +76,149 @@ def candidate_row_lens(
**pdl_kwargs,
)
return nblocks, valid
@cache_once
def _jit_block_amax_module():
args = make_cpp_args(is_arch_support_pdl())
return load_jit(
make_name("block_amax"),
*args,
cuda_files=["deepseek_v4/block_amax.cuh"],
cuda_wrappers=[("amax8_varlen", f"BlockAmaxKernel<{args}>::amax8_varlen")],
)
def amax8_varlen(
scores: torch.Tensor,
seq_lens: torch.Tensor,
topk: int = 0,
*,
max_seqlen: int = 0,
out: Optional[torch.Tensor] = None,
) -> torch.Tensor:
"""Level-one keys of the two-level indexer: ``out[b, i]`` is the max of
``scores[b, 8 i : 8 i + 8]`` for ``i < ceil(seq_lens[b] / 8)``, the last of
them ``+inf`` (the newest block is always selected), nothing written past
that count. Rows with at most ``topk`` blocks are skipped (every block is
selected anyway); ``topk=0`` never skips. ``out`` is allocated as
``[rows, ceil(max_seqlen / 8)]`` when not given, ``max_seqlen`` defaulting to
the width of ``scores``; every ``seq_lens[b]`` must fit in ``8 * out.shape[1]``.
fp32 only for now; ``scores`` rows must be 32-byte aligned (stride a multiple
of 8). Returns ``out``.
"""
if out is None:
num_tokens, max_len = scores.shape
if max_seqlen == 0:
max_seqlen = max_len
out = scores.new_empty(num_tokens, (max_seqlen + 7) // 8)
_jit_block_amax_module().amax8_varlen(scores, seq_lens, out, topk)
return out
def amax_topk_blocks(
logits: torch.Tensor,
seq_lens: torch.Tensor,
nblocks: torch.Tensor,
topk_blocks: int,
max_seq_len: Optional[int] = None,
) -> torch.Tensor:
"""Per row the ``topk_blocks`` blocks of 8 positions with the largest block
maximum among its first ``seq_lens[b]`` positions, the newest block always
included: block ids in no particular order, ``-1`` past the row's count.
``nblocks`` is ``ceil(seq_lens / 8)`` as int32."""
rows = logits.shape[0]
block = CANDIDATE_BLOCK_SIZE
if max_seq_len is None:
max_seq_len = logits.shape[1]
# NOTE: plan cannot be the previous kernel of topk_transform_paged_v2
plan = plan_topk_v2(nblocks)
# block maxima, the newest block +inf; the top-k reads each row up to nblocks
# only, so nothing past a row's keys is initialised (v2 needs stride % 4 == 0)
keys = logits.new_empty(rows, -(-max_seq_len // (4 * block)) * 4)
amax8_varlen(logits, seq_lens, out=keys)
blocks = torch.empty(rows, topk_blocks, dtype=torch.int32, device=logits.device)
topk_transform_paged_v2(keys, nblocks, None, blocks, 1, plan)
return blocks
def mask_topk_scores(
scores: torch.Tensor,
indices: torch.Tensor,
offsets: Optional[torch.Tensor] = None,
) -> torch.Tensor:
"""Keep masked indexer scores out of attention even when top-k underfills."""
columns = indices.to(torch.int64)
if offsets is not None:
columns = columns - offsets[:, None]
selected_scores = scores.gather(1, columns.clamp(0, scores.shape[1] - 1))
valid = (
(columns >= 0) & (columns < scores.shape[1]) & (selected_scores > -torch.inf)
)
return indices.masked_fill(~valid, -1)
def _candidate_block_topk(
logits: torch.Tensor,
compress_lens: Union[torch.Tensor, int],
topk_blocks: int,
block_size: int,
) -> torch.return_types.topk:
width = logits.size(-1)
padding = -width % block_size
scores = F.pad(logits, (0, padding), value=-torch.inf) if padding else logits
scores = scores.unflatten(-1, (-1, block_size)).amax(dim=-1)
num_blocks = scores.size(-1)
last = (compress_lens - 1) // block_size
scores = scores.masked_fill(
torch.arange(num_blocks, device=logits.device) == last, torch.inf
)
return scores.topk(min(topk_blocks, num_blocks), dim=-1)
def select_candidate_block_ids(
logits: torch.Tensor,
compress_lens: Union[torch.Tensor, int],
topk_blocks: int,
block_size: int,
) -> torch.Tensor:
top = _candidate_block_topk(
logits=logits,
compress_lens=compress_lens,
topk_blocks=topk_blocks,
block_size=block_size,
)
return top.indices.to(torch.int32).masked_fill_(~(top.values > -torch.inf), -1)
def candidate_block_mask(
blocks: torch.Tensor, width: int, block_size: int
) -> torch.Tensor:
num_blocks = (width + block_size - 1) // block_size
keep = torch.zeros(
(*blocks.shape[:-1], num_blocks + 1), dtype=torch.bool, device=blocks.device
)
keep.scatter_(-1, blocks.to(torch.int64).masked_fill(blocks < 0, num_blocks), True)
return keep[..., :num_blocks].repeat_interleave(block_size, dim=-1)[..., :width]
def select_candidate_blocks(
logits: torch.Tensor,
compress_lens: Union[torch.Tensor, int],
topk_blocks: int,
block_size: int,
) -> torch.Tensor:
top = _candidate_block_topk(
logits=logits,
compress_lens=compress_lens,
topk_blocks=topk_blocks,
block_size=block_size,
)
width = logits.shape[-1]
num_blocks = (width + block_size - 1) // block_size
keep = torch.zeros(
(*logits.shape[:-1], num_blocks), dtype=torch.bool, device=logits.device
).scatter_(-1, top.indices, top.values > -torch.inf)
return keep.repeat_interleave(block_size, dim=-1)[..., :width]
@@ -1,5 +1,5 @@
"""Candidate-table kernels of the two-level indexer: level-one block keys and
the sorted, page-transformed block table."""
"""The sparse table of the two-level indexer from a row's selected blocks: the
sorted, page-transformed block table and DeepGEMM's schedule for it."""
from __future__ import annotations
@@ -19,43 +19,7 @@ from .utils import make_name
if TYPE_CHECKING:
pass
@cache_once
def _jit_block_amax_module():
args = make_cpp_args(is_arch_support_pdl())
return load_jit(
make_name("block_amax"),
*args,
cuda_files=["deepseek_v4/block_amax.cuh"],
cuda_wrappers=[("amax8_varlen", f"BlockAmaxKernel<{args}>::amax8_varlen")],
)
def amax8_varlen(
scores: torch.Tensor,
seq_lens: torch.Tensor,
topk: int = 0,
*,
max_seqlen: int = 0,
out: Optional[torch.Tensor] = None,
) -> torch.Tensor:
"""Level-one keys of the two-level indexer: ``out[b, i]`` is the max of
``scores[b, 8 i : 8 i + 8]`` for ``i < ceil(seq_lens[b] / 8)``, the last of
them ``+inf`` (the newest block is always selected), nothing written past
that count. Rows with at most ``topk`` blocks are skipped (every block is
selected anyway); ``topk=0`` never skips. ``out`` is allocated as
``[rows, ceil(max_seqlen / 8)]`` when not given, ``max_seqlen`` defaulting to
the width of ``scores``; every ``seq_lens[b]`` must fit in ``8 * out.shape[1]``.
fp32 only for now; ``scores`` rows must be 32-byte aligned (stride a multiple
of 8). Returns ``out``.
"""
if out is None:
num_tokens, max_len = scores.shape
if max_seqlen == 0:
max_seqlen = max_len
out = scores.new_empty(num_tokens, (max_seqlen + 7) // 8)
_jit_block_amax_module().amax8_varlen(scores, seq_lens, out, topk)
return out
CANDIDATE_BLOCK_SIZE = 8 # positions per block; DeepGEMM accepts 8 or 16
@cache_once
@@ -91,3 +55,29 @@ def sort_candidate_blocks(
blocks, seq_lens, page_table, out_pages, page_size
)
return out_pages
def build_sparse_indexer_schedule(
blocks: torch.Tensor,
seq_lens: torch.Tensor,
page_table: torch.Tensor,
page_size: int,
q_dtype: torch.dtype,
request_ids: torch.Tensor,
) -> torch.Tensor:
"""DeepGEMM's schedule for the published blocks: ``seq_lens`` ``[rows]``
int32, ``page_table`` ``[rows, pages]`` int32 at the index pool's page size.
``request_ids`` ``[rows]`` int32 lets DeepGEMM pair two rows of a request on
one KV pass; each row keeps its own block list and output layout, and paired
rows must share their page-table row."""
import deep_gemm
return deep_gemm.get_paged_sparse_mqa_logits_metadata(
seq_lens.contiguous(),
page_table,
request_ids,
page_size,
blocks,
q_dtype,
CANDIDATE_BLOCK_SIZE,
)
@@ -0,0 +1,126 @@
"""The index logits of the DeepSeek V4.1 ratio-1/2 index layers, by the layout of
the index K they read: the flattened K of a prefill chunk (tile by tile under a
memory budget), the paged pool, or the published blocks of a sparse table."""
from __future__ import annotations
from typing import Iterator, Tuple
import torch
from sglang.srt.layers.attention.mqa_logits_utils import (
mqa_logits_row_bytes,
mqa_logits_rows_per_chunk,
)
from sglang.srt.utils.common import ceil_align
from .candidate_table import CANDIDATE_BLOCK_SIZE
def flat_index_logits_rows_per_tile(
rows: int, width: int, *, heads: int, budget_bytes: int
) -> int:
"""Rows per fp32 logits tile within ``budget_bytes``, at the kernel's row
alignment."""
row_alignment = 128 // heads
rows_per_chunk = mqa_logits_rows_per_chunk(
num_rows=ceil_align(rows, row_alignment),
row_bytes=mqa_logits_row_bytes(width),
budget_bytes=budget_bytes,
)
if rows_per_chunk is None:
return rows
return max(row_alignment, rows_per_chunk // row_alignment * row_alignment)
def flat_index_logits_tiles(
*,
q: tuple[torch.Tensor, torch.Tensor],
kv: tuple[torch.Tensor, torch.Tensor],
weights: torch.Tensor,
starts: torch.Tensor,
lengths: torch.Tensor,
context_lengths: list[int],
budget_bytes: int,
width_align: int = 4,
) -> Iterator[tuple[slice, torch.Tensor]]:
"""``(rows, logits)`` per row tile, each fp32 tile within ``budget_bytes``:
``logits[i, j]`` scores query row ``rows.start + i`` against ``kv[starts + j]``,
garbage past the row's ``lengths``; the width is ``max(context_lengths)``
aligned to ``width_align``."""
from deep_gemm import fp8_fp4_mqa_logits
rows = q[0].shape[0]
width = ceil_align(max(context_lengths, default=0), width_align)
if rows == 0 or width == 0:
return
rows_per_chunk = flat_index_logits_rows_per_tile(
rows, width, heads=q[0].shape[1], budget_bytes=budget_bytes
)
for offset in range(0, rows, rows_per_chunk):
tile = slice(offset, min(offset + rows_per_chunk, rows))
tile_starts = starts[tile]
yield (
tile,
fp8_fp4_mqa_logits(
(q[0][tile], q[1][tile]),
kv,
weights[tile],
tile_starts,
tile_starts + lengths[tile],
False,
width,
),
)
def deep_gemm_fp4_paged_mqa_logits(
q_fp4: Tuple[torch.Tensor, torch.Tensor],
k_cache: torch.Tensor,
weights: torch.Tensor,
seq_lens: torch.Tensor,
page_table: torch.Tensor,
deep_gemm_metadata,
max_seq_len: int,
) -> torch.Tensor:
"""DeepGEMM paged fp4 logits; no hadamard, the reference does not apply one."""
from deep_gemm import fp8_fp4_paged_mqa_logits
sl = seq_lens.to(torch.int32)
if sl.dim() == 1:
sl = sl.unsqueeze(-1)
return fp8_fp4_paged_mqa_logits(
q_fp4,
k_cache,
weights,
sl,
page_table,
deep_gemm_metadata,
max_seq_len,
False,
)
def sparse_logits(
q_fp4: torch.Tensor,
q_sf: torch.Tensor,
k_cache: torch.Tensor,
weights: torch.Tensor,
schedule: torch.Tensor,
topk_blocks: int,
) -> torch.Tensor:
"""bf16 logits ``[rows, topk_blocks * 8]`` of the published blocks: ``q_fp4``
``[rows, 1, heads, 64]`` int8 with ``q_sf`` ``[rows, 1, heads]`` int32 (packed
ue8m0), ``k_cache`` ``[pages, page_size, 1, 68]`` uint8 whose page stride is
a multiple of 512 bytes, ``weights`` ``[rows, heads]`` bf16, ``schedule`` the
``build_sparse_indexer_schedule`` of the ``topk_blocks`` published blocks."""
import deep_gemm
return deep_gemm.fp8_fp4_paged_sparse_mqa_logits(
(q_fp4, q_sf),
k_cache,
weights,
schedule,
topk_blocks,
CANDIDATE_BLOCK_SIZE,
)
@@ -13,6 +13,7 @@ from sglang.kernels.jit.utils import (
)
from sglang.srt.utils import is_xpu
from .candidate_table import CANDIDATE_BLOCK_SIZE
from .utils import make_name
@@ -123,6 +124,30 @@ def topk_transform_paged(
)
def topk_transform_paged_torch(
scores: torch.Tensor,
seq_lens: torch.Tensor,
page_tables: torch.Tensor,
out_page_indices: torch.Tensor,
page_size: int,
out_raw_indices: Optional[torch.Tensor] = None,
) -> None:
"""The torch ``topk_transform_paged``: top-``k`` (``k = out_page_indices.shape[1]``)
of each row within its first ``seq_lens[b]`` columns, ascending, as pool slots
through ``page_tables`` and as positions when given; ``-1`` padded."""
topk = out_page_indices.shape[1]
columns = torch.arange(scores.shape[1], device=seq_lens.device)
lens_c = seq_lens.unsqueeze(-1)
# Columns past a row's length hold garbage.
s = scores.masked_fill(columns[None, :] >= lens_c, -torch.inf)
idx = s.topk(topk, dim=-1, sorted=False).indices.sort(dim=-1).values
reach = idx < lens_c
slots = page_tables.gather(-1, idx // page_size) * page_size + (idx % page_size)
out_page_indices.copy_(torch.where(reach, slots, -1).to(torch.int32))
if out_raw_indices is not None:
out_raw_indices.copy_(torch.where(reach, idx, -1).to(torch.int32))
# metadata is (batch+1, 2) int32: row 0 = {cluster_threshold, num_cluster_items};
# rows 1..N = {batch_id, seq_len} of items routed to the persistent cluster pool.
_PLAN_METADATA_INTS_PER_BATCH = 2
@@ -301,3 +326,18 @@ def topk_transform_packed_v2(
page_size,
row_to_batch,
)
def topk_transform_sparse(
logits: torch.Tensor,
valid_lens: torch.Tensor,
phys_blocks: torch.Tensor,
page_indices: torch.Tensor,
) -> None:
"""Top-``k`` (``k = page_indices.shape[1]``) of each row of the bf16 sparse
``logits`` within its first ``valid_lens[b]`` columns, as pool slots through
``phys_blocks`` (the published blocks as pool slots / 8); ``-1`` padded,
unordered."""
topk_transform_bf16_small(
logits, valid_lens, phys_blocks, page_indices, CANDIDATE_BLOCK_SIZE
)
@@ -19,6 +19,10 @@ import msgspec
import torch
import torch.nn.functional as F
from sglang.kernels.ops.attention.dsv4.candidate_blocks import (
mask_topk_scores,
select_candidate_blocks,
)
from sglang.kernels.ops.attention.dsv4.decode_attention_sm100 import (
can_use_swapab_attention,
)
@@ -30,6 +34,9 @@ from sglang.kernels.ops.attention.dsv4.dequant_k_cache import (
q8kv8_padded_num_heads,
)
from sglang.kernels.ops.attention.dsv4.fp4_indexer import fp4_index_logits_decode
from sglang.kernels.ops.attention.dsv4.index_logits import (
deep_gemm_fp4_paged_mqa_logits,
)
from sglang.kernels.ops.attention.dsv4.kv_layout import KVLayout
from sglang.kernels.ops.attention.dsv4.metadata_kernel import (
fill_all_compressed_indices,
@@ -38,6 +45,7 @@ from sglang.kernels.ops.attention.dsv4.metadata_kernel import (
init_compression_metadata as _init_compression_metadata_triton,
)
from sglang.kernels.ops.attention.dsv4.online_c128_mtp import OnlineC128MTPController
from sglang.kernels.ops.attention.dsv4.topk import topk_transform_paged_torch
from sglang.kernels.ops.attention.dsv4_attn_metadata_kernels import (
BuildCausalSwaPageIndices,
BuildPageTablePositions,
@@ -64,9 +72,7 @@ from sglang.srt.layers.attention.dsv4.candidate_indexer import (
cut_request_masks,
expand_index_page_table,
make_candidate_indexer,
mask_topk_scores,
published_masks,
select_candidate_blocks,
)
from sglang.srt.layers.attention.dsv4.compressor_v2 import (
CompressorBackendMixin,
@@ -80,7 +86,6 @@ from sglang.srt.layers.attention.dsv4.dsv41_sparse import (
)
from sglang.srt.layers.attention.dsv4.indexer import (
C4IndexerBackendMixin,
deep_gemm_fp4_paged_mqa_logits,
topk_transform_paged_from_metadata,
)
from sglang.srt.layers.attention.dsv4.metadata import (
@@ -3340,7 +3345,6 @@ class DeepseekV4AttnBackend(
page_indices = core.sparse_page_indices(ratio)
raw_indices = core.sparse_raw_indices(ratio)
topk = min(indexer.index_topk, width)
columns = torch.arange(width, device=lens.device)
for rows, plan in metadata.row_chunks():
logits = deep_gemm_fp4_paged_mqa_logits(
(q_fp4[rows], q_sf[rows]),
@@ -3351,17 +3355,14 @@ class DeepseekV4AttnBackend(
plan,
width,
)
lens_c = lens[rows].unsqueeze(-1)
# Columns past a row's length hold garbage.
s = logits.masked_fill(columns[None, :] >= lens_c, -torch.inf)
idx = s.topk(topk, dim=-1, sorted=False).indices.sort(dim=-1).values
reach = idx < lens_c
slots = page_table[rows].gather(-1, idx // page_size) * page_size + (
idx % page_size
topk_transform_paged_torch(
logits,
lens[rows],
page_table[rows],
page_indices[rows, :topk],
page_size,
raw_indices[rows, :topk] if raw_indices is not None else None,
)
page_indices[rows, :topk] = torch.where(reach, slots, -1).to(torch.int32)
if raw_indices is not None:
raw_indices[rows, :topk] = torch.where(reach, idx, -1).to(torch.int32)
def _low_ratio_index_topk_decode(self, layer, x, q_lora, pos, req=None) -> None:
from sglang.srt.model_executor.runner_utils.capture_mode import (
@@ -2,11 +2,10 @@ from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import List, Optional, Tuple, Union
from typing import List, Optional, Tuple
import msgspec
import torch
import torch.nn.functional as F
from sglang.srt.layers.attention.dsv4.metadata import PagedIndexerMetadata
from sglang.srt.runtime_context import get_parallel, get_platform
@@ -193,85 +192,3 @@ class PrefillCandidateBlocks(CandidateMetadata, msgspec.Struct):
def published_masks(candidate) -> CandidateMasks:
assert isinstance(candidate, CandidateMasks), "candidate masks missing"
return candidate
def mask_topk_scores(
scores: torch.Tensor,
indices: torch.Tensor,
offsets: Optional[torch.Tensor] = None,
) -> torch.Tensor:
"""Keep masked indexer scores out of attention even when top-k underfills."""
columns = indices.to(torch.int64)
if offsets is not None:
columns = columns - offsets[:, None]
selected_scores = scores.gather(1, columns.clamp(0, scores.shape[1] - 1))
valid = (
(columns >= 0) & (columns < scores.shape[1]) & (selected_scores > -torch.inf)
)
return indices.masked_fill(~valid, -1)
def _candidate_block_topk(
logits: torch.Tensor,
compress_lens: Union[torch.Tensor, int],
topk_blocks: int,
block_size: int,
) -> torch.return_types.topk:
width = logits.size(-1)
padding = -width % block_size
scores = F.pad(logits, (0, padding), value=-torch.inf) if padding else logits
scores = scores.unflatten(-1, (-1, block_size)).amax(dim=-1)
num_blocks = scores.size(-1)
last = (compress_lens - 1) // block_size
scores = scores.masked_fill(
torch.arange(num_blocks, device=logits.device) == last, torch.inf
)
return scores.topk(min(topk_blocks, num_blocks), dim=-1)
def select_candidate_block_ids(
logits: torch.Tensor,
compress_lens: Union[torch.Tensor, int],
topk_blocks: int,
block_size: int,
) -> torch.Tensor:
top = _candidate_block_topk(
logits=logits,
compress_lens=compress_lens,
topk_blocks=topk_blocks,
block_size=block_size,
)
return top.indices.to(torch.int32).masked_fill_(~(top.values > -torch.inf), -1)
def candidate_block_mask(
blocks: torch.Tensor, width: int, block_size: int
) -> torch.Tensor:
num_blocks = (width + block_size - 1) // block_size
keep = torch.zeros(
(*blocks.shape[:-1], num_blocks + 1), dtype=torch.bool, device=blocks.device
)
keep.scatter_(-1, blocks.to(torch.int64).masked_fill(blocks < 0, num_blocks), True)
return keep[..., :num_blocks].repeat_interleave(block_size, dim=-1)[..., :width]
def select_candidate_blocks(
logits: torch.Tensor,
compress_lens: Union[torch.Tensor, int],
topk_blocks: int,
block_size: int,
) -> torch.Tensor:
top = _candidate_block_topk(
logits=logits,
compress_lens=compress_lens,
topk_blocks=topk_blocks,
block_size=block_size,
)
width = logits.shape[-1]
num_blocks = (width + block_size - 1) // block_size
keep = torch.zeros(
(*logits.shape[:-1], num_blocks), dtype=torch.bool, device=logits.device
).scatter_(-1, top.indices, top.values > -torch.inf)
return keep.repeat_interleave(block_size, dim=-1)[..., :width]
@@ -5,16 +5,26 @@ from typing import List, Optional
import torch
from sglang.kernels.ops.attention.dsv4.candidate_blocks import candidate_row_lens
from sglang.kernels.ops.attention.dsv4.candidate_table import (
from sglang.kernels.ops.attention.dsv4.candidate_blocks import (
amax8_varlen,
amax_topk_blocks,
candidate_row_lens,
)
from sglang.kernels.ops.attention.dsv4.candidate_table import (
CANDIDATE_BLOCK_SIZE,
build_sparse_indexer_schedule,
sort_candidate_blocks,
)
from sglang.kernels.ops.attention.dsv4.index_logits import (
deep_gemm_fp4_paged_mqa_logits,
flat_index_logits_tiles,
sparse_logits,
)
from sglang.kernels.ops.attention.dsv4.topk import (
plan_topk_v2,
topk_transform_bf16_small,
topk_transform_paged_v2,
topk_transform_ragged_v2,
topk_transform_sparse,
)
from sglang.srt.layers.attention.dsv4.candidate_indexer import (
CandidateIndexer,
@@ -24,16 +34,13 @@ from sglang.srt.layers.attention.dsv4.candidate_indexer import (
expand_index_page_table,
)
from sglang.srt.layers.attention.dsv4.dense_prefill_indexer import (
_SCORE_BUDGET_BYTES,
DenseCandidateIndexer,
score_tiles,
)
from sglang.srt.layers.attention.dsv4.indexer import (
deep_gemm_fp4_paged_mqa_logits,
topk_transform_paged_from_metadata,
)
CANDIDATE_BLOCK_SIZE = 8 # positions per block; DeepGEMM accepts 8 or 16
@dataclass
class SparseBlockTable(CandidateMetadata):
@@ -53,96 +60,6 @@ class SparseBlockTable(CandidateMetadata):
ready: torch.cuda.Event
def amax_topk_blocks(
logits: torch.Tensor,
seq_lens: torch.Tensor,
nblocks: torch.Tensor,
topk_blocks: int,
max_seq_len: Optional[int] = None,
) -> torch.Tensor:
"""Per row the ``topk_blocks`` blocks of 8 positions with the largest block
maximum among its first ``seq_lens[b]`` positions, the newest block always
included: block ids in no particular order, ``-1`` past the row's count.
``nblocks`` is ``ceil(seq_lens / 8)`` as int32."""
rows = logits.shape[0]
block = CANDIDATE_BLOCK_SIZE
if max_seq_len is None:
max_seq_len = logits.shape[1]
# NOTE: plan cannot be the previous kernel of topk_transform_paged_v2
plan = plan_topk_v2(nblocks)
# block maxima, the newest block +inf; the top-k reads each row up to nblocks
# only, so nothing past a row's keys is initialised (v2 needs stride % 4 == 0)
keys = logits.new_empty(rows, -(-max_seq_len // (4 * block)) * 4)
amax8_varlen(logits, seq_lens, out=keys)
blocks = torch.empty(rows, topk_blocks, dtype=torch.int32, device=logits.device)
topk_transform_paged_v2(keys, nblocks, None, blocks, 1, plan)
return blocks
def build_sparse_indexer_schedule(
blocks: torch.Tensor,
seq_lens: torch.Tensor,
page_table: torch.Tensor,
page_size: int,
q_dtype: torch.dtype,
request_ids: torch.Tensor,
) -> torch.Tensor:
"""DeepGEMM's schedule for the published blocks: ``seq_lens`` ``[rows]``
int32, ``page_table`` ``[rows, pages]`` int32 at the index pool's page size.
``request_ids`` ``[rows]`` int32 lets DeepGEMM pair two rows of a request on
one KV pass; each row keeps its own block list and output layout, and paired
rows must share their page-table row."""
import deep_gemm
return deep_gemm.get_paged_sparse_mqa_logits_metadata(
seq_lens.contiguous(),
page_table,
request_ids,
page_size,
blocks,
q_dtype,
CANDIDATE_BLOCK_SIZE,
)
def sparse_logits(
q_fp4: torch.Tensor,
q_sf: torch.Tensor,
k_cache: torch.Tensor,
weights: torch.Tensor,
table: SparseBlockTable,
) -> torch.Tensor:
"""bf16 logits ``[rows, topk_blocks * 8]`` of the published blocks: ``q_fp4``
``[rows, 1, heads, 64]`` int8 with ``q_sf`` ``[rows, 1, heads]`` int32 (packed
ue8m0), ``k_cache`` ``[pages, page_size, 1, 68]`` uint8 whose page stride is
a multiple of 512 bytes, ``weights`` ``[rows, heads]`` bf16."""
import deep_gemm
return deep_gemm.fp8_fp4_paged_sparse_mqa_logits(
(q_fp4, q_sf),
k_cache,
weights,
table.schedule,
table.blocks.shape[1],
CANDIDATE_BLOCK_SIZE,
)
def topk_transform_sparse(
logits: torch.Tensor,
valid_lens: torch.Tensor,
table: SparseBlockTable,
page_indices: torch.Tensor,
) -> None:
"""Top-``k`` (``k = page_indices.shape[1]``) of every row of the sparse
``logits`` (bf16 ``[rows, topk_blocks * 8]``) within its first ``valid_lens[b]``
columns, written as pool slots, ``-1`` where a row has fewer than ``k`` valid
columns, in no particular order."""
topk_transform_bf16_small(
logits, valid_lens, table.phys_blocks, page_indices, CANDIDATE_BLOCK_SIZE
)
@dataclass
class PrefillSparseBlockTable(SparseBlockTable):
"""Published by a prefill chunk's candidate source: one row per query token.
@@ -347,7 +264,8 @@ class DeepGemmCandidateIndexer(CandidateIndexer):
inputs.q_sf,
inputs.k_cache,
inputs.weights.to(torch.bfloat16),
table,
table.schedule,
table.blocks.shape[1],
)
def select_decode(
@@ -362,7 +280,7 @@ class DeepGemmCandidateIndexer(CandidateIndexer):
torch.cuda.current_stream().wait_event(table.ready)
logits = self._scores(table, inputs)
# decode carries no raw_indices; the kernel writes slots only
topk_transform_sparse(logits, table.valid_lens, table, page_indices)
topk_transform_sparse(logits, table.valid_lens, table.phys_blocks, page_indices)
def publish_prefill(
self, inputs: PrefillIndexerInputs, out_positions: torch.Tensor
@@ -377,7 +295,16 @@ class DeepGemmCandidateIndexer(CandidateIndexer):
nblocks, valid_lens = candidate_row_lens(inputs.compress_lens, self.topk_blocks)
blocks = torch.empty(rows, self.topk_blocks, dtype=torch.int32, device=device)
# the block keys read the score rows through 32-byte vectors
for tile, logits in score_tiles(inputs, width_align=8):
for tile, logits in flat_index_logits_tiles(
q=(inputs.q_fp4, inputs.q_sf),
kv=inputs.kv,
weights=inputs.weights,
starts=inputs.request_starts,
lengths=inputs.compress_lens,
context_lengths=inputs.lens_per_request,
budget_bytes=_SCORE_BUDGET_BYTES,
width_align=8,
):
lens = inputs.compress_lens[tile]
topk_transform_ragged_v2(
logits,
@@ -500,7 +427,8 @@ class DeepGemmCandidateIndexer(CandidateIndexer):
inputs.q_sf.view(rows, 1, heads),
inputs.k_cache,
inputs.weights.to(torch.bfloat16),
table,
table.schedule,
table.blocks.shape[1],
)
# request-relative compressed positions, -1 padded
topk_transform_bf16_small(
@@ -1,21 +1,22 @@
from __future__ import annotations
from typing import Iterator, List, Optional
from typing import List, Optional
import torch
from sglang.kernels.ops.attention.dsv4.candidate_blocks import (
candidate_block_mask,
mask_topk_scores,
select_candidate_block_ids,
)
from sglang.kernels.ops.attention.dsv4.index_logits import (
flat_index_logits_rows_per_tile,
)
from sglang.srt.layers.attention.dsv4.candidate_indexer import (
CandidateIndexer,
CandidateMetadata,
PrefillCandidateBlocks,
PrefillIndexerInputs,
candidate_block_mask,
mask_topk_scores,
select_candidate_block_ids,
)
from sglang.srt.layers.attention.mqa_logits_utils import (
mqa_logits_row_bytes,
mqa_logits_rows_per_chunk,
)
from sglang.srt.utils.common import ceil_align
@@ -63,7 +64,9 @@ def dense_prefill_topk(
width = ceil_align(max((n for _, n in request_lengths), default=0), 4)
if row == 0 or width == 0:
return selected, published
rows_per_chunk = _rows_per_chunk(row, width, heads=q[0].shape[1])
rows_per_chunk = flat_index_logits_rows_per_tile(
row, width, heads=q[0].shape[1], budget_bytes=_SCORE_BUDGET_BYTES
)
for offset in range(0, row, rows_per_chunk):
rows = slice(offset, min(offset + rows_per_chunk, row))
_select_tile(
@@ -83,52 +86,6 @@ def dense_prefill_topk(
return selected, published
def _rows_per_chunk(rows: int, width: int, *, heads: int) -> int:
"""Query rows per logits tile so one fp32 [rows, width] tile fits the
budget; the row count stays a multiple of the kernel's row alignment."""
row_alignment = 128 // heads
rows_per_chunk = mqa_logits_rows_per_chunk(
num_rows=ceil_align(rows, row_alignment),
row_bytes=mqa_logits_row_bytes(width),
budget_bytes=_SCORE_BUDGET_BYTES,
)
if rows_per_chunk is None:
return rows
return max(row_alignment, rows_per_chunk // row_alignment * row_alignment)
def score_tiles(
inputs: PrefillIndexerInputs, *, width_align: int = 4
) -> Iterator[tuple[slice, torch.Tensor]]:
"""The dense scores of ``inputs`` row tile by row tile under the budget:
``(rows, logits)`` with fp32 ``logits[i, j]`` the score of query row
``rows.start + i`` against ``kv[request_starts + j]``, garbage past the row's
``compress_lens``; the width is the batch's largest context aligned to
``width_align`` columns."""
from deep_gemm import fp8_fp4_mqa_logits
rows = inputs.num_rows
width = ceil_align(max(inputs.lens_per_request, default=0), width_align)
if rows == 0 or width == 0:
return
rows_per_chunk = _rows_per_chunk(rows, width, heads=inputs.q_fp4.shape[1])
for offset in range(0, rows, rows_per_chunk):
tile = slice(offset, min(offset + rows_per_chunk, rows))
starts = inputs.request_starts[tile]
yield (
tile,
fp8_fp4_mqa_logits(
(inputs.q_fp4[tile], inputs.q_sf[tile]),
inputs.kv,
inputs.weights[tile],
starts,
starts + inputs.compress_lens[tile],
False,
width,
),
)
def _dense_topk(
inputs: PrefillIndexerInputs,
out_positions: torch.Tensor,
@@ -235,7 +192,7 @@ def _select_tile(
) -> None:
from deep_gemm import fp8_fp4_mqa_logits
from sglang.kernels.ops.attention.dsv4 import topk_transform_ragged_v2
from sglang.kernels.ops.attention.dsv4.topk import topk_transform_ragged_v2
logits = fp8_fp4_mqa_logits(q, kv, weights, starts, starts + lengths, False, width)
if publish is not None or consume is not None:
@@ -443,33 +443,6 @@ def topk_transform_flashinfer_fused(
)
def deep_gemm_fp4_paged_mqa_logits(
q_fp4: Tuple[torch.Tensor, torch.Tensor],
k_cache: torch.Tensor,
weights: torch.Tensor,
seq_lens: torch.Tensor,
page_table: torch.Tensor,
deep_gemm_metadata,
max_seq_len: int,
) -> torch.Tensor:
"""DeepGEMM paged fp4 logits; no hadamard, the reference does not apply one."""
from deep_gemm import fp8_fp4_paged_mqa_logits
sl = seq_lens.to(torch.int32)
if sl.dim() == 1:
sl = sl.unsqueeze(-1)
return fp8_fp4_paged_mqa_logits(
q_fp4,
k_cache,
weights,
sl,
page_table,
deep_gemm_metadata,
max_seq_len,
False,
)
def topk_transform_paged_from_metadata(
logits: torch.Tensor,
metadata,