[DSv4.1] Score prefill consumer index layers on candidate blocks with DeepGEMM (#40352)

Co-authored-by: luoyuan.luo <luoyuan.luo@antgroup.com>
Co-authored-by: DarkSharpness <76582120+DarkSharpness@users.noreply.github.com>
This commit is contained in:
Yuan Luo
2026-09-22 22:06:06 +08:00
committed by GitHub
co-authored by luoyuan.luo DarkSharpness
parent 4a1b69abc8
commit c79510cc2a
5 changed files with 753 additions and 105 deletions
@@ -60,7 +60,9 @@ from sglang.srt.layers.attention.dsv4.candidate_indexer import (
CandidateMasks,
CandidateMetadata,
IndexerInputs,
PrefillCandidateBlocks,
PrefillIndexerInputs,
cut_request_masks,
expand_index_page_table,
make_candidate_indexer,
mask_topk_scores,
published_masks,
@@ -71,7 +73,7 @@ from sglang.srt.layers.attention.dsv4.compressor_v2 import (
FusedCompressMetadata,
create_paged_compressor_data,
)
from sglang.srt.layers.attention.dsv4.dense_prefill_indexer import dense_prefill_topk
from sglang.srt.layers.attention.dsv4.dense_prefill_indexer import plain_prefill_topk
from sglang.srt.layers.attention.dsv4.dsv41_sparse import (
_rope_fq4,
token_req_indices,
@@ -258,33 +260,6 @@ def _maybe_precompute_flashmla_sched_meta(
flashmla_metadata.num_splits = num_splits
def _expand_index_page_table(
page_table: torch.Tensor,
*,
full_page_size: int,
compress_ratio: int,
index_page_size: int,
) -> torch.Tensor:
"""Block table of a low-ratio indexer-K pool, which pages at `index_page_size`
slots: [bs, n] -> [bs, n * blocks_per_page] int32. The kernel reads compressed
slot j at page_table[b, j // index_page_size] * index_page_size + j %
index_page_size, which after this expansion is the c1/c2 pool slot of the same
position."""
slots_per_page = full_page_size // compress_ratio
assert slots_per_page % index_page_size == 0, (
f"{full_page_size = } / {compress_ratio = } must be a multiple of "
f"{index_page_size = }"
)
blocks_per_page = slots_per_page // index_page_size
if blocks_per_page == 1:
return page_table
bs, n = page_table.shape
base = page_table.to(torch.int64) * blocks_per_page
offsets = torch.arange(blocks_per_page, device=page_table.device, dtype=torch.int64)
expanded = base.unsqueeze(-1) + offsets # [bs, n, blocks_per_page]
return expanded.reshape(bs, n * blocks_per_page).to(torch.int32)
# Arbitrary cap on one bf16 [rows, heads, lc] score chunk; transients run ~3x this.
_TORCH_INDEXER_SCORE_BUDGET_BYTES = 1 << 30
@@ -1357,7 +1332,7 @@ class DeepseekV4AttnBackend(
index_page_size = self.token_to_kv_pool.get_index_k_page_size(
compress_ratio
)
page_table = _expand_index_page_table(
page_table = expand_index_page_table(
page_table,
full_page_size=self.page_size,
compress_ratio=compress_ratio,
@@ -1544,7 +1519,7 @@ class DeepseekV4AttnBackend(
if max_seq_len is not None:
num_pages = min(max_seq_len // self.page_size, num_pages)
index_page_size = self.token_to_kv_pool.get_index_k_page_size(compress_ratio)
page_table = _expand_index_page_table(
page_table = expand_index_page_table(
core.page_table[:, :num_pages],
full_page_size=self.page_size,
compress_ratio=compress_ratio,
@@ -1740,23 +1715,6 @@ class DeepseekV4AttnBackend(
get_local_dp_buffer_len(),
)
tail = tail_metadata.late_layer_tail
tail_lens_cpu = (
tail.local_lens_cpu
if tail.cp_metadata is not None
else tail.extend_seq_lens_cpu
)
# TODO(candidate): goes away once the source publishes its tail rows straight
# onto the tail metadata (publish_prefill); until then cut the full masks.
full_masks = self.forward_metadata.candidate_metadata
if isinstance(full_masks, PrefillCandidateBlocks):
tail_metadata.candidate_metadata = full_masks.tail(tail_lens_cpu)
elif isinstance(full_masks, CandidateMasks) and full_masks.request_masks:
tail_metadata.candidate_metadata = CandidateMasks(
request_masks=[
mask[mask.shape[0] - t :]
for mask, t in zip(full_masks.request_masks, tail_lens_cpu)
]
)
# The layers before the switch published top-k into the full metadata's
# buffers; carry the tail rows into the tail metadata's (padding stays -1).
full_core = saved[0].core_attn_metadata
@@ -3243,12 +3201,8 @@ class DeepseekV4AttnBackend(
num_tokens = pos.shape[0]
if not slot_chunks or num_tokens == 0:
if indexer.is_candidate_source:
self.forward_metadata.candidate_metadata = PrefillCandidateBlocks(
request_blocks=[
torch.empty((length, 0), dtype=torch.int32, device=device)
for length in q_lens_cpu
]
)
# no row to publish for; the consumers return here as well
self.forward_metadata.candidate_metadata = None
return
k_slots = torch.cat(slot_chunks)
k_fp4, k_sf = pool.get_low_ratio_index_k_fp4(layer.layer_id, k_slots)
@@ -3265,26 +3219,28 @@ class DeepseekV4AttnBackend(
q_lens.to(torch.int64),
output_size=num_tokens,
)
candidates = None
if indexer.uses_candidates and not indexer.is_candidate_source:
candidates = self.forward_metadata.candidate_metadata
assert isinstance(candidates, PrefillCandidateBlocks)
topk = indexer.index_topk
selected, published = dense_prefill_topk(
q=(q_fp4, q_sf),
kv=(k_fp4, k_sf),
weights=weights,
starts=ks,
lengths=compress_lens,
request_lengths=list(zip(q_lens_cpu, lc_per_req)),
topk=topk,
candidate_topk_blocks=indexer.candidate_topk_blocks,
candidate_block_size=indexer.candidate_block_size,
publish_candidates=indexer.is_candidate_source,
candidates=candidates,
inputs = self._prefill_indexer_inputs(
layer,
(q_fp4, q_sf),
(k_fp4, k_sf),
weights,
compress_lens,
ks,
lc_per_req,
list(q_lens_cpu),
)
if published is not None:
self.forward_metadata.candidate_metadata = published
topk = indexer.index_topk
selected = torch.full((num_tokens, topk), -1, dtype=torch.int32, device=device)
if indexer.is_candidate_source:
self._publish_prefill(
self.candidate_indexer.publish_prefill(inputs, selected)
)
elif indexer.uses_candidates:
self.candidate_indexer.select_prefill(
self.forward_metadata.candidate_metadata, inputs, selected
)
else:
plain_prefill_topk(inputs, selected)
# ascending positions, padding last: the layout the consumers expect
unselected = torch.iinfo(torch.int32).max
selected = selected.masked_fill(selected < 0, unselected).sort(dim=-1).values
@@ -3297,6 +3253,61 @@ class DeepseekV4AttnBackend(
chosen, selected - ks[:, None], -1
)
def _prefill_indexer_inputs(
self, layer, q, kv, weights, compress_lens, ks, lc_per_req, q_lens_cpu
) -> PrefillIndexerInputs:
pool = self.token_to_kv_pool
ratio = layer.compress_ratio
num_tokens = q[0].shape[0]
index_page_size = pool.get_index_k_page_size(ratio)
k_cache = pool.get_index_k_with_scale_buffer(layer.layer_id)
assert k_cache.dim() == 2
return PrefillIndexerInputs(
q_fp4=q[0],
q_sf=q[1],
weights=weights,
compress_lens=compress_lens,
request_starts=ks,
lens_per_request=lc_per_req,
rows_per_request=q_lens_cpu,
kv=kv,
k_cache=k_cache.view(k_cache.shape[0], index_page_size, 1, 68),
page_size=index_page_size,
kv_page_table=self.forward_metadata.core_metadata.page_table[:num_tokens],
kv_page_size=self.page_size,
compress_ratio=ratio,
)
def _tail_lens_to_publish(self) -> Optional[List[int]]:
"""Rows per request of the late-layer tail, when the layers after the
switch will consume what is published now."""
tail_metadata = self.tail_forward_metadata
if tail_metadata is None or tail_metadata is self.forward_metadata:
return None
tail = tail_metadata.late_layer_tail
return (
tail.local_lens_cpu
if tail.cp_metadata is not None
else tail.extend_seq_lens_cpu
)
def _publish_prefill(self, published: CandidateMetadata) -> None:
self.forward_metadata.candidate_metadata = published
tail_lens = self._tail_lens_to_publish()
if tail_lens is not None:
self.tail_forward_metadata.candidate_metadata = (
self.candidate_indexer.prefill_tail(published, tail_lens)
)
def _publish_prefill_masks(self, masks: CandidateMasks) -> None:
"""The torch prefill path's inline masks (see the TODO on CandidateMasks)."""
self.forward_metadata.candidate_metadata = masks
tail_lens = self._tail_lens_to_publish()
if tail_lens is not None:
self.tail_forward_metadata.candidate_metadata = cut_request_masks(
masks, tail_lens
)
def _low_ratio_index_topk_prefill_graph(self, layer, pos, q, w) -> None:
from sglang.kernels.ops.attention.dsv4.fp4_indexer import (
quantize_fp4_indexer_tensor,
@@ -3607,9 +3618,7 @@ class DeepseekV4AttnBackend(
if masks is not None:
publish.append(torch.cat(masks) if len(masks) > 1 else masks[0])
if publish is not None:
self.forward_metadata.candidate_metadata = CandidateMasks(
request_masks=publish
)
self._publish_prefill_masks(CandidateMasks(request_masks=publish))
def get_swa_out_cache_loc(self, forward_batch: ForwardBatch) -> torch.Tensor:
"""Idle always re-translates at store time: its metadata may be stale, and
@@ -1,19 +1,15 @@
from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import TYPE_CHECKING, List, Optional, Union
from typing import List, Optional, Tuple, Union
import msgspec
import torch
import torch.nn.functional as F
from sglang.srt.layers.attention.dsv4.metadata import PagedIndexerMetadata
from sglang.srt.runtime_context import get_platform
if TYPE_CHECKING:
from sglang.srt.layers.attention.dsv4.candidate_indexer_deep_gemm import (
DeepGemmCandidateIndexer,
)
from sglang.srt.runtime_context import get_parallel, get_platform
class CandidateMetadata:
@@ -40,11 +36,104 @@ class IndexerInputs:
return self.q_fp4.shape[0]
class PrefillIndexerInputs(msgspec.Struct, frozen=True):
"""One index layer's operands on the dense fp4 prefill path: the chunk's query
rows, one request's rows consecutive."""
q_fp4: torch.Tensor # [rows, heads, 64] int8, packed fp4
q_sf: torch.Tensor # [rows, heads] int32, packed ue8m0
weights: torch.Tensor # [rows, heads] fp32 head weights
compress_lens: torch.Tensor # [rows] int32, compressed positions the row sees
# [rows] int32: first flattened-K column of the row's request; selections
# are flattened-K columns, request start + compressed position
request_starts: torch.Tensor
lens_per_request: List[int] # compressed positions at each request's newest token
rows_per_request: List[int] # query rows of each request
# the flattened index K of the batch (int8 [n, 64], int32 [n]), column j of
# a row's dense scores against kv[request_starts[t] + j]
kv: Tuple[torch.Tensor, torch.Tensor]
k_cache: torch.Tensor # [pages, page_size, 1, 68] uint8, the layer's index-K pool
page_size: int # index-K pool page size, in slots
# [rows, kv pages] int32: the row's request in the KV pool, whose pages hold
# kv_page_size tokens; expand_index_page_table gives the index-K pool's
kv_page_table: torch.Tensor
kv_page_size: int
compress_ratio: int
@property
def num_rows(self) -> int:
return self.q_fp4.shape[0]
def expand_index_page_table(
page_table: torch.Tensor,
*,
full_page_size: int,
compress_ratio: int,
index_page_size: int,
) -> torch.Tensor:
"""Block table of a low-ratio indexer-K pool, which pages at `index_page_size`
slots: [bs, n] -> [bs, n * blocks_per_page] int32. The kernel reads compressed
slot j at page_table[b, j // index_page_size] * index_page_size + j %
index_page_size, which after this expansion is the c1/c2 pool slot of the same
position."""
slots_per_page = full_page_size // compress_ratio
assert slots_per_page % index_page_size == 0, (
f"{full_page_size = } / {compress_ratio = } must be a multiple of "
f"{index_page_size = }"
)
blocks_per_page = slots_per_page // index_page_size
if blocks_per_page == 1:
return page_table
bs, n = page_table.shape
base = page_table.to(torch.int64) * blocks_per_page
offsets = torch.arange(blocks_per_page, device=page_table.device, dtype=torch.int64)
expanded = base.unsqueeze(-1) + offsets # [bs, n, blocks_per_page]
return expanded.reshape(bs, n * blocks_per_page).to(torch.int32)
class CandidateIndexer(ABC):
"""The two-level low-ratio indexer. The candidate source publishes what its
consumers need to select among its top candidate blocks, on the forward
metadata; a consumer selects its top-k from that.
TODO(candidate): Hopper decode and the torch prefill path still publish and
consume masks inline in the backend; their decode side is not behind this
protocol yet."""
@abstractmethod
def publish_prefill(
self, inputs: PrefillIndexerInputs, out_positions: torch.Tensor
) -> CandidateMetadata:
"""The source layer's own top-``k`` (``k = out_positions.shape[1]``) as
flattened-K columns, ``-1`` padded, unordered, plus its candidates for
the chunk: both come from one pass over its dense scores."""
@abstractmethod
def select_prefill(
self,
published: CandidateMetadata,
inputs: PrefillIndexerInputs,
out_positions: torch.Tensor,
) -> None:
"""A consumer's top-``k`` over its published candidates, in the same
layout as ``publish_prefill`` writes."""
@abstractmethod
def prefill_tail(
self, published: CandidateMetadata, tail_lens: List[int]
) -> CandidateMetadata:
"""The candidates of the last ``tail_lens[b]`` rows of each request, for
the late layers that run on the tail only."""
def make_candidate_indexer(
topk_blocks: int, block_size: int
) -> Optional[DeepGemmCandidateIndexer]:
"""The paged fp4 decode path's two-level indexer; None on Hopper, whose decode
indexer selects through masks inline."""
) -> Optional[CandidateIndexer]:
"""DeepGEMM's paged sparse indexer on SM100, with the dense-score
implementation for prefill under context parallelism (a rank's local rows
are not the page table's rows); None on Hopper, whose decode and prefill
indexers select through masks inline."""
if topk_blocks <= 0 or get_platform().device_sm < 100:
return None
from sglang.srt.layers.deep_gemm_wrapper.configurer import (
@@ -59,18 +148,36 @@ def make_candidate_indexer(
from sglang.srt.layers.attention.dsv4.candidate_indexer_deep_gemm import (
DeepGemmCandidateIndexer,
)
from sglang.srt.layers.attention.dsv4.dense_prefill_indexer import (
DenseCandidateIndexer,
)
return DeepGemmCandidateIndexer(topk_blocks, block_size)
prefill_dense = None
if get_parallel().attn_cp_size > 1:
prefill_dense = DenseCandidateIndexer(topk_blocks, block_size)
return DeepGemmCandidateIndexer(
topk_blocks, block_size, prefill_dense=prefill_dense
)
# TODO(candidate): Hopper decode and prefill still select through these masks
# inline in the backend; move them behind the protocol as publish/select_prefill.
# TODO(candidate): Hopper decode and the torch prefill path still select through
# these masks inline in the backend.
@dataclass
class CandidateMasks(CandidateMetadata):
mask: Optional[torch.Tensor] = None # decode: [rows, width] bool
request_masks: Optional[List[torch.Tensor]] = None # prefill: [rows_b, lc_b] each
def cut_request_masks(masks: CandidateMasks, tail_lens: List[int]) -> CandidateMasks:
"""The last ``tail_lens[b]`` rows of each request's mask."""
assert masks.request_masks is not None, "prefill masks missing"
return CandidateMasks(
request_masks=[
mask[mask.shape[0] - t :] for mask, t in zip(masks.request_masks, tail_lens)
]
)
class PrefillCandidateBlocks(CandidateMetadata, msgspec.Struct):
request_blocks: List[torch.Tensor]
@@ -1,7 +1,7 @@
from __future__ import annotations
from dataclasses import dataclass
from typing import Optional
from typing import List, Optional
import torch
@@ -14,10 +14,18 @@ from sglang.kernels.ops.attention.dsv4.topk import (
plan_topk_v2,
topk_transform_bf16_small,
topk_transform_paged_v2,
topk_transform_ragged_v2,
)
from sglang.srt.layers.attention.dsv4.candidate_indexer import (
CandidateIndexer,
CandidateMetadata,
IndexerInputs,
PrefillIndexerInputs,
expand_index_page_table,
)
from sglang.srt.layers.attention.dsv4.dense_prefill_indexer import (
DenseCandidateIndexer,
score_tiles,
)
from sglang.srt.layers.attention.dsv4.indexer import (
deep_gemm_fp4_paged_mqa_logits,
@@ -134,16 +142,36 @@ def topk_transform_sparse(
)
# TODO(dark): support publish prefill/select prefill
@dataclass
class PrefillSparseBlockTable(SparseBlockTable):
"""Published by a prefill chunk's candidate source: one row per query token.
The inputs stay attached so the late-layer tail can rebuild the table for
its rows (the DeepGEMM schedule is per row set, it cannot be sliced)."""
compress_lens: torch.Tensor # [rows] int32
page_table: torch.Tensor # [rows, index pages] int32
request_ids: torch.Tensor # [rows] int32
q_dtype: torch.dtype
page_size: int # index-K pool page size (slots)
rows_per_request: List[int] # rows of each request, in row order
# TODO(dark): support fusion of publish + topk of publish layer
class DeepGemmCandidateIndexer:
def __init__(self, topk_blocks: int, block_size: int):
class DeepGemmCandidateIndexer(CandidateIndexer):
def __init__(
self,
topk_blocks: int,
block_size: int,
prefill_dense: Optional[DenseCandidateIndexer] = None,
):
assert block_size == CANDIDATE_BLOCK_SIZE, block_size
self.topk_blocks = topk_blocks
self.block_size = block_size
self.alt_stream = torch.cuda.Stream()
self._row_ids: Optional[torch.Tensor] = None
self._retired_row_ids: list = [] # captured graphs keep reading the buffers they saw
# set for the forwards whose prefill rows the page table does not describe
self._prefill_dense = prefill_dense
def _request_ids(
self, request_ids: Optional[torch.Tensor], rows: int, device: torch.device
@@ -253,3 +281,149 @@ class DeepGemmCandidateIndexer:
logits = self._scores(table, inputs)
# decode carries no raw_indices; the kernel writes slots only
topk_transform_sparse(logits, table.valid_lens, table, page_indices)
def publish_prefill(
self, inputs: PrefillIndexerInputs, out_positions: torch.Tensor
) -> CandidateMetadata:
"""The source layer's own top-k and the block table of the chunk from
one pass over its dense scores, tile by tile: per tile the plain top-k
into ``out_positions`` and one read for the block keys."""
if self._prefill_dense is not None:
return self._prefill_dense.publish_prefill(inputs, out_positions)
rows = inputs.num_rows
device = inputs.q_fp4.device
nblocks, valid_lens = candidate_row_lens(inputs.compress_lens, self.topk_blocks)
blocks = torch.empty(rows, self.topk_blocks, dtype=torch.int32, device=device)
# the block keys read the score rows through 32-byte vectors
for tile, logits in score_tiles(inputs, width_align=8):
lens = inputs.compress_lens[tile]
topk_transform_ragged_v2(
logits,
lens,
out_offsets=inputs.request_starts[tile],
out_indices=out_positions[tile],
)
# keys past a row's block count stay unset; the top-k reads a row
# up to its block count only (v2 wants the stride a multiple of 4)
width_blocks = (
logits.shape[1] + CANDIDATE_BLOCK_SIZE - 1
) // CANDIDATE_BLOCK_SIZE
keys = logits.new_empty(logits.shape[0], (width_blocks + 3) // 4 * 4)
amax8_varlen(logits, lens, out=keys)
topk_transform_ragged_v2(
keys,
nblocks[tile],
out_offsets=torch.zeros(
logits.shape[0], dtype=torch.int32, device=device
),
out_indices=blocks[tile],
)
request_ids = torch.repeat_interleave(
torch.arange(
len(inputs.rows_per_request), dtype=torch.int32, device=device
),
torch.tensor(inputs.rows_per_request, dtype=torch.int64, device=device),
output_size=rows,
)
# index-K pool pages of each row's request
page_table = expand_index_page_table(
inputs.kv_page_table,
full_page_size=inputs.kv_page_size,
compress_ratio=inputs.compress_ratio,
index_page_size=inputs.page_size,
).contiguous()
return self._prefill_table(
blocks,
inputs.compress_lens,
page_table,
inputs.page_size,
request_ids,
inputs.rows_per_request,
inputs.q_fp4.dtype,
valid_lens,
)
def _prefill_table(
self,
blocks,
compress_lens,
page_table,
page_size,
request_ids,
rows_per_request,
q_dtype,
valid_lens,
) -> PrefillSparseBlockTable:
# in place: ascending, INT32_MAX padded, plus the blocks as pool slots / 8
phys_blocks = sort_candidate_blocks(
blocks, compress_lens, page_table, page_size
)
schedule = build_sparse_indexer_schedule(
blocks, compress_lens, page_table, page_size, q_dtype, request_ids
)
ready = torch.cuda.Event()
ready.record()
return PrefillSparseBlockTable(
blocks=blocks,
schedule=schedule,
phys_blocks=phys_blocks,
valid_lens=valid_lens,
ready=ready,
compress_lens=compress_lens,
page_table=page_table,
request_ids=request_ids,
q_dtype=q_dtype,
page_size=page_size,
rows_per_request=rows_per_request,
)
def prefill_tail(
self, published: CandidateMetadata, tail_lens: List[int]
) -> CandidateMetadata:
if self._prefill_dense is not None:
return self._prefill_dense.prefill_tail(published, tail_lens)
table = published
assert isinstance(table, PrefillSparseBlockTable), "prefill block table missing"
rows, start = [], 0
for n, t in zip(table.rows_per_request, tail_lens):
rows.append(torch.arange(start + n - t, start + n))
start += n
rows = torch.cat(rows).to(table.blocks.device)
compress_lens = table.compress_lens[rows].contiguous()
_, valid_lens = candidate_row_lens(compress_lens, self.topk_blocks)
return self._prefill_table(
table.blocks[rows].contiguous(),
compress_lens,
table.page_table[rows].contiguous(),
table.page_size,
table.request_ids[rows].contiguous(),
tail_lens,
table.q_dtype,
valid_lens,
)
def select_prefill(
self,
published: CandidateMetadata,
inputs: PrefillIndexerInputs,
out_positions: torch.Tensor,
) -> None:
if self._prefill_dense is not None:
return self._prefill_dense.select_prefill(published, inputs, out_positions)
table = published
assert isinstance(table, PrefillSparseBlockTable), "prefill block table missing"
rows, heads = inputs.q_sf.shape
logits = sparse_logits(
inputs.q_fp4.view(rows, 1, heads, 64),
inputs.q_sf.view(rows, 1, heads),
inputs.k_cache,
inputs.weights.to(torch.bfloat16),
table,
)
# request-relative compressed positions, -1 padded
topk_transform_bf16_small(
logits, table.valid_lens, table.blocks, out_positions, CANDIDATE_BLOCK_SIZE
)
out_positions.add_(
torch.where(out_positions >= 0, inputs.request_starts[:, None], 0)
)
@@ -1,9 +1,14 @@
from __future__ import annotations
from typing import Iterator, List, Optional
import torch
from sglang.srt.layers.attention.dsv4.candidate_indexer import (
CandidateIndexer,
CandidateMetadata,
PrefillCandidateBlocks,
PrefillIndexerInputs,
candidate_block_mask,
mask_topk_scores,
select_candidate_block_ids,
@@ -58,18 +63,7 @@ def dense_prefill_topk(
width = ceil_align(max((n for _, n in request_lengths), default=0), 4)
if row == 0 or width == 0:
return selected, published
row_alignment = 128 // q[0].shape[1]
rows_per_chunk = mqa_logits_rows_per_chunk(
num_rows=ceil_align(row, row_alignment),
row_bytes=mqa_logits_row_bytes(width),
budget_bytes=_SCORE_BUDGET_BYTES,
)
if rows_per_chunk is None:
rows_per_chunk = row
else:
rows_per_chunk = max(
row_alignment, rows_per_chunk // row_alignment * row_alignment
)
rows_per_chunk = _rows_per_chunk(row, width, heads=q[0].shape[1])
for offset in range(0, row, rows_per_chunk):
rows = slice(offset, min(offset + rows_per_chunk, row))
_select_tile(
@@ -89,6 +83,141 @@ def dense_prefill_topk(
return selected, published
def _rows_per_chunk(rows: int, width: int, *, heads: int) -> int:
"""Query rows per logits tile so one fp32 [rows, width] tile fits the
budget; the row count stays a multiple of the kernel's row alignment."""
row_alignment = 128 // heads
rows_per_chunk = mqa_logits_rows_per_chunk(
num_rows=ceil_align(rows, row_alignment),
row_bytes=mqa_logits_row_bytes(width),
budget_bytes=_SCORE_BUDGET_BYTES,
)
if rows_per_chunk is None:
return rows
return max(row_alignment, rows_per_chunk // row_alignment * row_alignment)
def score_tiles(
inputs: PrefillIndexerInputs, *, width_align: int = 4
) -> Iterator[tuple[slice, torch.Tensor]]:
"""The dense scores of ``inputs`` row tile by row tile under the budget:
``(rows, logits)`` with fp32 ``logits[i, j]`` the score of query row
``rows.start + i`` against ``kv[request_starts + j]``, garbage past the row's
``compress_lens``; the width is the batch's largest context aligned to
``width_align`` columns."""
from deep_gemm import fp8_fp4_mqa_logits
rows = inputs.num_rows
width = ceil_align(max(inputs.lens_per_request, default=0), width_align)
if rows == 0 or width == 0:
return
rows_per_chunk = _rows_per_chunk(rows, width, heads=inputs.q_fp4.shape[1])
for offset in range(0, rows, rows_per_chunk):
tile = slice(offset, min(offset + rows_per_chunk, rows))
starts = inputs.request_starts[tile]
yield (
tile,
fp8_fp4_mqa_logits(
(inputs.q_fp4[tile], inputs.q_sf[tile]),
inputs.kv,
inputs.weights[tile],
starts,
starts + inputs.compress_lens[tile],
False,
width,
),
)
def _dense_topk(
inputs: PrefillIndexerInputs,
out_positions: torch.Tensor,
*,
topk_blocks: int,
block_size: int,
publish: bool,
candidates: Optional[PrefillCandidateBlocks],
) -> Optional[PrefillCandidateBlocks]:
selected, published = dense_prefill_topk(
q=(inputs.q_fp4, inputs.q_sf),
kv=inputs.kv,
weights=inputs.weights,
starts=inputs.request_starts,
lengths=inputs.compress_lens,
request_lengths=list(zip(inputs.rows_per_request, inputs.lens_per_request)),
topk=out_positions.shape[1],
candidate_topk_blocks=topk_blocks,
candidate_block_size=block_size,
publish_candidates=publish,
candidates=candidates,
)
out_positions.copy_(selected)
return published
def plain_prefill_topk(
inputs: PrefillIndexerInputs, out_positions: torch.Tensor
) -> None:
"""The top-k of an index layer outside the candidate scheme, over its dense
scores tile by tile."""
_dense_topk(
inputs,
out_positions,
topk_blocks=0,
block_size=1,
publish=False,
candidates=None,
)
class DenseCandidateIndexer(CandidateIndexer):
"""Candidates as block ids per request (``PrefillCandidateBlocks``): the
source keeps its best blocks from its dense scores, a consumer masks its own
dense scores to -inf outside them and runs the plain top-k, tile by tile.
The prefill implementation for the CP layout; Hopper still runs the same
selection inline in the backend."""
def __init__(self, topk_blocks: int, block_size: int):
self.topk_blocks = topk_blocks
self.block_size = block_size
def publish_prefill(
self, inputs: PrefillIndexerInputs, out_positions: torch.Tensor
) -> PrefillCandidateBlocks:
published = _dense_topk(
inputs,
out_positions,
topk_blocks=self.topk_blocks,
block_size=self.block_size,
publish=True,
candidates=None,
)
assert published is not None
return published
def select_prefill(
self,
published: CandidateMetadata,
inputs: PrefillIndexerInputs,
out_positions: torch.Tensor,
) -> None:
assert isinstance(published, PrefillCandidateBlocks), "candidate blocks missing"
_dense_topk(
inputs,
out_positions,
topk_blocks=self.topk_blocks,
block_size=self.block_size,
publish=False,
candidates=published,
)
def prefill_tail(
self, published: CandidateMetadata, tail_lens: List[int]
) -> PrefillCandidateBlocks:
assert isinstance(published, PrefillCandidateBlocks), "candidate blocks missing"
return published.tail(tail_lens)
def _select_tile(
*,
q: tuple[torch.Tensor, torch.Tensor],