[DSv4.1] Score prefill consumer index layers on candidate blocks with DeepGEMM (#40352)
Co-authored-by: luoyuan.luo <luoyuan.luo@antgroup.com> Co-authored-by: DarkSharpness <76582120+DarkSharpness@users.noreply.github.com>
This commit is contained in:
co-authored by
luoyuan.luo
DarkSharpness
parent
4a1b69abc8
commit
c79510cc2a
@@ -60,7 +60,9 @@ from sglang.srt.layers.attention.dsv4.candidate_indexer import (
|
||||
CandidateMasks,
|
||||
CandidateMetadata,
|
||||
IndexerInputs,
|
||||
PrefillCandidateBlocks,
|
||||
PrefillIndexerInputs,
|
||||
cut_request_masks,
|
||||
expand_index_page_table,
|
||||
make_candidate_indexer,
|
||||
mask_topk_scores,
|
||||
published_masks,
|
||||
@@ -71,7 +73,7 @@ from sglang.srt.layers.attention.dsv4.compressor_v2 import (
|
||||
FusedCompressMetadata,
|
||||
create_paged_compressor_data,
|
||||
)
|
||||
from sglang.srt.layers.attention.dsv4.dense_prefill_indexer import dense_prefill_topk
|
||||
from sglang.srt.layers.attention.dsv4.dense_prefill_indexer import plain_prefill_topk
|
||||
from sglang.srt.layers.attention.dsv4.dsv41_sparse import (
|
||||
_rope_fq4,
|
||||
token_req_indices,
|
||||
@@ -258,33 +260,6 @@ def _maybe_precompute_flashmla_sched_meta(
|
||||
flashmla_metadata.num_splits = num_splits
|
||||
|
||||
|
||||
def _expand_index_page_table(
|
||||
page_table: torch.Tensor,
|
||||
*,
|
||||
full_page_size: int,
|
||||
compress_ratio: int,
|
||||
index_page_size: int,
|
||||
) -> torch.Tensor:
|
||||
"""Block table of a low-ratio indexer-K pool, which pages at `index_page_size`
|
||||
slots: [bs, n] -> [bs, n * blocks_per_page] int32. The kernel reads compressed
|
||||
slot j at page_table[b, j // index_page_size] * index_page_size + j %
|
||||
index_page_size, which after this expansion is the c1/c2 pool slot of the same
|
||||
position."""
|
||||
slots_per_page = full_page_size // compress_ratio
|
||||
assert slots_per_page % index_page_size == 0, (
|
||||
f"{full_page_size = } / {compress_ratio = } must be a multiple of "
|
||||
f"{index_page_size = }"
|
||||
)
|
||||
blocks_per_page = slots_per_page // index_page_size
|
||||
if blocks_per_page == 1:
|
||||
return page_table
|
||||
bs, n = page_table.shape
|
||||
base = page_table.to(torch.int64) * blocks_per_page
|
||||
offsets = torch.arange(blocks_per_page, device=page_table.device, dtype=torch.int64)
|
||||
expanded = base.unsqueeze(-1) + offsets # [bs, n, blocks_per_page]
|
||||
return expanded.reshape(bs, n * blocks_per_page).to(torch.int32)
|
||||
|
||||
|
||||
# Arbitrary cap on one bf16 [rows, heads, lc] score chunk; transients run ~3x this.
|
||||
_TORCH_INDEXER_SCORE_BUDGET_BYTES = 1 << 30
|
||||
|
||||
@@ -1357,7 +1332,7 @@ class DeepseekV4AttnBackend(
|
||||
index_page_size = self.token_to_kv_pool.get_index_k_page_size(
|
||||
compress_ratio
|
||||
)
|
||||
page_table = _expand_index_page_table(
|
||||
page_table = expand_index_page_table(
|
||||
page_table,
|
||||
full_page_size=self.page_size,
|
||||
compress_ratio=compress_ratio,
|
||||
@@ -1544,7 +1519,7 @@ class DeepseekV4AttnBackend(
|
||||
if max_seq_len is not None:
|
||||
num_pages = min(max_seq_len // self.page_size, num_pages)
|
||||
index_page_size = self.token_to_kv_pool.get_index_k_page_size(compress_ratio)
|
||||
page_table = _expand_index_page_table(
|
||||
page_table = expand_index_page_table(
|
||||
core.page_table[:, :num_pages],
|
||||
full_page_size=self.page_size,
|
||||
compress_ratio=compress_ratio,
|
||||
@@ -1740,23 +1715,6 @@ class DeepseekV4AttnBackend(
|
||||
get_local_dp_buffer_len(),
|
||||
)
|
||||
tail = tail_metadata.late_layer_tail
|
||||
tail_lens_cpu = (
|
||||
tail.local_lens_cpu
|
||||
if tail.cp_metadata is not None
|
||||
else tail.extend_seq_lens_cpu
|
||||
)
|
||||
# TODO(candidate): goes away once the source publishes its tail rows straight
|
||||
# onto the tail metadata (publish_prefill); until then cut the full masks.
|
||||
full_masks = self.forward_metadata.candidate_metadata
|
||||
if isinstance(full_masks, PrefillCandidateBlocks):
|
||||
tail_metadata.candidate_metadata = full_masks.tail(tail_lens_cpu)
|
||||
elif isinstance(full_masks, CandidateMasks) and full_masks.request_masks:
|
||||
tail_metadata.candidate_metadata = CandidateMasks(
|
||||
request_masks=[
|
||||
mask[mask.shape[0] - t :]
|
||||
for mask, t in zip(full_masks.request_masks, tail_lens_cpu)
|
||||
]
|
||||
)
|
||||
# The layers before the switch published top-k into the full metadata's
|
||||
# buffers; carry the tail rows into the tail metadata's (padding stays -1).
|
||||
full_core = saved[0].core_attn_metadata
|
||||
@@ -3243,12 +3201,8 @@ class DeepseekV4AttnBackend(
|
||||
num_tokens = pos.shape[0]
|
||||
if not slot_chunks or num_tokens == 0:
|
||||
if indexer.is_candidate_source:
|
||||
self.forward_metadata.candidate_metadata = PrefillCandidateBlocks(
|
||||
request_blocks=[
|
||||
torch.empty((length, 0), dtype=torch.int32, device=device)
|
||||
for length in q_lens_cpu
|
||||
]
|
||||
)
|
||||
# no row to publish for; the consumers return here as well
|
||||
self.forward_metadata.candidate_metadata = None
|
||||
return
|
||||
k_slots = torch.cat(slot_chunks)
|
||||
k_fp4, k_sf = pool.get_low_ratio_index_k_fp4(layer.layer_id, k_slots)
|
||||
@@ -3265,26 +3219,28 @@ class DeepseekV4AttnBackend(
|
||||
q_lens.to(torch.int64),
|
||||
output_size=num_tokens,
|
||||
)
|
||||
candidates = None
|
||||
if indexer.uses_candidates and not indexer.is_candidate_source:
|
||||
candidates = self.forward_metadata.candidate_metadata
|
||||
assert isinstance(candidates, PrefillCandidateBlocks)
|
||||
topk = indexer.index_topk
|
||||
selected, published = dense_prefill_topk(
|
||||
q=(q_fp4, q_sf),
|
||||
kv=(k_fp4, k_sf),
|
||||
weights=weights,
|
||||
starts=ks,
|
||||
lengths=compress_lens,
|
||||
request_lengths=list(zip(q_lens_cpu, lc_per_req)),
|
||||
topk=topk,
|
||||
candidate_topk_blocks=indexer.candidate_topk_blocks,
|
||||
candidate_block_size=indexer.candidate_block_size,
|
||||
publish_candidates=indexer.is_candidate_source,
|
||||
candidates=candidates,
|
||||
inputs = self._prefill_indexer_inputs(
|
||||
layer,
|
||||
(q_fp4, q_sf),
|
||||
(k_fp4, k_sf),
|
||||
weights,
|
||||
compress_lens,
|
||||
ks,
|
||||
lc_per_req,
|
||||
list(q_lens_cpu),
|
||||
)
|
||||
if published is not None:
|
||||
self.forward_metadata.candidate_metadata = published
|
||||
topk = indexer.index_topk
|
||||
selected = torch.full((num_tokens, topk), -1, dtype=torch.int32, device=device)
|
||||
if indexer.is_candidate_source:
|
||||
self._publish_prefill(
|
||||
self.candidate_indexer.publish_prefill(inputs, selected)
|
||||
)
|
||||
elif indexer.uses_candidates:
|
||||
self.candidate_indexer.select_prefill(
|
||||
self.forward_metadata.candidate_metadata, inputs, selected
|
||||
)
|
||||
else:
|
||||
plain_prefill_topk(inputs, selected)
|
||||
# ascending positions, padding last: the layout the consumers expect
|
||||
unselected = torch.iinfo(torch.int32).max
|
||||
selected = selected.masked_fill(selected < 0, unselected).sort(dim=-1).values
|
||||
@@ -3297,6 +3253,61 @@ class DeepseekV4AttnBackend(
|
||||
chosen, selected - ks[:, None], -1
|
||||
)
|
||||
|
||||
def _prefill_indexer_inputs(
|
||||
self, layer, q, kv, weights, compress_lens, ks, lc_per_req, q_lens_cpu
|
||||
) -> PrefillIndexerInputs:
|
||||
pool = self.token_to_kv_pool
|
||||
ratio = layer.compress_ratio
|
||||
num_tokens = q[0].shape[0]
|
||||
index_page_size = pool.get_index_k_page_size(ratio)
|
||||
k_cache = pool.get_index_k_with_scale_buffer(layer.layer_id)
|
||||
assert k_cache.dim() == 2
|
||||
return PrefillIndexerInputs(
|
||||
q_fp4=q[0],
|
||||
q_sf=q[1],
|
||||
weights=weights,
|
||||
compress_lens=compress_lens,
|
||||
request_starts=ks,
|
||||
lens_per_request=lc_per_req,
|
||||
rows_per_request=q_lens_cpu,
|
||||
kv=kv,
|
||||
k_cache=k_cache.view(k_cache.shape[0], index_page_size, 1, 68),
|
||||
page_size=index_page_size,
|
||||
kv_page_table=self.forward_metadata.core_metadata.page_table[:num_tokens],
|
||||
kv_page_size=self.page_size,
|
||||
compress_ratio=ratio,
|
||||
)
|
||||
|
||||
def _tail_lens_to_publish(self) -> Optional[List[int]]:
|
||||
"""Rows per request of the late-layer tail, when the layers after the
|
||||
switch will consume what is published now."""
|
||||
tail_metadata = self.tail_forward_metadata
|
||||
if tail_metadata is None or tail_metadata is self.forward_metadata:
|
||||
return None
|
||||
tail = tail_metadata.late_layer_tail
|
||||
return (
|
||||
tail.local_lens_cpu
|
||||
if tail.cp_metadata is not None
|
||||
else tail.extend_seq_lens_cpu
|
||||
)
|
||||
|
||||
def _publish_prefill(self, published: CandidateMetadata) -> None:
|
||||
self.forward_metadata.candidate_metadata = published
|
||||
tail_lens = self._tail_lens_to_publish()
|
||||
if tail_lens is not None:
|
||||
self.tail_forward_metadata.candidate_metadata = (
|
||||
self.candidate_indexer.prefill_tail(published, tail_lens)
|
||||
)
|
||||
|
||||
def _publish_prefill_masks(self, masks: CandidateMasks) -> None:
|
||||
"""The torch prefill path's inline masks (see the TODO on CandidateMasks)."""
|
||||
self.forward_metadata.candidate_metadata = masks
|
||||
tail_lens = self._tail_lens_to_publish()
|
||||
if tail_lens is not None:
|
||||
self.tail_forward_metadata.candidate_metadata = cut_request_masks(
|
||||
masks, tail_lens
|
||||
)
|
||||
|
||||
def _low_ratio_index_topk_prefill_graph(self, layer, pos, q, w) -> None:
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer import (
|
||||
quantize_fp4_indexer_tensor,
|
||||
@@ -3607,9 +3618,7 @@ class DeepseekV4AttnBackend(
|
||||
if masks is not None:
|
||||
publish.append(torch.cat(masks) if len(masks) > 1 else masks[0])
|
||||
if publish is not None:
|
||||
self.forward_metadata.candidate_metadata = CandidateMasks(
|
||||
request_masks=publish
|
||||
)
|
||||
self._publish_prefill_masks(CandidateMasks(request_masks=publish))
|
||||
|
||||
def get_swa_out_cache_loc(self, forward_batch: ForwardBatch) -> torch.Tensor:
|
||||
"""Idle always re-translates at store time: its metadata may be stale, and
|
||||
|
||||
@@ -1,19 +1,15 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, List, Optional, Union
|
||||
from typing import List, Optional, Tuple, Union
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from sglang.srt.layers.attention.dsv4.metadata import PagedIndexerMetadata
|
||||
from sglang.srt.runtime_context import get_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.attention.dsv4.candidate_indexer_deep_gemm import (
|
||||
DeepGemmCandidateIndexer,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel, get_platform
|
||||
|
||||
|
||||
class CandidateMetadata:
|
||||
@@ -40,11 +36,104 @@ class IndexerInputs:
|
||||
return self.q_fp4.shape[0]
|
||||
|
||||
|
||||
class PrefillIndexerInputs(msgspec.Struct, frozen=True):
|
||||
"""One index layer's operands on the dense fp4 prefill path: the chunk's query
|
||||
rows, one request's rows consecutive."""
|
||||
|
||||
q_fp4: torch.Tensor # [rows, heads, 64] int8, packed fp4
|
||||
q_sf: torch.Tensor # [rows, heads] int32, packed ue8m0
|
||||
weights: torch.Tensor # [rows, heads] fp32 head weights
|
||||
compress_lens: torch.Tensor # [rows] int32, compressed positions the row sees
|
||||
# [rows] int32: first flattened-K column of the row's request; selections
|
||||
# are flattened-K columns, request start + compressed position
|
||||
request_starts: torch.Tensor
|
||||
lens_per_request: List[int] # compressed positions at each request's newest token
|
||||
rows_per_request: List[int] # query rows of each request
|
||||
# the flattened index K of the batch (int8 [n, 64], int32 [n]), column j of
|
||||
# a row's dense scores against kv[request_starts[t] + j]
|
||||
kv: Tuple[torch.Tensor, torch.Tensor]
|
||||
k_cache: torch.Tensor # [pages, page_size, 1, 68] uint8, the layer's index-K pool
|
||||
page_size: int # index-K pool page size, in slots
|
||||
# [rows, kv pages] int32: the row's request in the KV pool, whose pages hold
|
||||
# kv_page_size tokens; expand_index_page_table gives the index-K pool's
|
||||
kv_page_table: torch.Tensor
|
||||
kv_page_size: int
|
||||
compress_ratio: int
|
||||
|
||||
@property
|
||||
def num_rows(self) -> int:
|
||||
return self.q_fp4.shape[0]
|
||||
|
||||
|
||||
def expand_index_page_table(
|
||||
page_table: torch.Tensor,
|
||||
*,
|
||||
full_page_size: int,
|
||||
compress_ratio: int,
|
||||
index_page_size: int,
|
||||
) -> torch.Tensor:
|
||||
"""Block table of a low-ratio indexer-K pool, which pages at `index_page_size`
|
||||
slots: [bs, n] -> [bs, n * blocks_per_page] int32. The kernel reads compressed
|
||||
slot j at page_table[b, j // index_page_size] * index_page_size + j %
|
||||
index_page_size, which after this expansion is the c1/c2 pool slot of the same
|
||||
position."""
|
||||
slots_per_page = full_page_size // compress_ratio
|
||||
assert slots_per_page % index_page_size == 0, (
|
||||
f"{full_page_size = } / {compress_ratio = } must be a multiple of "
|
||||
f"{index_page_size = }"
|
||||
)
|
||||
blocks_per_page = slots_per_page // index_page_size
|
||||
if blocks_per_page == 1:
|
||||
return page_table
|
||||
bs, n = page_table.shape
|
||||
base = page_table.to(torch.int64) * blocks_per_page
|
||||
offsets = torch.arange(blocks_per_page, device=page_table.device, dtype=torch.int64)
|
||||
expanded = base.unsqueeze(-1) + offsets # [bs, n, blocks_per_page]
|
||||
return expanded.reshape(bs, n * blocks_per_page).to(torch.int32)
|
||||
|
||||
|
||||
class CandidateIndexer(ABC):
|
||||
"""The two-level low-ratio indexer. The candidate source publishes what its
|
||||
consumers need to select among its top candidate blocks, on the forward
|
||||
metadata; a consumer selects its top-k from that.
|
||||
|
||||
TODO(candidate): Hopper decode and the torch prefill path still publish and
|
||||
consume masks inline in the backend; their decode side is not behind this
|
||||
protocol yet."""
|
||||
|
||||
@abstractmethod
|
||||
def publish_prefill(
|
||||
self, inputs: PrefillIndexerInputs, out_positions: torch.Tensor
|
||||
) -> CandidateMetadata:
|
||||
"""The source layer's own top-``k`` (``k = out_positions.shape[1]``) as
|
||||
flattened-K columns, ``-1`` padded, unordered, plus its candidates for
|
||||
the chunk: both come from one pass over its dense scores."""
|
||||
|
||||
@abstractmethod
|
||||
def select_prefill(
|
||||
self,
|
||||
published: CandidateMetadata,
|
||||
inputs: PrefillIndexerInputs,
|
||||
out_positions: torch.Tensor,
|
||||
) -> None:
|
||||
"""A consumer's top-``k`` over its published candidates, in the same
|
||||
layout as ``publish_prefill`` writes."""
|
||||
|
||||
@abstractmethod
|
||||
def prefill_tail(
|
||||
self, published: CandidateMetadata, tail_lens: List[int]
|
||||
) -> CandidateMetadata:
|
||||
"""The candidates of the last ``tail_lens[b]`` rows of each request, for
|
||||
the late layers that run on the tail only."""
|
||||
|
||||
|
||||
def make_candidate_indexer(
|
||||
topk_blocks: int, block_size: int
|
||||
) -> Optional[DeepGemmCandidateIndexer]:
|
||||
"""The paged fp4 decode path's two-level indexer; None on Hopper, whose decode
|
||||
indexer selects through masks inline."""
|
||||
) -> Optional[CandidateIndexer]:
|
||||
"""DeepGEMM's paged sparse indexer on SM100, with the dense-score
|
||||
implementation for prefill under context parallelism (a rank's local rows
|
||||
are not the page table's rows); None on Hopper, whose decode and prefill
|
||||
indexers select through masks inline."""
|
||||
if topk_blocks <= 0 or get_platform().device_sm < 100:
|
||||
return None
|
||||
from sglang.srt.layers.deep_gemm_wrapper.configurer import (
|
||||
@@ -59,18 +148,36 @@ def make_candidate_indexer(
|
||||
from sglang.srt.layers.attention.dsv4.candidate_indexer_deep_gemm import (
|
||||
DeepGemmCandidateIndexer,
|
||||
)
|
||||
from sglang.srt.layers.attention.dsv4.dense_prefill_indexer import (
|
||||
DenseCandidateIndexer,
|
||||
)
|
||||
|
||||
return DeepGemmCandidateIndexer(topk_blocks, block_size)
|
||||
prefill_dense = None
|
||||
if get_parallel().attn_cp_size > 1:
|
||||
prefill_dense = DenseCandidateIndexer(topk_blocks, block_size)
|
||||
return DeepGemmCandidateIndexer(
|
||||
topk_blocks, block_size, prefill_dense=prefill_dense
|
||||
)
|
||||
|
||||
|
||||
# TODO(candidate): Hopper decode and prefill still select through these masks
|
||||
# inline in the backend; move them behind the protocol as publish/select_prefill.
|
||||
# TODO(candidate): Hopper decode and the torch prefill path still select through
|
||||
# these masks inline in the backend.
|
||||
@dataclass
|
||||
class CandidateMasks(CandidateMetadata):
|
||||
mask: Optional[torch.Tensor] = None # decode: [rows, width] bool
|
||||
request_masks: Optional[List[torch.Tensor]] = None # prefill: [rows_b, lc_b] each
|
||||
|
||||
|
||||
def cut_request_masks(masks: CandidateMasks, tail_lens: List[int]) -> CandidateMasks:
|
||||
"""The last ``tail_lens[b]`` rows of each request's mask."""
|
||||
assert masks.request_masks is not None, "prefill masks missing"
|
||||
return CandidateMasks(
|
||||
request_masks=[
|
||||
mask[mask.shape[0] - t :] for mask, t in zip(masks.request_masks, tail_lens)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
class PrefillCandidateBlocks(CandidateMetadata, msgspec.Struct):
|
||||
request_blocks: List[torch.Tensor]
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
from typing import List, Optional
|
||||
|
||||
import torch
|
||||
|
||||
@@ -14,10 +14,18 @@ from sglang.kernels.ops.attention.dsv4.topk import (
|
||||
plan_topk_v2,
|
||||
topk_transform_bf16_small,
|
||||
topk_transform_paged_v2,
|
||||
topk_transform_ragged_v2,
|
||||
)
|
||||
from sglang.srt.layers.attention.dsv4.candidate_indexer import (
|
||||
CandidateIndexer,
|
||||
CandidateMetadata,
|
||||
IndexerInputs,
|
||||
PrefillIndexerInputs,
|
||||
expand_index_page_table,
|
||||
)
|
||||
from sglang.srt.layers.attention.dsv4.dense_prefill_indexer import (
|
||||
DenseCandidateIndexer,
|
||||
score_tiles,
|
||||
)
|
||||
from sglang.srt.layers.attention.dsv4.indexer import (
|
||||
deep_gemm_fp4_paged_mqa_logits,
|
||||
@@ -134,16 +142,36 @@ def topk_transform_sparse(
|
||||
)
|
||||
|
||||
|
||||
# TODO(dark): support publish prefill/select prefill
|
||||
@dataclass
|
||||
class PrefillSparseBlockTable(SparseBlockTable):
|
||||
"""Published by a prefill chunk's candidate source: one row per query token.
|
||||
The inputs stay attached so the late-layer tail can rebuild the table for
|
||||
its rows (the DeepGEMM schedule is per row set, it cannot be sliced)."""
|
||||
|
||||
compress_lens: torch.Tensor # [rows] int32
|
||||
page_table: torch.Tensor # [rows, index pages] int32
|
||||
request_ids: torch.Tensor # [rows] int32
|
||||
q_dtype: torch.dtype
|
||||
page_size: int # index-K pool page size (slots)
|
||||
rows_per_request: List[int] # rows of each request, in row order
|
||||
|
||||
|
||||
# TODO(dark): support fusion of publish + topk of publish layer
|
||||
class DeepGemmCandidateIndexer:
|
||||
def __init__(self, topk_blocks: int, block_size: int):
|
||||
class DeepGemmCandidateIndexer(CandidateIndexer):
|
||||
def __init__(
|
||||
self,
|
||||
topk_blocks: int,
|
||||
block_size: int,
|
||||
prefill_dense: Optional[DenseCandidateIndexer] = None,
|
||||
):
|
||||
assert block_size == CANDIDATE_BLOCK_SIZE, block_size
|
||||
self.topk_blocks = topk_blocks
|
||||
self.block_size = block_size
|
||||
self.alt_stream = torch.cuda.Stream()
|
||||
self._row_ids: Optional[torch.Tensor] = None
|
||||
self._retired_row_ids: list = [] # captured graphs keep reading the buffers they saw
|
||||
# set for the forwards whose prefill rows the page table does not describe
|
||||
self._prefill_dense = prefill_dense
|
||||
|
||||
def _request_ids(
|
||||
self, request_ids: Optional[torch.Tensor], rows: int, device: torch.device
|
||||
@@ -253,3 +281,149 @@ class DeepGemmCandidateIndexer:
|
||||
logits = self._scores(table, inputs)
|
||||
# decode carries no raw_indices; the kernel writes slots only
|
||||
topk_transform_sparse(logits, table.valid_lens, table, page_indices)
|
||||
|
||||
def publish_prefill(
|
||||
self, inputs: PrefillIndexerInputs, out_positions: torch.Tensor
|
||||
) -> CandidateMetadata:
|
||||
"""The source layer's own top-k and the block table of the chunk from
|
||||
one pass over its dense scores, tile by tile: per tile the plain top-k
|
||||
into ``out_positions`` and one read for the block keys."""
|
||||
if self._prefill_dense is not None:
|
||||
return self._prefill_dense.publish_prefill(inputs, out_positions)
|
||||
rows = inputs.num_rows
|
||||
device = inputs.q_fp4.device
|
||||
nblocks, valid_lens = candidate_row_lens(inputs.compress_lens, self.topk_blocks)
|
||||
blocks = torch.empty(rows, self.topk_blocks, dtype=torch.int32, device=device)
|
||||
# the block keys read the score rows through 32-byte vectors
|
||||
for tile, logits in score_tiles(inputs, width_align=8):
|
||||
lens = inputs.compress_lens[tile]
|
||||
topk_transform_ragged_v2(
|
||||
logits,
|
||||
lens,
|
||||
out_offsets=inputs.request_starts[tile],
|
||||
out_indices=out_positions[tile],
|
||||
)
|
||||
# keys past a row's block count stay unset; the top-k reads a row
|
||||
# up to its block count only (v2 wants the stride a multiple of 4)
|
||||
width_blocks = (
|
||||
logits.shape[1] + CANDIDATE_BLOCK_SIZE - 1
|
||||
) // CANDIDATE_BLOCK_SIZE
|
||||
keys = logits.new_empty(logits.shape[0], (width_blocks + 3) // 4 * 4)
|
||||
amax8_varlen(logits, lens, out=keys)
|
||||
topk_transform_ragged_v2(
|
||||
keys,
|
||||
nblocks[tile],
|
||||
out_offsets=torch.zeros(
|
||||
logits.shape[0], dtype=torch.int32, device=device
|
||||
),
|
||||
out_indices=blocks[tile],
|
||||
)
|
||||
request_ids = torch.repeat_interleave(
|
||||
torch.arange(
|
||||
len(inputs.rows_per_request), dtype=torch.int32, device=device
|
||||
),
|
||||
torch.tensor(inputs.rows_per_request, dtype=torch.int64, device=device),
|
||||
output_size=rows,
|
||||
)
|
||||
# index-K pool pages of each row's request
|
||||
page_table = expand_index_page_table(
|
||||
inputs.kv_page_table,
|
||||
full_page_size=inputs.kv_page_size,
|
||||
compress_ratio=inputs.compress_ratio,
|
||||
index_page_size=inputs.page_size,
|
||||
).contiguous()
|
||||
return self._prefill_table(
|
||||
blocks,
|
||||
inputs.compress_lens,
|
||||
page_table,
|
||||
inputs.page_size,
|
||||
request_ids,
|
||||
inputs.rows_per_request,
|
||||
inputs.q_fp4.dtype,
|
||||
valid_lens,
|
||||
)
|
||||
|
||||
def _prefill_table(
|
||||
self,
|
||||
blocks,
|
||||
compress_lens,
|
||||
page_table,
|
||||
page_size,
|
||||
request_ids,
|
||||
rows_per_request,
|
||||
q_dtype,
|
||||
valid_lens,
|
||||
) -> PrefillSparseBlockTable:
|
||||
# in place: ascending, INT32_MAX padded, plus the blocks as pool slots / 8
|
||||
phys_blocks = sort_candidate_blocks(
|
||||
blocks, compress_lens, page_table, page_size
|
||||
)
|
||||
schedule = build_sparse_indexer_schedule(
|
||||
blocks, compress_lens, page_table, page_size, q_dtype, request_ids
|
||||
)
|
||||
ready = torch.cuda.Event()
|
||||
ready.record()
|
||||
return PrefillSparseBlockTable(
|
||||
blocks=blocks,
|
||||
schedule=schedule,
|
||||
phys_blocks=phys_blocks,
|
||||
valid_lens=valid_lens,
|
||||
ready=ready,
|
||||
compress_lens=compress_lens,
|
||||
page_table=page_table,
|
||||
request_ids=request_ids,
|
||||
q_dtype=q_dtype,
|
||||
page_size=page_size,
|
||||
rows_per_request=rows_per_request,
|
||||
)
|
||||
|
||||
def prefill_tail(
|
||||
self, published: CandidateMetadata, tail_lens: List[int]
|
||||
) -> CandidateMetadata:
|
||||
if self._prefill_dense is not None:
|
||||
return self._prefill_dense.prefill_tail(published, tail_lens)
|
||||
table = published
|
||||
assert isinstance(table, PrefillSparseBlockTable), "prefill block table missing"
|
||||
rows, start = [], 0
|
||||
for n, t in zip(table.rows_per_request, tail_lens):
|
||||
rows.append(torch.arange(start + n - t, start + n))
|
||||
start += n
|
||||
rows = torch.cat(rows).to(table.blocks.device)
|
||||
compress_lens = table.compress_lens[rows].contiguous()
|
||||
_, valid_lens = candidate_row_lens(compress_lens, self.topk_blocks)
|
||||
return self._prefill_table(
|
||||
table.blocks[rows].contiguous(),
|
||||
compress_lens,
|
||||
table.page_table[rows].contiguous(),
|
||||
table.page_size,
|
||||
table.request_ids[rows].contiguous(),
|
||||
tail_lens,
|
||||
table.q_dtype,
|
||||
valid_lens,
|
||||
)
|
||||
|
||||
def select_prefill(
|
||||
self,
|
||||
published: CandidateMetadata,
|
||||
inputs: PrefillIndexerInputs,
|
||||
out_positions: torch.Tensor,
|
||||
) -> None:
|
||||
if self._prefill_dense is not None:
|
||||
return self._prefill_dense.select_prefill(published, inputs, out_positions)
|
||||
table = published
|
||||
assert isinstance(table, PrefillSparseBlockTable), "prefill block table missing"
|
||||
rows, heads = inputs.q_sf.shape
|
||||
logits = sparse_logits(
|
||||
inputs.q_fp4.view(rows, 1, heads, 64),
|
||||
inputs.q_sf.view(rows, 1, heads),
|
||||
inputs.k_cache,
|
||||
inputs.weights.to(torch.bfloat16),
|
||||
table,
|
||||
)
|
||||
# request-relative compressed positions, -1 padded
|
||||
topk_transform_bf16_small(
|
||||
logits, table.valid_lens, table.blocks, out_positions, CANDIDATE_BLOCK_SIZE
|
||||
)
|
||||
out_positions.add_(
|
||||
torch.where(out_positions >= 0, inputs.request_starts[:, None], 0)
|
||||
)
|
||||
|
||||
@@ -1,9 +1,14 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Iterator, List, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.attention.dsv4.candidate_indexer import (
|
||||
CandidateIndexer,
|
||||
CandidateMetadata,
|
||||
PrefillCandidateBlocks,
|
||||
PrefillIndexerInputs,
|
||||
candidate_block_mask,
|
||||
mask_topk_scores,
|
||||
select_candidate_block_ids,
|
||||
@@ -58,18 +63,7 @@ def dense_prefill_topk(
|
||||
width = ceil_align(max((n for _, n in request_lengths), default=0), 4)
|
||||
if row == 0 or width == 0:
|
||||
return selected, published
|
||||
row_alignment = 128 // q[0].shape[1]
|
||||
rows_per_chunk = mqa_logits_rows_per_chunk(
|
||||
num_rows=ceil_align(row, row_alignment),
|
||||
row_bytes=mqa_logits_row_bytes(width),
|
||||
budget_bytes=_SCORE_BUDGET_BYTES,
|
||||
)
|
||||
if rows_per_chunk is None:
|
||||
rows_per_chunk = row
|
||||
else:
|
||||
rows_per_chunk = max(
|
||||
row_alignment, rows_per_chunk // row_alignment * row_alignment
|
||||
)
|
||||
rows_per_chunk = _rows_per_chunk(row, width, heads=q[0].shape[1])
|
||||
for offset in range(0, row, rows_per_chunk):
|
||||
rows = slice(offset, min(offset + rows_per_chunk, row))
|
||||
_select_tile(
|
||||
@@ -89,6 +83,141 @@ def dense_prefill_topk(
|
||||
return selected, published
|
||||
|
||||
|
||||
def _rows_per_chunk(rows: int, width: int, *, heads: int) -> int:
|
||||
"""Query rows per logits tile so one fp32 [rows, width] tile fits the
|
||||
budget; the row count stays a multiple of the kernel's row alignment."""
|
||||
row_alignment = 128 // heads
|
||||
rows_per_chunk = mqa_logits_rows_per_chunk(
|
||||
num_rows=ceil_align(rows, row_alignment),
|
||||
row_bytes=mqa_logits_row_bytes(width),
|
||||
budget_bytes=_SCORE_BUDGET_BYTES,
|
||||
)
|
||||
if rows_per_chunk is None:
|
||||
return rows
|
||||
return max(row_alignment, rows_per_chunk // row_alignment * row_alignment)
|
||||
|
||||
|
||||
def score_tiles(
|
||||
inputs: PrefillIndexerInputs, *, width_align: int = 4
|
||||
) -> Iterator[tuple[slice, torch.Tensor]]:
|
||||
"""The dense scores of ``inputs`` row tile by row tile under the budget:
|
||||
``(rows, logits)`` with fp32 ``logits[i, j]`` the score of query row
|
||||
``rows.start + i`` against ``kv[request_starts + j]``, garbage past the row's
|
||||
``compress_lens``; the width is the batch's largest context aligned to
|
||||
``width_align`` columns."""
|
||||
from deep_gemm import fp8_fp4_mqa_logits
|
||||
|
||||
rows = inputs.num_rows
|
||||
width = ceil_align(max(inputs.lens_per_request, default=0), width_align)
|
||||
if rows == 0 or width == 0:
|
||||
return
|
||||
rows_per_chunk = _rows_per_chunk(rows, width, heads=inputs.q_fp4.shape[1])
|
||||
for offset in range(0, rows, rows_per_chunk):
|
||||
tile = slice(offset, min(offset + rows_per_chunk, rows))
|
||||
starts = inputs.request_starts[tile]
|
||||
yield (
|
||||
tile,
|
||||
fp8_fp4_mqa_logits(
|
||||
(inputs.q_fp4[tile], inputs.q_sf[tile]),
|
||||
inputs.kv,
|
||||
inputs.weights[tile],
|
||||
starts,
|
||||
starts + inputs.compress_lens[tile],
|
||||
False,
|
||||
width,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _dense_topk(
|
||||
inputs: PrefillIndexerInputs,
|
||||
out_positions: torch.Tensor,
|
||||
*,
|
||||
topk_blocks: int,
|
||||
block_size: int,
|
||||
publish: bool,
|
||||
candidates: Optional[PrefillCandidateBlocks],
|
||||
) -> Optional[PrefillCandidateBlocks]:
|
||||
selected, published = dense_prefill_topk(
|
||||
q=(inputs.q_fp4, inputs.q_sf),
|
||||
kv=inputs.kv,
|
||||
weights=inputs.weights,
|
||||
starts=inputs.request_starts,
|
||||
lengths=inputs.compress_lens,
|
||||
request_lengths=list(zip(inputs.rows_per_request, inputs.lens_per_request)),
|
||||
topk=out_positions.shape[1],
|
||||
candidate_topk_blocks=topk_blocks,
|
||||
candidate_block_size=block_size,
|
||||
publish_candidates=publish,
|
||||
candidates=candidates,
|
||||
)
|
||||
out_positions.copy_(selected)
|
||||
return published
|
||||
|
||||
|
||||
def plain_prefill_topk(
|
||||
inputs: PrefillIndexerInputs, out_positions: torch.Tensor
|
||||
) -> None:
|
||||
"""The top-k of an index layer outside the candidate scheme, over its dense
|
||||
scores tile by tile."""
|
||||
_dense_topk(
|
||||
inputs,
|
||||
out_positions,
|
||||
topk_blocks=0,
|
||||
block_size=1,
|
||||
publish=False,
|
||||
candidates=None,
|
||||
)
|
||||
|
||||
|
||||
class DenseCandidateIndexer(CandidateIndexer):
|
||||
"""Candidates as block ids per request (``PrefillCandidateBlocks``): the
|
||||
source keeps its best blocks from its dense scores, a consumer masks its own
|
||||
dense scores to -inf outside them and runs the plain top-k, tile by tile.
|
||||
The prefill implementation for the CP layout; Hopper still runs the same
|
||||
selection inline in the backend."""
|
||||
|
||||
def __init__(self, topk_blocks: int, block_size: int):
|
||||
self.topk_blocks = topk_blocks
|
||||
self.block_size = block_size
|
||||
|
||||
def publish_prefill(
|
||||
self, inputs: PrefillIndexerInputs, out_positions: torch.Tensor
|
||||
) -> PrefillCandidateBlocks:
|
||||
published = _dense_topk(
|
||||
inputs,
|
||||
out_positions,
|
||||
topk_blocks=self.topk_blocks,
|
||||
block_size=self.block_size,
|
||||
publish=True,
|
||||
candidates=None,
|
||||
)
|
||||
assert published is not None
|
||||
return published
|
||||
|
||||
def select_prefill(
|
||||
self,
|
||||
published: CandidateMetadata,
|
||||
inputs: PrefillIndexerInputs,
|
||||
out_positions: torch.Tensor,
|
||||
) -> None:
|
||||
assert isinstance(published, PrefillCandidateBlocks), "candidate blocks missing"
|
||||
_dense_topk(
|
||||
inputs,
|
||||
out_positions,
|
||||
topk_blocks=self.topk_blocks,
|
||||
block_size=self.block_size,
|
||||
publish=False,
|
||||
candidates=published,
|
||||
)
|
||||
|
||||
def prefill_tail(
|
||||
self, published: CandidateMetadata, tail_lens: List[int]
|
||||
) -> PrefillCandidateBlocks:
|
||||
assert isinstance(published, PrefillCandidateBlocks), "candidate blocks missing"
|
||||
return published.tail(tail_lens)
|
||||
|
||||
|
||||
def _select_tile(
|
||||
*,
|
||||
q: tuple[torch.Tensor, torch.Tensor],
|
||||
|
||||
Reference in New Issue
Block a user