[Refactor] dsv4: Keep only the two-level V4.1 indexer in the prefill graph (#45783)
This commit is contained in:
@@ -126,30 +126,6 @@ def topk_transform_paged(
|
||||
)
|
||||
|
||||
|
||||
def topk_transform_paged_torch(
|
||||
scores: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
page_tables: torch.Tensor,
|
||||
out_page_indices: torch.Tensor,
|
||||
page_size: int,
|
||||
out_raw_indices: Optional[torch.Tensor] = None,
|
||||
) -> None:
|
||||
"""The torch ``topk_transform_paged``: top-``k`` (``k = out_page_indices.shape[1]``)
|
||||
of each row within its first ``seq_lens[b]`` columns, ascending, as pool slots
|
||||
through ``page_tables`` and as positions when given; ``-1`` padded."""
|
||||
topk = out_page_indices.shape[1]
|
||||
columns = torch.arange(scores.shape[1], device=seq_lens.device)
|
||||
lens_c = seq_lens.unsqueeze(-1)
|
||||
# Columns past a row's length hold garbage.
|
||||
s = scores.masked_fill(columns[None, :] >= lens_c, -torch.inf)
|
||||
idx = s.topk(topk, dim=-1, sorted=False).indices.sort(dim=-1).values
|
||||
reach = idx < lens_c
|
||||
slots = page_tables.gather(-1, idx // page_size) * page_size + (idx % page_size)
|
||||
out_page_indices.copy_(torch.where(reach, slots, -1).to(torch.int32))
|
||||
if out_raw_indices is not None:
|
||||
out_raw_indices.copy_(torch.where(reach, idx, -1).to(torch.int32))
|
||||
|
||||
|
||||
# metadata is (batch+1, 2) int32: row 0 = {cluster_threshold, num_cluster_items};
|
||||
# rows 1..N = {batch_id, seq_len} of items routed to the persistent cluster pool.
|
||||
_PLAN_METADATA_INTS_PER_BATCH = 2
|
||||
|
||||
@@ -1741,9 +1741,6 @@ class Envs:
|
||||
SGLANG_OPT_USE_ONLINE_COMPRESS = EnvBool(False)
|
||||
SGLANG_EXPERIMENTAL_ONLINE_C128_MTP = EnvBool(False)
|
||||
SGLANG_DSV4_COMPRESS_STATE_DTYPE = EnvStr("float32")
|
||||
# Run the DeepSeek-V4.1 ratio-1/2 prefill indexer on the torch path instead
|
||||
# of the DeepGEMM dense fp4 logits kernel (test oracle / fallback).
|
||||
SGLANG_DSV41_TORCH_PREFILL_INDEXER = EnvBool(False)
|
||||
# DeepSeek-V4.1 ratio-1/2 index layers: the DeepGEMM fp4 logits kernels store bf16 instead
|
||||
# of fp32 (the reference ranks bf16 scores); every consumer of those logits must take bf16.
|
||||
SGLANG_OPT_DSV41_BF16_INDEX_LOGITS = EnvBool(False)
|
||||
|
||||
@@ -83,7 +83,6 @@ from sglang.srt.layers.attention.dsv4.v41_indexer import (
|
||||
CapturedPrefillInputs,
|
||||
DecodeInputs,
|
||||
PrefillInputs,
|
||||
has_dense_fp4_indexer,
|
||||
is_sm100_or_newer,
|
||||
make_candidate_indexer,
|
||||
make_full_topk_indexer,
|
||||
@@ -1012,13 +1011,8 @@ def _pad_rows(t: torch.Tensor, *, num_rows: int) -> torch.Tensor:
|
||||
return torch.nn.functional.pad(t, (0, num_rows - t.shape[0]))
|
||||
|
||||
|
||||
# Rows per logits chunk of the ratio-1/2 indexer metadata under the prefill CUDA
|
||||
# graph; its width is _prefill_graph_context_width.
|
||||
_PREFILL_GRAPH_INDEXER_ROW_CHUNK = 2048
|
||||
|
||||
|
||||
# The candidate window (2,048 blocks of 8 tokens): inside it the captured full
|
||||
# top-k equals the two-level selection.
|
||||
# The replayed context limit when the prefill graph's metadata is built at a fixed
|
||||
# width (no two-level candidates); the candidate window, 2,048 blocks of 8 tokens.
|
||||
_PREFILL_GRAPH_DEFAULT_MAX_SEQ_LEN = 16 * 1024
|
||||
|
||||
|
||||
@@ -1085,30 +1079,11 @@ class DSV4Metadata:
|
||||
)
|
||||
self.sparse_prefill_cache = None
|
||||
|
||||
def refresh_for_breakable_cuda_graph_replay_(
|
||||
self,
|
||||
static_metadata: DSV4Metadata,
|
||||
*,
|
||||
two_level: bool,
|
||||
):
|
||||
def refresh_for_breakable_cuda_graph_replay_(self, static_metadata: DSV4Metadata):
|
||||
self.core_attn_metadata.refresh_for_breakable_cuda_graph_replay_(
|
||||
static_metadata.core_attn_metadata
|
||||
)
|
||||
maybe_copy_inplace(self.indexer_metadata, src=static_metadata.indexer_metadata)
|
||||
if not two_level:
|
||||
# The captured full top-k fallback reads the ratio-1/2 lengths and
|
||||
# page tables by address: refresh them in place.
|
||||
maybe_copy_inplace(
|
||||
self.c1_indexer_metadata, src=static_metadata.c1_indexer_metadata
|
||||
)
|
||||
maybe_copy_inplace(
|
||||
self.c2_indexer_metadata, src=static_metadata.c2_indexer_metadata
|
||||
)
|
||||
else:
|
||||
# Two-level candidates: only the eager full top-k breaks read these, so
|
||||
# they take the live objects; captured consumers read the bridge tensors.
|
||||
self.c1_indexer_metadata = static_metadata.c1_indexer_metadata
|
||||
self.c2_indexer_metadata = static_metadata.c2_indexer_metadata
|
||||
maybe_copy_inplace(
|
||||
self.low_ratio_req_indices, src=static_metadata.low_ratio_req_indices
|
||||
)
|
||||
@@ -1607,59 +1582,20 @@ class DeepseekV4AttnBackend(
|
||||
),
|
||||
)
|
||||
if use_prefill_cuda_graph and self.low_ratio_prefill_graph:
|
||||
low = core_attn_metadata.low_ratios
|
||||
metadata.c1_indexer_metadata = (
|
||||
self._low_ratio_prefill_indexer_metadata(core_attn_metadata, 1)
|
||||
if 1 in low
|
||||
else None
|
||||
)
|
||||
metadata.c2_indexer_metadata = (
|
||||
self._low_ratio_prefill_indexer_metadata(core_attn_metadata, 2)
|
||||
if 2 in low
|
||||
else None
|
||||
)
|
||||
metadata.low_ratio_req_indices = req_pool_indices_repeated.to(torch.int64)
|
||||
metadata.low_ratio_pos_i64 = core_attn_metadata.positions_casual.to(
|
||||
torch.int64
|
||||
)
|
||||
return metadata
|
||||
|
||||
def _low_ratio_prefill_indexer_metadata(
|
||||
self, core: DSV4AttnMetadata, compress_ratio: int
|
||||
) -> PagedIndexerMetadata:
|
||||
num_pages = core.page_table.shape[1]
|
||||
max_seq_len = self._prefill_graph_seq_limit()
|
||||
if max_seq_len is not None:
|
||||
num_pages = min(max_seq_len // self.page_size, num_pages)
|
||||
index_page_size = self.token_to_kv_pool.get_index_k_page_size(compress_ratio)
|
||||
page_table = expand_index_page_table(
|
||||
core.page_table[:, :num_pages],
|
||||
full_page_size=self.page_size,
|
||||
compress_ratio=compress_ratio,
|
||||
index_page_size=index_page_size,
|
||||
)
|
||||
# Unclamped: a token with no completed group scores nothing, as in eager.
|
||||
c_seq_lens = (core.seq_lens_casual // compress_ratio).to(torch.int32)
|
||||
row_chunk = _PREFILL_GRAPH_INDEXER_ROW_CHUNK
|
||||
return PagedIndexerMetadata(
|
||||
page_size=self.page_size,
|
||||
compressed_page_size=index_page_size,
|
||||
page_table=page_table,
|
||||
compressed_seq_lens=c_seq_lens,
|
||||
use_topk_v2=False,
|
||||
use_prefill_cuda_graph=True,
|
||||
compress_ratio=compress_ratio,
|
||||
row_chunk=row_chunk if row_chunk < c_seq_lens.shape[0] else 0,
|
||||
)
|
||||
|
||||
@property
|
||||
def low_ratio_prefill_graph(self) -> bool:
|
||||
"""Whether ratio-1/2 sources use captured projections and indexer metadata."""
|
||||
return (
|
||||
bool(self.low_ratios)
|
||||
and has_dense_fp4_indexer()
|
||||
and is_sm100_or_newer()
|
||||
and get_parallel().attn_cp_size == 1
|
||||
"""Whether ratio-1/2 sources run in the prefill graph with two-level candidates:
|
||||
the full top-k layers score at eager breaks, and the consumers score the
|
||||
published candidate blocks in the graph. Otherwise the sources are one eager
|
||||
break."""
|
||||
return bool(self.low_ratios) and isinstance(
|
||||
self.prefill_candidates, SparseTableBackend
|
||||
)
|
||||
|
||||
def can_run_prefill_cuda_graph(self, forward_batch: ForwardBatch) -> bool:
|
||||
@@ -3128,10 +3064,7 @@ class DeepseekV4AttnBackend(
|
||||
use_prefill_cuda_graph=True,
|
||||
)
|
||||
assert isinstance(capture_metadata, DSV4Metadata)
|
||||
capture_metadata.refresh_for_breakable_cuda_graph_replay_(
|
||||
static_metadata,
|
||||
two_level=self._prefill_graph_two_level(),
|
||||
)
|
||||
capture_metadata.refresh_for_breakable_cuda_graph_replay_(static_metadata)
|
||||
self.prefill_graph_replay_batch = forward_batch
|
||||
self.forward_metadata = capture_metadata
|
||||
self._activate_request_window()
|
||||
@@ -3156,10 +3089,7 @@ class DeepseekV4AttnBackend(
|
||||
if held is None:
|
||||
self._bcg_tail_metadata[rows] = fresh
|
||||
return fresh
|
||||
held.refresh_for_breakable_cuda_graph_replay_(
|
||||
fresh,
|
||||
two_level=self._prefill_graph_two_level(),
|
||||
)
|
||||
held.refresh_for_breakable_cuda_graph_replay_(fresh)
|
||||
held.core_attn_metadata.swa_out_cache_loc = (
|
||||
fresh.core_attn_metadata.swa_out_cache_loc
|
||||
)
|
||||
@@ -3302,11 +3232,10 @@ class DeepseekV4AttnBackend(
|
||||
and self._low_ratio_in_prefill_graph()
|
||||
):
|
||||
bufs = self._source_projection_buffers(x.shape[0], layer.compress_ratio)
|
||||
two_level = self._prefill_graph_two_level()
|
||||
captured_index = (
|
||||
run_indexer
|
||||
and layer.indexer is not None
|
||||
and (layer.indexer.uses_candidates or not two_level)
|
||||
and layer.indexer.uses_candidates
|
||||
)
|
||||
_bcg_low_ratio_source_projections(
|
||||
layer, x, q_lora, pos, bufs, run_compressor, captured_index
|
||||
@@ -3761,35 +3690,19 @@ class DeepseekV4AttnBackend(
|
||||
ratio = layer.compress_ratio
|
||||
indexer = layer.indexer
|
||||
core = self.forward_metadata.core_metadata
|
||||
metadata = (
|
||||
self.forward_metadata.c1_indexer_metadata
|
||||
if ratio == 1
|
||||
else self.forward_metadata.c2_indexer_metadata
|
||||
)
|
||||
assert metadata is not None, f"no prefill graph indexer metadata for {ratio = }"
|
||||
inputs = CapturedPrefillInputs(
|
||||
indexer=indexer,
|
||||
layer_id=layer.layer_id,
|
||||
q=projected_q,
|
||||
weights=projected_w,
|
||||
paged_metadata=metadata,
|
||||
index_page_size=self.token_to_kv_pool.get_index_k_page_size(ratio),
|
||||
out_page_indices=core.sparse_page_indices(ratio),
|
||||
out_raw_indices=core.sparse_raw_indices(ratio),
|
||||
)
|
||||
# At any width: inside the candidate window the published table is the
|
||||
# identity, and past it the full top-k would be wrong.
|
||||
if indexer.uses_candidates and self._prefill_graph_two_level():
|
||||
self.prefill_candidates.consume_prefill_captured(
|
||||
inputs, self.forward_metadata.candidate_metadata
|
||||
)
|
||||
return
|
||||
if indexer.uses_candidates or indexer.is_candidate_source:
|
||||
# Inside the window every block is a candidate: the full top-k applies.
|
||||
width = metadata.max_compressed_seq_len
|
||||
assert (
|
||||
width <= indexer.candidate_topk_blocks * indexer.candidate_block_size
|
||||
), f"prefill graph indexer width {width} exceeds the candidate window"
|
||||
self.full_topk_indexer.topk_prefill_captured(inputs)
|
||||
# At any width: the consumer scores only the blocks the source published.
|
||||
self.prefill_candidates.consume_prefill_captured(
|
||||
inputs, self.forward_metadata.candidate_metadata
|
||||
)
|
||||
|
||||
def _make_low_ratio_decode_indexer_inputs(self, layer, x, q_lora, req, pos, mode):
|
||||
ratio = layer.compress_ratio
|
||||
@@ -3888,25 +3801,15 @@ class DeepseekV4AttnBackend(
|
||||
return
|
||||
self.forward_metadata.candidate_metadata = published
|
||||
|
||||
def _prefill_graph_two_level(self) -> bool:
|
||||
"""Whether the full top-k layers score at eager breaks and the consumers score
|
||||
the published candidate blocks inside the prefill graph."""
|
||||
return isinstance(self.prefill_candidates, SparseTableBackend)
|
||||
|
||||
def _prefill_graph_replays_any_context(self) -> bool:
|
||||
"""Whether a prefill graph replay admits any context length: with two-level
|
||||
candidates only c4/c128 compress metadata would be captured at a width."""
|
||||
return (
|
||||
self.low_ratio_prefill_graph
|
||||
and self._prefill_graph_two_level()
|
||||
and not self.has_c4
|
||||
and not self.has_c128
|
||||
)
|
||||
return self.low_ratio_prefill_graph and not self.has_c4 and not self.has_c128
|
||||
|
||||
@property
|
||||
def prefill_cuda_graph_default_max_seq_len(self) -> Optional[int]:
|
||||
"""DeepSeek-V4.1's captured indexer width; prefill context parallelism builds
|
||||
its own metadata and needs none."""
|
||||
"""DeepSeek-V4.1's fixed metadata width without two-level candidates; prefill
|
||||
context parallelism builds its own metadata and needs none."""
|
||||
if (
|
||||
self.low_ratios != (1, 2)
|
||||
or get_parallel().attn_cp_size > 1
|
||||
|
||||
@@ -889,7 +889,8 @@ class DeepseekV4HipRadixBackend(
|
||||
forward_low_ratio_sources = DeepseekV4AttnBackend.forward_low_ratio_sources
|
||||
_forward_low_ratio_sources_cp = DeepseekV4AttnBackend._forward_low_ratio_sources_cp
|
||||
write_tail_source_kv = DeepseekV4AttnBackend.write_tail_source_kv
|
||||
low_ratio_prefill_graph = DeepseekV4AttnBackend.low_ratio_prefill_graph
|
||||
# No two-level candidates on HIP: the ratio-1/2 sources stay one eager break.
|
||||
low_ratio_prefill_graph = False
|
||||
# This attribute is used in python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py.
|
||||
# PrefillCudaGraphRunner looks this up via getattr; contexts over prefill.max_seq_len run eager.
|
||||
can_run_prefill_cuda_graph = DeepseekV4AttnBackend.can_run_prefill_cuda_graph
|
||||
@@ -3491,11 +3492,6 @@ class DeepseekV4HipRadixBackend(
|
||||
self, layer, x, q_lora, pos, forward_batch=forward_batch
|
||||
)
|
||||
elif forward_batch.forward_mode.is_extend():
|
||||
if envs.SGLANG_DSV41_TORCH_PREFILL_INDEXER.get():
|
||||
raise NotImplementedError(
|
||||
"SGLANG_DSV41_TORCH_PREFILL_INDEXER (the torch prefill indexer "
|
||||
"oracle) is not supported on HIP"
|
||||
)
|
||||
# HIP rejects V4.1 prefill CP, the only caller passing rows_per_request
|
||||
assert rows_per_request is None, "V4.1 prefill CP is not supported on HIP"
|
||||
assert (
|
||||
|
||||
@@ -185,9 +185,6 @@ class PagedIndexerMetadata:
|
||||
use_prefill_cuda_graph: bool = False
|
||||
# Indexer source compression ratio: 4 for c4, 1 or 2 for the dsv41 sources.
|
||||
compress_ratio: int = 4
|
||||
# Rows per logits chunk for the prefill CUDA graph low-ratio indexer; 0 plans
|
||||
# all rows at once.
|
||||
row_chunk: int = 0
|
||||
rows_per_request: int = 1
|
||||
# A list when the dynamic-budget forward is row-chunked: one schedule per chunk.
|
||||
deep_gemm_metadata: Any = field(init=False, repr=False)
|
||||
@@ -254,18 +251,7 @@ class PagedIndexerMetadata:
|
||||
self.rows_per_chunk,
|
||||
self.mqa_logits_budget_bytes,
|
||||
)
|
||||
if self.row_chunk > 0:
|
||||
self.deep_gemm_metadata = torch.stack(
|
||||
[
|
||||
get_paged_mqa_logits_metadata(
|
||||
compressed_seq_lens[_s : _s + self.row_chunk],
|
||||
self.compressed_page_size,
|
||||
deep_gemm.get_num_sms(),
|
||||
)
|
||||
for _s in range(0, compressed_seq_lens.shape[0], self.row_chunk)
|
||||
]
|
||||
)
|
||||
elif self.rows_per_chunk is not None:
|
||||
if self.rows_per_chunk is not None:
|
||||
# Chunk metadata is shared by all indexer layers in this forward.
|
||||
self.deep_gemm_metadata = [
|
||||
get_paged_mqa_logits_metadata(
|
||||
@@ -339,9 +325,7 @@ class PagedIndexerMetadata:
|
||||
|
||||
def row_chunks(self):
|
||||
num_rows = self.compressed_seq_lens.shape[0]
|
||||
if self.row_chunk > 0:
|
||||
rows_per_chunk = self.row_chunk
|
||||
elif isinstance(self.deep_gemm_metadata, list):
|
||||
if isinstance(self.deep_gemm_metadata, list):
|
||||
assert self.rows_per_chunk is not None, (
|
||||
"chunked DeepGEMM metadata requires rows_per_chunk"
|
||||
)
|
||||
@@ -385,7 +369,6 @@ class PagedIndexerMetadata:
|
||||
"page_size",
|
||||
"compressed_page_size",
|
||||
"compress_ratio",
|
||||
"row_chunk",
|
||||
"force_deep_gemm_metadata",
|
||||
"use_prefill_cuda_graph",
|
||||
"use_topk_v2",
|
||||
|
||||
@@ -59,18 +59,7 @@ def has_dense_fp4_indexer() -> bool:
|
||||
|
||||
@functools.cache
|
||||
def _use_deep_gemm_prefill() -> bool:
|
||||
# A CP rank scores rank-local rows, which only DeepGEMM serves, so CP ignores the env.
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
return (
|
||||
is_sm100_or_newer()
|
||||
and has_dense_fp4_indexer()
|
||||
and (
|
||||
get_parallel().attn_cp_size > 1
|
||||
or not envs.SGLANG_DSV41_TORCH_PREFILL_INDEXER.get()
|
||||
)
|
||||
)
|
||||
return is_sm100_or_newer() and has_dense_fp4_indexer()
|
||||
|
||||
|
||||
def make_full_topk_indexer(
|
||||
@@ -127,8 +116,8 @@ def make_candidate_indexer(
|
||||
candidate_topk_blocks=candidate_topk_blocks,
|
||||
candidate_block_size=candidate_block_size,
|
||||
)
|
||||
# A CP rank's rows are an interleaved subset of the batch; the torch prefill
|
||||
# env keeps the sparse table on decode only.
|
||||
# A CP rank's rows are an interleaved subset of the batch, and without the dense
|
||||
# fp4 indexer prefill scores on torch: both keep the sparse table on decode only.
|
||||
if not use_deep_gemm_prefill or get_parallel().attn_cp_size > 1:
|
||||
return dense_blocks, sparse_table
|
||||
return sparse_table, sparse_table
|
||||
|
||||
@@ -14,7 +14,6 @@ from sglang.kernels.ops.attention.deep_select import (
|
||||
from sglang.kernels.ops.attention.dsv4.index_logits import (
|
||||
deep_gemm_fp4_paged_mqa_logits,
|
||||
)
|
||||
from sglang.kernels.ops.attention.dsv4.topk import topk_transform_paged_torch
|
||||
from sglang.srt.layers.attention.dsv4.indexer import topk_transform_paged_from_metadata
|
||||
|
||||
from .scoring import (
|
||||
@@ -22,14 +21,11 @@ from .scoring import (
|
||||
dense_prefill_topk,
|
||||
get_deep_gemm_decode_data,
|
||||
get_deep_gemm_prefill_data,
|
||||
get_index_k_cache,
|
||||
prefill_requests,
|
||||
quantize_index_q,
|
||||
select_decode,
|
||||
write_prefill,
|
||||
)
|
||||
from .types import (
|
||||
CapturedPrefillInputs,
|
||||
DecodeInputs,
|
||||
PrefillInputs,
|
||||
)
|
||||
@@ -61,9 +57,6 @@ class FullTopKIndexer:
|
||||
else:
|
||||
self._torch_prefill(inputs)
|
||||
|
||||
def topk_prefill_captured(self, inputs: CapturedPrefillInputs) -> None:
|
||||
self._deep_gemm_prefill_captured(inputs)
|
||||
|
||||
def topk_decode(self, inputs: DecodeInputs) -> None:
|
||||
if self.use_deep_gemm_decode:
|
||||
self._deep_gemm_decode(inputs)
|
||||
@@ -156,42 +149,3 @@ class FullTopKIndexer:
|
||||
)
|
||||
return
|
||||
select_decode(inputs, d, inputs.indexer.index_topk)
|
||||
|
||||
def _deep_gemm_prefill_captured(self, inputs: CapturedPrefillInputs) -> None:
|
||||
indexer = inputs.indexer
|
||||
metadata = inputs.paged_metadata
|
||||
assert indexer.n_local_heads == indexer.n_heads
|
||||
q_fp4, q_sf = quantize_index_q(inputs.q)
|
||||
num_tokens, num_heads = q_fp4.shape[0], q_fp4.shape[1]
|
||||
q_fp4 = q_fp4.view(num_tokens, 1, num_heads, 64)
|
||||
q_sf = q_sf.view(num_tokens, 1, num_heads)
|
||||
weights = inputs.weights.float()
|
||||
page_size = metadata.compressed_page_size
|
||||
k_cache = get_index_k_cache(
|
||||
token_to_kv_pool=self.token_to_kv_pool,
|
||||
layer_id=inputs.layer_id,
|
||||
page_size=page_size,
|
||||
)
|
||||
|
||||
width = metadata.max_compressed_seq_len
|
||||
lens = metadata.compressed_seq_lens
|
||||
page_table = metadata.page_table
|
||||
topk = min(indexer.index_topk, width)
|
||||
for rows, plan in metadata.row_chunks():
|
||||
logits = deep_gemm_fp4_paged_mqa_logits(
|
||||
(q_fp4[rows], q_sf[rows]),
|
||||
k_cache,
|
||||
weights[rows],
|
||||
lens[rows],
|
||||
page_table[rows],
|
||||
plan,
|
||||
width,
|
||||
)
|
||||
topk_transform_paged_torch(
|
||||
logits,
|
||||
lens[rows],
|
||||
page_table[rows],
|
||||
inputs.out_page_indices[rows, :topk],
|
||||
page_size,
|
||||
inputs.out_raw_indices[rows, :topk],
|
||||
)
|
||||
|
||||
@@ -200,7 +200,7 @@ class SparseTableBackend:
|
||||
k_cache = get_index_k_cache(
|
||||
token_to_kv_pool=self.token_to_kv_pool,
|
||||
layer_id=inputs.layer_id,
|
||||
page_size=inputs.paged_metadata.compressed_page_size,
|
||||
page_size=inputs.index_page_size,
|
||||
)
|
||||
logits = _prefill_table_logits(published, q_fp4, q_sf, inputs.weights, k_cache)
|
||||
topk = min(indexer.index_topk, published.blocks.shape[1] * CANDIDATE_BLOCK_SIZE)
|
||||
|
||||
@@ -104,7 +104,7 @@ class CapturedPrefillInputs(msgspec.Struct, frozen=True, kw_only=True):
|
||||
layer_id: int
|
||||
q: torch.Tensor # [rows, heads, 128] bf16, roped
|
||||
weights: torch.Tensor # [rows, heads] bf16 head weights
|
||||
paged_metadata: PagedIndexerMetadata
|
||||
index_page_size: int # index-K pool page size at this ratio
|
||||
# The selection as pool slots and as request-local compressed positions.
|
||||
out_page_indices: torch.Tensor
|
||||
out_raw_indices: torch.Tensor
|
||||
|
||||
Reference in New Issue
Block a user