[Perf] dsv4: Gather V4.1 index-K slots once per prefill graph replay (#45741)

This commit is contained in:
Liangsheng Yin
2026-10-21 23:34:17 -07:00
committed by GitHub
parent a56024c267
commit 82f7249ce6
2 changed files with 28 additions and 19 deletions
@@ -1043,11 +1043,11 @@ class DSV4Metadata:
# Shared by all low-ratio source layers; graph replay refreshes them live.
low_ratio_req_indices: Optional[torch.Tensor] = None
low_ratio_pos_i64: Optional[torch.Tensor] = None
# Eager extend: each low ratio's index-K slots (scoring.prefill_k_slots),
# shared with the late-layer tail metadata.
# Each low ratio's index-K slots (scoring.prefill_k_slots), gathered once by the
# extend's metadata builder; the late-layer tail's metadata reads the same dict.
low_ratio_k_slots: Optional[Dict[int, torch.Tensor]] = None
# Eager extend: the indexer's batch-only plan per low ratio, filled by its first
# layer (scoring.prefill_index_plan); one per metadata, as the rows differ.
# The indexer's batch-only plan per low ratio, filled by its first layer
# (scoring.prefill_index_plan); one per metadata, as the rows differ.
low_ratio_index_plans: Optional[Dict[int, object]] = None
# Per-step scratch for TP-padded query heads, zeroed by the first user.
@@ -1115,6 +1115,8 @@ class DSV4Metadata:
maybe_copy_inplace(
self.low_ratio_pos_i64, src=static_metadata.low_ratio_pos_i64
)
self.low_ratio_k_slots = static_metadata.low_ratio_k_slots
self.low_ratio_index_plans = static_metadata.low_ratio_index_plans
maybe_copy_inplace(
self.c4_compress_metadata, src=static_metadata.c4_compress_metadata
)
@@ -1935,6 +1937,11 @@ class DeepseekV4AttnBackend(
req_global=metadata.low_ratio_req_indices,
pos_global=metadata.low_ratio_pos_i64,
)
# Same requests as the extend in use, so the same index-K slots; other rows,
# so its own plans.
if self.low_ratios and self.forward_metadata.low_ratio_k_slots is not None:
metadata.low_ratio_k_slots = self.forward_metadata.low_ratio_k_slots
metadata.low_ratio_index_plans = {}
return metadata
def _late_layer_tail_cp_layout(
@@ -2724,17 +2731,19 @@ class DeepseekV4AttnBackend(
and forward_batch.forward_mode.is_extend_without_speculative()
else None
)
if (
self.low_ratios
and forward_batch.forward_mode.is_extend_without_speculative()
):
self._snapshot_low_ratio_k_slots(forward_batch)
self._activate_request_window()
def _snapshot_low_ratio_k_slots(self, forward_batch: ForwardBatch) -> None:
def _snapshot_low_ratio_k_slots(
self, forward_batch: ForwardBatch, metadata: DSV4Metadata, *, publish_rows: bool
) -> None:
# Gathered once with the metadata instead of in every indexer layer, so no
# indexer layer reads req_to_token.
if not (
self.low_ratios
and forward_batch.forward_mode.is_extend_without_speculative()
):
return
seq_lens_cpu = _as_int_list(forward_batch.seq_lens_cpu)
k_slots = {
ratio: prefill_k_slots(
@@ -2745,10 +2754,9 @@ class DeepseekV4AttnBackend(
)
for ratio in self.low_ratios
}
metadata = self.forward_metadata
metadata.low_ratio_k_slots = k_slots
metadata.low_ratio_index_plans = {}
if metadata.low_ratio_req_indices is None:
if publish_rows:
# Every source layer would rebuild these rows; the tail and the
# prefill graph publish theirs at metadata init the same way.
num_tokens = sum(forward_batch.extend_seq_lens_cpu)
@@ -2758,9 +2766,6 @@ class DeepseekV4AttnBackend(
metadata.low_ratio_pos_i64 = forward_batch.positions[:num_tokens].to(
torch.int64
)
if self.tail_forward_metadata is not None:
self.tail_forward_metadata.low_ratio_k_slots = k_slots
self.tail_forward_metadata.low_ratio_index_plans = {}
def _activate_request_window(self) -> None:
if self.token_to_kv_pool.request_window is not None:
@@ -3009,6 +3014,9 @@ class DeepseekV4AttnBackend(
use_prefill_cuda_graph=use_prefill_cuda_graph,
forward_batch=forward_batch,
)
self._snapshot_low_ratio_k_slots(
forward_batch, metadata, publish_rows=not use_prefill_cuda_graph
)
else:
raise NotImplementedError(f"unsupported mode {forward_batch.forward_mode=}")
@@ -3834,7 +3842,7 @@ class DeepseekV4AttnBackend(
if k_slots is not None:
k_slots = k_slots[layer.compress_ratio]
else:
# Graph-captured metadata carries no snapshot.
# Only a non-speculative extend's metadata carries a snapshot.
k_slots = prefill_k_slots(
req_to_token=self.req_to_token,
req_pool_indices=forward_batch.req_pool_indices,
@@ -48,11 +48,12 @@ class PrefillInputs(msgspec.Struct, frozen=True, kw_only=True):
req_rows: Optional[torch.Tensor]
req_pool_indices: torch.Tensor # [requests] req_to_token row of each request
# [columns] int64 index-K pool slot of each flattened-K column, from
# scoring.prefill_k_slots; gathered with the metadata, never per layer.
# scoring.prefill_k_slots; gathered once with a non-speculative extend's
# metadata, per layer otherwise.
k_slots: torch.Tensor
# ratio -> scoring.PrefillIndexPlan, filled by the first layer of each ratio and
# reused by the rest of the forward; None when the metadata outlives a forward
# (captured graphs), so every layer builds its own.
# reused by the rest of the forward; None when the metadata has no snapshot, so
# every layer builds its own.
index_plans: Optional[Dict[int, object]]
# [rows, pages] int32 at the KV page size; the index-K pool pages it with
# expand_index_page_table.