[Perf] dsv4: Gather V4.1 index-K slots once per forward and cache the prefill index plan (#45577)

This commit is contained in:
Liangsheng Yin
2026-10-21 14:00:35 -07:00
committed by GitHub
parent 620718c8ea
commit bc412213c1
7 changed files with 140 additions and 33 deletions
@@ -88,6 +88,7 @@ from sglang.srt.layers.attention.dsv4.v41_indexer import (
make_candidate_indexer,
make_full_topk_indexer,
)
from sglang.srt.layers.attention.dsv4.v41_indexer.scoring import prefill_k_slots
from sglang.srt.layers.attention.dsv4.v41_indexer.sparse_table import (
SparseTableBackend,
)
@@ -1042,6 +1043,12 @@ class DSV4Metadata:
# Shared by all low-ratio source layers; graph replay refreshes them live.
low_ratio_req_indices: Optional[torch.Tensor] = None
low_ratio_pos_i64: Optional[torch.Tensor] = None
# Eager extend: each low ratio's index-K slots (scoring.prefill_k_slots),
# shared with the late-layer tail metadata.
low_ratio_k_slots: Optional[Dict[int, torch.Tensor]] = None
# Eager extend: the indexer's batch-only plan per low ratio, filled by its first
# layer (scoring.prefill_index_plan); one per metadata, as the rows differ.
low_ratio_index_plans: Optional[Dict[int, object]] = None
# Per-step scratch for TP-padded query heads, zeroed by the first user.
# Later layers overwrite real heads and preserve the zero padding.
@@ -2717,9 +2724,44 @@ class DeepseekV4AttnBackend(
and forward_batch.forward_mode.is_extend_without_speculative()
else None
)
if (
self.low_ratios
and forward_batch.forward_mode.is_extend_without_speculative()
):
self._snapshot_low_ratio_k_slots(forward_batch)
self._activate_request_window()
def _snapshot_low_ratio_k_slots(self, forward_batch: ForwardBatch) -> None:
# Gathered once with the metadata instead of in every indexer layer, so no
# indexer layer reads req_to_token.
seq_lens_cpu = _as_int_list(forward_batch.seq_lens_cpu)
k_slots = {
ratio: prefill_k_slots(
req_to_token=self.req_to_token,
req_pool_indices=forward_batch.req_pool_indices,
seq_lens_cpu=seq_lens_cpu,
ratio=ratio,
)
for ratio in self.low_ratios
}
metadata = self.forward_metadata
metadata.low_ratio_k_slots = k_slots
metadata.low_ratio_index_plans = {}
if metadata.low_ratio_req_indices is None:
# Every source layer would rebuild these rows; the tail and the
# prefill graph publish theirs at metadata init the same way.
num_tokens = sum(forward_batch.extend_seq_lens_cpu)
metadata.low_ratio_req_indices = token_req_indices(
forward_batch, num_tokens=num_tokens
)
metadata.low_ratio_pos_i64 = forward_batch.positions[:num_tokens].to(
torch.int64
)
if self.tail_forward_metadata is not None:
self.tail_forward_metadata.low_ratio_k_slots = k_slots
self.tail_forward_metadata.low_ratio_index_plans = {}
def _activate_request_window(self) -> None:
if self.token_to_kv_pool.request_window is not None:
self.token_to_kv_pool.request_window.activate(
@@ -3782,6 +3824,18 @@ class DeepseekV4AttnBackend(
else:
rows_per_request = _as_int_list(forward_batch.extend_seq_lens_cpu)
rows_per_request_device = forward_batch.extend_seq_lens
seq_lens_cpu = _as_int_list(forward_batch.seq_lens_cpu)
k_slots = self.forward_metadata.low_ratio_k_slots
if k_slots is not None:
k_slots = k_slots[layer.compress_ratio]
else:
# Graph-captured metadata carries no snapshot.
k_slots = prefill_k_slots(
req_to_token=self.req_to_token,
req_pool_indices=forward_batch.req_pool_indices,
seq_lens_cpu=seq_lens_cpu,
ratio=layer.compress_ratio,
)
return PrefillInputs(
indexer=layer.indexer,
layer_id=layer.layer_id,
@@ -3792,8 +3846,10 @@ class DeepseekV4AttnBackend(
positions=pos,
req_rows=req,
req_pool_indices=forward_batch.req_pool_indices,
k_slots=k_slots,
index_plans=self.forward_metadata.low_ratio_index_plans,
kv_page_table=core.page_table,
seq_lens_cpu=_as_int_list(forward_batch.seq_lens_cpu),
seq_lens_cpu=seq_lens_cpu,
rows_per_request=rows_per_request,
rows_per_request_device=rows_per_request_device,
out_raw_indices=core.sparse_raw_indices(layer.compress_ratio),
@@ -123,7 +123,6 @@ def make_candidate_indexer(
sparse_table = SparseTableBackend(
token_to_kv_pool=token_to_kv_pool,
req_to_token=req_to_token,
page_size=page_size,
candidate_topk_blocks=candidate_topk_blocks,
candidate_block_size=candidate_block_size,
@@ -146,7 +146,7 @@ class DenseBlocksBackend:
def _deep_gemm_publish_prefill(self, inputs: PrefillInputs):
inputs.reset_outputs()
data = get_deep_gemm_prefill_data(inputs, self.req_to_token)
data = get_deep_gemm_prefill_data(inputs)
if data is None:
return None
selected, blocks = _publish_prefill_blocks(
@@ -167,7 +167,7 @@ class DenseBlocksBackend:
published: Optional[BlockIds],
) -> None:
inputs.reset_outputs()
data = get_deep_gemm_prefill_data(inputs, self.req_to_token)
data = get_deep_gemm_prefill_data(inputs)
if data is None:
return
assert published is not None
@@ -189,7 +189,6 @@ class DenseBlocksBackend:
for request, chunks in prefill_requests(
inputs=inputs,
token_to_kv_pool=self.token_to_kv_pool,
req_to_token=self.req_to_token,
):
for chunk in chunks:
picked.append(
@@ -225,7 +224,6 @@ class DenseBlocksBackend:
for request, chunks in prefill_requests(
inputs=inputs,
token_to_kv_pool=self.token_to_kv_pool,
req_to_token=self.req_to_token,
):
for chunk in chunks:
idx = topk_among_blocks(
@@ -72,7 +72,7 @@ class FullTopKIndexer:
def _deep_gemm_prefill(self, inputs: PrefillInputs) -> None:
inputs.reset_outputs()
data = get_deep_gemm_prefill_data(inputs, self.req_to_token)
data = get_deep_gemm_prefill_data(inputs)
if data is None:
return
kv = self.token_to_kv_pool.get_low_ratio_index_k_fp4(
@@ -85,7 +85,6 @@ class FullTopKIndexer:
requests = prefill_requests(
inputs=inputs,
token_to_kv_pool=self.token_to_kv_pool,
req_to_token=self.req_to_token,
)
for request, chunks in requests:
for chunk in chunks:
@@ -173,9 +173,39 @@ def get_deep_gemm_decode_data(
return DeepGEMMDecodeData(q_fp4, q_sf, weights, k_cache)
def get_deep_gemm_prefill_data(
inputs: PrefillInputs, req_to_token: torch.Tensor
) -> Optional[DeepGEMMPrefillData]:
def prefill_k_slots(
*,
req_to_token: torch.Tensor,
req_pool_indices: torch.Tensor,
seq_lens_cpu: List[int],
ratio: int,
) -> torch.Tensor:
"""[columns] int64 index-K pool slot of each flattened-K column: request by
request, the compressed positions visible at its newest token."""
device = req_pool_indices.device
lens_per_request = [s // ratio for s in seq_lens_cpu]
columns = sum(lens_per_request)
starts = list(itertools.accumulate(lens_per_request, initial=0))[:-1]
lens = async_h2d(lens_per_request, dtype=torch.int64, device=device)
starts_dev = async_h2d(starts, dtype=torch.int64, device=device)
request = torch.repeat_interleave(
torch.arange(len(lens_per_request), device=device), lens, output_size=columns
)
position = torch.arange(columns, device=device) - starts_dev[request]
pool_rows = req_pool_indices.to(torch.int64)[request]
return req_to_token[pool_rows, position * ratio].to(torch.int64) // ratio
class PrefillIndexPlan(msgspec.Struct, frozen=True):
"""The batch-only part of DeepGEMMPrefillData, the same in every layer of one
compress ratio."""
request_starts: torch.Tensor # [rows] int32, first column of the row's request
lens_per_request: List[int] # compressed positions at each request's newest token
compress_lens: torch.Tensor # [rows] int32, compressed positions the row sees
def prefill_index_plan(inputs: PrefillInputs) -> Optional[PrefillIndexPlan]:
"""None when the chunk has no row or no visible compressed position."""
ratio = inputs.compress_ratio
pos = inputs.positions
@@ -192,30 +222,43 @@ def get_deep_gemm_prefill_data(
if columns == 0 or num_tokens == 0:
return None
starts = list(itertools.accumulate(lens_per_request, initial=0))[:-1]
lens = async_h2d(lens_per_request, dtype=torch.int64, device=device)
starts_dev = async_h2d(starts, dtype=torch.int64, device=device)
request = torch.repeat_interleave(
torch.arange(len(lens_per_request), device=device), lens, output_size=columns
starts_dev = async_h2d(starts, dtype=torch.int32, device=device)
return PrefillIndexPlan(
request_starts=torch.repeat_interleave(
starts_dev,
inputs.rows_per_request_device.to(torch.int64),
output_size=num_tokens,
),
lens_per_request=lens_per_request,
compress_lens=((pos + 1) // ratio).to(torch.int32),
)
position = torch.arange(columns, device=device) - starts_dev[request]
pool_rows = inputs.req_pool_indices.to(torch.int64)[request]
def get_deep_gemm_prefill_data(inputs: PrefillInputs) -> Optional[DeepGEMMPrefillData]:
"""None when the chunk has no row or no visible compressed position."""
ratio = inputs.compress_ratio
plans = inputs.index_plans
if plans is not None and ratio in plans:
plan = plans[ratio]
else:
plan = prefill_index_plan(inputs)
if plans is not None:
plans[ratio] = plan
if plan is None:
return None
q_fp4, q_sf, weights = _index_q_and_weights(
indexer=inputs.indexer,
x=inputs.x,
q_lora=inputs.q_lora,
freqs_cis=inputs.freqs_cis,
positions=pos,
positions=inputs.positions,
)
return DeepGEMMPrefillData(
k_slots=req_to_token[pool_rows, position * ratio].to(torch.int64) // ratio,
request_starts=torch.repeat_interleave(
starts_dev.to(torch.int32),
inputs.rows_per_request_device.to(torch.int64),
output_size=num_tokens,
),
lens_per_request=lens_per_request,
k_slots=inputs.k_slots,
request_starts=plan.request_starts,
lens_per_request=plan.lens_per_request,
rows_per_request=inputs.rows_per_request,
compress_lens=((pos + 1) // ratio).to(torch.int32),
compress_lens=plan.compress_lens,
q_fp4=q_fp4,
q_sf=q_sf,
weights=weights,
@@ -312,7 +355,6 @@ def prefill_requests(
*,
inputs: PrefillInputs,
token_to_kv_pool: DeepSeekV4TokenToKVPool,
req_to_token: torch.Tensor,
) -> Iterator[tuple[RequestScores, Iterator[ChunkScores]]]:
"""Requests with nothing visible yet are skipped."""
pool = token_to_kv_pool
@@ -320,6 +362,14 @@ def prefill_requests(
indexer = inputs.indexer
req, pos = inputs.req_rows, inputs.positions
assert req is not None, "the torch indexer needs the batch's rows"
assert inputs.seq_lens_cpu is not None
# First flattened-K column of each request row, as inputs.k_slots lays them out.
k_starts = dict(
zip(
inputs.req_pool_indices.tolist(),
itertools.accumulate((s // ratio for s in inputs.seq_lens_cpu), initial=0),
)
)
inputs.reset_outputs()
q = indexer.queries(inputs.q_lora, inputs.freqs_cis[pos])
weights = indexer.head_weights(inputs.x)
@@ -333,7 +383,7 @@ def prefill_requests(
if lc == 0:
continue
j = torch.arange(lc, device=pos.device)
slots = req_to_token[r, j * ratio].to(torch.int64) // ratio
slots = inputs.k_slots[k_starts[r] : k_starts[r] + lc]
# Dequantize only this request's visible K rows; the table is pool-sized.
index_k = pool.get_low_ratio_index_k_dequant(inputs.layer_id, slots)
request = RequestScores(
@@ -120,14 +120,12 @@ class SparseTableBackend:
self,
*,
token_to_kv_pool: DeepSeekV4TokenToKVPool,
req_to_token: torch.Tensor,
page_size: int,
candidate_topk_blocks: int,
candidate_block_size: int,
) -> None:
assert candidate_block_size == CANDIDATE_BLOCK_SIZE
self.token_to_kv_pool = token_to_kv_pool
self.req_to_token = req_to_token
self.page_size = page_size
self.topk_blocks = candidate_topk_blocks
self.alt_stream = torch.cuda.Stream()
@@ -139,7 +137,7 @@ class SparseTableBackend:
def publish_prefill(self, inputs: PrefillInputs):
inputs.reset_outputs()
data = get_deep_gemm_prefill_data(inputs, self.req_to_token)
data = get_deep_gemm_prefill_data(inputs)
if data is None:
return None
index_page_size = self.token_to_kv_pool.get_index_k_page_size(
@@ -169,7 +167,7 @@ class SparseTableBackend:
published: Optional[_SparsePrefillTable],
) -> None:
inputs.reset_outputs()
data = get_deep_gemm_prefill_data(inputs, self.req_to_token)
data = get_deep_gemm_prefill_data(inputs)
if data is None:
return
assert published is not None
@@ -2,7 +2,7 @@
from __future__ import annotations
from typing import TYPE_CHECKING, List, Optional, Protocol, TypeVar
from typing import TYPE_CHECKING, Dict, List, Optional, Protocol, TypeVar
import msgspec
import torch
@@ -47,6 +47,13 @@ class PrefillInputs(msgspec.Struct, frozen=True, kw_only=True):
# rank's rows are an interleaved subset of the batch's.
req_rows: Optional[torch.Tensor]
req_pool_indices: torch.Tensor # [requests] req_to_token row of each request
# [columns] int64 index-K pool slot of each flattened-K column, from
# scoring.prefill_k_slots; gathered with the metadata, never per layer.
k_slots: torch.Tensor
# ratio -> scoring.PrefillIndexPlan, filled by the first layer of each ratio and
# reused by the rest of the forward; None when the metadata outlives a forward
# (captured graphs), so every layer builds its own.
index_plans: Optional[Dict[int, object]]
# [rows, pages] int32 at the KV page size; the index-K pool pages it with
# expand_index_page_table.
kv_page_table: torch.Tensor