[Perf] dsv4: Gather V4.1 index-K slots once per forward and cache the prefill index plan (#45577)
This commit is contained in:
@@ -88,6 +88,7 @@ from sglang.srt.layers.attention.dsv4.v41_indexer import (
|
||||
make_candidate_indexer,
|
||||
make_full_topk_indexer,
|
||||
)
|
||||
from sglang.srt.layers.attention.dsv4.v41_indexer.scoring import prefill_k_slots
|
||||
from sglang.srt.layers.attention.dsv4.v41_indexer.sparse_table import (
|
||||
SparseTableBackend,
|
||||
)
|
||||
@@ -1042,6 +1043,12 @@ class DSV4Metadata:
|
||||
# Shared by all low-ratio source layers; graph replay refreshes them live.
|
||||
low_ratio_req_indices: Optional[torch.Tensor] = None
|
||||
low_ratio_pos_i64: Optional[torch.Tensor] = None
|
||||
# Eager extend: each low ratio's index-K slots (scoring.prefill_k_slots),
|
||||
# shared with the late-layer tail metadata.
|
||||
low_ratio_k_slots: Optional[Dict[int, torch.Tensor]] = None
|
||||
# Eager extend: the indexer's batch-only plan per low ratio, filled by its first
|
||||
# layer (scoring.prefill_index_plan); one per metadata, as the rows differ.
|
||||
low_ratio_index_plans: Optional[Dict[int, object]] = None
|
||||
|
||||
# Per-step scratch for TP-padded query heads, zeroed by the first user.
|
||||
# Later layers overwrite real heads and preserve the zero padding.
|
||||
@@ -2717,9 +2724,44 @@ class DeepseekV4AttnBackend(
|
||||
and forward_batch.forward_mode.is_extend_without_speculative()
|
||||
else None
|
||||
)
|
||||
if (
|
||||
self.low_ratios
|
||||
and forward_batch.forward_mode.is_extend_without_speculative()
|
||||
):
|
||||
self._snapshot_low_ratio_k_slots(forward_batch)
|
||||
|
||||
self._activate_request_window()
|
||||
|
||||
def _snapshot_low_ratio_k_slots(self, forward_batch: ForwardBatch) -> None:
|
||||
# Gathered once with the metadata instead of in every indexer layer, so no
|
||||
# indexer layer reads req_to_token.
|
||||
seq_lens_cpu = _as_int_list(forward_batch.seq_lens_cpu)
|
||||
k_slots = {
|
||||
ratio: prefill_k_slots(
|
||||
req_to_token=self.req_to_token,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
seq_lens_cpu=seq_lens_cpu,
|
||||
ratio=ratio,
|
||||
)
|
||||
for ratio in self.low_ratios
|
||||
}
|
||||
metadata = self.forward_metadata
|
||||
metadata.low_ratio_k_slots = k_slots
|
||||
metadata.low_ratio_index_plans = {}
|
||||
if metadata.low_ratio_req_indices is None:
|
||||
# Every source layer would rebuild these rows; the tail and the
|
||||
# prefill graph publish theirs at metadata init the same way.
|
||||
num_tokens = sum(forward_batch.extend_seq_lens_cpu)
|
||||
metadata.low_ratio_req_indices = token_req_indices(
|
||||
forward_batch, num_tokens=num_tokens
|
||||
)
|
||||
metadata.low_ratio_pos_i64 = forward_batch.positions[:num_tokens].to(
|
||||
torch.int64
|
||||
)
|
||||
if self.tail_forward_metadata is not None:
|
||||
self.tail_forward_metadata.low_ratio_k_slots = k_slots
|
||||
self.tail_forward_metadata.low_ratio_index_plans = {}
|
||||
|
||||
def _activate_request_window(self) -> None:
|
||||
if self.token_to_kv_pool.request_window is not None:
|
||||
self.token_to_kv_pool.request_window.activate(
|
||||
@@ -3782,6 +3824,18 @@ class DeepseekV4AttnBackend(
|
||||
else:
|
||||
rows_per_request = _as_int_list(forward_batch.extend_seq_lens_cpu)
|
||||
rows_per_request_device = forward_batch.extend_seq_lens
|
||||
seq_lens_cpu = _as_int_list(forward_batch.seq_lens_cpu)
|
||||
k_slots = self.forward_metadata.low_ratio_k_slots
|
||||
if k_slots is not None:
|
||||
k_slots = k_slots[layer.compress_ratio]
|
||||
else:
|
||||
# Graph-captured metadata carries no snapshot.
|
||||
k_slots = prefill_k_slots(
|
||||
req_to_token=self.req_to_token,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
seq_lens_cpu=seq_lens_cpu,
|
||||
ratio=layer.compress_ratio,
|
||||
)
|
||||
return PrefillInputs(
|
||||
indexer=layer.indexer,
|
||||
layer_id=layer.layer_id,
|
||||
@@ -3792,8 +3846,10 @@ class DeepseekV4AttnBackend(
|
||||
positions=pos,
|
||||
req_rows=req,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
k_slots=k_slots,
|
||||
index_plans=self.forward_metadata.low_ratio_index_plans,
|
||||
kv_page_table=core.page_table,
|
||||
seq_lens_cpu=_as_int_list(forward_batch.seq_lens_cpu),
|
||||
seq_lens_cpu=seq_lens_cpu,
|
||||
rows_per_request=rows_per_request,
|
||||
rows_per_request_device=rows_per_request_device,
|
||||
out_raw_indices=core.sparse_raw_indices(layer.compress_ratio),
|
||||
|
||||
@@ -123,7 +123,6 @@ def make_candidate_indexer(
|
||||
|
||||
sparse_table = SparseTableBackend(
|
||||
token_to_kv_pool=token_to_kv_pool,
|
||||
req_to_token=req_to_token,
|
||||
page_size=page_size,
|
||||
candidate_topk_blocks=candidate_topk_blocks,
|
||||
candidate_block_size=candidate_block_size,
|
||||
|
||||
@@ -146,7 +146,7 @@ class DenseBlocksBackend:
|
||||
|
||||
def _deep_gemm_publish_prefill(self, inputs: PrefillInputs):
|
||||
inputs.reset_outputs()
|
||||
data = get_deep_gemm_prefill_data(inputs, self.req_to_token)
|
||||
data = get_deep_gemm_prefill_data(inputs)
|
||||
if data is None:
|
||||
return None
|
||||
selected, blocks = _publish_prefill_blocks(
|
||||
@@ -167,7 +167,7 @@ class DenseBlocksBackend:
|
||||
published: Optional[BlockIds],
|
||||
) -> None:
|
||||
inputs.reset_outputs()
|
||||
data = get_deep_gemm_prefill_data(inputs, self.req_to_token)
|
||||
data = get_deep_gemm_prefill_data(inputs)
|
||||
if data is None:
|
||||
return
|
||||
assert published is not None
|
||||
@@ -189,7 +189,6 @@ class DenseBlocksBackend:
|
||||
for request, chunks in prefill_requests(
|
||||
inputs=inputs,
|
||||
token_to_kv_pool=self.token_to_kv_pool,
|
||||
req_to_token=self.req_to_token,
|
||||
):
|
||||
for chunk in chunks:
|
||||
picked.append(
|
||||
@@ -225,7 +224,6 @@ class DenseBlocksBackend:
|
||||
for request, chunks in prefill_requests(
|
||||
inputs=inputs,
|
||||
token_to_kv_pool=self.token_to_kv_pool,
|
||||
req_to_token=self.req_to_token,
|
||||
):
|
||||
for chunk in chunks:
|
||||
idx = topk_among_blocks(
|
||||
|
||||
@@ -72,7 +72,7 @@ class FullTopKIndexer:
|
||||
|
||||
def _deep_gemm_prefill(self, inputs: PrefillInputs) -> None:
|
||||
inputs.reset_outputs()
|
||||
data = get_deep_gemm_prefill_data(inputs, self.req_to_token)
|
||||
data = get_deep_gemm_prefill_data(inputs)
|
||||
if data is None:
|
||||
return
|
||||
kv = self.token_to_kv_pool.get_low_ratio_index_k_fp4(
|
||||
@@ -85,7 +85,6 @@ class FullTopKIndexer:
|
||||
requests = prefill_requests(
|
||||
inputs=inputs,
|
||||
token_to_kv_pool=self.token_to_kv_pool,
|
||||
req_to_token=self.req_to_token,
|
||||
)
|
||||
for request, chunks in requests:
|
||||
for chunk in chunks:
|
||||
|
||||
@@ -173,9 +173,39 @@ def get_deep_gemm_decode_data(
|
||||
return DeepGEMMDecodeData(q_fp4, q_sf, weights, k_cache)
|
||||
|
||||
|
||||
def get_deep_gemm_prefill_data(
|
||||
inputs: PrefillInputs, req_to_token: torch.Tensor
|
||||
) -> Optional[DeepGEMMPrefillData]:
|
||||
def prefill_k_slots(
|
||||
*,
|
||||
req_to_token: torch.Tensor,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens_cpu: List[int],
|
||||
ratio: int,
|
||||
) -> torch.Tensor:
|
||||
"""[columns] int64 index-K pool slot of each flattened-K column: request by
|
||||
request, the compressed positions visible at its newest token."""
|
||||
device = req_pool_indices.device
|
||||
lens_per_request = [s // ratio for s in seq_lens_cpu]
|
||||
columns = sum(lens_per_request)
|
||||
starts = list(itertools.accumulate(lens_per_request, initial=0))[:-1]
|
||||
lens = async_h2d(lens_per_request, dtype=torch.int64, device=device)
|
||||
starts_dev = async_h2d(starts, dtype=torch.int64, device=device)
|
||||
request = torch.repeat_interleave(
|
||||
torch.arange(len(lens_per_request), device=device), lens, output_size=columns
|
||||
)
|
||||
position = torch.arange(columns, device=device) - starts_dev[request]
|
||||
pool_rows = req_pool_indices.to(torch.int64)[request]
|
||||
return req_to_token[pool_rows, position * ratio].to(torch.int64) // ratio
|
||||
|
||||
|
||||
class PrefillIndexPlan(msgspec.Struct, frozen=True):
|
||||
"""The batch-only part of DeepGEMMPrefillData, the same in every layer of one
|
||||
compress ratio."""
|
||||
|
||||
request_starts: torch.Tensor # [rows] int32, first column of the row's request
|
||||
lens_per_request: List[int] # compressed positions at each request's newest token
|
||||
compress_lens: torch.Tensor # [rows] int32, compressed positions the row sees
|
||||
|
||||
|
||||
def prefill_index_plan(inputs: PrefillInputs) -> Optional[PrefillIndexPlan]:
|
||||
"""None when the chunk has no row or no visible compressed position."""
|
||||
ratio = inputs.compress_ratio
|
||||
pos = inputs.positions
|
||||
@@ -192,30 +222,43 @@ def get_deep_gemm_prefill_data(
|
||||
if columns == 0 or num_tokens == 0:
|
||||
return None
|
||||
starts = list(itertools.accumulate(lens_per_request, initial=0))[:-1]
|
||||
lens = async_h2d(lens_per_request, dtype=torch.int64, device=device)
|
||||
starts_dev = async_h2d(starts, dtype=torch.int64, device=device)
|
||||
request = torch.repeat_interleave(
|
||||
torch.arange(len(lens_per_request), device=device), lens, output_size=columns
|
||||
starts_dev = async_h2d(starts, dtype=torch.int32, device=device)
|
||||
return PrefillIndexPlan(
|
||||
request_starts=torch.repeat_interleave(
|
||||
starts_dev,
|
||||
inputs.rows_per_request_device.to(torch.int64),
|
||||
output_size=num_tokens,
|
||||
),
|
||||
lens_per_request=lens_per_request,
|
||||
compress_lens=((pos + 1) // ratio).to(torch.int32),
|
||||
)
|
||||
position = torch.arange(columns, device=device) - starts_dev[request]
|
||||
pool_rows = inputs.req_pool_indices.to(torch.int64)[request]
|
||||
|
||||
|
||||
def get_deep_gemm_prefill_data(inputs: PrefillInputs) -> Optional[DeepGEMMPrefillData]:
|
||||
"""None when the chunk has no row or no visible compressed position."""
|
||||
ratio = inputs.compress_ratio
|
||||
plans = inputs.index_plans
|
||||
if plans is not None and ratio in plans:
|
||||
plan = plans[ratio]
|
||||
else:
|
||||
plan = prefill_index_plan(inputs)
|
||||
if plans is not None:
|
||||
plans[ratio] = plan
|
||||
if plan is None:
|
||||
return None
|
||||
q_fp4, q_sf, weights = _index_q_and_weights(
|
||||
indexer=inputs.indexer,
|
||||
x=inputs.x,
|
||||
q_lora=inputs.q_lora,
|
||||
freqs_cis=inputs.freqs_cis,
|
||||
positions=pos,
|
||||
positions=inputs.positions,
|
||||
)
|
||||
return DeepGEMMPrefillData(
|
||||
k_slots=req_to_token[pool_rows, position * ratio].to(torch.int64) // ratio,
|
||||
request_starts=torch.repeat_interleave(
|
||||
starts_dev.to(torch.int32),
|
||||
inputs.rows_per_request_device.to(torch.int64),
|
||||
output_size=num_tokens,
|
||||
),
|
||||
lens_per_request=lens_per_request,
|
||||
k_slots=inputs.k_slots,
|
||||
request_starts=plan.request_starts,
|
||||
lens_per_request=plan.lens_per_request,
|
||||
rows_per_request=inputs.rows_per_request,
|
||||
compress_lens=((pos + 1) // ratio).to(torch.int32),
|
||||
compress_lens=plan.compress_lens,
|
||||
q_fp4=q_fp4,
|
||||
q_sf=q_sf,
|
||||
weights=weights,
|
||||
@@ -312,7 +355,6 @@ def prefill_requests(
|
||||
*,
|
||||
inputs: PrefillInputs,
|
||||
token_to_kv_pool: DeepSeekV4TokenToKVPool,
|
||||
req_to_token: torch.Tensor,
|
||||
) -> Iterator[tuple[RequestScores, Iterator[ChunkScores]]]:
|
||||
"""Requests with nothing visible yet are skipped."""
|
||||
pool = token_to_kv_pool
|
||||
@@ -320,6 +362,14 @@ def prefill_requests(
|
||||
indexer = inputs.indexer
|
||||
req, pos = inputs.req_rows, inputs.positions
|
||||
assert req is not None, "the torch indexer needs the batch's rows"
|
||||
assert inputs.seq_lens_cpu is not None
|
||||
# First flattened-K column of each request row, as inputs.k_slots lays them out.
|
||||
k_starts = dict(
|
||||
zip(
|
||||
inputs.req_pool_indices.tolist(),
|
||||
itertools.accumulate((s // ratio for s in inputs.seq_lens_cpu), initial=0),
|
||||
)
|
||||
)
|
||||
inputs.reset_outputs()
|
||||
q = indexer.queries(inputs.q_lora, inputs.freqs_cis[pos])
|
||||
weights = indexer.head_weights(inputs.x)
|
||||
@@ -333,7 +383,7 @@ def prefill_requests(
|
||||
if lc == 0:
|
||||
continue
|
||||
j = torch.arange(lc, device=pos.device)
|
||||
slots = req_to_token[r, j * ratio].to(torch.int64) // ratio
|
||||
slots = inputs.k_slots[k_starts[r] : k_starts[r] + lc]
|
||||
# Dequantize only this request's visible K rows; the table is pool-sized.
|
||||
index_k = pool.get_low_ratio_index_k_dequant(inputs.layer_id, slots)
|
||||
request = RequestScores(
|
||||
|
||||
@@ -120,14 +120,12 @@ class SparseTableBackend:
|
||||
self,
|
||||
*,
|
||||
token_to_kv_pool: DeepSeekV4TokenToKVPool,
|
||||
req_to_token: torch.Tensor,
|
||||
page_size: int,
|
||||
candidate_topk_blocks: int,
|
||||
candidate_block_size: int,
|
||||
) -> None:
|
||||
assert candidate_block_size == CANDIDATE_BLOCK_SIZE
|
||||
self.token_to_kv_pool = token_to_kv_pool
|
||||
self.req_to_token = req_to_token
|
||||
self.page_size = page_size
|
||||
self.topk_blocks = candidate_topk_blocks
|
||||
self.alt_stream = torch.cuda.Stream()
|
||||
@@ -139,7 +137,7 @@ class SparseTableBackend:
|
||||
|
||||
def publish_prefill(self, inputs: PrefillInputs):
|
||||
inputs.reset_outputs()
|
||||
data = get_deep_gemm_prefill_data(inputs, self.req_to_token)
|
||||
data = get_deep_gemm_prefill_data(inputs)
|
||||
if data is None:
|
||||
return None
|
||||
index_page_size = self.token_to_kv_pool.get_index_k_page_size(
|
||||
@@ -169,7 +167,7 @@ class SparseTableBackend:
|
||||
published: Optional[_SparsePrefillTable],
|
||||
) -> None:
|
||||
inputs.reset_outputs()
|
||||
data = get_deep_gemm_prefill_data(inputs, self.req_to_token)
|
||||
data = get_deep_gemm_prefill_data(inputs)
|
||||
if data is None:
|
||||
return
|
||||
assert published is not None
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, List, Optional, Protocol, TypeVar
|
||||
from typing import TYPE_CHECKING, Dict, List, Optional, Protocol, TypeVar
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
@@ -47,6 +47,13 @@ class PrefillInputs(msgspec.Struct, frozen=True, kw_only=True):
|
||||
# rank's rows are an interleaved subset of the batch's.
|
||||
req_rows: Optional[torch.Tensor]
|
||||
req_pool_indices: torch.Tensor # [requests] req_to_token row of each request
|
||||
# [columns] int64 index-K pool slot of each flattened-K column, from
|
||||
# scoring.prefill_k_slots; gathered with the metadata, never per layer.
|
||||
k_slots: torch.Tensor
|
||||
# ratio -> scoring.PrefillIndexPlan, filled by the first layer of each ratio and
|
||||
# reused by the rest of the forward; None when the metadata outlives a forward
|
||||
# (captured graphs), so every layer builds its own.
|
||||
index_plans: Optional[Dict[int, object]]
|
||||
# [rows, pages] int32 at the KV page size; the index-K pool pages it with
|
||||
# expand_index_page_table.
|
||||
kv_page_table: torch.Tensor
|
||||
|
||||
Reference in New Issue
Block a user