[Perf] DSv4.1: Reduce BCG metadata memory (#44920)

This commit is contained in:
Yuwei An
2026-10-17 16:05:28 -07:00
committed by GitHub
parent 563cd60345
commit e4512a5a17
2 changed files with 42 additions and 0 deletions
@@ -2907,6 +2907,10 @@ class DeepseekV4AttnBackend(
self, forward_batch: ForwardBatch
):
max_seq_len = forward_batch.max_seq_len_override or self.MAX_SEQ_LEN_FOR_CAPTURE
if self.token_to_kv_pool.request_window is None and self.low_ratios == (1, 2):
graph_context_limit = _prefill_graph_max_seq_len()
if graph_context_limit is not None:
max_seq_len = min(max_seq_len, graph_context_limit)
self.forward_metadata = self._build_forward_metadata(
forward_batch,
max_seq_len_override=max_seq_len,
@@ -2916,6 +2920,8 @@ class DeepseekV4AttnBackend(
self.enable_decoder_swa_bounded_replay
and forward_batch.forward_mode.is_extend_without_speculative()
)
if self.token_to_kv_pool.request_window is None and self.low_ratios == (1, 2):
self._share_bcg_low_ratio_index_buffers(self.forward_metadata.core_metadata)
self.forward_metadata.prefill_graph_tail = with_tail
self.tail_forward_metadata = (
self._prefill_graph_tail_metadata(forward_batch, max_seq_len)
@@ -2929,6 +2935,25 @@ class DeepseekV4AttnBackend(
)
return self.forward_metadata
def _share_bcg_low_ratio_index_buffers(self, core) -> None:
# BCG buckets replay serially. Share only full-metadata output buffers;
# both ratios must survive the later full-to-tail metadata transition.
# Keep old allocations alive if capture order ever grows the capacity.
pools = getattr(self, "_bcg_low_ratio_index_buffers", None)
if pools is None:
pools = self._bcg_low_ratio_index_buffers = {}
for ratio in (1, 2):
for kind in ("page", "raw"):
name = f"c{ratio}_sparse_{kind}_indices"
value = getattr(core, name)
if value is None:
continue
key = (name, value.shape[1], value.dtype, value.device)
retained = pools.setdefault(key, [])
if not retained or retained[0].shape[0] < value.shape[0]:
retained.insert(0, value)
setattr(core, name, retained[0][: value.shape[0]])
def _source_projection_buffers(self, num_tokens: int, ratio: int) -> dict:
cfg = self.model_runner.model_config.hf_text_config
heads, dim = int(cfg.index_n_heads), int(cfg.index_head_dim)
@@ -2971,6 +2996,10 @@ class DeepseekV4AttnBackend(
max_seq_len = (
metadata_batch.max_seq_len_override or self.MAX_SEQ_LEN_FOR_CAPTURE
)
if self.token_to_kv_pool.request_window is None and self.low_ratios == (1, 2):
graph_context_limit = _prefill_graph_max_seq_len()
if graph_context_limit is not None:
max_seq_len = min(max_seq_len, graph_context_limit)
static_metadata = self._build_forward_metadata(
metadata_batch,
max_seq_len_override=max_seq_len,
@@ -337,6 +337,9 @@ class SparsePrefillChunkCache:
# Compressed caches keyed by compress ratio: c128 (every block, combined once
# per chunk) and the top-k ratios (combined per layer).
compressed: Dict[int, CompressedGather] = field(default_factory=dict)
# Per-layer scratch only: combine overwrites all entries before attention.
# c0/c128 are cached across layers and must retain separate storage.
low_ratio_combined: dict = field(default_factory=dict)
swa_indices: Optional[torch.Tensor] = None
swa_lengths: Optional[torch.Tensor] = None
# Host lengths avoid a device synchronization when sizing ragged gathers.
@@ -602,6 +605,11 @@ class SparsePrefillChunkCache:
"""
gather = self.compressed[compress_ratio]
topk = sparse_raw_indices.shape[-1]
width = combined_topk_width(topk, self.swa_window_size)
if compress_ratio in (1, 2) and width in self.low_ratio_combined:
gather.combined_indices, gather.combined_lens = self.low_ratio_combined[
width
]
if gather.combined_indices is None:
device = self.seq_lens.device
gather.combined_indices = torch.full(
@@ -613,6 +621,11 @@ class SparsePrefillChunkCache:
gather.combined_lens = torch.zeros(
self.num_qo_tokens, dtype=torch.int32, device=device
)
if compress_ratio in (1, 2):
self.low_ratio_combined[width] = (
gather.combined_indices,
gather.combined_lens,
)
return combine_topk_swa_indices(
topk_indices=sparse_raw_indices,
query_start_loc=self.query_start_loc,