[Perf] DSv4.1: Reduce BCG metadata memory (#44920)
This commit is contained in:
@@ -2907,6 +2907,10 @@ class DeepseekV4AttnBackend(
|
||||
self, forward_batch: ForwardBatch
|
||||
):
|
||||
max_seq_len = forward_batch.max_seq_len_override or self.MAX_SEQ_LEN_FOR_CAPTURE
|
||||
if self.token_to_kv_pool.request_window is None and self.low_ratios == (1, 2):
|
||||
graph_context_limit = _prefill_graph_max_seq_len()
|
||||
if graph_context_limit is not None:
|
||||
max_seq_len = min(max_seq_len, graph_context_limit)
|
||||
self.forward_metadata = self._build_forward_metadata(
|
||||
forward_batch,
|
||||
max_seq_len_override=max_seq_len,
|
||||
@@ -2916,6 +2920,8 @@ class DeepseekV4AttnBackend(
|
||||
self.enable_decoder_swa_bounded_replay
|
||||
and forward_batch.forward_mode.is_extend_without_speculative()
|
||||
)
|
||||
if self.token_to_kv_pool.request_window is None and self.low_ratios == (1, 2):
|
||||
self._share_bcg_low_ratio_index_buffers(self.forward_metadata.core_metadata)
|
||||
self.forward_metadata.prefill_graph_tail = with_tail
|
||||
self.tail_forward_metadata = (
|
||||
self._prefill_graph_tail_metadata(forward_batch, max_seq_len)
|
||||
@@ -2929,6 +2935,25 @@ class DeepseekV4AttnBackend(
|
||||
)
|
||||
return self.forward_metadata
|
||||
|
||||
def _share_bcg_low_ratio_index_buffers(self, core) -> None:
|
||||
# BCG buckets replay serially. Share only full-metadata output buffers;
|
||||
# both ratios must survive the later full-to-tail metadata transition.
|
||||
# Keep old allocations alive if capture order ever grows the capacity.
|
||||
pools = getattr(self, "_bcg_low_ratio_index_buffers", None)
|
||||
if pools is None:
|
||||
pools = self._bcg_low_ratio_index_buffers = {}
|
||||
for ratio in (1, 2):
|
||||
for kind in ("page", "raw"):
|
||||
name = f"c{ratio}_sparse_{kind}_indices"
|
||||
value = getattr(core, name)
|
||||
if value is None:
|
||||
continue
|
||||
key = (name, value.shape[1], value.dtype, value.device)
|
||||
retained = pools.setdefault(key, [])
|
||||
if not retained or retained[0].shape[0] < value.shape[0]:
|
||||
retained.insert(0, value)
|
||||
setattr(core, name, retained[0][: value.shape[0]])
|
||||
|
||||
def _source_projection_buffers(self, num_tokens: int, ratio: int) -> dict:
|
||||
cfg = self.model_runner.model_config.hf_text_config
|
||||
heads, dim = int(cfg.index_n_heads), int(cfg.index_head_dim)
|
||||
@@ -2971,6 +2996,10 @@ class DeepseekV4AttnBackend(
|
||||
max_seq_len = (
|
||||
metadata_batch.max_seq_len_override or self.MAX_SEQ_LEN_FOR_CAPTURE
|
||||
)
|
||||
if self.token_to_kv_pool.request_window is None and self.low_ratios == (1, 2):
|
||||
graph_context_limit = _prefill_graph_max_seq_len()
|
||||
if graph_context_limit is not None:
|
||||
max_seq_len = min(max_seq_len, graph_context_limit)
|
||||
static_metadata = self._build_forward_metadata(
|
||||
metadata_batch,
|
||||
max_seq_len_override=max_seq_len,
|
||||
|
||||
@@ -337,6 +337,9 @@ class SparsePrefillChunkCache:
|
||||
# Compressed caches keyed by compress ratio: c128 (every block, combined once
|
||||
# per chunk) and the top-k ratios (combined per layer).
|
||||
compressed: Dict[int, CompressedGather] = field(default_factory=dict)
|
||||
# Per-layer scratch only: combine overwrites all entries before attention.
|
||||
# c0/c128 are cached across layers and must retain separate storage.
|
||||
low_ratio_combined: dict = field(default_factory=dict)
|
||||
swa_indices: Optional[torch.Tensor] = None
|
||||
swa_lengths: Optional[torch.Tensor] = None
|
||||
# Host lengths avoid a device synchronization when sizing ragged gathers.
|
||||
@@ -602,6 +605,11 @@ class SparsePrefillChunkCache:
|
||||
"""
|
||||
gather = self.compressed[compress_ratio]
|
||||
topk = sparse_raw_indices.shape[-1]
|
||||
width = combined_topk_width(topk, self.swa_window_size)
|
||||
if compress_ratio in (1, 2) and width in self.low_ratio_combined:
|
||||
gather.combined_indices, gather.combined_lens = self.low_ratio_combined[
|
||||
width
|
||||
]
|
||||
if gather.combined_indices is None:
|
||||
device = self.seq_lens.device
|
||||
gather.combined_indices = torch.full(
|
||||
@@ -613,6 +621,11 @@ class SparsePrefillChunkCache:
|
||||
gather.combined_lens = torch.zeros(
|
||||
self.num_qo_tokens, dtype=torch.int32, device=device
|
||||
)
|
||||
if compress_ratio in (1, 2):
|
||||
self.low_ratio_combined[width] = (
|
||||
gather.combined_indices,
|
||||
gather.combined_lens,
|
||||
)
|
||||
return combine_topk_swa_indices(
|
||||
topk_indices=sparse_raw_indices,
|
||||
query_start_loc=self.query_start_loc,
|
||||
|
||||
Reference in New Issue
Block a user