[Perf] dsv41: Translate SWA locations once per prefill graph step (#46229)

This commit is contained in:
Liangsheng Yin
2026-10-23 21:33:15 -07:00
committed by GitHub
parent 564aec61c8
commit 54df642fe6
4 changed files with 41 additions and 4 deletions
@@ -3151,6 +3151,13 @@ class DeepseekV4AttnBackend(
max_seq_len_override=max_seq_len,
use_prefill_cuda_graph=True,
)
if self._prefill_graph_translates_swa_loc(self.forward_metadata):
# Every layer's store reads this buffer; replay refreshes it in place.
self.forward_metadata.core_attn_metadata.swa_out_cache_loc = (
self.token_to_kv_pool.translate_loc_from_full_to_swa(
forward_batch.out_cache_loc
).to(torch.int32)
)
self._activate_request_window()
with_tail = (
self.enable_decoder_swa_bounded_replay
@@ -3171,6 +3178,13 @@ class DeepseekV4AttnBackend(
)
return self.forward_metadata
def _prefill_graph_translates_swa_loc(self, metadata: DSV4Metadata) -> bool:
return (
metadata.core_attn_metadata.swa_out_cache_loc is None
and self.token_to_kv_pool.request_window is None
and self.token_to_kv_pool.swa_req_ring_size is None
)
def _prefill_graph_context_width(self, forward_batch: ForwardBatch) -> int:
"""One context width for the full rows' and the late-layer tail's metadata:
the live context, unless c4/c128 compress metadata is captured at a width."""
@@ -3250,6 +3264,12 @@ class DeepseekV4AttnBackend(
)
assert isinstance(capture_metadata, DSV4Metadata)
capture_metadata.refresh_for_breakable_cuda_graph_replay_(static_metadata)
if self._prefill_graph_translates_swa_loc(static_metadata):
capture_metadata.core_attn_metadata.swa_out_cache_loc.copy_(
self.token_to_kv_pool.translate_loc_from_full_to_swa(
metadata_batch.out_cache_loc
)
)
self.prefill_graph_replay_batch = forward_batch
self.forward_metadata = capture_metadata
self._activate_request_window()
@@ -497,7 +497,9 @@ class DecoderReplayTail:
def _bridge_outputs(
self, block_rows: int, outputs: List[Optional[torch.Tensor]]
) -> Tuple[Optional[torch.Tensor], ...]:
"""Block-sized bridge views of the tail rows; rows past the tail are zero."""
"""Block-sized bridge views of the tail rows. Rows past the tail hold stale
values, which reach only rows that _scatter_tail_rows and the
real_rows_count cut drop."""
key = tuple(None if t is None else (t.dtype, t.shape[1:]) for t in outputs)
sets = self._bridge_buffers.setdefault(key, [])
if not sets or sets[0][0].shape[0] < block_rows:
@@ -515,7 +517,6 @@ class DecoderReplayTail:
continue
view = buf[:block_rows]
view[: t.shape[0]].copy_(t)
view[t.shape[0] :].zero_()
views.append(view)
return tuple(views)