[Perf] dsv41: Translate SWA locations once per prefill graph step (#46229)
This commit is contained in:
@@ -3151,6 +3151,13 @@ class DeepseekV4AttnBackend(
|
||||
max_seq_len_override=max_seq_len,
|
||||
use_prefill_cuda_graph=True,
|
||||
)
|
||||
if self._prefill_graph_translates_swa_loc(self.forward_metadata):
|
||||
# Every layer's store reads this buffer; replay refreshes it in place.
|
||||
self.forward_metadata.core_attn_metadata.swa_out_cache_loc = (
|
||||
self.token_to_kv_pool.translate_loc_from_full_to_swa(
|
||||
forward_batch.out_cache_loc
|
||||
).to(torch.int32)
|
||||
)
|
||||
self._activate_request_window()
|
||||
with_tail = (
|
||||
self.enable_decoder_swa_bounded_replay
|
||||
@@ -3171,6 +3178,13 @@ class DeepseekV4AttnBackend(
|
||||
)
|
||||
return self.forward_metadata
|
||||
|
||||
def _prefill_graph_translates_swa_loc(self, metadata: DSV4Metadata) -> bool:
|
||||
return (
|
||||
metadata.core_attn_metadata.swa_out_cache_loc is None
|
||||
and self.token_to_kv_pool.request_window is None
|
||||
and self.token_to_kv_pool.swa_req_ring_size is None
|
||||
)
|
||||
|
||||
def _prefill_graph_context_width(self, forward_batch: ForwardBatch) -> int:
|
||||
"""One context width for the full rows' and the late-layer tail's metadata:
|
||||
the live context, unless c4/c128 compress metadata is captured at a width."""
|
||||
@@ -3250,6 +3264,12 @@ class DeepseekV4AttnBackend(
|
||||
)
|
||||
assert isinstance(capture_metadata, DSV4Metadata)
|
||||
capture_metadata.refresh_for_breakable_cuda_graph_replay_(static_metadata)
|
||||
if self._prefill_graph_translates_swa_loc(static_metadata):
|
||||
capture_metadata.core_attn_metadata.swa_out_cache_loc.copy_(
|
||||
self.token_to_kv_pool.translate_loc_from_full_to_swa(
|
||||
metadata_batch.out_cache_loc
|
||||
)
|
||||
)
|
||||
self.prefill_graph_replay_batch = forward_batch
|
||||
self.forward_metadata = capture_metadata
|
||||
self._activate_request_window()
|
||||
|
||||
@@ -497,7 +497,9 @@ class DecoderReplayTail:
|
||||
def _bridge_outputs(
|
||||
self, block_rows: int, outputs: List[Optional[torch.Tensor]]
|
||||
) -> Tuple[Optional[torch.Tensor], ...]:
|
||||
"""Block-sized bridge views of the tail rows; rows past the tail are zero."""
|
||||
"""Block-sized bridge views of the tail rows. Rows past the tail hold stale
|
||||
values, which reach only rows that _scatter_tail_rows and the
|
||||
real_rows_count cut drop."""
|
||||
key = tuple(None if t is None else (t.dtype, t.shape[1:]) for t in outputs)
|
||||
sets = self._bridge_buffers.setdefault(key, [])
|
||||
if not sets or sets[0][0].shape[0] < block_rows:
|
||||
@@ -515,7 +517,6 @@ class DecoderReplayTail:
|
||||
continue
|
||||
view = buf[:block_rows]
|
||||
view[: t.shape[0]].copy_(t)
|
||||
view[t.shape[0] :].zero_()
|
||||
views.append(view)
|
||||
return tuple(views)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user