[Fix] Harden FlashAttention CUDA graph metadata bounds (#35454)
Co-authored-by: Yuwei An <ayw.sirius19@gmail.com>
This commit is contained in:
@@ -616,11 +616,14 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
# upper bound) so captured graphs keep a valid address; each replay
|
||||
# refills a [:num_tokens] view.
|
||||
if self.use_sliding_window_kv_pool:
|
||||
assert forward_batch.out_cache_loc is not None
|
||||
m.swa_page_table = torch.zeros(
|
||||
(bs, self.max_num_pages), dtype=torch.int32, device=device
|
||||
)
|
||||
self.full_cg_prefill_swa_out_cache_loc = torch.zeros(
|
||||
(self.max_context_len,), dtype=torch.int64, device=device
|
||||
(forward_batch.out_cache_loc.shape[0],),
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
)
|
||||
self.full_cg_prefill_metadata = m
|
||||
m = self.full_cg_prefill_metadata
|
||||
@@ -657,6 +660,11 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
# SWA write targets for the new tokens (KVWriteLoc.swa_loc), refilled
|
||||
# into the pointer-stable buffer and bound as a [:num_tokens] view.
|
||||
num_out = forward_batch.out_cache_loc.shape[0]
|
||||
assert_buffer_fits(
|
||||
num_out,
|
||||
self.full_cg_prefill_swa_out_cache_loc.shape[0],
|
||||
"full-CG prefill SWA write-location buffer",
|
||||
)
|
||||
self.full_cg_prefill_swa_out_cache_loc[:num_out].copy_(
|
||||
self.token_to_kv_pool.translate_loc_from_full_to_swa(
|
||||
forward_batch.out_cache_loc
|
||||
@@ -686,7 +694,7 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
seq_lens_cpu = forward_batch.seq_lens_cpu
|
||||
eager_max_k = (
|
||||
seq_lens_cpu.max().item()
|
||||
if seq_lens_cpu is not None
|
||||
if seq_lens_cpu is not None and seq_lens_cpu.numel() > 0
|
||||
else self.max_context_len
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user