[mem_cache] Drop the torch.unique sync from the SWA page expansion (#37463)

This commit is contained in:
Liangsheng Yin
2026-09-01 14:16:47 -07:00
committed by GitHub
parent b24c8f10e7
commit 83a9b5dd88
+12 -2
View File
@@ -407,11 +407,21 @@ class SWATokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
self.free_full(torch.cat(full_free_group))
def _expand_to_full_pages(self, indices: torch.Tensor) -> torch.Tensor:
pages = torch.unique(indices // self.page_size)
# Duplicates are kept: deduplicating would be a torch.unique whose
# data-dependent output shape synchronizes the scheduler stream, and
# every consumer ends in the paged free's own page dedup anyway.
base = (indices // self.page_size) * self.page_size
page_offsets = torch.arange(
self.page_size, dtype=indices.dtype, device=indices.device
)
return (pages[:, None] * self.page_size + page_offsets[None, :]).reshape(-1)
expanded = (base[:, None] + page_offsets[None, :]).reshape(-1)
if self.swa_attn_allocator.debug_mode:
# Reference unique on CPU: the expansion must cover exactly the
# touched pages, on every caller's real input.
got = torch.unique(expanded.cpu() // self.page_size)
ref = torch.unique(indices.cpu() // self.page_size)
assert torch.equal(got, ref), "expansion page set mismatch"
return expanded
def resize(self, config) -> None:
size_full = int(config.full_max_total_num_tokens)