[mem_cache] Drop the torch.unique sync from the SWA page expansion (#37463)
This commit is contained in:
@@ -407,11 +407,21 @@ class SWATokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
|
||||
self.free_full(torch.cat(full_free_group))
|
||||
|
||||
def _expand_to_full_pages(self, indices: torch.Tensor) -> torch.Tensor:
|
||||
pages = torch.unique(indices // self.page_size)
|
||||
# Duplicates are kept: deduplicating would be a torch.unique whose
|
||||
# data-dependent output shape synchronizes the scheduler stream, and
|
||||
# every consumer ends in the paged free's own page dedup anyway.
|
||||
base = (indices // self.page_size) * self.page_size
|
||||
page_offsets = torch.arange(
|
||||
self.page_size, dtype=indices.dtype, device=indices.device
|
||||
)
|
||||
return (pages[:, None] * self.page_size + page_offsets[None, :]).reshape(-1)
|
||||
expanded = (base[:, None] + page_offsets[None, :]).reshape(-1)
|
||||
if self.swa_attn_allocator.debug_mode:
|
||||
# Reference unique on CPU: the expansion must cover exactly the
|
||||
# touched pages, on every caller's real input.
|
||||
got = torch.unique(expanded.cpu() // self.page_size)
|
||||
ref = torch.unique(indices.cpu() // self.page_size)
|
||||
assert torch.equal(got, ref), "expansion page set mismatch"
|
||||
return expanded
|
||||
|
||||
def resize(self, config) -> None:
|
||||
size_full = int(config.full_max_total_num_tokens)
|
||||
|
||||
Reference in New Issue
Block a user