[mem_cache] Drop the torch.unique sync from the SWA page expansion (#37463)
This commit is contained in:
@@ -407,11 +407,21 @@ class SWATokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
|
|||||||
self.free_full(torch.cat(full_free_group))
|
self.free_full(torch.cat(full_free_group))
|
||||||
|
|
||||||
def _expand_to_full_pages(self, indices: torch.Tensor) -> torch.Tensor:
|
def _expand_to_full_pages(self, indices: torch.Tensor) -> torch.Tensor:
|
||||||
pages = torch.unique(indices // self.page_size)
|
# Duplicates are kept: deduplicating would be a torch.unique whose
|
||||||
|
# data-dependent output shape synchronizes the scheduler stream, and
|
||||||
|
# every consumer ends in the paged free's own page dedup anyway.
|
||||||
|
base = (indices // self.page_size) * self.page_size
|
||||||
page_offsets = torch.arange(
|
page_offsets = torch.arange(
|
||||||
self.page_size, dtype=indices.dtype, device=indices.device
|
self.page_size, dtype=indices.dtype, device=indices.device
|
||||||
)
|
)
|
||||||
return (pages[:, None] * self.page_size + page_offsets[None, :]).reshape(-1)
|
expanded = (base[:, None] + page_offsets[None, :]).reshape(-1)
|
||||||
|
if self.swa_attn_allocator.debug_mode:
|
||||||
|
# Reference unique on CPU: the expansion must cover exactly the
|
||||||
|
# touched pages, on every caller's real input.
|
||||||
|
got = torch.unique(expanded.cpu() // self.page_size)
|
||||||
|
ref = torch.unique(indices.cpu() // self.page_size)
|
||||||
|
assert torch.equal(got, ref), "expansion page set mismatch"
|
||||||
|
return expanded
|
||||||
|
|
||||||
def resize(self, config) -> None:
|
def resize(self, config) -> None:
|
||||||
size_full = int(config.full_max_total_num_tokens)
|
size_full = int(config.full_max_total_num_tokens)
|
||||||
|
|||||||
Reference in New Issue
Block a user