Fix paged SWA free mapping cleanup (#27779)
This commit is contained in:
@@ -338,10 +338,25 @@ class SWATokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
|
||||
self.full_to_swa_index_mapping[full_indices] = swa_indices
|
||||
|
||||
def free_swa(self, free_index: torch.Tensor):
|
||||
swa_indices = self.full_to_swa_index_mapping[free_index]
|
||||
if free_index.numel() == 0:
|
||||
return
|
||||
|
||||
if self.page_size == 1:
|
||||
mapping_indices = free_index
|
||||
else:
|
||||
mapping_indices = self._expand_to_full_pages(free_index)
|
||||
|
||||
swa_indices = self.full_to_swa_index_mapping[mapping_indices]
|
||||
swa_indices = swa_indices[swa_indices > 0]
|
||||
self.swa_attn_allocator.free(swa_indices)
|
||||
self.full_to_swa_index_mapping[free_index] = 0
|
||||
self.full_to_swa_index_mapping[mapping_indices] = 0
|
||||
|
||||
def _expand_to_full_pages(self, indices: torch.Tensor) -> torch.Tensor:
|
||||
pages = torch.unique(indices // self.page_size)
|
||||
page_offsets = torch.arange(
|
||||
self.page_size, dtype=indices.dtype, device=indices.device
|
||||
)
|
||||
return (pages[:, None] * self.page_size + page_offsets[None, :]).reshape(-1)
|
||||
|
||||
def backup_state(self):
|
||||
return [
|
||||
|
||||
@@ -246,6 +246,30 @@ class TestSWA(unittest.TestCase):
|
||||
result = alloc.translate_loc_from_full_to_swa(index)
|
||||
print(result)
|
||||
|
||||
def test_swa_memory_pool_paged_free_clears_full_page_mapping(self):
|
||||
page_size = 4
|
||||
_, allocator, _ = _build_swa_tree(
|
||||
is_eagle=False,
|
||||
page_size=page_size,
|
||||
kv_size=16,
|
||||
kv_size_swa=16,
|
||||
sliding_window_size=page_size,
|
||||
)
|
||||
|
||||
full_indices = _swa_alloc(allocator, page_size)
|
||||
self.assertEqual(allocator.swa_available_size(), 16 - page_size)
|
||||
|
||||
allocator.free_swa(full_indices[:1])
|
||||
self.assertEqual(allocator.swa_available_size(), 16)
|
||||
self.assertTrue(
|
||||
torch.all(
|
||||
allocator.full_to_swa_index_mapping[full_indices.to(torch.int64)] == 0
|
||||
)
|
||||
)
|
||||
|
||||
allocator.free_swa(full_indices[1:2])
|
||||
self.assertEqual(allocator.swa_available_size(), 16)
|
||||
|
||||
def test_swa_radix_cache_1(self):
|
||||
# args
|
||||
req_size = 10
|
||||
|
||||
Reference in New Issue
Block a user