Fix SWA ownership across grouped frees (#36381)
This commit is contained in:
@@ -354,10 +354,6 @@ class SWATokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
|
|||||||
if free_index.numel() == 0:
|
if free_index.numel() == 0:
|
||||||
return
|
return
|
||||||
|
|
||||||
if not self.is_not_in_free_group:
|
|
||||||
self.swa_free_group.append(self._copy_for_free_group(free_index))
|
|
||||||
return
|
|
||||||
|
|
||||||
if self.page_size == 1:
|
if self.page_size == 1:
|
||||||
mapping_indices = free_index
|
mapping_indices = free_index
|
||||||
else:
|
else:
|
||||||
@@ -365,9 +361,16 @@ class SWATokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
|
|||||||
|
|
||||||
swa_indices = self.full_to_swa_index_mapping[mapping_indices]
|
swa_indices = self.full_to_swa_index_mapping[mapping_indices]
|
||||||
swa_indices = swa_indices[swa_indices > 0]
|
swa_indices = swa_indices[swa_indices > 0]
|
||||||
self.swa_attn_allocator.free(swa_indices)
|
|
||||||
self.clear_full_to_swa_mapping(mapping_indices)
|
self.clear_full_to_swa_mapping(mapping_indices)
|
||||||
|
|
||||||
|
if not self.is_not_in_free_group:
|
||||||
|
# Resolve ownership now. A cache action later in this group may
|
||||||
|
# install a new mapping for the same full index.
|
||||||
|
self.swa_free_group.append(swa_indices)
|
||||||
|
return
|
||||||
|
|
||||||
|
self.swa_attn_allocator.free(swa_indices)
|
||||||
|
|
||||||
def free_group_begin(self):
|
def free_group_begin(self):
|
||||||
super().free_group_begin()
|
super().free_group_begin()
|
||||||
self.swa_free_group = []
|
self.swa_free_group = []
|
||||||
@@ -377,7 +380,7 @@ class SWATokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
|
|||||||
if self.swa_free_group:
|
if self.swa_free_group:
|
||||||
swa_free_group = self.swa_free_group
|
swa_free_group = self.swa_free_group
|
||||||
self.swa_free_group = []
|
self.swa_free_group = []
|
||||||
self.free_swa(torch.cat(swa_free_group))
|
self.swa_attn_allocator.free(torch.cat(swa_free_group))
|
||||||
|
|
||||||
def _expand_to_full_pages(self, indices: torch.Tensor) -> torch.Tensor:
|
def _expand_to_full_pages(self, indices: torch.Tensor) -> torch.Tensor:
|
||||||
pages = torch.unique(indices // self.page_size)
|
pages = torch.unique(indices // self.page_size)
|
||||||
|
|||||||
@@ -298,6 +298,38 @@ class TestSWA(unittest.TestCase):
|
|||||||
available_before_free + original_indices.numel(),
|
available_before_free + original_indices.numel(),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_free_swa_group_owns_mapping_at_enqueue_time(self):
|
||||||
|
_, allocator, _ = _build_swa_tree(
|
||||||
|
is_eagle=False,
|
||||||
|
kv_size=8,
|
||||||
|
kv_size_swa=8,
|
||||||
|
)
|
||||||
|
old_full = _swa_alloc(allocator, 1)
|
||||||
|
new_full = _swa_alloc(allocator, 1)
|
||||||
|
assert old_full is not None and new_full is not None
|
||||||
|
old_swa = allocator.full_to_swa_index_mapping[old_full].clone()
|
||||||
|
new_swa = allocator.full_to_swa_index_mapping[new_full].clone()
|
||||||
|
|
||||||
|
allocator.free_group_begin()
|
||||||
|
allocator.free_swa(old_full)
|
||||||
|
|
||||||
|
# Cache reconciliation can transfer a different SWA slot onto the same
|
||||||
|
# full slot before the group flushes. The deferred free still owns the
|
||||||
|
# mapping observed above, not this replacement mapping.
|
||||||
|
allocator.set_full_to_swa_mapping(old_full, new_swa)
|
||||||
|
allocator.clear_full_to_swa_mapping(new_full)
|
||||||
|
allocator.free_group_end()
|
||||||
|
|
||||||
|
torch.testing.assert_close(
|
||||||
|
allocator.full_to_swa_index_mapping[old_full], new_swa
|
||||||
|
)
|
||||||
|
self.assertTrue(
|
||||||
|
torch.isin(old_swa, allocator.swa_attn_allocator.free_pages).item()
|
||||||
|
)
|
||||||
|
self.assertFalse(
|
||||||
|
torch.isin(new_swa, allocator.swa_attn_allocator.free_pages).item()
|
||||||
|
)
|
||||||
|
|
||||||
def test_swa_radix_cache_1(self):
|
def test_swa_radix_cache_1(self):
|
||||||
# args
|
# args
|
||||||
req_size = 10
|
req_size = 10
|
||||||
|
|||||||
Reference in New Issue
Block a user