[Fix] Respect cache_protected_len in ChunkCache and disabled-radix release paths (#31662)
This commit is contained in:
@@ -23,6 +23,7 @@ class _FakeAllocator:
|
||||
class _FakeReq:
|
||||
req_pool_idx = 0
|
||||
swa_evict_floor = 3
|
||||
cache_protected_len = 0
|
||||
kv = SimpleNamespace(swa_evicted_seqlen=6)
|
||||
|
||||
def pop_committed_kv_cache(self):
|
||||
@@ -30,12 +31,16 @@ class _FakeReq:
|
||||
|
||||
|
||||
class TestPureSWAChunkCache(CustomTestCase):
|
||||
def test_finished_req_skips_already_evicted_swa_range(self):
|
||||
def _make_cache(self):
|
||||
cache = PureSWAChunkCache.__new__(PureSWAChunkCache)
|
||||
cache.req_to_token_pool = SimpleNamespace(
|
||||
req_to_token=torch.arange(10, dtype=torch.int64).unsqueeze(0)
|
||||
)
|
||||
cache.token_to_kv_pool_allocator = _FakeAllocator()
|
||||
return cache
|
||||
|
||||
def test_finished_req_skips_already_evicted_swa_range(self):
|
||||
cache = self._make_cache()
|
||||
|
||||
cache.cache_finished_req(_FakeReq(), kv_len_to_handle=8)
|
||||
|
||||
@@ -43,6 +48,16 @@ class TestPureSWAChunkCache(CustomTestCase):
|
||||
freed = cache.token_to_kv_pool_allocator.freed[0]
|
||||
self.assertTrue(torch.equal(freed, torch.tensor([0, 1, 2, 6, 7])))
|
||||
|
||||
def test_finished_req_skips_protected_prefix(self):
|
||||
cache = self._make_cache()
|
||||
req = _FakeReq()
|
||||
req.cache_protected_len = 2
|
||||
|
||||
cache.cache_finished_req(req, kv_len_to_handle=8)
|
||||
|
||||
freed = cache.token_to_kv_pool_allocator.freed[0]
|
||||
self.assertTrue(torch.equal(freed, torch.tensor([2, 6, 7])))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user