diff --git a/python/sglang/srt/mem_cache/base_prefix_cache.py b/python/sglang/srt/mem_cache/base_prefix_cache.py index 0d50b61711..f1992b095a 100644 --- a/python/sglang/srt/mem_cache/base_prefix_cache.py +++ b/python/sglang/srt/mem_cache/base_prefix_cache.py @@ -441,13 +441,15 @@ class BasePrefixCache(ABC, PrefixCacheTrait): """ from sglang.srt.mem_cache.common import coalesce_ranges, free_kv_row_segments + allocator = self.token_to_kv_pool_allocator row = self.req_to_token_pool.req_to_token[kv.req_pool_idx] # Adjacent pieces whose seam falls inside one (DCP-widened) page would # free that page twice; the allocator rejects that, so merge them first. free_kv_row_segments( - self.token_to_kv_pool_allocator, + allocator, [(row[start:end], start) for start, end in coalesce_ranges(ranges)], swa_evicted_seqlen=kv.swa_evicted_seqlen, + swa_dead_lo=kv.swa_dead_lo(allocator.page_size), ) @abstractmethod diff --git a/python/sglang/srt/mem_cache/chunk_cache.py b/python/sglang/srt/mem_cache/chunk_cache.py index 8c005ac390..d608ecdab4 100644 --- a/python/sglang/srt/mem_cache/chunk_cache.py +++ b/python/sglang/srt/mem_cache/chunk_cache.py @@ -137,38 +137,5 @@ class SWAChunkCache(ChunkCache): class PureSWAChunkCache(SWAChunkCache): - """ChunkCache for all-SWA models (no full attention layers). - - For hybrid models, full_to_swa_index_mapping prevents SWA double-free. - All-SWA models lack this mapping, so on request completion we must - explicitly skip the range already freed by ``free_swa_out_of_window_slots`` - (a.k.a. _evict_swa) during decode. - - ``req.kv.swa_evict_floor`` shields the prompt/image KV from window eviction - only while the request is active, so that range IS released here on - finish. Distinct from the ``cache_protected_len`` prefix, which is owned - elsewhere and never freed by this path. - """ - - def cache_finished_req( - self, req: Req, is_insert: bool = True, *, owned_kv_len: int - ): - kv_indices = self.req_to_token_pool.req_to_token[ - req.kv.req_pool_idx, :owned_kv_len - ] - # The cache_protected_len prefix is not this req's to free. - protected_len = req.kv.cache_protected_len - evict_floor = req.kv.swa_evict_floor - evicted_seqlen = req.kv.swa_evicted_seqlen - if evicted_seqlen > evict_floor: - parts = [] - if evict_floor > protected_len: - parts.append(kv_indices[protected_len:evict_floor]) - if evicted_seqlen < owned_kv_len: - parts.append( - kv_indices[max(evicted_seqlen, protected_len) : owned_kv_len] - ) - if parts: - self.token_to_kv_pool_allocator.free(torch.cat(parts)) - else: - self.token_to_kv_pool_allocator.free(kv_indices[protected_len:]) + """ChunkCache for all-SWA models (no full attention layers): no + full_to_swa_index_mapping, so free_kv_row must skip the window-evicted span.""" diff --git a/python/sglang/srt/mem_cache/common.py b/python/sglang/srt/mem_cache/common.py index e4c5b1c565..863111c408 100644 --- a/python/sglang/srt/mem_cache/common.py +++ b/python/sglang/srt/mem_cache/common.py @@ -123,28 +123,34 @@ def free_kv_row_segments( segments: list[tuple[torch.Tensor, int]], *, swa_evicted_seqlen: int, + swa_dead_lo: int = 0, ) -> None: """Free ascending disjoint ``(kv_indices, start_pos)`` segments of one - request's kv row, split at the SWA eviction floor.""" + request's kv row; ``[swa_dead_lo, swa_evicted_seqlen)`` goes back full-side only.""" + dead_lo, dead_hi = swa_dead_lo, max(swa_evicted_seqlen, swa_dead_lo) swa_dead: list[tuple[torch.Tensor, int]] = [] swa_alive: list[tuple[torch.Tensor, int]] = [] for kv_indices, start_pos in segments: - num_indices = kv_indices.numel() - if num_indices == 0: + end_pos = start_pos + kv_indices.numel() + lo = min(max(dead_lo, start_pos), end_pos) + hi = min(max(dead_hi, start_pos), end_pos) + # start_pos <= lo <= hi <= end_pos + # inside [lo, hi) is the swa dead segment + if hi <= lo: + swa_alive.append((kv_indices, start_pos)) continue - # Below the floor the SWA peers are already gone -- window eviction, or - # the deliberately unmapped prefix of a PD decode SWA-tail prealloc. - num_dead = min(max(swa_evicted_seqlen - start_pos, 0), num_indices) - if num_dead > 0: - swa_dead.append((kv_indices[:num_dead], start_pos)) - if num_dead < num_indices: - swa_alive.append((kv_indices[num_dead:], start_pos + num_dead)) + swa_dead.append((kv_indices[lo - start_pos : hi - start_pos], lo)) + + if lo > start_pos: + swa_alive.append((kv_indices[: lo - start_pos], start_pos)) + if end_pos > hi: + swa_alive.append((kv_indices[hi - start_pos :], hi)) if swa_dead and swa_alive: # The two sides are separate calls, so neither one's page-disjointness - # check sees a floor that splits a page between them. + # check sees a boundary that splits a page between them. assert swa_evicted_seqlen % allocator.page_size == 0, ( - f"SWA eviction floor {swa_evicted_seqlen} splits a page " + f"SWA eviction cursor {swa_evicted_seqlen} splits a page " f"(page_size {allocator.page_size})" ) if swa_dead: diff --git a/python/sglang/srt/mem_cache/pure_swa_radix_cache.py b/python/sglang/srt/mem_cache/pure_swa_radix_cache.py index 9926f5aa08..96f600d766 100644 --- a/python/sglang/srt/mem_cache/pure_swa_radix_cache.py +++ b/python/sglang/srt/mem_cache/pure_swa_radix_cache.py @@ -65,18 +65,10 @@ class PureSWARadixCache(RadixCache): def cache_finished_req( self, req: Req, is_insert: bool = True, *, owned_kv_len: int ): - """Cache request when it finishes. - - Only inserts the prefill portion [0, evict_floor) into the radix tree. - The window portion [swa_evicted_seqlen, committed_len) is freed back - to the allocator. The range [evict_floor, swa_evicted_seqlen) was already - freed by _evict_swa during decode — we skip it to avoid double-free. - """ + """Insert only the prefill portion [0, evict_floor); free_kv_row skips + the span _evict_swa already freed during decode.""" if self.disable: - kv_indices = self.req_to_token_pool.req_to_token[ - req.kv.req_pool_idx, :owned_kv_len - ] - self.token_to_kv_pool_allocator.free(kv_indices) + self.free_kv_row(req.kv, [(req.kv.cache_protected_len, owned_kv_len)]) return token_ids = (req.origin_input_ids + req.output_ids)[:owned_kv_len] @@ -94,42 +86,26 @@ class PureSWARadixCache(RadixCache): old_prefix_len = req.kv.cache_protected_len swa_evict_floor = req.kv.swa_evict_floor - swa_evicted_seqlen = req.kv.swa_evicted_seqlen - if self.page_size > 1 and swa_evict_floor > 0: swa_evict_floor = -(-swa_evict_floor // self.page_size) * self.page_size - if swa_evict_floor > 0: insert_end = min(swa_evict_floor, keys_len) else: insert_end = keys_len + release_from = old_prefix_len if is_insert and insert_end > 0: insert_values = kv_indices[:insert_end].to(dtype=torch.int64, copy=True) result = self.insert( InsertParams(key=radix_key[:insert_end], value=insert_values) ) - new_prefix_len = result.prefix_len - if new_prefix_len > old_prefix_len: - self.token_to_kv_pool_allocator.free( - kv_indices[old_prefix_len:new_prefix_len] - ) - alive_start = max(swa_evicted_seqlen, insert_end) - if alive_start < keys_len: - self.token_to_kv_pool_allocator.free(kv_indices[alive_start:keys_len]) - else: - free_end = ( - min(swa_evict_floor, keys_len) if swa_evict_floor > 0 else keys_len + self.token_to_kv_pool_allocator.free_segment( + kv_indices[old_prefix_len : result.prefix_len], + start_pos=old_prefix_len, ) - if free_end > old_prefix_len: - self.token_to_kv_pool_allocator.free( - kv_indices[old_prefix_len:free_end] - ) - alive_start = max(swa_evicted_seqlen, free_end) - if alive_start < keys_len: - self.token_to_kv_pool_allocator.free(kv_indices[alive_start:keys_len]) + release_from = insert_end - self.token_to_kv_pool_allocator.free(kv_indices[keys_len:]) + self.free_kv_row(req.kv, [(release_from, owned_kv_len)]) if req.last_node is not None: self.dec_lock_ref(req.last_node) diff --git a/test/registered/unit/mem_cache/test_free_kv_row_coalesce.py b/test/registered/unit/mem_cache/test_free_kv_row_coalesce.py index 74e9e2e0fd..9f1c4fa153 100644 --- a/test/registered/unit/mem_cache/test_free_kv_row_coalesce.py +++ b/test/registered/unit/mem_cache/test_free_kv_row_coalesce.py @@ -13,6 +13,7 @@ from types import SimpleNamespace import torch +from sglang.srt.managers.schedule_batch import ReqKvInfo from sglang.srt.mem_cache.allocator.paged import PagedTokenToKVPoolAllocator from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache from sglang.test.ci.ci_register import register_cpu_ci @@ -66,7 +67,7 @@ def _make_cache_with_row(num_tokens): row = alloc.alloc(num_tokens) req_to_token = torch.zeros((1, num_tokens), dtype=row.dtype) req_to_token[0] = row - kv = SimpleNamespace(req_pool_idx=0, swa_evicted_seqlen=0) + kv = ReqKvInfo(req_pool_idx=0) return _RowCache(alloc, req_to_token), alloc, row, kv diff --git a/test/registered/unit/mem_cache/test_page_interleave_shard.py b/test/registered/unit/mem_cache/test_page_interleave_shard.py index 90729d6c49..8a9ca91888 100644 --- a/test/registered/unit/mem_cache/test_page_interleave_shard.py +++ b/test/registered/unit/mem_cache/test_page_interleave_shard.py @@ -53,6 +53,7 @@ from sglang.srt.distributed import ( init_distributed_environment, initialize_model_parallel, ) +from sglang.srt.managers.schedule_batch import ReqKvInfo from sglang.srt.mem_cache import page_interleave from sglang.srt.mem_cache.allocator.page_interleave import ( PageInterleavePoolAllocator, @@ -766,11 +767,7 @@ class _GraftReq: self.fill_ids = list(fill_ids) self.origin_input_ids = array("q", fill_ids) self.output_ids = array("q", []) - self.kv = SimpleNamespace( - req_pool_idx=req_pool_idx, - cache_protected_len=0, - swa_evicted_seqlen=0, - ) + self.kv = ReqKvInfo(req_pool_idx=req_pool_idx) self.extra_key = None self.cache_salt = None self.prefix_indices = torch.empty(0, dtype=torch.int64) diff --git a/test/registered/unit/mem_cache/test_pure_swa_chunk_cache.py b/test/registered/unit/mem_cache/test_pure_swa_chunk_cache.py index 6fd818a16f..2d1cd97c86 100644 --- a/test/registered/unit/mem_cache/test_pure_swa_chunk_cache.py +++ b/test/registered/unit/mem_cache/test_pure_swa_chunk_cache.py @@ -5,6 +5,7 @@ from types import SimpleNamespace import torch +from sglang.srt.managers.schedule_batch import ReqKvInfo from sglang.srt.mem_cache.chunk_cache import PureSWAChunkCache from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -13,24 +14,35 @@ register_cpu_ci(est_time=10, suite="base-a-test-cpu") class _FakeAllocator: + """Rows routed to the full side are skipped: all-SWA has no full pool.""" + + page_size = 1 + def __init__(self): self.freed = [] + self.skipped = [] - def free(self, indices): - self.freed.append(indices.detach().cpu().clone()) + def free_segment(self, indices, *, start_pos): + self.freed.extend(indices.tolist()) + + def free_segments(self, segments): + for indices, start_pos in segments: + self.free_segment(indices, start_pos=start_pos) + + def free_full_segments(self, segments): + for indices, _ in segments: + self.skipped.extend(indices.tolist()) -class _FakeReq: - def __init__(self): - self.kv = SimpleNamespace( +def _make_req(): + return SimpleNamespace( + kv=ReqKvInfo( req_pool_idx=0, - swa_evicted_seqlen=6, + cache_protected_len=2, swa_evict_floor=3, - cache_protected_len=0, + swa_evicted_seqlen=6, ) - - def pop_committed_kv_cache(self): - return 8 + ) class TestPureSWAChunkCache(CustomTestCase): @@ -42,24 +54,14 @@ class TestPureSWAChunkCache(CustomTestCase): cache.token_to_kv_pool_allocator = _FakeAllocator() return cache - def test_finished_req_skips_already_evicted_swa_range(self): + def test_finished_req_skips_protected_prefix_and_evicted_range(self): cache = self._make_cache() - cache.cache_finished_req(_FakeReq(), owned_kv_len=8) + # protected 2, floor 3, cursor 6: [2, 3) and [6, 8) go back, [3, 6) is dead + cache.cache_finished_req(_make_req(), owned_kv_len=8) - self.assertEqual(len(cache.token_to_kv_pool_allocator.freed), 1) - freed = cache.token_to_kv_pool_allocator.freed[0] - self.assertTrue(torch.equal(freed, torch.tensor([0, 1, 2, 6, 7]))) - - def test_finished_req_skips_protected_prefix(self): - cache = self._make_cache() - req = _FakeReq() - req.kv.cache_protected_len = 2 - - cache.cache_finished_req(req, owned_kv_len=8) - - freed = cache.token_to_kv_pool_allocator.freed[0] - self.assertTrue(torch.equal(freed, torch.tensor([2, 6, 7]))) + self.assertEqual(cache.token_to_kv_pool_allocator.freed, [2, 6, 7]) + self.assertEqual(cache.token_to_kv_pool_allocator.skipped, [3, 4, 5]) if __name__ == "__main__": diff --git a/test/registered/unit/mem_cache/test_pure_swa_radix_cache.py b/test/registered/unit/mem_cache/test_pure_swa_radix_cache.py index 25d3911cf3..c4fdab8bdb 100644 --- a/test/registered/unit/mem_cache/test_pure_swa_radix_cache.py +++ b/test/registered/unit/mem_cache/test_pure_swa_radix_cache.py @@ -1,11 +1,16 @@ """Unit tests for all-SWA RadixCache release semantics.""" import unittest +from array import array from types import SimpleNamespace import torch +from sglang.srt.managers.schedule_batch import ReqKvInfo +from sglang.srt.mem_cache.base_prefix_cache import MatchPrefixParams +from sglang.srt.mem_cache.cache_init_params import CacheInitParams from sglang.srt.mem_cache.pure_swa_radix_cache import PureSWARadixCache +from sglang.srt.mem_cache.radix_cache import RadixKey from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -13,43 +18,59 @@ register_cpu_ci(est_time=10, suite="base-a-test-cpu") class _FakeAllocator: + """Rows routed to the full side are skipped: all-SWA has no full pool.""" + + page_size = 1 + device = "cpu" + def __init__(self): self.freed = [] + self.skipped = [] - def free(self, indices): - if indices.numel() == 0: - return - self.freed.append(indices.detach().cpu().tolist()) + def free_segment(self, indices, *, start_pos): + self.freed.extend(indices.tolist()) + + def free_segments(self, segments): + for indices, start_pos in segments: + self.free_segment(indices, start_pos=start_pos) + + def free_full_segments(self, segments): + for indices, _ in segments: + self.skipped.extend(indices.tolist()) class TestPureSWARadixCache(CustomTestCase): - def test_no_insert_frees_window_after_evict_floor_before_swa_eviction(self): + def test_finish_inserts_up_to_the_evict_floor_and_frees_the_rest(self): allocator = _FakeAllocator() - cache = PureSWARadixCache.__new__(PureSWARadixCache) - cache.disable = False - cache.is_eagle = False - cache.page_size = 1 - cache.req_to_token_pool = SimpleNamespace( - req_to_token=torch.arange(8, dtype=torch.int64).unsqueeze(0) + cache = PureSWARadixCache( + CacheInitParams( + disable=False, + req_to_token_pool=SimpleNamespace( + req_to_token=torch.arange(8, dtype=torch.int64).unsqueeze(0) + ), + token_to_kv_pool_allocator=allocator, + page_size=1, + sliding_window_size=4, + ) ) - cache.token_to_kv_pool_allocator = allocator + token_ids = array("q", range(8)) req = SimpleNamespace( - origin_input_ids=list(range(8)), - output_ids=[], + origin_input_ids=token_ids, + output_ids=array("q"), extra_key=None, cache_salt=None, last_node=None, - kv=SimpleNamespace( - req_pool_idx=0, - cache_protected_len=0, - swa_evict_floor=4, - swa_evicted_seqlen=0, - ), + priority=0, + kv=ReqKvInfo(req_pool_idx=0, swa_evict_floor=4, swa_evicted_seqlen=6), ) - cache.cache_finished_req(req, is_insert=False, owned_kv_len=8) + cache.cache_finished_req(req, is_insert=True, owned_kv_len=8) - self.assertEqual(allocator.freed, [[0, 1, 2, 3], [4, 5, 6, 7]]) + # [0, 4) went into the tree; [4, 6) was window-evicted; [6, 8) is freed. + match = cache.match_prefix(MatchPrefixParams(key=RadixKey(token_ids))) + self.assertEqual(len(match.device_indices), 4) + self.assertEqual(allocator.freed, [6, 7]) + self.assertEqual(allocator.skipped, [4, 5]) if __name__ == "__main__": diff --git a/test/registered/unit/mem_cache/test_swa_unittest.py b/test/registered/unit/mem_cache/test_swa_unittest.py index 863ceb11c8..7e6b9dd45e 100644 --- a/test/registered/unit/mem_cache/test_swa_unittest.py +++ b/test/registered/unit/mem_cache/test_swa_unittest.py @@ -6,6 +6,7 @@ from unittest.mock import patch import torch from sglang.srt.environ import InvariantCheckLevel, envs +from sglang.srt.managers.schedule_batch import ReqKvInfo from sglang.srt.mem_cache.allocator.base import BaseTokenToKVPoolAllocator from sglang.srt.mem_cache.allocator.swa import ( PureSWATokenToKVPoolAllocator, @@ -483,7 +484,7 @@ class TestFreeKvRow(CustomTestCase): def test_free_kv_row_reads_the_record_row_and_its_floor(self): indices = _swa_alloc(self.allocator, 8) cache = _RowCache(self.allocator, indices) - kv = SimpleNamespace(req_pool_idx=0, swa_evicted_seqlen=3) + kv = ReqKvInfo(req_pool_idx=0, swa_evicted_seqlen=3) self.allocator.free_swa(indices[:3]) cache.free_kv_row(kv, [(1, 5)]) @@ -495,7 +496,7 @@ class TestFreeKvRow(CustomTestCase): def test_single_pool_free_kv_row_still_frees_the_whole_range(self): allocator = _SinglePoolAllocator() cache = _RowCache(allocator, torch.arange(16, dtype=torch.int64)) - kv = SimpleNamespace(req_pool_idx=0, swa_evicted_seqlen=4) + kv = ReqKvInfo(req_pool_idx=0, swa_evicted_seqlen=4) cache.free_kv_row(kv, [(2, 6)]) @@ -506,6 +507,20 @@ class TestFreeKvRow(CustomTestCase): cache.free_kv_row(kv, [(6, 6)]) self.assertEqual(len(allocator.freed), 2) + def test_rows_below_the_dead_floor_are_still_alive(self): + # all-SWA: only [swa_dead_lo, cursor) is dead; [start, swa_dead_lo) is alive + allocator = _build_pure_swa_allocator(size_swa=16) + baseline = allocator.swa_available_size() + indices = _swa_alloc(allocator, 8) + allocator.free_swa(indices[4:6]) + self.assertEqual(allocator.swa_available_size(), baseline - 6) + + cache = _RowCache(allocator, indices) + kv = ReqKvInfo(req_pool_idx=0, swa_evicted_seqlen=6, swa_evict_floor=4) + cache.free_kv_row(kv, [(2, 8)]) + + self.assertEqual(allocator.swa_available_size(), baseline - 2) + class TestSWAPeerMappedContract(CustomTestCase): """page_size 1 gives back every peer the mapping names, without filtering: