From 47eeddcd180e2b5228ea35dae7a0ff327c5d07ab Mon Sep 17 00:00:00 2001 From: caihuali95 <42954765+caihuali95@users.noreply.github.com> Date: Sun, 18 Oct 2026 05:36:02 -0700 Subject: [PATCH] [Fix] swa: Retain a whole window behind insertion pages (#43168) Co-authored-by: Caihua Li --- python/sglang/srt/mem_cache/common.py | 11 ++-- .../test_unified_radix_cache_unittest.py | 53 +++++++++++++++++++ 2 files changed, 59 insertions(+), 5 deletions(-) diff --git a/python/sglang/srt/mem_cache/common.py b/python/sglang/srt/mem_cache/common.py index 27e0a961a5..b556dad8ab 100644 --- a/python/sglang/srt/mem_cache/common.py +++ b/python/sglang/srt/mem_cache/common.py @@ -109,11 +109,12 @@ def free_swa_out_of_window_slots( # up to the window boundary (the trailing floor keeps it page-aligned). evict_threshold = pre_len - sliding_window_size else: - # Prefix-sharing cache: keep max(window, page). The trailing floor page-aligns the - # frontier, and subtracting at least one page keeps it below the insert - # boundary (page_floor(seq_len)) so the last leaf is never all-tombstone. - # No extra page margin is needed. - evict_threshold = pre_len - max(sliding_window_size, page_size) + # Prefix-sharing cache: keep max(window, page) rounded up to whole pages, so + # a key that ends at page_floor(pre_len) still has a full window of live SWA, + # and the last leaf, below page_floor(seq_len), is never all-tombstone. + evict_threshold = pre_len - ceil_align( + max(sliding_window_size, page_size), page_size + ) if retain_floor is not None and supports_prefix_sharing: # The caller owns where the floor is (see BasePrefixCache.swa_retain_floor); # this only promises not to free past it. Without prefix sharing a retained diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py index f4eaf35d9c..aeed32db9d 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py +++ b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py @@ -11501,6 +11501,59 @@ class TestSWAWindowUnderBigramKey(CustomTestCase): cache.unlock(req.lock) cache.sanity_check() + def test_finish_after_decode_eviction_frees_only_the_requests_pages(self): + # Decode evicts at pre_len = seq_len - 1. When the last verify then commits + # one token, the finished insert ends at page_floor(pre_len): the frontier + # must sit a full window below it although the window is not page-aligned. + cache, allocator, req_to_token_pool = build_fixture(self.cfg) + page_size = self.cfg.page_size + full_total = allocator.full_available_size() + swa_total = allocator.swa_available_size() + prompt_len, owned = 4 * page_size, 6 * page_size + tokens = list(range(1, owned + 1)) + + req = Req( + rid=0, + origin_input_text="", + origin_input_ids=array("q", tokens[:prompt_len]), + sampling_params=SamplingParams( + temperature=0, max_new_tokens=owned - prompt_len + ), + ) + req_to_token_pool.alloc([req]) + req.output_ids = [] + req.full_untruncated_fill_ids = array("q", tokens[:prompt_len]) + req.extend_end = prompt_len + kv_indices = self._alloc_paged(allocator, owned) + req_to_token_pool.write((req.kv.req_pool_idx, slice(0, owned)), kv_indices) + req.kv.kv_committed_len = prompt_len + req.kv.kv_allocated_len = owned + req.last_node = cache.root_node_handle() + req.kv.cache_protected_len = 0 + req.lock = None + req.extra_key = None + cache.checkpoint(req, up_to=prompt_len) + + req.output_ids = tokens[prompt_len:] + req.kv.kv_committed_len = owned + cache.evict_sliding_windows(req, owned - 1) + insert_boundary = (owned - 1) // page_size * page_size + self.assertGreaterEqual( + insert_boundary - req.kv.get_evicted_seqlen(ComponentType.SWA), + self.cfg.sliding_window_size, + ) + req.finished_reason = FINISH_LENGTH(length=len(req.output_ids)) + release_kv_cache(req, cache, checkpoint=True) + + # Each page is back in the pool or held by the tree, never both. + self.assertEqual( + allocator.full_available_size() + cache.full_evictable_size(), full_total + ) + self.assertEqual( + allocator.swa_available_size() + cache.swa_evictable_size(), swa_total + ) + cache.sanity_check() + class TestUnifiedRadixCacheStorageAttachBackfill(CustomTestCase): """Enabling a storage backend must hash nodes that predate it.