[Fix] swa: Retain a whole window behind insertion pages (#43168)

Co-authored-by: Caihua Li <caihua.li@bytedance.com>
This commit is contained in:
caihuali95
2026-10-18 05:36:02 -07:00
committed by GitHub
co-authored by Caihua Li
parent fe1851c3c2
commit 47eeddcd18
2 changed files with 59 additions and 5 deletions
+6 -5
View File
@@ -109,11 +109,12 @@ def free_swa_out_of_window_slots(
# up to the window boundary (the trailing floor keeps it page-aligned).
evict_threshold = pre_len - sliding_window_size
else:
# Prefix-sharing cache: keep max(window, page). The trailing floor page-aligns the
# frontier, and subtracting at least one page keeps it below the insert
# boundary (page_floor(seq_len)) so the last leaf is never all-tombstone.
# No extra page margin is needed.
evict_threshold = pre_len - max(sliding_window_size, page_size)
# Prefix-sharing cache: keep max(window, page) rounded up to whole pages, so
# a key that ends at page_floor(pre_len) still has a full window of live SWA,
# and the last leaf, below page_floor(seq_len), is never all-tombstone.
evict_threshold = pre_len - ceil_align(
max(sliding_window_size, page_size), page_size
)
if retain_floor is not None and supports_prefix_sharing:
# The caller owns where the floor is (see BasePrefixCache.swa_retain_floor);
# this only promises not to free past it. Without prefix sharing a retained
@@ -11501,6 +11501,59 @@ class TestSWAWindowUnderBigramKey(CustomTestCase):
cache.unlock(req.lock)
cache.sanity_check()
def test_finish_after_decode_eviction_frees_only_the_requests_pages(self):
# Decode evicts at pre_len = seq_len - 1. When the last verify then commits
# one token, the finished insert ends at page_floor(pre_len): the frontier
# must sit a full window below it although the window is not page-aligned.
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
page_size = self.cfg.page_size
full_total = allocator.full_available_size()
swa_total = allocator.swa_available_size()
prompt_len, owned = 4 * page_size, 6 * page_size
tokens = list(range(1, owned + 1))
req = Req(
rid=0,
origin_input_text="",
origin_input_ids=array("q", tokens[:prompt_len]),
sampling_params=SamplingParams(
temperature=0, max_new_tokens=owned - prompt_len
),
)
req_to_token_pool.alloc([req])
req.output_ids = []
req.full_untruncated_fill_ids = array("q", tokens[:prompt_len])
req.extend_end = prompt_len
kv_indices = self._alloc_paged(allocator, owned)
req_to_token_pool.write((req.kv.req_pool_idx, slice(0, owned)), kv_indices)
req.kv.kv_committed_len = prompt_len
req.kv.kv_allocated_len = owned
req.last_node = cache.root_node_handle()
req.kv.cache_protected_len = 0
req.lock = None
req.extra_key = None
cache.checkpoint(req, up_to=prompt_len)
req.output_ids = tokens[prompt_len:]
req.kv.kv_committed_len = owned
cache.evict_sliding_windows(req, owned - 1)
insert_boundary = (owned - 1) // page_size * page_size
self.assertGreaterEqual(
insert_boundary - req.kv.get_evicted_seqlen(ComponentType.SWA),
self.cfg.sliding_window_size,
)
req.finished_reason = FINISH_LENGTH(length=len(req.output_ids))
release_kv_cache(req, cache, checkpoint=True)
# Each page is back in the pool or held by the tree, never both.
self.assertEqual(
allocator.full_available_size() + cache.full_evictable_size(), full_total
)
self.assertEqual(
allocator.swa_available_size() + cache.swa_evictable_size(), swa_total
)
cache.sanity_check()
class TestUnifiedRadixCacheStorageAttachBackfill(CustomTestCase):
"""Enabling a storage backend must hash nodes that predate it.