[Fix] swa: Retain a whole window behind insertion pages (#43168)
Co-authored-by: Caihua Li <caihua.li@bytedance.com>
This commit is contained in:
@@ -109,11 +109,12 @@ def free_swa_out_of_window_slots(
|
||||
# up to the window boundary (the trailing floor keeps it page-aligned).
|
||||
evict_threshold = pre_len - sliding_window_size
|
||||
else:
|
||||
# Prefix-sharing cache: keep max(window, page). The trailing floor page-aligns the
|
||||
# frontier, and subtracting at least one page keeps it below the insert
|
||||
# boundary (page_floor(seq_len)) so the last leaf is never all-tombstone.
|
||||
# No extra page margin is needed.
|
||||
evict_threshold = pre_len - max(sliding_window_size, page_size)
|
||||
# Prefix-sharing cache: keep max(window, page) rounded up to whole pages, so
|
||||
# a key that ends at page_floor(pre_len) still has a full window of live SWA,
|
||||
# and the last leaf, below page_floor(seq_len), is never all-tombstone.
|
||||
evict_threshold = pre_len - ceil_align(
|
||||
max(sliding_window_size, page_size), page_size
|
||||
)
|
||||
if retain_floor is not None and supports_prefix_sharing:
|
||||
# The caller owns where the floor is (see BasePrefixCache.swa_retain_floor);
|
||||
# this only promises not to free past it. Without prefix sharing a retained
|
||||
|
||||
@@ -11501,6 +11501,59 @@ class TestSWAWindowUnderBigramKey(CustomTestCase):
|
||||
cache.unlock(req.lock)
|
||||
cache.sanity_check()
|
||||
|
||||
def test_finish_after_decode_eviction_frees_only_the_requests_pages(self):
|
||||
# Decode evicts at pre_len = seq_len - 1. When the last verify then commits
|
||||
# one token, the finished insert ends at page_floor(pre_len): the frontier
|
||||
# must sit a full window below it although the window is not page-aligned.
|
||||
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
|
||||
page_size = self.cfg.page_size
|
||||
full_total = allocator.full_available_size()
|
||||
swa_total = allocator.swa_available_size()
|
||||
prompt_len, owned = 4 * page_size, 6 * page_size
|
||||
tokens = list(range(1, owned + 1))
|
||||
|
||||
req = Req(
|
||||
rid=0,
|
||||
origin_input_text="",
|
||||
origin_input_ids=array("q", tokens[:prompt_len]),
|
||||
sampling_params=SamplingParams(
|
||||
temperature=0, max_new_tokens=owned - prompt_len
|
||||
),
|
||||
)
|
||||
req_to_token_pool.alloc([req])
|
||||
req.output_ids = []
|
||||
req.full_untruncated_fill_ids = array("q", tokens[:prompt_len])
|
||||
req.extend_end = prompt_len
|
||||
kv_indices = self._alloc_paged(allocator, owned)
|
||||
req_to_token_pool.write((req.kv.req_pool_idx, slice(0, owned)), kv_indices)
|
||||
req.kv.kv_committed_len = prompt_len
|
||||
req.kv.kv_allocated_len = owned
|
||||
req.last_node = cache.root_node_handle()
|
||||
req.kv.cache_protected_len = 0
|
||||
req.lock = None
|
||||
req.extra_key = None
|
||||
cache.checkpoint(req, up_to=prompt_len)
|
||||
|
||||
req.output_ids = tokens[prompt_len:]
|
||||
req.kv.kv_committed_len = owned
|
||||
cache.evict_sliding_windows(req, owned - 1)
|
||||
insert_boundary = (owned - 1) // page_size * page_size
|
||||
self.assertGreaterEqual(
|
||||
insert_boundary - req.kv.get_evicted_seqlen(ComponentType.SWA),
|
||||
self.cfg.sliding_window_size,
|
||||
)
|
||||
req.finished_reason = FINISH_LENGTH(length=len(req.output_ids))
|
||||
release_kv_cache(req, cache, checkpoint=True)
|
||||
|
||||
# Each page is back in the pool or held by the tree, never both.
|
||||
self.assertEqual(
|
||||
allocator.full_available_size() + cache.full_evictable_size(), full_total
|
||||
)
|
||||
self.assertEqual(
|
||||
allocator.swa_available_size() + cache.swa_evictable_size(), swa_total
|
||||
)
|
||||
cache.sanity_check()
|
||||
|
||||
|
||||
class TestUnifiedRadixCacheStorageAttachBackfill(CustomTestCase):
|
||||
"""Enabling a storage backend must hash nodes that predate it.
|
||||
|
||||
Reference in New Issue
Block a user