diff --git a/python/sglang/srt/mem_cache/base_prefix_cache.py b/python/sglang/srt/mem_cache/base_prefix_cache.py index cfa94d5677..ea72c287f4 100644 --- a/python/sglang/srt/mem_cache/base_prefix_cache.py +++ b/python/sglang/srt/mem_cache/base_prefix_cache.py @@ -703,6 +703,7 @@ class BasePrefixCache(ABC, PrefixCacheTrait): supports_prefix_sharing=self.supports_prefix_sharing(), retain_floor=self.swa_retain_floor(req), eviction_interval=eviction_interval, + reprefill_tail_tokens=self.swa_reprefill_tail_tokens(req), ) def swa_retain_floor(self, req) -> int | None: diff --git a/python/sglang/srt/mem_cache/common.py b/python/sglang/srt/mem_cache/common.py index b556dad8ab..c4aa18e6f5 100644 --- a/python/sglang/srt/mem_cache/common.py +++ b/python/sglang/srt/mem_cache/common.py @@ -84,6 +84,7 @@ def free_swa_out_of_window_slots( component_type: ComponentType = ComponentType.SWA, free_segment: Callable[..., None] | None = None, eviction_interval: int = 1, + reprefill_tail_tokens: int = 0, ) -> None: if not req.kv.holds_kv: return @@ -115,6 +116,14 @@ def free_swa_out_of_window_slots( evict_threshold = pre_len - ceil_align( max(sliding_window_size, page_size), page_size ) + if reprefill_tail_tokens: + # Matching excludes the replay tail before page alignment. Keep the + # SWA window behind that earlier boundary too, or a first replay + # falls back to an older checkpoint despite a Full-KV cache hit. + replay_boundary = (pre_len - reprefill_tail_tokens) // page_size * page_size + evict_threshold = min( + evict_threshold, replay_boundary - sliding_window_size + ) if retain_floor is not None and supports_prefix_sharing: # The caller owns where the floor is (see BasePrefixCache.swa_retain_floor); # this only promises not to free past it. Without prefix sharing a retained diff --git a/python/sglang/srt/mem_cache/unified_cache/components/swa.py b/python/sglang/srt/mem_cache/unified_cache/components/swa.py index 0850925634..6416a2e662 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/swa.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/swa.py @@ -1046,6 +1046,7 @@ class SWAComponent(TreeComponent): req_to_token_pool=self.cache.req_to_token_pool, token_to_kv_pool_allocator=self.cache.token_to_kv_pool_allocator, retain_floor=self.cache.swa_retain_floor(req), + reprefill_tail_tokens=self.cache.swa_reprefill_tail_tokens(req), ) def free_out_of_window_slots( diff --git a/test/registered/unit/mem_cache/test_swa_replay_retention.py b/test/registered/unit/mem_cache/test_swa_replay_retention.py new file mode 100644 index 0000000000..16e048089a --- /dev/null +++ b/test/registered/unit/mem_cache/test_swa_replay_retention.py @@ -0,0 +1,155 @@ +"""Exercise SWA eviction frontiers without importing CUDA or model weights.""" + +import ast +import importlib.util +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock + +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +class ReplayRetentionTest(CustomTestCase): + @classmethod + def setUpClass(cls): + package = importlib.util.find_spec("sglang") + cls.source = ( + Path(next(iter(package.submodule_search_locations))) / "srt/mem_cache" + ) + tree = ast.parse((cls.source / "common.py").read_text()) + function = next( + node + for node in tree.body + if isinstance(node, ast.FunctionDef) + and node.name == "free_swa_out_of_window_slots" + ) + module = ast.Module( + body=[ + ast.ImportFrom( + module="__future__", names=[ast.alias(name="annotations")], level=0 + ), + function, + ], + type_ignores=[], + ) + ast.fix_missing_locations(module) + namespace = { + "ComponentType": SimpleNamespace(SWA="swa"), + "ceil_align": lambda n, alignment: ( + (n + alignment - 1) // alignment * alignment + ), + } + exec(compile(module, "swa-replay-retention", "exec"), namespace) + cls.free_window = staticmethod(namespace["free_swa_out_of_window_slots"]) + + def frontier( + self, + length, + window=128, + page=256, + replay=128, + sharing=True, + retain_floor=None, + previous=0, + ): + kv = SimpleNamespace( + holds_kv=True, + cache_protected_len=0, + req_pool_idx=0, + swa_dead_lo=lambda _: 0, + get_evicted_seqlen=lambda _: previous, + ) + kv.set_evicted_seqlen = lambda _, value: setattr(kv, "evicted", value) + table = Mock() + table.__getitem__ = Mock(return_value=object()) + allocator = SimpleNamespace(free_swa_segment=Mock()) + self.free_window( + SimpleNamespace(kv=kv), + length - 1, + sliding_window_size=window, + page_size=page, + req_to_token_pool=SimpleNamespace(req_to_token=table), + token_to_kv_pool_allocator=allocator, + supports_prefix_sharing=sharing, + retain_floor=retain_floor, + reprefill_tail_tokens=replay, + ) + return kv.evicted + + def test_first_replay_keeps_window_before_trimmed_match(self): + for length in (32576, 32448, 269913, 1029986): + with self.subTest(length=length): + boundary = (length - 128) // 256 * 256 + self.assertLessEqual(self.frontier(length), boundary - 128) + + def test_all_page_alignments_preserve_replay_history(self): + for page, window, replay in ( + (256, 128, 128), + (64, 128, 128), + (16, 8, 8), + (1, 4, 4), + ): + for remainder in range(page): + length = 64 * page + 2 * window + remainder + with self.subTest(page=page, window=window, remainder=remainder): + boundary = (length - replay) // page * page + self.assertLessEqual( + self.frontier(length, window, page, replay), boundary - window + ) + + def test_no_replay_keeps_existing_eviction_policy(self): + for length in range(32000, 32512): + expected = (length - 1 - 256) // 256 * 256 + self.assertEqual(self.frontier(length, replay=0), expected) + + def test_256_token_pages_retain_at_most_one_extra_page(self): + for length in range(32000, 32512): + before = self.frontier(length, replay=0) + after = self.frontier(length) + self.assertGreaterEqual(before - after, 0) + self.assertLessEqual(before - after, 256) + + def test_bad_alignment_exposes_original_frontier_mismatch(self): + self.assertEqual(self.frontier(32576, replay=0), 32256) + self.assertEqual(self.frontier(32576), 32000) + self.assertEqual(self.frontier(32448), 32000) + + def test_disabled_prefix_sharing_does_not_retain_replay_history(self): + for replay in (0, 128): + self.assertEqual(self.frontier(32576, replay=replay, sharing=False), 32256) + + def test_checkpoint_floor_and_eviction_cursor_still_apply(self): + self.assertEqual(self.frontier(32576, retain_floor=30000), 29952) + # Previously freed slots cannot be recovered by changing a frontier. + self.assertEqual(self.frontier(32576, previous=32256), 32256) + + def test_scheduler_and_checkpoint_paths_pass_the_replay_guard(self): + for relative, expected in ( + ("base_prefix_cache.py", "self.swa_reprefill_tail_tokens(req)"), + ( + "unified_cache/components/swa.py", + "self.cache.swa_reprefill_tail_tokens(req)", + ), + ): + tree = ast.parse((self.source / relative).read_text()) + calls = [ + node + for node in ast.walk(tree) + if isinstance(node, ast.Call) + and ast.unparse(node.func) == "free_swa_out_of_window_slots" + ] + self.assertEqual(len(calls), 1) + keyword = next( + (kw for kw in calls[0].keywords if kw.arg == "reprefill_tail_tokens"), + None, + ) + self.assertIsNotNone(keyword, relative) + self.assertEqual(ast.unparse(keyword.value), expected) + + +if __name__ == "__main__": + unittest.main()