From 17505e6cf692cc47b4dd3a52e5e61e45fd55c626 Mon Sep 17 00:00:00 2001 From: pzcyyj2017 Date: Fri, 30 Oct 2026 11:37:01 +0800 Subject: [PATCH] hicache: align prefetch and host restore boundaries Use the request radix-key cap for L3 prefetch so bounded replay cannot split a queried trailing-state checkpoint at the later rematch. Reject Host-only SWA-gap candidates before they mask a complete ancestor, while preserving device matching and the final load-back completeness guard. Add CPU regressions that import the real scheduler, cache and TreeCore methods. All 13 test methods pass; the same tests against the unmodified scheduler/SWA code fail six methods (ten assertions including subtests). The checks use synthetic cache state and controller boundaries, not GPU or live Store execution. --- python/sglang/srt/managers/scheduler.py | 14 +- .../mem_cache/unified_cache/components/swa.py | 10 +- .../test_hicache_restore_endpoint_matching.py | 350 ++++++++++++++++++ 3 files changed, 370 insertions(+), 4 deletions(-) create mode 100644 test/registered/unit/mem_cache/test_hicache_restore_endpoint_matching.py diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index e20dc87f42..a57924af00 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -294,6 +294,7 @@ from sglang.srt.mem_cache import kv_cache_builder from sglang.srt.mem_cache.allocator.page_interleave import page_interleave_shard_size from sglang.srt.mem_cache.base_prefix_cache import CacheRequestOutcome from sglang.srt.mem_cache.common import ( + _req_radix_key, checkpoint_kv_cache, discard_kv_cache_backup, release_kv_cache, @@ -3323,9 +3324,16 @@ class Scheduler( and tree_cache.get_last_hash_value(last_host_node) is not None ) ): - match_end = req._compute_max_prefix_len( - len(req.full_untruncated_fill_ids) - ) + # L3 candidates must use the same raw-token cap as the + # subsequent rematch, including SWA bounded replay/logprobs. + # Otherwise a published trailing-state checkpoint can be split + # away by rematch, leaving a FULL-only parent with no SWA state. + match_end = _req_radix_key( + tree_cache, + req, + req.full_untruncated_fill_ids, + req._compute_max_prefix_len(len(req.full_untruncated_fill_ids)), + )._raw_len() new_input_tokens = req.full_untruncated_fill_ids[matched_len:match_end] prefix_keys = ( tree_cache.get_prefix_hash_values(last_host_node) diff --git a/python/sglang/srt/mem_cache/unified_cache/components/swa.py b/python/sglang/srt/mem_cache/unified_cache/components/swa.py index ac8cdda1a8..0850925634 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/swa.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/swa.py @@ -369,7 +369,15 @@ class SWAComponent(TreeComponent): if cd.value is None and (match_device_only or cd.host_value is None): state["len"] = 0 if not gates_prefix_match and (node.backuped or not node.evicted): - return True + # Gate-off preserves device bounded replay. A Host-only + # candidate must still be loadable: otherwise it masks a + # complete ancestor/L3 opportunity and later produces an + # empty load-back spec. Keep zero-window/root no-gap cases. + return ( + match_device_only + or not node.evicted + or not self.has_load_back_gap(node) + ) return False state["len"] += len(node.key) return state["len"] >= sliding_window_size diff --git a/test/registered/unit/mem_cache/test_hicache_restore_endpoint_matching.py b/test/registered/unit/mem_cache/test_hicache_restore_endpoint_matching.py new file mode 100644 index 0000000000..0200bed03e --- /dev/null +++ b/test/registered/unit/mem_cache/test_hicache_restore_endpoint_matching.py @@ -0,0 +1,350 @@ +"""CPU regressions for the lookup and match boundaries of hybrid restoration.""" + +import unittest +from array import array +from collections import defaultdict +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import torch + +from sglang.srt.managers.schedule_batch import Req +from sglang.srt.managers.scheduler import Scheduler +from sglang.srt.mem_cache.base_prefix_cache import ( + CacheRequestHandle, + MatchPrefixParams, + MatchResult, +) +from sglang.srt.mem_cache.common import _req_radix_key +from sglang.srt.mem_cache.radix_cache import RadixKey +from sglang.srt.mem_cache.unified_cache.component_type import ComponentType +from sglang.srt.mem_cache.unified_cache.components.full import FullComponent +from sglang.srt.mem_cache.unified_cache.components.swa import SWAComponent +from sglang.srt.mem_cache.unified_cache.unified_tree_core import ( + UnifiedTreeCore, + UnifiedTreeNode, +) +from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +_PAGE_SIZE = 256 +_SWA_WINDOW = 128 +_FULL = ComponentType.FULL +_SWA = ComponentType.SWA + + +def _prefetch_fixture( + input_tokens, + *, + replay=True, + lend=False, + prefix_len=0, + logprob_start_len=-1, +): + # Use the real request cap, scheduler method, radix key and cache submission + # path. Initial local matching and the final asynchronous controller are the + # boundaries of this unit test; no model or native Store is instantiated. + req = Req.__new__(Req) + req.rid = "restore-endpoint" + req.cache_request_handle = CacheRequestHandle(req.rid, 0) + req.full_untruncated_fill_ids = array("q", range(input_tokens)) + req.extra_key = "model-adapter" + req.cache_salt = "cache-namespace" + req.dllm_config = None + req.return_logprob = logprob_start_len >= 0 + req.logprob_start_len = logprob_start_len + req.prefix_len = prefix_len + req.host_hit_length = 0 + req.last_host_node = 1 if prefix_len else 0 + req.last_node = req.last_host_node + req.init_next_round_input = Mock() + + cache = UnifiedRadixCache.__new__(UnifiedRadixCache) + cache.tree_core = SimpleNamespace( + is_eagle=False, + has_swa_host_pool=True, + prefetch_anchor_info=Mock(return_value=(None, None)), + ) + cache.enable_storage = True + cache.host_memory_mode = "cache" + cache.buffer_pipeline = None + cache.decoder_swa_bounded_replay = replay + cache.session = SimpleNamespace(will_lend_record=Mock(return_value=lend)) + cache.page_size = cache.prefetch_threshold = _PAGE_SIZE + cache.tree_components = (_FULL, _SWA) + cache.components = { + _SWA: SimpleNamespace( + sliding_window_size=_SWA_WINDOW, + prepare_prefetch=Mock(return_value=SimpleNamespace(staging_tokens=0)), + ) + } + cache.is_root = Mock(side_effect=lambda node: node == 0) + cache.is_backuped = Mock(return_value=prefix_len > 0) + cache.get_last_hash_value = Mock(return_value=None) + cache.hicache_storage_pass_prefix_keys = False + cache.storage_prefetch_retries = Mock() + cache._storage_prefetch_wait = {} + cache.ongoing_prefetch = {} + cache._prefetch_outcome_stats = defaultdict(int) + cache.inc_host_lock_ref = Mock() + cache._build_sidecar_transfers = Mock(return_value=[]) + cache.cache_controller = SimpleNamespace( + get_prefetch_submission=Mock(return_value=None), + prefetch_rate_limited=Mock(return_value=False), + submit_prefetch=Mock( + return_value=SimpleNamespace(operation=None, decision=False) + ), + append_host_mem_release=Mock(), + ) + scheduler = Scheduler.__new__(Scheduler) + scheduler.enable_hicache_storage = True + scheduler.enable_lmcache = False + scheduler.tree_cache = cache + return scheduler, cache, req + + +def _issue_prefetch(scheduler, req, *, storage_hit_end=None): + with ( + patch( + "sglang.srt.managers.scheduler.get_memory", + return_value=SimpleNamespace(hicache_host_memory_mode="cache"), + ), + patch( + "sglang.srt.layers.attention.dsv4.unified_kv_gate.is_unified_kv_triton", + return_value=False, + ), + ): + scheduler._prefetch_kvcache(req, storage_hit_end=storage_hit_end) + + +class TestPrefetchRematchBoundary(unittest.TestCase): + def test_bounded_replay_page_boundary_matches_later_lookup(self): + for remainder, expected_end in ( + (0, 32512), + (1, 32512), + (127, 32512), + (128, 32768), + (183, 32768), + (255, 32768), + ): + with self.subTest(remainder=remainder): + scheduler, cache, req = _prefetch_fixture(32768 + remainder) + _issue_prefetch(scheduler, req) + args, kwargs = cache.cache_controller.submit_prefetch.call_args + key = args[1] + self.assertEqual(len(key), expected_end) + self.assertEqual(key.token_ids[-1], expected_end - 1) + self.assertFalse(kwargs["assume_stored"]) + rematch = _req_radix_key( + cache, + req, + req.full_untruncated_fill_ids, + req._compute_max_prefix_len(len(req.full_untruncated_fill_ids)), + ).page_aligned(_PAGE_SIZE) + self.assertEqual(list(key), list(rematch)) + self.assertEqual(key.extra_key, req.extra_key) + self.assertEqual(key.cache_salt, req.cache_salt) + req.init_next_round_input.assert_called_once_with( + cache, cow_mamba=False + ) + + def test_replay_disabled_and_borrowed_stream_preserve_ordinary_cap(self): + for options in ({"replay": False}, {"lend": True}): + with self.subTest(options=options): + scheduler, cache, req = _prefetch_fixture(30825, **options) + _issue_prefetch(scheduler, req) + self.assertEqual( + len(cache.cache_controller.submit_prefetch.call_args.args[1]), + 30720, + ) + + def test_logprob_bound_remains_stricter_than_replay_bound(self): + scheduler, cache, req = _prefetch_fixture(30825, logprob_start_len=23000) + _issue_prefetch(scheduler, req) + self.assertEqual( + len(cache.cache_controller.submit_prefetch.call_args.args[1]), 22784 + ) + + def test_prefetch_suffix_preserves_existing_device_prefix(self): + scheduler, cache, req = _prefetch_fixture(30825, prefix_len=8192) + _issue_prefetch(scheduler, req) + args = cache.cache_controller.submit_prefetch.call_args.args + self.assertEqual(len(args[1]), 22272) + self.assertEqual(args[1].token_ids[0], 8192) + self.assertEqual(args[1].token_ids[-1], 30463) + self.assertEqual(list(args[4]), list(range(8192))) + self.assertEqual(req.storage_prefetch_last_match_len, 8192) + + def test_retry_beyond_replay_cap_requires_a_fresh_query(self): + for hit_end, expected_end, assumed in ( + (30720, 30464, False), + (24576, 24576, True), + ): + with self.subTest(hit_end=hit_end): + scheduler, cache, req = _prefetch_fixture(30825) + _issue_prefetch(scheduler, req, storage_hit_end=hit_end) + args, kwargs = cache.cache_controller.submit_prefetch.call_args + self.assertEqual(len(args[1]), expected_end) + self.assertIs(kwargs["assume_stored"], assumed) + + def test_short_replay_window_does_not_submit_empty_prefetch(self): + scheduler, cache, req = _prefetch_fixture(128) + _issue_prefetch(scheduler, req) + cache.cache_controller.submit_prefetch.assert_not_called() + cache.inc_host_lock_ref.assert_not_called() + + +def _host_tree( + *, + checkpoints=(), + device_until=0, + device_swa=False, + gates=False, + window=_SWA_WINDOW, + hicache=True, + storage=True, +): + # Snapshot availability is the fixture. The real TreeCore walk, component + # validators, finalizers and gap detection choose all returned boundaries. + # Include the split FULL-only 30464 boundary explicitly so this test does + # not need an allocator or mock the node-splitting algorithm. + endpoints = (8192, 16384, 24576, 30464, 30720, 32768) + component_types = (_FULL, _SWA) + root = UnifiedTreeNode(component_types) + root.key = RadixKey(array("q")) + nodes = {0: root} + previous = root + offset = 0 + for endpoint in endpoints: + node = UnifiedTreeNode(component_types) + node.parent = previous + node.key = RadixKey(array("q", range(offset, endpoint))) + indices = torch.arange(offset, endpoint, dtype=torch.int64, device="cpu") + if endpoint <= device_until: + node.component_data[_FULL].value = indices + if device_swa: + node.component_data[_SWA].value = indices.clone() + else: + node.component_data[_FULL].host_value = indices + if endpoint in checkpoints: + node.component_data[_SWA].host_value = torch.arange( + _PAGE_SIZE, dtype=torch.int64, device="cpu" + ) + previous.children[node.key.child_key(_PAGE_SIZE)] = node + nodes[endpoint] = node + previous, offset = node, endpoint + + tree = UnifiedTreeCore.__new__(UnifiedTreeCore) + tree.root_node = root + tree._node_arena = {node.id: node for node in nodes.values()} + tree.page_size = _PAGE_SIZE + tree.is_eagle = False + tree.enable_hicache = hicache + tree.enable_storage = storage + tree._empty_match_result = MatchResult( + device_prefix_len=0, + last_device_node=root.id, + last_host_node=root.id, + best_match_node=root.id, + ) + tree._split_node = Mock(side_effect=AssertionError("fixture has exact boundaries")) + full = FullComponent.__new__(FullComponent) + full.tree_core = tree + swa = SWAComponent.__new__(SWAComponent) + swa.tree_core = tree + swa.sliding_window_size = window + swa.gates_prefix_match = gates + swa.refresh_lru = Mock() + tree.components = (full, swa) + tree.components_by_type = {_FULL: full, _SWA: swa} + return tree, swa, nodes + + +def _match(tree, endpoint=30464): + return tree.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", range(32768)), limit=endpoint)) + ) + + +class TestHostRestoreMatchBoundary(unittest.TestCase): + def test_full_only_host_parent_falls_back_to_complete_ancestor(self): + for checkpoints, expected in ( + ((8192, 16384, 24576, 30720), 24576), + ((16384, 30720), 16384), + ((), 0), + ): + with self.subTest(checkpoints=checkpoints): + tree, swa, nodes = _host_tree(checkpoints=checkpoints) + result = _match(tree) + self.assertEqual(result.host_hit_length, expected) + self.assertEqual(result.last_host_node, nodes[expected].id) + self.assertEqual(result.best_match_node, nodes[expected].id) + self.assertEqual(result.device_prefix_len, 0) + self.assertEqual(result.full_kv_hit_length, 30464) + self.assertFalse(swa.has_load_back_gap(nodes[expected])) + + def test_host_fallback_does_not_shorten_device_match(self): + for checkpoints, host_tokens, host_endpoint in ( + ((24576, 30720), 8192, 24576), + ((), 0, 16384), + ): + with self.subTest(checkpoints=checkpoints): + tree, _, nodes = _host_tree(checkpoints=checkpoints, device_until=16384) + result = _match(tree) + self.assertEqual(result.device_prefix_len, 16384) + self.assertEqual(result.last_device_node, nodes[16384].id) + self.assertEqual(result.host_hit_length, host_tokens) + self.assertEqual(result.last_host_node, nodes[host_endpoint].id) + + def test_device_resident_matching_keeps_replay_and_gate_rules(self): + for gates, device_swa, expected in ( + (False, False, 30464), + (False, True, 30464), + (True, True, 30464), + (True, False, 0), + ): + with self.subTest(gates=gates, device_swa=device_swa): + tree, _, nodes = _host_tree( + device_until=32768, device_swa=device_swa, gates=gates + ) + result = _match(tree) + self.assertEqual(result.device_prefix_len, expected) + self.assertEqual(result.last_device_node, nodes[expected].id) + self.assertEqual(result.host_hit_length, 0) + + def test_complete_checkpoint_after_gap_remains_reachable(self): + tree, swa, nodes = _host_tree(checkpoints=(16384, 32768)) + result = _match(tree, 32768) + self.assertEqual(result.host_hit_length, 32768) + self.assertEqual(result.last_host_node, nodes[32768].id) + self.assertFalse(swa.has_load_back_gap(nodes[32768])) + + def test_zero_window_and_gates_enabled_keep_their_existing_rules(self): + for options, expected in ( + ({"window": 0}, 30464), + ({"checkpoints": (16384, 30720), "gates": True}, 16384), + ): + with self.subTest(options=options): + tree, _, nodes = _host_tree(**options) + result = _match(tree) + self.assertEqual(result.host_hit_length, expected) + self.assertEqual(result.last_host_node, nodes[expected].id) + + def test_local_host_matching_remains_safe_without_l3(self): + tree, _, nodes = _host_tree(checkpoints=(16384, 30720), storage=False) + result = _match(tree) + self.assertEqual(result.host_hit_length, 16384) + self.assertEqual(result.last_host_node, nodes[16384].id) + + def test_hicache_disabled_preserves_device_only_matching(self): + tree, _, nodes = _host_tree(device_until=16384, hicache=False) + result = _match(tree) + self.assertEqual(result.device_prefix_len, 16384) + self.assertEqual(result.last_device_node, nodes[16384].id) + self.assertEqual(result.host_hit_length, 0) + + +if __name__ == "__main__": + unittest.main()