From 4f96807ea85ae2c38f0e95cbe783b8d37af931fe Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Thu, 22 Oct 2026 13:04:23 -0700 Subject: [PATCH] [CI] radix-cache: Build unified radix cache test states through production paths (#45334) --- .../test/kits/unified_radix_cache/config.py | 11 +- .../test/kits/unified_radix_cache/core.py | 5 +- .../test/kits/unified_radix_cache/fixtures.py | 21 +- .../kits/unified_radix_cache/insert_walk.py | 2 - .../kits/unified_radix_cache/kv_events.py | 9 +- .../test/kits/unified_radix_cache/mixins.py | 56 +- .../test/kits/unified_radix_cache/storage.py | 64 +- .../unified_tree_core_inspection_interface.py | 6 - .../unified_tree_core_inspector.py | 27 - .../test_unified_radix_cache_core.py | 277 ++------- .../test_unified_radix_cache_hicache.py | 460 ++++---------- .../test_unified_radix_cache_insert_walk.py | 97 +-- .../test_unified_radix_cache_kv_events.py | 7 - .../test_unified_radix_cache_load_back.py | 176 ++---- .../test_unified_radix_cache_storage.py | 570 +++++++----------- 15 files changed, 521 insertions(+), 1267 deletions(-) diff --git a/python/sglang/test/kits/unified_radix_cache/config.py b/python/sglang/test/kits/unified_radix_cache/config.py index e1ba1a5895..f4bc6708b9 100644 --- a/python/sglang/test/kits/unified_radix_cache/config.py +++ b/python/sglang/test/kits/unified_radix_cache/config.py @@ -13,7 +13,6 @@ from sglang.test.test_utils import CustomTestCase @dataclass(frozen=True) class CacheConfig: - # Tree page_size: int = 1 components: tuple[ComponentType, ...] = (ComponentType.FULL,) @@ -21,12 +20,10 @@ class CacheConfig: num_layers: int = 24 full_attention_layer_ids: tuple[int, ...] = (3, 7, 11, 15, 19, 23) - # SWA sliding_window_size: Optional[int] = None draft_swa_window: int = 0 decoder_swa_bounded_replay: bool = False - # Mamba enable_mamba_extra_buffer: bool = False mamba_cache_size: int = 20 mamba_intermediate_size: int = 256 @@ -37,7 +34,6 @@ class CacheConfig: mamba_conv_kernel: int = 4 enable_int8_mamba_checkpoint: bool = False - # Model / pool kv_size: int = 256 max_num_reqs: int = 10 max_context_len: int = 512 @@ -193,11 +189,8 @@ CONFIGS: list[CacheConfig] = [ def requires(predicate, reason): """Keep a test only under the configs where ``predicate`` holds. - Under any other config the test is removed from the bound class, not - skipped. ``reason`` documents the requirement and is not read at runtime. - - Stacked requirements are checked top to bottom and stop at the first that - fails, so a lower one may assume the ones above it hold. + Elsewhere the test is removed, not skipped. Stacked requirements are checked + top to bottom, so a lower one may assume the ones above it hold. """ def decorate(test): diff --git a/python/sglang/test/kits/unified_radix_cache/core.py b/python/sglang/test/kits/unified_radix_cache/core.py index 7c62535431..008d951d40 100644 --- a/python/sglang/test/kits/unified_radix_cache/core.py +++ b/python/sglang/test/kits/unified_radix_cache/core.py @@ -86,9 +86,7 @@ class FakeFullComponent(TreeComponent): class PagedFullComponent(FullComponent): - """FULL component whose device values are not backed one-to-one by pool - rows: a value lists two entries per row, and the component can hand rows - of a node back to the allocator while the value stays on device.""" + """FULL component whose device values are not backed one-to-one by pool rows.""" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -99,7 +97,6 @@ class PagedFullComponent(FullComponent): def component_with_cache(component_type, cache): - """A registry component instance bound to a (mock) cache and its tree_core.""" component = object.__new__(COMPONENT_REGISTRY[component_type]) component.cache = cache component.tree_core = cache.tree_core diff --git a/python/sglang/test/kits/unified_radix_cache/fixtures.py b/python/sglang/test/kits/unified_radix_cache/fixtures.py index ba0336288a..149996b124 100644 --- a/python/sglang/test/kits/unified_radix_cache/fixtures.py +++ b/python/sglang/test/kits/unified_radix_cache/fixtures.py @@ -52,9 +52,7 @@ from sglang.test.kits.unified_radix_cache.unified_tree_core_inspector import ( UnifiedTreeCoreInspector, ) -# The Rust re-run entry point overrides this without changing the process-wide -# production backend selection. Every cache built by build_fixture follows it; -# tests that construct a Python core directly stay Python-only. +# The Rust re-run overrides this; every cache built by build_fixture follows it. TREE_CORE_TEST_BACKEND: Optional[str] = "python" @@ -63,9 +61,8 @@ def selected_tree_core_test_backend() -> str: def drop_hicache_atexit_pin(cache): - """init_hicache registers ``cache.shutdown`` with atexit, which pins the - fixture's device and host pools until process exit; hundreds of HiCache - fixtures in one test process would otherwise accumulate on the GPU.""" + """init_hicache registers cache.shutdown with atexit, which pins the fixture's GPU + memory until exit.""" atexit.unregister(cache.shutdown) @@ -128,19 +125,17 @@ def build_fixture( ): """Create (tree, allocator, req_to_token_pool) from a CacheConfig. - ``tree_page_size`` stands in for DCP, which widens the tree page past the - ``page_size`` the rest of the config still sees. It only reaches values - derived from the tree page: the allocator keeps ``cfg.page_size``, whereas - DCP sets the two equal, so do not read insert or match behaviour off it. + ``tree_page_size`` stands in for DCP but only reaches values derived from the + tree page; the allocator keeps ``cfg.page_size``, so do not read insert or + match behaviour off it. """ server_args = ServerArgs( model_path="dummy", page_size=cfg.page_size, enable_int8_mamba_checkpoint=cfg.enable_int8_mamba_checkpoint, ) - # The mamba component reads mamba_cache_chunk_size, whose property otherwise - # loads the HF config for self.model_path — impossible for the dummy model. - # Mirror the property's default for a dummy HF config: FLA_CHUNK_SIZE. + # The dummy HF config cannot supply mamba_cache_chunk_size, so preset + # FLA_CHUNK_SIZE. server_args._mamba_cache_chunk_size = ( max(FLA_CHUNK_SIZE, cfg.page_size) if mamba_cache_chunk_size is None diff --git a/python/sglang/test/kits/unified_radix_cache/insert_walk.py b/python/sglang/test/kits/unified_radix_cache/insert_walk.py index 5186e8e375..bb25cc4cad 100644 --- a/python/sglang/test/kits/unified_radix_cache/insert_walk.py +++ b/python/sglang/test/kits/unified_radix_cache/insert_walk.py @@ -33,7 +33,6 @@ class ResumableInsertWalkSWAFixtures(UnifiedRadixCacheFixtures): seq = list(range(1, self.cfg.sliding_window_size + 1)) key = RadixKey(array("q", seq)) evicted = self._alloc(allocator, len(seq)) - # Window eviction already released the peers below the floor. allocator.free_swa(evicted) cache.insert( InsertParams( @@ -229,7 +228,6 @@ class SegmentLockFuzzFixtures(UnifiedRadixCacheFixtures): raise cache.sanity_check() - # Drain remaining locks; the tree must come back exactly balanced. for node_id, receipt, released in held: cache.dec_lock_ref(node_id, receipt.to_dec_params(), skip_swa=released) cache.sanity_check() diff --git a/python/sglang/test/kits/unified_radix_cache/kv_events.py b/python/sglang/test/kits/unified_radix_cache/kv_events.py index 2185242d72..76f0b5e91d 100644 --- a/python/sglang/test/kits/unified_radix_cache/kv_events.py +++ b/python/sglang/test/kits/unified_radix_cache/kv_events.py @@ -81,11 +81,10 @@ class UnifiedRadixCacheKVEventsFixtures: return leaf def _init_hicache(self, cache, *, write_policy: str = "write_through"): - # Production config: kernel IO backend + page_first layout with - # PINNED host pools (kernel GPU DMA requires them). Pools track their - # cudaHostRegister'd pointers and unregister on destroy()/GC, so the - # many fixtures sharing this process cannot collide on recycled - # address ranges (rc=712). + # Production config: kernel IO backend + page_first layout. + # Pinned host pools (kernel GPU DMA requires them) track their registered + # pointers and unregister on destroy/GC, so fixtures sharing a process + # cannot collide on recycled address ranges. server_args = ServerArgs( model_path="dummy", page_size=self.cfg.page_size, diff --git a/python/sglang/test/kits/unified_radix_cache/mixins.py b/python/sglang/test/kits/unified_radix_cache/mixins.py index 97faf60dca..550632a923 100644 --- a/python/sglang/test/kits/unified_radix_cache/mixins.py +++ b/python/sglang/test/kits/unified_radix_cache/mixins.py @@ -11,11 +11,14 @@ from sglang.srt.managers.schedule_batch import ( Req, ) from sglang.srt.mem_cache.base_prefix_cache import ( + InitLoadBackParams, InsertParams, + MatchPrefixParams, ) from sglang.srt.mem_cache.radix_cache import RadixKey from sglang.srt.mem_cache.unified_cache.components.base import ( ComponentType, + EvictLayer, ) from sglang.srt.sampling.sampling_params import SamplingParams from sglang.srt.server_args import ( @@ -73,9 +76,8 @@ class UnifiedRadixCacheFixtures: if not (self.cfg.has_swa and self.cfg.page_size > 1): return allocator.alloc(need_size) - # SWATokenToKVPoolAllocator.alloc() asserts page_size == 1, and - # alloc_extend() requires batch tensors unsuitable for unit tests. - # Replicate alloc_extend's core logic here. + # The SWA allocator's alloc asserts page_size == 1 and alloc_extend needs + # batch tensors, so replicate alloc_extend's core logic. ps = self.cfg.page_size aligned = ((need_size + ps - 1) // ps) * ps if aligned > allocator.full_attn_allocator.available_size(): @@ -195,9 +197,8 @@ class UnifiedRadixCacheFixtures: if storage_backend == "file": from sglang.srt.runtime_context import get_parallel - # The file-backend storage config records TP/PP rank/size. These unit - # fixtures run without initializing distributed parallel state, so - # force the local single-rank topology the fixture represents. + # Distributed state is not initialized in unit fixtures; force the + # local single-rank topology. parallel_override = get_parallel().override( tp_rank=0, tp_size=1, pp_rank=0, pp_size=1 ) @@ -216,7 +217,6 @@ class UnifiedRadixCacheFixtures: extra.update(storage_extra) storage_extra_config = json.dumps(extra) if extra else None - # See _init_hicache: production kernel IO backend, pinned pools. server_args = ServerArgs( model_path="dummy", page_size=self.cfg.page_size, @@ -228,7 +228,6 @@ class UnifiedRadixCacheFixtures: hicache_host_memory_mode=host_memory_mode, context_length=context_length, ) - # See build_fixture for why _mamba_cache_chunk_size is preset. server_args._mamba_cache_chunk_size = max(FLA_CHUNK_SIZE, self.cfg.page_size) set_global_server_args_for_scheduler(server_args) cache.init_hicache(server_args, cache.cache_init_params) @@ -237,15 +236,12 @@ class UnifiedRadixCacheFixtures: cache.write_through_threshold = 1 << 30 cache.load_back_threshold = 0 if storage_backend is not None: - # Unit fixtures size host/device pools equally, which makes the - # production prefetch capacity limit (host - device) zero. Keep the - # L3 tests focused on storage round trips by allowing one fixture - # worth of prefetch tokens. + # Fixtures size host and device pools equally, which zeroes the + # prefetch capacity limit; allow one fixture of prefetch tokens. cache.cache_controller.prefetch_capacity_limit = max( cache.cache_controller.prefetch_capacity_limit, cache.cache_controller.mem_pool_host.size, ) - # Background prefetch/backup threads are daemon; stop them per-test. self.addCleanup(cache.cache_controller._stop_storage_threads) def _build_hicache_fixture(self, tree_core_backend: Optional[str] = None): @@ -307,10 +303,9 @@ class UnifiedRadixCacheFixtures: ) def _build_chain_pages(self, cache, allocator, req_to_token_pool, num_pages): - """Insert an incremental chain of single-page extensions. + """Insert a chain of single-page extensions; return it root-to-leaf. - Returns the chain root-to-leaf. Length may differ from num_pages - when the radix tree merges or splits nodes. + The length may differ from num_pages when the tree merges or splits nodes. """ seq: list[int] = [] for i in range(num_pages): @@ -340,6 +335,31 @@ class UnifiedRadixCacheFixtures: ack.finish_event.synchronize() cache.loading_check() + def _evict_device_component(self, cache, node, component_type): + """Evict one node's device value of one component, as the eviction + walk does for the victim it picks.""" + result = cache.tree_core.evict_component( + node, component_type, EvictLayer.DEVICE + ) + cache._free_values(result.device_frees, result.host_frees) + + def _load_back_through_admission(self, cache, req_to_token_pool, tokens): + """Match ``tokens`` and start the scheduler's load-back of the host + hit. The load-back holds its path lock until it is released.""" + req = self._make_req(req_to_token_pool) + match = cache.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", tokens)), req=req) + ) + self._apply_match_to_req(req, match) + cache.init_load_back( + InitLoadBackParams( + best_match_node=req.best_match_node, + host_hit_length=req.host_hit_length, + req=req, + ) + ) + return req + def _match_tokens_for_chain(self, cache, chain): tokens: list[int] = [] for node in chain: @@ -363,9 +383,7 @@ class UnifiedRadixCacheFixtures: ) def _swa_finalize_setup(self): - """Build a SWA chain long enough to fill at least the window - plus one extra page, and host-back every node so we can flip - SWA tombstones at will.""" + """Build a SWA chain longer than the window, host-backed at every node.""" ps = self.cfg.page_size sw = self.cfg.sliding_window_size window_pages = (sw + ps - 1) // ps diff --git a/python/sglang/test/kits/unified_radix_cache/storage.py b/python/sglang/test/kits/unified_radix_cache/storage.py index 1d81154d85..a9b8fbfdc8 100644 --- a/python/sglang/test/kits/unified_radix_cache/storage.py +++ b/python/sglang/test/kits/unified_radix_cache/storage.py @@ -134,10 +134,9 @@ class UnifiedRadixCacheStorageFixtures(UnifiedRadixCacheFixtures): def _run_prefetch_to_completion(self, cache, req_id, timeout: float = 10.0): deadline = time.time() + timeout while time.time() < deadline: - # Host memory is reserved (and IO started) by the scheduler-thread - # drain once the L3 hit count is known, and the buffer-mode fill - # commits at its H2D ack in loading_check — pump the full event - # round like the real scheduler loop does. + # Pump the full event round like the scheduler loop: host reservation + # happens at the scheduler-thread drain, and the buffer-mode fill + # commits at its H2D ack. cache.check_hicache_events() if cache.check_prefetch_progress(req_id): # Buffer mode parks a completed fetch as a staged prefetch; @@ -175,7 +174,7 @@ class UnifiedRadixCacheStorageFixtures(UnifiedRadixCacheFixtures): f = cache.buffer_pipeline.staged_prefetches[req_id] if prefix_len is None: prefix_len = f.matched_len - req = mock.Mock() + req = self._make_req(cache.req_to_token_pool) req.rid = req_id.rid req.cache_request_handle = req_id req.extra_key = extra_key @@ -391,9 +390,8 @@ class UnifiedRadixCacheStorageFixtures(UnifiedRadixCacheFixtures): cons.enable_storage_metrics = True cons.storage_metrics_collector = mock.Mock() - # Masked state: nodes born with live FULL under SWA tombstones (a - # sibling insert whose SWA ring had slid past the span). live_tail is - # how far short of the span end the ring stopped. + # Nodes born with live FULL under SWA tombstones; live_tail is how far short + # of the span end the ring stopped. live_from = len(seq) - live_tail value = self._alloc(cons_alloc, len(seq)) cons.insert( @@ -407,8 +405,7 @@ class UnifiedRadixCacheStorageFixtures(UnifiedRadixCacheFixtures): self.assertEqual(masked.device_prefix_len, 0, "unified match not masked") self.assertEqual(masked.full_kv_hit_length, len(seq), "live FULL not resident") masked_full = value[: len(seq)].clone() - # Model the real SWA-eviction path: tombstoned positions have their - # allocator mapping cleared, while the live trailing portion remains. + # Tombstoned positions have their allocator mapping cleared; the tail stays. cons_alloc.free_swa(masked_full[:live_from]) self._fill_full_kv(cons_alloc, masked_full, marker=7) masked_kv = self._snapshot_full_kv(cons_alloc, masked_full) @@ -421,7 +418,6 @@ class UnifiedRadixCacheStorageFixtures(UnifiedRadixCacheFixtures): cons.prefetch_from_storage( req_id, cons.root_node_handle(), array("q", seq), None, None ) - # A parked SWA-only retry has no FULL suffix, but must re-pin its prefix. with mock.patch.object(cons.swa_kv_pool_host, "alloc", return_value=None): self._pump_hicache_until( cons, @@ -500,7 +496,6 @@ class UnifiedRadixCacheStorageFixtures(UnifiedRadixCacheFixtures): after = match_prefix(cons, seq) self.assertEqual(after.full_kv_hit_length, len(seq)) self.assertEqual(after.device_prefix_len, len(seq)) - # Only the fetched SWA tail made the resident FULL reusable. self.assertEqual((req.storage_hit_start, req.storage_hit_length), (0, len(seq))) cons.storage_metrics_collector.log_storage_prefetch_unfulfilled_tokens.assert_not_called() cons.sanity_check() @@ -552,7 +547,6 @@ class UnifiedRadixCacheStorageFixtures(UnifiedRadixCacheFixtures): ) self.assertGreater(aux_max, len(seq) - len(head)) - # The surfaced host hit is the splice-able tail, not the full span. req = SimpleNamespace( rid=req_id.rid, cache_request_handle=req_id, @@ -585,7 +579,6 @@ class UnifiedRadixCacheStorageFixtures(UnifiedRadixCacheFixtures): self.assertTrue(torch.equal(hk, head_k)) self.assertTrue(torch.equal(hv, head_v)) - # The whole bounce (trimmed head included) frees at the ack. self.assertEqual(self._host_avail_sizes(cons), avail0) self.assertEqual(cons.cache_controller.prefetch_tokens_occupied, 0) cons.sanity_check() @@ -596,7 +589,6 @@ class UnifiedRadixCacheStorageFixtures(UnifiedRadixCacheFixtures): cc = cache.cache_controller - # Fake _reduce_prefetch_ack to drop SWA pool_hits when drop_swa is True. def fake_reduce(ack): if drop_swa and ack.pool_hits is not None: if PoolName.SWA.value in ack.pool_hits: @@ -608,7 +600,6 @@ class UnifiedRadixCacheStorageFixtures(UnifiedRadixCacheFixtures): p_reduce.start() self.addCleanup(p_reduce.stop) - # Make all_reduce no-op. The real all_reduce fails in unit tests with tp_world_size=2. p_dist = mock.patch.object(dist, "all_reduce", return_value=None) p_dist.start() self.addCleanup(p_dist.stop) @@ -774,25 +765,26 @@ class UnifiedRadixCacheStorageAttachBackfillFixtures: return hashes -class AnchorLockOutcomePolicyFixtures: - def _make_pipeline(self, cache, cap_tokens=10_000): - from sglang.srt.mem_cache.buffer_mode.pipeline import BufferModePipeline +class AnchorLockOutcomePolicyFixtures(UnifiedRadixCacheStorageFixtures): + def _insert_tokens(self, cache, tokens): + value = cache.token_to_kv_pool_allocator.alloc(len(tokens)) + self.assertIsNotNone(value) + cache.insert(InsertParams(key=RadixKey(array("q", tokens)), value=value)) - pipeline = BufferModePipeline.__new__(BufferModePipeline) - pipeline.anchor_locks = {} - pipeline.anchor_locked_tokens_ = 0 - pipeline.anchor_lock_cap_tokens = cap_tokens - pipeline._anchor_lock_cap_skips = 0 - pipeline._prefetch_prefix_ctx = {self._REQ: (list(self._PREFIX), None, None)} - pipeline._cache = cache - return pipeline - - def _make_cache(self, live_match_len, pinned_len=None): - cache = mock.MagicMock() - cache.tree_core.is_eagle = False - cache.tree_core.match_full_device_prefix.return_value = ( - live_match_len, - 99, - live_match_len if pinned_len is None else pinned_len, + def _make_buffer_cache( + self, resident=(), cap_tokens=None, prefix=(), storage_dir=None + ): + """A buffer-mode cache holding ``resident`` on device, with the + request's matched prefix recorded as prefetch enqueue records it.""" + cache, _, _ = self._build_buffer_hicache_fixture( + storage_dir or self._make_storage_dir() ) - return cache + if resident: + self._insert_tokens(cache, list(resident)) + pipeline = cache.buffer_pipeline + if cap_tokens is not None: + pipeline.anchor_lock_cap_tokens = cap_tokens + pipeline.set_prefix_ctx( + self._REQ, self._PREFIX if prefix == () else (prefix or []) + ) + return cache, pipeline diff --git a/python/sglang/test/kits/unified_radix_cache/unified_tree_core_inspection_interface.py b/python/sglang/test/kits/unified_radix_cache/unified_tree_core_inspection_interface.py index 6f6a5c0e29..dcc8b1d7e9 100644 --- a/python/sglang/test/kits/unified_radix_cache/unified_tree_core_inspection_interface.py +++ b/python/sglang/test/kits/unified_radix_cache/unified_tree_core_inspection_interface.py @@ -27,8 +27,6 @@ if TYPE_CHECKING: class UnifiedTreeCoreInspectionInterface(UnifiedTreeCoreInterface): """Test-only inspection and control contract for shared backend tests.""" - # ==== Read-only inspection ==== - @abstractmethod def contains_node(self, node_id: NodeId) -> bool: ... @@ -125,8 +123,6 @@ class UnifiedTreeCoreInspectionInterface(UnifiedTreeCoreInterface): """Protected token count for one component (0 if the component is absent).""" ... - # ==== White-box state controls ==== - @abstractmethod def set_node_hash_values( self, node_id: NodeId, hash_values: Optional[list[str]] @@ -194,8 +190,6 @@ class UnifiedTreeCoreInspectionInterface(UnifiedTreeCoreInterface): """Refresh duplicate-host tracking for the node.""" ... - # ==== Targeted white-box operations ==== - @abstractmethod def advance_insert_walk_once(self) -> None: """Advance one suspended insert walk step without flushing its actions.""" diff --git a/python/sglang/test/kits/unified_radix_cache/unified_tree_core_inspector.py b/python/sglang/test/kits/unified_radix_cache/unified_tree_core_inspector.py index eb45304f1a..bc7f7bc476 100644 --- a/python/sglang/test/kits/unified_radix_cache/unified_tree_core_inspector.py +++ b/python/sglang/test/kits/unified_radix_cache/unified_tree_core_inspector.py @@ -30,7 +30,6 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa return node_id in self._node_arena def get_parent_node_id(self, node_id: NodeId) -> Optional[NodeId]: - """The parent node id, or None for the root.""" parent = self.node_by_id(node_id).parent return None if parent is None else parent.id @@ -45,13 +44,11 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa return len(key) def get_node_token_ids(self, node_id: NodeId) -> list[int]: - """The raw token ids spanned by the node's radix key.""" key = self.node_by_id(node_id).key assert key is not None return list(key.raw_token_ids()) def is_node_key_bigram(self, node_id: NodeId) -> bool: - """Whether the node's radix key uses bigram atoms.""" key = self.node_by_id(node_id).key assert key is not None return key.is_bigram @@ -59,19 +56,16 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa def get_component_host_value( self, node_id: NodeId, component_type: ComponentType ) -> Optional[torch.Tensor]: - """The component's host value on the node, or None if absent.""" return self.node_by_id(node_id).component_data[component_type].host_value def get_component_device_lock_ref( self, node_id: NodeId, component_type: ComponentType ) -> int: - """The component's device lock count on the node.""" return self.node_by_id(node_id).component_data[component_type].lock_ref def get_component_host_lock_ref( self, node_id: NodeId, component_type: ComponentType ) -> int: - """The component's host lock count on the node.""" return self.node_by_id(node_id).component_data[component_type].host_lock_ref def get_node_hit_count(self, node_id: NodeId) -> int: @@ -83,20 +77,17 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa return self.node_by_id(node_id).write_through_pending_id def is_external_cache_stored(self, node_id: NodeId) -> bool: - """Whether the node is known to be stored in the external cache.""" return self.node_by_id(node_id).external_cache_stored def is_node_in_device_lru( self, node_id: NodeId, component_type: ComponentType ) -> bool: - """Whether the node belongs to the component's device LRU.""" lru = self.lru_lists.get(component_type) return lru is not None and lru.in_list(self.node_by_id(node_id)) def is_node_in_host_lru( self, node_id: NodeId, component_type: ComponentType ) -> bool: - """Whether the node belongs to the component's host LRU.""" lru = self.host_lru_lists.get(component_type) return lru is not None and lru.in_list(self.node_by_id(node_id)) @@ -114,7 +105,6 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa def get_component_device_lru_node_ids( self, component_type: ComponentType ) -> list[NodeId]: - """The component's device LRU members from most to least recent.""" lru = self.lru_lists.get(component_type) return [] if lru is None else self._lru_node_ids(lru) @@ -129,7 +119,6 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa return node is not None and node in self.evictable_host_leaves def is_device_leaf(self, node_id: NodeId) -> bool: - """Whether the node has no device-resident descendants.""" return self._is_device_leaf(self.node_by_id(node_id)) def get_all_node_ids(self) -> list[NodeId]: @@ -137,13 +126,11 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa return [node.id for node in self._collect_all_nodes()] def component_protected_size(self, component_type: ComponentType) -> int: - """Protected token count for one component (0 if the component is absent).""" return self.component_protected_size_.get(component_type, 0) def set_node_hash_values( self, node_id: NodeId, hash_values: Optional[list[str]] ) -> None: - """Replace the node's page-hash field.""" self.node_by_id(node_id).hash_value = hash_values def set_component_device_value_raw( @@ -152,7 +139,6 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa component_type: ComponentType, value: Optional[torch.Tensor], ) -> None: - """Replace the device-value field without updating tree bookkeeping.""" self.node_by_id(node_id).component_data[component_type].value = value def set_component_host_value_raw( @@ -161,48 +147,40 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa component_type: ComponentType, value: Optional[torch.Tensor], ) -> None: - """Replace the host-value field without updating tree bookkeeping.""" self.node_by_id(node_id).component_data[component_type].host_value = value def set_component_device_lock_ref( self, node_id: NodeId, component_type: ComponentType, lock_ref: int ) -> None: - """Replace the component's device lock count.""" assert lock_ref >= 0 self.node_by_id(node_id).component_data[component_type].lock_ref = lock_ref def remove_node_from_device_lru( self, node_id: NodeId, component_type: ComponentType ) -> None: - """Remove the node from the component's device LRU.""" self.lru_lists[component_type].remove_node(self.node_by_id(node_id)) def insert_node_into_host_lru( self, node_id: NodeId, component_type: ComponentType ) -> None: - """Insert the node as the component's most-recent host entry.""" self.host_lru_lists[component_type].insert_mru(self.node_by_id(node_id)) def set_component_evictable_size( self, component_type: ComponentType, value: int ) -> None: - """Replace the component's evictable device-token count.""" assert value >= 0 self.component_evictable_size_[component_type] = value def set_component_protected_size( self, component_type: ComponentType, value: int ) -> None: - """Replace the component's protected device-token count.""" assert value >= 0 self.component_protected_size_[component_type] = value def update_duplicate_tracking(self, node_id: NodeId) -> None: - """Refresh duplicate-host tracking for the node.""" self._update_duplicate_tracking(self.node_by_id(node_id)) def advance_insert_walk_once(self) -> None: - """Advance one suspended insert walk step without flushing its actions.""" state = self._ongoing_insert_walk_state if state is None: raise RuntimeError("no in-flight insert") @@ -216,7 +194,6 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa component_type: ComponentType, target: EvictLayer, ) -> BaseEvictionResult: - """Evict one component layer from a node and detach its LRU entry.""" result = BaseEvictionResult() self._evict_component_and_detach_lru( self.node_by_id(node_id), @@ -235,7 +212,6 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa component_type: ComponentType, target: EvictLayer, ) -> None: - """Validate the locks for a component-triggered cascade eviction.""" node = self.node_by_id(node_id) trigger = self.components_by_type[component_type] is_leaf = self._is_cascade_evict_leaf(node, target) @@ -243,7 +219,6 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa self._should_cascade_evict_component(node, trigger, comp, target, is_leaf) def cleanup_tombstone_ancestors(self, node_id: NodeId) -> BaseEvictionResult: - """Delete childless tombstone ancestors until a live or locked node is reached.""" result = BaseEvictionResult() self._iteratively_delete_tombstone_ancestors( self.node_by_id(node_id), @@ -262,7 +237,6 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa value_chunks: list[torch.Tensor], best_value_len: int, ) -> MatchResult: - """Run one component's match finalizer with NodeId boundaries.""" node_result = result._replace( last_device_node=self.node_by_id(result.last_device_node), last_host_node=self.node_by_id(result.last_host_node), @@ -285,7 +259,6 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa def build_backup_node_ids( self, node_id: NodeId, write_back: bool = False ) -> list[NodeId]: - """Build the ordered node list for a device-to-host backup.""" return self._build_backup_kv_action( self.node_by_id(node_id), write_back ).node_ids diff --git a/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_core.py b/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_core.py index 894cdf6c3f..6270f26a0a 100644 --- a/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_core.py +++ b/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_core.py @@ -1,5 +1,3 @@ -"""Insert, match, lock, eviction, checkpoint, SWA and Mamba tests for UnifiedRadixCache.""" - import unittest from array import array from dataclasses import replace @@ -121,13 +119,8 @@ class TestUnifiedRadixComponentRegistryOverride(CustomTestCase): class TestUnifiedRadixComponentLedgerUnits(CustomTestCase): def test_ledgers_count_reclaimable_tokens(self): - """The evictable and protected ledgers are in the allocator's units: - prefill admission adds the evictable ledger to the allocator's free - space and the eviction walk counts them toward its request. So every - ledger site (insert, lock, release, an in-place change, eviction) - reads ``TreeComponent.reclaimable_value_len`` rather than the value - length. A site that goes back to ``len(value)`` drifts the ledger from - the recount in ``sanity_check`` for a component that overrides it.""" + """Ledgers count allocator units: every site reads reclaimable_value_len, + not len(value), or the ledger drifts from sanity_check's recount.""" cache, allocator, _ = build_fixture( CacheConfig(), component_registry_override={ComponentType.FULL: PagedFullComponent}, @@ -183,7 +176,7 @@ class TestDisabledUnifiedRadixCache(DisabledUnifiedRadixCacheFixtures, CustomTes class TestUnifiedTreeNodeGetPrefixHashValues(CustomTestCase): def test_get_prefix_hash_values_not_shared_across_calls(self): - """Regression guard for cached mutable prefix hash lists (#26177).""" + """Cached prefix hash lists must not be mutated by callers.""" def make_node(): return UnifiedTreeNode(tree_components=(ComponentType.FULL,)) @@ -458,7 +451,6 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures): self._insert(cache, allocator, req_to_token_pool, seq_1p) self.assertEqual(allocator.available_size(), initial_avail - len(seq_1p)) - # Frees the 1-page overlap. key_2p = RadixKey(array("q", seq_2p)) value_2p = self._alloc(allocator, len(seq_2p)) params = InsertParams( @@ -481,7 +473,6 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures): initial_avail - len(seq_1p) - (len(seq_2p) - len(seq_1p)), ) - # prev_prefix_len covers the whole overlap: nothing freed. avail_before = allocator.available_size() key_3p = RadixKey(array("q", seq_3p)) value_3p = self._alloc(allocator, len(seq_3p)) @@ -637,10 +628,9 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures): @requires_swa_only @requires_ps1_window4 def test_swa_unfinished_req_repoints_whole_insert_below_window(self): - # A request whose live SWA tail inside the inserted range is shorter - # than the window must still end up protected for the whole insert: - # the tree owns those slots now, and a shorter protected length lets - # the request free tree-owned SWA pages and re-insert them at finish. + # Even a live SWA tail shorter than the window must be protected for the whole + # insert, else the request frees tree-owned SWA pages and re-inserts them at + # finish. cache, allocator, req_to_token_pool = build_fixture(self.cfg) full_avail = allocator.full_attn_allocator.available_size() swa_avail = allocator.swa_attn_allocator.available_size() @@ -730,12 +720,10 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures): seq = self._make_seq(1, 2) self._insert(cache, allocator, req_to_token_pool, seq) - # Tree truncates unaligned tail internally, so it matches the seq prefix. unaligned = seq + list(range(9000, 9000 + ps - 1)) m = match_prefix(cache, unaligned) self.assertEqual(m.device_prefix_len, len(seq)) - # Below-page-size key aligns to 0 -> no match. m = match_prefix(cache, seq[: ps - 1]) self.assertEqual(m.device_prefix_len, 0) @@ -752,12 +740,10 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures): self._insert(cache, allocator, req_to_token_pool, first_page) self._insert(cache, allocator, req_to_token_pool, seq) - # Mismatch in second page → only first page matches bad_page2 = seq[:ps] + [9999] * ps m = match_prefix(cache, bad_page2) self.assertEqual(m.device_prefix_len, ps) - # Mismatch in first page → 0 match bad_page1 = [9999] + seq[1:] m = match_prefix(cache, bad_page1) self.assertEqual(m.device_prefix_len, 0) @@ -896,7 +882,6 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures): # past the SWA window (swa_evicted_seqlen >= total), the Full leaf must # be materialized (and the Full KV kept) so the prefix stays cacheable. # Runs across all SWA configs, including page_size > sliding_window_size - # (the dsv4-style edge case). cache, allocator, _ = build_fixture(self.cfg) tokens = self._make_seq(1, 2) @@ -938,11 +923,8 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures): node_a = m.last_device_node self.assertGreater(len(node_children(cache, node_a)), 0, "A must be internal") - # A request locks A, then decodes past the window → early-release the SWA - # portion. On this internal node, dec_swa_lock_only also drops the - # strictly-lower-tier Mamba lock (the co-located Mamba is useless once SWA - # is gone), leaving only the Full path-lock held. This is what guarantees - # the later SWA-eviction cascade never meets a legitimately-locked Mamba. + # Early SWA release on an internal node also drops the lower-tier Mamba lock, + # so later SWA eviction never meets a locked Mamba. lock_result = cache.inc_lock_ref(node_a) self.assertGreaterEqual( node_device_lock_ref(cache, node_a, ComponentType.MAMBA), @@ -963,9 +945,7 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures): cache.tree_core.is_node_in_device_lru(node_a, ComponentType.SWA) ) - # Evict the child branch (Full/device eviction only) → A becomes a - # Full-locked leaf with its now-unlocked SWA still in the SWA LRU. We do - # NOT tombstone aux at the leaf-transition; the held Full pins the node. + # Evicting the child makes A a Full-locked leaf with its SWA still in the LRU. cache.evict(EvictParams(num_tokens=len(seq_ab))) self.assertEqual(len(node_children(cache, node_a)), 0, "A should now be a leaf") @@ -979,10 +959,7 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures): "A's unlocked SWA stays in the LRU (not tombstoned at transition)", ) - # SWA eviction now selects A (it is in the SWA LRU). A's Full lock is a - # higher-or-equal internal tier, so the cascade skips it and spares the - # Full KV. The unlocked lower-tier Mamba is cascaded as part of the atomic - # leaf teardown. This used to assert `cd.lock_ref == 0` on the locked Full. + # SWA eviction skips A's Full lock and cascades the unlocked lower-tier Mamba. cache.evict(EvictParams(num_tokens=0, swa_num_tokens=len(seq_a))) self.assertGreaterEqual( node_device_lock_ref(cache, node_a, ComponentType.FULL), @@ -1006,8 +983,7 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures): def test_mamba_opt_out_holder_cannot_release_another_holders_mamba_lock(self): """Holder A takes the mamba lock; holder B opts out (skip_lock_components=(ComponentType.MAMBA,)) on the same node. B's early SWA release and final release must leave - A's mamba lock intact -- a lost/defaulted receipt on B's side used to - decrement A's lock without tripping any assert.""" + A's mamba lock intact.""" cache, allocator, req_to_token_pool = build_fixture(self.cfg) seq = self._make_seq( @@ -1064,7 +1040,6 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures): "A must hold a Mamba checkpoint", ) - # Natural lock acquisition — records inc_lock_ref in the lock trace. lock_result = cache.inc_lock_ref(node_a) self.assertGreaterEqual( node_device_lock_ref(cache, node_a, ComponentType.SWA), 1, "SWA locked" @@ -1096,46 +1071,6 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures): "Full stays locked", ) - @requires_swa_and_mamba - def test_cascade_evict_asserts_on_locked_internal_mamba(self): - cache, allocator, req_to_token_pool = build_fixture(self.cfg) - - n_short = (self.cfg.sliding_window_size // self.cfg.page_size) + 4 - seq_a = self._make_seq(1, n_short) - seq_ab = seq_a + self._make_seq(7000, 2) - self._insert(cache, allocator, req_to_token_pool, seq_a) - self._insert(cache, allocator, req_to_token_pool, seq_ab) - - node_a = match_prefix(cache, seq_a).last_device_node - self.assertGreater(len(node_children(cache, node_a)), 0, "A must be internal") - self.assertIsNotNone( - node_device_value(cache, node_a, ComponentType.MAMBA), - "A must hold a Mamba checkpoint", - ) - - # Lock ONLY Mamba — a stranded lower-priority lock that no supported path - # produces. The cascade must surface it rather than silently skip. - cache.tree_core.set_component_device_lock_ref(node_a, ComponentType.MAMBA, 1) - self.assertGreaterEqual( - node_device_lock_ref(cache, node_a, ComponentType.MAMBA), 1, "Mamba locked" - ) - self.assertEqual( - node_device_lock_ref(cache, node_a, ComponentType.FULL), 0, "Full unlocked" - ) - - result = cache.tree_core.evict_component( - node_a, ComponentType.SWA, EvictLayer.DEVICE - ) - cache._free_values(result.device_frees, result.host_frees) - # No higher-or-equal tier pins the node, so even with early-release on - # the stranded Mamba lock must trip the hard-invariant assert. - with self.assertRaises(AssertionError): - cache.tree_core.validate_cascade_evict( - node_a, ComponentType.SWA, EvictLayer.DEVICE - ) - - cache.tree_core.set_component_device_lock_ref(node_a, ComponentType.MAMBA, 0) - @requires_swa_and_mamba def test_cascade_evict_asserts_on_locked_leaf_mamba(self): cache, allocator, req_to_token_pool = build_fixture(self.cfg) @@ -1173,9 +1108,6 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures): @requires_swa def test_swa_leaf_capped_to_window_on_insert(self): - """A long SWA leaf is split so locking it protects one window of SWA - while full attention still protects the whole sequence.""" - ps = self.cfg.page_size window = self.cfg.sliding_window_size tail_size = ((window + ps - 1) // ps) * ps @@ -1194,13 +1126,11 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures): self.assertIsNotNone(swa_val) if case == "long_splits": - # Capped to one page-aligned window; prefix is a real ancestor. self.assertEqual(len(swa_val), tail_size) self.assertNotEqual( node_parent(cache, leaf), cache.root_node_handle() ) else: - # Already within one window — no split. self.assertEqual(len(swa_val), len(seq)) self.assertEqual(node_parent(cache, leaf), cache.root_node_handle()) @@ -1447,19 +1377,6 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures): cache.unlock(req.lock) cache.sanity_check() - def test_tombstone_cleanup_respects_locked_parent(self): - cache, allocator, req_to_token_pool = build_fixture(self.cfg) - tokens = self._make_seq(1, 1) - self._insert(cache, allocator, req_to_token_pool, tokens) - parent = match_prefix(cache, tokens).last_device_node - cache.tree_core.set_component_device_lock_ref(parent, ComponentType.FULL, 1) - - result = cache.tree_core.cleanup_tombstone_ancestors(parent) - cache._free_values(result.device_frees, result.host_frees) - - self.assertIn(parent, node_children(cache, cache.root_node_handle())) - self.assertTrue(all(evicted == 0 for evicted in result.tracker.values())) - def test_aux_evict_full_locked_leaf_tombstones_aux_only(self): aux_types = [ ct @@ -1473,7 +1390,6 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures): aux = aux_types[0] cache, allocator, req_to_token_pool = build_fixture(self.cfg) - # One page stays within a single SWA window (no leaf-cap split). seq = self._make_seq(1, 1) self._insert(cache, allocator, req_to_token_pool, seq) @@ -1483,9 +1399,8 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures): self.assertIsNotNone(node_device_value(cache, node, ComponentType.FULL)) self.assertIsNotNone(node_device_value(cache, node, aux)) - # Reach the "FULL locked, aux unlocked" state the way production does: - # mamba via the decode-hold opt-out (skip_lock_components=(ComponentType.MAMBA,)), SWA via the - # early window release (its own first-class op). + # Reach "FULL locked, aux unlocked" as production does: Mamba via the + # decode-hold opt-out, SWA via early window release. aux_len = len(node_device_value(cache, node, aux)) if aux == ComponentType.MAMBA: lock_result = cache.inc_lock_ref( @@ -1495,8 +1410,6 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures): lock_result = cache.inc_lock_ref(node) self.assertGreater(node_device_lock_ref(cache, node, aux), 0) cache.release_swa(TreeLock(node, lock_result.to_dec_params())) - # FULL still locked -> not a device leaf -> no inline evict; the - # value stays evictable for the explicit aux eviction below. self.assertIsNotNone(node_device_value(cache, node, aux)) self.assertGreater(node_device_lock_ref(cache, node, ComponentType.FULL), 0) self.assertEqual(node_device_lock_ref(cache, node, aux), 0) @@ -1525,25 +1438,6 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures): ) cache.sanity_check() - def test_evict_leaf_frees_all_components(self): - cache, allocator, req_to_token_pool = build_fixture(self.cfg) - seq = self._make_seq(1, 3) - self._insert(cache, allocator, req_to_token_pool, seq) - - full_before = cache.full_evictable_size() - mamba_before = cache.mamba_evictable_size() if self.cfg.has_mamba else 0 - swa_before = cache.swa_evictable_size() if self.cfg.has_swa else 0 - self.assertGreater(full_before, 0) - - result = cache.evict(EvictParams(num_tokens=full_before * 2)) - self.assertGreaterEqual(result.num_tokens_evicted, full_before) - self.assertEqual(cache.full_evictable_size(), 0) - if self.cfg.has_mamba: - self.assertEqual(cache.mamba_evictable_size(), 0) - if self.cfg.has_swa: - self.assertEqual(cache.swa_evictable_size(), 0) - cache.sanity_check() - def test_evict_cascade_parent_becomes_d_leaf(self): cache, allocator, req_to_token_pool = build_fixture(self.cfg) base = self._make_seq(1, 2) @@ -1580,6 +1474,10 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures): result = cache.evict(EvictParams(num_tokens=initial_evictable * 2)) self.assertGreaterEqual(result.num_tokens_evicted, initial_evictable) self.assertEqual(cache.full_evictable_size(), 0) + if self.cfg.has_mamba: + self.assertEqual(cache.mamba_evictable_size(), 0) + if self.cfg.has_swa: + self.assertEqual(cache.swa_evictable_size(), 0) self.assertEqual(len(node_children(cache, cache.root_node_handle())), 0) cache.sanity_check() @@ -1623,27 +1521,6 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures): self.assertEqual(m_low.device_prefix_len, 0) cache.sanity_check() - def test_evict_multiple_independent_leaves(self): - cache, allocator, req_to_token_pool = build_fixture(self.cfg) - seqs = [self._make_seq(i * 100, 2) for i in range(4)] - for s in seqs: - self._insert(cache, allocator, req_to_token_pool, s) - - total = sum(len(s) for s in seqs) - self.assertEqual(cache.full_evictable_size(), total) - - half = total // 2 - result = cache.evict(EvictParams(num_tokens=half)) - self.assertGreaterEqual(result.num_tokens_evicted, half) - self.assertLessEqual(cache.full_evictable_size(), total - half) - cache.sanity_check() - - remaining = cache.full_evictable_size() - result = cache.evict(EvictParams(num_tokens=remaining * 2)) - self.assertGreaterEqual(result.num_tokens_evicted, remaining) - self.assertEqual(cache.full_evictable_size(), 0) - cache.sanity_check() - def test_evict_result_accounting_matches_actual(self): cache, allocator, req_to_token_pool = build_fixture(self.cfg) seqs = [self._make_seq(i * 100, 2) for i in range(5)] @@ -1651,9 +1528,15 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures): self._insert(cache, allocator, req_to_token_pool, s) before = cache.full_evictable_size() - result = cache.evict(EvictParams(num_tokens=before)) + half = before // 2 + result = cache.evict(EvictParams(num_tokens=half)) after = cache.full_evictable_size() + self.assertGreaterEqual(result.num_tokens_evicted, half) self.assertEqual(result.num_tokens_evicted, before - after) + + remaining = cache.full_evictable_size() + cache.evict(EvictParams(num_tokens=remaining * 2)) + self.assertEqual(cache.full_evictable_size(), 0) cache.sanity_check() @requires_swa @@ -1706,54 +1589,37 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures): @requires_full_swa_ps1_window4 def test_swa_branch_insert_releases_forward_overshoot(self): - cache, allocator, req_to_token_pool = build_fixture(self.cfg) - swa = cache.components[ComponentType.SWA] - branching_seqlen = 8 forward_len = 20 - req = self._make_req(req_to_token_pool) tokens = self._make_seq(1, forward_len) - req.origin_input_ids = tokens - req.output_ids = [] - req.full_untruncated_fill_ids = array("q", tokens) - req.extend_end = forward_len - req.kv.cache_protected_len = branching_seqlen - - kv_indices = self._alloc(allocator, forward_len) - req_to_token_pool.write( - (req.kv.req_pool_idx, slice(0, forward_len)), - kv_indices, + req, kv_indices = self._make_inflight_req( + cache, allocator, req_to_token_pool, tokens ) - - params = InsertParams( - key=RadixKey(array("q", tokens[:branching_seqlen])), - swa_branching_seqlen=branching_seqlen, - ) - result = mock.Mock(swa_branch_inserted=False) - swa.commit_insert_component_data( - mock.Mock(), - False, - params, - result, - [], - ) - self.assertTrue(result.swa_branch_inserted) + req.swa_branching_seqlen = branching_seqlen with envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS.override(True): - swa.cleanup_after_caching_req( - req, - is_finished=False, - insert_result=result, - ) + cache.checkpoint(req, up_to=forward_len) + # Only the branch point is cached; the forward past it is released. + self.assertEqual( + match_prefix(cache, tokens).device_prefix_len, branching_seqlen + ) expected_evicted = forward_len - 1 - self.cfg.sliding_window_size self.assertEqual(req.kv.get_evicted_seqlen(ComponentType.SWA), expected_evicted) + # The cached branch keeps SWA for its last window; the forward overshoot + # past it is released, and the request keeps its own live window. + branch_freed = branching_seqlen - 1 - self.cfg.sliding_window_size mapping = allocator.full_to_swa_index_mapping self.assertEqual( - torch.count_nonzero(mapping[kv_indices[:branching_seqlen]]).item(), - branching_seqlen, + torch.count_nonzero(mapping[kv_indices[:branch_freed]]).item(), 0 + ) + self.assertEqual( + torch.count_nonzero( + mapping[kv_indices[branch_freed:branching_seqlen]] + ).item(), + branching_seqlen - branch_freed, ) self.assertEqual( torch.count_nonzero( @@ -1765,6 +1631,8 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures): torch.count_nonzero(mapping[kv_indices[expected_evicted:]]).item(), forward_len - expected_evicted, ) + cache.unlock(req.lock) + cache.sanity_check() define_config_test_classes(UnifiedRadixCacheCoreSuite, globals()) @@ -1786,22 +1654,18 @@ class UnifiedLRUListBoundedRefreshTest( lru.reset_node_and_window_ancestors_mru( d, root, window_size=5, should_include=lambda n: n in included ) - # D, C refreshed; B contributes 2 to acc (now 6 >= 5) but is skipped; # A is not visited because the walk stops at B. self.assertEqual(self._lru_order(lru), [d, c, a]) def test_bounded_refresh_visits_only_until_window_filled(self): root, [a, b, c, d] = self._build_chain([3, 3, 3, 3]) lru = UnifiedLRUList(ComponentType.SWA, self.components) - # MRU->LRU: A, B, C, D (deepest is at the LRU tail, oldest) for n in (d, c, b, a): lru.insert_mru(n) self.assertEqual(self._lru_order(lru), [a, b, c, d]) - # window=5: walking up from D visits D(acc=3<5) and C(acc=6>=5, stop). - # Order expected: [D, C, A, B]. Why: D and C move to head in that - # order (D first, then C right after D). A and B keep their relative - # positions (they were the surviving prefix [A, B] before). + # D and C move to the head in that order; A and B keep their relative + # positions. lru.reset_node_and_window_ancestors_mru( d, root, window_size=5, should_include=lambda _n: True ) @@ -1835,13 +1699,8 @@ class UnifiedLRUListBoundedRefreshTest( class TestUnifiedMambaLRUMatchRefresh( UnifiedMambaLRUMatchRefreshFixtures, CustomTestCase ): - """A prefix-cache hit must refresh only best_match_node's mamba state in the - mamba LRU, not its ancestors. The base TreeComponent refresh is whole-chain - (reset_node_and_parents_mru), which clusters a session's states so that under - mamba-pool pressure eviction drops whole cold sessions instead of the - intermediate states reuse never needs. Guards MambaComponent.refresh_lru's - single-node MATCH_END override. - """ + """A hit refreshes only best_match_node's mamba state, so pressure evicts whole + cold sessions rather than intermediate states.""" cfg = CacheConfig(page_size=1, components=(ComponentType.FULL, ComponentType.MAMBA)) @@ -1917,8 +1776,6 @@ class TestMambaCheckpointGrid(MambaCheckpointGridFixtures, CustomTestCase): class TestMambaFinishedOvershootCheckpoint( MambaFinishedOvershootCheckpointFixtures, CustomTestCase ): - """A donated state must have consumed exactly the prefix its key names.""" - _rid = 0 cfg = CacheConfig( page_size=4, @@ -2001,7 +1858,6 @@ class TestUnifiedRadixCacheInt8MambaCheckpoint( def _component_with_cache(component_type, cache): - """A registry component instance bound to a (mock) cache and its tree_core.""" component = object.__new__(COMPONENT_REGISTRY[component_type]) component.cache = cache component.tree_core = cache.tree_core @@ -2009,8 +1865,6 @@ def _component_with_cache(component_type, cache): class TestUnifiedRadixCacheActionRouting(CustomTestCase): - """CacheAction routing: each type forwards to the right Controller API.""" - def test_retraction_uses_engine_transfer_index_domains(self): cache = object.__new__(UnifiedRadixCache) cache.is_swa_enabled = True @@ -2087,19 +1941,6 @@ class TestUnifiedRadixCacheActionRouting(CustomTestCase): 5, ComponentType.SWA, swa_value ) - def test_apply_cache_actions_applies_each_in_order(self): - cache = mock.MagicMock() - first = UpdatePendingBackupOnNodeSplit( - ack_id=1, old_node_id=1, new_node_id=2, new_child_node_id=1 - ) - second = UpdatePendingBackupOnNodeSplit( - ack_id=2, old_node_id=3, new_node_id=4, new_child_node_id=3 - ) - UnifiedRadixCache._apply_cache_actions(cache, [first, second]) - cache._apply_cache_action.assert_has_calls( - [mock.call(first), mock.call(second)] - ) - def test_eviction_free_failure_still_drains_actions_in_order(self): cache = object.__new__(UnifiedRadixCache) first = UpdatePendingBackupOnNodeSplit( @@ -2147,7 +1988,6 @@ class TestUnifiedRadixCacheActionRouting(CustomTestCase): def apply(cache, action): UnifiedRadixCache._apply_cache_action(cache, action) - # split X(10) -> [A(11), X(10)], then fragment A(11) -> [B(12), A(11)] first = UpdatePendingBackupOnNodeSplit( ack_id=7, old_node_id=10, new_node_id=11, new_child_node_id=10 ) @@ -2155,7 +1995,6 @@ class TestUnifiedRadixCacheActionRouting(CustomTestCase): ack_id=7, old_node_id=11, new_node_id=12, new_child_node_id=11 ) - # in list order, the publish list threads through both replaces cache = make_cache() apply(cache, first) apply(cache, second) @@ -2169,8 +2008,8 @@ class TestUnifiedRadixCacheActionRouting(CustomTestCase): self.assertEqual(receiver.call_args_list, [mock.call(first), mock.call(second)]) self.assertEqual(cache.linker.replace_pending_offload_node.call_count, 2) - # reversed order silently drops the second fragment - documents why - # emission order must be preserved end to end + # Reversed order silently drops the second fragment, so emission order + # must be preserved end to end. cache = make_cache() apply(cache, second) apply(cache, first) @@ -2195,8 +2034,7 @@ class TestUnifiedRadixCacheActionRouting(CustomTestCase): class TestSWADraftHistory(SWADraftHistoryFixtures, CustomTestCase): """A widened draft needs SWA history before the target window. A running checkpoint must lock that history, and an incomplete cached window must - not be reused even when the target alone could read it. The Rust rerun module - runs these contracts against both Python and Rust TreeCore. + not be reused even when the target alone could read it. """ cfg = CacheConfig( @@ -2271,15 +2109,10 @@ class TestSWADraftHistory(SWADraftHistoryFixtures, CustomTestCase): class TestSWAWindowUnderBigramKey(SWAWindowUnderBigramKeyFixtures, CustomTestCase): - """`checkpoint` has to leave the leaf it inserts holding a full - sliding window of live SWA. Otherwise the match that follows the insert - refuses that leaf, `cache_protected_len` never advances, and the next insert - frees KV the tree already owns as if it were the request's duplicate. - - An EAGLE bigram key holds one entry less than the tokens it spans, so the - leaf stops at page_floor(len(key)) rather than page_floor(seq_len), a page - lower. The eviction frontier has to be measured against the key. - """ + """checkpoint must leave its leaf with a full live SWA window, else + cache_protected_len stalls and the next insert frees tree-owned KV. EAGLE + bigram keys hold one entry fewer than seq_len, so measure the frontier + against the key.""" cfg = CacheConfig( page_size=4, diff --git a/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_hicache.py b/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_hicache.py index a21f989761..0a9ff65d45 100644 --- a/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_hicache.py +++ b/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_hicache.py @@ -1,12 +1,8 @@ -"""HiCache device <-> host backup, demotion and eviction tests for UnifiedRadixCache.""" - import unittest from array import array from dataclasses import replace from unittest import mock -import torch - from sglang.srt.mem_cache.base_prefix_cache import ( EvictParams, InitLoadBackParams, @@ -18,10 +14,8 @@ from sglang.srt.mem_cache.base_prefix_cache import ( from sglang.srt.mem_cache.radix_cache import RadixKey from sglang.srt.mem_cache.unified_cache.cache_action import ( BackupKV, - FreeDeviceKV, ) from sglang.srt.mem_cache.unified_cache.components.base import ( - CacheTransferPhase, ComponentType, ) from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci @@ -36,7 +30,6 @@ from sglang.test.kits.unified_radix_cache.config import ( requires_mamba, requires_swa, requires_swa_and_mamba, - requires_swa_only, ) from sglang.test.kits.unified_radix_cache.fixtures import ( build_fixture, @@ -158,8 +151,7 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): @requires_hicache def test_hicache_write_back_evict_drops_unbacked_leaf_when_host_full(self): - """Write-back eviction will keep freeing device KV when the host pool - is exhausted and host eviction cannot free space to prevent OOM.""" + """Write-back eviction keeps freeing device KV when the host pool is full.""" cache, allocator, req_to_token_pool = build_fixture(self.cfg) self._init_hicache(cache, write_policy="write_back") @@ -179,7 +171,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): result = cache.evict(EvictParams(num_tokens=len(seq))) self.assertGreaterEqual(result.num_tokens_evicted, len(seq)) - # The chain is gone entirely: no device hit, no host hit. m = match_prefix(cache, seq) self.assertEqual(m.device_prefix_len, 0) self.assertEqual(m.host_hit_length, 0) @@ -187,9 +178,8 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): @requires_hicache def test_hicache_write_back_drop_respects_pins_then_frees_subtree(self): - """The host-pressure drop fallback must decline while any node in the - unbacked subtree is host-pinned, then reclaim the whole subtree -- - including a demoted child's host backup -- once unpinned.""" + """The host-pressure drop fallback declines while any node in the unbacked + subtree is host-pinned, then reclaims the whole subtree once unpinned.""" cache, allocator, req_to_token_pool = build_fixture(self.cfg) self._init_hicache(cache, write_policy="write_back") host_pool = cache.cache_controller.mem_pool_host @@ -217,13 +207,11 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): # From here every backup fails (controller.write returns None), so # each evict() attempts the drop fallback on the parent. with mock.patch.object(cache.cache_controller, "write", return_value=None): - # Pinned subtree root: drop declines, chain stays intact. parent_lock_params = cache.inc_host_lock_ref(parent).to_dec_params() result = cache.evict(EvictParams(num_tokens=len(parent_seq))) self.assertEqual(result.num_tokens_evicted, 0) cache.dec_host_lock_ref(parent, parent_lock_params) - # Pinned host-only descendant: drop declines as well. child_lock_params = cache.inc_host_lock_ref(child).to_dec_params() result = cache.evict(EvictParams(num_tokens=len(parent_seq))) self.assertEqual(result.num_tokens_evicted, 0) @@ -231,7 +219,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): self.assertEqual(m.device_prefix_len, len(parent_seq)) cache.dec_host_lock_ref(child, child_lock_params) - # Unpinned: the subtree drops and the child's host slots return. result = cache.evict(EvictParams(num_tokens=len(parent_seq))) self.assertGreaterEqual(result.num_tokens_evicted, len(parent_seq)) self.assertEqual(host_pool.available_size(), baseline_host) @@ -244,8 +231,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): lambda cfg: cfg == CacheConfig(), "single Full page-size-1 ownership regression" ) def test_hicache_split_preserves_inflight_full_host_pin(self): - """A split prefix keeps an outstanding storage backup's host ownership.""" - cache, allocator, req_to_token_pool = build_fixture(self.cfg) self._init_hicache(cache, write_policy="write_back") host_pool = cache.cache_controller.mem_pool_host @@ -266,8 +251,8 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): self.assertEqual(node_host_lock_ref(cache, split_parent, ComponentType.FULL), 1) self.assertEqual(node_host_lock_ref(cache, leaf, ComponentType.FULL), 1) - # Before the fix the prefix had host_lock_ref == 0 and these slots were - # reclaimed while the storage thread could still be reading them. + # The prefix must hold a host lock, else these slots are reclaimed while + # the storage thread still reads them. self.assertEqual(cache.evict_host(len(split_host)), 0) # A lock acquired after the split covers only the suffix and gets a new @@ -293,7 +278,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): cache.sanity_check() def test_hicache_split_preserves_inflight_swa_host_pin(self): - """A split SWA host segment stays pinned and releases without leaks.""" expected_cfg = CacheConfig( page_size=1, components=(ComponentType.FULL, ComponentType.SWA), @@ -321,9 +305,8 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): lock_params.get_lock_uuid(ComponentType.SWA, lock_host=True) ) - # Diverge inside the locked SWA window. The leaf handle stays on the - # suffix while the lock boundary and copied host refs move to the new - # prefix, so neither half may be reclaimed by component host eviction. + # Diverging inside the locked SWA window moves the lock boundary and copied + # host refs to the new prefix; neither half may be reclaimed. self._insert(cache, allocator, req_to_token_pool, [1, 2, 3, 4, 5, 6, 9, 10]) split_parent = node_parent(cache, leaf) self.assertNotEqual(split_parent, parent_before_split) @@ -339,9 +322,8 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): @requires_hicache def test_hicache_evict_device_leaf_aborts_demote_when_backup_fails(self): - """The tree op defers a write-back victim's demote: it returns a BackupKV - without freeing device, and _demote requires a completed backup, so a - failed backup leaves the node recoverable on device.""" + """The tree op defers a write-back victim's demote, so a failed backup + leaves the node recoverable on device.""" cache, allocator, req_to_token_pool = build_fixture(self.cfg) self._init_hicache(cache, write_policy="write_back") ct = ComponentType.FULL @@ -354,7 +336,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): self.assertFalse(cache.tree_core.is_backuped(node)) self.assertFalse(cache.tree_core.is_full_device_evicted(node)) - # Tree op defers: returns a BackupKV and leaves the node on device (no demote). leaf_result = cache.tree_core.evict_device_leaf(node, is_write_back=True) self.assertIsNotNone(leaf_result.backup_kv) cache._free_values(leaf_result.device_frees, leaf_result.host_frees) @@ -369,33 +350,8 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): cache.sanity_check() - @requires_hicache - def test_hicache_evict_to_host(self): - cache, allocator, req_to_token_pool = self._build_hicache_fixture() - seq = self._make_seq(1, 2) - self._insert(cache, allocator, req_to_token_pool, seq) - - m = match_prefix(cache, seq) - node = m.last_device_node - - self._backup_node(cache, node) - self.assertTrue(cache.tree_core.is_backuped(node)) - - result = cache.evict(EvictParams(num_tokens=len(seq))) - self.assertGreaterEqual(result.num_tokens_evicted, len(seq)) - - self.assertTrue(cache.tree_core.is_full_device_evicted(node)) - self.assertTrue(cache.tree_core.is_backuped(node)) - self.assertIsNone(node_device_value(cache, node, ComponentType.FULL)) - self.assertIsNotNone(node_host_value(cache, node, ComponentType.FULL)) - - self.assertFalse(cache.tree_core.is_device_evictable_leaf(node)) - self.assertTrue(cache.tree_core.is_host_evictable_leaf(node)) - cache.sanity_check() - @requires_hicache def test_hicache_partial_match_splits_evicted_backed_up_node(self): - """Partial matches on host-only nodes must keep the host prefix usable.""" cache, allocator, req_to_token_pool = self._build_hicache_fixture() ps = self.cfg.page_size seq = self._make_seq(1, 4) @@ -428,7 +384,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): self.assertNotEqual(split_parent, cache.root_node_handle()) self.assertTrue(cache.tree_core.is_full_device_evicted(split_parent)) self.assertTrue(cache.tree_core.is_backuped(split_parent)) - # root -> split_parent keys reconstruct expected_prefix prefix_tokens: list[int] = [] chain = [] cur = split_parent @@ -438,7 +393,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): for n in reversed(chain): prefix_tokens.extend(node_token_ids(cache, n)) self.assertEqual(prefix_tokens, expected_prefix) - # the diverged suffix stays as evicted+backuped descendant(s) suffix_tokens: list[int] = [] cur = split_parent while children := node_children(cache, cur): @@ -462,6 +416,10 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): self._backup_node(cache, node) cache.evict(EvictParams(num_tokens=len(seq))) + self.assertTrue(cache.tree_core.is_backuped(node)) + self.assertIsNone(node_device_value(cache, node, ComponentType.FULL)) + self.assertIsNotNone(node_host_value(cache, node, ComponentType.FULL)) + self.assertFalse(cache.tree_core.is_device_evictable_leaf(node)) self.assertTrue(cache.tree_core.is_full_device_evicted(node)) self.assertTrue(cache.tree_core.is_host_evictable_leaf(node)) @@ -483,7 +441,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): self.assertFalse(cache.tree_core.is_backuped(node)) self.assertFalse(cache.tree_core.is_full_device_evicted(node)) - # Backup failure falls back to dropping the unlocked subtree. with mock.patch.object(cache, "_execute_and_commit_kv_backup", return_value=0): result = cache.evict(EvictParams(num_tokens=len(seq))) @@ -497,7 +454,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): lambda cfg: not cfg.has_mamba, "SWA-only path keeps the split setup simple" ) def test_hicache_write_through_offloads_swa_split_leaf(self): - ps = self.cfg.page_size cache, allocator, _ = build_fixture(self.cfg) self._init_hicache(cache) @@ -525,7 +481,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): cache.sanity_check() def test_hicache_evict_to_host_updates_aux_lru(self): - """Aux components (MAMBA / SWA) move from device LRU to host LRU on D->H eviction.""" aux_types = [ ct for ct in (ComponentType.MAMBA, ComponentType.SWA) @@ -556,9 +511,8 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): @requires_mamba def test_hicache_write_back_internal_mamba_evict_demotes_state(self): - """write_back: an internal node's tombstoned mamba state is demoted - to host, keeping the node a valid match boundary so the KV beneath - it stays servable.""" + """write_back: an internal node's tombstoned mamba state is demoted to host, + keeping the node a valid match boundary.""" cache, req_to_token_pool, seq_a = self._build_internal_mamba_fixture( "write_back" ) @@ -638,8 +592,8 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): # The backed parent stops the Full backup chain, while the SWA window # reaches the unbacked grandparent and marks it under the anchor's ack. insert(3 * page) - # Queue the asynchronous window backup explicitly: write-back insertion - # no longer publishes it, but Mamba eviction must still respect its ack. + # Queue the asynchronous window backup explicitly; Mamba eviction must + # respect its ack. cache._execute_and_commit_kv_backup(BackupKV(node_ids=[anchor])) self.assertEqual( cache.ongoing_write_through[anchor].publish_node_ids, nodes[:3] @@ -666,7 +620,7 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): @requires_mamba def test_hicache_write_through_internal_mamba_evict_keeps_drop(self): - """Non-write_back policies keep the legacy tombstone-and-drop.""" + """Non-write_back policies tombstone and drop.""" cache, _, seq_a = self._build_internal_mamba_fixture("write_through") cache.evict(EvictParams(num_tokens=0, mamba_num=10)) @@ -675,15 +629,14 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): self.assertEqual( m.device_prefix_len, 0, - "write_through keeps the legacy drop: frontier capped at root", + "write_through keeps the drop: frontier capped at root", ) cache.sanity_check() @requires_swa def test_hicache_write_back_internal_swa_evict_demotes_kv(self): - """write_back: an internal node's tombstoned SWA KV is demoted to - host, keeping the node a valid match boundary so the KV beneath it - stays servable instead of losing a window of matchable prefix.""" + """write_back: an internal node's tombstoned SWA KV is demoted to host, + keeping the node a valid match boundary.""" cache, req_to_token_pool, seq_a, seq_b = self._build_internal_swa_fixture( "write_back" ) @@ -753,44 +706,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): self.assertEqual(result.last_host_node, result.last_device_node) self.assertEqual(result.host_hit_length, 0) - @requires_swa_only - @requires( - lambda cfg: cfg.sliding_window_size > cfg.page_size, - "the window must reach past the leaf's own page", - ) - def test_swa_backup_collector_is_shared_by_both_call_sites(self): - """needs_incremental_backup and the BACKUP_HOST transfer read one - collector: cache mode walks the window past a host-backed target to - its device-only ancestor, buffer mode stages the target alone.""" - # This test exercises the Python component's collector directly. - cache, allocator, req_to_token_pool = self._build_hicache_fixture( - tree_core_backend="python" - ) - chain = self._build_chain_pages(cache, allocator, req_to_token_pool, 2) - if len(chain) < 2: - self.skipTest("chain too short") - parent, leaf = chain[-2], chain[-1] - swa = cache.components[ComponentType.SWA] - leaf_node = cache.tree_core.node_by_id(leaf) - cache.tree_core.set_component_host_value_raw( - leaf, - ComponentType.SWA, - node_device_value(cache, leaf, ComponentType.SWA).clone(), - ) - - self.assertTrue(swa.needs_incremental_backup(leaf_node)) - xfer = cache.tree_core.build_hicache_transfers( - ComponentType.SWA, leaf, CacheTransferPhase.BACKUP_HOST - )[0] - self.assertEqual(xfer.nodes_to_load, [parent]) - - cache.tree_core.set_host_memory_buffer_only() - self.assertTrue(swa.needs_incremental_backup(leaf_node)) - xfer = cache.tree_core.build_hicache_transfers( - ComponentType.SWA, leaf, CacheTransferPhase.BACKUP_HOST - )[0] - self.assertEqual(xfer.nodes_to_load, [leaf]) - def test_evict_host_drains_freed_host_values_to_the_pools(self): cache, allocator, req_to_token_pool = self._build_hicache_fixture() chain = self._build_chain_pages(cache, allocator, req_to_token_pool, 3) @@ -807,7 +722,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): ComponentType.MAMBA: "mamba_pool_host", } for ct in cache.tree_components: - # set by the hicache assembler for every enabled component pool = getattr(cache, host_pool_attr[ct]) if pool is None: continue @@ -815,7 +729,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): evicted = cache.evict_host(pool.size, ct) if evicted == 0: continue - # the drain must return every freed host value to the pool self.assertGreater(pool.available_size(), available_before) @requires_full_mamba_ps1 @@ -841,38 +754,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): ) self.assertEqual(result.host_hit_length, node_key_length(cache, leaf)) - @requires_full_mamba_ps1 - def test_mamba_has_host_value_only_predicate(self): - """Needs a slot only when mamba is host-only (device evicted, host backed up).""" - cache, allocator, req_to_token_pool = self._build_hicache_fixture() - chain = self._build_chain_pages(cache, allocator, req_to_token_pool, 3) - if not chain: - self.skipTest("chain too short") - node = chain[-1] - dev = torch.tensor([0], dtype=torch.int64) - host = torch.tensor([0], dtype=torch.int64) - - # host-only: device evicted, host backup present -> restore needs a slot - cache.tree_core.set_component_device_value_raw(node, ComponentType.MAMBA, None) - cache.tree_core.set_component_host_value_raw(node, ComponentType.MAMBA, host) - self.assertTrue( - cache.tree_core.component_has_host_value_only(node, ComponentType.MAMBA) - ) - # device + host: device value present -> no restore, no slot (the D+H bug case) - cache.tree_core.set_component_device_value_raw(node, ComponentType.MAMBA, dev) - self.assertFalse( - cache.tree_core.component_has_host_value_only(node, ComponentType.MAMBA) - ) - # device-only / neither: nothing to restore - cache.tree_core.set_component_host_value_raw(node, ComponentType.MAMBA, None) - self.assertFalse( - cache.tree_core.component_has_host_value_only(node, ComponentType.MAMBA) - ) - cache.tree_core.set_component_device_value_raw(node, ComponentType.MAMBA, None) - self.assertFalse( - cache.tree_core.component_has_host_value_only(node, ComponentType.MAMBA) - ) - @requires_full_swa_ps1 def test_hicache_swa_host_best_match_keeps_device_anchor(self): cache, allocator, req_to_token_pool = self._build_hicache_fixture() @@ -919,25 +800,16 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): @requires_full_swa_ps1_window4 def test_hicache_swa_backup_window_stops_at_pending_ancestor(self): - cache, allocator, req_to_token_pool = self._build_hicache_fixture() chain = self._build_chain_pages(cache, allocator, req_to_token_pool, 3) if len(chain) < 3: self.skipTest("chain collapsed below the pending ancestor test") - c = chain[-1] - c_swa = node_device_value(cache, c, ComponentType.SWA).clone() - - # First transfer: publish Full for C only. C's SWA is a device tombstone - # (decode-evicted, never backed up), so the write-through ack stays - # pending on a node the SWA backup window has nothing to send for. - cache.tree_core.set_component_device_value_raw(c, ComponentType.SWA, None) - if cache.tree_core.is_node_in_device_lru(c, ComponentType.SWA): - cache.tree_core.remove_node_from_device_lru(c, ComponentType.SWA) - cache.tree_core.set_component_evictable_size( - ComponentType.SWA, - cache.tree_core.component_evictable_size(ComponentType.SWA) - len(c_swa), - ) + tokens = self._match_tokens_for_chain(cache, chain) + # C's SWA is decode-evicted before it is ever backed up, so the + # write-through of C publishes Full only and its ack stays pending on a + # node the SWA backup window has nothing to send for. + self._evict_device_component(cache, c, ComponentType.SWA) self.assertGreater( cache._execute_and_commit_kv_backup(BackupKV(node_ids=[c])), 0, @@ -945,25 +817,19 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): self.assertEqual(cache.tree_core.get_write_through_pending_id(c), c) self.assertIsNotNone(node_host_value(cache, c, ComponentType.FULL)) self.assertIsNone(node_host_value(cache, c, ComponentType.SWA)) - - # SWA is reconstructed on device before the first ack, the way a - # load-back commit stores it: under the pending segment lock the value - # counts as protected until the ack releases it. The next incremental - # SWA backup must treat C as the boundary and back up only the newly - # inserted descendant. - cache.tree_core.set_component_device_value(c, ComponentType.SWA, c_swa) - tokens = self._match_tokens_for_chain(cache, chain) + # A request re-inserting C's tokens recovers its SWA on device before + # the first ack. The next incremental SWA backup must treat C as the + # boundary and back up only the newly inserted descendant. + self._insert(cache, allocator, req_to_token_pool, tokens) + self.assertIsNotNone(node_device_value(cache, c, ComponentType.SWA)) next_tokens = tokens + self._make_seq(9000, 1) cache.write_through_threshold = 1 - self._insert(cache, allocator, req_to_token_pool, next_tokens) - d = match_prefix(cache, next_tokens).last_device_node self.assertEqual(cache.tree_core.get_write_through_pending_id(c), c) self.assertEqual(cache.tree_core.get_write_through_pending_id(d), d) self.assertIsNone(node_host_value(cache, c, ComponentType.SWA)) self.assertIsNotNone(node_host_value(cache, d, ComponentType.SWA)) - cache.writing_check(write_back=True) self.assertIsNone(cache.tree_core.get_write_through_pending_id(c)) self.assertIsNone(cache.tree_core.get_write_through_pending_id(d)) @@ -974,59 +840,39 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): lambda cfg: not cfg.has_mamba, "SWA-only path keeps the chain construction simple", ) - def test_hicache_swa_finalize_match_result(self): - """finalize_match_result accumulates host_value lengths of SWA tombstones - within the trailing sliding window into ``swa_host_hit_length``. Out-of-window - tombstones and chains fully on device must leave ``swa_host_hit_length`` at 0. - ``host_hit_length`` is Full-KV only and is never written by SWA. - """ - - cache, _, _, chain, window_pages = self._swa_finalize_setup() - leaf = chain[-1] + def test_hicache_swa_host_hit_counts_tombstones_inside_the_window(self): + """A host-backed SWA tombstone inside the trailing sliding window stops + the device match before it: the Full-backed tail from the tombstone on + is a host hit, and the tombstone's own SWA counts in + ``swa_host_hit_length``.""" ps = self.cfg.page_size - - cases = [ - ("all_on_device", None, 0), - ("tombstone_in_window", chain[-window_pages], ps), - ("tombstone_outside_window", chain[-(window_pages + 1)], 0), - ] - for name, victim, expected in cases: + window_pages = (self.cfg.sliding_window_size + ps - 1) // ps + for name, victim in ( + ("all_on_device", None), + ("tombstone_in_window", -window_pages), + ("tombstone_outside_window", -(window_pages + 1)), + ): with self.subTest(name): - # Reset SWA state for each subcase. - for n in chain: - device_value = node_device_value(cache, n, ComponentType.SWA) - host_value = node_host_value(cache, n, ComponentType.SWA) - if device_value is None and host_value is not None: - cache.tree_core.set_component_device_value_raw( - n, ComponentType.SWA, host_value.clone() - ) + cache, allocator, req_to_token_pool = self._build_hicache_fixture() + chain = self._build_chain_pages( + cache, allocator, req_to_token_pool, window_pages + 2 + ) + if len(chain) <= window_pages: + self.skipTest("chain collapsed below the window length") + tokens = self._match_tokens_for_chain(cache, chain) + self._backup_node(cache, chain[-1]) + host_tail, swa_host = 0, 0 if victim is not None: - cache.tree_core.set_component_device_value_raw( - victim, ComponentType.SWA, None + self._evict_device_component( + cache, chain[victim], ComponentType.SWA ) - - result = MatchResult( - device_prefix_len=0, - last_device_node=leaf, - last_host_node=leaf, - best_match_node=leaf, - host_hit_length=0, - cache_protected_len=7, - cache_actions=(FreeDeviceKV(indices=[]),), - ) - result = cache.tree_core.finalize_component_match_result( - ComponentType.SWA, - result=result, - params=MatchPrefixParams( - key=RadixKey(array("q", self._make_seq(1, 1))) - ), - value_chunks=[], - best_value_len=0, - ) - self.assertEqual(result.host_hit_length, 0) - self.assertEqual(result.swa_host_hit_length, expected) - self.assertEqual(result.cache_protected_len, 7) - self.assertEqual(result.cache_actions, (FreeDeviceKV(indices=[]),)) + if name == "tombstone_in_window": + host_tail = sum(node_key_length(cache, n) for n in chain[victim:]) + swa_host = node_key_length(cache, chain[victim]) + result = match_prefix(cache, tokens) + self.assertEqual(result.device_prefix_len, len(tokens) - host_tail) + self.assertEqual(result.host_hit_length, host_tail) + self.assertEqual(result.swa_host_hit_length, swa_host) def test_hicache_swa_finalize_anchored_on_best_match_node(self): cache, _, _, y, x, _ = self._swa_anchor_setup() @@ -1057,62 +903,41 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): ) def test_hicache_swa_temp_lock_does_not_release_restored_tombstone(self): """A temporary scheduler lock that skipped a SWA tombstone must not - release later load-back/request locks after the tombstone is restored. - """ - - cache, allocator, _, chain, _ = self._swa_finalize_setup() + release later load-back/request locks after the tombstone is restored.""" + cache, allocator, req_to_token_pool = self._build_hicache_fixture() + chain = self._build_chain_pages(cache, allocator, req_to_token_pool, 3) + if len(chain) < 3: + self.skipTest("chain too short") leaf = chain[-1] - tombstone = leaf - old_swa = node_device_value(cache, tombstone, ComponentType.SWA) - self.assertIsNotNone(old_swa) + tokens = self._match_tokens_for_chain(cache, chain) + self._backup_node(cache, leaf) + self.assertIsNotNone(node_host_value(cache, leaf, ComponentType.SWA)) + self._evict_device_component(cache, leaf, ComponentType.SWA) + self.assertIsNone(node_device_value(cache, leaf, ComponentType.SWA)) - cache.tree_core.set_component_device_value_raw( - tombstone, ComponentType.SWA, None - ) - cache.tree_core.remove_node_from_device_lru(tombstone, ComponentType.SWA) - cache.tree_core.insert_node_into_host_lru(tombstone, ComponentType.SWA) - cache.tree_core.set_component_evictable_size( - ComponentType.SWA, - cache.tree_core.component_evictable_size(ComponentType.SWA) - len(old_swa), - ) + def swa_ref(): + return node_device_lock_ref(cache, leaf, ComponentType.SWA) temp_lock = cache.inc_lock_ref(leaf) - self.assertEqual(node_device_lock_ref(cache, tombstone, ComponentType.SWA), 1) - - xfer = cache.tree_core.build_hicache_transfers( - ComponentType.SWA, leaf, CacheTransferPhase.LOAD_BACK - )[0] - new_swa = allocator.swa_attn_allocator.alloc(int(xfer.host_indices.numel())) - self.assertIsNotNone(new_swa) - xfer.device_indices = new_swa - load_actions = [] - cache.tree_core.commit_hicache_transfers( - leaf, - CacheTransferPhase.LOAD_BACK, - {ComponentType.SWA: [xfer]}, - cache_actions=load_actions, - ) - cache._apply_cache_actions(load_actions) - - load_back_lock = cache.inc_lock_ref(leaf) + self.assertEqual(swa_ref(), 1) + self._load_back_through_admission(cache, req_to_token_pool, tokens) + self.assertIsNotNone(node_device_value(cache, leaf, ComponentType.SWA)) request_lock = cache.inc_lock_ref(leaf) - self.assertEqual(node_device_lock_ref(cache, tombstone, ComponentType.SWA), 3) - + self.assertEqual(swa_ref(), 3) cache.dec_lock_ref(leaf, temp_lock.to_dec_params()) - self.assertEqual(node_device_lock_ref(cache, tombstone, ComponentType.SWA), 2) - - cache.dec_lock_ref(leaf, load_back_lock.to_dec_params()) + self.assertEqual(swa_ref(), 2) + self._finish_pending_loads(cache) + self._release_ongoing_load_back_locks(cache) cache.dec_lock_ref(leaf, request_lock.to_dec_params()) - self.assertEqual(node_device_lock_ref(cache, tombstone, ComponentType.SWA), 0) + self.assertEqual(swa_ref(), 0) + cache.sanity_check() @requires_hicache def test_hicache_full_temp_lock_covers_evicted_anchor_and_mirrors_on_release( self, ): - """Segment locks count the evicted anchor too (no skip receipts), so - a value restored mid-hold stays correctly attributed: each release - takes back exactly its own ref regardless of interleaved holders. - """ + """Segment locks count the evicted anchor too (no skip receipts), so a value + restored mid-hold stays correctly attributed.""" ps = self.cfg.page_size if 3 * ps > self.cfg.kv_size // 2: self.skipTest("kv_size too small") @@ -1120,53 +945,26 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): chain = self._build_chain_pages(cache, allocator, req_to_token_pool, 3) if len(chain) < 3: self.skipTest("chain too short") - a, y, anchor = chain - self._simulate_backup_tree(cache) + anchor = chain[-1] + self._backup_node(cache, anchor) + cache.evict(EvictParams(num_tokens=node_key_length(cache, anchor))) + self.assertTrue(cache.tree_core.is_full_device_evicted(anchor)) - anchor_value = node_device_value(cache, anchor, ComponentType.FULL) - # Simulate the anchor's FULL device eviction: drop the value and take - # its tokens out of the evictable ledger, as a real evict would. - cache.tree_core.set_component_device_value_raw(anchor, ComponentType.FULL, None) - cache.tree_core.set_component_evictable_size( - ComponentType.FULL, - cache.tree_core.component_evictable_size(ComponentType.FULL) - - len(anchor_value), - ) - - self.assertEqual(node_device_lock_ref(cache, anchor, ComponentType.FULL), 0) - self.assertEqual(node_device_lock_ref(cache, y, ComponentType.FULL), 0) - self.assertEqual(node_device_lock_ref(cache, a, ComponentType.FULL), 0) + def full_refs(): + return [node_device_lock_ref(cache, n, ComponentType.FULL) for n in chain] + self.assertEqual(full_refs(), [0, 0, 0]) temp_lock = cache.inc_lock_ref(anchor) - self.assertEqual(node_device_lock_ref(cache, anchor, ComponentType.FULL), 1) - self.assertEqual(node_device_lock_ref(cache, y, ComponentType.FULL), 1) - self.assertEqual(node_device_lock_ref(cache, a, ComponentType.FULL), 1) - - # Restore the value mid-hold: a value materialized under lock is - # protected until the last release, exactly as a load-back credits it. - cache.tree_core.set_component_device_value_raw( - anchor, ComponentType.FULL, anchor_value - ) - cache.tree_core.set_component_protected_size( - ComponentType.FULL, - cache.tree_core.component_protected_size(ComponentType.FULL) - + len(anchor_value), - ) - + self.assertEqual(full_refs(), [1, 1, 1]) + # Restore the anchor mid-hold. + self._load_back_node(cache, anchor) second_lock = cache.inc_lock_ref(anchor) - self.assertEqual(node_device_lock_ref(cache, anchor, ComponentType.FULL), 2) - self.assertEqual(node_device_lock_ref(cache, y, ComponentType.FULL), 2) - self.assertEqual(node_device_lock_ref(cache, a, ComponentType.FULL), 2) - + self.assertEqual(full_refs(), [2, 2, 2]) cache.dec_lock_ref(anchor, temp_lock.to_dec_params()) - self.assertEqual(node_device_lock_ref(cache, anchor, ComponentType.FULL), 1) - self.assertEqual(node_device_lock_ref(cache, y, ComponentType.FULL), 1) - self.assertEqual(node_device_lock_ref(cache, a, ComponentType.FULL), 1) - + self.assertEqual(full_refs(), [1, 1, 1]) cache.dec_lock_ref(anchor, second_lock.to_dec_params()) - self.assertEqual(node_device_lock_ref(cache, anchor, ComponentType.FULL), 0) - self.assertEqual(node_device_lock_ref(cache, y, ComponentType.FULL), 0) - self.assertEqual(node_device_lock_ref(cache, a, ComponentType.FULL), 0) + self.assertEqual(full_refs(), [0, 0, 0]) + cache.sanity_check() @requires_mamba @requires( @@ -1175,54 +973,32 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): ) def test_hicache_mamba_temp_lock_does_not_release_restored_tombstone(self): """A temporary scheduler lock that skipped a Mamba tombstone must not - release later load-back/request locks after the tombstone is restored. - """ - - cache, allocator, req_to_token_pool = build_fixture(self.cfg) + release later load-back/request locks after the tombstone is restored.""" + cache, allocator, req_to_token_pool = self._build_hicache_fixture() seq = self._make_seq(1, 2) self._insert(cache, allocator, req_to_token_pool, seq) - m = match_prefix(cache, seq) - node = m.last_device_node - old_mamba = node_device_value(cache, node, ComponentType.MAMBA) - self.assertIsNotNone(old_mamba) - self._simulate_backup(cache, node) + node = match_prefix(cache, seq).last_device_node + self._backup_node(cache, node) + self.assertIsNotNone(node_host_value(cache, node, ComponentType.MAMBA)) + self._evict_device_component(cache, node, ComponentType.MAMBA) + self.assertIsNone(node_device_value(cache, node, ComponentType.MAMBA)) - cache.tree_core.set_component_device_value_raw(node, ComponentType.MAMBA, None) - cache.tree_core.remove_node_from_device_lru(node, ComponentType.MAMBA) - cache.tree_core.insert_node_into_host_lru(node, ComponentType.MAMBA) - cache.tree_core.set_component_evictable_size( - ComponentType.MAMBA, - cache.tree_core.component_evictable_size(ComponentType.MAMBA) - - len(old_mamba), - ) + def mamba_ref(): + return node_device_lock_ref(cache, node, ComponentType.MAMBA) temp_lock = cache.inc_lock_ref(node) - self.assertEqual(node_device_lock_ref(cache, node, ComponentType.MAMBA), 1) - - xfer = cache.tree_core.build_hicache_transfers( - ComponentType.MAMBA, node, CacheTransferPhase.LOAD_BACK - )[0] - new_mamba = req_to_token_pool.mamba_allocator.alloc(1) - self.assertIsNotNone(new_mamba) - xfer.device_indices = new_mamba - cache.tree_core.commit_hicache_transfers( - node, - CacheTransferPhase.LOAD_BACK, - {ComponentType.MAMBA: [xfer]}, - cache_actions=[], - ) - - load_back_lock = cache.inc_lock_ref(node) + self.assertEqual(mamba_ref(), 1) + self._load_back_through_admission(cache, req_to_token_pool, seq) + self.assertIsNotNone(node_device_value(cache, node, ComponentType.MAMBA)) request_lock = cache.inc_lock_ref(node) - self.assertEqual(node_device_lock_ref(cache, node, ComponentType.MAMBA), 3) - + self.assertEqual(mamba_ref(), 3) cache.dec_lock_ref(node, temp_lock.to_dec_params()) - self.assertEqual(node_device_lock_ref(cache, node, ComponentType.MAMBA), 2) - - cache.dec_lock_ref(node, load_back_lock.to_dec_params()) + self.assertEqual(mamba_ref(), 2) + self._finish_pending_loads(cache) + self._release_ongoing_load_back_locks(cache) cache.dec_lock_ref(node, request_lock.to_dec_params()) - self.assertEqual(node_device_lock_ref(cache, node, ComponentType.MAMBA), 0) - # The commit ran under held locks: the restored value must have been + self.assertEqual(mamba_ref(), 0) + # The restore ran under held locks: the restored value must have been # credited to protected, or the ledger drifts on the final release. cache.sanity_check() @@ -1274,7 +1050,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): "SWA boundary splits deepen the chain non-deterministically", ) def test_build_backup_kv_action_orders_ancestors_first(self): - """write-through backs up unbacked ancestors parent-first; write-back only the leaf.""" cache, allocator, req_to_token_pool = self._build_hicache_fixture() top_seq = self._make_seq(1, 1) @@ -1293,11 +1068,9 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): self.assertFalse(cache.tree_core.is_backuped(mid)) self.assertFalse(cache.tree_core.is_backuped(top)) - # write-through: every unbacked ancestor, top-most first, child last write_through = cache.tree_core.build_backup_node_ids(leaf, write_back=False) self.assertEqual(write_through, [top, mid, leaf]) - # write-back: only the eviction victim, even with unbacked ancestors write_back = cache.tree_core.build_backup_node_ids(leaf, write_back=True) self.assertEqual(write_back, [leaf]) @@ -1306,7 +1079,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): @requires_swa @requires_hicache def test_write_back_defers_swa_window_backup_until_eviction(self): - """Reinserting backed Full nodes must not publish freshly recovered SWA.""" ps = self.cfg.page_size if self.cfg.sliding_window_size <= 2 * ps: self.skipTest("window must span past the leaf and its parent to reach c") @@ -1338,7 +1110,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): self.assertIsNotNone(node_device_value(cache, node, ComponentType.SWA)) self.assertIsNone(node_host_value(cache, node, ComponentType.SWA)) - # Two requests in one batch finish at t1 and t2; neither ack drains. with mock.patch.object( cache, "_execute_and_commit_kv_backup", @@ -1352,7 +1123,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures): self.assertIsNone(cache.tree_core.get_write_through_pending_id(node)) self.assertIsNone(node_host_value(cache, node, ComponentType.SWA)) self.assertEqual(cache.cache_controller.write_queue, []) - # Write-back defers the unbacked Full prefix to eviction. self.assertFalse(cache.tree_core.is_backuped(c)) self.assertFalse(cache.tree_core.is_backuped(d)) diff --git a/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_insert_walk.py b/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_insert_walk.py index 199425b0d8..389d70f5fc 100644 --- a/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_insert_walk.py +++ b/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_insert_walk.py @@ -1,5 +1,3 @@ -"""Resumable insert walk, returned-value drain and segment lock tests for UnifiedRadixCache.""" - import sys import unittest from array import array @@ -134,7 +132,6 @@ class TestResumableInsertWalk(UnifiedRadixCacheFixtures, CustomTestCase): self._insert(cache, allocator, req_to_token_pool, [1, 2, 3, 4]) self.assertFalse(cache.tree_core.has_ongoing_insert()) - # The tree stays usable and the crossing re-fires on the next walk. self._insert(cache, allocator, req_to_token_pool, [1, 2, 3, 4]) cache.writing_check(write_back=True) (ancestor,) = node_children(cache, cache.root_node_handle()) @@ -231,7 +228,6 @@ class TestResumableInsertWalk(UnifiedRadixCacheFixtures, CustomTestCase): self.assertTrue(cache.tree_core.is_backuped(middle)) self.assertFalse(cache.tree_core.is_backuped(top)) - # The device insert unevicts the middle and adds the deep leaf. self._insert(cache, allocator, req_to_token_pool, list(range(1, 13))) (deep,) = node_children(cache, middle) @@ -386,11 +382,9 @@ class TestResumableInsertWalkSWA(ResumableInsertWalkSWAFixtures, CustomTestCase) (window_node,) = node_children(cache, prefix_node) self.assertIsNone(node_device_value(cache, prefix_node, ComponentType.SWA)) - # Re-inserting fully in-window recovers the prefix span's SWA data. cache.insert(InsertParams(key=key, value=self._alloc(allocator, len(seq)))) self.assertIsNotNone(node_device_value(cache, prefix_node, ComponentType.SWA)) - # SWA eviction takes the recovered span and keeps the window leaf. cache.evict(EvictParams(num_tokens=0, swa_num_tokens=sw)) self.assertIsNone(node_device_value(cache, prefix_node, ComponentType.SWA)) self.assertIsNotNone(node_device_value(cache, window_node, ComponentType.SWA)) @@ -457,14 +451,13 @@ class TestResumableInsertWalkWriteBack(UnifiedRadixCacheFixtures, CustomTestCase def test_drop_fallback_frees_host_for_later_backups_same_round(self): """Host slots reclaimed by the drop fallback itself (an interior host-only descendant) must be reusable by later write-back backups in - the same eviction round (pre-split freed them inline).""" + the same eviction round.""" if self._skip_unsupported_hicache_test(): return cache, allocator, req_to_token_pool = build_fixture(self.cfg) self._init_hicache(cache, write_policy="write_back") host_pool = cache.cache_controller.mem_pool_host - # Chain 1: unbacked parent -> host-only child -> host-only grandchild. p1 = self._make_seq(1, 2) self._insert(cache, allocator, req_to_token_pool, p1) c1 = p1 + self._make_seq(1000, 2) @@ -518,8 +511,7 @@ class TestResumableInsertWalkWriteBack(UnifiedRadixCacheFixtures, CustomTestCase @unittest.skipUnless(torch.cuda.is_available(), "cache fixtures need CUDA") class TestReturnedValuesDrain(UnifiedRadixCacheFixtures, CustomTestCase): - """Drain contract of the returned-values eviction API: every tree-core step - result is drained exactly once, in per-component insertion order.""" + """Each tree-core step result is drained exactly once, in insertion order.""" cfg = CacheConfig() @@ -629,45 +621,6 @@ class TestReturnedValuesDrain(UnifiedRadixCacheFixtures, CustomTestCase): self.assertIs(device_frees[ComponentType.FULL][0], sentinel) self.assertIs(host_frees[ComponentType.FULL][0], sentinel) - def test_free_values_frees_in_component_insertion_order(self): - """Device frees apply before host frees, each in per-component - insertion order — the allocator free-list order the pre-split inline - frees produced.""" - cache, allocator, req_to_token_pool = build_fixture(self.cfg) - order = [ComponentType.FULL, ComponentType.SWA, ComponentType.MAMBA] - device_frees = defaultdict(list) - host_frees = defaultdict(list) - for ct in order: - device_frees[ct].append(torch.tensor([1])) - host_frees[ct].append(torch.tensor([2])) - - freed = [] - fake_components = { - ct: mock.MagicMock( - free_host_values=mock.MagicMock( - side_effect=lambda values, ct=ct: freed.append(("host", ct)) - ) - ) - for ct in order - } - with ( - mock.patch.object( - cache, - "_apply_cache_action", - side_effect=lambda action: freed.append( - ("device", action.component_type) - ), - ), - mock.patch.dict(cache.components, fake_components), - ): - cache._free_values(device_frees, host_frees) - - self.assertEqual( - freed, [("device", ct) for ct in order] + [("host", ct) for ct in order] - ) - self.assertFalse(device_frees) - self.assertFalse(host_frees) - def test_free_values_mid_drain_failure_cannot_replay_freed_entries(self): """A device free that raises must leave only un-attempted entries in the dict (no replay of successes) while host frees still drain.""" @@ -723,25 +676,17 @@ class TestReturnedValuesDrain(UnifiedRadixCacheFixtures, CustomTestCase): @unittest.skipUnless(torch.cuda.is_available(), "cache fixtures need CUDA") class TestSegmentLockProtocol(SegmentLockProtocolFixtures, CustomTestCase): - """Segment-lock protocol regressions, replaying the production lock-theft - failure classes (F1/F2) and the split hazards. - - The protocol: a lock covers the contiguous node segment - [start, boundary-uuid], counting every node (tombstones included), so a - release needs only the receipt (anchor node, boundary uuid, skipped - components) and any ref==0 met inside the segment is a hard protocol - violation. The replays read the tree through the inspection interface, so - they run unchanged against the Python and Rust cores. - """ + """A lock covers the node segment [start, boundary-uuid] counting tombstones; + a release needs only its receipt, and any ref==0 inside the segment is a + violation.""" cfg = CacheConfig( components=(ComponentType.FULL, ComponentType.SWA), sliding_window_size=8 ) def test_rebuilt_tombstone_relock_release_no_theft(self): - """F1 attribution replay: A locks a window containing a tombstone; the - tombstone is rebuilt and locked by B mid-hold; A's release must leave - B's refs intact (the old skip-set protocol decremented B's lock).""" + """A locks a window containing a tombstone; B rebuilds and locks it + mid-hold; A's release must leave B's refs intact.""" sw = self.cfg.sliding_window_size cache, allocator, req_to_token_pool = build_fixture(self.cfg) seq = self._make_seq(1, 2 * sw) @@ -794,7 +739,6 @@ class TestSegmentLockProtocol(SegmentLockProtocolFixtures, CustomTestCase): for n in segment: self.assertEqual(self._swa_ref(cache, n), 2) - # THE regression: A's release takes back exactly A's refs. cache.dec_lock_ref(leaf, lock_a.to_dec_params()) for n in segment: self.assertEqual(self._swa_ref(cache, n), 1) @@ -821,7 +765,6 @@ class TestSegmentLockProtocol(SegmentLockProtocolFixtures, CustomTestCase): cache.sanity_check() def test_root_acquire_receipt_releases_on_root(self): - """A cold request locks the root, so its receipt is anchored on root.""" cache, _, _ = build_fixture(self.cfg) root = cache.root_node_handle() lock = cache.inc_lock_ref(root).to_dec_params() @@ -830,7 +773,6 @@ class TestSegmentLockProtocol(SegmentLockProtocolFixtures, CustomTestCase): cache.sanity_check() def test_release_with_incorrect_root_boundary_fails_loud(self): - """An incorrect root boundary must fail at the acquired segment's edge.""" sw = self.cfg.sliding_window_size cache, allocator, req_to_token_pool = build_fixture(self.cfg) seq = self._make_seq(1, 3 * sw) @@ -869,8 +811,8 @@ class TestSegmentLockProtocol(SegmentLockProtocolFixtures, CustomTestCase): ) def test_unanchored_device_release_fails_loud(self): - """#41261 replay: a receipt that lost its anchor must not walk the - node's segment and drop locks other holders own.""" + """A receipt that lost its anchor must not walk the node's segment and + drop locks other holders own.""" cache, allocator, leaf = self._leaf_with_lock_path() lock = cache.inc_lock_ref(leaf).to_dec_params() self._assert_unanchored_release_rejected( @@ -920,8 +862,6 @@ class TestSegmentLockProtocol(SegmentLockProtocolFixtures, CustomTestCase): ) def test_unanchored_root_release_fails_loud(self): - """Every acquire anchors its receipt, root included; an unanchored - receipt lost its anchor upstream even where releasing is a no-op.""" cache, _, _ = build_fixture(self.cfg) root = cache.root_node_handle() self._assert_protocol_violation( @@ -941,8 +881,8 @@ class TestSegmentLockProtocol(SegmentLockProtocolFixtures, CustomTestCase): ) def test_finish_after_early_release_without_skip_swa_fails_loud(self): - """F2 replay: retraction-after-early-release used to run a second SWA - walk that stole ancestors' locks; now it dies at the first node.""" + """A finish after early SWA release without skip_swa asserts at the first + node instead of stealing ancestors' locks.""" sw = self.cfg.sliding_window_size cache, allocator, req_to_token_pool = build_fixture(self.cfg) seq = self._make_seq(1, 2 * sw) @@ -955,7 +895,7 @@ class TestSegmentLockProtocol(SegmentLockProtocolFixtures, CustomTestCase): ) def test_finish_after_early_release_with_skip_swa(self): - """The correct F2 flow: unlock honors the early release.""" + """Finish with skip_swa honors the early release.""" sw = self.cfg.sliding_window_size cache, allocator, req_to_token_pool = build_fixture(self.cfg) seq = self._make_seq(1, 2 * sw) @@ -983,7 +923,6 @@ class TestSegmentLockProtocol(SegmentLockProtocolFixtures, CustomTestCase): pre_segment = self._segment(cache, leaf, sw) pre_full_path = self._full_path(cache, leaf) - # Diverge inside the window to force a split of a locked node. fork = seq[: len(seq) - sw // 2] + self._make_seq(9000, sw) self._insert(cache, allocator, req_to_token_pool, fork) post_segment = self._segment(cache, leaf, sw) @@ -1018,7 +957,6 @@ class TestSegmentLockProtocol(SegmentLockProtocolFixtures, CustomTestCase): params = cache.inc_lock_ref(leaf).to_dec_params() self.assertNotIn(node, cache.tree_core.evictable_device_leaves) - # Full first: its walk still sees the SWA lock, so the leaf stays out. cache.components[ComponentType.FULL].release_component_lock(node, params) self.assertNotIn(node, cache.tree_core.evictable_device_leaves) # The SWA release drops the last lock and must readmit the leaf itself. @@ -1029,8 +967,7 @@ class TestSegmentLockProtocol(SegmentLockProtocolFixtures, CustomTestCase): @unittest.skipUnless(torch.cuda.is_available(), "cache fixtures need CUDA") class TestSegmentLockFuzz(SegmentLockFuzzFixtures, CustomTestCase): - """Random lock/insert/evict interleavings with the tree's own ledger - recomputation (sanity_check) as the per-step oracle, on both cores.""" + """Random lock/insert/evict interleavings, with sanity_check as the oracle.""" cfg = CacheConfig( components=(ComponentType.FULL, ComponentType.SWA), @@ -1051,9 +988,7 @@ class TestSegmentLockFuzz(SegmentLockFuzzFixtures, CustomTestCase): @unittest.skipUnless(torch.cuda.is_available(), "cache fixtures need CUDA") class TestSegmentLockFuzzWithMamba(TestSegmentLockFuzz): - """The FULL+SWA+MAMBA (Inkling) shape: the Mamba opt-out receipt, the - lower-priority cascade on early SWA release, and Mamba evictions all join - the interleavings.""" + """The FULL+SWA+MAMBA shape adds the Mamba opt-out and Mamba evictions.""" cfg = CacheConfig( components=(ComponentType.FULL, ComponentType.SWA, ComponentType.MAMBA), @@ -1067,8 +1002,8 @@ class TestSegmentLockFuzzWithMamba(TestSegmentLockFuzz): class TestStreamingSessionLockLifecycle( StreamingSessionLockLifecycleFixtures, CustomTestCase ): - """A streaming session owns its record and prefix lock from the first - turn's row allocation; every move of that lock keeps the tree balanced.""" + """A streaming session's prefix lock keeps the tree balanced through every + move.""" cfg = CacheConfig( page_size=1, diff --git a/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_kv_events.py b/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_kv_events.py index f68125cb6d..d9a2374d87 100644 --- a/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_kv_events.py +++ b/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_kv_events.py @@ -1,5 +1,3 @@ -"""KV cache event tests for UnifiedRadixCache.""" - import unittest from array import array from unittest import mock @@ -174,7 +172,6 @@ class TestUnifiedRadixCacheKVEvents(UnifiedRadixCacheKVEventsFixtures, CustomTes ) def test_lora_external_ids_separate_adapter_loads(self): - """External IDs distinguish loads; routers match tokens and public names.""" lora_names, _reqs = make_lora_names( **{ "uuid-a": "adapter-a", @@ -207,7 +204,6 @@ class TestUnifiedRadixCacheKVEvents(UnifiedRadixCacheKVEventsFixtures, CustomTes for h in self._event_hashes(base) ], ) - # Another load of the same public name has distinct external IDs. self.assertNotEqual( self._event_hashes(store_then_evict("user|uuid-a2")), self._event_hashes(lora_a), @@ -385,7 +381,6 @@ class TestUnifiedRadixCacheKVEvents(UnifiedRadixCacheKVEventsFixtures, CustomTes self.assertCountEqual(self._event_hashes(removed_cpu), stored_hashes) def test_hicache_storage_prefetch_publishes_host_only_suffix(self): - """A storage-prefetch refill has no write-through ack to publish it.""" from sglang.srt.mem_cache.utils import get_hash_str, hash_str_to_int64 cache, _, _ = build_fixture(self.cfg, enable_kv_cache_events=True) @@ -403,7 +398,6 @@ class TestUnifiedRadixCacheKVEvents(UnifiedRadixCacheKVEventsFixtures, CustomTes self.assertFalse(result.host_insert_dropped) self.assertIsNotNone(result.inserted_host_node) - # The recorder coalesces the parent-linked pages into one event. stored_cpu = self._stored_events(cache, StorageMedium.CPU) self.assertEqual(len(stored_cpu), 1) self.assertEqual(list(stored_cpu[0].token_ids), tokens) @@ -441,7 +435,6 @@ class TestUnifiedRadixCacheKVEvents(UnifiedRadixCacheKVEventsFixtures, CustomTes [[1, 2], [3, 4]], ) - # Both split fragments must be published as one parent-linked batch. stored_cpu = self._stored_events(cache, StorageMedium.CPU) self.assertEqual(len(stored_cpu), 1) self.assertEqual(list(stored_cpu[0].token_ids), [1, 2, 3, 4]) diff --git a/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_load_back.py b/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_load_back.py index 3fc645133d..c5be8dce72 100644 --- a/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_load_back.py +++ b/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_load_back.py @@ -1,5 +1,3 @@ -"""HiCache load-back, admission and branching tests for UnifiedRadixCache.""" - import unittest from array import array from dataclasses import replace @@ -9,7 +7,6 @@ import torch from sglang.srt.managers.schedule_batch import ( Req, - ReqKvInfo, ) from sglang.srt.managers.schedule_policy import PrefillAdder from sglang.srt.mem_cache.base_prefix_cache import ( @@ -141,7 +138,7 @@ class TestUnifiedTreeCoreLoadBackOwnershipBackends(CustomTestCase): sliding_window_size=4, ) - def test_auxiliary_load_does_not_reuse_full_pending_pin(self): + def test_auxiliary_load_restores_swa_and_rebuilds_mapping(self): cache, _, _ = build_fixture(self.cfg) core = cache.tree_core core.is_write_back = True @@ -240,7 +237,6 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures): self._load_back_node(cache, node) self.assertFalse(cache.tree_core.is_full_device_evicted(node)) self.assertIsNotNone(node_device_value(cache, node, ComponentType.FULL)) - # Gather the whole reloaded prefix via match (a leaf may be split). loaded_indices = cache.path_device_indices( match_prefix(cache, base).last_device_node ) @@ -288,7 +284,6 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures): self.assertEqual(result.swa_branching_seqlen, len(tokens)) self.assertEqual(result.swa_branching_seqlen % self.cfg.page_size, 0) - # Simulate forward producing fresh SWA KV at the branching point. self._insert( cache, allocator, @@ -352,15 +347,12 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures): @requires_full_swa_ps1_window4 def test_swa_branching_seqlen_caps_insert_after_forward(self): - cache, _, _ = build_fixture(self.cfg) + cache, _, req_to_token_pool = build_fixture(self.cfg) swa = cache.components[ComponentType.SWA] - req = mock.Mock( - swa_branching_seqlen=8, - kv=ReqKvInfo( - cache_protected_len=4, - component_evicted_seqlens={ComponentType.SWA: 3}, - ), - ) + req = self._make_req(req_to_token_pool) + req.swa_branching_seqlen = 8 + req.kv.cache_protected_len = 4 + req.kv.set_evicted_seqlen(ComponentType.SWA, 3) for is_finished in (False, True): params = InsertParams(component_evicted_seqlens={ComponentType.MAMBA: 6}) @@ -378,12 +370,7 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures): cache.tree_core.is_eagle = True req.kv.cache_protected_len = 4 - params = InsertParams() - self.assertEqual(swa.prepare_for_caching_req(req, params, 12, False), 9) - params.key = RadixKey(array("q", range(9)), is_bigram=True) - result = mock.Mock(swa_branch_inserted=False) - swa.commit_insert_component_data(mock.Mock(), False, params, result, []) - self.assertTrue(result.swa_branch_inserted) + self.assertEqual(swa.prepare_for_caching_req(req, InsertParams(), 12, False), 9) @requires_full_mamba_ps1 def test_mamba_branching_seqlen_disabled_under_hicache(self): @@ -615,7 +602,6 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures): self.assertEqual(new_node, match.last_device_node) self.assertIsNone(node_device_value(cache, leaf, ComponentType.FULL)) self.assertIsNone(node_device_value(cache, leaf, ComponentType.MAMBA)) - # A failed load-back must roll back the pre-allocated mamba slot. self.assertIsNone(req.kv.mamba_pool_idx) self.assertEqual( req_to_token_pool.mamba_allocator.available_size(), avail_before @@ -639,10 +625,8 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures): ) self._apply_match_to_req(req, match) - # Simulate a request without its own mamba slot so load-back allocates one. req.kv.mamba_pool_idx = None avail_before = req_to_token_pool.mamba_allocator.available_size() - # H->D load fails after the mamba slot is pre-allocated -> must free it. with mock.patch.object(cache.cache_controller, "load", return_value=None): loaded_len, _ = cache.init_load_back( InitLoadBackParams( @@ -677,7 +661,6 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures): ) self._apply_match_to_req(req, match) - # Simulate a request without its own mamba slot so load-back allocates one. req.kv.mamba_pool_idx = None avail_before = req_to_token_pool.mamba_allocator.available_size() # No device room and eviction frees nothing -> load-back bails after the @@ -762,36 +745,6 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures): ) self._release_ongoing_load_back_locks(cache) - @requires_full_mamba_ps1 - def test_load_back_success_publishes_fresh_mamba_slot(self): - cache, allocator, req_to_token_pool = self._build_hicache_fixture() - chain = self._build_chain_pages(cache, allocator, req_to_token_pool, 3) - if len(chain) < 3: - self.skipTest("chain too short") - leaf = chain[-1] - - self._backup_node(cache, leaf) - cache.evict(EvictParams(num_tokens=node_key_length(cache, leaf))) - self.assertTrue(cache.tree_core.is_full_device_evicted(leaf)) - - req = self._make_req(req_to_token_pool) - req_to_token_pool.mamba_allocator.free(req.kv.mamba_pool_idx.unsqueeze(0)) - req.kv.mamba_pool_idx = None - mamba_avail = req_to_token_pool.mamba_allocator.available_size() - - loaded = cache.load_back(leaf, req=req) - - self.assertTrue(loaded) - # the successful load must keep the freshly allocated slot published - self.assertIsNotNone(req.kv.mamba_pool_idx) - self.assertIsNotNone(node_device_value(cache, leaf, ComponentType.MAMBA)) - # one slot restores the node's mamba value, one is the request's CoW slot - self.assertEqual( - req_to_token_pool.mamba_allocator.available_size(), mamba_avail - 2 - ) - self._finish_pending_loads(cache) - self._release_ongoing_load_back_locks(cache) - @requires( lambda cfg: ( cfg.has_mamba @@ -835,9 +788,8 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures): ) adder.add_one_req(req, False, None) - # The load pins 12 FULL slots, leaving 8 free. Rechecking the original - # 18-token demand would reject and scheduler cleanup would free the CoW - # destination even though the queued H2D still targets it. + # Rechecking the original 18-token demand would reject, and scheduler + # cleanup would free the CoW destination while the queued H2D targets it. self.assertEqual(adder.can_run_list, [req]) self.assertEqual(req.prefix_len, 12) self.assertEqual(req.kv.cache_protected_len, 12) @@ -858,7 +810,6 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures): self.skipTest("chain too short") leaf = chain[-1] - # Stamp the node's mamba state so the host backup carries it. node_mamba_indices = node_device_value(cache, leaf, ComponentType.MAMBA).clone() self._fill_mamba_state(req_to_token_pool, node_mamba_indices, marker=11) expected_temporal, expected_conv = self._snapshot_mamba_state( @@ -872,10 +823,16 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures): req = self._make_req(req_to_token_pool) req_to_token_pool.mamba_allocator.free(req.kv.mamba_pool_idx.unsqueeze(0)) req.kv.mamba_pool_idx = None + mamba_avail = req_to_token_pool.mamba_allocator.available_size() loaded = cache.load_back(leaf, req=req) self.assertTrue(loaded) self.assertIsNotNone(req.kv.mamba_pool_idx) + self.assertIsNotNone(node_device_value(cache, leaf, ComponentType.MAMBA)) + # one slot restores the node's mamba value, one is the request's CoW slot + self.assertEqual( + req_to_token_pool.mamba_allocator.available_size(), mamba_avail - 2 + ) self._finish_pending_loads(cache) # The CoW slot must actually hold the backed-up mamba state, not merely exist. @@ -902,28 +859,23 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures): req_to_token_pool.mamba_allocator.free(req.kv.mamba_pool_idx.unsqueeze(0)) req.kv.mamba_pool_idx = None - # device value still present -> nothing to prepare even though host-backed self.assertIsNone(comp.prepare_load_back(leaf, req=req).allocated_mamba_slot) self.assertIsNone(req.kv.mamba_pool_idx) cache.evict(EvictParams(num_tokens=node_key_length(cache, leaf))) self.assertTrue(cache.tree_core.is_full_device_evicted(leaf)) - # a request that already owns a slot -> nothing to prepare req_owned = self._make_req(req_to_token_pool) self.assertIsNone( comp.prepare_load_back(leaf, req=req_owned).allocated_mamba_slot ) - # no request -> nothing to prepare self.assertIsNone(comp.prepare_load_back(leaf, req=None).allocated_mamba_slot) - # fresh request + host-only mamba -> allocates and publishes onto req prep = comp.prepare_load_back(leaf, req=req) self.assertIsNotNone(prep.allocated_mamba_slot) self.assertEqual(int(req.kv.mamba_pool_idx), int(prep.allocated_mamba_slot[0])) - # node without host-backed mamba -> nothing to prepare req2 = self._make_req(req_to_token_pool) req_to_token_pool.mamba_allocator.free(req2.kv.mamba_pool_idx.unsqueeze(0)) req2.kv.mamba_pool_idx = None @@ -987,9 +939,7 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures): self._simulate_backup_tree(cache) - # Tombstone every chain node on the device side without going through - # the tree-wide eviction loop. This isolates build_hicache_transfers - # from LRU and cascade ordering. + # Tombstone every chain node directly, bypassing the eviction loop. for n in chain: cache.tree_core.set_component_device_value_raw(n, ComponentType.FULL, None) cache.tree_core.set_component_device_value_raw(n, ComponentType.SWA, None) @@ -1003,7 +953,6 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures): xfer = transfers[0] self.assertEqual(xfer.name, PoolName.SWA) self.assertEqual(len(xfer.nodes_to_load), expected_pages) - # host_indices must cover exactly the expected suffix tokens (>= sw). self.assertEqual(int(xfer.host_indices.numel()), expected_pages * ps) self.assertGreaterEqual(int(xfer.host_indices.numel()), sw) self.assertEqual(xfer.nodes_to_load, chain[-expected_pages:]) @@ -1018,17 +967,11 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures): "window must span past the leaf to reach the pin", ) def test_hicache_swa_load_back_rejects_foreign_pinned_window(self): - """A second load-back must not claim nodes pinned by an in-flight one. - - commit_load_back republishes Full device values before the DMA acks, so - a later anchor can claim just the still-host-only SWA window of a - pinned node and hit the commit pin assert. The spec build must degrade - to an empty spec instead (the caller recomputes). - """ + """A second load-back must not claim nodes pinned by an in-flight one; + the spec build degrades to empty (caller recomputes).""" cache, allocator, req_to_token_pool = build_fixture(self.cfg) - # commit_load_back pins its source nodes only under write-back; run the - # scenario in that mode so the foreign-pin path is reachable. + # commit_load_back pins its source nodes only under write-back. cache.is_write_back = True chain = self._build_chain_pages(cache, allocator, req_to_token_pool, 3) if len(chain) < 3: @@ -1058,7 +1001,6 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures): self.assertEqual(kv_xfer.nodes_to_load, []) self.assertEqual(comp_xfers, {}) - # After the ack unpins, the same spec builds fully. cache.tree_core.finish_load_back(a) kv_xfer, comp_xfers = cache.tree_core.build_load_back_spec(b) self.assertEqual(kv_xfer.nodes_to_load, [b]) @@ -1070,78 +1012,42 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures): "SWA-only path keeps the chain construction simple", ) def test_hicache_swa_commit_load_back_rebuilds_mapping(self): - """LOAD_BACK commit must: - (1) restore SWA cd.value via _restore_device_value (host LRU -> device LRU), - (2) rewrite full_to_swa_index_mapping[full_idx] = new_swa_idx for every - loaded chunk so subsequent SWA reads via translate_loc_from_full_to_swa - return the freshly allocated SWA slot.""" - - cache, allocator, _, chain, window_pages = self._swa_finalize_setup() - - # Tombstone every SWA node in the trailing window. + """A SWA load-back restores each tombstoned window node's SWA to device + and maps restored Full slots to their SWA slots.""" + ps = self.cfg.page_size + window_pages = (self.cfg.sliding_window_size + ps - 1) // ps + cache, allocator, req_to_token_pool = self._build_hicache_fixture() + chain = self._build_chain_pages( + cache, allocator, req_to_token_pool, window_pages + 2 + ) + if len(chain) <= window_pages: + self.skipTest("chain collapsed below the window length") + tokens = self._match_tokens_for_chain(cache, chain) + self._backup_node(cache, chain[-1]) loaded_nodes = chain[-window_pages:] for n in loaded_nodes: - cache.tree_core.set_component_device_value_raw(n, ComponentType.SWA, None) - # SWA LRU bookkeeping must reflect tombstone state for the - # _restore_device_value path to exercise the host->device move. - cache.tree_core.remove_node_from_device_lru(n, ComponentType.SWA) - cache.tree_core.insert_node_into_host_lru(n, ComponentType.SWA) - - # Build the LOAD_BACK transfer the same way load_back() would. - transfers = cache.tree_core.build_hicache_transfers( - ComponentType.SWA, chain[-1], CacheTransferPhase.LOAD_BACK - ) - self.assertIsNotNone(transfers) - xfer = transfers[0] - self.assertEqual(xfer.nodes_to_load, loaded_nodes) - - # Allocate SWA device slots from the inner allocator (mirrors how - # _resolve_device_transfers routes via device_alloc_fn -> - # swa_attn_allocator.alloc on the load-back path). - n_swa = int(xfer.host_indices.numel()) - new_swa = allocator.swa_attn_allocator.alloc(n_swa) - self.assertIsNotNone(new_swa) - xfer.device_indices = new_swa - - # Snapshot pre-commit state for invariants checks. + self._evict_device_component(cache, n, ComponentType.SWA) pre_evictable = cache.tree_core.component_evictable_size(ComponentType.SWA) - load_actions = [] - cache.tree_core.commit_hicache_transfers( - chain[-1], - CacheTransferPhase.LOAD_BACK, - {ComponentType.SWA: transfers}, - cache_actions=load_actions, - ) - cache._apply_cache_actions(load_actions) + self._load_back_through_admission(cache, req_to_token_pool, tokens) + self._finish_pending_loads(cache) + self._release_ongoing_load_back_locks(cache) - # (1) cd.value restored, host LRU -> device LRU swap done. - offset = 0 + restored = 0 for n in loaded_nodes: - value = node_device_value(cache, n, ComponentType.SWA) - self.assertIsNotNone(value) - chunk_len = int(value.numel()) - self.assertEqual( - value.tolist(), - new_swa[offset : offset + chunk_len].tolist(), - ) - offset += chunk_len + swa_idx = node_device_value(cache, n, ComponentType.SWA) + self.assertIsNotNone(swa_idx) self.assertTrue(cache.tree_core.is_node_in_device_lru(n, ComponentType.SWA)) self.assertFalse(cache.tree_core.is_node_in_host_lru(n, ComponentType.SWA)) - self.assertEqual(offset, n_swa) - - # (2) full_to_swa_index_mapping rebuilt for every loaded chunk. - for n in loaded_nodes: full_idx = node_device_value(cache, n, ComponentType.FULL) - swa_idx = node_device_value(cache, n, ComponentType.SWA) translated = allocator.translate_loc_from_full_to_swa(full_idx) self.assertEqual(translated.tolist(), swa_idx.tolist()) - - # Evictable size moved up by the restored token count. + restored += int(swa_idx.numel()) self.assertEqual( cache.tree_core.component_evictable_size(ComponentType.SWA) - pre_evictable, - n_swa, + restored, ) + cache.sanity_check() def test_hicache_swa_load_back_anchored_on_best_match_node(self): cache, _, _, y, x, _ = self._swa_anchor_setup() diff --git a/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_storage.py b/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_storage.py index e7b6899ed4..950fe7b95b 100644 --- a/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_storage.py +++ b/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_storage.py @@ -30,7 +30,6 @@ from sglang.srt.mem_cache.hicache_storage import ( PoolTransferResult, ) from sglang.srt.mem_cache.radix_cache import RadixKey -from sglang.srt.mem_cache.storage_prefetch import StoragePrefetchRetries from sglang.srt.mem_cache.unified_cache.cache_action import ( FreeComponentHostSlot, ) @@ -80,7 +79,9 @@ register_cuda_ci(est_time=28, stage="base-b", runner_config="1-gpu-small") register_amd_ci(est_time=28, suite="stage-b-test-1-gpu-small-amd") -class TestUnifiedRadixCacheEagleHiCacheStorageKey(CustomTestCase): +class TestUnifiedRadixCacheEagleHiCacheStorageKey( + UnifiedRadixCacheStorageFixtures, CustomTestCase +): cfg = CacheConfig( page_size=4, components=(ComponentType.FULL,), @@ -178,61 +179,38 @@ class TestUnifiedRadixCacheEagleHiCacheStorageKey(CustomTestCase): self.assertNotEqual(canonical_hashes, cache.tree_core.get_hash_values(leaf)) def test_buffer_anchor_rematch_preserves_bigram_boundary(self): - from sglang.srt.mem_cache.buffer_mode.pipeline import BufferModePipeline - - cache, allocator, _ = build_fixture(self.cfg) + cache, allocator, _ = self._build_buffer_hicache_fixture( + self._make_storage_dir() + ) + pipeline = cache.buffer_pipeline extra_key = "adapter-a" cache_salt = "tenant-a" prefix_tokens = array("q", [1, 2, 3, 4, 5]) value = allocator.alloc(len(prefix_tokens) - 1) self.assertIsNotNone(value) - cache.insert( - InsertParams( - key=RadixKey( - prefix_tokens, - extra_key=extra_key, - cache_salt=cache_salt, - ), - value=value, - ) - ) - match = cache.match_prefix( - MatchPrefixParams( - key=RadixKey( - prefix_tokens, - extra_key=extra_key, - cache_salt=cache_salt, - ) - ) - ) + key = RadixKey(prefix_tokens, extra_key=extra_key, cache_salt=cache_salt) + cache.insert(InsertParams(key=key, value=value)) + match = cache.match_prefix(MatchPrefixParams(key=key)) self.assertEqual(match.device_prefix_len, len(prefix_tokens) - 1) - req_id = CacheRequestHandle("bigram-anchor", 0) - prefetch_key = RadixKey( - array("q", [prefix_tokens[-1], 6, 7, 8, 9]), - extra_key=extra_key, - is_bigram=True, - cache_salt=cache_salt, - ) + # What prefetch_from_storage registers for a bigram request anchored + # at the matched node: the suffix key starts at the boundary token. cache.ongoing_prefetch[req_id] = _OngoingPrefetch( anchor_node_id=match.last_device_node, - prefetch_key=prefetch_key, + prefetch_key=RadixKey( + array("q", [prefix_tokens[-1], 6, 7, 8, 9]), + extra_key=extra_key, + is_bigram=True, + cache_salt=cache_salt, + ), host_indices=None, operation=None, anchor_lock_params=None, comp_xfers={}, ) - - pipeline = BufferModePipeline.__new__(BufferModePipeline) - pipeline.anchor_locks = {} - pipeline.anchor_locked_tokens_ = 0 - pipeline.anchor_lock_cap_tokens = 10_000 - pipeline._anchor_lock_cap_skips = 0 - pipeline._prefetch_prefix_ctx = { - req_id: (list(prefix_tokens[:-1]), extra_key, cache_salt) - } - pipeline._cache = cache - + pipeline.set_prefix_ctx( + req_id, list(prefix_tokens[:-1]), extra_key=extra_key, cache_salt=cache_salt + ) lock_ref = node_device_lock_ref( cache, match.last_device_node, ComponentType.FULL ) @@ -244,8 +222,8 @@ class TestUnifiedRadixCacheEagleHiCacheStorageKey(CustomTestCase): node_device_lock_ref(cache, match.last_device_node, ComponentType.FULL), lock_ref + 1, ) - pipeline.release_anchor_lock(req_id) + pipeline.pop_prefix_ctx(req_id) cache.ongoing_prefetch.pop(req_id) self.assertEqual( node_device_lock_ref(cache, match.last_device_node, ComponentType.FULL), @@ -287,52 +265,23 @@ class TestUnifiedRadixCacheEagleHiCacheStorageKey(CustomTestCase): def test_sanity_check_reads_buffer_backup_node_id_from_snapshot(self): """A buffer-mode backup entry keeps its node id inside the detached - snapshot; sanity_check must read it from there. It used to read - ``entry.intent.node_id``, an attribute the intent no longer has, so the - idle check raised AttributeError whenever a backup was in flight.""" - from sglang.srt.mem_cache.buffer_mode.pipeline import ( - BufferModePipeline, - _UnifiedBackupIntent, - _UnifiedBufferBackupEntry, + snapshot; sanity_check must read it from there while the write is in + flight.""" + cache, allocator, _ = self._build_buffer_hicache_fixture( + self._make_storage_dir() ) - - cache, allocator, _ = build_fixture(self.cfg) - cache.enable_storage = True tokens = array("q", [1, 2, 3, 4, 5, 6, 7, 8, 9]) value = allocator.alloc(len(tokens) - 1) self.assertIsNotNone(value) cache.insert(InsertParams(key=RadixKey(tokens), value=value)) - leaf_id = cache.match_prefix( + leaf = cache.match_prefix( MatchPrefixParams(key=RadixKey(tokens)) ).last_device_node - snapshot = cache.tree_core.snapshot_buffer_backup( - leaf_id, pass_prefix_keys=False - ) - self.assertIsNotNone(snapshot) - - # The post-launch entry pins its node, as sanity_check requires of - # every in-flight backup. - lock_params = cache.inc_lock_ref(leaf_id).to_dec_params() - pipeline = BufferModePipeline.__new__(BufferModePipeline) - keys = list(snapshot.hash_values) - # Entries are keyed by the write's ack id, not by the node. - pipeline.ongoing_write_through = { - -1: _UnifiedBufferBackupEntry( - intent=_UnifiedBackupIntent( - snapshot=snapshot, pool=PoolName.KV, keys=keys - ), - host_indices=torch.empty(0, dtype=torch.int64), - aux_xfers=[], - lock_params=lock_params, - occupied_units=0, - keys_by_pool={PoolName.KV: keys}, - ) - } - cache.buffer_pipeline = pipeline + write_backup(cache, leaf) + cache.buffer_pipeline.flush_pending_writes() + self.assertTrue(cache.buffer_pipeline.ongoing_write_through) cache.sanity_check() - - cache.buffer_pipeline = None - cache.dec_lock_ref(leaf_id, lock_params) + self._drain_buffer_writes(cache) cache.sanity_check() @@ -343,18 +292,14 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): "mamba L3 prefetch is out of scope for this unit fixture", ) def test_hicache_l3_prefetch(self): - """L3 round trip: write with one tree, prefetch into a fresh tree. - - Uses two independent trees that share the same file storage dir so the - prefetch path genuinely reloads from L3 (no host/device residue). - """ + """Write with one tree, prefetch into a fresh tree sharing the storage dir, + so the path reloads from L3.""" storage_dir = self._make_storage_dir() num_pages = 4 if self.cfg.has_swa: - # SWA L3 prefetch is all-or-nothing over one full sliding window. - # Keep the generic L3 round trip valid for SWA configs whose window - # is larger than the old fixed 4-page request. + # SWA L3 prefetch is all-or-nothing over one full sliding window; + # size the request to cover it. sw_pages = ( self.cfg.sliding_window_size + self.cfg.page_size - 1 ) // self.cfg.page_size @@ -369,14 +314,12 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): ) self._run_prefetch_to_completion(cons, req_id) - # The full prefix must now be a host hit (loaded from L3). mc = match_prefix(cons, seq) self.assertEqual(mc.host_hit_length, len(seq)) host_node = mc.last_host_node self.assertNotEqual(host_node, cons.root_node_handle()) self.assertTrue(cons.tree_core.is_full_device_evicted(host_node)) - # Load the reloaded host prefix back to device and verify KV bytes. self._load_back_node(cons, host_node) loaded_indices = cons.path_device_indices( match_prefix(cons, seq).last_device_node @@ -393,13 +336,8 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): "SWA-only fixture required to exercise extra pool", ) def test_release_aborted_request_l3_prefetch_io_in_progress(self): - """Test release_aborted_request while a prefetch IO is still in-progress. - 1. Fill KV and SWA to L3. - 2. Trigger L3 prefetch. - 3. Hack IO thread, blocking at _page_transfer. - 4. Call release_aborted_request. Assert that the KV and SWA buffers are not released (owned by the IO thread). - 5. Unlbock IO thread. Assert that KV and SWA buffers are eventually released. - """ + """release_aborted_request while a prefetch IO is still in progress; the IO + thread owns the buffers until it finishes.""" # SWA prefetch is all-or-nothing over one full sliding window: size the # request at sw_pages + 1 pages so prepare_prefetch actually materializes @@ -433,7 +371,6 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): ) self.assertIn(req_id, cons.ongoing_prefetch) - # Block IO thread at the entry of _page_transfer. import threading from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import ( @@ -446,19 +383,16 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): swa_release_q = cc.extra_host_mem_release_queues.get(PoolName.SWA) def _page_transfer_hook(_self, op): - # Signal that the worker has reached the sidecar step, then block - # until the main thread releases the gate. + # Block the worker at the sidecar step until the main thread + # releases the gate. entered.set() gate.wait(timeout=10.0) - # Forward to the real implementation so the worker runs the - # terminate-aware release path itself. + # The worker runs the terminate-aware release path itself. original_sidecar(_self, op) - # Simulate a slow prefetch IO. Hook on _page_transfer. with mock.patch.object( HybridCacheController, "_page_transfer", _page_transfer_hook ): - # Pump until the IO aux thread has entered the sidecar barrier. deadline = time.time() + 10.0 while time.time() < deadline: cons.drain_storage_control_queues() @@ -469,9 +403,6 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): else: self.fail("prefetch did not reach the sidecar barrier in time") - # Now, the prefetch IO thread is stopping at _page_transfer. - # Let the scheduler thread calls release_aborted_request. - # Assert that everything will not be released. op = cons.ongoing_prefetch[req_id].operation self.assertFalse(op.pool_transfers_done) self.assertEqual(cc.host_mem_release_queue.qsize(), 0) @@ -480,12 +411,8 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): self.assertEqual(cc.host_mem_release_queue.qsize(), 0) self.assertEqual(swa_release_q.qsize(), 0) - # Let the prefetch IO thread continue to run. gate.set() - # Wait for the IO thread has completed prefetch. - # We don't consume release queue, as we will use that later to check whether - # the host memory was released yet (as used by the prefetch IO thread). deadline = time.time() + 10.0 while time.time() < deadline: cons._drain_storage_control_queues_impl( @@ -502,7 +429,6 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): else: self.fail("SWA extra pool was not released after the abort") - # Asserts that everything is correctly released. self.assertFalse(op.pool_transfers_done) self.assertGreater(cc.host_mem_release_queue.qsize(), 0) self.assertGreater(swa_release_q.qsize(), 0) @@ -527,13 +453,8 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): "SWA-only fixture required to exercise extra pool", ) def test_release_aborted_request_l3_prefetch_io_done(self): - """Test release_aborted_request is called after the IO thread has completed - prefetch. - 1. Fill KV and SWA to L3. - 2. Trigger L3 prefetch. - 3. Wait until the completion of L3 prefetch IO. - 4. Call release_aborted_request. Assert that KV and SWA buffers are released. - """ + """release_aborted_request after the prefetch IO completed releases the + buffers.""" # SWA prefetch is all-or-nothing over one full sliding window: size the # request at sw_pages + 1 pages so prepare_prefetch actually materializes @@ -570,14 +491,8 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): swa_release_q = cc.extra_host_mem_release_queues.get(PoolName.SWA) self.assertIsNotNone(swa_release_q) - # Simulate polling check_hicache_events. - # There will be a sequence of events populated from queue: - # 1. a storage hit notification (from cc.prefetch_hit_queue). - # 2. a HiCacheAck, indicating the completion of KV pool read. - # 3. a HiCacheAck, indicating the completion of SWA pool read. - # 4. a HiCacheACk, idnicating the completion of entire prefetch request. - # We are going to stop at the exact timing-window between 3 and 4. So we have to - # consume ONE event from the queue at each iteration. + # Consume one queued event per iteration to stop in the window between the + # SWA pool ack and the whole-request ack. deadline = time.time() + 10.0 while time.time() < deadline: cons._drain_storage_control_queues_impl( @@ -599,7 +514,6 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): self.assertEqual(cc.host_mem_release_queue.qsize(), 0) self.assertEqual(swa_release_q.qsize(), 0) - # --- Act: abort without committing the prefetch. --- cons.finish(req_id, CacheRequestOutcome.ABORT) self.assertTrue(op.pool_transfers_done) @@ -621,10 +535,6 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): cons.sanity_check() - # ================================================================ - # Buffer-only host memory mode (host = transient staging, L3 = cache) - # ================================================================ - @requires_hicache @requires_swa def test_buffer_only_swa_tail_only_backups(self): @@ -794,7 +704,6 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): pipeline = cache.buffer_pipeline controller = cache.cache_controller write_backup(cache, leaf) - # One intent per pool of every node on the path. intents = list(pipeline.pending_write_queue) node_ids = sorted({intent.snapshot.node_id for intent in intents}) self.assertGreaterEqual(len(node_ids), 2) @@ -833,7 +742,6 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): self.assertFalse(pipeline.pending_write_queue) self.assertFalse(controller.write_queue) self.assertEqual(len(controller.ack_write_queue), 1) - # Every intent is a write of its own, acked under its own id. ack = controller.ack_write_queue[0] self.assertEqual(len(ack.node_ids), len(intents)) self.assertEqual(list(ack.node_ids), list(pipeline.ongoing_write_through)) @@ -896,7 +804,6 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): transfer.keys ) ) - # Re-hit absorbed when every pool is believed present. write_backup(cache, leaf) self.assertNotIn(leaf, cache.buffer_pipeline.inflight_backup_node_ids) @@ -939,7 +846,6 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): ], [(leaf, PoolName.SWA)], ) - # One admission per node per walk. self.assertEqual(admissions.call_count, len(node_ids)) self.assertTrue(cache.buffer_backup_pending(leaf)) self._pump_hicache_until( @@ -983,7 +889,6 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): @requires_hicache def test_buffer_only_split_intent_writes_every_piece(self): - """Repeated splits preserve every pool's write once, parents first.""" storage_dir = self._make_storage_dir() cache, allocator, req_to_token_pool = self._build_buffer_hicache_fixture( storage_dir @@ -1044,7 +949,6 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): "only FULL-only caches split the leaf at the prompt", ) def test_buffer_only_finish_writes_prompt_and_output(self): - """The finish-path split preserves both prompt and output writes.""" storage_dir = self._make_storage_dir() cache, allocator, req_to_token_pool = self._build_buffer_hicache_fixture( storage_dir @@ -1071,9 +975,8 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): @requires_hicache def test_buffer_only_writes_component_pools_independently(self): - """A component's own pool (aliased host entry, per-pool keys, - BACKUP_HOST transfers) is written, believed and re-written - independently of the KV pool.""" + """A component's own pool is written, believed and re-written independently + of the KV pool.""" storage_dir = self._make_storage_dir() # Component buffer-mode hooks are Python-core only. @@ -1203,19 +1106,14 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): ) self.assertEqual(self._host_avail_sizes(cache), avail0) - # Everything believed stored: a requested backup is absorbed. cache.request_buffer_backup(leaf) self.assertNotIn(leaf, pipeline.inflight_backup_node_ids) cache.sanity_check() @requires_hicache def test_buffer_only_read_path_roundtrip(self): - """Read path end to end: prefetch -> staged (host bounce only, - nothing device-side, unmatchable, stable readiness, counters fed) -> - admission-time load-back against a saturated-but-evictable pool - (evict-before-alloc) publishing pre-ack -> ack frees the bounce. - Data bytes match the producer's; no CPU-tier KV events anywhere; - declines feed the outcome counters.""" + """Read path end to end: prefetch -> staged -> load-back that evicts before + alloc and publishes pre-ack -> ack frees the bounce.""" storage_dir = self._make_storage_dir() seq = self._buffer_swa_seq() @@ -1260,8 +1158,8 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): self.assertEqual(stats["l3_demand_requests"], 1) self.assertEqual(stats["l3_miss_tokens"], 0) - # Saturate the device pool with unrelated evictable spans: the - # load-back must evict, not degrade to recompute (run-2 regression). + # Saturate the device pool with unrelated evictable spans: load-back must + # evict, not degrade to recompute. def _avail(): if cons.supports_swa(): return cons.token_to_kv_pool_allocator.full_available_size() @@ -1279,7 +1177,7 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): from sglang.srt.mem_cache.base_prefix_cache import InitLoadBackParams held = cons.buffer_pipeline.staged_prefetches[req_id] - req = mock.Mock() + req = self._make_req(cons.req_to_token_pool) req.rid = req_id.rid req.cache_request_handle = req_id req.extra_key = None @@ -1299,8 +1197,6 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): (m.device_prefix_len, m.last_device_node), (len(seq), spliced_node) ) - # Ack: bounce freed, beliefs fed, tree holds no host values, and the - # loaded KV bytes equal the producer's. self._pump_hicache_until( cons, lambda: ( @@ -1321,8 +1217,6 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): self._all_page_hashes(cons, leaf) ) ) - # The fetch is evidence for every pool it delivered, so the - # consumer's own re-write of the span is skipped pool by pool. for transfer in aux_storage_key_transfers(cons, leaf) or (): self.assertTrue( cons.storage_existence_cache.pool(transfer.name).contains_all( @@ -1502,7 +1396,6 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): cons, cons_alloc, cons_rtp = self._build_buffer_hicache_fixture(storage_dir) stats = cons._prefetch_outcome_stats - # Query BEFORE any producer wrote the span: full miss -> revoked. req_id = CacheRequestHandle("early-query-miss", 0) cons.prefetch_from_storage( req_id, cons.root_node_handle(), array("q", seq), None, None @@ -1542,7 +1435,6 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): self.assertNotIn(req_id.rid, cons.storage_prefetch_retries._pending) self.assertEqual(cons.pop_prefetch_loaded_tokens(req_id), len(seq)) - # Unserved markers must not leak: abort cleanup ... aborted_rid = CacheRequestHandle("aborted-miss", 0) cons.prefetch_from_storage( aborted_rid, @@ -1575,11 +1467,9 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): @requires_hicache @requires_ps1_window4 def test_buffer_only_anchor_lock_cap_clamped_by_context_headroom(self): - """Deadlock invariant: cap <= pool - context_length (floor 0), so - pinned anchors always leave room to admit the largest request.""" - # The cap is pool-size arithmetic, not a layout property; pin to one - # SWA config (which also covers the SWAKVPool full_kv_pool branch) so - # the retained per-fixture device memory stays bounded. + """The anchor lock cap is clamped to pool - context_length (floor 0), so + pinned anchors leave room for the largest request.""" + # Layout-independent; pinned to one config to bound GPU memory. storage_dir = self._make_storage_dir() from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool @@ -1603,14 +1493,8 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): @requires_hicache @requires(lambda cfg: cfg.has_swa, "SWA-specific admission accounting") def test_buffer_load_back_swa_window_charged_at_admission(self): - """Admission contract: a request the SWA budget gate accepts must be - allocatable at batch time (estimate_swa_kv_tokens: "an admitted request - cannot OOM"). Regression: buffer mode surfaced a staged prefetch as - host_hit_length only, so the gate never charged the SWA window that - consumption (init_load_back -> cc.load) allocates and the request - lock pins; with the rest of the SWA pool batch-held, the batch alloc - fell short by up to one window and raised the fail-loud prefill OOM - (prod scheduler crash, 2026-08-17).""" + """A request the SWA budget gate accepts must be allocatable at batch time: + a staged prefetch's SWA window must be charged at admission.""" from sglang.srt.mem_cache.allocation import ( alloc_paged_token_slots_extend, alloc_token_slots, @@ -1645,7 +1529,7 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): hold = cons_alloc.swa_available_size() - (window + extend_need - 1) self.assertIsNotNone(cons_alloc.swa_attn_allocator.alloc(hold // ps * ps)) - # The adder's SWA gate for this request (_swa_budget_for_req). + # The adder's SWA budget gate for this request. req = SimpleNamespace( rid=req_id.rid, cache_request_handle=req_id, @@ -1662,9 +1546,8 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): ) budget = cons_alloc.swa_available_size() + cons.swa_evictable_size() - # Consume at admission (init_load_back + request lock). held = cons.buffer_pipeline.staged_prefetches[req_id] - req = mock.Mock() + req = self._make_req(cons.req_to_token_pool) req.rid = req_id.rid req.cache_request_handle = req_id req.extra_key = None @@ -1683,7 +1566,6 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): cons.inc_lock_ref(last_node) # the admission lock self.assertEqual(cons.swa_evictable_size(), 0) # window is protected - # FULL stays roomy: only SWA can fail below. self.assertGreaterEqual(cons_alloc.full_available_size(), extend_need + ps) if reserved <= budget: @@ -1715,12 +1597,11 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): f"but the batch alloc OOMed: {e}" ) else: - # Rejection must come from the surfaced window charge. self.assertGreaterEqual(surfaced_swa_hit, window) @requires_hicache def test_buffer_only_load_back_drops_on_sibling_published_span(self): - """Queued-UAF regression: a sibling publishes the staged span between + """A sibling publishes the staged span between staging and consumption; the live pre-check must drop the hold before any device allocation or H2D exists.""" storage_dir = self._make_storage_dir() @@ -1747,7 +1628,6 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): ) cons.pop_prefetch_loaded_tokens(req_id) - # Sibling publishes the identical span (live FULL + SWA). self._insert(cons, cons_alloc, cons_rtp, seq) sib = match_prefix(cons, seq) self.assertEqual(sib.device_prefix_len, len(seq)) @@ -1764,7 +1644,6 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): spliced = self._consume_staged_prefetch(cons, req_id, prefix_len=0) self.assertEqual(int(spliced.numel()), 0) - # Nothing device-side happened; sibling slots and staging intact. self.assertEqual(cons.token_to_kv_pool_allocator.available_size(), dev_avail0) self.assertEqual(cons.buffer_pipeline.ongoing_buffer_load_back, {}) self.assertFalse(cons.buffer_pipeline.has_staged(req_id)) @@ -1790,7 +1669,6 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): @requires_hicache @requires(lambda cfg: cfg.has_swa, "masked overlap requires an SWA component") def test_buffer_only_load_back_uses_full_behind_swa_tombstone(self): - """FULL-only rematch keeps resident FULL and repairs missing SWA rows.""" storage_dir = self._make_storage_dir() seq = self._buffer_swa_seq() self._produce_buffer_l3(storage_dir, seq) @@ -1809,7 +1687,6 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): @requires_hicache @requires(lambda cfg: cfg.has_swa, "masked overlap requires an SWA component") def test_buffer_only_load_back_reuses_partial_masked_full(self): - """FULL-only rematch reuses a resident head and fetches only its tail.""" page_size = self.cfg.page_size storage_dir = self._make_storage_dir() @@ -1886,7 +1763,6 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): @requires_hicache @requires(lambda cfg: cfg.has_swa, "masked overlap requires an SWA component") def test_buffer_only_masked_head_is_not_evicted_for_tail_load(self): - """Loading past a masked FULL head preserves its existing ownership.""" page_size = self.cfg.page_size storage_dir = self._make_storage_dir() @@ -1921,10 +1797,8 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): ) cons.pop_prefetch_loaded_tokens(req_id) - # Admission's temporary request lock must protect the head on its own. cons.buffer_pipeline.release_anchor_lock(req_id) self.assertEqual(cons.buffer_pipeline.anchor_locks, {}) - # Starve FULL only; SWA stays free so the window still loads. ballast = cons_alloc.full_attn_allocator.alloc(cons_alloc.full_available_size()) self.assertIsNotNone(ballast) self.addCleanup(cons_alloc.full_attn_allocator.free, ballast) @@ -1940,7 +1814,6 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): self._consume_staged_prefetch(cons, req_id, prefix_len=0) self.assertEqual(accounted0, full_accounted(), "a FULL page is free and owned") - # The resident FULL head was reused rather than evicted for the tail. live = match_prefix(cons, seq) self.assertEqual(live.full_kv_hit_length, split_at) cons.sanity_check() @@ -1981,7 +1854,7 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): return real_load(*args, **kwargs) f = cons.buffer_pipeline.staged_prefetches[req_id] - req = mock.Mock() + req = self._make_req(cons.req_to_token_pool) req.rid = req_id.rid req.cache_request_handle = req_id req.extra_key = None @@ -2000,19 +1873,10 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): @requires_hicache @requires_ps1_window4 def test_buffer_only_load_back_trims_head_published_by_sibling(self): - """Growth-waste regression: a sibling publishing the span HEAD after - enqueue used to invalidate the whole staged fetch at consumption - (splice base moved past matched_len -> full drop, every fetched byte - wasted). Consumption must instead splice the tail beyond the live - prefix: sibling head slots stay untouched (add-only insert), the - tail carries the producer's bytes, and the ack frees the entire - bounce including the trimmed head. - - A window-sized fetch still passes the global threshold, but after - sibling growth its FULL splice is shorter than its SWA transfer.""" - # Buffer-mode plan/commit logic is layout-independent, and each - # hicache fixture retains its pools for the whole file run. Pin to - # one config so the matrix does not exhaust a small CI GPU. + """A sibling publishing the span head after enqueue must not drop the staged + fetch; consumption splices the tail beyond the live prefix and the ack frees + the whole bounce, trimmed head included.""" + # Layout-independent; pinned to one config to bound GPU memory. storage_dir = self._make_storage_dir() with self.subTest(span="over-window"): @@ -2032,9 +1896,7 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): the surface-time plan must report (0, 0) and free it (bounce, anchor pin, occupancy) — a kept hold would leak, since admission without a host hit never calls init_load_back.""" - # Buffer-mode plan/commit logic is layout-independent, and each - # hicache fixture retains its pools for the whole file run. Pin to - # one config so the matrix does not exhaust a small CI GPU. + # Layout-independent; pinned to one config to bound GPU memory. storage_dir = self._make_storage_dir() seq = self._buffer_swa_seq() @@ -2059,19 +1921,17 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): self._insert(cons, cons_alloc, cons_rtp, seq) live = match_prefix(cons, seq) - req = mock.Mock( - rid=req_id.rid, - cache_request_handle=req_id, - prefix_len=live.device_prefix_len, - last_node=live.last_device_node, - ) + req = self._make_req(cons.req_to_token_pool) + req.rid = req_id.rid + req.cache_request_handle = req_id + req.prefix_len = live.device_prefix_len + req.last_node = live.last_device_node self.assertTrue(cons.buffer_pipeline.prepare_staged_prefetch(req)) self.assertEqual((req.host_hit_length, req.swa_host_hit_length), (0, 0)) self.assertFalse(cons.buffer_pipeline.has_staged(req_id)) self.assertEqual(cons.buffer_pipeline.anchor_locks, {}) self.assertEqual(cons.cache_controller.prefetch_tokens_occupied, 0) self.assertEqual(self._host_avail_sizes(cons), avail0) - # Idempotent once freed. self.assertTrue(cons.buffer_pipeline.prepare_staged_prefetch(req)) self.assertIsNone(req.staged_prefetch_plan) cons.sanity_check() @@ -2083,9 +1943,7 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): in flight makes the fetch unconsumable; the IO-commit gate must cancel it BEFORE the bounce alloc and the storage read (counted as declined_device_covered), leaving no staging or occupancy behind.""" - # Buffer-mode plan/commit logic is layout-independent, and each - # hicache fixture retains its pools for the whole file run. Pin to - # one config so the matrix does not exhaust a small CI GPU. + # Layout-independent; pinned to one config to bound GPU memory. storage_dir = self._make_storage_dir() seq = self._buffer_swa_seq() @@ -2127,14 +1985,13 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): @requires_hicache @requires_swa def test_buffer_only_swa_window_semantics(self): - """Root and mid-tree requests and hits share a complete-window minimum.""" window = self.cfg.sliding_window_size if window <= self.cfg.page_size: self.skipTest("window fits in one page") sw_pages = (window + self.cfg.page_size - 1) // self.cfg.page_size storage_dir = self._make_storage_dir() - # 1. A root-anchored sub-window request is declined before querying. + # A root-anchored sub-window request is declined before querying. seq = self._make_seq(1, sw_pages - 1) self._produce_buffer_l3(storage_dir, seq, marker=5) cons, _, _ = self._build_buffer_hicache_fixture(storage_dir) @@ -2156,7 +2013,7 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): self.assertEqual(self._host_avail_sizes(cons), avail) cons.sanity_check() - # 2. The same threshold applies mid-tree; an exact window is accepted. + # Mid-tree uses the same threshold; an exact window is accepted. if sw_pages >= 2: seq_a = self._make_seq(1, max(2, sw_pages)) seq_ab = seq_a + self._make_seq(900, sw_pages) @@ -2227,7 +2084,7 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): self.assertEqual(self._host_avail_sizes(cons2), avail2) cons2.sanity_check() - # 3. An accepted request with a sub-window L3 hit is also declined. + # An accepted request with a sub-window L3 hit is also declined. full = self._make_seq(6000, sw_pages + 1) stored = full[: (sw_pages - 1) * self.cfg.page_size] self._produce_buffer_l3(storage_dir, stored, marker=6) @@ -2261,11 +2118,7 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): self.assertEqual(self._host_avail_sizes(cons3), avail3) cons3.sanity_check() - # TP consistency for SWA prefetch - def test_tp_swa_prefetch_dropped_when_peer_misses(self): - """A peer rank missing the SWA window drops the whole prefetch result - on every rank (TP-consistent all-or-nothing).""" setup = self._setup_swa_tp_prefetch() if setup is None: return @@ -2274,6 +2127,7 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): cons, _, _ = self._l3_consumer(storage_dir) cons.tp_world_size = 2 self._patch_tp_prefetch_sync(cons, drop_swa=True) + avail_before = cons.swa_kv_pool_host.available_size() self._consume_prefetch(cons, seq, CacheRequestHandle("drop", 0)) m = match_prefix(cons, seq) @@ -2281,11 +2135,11 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): self.assertFalse( self._swa_host_on_path(cons, seq), "SWA must be dropped when a peer misses" ) + cons.drain_storage_control_queues() # Drain the release queue. + self.assertEqual(cons.swa_kv_pool_host.available_size(), avail_before) cons.sanity_check() def test_tp_swa_prefetch_adopted_when_peer_present(self): - """When every rank has the full SWA window, SWA is adopted under tp>1 - (the single packed all_reduce path still works).""" setup = self._setup_swa_tp_prefetch() if setup is None: return @@ -2304,35 +2158,12 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): ) cons.sanity_check() - def test_tp_swa_prefetch_drop_frees_host_pool(self): - """A dropped SWA prefetch must return its whole host buffer to the pool - (no leak, no over-free).""" - setup = self._setup_swa_tp_prefetch() - if setup is None: - return - storage_dir, seq = setup - - cons, _, _ = self._l3_consumer(storage_dir) - cons.tp_world_size = 2 - self._patch_tp_prefetch_sync(cons, drop_swa=True) - avail_before = cons.swa_kv_pool_host.available_size() - self._consume_prefetch(cons, seq, CacheRequestHandle("drop", 0)) - - self.assertEqual( - match_prefix(cons, seq).host_hit_length, - 0, - ) - # Whole window dropped -> its host buffer is fully released back. - cons.drain_storage_control_queues() # Drain the release queue. - self.assertEqual(cons.swa_kv_pool_host.available_size(), avail_before) - @requires_full_swa_ps1 def test_prepare_prefetch_swa(self): cache, _, _ = self._build_hicache_fixture() sw = cache.sliding_window_size swa = cache.components[ComponentType.SWA] avail = cache.swa_kv_pool_host.available_size() - # zero-length prefetch -> does not participate self.assertEqual( swa.prepare_prefetch( cache.root_node_handle(), prefetch_tokens=0 @@ -2347,14 +2178,12 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): ).staging_tokens, sw - 1, ) - # a full window available -> one window of host pages self.assertEqual( swa.prepare_prefetch( cache.root_node_handle(), prefetch_tokens=sw ).staging_tokens, sw, ) - # a non-participating component never stages self.assertEqual( cache.components[ComponentType.FULL] .prepare_prefetch(cache.root_node_handle(), prefetch_tokens=sw) @@ -2369,9 +2198,8 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): @requires_swa_only @requires_hicache def test_buffer_only_hit_parks_on_swa_staging_shortfall(self): - """A hit whose SWA window cannot be staged parks whole and launches - once the pool frees. It is never forfeited, and it never launches with - the KV staged and the window not; a parked hit holds no host memory.""" + """A hit whose SWA window cannot be staged parks whole, holding no host + memory, and launches once the pool frees.""" storage_dir = self._make_storage_dir() seq = self._buffer_swa_seq() self._produce_buffer_l3(storage_dir, seq) @@ -2396,7 +2224,6 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): None, None, ) - # The query holds no SWA staging. self.assertEqual(pool.available_size(), avail) self._pump_hicache_until( cons, @@ -2405,7 +2232,6 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): ) self.assertIn(CacheRequestHandle("r", 0), cons.ongoing_prefetch) self.assertEqual(cons._prefetch_outcome_stats["declined_rate_limited"], 1) - # Parked, before the next drain retries it: nothing is held. self.assertEqual(pool.available_size(), avail) self._run_prefetch_to_completion(cons, CacheRequestHandle("r", 0)) cons.drain_storage_control_queues() @@ -2413,6 +2239,40 @@ class UnifiedRadixCacheStorageSuite(UnifiedRadixCacheStorageFixtures): self.assertEqual(m.device_prefix_len, len(seq)) cons.sanity_check() + @requires_swa_only + @requires( + lambda cfg: cfg.sliding_window_size > cfg.page_size, + "the window must reach past the leaf's own page", + ) + def test_swa_backup_reaches_device_ancestor_in_cache_mode_only(self): + """Cache mode carries a leaf's SWA window into the device-only parent; + buffer mode writes each node's SWA under its own pages.""" + cache, allocator, req_to_token_pool = self._build_hicache_fixture() + chain = self._build_chain_pages(cache, allocator, req_to_token_pool, 2) + if len(chain) < 2: + self.skipTest("chain too short") + parent, leaf = chain[-2], chain[-1] + write_backup(cache, leaf, write_back=True) + cache.writing_check(write_back=True) + self.assertIsNotNone(node_host_value(cache, leaf, ComponentType.SWA)) + self.assertIsNotNone(node_host_value(cache, parent, ComponentType.SWA)) + self.assertIsNone(node_host_value(cache, parent, ComponentType.FULL)) + + cache, allocator, req_to_token_pool = self._build_buffer_hicache_fixture( + self._make_storage_dir() + ) + chain = self._build_chain_pages(cache, allocator, req_to_token_pool, 2) + parent, leaf = chain[-2], chain[-1] + write_backup(cache, leaf) + swa_keys = { + intent.snapshot.node_id: list(intent.keys) + for intent in cache.buffer_pipeline.pending_write_queue + if intent.pool == PoolName.SWA + } + self.assertEqual(swa_keys[leaf], cache.tree_core.get_hash_values(leaf)) + self._drain_buffer_writes(cache) + cache.sanity_check() + define_config_test_classes(UnifiedRadixCacheStorageSuite, globals()) @@ -2544,12 +2404,8 @@ class TestPrefetchCommitOrdering(CustomTestCase): class TestUnifiedRadixPrefetchCorruption( UnifiedRadixPrefetchCorruptionFixtures, CustomTestCase ): - """Prefetch must not hang a backed-up host child under an un-backed-up parent. - - Under write-through that broke the "child backed up => parent backed up" - invariant, failing as an idle-sanity error and, on eviction, as - `_remove_leaf_from_parent -> assert v == node`. Fix: drop the refill. - """ + """Prefetch must not hang a backed-up host child under an un-backed-up parent + (breaks "child backed up => parent backed up"); the refill is dropped.""" ps = 16 cfg = CacheConfig( @@ -2716,13 +2572,8 @@ class TestUnifiedRadixPrefetchCorruption( class TestUnifiedRadixCacheStorageAttachBackfill( UnifiedRadixCacheStorageAttachBackfillFixtures, CustomTestCase ): - """Enabling a storage backend must hash nodes that predate it. - - Page hashes chain off the parent's last hash, so a node hashed against an - unhashed parent restarts the chain mid-sequence: its L3 keys then cover only a - suffix of the prefix they claim to represent, which aliases any unrelated - request that happens to start with those tokens. - """ + """Enabling storage must hash nodes that predate it: page hashes chain off the + parent, so an unhashed parent yields keys that alias unrelated requests.""" cfg = CacheConfig( page_size=4, @@ -2780,7 +2631,6 @@ class TestUnifiedRadixCacheStorageAttachBackfill( cache.tree_core.set_node_hash_values(parent, parent_hashes) def test_enabling_storage_backfills_the_tree(self): - """The tree is hashed by the time `enable_storage` flips on.""" cache = self._build_two_level_tree(storage_on_from_the_start=False) StorageAttachment(cache).apply_runtime_config( storage_backend="file", @@ -2800,134 +2650,142 @@ class TestUnifiedRadixCacheStorageAttachBackfill( class TestAnchorLockOutcomePolicy(AnchorLockOutcomePolicyFixtures, CustomTestCase): - """try_lock_anchor finds the anchor by re-matching the live tree (no - carried node id to go stale): prefix intact -> lock the live node; - prefix shrunk -> anchor_lost so the caller re-plans; over the whole cap - -> cap_skip (launch unlocked, since no drain can ever fit it), merely - over budget -> cap_busy (park).""" + """try_lock_anchor re-matches the live tree: intact -> lock; shrunk -> + anchor_lost; over cap -> cap_skip; over budget -> cap_busy.""" + cfg = CacheConfig() _REQ = CacheRequestHandle("req-1", 0) _PREFIX = list(range(100, 100 + 8)) def test_intact_prefix_locks_live_node(self): - cache = self._make_cache(live_match_len=len(self._PREFIX)) - pipeline = self._make_pipeline(cache) - self.assertEqual(pipeline.try_lock_anchor(self._REQ, 0)[0], "locked") - self.assertEqual(pipeline.anchor_locks[self._REQ].node_id, 99) + cache, pipeline = self._make_buffer_cache(resident=self._PREFIX) + leaf = match_prefix(cache, self._PREFIX).last_device_node + self.assertEqual( + pipeline.try_lock_anchor(self._REQ, 0), ("locked", len(self._PREFIX)) + ) + self.assertEqual(pipeline.anchor_locks[self._REQ].node_id, leaf) self.assertEqual(pipeline.anchor_locked_tokens_, len(self._PREFIX)) - cache.tree_core.inc_full_pin.assert_called_once_with(99) - cache.inc_lock_ref.assert_not_called() + self.assertEqual(node_device_lock_ref(cache, leaf, ComponentType.FULL), 1) + pipeline.release_anchor_lock(self._REQ) + self.assertEqual(node_device_lock_ref(cache, leaf, ComponentType.FULL), 0) def test_shrunk_prefix_reports_anchor_lost(self): - cache = self._make_cache(live_match_len=len(self._PREFIX) - 2) - pipeline = self._make_pipeline(cache) + cache, pipeline = self._make_buffer_cache(resident=self._PREFIX[:-2]) self.assertEqual(pipeline.try_lock_anchor(self._REQ, 0)[0], "anchor_lost") self.assertEqual(pipeline.anchor_locks, {}) self.assertEqual(pipeline.anchor_locked_tokens_, 0) def test_storage_cleanup_releases_buffer_prefetch_anchor(self): - cache = self._make_cache(live_match_len=len(self._PREFIX)) - pipeline = self._make_pipeline(cache) - cache.buffer_pipeline = pipeline - self.assertEqual(pipeline.try_lock_anchor(self._REQ, 0)[0], "locked") - host_indices = torch.arange(4) - cache.ongoing_prefetch = { - self._REQ: _OngoingPrefetch( - anchor_node_id=99, - prefetch_key=RadixKey(array("q", range(16))), - host_indices=host_indices, - operation=mock.Mock( - buffer_host_occupied_units=4, pool_transfers_done=True - ), - anchor_lock_params=None, - comp_xfers={}, - ) - } - cache.ongoing_backup = {} - cache.host_memory_mode = "buffer_only" - cache._prefetch_occupied_span.side_effect = ( - lambda key, indices, *, operation=None: ( - UnifiedRadixCache._prefetch_occupied_span( - cache, key, indices, operation=operation - ) - ) + storage_dir = self._make_storage_dir() + suffix = list(range(200, 208)) + self._produce_buffer_l3(storage_dir, self._PREFIX + suffix) + cache, pipeline = self._make_buffer_cache( + resident=self._PREFIX, storage_dir=storage_dir, prefix=None ) - cache._retire_ongoing_prefetch.side_effect = lambda *args: ( - UnifiedRadixCache._retire_ongoing_prefetch(cache, *args) + anchor = match_prefix(cache, self._PREFIX).last_device_node + pipeline.pop_prefix_ctx(self._REQ) + cache.prefetch_from_storage( + self._REQ, + anchor, + array("q", suffix), + cache.get_last_hash_value(anchor), + None, + matched_prefix_tokens=list(self._PREFIX), ) - controller = cache.cache_controller - controller.terminate_prefetch.return_value = (4, None) - controller.prefetch_tokens_occupied = 12 - - StorageAttachment(cache)._release_pending_storage_ops() - + self._pump_hicache_until( + cache, + lambda: self._REQ in pipeline.anchor_locks, + "the storage hit did not pin its anchor", + ) + self.assertEqual(node_device_lock_ref(cache, anchor, ComponentType.FULL), 1) + detached, message = StorageAttachment(cache).detach() + self.assertTrue(detached, message) self.assertEqual(cache.ongoing_prefetch, {}) self.assertEqual(pipeline.anchor_locks, {}) self.assertEqual(pipeline.anchor_locked_tokens_, 0) self.assertNotIn(self._REQ, pipeline._prefetch_prefix_ctx) - cache.tree_core.dec_full_pin.assert_called_once_with(99) - cache.dec_lock_ref.assert_not_called() - cache.dec_host_lock_ref.assert_not_called() - self.assertEqual(controller.prefetch_tokens_occupied, 8) + self.assertEqual(node_device_lock_ref(cache, anchor, ComponentType.FULL), 0) + self.assertEqual(cache.cache_controller.prefetch_tokens_occupied, 0) def test_positive_hit_with_lost_anchor_reports_anchor_lost(self): - cache = UnifiedRadixCache.__new__(UnifiedRadixCache) - cache.storage_prefetch_retries = StoragePrefetchRetries() - cache.ongoing_prefetch = { - self._REQ: SimpleNamespace( - operation=SimpleNamespace(storage_start=4, storage_hit_count=8) - ) - } - cache._finish_storage_prefetch = mock.Mock() - cache.revoke_pending_prefetch = mock.Mock() - - cache._handle_storage_prefetch_anchor_loss(self._REQ) - - cache._finish_storage_prefetch.assert_called_once_with( - self._REQ, fulfilled_tokens=0, reason="anchor_lost" + """A storage hit whose anchor was evicted before the progress check + pinned it is declined as anchor_lost, and the request replans the + widened span right away -- skipping the query, the prior hit having + proved it stored, and the poll interval.""" + storage_dir = self._make_storage_dir() + suffix = list(range(200, 208)) + self._produce_buffer_l3(storage_dir, self._PREFIX + suffix) + cache, pipeline = self._make_buffer_cache( + resident=self._PREFIX, storage_dir=storage_dir, prefix=None ) - # The eviction widened the span, so the request replans over it -- - # skipping the query, the prior hit having proved it stored, and the - # poll interval, which would only let admission run without it. - req = SimpleNamespace(rid=self._REQ.rid, storage_prefetch_retry_attempts=0) + anchor = match_prefix(cache, self._PREFIX).last_device_node + pipeline.pop_prefix_ctx(self._REQ) + cache.prefetch_from_storage( + self._REQ, + anchor, + array("q", suffix), + cache.get_last_hash_value(anchor), + None, + matched_prefix_tokens=list(self._PREFIX), + ) + self.assertIn(self._REQ, cache.ongoing_prefetch) + cache.evict(EvictParams(num_tokens=len(self._PREFIX))) + self.assertEqual(match_prefix(cache, self._PREFIX).device_prefix_len, 0) + self._pump_hicache_until( + cache, + lambda: self._REQ not in cache.ongoing_prefetch, + "the anchor loss did not end the prefetch", + ) + self.assertEqual(pipeline.anchor_locks, {}) + req = self._make_req(cache.req_to_token_pool) + req.rid = self._REQ.rid self.assertEqual( - cache.storage_prefetch_retries.pop_ready([req], 0, 8), [(req, 12)] + cache.storage_prefetch_retries.pop_ready([req], 0, 8), + [(req, len(self._PREFIX) + len(suffix))], ) - cache.revoke_pending_prefetch.assert_called_once_with(self._REQ) def test_over_cap_reports_cap_skip_after_full_rematch(self): - cache = self._make_cache(live_match_len=len(self._PREFIX)) - pipeline = self._make_pipeline(cache, cap_tokens=len(self._PREFIX) - 1) + cache, pipeline = self._make_buffer_cache( + resident=self._PREFIX, cap_tokens=len(self._PREFIX) - 1 + ) + leaf = match_prefix(cache, self._PREFIX).last_device_node self.assertEqual(pipeline.try_lock_anchor(self._REQ, 0)[0], "cap_skip") self.assertEqual(pipeline.anchor_locks, {}) - cache.tree_core.match_full_device_prefix.assert_called_once() + self.assertEqual(node_device_lock_ref(cache, leaf, ComponentType.FULL), 0) def test_root_anchor_reports_no_anchor(self): - cache = self._make_cache(live_match_len=0) - cache.ongoing_prefetch[self._REQ].prefetch_key = RadixKey(array("q", range(8))) - pipeline = self._make_pipeline(cache) - pipeline._prefetch_prefix_ctx[self._REQ] = ([], None, None) + cache, pipeline = self._make_buffer_cache(prefix=[]) + cache.ongoing_prefetch[self._REQ] = _OngoingPrefetch( + anchor_node_id=cache.root_node_handle(), + prefetch_key=RadixKey(array("q", range(8))), + host_indices=None, + operation=None, + anchor_lock_params=None, + comp_xfers={}, + ) self.assertEqual(pipeline.try_lock_anchor(self._REQ, 8), ("no_anchor", 0)) - cache.tree_core.match_full_device_prefix.assert_called_once() + cache.ongoing_prefetch.pop(self._REQ) def test_anchor_lock_requires_release_before_retry(self): - cache = self._make_cache(live_match_len=8, pinned_len=12) - pipeline = self._make_pipeline(cache) + tokens = self._PREFIX + [900, 901, 902, 903] + cache, pipeline = self._make_buffer_cache(resident=tokens) + node = match_prefix(cache, tokens).last_device_node self.assertEqual(pipeline.try_lock_anchor(self._REQ, 0), ("locked", 8)) with self.assertRaisesRegex(AssertionError, "prefetch anchor already locked"): pipeline.try_lock_anchor(self._REQ, 0) - self.assertEqual(pipeline.anchor_locked_tokens_, 12) - cache.tree_core.match_full_device_prefix.assert_called_once() - cache.tree_core.inc_full_pin.assert_called_once_with(99) - + self.assertEqual(pipeline.anchor_locked_tokens_, len(tokens)) + self.assertEqual(node_device_lock_ref(cache, node, ComponentType.FULL), 1) pipeline.release_anchor_lock(self._REQ) self.assertEqual(pipeline.anchor_locked_tokens_, 0) - cache.tree_core.dec_full_pin.assert_called_once_with(99) - cache.tree_core.match_full_device_prefix.return_value = (8, 100, 8) + self.assertEqual(node_device_lock_ref(cache, node, ComponentType.FULL), 0) + # A branch at the prefix boundary splits the node; the retry re-matches + # the live tree and pins the new prefix node alone. + self._insert_tokens(cache, self._PREFIX + [800, 801]) + prefix_node = match_prefix(cache, self._PREFIX).last_device_node self.assertEqual(pipeline.try_lock_anchor(self._REQ, 0), ("locked", 8)) - self.assertEqual(pipeline.anchor_locks[self._REQ].node_id, 100) - self.assertEqual(pipeline.anchor_locked_tokens_, 8) - self.assertEqual(cache.tree_core.match_full_device_prefix.call_count, 2) + self.assertEqual(pipeline.anchor_locks[self._REQ].node_id, prefix_node) + self.assertEqual(pipeline.anchor_locked_tokens_, len(self._PREFIX)) + pipeline.release_anchor_lock(self._REQ) if __name__ == "__main__":