From 77173ffb08e94e474a719c1efd01fa534131c6a8 Mon Sep 17 00:00:00 2001 From: Shangming Cai Date: Thu, 24 Sep 2026 23:56:17 +0800 Subject: [PATCH] [HiCache] Demote SWA KV to host on write_back eviction instead of dropping it (#40712) Co-authored-by: Claude Fable 5 --- .../mem_cache/unified_cache/components/swa.py | 41 ++++++++ .../test_unified_radix_cache_unittest.py | 96 +++++++++++++++++++ 2 files changed, 137 insertions(+) diff --git a/python/sglang/srt/mem_cache/unified_cache/components/swa.py b/python/sglang/srt/mem_cache/unified_cache/components/swa.py index 0fdac250ab..054fb94223 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/swa.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/swa.py @@ -751,6 +751,8 @@ class SWAComponent(TreeComponent): return x.id if not enabled: x_next = lru.get_prev_no_lock(x) + # write_back: demote the SWA KV to host before the internal tombstone. + self._maybe_backup_node_before_swa_tombstone(x) self.tree_core._evict_component_and_detach_lru( x, self, @@ -765,6 +767,45 @@ class SWAComponent(TreeComponent): self._evict_device_cursor = lru.cursor_next() if enabled else x_next return None + def _maybe_backup_node_before_swa_tombstone(self, node: UnifiedTreeNode) -> None: + """Demote an internal node's SWA KV to host before its tombstone + (write_back only), mirroring the leaf deferred-demote path. + + The match validator treats an unbacked tombstone as a window reset, + so a dropped internal SWA segment caps the match frontier until a + full sliding window re-accumulates below it, leaving up to one + window of still-resident KV unservable. The leaf backup walk covers + ancestors only within one window of the evicted leaf, so an + internal node whose child spans the window arrives here unbacked. + Best-effort: this walk must make progress (it satisfies an imminent + allocation), so any failure falls back to the legacy drop. + """ + cache = self.cache + cd = node.component_data[self.component_type] + if ( + cache.cache_controller is None + or not cache.is_write_back + or not self.tree_core.has_swa_host_pool + or cd.host_value is not None + or node.backuped + or node.component_data[BASE_COMPONENT_TYPE].value is None + ): + return + # The backup executor pre-evicts only the KV host pool; make room + # in the SWA host pool the way the PREFETCH hook does. + needed = sum( + len(n.component_data[self.component_type].value) + for n in self._collect_unbacked_swa_nodes(node) + ) + if needed == 0: + return + if ( + self._swa_kv_pool_host is not None + and self._swa_kv_pool_host.available_size() < needed + ): + cache.evict_host(needed, self.component_type) + cache.backup_node_for_write_back(node.id) + def _evict_device_end(self) -> None: """Clear the device-eviction walk cursor state.""" if self.tree_core.enable_session_radix_cache: diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py index 2bf995a606..73abcecf7a 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py +++ b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py @@ -6207,6 +6207,102 @@ class UnifiedRadixCacheSuite: ) cache.sanity_check() + def _build_internal_swa_fixture(self, write_policy): + """HiCache fixture where seq_a's node is INTERNAL and its child spans + the sliding window, so the leaf backup walk cannot cover it.""" + cache, allocator, req_to_token_pool = build_fixture(self.cfg) + self._init_hicache(cache, write_policy=write_policy) + ps = self.cfg.page_size + tail_pages = max(1, -(-self.cfg.sliding_window_size // ps)) + seq_a = self._make_seq(1, 2) + seq_b = seq_a + self._make_seq(1000, tail_pages) + self._insert(cache, allocator, req_to_token_pool, seq_a) + self._insert(cache, allocator, req_to_token_pool, seq_b) + return cache, req_to_token_pool, seq_a, seq_b + + def test_hicache_write_back_internal_swa_evict_demotes_kv(self): + """write_back: an internal node's tombstoned SWA KV is demoted to + host, keeping the node a valid match boundary so the KV beneath it + stays servable instead of losing a window of matchable prefix.""" + if not self.cfg.has_swa: + self.skipTest("requires SWA component") + if self.cfg.has_mamba: + self.skipTest("no hicache strategy covers FULL+SWA+MAMBA") + # TODO(ShangmingCai): port the internal-node demote to the Rust core; + # its eviction walk still tombstones the SWA KV without a host backup. + if _selected_tree_core_test_backend() == "rust": + self.skipTest("internal-node SWA demote is Python-core only") + cache, req_to_token_pool, seq_a, seq_b = self._build_internal_swa_fixture( + "write_back" + ) + + result = cache.evict(EvictParams(num_tokens=0, swa_num_tokens=len(seq_b))) + self.assertGreaterEqual(result.swa_num_tokens_evicted, len(seq_a)) + + m = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq_a)))) + node = m.best_match_node + self.assertNotEqual( + node, + cache.root_node_handle(), + "host-demoted SWA must keep the node a valid match boundary", + ) + self.assertIsNone( + _device_value(cache, node, ComponentType.SWA), + "device SWA was tombstoned", + ) + self.assertIsNotNone( + _host_value(cache, node, ComponentType.SWA), + "SWA KV demoted to host, not dropped", + ) + self.assertEqual( + len(m.device_indices) + m.host_hit_length, + len(seq_a), + "the full prefix stays servable", + ) + self.assertGreaterEqual( + m.swa_host_hit_length, 1, "load-back armed for the host SWA KV" + ) + + # Scheduler-side serve path: init_load_back revives the SWA KV and + # the node's still-resident full KV serves the prefix. + req = self._make_req(req_to_token_pool) + self._apply_match_to_req(req, m) + new_indices, last_node = cache.init_load_back( + InitLoadBackParams( + best_match_node=m.best_match_node, + host_hit_length=m.host_hit_length, + req=req, + ) + ) + self.assertEqual(last_node, node) + self.assertEqual(len(m.device_indices) + len(new_indices), len(seq_a)) + self.assertIsNotNone( + _device_value(cache, node, ComponentType.SWA), + "SWA KV revived on device", + ) + self._finish_pending_loads(cache) + self._release_ongoing_load_back_locks(cache) + cache.sanity_check() + + def test_hicache_write_through_internal_swa_evict_keeps_drop(self): + """Non-write_back policies keep the legacy tombstone-and-drop.""" + if not self.cfg.has_swa: + self.skipTest("requires SWA component") + if self.cfg.has_mamba: + self.skipTest("no hicache strategy covers FULL+SWA+MAMBA") + cache, _, seq_a, seq_b = self._build_internal_swa_fixture("write_through") + + cache.evict(EvictParams(num_tokens=0, swa_num_tokens=len(seq_b))) + + m = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq_a)))) + if m.best_match_node != cache.root_node_handle(): + node = m.best_match_node + self.assertIsNone( + _host_value(cache, node, ComponentType.SWA), + "write_through keeps the legacy drop: no host demote", + ) + cache.sanity_check() + def _build_chain_pages(self, cache, allocator, req_to_token_pool, num_pages): """Insert an incremental chain of single-page extensions.