From 2bae12b9b355b6d149b2db4ca30352aa20755f1c Mon Sep 17 00:00:00 2001 From: metamergebot Date: Sat, 3 Oct 2026 09:45:56 -0700 Subject: [PATCH] [HiCache] Fix write-back SWA insert backups tripping the write-through pending-ack assert (#42264) Co-authored-by: merrymercy Co-authored-by: 842974287 <842974287@users.noreply.github.com> --- .../unified_cache/unified_tree_core.py | 10 +- .../src/tests/unified_tree_core.rs | 101 ++++++++++++++++++ .../src/unified_tree_core.rs | 12 +-- .../test_rust_tree_core_integration.py | 33 +++--- .../test_unified_radix_cache_unittest.py | 62 +++++++++++ 5 files changed, 191 insertions(+), 27 deletions(-) diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py index deffec7fba..d4f088a64f 100644 --- a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py +++ b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py @@ -1383,7 +1383,8 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): ) def _insert_tail_step(self, state: _InsertWalkState) -> None: - """Refresh the LRUs and append terminal backup actions.""" + """Refresh the LRUs and append the insert backup: a new-leaf write-through, + or an SWA window publish on an existing backed node.""" if state.target_node is not self.root_node: for component in self.components: if component.component_type == BASE_COMPONENT_TYPE: @@ -1394,7 +1395,9 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): if self._should_backup_after_insert(state): state.pending_actions.append( - self._build_backup_kv_action(state.target_node) + self._build_backup_kv_action( + state.target_node, write_back=self.is_write_back + ) ) def _split_node( @@ -2463,7 +2466,8 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): def _build_backup_kv_action( self, node: UnifiedTreeNode, write_back: bool = False ) -> BackupKV: - """Build the backup action for a node and its not-yet-persisted ancestors.""" + """Build the backup action for a node; write-through also chains its + unbacked ancestors.""" chain = [node] if not write_back: ancestor = node.parent diff --git a/rust/sglang-radix-tree/src/tests/unified_tree_core.rs b/rust/sglang-radix-tree/src/tests/unified_tree_core.rs index 3f43e267d9..e8468bd1fe 100644 --- a/rust/sglang-radix-tree/src/tests/unified_tree_core.rs +++ b/rust/sglang-radix-tree/src/tests/unified_tree_core.rs @@ -2813,6 +2813,107 @@ fn backup_kv_action_chains_unbacked_ancestors_first() { assert_eq!(action.node_ids, vec![c]); } +// Mirrors UnifiedRadixCache._execute_and_commit_kv_backup: back up each chain +// node and stamp every node its transfers cover with that node's ack. +// It skips the inc_lock_ref the Python side takes; the asserts do not depend on it. +fn execute_backup_kv_for_test(tc: &mut UnifiedTreeCore>, backup: &BackupKV) { + for &node_id in &backup.node_ids { + let (device_value, mut comp_xfers) = tc.build_backup_spec(node_id).expect("backup spec"); + if device_value.numel() == 0 && comp_xfers.is_empty() { + continue; + } + let mut publish_node_ids = Vec::new(); + for transfer in comp_xfers.values_mut().flatten() { + transfer.host_indices = transfer.device_indices.as_ref().map(Tensor::copy); + publish_node_ids.extend(transfer.nodes_to_load.iter().flatten().copied()); + } + if !publish_node_ids.contains(&node_id) { + publish_node_ids.push(node_id); + } + tc.commit_backup(node_id, device_value.copy(), comp_xfers) + .expect("live test node"); + tc.mark_write_through_pending(publish_node_ids, /* ack_id = */ node_id) + .expect("live test nodes"); + } +} + +#[test] +fn write_back_swa_publish_leaves_an_ancestor_pending_under_another_ack_alone() { + // c(Full unbacked) -> b(Full backed) -> t1(Full backed) + // -> d(Full unbacked) -> t2(Full backed) + // Every node holds a device-only SWA value inside the window. + let mut tc = UnifiedTreeCore::new( + CacheInitParams { + is_write_back: true, + enable_hicache: true, + has_swa_host_pool: true, + swa_sliding_window_size: Some(8), + ..Default::default() + }, + vec![FULL, SWA], + ); + let (key_t1, key_t2) = (vec![1, 2, 3, 4], vec![1, 2, 5, 6]); + tc.insert(&insert_params(&key_t1, &[10, 11, 12, 13])); + tc.insert(&insert_params(&key_t2, &[10, 11, 14, 15])); + tc.insert(&insert_params(&vec![1, 2, 3], &[10, 11, 12])); + tc.insert(&insert_params(&vec![1, 2, 5], &[10, 11, 14])); + let child = |parent, page: &[i64]| { + tc.arena + .child_on_page(parent, /* extra_key = */ None, page) + .expect("child on page") + }; + let c_idx = child(tc.arena.root(), &[1]); + let b_idx = child(c_idx, &[3]); + let t1_idx = child(b_idx, &[4]); + let d_idx = child(c_idx, &[5]); + let t2_idx = child(d_idx, &[6]); + let [c, b, t1, d, t2] = [c_idx, b_idx, t1_idx, d_idx, t2_idx].map(|idx| tc.arena.node(idx).id); + for idx in [c_idx, b_idx, t1_idx, d_idx, t2_idx] { + let full = tc.arena.node(idx).device_value(FULL).copy(); + tc.set_component_device_value(tc.arena.node(idx).id, SWA, full.copy()) + .expect("live test node"); + if [b_idx, t1_idx, t2_idx].contains(&idx) { + tc.arena.set_host_value(idx, FULL, full); + } + } + + // Two requests in one batch finish at t1 and t2; neither ack drains. + for (key, target) in [(&key_t1, t1), (&key_t2, t2)] { + let result = tc.insert(&insert_params(key, &[20, 21, 22, 23])); + let backups: Vec<&BackupKV> = result + .cache_actions + .iter() + .filter_map(|action| match action { + CacheAction::BackupKV(backup) => Some(backup), + _ => None, + }) + .collect(); + assert_eq!(backups.len(), 1); + assert_eq!(backups[0].node_ids, vec![target]); + execute_backup_kv_for_test(&mut tc, backups[0]); + } + + let pending = |node_id| { + tc.arena + .node(tc.arena.resolve(node_id).expect("live test node")) + .write_through_pending_id + }; + for (node_id, ack_id) in [(c, t1), (b, t1), (t1, t1), (d, t2), (t2, t2)] { + assert_eq!(pending(node_id), Some(ack_id)); + } + // Write-back defers the unbacked Full prefix to eviction. + assert!( + !tc.arena + .node(tc.arena.resolve(c).expect("live test node")) + .backuped() + ); + assert!( + !tc.arena + .node(tc.arena.resolve(d).expect("live test node")) + .backuped() + ); +} + #[test] fn backup_kv_action_stops_at_an_externally_stored_or_pending_ancestor() { let mut tc = core(); diff --git a/rust/sglang-radix-tree/src/unified_tree_core.rs b/rust/sglang-radix-tree/src/unified_tree_core.rs index f1a4ee175e..2436eafc67 100644 --- a/rust/sglang-radix-tree/src/unified_tree_core.rs +++ b/rust/sglang-radix-tree/src/unified_tree_core.rs @@ -1980,7 +1980,8 @@ impl UnifiedTreeCore { && self.needs_incremental_component_backup_(target_node_id) } - /// Refresh the LRUs and append terminal backup actions. + /// Refresh the LRUs and append the insert backup: a new-leaf write-through, + /// or an SWA window publish on an existing backed node. fn insert_tail_step_(&mut self, state: &mut InsertWalkState) { let target_node_id = state .target_node_id @@ -1997,10 +1998,8 @@ impl UnifiedTreeCore { } if self.should_backup_after_insert_(state, target_node_id) { - let backup = self.build_backup_kv_action_( - self.arena.node(target_node_id), - /* write_back = */ false, - ); + let backup = + self.build_backup_kv_action_(self.arena.node(target_node_id), self.is_write_back); state.pending_actions.push(CacheAction::BackupKV(backup)); } } @@ -3927,7 +3926,8 @@ impl UnifiedTreeCore { Ok(order) } - /// Build the backup action for a node and its not-yet-persisted ancestors. + /// Build the backup action for a node; write-through also chains its + /// unbacked ancestors. pub fn build_backup_kv_action_(&self, node: &Node, write_back: bool) -> BackupKV { let mut chain = vec![node.id]; if !write_back { diff --git a/test/registered/unit/mem_cache/test_rust_tree_core_integration.py b/test/registered/unit/mem_cache/test_rust_tree_core_integration.py index 043edae520..075e1e8b9d 100644 --- a/test/registered/unit/mem_cache/test_rust_tree_core_integration.py +++ b/test/registered/unit/mem_cache/test_rust_tree_core_integration.py @@ -3437,7 +3437,9 @@ def test_full_load_back_does_not_pin_restored_ancestor_aux( @pytest.mark.parametrize("backend", ["python", "rust"]) -def test_full_host_duplicates_follow_ack_order_after_pending_swa_split(backend): +def test_write_back_swa_repair_backup_leaves_the_unbacked_parent_to_eviction(backend): + """Write-back builds the SWA repair backup as the repaired node alone; the + parent's missing Full copy waits for eviction instead of joining at insert.""" cache, allocator = _swa_cache(window=4, backend=backend) core = cache.tree_core values = allocator.alloc(12) @@ -3458,7 +3460,6 @@ def test_full_host_duplicates_follow_ack_order_after_pending_swa_split(backend): for node in nodes: _complete_backup(core, node, full_offset=10000, aux_offset=20000) - # Repairing SWA later must issue an async backup even in write-back mode. # Reclaim the two internal SWA states and the parent's Full host copy. tracker = {} core.evict_device_start(ComponentType.SWA, 8) @@ -3478,8 +3479,8 @@ def test_full_host_duplicates_follow_ack_order_after_pending_swa_split(backend): _accumulate_step(step, {}, {}, {}) assert not core.is_backuped(parent) and core.is_backuped(child) - # Mock DMA submission only; insertion, backup publication, the pending - # split and FIFO acknowledgment all use the real shared controller. + # Mock DMA submission only; insertion, backup publication and FIFO + # acknowledgment all use the real shared controller. cache.cache_controller = SimpleNamespace(write_policy="write_back") cache._build_backup_sidecar = Mock(return_value=[]) submissions = [] @@ -3499,23 +3500,19 @@ def test_full_host_duplicates_follow_ack_order_after_pending_swa_split(backend): component_evicted_seqlens={ComponentType.SWA: 4}, ) ) - assert submissions == [(parent, 4), (child, 0)] - assert set(cache.ongoing_write_through) == {parent, child} - core.sanity_check([(node, node) for node in cache.ongoing_write_through], []) - cache.match_prefix(MatchPrefixParams(key=_key([0, 1]))) - prefix, suffix = cache.ongoing_write_through[parent].publish_node_ids - assert suffix == parent and prefix != parent - core.sanity_check([(node, node) for node in cache.ongoing_write_through], []) - for node in (parent, child): - cache._finish_write_through_ack(node) + # Repairing SWA still issues the async backup in write-back mode, but for + # the repaired node alone: a chained parent could already be pending under + # another request's window publish. + assert submissions == [(child, 0)] + assert set(cache.ongoing_write_through) == {child} + core.sanity_check([(child, child)], []) + cache._finish_write_through_ack(child) core.sanity_check([], []) - # The existing child keeps its place. New Full copies join at ack in - # prefix/suffix order, including when the pending node was internal. + # The existing Full copies keep their order; the parent stays unbacked + # until write-back eviction backs it up. expected = [ (values[4:8] + 10000).tolist(), - (values[:2] + 40000).tolist(), - (values[2:4] + 40000).tolist(), (values[8:] + 10000).tolist(), ] for host_indices in expected: @@ -3526,7 +3523,7 @@ def test_full_host_duplicates_follow_ack_order_after_pending_swa_split(backend): assert [value.tolist() for value in host_frees[ComponentType.FULL]] == [ host_indices ] - assert all(not core.is_backuped(node) for node in (prefix, parent, child, leaf)) + assert all(not core.is_backuped(node) for node in nodes) core.sanity_check([], []) diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py index 43e6af9a55..4b3c9bbcdb 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py +++ b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py @@ -8522,6 +8522,68 @@ class UnifiedRadixCacheSuite: cache.sanity_check() + def test_write_back_swa_publish_leaves_an_ancestor_pending_under_another_ack_alone( + self, + ): + """Write-back builds the insert backup as [target] only, so a second SWA + window publish never re-marks an ancestor another ack still holds.""" + if not self.cfg.has_swa: + self.skipTest("requires SWA") + if self._skip_unsupported_hicache_test(): + return + ps = self.cfg.page_size + if self.cfg.sliding_window_size <= 2 * ps: + self.skipTest("window must span past the leaf and its parent to reach c") + + cache, allocator, req_to_token_pool = build_fixture(self.cfg) + self._init_hicache(cache, write_policy="write_back") + + # c(Full unbacked) -> b(Full backed) -> t1(Full backed) + # -> d(Full unbacked) -> t2(Full backed) + # Every node holds a device-only SWA value inside the window. + c_seq = self._make_seq(1, 2) + t1_seq = c_seq + self._make_seq(1000, 1) + self._make_seq(2000, 1) + t2_seq = c_seq + self._make_seq(3000, 1) + self._make_seq(4000, 1) + for seq in (t1_seq, t2_seq, t1_seq[:-ps], t2_seq[:-ps]): + self._insert(cache, allocator, req_to_token_pool, seq) + + def leaf(seq): + return cache.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", seq))) + ).last_device_node + + c, b, t1 = self._path_chain(cache, leaf(t1_seq)) + _, d, t2 = self._path_chain(cache, leaf(t2_seq)) + for node in (b, t1, t2): + cache.tree_core.set_component_host_value_raw( + node, + ComponentType.FULL, + _device_value(cache, node, ComponentType.FULL).clone(), + ) + for node in (c, b, t1, d, t2): + self.assertIsNotNone(_device_value(cache, node, ComponentType.SWA)) + self.assertIsNone(_host_value(cache, node, ComponentType.SWA)) + + # Two requests in one batch finish at t1 and t2; neither ack drains. + with mock.patch.object( + cache, + "_execute_and_commit_kv_backup", + wraps=cache._execute_and_commit_kv_backup, + ) as backups: + for seq, target in ((t1_seq, t1), (t2_seq, t2)): + backups.reset_mock() + self._insert(cache, allocator, req_to_token_pool, seq) + self.assertEqual( + [call.args[0].node_ids for call in backups.call_args_list], + [[target]], + ) + + for node, ack in ((c, t1), (b, t1), (t1, t1), (d, t2), (t2, t2)): + self.assertEqual(cache.tree_core.get_write_through_pending_id(node), ack) + # Write-back defers the unbacked Full prefix to eviction. + self.assertFalse(cache.tree_core.is_backuped(c)) + self.assertFalse(cache.tree_core.is_backuped(d)) + class UnifiedLRUListBoundedRefreshTest(CustomTestCase): components = (ComponentType.FULL, ComponentType.SWA)