From aae2145eb24ec3087961387cc2f0c908717e1d47 Mon Sep 17 00:00:00 2001 From: metamergebot Date: Thu, 22 Oct 2026 13:11:33 -0700 Subject: [PATCH] [Feat] radix-tree: Report the last visited node and a shorter FULL device match in match results (#45703) Co-authored-by: metamergebot Co-authored-by: Jialin --- .../sglang/srt/mem_cache/base_prefix_cache.py | 7 ++ .../srt/mem_cache/rust_tree_core/adapter.py | 2 + .../unified_cache/unified_tree_core.py | 38 +++------ rust/sglang-radix-tree/src/python_bindings.rs | 12 +++ .../src/tests/unified_tree_core.rs | 13 +++ .../src/unified_tree_core.rs | 16 ++++ .../test_rust_tree_core_integration.py | 82 +++++++++++++++++-- 7 files changed, 137 insertions(+), 33 deletions(-) diff --git a/python/sglang/srt/mem_cache/base_prefix_cache.py b/python/sglang/srt/mem_cache/base_prefix_cache.py index 748594ee2b..cfa94d5677 100644 --- a/python/sglang/srt/mem_cache/base_prefix_cache.py +++ b/python/sglang/srt/mem_cache/base_prefix_cache.py @@ -315,6 +315,11 @@ class MatchResult(NamedTuple): full_kv_hit_length: int = 0 # Actions the Controller applies: CacheActions itself, ComponentActions routed to the owning component. cache_actions: Sequence[CacheAction | ComponentAction] = () + # Deepest non-root node visited, including nodes rejected by validators. + last_visited_node_id: Optional[int] = None + # Reusable prefix length in FULL's device-value scan, before node-boundary + # or auxiliary constraints. None means no shorter prefix was reported. + full_device_match_len: Optional[int] = None def zero_match_result( @@ -334,6 +339,8 @@ def zero_match_result( swa_branching_seqlen=None, mamba_host_hit_length=0, full_kv_hit_length=0, + last_visited_node_id=None, + full_device_match_len=None, ) diff --git a/python/sglang/srt/mem_cache/rust_tree_core/adapter.py b/python/sglang/srt/mem_cache/rust_tree_core/adapter.py index 227ba02e19..4cf43a2c20 100644 --- a/python/sglang/srt/mem_cache/rust_tree_core/adapter.py +++ b/python/sglang/srt/mem_cache/rust_tree_core/adapter.py @@ -352,6 +352,8 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): last_device_node=result.last_device_node_id, last_host_node=result.last_host_node_id, best_match_node=result.best_match_node_id, + last_visited_node_id=result.last_visited_node_id, + full_device_match_len=result.full_device_match_len, host_hit_length=result.host_hit_length, swa_host_hit_length=result.swa_host_hit_length, swa_branching_seqlen=result.swa_branching_seqlen, diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py index c921d323bb..db7a917936 100644 --- a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py +++ b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py @@ -876,34 +876,11 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): if len(key) == 0: return self._empty_match_result - ( - value, - best_match_node, - best_match_device_node, - best_match_device_value_len, - full_kv_hit_length, - action, - ) = self._match_prefix_helper(key) - return self._match_post_processor( - params, - value, - best_match_node, - best_match_device_node, - best_match_device_value_len, - full_kv_hit_length, - action, - ) + return self._match_prefix_helper(key, params) def _match_prefix_helper( - self, key: RadixKey - ) -> tuple[ - list[torch.Tensor], - UnifiedTreeNode, - UnifiedTreeNode, - int, - int, - Optional[CacheAction | ComponentAction], - ]: + self, key: RadixKey, params: MatchPrefixParams + ) -> MatchResult: # Non-HiCache mode has only device-resident matches, so the scheduler # device anchor follows the best match. In HiCache mode, host-backed # nodes can also match, so we separately track the best device-resident @@ -931,6 +908,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): comp.create_match_validator(match_device_only=True) for comp in self.components ) + device_validators = validators def _all_valid(validators, node): return all([v(node) for v in validators]) @@ -975,13 +953,16 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): if key_offset < len(key): child_key = key.child_key_at(key_offset, self.page_size) - return ( + return self._match_post_processor( + params, value, best_match_node, best_match_device_node, best_match_device_value_len, full_kv_hit_length, action, + last_visited_node_id=node.id if node is not self.root_node else None, + device_validators=device_validators, ) def match_full_device_prefix(self, key: RadixKey) -> tuple[int, NodeId, int]: @@ -1019,6 +1000,8 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): best_match_device_value_len: int, full_kv_hit_length: int, action: Optional[CacheAction | ComponentAction], + last_visited_node_id: Optional[NodeId], + device_validators: tuple[Callable[[UnifiedTreeNode], bool], ...], ) -> MatchResult: node_update = best_match_node for comp in self.components: @@ -1045,6 +1028,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): last_device_node=best_match_device_node, last_host_node=last_host_node, best_match_node=best_match_node, + last_visited_node_id=last_visited_node_id, host_hit_length=0, full_kv_hit_length=full_kv_hit_length, ) diff --git a/rust/sglang-radix-tree/src/python_bindings.rs b/rust/sglang-radix-tree/src/python_bindings.rs index ca39548c5a..7e6d75d4eb 100644 --- a/rust/sglang-radix-tree/src/python_bindings.rs +++ b/rust/sglang-radix-tree/src/python_bindings.rs @@ -339,6 +339,10 @@ struct InspectionMatchResultInput { #[pyo3(attribute)] best_match_node: NodeId, #[pyo3(attribute)] + last_visited_node_id: Option, + #[pyo3(attribute)] + full_device_match_len: Option, + #[pyo3(attribute)] host_hit_length: usize, #[pyo3(attribute)] swa_host_hit_length: usize, @@ -665,6 +669,8 @@ pub struct MatchResultBinding { last_device_node_id: NodeId, last_host_node_id: NodeId, best_match_node_id: NodeId, + last_visited_node_id: Option, + full_device_match_len: Option, host_hit_length: usize, swa_host_hit_length: usize, swa_branching_seqlen: Option, @@ -682,6 +688,8 @@ impl MatchResultBinding { last_device_node_id: result.last_device_node_id, last_host_node_id: result.last_host_node_id, best_match_node_id: result.best_match_node_id, + last_visited_node_id: result.last_visited_node_id, + full_device_match_len: result.full_device_match_len, host_hit_length: result.host_hit_length, swa_host_hit_length: result.swa_host_hit_length, swa_branching_seqlen: result.swa_branching_seqlen, @@ -2709,6 +2717,8 @@ macro_rules! tree_core_methods { last_device_node: last_device_node_id, last_host_node: last_host_node_id, best_match_node: best_match_node_id, + last_visited_node_id, + full_device_match_len, host_hit_length, swa_host_hit_length, swa_branching_seqlen, @@ -2721,6 +2731,8 @@ macro_rules! tree_core_methods { last_device_node_id, last_host_node_id, best_match_node_id, + last_visited_node_id, + full_device_match_len, host_hit_length, swa_host_hit_length, swa_branching_seqlen, diff --git a/rust/sglang-radix-tree/src/tests/unified_tree_core.rs b/rust/sglang-radix-tree/src/tests/unified_tree_core.rs index 85bdedffc3..f451373b09 100644 --- a/rust/sglang-radix-tree/src/tests/unified_tree_core.rs +++ b/rust/sglang-radix-tree/src/tests/unified_tree_core.rs @@ -1523,6 +1523,7 @@ fn match_prefix_returns_the_full_hit() { assert_eq!(result.best_match_node_id, tc.arena.node(b).id); assert_eq!(result.host_hit_length, 0); assert!(result.cache_actions.is_empty()); + assert_eq!(result.last_visited_node_id, Some(tc.arena.node(b).id)); } #[test] @@ -1532,6 +1533,7 @@ fn match_prefix_stops_at_the_matched_depth() { let result = tc.match_prefix(&match_params(&vec![1, 2, 9])); assert!(matched_device_indices(&tc, &result).equal(&Tensor::from_slice(&[10i64, 11]))); assert_eq!(result.best_match_node_id, tc.arena.node(a).id); + assert_eq!(result.last_visited_node_id, Some(tc.arena.node(a).id)); } #[test] @@ -1543,6 +1545,7 @@ fn match_prefix_miss_anchors_at_the_root() { assert_eq!(result.device_prefix_len, 0); assert_eq!(result.best_match_node_id, tc.arena.node(root).id); assert_eq!(result.last_device_node_id, tc.arena.node(root).id); + assert_eq!(result.last_visited_node_id, None); } #[test] @@ -1565,6 +1568,7 @@ fn match_prefix_splits_on_a_partial_match() { tc.arena.node(a).parent(), tc.arena.resolve(prefix_node).expect("live test node") ); + assert_eq!(result.last_visited_node_id, Some(prefix_node)); } #[test] @@ -1604,6 +1608,7 @@ fn match_prefix_stops_at_a_dead_node() { let result = tc.match_prefix(&match_params(&vec![1, 2])); assert_eq!(result.device_prefix_len, 0); assert_eq!(result.best_match_node_id, tc.arena.node(root).id); + assert_eq!(result.last_visited_node_id, None); } #[test] @@ -1670,6 +1675,8 @@ fn match_prefix_empty_query_anchors_at_the_root() { key: &vec![], namespace: KeyNamespaceRef::new(extra_key, None), }); + assert_eq!(result.last_visited_node_id, None); + assert_eq!(result.full_device_match_len, None); assert_eq!(result.best_match_node_id, root_handle); assert_eq!(result.last_device_node_id, root_handle); assert_eq!(result.last_host_node_id, root_handle); @@ -1690,6 +1697,7 @@ fn match_prefix_skips_host_only_nodes_without_hicache() { assert!(matched_device_indices(&tc, &result).equal(&Tensor::from_slice(&[10i64, 11]))); assert_eq!(result.best_match_node_id, tc.arena.node(a).id); assert_eq!(result.last_device_node_id, tc.arena.node(a).id); + assert_eq!(result.last_visited_node_id, Some(tc.arena.node(b).id)); } #[test] @@ -1708,6 +1716,7 @@ fn match_prefix_with_hicache_advances_best_match_onto_host_nodes() { assert_eq!(result.last_host_node_id, tc.arena.node(b).id); assert_eq!(result.host_hit_length, 1); assert!(result.cache_actions.is_empty()); + assert_eq!(result.last_visited_node_id, Some(tc.arena.node(b).id)); } #[test] @@ -1890,6 +1899,8 @@ fn match_prefix_swa_tombstone_holds_the_best_match_below_the_window() { let result = tc.match_prefix(&match_params(&vec![1, 2, 3, 4])); assert_eq!(result.best_match_node_id, tc.arena.node(a).id); assert!(matched_device_indices(&tc, &result).equal(&Tensor::from_slice(&[10i64, 11]))); + assert_eq!(result.last_visited_node_id, Some(tc.arena.node(c).id)); + assert_eq!(result.full_device_match_len, None); } #[test] @@ -9474,8 +9485,10 @@ fn match_with_borrowed_validators_retains_walk_state_and_common_postprocessing() result.best_match_node_id, if enable_hicache { tail_id } else { head_id } ); + assert_eq!(result.last_visited_node_id, Some(tail_id)); assert_eq!(result.full_kv_hit_length, 4); assert_eq!(result.device_prefix_len, 2); + assert!(result.full_device_match_len.is_none()); tc.sanity_check(&[], &[]); } } diff --git a/rust/sglang-radix-tree/src/unified_tree_core.rs b/rust/sglang-radix-tree/src/unified_tree_core.rs index 358495fc78..4fbeea8390 100644 --- a/rust/sglang-radix-tree/src/unified_tree_core.rs +++ b/rust/sglang-radix-tree/src/unified_tree_core.rs @@ -139,6 +139,12 @@ pub struct MatchResult { pub last_host_node_id: NodeId, /// Deepest node accepted by all component validators; anchors host->device load-back. pub best_match_node_id: NodeId, + /// Deepest node stepped into, even if rejected; None when no node was stepped into. + pub last_visited_node_id: Option, + /// Optional reusable prefix length in the FULL validator's device-value scan, + /// before node-boundary or auxiliary constraints. None means no shorter prefix + /// was reported. + pub full_device_match_len: Option, /// Full-KV tokens that hit on host and must be loaded back to device. pub host_hit_length: usize, /// SWA tokens that hit on host (within the sliding window) and will be @@ -1398,6 +1404,7 @@ impl UnifiedTreeCore { best_match_device_node_id, best_match_device_value_len, full_kv_hit_length, + last_visited_node_id, action, ) = self.match_prefix_helper_( root_id, @@ -1415,6 +1422,7 @@ impl UnifiedTreeCore { best_match_device_node_id, best_match_device_value_len, full_kv_hit_length, + last_visited_node_id, action, ) } @@ -1501,6 +1509,7 @@ impl UnifiedTreeCore { NodeIdx_, usize, usize, + Option, Option, ) { // Non-HiCache mode has only device-resident matches, so the scheduler @@ -1607,12 +1616,14 @@ impl UnifiedTreeCore { offset += prefix_len; } + let last_visited_node_id = (node_id != root_id).then(|| self.arena.node(node_id).id); ( value, best_match_node_id, best_match_device_node_id, best_match_device_value_len, full_kv_hit_length, + last_visited_node_id, action, ) } @@ -1627,6 +1638,7 @@ impl UnifiedTreeCore { best_match_device_node_id: NodeIdx_, best_match_device_value_len: usize, full_kv_hit_length: usize, + last_visited_node_id: Option, action: Option, ) -> MatchResult { for i in 0..self.components.len() { @@ -1671,6 +1683,8 @@ impl UnifiedTreeCore { last_device_node_id: self.arena.node(best_match_device_node_id).id, last_host_node_id: self.arena.node(last_host_node_id).id, best_match_node_id: self.arena.node(best_match_node_id).id, + last_visited_node_id, + full_device_match_len: None, host_hit_length: 0, mamba_host_hit_length: 0, mamba_branching_seqlen: None, @@ -1703,6 +1717,8 @@ impl UnifiedTreeCore { last_device_node_id: root_id, last_host_node_id: root_id, best_match_node_id: root_id, + last_visited_node_id: None, + full_device_match_len: None, host_hit_length: 0, swa_host_hit_length: 0, swa_branching_seqlen: None, diff --git a/test/registered/unit/mem_cache/test_rust_tree_core_integration.py b/test/registered/unit/mem_cache/test_rust_tree_core_integration.py index 451c1da07b..63ada14749 100644 --- a/test/registered/unit/mem_cache/test_rust_tree_core_integration.py +++ b/test/registered/unit/mem_cache/test_rust_tree_core_integration.py @@ -40,6 +40,7 @@ from sglang.srt.mem_cache.base_prefix_cache import ( MatchPrefixParams, MatchResult, TreeLock, + zero_match_result, ) from sglang.srt.mem_cache.cache_init_params import CacheInitParams from sglang.srt.mem_cache.common import release_kv_cache @@ -161,10 +162,16 @@ def _accumulate_step(result, tracker, device_frees, host_frees): result.host_frees.clear() -def test_match_on_the_empty_tree_returns_no_indices(): - core = _tree_core() - result = core.match_prefix(MatchPrefixParams(key=_key([1, 2, 3]))) - assert result.device_prefix_len == 0 +@pytest.mark.parametrize("backend", ["python", "rust"]) +def test_match_on_the_empty_tree_returns_no_indices(backend): + core = _tree_core(backend, page_size=2) + for key in ([], [1], [1, 2, 3]): + result = core.match_prefix(MatchPrefixParams(key=_key(key))) + assert result.device_prefix_len == 0 + assert result.last_visited_node_id is None + assert result.full_device_match_len is None + assert core.empty_match_result.last_visited_node_id is None + assert core.empty_match_result.full_device_match_len is None @pytest.mark.parametrize("backend", ["rust", "python"]) @@ -232,14 +239,61 @@ def test_default_backend_constructs_real_rust_cpu_cache(monkeypatch, instance_ba cache.tree_core.sanity_check([], []) -def test_insert_then_match_back_returns_the_exact_indices(): - core = _tree_core() +@pytest.mark.parametrize("backend", ["python", "rust"]) +def test_insert_then_match_back_returns_the_exact_indices(backend): + core = _tree_core(backend) result = _insert(core, [1, 2, 3], [10, 11, 12]) assert result.prefix_len == 0 assert result.cache_actions == [] matched = core.match_prefix(MatchPrefixParams(key=_key([1, 2, 3]))) assert result.last_device_node == matched.last_device_node assert _path_indices(core, matched.last_device_node).tolist() == [10, 11, 12] + assert matched.last_visited_node_id == result.last_device_node + assert matched.full_device_match_len is None + + +@pytest.mark.parametrize("backend", ["python", "rust"]) +def test_match_reports_split_prefix_and_zero_match_clears_walk_facts(backend): + core = _tree_core(backend) + leaf = _insert(core, [1, 2, 3], [10, 11, 12]).last_device_node + matched = core.match_prefix(MatchPrefixParams(key=_key([1, 2, 9]))) + assert matched.last_visited_node_id == matched.best_match_node + assert matched.last_visited_node_id != leaf + assert matched.device_prefix_len == 2 + assert _path_indices(core, matched.last_device_node).tolist() == [10, 11] + path = core.collect_path_info([leaf])[0] + assert path[0].node_id == matched.last_visited_node_id + assert path[0].path_end == 2 + + cache = SimpleNamespace( + supports_prefix_sharing=lambda: True, + root_node_handle=core.root_node_handle, + ) + zeroed = zero_match_result(cache, matched._replace(full_device_match_len=2)) + assert zeroed.last_device_node == core.root_node_handle() + assert zeroed.last_visited_node_id is None + assert zeroed.full_device_match_len is None + core.sanity_check([], []) + + +@pytest.mark.parametrize("backend", ["python", "rust"]) +@pytest.mark.parametrize("hicache", [False, True]) +def test_match_reports_host_only_node_independent_of_device_acceptance( + backend, hicache +): + core = _tree_core(backend) + if hicache: + core.set_hicache_enabled() + host = core.insert_host( + core.root_node_handle(), _key([1, 2]), torch.tensor([100, 101]), ["h0", "h1"] + ).inserted_host_node + matched = core.match_prefix(MatchPrefixParams(key=_key([1, 2]))) + assert matched.last_visited_node_id == host + assert matched.last_device_node == core.root_node_handle() + assert matched.best_match_node == (host if hicache else core.root_node_handle()) + assert matched.device_prefix_len == 0 + assert matched.full_device_match_len is None + core.sanity_check([], []) def test_root_node_handle_is_namespace_independent(): @@ -784,11 +838,14 @@ def test_stale_match_finalizer_handles_raise_key_error_without_poisoning_the_cor stale_root = core.root_node_handle() core.reset() live_root = core.root_node_handle() + visited = _insert(core, [1], [10]).last_device_node result = MatchResult( device_prefix_len=0, last_device_node=live_root, last_host_node=live_root, best_match_node=live_root, + last_visited_node_id=visited, + full_device_match_len=0, ) params = MatchPrefixParams(key=_key([])) @@ -812,6 +869,8 @@ def test_stale_match_finalizer_handles_raise_key_error_without_poisoning_the_cor best_value_len=0, ) assert finalized.best_match_node == live_root + assert finalized.last_visited_node_id == visited + assert finalized.full_device_match_len == 0 def test_stale_handle_operations_raise_key_error_without_poisoning_the_core(): @@ -3934,6 +3993,17 @@ def test_swa_match_uses_allocator_layout(backend, ring, hicache, has_swa_host_po assert cache.path_device_indices(matched.last_device_node).tolist() == ( indices[:8].tolist() if ring else [] ) + assert matched.last_visited_node_id != cache.tree_core.root_node_handle() + assert matched.last_visited_node_id is not None + if not ring: + assert matched.last_visited_node_id != matched.last_device_node + assert ( + cache.tree_core.collect_path_info([matched.last_visited_node_id])[0][ + -1 + ].path_end + == 8 + ) + assert matched.full_device_match_len is None @pytest.mark.parametrize("backend", ["python", "rust"])