From 77a15cf4e24f239619e46a109a32d812c3c4ebd4 Mon Sep 17 00:00:00 2001 From: metamergebot Date: Sun, 11 Oct 2026 20:14:07 -0700 Subject: [PATCH] [MemCache] Add shared component size and path APIs to the tree core interface (#43722) Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com> Co-authored-by: Jialin --- .../srt/mem_cache/rust_tree_core/adapter.py | 12 + .../unified_cache/components/base.py | 2 +- .../unified_cache/unified_tree_core.py | 44 +++- .../unified_tree_core_interface.py | 34 +++ rust/sglang-radix-tree/src/node.rs | 14 + rust/sglang-radix-tree/src/python_bindings.rs | 71 +++++ .../src/tests/unified_tree_core.rs | 245 ++++++++++++++++++ .../src/unified_tree_core.rs | 97 +++++++ .../test_rust_tree_core_integration.py | 170 +++++++++++- .../test_unified_radix_cache_unittest.py | 6 +- 10 files changed, 677 insertions(+), 18 deletions(-) diff --git a/python/sglang/srt/mem_cache/rust_tree_core/adapter.py b/python/sglang/srt/mem_cache/rust_tree_core/adapter.py index 7b6ba48cd1..6fb481ccd5 100644 --- a/python/sglang/srt/mem_cache/rust_tree_core/adapter.py +++ b/python/sglang/srt/mem_cache/rust_tree_core/adapter.py @@ -57,6 +57,7 @@ from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import ( InsertStepResult, NodeId, RadixCacheWalkResult, + TreePathNodeInfo, UnifiedTreeCoreInterface, ) from sglang.srt.mem_cache.utils import get_eviction_strategy @@ -1138,6 +1139,17 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): def write_back_duplicate_reclaim_digest(self) -> int: return self._binding.write_back_duplicate_reclaim_digest() + def adjust_component_size( + self, node_id: NodeId, component_type: ComponentType, delta: int + ) -> None: + self._binding.adjust_component_size(node_id, int(component_type), delta) + + def collect_path_info( + self, end_node_ids: Sequence[NodeId] + ) -> tuple[tuple[TreePathNodeInfo, ...], ...]: + paths = self._binding.collect_path_info(list(end_node_ids)) + return tuple(tuple(TreePathNodeInfo(*node) for node in path) for path in paths) + def set_component_device_value( self, node_id: NodeId, component_type: ComponentType, value: torch.Tensor ) -> None: diff --git a/python/sglang/srt/mem_cache/unified_cache/components/base.py b/python/sglang/srt/mem_cache/unified_cache/components/base.py index 9d6a85cae1..c627974c38 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/base.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/base.py @@ -354,7 +354,7 @@ class TreeComponent(ABC): the allocator's units: the value length by default, less for a component whose values are not backed one-to-one by pool rows. A component whose per-node figure changes in place reports the delta - through ``UnifiedTreeCore.adjust_component_ledger``.""" + through ``UnifiedTreeCore.adjust_component_size``.""" return self.value_len(node) def has_host_value_only(self, node: UnifiedTreeNode) -> bool: diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py index 4f989087b4..c917ecf791 100644 --- a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py +++ b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py @@ -78,6 +78,7 @@ from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import ( InsertStepResult, NodeId, RadixCacheWalkResult, + TreePathNodeInfo, UnifiedTreeCoreInterface, ) from sglang.srt.mem_cache.utils import ( @@ -667,6 +668,34 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): def dfs_weight_order(self, node_ids: Sequence[NodeId]) -> list[int]: return _dfs_weight_order(self.root_node, node_ids, self.node_by_id) + def collect_path_info( + self, end_node_ids: Sequence[NodeId] + ) -> tuple[tuple[TreePathNodeInfo, ...], ...]: + paths = [] + for node_id in end_node_ids: + node = self.node_by_id(node_id) + chain = [] + while node is not self.root_node: + chain.append(node) + node = node.parent + path = [] + start = 0 + for node in reversed(chain): + end = start + len(node.key) + value = node.component_data[BASE_COMPONENT_TYPE].value + path.append( + TreePathNodeInfo( + node.id, + node.parent.id, + start, + end, + None if value is None else len(value), + ) + ) + start = end + paths.append(tuple(path)) + return tuple(paths) + def _new_node(self, priority: int = 0) -> UnifiedTreeNode: """Create and register a tree node in the arena.""" node = UnifiedTreeNode(self.component_types, priority=priority) @@ -2858,17 +2887,14 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): else: self.component_evictable_size_[component_type] += ledger - def adjust_component_ledger( - self, node: UnifiedTreeNode, component_type: ComponentType, delta: int + def adjust_component_size( + self, node_id: NodeId, component_type: ComponentType, delta: int ) -> None: - """A component changed how many pool tokens evicting ``node`` would - reclaim (``TreeComponent.reclaimable_tokens``) while the value stays - on device: move the ledger the node is on by ``delta``. A tombstoned - value is on neither ledger.""" - if delta == 0: - return + node = self.node_by_id(node_id) + if component_type not in self.components_by_type: + raise ValueError(f"component is not enabled: {component_type}") cd = node.component_data[component_type] - if cd.value is None: + if node is self.root_node or cd.value is None: return if cd.lock_ref > 0: self.component_protected_size_[component_type] += delta diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py index 113080b883..304318b18f 100644 --- a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py +++ b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py @@ -104,6 +104,18 @@ class BufferBackupState(msgspec.Struct, frozen=True): parent_last_hash: Optional[str] +class TreePathNodeInfo(msgspec.Struct, frozen=True): + """A node's key span on its root-to-node path.""" + + node_id: NodeId + parent_id: NodeId + # Half-open offsets from the root, in tokens (bigrams for bigram trees). + path_start: int + path_end: int + # Logical FULL device indices; None when the value is absent. + full_value_len: Optional[int] + + class InsertStepResult(msgspec.Struct, frozen=True): """One step of a resumable insert: the Controller executes ``actions``, then resumes while ``result`` is None; ``result`` is set on the final step.""" @@ -270,6 +282,17 @@ class UnifiedTreeCoreInterface(ABC): """Return input indices in depth-first, subtree-weight order.""" ... + @abstractmethod + def collect_path_info( + self, end_node_ids: Sequence[NodeId] + ) -> tuple[tuple[TreePathNodeInfo, ...], ...]: + """Copy root-to-node paths, excluding roots, in input order. + + Roots produce empty paths; missing handles raise KeyError. Callers must + serialize with tree mutations; native backends hold one lock per batch. + """ + ... + @abstractmethod def inc_lock_ref( self, node_id: NodeId, skip_lock_components: Sequence[ComponentType] = () @@ -398,6 +421,17 @@ class UnifiedTreeCoreInterface(ABC): """Protected token count for one component (0 if absent).""" raise NotImplementedError("This tree core does not expose per-component sizes") + @abstractmethod + def adjust_component_size( + self, node_id: NodeId, component_type: ComponentType, delta: int + ) -> None: + """Adjust protected/evictable tokens after the component backing changes. + + Roots and nodes without device values are no-ops. Missing handles raise + KeyError; disabled components raise ValueError. + """ + ... + @abstractmethod def full_evictable_size(self) -> int: ... diff --git a/rust/sglang-radix-tree/src/node.rs b/rust/sglang-radix-tree/src/node.rs index 2c21f6ec35..60ddc6a9f4 100644 --- a/rust/sglang-radix-tree/src/node.rs +++ b/rust/sglang-radix-tree/src/node.rs @@ -736,6 +736,20 @@ pub enum TreeCoreRuntimeError { /// A public NodeId no longer names a live arena node. #[error(transparent)] NodeAccess(#[from] NodeAccessError), + /// The requested component is not configured for this core. + #[error("component {component_type:?} is not enabled")] + ComponentNotEnabled { component_type: ComponentType }, + /// A signed adjustment would underflow or overflow the selected device ledger. + #[error( + "invalid size delta {delta} for node {node_id} component {component_type:?}: current={current}, protected={protected}" + )] + InvalidComponentSizeDelta { + node_id: NodeId, + component_type: ComponentType, + current: usize, + delta: i64, + protected: bool, + }, /// `begin_insert`/`insert` called while a resumable insert is suspended. #[error("concurrent insert walks")] ConcurrentInsertWalk, diff --git a/rust/sglang-radix-tree/src/python_bindings.rs b/rust/sglang-radix-tree/src/python_bindings.rs index 04456370a8..168c98120c 100644 --- a/rust/sglang-radix-tree/src/python_bindings.rs +++ b/rust/sglang-radix-tree/src/python_bindings.rs @@ -97,6 +97,10 @@ fn node_access_error(error: NodeAccessError) -> PyErr { fn tree_core_runtime_error(error: TreeCoreRuntimeError) -> PyErr { match error { TreeCoreRuntimeError::NodeAccess(error) => node_access_error(error), + error @ (TreeCoreRuntimeError::ComponentNotEnabled { .. } + | TreeCoreRuntimeError::InvalidComponentSizeDelta { .. }) => { + PyValueError::new_err(error.to_string()) + } error => PyRuntimeError::new_err(error.to_string()), } } @@ -1007,6 +1011,8 @@ impl HostEvictionResultBinding { } } +type TreePathNodeInfoTuple = (NodeId, NodeId, usize, usize, Option); + /// The generic UnifiedTreeCore adapter the per-key-type pyclasses delegate to; /// the Mutex makes the Send-only core satisfy pyclass's Sync bound. struct TreeCoreBinding { @@ -1375,6 +1381,49 @@ impl TreeCoreBinding { Ok((frees_to_py(py, device_frees)?, frees_to_py(py, host_frees)?)) } + /// Adjust the component's reclaimable device size. + fn adjust_component_size( + &self, + py: Python<'_>, + node_id: NodeId, + component_type: u8, + delta: i64, + ) -> PyResult<()> { + let component_type = parse_component_type(component_type)?; + py.detach(|| { + self.core() + .adjust_component_size(node_id, component_type, delta) + }) + .map_err(tree_core_runtime_error) + } + + /// Collect paths under one tree lock. + fn collect_path_info( + &self, + py: Python<'_>, + end_node_ids: Vec, + ) -> PyResult>> { + let paths = py + .detach(|| self.core().collect_path_info(&end_node_ids)) + .map_err(tree_core_runtime_error)?; + Ok(paths + .into_iter() + .map(|path| { + path.into_iter() + .map(|node| { + ( + node.node_id, + node.parent_id, + node.path_start, + node.path_end, + node.full_value_len, + ) + }) + .collect() + }) + .collect()) + } + /// Store a component's device value on a node (the SWA rebuild write-back). fn set_component_device_value( &self, @@ -2876,6 +2925,28 @@ macro_rules! tree_core_binding { catch_native_panic(|| self.inner.dec_swa_lock_only(py, node_id, params)) } + /// Adjust the component's reclaimable device size. + fn adjust_component_size( + &self, + py: Python<'_>, + node_id: NodeId, + component_type: u8, + delta: i64, + ) -> PyResult<()> { + catch_native_panic(|| { + self.inner.adjust_component_size(py, node_id, component_type, delta) + }) + } + + /// Collect paths under one tree lock. + fn collect_path_info( + &self, + py: Python<'_>, + end_node_ids: Vec, + ) -> PyResult>> { + catch_native_panic(|| self.inner.collect_path_info(py, end_node_ids)) + } + /// Store a component's device value on a node (the SWA rebuild write-back). fn set_component_device_value( &self, diff --git a/rust/sglang-radix-tree/src/tests/unified_tree_core.rs b/rust/sglang-radix-tree/src/tests/unified_tree_core.rs index 5c7d8da76e..1027a43025 100644 --- a/rust/sglang-radix-tree/src/tests/unified_tree_core.rs +++ b/rust/sglang-radix-tree/src/tests/unified_tree_core.rs @@ -11,6 +11,251 @@ fn core() -> UnifiedTreeCore> { UnifiedTreeCore::new(CacheInitParams::default(), vec![FULL]) } +fn add_leaf(core: &mut UnifiedTreeCore, parent: NodeIdx_, key: K) -> NodeIdx_ { + let values = Tensor::arange(key.atom_len() as i64, (Kind::Int64, Device::Cpu)); + core.add_new_node_(parent, key, &values, 0, None) +} + +#[test] +fn component_size_adjustment_tracks_changed_backing_through_locks_and_eviction() { + let mut core = core(); + let root = core.arena.root(); + let node = add_leaf(&mut core, root, vec![1, 2, 3, 4]); + let id = core.arena.node(node).id; + // Resize key and backing together, keeping the same child edge. + let _ = core.arena.take_device_value(node, FULL); + core.arena.node_mut(node).key = vec![1, 2, 3, 4, 5]; + core.arena + .set_device_value(node, FULL, Tensor::from_slice(&[10i64, 11, 12, 13, 14])); + core.adjust_component_size(id, FULL, 1).unwrap(); + assert_eq!(core.evictable_size_(FULL), 5); + assert_eq!(core.protected_size_(FULL), 0); + core.sanity_check(&[], &[]); + + let receipt = core.inc_lock_ref(id, ComponentSet::EMPTY).unwrap(); + assert_eq!(core.evictable_size_(FULL), 0); + assert_eq!(core.protected_size_(FULL), 5); + + let _ = core.arena.take_device_value(node, FULL); + core.arena.node_mut(node).key = vec![1, 2, 3, 4]; + core.arena + .set_device_value(node, FULL, Tensor::from_slice(&[20i64, 21, 22, 23])); + core.adjust_component_size(id, FULL, -1).unwrap(); + assert_eq!(core.protected_size_(FULL), 4); + core.sanity_check(&[], &[]); + + core.dec_lock_ref(id, &receipt.to_dec_params(), false) + .unwrap(); + assert_eq!(core.evictable_size_(FULL), 4); + assert_eq!(core.protected_size_(FULL), 0); + let (_, evicted) = core.evict_device_leaf(id, false).unwrap(); + assert_eq!(evicted.tracker[&FULL], 4); + assert!(evicted.device_frees[&FULL][0].equal(&Tensor::from_slice(&[20i64, 21, 22, 23]))); + assert_eq!(core.evictable_size_(FULL), 0); + core.sanity_check(&[], &[]); +} + +#[test] +fn component_size_adjustment_errors_preserve_both_ledgers() { + let mut core = core(); + let root = core.arena.root(); + let node = add_leaf(&mut core, root, vec![1, 2, 3, 4]); + let id = core.arena.node(node).id; + for protected in [false, true] { + let receipt = protected.then(|| core.inc_lock_ref(id, ComponentSet::EMPTY).unwrap()); + let before = (core.evictable_size_(FULL), core.protected_size_(FULL)); + for delta in [-5, i64::MIN] { + assert!(matches!( + core.adjust_component_size(id, FULL, delta), + Err(TreeCoreRuntimeError::InvalidComponentSizeDelta { + current: 4, + protected: actual, + .. + }) if actual == protected + )); + assert_eq!( + (core.evictable_size_(FULL), core.protected_size_(FULL)), + before + ); + } + let state = core.component_state_mut(FULL); + if protected { + state.protected_size = usize::MAX; + } else { + state.evictable_size = usize::MAX; + } + assert!(matches!( + core.adjust_component_size(id, FULL, 1), + Err(TreeCoreRuntimeError::InvalidComponentSizeDelta { .. }) + )); + assert_eq!( + (core.evictable_size_(FULL), core.protected_size_(FULL)), + if protected { + (0, usize::MAX) + } else { + (usize::MAX, 0) + } + ); + let state = core.component_state_mut(FULL); + state.evictable_size = before.0; + state.protected_size = before.1; + if let Some(receipt) = receipt { + core.dec_lock_ref(id, &receipt.to_dec_params(), false) + .unwrap(); + } + } + assert!(matches!( + core.adjust_component_size(id, SWA, 0), + Err(TreeCoreRuntimeError::ComponentNotEnabled { + component_type: SWA + }) + )); + assert!(matches!( + core.adjust_component_size(usize::MAX, FULL, 0), + Err(TreeCoreRuntimeError::NodeAccess(_)) + )); + assert_eq!( + (core.evictable_size_(FULL), core.protected_size_(FULL)), + (4, 0) + ); + core.sanity_check(&[], &[]); + + core.evict_device_leaf(id, false).unwrap(); + assert!(matches!( + core.adjust_component_size(id, FULL, 0), + Err(TreeCoreRuntimeError::NodeAccess(_)) + )); +} + +#[test] +fn component_size_adjustment_ignores_valid_roots_and_device_tombstones() { + let mut core = core(); + core.set_hicache_enabled(); + let root = core.arena.root(); + let root_id = core.arena.node(root).id; + let node = add_leaf(&mut core, root, vec![1, 2]); + let id = core.arena.node(node).id; + core.commit_backup(id, Tensor::from_slice(&[20i64, 21]), HashMap::new()) + .unwrap(); + core.demote(id).unwrap(); + for id in [root_id, id] { + for delta in [i64::MIN, 0, i64::MAX] { + core.adjust_component_size(id, FULL, delta).unwrap(); + } + assert!(matches!( + core.adjust_component_size(id, SWA, 0), + Err(TreeCoreRuntimeError::ComponentNotEnabled { .. }) + )); + } + assert_eq!( + (core.evictable_size_(FULL), core.protected_size_(FULL)), + (0, 0) + ); + core.sanity_check(&[], &[]); +} + +fn assert_path_info() { + let mut core = UnifiedTreeCore::::new(CacheInitParams::default(), vec![FULL]); + core.set_hicache_enabled(); + let root = core.arena.root(); + let root_id = core.arena.node(root).id; + let key = K::key_from(std::borrow::Cow::Owned(vec![1, 2, 3, 4, 5, 6, 7])).into_owned(); + let total = key.atom_len(); + let (head_key, tail_key) = key.split_at(2); + let head = add_leaf(&mut core, root, head_key); + let tail = add_leaf(&mut core, head, tail_key); + let other_key = K::key_from(std::borrow::Cow::Owned(vec![20, 21, 22])).into_owned(); + let other_len = other_key.atom_len(); + let other = add_leaf(&mut core, root, other_key); + let head_id = core.arena.node(head).id; + let tail_id = core.arena.node(tail).id; + let other_id = core.arena.node(other).id; + let full_path = vec![ + TreePathNodeInfo { + node_id: head_id, + parent_id: root_id, + path_start: 0, + path_end: 2, + full_value_len: Some(2), + }, + TreePathNodeInfo { + node_id: tail_id, + parent_id: head_id, + path_start: 2, + path_end: total, + full_value_len: Some(total - 2), + }, + ]; + let snapshot = core + .collect_path_info(&[tail_id, root_id, other_id, tail_id]) + .unwrap(); + assert_eq!( + snapshot, + vec![ + full_path.clone(), + vec![], + vec![TreePathNodeInfo { + node_id: other_id, + parent_id: root_id, + path_start: 0, + path_end: other_len, + full_value_len: Some(other_len), + }], + full_path.clone(), + ] + ); + assert!(core.collect_path_info(&[]).unwrap().is_empty()); + + core.commit_backup(head_id, Tensor::from_slice(&[30i64, 31]), HashMap::new()) + .unwrap(); + core.commit_backup( + tail_id, + Tensor::arange((total - 2) as i64, (Kind::Int64, Device::Cpu)), + HashMap::new(), + ) + .unwrap(); + core.demote(tail_id).unwrap(); + core.demote(head_id).unwrap(); + let tombstone_path = core.collect_path_info(&[tail_id]).unwrap(); + assert_eq!( + tombstone_path[0][0], + TreePathNodeInfo { + full_value_len: None, + ..full_path[0] + } + ); + assert_eq!( + tombstone_path[0][1], + TreePathNodeInfo { + full_value_len: None, + ..full_path[1] + } + ); + assert_eq!(snapshot[0], full_path); // Earlier snapshot stays unchanged. + assert!(matches!( + core.collect_path_info(&[tail_id, usize::MAX, other_id]), + Err(TreeCoreRuntimeError::NodeAccess(_)) + )); + core.sanity_check(&[], &[]); + + core.reset(); + assert!(matches!( + core.collect_path_info(&[tail_id]), + Err(TreeCoreRuntimeError::NodeAccess(_)) + )); + assert_eq!(snapshot[0], full_path); +} + +#[test] +fn path_info_plain_keys() { + assert_path_info::>(); +} + +#[test] +fn path_info_bigram_keys_use_normalized_atoms() { + assert_path_info::>(); +} + // Records every refresh_lru dispatch; unrelated hooks stay unimplemented. #[derive(Default)] struct RecordingComponentForTest { diff --git a/rust/sglang-radix-tree/src/unified_tree_core.rs b/rust/sglang-radix-tree/src/unified_tree_core.rs index 914ae1a202..3c9df474d4 100644 --- a/rust/sglang-radix-tree/src/unified_tree_core.rs +++ b/rust/sglang-radix-tree/src/unified_tree_core.rs @@ -53,6 +53,18 @@ fn next_coexist_reclaim_digest(current: i64, node_id: NodeId, component_idx: usi // ---- interface types ---- +/// A node's position and FULL value length on a root-to-node path. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TreePathNodeInfo { + pub node_id: NodeId, + pub parent_id: NodeId, + /// Half-open offsets from the root, in tokens (or bigrams for bigram keys). + pub path_start: usize, + pub path_end: usize, + /// Logical FULL device indices; None when absent. + pub full_value_len: Option, +} + /// Receipt returned by `inc_lock_ref` and replayed by `dec_lock_ref`. #[derive(Default)] pub struct IncLockRefResult { @@ -660,6 +672,91 @@ pub struct UnifiedTreeCore { } impl UnifiedTreeCore { + /// Adjust the component's evictable or protected count of reclaimable tokens. + /// The caller updates backing and size together, and restores backing on error. + /// This changes counters only; roots and device tombstones are ignored. + pub fn adjust_component_size( + &mut self, + node_id: NodeId, + component_type: ComponentType, + delta: i64, + ) -> Result<(), TreeCoreRuntimeError> { + let node_idx = self.arena.resolve(node_id)?; + if self.components_by_type[component_type.idx()].is_none() { + return Err(TreeCoreRuntimeError::ComponentNotEnabled { component_type }); + } + let node = self.arena.node(node_idx); + if node.is_root() || !node.has_device_value(component_type) { + return Ok(()); + } + let protected = node.device_lock_ref(component_type) > 0; + let state = self.component_state_mut(component_type); + let size = if protected { + &mut state.protected_size + } else { + &mut state.evictable_size + }; + let updated = usize::try_from(delta.unsigned_abs()) + .ok() + .and_then(|magnitude| { + if delta >= 0 { + size.checked_add(magnitude) + } else { + size.checked_sub(magnitude) + } + }) + .ok_or(TreeCoreRuntimeError::InvalidComponentSizeDelta { + node_id, + component_type, + current: *size, + delta, + protected, + })?; + *size = updated; + Ok(()) + } + + /// Copy root-to-node paths in input order, keeping duplicates and excluding roots. + /// An invalid node id fails the whole call. + pub fn collect_path_info( + &self, + node_ids: &[NodeId], + ) -> Result>, TreeCoreRuntimeError> { + node_ids + .iter() + .map(|&node_id| { + let mut current = self.arena.resolve(node_id)?; + let mut path = Vec::new(); + loop { + let node = self.arena.node(current); + if node.is_root() { + break; + } + let parent = node.parent(); + path.push(TreePathNodeInfo { + node_id: node.id, + parent_id: self.arena.node(parent).id, + path_start: 0, + path_end: node.key.atom_len(), + full_value_len: node + .has_device_value(FULL) + .then(|| node.device_value_len(FULL)), + }); + current = parent; + } + path.reverse(); + let mut start = 0; + for info in &mut path { + let length = info.path_end; + info.path_start = start; + start += length; + info.path_end = start; + } + Ok(path) + }) + .collect() + } + /// Build a tree core for the given component types with a fresh arena. /// Fresh per-slot LRU lists. pub(crate) fn new_lru_lists() -> [UnifiedLRUList; NUM_VALUE_SLOTS] { diff --git a/test/registered/unit/mem_cache/test_rust_tree_core_integration.py b/test/registered/unit/mem_cache/test_rust_tree_core_integration.py index 30675cebc7..026747a10d 100644 --- a/test/registered/unit/mem_cache/test_rust_tree_core_integration.py +++ b/test/registered/unit/mem_cache/test_rust_tree_core_integration.py @@ -13,7 +13,7 @@ import torch from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=17, suite="base-a-test-cpu") +register_cpu_ci(est_time=20, suite="base-a-test-cpu") from sglang.srt.disaggregation.kv_events import ( AllBlocksCleared, @@ -59,11 +59,15 @@ from sglang.srt.mem_cache.unified_cache.cache_action import ( SWARebuild, ) from sglang.srt.mem_cache.unified_cache.component_type import ComponentType +from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import ( + TreePathNodeInfo, + UnifiedTreeCoreInterface, +) from sglang.srt.mem_cache.utils import get_storage_hash_str, hash_str_to_int64 from sglang.srt.runtime_context import get_context -def _tree_core(**params_overrides) -> RustUnifiedTreeCore: +def _tree_core(backend="rust", **params_overrides) -> UnifiedTreeCoreInterface: params = dict( disable=False, req_to_token_pool=None, @@ -72,7 +76,15 @@ def _tree_core(**params_overrides) -> RustUnifiedTreeCore: tree_components=(ComponentType.FULL,), ) params.update(params_overrides) - return RustUnifiedTreeCore(CacheInitParams(**params)) + params = CacheInitParams(**params) + if backend == "rust": + return RustUnifiedTreeCore(params) + + from sglang.srt.mem_cache.unified_cache.components.full import FullComponent + from sglang.srt.mem_cache.unified_cache.unified_tree_core import UnifiedTreeCore + + cache = SimpleNamespace(enable_session_radix_cache=False) + return UnifiedTreeCore(params, {ComponentType.FULL: FullComponent(cache, params)}) def _path_indices(core: RustUnifiedTreeCore, node_id) -> torch.Tensor: @@ -961,6 +973,158 @@ def test_extension_insert_frees_the_duplicate_overlap(): ] +@pytest.mark.parametrize("backend", ["python", "rust"]) +@pytest.mark.parametrize("is_eagle", [False, True], ids=["unigram", "bigram"]) +def test_path_order_namespaces_and_detached_snapshots(backend, is_eagle): + """Both backends report root-relative half-open spans per node in input + order, keep namespaces apart, and return frozen records that a later + split or reset does not rewrite.""" + core = _tree_core(backend, is_eagle=is_eagle) + assert isinstance(core, UnifiedTreeCoreInterface) + root = core.root_node_handle() + assert core.collect_path_info(()) == () + assert core.collect_path_info([root]) == ((),) + + def insert(tokens, namespace): + key = RadixKey(array("q", tokens), extra_key=namespace, is_bigram=is_eagle) + return _pump_insert( + core, InsertParams(key=key, value=torch.arange(10, 10 + len(key))) + ).last_device_node + + tokens = [1, 2, 3, 4, 5, 6] + length = len(tokens) - int(is_eagle) + leaf = insert(tokens, "first") + other = insert(tokens, "second") + before = core.collect_path_info((other, root, leaf, other)) + assert before == ( + (TreePathNodeInfo(other, root, 0, length, length),), + (), + (TreePathNodeInfo(leaf, root, 0, length, length),), + (TreePathNodeInfo(other, root, 0, length, length),), + ) + with pytest.raises(AttributeError): + before[0][0].path_end = 99 + + insert([1, 2, 3, 7, 8, 9], "first") + path = core.collect_path_info([leaf])[0] + split = 3 - int(is_eagle) + assert len(path) == 2 + assert path[0] == TreePathNodeInfo(path[0].node_id, root, 0, split, split) + assert path[1] == TreePathNodeInfo( + leaf, path[0].node_id, split, length, length - split + ) + assert before[2] == (TreePathNodeInfo(leaf, root, 0, length, length),) + core.reset() + with pytest.raises(KeyError): + core.collect_path_info([leaf]) + assert before[0] == before[3] + assert before[2][0].full_value_len == length + assert core.collect_path_info([core.root_node_handle()]) == ((),) + + +@pytest.mark.parametrize("backend", ["python", "rust"]) +@pytest.mark.parametrize("is_eagle", [False, True], ids=["unigram", "bigram"]) +def test_host_only_and_root_adjustments_are_noops(backend, is_eagle): + """A root or a node with no device value is on neither ledger, so an + adjustment must leave both ledgers at zero instead of underflowing.""" + core = _tree_core(backend, is_eagle=is_eagle) + root = core.root_node_handle() + key = RadixKey(array("q", [1, 2, 3, 4]), extra_key="host", is_bigram=is_eagle) + host = core.insert_host( + root, key, torch.arange(100, 100 + len(key)), ["hash"] * len(key) + ).inserted_host_node + assert core.collect_path_info([host]) == ( + (TreePathNodeInfo(host, root, 0, len(key), None),), + ) + for node in (root, host): + for delta in (-10, 0, 10): + core.adjust_component_size(node, ComponentType.FULL, delta) + assert (core.full_evictable_size(), core.full_protected_size()) == (0, 0) + assert core.collect_path_info([host])[0][0].full_value_len is None + + +@pytest.mark.parametrize("backend", ["python", "rust"]) +@pytest.mark.parametrize("is_eagle", [False, True], ids=["unigram", "bigram"]) +@pytest.mark.parametrize("locked", [False, True], ids=["evictable", "protected"]) +def test_failed_adjustments_preserve_accounting_and_tree(backend, is_eagle, locked): + """Rejected handles and disabled components raise the documented error + classes on both backends without moving either ledger or the path.""" + core = _tree_core(backend, is_eagle=is_eagle) + key = RadixKey(array("q", [1, 2, 3, 4]), is_bigram=is_eagle) + leaf = _pump_insert( + core, InsertParams(key=key, value=torch.arange(10, 10 + len(key))) + ).last_device_node + if locked: + lock = core.inc_lock_ref(leaf) + before = core.full_evictable_size(), core.full_protected_size() + path = core.collect_path_info([leaf]) + invalid_node = (1 << 63) - 1 + for node, component, delta, error in ( + (invalid_node, ComponentType.FULL, 0, KeyError), + (leaf, ComponentType.SWA, 0, ValueError), + (leaf, 99, 0, ValueError), + ): + with pytest.raises(error): + core.adjust_component_size(node, component, delta) + assert (core.full_evictable_size(), core.full_protected_size()) == before + assert core.collect_path_info([leaf]) == path + with pytest.raises(KeyError): + core.collect_path_info((leaf, invalid_node, leaf)) + core.adjust_component_size(leaf, ComponentType.FULL, 0) + assert (core.full_evictable_size(), core.full_protected_size()) == before + if locked: + core.dec_lock_ref(leaf, lock.to_dec_params()) + core.sanity_check([], []) + _accumulate_step(core.evict_device_leaf(leaf, False), {}, {}, {}) + assert (core.full_evictable_size(), core.full_protected_size()) == (0, 0) + with pytest.raises(KeyError): + core.adjust_component_size(leaf, ComponentType.FULL, 0) + + +def test_adjustment_follows_real_backing_changes_across_lock_transitions(): + """The adjusted figure moves with the node between the evictable and + protected ledgers, and eviction reclaims the component's own figure.""" + from sglang.srt.mem_cache.unified_cache.components.full import FullComponent + from sglang.srt.mem_cache.unified_cache.unified_tree_core import UnifiedTreeCore + + class SideBackingFullComponent(FullComponent): + def reclaimable_tokens(self, node): + backing = node.component_data[ComponentType.FULL].metadata.get( + "side_backing" + ) + return super().reclaimable_tokens(node) + ( + 0 if backing is None else backing.numel() + ) + + params = CacheInitParams( + False, None, None, 1, tree_components=(ComponentType.FULL,) + ) + cache = SimpleNamespace(enable_session_radix_cache=False) + core = UnifiedTreeCore( + params, {ComponentType.FULL: SideBackingFullComponent(cache, params)} + ) + leaf = _insert(core, [1, 2, 3, 4], [10, 11, 12, 13]).last_device_node + metadata = core.node_by_id(leaf).component_data[ComponentType.FULL].metadata + metadata["side_backing"] = torch.tensor([100, 101]) + core.adjust_component_size(leaf, ComponentType.FULL, 2) + assert (core.full_evictable_size(), core.full_protected_size()) == (6, 0) + + lock = core.inc_lock_ref(leaf) + assert (core.full_evictable_size(), core.full_protected_size()) == (0, 6) + metadata["side_backing"] = metadata["side_backing"][:1] + core.adjust_component_size(leaf, ComponentType.FULL, -1) + assert (core.full_evictable_size(), core.full_protected_size()) == (0, 5) + del metadata["side_backing"] + core.adjust_component_size(leaf, ComponentType.FULL, -1) + core.dec_lock_ref(leaf, lock.to_dec_params()) + assert (core.full_evictable_size(), core.full_protected_size()) == (4, 0) + core.sanity_check([], []) + tracker = {} + _accumulate_step(core.evict_device_leaf(leaf, False), tracker, {}, {}) + assert tracker[ComponentType.FULL] == 4 + assert (core.full_evictable_size(), core.full_protected_size()) == (0, 0) + + def test_lock_and_unlock_move_tokens_between_protected_and_evictable(): core = _tree_core() _insert(core, [1, 2, 3], [10, 11, 12]) diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py index f4775a9f99..3334ace755 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py +++ b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py @@ -357,11 +357,7 @@ class TestUnifiedRadixComponentLedgerUnits(CustomTestCase): # The figure changes in place, without a lock transition. component.rows_handed_back[leaf] = 1 - cache.tree_core.adjust_component_ledger( - node=cache.tree_core.node_by_id(leaf), - component_type=ComponentType.FULL, - delta=-1, - ) + cache.tree_core.adjust_component_size(leaf, ComponentType.FULL, -1) self.assertEqual(ledgers(), (0, 3)) cache.dec_lock_ref(leaf, lock_params)