From 0b536ca112652008d5fa8766df7258b0700109f3 Mon Sep 17 00:00:00 2001 From: metamergebot Date: Tue, 20 Oct 2026 13:24:20 -0700 Subject: [PATCH] [Fix] radix-cache: Separate logical and reclaimable value lengths in tree components (#45238) Co-authored-by: metamergebot Co-authored-by: Jialin --- .../unified_cache/components/base.py | 3 +- .../unified_cache/components/full.py | 10 +- .../unified_cache/components/mamba.py | 10 +- .../mem_cache/unified_cache/components/swa.py | 8 +- .../unified_cache/unified_tree_core.py | 23 +- .../unified_tree_core_interface.py | 2 + .../test/kits/unified_radix_cache/core.py | 2 +- rust/sglang-radix-tree/src/components/full.rs | 13 +- .../sglang-radix-tree/src/components/mamba.rs | 17 +- rust/sglang-radix-tree/src/components/swa.rs | 16 +- rust/sglang-radix-tree/src/node.rs | 13 +- rust/sglang-radix-tree/src/python_bindings.rs | 4 +- rust/sglang-radix-tree/src/tests/node.rs | 9 + .../src/unified_tree_core.rs | 32 +-- .../test_rust_tree_core_integration.py | 205 ++++++++++++++++-- .../test_unified_radix_cache_core.py | 2 +- 16 files changed, 285 insertions(+), 84 deletions(-) diff --git a/python/sglang/srt/mem_cache/unified_cache/components/base.py b/python/sglang/srt/mem_cache/unified_cache/components/base.py index 20859dfc36..f0da805345 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/base.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/base.py @@ -368,10 +368,11 @@ class TreeComponent(ABC): return cd.host_value is not None def value_len(self, node: UnifiedTreeNode) -> int: + """Length of the stored device indices, including entries without backing.""" value = node.component_data[self.component_type].value return len(value) if value is not None else 0 - def reclaimable_tokens(self, node: UnifiedTreeNode) -> int: + def reclaimable_value_len(self, node: UnifiedTreeNode) -> int: """Pool tokens the allocator gets back when this node's device value is evicted. The evictable/protected ledgers count these (prefill admission adds the evictable ledger to the allocator's free space, and diff --git a/python/sglang/srt/mem_cache/unified_cache/components/full.py b/python/sglang/srt/mem_cache/unified_cache/components/full.py index 06dbed7a40..a071c26a07 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/full.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/full.py @@ -181,8 +181,8 @@ class FullComponent(TreeComponent): if EvictLayer.DEVICE in target and cd.value is not None: device_frees[self.component_type].append(cd.value) # Progress toward an eviction request and the ledger move are - # both in allocator units (see reclaimable_tokens). - freed = self.reclaimable_tokens(node) + # both in allocator units (see reclaimable_value_len). + freed = self.reclaimable_value_len(node) self.tree_core.component_evictable_size_[self.component_type] -= freed # NOTE: cd.value = None is deferred to _cascade_evict (Full as trigger) # because SWA's free_swa still needs to read Full.value. @@ -310,7 +310,7 @@ class FullComponent(TreeComponent): f"FULL invariant broken: evicted ancestor {cur.id} above device-on segment" ) if cd.lock_ref == 0: - key_len = self.reclaimable_tokens(cur) + key_len = self.reclaimable_value_len(cur) self.tree_core.component_evictable_size_[ct] -= key_len self.tree_core.component_protected_size_[ct] += key_len delta += key_len @@ -359,7 +359,7 @@ class FullComponent(TreeComponent): f"FULL segment release hit lock_ref=0 on node {cur.id}" ) if cd.lock_ref == 1 and cd.value is not None: - key_len = self.reclaimable_tokens(cur) + key_len = self.reclaimable_value_len(cur) self.tree_core.component_evictable_size_[ct] += key_len self.tree_core.component_protected_size_[ct] -= key_len cd.lock_ref -= 1 @@ -453,7 +453,7 @@ class FullComponent(TreeComponent): offset += n_len # Full uses leaf sets, not LRU. A value materialized under # lock is protected; the last release moves it to evictable. - ledger = self.reclaimable_tokens(n) + ledger = self.reclaimable_value_len(n) if cd.lock_ref > 0: self.tree_core.component_protected_size_[ct] += ledger else: diff --git a/python/sglang/srt/mem_cache/unified_cache/components/mamba.py b/python/sglang/srt/mem_cache/unified_cache/components/mamba.py index 762a5d45bc..df1419e6e1 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/mamba.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/mamba.py @@ -234,8 +234,8 @@ class MambaComponent(TreeComponent): if is_new_leaf: node.component_data[self.component_type].value = params.mamba_value self.tree_core.lru_lists[self.component_type].insert_mru(node) - self.tree_core.component_evictable_size_[self.component_type] += len( - params.mamba_value + self.tree_core.component_evictable_size_[self.component_type] += ( + self.reclaimable_value_len(node) ) self._emit_excess_path_states_eviction(node, cache_actions) return @@ -334,7 +334,7 @@ class MambaComponent(TreeComponent): # Device layer if EvictLayer.DEVICE in target and cd.value is not None: device_frees[self.component_type].append(cd.value) - freed = len(cd.value) + freed = self.reclaimable_value_len(node) self.tree_core.component_evictable_size_[self.component_type] -= freed cd.value = None @@ -465,7 +465,7 @@ class MambaComponent(TreeComponent): cd.host_lock_ref += 1 else: if cd.lock_ref == 0 and value is not None: - vlen = len(value) + vlen = self.reclaimable_value_len(node) self.tree_core.component_evictable_size_[ct] -= vlen self.tree_core.component_protected_size_[ct] += vlen cd.lock_ref += 1 @@ -498,7 +498,7 @@ class MambaComponent(TreeComponent): assert cd.lock_ref > 0, f"Mamba release hit lock_ref=0 on node {node.id}" if cd.lock_ref == 1 and value is not None: - vlen = len(value) + vlen = self.reclaimable_value_len(node) self.tree_core.component_evictable_size_[ct] += vlen self.tree_core.component_protected_size_[ct] -= vlen cd.lock_ref -= 1 diff --git a/python/sglang/srt/mem_cache/unified_cache/components/swa.py b/python/sglang/srt/mem_cache/unified_cache/components/swa.py index 96176b8090..ef6fa890b8 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/swa.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/swa.py @@ -701,7 +701,7 @@ class SWAComponent(TreeComponent): device_frees[self.component_type].append( node.component_data[BASE_COMPONENT_TYPE].value ) - freed = len(cd.value) + freed = self.reclaimable_value_len(node) self.tree_core.component_evictable_size_[ct] -= freed cd.value = None @@ -866,7 +866,7 @@ class SWAComponent(TreeComponent): if lru.in_list(cur): lru.remove_node(cur) else: - key_len = len(value) + key_len = self.reclaimable_value_len(cur) self.tree_core.component_evictable_size_[ct] -= key_len self.tree_core.component_protected_size_[ct] += key_len if lock_host: @@ -915,7 +915,7 @@ class SWAComponent(TreeComponent): if not host_lru.in_list(cur): host_lru.insert_mru(cur) else: - key_len = len(value) + key_len = self.reclaimable_value_len(cur) self.tree_core.component_evictable_size_[ct] += key_len self.tree_core.component_protected_size_[ct] -= key_len if lock_host: @@ -960,7 +960,7 @@ class SWAComponent(TreeComponent): if cd.lock_ref == 0: self.tree_core._update_evictable_leaf_sets(cur) if cd.lock_ref == 0 and cd.value is not None: - value_len = len(cd.value) + value_len = self.reclaimable_value_len(cur) self.tree_core.component_protected_size_[ct] -= value_len self.tree_core.component_evictable_size_[ct] += value_len if self.tree_core._is_device_leaf(cur): diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py index 15307a0d19..7a4203f409 100644 --- a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py +++ b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py @@ -1529,7 +1529,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): parent.children[key.child_key(self.page_size)] = new_node self.component_evictable_size_[BASE_COMPONENT_TYPE] += self.components_by_type[ BASE_COMPONENT_TYPE - ].reclaimable_tokens(new_node) + ].reclaimable_value_len(new_node) if self.enable_storage or self.enable_external_cache_linker: new_node.hash_value = compute_node_hash_values(new_node, self.page_size) @@ -1550,7 +1550,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): cd = node.component_data[ct] assert cd.value is None cd.value = fresh_value.clone() - n = self.components_by_type[ct].reclaimable_tokens(node) + n = self.components_by_type[ct].reclaimable_value_len(node) if cd.lock_ref > 0: self.component_protected_size_[ct] += n else: @@ -1675,6 +1675,8 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): or result.mamba_backup_node_id is not None or result.swa_backup_node_id is not None or bool(result.tracker) + or any(result.device_frees.values()) + or any(result.host_frees.values()) ) return result @@ -2887,7 +2889,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): self.lru_lists[component_type].insert_mru(node) # A value materialized under lock is protected; the last release # moves it to evictable. - ledger = self.components_by_type[component_type].reclaimable_tokens(node) + ledger = self.components_by_type[component_type].reclaimable_value_len(node) if cd.lock_ref > 0: self.component_protected_size_[component_type] += ledger else: @@ -2902,10 +2904,15 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): cd = node.component_data[component_type] if node is self.root_node or cd.value is None: return - if cd.lock_ref > 0: - self.component_protected_size_[component_type] += delta - else: - self.component_evictable_size_[component_type] += delta + sizes = ( + self.component_protected_size_ + if cd.lock_ref > 0 + else self.component_evictable_size_ + ) + updated = sizes[component_type] + delta + if updated < 0: + raise ValueError(f"component size cannot be negative: {updated}") + sizes[component_type] = updated def get_component_device_value( self, node_id: NodeId, component_type: ComponentType @@ -3126,7 +3133,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): continue cd = n.component_data[ct] if cd.value is not None: - toks = self.components_by_type[ct].reclaimable_tokens(n) + toks = self.components_by_type[ct].reclaimable_value_len(n) if cd.lock_ref > 0: protected += toks else: diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py index ed83ca91ca..d25e2301b2 100644 --- a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py +++ b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py @@ -431,6 +431,8 @@ class UnifiedTreeCoreInterface(ABC): Roots and nodes without device values are no-ops. Missing handles raise KeyError; disabled components raise ValueError. + Negative totals raise ValueError without updating the count. Rust also + rejects fixed-width overflow; Python integers are unbounded. """ ... diff --git a/python/sglang/test/kits/unified_radix_cache/core.py b/python/sglang/test/kits/unified_radix_cache/core.py index 1df20d6939..38e8a43c39 100644 --- a/python/sglang/test/kits/unified_radix_cache/core.py +++ b/python/sglang/test/kits/unified_radix_cache/core.py @@ -89,7 +89,7 @@ class PagedFullComponent(FullComponent): super().__init__(*args, **kwargs) self.rows_handed_back: dict[int, int] = {} - def reclaimable_tokens(self, node): + def reclaimable_value_len(self, node): return self.value_len(node) // 2 - self.rows_handed_back.get(node.id, 0) diff --git a/rust/sglang-radix-tree/src/components/full.rs b/rust/sglang-radix-tree/src/components/full.rs index 1eb72901e3..acfe6c3501 100644 --- a/rust/sglang-radix-tree/src/components/full.rs +++ b/rust/sglang-radix-tree/src/components/full.rs @@ -111,7 +111,7 @@ impl TreeComponent for FullComponent { let mut host_freed = 0; if target.contains(EvictLayer::Device) && node.has_device_value(FULL) { let value = node.device_value(FULL); - freed = node.device_value_len(FULL); + freed = node.device_reclaimable_value_len(FULL); device_frees .entry(FULL) .or_default() @@ -320,7 +320,7 @@ impl TreeComponent for FullComponent { ); let parent = node.parent(); let newly_locked_len = if node.device_lock_ref(FULL) == 0 { - Some(node.device_value_len(FULL)) + Some(node.device_reclaimable_value_len(FULL)) } else { None }; @@ -397,8 +397,8 @@ impl TreeComponent for FullComponent { old_lock_ref > 0, "FULL segment release hit lock_ref=0 on node {cur}" ); - let newly_unlocked_len = if old_lock_ref == 1 && node.has_device_value(FULL) { - Some(node.device_value_len(FULL)) + let newly_unlocked_len = if old_lock_ref == 1 { + Some(node.device_reclaimable_value_len(FULL)) } else { None }; @@ -514,14 +514,15 @@ impl TreeComponent for FullComponent { loaded .set_device_value(FULL, device_indices.narrow(0, offset, n_len).copy()); let locked = loaded.device_lock_ref(FULL) > 0; + let tokens = loaded.device_reclaimable_value_len(FULL); offset += n_len; // Full uses leaf sets, not LRU. A value materialized // under lock is protected; the last release moves it // to evictable. if locked { - tree_core.inc_protected_size(FULL, n_len as usize); + tree_core.inc_protected_size(FULL, tokens); } else { - tree_core.inc_evictable_size(FULL, n_len as usize); + tree_core.inc_evictable_size(FULL, tokens); } tree_core.update_evictable_leaf_sets_(loaded_idx); } diff --git a/rust/sglang-radix-tree/src/components/mamba.rs b/rust/sglang-radix-tree/src/components/mamba.rs index 142a2d2474..03ee89d7d0 100644 --- a/rust/sglang-radix-tree/src/components/mamba.rs +++ b/rust/sglang-radix-tree/src/components/mamba.rs @@ -166,14 +166,17 @@ impl TreeComponent for MambaComponent { .mamba_value .as_ref() .expect("mamba insert requires a donated mamba_value"); - let slot_len = mamba_value.size()[0] as usize; if is_new_leaf { tree_core .arena .set_device_value(node_id, MAMBA, mamba_value.shallow_clone()); tree_core.device_lru_list_mut(MAMBA).insert_mru(node_id); - tree_core.inc_evictable_size(MAMBA, slot_len); + let tokens = tree_core + .arena + .node(node_id) + .device_reclaimable_value_len(MAMBA); + tree_core.inc_evictable_size(MAMBA, tokens); self.emit_excess_path_states_eviction_(tree_core.arena.node(node_id).id, cache_actions); return; } @@ -288,7 +291,7 @@ impl TreeComponent for MambaComponent { // Device layer if target.contains(EvictLayer::Device) && node.has_device_value(MAMBA) { - freed = node.device_value_len(MAMBA); + freed = node.device_reclaimable_value_len(MAMBA); device_frees .entry(ct) .or_default() @@ -449,8 +452,8 @@ impl TreeComponent for MambaComponent { } tree_core.arena.inc_host_lock_ref(node_id, MAMBA); } else { - let value_len = node.device_value_len(MAMBA); - if node.device_lock_ref(MAMBA) == 0 && has_value { + let value_len = node.device_reclaimable_value_len(MAMBA); + if node.device_lock_ref(MAMBA) == 0 { tree_core.dec_evictable_size(MAMBA, value_len); tree_core.inc_protected_size(MAMBA, value_len); } @@ -494,8 +497,8 @@ impl TreeComponent for MambaComponent { device_lock_ref > 0, "Mamba release hit lock_ref=0 on node {node_id}" ); - if device_lock_ref == 1 && node.has_device_value(MAMBA) { - let value_len = node.device_value_len(MAMBA); + if device_lock_ref == 1 { + let value_len = node.device_reclaimable_value_len(MAMBA); tree_core.inc_evictable_size(MAMBA, value_len); tree_core.dec_protected_size(MAMBA, value_len); } diff --git a/rust/sglang-radix-tree/src/components/swa.rs b/rust/sglang-radix-tree/src/components/swa.rs index 509269fe44..003dc447eb 100644 --- a/rust/sglang-radix-tree/src/components/swa.rs +++ b/rust/sglang-radix-tree/src/components/swa.rs @@ -118,14 +118,6 @@ impl SwaComponent { } } - fn value_len(node: &Node, host: bool) -> usize { - if host { - node.host_value_len(SWA) - } else { - node.device_value_len(SWA) - } - } - fn swa_uuid(node: &Node, host: bool) -> Option { if host { node.swa_host_uuid @@ -682,7 +674,7 @@ impl TreeComponent for SwaComponent { .entry(ct) .or_default() .push(node.device_value(FULL).shallow_clone()); - freed = node.device_value_len(SWA); + freed = node.device_reclaimable_value_len(SWA); let _ = node.take_device_value(SWA); tree_core.dec_evictable_size(SWA, freed); } @@ -1172,7 +1164,7 @@ impl TreeComponent for SwaComponent { let parent = node.parent(); let key_len = node.key.atom_len(); let has_value = Self::has_value(node, lock_host); - let value_len = Self::value_len(node, lock_host); + let value_len = node.device_reclaimable_value_len(SWA); let newly_locked = Self::lock_ref(node, lock_host) == 0; Self::inc_lock_ref(node, lock_host); if newly_locked && has_value { @@ -1225,7 +1217,7 @@ impl TreeComponent for SwaComponent { if lock_host { "host_" } else { "" } ); let has_value = Self::has_value(node, lock_host); - let value_len = Self::value_len(node, lock_host); + let value_len = node.device_reclaimable_value_len(SWA); if lock_ref == 1 && has_value { if lock_host { if !node.has_device_value(SWA) { @@ -1288,7 +1280,7 @@ impl TreeComponent for SwaComponent { "SWA window release hit lock_ref=0 on node {cur}" ); let has_value = node.has_device_value(SWA); - let value_len = node.device_value_len(SWA); + let value_len = node.device_reclaimable_value_len(SWA); node.dec_device_lock_ref(SWA); let now_unlocked = node.device_lock_ref(SWA) == 0; if now_unlocked { diff --git a/rust/sglang-radix-tree/src/node.rs b/rust/sglang-radix-tree/src/node.rs index 4c0b563307..ca2a20f1fb 100644 --- a/rust/sglang-radix-tree/src/node.rs +++ b/rust/sglang-radix-tree/src/node.rs @@ -264,11 +264,20 @@ impl Node { self.has_value_(ValueSlotIdx::device(component_type)) } - /// The component's device value length, or 0 when value-less. + /// Logical length of the stored device-index tensor, or 0 when absent. pub fn device_value_len(&self, component_type: ComponentType) -> usize { self.value_len_(ValueSlotIdx::device(component_type)) } + /// Allocator slots reclaimed on eviction; roots and absent device values contribute zero. + /// Ordinary backing uses logical length; released/shared backing needs its own accounting. + pub fn device_reclaimable_value_len(&self, component_type: ComponentType) -> usize { + if self.is_root() { + return 0; + } + self.device_value_len(component_type) + } + /// Set the component's device value; panics if already set. pub fn set_device_value(&mut self, component_type: ComponentType, value: Tensor) { self.set_value_(ValueSlotIdx::device(component_type), value); @@ -1489,7 +1498,7 @@ impl NodeArena { self.node(id).has_device_value(component_type) } - /// The node's device value length for the component, or 0 when value-less. + /// Logical length of the node's stored device-index tensor, or 0 when absent. pub fn device_value_len(&self, id: NodeIdx_, component_type: ComponentType) -> usize { self.node(id).device_value_len(component_type) } diff --git a/rust/sglang-radix-tree/src/python_bindings.rs b/rust/sglang-radix-tree/src/python_bindings.rs index 4f93548460..935d6c71e4 100644 --- a/rust/sglang-radix-tree/src/python_bindings.rs +++ b/rust/sglang-radix-tree/src/python_bindings.rs @@ -1512,7 +1512,9 @@ impl TreeCoreBinding { let made_progress = node_id.is_some() || result.mamba_backup_node_id.is_some() || result.swa_backup_node_id.is_some() - || !result.tracker.is_empty(); + || !result.tracker.is_empty() + || result.device_frees.values().any(|frees| !frees.is_empty()) + || result.host_frees.values().any(|frees| !frees.is_empty()); Ok(EvictDeviceNextNodeResultBinding { node_id, mamba_backup_node_id: result.mamba_backup_node_id, diff --git a/rust/sglang-radix-tree/src/tests/node.rs b/rust/sglang-radix-tree/src/tests/node.rs index c5ec7de3b6..6e5f0ca532 100644 --- a/rust/sglang-radix-tree/src/tests/node.rs +++ b/rust/sglang-radix-tree/src/tests/node.rs @@ -67,6 +67,7 @@ impl ChildKeyType for CountedKey { fn evicted_only_for_attached_nodes_without_full_device_value() { let mut parent: Node> = Node::new_root(/* id = */ 0); assert!(!parent.evicted()); + assert_eq!(parent.device_reclaimable_value_len(FULL), 0); let mut child = Node::new_child( /* id = */ 1, /* key = */ vec![7], @@ -78,8 +79,12 @@ fn evicted_only_for_attached_nodes_without_full_device_value() { .attach_child(&mut child, /* page_size = */ 1) .unwrap(); assert!(child.evicted()); + assert_eq!(child.device_reclaimable_value_len(FULL), 0); child.set_device_value(FULL, Tensor::from_slice(&[1i64])); assert!(!child.evicted()); + assert_eq!(child.device_reclaimable_value_len(FULL), 1); + let _ = child.take_device_value(FULL); + assert_eq!(child.device_reclaimable_value_len(FULL), 0); } #[test] @@ -470,6 +475,10 @@ fn present_but_empty_tensor_has_value_true_len_zero() { node.state_mut_(ValueSlotIdx::device(FULL)).value = Some(Tensor::from_slice(&empty)); assert!(node.has_device_value(FULL)); assert_eq!(node.device_value_len(FULL), 0); + assert_eq!(node.device_reclaimable_value_len(FULL), 0); + node.state_mut_(ValueSlotIdx::device(FULL)).value = Some(Tensor::from_slice(&[1i64])); + assert_eq!(node.device_value_len(FULL), 1); + assert_eq!(node.device_reclaimable_value_len(FULL), 0); } #[test] diff --git a/rust/sglang-radix-tree/src/unified_tree_core.rs b/rust/sglang-radix-tree/src/unified_tree_core.rs index 2729528968..0ebfa57f65 100644 --- a/rust/sglang-radix-tree/src/unified_tree_core.rs +++ b/rust/sglang-radix-tree/src/unified_tree_core.rs @@ -676,7 +676,7 @@ pub struct UnifiedTreeCore { impl UnifiedTreeCore { /// Adjust the component's evictable or protected count of reclaimable tokens. - /// The caller updates backing and size together, and restores backing on error. + /// Counters are unchanged on error; the caller keeps backing consistent. /// This changes counters only; roots and device tombstones are ignored. pub fn adjust_component_size( &mut self, @@ -2311,7 +2311,11 @@ impl UnifiedTreeCore { displaced.is_none(), "add_new_node_: parent {parent_id} already has a child on the new node's page" ); - self.inc_evictable_size(FULL, value.size()[0] as usize); + let tokens = self + .arena + .node(new_node_id) + .device_reclaimable_value_len(FULL); + self.inc_evictable_size(FULL, tokens); if self.enable_storage || self.enable_external_cache_linker { let hash_values = self.arena.compute_node_hash_values(new_node_id, page_size); self.arena.node_mut(new_node_id).hash_value = Some(hash_values); @@ -2333,7 +2337,7 @@ impl UnifiedTreeCore { ) { self.arena .set_device_value(node_id, FULL, fresh_value.copy()); - let tokens = fresh_value.size()[0] as usize; + let tokens = self.arena.node(node_id).device_reclaimable_value_len(FULL); // A value materialized under lock is protected; the last release // moves it to evictable. if self.arena.device_lock_ref(node_id, FULL) > 0 { @@ -4675,8 +4679,11 @@ impl UnifiedTreeCore { component_type != BASE_COMPONENT_TYPE, "set_component_device_value: auxiliary components only" ); - let tokens = value.size()[0] as usize; self.arena.set_device_value(node_id, component_type, value); + let tokens = self + .arena + .node(node_id) + .device_reclaimable_value_len(component_type); let host_lru = self.host_lru_list_mut(component_type); if host_lru.in_list(Some(node_id)) { host_lru.remove_node(node_id); @@ -5136,17 +5143,12 @@ impl UnifiedTreeCore { let mut evictable = 0usize; let mut protected = 0usize; for &node_id in &all_nodes { - if self.arena.node(node_id).is_root() { - continue; - } - let state = &self.arena.node(node_id).values[ct.idx()]; - if let Some(value) = &state.value { - let tokens = value.size()[0] as usize; - if state.lock_ref > 0 { - protected += tokens; - } else { - evictable += tokens; - } + let node = self.arena.node(node_id); + let tokens = node.device_reclaimable_value_len(ct); + if node.device_lock_ref(ct) > 0 { + protected += tokens; + } else { + evictable += tokens; } } let recorded = self.component_state(ct); diff --git a/test/registered/unit/mem_cache/test_rust_tree_core_integration.py b/test/registered/unit/mem_cache/test_rust_tree_core_integration.py index 512915e9e7..a96dd8611c 100644 --- a/test/registered/unit/mem_cache/test_rust_tree_core_integration.py +++ b/test/registered/unit/mem_cache/test_rust_tree_core_integration.py @@ -1105,6 +1105,8 @@ def test_host_only_and_root_adjustments_are_noops(backend, is_eagle): for node in (root, host): for delta in (-10, 0, 10): core.adjust_component_size(node, ComponentType.FULL, delta) + with pytest.raises(ValueError): + core.adjust_component_size(node, ComponentType.SWA, 0) assert (core.full_evictable_size(), core.full_protected_size()) == (0, 0) assert core.collect_path_info([host])[0][0].full_value_len is None @@ -1129,6 +1131,7 @@ def test_failed_adjustments_preserve_accounting_and_tree(backend, is_eagle, lock (invalid_node, ComponentType.FULL, 0, KeyError), (leaf, ComponentType.SWA, 0, ValueError), (leaf, 99, 0, ValueError), + (leaf, ComponentType.FULL, -len(key) - 1, ValueError), ): with pytest.raises(error): core.adjust_component_size(node, component, delta) @@ -1154,29 +1157,55 @@ def test_adjustment_follows_real_backing_changes_across_lock_transitions(): from sglang.srt.mem_cache.unified_cache.unified_tree_core import UnifiedTreeCore class SideBackingFullComponent(FullComponent): - def reclaimable_tokens(self, node): - backing = node.component_data[ComponentType.FULL].metadata.get( - "side_backing" - ) - return super().reclaimable_tokens(node) + ( - 0 if backing is None else backing.numel() - ) + def reclaimable_value_len(self, node): + metadata = node.component_data[ComponentType.FULL].metadata + tokens = super().reclaimable_value_len(node) + if "raw_backing" in metadata: + tokens = metadata["raw_backing"].numel() + side_backing = metadata.get("side_backing") + return tokens + (0 if side_backing is None else side_backing.numel()) params = CacheInitParams( False, None, None, 1, tree_components=(ComponentType.FULL,) ) cache = SimpleNamespace(enable_session_radix_cache=False) - core = UnifiedTreeCore( - params, {ComponentType.FULL: SideBackingFullComponent(cache, params)} - ) + driver = SideBackingFullComponent(cache, params) + core = UnifiedTreeCore(params, {ComponentType.FULL: driver}) leaf = _insert(core, [1, 2, 3, 4], [10, 11, 12, 13]).last_device_node - metadata = core.node_by_id(leaf).component_data[ComponentType.FULL].metadata + node = core.node_by_id(leaf) + metadata = node.component_data[ComponentType.FULL].metadata metadata["side_backing"] = torch.tensor([100, 101]) core.adjust_component_size(leaf, ComponentType.FULL, 2) assert (core.full_evictable_size(), core.full_protected_size()) == (6, 0) + assert driver.value_len(node) == 4 + assert driver.reclaimable_value_len(node) == 6 + assert core.collect_path_info([leaf])[0][0].full_value_len == 4 + matched = core.match_prefix(MatchPrefixParams(key=_key([1, 2, 3, 4]))) + assert matched.device_prefix_len == 4 + assert _path_indices(core, matched.last_device_node).tolist() == [10, 11, 12, 13] lock = core.inc_lock_ref(leaf) assert (core.full_evictable_size(), core.full_protected_size()) == (0, 6) + # A locked split follows logical indices; the tail keeps the side backing. + prefix_match = core.match_prefix(MatchPrefixParams(key=_key([1, 2]))) + prefix = prefix_match.last_device_node + prefix_node = core.node_by_id(prefix) + assert prefix_match.device_prefix_len == 2 + assert _path_indices(core, prefix).tolist() == [10, 11] + assert core.get_component_device_value(leaf, ComponentType.FULL).tolist() == [ + 12, + 13, + ] + assert (driver.value_len(prefix_node), driver.value_len(node)) == (2, 2) + assert ( + driver.reclaimable_value_len(prefix_node), + driver.reclaimable_value_len(node), + ) == (2, 4) + assert [ + (span.path_start, span.path_end, span.full_value_len) + for span in core.collect_path_info([leaf])[0] + ] == [(0, 2, 2), (2, 4, 2)] + assert (core.full_evictable_size(), core.full_protected_size()) == (0, 6) metadata["side_backing"] = metadata["side_backing"][:1] core.adjust_component_size(leaf, ComponentType.FULL, -1) assert (core.full_evictable_size(), core.full_protected_size()) == (0, 5) @@ -1185,9 +1214,27 @@ def test_adjustment_follows_real_backing_changes_across_lock_transitions(): core.dec_lock_ref(leaf, lock.to_dec_params()) assert (core.full_evictable_size(), core.full_protected_size()) == (4, 0) core.sanity_check([], []) + + for fragment in (prefix_node, node): + fragment.component_data[ComponentType.FULL].metadata["raw_backing"] = ( + torch.empty(0, dtype=torch.int64) + ) + core.adjust_component_size(fragment.id, ComponentType.FULL, -2) + assert driver.value_len(fragment) == 2 + assert driver.reclaimable_value_len(fragment) == 0 + assert (core.full_evictable_size(), core.full_protected_size()) == (0, 0) + assert [span.full_value_len for span in core.collect_path_info([leaf])[0]] == [2, 2] + matched = core.match_prefix(MatchPrefixParams(key=_key([1, 2, 3, 4]))) + assert matched.device_prefix_len == 4 + assert _path_indices(core, matched.last_device_node).tolist() == [10, 11, 12, 13] + core.sanity_check([], []) tracker = {} - _accumulate_step(core.evict_device_leaf(leaf, False), tracker, {}, {}) - assert tracker[ComponentType.FULL] == 4 + device_frees = {} + _accumulate_step(core.evict_device_leaf(leaf, False), tracker, device_frees, {}) + assert torch.cat(device_frees[ComponentType.FULL]).tolist() == [12, 13] + _accumulate_step(core.evict_device_leaf(prefix, False), tracker, device_frees, {}) + assert tracker[ComponentType.FULL] == 0 + assert torch.cat(device_frees[ComponentType.FULL]).tolist() == [12, 13, 10, 11] assert (core.full_evictable_size(), core.full_protected_size()) == (0, 0) @@ -2692,6 +2739,7 @@ def _swa_cache( backend="rust", ring=False, request_window=False, + component_registry_override=None, ): """A real UnifiedRadixCache and SWA allocator, defaulting to the Rust core.""" from sglang.srt.mem_cache.allocator.swa import SWATokenToKVPoolAllocator @@ -2742,6 +2790,7 @@ def _swa_cache( sliding_window_size=window, swa_gates_prefix_match=not (ring or request_window), tree_components=(ComponentType.FULL, ComponentType.SWA), + component_registry_override=component_registry_override, ) ) return cache, allocator @@ -2758,7 +2807,9 @@ def _complete_backup(core, node, full_offset=1000, aux_offset=1000): return full + full_offset, aux -def _swa_mamba_cache(backend, window, enable_session=False): +def _swa_mamba_cache( + backend, window, enable_session=False, component_registry_override=None +): from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache @@ -2779,6 +2830,7 @@ def _swa_mamba_cache(backend, window, enable_session=False): page_size=1, sliding_window_size=window, enable_session_radix_cache=enable_session, + component_registry_override=component_registry_override, tree_components=( ComponentType.FULL, ComponentType.SWA, @@ -2796,11 +2848,17 @@ def _internal_swa_write_back_case( with_mamba=False, prefix_nodes=1, enable_session=False, + component_registry_override=None, ): cache, allocator = ( - _swa_mamba_cache(backend, window, enable_session) + _swa_mamba_cache(backend, window, enable_session, component_registry_override) if with_mamba - else _swa_cache(window=window, page_size=page_size, backend=backend) + else _swa_cache( + window=window, + page_size=page_size, + backend=backend, + component_registry_override=component_registry_override, + ) ) assert cache._tree_core_backend == backend segment = 2 * page_size @@ -3140,6 +3198,121 @@ def _assert_internal_eviction_pending(step, component, node, swa_tokens=0): assert step.unbacked_tokens == 0 +@pytest.mark.parametrize("component", [ComponentType.MAMBA, ComponentType.SWA]) +@pytest.mark.parametrize("early_release", [False, True], ids=["final", "early-swa"]) +def test_python_component_locks_use_reclaimable_backing(component, early_release): + from sglang.srt.mem_cache.unified_cache.components.mamba import MambaComponent + from sglang.srt.mem_cache.unified_cache.components.swa import SWAComponent + + base = MambaComponent if component == ComponentType.MAMBA else SWAComponent + + class SideBackingComponent(base): + def reclaimable_value_len(self, node): + backing = node.component_data[self.component_type].metadata.get("backing") + if backing is not None: + return backing.numel() + return super().reclaimable_value_len(node) + + case = _internal_swa_write_back_case( + "python", + with_mamba=component == ComponentType.MAMBA, + component_registry_override={component: SideBackingComponent}, + ) + cache, core = case.cache, case.core + node_id, leaf = case.nodes + node = core.node_by_id(node_id) + state = node.component_data[component] + logical = state.value.clone() + state.metadata["backing"] = torch.arange(len(logical) + 2) + core.adjust_component_size(node_id, component, 2) + + def ledgers(): + return ( + core.component_evictable_size_[component], + core.component_protected_size_[component], + ) + + before = ledgers() + reclaimed = len(logical) + 2 + lock = core.inc_lock_ref(node_id).to_dec_params() + assert state.lock_ref == 1 + assert ledgers() == (before[0] - reclaimed, before[1] + reclaimed) + assert cache.components[component].value_len(node) == len(logical) + assert torch.equal(state.value, logical) + + # The final release uses current backing, not the original lock delta. + state.metadata["backing"] = state.metadata["backing"][:-1] + core.adjust_component_size(node_id, component, -1) + if early_release: + released = core.dec_swa_lock_only(node_id, lock) + assert not released.device_frees and not released.host_frees + core.dec_lock_ref(node_id, lock, skip_swa=True) + else: + core.dec_lock_ref(node_id, lock) + assert state.lock_ref == 0 + assert ledgers() == (before[0] - 1, before[1]) + assert torch.equal(state.value, logical) + cache.dec_lock_ref(leaf, case.leaf_lock) + core.sanity_check([], []) + + +@pytest.mark.parametrize("component", [ComponentType.MAMBA, ComponentType.SWA]) +def test_python_zero_accounted_internal_eviction_continues(component): + from sglang.srt.mem_cache.unified_cache.components.mamba import MambaComponent + from sglang.srt.mem_cache.unified_cache.components.swa import SWAComponent + + base = MambaComponent if component == ComponentType.MAMBA else SWAComponent + + class ReleasedBackingComponent(base): + def reclaimable_value_len(self, node): + if node.component_data[self.component_type].metadata.get("released"): + return 0 + return super().reclaimable_value_len(node) + + case = _internal_swa_write_back_case( + "python", + with_mamba=component == ComponentType.MAMBA, + prefix_nodes=2, + component_registry_override={component: ReleasedBackingComponent}, + ) + cache, core = case.cache, case.core + first, second, leaf = case.nodes + cache.is_write_back = False + driver = cache.components[component] + expected_values = [ + core.get_component_device_value(node, component).clone() + for node in (first, second) + ] + first_size = driver.reclaimable_value_len(core.node_by_id(first)) + expected = driver.reclaimable_value_len(core.node_by_id(second)) + core.node_by_id(first).component_data[component].metadata["released"] = True + core.adjust_component_size(first, component, -first_size) + assert driver.value_len(core.node_by_id(first)) == len(expected_values[0]) > 0 + assert driver.reclaimable_value_len(core.node_by_id(first)) == 0 + tracker, freed = {}, {} + core.evict_device_start(component, expected) + try: + first_step = core.evict_device_next_node(component, tracker) + assert first_step.node_id is None and first_step.made_progress + assert not first_step.tracker and first_step.device_frees[component] + _accumulate_step(first_step, tracker, freed, {}) + assert core.get_component_device_value(first, component) is None + assert torch.equal(torch.cat(freed[component]), expected_values[0]) + + second_step = core.evict_device_next_node(component, tracker) + assert second_step.node_id is None and second_step.made_progress + _accumulate_step(second_step, tracker, freed, {}) + assert tracker[component] == expected + assert core.get_component_device_value(second, component) is None + assert len(freed[component]) == 2 + assert torch.equal(torch.cat(freed[component]), torch.cat(expected_values)) + assert not core.evict_device_next_node(component, tracker).made_progress + finally: + core.evict_device_end(component) + cache.dec_lock_ref(leaf, case.leaf_lock) + core.sanity_check([], []) + + @pytest.mark.parametrize("component", [ComponentType.MAMBA, ComponentType.SWA]) def test_python_component_returns_immutable_internal_backup_request(component): from dataclasses import FrozenInstanceError diff --git a/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_core.py b/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_core.py index 1f444bec7b..759b3030d8 100644 --- a/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_core.py +++ b/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_core.py @@ -122,7 +122,7 @@ class TestUnifiedRadixComponentLedgerUnits(CustomTestCase): prefill admission adds the evictable ledger to the allocator's free space and the eviction walk counts them toward its request. So every ledger site (insert, lock, release, an in-place change, eviction) - reads ``TreeComponent.reclaimable_tokens`` rather than the value + reads ``TreeComponent.reclaimable_value_len`` rather than the value length. A site that goes back to ``len(value)`` drifts the ledger from the recount in ``sanity_check`` for a component that overrides it.""" cache, allocator, _ = build_fixture(