[Fix] radix-cache: Separate logical and reclaimable value lengths in tree components (#45238)

Co-authored-by: metamergebot <metamergebot@users.noreply.github.com>
Co-authored-by: Jialin <Jialin@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-20 13:24:20 -07:00
committed by GitHub
co-authored by metamergebot Jialin
parent 468e26f253
commit 0b536ca112
16 changed files with 285 additions and 84 deletions
@@ -368,10 +368,11 @@ class TreeComponent(ABC):
return cd.host_value is not None
def value_len(self, node: UnifiedTreeNode) -> int:
"""Length of the stored device indices, including entries without backing."""
value = node.component_data[self.component_type].value
return len(value) if value is not None else 0
def reclaimable_tokens(self, node: UnifiedTreeNode) -> int:
def reclaimable_value_len(self, node: UnifiedTreeNode) -> int:
"""Pool tokens the allocator gets back when this node's device value
is evicted. The evictable/protected ledgers count these (prefill
admission adds the evictable ledger to the allocator's free space, and
@@ -181,8 +181,8 @@ class FullComponent(TreeComponent):
if EvictLayer.DEVICE in target and cd.value is not None:
device_frees[self.component_type].append(cd.value)
# Progress toward an eviction request and the ledger move are
# both in allocator units (see reclaimable_tokens).
freed = self.reclaimable_tokens(node)
# both in allocator units (see reclaimable_value_len).
freed = self.reclaimable_value_len(node)
self.tree_core.component_evictable_size_[self.component_type] -= freed
# NOTE: cd.value = None is deferred to _cascade_evict (Full as trigger)
# because SWA's free_swa still needs to read Full.value.
@@ -310,7 +310,7 @@ class FullComponent(TreeComponent):
f"FULL invariant broken: evicted ancestor {cur.id} above device-on segment"
)
if cd.lock_ref == 0:
key_len = self.reclaimable_tokens(cur)
key_len = self.reclaimable_value_len(cur)
self.tree_core.component_evictable_size_[ct] -= key_len
self.tree_core.component_protected_size_[ct] += key_len
delta += key_len
@@ -359,7 +359,7 @@ class FullComponent(TreeComponent):
f"FULL segment release hit lock_ref=0 on node {cur.id}"
)
if cd.lock_ref == 1 and cd.value is not None:
key_len = self.reclaimable_tokens(cur)
key_len = self.reclaimable_value_len(cur)
self.tree_core.component_evictable_size_[ct] += key_len
self.tree_core.component_protected_size_[ct] -= key_len
cd.lock_ref -= 1
@@ -453,7 +453,7 @@ class FullComponent(TreeComponent):
offset += n_len
# Full uses leaf sets, not LRU. A value materialized under
# lock is protected; the last release moves it to evictable.
ledger = self.reclaimable_tokens(n)
ledger = self.reclaimable_value_len(n)
if cd.lock_ref > 0:
self.tree_core.component_protected_size_[ct] += ledger
else:
@@ -234,8 +234,8 @@ class MambaComponent(TreeComponent):
if is_new_leaf:
node.component_data[self.component_type].value = params.mamba_value
self.tree_core.lru_lists[self.component_type].insert_mru(node)
self.tree_core.component_evictable_size_[self.component_type] += len(
params.mamba_value
self.tree_core.component_evictable_size_[self.component_type] += (
self.reclaimable_value_len(node)
)
self._emit_excess_path_states_eviction(node, cache_actions)
return
@@ -334,7 +334,7 @@ class MambaComponent(TreeComponent):
# Device layer
if EvictLayer.DEVICE in target and cd.value is not None:
device_frees[self.component_type].append(cd.value)
freed = len(cd.value)
freed = self.reclaimable_value_len(node)
self.tree_core.component_evictable_size_[self.component_type] -= freed
cd.value = None
@@ -465,7 +465,7 @@ class MambaComponent(TreeComponent):
cd.host_lock_ref += 1
else:
if cd.lock_ref == 0 and value is not None:
vlen = len(value)
vlen = self.reclaimable_value_len(node)
self.tree_core.component_evictable_size_[ct] -= vlen
self.tree_core.component_protected_size_[ct] += vlen
cd.lock_ref += 1
@@ -498,7 +498,7 @@ class MambaComponent(TreeComponent):
assert cd.lock_ref > 0, f"Mamba release hit lock_ref=0 on node {node.id}"
if cd.lock_ref == 1 and value is not None:
vlen = len(value)
vlen = self.reclaimable_value_len(node)
self.tree_core.component_evictable_size_[ct] += vlen
self.tree_core.component_protected_size_[ct] -= vlen
cd.lock_ref -= 1
@@ -701,7 +701,7 @@ class SWAComponent(TreeComponent):
device_frees[self.component_type].append(
node.component_data[BASE_COMPONENT_TYPE].value
)
freed = len(cd.value)
freed = self.reclaimable_value_len(node)
self.tree_core.component_evictable_size_[ct] -= freed
cd.value = None
@@ -866,7 +866,7 @@ class SWAComponent(TreeComponent):
if lru.in_list(cur):
lru.remove_node(cur)
else:
key_len = len(value)
key_len = self.reclaimable_value_len(cur)
self.tree_core.component_evictable_size_[ct] -= key_len
self.tree_core.component_protected_size_[ct] += key_len
if lock_host:
@@ -915,7 +915,7 @@ class SWAComponent(TreeComponent):
if not host_lru.in_list(cur):
host_lru.insert_mru(cur)
else:
key_len = len(value)
key_len = self.reclaimable_value_len(cur)
self.tree_core.component_evictable_size_[ct] += key_len
self.tree_core.component_protected_size_[ct] -= key_len
if lock_host:
@@ -960,7 +960,7 @@ class SWAComponent(TreeComponent):
if cd.lock_ref == 0:
self.tree_core._update_evictable_leaf_sets(cur)
if cd.lock_ref == 0 and cd.value is not None:
value_len = len(cd.value)
value_len = self.reclaimable_value_len(cur)
self.tree_core.component_protected_size_[ct] -= value_len
self.tree_core.component_evictable_size_[ct] += value_len
if self.tree_core._is_device_leaf(cur):
@@ -1529,7 +1529,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
parent.children[key.child_key(self.page_size)] = new_node
self.component_evictable_size_[BASE_COMPONENT_TYPE] += self.components_by_type[
BASE_COMPONENT_TYPE
].reclaimable_tokens(new_node)
].reclaimable_value_len(new_node)
if self.enable_storage or self.enable_external_cache_linker:
new_node.hash_value = compute_node_hash_values(new_node, self.page_size)
@@ -1550,7 +1550,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
cd = node.component_data[ct]
assert cd.value is None
cd.value = fresh_value.clone()
n = self.components_by_type[ct].reclaimable_tokens(node)
n = self.components_by_type[ct].reclaimable_value_len(node)
if cd.lock_ref > 0:
self.component_protected_size_[ct] += n
else:
@@ -1675,6 +1675,8 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
or result.mamba_backup_node_id is not None
or result.swa_backup_node_id is not None
or bool(result.tracker)
or any(result.device_frees.values())
or any(result.host_frees.values())
)
return result
@@ -2887,7 +2889,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
self.lru_lists[component_type].insert_mru(node)
# A value materialized under lock is protected; the last release
# moves it to evictable.
ledger = self.components_by_type[component_type].reclaimable_tokens(node)
ledger = self.components_by_type[component_type].reclaimable_value_len(node)
if cd.lock_ref > 0:
self.component_protected_size_[component_type] += ledger
else:
@@ -2902,10 +2904,15 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
cd = node.component_data[component_type]
if node is self.root_node or cd.value is None:
return
if cd.lock_ref > 0:
self.component_protected_size_[component_type] += delta
else:
self.component_evictable_size_[component_type] += delta
sizes = (
self.component_protected_size_
if cd.lock_ref > 0
else self.component_evictable_size_
)
updated = sizes[component_type] + delta
if updated < 0:
raise ValueError(f"component size cannot be negative: {updated}")
sizes[component_type] = updated
def get_component_device_value(
self, node_id: NodeId, component_type: ComponentType
@@ -3126,7 +3133,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
continue
cd = n.component_data[ct]
if cd.value is not None:
toks = self.components_by_type[ct].reclaimable_tokens(n)
toks = self.components_by_type[ct].reclaimable_value_len(n)
if cd.lock_ref > 0:
protected += toks
else:
@@ -431,6 +431,8 @@ class UnifiedTreeCoreInterface(ABC):
Roots and nodes without device values are no-ops. Missing handles raise
KeyError; disabled components raise ValueError.
Negative totals raise ValueError without updating the count. Rust also
rejects fixed-width overflow; Python integers are unbounded.
"""
...
@@ -89,7 +89,7 @@ class PagedFullComponent(FullComponent):
super().__init__(*args, **kwargs)
self.rows_handed_back: dict[int, int] = {}
def reclaimable_tokens(self, node):
def reclaimable_value_len(self, node):
return self.value_len(node) // 2 - self.rows_handed_back.get(node.id, 0)
@@ -111,7 +111,7 @@ impl<K: ChildKeyType> TreeComponent<K> for FullComponent {
let mut host_freed = 0;
if target.contains(EvictLayer::Device) && node.has_device_value(FULL) {
let value = node.device_value(FULL);
freed = node.device_value_len(FULL);
freed = node.device_reclaimable_value_len(FULL);
device_frees
.entry(FULL)
.or_default()
@@ -320,7 +320,7 @@ impl<K: ChildKeyType> TreeComponent<K> for FullComponent {
);
let parent = node.parent();
let newly_locked_len = if node.device_lock_ref(FULL) == 0 {
Some(node.device_value_len(FULL))
Some(node.device_reclaimable_value_len(FULL))
} else {
None
};
@@ -397,8 +397,8 @@ impl<K: ChildKeyType> TreeComponent<K> for FullComponent {
old_lock_ref > 0,
"FULL segment release hit lock_ref=0 on node {cur}"
);
let newly_unlocked_len = if old_lock_ref == 1 && node.has_device_value(FULL) {
Some(node.device_value_len(FULL))
let newly_unlocked_len = if old_lock_ref == 1 {
Some(node.device_reclaimable_value_len(FULL))
} else {
None
};
@@ -514,14 +514,15 @@ impl<K: ChildKeyType> TreeComponent<K> for FullComponent {
loaded
.set_device_value(FULL, device_indices.narrow(0, offset, n_len).copy());
let locked = loaded.device_lock_ref(FULL) > 0;
let tokens = loaded.device_reclaimable_value_len(FULL);
offset += n_len;
// Full uses leaf sets, not LRU. A value materialized
// under lock is protected; the last release moves it
// to evictable.
if locked {
tree_core.inc_protected_size(FULL, n_len as usize);
tree_core.inc_protected_size(FULL, tokens);
} else {
tree_core.inc_evictable_size(FULL, n_len as usize);
tree_core.inc_evictable_size(FULL, tokens);
}
tree_core.update_evictable_leaf_sets_(loaded_idx);
}
+10 -7
View File
@@ -166,14 +166,17 @@ impl<K: ChildKeyType> TreeComponent<K> for MambaComponent {
.mamba_value
.as_ref()
.expect("mamba insert requires a donated mamba_value");
let slot_len = mamba_value.size()[0] as usize;
if is_new_leaf {
tree_core
.arena
.set_device_value(node_id, MAMBA, mamba_value.shallow_clone());
tree_core.device_lru_list_mut(MAMBA).insert_mru(node_id);
tree_core.inc_evictable_size(MAMBA, slot_len);
let tokens = tree_core
.arena
.node(node_id)
.device_reclaimable_value_len(MAMBA);
tree_core.inc_evictable_size(MAMBA, tokens);
self.emit_excess_path_states_eviction_(tree_core.arena.node(node_id).id, cache_actions);
return;
}
@@ -288,7 +291,7 @@ impl<K: ChildKeyType> TreeComponent<K> for MambaComponent {
// Device layer
if target.contains(EvictLayer::Device) && node.has_device_value(MAMBA) {
freed = node.device_value_len(MAMBA);
freed = node.device_reclaimable_value_len(MAMBA);
device_frees
.entry(ct)
.or_default()
@@ -449,8 +452,8 @@ impl<K: ChildKeyType> TreeComponent<K> for MambaComponent {
}
tree_core.arena.inc_host_lock_ref(node_id, MAMBA);
} else {
let value_len = node.device_value_len(MAMBA);
if node.device_lock_ref(MAMBA) == 0 && has_value {
let value_len = node.device_reclaimable_value_len(MAMBA);
if node.device_lock_ref(MAMBA) == 0 {
tree_core.dec_evictable_size(MAMBA, value_len);
tree_core.inc_protected_size(MAMBA, value_len);
}
@@ -494,8 +497,8 @@ impl<K: ChildKeyType> TreeComponent<K> for MambaComponent {
device_lock_ref > 0,
"Mamba release hit lock_ref=0 on node {node_id}"
);
if device_lock_ref == 1 && node.has_device_value(MAMBA) {
let value_len = node.device_value_len(MAMBA);
if device_lock_ref == 1 {
let value_len = node.device_reclaimable_value_len(MAMBA);
tree_core.inc_evictable_size(MAMBA, value_len);
tree_core.dec_protected_size(MAMBA, value_len);
}
+4 -12
View File
@@ -118,14 +118,6 @@ impl SwaComponent {
}
}
fn value_len<K: ChildKeyType>(node: &Node<K>, host: bool) -> usize {
if host {
node.host_value_len(SWA)
} else {
node.device_value_len(SWA)
}
}
fn swa_uuid<K: ChildKeyType>(node: &Node<K>, host: bool) -> Option<i64> {
if host {
node.swa_host_uuid
@@ -682,7 +674,7 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
.entry(ct)
.or_default()
.push(node.device_value(FULL).shallow_clone());
freed = node.device_value_len(SWA);
freed = node.device_reclaimable_value_len(SWA);
let _ = node.take_device_value(SWA);
tree_core.dec_evictable_size(SWA, freed);
}
@@ -1172,7 +1164,7 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
let parent = node.parent();
let key_len = node.key.atom_len();
let has_value = Self::has_value(node, lock_host);
let value_len = Self::value_len(node, lock_host);
let value_len = node.device_reclaimable_value_len(SWA);
let newly_locked = Self::lock_ref(node, lock_host) == 0;
Self::inc_lock_ref(node, lock_host);
if newly_locked && has_value {
@@ -1225,7 +1217,7 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
if lock_host { "host_" } else { "" }
);
let has_value = Self::has_value(node, lock_host);
let value_len = Self::value_len(node, lock_host);
let value_len = node.device_reclaimable_value_len(SWA);
if lock_ref == 1 && has_value {
if lock_host {
if !node.has_device_value(SWA) {
@@ -1288,7 +1280,7 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
"SWA window release hit lock_ref=0 on node {cur}"
);
let has_value = node.has_device_value(SWA);
let value_len = node.device_value_len(SWA);
let value_len = node.device_reclaimable_value_len(SWA);
node.dec_device_lock_ref(SWA);
let now_unlocked = node.device_lock_ref(SWA) == 0;
if now_unlocked {
+11 -2
View File
@@ -264,11 +264,20 @@ impl<K: ChildKeyType> Node<K> {
self.has_value_(ValueSlotIdx::device(component_type))
}
/// The component's device value length, or 0 when value-less.
/// Logical length of the stored device-index tensor, or 0 when absent.
pub fn device_value_len(&self, component_type: ComponentType) -> usize {
self.value_len_(ValueSlotIdx::device(component_type))
}
/// Allocator slots reclaimed on eviction; roots and absent device values contribute zero.
/// Ordinary backing uses logical length; released/shared backing needs its own accounting.
pub fn device_reclaimable_value_len(&self, component_type: ComponentType) -> usize {
if self.is_root() {
return 0;
}
self.device_value_len(component_type)
}
/// Set the component's device value; panics if already set.
pub fn set_device_value(&mut self, component_type: ComponentType, value: Tensor) {
self.set_value_(ValueSlotIdx::device(component_type), value);
@@ -1489,7 +1498,7 @@ impl<K: ChildKeyType> NodeArena<K> {
self.node(id).has_device_value(component_type)
}
/// The node's device value length for the component, or 0 when value-less.
/// Logical length of the node's stored device-index tensor, or 0 when absent.
pub fn device_value_len(&self, id: NodeIdx_, component_type: ComponentType) -> usize {
self.node(id).device_value_len(component_type)
}
@@ -1512,7 +1512,9 @@ impl<K: ChildKeyType + Send + Sync> TreeCoreBinding<K> {
let made_progress = node_id.is_some()
|| result.mamba_backup_node_id.is_some()
|| result.swa_backup_node_id.is_some()
|| !result.tracker.is_empty();
|| !result.tracker.is_empty()
|| result.device_frees.values().any(|frees| !frees.is_empty())
|| result.host_frees.values().any(|frees| !frees.is_empty());
Ok(EvictDeviceNextNodeResultBinding {
node_id,
mamba_backup_node_id: result.mamba_backup_node_id,
+9
View File
@@ -67,6 +67,7 @@ impl ChildKeyType for CountedKey {
fn evicted_only_for_attached_nodes_without_full_device_value() {
let mut parent: Node<Vec<i64>> = Node::new_root(/* id = */ 0);
assert!(!parent.evicted());
assert_eq!(parent.device_reclaimable_value_len(FULL), 0);
let mut child = Node::new_child(
/* id = */ 1,
/* key = */ vec![7],
@@ -78,8 +79,12 @@ fn evicted_only_for_attached_nodes_without_full_device_value() {
.attach_child(&mut child, /* page_size = */ 1)
.unwrap();
assert!(child.evicted());
assert_eq!(child.device_reclaimable_value_len(FULL), 0);
child.set_device_value(FULL, Tensor::from_slice(&[1i64]));
assert!(!child.evicted());
assert_eq!(child.device_reclaimable_value_len(FULL), 1);
let _ = child.take_device_value(FULL);
assert_eq!(child.device_reclaimable_value_len(FULL), 0);
}
#[test]
@@ -470,6 +475,10 @@ fn present_but_empty_tensor_has_value_true_len_zero() {
node.state_mut_(ValueSlotIdx::device(FULL)).value = Some(Tensor::from_slice(&empty));
assert!(node.has_device_value(FULL));
assert_eq!(node.device_value_len(FULL), 0);
assert_eq!(node.device_reclaimable_value_len(FULL), 0);
node.state_mut_(ValueSlotIdx::device(FULL)).value = Some(Tensor::from_slice(&[1i64]));
assert_eq!(node.device_value_len(FULL), 1);
assert_eq!(node.device_reclaimable_value_len(FULL), 0);
}
#[test]
+17 -15
View File
@@ -676,7 +676,7 @@ pub struct UnifiedTreeCore<K: ChildKeyType> {
impl<K: ChildKeyType> UnifiedTreeCore<K> {
/// Adjust the component's evictable or protected count of reclaimable tokens.
/// The caller updates backing and size together, and restores backing on error.
/// Counters are unchanged on error; the caller keeps backing consistent.
/// This changes counters only; roots and device tombstones are ignored.
pub fn adjust_component_size(
&mut self,
@@ -2311,7 +2311,11 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
displaced.is_none(),
"add_new_node_: parent {parent_id} already has a child on the new node's page"
);
self.inc_evictable_size(FULL, value.size()[0] as usize);
let tokens = self
.arena
.node(new_node_id)
.device_reclaimable_value_len(FULL);
self.inc_evictable_size(FULL, tokens);
if self.enable_storage || self.enable_external_cache_linker {
let hash_values = self.arena.compute_node_hash_values(new_node_id, page_size);
self.arena.node_mut(new_node_id).hash_value = Some(hash_values);
@@ -2333,7 +2337,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
) {
self.arena
.set_device_value(node_id, FULL, fresh_value.copy());
let tokens = fresh_value.size()[0] as usize;
let tokens = self.arena.node(node_id).device_reclaimable_value_len(FULL);
// A value materialized under lock is protected; the last release
// moves it to evictable.
if self.arena.device_lock_ref(node_id, FULL) > 0 {
@@ -4675,8 +4679,11 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
component_type != BASE_COMPONENT_TYPE,
"set_component_device_value: auxiliary components only"
);
let tokens = value.size()[0] as usize;
self.arena.set_device_value(node_id, component_type, value);
let tokens = self
.arena
.node(node_id)
.device_reclaimable_value_len(component_type);
let host_lru = self.host_lru_list_mut(component_type);
if host_lru.in_list(Some(node_id)) {
host_lru.remove_node(node_id);
@@ -5136,17 +5143,12 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
let mut evictable = 0usize;
let mut protected = 0usize;
for &node_id in &all_nodes {
if self.arena.node(node_id).is_root() {
continue;
}
let state = &self.arena.node(node_id).values[ct.idx()];
if let Some(value) = &state.value {
let tokens = value.size()[0] as usize;
if state.lock_ref > 0 {
protected += tokens;
} else {
evictable += tokens;
}
let node = self.arena.node(node_id);
let tokens = node.device_reclaimable_value_len(ct);
if node.device_lock_ref(ct) > 0 {
protected += tokens;
} else {
evictable += tokens;
}
}
let recorded = self.component_state(ct);
@@ -1105,6 +1105,8 @@ def test_host_only_and_root_adjustments_are_noops(backend, is_eagle):
for node in (root, host):
for delta in (-10, 0, 10):
core.adjust_component_size(node, ComponentType.FULL, delta)
with pytest.raises(ValueError):
core.adjust_component_size(node, ComponentType.SWA, 0)
assert (core.full_evictable_size(), core.full_protected_size()) == (0, 0)
assert core.collect_path_info([host])[0][0].full_value_len is None
@@ -1129,6 +1131,7 @@ def test_failed_adjustments_preserve_accounting_and_tree(backend, is_eagle, lock
(invalid_node, ComponentType.FULL, 0, KeyError),
(leaf, ComponentType.SWA, 0, ValueError),
(leaf, 99, 0, ValueError),
(leaf, ComponentType.FULL, -len(key) - 1, ValueError),
):
with pytest.raises(error):
core.adjust_component_size(node, component, delta)
@@ -1154,29 +1157,55 @@ def test_adjustment_follows_real_backing_changes_across_lock_transitions():
from sglang.srt.mem_cache.unified_cache.unified_tree_core import UnifiedTreeCore
class SideBackingFullComponent(FullComponent):
def reclaimable_tokens(self, node):
backing = node.component_data[ComponentType.FULL].metadata.get(
"side_backing"
)
return super().reclaimable_tokens(node) + (
0 if backing is None else backing.numel()
)
def reclaimable_value_len(self, node):
metadata = node.component_data[ComponentType.FULL].metadata
tokens = super().reclaimable_value_len(node)
if "raw_backing" in metadata:
tokens = metadata["raw_backing"].numel()
side_backing = metadata.get("side_backing")
return tokens + (0 if side_backing is None else side_backing.numel())
params = CacheInitParams(
False, None, None, 1, tree_components=(ComponentType.FULL,)
)
cache = SimpleNamespace(enable_session_radix_cache=False)
core = UnifiedTreeCore(
params, {ComponentType.FULL: SideBackingFullComponent(cache, params)}
)
driver = SideBackingFullComponent(cache, params)
core = UnifiedTreeCore(params, {ComponentType.FULL: driver})
leaf = _insert(core, [1, 2, 3, 4], [10, 11, 12, 13]).last_device_node
metadata = core.node_by_id(leaf).component_data[ComponentType.FULL].metadata
node = core.node_by_id(leaf)
metadata = node.component_data[ComponentType.FULL].metadata
metadata["side_backing"] = torch.tensor([100, 101])
core.adjust_component_size(leaf, ComponentType.FULL, 2)
assert (core.full_evictable_size(), core.full_protected_size()) == (6, 0)
assert driver.value_len(node) == 4
assert driver.reclaimable_value_len(node) == 6
assert core.collect_path_info([leaf])[0][0].full_value_len == 4
matched = core.match_prefix(MatchPrefixParams(key=_key([1, 2, 3, 4])))
assert matched.device_prefix_len == 4
assert _path_indices(core, matched.last_device_node).tolist() == [10, 11, 12, 13]
lock = core.inc_lock_ref(leaf)
assert (core.full_evictable_size(), core.full_protected_size()) == (0, 6)
# A locked split follows logical indices; the tail keeps the side backing.
prefix_match = core.match_prefix(MatchPrefixParams(key=_key([1, 2])))
prefix = prefix_match.last_device_node
prefix_node = core.node_by_id(prefix)
assert prefix_match.device_prefix_len == 2
assert _path_indices(core, prefix).tolist() == [10, 11]
assert core.get_component_device_value(leaf, ComponentType.FULL).tolist() == [
12,
13,
]
assert (driver.value_len(prefix_node), driver.value_len(node)) == (2, 2)
assert (
driver.reclaimable_value_len(prefix_node),
driver.reclaimable_value_len(node),
) == (2, 4)
assert [
(span.path_start, span.path_end, span.full_value_len)
for span in core.collect_path_info([leaf])[0]
] == [(0, 2, 2), (2, 4, 2)]
assert (core.full_evictable_size(), core.full_protected_size()) == (0, 6)
metadata["side_backing"] = metadata["side_backing"][:1]
core.adjust_component_size(leaf, ComponentType.FULL, -1)
assert (core.full_evictable_size(), core.full_protected_size()) == (0, 5)
@@ -1185,9 +1214,27 @@ def test_adjustment_follows_real_backing_changes_across_lock_transitions():
core.dec_lock_ref(leaf, lock.to_dec_params())
assert (core.full_evictable_size(), core.full_protected_size()) == (4, 0)
core.sanity_check([], [])
for fragment in (prefix_node, node):
fragment.component_data[ComponentType.FULL].metadata["raw_backing"] = (
torch.empty(0, dtype=torch.int64)
)
core.adjust_component_size(fragment.id, ComponentType.FULL, -2)
assert driver.value_len(fragment) == 2
assert driver.reclaimable_value_len(fragment) == 0
assert (core.full_evictable_size(), core.full_protected_size()) == (0, 0)
assert [span.full_value_len for span in core.collect_path_info([leaf])[0]] == [2, 2]
matched = core.match_prefix(MatchPrefixParams(key=_key([1, 2, 3, 4])))
assert matched.device_prefix_len == 4
assert _path_indices(core, matched.last_device_node).tolist() == [10, 11, 12, 13]
core.sanity_check([], [])
tracker = {}
_accumulate_step(core.evict_device_leaf(leaf, False), tracker, {}, {})
assert tracker[ComponentType.FULL] == 4
device_frees = {}
_accumulate_step(core.evict_device_leaf(leaf, False), tracker, device_frees, {})
assert torch.cat(device_frees[ComponentType.FULL]).tolist() == [12, 13]
_accumulate_step(core.evict_device_leaf(prefix, False), tracker, device_frees, {})
assert tracker[ComponentType.FULL] == 0
assert torch.cat(device_frees[ComponentType.FULL]).tolist() == [12, 13, 10, 11]
assert (core.full_evictable_size(), core.full_protected_size()) == (0, 0)
@@ -2692,6 +2739,7 @@ def _swa_cache(
backend="rust",
ring=False,
request_window=False,
component_registry_override=None,
):
"""A real UnifiedRadixCache and SWA allocator, defaulting to the Rust core."""
from sglang.srt.mem_cache.allocator.swa import SWATokenToKVPoolAllocator
@@ -2742,6 +2790,7 @@ def _swa_cache(
sliding_window_size=window,
swa_gates_prefix_match=not (ring or request_window),
tree_components=(ComponentType.FULL, ComponentType.SWA),
component_registry_override=component_registry_override,
)
)
return cache, allocator
@@ -2758,7 +2807,9 @@ def _complete_backup(core, node, full_offset=1000, aux_offset=1000):
return full + full_offset, aux
def _swa_mamba_cache(backend, window, enable_session=False):
def _swa_mamba_cache(
backend, window, enable_session=False, component_registry_override=None
):
from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
@@ -2779,6 +2830,7 @@ def _swa_mamba_cache(backend, window, enable_session=False):
page_size=1,
sliding_window_size=window,
enable_session_radix_cache=enable_session,
component_registry_override=component_registry_override,
tree_components=(
ComponentType.FULL,
ComponentType.SWA,
@@ -2796,11 +2848,17 @@ def _internal_swa_write_back_case(
with_mamba=False,
prefix_nodes=1,
enable_session=False,
component_registry_override=None,
):
cache, allocator = (
_swa_mamba_cache(backend, window, enable_session)
_swa_mamba_cache(backend, window, enable_session, component_registry_override)
if with_mamba
else _swa_cache(window=window, page_size=page_size, backend=backend)
else _swa_cache(
window=window,
page_size=page_size,
backend=backend,
component_registry_override=component_registry_override,
)
)
assert cache._tree_core_backend == backend
segment = 2 * page_size
@@ -3140,6 +3198,121 @@ def _assert_internal_eviction_pending(step, component, node, swa_tokens=0):
assert step.unbacked_tokens == 0
@pytest.mark.parametrize("component", [ComponentType.MAMBA, ComponentType.SWA])
@pytest.mark.parametrize("early_release", [False, True], ids=["final", "early-swa"])
def test_python_component_locks_use_reclaimable_backing(component, early_release):
from sglang.srt.mem_cache.unified_cache.components.mamba import MambaComponent
from sglang.srt.mem_cache.unified_cache.components.swa import SWAComponent
base = MambaComponent if component == ComponentType.MAMBA else SWAComponent
class SideBackingComponent(base):
def reclaimable_value_len(self, node):
backing = node.component_data[self.component_type].metadata.get("backing")
if backing is not None:
return backing.numel()
return super().reclaimable_value_len(node)
case = _internal_swa_write_back_case(
"python",
with_mamba=component == ComponentType.MAMBA,
component_registry_override={component: SideBackingComponent},
)
cache, core = case.cache, case.core
node_id, leaf = case.nodes
node = core.node_by_id(node_id)
state = node.component_data[component]
logical = state.value.clone()
state.metadata["backing"] = torch.arange(len(logical) + 2)
core.adjust_component_size(node_id, component, 2)
def ledgers():
return (
core.component_evictable_size_[component],
core.component_protected_size_[component],
)
before = ledgers()
reclaimed = len(logical) + 2
lock = core.inc_lock_ref(node_id).to_dec_params()
assert state.lock_ref == 1
assert ledgers() == (before[0] - reclaimed, before[1] + reclaimed)
assert cache.components[component].value_len(node) == len(logical)
assert torch.equal(state.value, logical)
# The final release uses current backing, not the original lock delta.
state.metadata["backing"] = state.metadata["backing"][:-1]
core.adjust_component_size(node_id, component, -1)
if early_release:
released = core.dec_swa_lock_only(node_id, lock)
assert not released.device_frees and not released.host_frees
core.dec_lock_ref(node_id, lock, skip_swa=True)
else:
core.dec_lock_ref(node_id, lock)
assert state.lock_ref == 0
assert ledgers() == (before[0] - 1, before[1])
assert torch.equal(state.value, logical)
cache.dec_lock_ref(leaf, case.leaf_lock)
core.sanity_check([], [])
@pytest.mark.parametrize("component", [ComponentType.MAMBA, ComponentType.SWA])
def test_python_zero_accounted_internal_eviction_continues(component):
from sglang.srt.mem_cache.unified_cache.components.mamba import MambaComponent
from sglang.srt.mem_cache.unified_cache.components.swa import SWAComponent
base = MambaComponent if component == ComponentType.MAMBA else SWAComponent
class ReleasedBackingComponent(base):
def reclaimable_value_len(self, node):
if node.component_data[self.component_type].metadata.get("released"):
return 0
return super().reclaimable_value_len(node)
case = _internal_swa_write_back_case(
"python",
with_mamba=component == ComponentType.MAMBA,
prefix_nodes=2,
component_registry_override={component: ReleasedBackingComponent},
)
cache, core = case.cache, case.core
first, second, leaf = case.nodes
cache.is_write_back = False
driver = cache.components[component]
expected_values = [
core.get_component_device_value(node, component).clone()
for node in (first, second)
]
first_size = driver.reclaimable_value_len(core.node_by_id(first))
expected = driver.reclaimable_value_len(core.node_by_id(second))
core.node_by_id(first).component_data[component].metadata["released"] = True
core.adjust_component_size(first, component, -first_size)
assert driver.value_len(core.node_by_id(first)) == len(expected_values[0]) > 0
assert driver.reclaimable_value_len(core.node_by_id(first)) == 0
tracker, freed = {}, {}
core.evict_device_start(component, expected)
try:
first_step = core.evict_device_next_node(component, tracker)
assert first_step.node_id is None and first_step.made_progress
assert not first_step.tracker and first_step.device_frees[component]
_accumulate_step(first_step, tracker, freed, {})
assert core.get_component_device_value(first, component) is None
assert torch.equal(torch.cat(freed[component]), expected_values[0])
second_step = core.evict_device_next_node(component, tracker)
assert second_step.node_id is None and second_step.made_progress
_accumulate_step(second_step, tracker, freed, {})
assert tracker[component] == expected
assert core.get_component_device_value(second, component) is None
assert len(freed[component]) == 2
assert torch.equal(torch.cat(freed[component]), torch.cat(expected_values))
assert not core.evict_device_next_node(component, tracker).made_progress
finally:
core.evict_device_end(component)
cache.dec_lock_ref(leaf, case.leaf_lock)
core.sanity_check([], [])
@pytest.mark.parametrize("component", [ComponentType.MAMBA, ComponentType.SWA])
def test_python_component_returns_immutable_internal_backup_request(component):
from dataclasses import FrozenInstanceError
@@ -122,7 +122,7 @@ class TestUnifiedRadixComponentLedgerUnits(CustomTestCase):
prefill admission adds the evictable ledger to the allocator's free
space and the eviction walk counts them toward its request. So every
ledger site (insert, lock, release, an in-place change, eviction)
reads ``TreeComponent.reclaimable_tokens`` rather than the value
reads ``TreeComponent.reclaimable_value_len`` rather than the value
length. A site that goes back to ``len(value)`` drifts the ledger from
the recount in ``sanity_check`` for a component that overrides it."""
cache, allocator, _ = build_fixture(