[Fix] radix-cache: Separate logical and reclaimable value lengths in tree components (#45238)
Co-authored-by: metamergebot <metamergebot@users.noreply.github.com> Co-authored-by: Jialin <Jialin@users.noreply.github.com>
This commit is contained in:
co-authored by
metamergebot
Jialin
parent
468e26f253
commit
0b536ca112
@@ -368,10 +368,11 @@ class TreeComponent(ABC):
|
||||
return cd.host_value is not None
|
||||
|
||||
def value_len(self, node: UnifiedTreeNode) -> int:
|
||||
"""Length of the stored device indices, including entries without backing."""
|
||||
value = node.component_data[self.component_type].value
|
||||
return len(value) if value is not None else 0
|
||||
|
||||
def reclaimable_tokens(self, node: UnifiedTreeNode) -> int:
|
||||
def reclaimable_value_len(self, node: UnifiedTreeNode) -> int:
|
||||
"""Pool tokens the allocator gets back when this node's device value
|
||||
is evicted. The evictable/protected ledgers count these (prefill
|
||||
admission adds the evictable ledger to the allocator's free space, and
|
||||
|
||||
@@ -181,8 +181,8 @@ class FullComponent(TreeComponent):
|
||||
if EvictLayer.DEVICE in target and cd.value is not None:
|
||||
device_frees[self.component_type].append(cd.value)
|
||||
# Progress toward an eviction request and the ledger move are
|
||||
# both in allocator units (see reclaimable_tokens).
|
||||
freed = self.reclaimable_tokens(node)
|
||||
# both in allocator units (see reclaimable_value_len).
|
||||
freed = self.reclaimable_value_len(node)
|
||||
self.tree_core.component_evictable_size_[self.component_type] -= freed
|
||||
# NOTE: cd.value = None is deferred to _cascade_evict (Full as trigger)
|
||||
# because SWA's free_swa still needs to read Full.value.
|
||||
@@ -310,7 +310,7 @@ class FullComponent(TreeComponent):
|
||||
f"FULL invariant broken: evicted ancestor {cur.id} above device-on segment"
|
||||
)
|
||||
if cd.lock_ref == 0:
|
||||
key_len = self.reclaimable_tokens(cur)
|
||||
key_len = self.reclaimable_value_len(cur)
|
||||
self.tree_core.component_evictable_size_[ct] -= key_len
|
||||
self.tree_core.component_protected_size_[ct] += key_len
|
||||
delta += key_len
|
||||
@@ -359,7 +359,7 @@ class FullComponent(TreeComponent):
|
||||
f"FULL segment release hit lock_ref=0 on node {cur.id}"
|
||||
)
|
||||
if cd.lock_ref == 1 and cd.value is not None:
|
||||
key_len = self.reclaimable_tokens(cur)
|
||||
key_len = self.reclaimable_value_len(cur)
|
||||
self.tree_core.component_evictable_size_[ct] += key_len
|
||||
self.tree_core.component_protected_size_[ct] -= key_len
|
||||
cd.lock_ref -= 1
|
||||
@@ -453,7 +453,7 @@ class FullComponent(TreeComponent):
|
||||
offset += n_len
|
||||
# Full uses leaf sets, not LRU. A value materialized under
|
||||
# lock is protected; the last release moves it to evictable.
|
||||
ledger = self.reclaimable_tokens(n)
|
||||
ledger = self.reclaimable_value_len(n)
|
||||
if cd.lock_ref > 0:
|
||||
self.tree_core.component_protected_size_[ct] += ledger
|
||||
else:
|
||||
|
||||
@@ -234,8 +234,8 @@ class MambaComponent(TreeComponent):
|
||||
if is_new_leaf:
|
||||
node.component_data[self.component_type].value = params.mamba_value
|
||||
self.tree_core.lru_lists[self.component_type].insert_mru(node)
|
||||
self.tree_core.component_evictable_size_[self.component_type] += len(
|
||||
params.mamba_value
|
||||
self.tree_core.component_evictable_size_[self.component_type] += (
|
||||
self.reclaimable_value_len(node)
|
||||
)
|
||||
self._emit_excess_path_states_eviction(node, cache_actions)
|
||||
return
|
||||
@@ -334,7 +334,7 @@ class MambaComponent(TreeComponent):
|
||||
# Device layer
|
||||
if EvictLayer.DEVICE in target and cd.value is not None:
|
||||
device_frees[self.component_type].append(cd.value)
|
||||
freed = len(cd.value)
|
||||
freed = self.reclaimable_value_len(node)
|
||||
self.tree_core.component_evictable_size_[self.component_type] -= freed
|
||||
cd.value = None
|
||||
|
||||
@@ -465,7 +465,7 @@ class MambaComponent(TreeComponent):
|
||||
cd.host_lock_ref += 1
|
||||
else:
|
||||
if cd.lock_ref == 0 and value is not None:
|
||||
vlen = len(value)
|
||||
vlen = self.reclaimable_value_len(node)
|
||||
self.tree_core.component_evictable_size_[ct] -= vlen
|
||||
self.tree_core.component_protected_size_[ct] += vlen
|
||||
cd.lock_ref += 1
|
||||
@@ -498,7 +498,7 @@ class MambaComponent(TreeComponent):
|
||||
|
||||
assert cd.lock_ref > 0, f"Mamba release hit lock_ref=0 on node {node.id}"
|
||||
if cd.lock_ref == 1 and value is not None:
|
||||
vlen = len(value)
|
||||
vlen = self.reclaimable_value_len(node)
|
||||
self.tree_core.component_evictable_size_[ct] += vlen
|
||||
self.tree_core.component_protected_size_[ct] -= vlen
|
||||
cd.lock_ref -= 1
|
||||
|
||||
@@ -701,7 +701,7 @@ class SWAComponent(TreeComponent):
|
||||
device_frees[self.component_type].append(
|
||||
node.component_data[BASE_COMPONENT_TYPE].value
|
||||
)
|
||||
freed = len(cd.value)
|
||||
freed = self.reclaimable_value_len(node)
|
||||
self.tree_core.component_evictable_size_[ct] -= freed
|
||||
cd.value = None
|
||||
|
||||
@@ -866,7 +866,7 @@ class SWAComponent(TreeComponent):
|
||||
if lru.in_list(cur):
|
||||
lru.remove_node(cur)
|
||||
else:
|
||||
key_len = len(value)
|
||||
key_len = self.reclaimable_value_len(cur)
|
||||
self.tree_core.component_evictable_size_[ct] -= key_len
|
||||
self.tree_core.component_protected_size_[ct] += key_len
|
||||
if lock_host:
|
||||
@@ -915,7 +915,7 @@ class SWAComponent(TreeComponent):
|
||||
if not host_lru.in_list(cur):
|
||||
host_lru.insert_mru(cur)
|
||||
else:
|
||||
key_len = len(value)
|
||||
key_len = self.reclaimable_value_len(cur)
|
||||
self.tree_core.component_evictable_size_[ct] += key_len
|
||||
self.tree_core.component_protected_size_[ct] -= key_len
|
||||
if lock_host:
|
||||
@@ -960,7 +960,7 @@ class SWAComponent(TreeComponent):
|
||||
if cd.lock_ref == 0:
|
||||
self.tree_core._update_evictable_leaf_sets(cur)
|
||||
if cd.lock_ref == 0 and cd.value is not None:
|
||||
value_len = len(cd.value)
|
||||
value_len = self.reclaimable_value_len(cur)
|
||||
self.tree_core.component_protected_size_[ct] -= value_len
|
||||
self.tree_core.component_evictable_size_[ct] += value_len
|
||||
if self.tree_core._is_device_leaf(cur):
|
||||
|
||||
@@ -1529,7 +1529,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
parent.children[key.child_key(self.page_size)] = new_node
|
||||
self.component_evictable_size_[BASE_COMPONENT_TYPE] += self.components_by_type[
|
||||
BASE_COMPONENT_TYPE
|
||||
].reclaimable_tokens(new_node)
|
||||
].reclaimable_value_len(new_node)
|
||||
if self.enable_storage or self.enable_external_cache_linker:
|
||||
new_node.hash_value = compute_node_hash_values(new_node, self.page_size)
|
||||
|
||||
@@ -1550,7 +1550,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
cd = node.component_data[ct]
|
||||
assert cd.value is None
|
||||
cd.value = fresh_value.clone()
|
||||
n = self.components_by_type[ct].reclaimable_tokens(node)
|
||||
n = self.components_by_type[ct].reclaimable_value_len(node)
|
||||
if cd.lock_ref > 0:
|
||||
self.component_protected_size_[ct] += n
|
||||
else:
|
||||
@@ -1675,6 +1675,8 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
or result.mamba_backup_node_id is not None
|
||||
or result.swa_backup_node_id is not None
|
||||
or bool(result.tracker)
|
||||
or any(result.device_frees.values())
|
||||
or any(result.host_frees.values())
|
||||
)
|
||||
return result
|
||||
|
||||
@@ -2887,7 +2889,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
self.lru_lists[component_type].insert_mru(node)
|
||||
# A value materialized under lock is protected; the last release
|
||||
# moves it to evictable.
|
||||
ledger = self.components_by_type[component_type].reclaimable_tokens(node)
|
||||
ledger = self.components_by_type[component_type].reclaimable_value_len(node)
|
||||
if cd.lock_ref > 0:
|
||||
self.component_protected_size_[component_type] += ledger
|
||||
else:
|
||||
@@ -2902,10 +2904,15 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
cd = node.component_data[component_type]
|
||||
if node is self.root_node or cd.value is None:
|
||||
return
|
||||
if cd.lock_ref > 0:
|
||||
self.component_protected_size_[component_type] += delta
|
||||
else:
|
||||
self.component_evictable_size_[component_type] += delta
|
||||
sizes = (
|
||||
self.component_protected_size_
|
||||
if cd.lock_ref > 0
|
||||
else self.component_evictable_size_
|
||||
)
|
||||
updated = sizes[component_type] + delta
|
||||
if updated < 0:
|
||||
raise ValueError(f"component size cannot be negative: {updated}")
|
||||
sizes[component_type] = updated
|
||||
|
||||
def get_component_device_value(
|
||||
self, node_id: NodeId, component_type: ComponentType
|
||||
@@ -3126,7 +3133,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
continue
|
||||
cd = n.component_data[ct]
|
||||
if cd.value is not None:
|
||||
toks = self.components_by_type[ct].reclaimable_tokens(n)
|
||||
toks = self.components_by_type[ct].reclaimable_value_len(n)
|
||||
if cd.lock_ref > 0:
|
||||
protected += toks
|
||||
else:
|
||||
|
||||
@@ -431,6 +431,8 @@ class UnifiedTreeCoreInterface(ABC):
|
||||
|
||||
Roots and nodes without device values are no-ops. Missing handles raise
|
||||
KeyError; disabled components raise ValueError.
|
||||
Negative totals raise ValueError without updating the count. Rust also
|
||||
rejects fixed-width overflow; Python integers are unbounded.
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
@@ -89,7 +89,7 @@ class PagedFullComponent(FullComponent):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.rows_handed_back: dict[int, int] = {}
|
||||
|
||||
def reclaimable_tokens(self, node):
|
||||
def reclaimable_value_len(self, node):
|
||||
return self.value_len(node) // 2 - self.rows_handed_back.get(node.id, 0)
|
||||
|
||||
|
||||
|
||||
@@ -111,7 +111,7 @@ impl<K: ChildKeyType> TreeComponent<K> for FullComponent {
|
||||
let mut host_freed = 0;
|
||||
if target.contains(EvictLayer::Device) && node.has_device_value(FULL) {
|
||||
let value = node.device_value(FULL);
|
||||
freed = node.device_value_len(FULL);
|
||||
freed = node.device_reclaimable_value_len(FULL);
|
||||
device_frees
|
||||
.entry(FULL)
|
||||
.or_default()
|
||||
@@ -320,7 +320,7 @@ impl<K: ChildKeyType> TreeComponent<K> for FullComponent {
|
||||
);
|
||||
let parent = node.parent();
|
||||
let newly_locked_len = if node.device_lock_ref(FULL) == 0 {
|
||||
Some(node.device_value_len(FULL))
|
||||
Some(node.device_reclaimable_value_len(FULL))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -397,8 +397,8 @@ impl<K: ChildKeyType> TreeComponent<K> for FullComponent {
|
||||
old_lock_ref > 0,
|
||||
"FULL segment release hit lock_ref=0 on node {cur}"
|
||||
);
|
||||
let newly_unlocked_len = if old_lock_ref == 1 && node.has_device_value(FULL) {
|
||||
Some(node.device_value_len(FULL))
|
||||
let newly_unlocked_len = if old_lock_ref == 1 {
|
||||
Some(node.device_reclaimable_value_len(FULL))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -514,14 +514,15 @@ impl<K: ChildKeyType> TreeComponent<K> for FullComponent {
|
||||
loaded
|
||||
.set_device_value(FULL, device_indices.narrow(0, offset, n_len).copy());
|
||||
let locked = loaded.device_lock_ref(FULL) > 0;
|
||||
let tokens = loaded.device_reclaimable_value_len(FULL);
|
||||
offset += n_len;
|
||||
// Full uses leaf sets, not LRU. A value materialized
|
||||
// under lock is protected; the last release moves it
|
||||
// to evictable.
|
||||
if locked {
|
||||
tree_core.inc_protected_size(FULL, n_len as usize);
|
||||
tree_core.inc_protected_size(FULL, tokens);
|
||||
} else {
|
||||
tree_core.inc_evictable_size(FULL, n_len as usize);
|
||||
tree_core.inc_evictable_size(FULL, tokens);
|
||||
}
|
||||
tree_core.update_evictable_leaf_sets_(loaded_idx);
|
||||
}
|
||||
|
||||
@@ -166,14 +166,17 @@ impl<K: ChildKeyType> TreeComponent<K> for MambaComponent {
|
||||
.mamba_value
|
||||
.as_ref()
|
||||
.expect("mamba insert requires a donated mamba_value");
|
||||
let slot_len = mamba_value.size()[0] as usize;
|
||||
|
||||
if is_new_leaf {
|
||||
tree_core
|
||||
.arena
|
||||
.set_device_value(node_id, MAMBA, mamba_value.shallow_clone());
|
||||
tree_core.device_lru_list_mut(MAMBA).insert_mru(node_id);
|
||||
tree_core.inc_evictable_size(MAMBA, slot_len);
|
||||
let tokens = tree_core
|
||||
.arena
|
||||
.node(node_id)
|
||||
.device_reclaimable_value_len(MAMBA);
|
||||
tree_core.inc_evictable_size(MAMBA, tokens);
|
||||
self.emit_excess_path_states_eviction_(tree_core.arena.node(node_id).id, cache_actions);
|
||||
return;
|
||||
}
|
||||
@@ -288,7 +291,7 @@ impl<K: ChildKeyType> TreeComponent<K> for MambaComponent {
|
||||
|
||||
// Device layer
|
||||
if target.contains(EvictLayer::Device) && node.has_device_value(MAMBA) {
|
||||
freed = node.device_value_len(MAMBA);
|
||||
freed = node.device_reclaimable_value_len(MAMBA);
|
||||
device_frees
|
||||
.entry(ct)
|
||||
.or_default()
|
||||
@@ -449,8 +452,8 @@ impl<K: ChildKeyType> TreeComponent<K> for MambaComponent {
|
||||
}
|
||||
tree_core.arena.inc_host_lock_ref(node_id, MAMBA);
|
||||
} else {
|
||||
let value_len = node.device_value_len(MAMBA);
|
||||
if node.device_lock_ref(MAMBA) == 0 && has_value {
|
||||
let value_len = node.device_reclaimable_value_len(MAMBA);
|
||||
if node.device_lock_ref(MAMBA) == 0 {
|
||||
tree_core.dec_evictable_size(MAMBA, value_len);
|
||||
tree_core.inc_protected_size(MAMBA, value_len);
|
||||
}
|
||||
@@ -494,8 +497,8 @@ impl<K: ChildKeyType> TreeComponent<K> for MambaComponent {
|
||||
device_lock_ref > 0,
|
||||
"Mamba release hit lock_ref=0 on node {node_id}"
|
||||
);
|
||||
if device_lock_ref == 1 && node.has_device_value(MAMBA) {
|
||||
let value_len = node.device_value_len(MAMBA);
|
||||
if device_lock_ref == 1 {
|
||||
let value_len = node.device_reclaimable_value_len(MAMBA);
|
||||
tree_core.inc_evictable_size(MAMBA, value_len);
|
||||
tree_core.dec_protected_size(MAMBA, value_len);
|
||||
}
|
||||
|
||||
@@ -118,14 +118,6 @@ impl SwaComponent {
|
||||
}
|
||||
}
|
||||
|
||||
fn value_len<K: ChildKeyType>(node: &Node<K>, host: bool) -> usize {
|
||||
if host {
|
||||
node.host_value_len(SWA)
|
||||
} else {
|
||||
node.device_value_len(SWA)
|
||||
}
|
||||
}
|
||||
|
||||
fn swa_uuid<K: ChildKeyType>(node: &Node<K>, host: bool) -> Option<i64> {
|
||||
if host {
|
||||
node.swa_host_uuid
|
||||
@@ -682,7 +674,7 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
|
||||
.entry(ct)
|
||||
.or_default()
|
||||
.push(node.device_value(FULL).shallow_clone());
|
||||
freed = node.device_value_len(SWA);
|
||||
freed = node.device_reclaimable_value_len(SWA);
|
||||
let _ = node.take_device_value(SWA);
|
||||
tree_core.dec_evictable_size(SWA, freed);
|
||||
}
|
||||
@@ -1172,7 +1164,7 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
|
||||
let parent = node.parent();
|
||||
let key_len = node.key.atom_len();
|
||||
let has_value = Self::has_value(node, lock_host);
|
||||
let value_len = Self::value_len(node, lock_host);
|
||||
let value_len = node.device_reclaimable_value_len(SWA);
|
||||
let newly_locked = Self::lock_ref(node, lock_host) == 0;
|
||||
Self::inc_lock_ref(node, lock_host);
|
||||
if newly_locked && has_value {
|
||||
@@ -1225,7 +1217,7 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
|
||||
if lock_host { "host_" } else { "" }
|
||||
);
|
||||
let has_value = Self::has_value(node, lock_host);
|
||||
let value_len = Self::value_len(node, lock_host);
|
||||
let value_len = node.device_reclaimable_value_len(SWA);
|
||||
if lock_ref == 1 && has_value {
|
||||
if lock_host {
|
||||
if !node.has_device_value(SWA) {
|
||||
@@ -1288,7 +1280,7 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
|
||||
"SWA window release hit lock_ref=0 on node {cur}"
|
||||
);
|
||||
let has_value = node.has_device_value(SWA);
|
||||
let value_len = node.device_value_len(SWA);
|
||||
let value_len = node.device_reclaimable_value_len(SWA);
|
||||
node.dec_device_lock_ref(SWA);
|
||||
let now_unlocked = node.device_lock_ref(SWA) == 0;
|
||||
if now_unlocked {
|
||||
|
||||
@@ -264,11 +264,20 @@ impl<K: ChildKeyType> Node<K> {
|
||||
self.has_value_(ValueSlotIdx::device(component_type))
|
||||
}
|
||||
|
||||
/// The component's device value length, or 0 when value-less.
|
||||
/// Logical length of the stored device-index tensor, or 0 when absent.
|
||||
pub fn device_value_len(&self, component_type: ComponentType) -> usize {
|
||||
self.value_len_(ValueSlotIdx::device(component_type))
|
||||
}
|
||||
|
||||
/// Allocator slots reclaimed on eviction; roots and absent device values contribute zero.
|
||||
/// Ordinary backing uses logical length; released/shared backing needs its own accounting.
|
||||
pub fn device_reclaimable_value_len(&self, component_type: ComponentType) -> usize {
|
||||
if self.is_root() {
|
||||
return 0;
|
||||
}
|
||||
self.device_value_len(component_type)
|
||||
}
|
||||
|
||||
/// Set the component's device value; panics if already set.
|
||||
pub fn set_device_value(&mut self, component_type: ComponentType, value: Tensor) {
|
||||
self.set_value_(ValueSlotIdx::device(component_type), value);
|
||||
@@ -1489,7 +1498,7 @@ impl<K: ChildKeyType> NodeArena<K> {
|
||||
self.node(id).has_device_value(component_type)
|
||||
}
|
||||
|
||||
/// The node's device value length for the component, or 0 when value-less.
|
||||
/// Logical length of the node's stored device-index tensor, or 0 when absent.
|
||||
pub fn device_value_len(&self, id: NodeIdx_, component_type: ComponentType) -> usize {
|
||||
self.node(id).device_value_len(component_type)
|
||||
}
|
||||
|
||||
@@ -1512,7 +1512,9 @@ impl<K: ChildKeyType + Send + Sync> TreeCoreBinding<K> {
|
||||
let made_progress = node_id.is_some()
|
||||
|| result.mamba_backup_node_id.is_some()
|
||||
|| result.swa_backup_node_id.is_some()
|
||||
|| !result.tracker.is_empty();
|
||||
|| !result.tracker.is_empty()
|
||||
|| result.device_frees.values().any(|frees| !frees.is_empty())
|
||||
|| result.host_frees.values().any(|frees| !frees.is_empty());
|
||||
Ok(EvictDeviceNextNodeResultBinding {
|
||||
node_id,
|
||||
mamba_backup_node_id: result.mamba_backup_node_id,
|
||||
|
||||
@@ -67,6 +67,7 @@ impl ChildKeyType for CountedKey {
|
||||
fn evicted_only_for_attached_nodes_without_full_device_value() {
|
||||
let mut parent: Node<Vec<i64>> = Node::new_root(/* id = */ 0);
|
||||
assert!(!parent.evicted());
|
||||
assert_eq!(parent.device_reclaimable_value_len(FULL), 0);
|
||||
let mut child = Node::new_child(
|
||||
/* id = */ 1,
|
||||
/* key = */ vec![7],
|
||||
@@ -78,8 +79,12 @@ fn evicted_only_for_attached_nodes_without_full_device_value() {
|
||||
.attach_child(&mut child, /* page_size = */ 1)
|
||||
.unwrap();
|
||||
assert!(child.evicted());
|
||||
assert_eq!(child.device_reclaimable_value_len(FULL), 0);
|
||||
child.set_device_value(FULL, Tensor::from_slice(&[1i64]));
|
||||
assert!(!child.evicted());
|
||||
assert_eq!(child.device_reclaimable_value_len(FULL), 1);
|
||||
let _ = child.take_device_value(FULL);
|
||||
assert_eq!(child.device_reclaimable_value_len(FULL), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -470,6 +475,10 @@ fn present_but_empty_tensor_has_value_true_len_zero() {
|
||||
node.state_mut_(ValueSlotIdx::device(FULL)).value = Some(Tensor::from_slice(&empty));
|
||||
assert!(node.has_device_value(FULL));
|
||||
assert_eq!(node.device_value_len(FULL), 0);
|
||||
assert_eq!(node.device_reclaimable_value_len(FULL), 0);
|
||||
node.state_mut_(ValueSlotIdx::device(FULL)).value = Some(Tensor::from_slice(&[1i64]));
|
||||
assert_eq!(node.device_value_len(FULL), 1);
|
||||
assert_eq!(node.device_reclaimable_value_len(FULL), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -676,7 +676,7 @@ pub struct UnifiedTreeCore<K: ChildKeyType> {
|
||||
|
||||
impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
/// Adjust the component's evictable or protected count of reclaimable tokens.
|
||||
/// The caller updates backing and size together, and restores backing on error.
|
||||
/// Counters are unchanged on error; the caller keeps backing consistent.
|
||||
/// This changes counters only; roots and device tombstones are ignored.
|
||||
pub fn adjust_component_size(
|
||||
&mut self,
|
||||
@@ -2311,7 +2311,11 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
displaced.is_none(),
|
||||
"add_new_node_: parent {parent_id} already has a child on the new node's page"
|
||||
);
|
||||
self.inc_evictable_size(FULL, value.size()[0] as usize);
|
||||
let tokens = self
|
||||
.arena
|
||||
.node(new_node_id)
|
||||
.device_reclaimable_value_len(FULL);
|
||||
self.inc_evictable_size(FULL, tokens);
|
||||
if self.enable_storage || self.enable_external_cache_linker {
|
||||
let hash_values = self.arena.compute_node_hash_values(new_node_id, page_size);
|
||||
self.arena.node_mut(new_node_id).hash_value = Some(hash_values);
|
||||
@@ -2333,7 +2337,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
) {
|
||||
self.arena
|
||||
.set_device_value(node_id, FULL, fresh_value.copy());
|
||||
let tokens = fresh_value.size()[0] as usize;
|
||||
let tokens = self.arena.node(node_id).device_reclaimable_value_len(FULL);
|
||||
// A value materialized under lock is protected; the last release
|
||||
// moves it to evictable.
|
||||
if self.arena.device_lock_ref(node_id, FULL) > 0 {
|
||||
@@ -4675,8 +4679,11 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
component_type != BASE_COMPONENT_TYPE,
|
||||
"set_component_device_value: auxiliary components only"
|
||||
);
|
||||
let tokens = value.size()[0] as usize;
|
||||
self.arena.set_device_value(node_id, component_type, value);
|
||||
let tokens = self
|
||||
.arena
|
||||
.node(node_id)
|
||||
.device_reclaimable_value_len(component_type);
|
||||
let host_lru = self.host_lru_list_mut(component_type);
|
||||
if host_lru.in_list(Some(node_id)) {
|
||||
host_lru.remove_node(node_id);
|
||||
@@ -5136,17 +5143,12 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
let mut evictable = 0usize;
|
||||
let mut protected = 0usize;
|
||||
for &node_id in &all_nodes {
|
||||
if self.arena.node(node_id).is_root() {
|
||||
continue;
|
||||
}
|
||||
let state = &self.arena.node(node_id).values[ct.idx()];
|
||||
if let Some(value) = &state.value {
|
||||
let tokens = value.size()[0] as usize;
|
||||
if state.lock_ref > 0 {
|
||||
protected += tokens;
|
||||
} else {
|
||||
evictable += tokens;
|
||||
}
|
||||
let node = self.arena.node(node_id);
|
||||
let tokens = node.device_reclaimable_value_len(ct);
|
||||
if node.device_lock_ref(ct) > 0 {
|
||||
protected += tokens;
|
||||
} else {
|
||||
evictable += tokens;
|
||||
}
|
||||
}
|
||||
let recorded = self.component_state(ct);
|
||||
|
||||
@@ -1105,6 +1105,8 @@ def test_host_only_and_root_adjustments_are_noops(backend, is_eagle):
|
||||
for node in (root, host):
|
||||
for delta in (-10, 0, 10):
|
||||
core.adjust_component_size(node, ComponentType.FULL, delta)
|
||||
with pytest.raises(ValueError):
|
||||
core.adjust_component_size(node, ComponentType.SWA, 0)
|
||||
assert (core.full_evictable_size(), core.full_protected_size()) == (0, 0)
|
||||
assert core.collect_path_info([host])[0][0].full_value_len is None
|
||||
|
||||
@@ -1129,6 +1131,7 @@ def test_failed_adjustments_preserve_accounting_and_tree(backend, is_eagle, lock
|
||||
(invalid_node, ComponentType.FULL, 0, KeyError),
|
||||
(leaf, ComponentType.SWA, 0, ValueError),
|
||||
(leaf, 99, 0, ValueError),
|
||||
(leaf, ComponentType.FULL, -len(key) - 1, ValueError),
|
||||
):
|
||||
with pytest.raises(error):
|
||||
core.adjust_component_size(node, component, delta)
|
||||
@@ -1154,29 +1157,55 @@ def test_adjustment_follows_real_backing_changes_across_lock_transitions():
|
||||
from sglang.srt.mem_cache.unified_cache.unified_tree_core import UnifiedTreeCore
|
||||
|
||||
class SideBackingFullComponent(FullComponent):
|
||||
def reclaimable_tokens(self, node):
|
||||
backing = node.component_data[ComponentType.FULL].metadata.get(
|
||||
"side_backing"
|
||||
)
|
||||
return super().reclaimable_tokens(node) + (
|
||||
0 if backing is None else backing.numel()
|
||||
)
|
||||
def reclaimable_value_len(self, node):
|
||||
metadata = node.component_data[ComponentType.FULL].metadata
|
||||
tokens = super().reclaimable_value_len(node)
|
||||
if "raw_backing" in metadata:
|
||||
tokens = metadata["raw_backing"].numel()
|
||||
side_backing = metadata.get("side_backing")
|
||||
return tokens + (0 if side_backing is None else side_backing.numel())
|
||||
|
||||
params = CacheInitParams(
|
||||
False, None, None, 1, tree_components=(ComponentType.FULL,)
|
||||
)
|
||||
cache = SimpleNamespace(enable_session_radix_cache=False)
|
||||
core = UnifiedTreeCore(
|
||||
params, {ComponentType.FULL: SideBackingFullComponent(cache, params)}
|
||||
)
|
||||
driver = SideBackingFullComponent(cache, params)
|
||||
core = UnifiedTreeCore(params, {ComponentType.FULL: driver})
|
||||
leaf = _insert(core, [1, 2, 3, 4], [10, 11, 12, 13]).last_device_node
|
||||
metadata = core.node_by_id(leaf).component_data[ComponentType.FULL].metadata
|
||||
node = core.node_by_id(leaf)
|
||||
metadata = node.component_data[ComponentType.FULL].metadata
|
||||
metadata["side_backing"] = torch.tensor([100, 101])
|
||||
core.adjust_component_size(leaf, ComponentType.FULL, 2)
|
||||
assert (core.full_evictable_size(), core.full_protected_size()) == (6, 0)
|
||||
assert driver.value_len(node) == 4
|
||||
assert driver.reclaimable_value_len(node) == 6
|
||||
assert core.collect_path_info([leaf])[0][0].full_value_len == 4
|
||||
matched = core.match_prefix(MatchPrefixParams(key=_key([1, 2, 3, 4])))
|
||||
assert matched.device_prefix_len == 4
|
||||
assert _path_indices(core, matched.last_device_node).tolist() == [10, 11, 12, 13]
|
||||
|
||||
lock = core.inc_lock_ref(leaf)
|
||||
assert (core.full_evictable_size(), core.full_protected_size()) == (0, 6)
|
||||
# A locked split follows logical indices; the tail keeps the side backing.
|
||||
prefix_match = core.match_prefix(MatchPrefixParams(key=_key([1, 2])))
|
||||
prefix = prefix_match.last_device_node
|
||||
prefix_node = core.node_by_id(prefix)
|
||||
assert prefix_match.device_prefix_len == 2
|
||||
assert _path_indices(core, prefix).tolist() == [10, 11]
|
||||
assert core.get_component_device_value(leaf, ComponentType.FULL).tolist() == [
|
||||
12,
|
||||
13,
|
||||
]
|
||||
assert (driver.value_len(prefix_node), driver.value_len(node)) == (2, 2)
|
||||
assert (
|
||||
driver.reclaimable_value_len(prefix_node),
|
||||
driver.reclaimable_value_len(node),
|
||||
) == (2, 4)
|
||||
assert [
|
||||
(span.path_start, span.path_end, span.full_value_len)
|
||||
for span in core.collect_path_info([leaf])[0]
|
||||
] == [(0, 2, 2), (2, 4, 2)]
|
||||
assert (core.full_evictable_size(), core.full_protected_size()) == (0, 6)
|
||||
metadata["side_backing"] = metadata["side_backing"][:1]
|
||||
core.adjust_component_size(leaf, ComponentType.FULL, -1)
|
||||
assert (core.full_evictable_size(), core.full_protected_size()) == (0, 5)
|
||||
@@ -1185,9 +1214,27 @@ def test_adjustment_follows_real_backing_changes_across_lock_transitions():
|
||||
core.dec_lock_ref(leaf, lock.to_dec_params())
|
||||
assert (core.full_evictable_size(), core.full_protected_size()) == (4, 0)
|
||||
core.sanity_check([], [])
|
||||
|
||||
for fragment in (prefix_node, node):
|
||||
fragment.component_data[ComponentType.FULL].metadata["raw_backing"] = (
|
||||
torch.empty(0, dtype=torch.int64)
|
||||
)
|
||||
core.adjust_component_size(fragment.id, ComponentType.FULL, -2)
|
||||
assert driver.value_len(fragment) == 2
|
||||
assert driver.reclaimable_value_len(fragment) == 0
|
||||
assert (core.full_evictable_size(), core.full_protected_size()) == (0, 0)
|
||||
assert [span.full_value_len for span in core.collect_path_info([leaf])[0]] == [2, 2]
|
||||
matched = core.match_prefix(MatchPrefixParams(key=_key([1, 2, 3, 4])))
|
||||
assert matched.device_prefix_len == 4
|
||||
assert _path_indices(core, matched.last_device_node).tolist() == [10, 11, 12, 13]
|
||||
core.sanity_check([], [])
|
||||
tracker = {}
|
||||
_accumulate_step(core.evict_device_leaf(leaf, False), tracker, {}, {})
|
||||
assert tracker[ComponentType.FULL] == 4
|
||||
device_frees = {}
|
||||
_accumulate_step(core.evict_device_leaf(leaf, False), tracker, device_frees, {})
|
||||
assert torch.cat(device_frees[ComponentType.FULL]).tolist() == [12, 13]
|
||||
_accumulate_step(core.evict_device_leaf(prefix, False), tracker, device_frees, {})
|
||||
assert tracker[ComponentType.FULL] == 0
|
||||
assert torch.cat(device_frees[ComponentType.FULL]).tolist() == [12, 13, 10, 11]
|
||||
assert (core.full_evictable_size(), core.full_protected_size()) == (0, 0)
|
||||
|
||||
|
||||
@@ -2692,6 +2739,7 @@ def _swa_cache(
|
||||
backend="rust",
|
||||
ring=False,
|
||||
request_window=False,
|
||||
component_registry_override=None,
|
||||
):
|
||||
"""A real UnifiedRadixCache and SWA allocator, defaulting to the Rust core."""
|
||||
from sglang.srt.mem_cache.allocator.swa import SWATokenToKVPoolAllocator
|
||||
@@ -2742,6 +2790,7 @@ def _swa_cache(
|
||||
sliding_window_size=window,
|
||||
swa_gates_prefix_match=not (ring or request_window),
|
||||
tree_components=(ComponentType.FULL, ComponentType.SWA),
|
||||
component_registry_override=component_registry_override,
|
||||
)
|
||||
)
|
||||
return cache, allocator
|
||||
@@ -2758,7 +2807,9 @@ def _complete_backup(core, node, full_offset=1000, aux_offset=1000):
|
||||
return full + full_offset, aux
|
||||
|
||||
|
||||
def _swa_mamba_cache(backend, window, enable_session=False):
|
||||
def _swa_mamba_cache(
|
||||
backend, window, enable_session=False, component_registry_override=None
|
||||
):
|
||||
from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool
|
||||
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
|
||||
|
||||
@@ -2779,6 +2830,7 @@ def _swa_mamba_cache(backend, window, enable_session=False):
|
||||
page_size=1,
|
||||
sliding_window_size=window,
|
||||
enable_session_radix_cache=enable_session,
|
||||
component_registry_override=component_registry_override,
|
||||
tree_components=(
|
||||
ComponentType.FULL,
|
||||
ComponentType.SWA,
|
||||
@@ -2796,11 +2848,17 @@ def _internal_swa_write_back_case(
|
||||
with_mamba=False,
|
||||
prefix_nodes=1,
|
||||
enable_session=False,
|
||||
component_registry_override=None,
|
||||
):
|
||||
cache, allocator = (
|
||||
_swa_mamba_cache(backend, window, enable_session)
|
||||
_swa_mamba_cache(backend, window, enable_session, component_registry_override)
|
||||
if with_mamba
|
||||
else _swa_cache(window=window, page_size=page_size, backend=backend)
|
||||
else _swa_cache(
|
||||
window=window,
|
||||
page_size=page_size,
|
||||
backend=backend,
|
||||
component_registry_override=component_registry_override,
|
||||
)
|
||||
)
|
||||
assert cache._tree_core_backend == backend
|
||||
segment = 2 * page_size
|
||||
@@ -3140,6 +3198,121 @@ def _assert_internal_eviction_pending(step, component, node, swa_tokens=0):
|
||||
assert step.unbacked_tokens == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("component", [ComponentType.MAMBA, ComponentType.SWA])
|
||||
@pytest.mark.parametrize("early_release", [False, True], ids=["final", "early-swa"])
|
||||
def test_python_component_locks_use_reclaimable_backing(component, early_release):
|
||||
from sglang.srt.mem_cache.unified_cache.components.mamba import MambaComponent
|
||||
from sglang.srt.mem_cache.unified_cache.components.swa import SWAComponent
|
||||
|
||||
base = MambaComponent if component == ComponentType.MAMBA else SWAComponent
|
||||
|
||||
class SideBackingComponent(base):
|
||||
def reclaimable_value_len(self, node):
|
||||
backing = node.component_data[self.component_type].metadata.get("backing")
|
||||
if backing is not None:
|
||||
return backing.numel()
|
||||
return super().reclaimable_value_len(node)
|
||||
|
||||
case = _internal_swa_write_back_case(
|
||||
"python",
|
||||
with_mamba=component == ComponentType.MAMBA,
|
||||
component_registry_override={component: SideBackingComponent},
|
||||
)
|
||||
cache, core = case.cache, case.core
|
||||
node_id, leaf = case.nodes
|
||||
node = core.node_by_id(node_id)
|
||||
state = node.component_data[component]
|
||||
logical = state.value.clone()
|
||||
state.metadata["backing"] = torch.arange(len(logical) + 2)
|
||||
core.adjust_component_size(node_id, component, 2)
|
||||
|
||||
def ledgers():
|
||||
return (
|
||||
core.component_evictable_size_[component],
|
||||
core.component_protected_size_[component],
|
||||
)
|
||||
|
||||
before = ledgers()
|
||||
reclaimed = len(logical) + 2
|
||||
lock = core.inc_lock_ref(node_id).to_dec_params()
|
||||
assert state.lock_ref == 1
|
||||
assert ledgers() == (before[0] - reclaimed, before[1] + reclaimed)
|
||||
assert cache.components[component].value_len(node) == len(logical)
|
||||
assert torch.equal(state.value, logical)
|
||||
|
||||
# The final release uses current backing, not the original lock delta.
|
||||
state.metadata["backing"] = state.metadata["backing"][:-1]
|
||||
core.adjust_component_size(node_id, component, -1)
|
||||
if early_release:
|
||||
released = core.dec_swa_lock_only(node_id, lock)
|
||||
assert not released.device_frees and not released.host_frees
|
||||
core.dec_lock_ref(node_id, lock, skip_swa=True)
|
||||
else:
|
||||
core.dec_lock_ref(node_id, lock)
|
||||
assert state.lock_ref == 0
|
||||
assert ledgers() == (before[0] - 1, before[1])
|
||||
assert torch.equal(state.value, logical)
|
||||
cache.dec_lock_ref(leaf, case.leaf_lock)
|
||||
core.sanity_check([], [])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("component", [ComponentType.MAMBA, ComponentType.SWA])
|
||||
def test_python_zero_accounted_internal_eviction_continues(component):
|
||||
from sglang.srt.mem_cache.unified_cache.components.mamba import MambaComponent
|
||||
from sglang.srt.mem_cache.unified_cache.components.swa import SWAComponent
|
||||
|
||||
base = MambaComponent if component == ComponentType.MAMBA else SWAComponent
|
||||
|
||||
class ReleasedBackingComponent(base):
|
||||
def reclaimable_value_len(self, node):
|
||||
if node.component_data[self.component_type].metadata.get("released"):
|
||||
return 0
|
||||
return super().reclaimable_value_len(node)
|
||||
|
||||
case = _internal_swa_write_back_case(
|
||||
"python",
|
||||
with_mamba=component == ComponentType.MAMBA,
|
||||
prefix_nodes=2,
|
||||
component_registry_override={component: ReleasedBackingComponent},
|
||||
)
|
||||
cache, core = case.cache, case.core
|
||||
first, second, leaf = case.nodes
|
||||
cache.is_write_back = False
|
||||
driver = cache.components[component]
|
||||
expected_values = [
|
||||
core.get_component_device_value(node, component).clone()
|
||||
for node in (first, second)
|
||||
]
|
||||
first_size = driver.reclaimable_value_len(core.node_by_id(first))
|
||||
expected = driver.reclaimable_value_len(core.node_by_id(second))
|
||||
core.node_by_id(first).component_data[component].metadata["released"] = True
|
||||
core.adjust_component_size(first, component, -first_size)
|
||||
assert driver.value_len(core.node_by_id(first)) == len(expected_values[0]) > 0
|
||||
assert driver.reclaimable_value_len(core.node_by_id(first)) == 0
|
||||
tracker, freed = {}, {}
|
||||
core.evict_device_start(component, expected)
|
||||
try:
|
||||
first_step = core.evict_device_next_node(component, tracker)
|
||||
assert first_step.node_id is None and first_step.made_progress
|
||||
assert not first_step.tracker and first_step.device_frees[component]
|
||||
_accumulate_step(first_step, tracker, freed, {})
|
||||
assert core.get_component_device_value(first, component) is None
|
||||
assert torch.equal(torch.cat(freed[component]), expected_values[0])
|
||||
|
||||
second_step = core.evict_device_next_node(component, tracker)
|
||||
assert second_step.node_id is None and second_step.made_progress
|
||||
_accumulate_step(second_step, tracker, freed, {})
|
||||
assert tracker[component] == expected
|
||||
assert core.get_component_device_value(second, component) is None
|
||||
assert len(freed[component]) == 2
|
||||
assert torch.equal(torch.cat(freed[component]), torch.cat(expected_values))
|
||||
assert not core.evict_device_next_node(component, tracker).made_progress
|
||||
finally:
|
||||
core.evict_device_end(component)
|
||||
cache.dec_lock_ref(leaf, case.leaf_lock)
|
||||
core.sanity_check([], [])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("component", [ComponentType.MAMBA, ComponentType.SWA])
|
||||
def test_python_component_returns_immutable_internal_backup_request(component):
|
||||
from dataclasses import FrozenInstanceError
|
||||
|
||||
@@ -122,7 +122,7 @@ class TestUnifiedRadixComponentLedgerUnits(CustomTestCase):
|
||||
prefill admission adds the evictable ledger to the allocator's free
|
||||
space and the eviction walk counts them toward its request. So every
|
||||
ledger site (insert, lock, release, an in-place change, eviction)
|
||||
reads ``TreeComponent.reclaimable_tokens`` rather than the value
|
||||
reads ``TreeComponent.reclaimable_value_len`` rather than the value
|
||||
length. A site that goes back to ``len(value)`` drifts the ledger from
|
||||
the recount in ``sanity_check`` for a component that overrides it."""
|
||||
cache, allocator, _ = build_fixture(
|
||||
|
||||
Reference in New Issue
Block a user