[Refactor] radix-tree: Make each cache action a typed CacheAction implementation (#45444)

Co-authored-by: metamergebot <metamergebot@users.noreply.github.com>
Co-authored-by: Jialin <Jialin@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-23 12:49:30 -07:00
committed by GitHub
co-authored by metamergebot Jialin
parent d0022e1df3
commit 005333d029
16 changed files with 836 additions and 669 deletions
+157
View File
@@ -0,0 +1,157 @@
//! Cache actions returned to the cache for execution.
use std::fmt::Debug;
use tch::Tensor;
use crate::components::ComponentType;
use crate::node::NodeId;
pub trait CacheAction: Debug + Send {
fn tag(&self) -> &'static str;
/// Whether insertion must pause for the cache to apply this action.
fn requires_insert_barrier(&self) -> bool {
true
}
#[cfg(feature = "python-extension")]
fn into_py_tagged(
self: Box<Self>,
py: pyo3::Python<'_>,
) -> pyo3::PyResult<pyo3::Py<pyo3::PyAny>>;
/// Rust tests inspect emitted payloads without a production downcast API.
#[cfg(test)]
fn as_any(&self) -> &dyn std::any::Any;
}
pub type CacheActions = Vec<Box<dyn CacheAction>>;
macro_rules! impl_cache_action {
($action:ty, $tag:literal $(, insert_barrier = $barrier:expr)? $(,)?) => {
impl $crate::cache_action::CacheAction for $action {
fn tag(&self) -> &'static str {
$tag
}
$(fn requires_insert_barrier(&self) -> bool {
$barrier
})?
#[cfg(feature = "python-extension")]
fn into_py_tagged(
self: Box<Self>,
py: pyo3::Python<'_>,
) -> pyo3::PyResult<pyo3::Py<pyo3::PyAny>> {
<$action>::into_py_tagged(*self, py)
}
#[cfg(test)]
fn as_any(&self) -> &dyn std::any::Any {
self
}
}
};
}
pub(crate) use impl_cache_action;
/// Duplicate device KV slices the cache frees after the insert.
#[derive(Debug)]
pub struct FreeDeviceKV {
pub indices: Vec<Tensor>,
}
/// Free the full side only, for a tombstoned node whose SWA peers are gone;
/// FreeDeviceKV would release the SWA side twice.
#[derive(Debug)]
pub struct FreeDeviceKVFullOnly {
pub indices: Vec<Tensor>,
}
/// Update cache-side write bookkeeping after a node split:
///
/// parent -> node => parent -> new_node -> new_child
///
/// old_node_id (the pre-split node) is replaced by new_node_id + new_child_node_id.
#[derive(Debug)]
pub struct UpdatePendingBackupOnNodeSplit {
/// None when buffer-only bookkeeping has no tree-tracked write.
pub ack_id: Option<usize>,
pub old_node_id: NodeId,
pub new_node_id: NodeId,
pub new_child_node_id: NodeId,
}
/// Per-path Mamba state-cap eviction from the tail's root path; applied at
/// the insert's commit barrier, after the walk-time backups whose
/// write-through locks shield the backed-up chain.
#[derive(Debug)]
pub struct MambaEvictExcessPathStates {
pub tail_node_id: NodeId,
}
/// Free only the given component's device KV slots.
#[derive(Debug)]
pub struct FreeComponentDeviceSlot {
pub component_type: ComponentType,
pub indices: Vec<Tensor>,
}
/// Free the given component's host KV pages.
#[derive(Debug)]
pub struct FreeComponentHostSlot {
pub component_type: ComponentType,
pub host_indices: Vec<Tensor>,
}
/// Rebuild the SWA allocator's full->swa index mapping for loaded chunks.
#[derive(Debug)]
pub struct RebuildFullToSwaMapping {
pub full_indices: Vec<Tensor>,
pub swa_indices: Vec<Tensor>,
}
/// Recover an SWA tombstone whose full is locked: keep the locked full, remap
/// it onto the incoming full's SWA translation, and free only the incoming full.
#[derive(Debug)]
pub struct RecoverSwaWithLockedFull {
pub node_id: NodeId,
pub kept_full: Tensor,
pub incoming_full: Tensor,
}
/// Rebuild a node's SWA value by translating its source full value, then store it.
#[derive(Debug)]
pub struct SwaRebuild {
pub node_id: NodeId,
pub source_value: Tensor,
}
/// A device->host backup work item for the cache to execute.
#[derive(Default, Debug)]
pub struct BackupKV {
/// Backup these nodes device->host in order, stopping at the first failure; the
/// caller orders them parent-before-child for write-through and child-first for
/// write-back. External handles: the list crosses to the orchestrator.
pub node_ids: Vec<NodeId>,
}
self::impl_cache_action!(FreeDeviceKV, "free_device_kv", insert_barrier = false);
self::impl_cache_action!(
FreeDeviceKVFullOnly,
"free_device_kv_full_only",
insert_barrier = false
);
self::impl_cache_action!(BackupKV, "backup_kv");
self::impl_cache_action!(
UpdatePendingBackupOnNodeSplit,
"update_pending_backup_on_node_split",
insert_barrier = false
);
self::impl_cache_action!(MambaEvictExcessPathStates, "mamba_evict_excess_path_states");
self::impl_cache_action!(FreeComponentDeviceSlot, "free_component_device_slot");
self::impl_cache_action!(FreeComponentHostSlot, "free_component_host_slot");
self::impl_cache_action!(RebuildFullToSwaMapping, "rebuild_full_to_swa_mapping");
self::impl_cache_action!(RecoverSwaWithLockedFull, "recover_swa_with_locked_full");
self::impl_cache_action!(SwaRebuild, "swa_rebuild");
@@ -13,7 +13,7 @@ use crate::node::Node;
use crate::node::{NodeId, NodeIdx_, TreeCoreRuntimeError, ValueSlotIdx};
use crate::unified_lru_list::PriorityKey;
use crate::unified_tree_core::{
CacheAction, CacheTransferPhase, DecLockRefParams, EvictLayer, IncLockRefResult, InsertResult,
CacheActions, CacheTransferPhase, DecLockRefParams, EvictLayer, IncLockRefResult, InsertResult,
MatchPrefixParams, MatchResult, PoolName, PoolTransfer, PoolTransferResult, UnifiedTreeCore,
};
@@ -104,7 +104,7 @@ impl<K: ChildKeyType> TreeComponent<K> for FullComponent {
node_id: NodeIdx_,
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
target: EvictLayer,
) -> (usize, usize) {
let node = tree_core.arena.node(node_id);
@@ -156,7 +156,7 @@ impl<K: ChildKeyType> TreeComponent<K> for FullComponent {
tracker: &mut HashMap<ComponentType, usize>,
_device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
_host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
_cache_actions: &mut Vec<CacheAction>,
_cache_actions: &mut CacheActions,
) -> Option<NodeIdx_> {
let ct = FULL;
assert!(
@@ -205,7 +205,7 @@ impl<K: ChildKeyType> TreeComponent<K> for FullComponent {
tracker: &mut HashMap<ComponentType, usize>,
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
) {
for spare_imminent_demotes in [true, false] {
if tracker[&FULL] >= num_tokens {
@@ -248,7 +248,7 @@ impl<K: ChildKeyType> TreeComponent<K> for FullComponent {
tracker: &mut HashMap<ComponentType, usize>,
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
) {
let ct = FULL;
let arena = &tree_core.arena;
@@ -491,7 +491,7 @@ impl<K: ChildKeyType> TreeComponent<K> for FullComponent {
node_id: NodeIdx_,
phase: CacheTransferPhase,
transfers: Vec<PoolTransfer>,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
insert_result: Option<&mut InsertResult>,
pool_storage_result: Option<&PoolTransferResult>,
) {
+12 -11
View File
@@ -6,13 +6,14 @@ use std::collections::HashMap;
use tch::Tensor;
use crate::cache_action::{FreeComponentHostSlot, MambaEvictExcessPathStates};
use crate::components::TreeComponent;
use crate::components::{ComponentType, FULL, MAMBA};
use crate::node::ChildKeyType;
use crate::node::Node;
use crate::node::{NodeId, NodeIdx_, TreeCoreRuntimeError, ValueSlotIdx};
use crate::unified_tree_core::{
CacheAction, CacheInitParams, CacheTransferPhase, DecLockRefParams, EvictLayer,
CacheActions, CacheInitParams, CacheTransferPhase, DecLockRefParams, EvictLayer,
IncLockRefResult, InsertParams, InsertResult, LRURefreshPhase, MatchPrefixParams, MatchResult,
PoolHitPolicy, PoolName, PoolTransfer, PoolTransferResult, UnifiedTreeCore,
};
@@ -70,12 +71,12 @@ impl MambaComponent {
fn emit_excess_path_states_eviction_(
&self,
tail_node_id: NodeId,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
) {
if self.mamba_max_states_per_path.is_none() {
return;
}
cache_actions.push(CacheAction::MambaEvictExcessPathStates { tail_node_id });
cache_actions.push(Box::new(MambaEvictExcessPathStates { tail_node_id }));
}
}
@@ -160,7 +161,7 @@ impl<K: ChildKeyType> TreeComponent<K> for MambaComponent {
is_new_leaf: bool,
params: &InsertParams<'_, K>,
result: &mut InsertResult,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
) {
let mamba_value = params
.mamba_value
@@ -200,7 +201,7 @@ impl<K: ChildKeyType> TreeComponent<K> for MambaComponent {
tail_node_id: NodeIdx_,
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
) {
let Some(cap) = self.mamba_max_states_per_path else {
return;
@@ -282,7 +283,7 @@ impl<K: ChildKeyType> TreeComponent<K> for MambaComponent {
node_id: NodeIdx_,
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
target: EvictLayer,
) -> (usize, usize) {
let ct = MAMBA;
@@ -348,7 +349,7 @@ impl<K: ChildKeyType> TreeComponent<K> for MambaComponent {
tracker: &mut HashMap<ComponentType, usize>,
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
) -> Option<NodeIdx_> {
let ct = MAMBA;
assert!(
@@ -636,7 +637,7 @@ impl<K: ChildKeyType> TreeComponent<K> for MambaComponent {
node_id: NodeIdx_,
phase: CacheTransferPhase,
transfers: Vec<PoolTransfer>,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
insert_result: Option<&mut InsertResult>,
pool_storage_result: Option<&PoolTransferResult>,
) {
@@ -696,12 +697,12 @@ impl<K: ChildKeyType> TreeComponent<K> for MambaComponent {
let Some(target) = attach_target else {
// The buffer cannot attach: free it and let the caller keep
// its own donated slot bookkeeping.
cache_actions.push(CacheAction::FreeComponentHostSlot {
cache_actions.push(Box::new(FreeComponentHostSlot {
component_type: MAMBA,
host_indices: host_indices
.map(|host| vec![host.shallow_clone()])
.unwrap_or_default(),
});
}));
if let Some(insert_result) = insert_result {
insert_result.mamba_exist = true;
}
@@ -733,7 +734,7 @@ impl<K: ChildKeyType> TreeComponent<K> for MambaComponent {
tracker: &mut HashMap<ComponentType, usize>,
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
) {
let ct = MAMBA;
let mut x = tree_core
+11 -11
View File
@@ -7,7 +7,7 @@ use tch::Tensor;
use crate::node::{ChildKeyType, Node, NodeArena, NodeId, NodeIdx_, TreeCoreRuntimeError};
use crate::unified_tree_core::{
CacheAction, CacheTransferPhase, DecLockRefParams, EvictLayer, IncLockRefResult, InsertParams,
CacheActions, CacheTransferPhase, DecLockRefParams, EvictLayer, IncLockRefResult, InsertParams,
InsertResult, LRURefreshPhase, MatchPrefixParams, MatchResult, PoolTransfer,
PoolTransferResult, UnifiedTreeCore,
};
@@ -199,7 +199,7 @@ pub trait TreeComponent<K: ChildKeyType> {
params: &InsertParams<'_, K>,
preserve_full_ids: bool,
result: &mut InsertResult,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
) -> usize {
prefix_len
}
@@ -216,7 +216,7 @@ pub trait TreeComponent<K: ChildKeyType> {
total_prefix_len: usize,
params: &InsertParams<'_, K>,
result: &mut InsertResult,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
) {
}
@@ -239,7 +239,7 @@ pub trait TreeComponent<K: ChildKeyType> {
is_new_leaf: bool,
params: &InsertParams<'_, K>,
result: &mut InsertResult,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
) {
}
@@ -251,7 +251,7 @@ pub trait TreeComponent<K: ChildKeyType> {
tail_node_id: NodeIdx_,
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
) {
}
@@ -307,7 +307,7 @@ pub trait TreeComponent<K: ChildKeyType> {
node_id: NodeIdx_,
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
target: EvictLayer,
) -> (usize, usize);
@@ -330,7 +330,7 @@ pub trait TreeComponent<K: ChildKeyType> {
tracker: &mut HashMap<ComponentType, usize>,
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
) -> Option<NodeIdx_>;
/// Clear this component's device-eviction walk state.
@@ -406,7 +406,7 @@ pub trait TreeComponent<K: ChildKeyType> {
_params: &DecLockRefParams,
_device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
_host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
_cache_actions: &mut Vec<CacheAction>,
_cache_actions: &mut CacheActions,
) {
unimplemented!("release_window_lock is SWA-only")
}
@@ -460,7 +460,7 @@ pub trait TreeComponent<K: ChildKeyType> {
node_id: NodeIdx_,
phase: CacheTransferPhase,
transfers: Vec<PoolTransfer>,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
insert_result: Option<&mut InsertResult>,
pool_storage_result: Option<&PoolTransferResult>,
) {
@@ -489,7 +489,7 @@ pub trait TreeComponent<K: ChildKeyType> {
_tracker: &mut HashMap<ComponentType, usize>,
_device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
_host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
_cache_actions: &mut Vec<CacheAction>,
_cache_actions: &mut CacheActions,
) {
}
@@ -503,7 +503,7 @@ pub trait TreeComponent<K: ChildKeyType> {
_tracker: &mut HashMap<ComponentType, usize>,
_device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
_host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
_cache_actions: &mut Vec<CacheAction>,
_cache_actions: &mut CacheActions,
) {
}
}
+46 -44
View File
@@ -7,13 +7,17 @@ use std::collections::HashMap;
use tch::{Kind, Tensor};
use crate::cache_action::{
FreeComponentHostSlot, FreeDeviceKVFullOnly, RebuildFullToSwaMapping, RecoverSwaWithLockedFull,
SwaRebuild,
};
use crate::components::TreeComponent;
use crate::components::{ComponentType, FULL, SWA};
use crate::node::ChildKeyType;
use crate::node::Node;
use crate::node::{NodeId, NodeIdx_, TreeCoreRuntimeError, ValueSlotIdx};
use crate::unified_tree_core::{
CacheAction, CacheInitParams, CacheTransferPhase, DecLockRefParams, EvictLayer,
CacheActions, CacheInitParams, CacheTransferPhase, DecLockRefParams, EvictLayer,
IncLockRefResult, InsertParams, InsertResult, LRURefreshPhase, MatchPrefixParams, MatchResult,
PoolHitPolicy, PoolName, PoolTransfer, PoolTransferResult, UnifiedTreeCore,
};
@@ -57,7 +61,7 @@ impl SwaComponent {
&self,
tree_core: &mut UnifiedTreeCore<K>,
leaf_id: NodeIdx_,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
) -> Option<NodeIdx_> {
let leaf = tree_core.arena.node(leaf_id);
if leaf.is_root() || leaf.device_lock_ref(SWA) > 0 {
@@ -80,17 +84,11 @@ impl SwaComponent {
let (new_parent, action) = tree_core.split_node_(leaf_id, split_at);
assert!(
!matches!(
action,
Some(CacheAction::UpdatePendingBackupOnNodeSplit {
ack_id: Some(_),
..
})
),
action.as_ref().is_none_or(|action| action.ack_id.is_none()),
"fresh SWA leaf cannot be write-through-pending"
);
if let Some(action) = action {
cache_actions.push(action);
cache_actions.push(Box::new(action));
}
Some(new_parent)
}
@@ -193,12 +191,12 @@ impl SwaComponent {
impl SwaComponent {
/// Queue a free of the given SWA host slots; empty tensors are dropped.
fn release_swa_host_(&self, host_indices: Tensor, cache_actions: &mut Vec<CacheAction>) {
fn release_swa_host_(&self, host_indices: Tensor, cache_actions: &mut CacheActions) {
if host_indices.numel() > 0 {
cache_actions.push(CacheAction::FreeComponentHostSlot {
cache_actions.push(Box::new(FreeComponentHostSlot {
component_type: SWA,
host_indices: vec![host_indices],
});
}));
}
}
@@ -234,7 +232,7 @@ impl SwaComponent {
tree_core: &mut UnifiedTreeCore<K>,
node_id: NodeIdx_,
transfers: Vec<PoolTransfer>,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
insert_result: Option<&InsertResult>,
pool_storage_result: Option<&PoolTransferResult>,
) {
@@ -307,7 +305,7 @@ impl SwaComponent {
if fill_start > node_start {
let (_, action) = tree_core.split_node_(cur, fill_start - node_start);
if let Some(action) = action {
cache_actions.push(action);
cache_actions.push(Box::new(action));
}
}
self.attach_swa_host_value_(tree_core, cur, slice);
@@ -441,7 +439,7 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
params: &InsertParams<'_, K>,
preserve_full_ids: bool,
result: &mut InsertResult,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
) -> usize {
if params.prev_prefix_len >= total_prefix_len + prefix_len {
return prefix_len;
@@ -467,21 +465,23 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
// Branch 1: entire value_slice is within SWA window — recover
result.record_adopted_range(SWA, total_prefix_len, total_prefix_len + prefix_len);
if node.device_lock_ref(FULL) > 0 || preserve_full_ids {
cache_actions.push(CacheAction::RecoverSwaWithLockedFull {
cache_actions.push(Box::new(RecoverSwaWithLockedFull {
node_id: node.id,
kept_full: node.device_value(FULL).shallow_clone(),
incoming_full: value_slice,
});
}));
return 0;
}
result.record_adopted_range(FULL, total_prefix_len, total_prefix_len + prefix_len);
let old_full = node.take_device_value(FULL);
node.set_device_value(FULL, value_slice.copy());
cache_actions.push(CacheAction::FreeDeviceKVFullOnly(vec![old_full]));
cache_actions.push(CacheAction::SwaRebuild {
cache_actions.push(Box::new(FreeDeviceKVFullOnly {
indices: vec![old_full],
}));
cache_actions.push(Box::new(SwaRebuild {
node_id: node.id,
source_value: value_slice,
});
}));
0
} else if swa_evicted_seqlen < total_prefix_len + prefix_len {
// Branch 2: value_slice[start_idx:] is within SWA window — partial recover
@@ -495,26 +495,28 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
.narrow(0, start_idx as i64, (full_len - start_idx) as i64);
let (_, action) = tree_core.split_node_(node_id, start_idx);
if let Some(action) = action {
cache_actions.push(action);
cache_actions.push(Box::new(action));
}
let new_full = value_slice.narrow(0, start_idx as i64, (prefix_len - start_idx) as i64);
if preserve_full {
cache_actions.push(CacheAction::RecoverSwaWithLockedFull {
cache_actions.push(Box::new(RecoverSwaWithLockedFull {
node_id: node_ext_id,
kept_full: old_full,
incoming_full: new_full,
});
}));
return start_idx;
}
result.record_adopted_range(FULL, swa_evicted_seqlen, total_prefix_len + prefix_len);
let node = tree_core.arena.node_mut(node_id);
let _ = node.take_device_value(FULL);
node.set_device_value(FULL, new_full.copy());
cache_actions.push(CacheAction::FreeDeviceKVFullOnly(vec![old_full]));
cache_actions.push(CacheAction::SwaRebuild {
cache_actions.push(Box::new(FreeDeviceKVFullOnly {
indices: vec![old_full],
}));
cache_actions.push(Box::new(SwaRebuild {
node_id: node_ext_id,
source_value: new_full,
});
}));
start_idx
} else {
// Branch 3: entire value_slice is outside SWA window — not consumed
@@ -530,7 +532,7 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
total_prefix_len: usize,
params: &InsertParams<'_, K>,
result: &mut InsertResult,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
) {
// _unevict_node_on_insert already wrote the request's fresh KV slice
// into the base value. We just need to rebuild SWA from that slice for
@@ -552,7 +554,7 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
let start_idx = swa_evicted_seqlen - total_prefix_len;
let (_, action) = tree_core.split_node_(node_id, start_idx);
if let Some(action) = action {
cache_actions.push(action);
cache_actions.push(Box::new(action));
}
} else {
return;
@@ -562,10 +564,10 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
total_prefix_len.max(swa_evicted_seqlen),
total_prefix_len + prefix_len,
);
cache_actions.push(CacheAction::SwaRebuild {
cache_actions.push(Box::new(SwaRebuild {
node_id: tree_core.arena.node(node_id).id,
source_value: tree_core.arena.device_value(node_id, FULL).shallow_clone(),
});
}));
}
fn commit_insert_component_data(
@@ -575,7 +577,7 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
is_new_leaf: bool,
params: &InsertParams<'_, K>,
result: &mut InsertResult,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
) {
if let Some(branching_seqlen) = params.swa_branching_seqlen {
result.swa_branch_inserted = params.key.atom_len() >= branching_seqlen;
@@ -599,7 +601,7 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
// (tombstone) and an in-window child; `node` becomes the child.
let (_, action) = tree_core.split_node_(node_id, split_pos);
if let Some(action) = action {
cache_actions.push(action);
cache_actions.push(Box::new(action));
}
}
}
@@ -609,18 +611,18 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
// the in-window tail lands more-MRU.
let capped_parent = self.maybe_split_leaf_for_swa_lock_(tree_core, node_id, cache_actions);
if let Some(capped_parent) = capped_parent {
cache_actions.push(CacheAction::SwaRebuild {
cache_actions.push(Box::new(SwaRebuild {
node_id: tree_core.arena.node(capped_parent).id,
source_value: tree_core
.arena
.device_value(capped_parent, FULL)
.shallow_clone(),
});
}));
}
cache_actions.push(CacheAction::SwaRebuild {
cache_actions.push(Box::new(SwaRebuild {
node_id: tree_core.arena.node(node_id).id,
source_value: tree_core.arena.device_value(node_id, FULL).shallow_clone(),
});
}));
}
fn redistribute_on_node_split(
@@ -671,7 +673,7 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
node_id: NodeIdx_,
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
target: EvictLayer,
) -> (usize, usize) {
let ct = SWA;
@@ -741,7 +743,7 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
tracker: &mut HashMap<ComponentType, usize>,
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
) -> Option<NodeIdx_> {
let ct = SWA;
assert!(
@@ -848,7 +850,7 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
tracker: &mut HashMap<ComponentType, usize>,
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
) {
let ct = SWA;
let mut x = tree_core
@@ -1057,7 +1059,7 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
node_id: NodeIdx_,
phase: CacheTransferPhase,
transfers: Vec<PoolTransfer>,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
insert_result: Option<&mut InsertResult>,
pool_storage_result: Option<&PoolTransferResult>,
) {
@@ -1142,10 +1144,10 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
assert_eq!(offset, host_indices.size()[0]);
// Rebuild the full->swa mapping for the loaded chunks at the orchestration layer.
if !full_chunks.is_empty() {
cache_actions.push(CacheAction::RebuildFullToSwaMapping {
cache_actions.push(Box::new(RebuildFullToSwaMapping {
full_indices: full_chunks,
swa_indices: swa_chunks,
});
}));
}
}
CacheTransferPhase::Prefetch => {
@@ -1289,7 +1291,7 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
params: &DecLockRefParams,
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
) {
if tree_core.arena.node(node_id).is_root() {
return;
+1
View File
@@ -16,6 +16,7 @@
clippy::useless_conversion
)]
mod cache_action;
mod components;
mod node;
#[cfg(feature = "python-extension")]
+9 -79
View File
@@ -18,10 +18,10 @@ use crate::node::{KeyNamespaceRef, NodeAccessError, NodeId, TreeCoreRuntimeError
use crate::unified_lru_list::{TlruFloatConfig, TlruPromptEstimate};
use crate::unified_tree_core::KvCacheEvent;
use crate::unified_tree_core::{
BufferBackupSnapshot, BufferBackupState, CacheAction, CacheInitParams, CacheTransferPhase,
DecLockRefParams, EvictLayer, EvictionStepResult, InsertParams, InsertResult, InsertStepResult,
MatchPrefixParams, MatchResult, PoolHitPolicy, PoolName, PoolTransfer, PoolTransferResult, Req,
UnifiedTreeCore,
BufferBackupSnapshot, BufferBackupState, CacheAction, CacheActions, CacheInitParams,
CacheTransferPhase, DecLockRefParams, EvictLayer, EvictionStepResult, InsertParams,
InsertResult, InsertStepResult, MatchPrefixParams, MatchResult, PoolHitPolicy, PoolName,
PoolTransfer, PoolTransferResult, Req, UnifiedTreeCore,
};
/// Translate only Rust panics; ordinary Python errors pass through unchanged.
@@ -173,83 +173,13 @@ fn py_array_to_vec_i64(py: Python<'_>, key: &Bound<'_, PyAny>) -> PyResult<Vec<i
buffer.to_vec(py)
}
/// The tagged-tuple tag of a cache action.
fn cache_action_tag(action: &CacheAction) -> &'static str {
match action {
CacheAction::FreeDeviceKV(_) => "free_device_kv",
CacheAction::FreeDeviceKVFullOnly(_) => "free_device_kv_full_only",
CacheAction::BackupKV(_) => "backup_kv",
CacheAction::UpdatePendingBackupOnNodeSplit { .. } => "update_pending_backup_on_node_split",
CacheAction::MambaEvictExcessPathStates { .. } => "mamba_evict_excess_path_states",
CacheAction::FreeComponentDeviceSlot { .. } => "free_component_device_slot",
CacheAction::FreeComponentHostSlot { .. } => "free_component_host_slot",
CacheAction::RebuildFullToSwaMapping { .. } => "rebuild_full_to_swa_mapping",
CacheAction::RecoverSwaWithLockedFull { .. } => "recover_swa_with_locked_full",
CacheAction::SwaRebuild { .. } => "swa_rebuild",
}
}
/// Convert a cache action into its Python tagged tuple.
fn cache_action_to_py(py: Python<'_>, action: CacheAction) -> PyResult<Py<PyAny>> {
let tag = cache_action_tag(&action);
match action {
CacheAction::FreeDeviceKV(tensors) => {
let tensors = PyList::new(py, tensors.into_iter().map(PyTensor))?;
(tag, tensors).into_py_any(py)
}
CacheAction::FreeDeviceKVFullOnly(tensors) => {
let tensors = PyList::new(py, tensors.into_iter().map(PyTensor))?;
(tag, tensors).into_py_any(py)
}
CacheAction::BackupKV(backup) => (tag, backup.node_ids).into_py_any(py),
CacheAction::UpdatePendingBackupOnNodeSplit {
ack_id,
old_node_id,
new_node_id,
new_child_node_id,
} => (tag, ack_id, old_node_id, new_node_id, new_child_node_id).into_py_any(py),
CacheAction::MambaEvictExcessPathStates { tail_node_id } => {
(tag, tail_node_id).into_py_any(py)
}
CacheAction::FreeComponentDeviceSlot {
component_type,
indices,
} => {
let indices = PyList::new(py, indices.into_iter().map(PyTensor))?;
(tag, component_type_to_u8(component_type), indices).into_py_any(py)
}
CacheAction::FreeComponentHostSlot {
component_type,
host_indices,
} => {
let host_indices = PyList::new(py, host_indices.into_iter().map(PyTensor))?;
(tag, component_type_to_u8(component_type), host_indices).into_py_any(py)
}
CacheAction::RebuildFullToSwaMapping {
full_indices,
swa_indices,
} => {
let full_indices = PyList::new(py, full_indices.into_iter().map(PyTensor))?;
let swa_indices = PyList::new(py, swa_indices.into_iter().map(PyTensor))?;
(tag, full_indices, swa_indices).into_py_any(py)
}
CacheAction::RecoverSwaWithLockedFull {
node_id,
kept_full,
incoming_full,
} => (tag, node_id, PyTensor(kept_full), PyTensor(incoming_full)).into_py_any(py),
CacheAction::SwaRebuild {
node_id,
source_value,
} => (tag, node_id, PyTensor(source_value)).into_py_any(py),
}
}
mod cache_action;
/// Convert cache actions into a Python list of tagged tuples.
fn cache_actions_to_py(py: Python<'_>, actions: Vec<CacheAction>) -> PyResult<Py<PyList>> {
fn cache_actions_to_py(py: Python<'_>, actions: CacheActions) -> PyResult<Py<PyList>> {
let list = PyList::empty(py);
for action in actions {
list.append(cache_action_to_py(py, action)?)?;
list.append(action.into_py_tagged(py)?)?;
}
Ok(list.unbind())
}
@@ -1587,7 +1517,7 @@ macro_rules! tree_core_methods {
Ok(EvictDeviceLeafResultBinding {
unbacked_tokens: result.unbacked_tokens,
backup_kv: backup
.map(|backup| cache_action_to_py(py, CacheAction::BackupKV(backup)))
.map(|backup| backup.into_py_tagged(py))
.transpose()?,
tracker: tracker_to_py(result.tracker),
new_device_frees: frees_to_py(py, result.device_frees)?,
@@ -1774,7 +1704,7 @@ macro_rules! tree_core_methods {
let backup = py
.detach(|| self.core().build_backup_kv_action(node_id))
.map_err(node_access_error)?;
cache_action_to_py(py, CacheAction::BackupKV(backup))
backup.into_py_tagged(py)
}
/// Gather a node's device value plus per-component BACKUP_HOST transfers.
@@ -0,0 +1,97 @@
//! Python encoding for the common cache actions.
use super::*;
use crate::cache_action::{
BackupKV, FreeComponentDeviceSlot, FreeComponentHostSlot, FreeDeviceKV, FreeDeviceKVFullOnly,
MambaEvictExcessPathStates, RebuildFullToSwaMapping, RecoverSwaWithLockedFull, SwaRebuild,
UpdatePendingBackupOnNodeSplit,
};
impl FreeDeviceKV {
pub(crate) fn into_py_tagged(self, py: Python<'_>) -> PyResult<Py<PyAny>> {
let tag = self.tag();
let indices = PyList::new(py, self.indices.into_iter().map(PyTensor))?;
(tag, indices).into_py_any(py)
}
}
impl FreeDeviceKVFullOnly {
pub(crate) fn into_py_tagged(self, py: Python<'_>) -> PyResult<Py<PyAny>> {
let tag = self.tag();
let indices = PyList::new(py, self.indices.into_iter().map(PyTensor))?;
(tag, indices).into_py_any(py)
}
}
impl BackupKV {
pub(crate) fn into_py_tagged(self, py: Python<'_>) -> PyResult<Py<PyAny>> {
let tag = self.tag();
(tag, self.node_ids).into_py_any(py)
}
}
impl UpdatePendingBackupOnNodeSplit {
pub(crate) fn into_py_tagged(self, py: Python<'_>) -> PyResult<Py<PyAny>> {
let tag = self.tag();
(
tag,
self.ack_id,
self.old_node_id,
self.new_node_id,
self.new_child_node_id,
)
.into_py_any(py)
}
}
impl MambaEvictExcessPathStates {
pub(crate) fn into_py_tagged(self, py: Python<'_>) -> PyResult<Py<PyAny>> {
let tag = self.tag();
(tag, self.tail_node_id).into_py_any(py)
}
}
impl FreeComponentDeviceSlot {
pub(crate) fn into_py_tagged(self, py: Python<'_>) -> PyResult<Py<PyAny>> {
let tag = self.tag();
let indices = PyList::new(py, self.indices.into_iter().map(PyTensor))?;
(tag, component_type_to_u8(self.component_type), indices).into_py_any(py)
}
}
impl FreeComponentHostSlot {
pub(crate) fn into_py_tagged(self, py: Python<'_>) -> PyResult<Py<PyAny>> {
let tag = self.tag();
let indices = PyList::new(py, self.host_indices.into_iter().map(PyTensor))?;
(tag, component_type_to_u8(self.component_type), indices).into_py_any(py)
}
}
impl RebuildFullToSwaMapping {
pub(crate) fn into_py_tagged(self, py: Python<'_>) -> PyResult<Py<PyAny>> {
let tag = self.tag();
let full_indices = PyList::new(py, self.full_indices.into_iter().map(PyTensor))?;
let swa_indices = PyList::new(py, self.swa_indices.into_iter().map(PyTensor))?;
(tag, full_indices, swa_indices).into_py_any(py)
}
}
impl RecoverSwaWithLockedFull {
pub(crate) fn into_py_tagged(self, py: Python<'_>) -> PyResult<Py<PyAny>> {
let tag = self.tag();
(
tag,
self.node_id,
PyTensor(self.kept_full),
PyTensor(self.incoming_full),
)
.into_py_any(py)
}
}
impl SwaRebuild {
pub(crate) fn into_py_tagged(self, py: Python<'_>) -> PyResult<Py<PyAny>> {
let tag = self.tag();
(tag, self.node_id, PyTensor(self.source_value)).into_py_any(py)
}
}
@@ -1,5 +1,7 @@
use super::*;
use crate::cache_action::{CacheAction, MambaEvictExcessPathStates};
use crate::components::{FULL, FullComponent, MAMBA, MambaComponent, SWA, SwaComponent};
use crate::test_utils::expect_action;
use crate::unified_tree_core::CacheInitParams;
use std::sync::{Arc, Mutex};
@@ -56,12 +58,12 @@ impl TreeComponent<Vec<i64>> for DefaultComponentForTest {
node_id: NodeIdx_,
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
target: EvictLayer,
) -> (usize, usize) {
cache_actions.push(CacheAction::MambaEvictExcessPathStates {
cache_actions.push(Box::new(MambaEvictExcessPathStates {
tail_node_id: tree_core.arena.node(node_id).id,
});
}));
FullComponent.evict_component(
tree_core,
node_id,
@@ -82,7 +84,7 @@ impl TreeComponent<Vec<i64>> for DefaultComponentForTest {
tracker: &mut HashMap<ComponentType, usize>,
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
) -> Option<NodeIdx_> {
unimplemented!()
}
@@ -234,13 +236,10 @@ fn action_node(
node
}
fn action_node_ids(actions: &[CacheAction]) -> Vec<usize> {
fn action_node_ids(actions: &[Box<dyn CacheAction>]) -> Vec<usize> {
actions
.iter()
.map(|action| match action {
CacheAction::MambaEvictExcessPathStates { tail_node_id } => *tail_node_id,
_ => panic!("unexpected component action"),
})
.map(|action| expect_action::<MambaEvictExcessPathStates>(action.as_ref()).tail_node_id)
.collect()
}
@@ -1,4 +1,5 @@
use super::*;
use crate::cache_action::UpdatePendingBackupOnNodeSplit;
use crate::components::{ComponentSet, FULL};
use crate::node::NodeAccessError;
use crate::test_utils::accumulate_step;
@@ -496,7 +497,7 @@ fn host_reclaim_orders_pending_internal_split_fragments_by_ack() {
// asynchronously under write_back; matching can split it before the ack.
let (prefix_idx, action) = tc.split_node_(tc.arena.resolve(a).unwrap(), 2);
let published_a = match action {
Some(CacheAction::UpdatePendingBackupOnNodeSplit {
Some(UpdatePendingBackupOnNodeSplit {
ack_id,
old_node_id,
new_node_id,
@@ -1,6 +1,7 @@
use super::*;
use crate::cache_action::{BackupKV, FreeComponentHostSlot, MambaEvictExcessPathStates};
use crate::components::{ComponentSet, FULL, MAMBA, SWA};
use crate::test_utils::{accumulate_step, action_kinds};
use crate::test_utils::{accumulate_step, action_kinds, expect_action};
use crate::unified_lru_list::UnifiedLRUList;
fn mamba_core(page_size: usize) -> UnifiedTreeCore<Vec<i64>> {
@@ -560,10 +561,7 @@ fn reinsert_full_backed_target_schedules_mamba_only_backup() {
let backups = result
.cache_actions
.iter()
.filter_map(|action| match action {
CacheAction::BackupKV(backup) => Some(backup),
_ => None,
})
.filter_map(|action| action.as_any().downcast_ref::<BackupKV>())
.collect::<Vec<_>>();
assert_eq!(backups.len(), 1);
assert_eq!(backups[0].node_ids, vec![leaf]);
@@ -587,7 +585,7 @@ fn reinsert_full_backed_target_schedules_mamba_only_backup() {
!pending
.cache_actions
.iter()
.any(|action| matches!(action, CacheAction::BackupKV(_)))
.any(|action| action.as_any().is::<BackupKV>())
);
}
@@ -607,7 +605,7 @@ fn write_back_reinsert_defers_the_mamba_backup_to_eviction() {
!result
.cache_actions
.iter()
.any(|action| matches!(action, CacheAction::BackupKV(_)))
.any(|action| action.as_any().is::<BackupKV>())
);
}
@@ -984,7 +982,7 @@ fn check_mamba_restored_during_expanded_swa_backup(pin_ancestor: bool) {
!inserted
.cache_actions
.iter()
.any(|action| matches!(action, CacheAction::BackupKV(_)))
.any(|action| action.as_any().is::<BackupKV>())
);
assert!(
tc.arena
@@ -2123,13 +2121,10 @@ fn prefetch_commit_frees_the_buffer_when_it_cannot_attach() {
.has_host_value(MAMBA)
);
assert!(insert_result.mamba_exist);
let CacheAction::FreeComponentHostSlot {
let FreeComponentHostSlot {
component_type,
host_indices,
} = &cache_actions[0]
else {
panic!("expected a FreeComponentHostSlot action");
};
} = expect_action::<FreeComponentHostSlot>(cache_actions[0].as_ref());
assert_eq!(*component_type, MAMBA);
assert!(host_indices[0].equal(&Tensor::from_slice(&[50i64])));
@@ -2260,14 +2255,14 @@ fn insert_commit_emits_the_path_cap_action_only_when_capped() {
let leaf = tc
.match_prefix(&match_params(&vec![1, 2]))
.best_match_node_id;
let [CacheAction::MambaEvictExcessPathStates { tail_node_id }] =
result.cache_actions.as_slice()
else {
let [action] = result.cache_actions.as_slice() else {
panic!(
"expected one MambaEvictExcessPathStates, got {:?}",
action_kinds(&result.cache_actions)
);
};
let MambaEvictExcessPathStates { tail_node_id } =
expect_action::<MambaEvictExcessPathStates>(action.as_ref());
assert_eq!(*tail_node_id, leaf);
let mut uncapped = mamba_core(/* page_size = */ 1);
+249 -279
View File
@@ -1,6 +1,10 @@
use super::*;
use crate::cache_action::{
BackupKV, FreeComponentHostSlot, FreeDeviceKV, FreeDeviceKVFullOnly, RebuildFullToSwaMapping,
RecoverSwaWithLockedFull, SwaRebuild, UpdatePendingBackupOnNodeSplit,
};
use crate::components::{ComponentSet, FULL, MAMBA, SWA};
use crate::test_utils::{accumulate_step, action_kinds, matched_device_indices};
use crate::test_utils::{accumulate_step, action_kinds, expect_action, matched_device_indices};
use crate::unified_tree_core::CacheInitParams;
#[test]
@@ -76,7 +80,7 @@ fn swa_window_attach_splits_both_edges_preserving_full_and_pending_actions() {
assert_eq!(actions.len(), 2);
for action in &actions {
assert!(
matches!(action, CacheAction::UpdatePendingBackupOnNodeSplit { ack_id: Some(91), old_node_id, new_child_node_id, .. } if *old_node_id == leaf_id && *new_child_node_id == leaf_id)
matches!(action.as_any().downcast_ref::<UpdatePendingBackupOnNodeSplit>(), Some(UpdatePendingBackupOnNodeSplit { ack_id: Some(91), old_node_id, new_child_node_id, .. }) if *old_node_id == leaf_id && *new_child_node_id == leaf_id)
);
}
let head = child_of(&tc, tc.arena.root(), &[1, 2]);
@@ -904,15 +908,13 @@ fn insert_new_leaf_in_window_emits_one_leaf_rebuild() {
assert_eq!(result.prefix_len, 0);
let root = tc.arena.root();
let leaf = child_of(&tc, root, &[1]);
let [
CacheAction::SwaRebuild {
node_id,
source_value,
},
] = result.cache_actions.as_slice()
else {
let [action] = result.cache_actions.as_slice() else {
panic!("expected one SwaRebuild action");
};
let SwaRebuild {
node_id,
source_value,
} = expect_action::<SwaRebuild>(action.as_ref());
assert_eq!(*node_id, tc.arena.node(leaf).id);
assert!(source_value.equal(&Tensor::from_slice(&[10i64, 11, 12])));
// The rebuild is deferred to apply time: the leaf is still an SWA tombstone.
@@ -940,15 +942,13 @@ fn insert_new_leaf_straddling_the_boundary_splits_and_rebuilds_the_tail() {
.device_value(parent, FULL)
.equal(&Tensor::from_slice(&[10i64, 11]))
);
let [
CacheAction::SwaRebuild {
node_id,
source_value,
},
] = result.cache_actions.as_slice()
else {
let [action] = result.cache_actions.as_slice() else {
panic!("expected one SwaRebuild action");
};
let SwaRebuild {
node_id,
source_value,
} = expect_action::<SwaRebuild>(action.as_ref());
assert_eq!(*node_id, tc.arena.node(child).id);
assert!(source_value.equal(&Tensor::from_slice(&[12i64, 13])));
}
@@ -984,19 +984,17 @@ fn insert_long_leaf_caps_the_window_and_rebuilds_the_older_prefix_first() {
let tail = child_of(&tc, capped, &[5]);
assert_eq!(tc.arena.node(capped).key, vec![1, 2, 3, 4]);
assert_eq!(tc.arena.node(tail).key, vec![5, 6]);
let [
CacheAction::SwaRebuild {
node_id: first_id,
source_value: first_value,
},
CacheAction::SwaRebuild {
node_id: second_id,
source_value: second_value,
},
] = result.cache_actions.as_slice()
else {
let [action0, action1] = result.cache_actions.as_slice() else {
panic!("expected two SwaRebuild actions");
};
let SwaRebuild {
node_id: first_id,
source_value: first_value,
} = expect_action::<SwaRebuild>(action0.as_ref());
let SwaRebuild {
node_id: second_id,
source_value: second_value,
} = expect_action::<SwaRebuild>(action1.as_ref());
assert_eq!(*first_id, tc.arena.node(capped).id);
assert!(first_value.equal(&Tensor::from_slice(&[10i64, 11, 12, 13])));
assert_eq!(*second_id, tc.arena.node(tail).id);
@@ -1035,9 +1033,10 @@ fn insert_overlap_with_live_swa_frees_the_whole_duplicate() {
.expect("live test node");
let result = tc.insert(&insert_params_swa(&vec![1, 2, 3], &[20, 21, 22], 0, 0));
assert_eq!(result.prefix_len, 3);
let [CacheAction::FreeDeviceKV(freed)] = result.cache_actions.as_slice() else {
let [action] = result.cache_actions.as_slice() else {
panic!("expected one FreeDeviceKV action");
};
let FreeDeviceKV { indices: freed } = expect_action::<FreeDeviceKV>(action.as_ref());
assert_eq!(freed.len(), 1);
assert!(freed[0].equal(&Tensor::from_slice(&[20i64, 21, 22])));
// The node keeps its original Full and SWA values.
@@ -1071,16 +1070,15 @@ fn insert_overlap_recovers_a_tombstone_inside_the_window() {
.device_value(leaf, FULL)
.equal(&Tensor::from_slice(&[20i64, 21, 22]))
);
let [
CacheAction::FreeDeviceKVFullOnly(freed),
CacheAction::SwaRebuild {
node_id,
source_value,
},
] = result.cache_actions.as_slice()
else {
let [action0, action1] = result.cache_actions.as_slice() else {
panic!("expected FreeDeviceKVFullOnly then SwaRebuild");
};
let FreeDeviceKVFullOnly { indices: freed } =
expect_action::<FreeDeviceKVFullOnly>(action0.as_ref());
let SwaRebuild {
node_id,
source_value,
} = expect_action::<SwaRebuild>(action1.as_ref());
assert!(freed[0].equal(&Tensor::from_slice(&[10i64, 11, 12])));
assert_eq!(*node_id, tc.arena.node(leaf).id);
assert!(source_value.equal(&Tensor::from_slice(&[20i64, 21, 22])));
@@ -1110,12 +1108,10 @@ fn insert_overlap_recovers_a_locked_swa_tombstone() {
.device_value(leaf, FULL)
.equal(&Tensor::from_slice(&[10i64, 11, 12]))
);
assert!(
result
.cache_actions
.iter()
.any(|action| matches!(action, CacheAction::RecoverSwaWithLockedFull { .. }))
);
assert!(result.cache_actions.iter().any(|action| matches!(
action.as_any().downcast_ref::<RecoverSwaWithLockedFull>(),
Some(RecoverSwaWithLockedFull { .. })
)));
}
#[test]
@@ -1147,16 +1143,14 @@ fn insert_overlap_can_preserve_unlocked_full_ids() {
assert_eq!(consumed_from, window_start);
assert_eq!(tc.arena.node(leaf).id, original_id);
assert_eq!(tc.arena.node(leaf).device_lock_ref(FULL), 0);
let [
CacheAction::RecoverSwaWithLockedFull {
node_id,
kept_full,
incoming_full,
},
] = actions.as_slice()
else {
let [action] = actions.as_slice() else {
panic!("expected recovery with canonical FULL IDs");
};
let RecoverSwaWithLockedFull {
node_id,
kept_full,
incoming_full,
} = expect_action::<RecoverSwaWithLockedFull>(action.as_ref());
assert_eq!(*node_id, original_id);
assert!(kept_full.equal(&Tensor::from_slice(&[10i64, 11, 12, 13][window_start..])));
assert!(incoming_full.equal(&Tensor::from_slice(&[20i64, 21, 22, 23][window_start..])));
@@ -1187,16 +1181,14 @@ fn insert_overlap_with_a_locked_full_emits_the_recover_action() {
.device_value(leaf, FULL)
.equal(&Tensor::from_slice(&[10i64, 11, 12]))
);
let [
CacheAction::RecoverSwaWithLockedFull {
node_id,
kept_full,
incoming_full,
},
] = result.cache_actions.as_slice()
else {
let [action] = result.cache_actions.as_slice() else {
panic!("expected one RecoverSwaWithLockedFull action");
};
let RecoverSwaWithLockedFull {
node_id,
kept_full,
incoming_full,
} = expect_action::<RecoverSwaWithLockedFull>(action.as_ref());
assert_eq!(*node_id, tc.arena.node(leaf).id);
assert!(kept_full.equal(&Tensor::from_slice(&[10i64, 11, 12])));
assert!(incoming_full.equal(&Tensor::from_slice(&[20i64, 21, 22])));
@@ -1238,20 +1230,21 @@ fn insert_overlap_straddling_the_boundary_splits_and_recovers_the_tail() {
.device_value(node, FULL)
.equal(&Tensor::from_slice(&[22i64, 23]))
);
let [
CacheAction::FreeDeviceKVFullOnly(old_tail),
CacheAction::SwaRebuild {
node_id,
source_value,
},
CacheAction::FreeDeviceKVFullOnly(duplicates),
] = result.cache_actions.as_slice()
else {
let [action0, action1, action2] = result.cache_actions.as_slice() else {
panic!(
"expected FreeDeviceKVFullOnly, SwaRebuild, FreeDeviceKVFullOnly, got {:?}",
action_kinds(&result.cache_actions)
);
};
let FreeDeviceKVFullOnly { indices: old_tail } =
expect_action::<FreeDeviceKVFullOnly>(action0.as_ref());
let SwaRebuild {
node_id,
source_value,
} = expect_action::<SwaRebuild>(action1.as_ref());
let FreeDeviceKVFullOnly {
indices: duplicates,
} = expect_action::<FreeDeviceKVFullOnly>(action2.as_ref());
assert!(old_tail[0].equal(&Tensor::from_slice(&[12i64, 13])));
assert_eq!(*node_id, tc.arena.node(node).id);
assert!(source_value.equal(&Tensor::from_slice(&[22i64, 23])));
@@ -1285,17 +1278,17 @@ fn insert_overlap_straddling_with_a_locked_full_defers_the_tail() {
.device_value(node, FULL)
.equal(&Tensor::from_slice(&[12i64, 13]))
);
let [
CacheAction::RecoverSwaWithLockedFull {
node_id,
kept_full,
incoming_full,
},
CacheAction::FreeDeviceKVFullOnly(duplicates),
] = result.cache_actions.as_slice()
else {
let [action0, action1] = result.cache_actions.as_slice() else {
panic!("expected RecoverSwaWithLockedFull then FreeDeviceKVFullOnly");
};
let RecoverSwaWithLockedFull {
node_id,
kept_full,
incoming_full,
} = expect_action::<RecoverSwaWithLockedFull>(action0.as_ref());
let FreeDeviceKVFullOnly {
indices: duplicates,
} = expect_action::<FreeDeviceKVFullOnly>(action1.as_ref());
assert_eq!(*node_id, tc.arena.node(node).id);
assert!(kept_full.equal(&Tensor::from_slice(&[12i64, 13])));
assert!(incoming_full.equal(&Tensor::from_slice(&[22i64, 23])));
@@ -1313,9 +1306,11 @@ fn insert_overlap_entirely_outside_the_window_is_all_duplicate() {
/* prev_prefix_len = */ 0,
/* swa_evicted_seqlen = */ 3,
));
let [CacheAction::FreeDeviceKVFullOnly(freed)] = result.cache_actions.as_slice() else {
let [action] = result.cache_actions.as_slice() else {
panic!("expected one FreeDeviceKVFullOnly action");
};
let FreeDeviceKVFullOnly { indices: freed } =
expect_action::<FreeDeviceKVFullOnly>(action.as_ref());
assert!(freed[0].equal(&Tensor::from_slice(&[20i64, 21, 22])));
}
@@ -1365,20 +1360,21 @@ fn insert_overlap_boundary_at_the_node_start_recovers_the_whole_node() {
.device_value(b, FULL)
.equal(&Tensor::from_slice(&[23i64, 24]))
);
let [
CacheAction::FreeDeviceKVFullOnly(duplicates),
CacheAction::FreeDeviceKVFullOnly(old_full),
CacheAction::SwaRebuild {
node_id,
source_value,
},
] = result.cache_actions.as_slice()
else {
let [action0, action1, action2] = result.cache_actions.as_slice() else {
panic!(
"expected FreeDeviceKVFullOnly, FreeDeviceKVFullOnly, SwaRebuild, got {:?}",
action_kinds(&result.cache_actions)
);
};
let FreeDeviceKVFullOnly {
indices: duplicates,
} = expect_action::<FreeDeviceKVFullOnly>(action0.as_ref());
let FreeDeviceKVFullOnly { indices: old_full } =
expect_action::<FreeDeviceKVFullOnly>(action1.as_ref());
let SwaRebuild {
node_id,
source_value,
} = expect_action::<SwaRebuild>(action2.as_ref());
assert!(duplicates[0].equal(&Tensor::from_slice(&[20i64, 21, 22])));
assert!(old_full[0].equal(&Tensor::from_slice(&[13i64, 14])));
assert_eq!(*node_id, tc.arena.node(b).id);
@@ -1420,21 +1416,24 @@ fn insert_overlap_straddling_a_second_level_node_recovers_the_tail() {
.device_value(b, FULL)
.equal(&Tensor::from_slice(&[24i64]))
);
let [
CacheAction::FreeDeviceKVFullOnly(duplicates_head),
CacheAction::FreeDeviceKVFullOnly(old_tail),
CacheAction::SwaRebuild {
node_id,
source_value,
},
CacheAction::FreeDeviceKVFullOnly(duplicates_tail),
] = result.cache_actions.as_slice()
else {
let [action0, action1, action2, action3] = result.cache_actions.as_slice() else {
panic!(
"expected FreeDeviceKVFullOnly, FreeDeviceKVFullOnly, SwaRebuild, FreeDeviceKVFullOnly, got {:?}",
action_kinds(&result.cache_actions)
);
};
let FreeDeviceKVFullOnly {
indices: duplicates_head,
} = expect_action::<FreeDeviceKVFullOnly>(action0.as_ref());
let FreeDeviceKVFullOnly { indices: old_tail } =
expect_action::<FreeDeviceKVFullOnly>(action1.as_ref());
let SwaRebuild {
node_id,
source_value,
} = expect_action::<SwaRebuild>(action2.as_ref());
let FreeDeviceKVFullOnly {
indices: duplicates_tail,
} = expect_action::<FreeDeviceKVFullOnly>(action3.as_ref());
assert!(duplicates_head[0].equal(&Tensor::from_slice(&[20i64, 21, 22])));
assert!(old_tail[0].equal(&Tensor::from_slice(&[14i64])));
assert_eq!(*node_id, tc.arena.node(b).id);
@@ -1473,21 +1472,23 @@ fn insert_overlap_straddling_a_second_level_locked_node_defers_the_tail() {
.device_value(b, FULL)
.equal(&Tensor::from_slice(&[14i64]))
);
let [
CacheAction::FreeDeviceKVFullOnly(duplicates_head),
CacheAction::RecoverSwaWithLockedFull {
node_id,
kept_full,
incoming_full,
},
CacheAction::FreeDeviceKVFullOnly(duplicates_tail),
] = result.cache_actions.as_slice()
else {
let [action0, action1, action2] = result.cache_actions.as_slice() else {
panic!(
"expected FreeDeviceKVFullOnly, RecoverSwaWithLockedFull, FreeDeviceKVFullOnly, got {:?}",
action_kinds(&result.cache_actions)
);
};
let FreeDeviceKVFullOnly {
indices: duplicates_head,
} = expect_action::<FreeDeviceKVFullOnly>(action0.as_ref());
let RecoverSwaWithLockedFull {
node_id,
kept_full,
incoming_full,
} = expect_action::<RecoverSwaWithLockedFull>(action1.as_ref());
let FreeDeviceKVFullOnly {
indices: duplicates_tail,
} = expect_action::<FreeDeviceKVFullOnly>(action2.as_ref());
assert!(duplicates_head[0].equal(&Tensor::from_slice(&[20i64, 21, 22])));
assert_eq!(*node_id, tc.arena.node(b).id);
assert!(kept_full.equal(&Tensor::from_slice(&[14i64])));
@@ -1518,10 +1519,7 @@ fn insert_overlap_prev_prefix_strictly_inside_the_node_still_recovers() {
.device_value(node, FULL)
.equal(&Tensor::from_slice(&[20i64, 21, 22, 23]))
);
assert!(result.cache_actions.iter().any(|action| matches!(
action,
CacheAction::SwaRebuild { node_id, .. } if *node_id == tc.arena.node(node).id
)));
assert!(result.cache_actions.iter().any(|action| matches!(action.as_any().downcast_ref::<SwaRebuild>(), Some(SwaRebuild { node_id, .. }) if *node_id == tc.arena.node(node).id)));
}
#[test]
@@ -1551,15 +1549,13 @@ fn reinsert_after_full_eviction_rebuilds_swa_from_the_fresh_kv() {
.device_value(leaf, FULL)
.equal(&Tensor::from_slice(&[20i64, 21, 22]))
);
let [
CacheAction::SwaRebuild {
node_id,
source_value,
},
] = result.cache_actions.as_slice()
else {
let [action] = result.cache_actions.as_slice() else {
panic!("expected one SwaRebuild action");
};
let SwaRebuild {
node_id,
source_value,
} = expect_action::<SwaRebuild>(action.as_ref());
assert_eq!(*node_id, tc.arena.node(leaf).id);
assert!(source_value.equal(&Tensor::from_slice(&[20i64, 21, 22])));
}
@@ -1592,15 +1588,13 @@ fn reinsert_straddling_the_boundary_splits_before_the_rebuild() {
.equal(&Tensor::from_slice(&[20i64, 21]))
);
assert!(!tc.arena.has_device_value(parent, SWA));
let [
CacheAction::SwaRebuild {
node_id,
source_value,
},
] = result.cache_actions.as_slice()
else {
let [action] = result.cache_actions.as_slice() else {
panic!("expected one SwaRebuild action");
};
let SwaRebuild {
node_id,
source_value,
} = expect_action::<SwaRebuild>(action.as_ref());
assert_eq!(*node_id, tc.arena.node(node).id);
assert!(source_value.equal(&Tensor::from_slice(&[22i64, 23])));
}
@@ -1641,15 +1635,9 @@ fn reinsert_straddling_the_boundary_at_a_second_level_node_splits_before_the_reb
.equal(&Tensor::from_slice(&[24i64]))
);
assert!(!tc.arena.has_device_value(p, SWA));
assert!(result.cache_actions.iter().any(|action| matches!(
action,
CacheAction::SwaRebuild { node_id, source_value }
if *node_id == tc.arena.node(b).id && source_value.equal(&Tensor::from_slice(&[24i64]))
)));
assert!(!result.cache_actions.iter().any(|action| matches!(
action,
CacheAction::SwaRebuild { node_id, .. } if *node_id == tc.arena.node(p).id
)));
assert!(result.cache_actions.iter().any(|action| matches!(action.as_any().downcast_ref::<SwaRebuild>(), Some(SwaRebuild { node_id, source_value })
if *node_id == tc.arena.node(b).id && source_value.equal(&Tensor::from_slice(&[24i64])))));
assert!(!result.cache_actions.iter().any(|action| matches!(action.as_any().downcast_ref::<SwaRebuild>(), Some(SwaRebuild { node_id, .. }) if *node_id == tc.arena.node(p).id)));
}
#[test]
@@ -1715,19 +1703,19 @@ fn reinsert_boundary_at_the_node_start_rebuilds_the_whole_node() {
/* swa_evicted_seqlen = */ 3,
));
assert_eq!(tc.arena.node(b).key, vec![4, 5]);
let [
CacheAction::FreeDeviceKVFullOnly(duplicates),
CacheAction::SwaRebuild {
node_id,
source_value,
},
] = result.cache_actions.as_slice()
else {
let [action0, action1] = result.cache_actions.as_slice() else {
panic!(
"expected FreeDeviceKVFullOnly then SwaRebuild, got {:?}",
action_kinds(&result.cache_actions)
);
};
let FreeDeviceKVFullOnly {
indices: duplicates,
} = expect_action::<FreeDeviceKVFullOnly>(action0.as_ref());
let SwaRebuild {
node_id,
source_value,
} = expect_action::<SwaRebuild>(action1.as_ref());
assert!(duplicates[0].equal(&Tensor::from_slice(&[20i64, 21, 22])));
assert_eq!(*node_id, tc.arena.node(b).id);
assert!(source_value.equal(&Tensor::from_slice(&[23i64, 24])));
@@ -1768,16 +1756,16 @@ fn walk_split_redistributes_the_live_swa_value() {
);
assert!(tc.device_lru_list(SWA).in_list(Some(parent)));
assert!(tc.device_lru_list(SWA).in_list(Some(node)));
let [
CacheAction::FreeDeviceKV(duplicates),
CacheAction::SwaRebuild {
node_id,
source_value,
},
] = result.cache_actions.as_slice()
else {
let [action0, action1] = result.cache_actions.as_slice() else {
panic!("expected FreeDeviceKV then SwaRebuild");
};
let FreeDeviceKV {
indices: duplicates,
} = expect_action::<FreeDeviceKV>(action0.as_ref());
let SwaRebuild {
node_id,
source_value,
} = expect_action::<SwaRebuild>(action1.as_ref());
assert!(duplicates[0].equal(&Tensor::from_slice(&[20i64, 21])));
assert_eq!(*node_id, tc.arena.node(leaf).id);
assert!(source_value.equal(&Tensor::from_slice(&[29i64])));
@@ -3968,15 +3956,9 @@ fn new_leaf_after_a_cached_prefix_splits_at_the_leaf_relative_boundary() {
.device_value(leaf, FULL)
.equal(&Tensor::from_slice(&[24i64, 25]))
);
assert!(result.cache_actions.iter().any(|action| matches!(
action,
CacheAction::SwaRebuild { node_id, source_value }
if *node_id == tc.arena.node(leaf).id && source_value.equal(&Tensor::from_slice(&[24i64, 25]))
)));
assert!(!result.cache_actions.iter().any(|action| matches!(
action,
CacheAction::SwaRebuild { node_id, .. } if *node_id == tc.arena.node(p).id
)));
assert!(result.cache_actions.iter().any(|action| matches!(action.as_any().downcast_ref::<SwaRebuild>(), Some(SwaRebuild { node_id, source_value })
if *node_id == tc.arena.node(leaf).id && source_value.equal(&Tensor::from_slice(&[24i64, 25])))));
assert!(!result.cache_actions.iter().any(|action| matches!(action.as_any().downcast_ref::<SwaRebuild>(), Some(SwaRebuild { node_id, .. }) if *node_id == tc.arena.node(p).id)));
}
#[test]
@@ -4006,41 +3988,42 @@ fn new_leaf_boundary_split_and_window_cap_compose_in_one_commit() {
let rebuilds: Vec<NodeId> = result
.cache_actions
.iter()
.filter_map(|action| match action {
CacheAction::SwaRebuild { node_id, .. } => Some(*node_id),
_ => None,
})
.filter_map(
|action| match action.as_any().downcast_ref::<SwaRebuild>() {
Some(SwaRebuild { node_id, .. }) => Some(*node_id),
_ => None,
},
)
.collect();
assert_eq!(
rebuilds,
vec![tc.arena.node(capped).id, tc.arena.node(leaf).id]
);
assert!(result.cache_actions.iter().any(|action| matches!(
action,
CacheAction::SwaRebuild { node_id, source_value }
if *node_id == tc.arena.node(capped).id && source_value.equal(&Tensor::from_slice(&[23i64, 24]))
)));
assert!(result.cache_actions.iter().any(|action| matches!(
action,
CacheAction::SwaRebuild { node_id, source_value }
if *node_id == tc.arena.node(leaf).id && source_value.equal(&Tensor::from_slice(&[25i64, 26]))
)));
assert!(result.cache_actions.iter().any(|action| matches!(action.as_any().downcast_ref::<SwaRebuild>(), Some(SwaRebuild { node_id, source_value })
if *node_id == tc.arena.node(capped).id && source_value.equal(&Tensor::from_slice(&[23i64, 24])))));
assert!(result.cache_actions.iter().any(|action| matches!(action.as_any().downcast_ref::<SwaRebuild>(), Some(SwaRebuild { node_id, source_value })
if *node_id == tc.arena.node(leaf).id && source_value.equal(&Tensor::from_slice(&[25i64, 26])))));
let split_ids: Vec<NodeId> = result
.cache_actions
.iter()
.filter_map(|action| match action {
CacheAction::UpdatePendingBackupOnNodeSplit {
ack_id,
old_node_id,
new_node_id,
new_child_node_id,
} => {
assert_eq!(*ack_id, None);
assert_eq!(*old_node_id, tc.arena.node(leaf).id);
assert_eq!(*new_child_node_id, tc.arena.node(leaf).id);
Some(*new_node_id)
.filter_map(|action| {
match action
.as_any()
.downcast_ref::<UpdatePendingBackupOnNodeSplit>()
{
Some(UpdatePendingBackupOnNodeSplit {
ack_id,
old_node_id,
new_node_id,
new_child_node_id,
}) => {
assert_eq!(*ack_id, None);
assert_eq!(*old_node_id, tc.arena.node(leaf).id);
assert_eq!(*new_child_node_id, tc.arena.node(leaf).id);
Some(*new_node_id)
}
_ => None,
}
_ => None,
})
.collect();
if buffer_only {
@@ -4051,13 +4034,23 @@ fn new_leaf_boundary_split_and_window_cap_compose_in_one_commit() {
let first_rebuild = result
.cache_actions
.iter()
.position(|action| matches!(action, CacheAction::SwaRebuild { .. }))
.position(|action| {
matches!(
action.as_any().downcast_ref::<SwaRebuild>(),
Some(SwaRebuild { .. })
)
})
.unwrap();
assert_eq!(
result.cache_actions[..first_rebuild]
.iter()
.filter(|action| {
matches!(action, CacheAction::UpdatePendingBackupOnNodeSplit { .. })
matches!(
action
.as_any()
.downcast_ref::<UpdatePendingBackupOnNodeSplit>(),
Some(UpdatePendingBackupOnNodeSplit { .. })
)
})
.count(),
2,
@@ -4199,11 +4192,8 @@ fn new_leaf_with_the_boundary_above_its_start_skips_the_split() {
// in-window and stays unsplit.
let leaf = child_of(&tc, a, &[4]);
assert_eq!(tc.arena.node(leaf).key, vec![4, 5]);
assert!(result.cache_actions.iter().any(|action| matches!(
action,
CacheAction::SwaRebuild { node_id, source_value }
if *node_id == tc.arena.node(leaf).id && source_value.equal(&Tensor::from_slice(&[23i64, 24]))
)));
assert!(result.cache_actions.iter().any(|action| matches!(action.as_any().downcast_ref::<SwaRebuild>(), Some(SwaRebuild { node_id, source_value })
if *node_id == tc.arena.node(leaf).id && source_value.equal(&Tensor::from_slice(&[23i64, 24])))));
}
#[test]
@@ -4237,20 +4227,21 @@ fn insert_overlap_straddling_with_a_partial_prev_prefix_recovers_the_tail() {
.device_value(node, FULL)
.equal(&Tensor::from_slice(&[22i64, 23]))
);
let [
CacheAction::FreeDeviceKVFullOnly(old_tail),
CacheAction::SwaRebuild {
node_id,
source_value,
},
CacheAction::FreeDeviceKVFullOnly(duplicates),
] = result.cache_actions.as_slice()
else {
let [action0, action1, action2] = result.cache_actions.as_slice() else {
panic!(
"expected FreeDeviceKVFullOnly, SwaRebuild, FreeDeviceKVFullOnly, got {:?}",
action_kinds(&result.cache_actions)
);
};
let FreeDeviceKVFullOnly { indices: old_tail } =
expect_action::<FreeDeviceKVFullOnly>(action0.as_ref());
let SwaRebuild {
node_id,
source_value,
} = expect_action::<SwaRebuild>(action1.as_ref());
let FreeDeviceKVFullOnly {
indices: duplicates,
} = expect_action::<FreeDeviceKVFullOnly>(action2.as_ref());
assert!(old_tail[0].equal(&Tensor::from_slice(&[12i64, 13])));
assert_eq!(*node_id, tc.arena.node(node).id);
assert!(source_value.equal(&Tensor::from_slice(&[22i64, 23])));
@@ -4284,11 +4275,8 @@ fn insert_overlap_straddling_recovers_across_page_two() {
.device_value(node, FULL)
.equal(&Tensor::from_slice(&[22i64, 23]))
);
assert!(result.cache_actions.iter().any(|action| matches!(
action,
CacheAction::SwaRebuild { node_id, source_value }
if *node_id == tc.arena.node(node).id && source_value.equal(&Tensor::from_slice(&[22i64, 23]))
)));
assert!(result.cache_actions.iter().any(|action| matches!(action.as_any().downcast_ref::<SwaRebuild>(), Some(SwaRebuild { node_id, source_value })
if *node_id == tc.arena.node(node).id && source_value.equal(&Tensor::from_slice(&[22i64, 23])))));
}
#[test]
@@ -4322,15 +4310,9 @@ fn reinsert_straddling_recovers_across_page_two() {
.device_value(node, FULL)
.equal(&Tensor::from_slice(&[22i64, 23]))
);
assert!(result.cache_actions.iter().any(|action| matches!(
action,
CacheAction::SwaRebuild { node_id, source_value }
if *node_id == tc.arena.node(node).id && source_value.equal(&Tensor::from_slice(&[22i64, 23]))
)));
assert!(!result.cache_actions.iter().any(|action| matches!(
action,
CacheAction::SwaRebuild { node_id, .. } if *node_id == tc.arena.node(parent).id
)));
assert!(result.cache_actions.iter().any(|action| matches!(action.as_any().downcast_ref::<SwaRebuild>(), Some(SwaRebuild { node_id, source_value })
if *node_id == tc.arena.node(node).id && source_value.equal(&Tensor::from_slice(&[22i64, 23])))));
assert!(!result.cache_actions.iter().any(|action| matches!(action.as_any().downcast_ref::<SwaRebuild>(), Some(SwaRebuild { node_id, .. }) if *node_id == tc.arena.node(parent).id)));
}
#[test]
@@ -4362,12 +4344,10 @@ fn reinsert_rebuilds_a_locked_tombstone() {
.node_mut(node)
.set_lock_ref_(ValueSlotIdx::device(SWA), 1);
let result = tc.insert(&insert_params_swa(&vec![1, 2], &[20, 21], 0, 0));
assert!(
result
.cache_actions
.iter()
.any(|action| matches!(action, CacheAction::SwaRebuild { .. }))
);
assert!(result.cache_actions.iter().any(|action| matches!(
action.as_any().downcast_ref::<SwaRebuild>(),
Some(SwaRebuild { .. })
)));
}
fn set_full_host(tc: &mut UnifiedTreeCore<Vec<i64>>, node: NodeIdx_) {
@@ -4955,13 +4935,10 @@ fn load_back_commit_attaches_chunks_and_emits_the_rebuild_action() {
assert!(tc.device_lru_list(SWA).in_list(Some(b)));
assert_eq!(tc.swa_evictable_size(), 2);
assert_eq!(cache_actions.len(), 1);
let CacheAction::RebuildFullToSwaMapping {
let RebuildFullToSwaMapping {
full_indices,
swa_indices,
} = &cache_actions[0]
else {
panic!("expected a RebuildFullToSwaMapping action");
};
} = expect_action::<RebuildFullToSwaMapping>(cache_actions[0].as_ref());
assert_eq!(full_indices.len(), 2);
assert!(full_indices[0].equal(&Tensor::from_slice(&[40i64])));
assert!(full_indices[1].equal(&Tensor::from_slice(&[41i64])));
@@ -5196,9 +5173,8 @@ fn prefetch_commit_drops_the_whole_window_when_underloaded() {
);
assert!(!tc.arena.node(a).has_host_value(SWA));
assert_eq!(cache_actions.len(), 1);
let CacheAction::FreeComponentHostSlot { host_indices, .. } = &cache_actions[0] else {
panic!("expected a host free");
};
let FreeComponentHostSlot { host_indices, .. } =
expect_action::<FreeComponentHostSlot>(cache_actions[0].as_ref());
assert!(host_indices[0].equal(&Tensor::from_slice(&[30i64, 31])));
}
@@ -5227,9 +5203,8 @@ fn prefetch_commit_without_a_target_releases_the_whole_buffer() {
Some(&storage_result),
);
assert_eq!(cache_actions.len(), 1);
let CacheAction::FreeComponentHostSlot { host_indices, .. } = &cache_actions[0] else {
panic!("expected a host free");
};
let FreeComponentHostSlot { host_indices, .. } =
expect_action::<FreeComponentHostSlot>(cache_actions[0].as_ref());
assert!(host_indices[0].equal(&Tensor::from_slice(&[30i64, 31])));
// An insert result without an inserted host node releases the same way.
@@ -5253,9 +5228,8 @@ fn prefetch_commit_without_a_target_releases_the_whole_buffer() {
Some(&storage_result),
);
assert_eq!(cache_actions.len(), 1);
let CacheAction::FreeComponentHostSlot { host_indices, .. } = &cache_actions[0] else {
panic!("expected a host free");
};
let FreeComponentHostSlot { host_indices, .. } =
expect_action::<FreeComponentHostSlot>(cache_actions[0].as_ref());
assert!(host_indices[0].equal(&Tensor::from_slice(&[30i64, 31])));
assert!(!tc.arena.node(a).has_host_value(SWA));
}
@@ -5294,9 +5268,8 @@ fn prefetch_commit_releases_a_shortened_window_under_a_non_root_anchor() {
assert!(!tc.arena.node(a).has_host_value(SWA));
assert!(!tc.host_lru_list(SWA).in_list(Some(c)));
assert_eq!(cache_actions.len(), 1);
let CacheAction::FreeComponentHostSlot { host_indices, .. } = &cache_actions[0] else {
panic!("expected a host free");
};
let FreeComponentHostSlot { host_indices, .. } =
expect_action::<FreeComponentHostSlot>(cache_actions[0].as_ref());
assert!(host_indices[0].equal(&Tensor::from_slice(&[30i64, 31])));
}
@@ -5346,9 +5319,8 @@ fn prefetch_commit_fills_tombstoned_nodes_and_releases_covered_ones() {
.equal(&Tensor::from_slice(&[40i64]))
);
assert_eq!(cache_actions.len(), 1);
let CacheAction::FreeComponentHostSlot { host_indices, .. } = &cache_actions[0] else {
panic!("expected a host free");
};
let FreeComponentHostSlot { host_indices, .. } =
expect_action::<FreeComponentHostSlot>(cache_actions[0].as_ref());
assert!(host_indices[0].equal(&Tensor::from_slice(&[31i64])));
}
@@ -5405,13 +5377,10 @@ fn release_swa_host_queues_a_free_action_for_non_empty_indices() {
let mut cache_actions = Vec::new();
swa_component(4).release_swa_host_(Tensor::from_slice(&[7i64]), &mut cache_actions);
assert_eq!(cache_actions.len(), 1);
let CacheAction::FreeComponentHostSlot {
let FreeComponentHostSlot {
component_type,
host_indices,
} = &cache_actions[0]
else {
panic!("expected a FreeComponentHostSlot action");
};
} = expect_action::<FreeComponentHostSlot>(cache_actions[0].as_ref());
assert_eq!(*component_type, SWA);
assert_eq!(host_indices.len(), 1);
assert!(host_indices[0].equal(&Tensor::from_slice(&[7i64])));
@@ -5499,8 +5468,10 @@ fn build_load_back_spec_includes_the_swa_transfers() {
);
assert_eq!(actions.len(), 1);
assert!(matches!(
actions[0],
CacheAction::RebuildFullToSwaMapping { .. }
(actions[0])
.as_any()
.downcast_ref::<RebuildFullToSwaMapping>(),
Some(RebuildFullToSwaMapping { .. })
));
}
@@ -5642,16 +5613,14 @@ fn check_swa_device_restored_during_full_load_back(pin_ancestor: bool) {
// A is outside the loaded SWA window. A new request emits the normal
// locked-Full repair action; resolve only its allocator/mapping boundary.
let inserted = tc.insert(&insert_params_swa(&vec![1], &[90], 0, 0));
let [
CacheAction::RecoverSwaWithLockedFull {
node_id,
kept_full,
incoming_full,
},
] = inserted.cache_actions.as_slice()
else {
let [action] = inserted.cache_actions.as_slice() else {
panic!("expected one RecoverSwaWithLockedFull action");
};
let RecoverSwaWithLockedFull {
node_id,
kept_full,
incoming_full,
} = expect_action::<RecoverSwaWithLockedFull>(action.as_ref());
assert_eq!(*node_id, a_id);
assert!(kept_full.equal(&Tensor::from_slice(&[50i64])));
assert!(incoming_full.equal(&Tensor::from_slice(&[90i64])));
@@ -6009,9 +5978,11 @@ fn write_through_offloads_a_boundary_split_leaf() {
let backups: Vec<_> = result
.cache_actions
.iter()
.filter_map(|action| match action {
CacheAction::BackupKV(backup) => Some(backup.node_ids.clone()),
_ => None,
.filter_map(|action| {
action
.as_any()
.downcast_ref::<BackupKV>()
.map(|backup| backup.node_ids.clone())
})
.collect();
assert_eq!(
@@ -6083,10 +6054,10 @@ fn deep_request_ring_tree_survives_full_backup_evict_and_load_back_rounds() {
let result = tc.insert(&insert_params_swa(&key, &kv, 0, swa_evicted_seqlen));
// Apply the emitted rebuilds the way the cache would.
for action in &result.cache_actions {
if let CacheAction::SwaRebuild {
if let Some(SwaRebuild {
node_id,
source_value,
} = action
}) = action.as_any().downcast_ref::<SwaRebuild>()
{
tc.set_component_device_value(*node_id, SWA, source_value.copy())
.expect("live test node");
@@ -6198,18 +6169,16 @@ fn recovered_swa_span_evicts_before_the_window_leaf() {
let prefix = child_of(&tc, root, &[1]);
let leaf = child_of(&tc, prefix, &[9]);
assert_eq!(tc.arena.node(prefix).key, vec![1, 2, 3, 4, 5, 6, 7, 8]);
let [
CacheAction::SwaRebuild {
node_id,
source_value,
},
] = result.cache_actions.as_slice()
else {
let [action] = result.cache_actions.as_slice() else {
panic!(
"expected one SwaRebuild action, got {:?}",
action_kinds(&result.cache_actions)
);
};
let SwaRebuild {
node_id,
source_value,
} = expect_action::<SwaRebuild>(action.as_ref());
assert_eq!(*node_id, tc.arena.node(leaf).id);
tc.set_component_device_value(*node_id, SWA, source_value.copy())
.expect("live test node");
@@ -6225,19 +6194,18 @@ fn recovered_swa_span_evicts_before_the_window_leaf() {
/* swa_evicted_seqlen = */ 0,
));
assert!(step.result.is_none());
let [
CacheAction::FreeDeviceKVFullOnly(old_full),
CacheAction::SwaRebuild {
node_id,
source_value,
},
] = step.actions.as_slice()
else {
let [action0, action1] = step.actions.as_slice() else {
panic!(
"expected FreeDeviceKVFullOnly then SwaRebuild, got {:?}",
action_kinds(&step.actions)
);
};
let FreeDeviceKVFullOnly { indices: old_full } =
expect_action::<FreeDeviceKVFullOnly>(action0.as_ref());
let SwaRebuild {
node_id,
source_value,
} = expect_action::<SwaRebuild>(action1.as_ref());
assert!(old_full[0].equal(&Tensor::from_slice(&[
100i64, 101, 102, 103, 104, 105, 106, 107
])));
@@ -6613,7 +6581,7 @@ fn write_back_reinsert_defers_swa_backup_while_write_through_publishes() {
!result
.cache_actions
.iter()
.any(|action| matches!(action, CacheAction::BackupKV(_)))
.any(|action| action.as_any().is::<BackupKV>())
);
tc.is_write_back = false;
@@ -6621,9 +6589,11 @@ fn write_back_reinsert_defers_swa_backup_while_write_through_publishes() {
let backups: Vec<_> = result
.cache_actions
.iter()
.filter_map(|action| match action {
CacheAction::BackupKV(backup) => Some(backup.node_ids.clone()),
_ => None,
.filter_map(|action| {
action
.as_any()
.downcast_ref::<BackupKV>()
.map(|backup| backup.node_ids.clone())
})
.collect();
assert_eq!(backups, vec![vec![leaf]]);
+10 -17
View File
@@ -35,21 +35,14 @@ pub(crate) fn accumulate_step(
}
}
/// Short variant names for diagnosing an action sequence's shape.
pub(crate) fn action_kinds(actions: &[CacheAction]) -> Vec<&'static str> {
actions
.iter()
.map(|action| match action {
CacheAction::FreeDeviceKV(_) => "FreeDeviceKV",
CacheAction::FreeDeviceKVFullOnly(_) => "FreeDeviceKVFullOnly",
CacheAction::BackupKV(_) => "BackupKV",
CacheAction::UpdatePendingBackupOnNodeSplit { .. } => "UpdatePendingBackupOnNodeSplit",
CacheAction::MambaEvictExcessPathStates { .. } => "MambaEvictExcessPathStates",
CacheAction::FreeComponentDeviceSlot { .. } => "FreeComponentDeviceSlot",
CacheAction::FreeComponentHostSlot { .. } => "FreeComponentHostSlot",
CacheAction::RebuildFullToSwaMapping { .. } => "RebuildFullToSwaMapping",
CacheAction::RecoverSwaWithLockedFull { .. } => "RecoverSwaWithLockedFull",
CacheAction::SwaRebuild { .. } => "SwaRebuild",
})
.collect()
/// Tags for diagnosing an action sequence's shape.
pub(crate) fn action_kinds(actions: &[Box<dyn CacheAction>]) -> Vec<&'static str> {
actions.iter().map(|action| action.tag()).collect()
}
pub(crate) fn expect_action<T: std::any::Any>(action: &dyn CacheAction) -> &T {
action
.as_any()
.downcast_ref::<T>()
.unwrap_or_else(|| panic!("expected {}, got {:?}", std::any::type_name::<T>(), action))
}
@@ -1,3 +1,6 @@
use crate::cache_action::{
BackupKV, FreeDeviceKV, RecoverSwaWithLockedFull, UpdatePendingBackupOnNodeSplit,
};
use std::sync::Mutex;
use tch::Tensor;
@@ -5,7 +8,7 @@ use tch::Tensor;
use super::*;
use crate::components::{ComponentSet, FULL, MAMBA, SWA};
use crate::node::{NodeAccessError, ValueSlotIdx};
use crate::test_utils::{accumulate_step, action_kinds, matched_device_indices};
use crate::test_utils::{accumulate_step, action_kinds, expect_action, matched_device_indices};
fn core() -> UnifiedTreeCore<Vec<i64>> {
UnifiedTreeCore::new(CacheInitParams::default(), vec![FULL])
@@ -314,7 +317,7 @@ impl TreeComponent<Vec<i64>> for RecordingComponentForTest {
_node_id: NodeIdx_,
_device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
_host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
_cache_actions: &mut Vec<CacheAction>,
_cache_actions: &mut CacheActions,
_target: EvictLayer,
) -> (usize, usize) {
unimplemented!()
@@ -330,7 +333,7 @@ impl TreeComponent<Vec<i64>> for RecordingComponentForTest {
_tracker: &mut HashMap<ComponentType, usize>,
_device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
_host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
_cache_actions: &mut Vec<CacheAction>,
_cache_actions: &mut CacheActions,
) -> Option<NodeIdx_> {
unimplemented!()
}
@@ -366,7 +369,7 @@ impl TreeComponent<Vec<i64>> for RecordingComponentForTest {
tracker: &mut HashMap<ComponentType, usize>,
_device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
_host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
_cache_actions: &mut Vec<CacheAction>,
_cache_actions: &mut CacheActions,
) {
let tracked = tracker[&SWA];
self.host_eviction_calls
@@ -383,7 +386,7 @@ impl TreeComponent<Vec<i64>> for RecordingComponentForTest {
tracker: &mut HashMap<ComponentType, usize>,
_device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
_host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
_cache_actions: &mut Vec<CacheAction>,
_cache_actions: &mut CacheActions,
) {
let tracked = tracker[&SWA];
self.host_eviction_calls
@@ -440,7 +443,7 @@ impl TreeComponent<Vec<i64>> for CountingComponentForTest {
_node_id: NodeIdx_,
_device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
_host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
_cache_actions: &mut Vec<CacheAction>,
_cache_actions: &mut CacheActions,
_target: EvictLayer,
) -> (usize, usize) {
unimplemented!()
@@ -456,7 +459,7 @@ impl TreeComponent<Vec<i64>> for CountingComponentForTest {
_tracker: &mut HashMap<ComponentType, usize>,
_device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
_host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
_cache_actions: &mut Vec<CacheAction>,
_cache_actions: &mut CacheActions,
) -> Option<NodeIdx_> {
unimplemented!()
}
@@ -522,7 +525,7 @@ impl TreeComponent<Vec<i64>> for LowPriorityComponentForTest {
_node_id: NodeIdx_,
_device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
_host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
_cache_actions: &mut Vec<CacheAction>,
_cache_actions: &mut CacheActions,
_target: EvictLayer,
) -> (usize, usize) {
unimplemented!()
@@ -538,7 +541,7 @@ impl TreeComponent<Vec<i64>> for LowPriorityComponentForTest {
_tracker: &mut HashMap<ComponentType, usize>,
_device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
_host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
_cache_actions: &mut Vec<CacheAction>,
_cache_actions: &mut CacheActions,
) -> Option<NodeIdx_> {
unimplemented!()
}
@@ -599,7 +602,7 @@ impl TreeComponent<Vec<i64>> for SwaComponentForTest {
_node_id: NodeIdx_,
_device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
_host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
_cache_actions: &mut Vec<CacheAction>,
_cache_actions: &mut CacheActions,
_target: EvictLayer,
) -> (usize, usize) {
unimplemented!()
@@ -615,7 +618,7 @@ impl TreeComponent<Vec<i64>> for SwaComponentForTest {
_tracker: &mut HashMap<ComponentType, usize>,
_device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
_host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
_cache_actions: &mut Vec<CacheAction>,
_cache_actions: &mut CacheActions,
) -> Option<NodeIdx_> {
unimplemented!()
}
@@ -682,7 +685,7 @@ impl TreeComponent<Vec<i64>> for SwaEvictionComponentForTest {
node_id: NodeIdx_,
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
_cache_actions: &mut Vec<CacheAction>,
_cache_actions: &mut CacheActions,
target: EvictLayer,
) -> (usize, usize) {
self.evictions.lock().unwrap().push((node_id, target));
@@ -730,7 +733,7 @@ impl TreeComponent<Vec<i64>> for SwaEvictionComponentForTest {
_tracker: &mut HashMap<ComponentType, usize>,
_device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
_host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
_cache_actions: &mut Vec<CacheAction>,
_cache_actions: &mut CacheActions,
) -> Option<NodeIdx_> {
unimplemented!()
}
@@ -2444,12 +2447,13 @@ fn insert_full_overlap_frees_the_duplicates() {
let result = tc.insert(&insert_params(&vec![1, 2, 3], &[20, 21, 22]));
assert_eq!(result.prefix_len, 3);
// Nothing was consumed by any component: the whole overlap is duplicate.
let [CacheAction::FreeDeviceKV(freed)] = result.cache_actions.as_slice() else {
let [action] = result.cache_actions.as_slice() else {
panic!(
"expected one FreeDeviceKV action, got {:?}",
action_kinds(&result.cache_actions)
);
};
let FreeDeviceKV { indices: freed } = expect_action::<FreeDeviceKV>(action.as_ref());
assert_eq!(freed.len(), 1);
assert!(freed[0].equal(&Tensor::from_slice(&[20i64, 21, 22])));
}
@@ -2463,12 +2467,13 @@ fn insert_prev_prefix_len_narrows_the_dup_window() {
prev_prefix_len: 2,
..insert_params(&vec![1, 2, 3], &[20, 21, 22])
});
let [CacheAction::FreeDeviceKV(freed)] = result.cache_actions.as_slice() else {
let [action] = result.cache_actions.as_slice() else {
panic!(
"expected one FreeDeviceKV action, got {:?}",
action_kinds(&result.cache_actions)
);
};
let FreeDeviceKV { indices: freed } = expect_action::<FreeDeviceKV>(action.as_ref());
assert!(freed[0].equal(&Tensor::from_slice(&[22i64])));
}
@@ -2479,12 +2484,13 @@ fn insert_extends_an_existing_prefix() {
let result = tc.insert(&insert_params(&vec![1, 2, 3, 4, 5], &[20, 21, 22, 13, 14]));
assert_eq!(result.prefix_len, 3);
// The existing node's overlap is duplicate; the new suffix is kept.
let [CacheAction::FreeDeviceKV(freed)] = result.cache_actions.as_slice() else {
let [action] = result.cache_actions.as_slice() else {
panic!(
"expected one FreeDeviceKV action, got {:?}",
action_kinds(&result.cache_actions)
);
};
let FreeDeviceKV { indices: freed } = expect_action::<FreeDeviceKV>(action.as_ref());
assert_eq!(freed.len(), 1);
assert!(freed[0].equal(&Tensor::from_slice(&[20i64, 21, 22])));
let matched = tc.match_prefix(&match_params(&vec![1, 2, 3, 4, 5]));
@@ -2506,12 +2512,13 @@ fn insert_prev_prefix_len_spans_a_multi_node_walk() {
..insert_params(&vec![1, 2, 3, 4, 5], &[30, 31, 32, 33, 34])
});
assert_eq!(result.prefix_len, 5);
let [CacheAction::FreeDeviceKV(freed)] = result.cache_actions.as_slice() else {
let [action] = result.cache_actions.as_slice() else {
panic!(
"expected one FreeDeviceKV action, got {:?}",
action_kinds(&result.cache_actions)
);
};
let FreeDeviceKV { indices: freed } = expect_action::<FreeDeviceKV>(action.as_ref());
assert_eq!(freed.len(), 1);
assert!(freed[0].equal(&Tensor::from_slice(&[33i64, 34])));
}
@@ -2528,12 +2535,13 @@ fn insert_prev_prefix_len_narrows_mid_node_on_a_multi_node_walk() {
..insert_params(&vec![1, 2, 3, 4, 5], &[30, 31, 32, 33, 34])
});
assert_eq!(result.prefix_len, 5);
let [CacheAction::FreeDeviceKV(freed)] = result.cache_actions.as_slice() else {
let [action] = result.cache_actions.as_slice() else {
panic!(
"expected one FreeDeviceKV action, got {:?}",
action_kinds(&result.cache_actions)
);
};
let FreeDeviceKV { indices: freed } = expect_action::<FreeDeviceKV>(action.as_ref());
assert_eq!(freed.len(), 1);
assert!(freed[0].equal(&Tensor::from_slice(&[34i64])));
}
@@ -2680,9 +2688,11 @@ fn insert_threshold_crossing_emits_the_backup_kv_action() {
let backups: Vec<_> = result
.cache_actions
.iter()
.filter_map(|action| match action {
CacheAction::BackupKV(backup) => Some(backup.node_ids.clone()),
_ => None,
.filter_map(|action| {
action
.as_any()
.downcast_ref::<BackupKV>()
.map(|backup| backup.node_ids.clone())
})
.collect();
assert_eq!(backups, vec![vec![leaf]]);
@@ -2701,7 +2711,7 @@ fn external_linker_hashes_new_nodes_and_triggers_offload_action() {
let result = tc.insert(&insert_params(&vec![1, 2], &[10, 11]));
let leaf = result.last_device_node_id.unwrap();
assert!(result.cache_actions.iter().any(|action| {
matches!(action, CacheAction::BackupKV(backup) if backup.node_ids == vec![leaf])
matches!(action.as_any().downcast_ref::<BackupKV>(), Some(backup) if backup.node_ids == vec![leaf])
}));
let transfers = tc
@@ -3090,10 +3100,7 @@ fn write_back_defers_swa_window_backup_until_eviction() {
let backups: Vec<&BackupKV> = result
.cache_actions
.iter()
.filter_map(|action| match action {
CacheAction::BackupKV(backup) => Some(backup),
_ => None,
})
.filter_map(|action| action.as_any().downcast_ref::<BackupKV>())
.collect();
assert!(backups.is_empty());
}
@@ -3168,16 +3175,15 @@ fn split_updates_pending_backup_with_or_without_a_tree_ack() {
let prefix_id = result.best_match_node_id;
assert_ne!(prefix_id, tail_id);
if pending || buffer_only {
assert!(matches!(
result.cache_actions.as_slice(),
[CacheAction::UpdatePendingBackupOnNodeSplit {
assert!(
matches!(result.cache_actions.as_slice(), [action] if matches!(action.as_any().downcast_ref::<UpdatePendingBackupOnNodeSplit>(), Some(UpdatePendingBackupOnNodeSplit {
ack_id,
old_node_id,
new_node_id,
new_child_node_id,
}] if *ack_id == pending.then_some(tail_id) && *old_node_id == tail_id
&& *new_node_id == prefix_id && *new_child_node_id == tail_id
));
}) if *ack_id == pending.then_some(tail_id) && *old_node_id == tail_id
&& *new_node_id == prefix_id && *new_child_node_id == tail_id))
);
} else {
assert!(result.cache_actions.is_empty());
}
@@ -4063,16 +4069,21 @@ fn finish_write_through_after_a_split_publishes_both_fragments() {
let new_node_id = result
.cache_actions
.iter()
.find_map(|action| match action {
CacheAction::UpdatePendingBackupOnNodeSplit {
ack_id,
new_node_id,
..
} => {
assert_eq!(*ack_id, Some(leaf));
Some(*new_node_id)
.find_map(|action| {
match action
.as_any()
.downcast_ref::<UpdatePendingBackupOnNodeSplit>()
{
Some(UpdatePendingBackupOnNodeSplit {
ack_id,
new_node_id,
..
}) => {
assert_eq!(*ack_id, Some(leaf));
Some(*new_node_id)
}
_ => None,
}
_ => None,
})
.expect("the split relocates the pending write-through");
// Nothing reaches the host tier before the ack.
@@ -4531,10 +4542,9 @@ fn insert_host_refill_preserves_split_actions_and_lengths() {
assert_eq!(result.total_len, 2);
assert!(result.inserted_host_node.is_some());
assert!(!result.host_insert_dropped);
assert!(matches!(
result.cache_actions.as_slice(),
[CacheAction::UpdatePendingBackupOnNodeSplit { ack_id, .. }] if *ack_id == Some(leaf)
));
assert!(
matches!(result.cache_actions.as_slice(), [action] if matches!(action.as_any().downcast_ref::<UpdatePendingBackupOnNodeSplit>(), Some(UpdatePendingBackupOnNodeSplit { ack_id, .. }) if *ack_id == Some(leaf)))
);
}
#[test]
@@ -5614,12 +5624,13 @@ fn insert_page_size_two_splits_mid_page_divergence_at_the_page_boundary() {
// The keys share 3 atoms but pages quantize the overlap down to 2.
let result = tc.insert(&insert_params(&vec![1, 2, 3, 9], &[20, 21, 22, 29]));
assert_eq!(result.prefix_len, 2);
let [CacheAction::FreeDeviceKV(freed)] = result.cache_actions.as_slice() else {
let [action] = result.cache_actions.as_slice() else {
panic!(
"expected one FreeDeviceKV action, got {:?}",
action_kinds(&result.cache_actions)
);
};
let FreeDeviceKV { indices: freed } = expect_action::<FreeDeviceKV>(action.as_ref());
assert_eq!(freed.len(), 1);
assert!(freed[0].equal(&Tensor::from_slice(&[20i64, 21])));
let prefix = tc
@@ -8641,12 +8652,13 @@ fn insert_unevicts_a_tombstoned_deep_node() {
.device_value(tc.arena.resolve(leaf).expect("live test node"), FULL)
.equal(&Tensor::from_slice(&[33i64, 34]))
);
let [CacheAction::FreeDeviceKV(freed)] = result.cache_actions.as_slice() else {
let [action] = result.cache_actions.as_slice() else {
panic!(
"expected one FreeDeviceKV action, got {:?}",
action_kinds(&result.cache_actions)
);
};
let FreeDeviceKV { indices: freed } = expect_action::<FreeDeviceKV>(action.as_ref());
assert!(freed[0].equal(&Tensor::from_slice(&[30i64, 31, 32])));
}
@@ -8658,12 +8670,13 @@ fn insert_ragged_key_onto_an_existing_prefix_page_size_two() {
assert_eq!(result.prefix_len, 2);
// The ragged tail never enters: no new leaf, the aligned span is duplicate.
assert_eq!(tc.arena.len(), 2);
let [CacheAction::FreeDeviceKV(freed)] = result.cache_actions.as_slice() else {
let [action] = result.cache_actions.as_slice() else {
panic!(
"expected one FreeDeviceKV action, got {:?}",
action_kinds(&result.cache_actions)
);
};
let FreeDeviceKV { indices: freed } = expect_action::<FreeDeviceKV>(action.as_ref());
assert!(freed[0].equal(&Tensor::from_slice(&[20i64, 21])));
}
@@ -8675,16 +8688,18 @@ fn insert_ragged_key_traverses_a_node_before_the_tail_page_size_two() {
let result = tc.insert(&insert_params(&vec![1, 2, 3, 4, 5], &[30, 31, 32, 33, 34]));
assert_eq!(result.prefix_len, 4);
assert_eq!(tc.arena.len(), 3);
let [
CacheAction::FreeDeviceKV(freed_head),
CacheAction::FreeDeviceKV(freed_tail),
] = result.cache_actions.as_slice()
else {
let [action0, action1] = result.cache_actions.as_slice() else {
panic!(
"expected one FreeDeviceKV per walked node, got {:?}",
action_kinds(&result.cache_actions)
);
};
let FreeDeviceKV {
indices: freed_head,
} = expect_action::<FreeDeviceKV>(action0.as_ref());
let FreeDeviceKV {
indices: freed_tail,
} = expect_action::<FreeDeviceKV>(action1.as_ref());
assert!(freed_head[0].equal(&Tensor::from_slice(&[30i64, 31])));
assert!(freed_tail[0].equal(&Tensor::from_slice(&[32i64, 33])));
}
@@ -8714,7 +8729,7 @@ fn deferrable_dup_frees_ride_the_final_step_without_suspension() {
let mut tc = core();
tc.insert(&insert_params(&vec![1, 2, 3], &[10, 11, 12]));
let step = tc.begin_insert(&insert_params(&vec![1, 2, 3, 4], &[20, 21, 22, 13]));
assert_eq!(action_kinds(&step.actions), vec!["FreeDeviceKV"]);
assert_eq!(action_kinds(&step.actions), vec!["free_device_kv"]);
let result = step
.result
.expect("a deferrable-only walk completes in one step");
@@ -8749,11 +8764,9 @@ fn walk_backup_crossing_suspends_then_resume_completes() {
assert!(tc.has_ongoing_insert());
assert_eq!(
action_kinds(&step.actions),
vec!["FreeDeviceKV", "BackupKV"]
vec!["free_device_kv", "backup_kv"]
);
let CacheAction::BackupKV(backup) = &step.actions[1] else {
unreachable!();
};
let backup = expect_action::<BackupKV>(step.actions[1].as_ref());
assert_eq!(backup.node_ids, vec![tc.arena.node(a).id]);
let done = tc.resume_insert();
@@ -8841,7 +8854,7 @@ fn resume_insert_completes_after_an_on_path_host_leaf_is_evicted() {
assert!(step.result.is_none());
assert_eq!(
action_kinds(&step.actions),
vec!["FreeDeviceKV", "BackupKV"]
vec!["free_device_kv", "backup_kv"]
);
// The barrier's backup host-evicts the on-path H-leaf before committing.
let mut tracker = HashMap::from([(FULL, 0)]);
@@ -8889,9 +8902,11 @@ fn aborted_barrier_crossing_refires_on_the_next_insert() {
let backups: Vec<_> = step
.actions
.iter()
.filter_map(|action| match action {
CacheAction::BackupKV(backup) => Some(backup.node_ids.clone()),
_ => None,
.filter_map(|action| {
action
.as_any()
.downcast_ref::<BackupKV>()
.map(|backup| backup.node_ids.clone())
})
.collect();
assert_eq!(backups, vec![vec![tc.arena.node(a).id]]);
@@ -8959,9 +8974,11 @@ fn one_insert_walk_fires_two_crossings_around_a_backuped_middle() {
let backups: Vec<_> = result
.cache_actions
.iter()
.filter_map(|action| match action {
CacheAction::BackupKV(backup) => Some(backup.node_ids.clone()),
_ => None,
.filter_map(|action| {
action
.as_any()
.downcast_ref::<BackupKV>()
.map(|backup| backup.node_ids.clone())
})
.collect();
assert_eq!(backups, vec![vec![top], vec![deep]]);
@@ -9518,17 +9535,17 @@ fn caller_owned_overlap_drains_split_and_backup_before_duplicate_frees() {
);
assert_eq!(
action_kinds(&walk.pending_actions),
vec!["UpdatePendingBackupOnNodeSplit"]
vec!["update_pending_backup_on_node_split"]
);
assert!(!walk.has_barrier_actions_since(0));
walk.pending_actions.push(CacheAction::BackupKV(BackupKV {
walk.pending_actions.push(Box::new(BackupKV {
node_ids: vec![node.id],
}));
assert!(walk.has_barrier_actions_since(0));
let actions = std::mem::take(&mut walk.pending_actions);
assert_eq!(
action_kinds(&actions),
vec!["UpdatePendingBackupOnNodeSplit", "BackupKV"]
vec!["update_pending_backup_on_node_split", "backup_kv"]
);
assert_eq!(tc.arena.node(head).parent(), tc.arena.root());
assert_eq!(
@@ -9550,10 +9567,8 @@ fn caller_owned_overlap_drains_split_and_backup_before_duplicate_frees() {
));
let step = walk.finish();
assert_eq!(step.result.unwrap().prefix_len, 2);
assert_eq!(action_kinds(&step.actions), vec!["FreeDeviceKV"]);
let CacheAction::FreeDeviceKV(frees) = &step.actions[0] else {
unreachable!()
};
assert_eq!(action_kinds(&step.actions), vec!["free_device_kv"]);
let FreeDeviceKV { indices: frees } = expect_action::<FreeDeviceKV>(step.actions[0].as_ref());
assert!(frees[0].equal(&Tensor::from_slice(&[20i64, 21])));
assert!(!tc.has_ongoing_insert());
tc.sanity_check(&[], &[]);
@@ -9577,7 +9592,7 @@ fn caller_owned_walk_can_abort_after_split_without_claiming_donor() {
drop(walk);
assert_eq!(
action_kinds(&actions),
vec!["UpdatePendingBackupOnNodeSplit"]
vec!["update_pending_backup_on_node_split"]
);
assert!(!tc.has_ongoing_insert());
assert!(tc.end_insert().is_empty());
@@ -9590,7 +9605,7 @@ fn caller_owned_walk_can_abort_after_split_without_claiming_donor() {
assert!(matched_device_indices(&tc, &matched).equal(&Tensor::from_slice(&[10i64, 11, 12, 13])));
let result = tc.insert(&params);
assert_eq!(result.prefix_len, 2);
assert_eq!(action_kinds(&result.cache_actions), vec!["FreeDeviceKV"]);
assert_eq!(action_kinds(&result.cache_actions), vec!["free_device_kv"]);
tc.sanity_check(&[], &[]);
}
@@ -9675,15 +9690,12 @@ fn overlap_context_reaches_swa_before_it_replaces_unlocked_full_ids() {
);
assert!(walk.has_barrier_actions_since(start));
let actions = std::mem::take(&mut walk.pending_actions);
assert_eq!(action_kinds(&actions), vec!["RecoverSwaWithLockedFull"]);
let CacheAction::RecoverSwaWithLockedFull {
assert_eq!(action_kinds(&actions), vec!["recover_swa_with_locked_full"]);
let RecoverSwaWithLockedFull {
node_id: actual_node_id,
kept_full,
incoming_full,
} = &actions[0]
else {
unreachable!()
};
} = expect_action::<RecoverSwaWithLockedFull>(actions[0].as_ref());
assert_eq!(*actual_node_id, node_id);
assert!(kept_full.equal(&kept));
assert!(incoming_full.equal(&Tensor::from_slice(&[20i64, 21])));
+56 -116
View File
@@ -1,6 +1,8 @@
//! The radix prefix tree of cached KV.
#![allow(unused_variables)]
pub use crate::cache_action::{BackupKV, CacheAction, CacheActions};
use crate::cache_action::{FreeDeviceKV, FreeDeviceKVFullOnly, UpdatePendingBackupOnNodeSplit};
use std::cmp::Reverse;
use std::collections::{BinaryHeap, HashMap, HashSet};
use std::sync::Arc;
@@ -160,7 +162,7 @@ pub struct MatchResult {
/// of other components.
pub full_kv_hit_length: usize,
/// Actions for the controller to apply.
pub cache_actions: Vec<CacheAction>,
pub cache_actions: CacheActions,
}
/// Params for a prefix match; the key is borrowed from the caller.
@@ -229,7 +231,7 @@ pub struct InsertResult {
/// Incoming ranges retained by each component, in key-relative atom offsets.
pub adopted_ranges: Option<HashMap<ComponentType, Vec<(usize, usize)>>>,
/// Actions for the controller to apply.
pub cache_actions: Vec<CacheAction>,
pub cache_actions: CacheActions,
}
impl InsertResult {
@@ -260,7 +262,7 @@ impl InsertResult {
/// One step of a resumable insert: the Controller executes `actions`, then
/// resumes while `result` is None; `result` is set on the final step.
pub struct InsertStepResult {
pub actions: Vec<CacheAction>,
pub actions: CacheActions,
pub result: Option<InsertResult>,
}
@@ -305,7 +307,7 @@ pub(crate) struct InsertWalkState<K: ChildKeyType> {
target_node_id: Option<NodeIdx_>,
result: Option<InsertResult>,
/// Emitted actions awaiting the next barrier flush (or the final step).
pub(crate) pending_actions: Vec<CacheAction>,
pub(crate) pending_actions: CacheActions,
}
/// A new walk or the result of an empty or declined insert.
@@ -331,14 +333,9 @@ impl<K: ChildKeyType> InsertWalkState<K> {
/// Free and split bookkeeping can wait; other actions pause the walk.
pub(crate) fn has_barrier_actions_since(&self, start: usize) -> bool {
self.pending_actions[start..].iter().any(|action| {
!matches!(
action,
CacheAction::FreeDeviceKV(_)
| CacheAction::FreeDeviceKVFullOnly(_)
| CacheAction::UpdatePendingBackupOnNodeSplit { .. }
)
})
self.pending_actions[start..]
.iter()
.any(|action| action.requires_insert_barrier())
}
}
@@ -352,61 +349,6 @@ pub struct KvCanaryWalkResult {
pub prev_slot_indices: Vec<i64>,
}
/// A queued cache IO action.
#[derive(Debug)]
pub enum CacheAction {
/// Duplicate device KV slices the cache frees after the insert.
FreeDeviceKV(Vec<Tensor>),
/// Free the full side only, for a tombstoned node whose SWA peers are gone;
/// FreeDeviceKV would release the SWA side twice.
FreeDeviceKVFullOnly(Vec<Tensor>),
/// A device->host backup work item (the write-through threshold fired).
BackupKV(BackupKV),
/// Update cache-side write bookkeeping after a node split:
///
/// parent -> node => parent -> new_node -> new_child
///
/// old_node_id (the pre-split node) is replaced by new_node_id + new_child_node_id.
UpdatePendingBackupOnNodeSplit {
/// None when buffer-only bookkeeping has no tree-tracked write.
ack_id: Option<usize>,
old_node_id: NodeId,
new_node_id: NodeId,
new_child_node_id: NodeId,
},
/// Per-path Mamba state-cap eviction from the tail's root path; applied at
/// the insert's commit barrier, after the walk-time backups whose
/// write-through locks shield the backed-up chain.
MambaEvictExcessPathStates { tail_node_id: NodeId },
/// Free only the given component's device KV slots.
FreeComponentDeviceSlot {
component_type: ComponentType,
indices: Vec<Tensor>,
},
/// Free the given component's host KV pages.
FreeComponentHostSlot {
component_type: ComponentType,
host_indices: Vec<Tensor>,
},
/// Rebuild the SWA allocator's full->swa index mapping for loaded chunks.
RebuildFullToSwaMapping {
full_indices: Vec<Tensor>,
swa_indices: Vec<Tensor>,
},
/// Recover an SWA tombstone whose full is locked: keep the locked full, remap
/// it onto the incoming full's SWA translation, and free only the incoming full.
RecoverSwaWithLockedFull {
node_id: NodeId,
kept_full: Tensor,
incoming_full: Tensor,
},
/// Rebuild a node's SWA value by translating its source full value, then store it.
SwaRebuild {
node_id: NodeId,
source_value: Tensor,
},
}
/// A HiCache pool transfer descriptor.
#[derive(Default)]
pub struct PoolTransfer {
@@ -482,15 +424,6 @@ pub struct PoolTransferResult {
pub extra_pool_hit_pages: HashMap<PoolName, usize>,
}
/// A device->host backup work item for the cache to execute.
#[derive(Default, Debug)]
pub struct BackupKV {
/// Backup these nodes device->host in order, stopping at the first failure; the
/// caller orders them parent-before-child for write-through and child-first for
/// write-back. External handles: the list crosses to the orchestrator.
pub node_ids: Vec<NodeId>,
}
/// A device->storage backup spec.
#[derive(Default)]
pub struct StorageBackupSpec {
@@ -654,7 +587,7 @@ pub struct EvictionStepResult {
pub tracker: HashMap<ComponentType, usize>,
pub device_frees: HashMap<ComponentType, Vec<Tensor>>,
pub host_frees: HashMap<ComponentType, Vec<Tensor>>,
pub cache_actions: Vec<CacheAction>,
pub cache_actions: CacheActions,
pub backup_kv: Option<BackupKV>,
/// Full device tokens freed without a host copy during this device step.
pub unbacked_tokens: usize,
@@ -1258,7 +1191,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
params: &DecLockRefParams,
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
) -> Result<(), NodeAccessError> {
let node_idx = self.arena.resolve(node_id)?;
self.assert_receipt_anchor_(node_idx, params);
@@ -1286,7 +1219,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
params: &DecLockRefParams,
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
) -> Result<(), NodeAccessError> {
let node_idx = self.arena.resolve(node_id)?;
self.assert_receipt_anchor_(node_idx, params);
@@ -1510,7 +1443,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
usize,
usize,
Option<NodeId>,
Option<CacheAction>,
Option<UpdatePendingBackupOnNodeSplit>,
) {
// Non-HiCache mode has only device-resident matches, so the scheduler
// device anchor follows the best match. In HiCache mode, host-backed
@@ -1524,7 +1457,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
let mut best_match_device_node_id = node_id;
let mut best_match_device_value_len = 0;
let mut full_kv_hit_length = 0;
let mut action: Option<CacheAction> = None;
let mut action: Option<UpdatePendingBackupOnNodeSplit> = None;
let separate_device_match = self.enable_hicache;
fn update_best_if_valid<K: ChildKeyType>(
@@ -1639,7 +1572,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
best_match_device_value_len: usize,
full_kv_hit_length: usize,
last_visited_node_id: Option<NodeId>,
action: Option<CacheAction>,
action: Option<UpdatePendingBackupOnNodeSplit>,
) -> MatchResult {
for i in 0..self.components.len() {
// Full uses last_access ticks, not LRU.
@@ -1705,7 +1638,10 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
best_match_device_value_len,
);
}
result.cache_actions = action.into_iter().collect();
result.cache_actions = action
.into_iter()
.map(|action| Box::new(action) as Box<dyn CacheAction>)
.collect();
result
}
@@ -1808,7 +1744,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
) -> Result<InsertResult, TreeCoreRuntimeError> {
// Single-shot pump over the resumable walk: run every step inline and
// fold the step actions into the result for the caller to apply.
let mut actions = Vec::new();
let mut actions: CacheActions = Vec::new();
let mut step = self.try_begin_insert(params)?;
loop {
actions.append(&mut step.actions);
@@ -1988,7 +1924,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
}
/// Finish the insert (idempotent); returns still-pending actions to drain.
pub fn end_insert(&mut self) -> Vec<CacheAction> {
pub fn end_insert(&mut self) -> CacheActions {
self.ongoing_insert_walk_state
.take()
.map(|state| state.pending_actions)
@@ -2050,7 +1986,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
let (split_node_id, action) = self.split_node_(node_id, prefix_len);
node_id = split_node_id;
if let Some(action) = action {
state.pending_actions.push(action);
state.pending_actions.push(Box::new(action));
}
}
{
@@ -2166,22 +2102,22 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
.saturating_sub(cursor + dup_start)
.min(dup_len);
if swa_already_freed > 0 {
state
.pending_actions
.push(CacheAction::FreeDeviceKVFullOnly(vec![value_slice.narrow(
state.pending_actions.push(Box::new(FreeDeviceKVFullOnly {
indices: vec![value_slice.narrow(
0,
dup_start as i64,
swa_already_freed as i64,
)]));
)],
}));
}
if swa_already_freed < dup_len {
state
.pending_actions
.push(CacheAction::FreeDeviceKV(vec![value_slice.narrow(
state.pending_actions.push(Box::new(FreeDeviceKV {
indices: vec![value_slice.narrow(
0,
(dup_start + swa_already_freed) as i64,
(dup_len - swa_already_freed) as i64,
)]));
)],
}));
}
}
state.overlap_node_id = None;
@@ -2200,7 +2136,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
if node_end > state.inserted_len && self.inc_hit_count_and_check_(node_id) {
let backup = self
.build_backup_kv_action_(self.arena.node(node_id), /* write_back = */ false);
state.pending_actions.push(CacheAction::BackupKV(backup));
state.pending_actions.push(Box::new(backup));
}
state.node_id = node_id;
state.total_prefix_length += prefix_len;
@@ -2335,7 +2271,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
if self.should_backup_after_insert_(state, target_node_id) {
let backup =
self.build_backup_kv_action_(self.arena.node(target_node_id), self.is_write_back);
state.pending_actions.push(CacheAction::BackupKV(backup));
state.pending_actions.push(Box::new(backup));
}
}
@@ -2344,7 +2280,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
&mut self,
child_id: NodeIdx_,
split_len: usize,
) -> (NodeIdx_, Option<CacheAction>) {
) -> (NodeIdx_, Option<UpdatePendingBackupOnNodeSplit>) {
assert!(
split_len > 0 && split_len.is_multiple_of(self.page_size),
"split_node_: split_len {split_len} must be a nonzero page multiple"
@@ -2429,7 +2365,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
}
// Buffer-only writes are tracked by the cache rather than a node ack.
let action = if ack_id.is_some() || self.is_host_memory_buffer_only {
Some(CacheAction::UpdatePendingBackupOnNodeSplit {
Some(UpdatePendingBackupOnNodeSplit {
ack_id,
old_node_id: self.arena.node(child_id).id,
new_node_id: self.arena.node(new_node_id).id,
@@ -2951,7 +2887,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
tracker: &mut HashMap<ComponentType, usize>,
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
) {
self.record_remove_event_(node_id, medium);
for i in 0..self.components.len() {
@@ -2977,7 +2913,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
tracker: &mut HashMap<ComponentType, usize>,
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
) {
self.release_all_component_layers_(
node_id,
@@ -3066,7 +3002,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
tracker: &mut HashMap<ComponentType, usize>,
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
) {
assert!(
self.can_reclaim_coexisting_host_value_(node_id, component_type),
@@ -3102,7 +3038,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
tracker: &mut HashMap<ComponentType, usize>,
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
) {
assert!(
self.is_evictable_host_leaf_(self.arena.node(node_id)),
@@ -3170,7 +3106,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
tracker: &mut HashMap<ComponentType, usize>,
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
) {
{
let node = self.arena.node(node_id);
@@ -3215,7 +3151,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
tracker: &mut HashMap<ComponentType, usize>,
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
target: EvictLayer,
) {
let is_leaf = match target {
@@ -3347,7 +3283,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
component_type: ComponentType,
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
target: EvictLayer,
tracker: Option<&mut HashMap<ComponentType, usize>>,
) -> (usize, usize) {
@@ -3404,7 +3340,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
tracker: &mut HashMap<ComponentType, usize>,
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
) {
let ct = BASE_COMPONENT_TYPE;
let mut cur = deleted_node_parent_id;
@@ -3895,7 +3831,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
let mut host_prefix_len = 0;
let mut refilled_unbacked_node = false;
let mut inserted_host_node = None;
let mut cache_actions: Vec<CacheAction> = Vec::new();
let mut cache_actions: CacheActions = Vec::new();
while matched_length < total_len {
let Some(child_id) = self.arena.child_on_page_in_namespace(
node_id,
@@ -3916,7 +3852,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
let (split_node_id, action) = self.split_node_(node_id, prefix_len);
node_id = split_node_id;
if let Some(action) = action {
cache_actions.push(action);
cache_actions.push(Box::new(action));
}
}
@@ -4472,7 +4408,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
node_id: NodeId,
phase: CacheTransferPhase,
comp_xfers: HashMap<ComponentType, Vec<PoolTransfer>>,
cache_actions: &mut Vec<CacheAction>,
cache_actions: &mut CacheActions,
mut insert_result: Option<&mut InsertResult>,
pool_storage_result: Option<&PoolTransferResult>,
) -> Result<(), NodeAccessError> {
@@ -4510,7 +4446,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
) -> Result<(), NodeAccessError> {
let node_id = self.arena.resolve(node_id)?;
self.validate_pool_transfer_handles(comp_xfers.values().flatten())?;
let mut cache_actions: Vec<CacheAction> = Vec::new();
let mut cache_actions: CacheActions = Vec::new();
if host_indices.numel() > 0 {
let kv_xfer = PoolTransfer {
name: PoolName::Kv,
@@ -4555,7 +4491,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
device_indices: Tensor,
mut kv_xfer: PoolTransfer,
comp_xfers: HashMap<ComponentType, Vec<PoolTransfer>>,
) -> Result<Vec<CacheAction>, NodeAccessError> {
) -> Result<CacheActions, NodeAccessError> {
let anchor_id = node_id;
let node_id = self.arena.resolve(node_id)?;
let loaded_node_indices = kv_xfer
@@ -4565,7 +4501,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
.map(|&loaded_id| self.arena.resolve(loaded_id))
.collect::<Result<Vec<_>, _>>()?;
self.validate_pool_transfer_handles(comp_xfers.values().flatten())?;
let mut cache_actions: Vec<CacheAction> = Vec::new();
let mut cache_actions: CacheActions = Vec::new();
kv_xfer.device_indices = Some(device_indices);
let nodes_to_load = kv_xfer.nodes_to_load.clone();
if self.is_write_back {
@@ -4894,7 +4830,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
window_start: usize,
window_end: usize,
swa_values: &Tensor,
) -> Result<Vec<CacheAction>, String> {
) -> Result<CacheActions, String> {
self.validate_swa_span_(key, window_start, window_end)?;
if swa_values.kind() != Kind::Int64 || swa_values.device() != self.device {
return Err(format!(
@@ -4940,7 +4876,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
));
}
let mut actions = Vec::new();
let mut actions: CacheActions = Vec::new();
for (child_id, pos, prefix_len) in spans {
let seg_end = window_end.min(pos + prefix_len);
if seg_end <= window_start {
@@ -4951,11 +4887,15 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
if seg_start > pos {
// The original handle remains on the suffix, our target.
let (_, action) = self.split_node_(target, seg_start - pos);
actions.extend(action);
if let Some(action) = action {
actions.push(Box::new(action));
}
}
if seg_start + self.arena.node(target).key.atom_len() > seg_end {
let (fragment, action) = self.split_node_(target, seg_end - seg_start);
actions.extend(action);
if let Some(action) = action {
actions.push(Box::new(action));
}
target = fragment;
}
let values = swa_values
@@ -5611,3 +5611,72 @@ def test_finished_request_prompt_priority_saturates_at_int64_max(priority):
if __name__ == "__main__":
sys.exit(pytest.main([__file__]))
def test_load_back_action_tuple_preserves_chunk_order_and_tensor_ownership():
binding = mem_cache.RustUnifiedTreeCoreBinding(
mem_cache.TreeCoreInitParamsBinding(
page_size=1, swa_sliding_window_size=8, enable_hicache=True
),
[0, 1],
)
binding.set_has_swa_host_pool()
key = array("q", range(5))
prefix = binding.insert(
mem_cache.InsertParamsBinding(key=key[:2], value=torch.tensor([10, 11]))
).last_device_node
binding.set_component_device_value(prefix, 1, torch.tensor([110, 111]))
tail = binding.insert(
mem_cache.InsertParamsBinding(
key=key, value=torch.arange(10, 15), prev_prefix_len=2
)
).last_device_node
binding.set_component_device_value(tail, 1, torch.tensor([112, 113, 114]))
for node, host in (
(prefix, torch.arange(310, 312)),
(tail, torch.arange(312, 315)),
):
_, transfers = binding.build_backup_spec(node)
transfer = transfers[1][0]
transfers[1][0] = (transfer[0], transfer[2] + 100, *transfer[2:])
binding.mark_write_through_pending([node], node)
binding.commit_backup(node, host, transfers)
binding.finish_write_through([node], node)
binding.demote(tail)
binding.demote(prefix)
kv_transfer, transfers = binding.build_load_back_spec(tail)
transfer = transfers[1][0]
transfers[1][0] = (*transfer[:2], torch.arange(50, 55), *transfer[3:])
actions = binding.commit_load_back(
tail, torch.arange(40, 45), kv_transfer, transfers
)
receipt = binding.inc_lock_ref(tail)
assert isinstance(actions, list) and len(actions) == 1
action = actions[0]
assert isinstance(action, tuple) and len(action) == 3
assert action[0] == "rebuild_full_to_swa_mapping"
assert isinstance(action[1], list) and isinstance(action[2], list)
assert [value.tolist() for value in action[1]] == [[40, 41], [42, 43, 44]]
assert [value.tolist() for value in action[2]] == [[50, 51], [52, 53, 54]]
binding.dec_lock_ref(
tail,
mem_cache.DecLockRefParamsBinding(
node_id=receipt.node_id,
skipped_lock_components=receipt.skipped_lock_components,
component_lock_uuids=receipt.component_lock_uuids,
component_host_lock_uuids=receipt.component_host_lock_uuids,
),
)
binding.finish_load_back(tail)
binding.sanity_check([], [])
chunk = action[2][1]
storage_ref = chunk.untyped_storage()._weak_ref()
try:
binding.reset()
assert chunk.tolist() == [52, 53, 54]
del chunk, action, actions
assert torch.UntypedStorage._expired(storage_ref)
finally:
torch.UntypedStorage._free_weak_ref(storage_ref)