[Refactor] radix-tree: Make each cache action a typed CacheAction implementation (#45444)
Co-authored-by: metamergebot <metamergebot@users.noreply.github.com> Co-authored-by: Jialin <Jialin@users.noreply.github.com>
This commit is contained in:
co-authored by
metamergebot
Jialin
parent
d0022e1df3
commit
005333d029
@@ -0,0 +1,157 @@
|
||||
//! Cache actions returned to the cache for execution.
|
||||
|
||||
use std::fmt::Debug;
|
||||
|
||||
use tch::Tensor;
|
||||
|
||||
use crate::components::ComponentType;
|
||||
use crate::node::NodeId;
|
||||
|
||||
pub trait CacheAction: Debug + Send {
|
||||
fn tag(&self) -> &'static str;
|
||||
|
||||
/// Whether insertion must pause for the cache to apply this action.
|
||||
fn requires_insert_barrier(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
#[cfg(feature = "python-extension")]
|
||||
fn into_py_tagged(
|
||||
self: Box<Self>,
|
||||
py: pyo3::Python<'_>,
|
||||
) -> pyo3::PyResult<pyo3::Py<pyo3::PyAny>>;
|
||||
|
||||
/// Rust tests inspect emitted payloads without a production downcast API.
|
||||
#[cfg(test)]
|
||||
fn as_any(&self) -> &dyn std::any::Any;
|
||||
}
|
||||
|
||||
pub type CacheActions = Vec<Box<dyn CacheAction>>;
|
||||
|
||||
macro_rules! impl_cache_action {
|
||||
($action:ty, $tag:literal $(, insert_barrier = $barrier:expr)? $(,)?) => {
|
||||
impl $crate::cache_action::CacheAction for $action {
|
||||
fn tag(&self) -> &'static str {
|
||||
$tag
|
||||
}
|
||||
|
||||
$(fn requires_insert_barrier(&self) -> bool {
|
||||
$barrier
|
||||
})?
|
||||
|
||||
#[cfg(feature = "python-extension")]
|
||||
fn into_py_tagged(
|
||||
self: Box<Self>,
|
||||
py: pyo3::Python<'_>,
|
||||
) -> pyo3::PyResult<pyo3::Py<pyo3::PyAny>> {
|
||||
<$action>::into_py_tagged(*self, py)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn as_any(&self) -> &dyn std::any::Any {
|
||||
self
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
pub(crate) use impl_cache_action;
|
||||
|
||||
/// Duplicate device KV slices the cache frees after the insert.
|
||||
#[derive(Debug)]
|
||||
pub struct FreeDeviceKV {
|
||||
pub indices: Vec<Tensor>,
|
||||
}
|
||||
|
||||
/// Free the full side only, for a tombstoned node whose SWA peers are gone;
|
||||
/// FreeDeviceKV would release the SWA side twice.
|
||||
#[derive(Debug)]
|
||||
pub struct FreeDeviceKVFullOnly {
|
||||
pub indices: Vec<Tensor>,
|
||||
}
|
||||
|
||||
/// Update cache-side write bookkeeping after a node split:
|
||||
///
|
||||
/// parent -> node => parent -> new_node -> new_child
|
||||
///
|
||||
/// old_node_id (the pre-split node) is replaced by new_node_id + new_child_node_id.
|
||||
#[derive(Debug)]
|
||||
pub struct UpdatePendingBackupOnNodeSplit {
|
||||
/// None when buffer-only bookkeeping has no tree-tracked write.
|
||||
pub ack_id: Option<usize>,
|
||||
pub old_node_id: NodeId,
|
||||
pub new_node_id: NodeId,
|
||||
pub new_child_node_id: NodeId,
|
||||
}
|
||||
|
||||
/// Per-path Mamba state-cap eviction from the tail's root path; applied at
|
||||
/// the insert's commit barrier, after the walk-time backups whose
|
||||
/// write-through locks shield the backed-up chain.
|
||||
#[derive(Debug)]
|
||||
pub struct MambaEvictExcessPathStates {
|
||||
pub tail_node_id: NodeId,
|
||||
}
|
||||
|
||||
/// Free only the given component's device KV slots.
|
||||
#[derive(Debug)]
|
||||
pub struct FreeComponentDeviceSlot {
|
||||
pub component_type: ComponentType,
|
||||
pub indices: Vec<Tensor>,
|
||||
}
|
||||
|
||||
/// Free the given component's host KV pages.
|
||||
#[derive(Debug)]
|
||||
pub struct FreeComponentHostSlot {
|
||||
pub component_type: ComponentType,
|
||||
pub host_indices: Vec<Tensor>,
|
||||
}
|
||||
|
||||
/// Rebuild the SWA allocator's full->swa index mapping for loaded chunks.
|
||||
#[derive(Debug)]
|
||||
pub struct RebuildFullToSwaMapping {
|
||||
pub full_indices: Vec<Tensor>,
|
||||
pub swa_indices: Vec<Tensor>,
|
||||
}
|
||||
|
||||
/// Recover an SWA tombstone whose full is locked: keep the locked full, remap
|
||||
/// it onto the incoming full's SWA translation, and free only the incoming full.
|
||||
#[derive(Debug)]
|
||||
pub struct RecoverSwaWithLockedFull {
|
||||
pub node_id: NodeId,
|
||||
pub kept_full: Tensor,
|
||||
pub incoming_full: Tensor,
|
||||
}
|
||||
|
||||
/// Rebuild a node's SWA value by translating its source full value, then store it.
|
||||
#[derive(Debug)]
|
||||
pub struct SwaRebuild {
|
||||
pub node_id: NodeId,
|
||||
pub source_value: Tensor,
|
||||
}
|
||||
|
||||
/// A device->host backup work item for the cache to execute.
|
||||
#[derive(Default, Debug)]
|
||||
pub struct BackupKV {
|
||||
/// Backup these nodes device->host in order, stopping at the first failure; the
|
||||
/// caller orders them parent-before-child for write-through and child-first for
|
||||
/// write-back. External handles: the list crosses to the orchestrator.
|
||||
pub node_ids: Vec<NodeId>,
|
||||
}
|
||||
|
||||
self::impl_cache_action!(FreeDeviceKV, "free_device_kv", insert_barrier = false);
|
||||
self::impl_cache_action!(
|
||||
FreeDeviceKVFullOnly,
|
||||
"free_device_kv_full_only",
|
||||
insert_barrier = false
|
||||
);
|
||||
self::impl_cache_action!(BackupKV, "backup_kv");
|
||||
self::impl_cache_action!(
|
||||
UpdatePendingBackupOnNodeSplit,
|
||||
"update_pending_backup_on_node_split",
|
||||
insert_barrier = false
|
||||
);
|
||||
self::impl_cache_action!(MambaEvictExcessPathStates, "mamba_evict_excess_path_states");
|
||||
self::impl_cache_action!(FreeComponentDeviceSlot, "free_component_device_slot");
|
||||
self::impl_cache_action!(FreeComponentHostSlot, "free_component_host_slot");
|
||||
self::impl_cache_action!(RebuildFullToSwaMapping, "rebuild_full_to_swa_mapping");
|
||||
self::impl_cache_action!(RecoverSwaWithLockedFull, "recover_swa_with_locked_full");
|
||||
self::impl_cache_action!(SwaRebuild, "swa_rebuild");
|
||||
@@ -13,7 +13,7 @@ use crate::node::Node;
|
||||
use crate::node::{NodeId, NodeIdx_, TreeCoreRuntimeError, ValueSlotIdx};
|
||||
use crate::unified_lru_list::PriorityKey;
|
||||
use crate::unified_tree_core::{
|
||||
CacheAction, CacheTransferPhase, DecLockRefParams, EvictLayer, IncLockRefResult, InsertResult,
|
||||
CacheActions, CacheTransferPhase, DecLockRefParams, EvictLayer, IncLockRefResult, InsertResult,
|
||||
MatchPrefixParams, MatchResult, PoolName, PoolTransfer, PoolTransferResult, UnifiedTreeCore,
|
||||
};
|
||||
|
||||
@@ -104,7 +104,7 @@ impl<K: ChildKeyType> TreeComponent<K> for FullComponent {
|
||||
node_id: NodeIdx_,
|
||||
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
target: EvictLayer,
|
||||
) -> (usize, usize) {
|
||||
let node = tree_core.arena.node(node_id);
|
||||
@@ -156,7 +156,7 @@ impl<K: ChildKeyType> TreeComponent<K> for FullComponent {
|
||||
tracker: &mut HashMap<ComponentType, usize>,
|
||||
_device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
_host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
_cache_actions: &mut Vec<CacheAction>,
|
||||
_cache_actions: &mut CacheActions,
|
||||
) -> Option<NodeIdx_> {
|
||||
let ct = FULL;
|
||||
assert!(
|
||||
@@ -205,7 +205,7 @@ impl<K: ChildKeyType> TreeComponent<K> for FullComponent {
|
||||
tracker: &mut HashMap<ComponentType, usize>,
|
||||
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
) {
|
||||
for spare_imminent_demotes in [true, false] {
|
||||
if tracker[&FULL] >= num_tokens {
|
||||
@@ -248,7 +248,7 @@ impl<K: ChildKeyType> TreeComponent<K> for FullComponent {
|
||||
tracker: &mut HashMap<ComponentType, usize>,
|
||||
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
) {
|
||||
let ct = FULL;
|
||||
let arena = &tree_core.arena;
|
||||
@@ -491,7 +491,7 @@ impl<K: ChildKeyType> TreeComponent<K> for FullComponent {
|
||||
node_id: NodeIdx_,
|
||||
phase: CacheTransferPhase,
|
||||
transfers: Vec<PoolTransfer>,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
insert_result: Option<&mut InsertResult>,
|
||||
pool_storage_result: Option<&PoolTransferResult>,
|
||||
) {
|
||||
|
||||
@@ -6,13 +6,14 @@ use std::collections::HashMap;
|
||||
|
||||
use tch::Tensor;
|
||||
|
||||
use crate::cache_action::{FreeComponentHostSlot, MambaEvictExcessPathStates};
|
||||
use crate::components::TreeComponent;
|
||||
use crate::components::{ComponentType, FULL, MAMBA};
|
||||
use crate::node::ChildKeyType;
|
||||
use crate::node::Node;
|
||||
use crate::node::{NodeId, NodeIdx_, TreeCoreRuntimeError, ValueSlotIdx};
|
||||
use crate::unified_tree_core::{
|
||||
CacheAction, CacheInitParams, CacheTransferPhase, DecLockRefParams, EvictLayer,
|
||||
CacheActions, CacheInitParams, CacheTransferPhase, DecLockRefParams, EvictLayer,
|
||||
IncLockRefResult, InsertParams, InsertResult, LRURefreshPhase, MatchPrefixParams, MatchResult,
|
||||
PoolHitPolicy, PoolName, PoolTransfer, PoolTransferResult, UnifiedTreeCore,
|
||||
};
|
||||
@@ -70,12 +71,12 @@ impl MambaComponent {
|
||||
fn emit_excess_path_states_eviction_(
|
||||
&self,
|
||||
tail_node_id: NodeId,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
) {
|
||||
if self.mamba_max_states_per_path.is_none() {
|
||||
return;
|
||||
}
|
||||
cache_actions.push(CacheAction::MambaEvictExcessPathStates { tail_node_id });
|
||||
cache_actions.push(Box::new(MambaEvictExcessPathStates { tail_node_id }));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -160,7 +161,7 @@ impl<K: ChildKeyType> TreeComponent<K> for MambaComponent {
|
||||
is_new_leaf: bool,
|
||||
params: &InsertParams<'_, K>,
|
||||
result: &mut InsertResult,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
) {
|
||||
let mamba_value = params
|
||||
.mamba_value
|
||||
@@ -200,7 +201,7 @@ impl<K: ChildKeyType> TreeComponent<K> for MambaComponent {
|
||||
tail_node_id: NodeIdx_,
|
||||
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
) {
|
||||
let Some(cap) = self.mamba_max_states_per_path else {
|
||||
return;
|
||||
@@ -282,7 +283,7 @@ impl<K: ChildKeyType> TreeComponent<K> for MambaComponent {
|
||||
node_id: NodeIdx_,
|
||||
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
target: EvictLayer,
|
||||
) -> (usize, usize) {
|
||||
let ct = MAMBA;
|
||||
@@ -348,7 +349,7 @@ impl<K: ChildKeyType> TreeComponent<K> for MambaComponent {
|
||||
tracker: &mut HashMap<ComponentType, usize>,
|
||||
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
) -> Option<NodeIdx_> {
|
||||
let ct = MAMBA;
|
||||
assert!(
|
||||
@@ -636,7 +637,7 @@ impl<K: ChildKeyType> TreeComponent<K> for MambaComponent {
|
||||
node_id: NodeIdx_,
|
||||
phase: CacheTransferPhase,
|
||||
transfers: Vec<PoolTransfer>,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
insert_result: Option<&mut InsertResult>,
|
||||
pool_storage_result: Option<&PoolTransferResult>,
|
||||
) {
|
||||
@@ -696,12 +697,12 @@ impl<K: ChildKeyType> TreeComponent<K> for MambaComponent {
|
||||
let Some(target) = attach_target else {
|
||||
// The buffer cannot attach: free it and let the caller keep
|
||||
// its own donated slot bookkeeping.
|
||||
cache_actions.push(CacheAction::FreeComponentHostSlot {
|
||||
cache_actions.push(Box::new(FreeComponentHostSlot {
|
||||
component_type: MAMBA,
|
||||
host_indices: host_indices
|
||||
.map(|host| vec![host.shallow_clone()])
|
||||
.unwrap_or_default(),
|
||||
});
|
||||
}));
|
||||
if let Some(insert_result) = insert_result {
|
||||
insert_result.mamba_exist = true;
|
||||
}
|
||||
@@ -733,7 +734,7 @@ impl<K: ChildKeyType> TreeComponent<K> for MambaComponent {
|
||||
tracker: &mut HashMap<ComponentType, usize>,
|
||||
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
) {
|
||||
let ct = MAMBA;
|
||||
let mut x = tree_core
|
||||
|
||||
@@ -7,7 +7,7 @@ use tch::Tensor;
|
||||
|
||||
use crate::node::{ChildKeyType, Node, NodeArena, NodeId, NodeIdx_, TreeCoreRuntimeError};
|
||||
use crate::unified_tree_core::{
|
||||
CacheAction, CacheTransferPhase, DecLockRefParams, EvictLayer, IncLockRefResult, InsertParams,
|
||||
CacheActions, CacheTransferPhase, DecLockRefParams, EvictLayer, IncLockRefResult, InsertParams,
|
||||
InsertResult, LRURefreshPhase, MatchPrefixParams, MatchResult, PoolTransfer,
|
||||
PoolTransferResult, UnifiedTreeCore,
|
||||
};
|
||||
@@ -199,7 +199,7 @@ pub trait TreeComponent<K: ChildKeyType> {
|
||||
params: &InsertParams<'_, K>,
|
||||
preserve_full_ids: bool,
|
||||
result: &mut InsertResult,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
) -> usize {
|
||||
prefix_len
|
||||
}
|
||||
@@ -216,7 +216,7 @@ pub trait TreeComponent<K: ChildKeyType> {
|
||||
total_prefix_len: usize,
|
||||
params: &InsertParams<'_, K>,
|
||||
result: &mut InsertResult,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
) {
|
||||
}
|
||||
|
||||
@@ -239,7 +239,7 @@ pub trait TreeComponent<K: ChildKeyType> {
|
||||
is_new_leaf: bool,
|
||||
params: &InsertParams<'_, K>,
|
||||
result: &mut InsertResult,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
) {
|
||||
}
|
||||
|
||||
@@ -251,7 +251,7 @@ pub trait TreeComponent<K: ChildKeyType> {
|
||||
tail_node_id: NodeIdx_,
|
||||
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
) {
|
||||
}
|
||||
|
||||
@@ -307,7 +307,7 @@ pub trait TreeComponent<K: ChildKeyType> {
|
||||
node_id: NodeIdx_,
|
||||
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
target: EvictLayer,
|
||||
) -> (usize, usize);
|
||||
|
||||
@@ -330,7 +330,7 @@ pub trait TreeComponent<K: ChildKeyType> {
|
||||
tracker: &mut HashMap<ComponentType, usize>,
|
||||
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
) -> Option<NodeIdx_>;
|
||||
|
||||
/// Clear this component's device-eviction walk state.
|
||||
@@ -406,7 +406,7 @@ pub trait TreeComponent<K: ChildKeyType> {
|
||||
_params: &DecLockRefParams,
|
||||
_device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
_host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
_cache_actions: &mut Vec<CacheAction>,
|
||||
_cache_actions: &mut CacheActions,
|
||||
) {
|
||||
unimplemented!("release_window_lock is SWA-only")
|
||||
}
|
||||
@@ -460,7 +460,7 @@ pub trait TreeComponent<K: ChildKeyType> {
|
||||
node_id: NodeIdx_,
|
||||
phase: CacheTransferPhase,
|
||||
transfers: Vec<PoolTransfer>,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
insert_result: Option<&mut InsertResult>,
|
||||
pool_storage_result: Option<&PoolTransferResult>,
|
||||
) {
|
||||
@@ -489,7 +489,7 @@ pub trait TreeComponent<K: ChildKeyType> {
|
||||
_tracker: &mut HashMap<ComponentType, usize>,
|
||||
_device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
_host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
_cache_actions: &mut Vec<CacheAction>,
|
||||
_cache_actions: &mut CacheActions,
|
||||
) {
|
||||
}
|
||||
|
||||
@@ -503,7 +503,7 @@ pub trait TreeComponent<K: ChildKeyType> {
|
||||
_tracker: &mut HashMap<ComponentType, usize>,
|
||||
_device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
_host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
_cache_actions: &mut Vec<CacheAction>,
|
||||
_cache_actions: &mut CacheActions,
|
||||
) {
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,13 +7,17 @@ use std::collections::HashMap;
|
||||
|
||||
use tch::{Kind, Tensor};
|
||||
|
||||
use crate::cache_action::{
|
||||
FreeComponentHostSlot, FreeDeviceKVFullOnly, RebuildFullToSwaMapping, RecoverSwaWithLockedFull,
|
||||
SwaRebuild,
|
||||
};
|
||||
use crate::components::TreeComponent;
|
||||
use crate::components::{ComponentType, FULL, SWA};
|
||||
use crate::node::ChildKeyType;
|
||||
use crate::node::Node;
|
||||
use crate::node::{NodeId, NodeIdx_, TreeCoreRuntimeError, ValueSlotIdx};
|
||||
use crate::unified_tree_core::{
|
||||
CacheAction, CacheInitParams, CacheTransferPhase, DecLockRefParams, EvictLayer,
|
||||
CacheActions, CacheInitParams, CacheTransferPhase, DecLockRefParams, EvictLayer,
|
||||
IncLockRefResult, InsertParams, InsertResult, LRURefreshPhase, MatchPrefixParams, MatchResult,
|
||||
PoolHitPolicy, PoolName, PoolTransfer, PoolTransferResult, UnifiedTreeCore,
|
||||
};
|
||||
@@ -57,7 +61,7 @@ impl SwaComponent {
|
||||
&self,
|
||||
tree_core: &mut UnifiedTreeCore<K>,
|
||||
leaf_id: NodeIdx_,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
) -> Option<NodeIdx_> {
|
||||
let leaf = tree_core.arena.node(leaf_id);
|
||||
if leaf.is_root() || leaf.device_lock_ref(SWA) > 0 {
|
||||
@@ -80,17 +84,11 @@ impl SwaComponent {
|
||||
|
||||
let (new_parent, action) = tree_core.split_node_(leaf_id, split_at);
|
||||
assert!(
|
||||
!matches!(
|
||||
action,
|
||||
Some(CacheAction::UpdatePendingBackupOnNodeSplit {
|
||||
ack_id: Some(_),
|
||||
..
|
||||
})
|
||||
),
|
||||
action.as_ref().is_none_or(|action| action.ack_id.is_none()),
|
||||
"fresh SWA leaf cannot be write-through-pending"
|
||||
);
|
||||
if let Some(action) = action {
|
||||
cache_actions.push(action);
|
||||
cache_actions.push(Box::new(action));
|
||||
}
|
||||
Some(new_parent)
|
||||
}
|
||||
@@ -193,12 +191,12 @@ impl SwaComponent {
|
||||
|
||||
impl SwaComponent {
|
||||
/// Queue a free of the given SWA host slots; empty tensors are dropped.
|
||||
fn release_swa_host_(&self, host_indices: Tensor, cache_actions: &mut Vec<CacheAction>) {
|
||||
fn release_swa_host_(&self, host_indices: Tensor, cache_actions: &mut CacheActions) {
|
||||
if host_indices.numel() > 0 {
|
||||
cache_actions.push(CacheAction::FreeComponentHostSlot {
|
||||
cache_actions.push(Box::new(FreeComponentHostSlot {
|
||||
component_type: SWA,
|
||||
host_indices: vec![host_indices],
|
||||
});
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -234,7 +232,7 @@ impl SwaComponent {
|
||||
tree_core: &mut UnifiedTreeCore<K>,
|
||||
node_id: NodeIdx_,
|
||||
transfers: Vec<PoolTransfer>,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
insert_result: Option<&InsertResult>,
|
||||
pool_storage_result: Option<&PoolTransferResult>,
|
||||
) {
|
||||
@@ -307,7 +305,7 @@ impl SwaComponent {
|
||||
if fill_start > node_start {
|
||||
let (_, action) = tree_core.split_node_(cur, fill_start - node_start);
|
||||
if let Some(action) = action {
|
||||
cache_actions.push(action);
|
||||
cache_actions.push(Box::new(action));
|
||||
}
|
||||
}
|
||||
self.attach_swa_host_value_(tree_core, cur, slice);
|
||||
@@ -441,7 +439,7 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
|
||||
params: &InsertParams<'_, K>,
|
||||
preserve_full_ids: bool,
|
||||
result: &mut InsertResult,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
) -> usize {
|
||||
if params.prev_prefix_len >= total_prefix_len + prefix_len {
|
||||
return prefix_len;
|
||||
@@ -467,21 +465,23 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
|
||||
// Branch 1: entire value_slice is within SWA window — recover
|
||||
result.record_adopted_range(SWA, total_prefix_len, total_prefix_len + prefix_len);
|
||||
if node.device_lock_ref(FULL) > 0 || preserve_full_ids {
|
||||
cache_actions.push(CacheAction::RecoverSwaWithLockedFull {
|
||||
cache_actions.push(Box::new(RecoverSwaWithLockedFull {
|
||||
node_id: node.id,
|
||||
kept_full: node.device_value(FULL).shallow_clone(),
|
||||
incoming_full: value_slice,
|
||||
});
|
||||
}));
|
||||
return 0;
|
||||
}
|
||||
result.record_adopted_range(FULL, total_prefix_len, total_prefix_len + prefix_len);
|
||||
let old_full = node.take_device_value(FULL);
|
||||
node.set_device_value(FULL, value_slice.copy());
|
||||
cache_actions.push(CacheAction::FreeDeviceKVFullOnly(vec![old_full]));
|
||||
cache_actions.push(CacheAction::SwaRebuild {
|
||||
cache_actions.push(Box::new(FreeDeviceKVFullOnly {
|
||||
indices: vec![old_full],
|
||||
}));
|
||||
cache_actions.push(Box::new(SwaRebuild {
|
||||
node_id: node.id,
|
||||
source_value: value_slice,
|
||||
});
|
||||
}));
|
||||
0
|
||||
} else if swa_evicted_seqlen < total_prefix_len + prefix_len {
|
||||
// Branch 2: value_slice[start_idx:] is within SWA window — partial recover
|
||||
@@ -495,26 +495,28 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
|
||||
.narrow(0, start_idx as i64, (full_len - start_idx) as i64);
|
||||
let (_, action) = tree_core.split_node_(node_id, start_idx);
|
||||
if let Some(action) = action {
|
||||
cache_actions.push(action);
|
||||
cache_actions.push(Box::new(action));
|
||||
}
|
||||
let new_full = value_slice.narrow(0, start_idx as i64, (prefix_len - start_idx) as i64);
|
||||
if preserve_full {
|
||||
cache_actions.push(CacheAction::RecoverSwaWithLockedFull {
|
||||
cache_actions.push(Box::new(RecoverSwaWithLockedFull {
|
||||
node_id: node_ext_id,
|
||||
kept_full: old_full,
|
||||
incoming_full: new_full,
|
||||
});
|
||||
}));
|
||||
return start_idx;
|
||||
}
|
||||
result.record_adopted_range(FULL, swa_evicted_seqlen, total_prefix_len + prefix_len);
|
||||
let node = tree_core.arena.node_mut(node_id);
|
||||
let _ = node.take_device_value(FULL);
|
||||
node.set_device_value(FULL, new_full.copy());
|
||||
cache_actions.push(CacheAction::FreeDeviceKVFullOnly(vec![old_full]));
|
||||
cache_actions.push(CacheAction::SwaRebuild {
|
||||
cache_actions.push(Box::new(FreeDeviceKVFullOnly {
|
||||
indices: vec![old_full],
|
||||
}));
|
||||
cache_actions.push(Box::new(SwaRebuild {
|
||||
node_id: node_ext_id,
|
||||
source_value: new_full,
|
||||
});
|
||||
}));
|
||||
start_idx
|
||||
} else {
|
||||
// Branch 3: entire value_slice is outside SWA window — not consumed
|
||||
@@ -530,7 +532,7 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
|
||||
total_prefix_len: usize,
|
||||
params: &InsertParams<'_, K>,
|
||||
result: &mut InsertResult,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
) {
|
||||
// _unevict_node_on_insert already wrote the request's fresh KV slice
|
||||
// into the base value. We just need to rebuild SWA from that slice for
|
||||
@@ -552,7 +554,7 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
|
||||
let start_idx = swa_evicted_seqlen - total_prefix_len;
|
||||
let (_, action) = tree_core.split_node_(node_id, start_idx);
|
||||
if let Some(action) = action {
|
||||
cache_actions.push(action);
|
||||
cache_actions.push(Box::new(action));
|
||||
}
|
||||
} else {
|
||||
return;
|
||||
@@ -562,10 +564,10 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
|
||||
total_prefix_len.max(swa_evicted_seqlen),
|
||||
total_prefix_len + prefix_len,
|
||||
);
|
||||
cache_actions.push(CacheAction::SwaRebuild {
|
||||
cache_actions.push(Box::new(SwaRebuild {
|
||||
node_id: tree_core.arena.node(node_id).id,
|
||||
source_value: tree_core.arena.device_value(node_id, FULL).shallow_clone(),
|
||||
});
|
||||
}));
|
||||
}
|
||||
|
||||
fn commit_insert_component_data(
|
||||
@@ -575,7 +577,7 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
|
||||
is_new_leaf: bool,
|
||||
params: &InsertParams<'_, K>,
|
||||
result: &mut InsertResult,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
) {
|
||||
if let Some(branching_seqlen) = params.swa_branching_seqlen {
|
||||
result.swa_branch_inserted = params.key.atom_len() >= branching_seqlen;
|
||||
@@ -599,7 +601,7 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
|
||||
// (tombstone) and an in-window child; `node` becomes the child.
|
||||
let (_, action) = tree_core.split_node_(node_id, split_pos);
|
||||
if let Some(action) = action {
|
||||
cache_actions.push(action);
|
||||
cache_actions.push(Box::new(action));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -609,18 +611,18 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
|
||||
// the in-window tail lands more-MRU.
|
||||
let capped_parent = self.maybe_split_leaf_for_swa_lock_(tree_core, node_id, cache_actions);
|
||||
if let Some(capped_parent) = capped_parent {
|
||||
cache_actions.push(CacheAction::SwaRebuild {
|
||||
cache_actions.push(Box::new(SwaRebuild {
|
||||
node_id: tree_core.arena.node(capped_parent).id,
|
||||
source_value: tree_core
|
||||
.arena
|
||||
.device_value(capped_parent, FULL)
|
||||
.shallow_clone(),
|
||||
});
|
||||
}));
|
||||
}
|
||||
cache_actions.push(CacheAction::SwaRebuild {
|
||||
cache_actions.push(Box::new(SwaRebuild {
|
||||
node_id: tree_core.arena.node(node_id).id,
|
||||
source_value: tree_core.arena.device_value(node_id, FULL).shallow_clone(),
|
||||
});
|
||||
}));
|
||||
}
|
||||
|
||||
fn redistribute_on_node_split(
|
||||
@@ -671,7 +673,7 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
|
||||
node_id: NodeIdx_,
|
||||
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
target: EvictLayer,
|
||||
) -> (usize, usize) {
|
||||
let ct = SWA;
|
||||
@@ -741,7 +743,7 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
|
||||
tracker: &mut HashMap<ComponentType, usize>,
|
||||
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
) -> Option<NodeIdx_> {
|
||||
let ct = SWA;
|
||||
assert!(
|
||||
@@ -848,7 +850,7 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
|
||||
tracker: &mut HashMap<ComponentType, usize>,
|
||||
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
) {
|
||||
let ct = SWA;
|
||||
let mut x = tree_core
|
||||
@@ -1057,7 +1059,7 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
|
||||
node_id: NodeIdx_,
|
||||
phase: CacheTransferPhase,
|
||||
transfers: Vec<PoolTransfer>,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
insert_result: Option<&mut InsertResult>,
|
||||
pool_storage_result: Option<&PoolTransferResult>,
|
||||
) {
|
||||
@@ -1142,10 +1144,10 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
|
||||
assert_eq!(offset, host_indices.size()[0]);
|
||||
// Rebuild the full->swa mapping for the loaded chunks at the orchestration layer.
|
||||
if !full_chunks.is_empty() {
|
||||
cache_actions.push(CacheAction::RebuildFullToSwaMapping {
|
||||
cache_actions.push(Box::new(RebuildFullToSwaMapping {
|
||||
full_indices: full_chunks,
|
||||
swa_indices: swa_chunks,
|
||||
});
|
||||
}));
|
||||
}
|
||||
}
|
||||
CacheTransferPhase::Prefetch => {
|
||||
@@ -1289,7 +1291,7 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
|
||||
params: &DecLockRefParams,
|
||||
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
) {
|
||||
if tree_core.arena.node(node_id).is_root() {
|
||||
return;
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
clippy::useless_conversion
|
||||
)]
|
||||
|
||||
mod cache_action;
|
||||
mod components;
|
||||
mod node;
|
||||
#[cfg(feature = "python-extension")]
|
||||
|
||||
@@ -18,10 +18,10 @@ use crate::node::{KeyNamespaceRef, NodeAccessError, NodeId, TreeCoreRuntimeError
|
||||
use crate::unified_lru_list::{TlruFloatConfig, TlruPromptEstimate};
|
||||
use crate::unified_tree_core::KvCacheEvent;
|
||||
use crate::unified_tree_core::{
|
||||
BufferBackupSnapshot, BufferBackupState, CacheAction, CacheInitParams, CacheTransferPhase,
|
||||
DecLockRefParams, EvictLayer, EvictionStepResult, InsertParams, InsertResult, InsertStepResult,
|
||||
MatchPrefixParams, MatchResult, PoolHitPolicy, PoolName, PoolTransfer, PoolTransferResult, Req,
|
||||
UnifiedTreeCore,
|
||||
BufferBackupSnapshot, BufferBackupState, CacheAction, CacheActions, CacheInitParams,
|
||||
CacheTransferPhase, DecLockRefParams, EvictLayer, EvictionStepResult, InsertParams,
|
||||
InsertResult, InsertStepResult, MatchPrefixParams, MatchResult, PoolHitPolicy, PoolName,
|
||||
PoolTransfer, PoolTransferResult, Req, UnifiedTreeCore,
|
||||
};
|
||||
|
||||
/// Translate only Rust panics; ordinary Python errors pass through unchanged.
|
||||
@@ -173,83 +173,13 @@ fn py_array_to_vec_i64(py: Python<'_>, key: &Bound<'_, PyAny>) -> PyResult<Vec<i
|
||||
buffer.to_vec(py)
|
||||
}
|
||||
|
||||
/// The tagged-tuple tag of a cache action.
|
||||
fn cache_action_tag(action: &CacheAction) -> &'static str {
|
||||
match action {
|
||||
CacheAction::FreeDeviceKV(_) => "free_device_kv",
|
||||
CacheAction::FreeDeviceKVFullOnly(_) => "free_device_kv_full_only",
|
||||
CacheAction::BackupKV(_) => "backup_kv",
|
||||
CacheAction::UpdatePendingBackupOnNodeSplit { .. } => "update_pending_backup_on_node_split",
|
||||
CacheAction::MambaEvictExcessPathStates { .. } => "mamba_evict_excess_path_states",
|
||||
CacheAction::FreeComponentDeviceSlot { .. } => "free_component_device_slot",
|
||||
CacheAction::FreeComponentHostSlot { .. } => "free_component_host_slot",
|
||||
CacheAction::RebuildFullToSwaMapping { .. } => "rebuild_full_to_swa_mapping",
|
||||
CacheAction::RecoverSwaWithLockedFull { .. } => "recover_swa_with_locked_full",
|
||||
CacheAction::SwaRebuild { .. } => "swa_rebuild",
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert a cache action into its Python tagged tuple.
|
||||
fn cache_action_to_py(py: Python<'_>, action: CacheAction) -> PyResult<Py<PyAny>> {
|
||||
let tag = cache_action_tag(&action);
|
||||
match action {
|
||||
CacheAction::FreeDeviceKV(tensors) => {
|
||||
let tensors = PyList::new(py, tensors.into_iter().map(PyTensor))?;
|
||||
(tag, tensors).into_py_any(py)
|
||||
}
|
||||
CacheAction::FreeDeviceKVFullOnly(tensors) => {
|
||||
let tensors = PyList::new(py, tensors.into_iter().map(PyTensor))?;
|
||||
(tag, tensors).into_py_any(py)
|
||||
}
|
||||
CacheAction::BackupKV(backup) => (tag, backup.node_ids).into_py_any(py),
|
||||
CacheAction::UpdatePendingBackupOnNodeSplit {
|
||||
ack_id,
|
||||
old_node_id,
|
||||
new_node_id,
|
||||
new_child_node_id,
|
||||
} => (tag, ack_id, old_node_id, new_node_id, new_child_node_id).into_py_any(py),
|
||||
CacheAction::MambaEvictExcessPathStates { tail_node_id } => {
|
||||
(tag, tail_node_id).into_py_any(py)
|
||||
}
|
||||
CacheAction::FreeComponentDeviceSlot {
|
||||
component_type,
|
||||
indices,
|
||||
} => {
|
||||
let indices = PyList::new(py, indices.into_iter().map(PyTensor))?;
|
||||
(tag, component_type_to_u8(component_type), indices).into_py_any(py)
|
||||
}
|
||||
CacheAction::FreeComponentHostSlot {
|
||||
component_type,
|
||||
host_indices,
|
||||
} => {
|
||||
let host_indices = PyList::new(py, host_indices.into_iter().map(PyTensor))?;
|
||||
(tag, component_type_to_u8(component_type), host_indices).into_py_any(py)
|
||||
}
|
||||
CacheAction::RebuildFullToSwaMapping {
|
||||
full_indices,
|
||||
swa_indices,
|
||||
} => {
|
||||
let full_indices = PyList::new(py, full_indices.into_iter().map(PyTensor))?;
|
||||
let swa_indices = PyList::new(py, swa_indices.into_iter().map(PyTensor))?;
|
||||
(tag, full_indices, swa_indices).into_py_any(py)
|
||||
}
|
||||
CacheAction::RecoverSwaWithLockedFull {
|
||||
node_id,
|
||||
kept_full,
|
||||
incoming_full,
|
||||
} => (tag, node_id, PyTensor(kept_full), PyTensor(incoming_full)).into_py_any(py),
|
||||
CacheAction::SwaRebuild {
|
||||
node_id,
|
||||
source_value,
|
||||
} => (tag, node_id, PyTensor(source_value)).into_py_any(py),
|
||||
}
|
||||
}
|
||||
mod cache_action;
|
||||
|
||||
/// Convert cache actions into a Python list of tagged tuples.
|
||||
fn cache_actions_to_py(py: Python<'_>, actions: Vec<CacheAction>) -> PyResult<Py<PyList>> {
|
||||
fn cache_actions_to_py(py: Python<'_>, actions: CacheActions) -> PyResult<Py<PyList>> {
|
||||
let list = PyList::empty(py);
|
||||
for action in actions {
|
||||
list.append(cache_action_to_py(py, action)?)?;
|
||||
list.append(action.into_py_tagged(py)?)?;
|
||||
}
|
||||
Ok(list.unbind())
|
||||
}
|
||||
@@ -1587,7 +1517,7 @@ macro_rules! tree_core_methods {
|
||||
Ok(EvictDeviceLeafResultBinding {
|
||||
unbacked_tokens: result.unbacked_tokens,
|
||||
backup_kv: backup
|
||||
.map(|backup| cache_action_to_py(py, CacheAction::BackupKV(backup)))
|
||||
.map(|backup| backup.into_py_tagged(py))
|
||||
.transpose()?,
|
||||
tracker: tracker_to_py(result.tracker),
|
||||
new_device_frees: frees_to_py(py, result.device_frees)?,
|
||||
@@ -1774,7 +1704,7 @@ macro_rules! tree_core_methods {
|
||||
let backup = py
|
||||
.detach(|| self.core().build_backup_kv_action(node_id))
|
||||
.map_err(node_access_error)?;
|
||||
cache_action_to_py(py, CacheAction::BackupKV(backup))
|
||||
backup.into_py_tagged(py)
|
||||
}
|
||||
|
||||
/// Gather a node's device value plus per-component BACKUP_HOST transfers.
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
//! Python encoding for the common cache actions.
|
||||
|
||||
use super::*;
|
||||
use crate::cache_action::{
|
||||
BackupKV, FreeComponentDeviceSlot, FreeComponentHostSlot, FreeDeviceKV, FreeDeviceKVFullOnly,
|
||||
MambaEvictExcessPathStates, RebuildFullToSwaMapping, RecoverSwaWithLockedFull, SwaRebuild,
|
||||
UpdatePendingBackupOnNodeSplit,
|
||||
};
|
||||
|
||||
impl FreeDeviceKV {
|
||||
pub(crate) fn into_py_tagged(self, py: Python<'_>) -> PyResult<Py<PyAny>> {
|
||||
let tag = self.tag();
|
||||
let indices = PyList::new(py, self.indices.into_iter().map(PyTensor))?;
|
||||
(tag, indices).into_py_any(py)
|
||||
}
|
||||
}
|
||||
|
||||
impl FreeDeviceKVFullOnly {
|
||||
pub(crate) fn into_py_tagged(self, py: Python<'_>) -> PyResult<Py<PyAny>> {
|
||||
let tag = self.tag();
|
||||
let indices = PyList::new(py, self.indices.into_iter().map(PyTensor))?;
|
||||
(tag, indices).into_py_any(py)
|
||||
}
|
||||
}
|
||||
|
||||
impl BackupKV {
|
||||
pub(crate) fn into_py_tagged(self, py: Python<'_>) -> PyResult<Py<PyAny>> {
|
||||
let tag = self.tag();
|
||||
(tag, self.node_ids).into_py_any(py)
|
||||
}
|
||||
}
|
||||
|
||||
impl UpdatePendingBackupOnNodeSplit {
|
||||
pub(crate) fn into_py_tagged(self, py: Python<'_>) -> PyResult<Py<PyAny>> {
|
||||
let tag = self.tag();
|
||||
(
|
||||
tag,
|
||||
self.ack_id,
|
||||
self.old_node_id,
|
||||
self.new_node_id,
|
||||
self.new_child_node_id,
|
||||
)
|
||||
.into_py_any(py)
|
||||
}
|
||||
}
|
||||
|
||||
impl MambaEvictExcessPathStates {
|
||||
pub(crate) fn into_py_tagged(self, py: Python<'_>) -> PyResult<Py<PyAny>> {
|
||||
let tag = self.tag();
|
||||
(tag, self.tail_node_id).into_py_any(py)
|
||||
}
|
||||
}
|
||||
|
||||
impl FreeComponentDeviceSlot {
|
||||
pub(crate) fn into_py_tagged(self, py: Python<'_>) -> PyResult<Py<PyAny>> {
|
||||
let tag = self.tag();
|
||||
let indices = PyList::new(py, self.indices.into_iter().map(PyTensor))?;
|
||||
(tag, component_type_to_u8(self.component_type), indices).into_py_any(py)
|
||||
}
|
||||
}
|
||||
|
||||
impl FreeComponentHostSlot {
|
||||
pub(crate) fn into_py_tagged(self, py: Python<'_>) -> PyResult<Py<PyAny>> {
|
||||
let tag = self.tag();
|
||||
let indices = PyList::new(py, self.host_indices.into_iter().map(PyTensor))?;
|
||||
(tag, component_type_to_u8(self.component_type), indices).into_py_any(py)
|
||||
}
|
||||
}
|
||||
|
||||
impl RebuildFullToSwaMapping {
|
||||
pub(crate) fn into_py_tagged(self, py: Python<'_>) -> PyResult<Py<PyAny>> {
|
||||
let tag = self.tag();
|
||||
let full_indices = PyList::new(py, self.full_indices.into_iter().map(PyTensor))?;
|
||||
let swa_indices = PyList::new(py, self.swa_indices.into_iter().map(PyTensor))?;
|
||||
(tag, full_indices, swa_indices).into_py_any(py)
|
||||
}
|
||||
}
|
||||
|
||||
impl RecoverSwaWithLockedFull {
|
||||
pub(crate) fn into_py_tagged(self, py: Python<'_>) -> PyResult<Py<PyAny>> {
|
||||
let tag = self.tag();
|
||||
(
|
||||
tag,
|
||||
self.node_id,
|
||||
PyTensor(self.kept_full),
|
||||
PyTensor(self.incoming_full),
|
||||
)
|
||||
.into_py_any(py)
|
||||
}
|
||||
}
|
||||
|
||||
impl SwaRebuild {
|
||||
pub(crate) fn into_py_tagged(self, py: Python<'_>) -> PyResult<Py<PyAny>> {
|
||||
let tag = self.tag();
|
||||
(tag, self.node_id, PyTensor(self.source_value)).into_py_any(py)
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,7 @@
|
||||
use super::*;
|
||||
use crate::cache_action::{CacheAction, MambaEvictExcessPathStates};
|
||||
use crate::components::{FULL, FullComponent, MAMBA, MambaComponent, SWA, SwaComponent};
|
||||
use crate::test_utils::expect_action;
|
||||
use crate::unified_tree_core::CacheInitParams;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
@@ -56,12 +58,12 @@ impl TreeComponent<Vec<i64>> for DefaultComponentForTest {
|
||||
node_id: NodeIdx_,
|
||||
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
target: EvictLayer,
|
||||
) -> (usize, usize) {
|
||||
cache_actions.push(CacheAction::MambaEvictExcessPathStates {
|
||||
cache_actions.push(Box::new(MambaEvictExcessPathStates {
|
||||
tail_node_id: tree_core.arena.node(node_id).id,
|
||||
});
|
||||
}));
|
||||
FullComponent.evict_component(
|
||||
tree_core,
|
||||
node_id,
|
||||
@@ -82,7 +84,7 @@ impl TreeComponent<Vec<i64>> for DefaultComponentForTest {
|
||||
tracker: &mut HashMap<ComponentType, usize>,
|
||||
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
) -> Option<NodeIdx_> {
|
||||
unimplemented!()
|
||||
}
|
||||
@@ -234,13 +236,10 @@ fn action_node(
|
||||
node
|
||||
}
|
||||
|
||||
fn action_node_ids(actions: &[CacheAction]) -> Vec<usize> {
|
||||
fn action_node_ids(actions: &[Box<dyn CacheAction>]) -> Vec<usize> {
|
||||
actions
|
||||
.iter()
|
||||
.map(|action| match action {
|
||||
CacheAction::MambaEvictExcessPathStates { tail_node_id } => *tail_node_id,
|
||||
_ => panic!("unexpected component action"),
|
||||
})
|
||||
.map(|action| expect_action::<MambaEvictExcessPathStates>(action.as_ref()).tail_node_id)
|
||||
.collect()
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use super::*;
|
||||
use crate::cache_action::UpdatePendingBackupOnNodeSplit;
|
||||
use crate::components::{ComponentSet, FULL};
|
||||
use crate::node::NodeAccessError;
|
||||
use crate::test_utils::accumulate_step;
|
||||
@@ -496,7 +497,7 @@ fn host_reclaim_orders_pending_internal_split_fragments_by_ack() {
|
||||
// asynchronously under write_back; matching can split it before the ack.
|
||||
let (prefix_idx, action) = tc.split_node_(tc.arena.resolve(a).unwrap(), 2);
|
||||
let published_a = match action {
|
||||
Some(CacheAction::UpdatePendingBackupOnNodeSplit {
|
||||
Some(UpdatePendingBackupOnNodeSplit {
|
||||
ack_id,
|
||||
old_node_id,
|
||||
new_node_id,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use super::*;
|
||||
use crate::cache_action::{BackupKV, FreeComponentHostSlot, MambaEvictExcessPathStates};
|
||||
use crate::components::{ComponentSet, FULL, MAMBA, SWA};
|
||||
use crate::test_utils::{accumulate_step, action_kinds};
|
||||
use crate::test_utils::{accumulate_step, action_kinds, expect_action};
|
||||
use crate::unified_lru_list::UnifiedLRUList;
|
||||
|
||||
fn mamba_core(page_size: usize) -> UnifiedTreeCore<Vec<i64>> {
|
||||
@@ -560,10 +561,7 @@ fn reinsert_full_backed_target_schedules_mamba_only_backup() {
|
||||
let backups = result
|
||||
.cache_actions
|
||||
.iter()
|
||||
.filter_map(|action| match action {
|
||||
CacheAction::BackupKV(backup) => Some(backup),
|
||||
_ => None,
|
||||
})
|
||||
.filter_map(|action| action.as_any().downcast_ref::<BackupKV>())
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(backups.len(), 1);
|
||||
assert_eq!(backups[0].node_ids, vec![leaf]);
|
||||
@@ -587,7 +585,7 @@ fn reinsert_full_backed_target_schedules_mamba_only_backup() {
|
||||
!pending
|
||||
.cache_actions
|
||||
.iter()
|
||||
.any(|action| matches!(action, CacheAction::BackupKV(_)))
|
||||
.any(|action| action.as_any().is::<BackupKV>())
|
||||
);
|
||||
}
|
||||
|
||||
@@ -607,7 +605,7 @@ fn write_back_reinsert_defers_the_mamba_backup_to_eviction() {
|
||||
!result
|
||||
.cache_actions
|
||||
.iter()
|
||||
.any(|action| matches!(action, CacheAction::BackupKV(_)))
|
||||
.any(|action| action.as_any().is::<BackupKV>())
|
||||
);
|
||||
}
|
||||
|
||||
@@ -984,7 +982,7 @@ fn check_mamba_restored_during_expanded_swa_backup(pin_ancestor: bool) {
|
||||
!inserted
|
||||
.cache_actions
|
||||
.iter()
|
||||
.any(|action| matches!(action, CacheAction::BackupKV(_)))
|
||||
.any(|action| action.as_any().is::<BackupKV>())
|
||||
);
|
||||
assert!(
|
||||
tc.arena
|
||||
@@ -2123,13 +2121,10 @@ fn prefetch_commit_frees_the_buffer_when_it_cannot_attach() {
|
||||
.has_host_value(MAMBA)
|
||||
);
|
||||
assert!(insert_result.mamba_exist);
|
||||
let CacheAction::FreeComponentHostSlot {
|
||||
let FreeComponentHostSlot {
|
||||
component_type,
|
||||
host_indices,
|
||||
} = &cache_actions[0]
|
||||
else {
|
||||
panic!("expected a FreeComponentHostSlot action");
|
||||
};
|
||||
} = expect_action::<FreeComponentHostSlot>(cache_actions[0].as_ref());
|
||||
assert_eq!(*component_type, MAMBA);
|
||||
assert!(host_indices[0].equal(&Tensor::from_slice(&[50i64])));
|
||||
|
||||
@@ -2260,14 +2255,14 @@ fn insert_commit_emits_the_path_cap_action_only_when_capped() {
|
||||
let leaf = tc
|
||||
.match_prefix(&match_params(&vec![1, 2]))
|
||||
.best_match_node_id;
|
||||
let [CacheAction::MambaEvictExcessPathStates { tail_node_id }] =
|
||||
result.cache_actions.as_slice()
|
||||
else {
|
||||
let [action] = result.cache_actions.as_slice() else {
|
||||
panic!(
|
||||
"expected one MambaEvictExcessPathStates, got {:?}",
|
||||
action_kinds(&result.cache_actions)
|
||||
);
|
||||
};
|
||||
let MambaEvictExcessPathStates { tail_node_id } =
|
||||
expect_action::<MambaEvictExcessPathStates>(action.as_ref());
|
||||
assert_eq!(*tail_node_id, leaf);
|
||||
|
||||
let mut uncapped = mamba_core(/* page_size = */ 1);
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
use super::*;
|
||||
use crate::cache_action::{
|
||||
BackupKV, FreeComponentHostSlot, FreeDeviceKV, FreeDeviceKVFullOnly, RebuildFullToSwaMapping,
|
||||
RecoverSwaWithLockedFull, SwaRebuild, UpdatePendingBackupOnNodeSplit,
|
||||
};
|
||||
use crate::components::{ComponentSet, FULL, MAMBA, SWA};
|
||||
use crate::test_utils::{accumulate_step, action_kinds, matched_device_indices};
|
||||
use crate::test_utils::{accumulate_step, action_kinds, expect_action, matched_device_indices};
|
||||
use crate::unified_tree_core::CacheInitParams;
|
||||
|
||||
#[test]
|
||||
@@ -76,7 +80,7 @@ fn swa_window_attach_splits_both_edges_preserving_full_and_pending_actions() {
|
||||
assert_eq!(actions.len(), 2);
|
||||
for action in &actions {
|
||||
assert!(
|
||||
matches!(action, CacheAction::UpdatePendingBackupOnNodeSplit { ack_id: Some(91), old_node_id, new_child_node_id, .. } if *old_node_id == leaf_id && *new_child_node_id == leaf_id)
|
||||
matches!(action.as_any().downcast_ref::<UpdatePendingBackupOnNodeSplit>(), Some(UpdatePendingBackupOnNodeSplit { ack_id: Some(91), old_node_id, new_child_node_id, .. }) if *old_node_id == leaf_id && *new_child_node_id == leaf_id)
|
||||
);
|
||||
}
|
||||
let head = child_of(&tc, tc.arena.root(), &[1, 2]);
|
||||
@@ -904,15 +908,13 @@ fn insert_new_leaf_in_window_emits_one_leaf_rebuild() {
|
||||
assert_eq!(result.prefix_len, 0);
|
||||
let root = tc.arena.root();
|
||||
let leaf = child_of(&tc, root, &[1]);
|
||||
let [
|
||||
CacheAction::SwaRebuild {
|
||||
node_id,
|
||||
source_value,
|
||||
},
|
||||
] = result.cache_actions.as_slice()
|
||||
else {
|
||||
let [action] = result.cache_actions.as_slice() else {
|
||||
panic!("expected one SwaRebuild action");
|
||||
};
|
||||
let SwaRebuild {
|
||||
node_id,
|
||||
source_value,
|
||||
} = expect_action::<SwaRebuild>(action.as_ref());
|
||||
assert_eq!(*node_id, tc.arena.node(leaf).id);
|
||||
assert!(source_value.equal(&Tensor::from_slice(&[10i64, 11, 12])));
|
||||
// The rebuild is deferred to apply time: the leaf is still an SWA tombstone.
|
||||
@@ -940,15 +942,13 @@ fn insert_new_leaf_straddling_the_boundary_splits_and_rebuilds_the_tail() {
|
||||
.device_value(parent, FULL)
|
||||
.equal(&Tensor::from_slice(&[10i64, 11]))
|
||||
);
|
||||
let [
|
||||
CacheAction::SwaRebuild {
|
||||
node_id,
|
||||
source_value,
|
||||
},
|
||||
] = result.cache_actions.as_slice()
|
||||
else {
|
||||
let [action] = result.cache_actions.as_slice() else {
|
||||
panic!("expected one SwaRebuild action");
|
||||
};
|
||||
let SwaRebuild {
|
||||
node_id,
|
||||
source_value,
|
||||
} = expect_action::<SwaRebuild>(action.as_ref());
|
||||
assert_eq!(*node_id, tc.arena.node(child).id);
|
||||
assert!(source_value.equal(&Tensor::from_slice(&[12i64, 13])));
|
||||
}
|
||||
@@ -984,19 +984,17 @@ fn insert_long_leaf_caps_the_window_and_rebuilds_the_older_prefix_first() {
|
||||
let tail = child_of(&tc, capped, &[5]);
|
||||
assert_eq!(tc.arena.node(capped).key, vec![1, 2, 3, 4]);
|
||||
assert_eq!(tc.arena.node(tail).key, vec![5, 6]);
|
||||
let [
|
||||
CacheAction::SwaRebuild {
|
||||
node_id: first_id,
|
||||
source_value: first_value,
|
||||
},
|
||||
CacheAction::SwaRebuild {
|
||||
node_id: second_id,
|
||||
source_value: second_value,
|
||||
},
|
||||
] = result.cache_actions.as_slice()
|
||||
else {
|
||||
let [action0, action1] = result.cache_actions.as_slice() else {
|
||||
panic!("expected two SwaRebuild actions");
|
||||
};
|
||||
let SwaRebuild {
|
||||
node_id: first_id,
|
||||
source_value: first_value,
|
||||
} = expect_action::<SwaRebuild>(action0.as_ref());
|
||||
let SwaRebuild {
|
||||
node_id: second_id,
|
||||
source_value: second_value,
|
||||
} = expect_action::<SwaRebuild>(action1.as_ref());
|
||||
assert_eq!(*first_id, tc.arena.node(capped).id);
|
||||
assert!(first_value.equal(&Tensor::from_slice(&[10i64, 11, 12, 13])));
|
||||
assert_eq!(*second_id, tc.arena.node(tail).id);
|
||||
@@ -1035,9 +1033,10 @@ fn insert_overlap_with_live_swa_frees_the_whole_duplicate() {
|
||||
.expect("live test node");
|
||||
let result = tc.insert(&insert_params_swa(&vec![1, 2, 3], &[20, 21, 22], 0, 0));
|
||||
assert_eq!(result.prefix_len, 3);
|
||||
let [CacheAction::FreeDeviceKV(freed)] = result.cache_actions.as_slice() else {
|
||||
let [action] = result.cache_actions.as_slice() else {
|
||||
panic!("expected one FreeDeviceKV action");
|
||||
};
|
||||
let FreeDeviceKV { indices: freed } = expect_action::<FreeDeviceKV>(action.as_ref());
|
||||
assert_eq!(freed.len(), 1);
|
||||
assert!(freed[0].equal(&Tensor::from_slice(&[20i64, 21, 22])));
|
||||
// The node keeps its original Full and SWA values.
|
||||
@@ -1071,16 +1070,15 @@ fn insert_overlap_recovers_a_tombstone_inside_the_window() {
|
||||
.device_value(leaf, FULL)
|
||||
.equal(&Tensor::from_slice(&[20i64, 21, 22]))
|
||||
);
|
||||
let [
|
||||
CacheAction::FreeDeviceKVFullOnly(freed),
|
||||
CacheAction::SwaRebuild {
|
||||
node_id,
|
||||
source_value,
|
||||
},
|
||||
] = result.cache_actions.as_slice()
|
||||
else {
|
||||
let [action0, action1] = result.cache_actions.as_slice() else {
|
||||
panic!("expected FreeDeviceKVFullOnly then SwaRebuild");
|
||||
};
|
||||
let FreeDeviceKVFullOnly { indices: freed } =
|
||||
expect_action::<FreeDeviceKVFullOnly>(action0.as_ref());
|
||||
let SwaRebuild {
|
||||
node_id,
|
||||
source_value,
|
||||
} = expect_action::<SwaRebuild>(action1.as_ref());
|
||||
assert!(freed[0].equal(&Tensor::from_slice(&[10i64, 11, 12])));
|
||||
assert_eq!(*node_id, tc.arena.node(leaf).id);
|
||||
assert!(source_value.equal(&Tensor::from_slice(&[20i64, 21, 22])));
|
||||
@@ -1110,12 +1108,10 @@ fn insert_overlap_recovers_a_locked_swa_tombstone() {
|
||||
.device_value(leaf, FULL)
|
||||
.equal(&Tensor::from_slice(&[10i64, 11, 12]))
|
||||
);
|
||||
assert!(
|
||||
result
|
||||
.cache_actions
|
||||
.iter()
|
||||
.any(|action| matches!(action, CacheAction::RecoverSwaWithLockedFull { .. }))
|
||||
);
|
||||
assert!(result.cache_actions.iter().any(|action| matches!(
|
||||
action.as_any().downcast_ref::<RecoverSwaWithLockedFull>(),
|
||||
Some(RecoverSwaWithLockedFull { .. })
|
||||
)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1147,16 +1143,14 @@ fn insert_overlap_can_preserve_unlocked_full_ids() {
|
||||
assert_eq!(consumed_from, window_start);
|
||||
assert_eq!(tc.arena.node(leaf).id, original_id);
|
||||
assert_eq!(tc.arena.node(leaf).device_lock_ref(FULL), 0);
|
||||
let [
|
||||
CacheAction::RecoverSwaWithLockedFull {
|
||||
node_id,
|
||||
kept_full,
|
||||
incoming_full,
|
||||
},
|
||||
] = actions.as_slice()
|
||||
else {
|
||||
let [action] = actions.as_slice() else {
|
||||
panic!("expected recovery with canonical FULL IDs");
|
||||
};
|
||||
let RecoverSwaWithLockedFull {
|
||||
node_id,
|
||||
kept_full,
|
||||
incoming_full,
|
||||
} = expect_action::<RecoverSwaWithLockedFull>(action.as_ref());
|
||||
assert_eq!(*node_id, original_id);
|
||||
assert!(kept_full.equal(&Tensor::from_slice(&[10i64, 11, 12, 13][window_start..])));
|
||||
assert!(incoming_full.equal(&Tensor::from_slice(&[20i64, 21, 22, 23][window_start..])));
|
||||
@@ -1187,16 +1181,14 @@ fn insert_overlap_with_a_locked_full_emits_the_recover_action() {
|
||||
.device_value(leaf, FULL)
|
||||
.equal(&Tensor::from_slice(&[10i64, 11, 12]))
|
||||
);
|
||||
let [
|
||||
CacheAction::RecoverSwaWithLockedFull {
|
||||
node_id,
|
||||
kept_full,
|
||||
incoming_full,
|
||||
},
|
||||
] = result.cache_actions.as_slice()
|
||||
else {
|
||||
let [action] = result.cache_actions.as_slice() else {
|
||||
panic!("expected one RecoverSwaWithLockedFull action");
|
||||
};
|
||||
let RecoverSwaWithLockedFull {
|
||||
node_id,
|
||||
kept_full,
|
||||
incoming_full,
|
||||
} = expect_action::<RecoverSwaWithLockedFull>(action.as_ref());
|
||||
assert_eq!(*node_id, tc.arena.node(leaf).id);
|
||||
assert!(kept_full.equal(&Tensor::from_slice(&[10i64, 11, 12])));
|
||||
assert!(incoming_full.equal(&Tensor::from_slice(&[20i64, 21, 22])));
|
||||
@@ -1238,20 +1230,21 @@ fn insert_overlap_straddling_the_boundary_splits_and_recovers_the_tail() {
|
||||
.device_value(node, FULL)
|
||||
.equal(&Tensor::from_slice(&[22i64, 23]))
|
||||
);
|
||||
let [
|
||||
CacheAction::FreeDeviceKVFullOnly(old_tail),
|
||||
CacheAction::SwaRebuild {
|
||||
node_id,
|
||||
source_value,
|
||||
},
|
||||
CacheAction::FreeDeviceKVFullOnly(duplicates),
|
||||
] = result.cache_actions.as_slice()
|
||||
else {
|
||||
let [action0, action1, action2] = result.cache_actions.as_slice() else {
|
||||
panic!(
|
||||
"expected FreeDeviceKVFullOnly, SwaRebuild, FreeDeviceKVFullOnly, got {:?}",
|
||||
action_kinds(&result.cache_actions)
|
||||
);
|
||||
};
|
||||
let FreeDeviceKVFullOnly { indices: old_tail } =
|
||||
expect_action::<FreeDeviceKVFullOnly>(action0.as_ref());
|
||||
let SwaRebuild {
|
||||
node_id,
|
||||
source_value,
|
||||
} = expect_action::<SwaRebuild>(action1.as_ref());
|
||||
let FreeDeviceKVFullOnly {
|
||||
indices: duplicates,
|
||||
} = expect_action::<FreeDeviceKVFullOnly>(action2.as_ref());
|
||||
assert!(old_tail[0].equal(&Tensor::from_slice(&[12i64, 13])));
|
||||
assert_eq!(*node_id, tc.arena.node(node).id);
|
||||
assert!(source_value.equal(&Tensor::from_slice(&[22i64, 23])));
|
||||
@@ -1285,17 +1278,17 @@ fn insert_overlap_straddling_with_a_locked_full_defers_the_tail() {
|
||||
.device_value(node, FULL)
|
||||
.equal(&Tensor::from_slice(&[12i64, 13]))
|
||||
);
|
||||
let [
|
||||
CacheAction::RecoverSwaWithLockedFull {
|
||||
node_id,
|
||||
kept_full,
|
||||
incoming_full,
|
||||
},
|
||||
CacheAction::FreeDeviceKVFullOnly(duplicates),
|
||||
] = result.cache_actions.as_slice()
|
||||
else {
|
||||
let [action0, action1] = result.cache_actions.as_slice() else {
|
||||
panic!("expected RecoverSwaWithLockedFull then FreeDeviceKVFullOnly");
|
||||
};
|
||||
let RecoverSwaWithLockedFull {
|
||||
node_id,
|
||||
kept_full,
|
||||
incoming_full,
|
||||
} = expect_action::<RecoverSwaWithLockedFull>(action0.as_ref());
|
||||
let FreeDeviceKVFullOnly {
|
||||
indices: duplicates,
|
||||
} = expect_action::<FreeDeviceKVFullOnly>(action1.as_ref());
|
||||
assert_eq!(*node_id, tc.arena.node(node).id);
|
||||
assert!(kept_full.equal(&Tensor::from_slice(&[12i64, 13])));
|
||||
assert!(incoming_full.equal(&Tensor::from_slice(&[22i64, 23])));
|
||||
@@ -1313,9 +1306,11 @@ fn insert_overlap_entirely_outside_the_window_is_all_duplicate() {
|
||||
/* prev_prefix_len = */ 0,
|
||||
/* swa_evicted_seqlen = */ 3,
|
||||
));
|
||||
let [CacheAction::FreeDeviceKVFullOnly(freed)] = result.cache_actions.as_slice() else {
|
||||
let [action] = result.cache_actions.as_slice() else {
|
||||
panic!("expected one FreeDeviceKVFullOnly action");
|
||||
};
|
||||
let FreeDeviceKVFullOnly { indices: freed } =
|
||||
expect_action::<FreeDeviceKVFullOnly>(action.as_ref());
|
||||
assert!(freed[0].equal(&Tensor::from_slice(&[20i64, 21, 22])));
|
||||
}
|
||||
|
||||
@@ -1365,20 +1360,21 @@ fn insert_overlap_boundary_at_the_node_start_recovers_the_whole_node() {
|
||||
.device_value(b, FULL)
|
||||
.equal(&Tensor::from_slice(&[23i64, 24]))
|
||||
);
|
||||
let [
|
||||
CacheAction::FreeDeviceKVFullOnly(duplicates),
|
||||
CacheAction::FreeDeviceKVFullOnly(old_full),
|
||||
CacheAction::SwaRebuild {
|
||||
node_id,
|
||||
source_value,
|
||||
},
|
||||
] = result.cache_actions.as_slice()
|
||||
else {
|
||||
let [action0, action1, action2] = result.cache_actions.as_slice() else {
|
||||
panic!(
|
||||
"expected FreeDeviceKVFullOnly, FreeDeviceKVFullOnly, SwaRebuild, got {:?}",
|
||||
action_kinds(&result.cache_actions)
|
||||
);
|
||||
};
|
||||
let FreeDeviceKVFullOnly {
|
||||
indices: duplicates,
|
||||
} = expect_action::<FreeDeviceKVFullOnly>(action0.as_ref());
|
||||
let FreeDeviceKVFullOnly { indices: old_full } =
|
||||
expect_action::<FreeDeviceKVFullOnly>(action1.as_ref());
|
||||
let SwaRebuild {
|
||||
node_id,
|
||||
source_value,
|
||||
} = expect_action::<SwaRebuild>(action2.as_ref());
|
||||
assert!(duplicates[0].equal(&Tensor::from_slice(&[20i64, 21, 22])));
|
||||
assert!(old_full[0].equal(&Tensor::from_slice(&[13i64, 14])));
|
||||
assert_eq!(*node_id, tc.arena.node(b).id);
|
||||
@@ -1420,21 +1416,24 @@ fn insert_overlap_straddling_a_second_level_node_recovers_the_tail() {
|
||||
.device_value(b, FULL)
|
||||
.equal(&Tensor::from_slice(&[24i64]))
|
||||
);
|
||||
let [
|
||||
CacheAction::FreeDeviceKVFullOnly(duplicates_head),
|
||||
CacheAction::FreeDeviceKVFullOnly(old_tail),
|
||||
CacheAction::SwaRebuild {
|
||||
node_id,
|
||||
source_value,
|
||||
},
|
||||
CacheAction::FreeDeviceKVFullOnly(duplicates_tail),
|
||||
] = result.cache_actions.as_slice()
|
||||
else {
|
||||
let [action0, action1, action2, action3] = result.cache_actions.as_slice() else {
|
||||
panic!(
|
||||
"expected FreeDeviceKVFullOnly, FreeDeviceKVFullOnly, SwaRebuild, FreeDeviceKVFullOnly, got {:?}",
|
||||
action_kinds(&result.cache_actions)
|
||||
);
|
||||
};
|
||||
let FreeDeviceKVFullOnly {
|
||||
indices: duplicates_head,
|
||||
} = expect_action::<FreeDeviceKVFullOnly>(action0.as_ref());
|
||||
let FreeDeviceKVFullOnly { indices: old_tail } =
|
||||
expect_action::<FreeDeviceKVFullOnly>(action1.as_ref());
|
||||
let SwaRebuild {
|
||||
node_id,
|
||||
source_value,
|
||||
} = expect_action::<SwaRebuild>(action2.as_ref());
|
||||
let FreeDeviceKVFullOnly {
|
||||
indices: duplicates_tail,
|
||||
} = expect_action::<FreeDeviceKVFullOnly>(action3.as_ref());
|
||||
assert!(duplicates_head[0].equal(&Tensor::from_slice(&[20i64, 21, 22])));
|
||||
assert!(old_tail[0].equal(&Tensor::from_slice(&[14i64])));
|
||||
assert_eq!(*node_id, tc.arena.node(b).id);
|
||||
@@ -1473,21 +1472,23 @@ fn insert_overlap_straddling_a_second_level_locked_node_defers_the_tail() {
|
||||
.device_value(b, FULL)
|
||||
.equal(&Tensor::from_slice(&[14i64]))
|
||||
);
|
||||
let [
|
||||
CacheAction::FreeDeviceKVFullOnly(duplicates_head),
|
||||
CacheAction::RecoverSwaWithLockedFull {
|
||||
node_id,
|
||||
kept_full,
|
||||
incoming_full,
|
||||
},
|
||||
CacheAction::FreeDeviceKVFullOnly(duplicates_tail),
|
||||
] = result.cache_actions.as_slice()
|
||||
else {
|
||||
let [action0, action1, action2] = result.cache_actions.as_slice() else {
|
||||
panic!(
|
||||
"expected FreeDeviceKVFullOnly, RecoverSwaWithLockedFull, FreeDeviceKVFullOnly, got {:?}",
|
||||
action_kinds(&result.cache_actions)
|
||||
);
|
||||
};
|
||||
let FreeDeviceKVFullOnly {
|
||||
indices: duplicates_head,
|
||||
} = expect_action::<FreeDeviceKVFullOnly>(action0.as_ref());
|
||||
let RecoverSwaWithLockedFull {
|
||||
node_id,
|
||||
kept_full,
|
||||
incoming_full,
|
||||
} = expect_action::<RecoverSwaWithLockedFull>(action1.as_ref());
|
||||
let FreeDeviceKVFullOnly {
|
||||
indices: duplicates_tail,
|
||||
} = expect_action::<FreeDeviceKVFullOnly>(action2.as_ref());
|
||||
assert!(duplicates_head[0].equal(&Tensor::from_slice(&[20i64, 21, 22])));
|
||||
assert_eq!(*node_id, tc.arena.node(b).id);
|
||||
assert!(kept_full.equal(&Tensor::from_slice(&[14i64])));
|
||||
@@ -1518,10 +1519,7 @@ fn insert_overlap_prev_prefix_strictly_inside_the_node_still_recovers() {
|
||||
.device_value(node, FULL)
|
||||
.equal(&Tensor::from_slice(&[20i64, 21, 22, 23]))
|
||||
);
|
||||
assert!(result.cache_actions.iter().any(|action| matches!(
|
||||
action,
|
||||
CacheAction::SwaRebuild { node_id, .. } if *node_id == tc.arena.node(node).id
|
||||
)));
|
||||
assert!(result.cache_actions.iter().any(|action| matches!(action.as_any().downcast_ref::<SwaRebuild>(), Some(SwaRebuild { node_id, .. }) if *node_id == tc.arena.node(node).id)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1551,15 +1549,13 @@ fn reinsert_after_full_eviction_rebuilds_swa_from_the_fresh_kv() {
|
||||
.device_value(leaf, FULL)
|
||||
.equal(&Tensor::from_slice(&[20i64, 21, 22]))
|
||||
);
|
||||
let [
|
||||
CacheAction::SwaRebuild {
|
||||
node_id,
|
||||
source_value,
|
||||
},
|
||||
] = result.cache_actions.as_slice()
|
||||
else {
|
||||
let [action] = result.cache_actions.as_slice() else {
|
||||
panic!("expected one SwaRebuild action");
|
||||
};
|
||||
let SwaRebuild {
|
||||
node_id,
|
||||
source_value,
|
||||
} = expect_action::<SwaRebuild>(action.as_ref());
|
||||
assert_eq!(*node_id, tc.arena.node(leaf).id);
|
||||
assert!(source_value.equal(&Tensor::from_slice(&[20i64, 21, 22])));
|
||||
}
|
||||
@@ -1592,15 +1588,13 @@ fn reinsert_straddling_the_boundary_splits_before_the_rebuild() {
|
||||
.equal(&Tensor::from_slice(&[20i64, 21]))
|
||||
);
|
||||
assert!(!tc.arena.has_device_value(parent, SWA));
|
||||
let [
|
||||
CacheAction::SwaRebuild {
|
||||
node_id,
|
||||
source_value,
|
||||
},
|
||||
] = result.cache_actions.as_slice()
|
||||
else {
|
||||
let [action] = result.cache_actions.as_slice() else {
|
||||
panic!("expected one SwaRebuild action");
|
||||
};
|
||||
let SwaRebuild {
|
||||
node_id,
|
||||
source_value,
|
||||
} = expect_action::<SwaRebuild>(action.as_ref());
|
||||
assert_eq!(*node_id, tc.arena.node(node).id);
|
||||
assert!(source_value.equal(&Tensor::from_slice(&[22i64, 23])));
|
||||
}
|
||||
@@ -1641,15 +1635,9 @@ fn reinsert_straddling_the_boundary_at_a_second_level_node_splits_before_the_reb
|
||||
.equal(&Tensor::from_slice(&[24i64]))
|
||||
);
|
||||
assert!(!tc.arena.has_device_value(p, SWA));
|
||||
assert!(result.cache_actions.iter().any(|action| matches!(
|
||||
action,
|
||||
CacheAction::SwaRebuild { node_id, source_value }
|
||||
if *node_id == tc.arena.node(b).id && source_value.equal(&Tensor::from_slice(&[24i64]))
|
||||
)));
|
||||
assert!(!result.cache_actions.iter().any(|action| matches!(
|
||||
action,
|
||||
CacheAction::SwaRebuild { node_id, .. } if *node_id == tc.arena.node(p).id
|
||||
)));
|
||||
assert!(result.cache_actions.iter().any(|action| matches!(action.as_any().downcast_ref::<SwaRebuild>(), Some(SwaRebuild { node_id, source_value })
|
||||
if *node_id == tc.arena.node(b).id && source_value.equal(&Tensor::from_slice(&[24i64])))));
|
||||
assert!(!result.cache_actions.iter().any(|action| matches!(action.as_any().downcast_ref::<SwaRebuild>(), Some(SwaRebuild { node_id, .. }) if *node_id == tc.arena.node(p).id)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1715,19 +1703,19 @@ fn reinsert_boundary_at_the_node_start_rebuilds_the_whole_node() {
|
||||
/* swa_evicted_seqlen = */ 3,
|
||||
));
|
||||
assert_eq!(tc.arena.node(b).key, vec![4, 5]);
|
||||
let [
|
||||
CacheAction::FreeDeviceKVFullOnly(duplicates),
|
||||
CacheAction::SwaRebuild {
|
||||
node_id,
|
||||
source_value,
|
||||
},
|
||||
] = result.cache_actions.as_slice()
|
||||
else {
|
||||
let [action0, action1] = result.cache_actions.as_slice() else {
|
||||
panic!(
|
||||
"expected FreeDeviceKVFullOnly then SwaRebuild, got {:?}",
|
||||
action_kinds(&result.cache_actions)
|
||||
);
|
||||
};
|
||||
let FreeDeviceKVFullOnly {
|
||||
indices: duplicates,
|
||||
} = expect_action::<FreeDeviceKVFullOnly>(action0.as_ref());
|
||||
let SwaRebuild {
|
||||
node_id,
|
||||
source_value,
|
||||
} = expect_action::<SwaRebuild>(action1.as_ref());
|
||||
assert!(duplicates[0].equal(&Tensor::from_slice(&[20i64, 21, 22])));
|
||||
assert_eq!(*node_id, tc.arena.node(b).id);
|
||||
assert!(source_value.equal(&Tensor::from_slice(&[23i64, 24])));
|
||||
@@ -1768,16 +1756,16 @@ fn walk_split_redistributes_the_live_swa_value() {
|
||||
);
|
||||
assert!(tc.device_lru_list(SWA).in_list(Some(parent)));
|
||||
assert!(tc.device_lru_list(SWA).in_list(Some(node)));
|
||||
let [
|
||||
CacheAction::FreeDeviceKV(duplicates),
|
||||
CacheAction::SwaRebuild {
|
||||
node_id,
|
||||
source_value,
|
||||
},
|
||||
] = result.cache_actions.as_slice()
|
||||
else {
|
||||
let [action0, action1] = result.cache_actions.as_slice() else {
|
||||
panic!("expected FreeDeviceKV then SwaRebuild");
|
||||
};
|
||||
let FreeDeviceKV {
|
||||
indices: duplicates,
|
||||
} = expect_action::<FreeDeviceKV>(action0.as_ref());
|
||||
let SwaRebuild {
|
||||
node_id,
|
||||
source_value,
|
||||
} = expect_action::<SwaRebuild>(action1.as_ref());
|
||||
assert!(duplicates[0].equal(&Tensor::from_slice(&[20i64, 21])));
|
||||
assert_eq!(*node_id, tc.arena.node(leaf).id);
|
||||
assert!(source_value.equal(&Tensor::from_slice(&[29i64])));
|
||||
@@ -3968,15 +3956,9 @@ fn new_leaf_after_a_cached_prefix_splits_at_the_leaf_relative_boundary() {
|
||||
.device_value(leaf, FULL)
|
||||
.equal(&Tensor::from_slice(&[24i64, 25]))
|
||||
);
|
||||
assert!(result.cache_actions.iter().any(|action| matches!(
|
||||
action,
|
||||
CacheAction::SwaRebuild { node_id, source_value }
|
||||
if *node_id == tc.arena.node(leaf).id && source_value.equal(&Tensor::from_slice(&[24i64, 25]))
|
||||
)));
|
||||
assert!(!result.cache_actions.iter().any(|action| matches!(
|
||||
action,
|
||||
CacheAction::SwaRebuild { node_id, .. } if *node_id == tc.arena.node(p).id
|
||||
)));
|
||||
assert!(result.cache_actions.iter().any(|action| matches!(action.as_any().downcast_ref::<SwaRebuild>(), Some(SwaRebuild { node_id, source_value })
|
||||
if *node_id == tc.arena.node(leaf).id && source_value.equal(&Tensor::from_slice(&[24i64, 25])))));
|
||||
assert!(!result.cache_actions.iter().any(|action| matches!(action.as_any().downcast_ref::<SwaRebuild>(), Some(SwaRebuild { node_id, .. }) if *node_id == tc.arena.node(p).id)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4006,41 +3988,42 @@ fn new_leaf_boundary_split_and_window_cap_compose_in_one_commit() {
|
||||
let rebuilds: Vec<NodeId> = result
|
||||
.cache_actions
|
||||
.iter()
|
||||
.filter_map(|action| match action {
|
||||
CacheAction::SwaRebuild { node_id, .. } => Some(*node_id),
|
||||
_ => None,
|
||||
})
|
||||
.filter_map(
|
||||
|action| match action.as_any().downcast_ref::<SwaRebuild>() {
|
||||
Some(SwaRebuild { node_id, .. }) => Some(*node_id),
|
||||
_ => None,
|
||||
},
|
||||
)
|
||||
.collect();
|
||||
assert_eq!(
|
||||
rebuilds,
|
||||
vec![tc.arena.node(capped).id, tc.arena.node(leaf).id]
|
||||
);
|
||||
assert!(result.cache_actions.iter().any(|action| matches!(
|
||||
action,
|
||||
CacheAction::SwaRebuild { node_id, source_value }
|
||||
if *node_id == tc.arena.node(capped).id && source_value.equal(&Tensor::from_slice(&[23i64, 24]))
|
||||
)));
|
||||
assert!(result.cache_actions.iter().any(|action| matches!(
|
||||
action,
|
||||
CacheAction::SwaRebuild { node_id, source_value }
|
||||
if *node_id == tc.arena.node(leaf).id && source_value.equal(&Tensor::from_slice(&[25i64, 26]))
|
||||
)));
|
||||
assert!(result.cache_actions.iter().any(|action| matches!(action.as_any().downcast_ref::<SwaRebuild>(), Some(SwaRebuild { node_id, source_value })
|
||||
if *node_id == tc.arena.node(capped).id && source_value.equal(&Tensor::from_slice(&[23i64, 24])))));
|
||||
assert!(result.cache_actions.iter().any(|action| matches!(action.as_any().downcast_ref::<SwaRebuild>(), Some(SwaRebuild { node_id, source_value })
|
||||
if *node_id == tc.arena.node(leaf).id && source_value.equal(&Tensor::from_slice(&[25i64, 26])))));
|
||||
let split_ids: Vec<NodeId> = result
|
||||
.cache_actions
|
||||
.iter()
|
||||
.filter_map(|action| match action {
|
||||
CacheAction::UpdatePendingBackupOnNodeSplit {
|
||||
ack_id,
|
||||
old_node_id,
|
||||
new_node_id,
|
||||
new_child_node_id,
|
||||
} => {
|
||||
assert_eq!(*ack_id, None);
|
||||
assert_eq!(*old_node_id, tc.arena.node(leaf).id);
|
||||
assert_eq!(*new_child_node_id, tc.arena.node(leaf).id);
|
||||
Some(*new_node_id)
|
||||
.filter_map(|action| {
|
||||
match action
|
||||
.as_any()
|
||||
.downcast_ref::<UpdatePendingBackupOnNodeSplit>()
|
||||
{
|
||||
Some(UpdatePendingBackupOnNodeSplit {
|
||||
ack_id,
|
||||
old_node_id,
|
||||
new_node_id,
|
||||
new_child_node_id,
|
||||
}) => {
|
||||
assert_eq!(*ack_id, None);
|
||||
assert_eq!(*old_node_id, tc.arena.node(leaf).id);
|
||||
assert_eq!(*new_child_node_id, tc.arena.node(leaf).id);
|
||||
Some(*new_node_id)
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
if buffer_only {
|
||||
@@ -4051,13 +4034,23 @@ fn new_leaf_boundary_split_and_window_cap_compose_in_one_commit() {
|
||||
let first_rebuild = result
|
||||
.cache_actions
|
||||
.iter()
|
||||
.position(|action| matches!(action, CacheAction::SwaRebuild { .. }))
|
||||
.position(|action| {
|
||||
matches!(
|
||||
action.as_any().downcast_ref::<SwaRebuild>(),
|
||||
Some(SwaRebuild { .. })
|
||||
)
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
result.cache_actions[..first_rebuild]
|
||||
.iter()
|
||||
.filter(|action| {
|
||||
matches!(action, CacheAction::UpdatePendingBackupOnNodeSplit { .. })
|
||||
matches!(
|
||||
action
|
||||
.as_any()
|
||||
.downcast_ref::<UpdatePendingBackupOnNodeSplit>(),
|
||||
Some(UpdatePendingBackupOnNodeSplit { .. })
|
||||
)
|
||||
})
|
||||
.count(),
|
||||
2,
|
||||
@@ -4199,11 +4192,8 @@ fn new_leaf_with_the_boundary_above_its_start_skips_the_split() {
|
||||
// in-window and stays unsplit.
|
||||
let leaf = child_of(&tc, a, &[4]);
|
||||
assert_eq!(tc.arena.node(leaf).key, vec![4, 5]);
|
||||
assert!(result.cache_actions.iter().any(|action| matches!(
|
||||
action,
|
||||
CacheAction::SwaRebuild { node_id, source_value }
|
||||
if *node_id == tc.arena.node(leaf).id && source_value.equal(&Tensor::from_slice(&[23i64, 24]))
|
||||
)));
|
||||
assert!(result.cache_actions.iter().any(|action| matches!(action.as_any().downcast_ref::<SwaRebuild>(), Some(SwaRebuild { node_id, source_value })
|
||||
if *node_id == tc.arena.node(leaf).id && source_value.equal(&Tensor::from_slice(&[23i64, 24])))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4237,20 +4227,21 @@ fn insert_overlap_straddling_with_a_partial_prev_prefix_recovers_the_tail() {
|
||||
.device_value(node, FULL)
|
||||
.equal(&Tensor::from_slice(&[22i64, 23]))
|
||||
);
|
||||
let [
|
||||
CacheAction::FreeDeviceKVFullOnly(old_tail),
|
||||
CacheAction::SwaRebuild {
|
||||
node_id,
|
||||
source_value,
|
||||
},
|
||||
CacheAction::FreeDeviceKVFullOnly(duplicates),
|
||||
] = result.cache_actions.as_slice()
|
||||
else {
|
||||
let [action0, action1, action2] = result.cache_actions.as_slice() else {
|
||||
panic!(
|
||||
"expected FreeDeviceKVFullOnly, SwaRebuild, FreeDeviceKVFullOnly, got {:?}",
|
||||
action_kinds(&result.cache_actions)
|
||||
);
|
||||
};
|
||||
let FreeDeviceKVFullOnly { indices: old_tail } =
|
||||
expect_action::<FreeDeviceKVFullOnly>(action0.as_ref());
|
||||
let SwaRebuild {
|
||||
node_id,
|
||||
source_value,
|
||||
} = expect_action::<SwaRebuild>(action1.as_ref());
|
||||
let FreeDeviceKVFullOnly {
|
||||
indices: duplicates,
|
||||
} = expect_action::<FreeDeviceKVFullOnly>(action2.as_ref());
|
||||
assert!(old_tail[0].equal(&Tensor::from_slice(&[12i64, 13])));
|
||||
assert_eq!(*node_id, tc.arena.node(node).id);
|
||||
assert!(source_value.equal(&Tensor::from_slice(&[22i64, 23])));
|
||||
@@ -4284,11 +4275,8 @@ fn insert_overlap_straddling_recovers_across_page_two() {
|
||||
.device_value(node, FULL)
|
||||
.equal(&Tensor::from_slice(&[22i64, 23]))
|
||||
);
|
||||
assert!(result.cache_actions.iter().any(|action| matches!(
|
||||
action,
|
||||
CacheAction::SwaRebuild { node_id, source_value }
|
||||
if *node_id == tc.arena.node(node).id && source_value.equal(&Tensor::from_slice(&[22i64, 23]))
|
||||
)));
|
||||
assert!(result.cache_actions.iter().any(|action| matches!(action.as_any().downcast_ref::<SwaRebuild>(), Some(SwaRebuild { node_id, source_value })
|
||||
if *node_id == tc.arena.node(node).id && source_value.equal(&Tensor::from_slice(&[22i64, 23])))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4322,15 +4310,9 @@ fn reinsert_straddling_recovers_across_page_two() {
|
||||
.device_value(node, FULL)
|
||||
.equal(&Tensor::from_slice(&[22i64, 23]))
|
||||
);
|
||||
assert!(result.cache_actions.iter().any(|action| matches!(
|
||||
action,
|
||||
CacheAction::SwaRebuild { node_id, source_value }
|
||||
if *node_id == tc.arena.node(node).id && source_value.equal(&Tensor::from_slice(&[22i64, 23]))
|
||||
)));
|
||||
assert!(!result.cache_actions.iter().any(|action| matches!(
|
||||
action,
|
||||
CacheAction::SwaRebuild { node_id, .. } if *node_id == tc.arena.node(parent).id
|
||||
)));
|
||||
assert!(result.cache_actions.iter().any(|action| matches!(action.as_any().downcast_ref::<SwaRebuild>(), Some(SwaRebuild { node_id, source_value })
|
||||
if *node_id == tc.arena.node(node).id && source_value.equal(&Tensor::from_slice(&[22i64, 23])))));
|
||||
assert!(!result.cache_actions.iter().any(|action| matches!(action.as_any().downcast_ref::<SwaRebuild>(), Some(SwaRebuild { node_id, .. }) if *node_id == tc.arena.node(parent).id)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4362,12 +4344,10 @@ fn reinsert_rebuilds_a_locked_tombstone() {
|
||||
.node_mut(node)
|
||||
.set_lock_ref_(ValueSlotIdx::device(SWA), 1);
|
||||
let result = tc.insert(&insert_params_swa(&vec![1, 2], &[20, 21], 0, 0));
|
||||
assert!(
|
||||
result
|
||||
.cache_actions
|
||||
.iter()
|
||||
.any(|action| matches!(action, CacheAction::SwaRebuild { .. }))
|
||||
);
|
||||
assert!(result.cache_actions.iter().any(|action| matches!(
|
||||
action.as_any().downcast_ref::<SwaRebuild>(),
|
||||
Some(SwaRebuild { .. })
|
||||
)));
|
||||
}
|
||||
|
||||
fn set_full_host(tc: &mut UnifiedTreeCore<Vec<i64>>, node: NodeIdx_) {
|
||||
@@ -4955,13 +4935,10 @@ fn load_back_commit_attaches_chunks_and_emits_the_rebuild_action() {
|
||||
assert!(tc.device_lru_list(SWA).in_list(Some(b)));
|
||||
assert_eq!(tc.swa_evictable_size(), 2);
|
||||
assert_eq!(cache_actions.len(), 1);
|
||||
let CacheAction::RebuildFullToSwaMapping {
|
||||
let RebuildFullToSwaMapping {
|
||||
full_indices,
|
||||
swa_indices,
|
||||
} = &cache_actions[0]
|
||||
else {
|
||||
panic!("expected a RebuildFullToSwaMapping action");
|
||||
};
|
||||
} = expect_action::<RebuildFullToSwaMapping>(cache_actions[0].as_ref());
|
||||
assert_eq!(full_indices.len(), 2);
|
||||
assert!(full_indices[0].equal(&Tensor::from_slice(&[40i64])));
|
||||
assert!(full_indices[1].equal(&Tensor::from_slice(&[41i64])));
|
||||
@@ -5196,9 +5173,8 @@ fn prefetch_commit_drops_the_whole_window_when_underloaded() {
|
||||
);
|
||||
assert!(!tc.arena.node(a).has_host_value(SWA));
|
||||
assert_eq!(cache_actions.len(), 1);
|
||||
let CacheAction::FreeComponentHostSlot { host_indices, .. } = &cache_actions[0] else {
|
||||
panic!("expected a host free");
|
||||
};
|
||||
let FreeComponentHostSlot { host_indices, .. } =
|
||||
expect_action::<FreeComponentHostSlot>(cache_actions[0].as_ref());
|
||||
assert!(host_indices[0].equal(&Tensor::from_slice(&[30i64, 31])));
|
||||
}
|
||||
|
||||
@@ -5227,9 +5203,8 @@ fn prefetch_commit_without_a_target_releases_the_whole_buffer() {
|
||||
Some(&storage_result),
|
||||
);
|
||||
assert_eq!(cache_actions.len(), 1);
|
||||
let CacheAction::FreeComponentHostSlot { host_indices, .. } = &cache_actions[0] else {
|
||||
panic!("expected a host free");
|
||||
};
|
||||
let FreeComponentHostSlot { host_indices, .. } =
|
||||
expect_action::<FreeComponentHostSlot>(cache_actions[0].as_ref());
|
||||
assert!(host_indices[0].equal(&Tensor::from_slice(&[30i64, 31])));
|
||||
|
||||
// An insert result without an inserted host node releases the same way.
|
||||
@@ -5253,9 +5228,8 @@ fn prefetch_commit_without_a_target_releases_the_whole_buffer() {
|
||||
Some(&storage_result),
|
||||
);
|
||||
assert_eq!(cache_actions.len(), 1);
|
||||
let CacheAction::FreeComponentHostSlot { host_indices, .. } = &cache_actions[0] else {
|
||||
panic!("expected a host free");
|
||||
};
|
||||
let FreeComponentHostSlot { host_indices, .. } =
|
||||
expect_action::<FreeComponentHostSlot>(cache_actions[0].as_ref());
|
||||
assert!(host_indices[0].equal(&Tensor::from_slice(&[30i64, 31])));
|
||||
assert!(!tc.arena.node(a).has_host_value(SWA));
|
||||
}
|
||||
@@ -5294,9 +5268,8 @@ fn prefetch_commit_releases_a_shortened_window_under_a_non_root_anchor() {
|
||||
assert!(!tc.arena.node(a).has_host_value(SWA));
|
||||
assert!(!tc.host_lru_list(SWA).in_list(Some(c)));
|
||||
assert_eq!(cache_actions.len(), 1);
|
||||
let CacheAction::FreeComponentHostSlot { host_indices, .. } = &cache_actions[0] else {
|
||||
panic!("expected a host free");
|
||||
};
|
||||
let FreeComponentHostSlot { host_indices, .. } =
|
||||
expect_action::<FreeComponentHostSlot>(cache_actions[0].as_ref());
|
||||
assert!(host_indices[0].equal(&Tensor::from_slice(&[30i64, 31])));
|
||||
}
|
||||
|
||||
@@ -5346,9 +5319,8 @@ fn prefetch_commit_fills_tombstoned_nodes_and_releases_covered_ones() {
|
||||
.equal(&Tensor::from_slice(&[40i64]))
|
||||
);
|
||||
assert_eq!(cache_actions.len(), 1);
|
||||
let CacheAction::FreeComponentHostSlot { host_indices, .. } = &cache_actions[0] else {
|
||||
panic!("expected a host free");
|
||||
};
|
||||
let FreeComponentHostSlot { host_indices, .. } =
|
||||
expect_action::<FreeComponentHostSlot>(cache_actions[0].as_ref());
|
||||
assert!(host_indices[0].equal(&Tensor::from_slice(&[31i64])));
|
||||
}
|
||||
|
||||
@@ -5405,13 +5377,10 @@ fn release_swa_host_queues_a_free_action_for_non_empty_indices() {
|
||||
let mut cache_actions = Vec::new();
|
||||
swa_component(4).release_swa_host_(Tensor::from_slice(&[7i64]), &mut cache_actions);
|
||||
assert_eq!(cache_actions.len(), 1);
|
||||
let CacheAction::FreeComponentHostSlot {
|
||||
let FreeComponentHostSlot {
|
||||
component_type,
|
||||
host_indices,
|
||||
} = &cache_actions[0]
|
||||
else {
|
||||
panic!("expected a FreeComponentHostSlot action");
|
||||
};
|
||||
} = expect_action::<FreeComponentHostSlot>(cache_actions[0].as_ref());
|
||||
assert_eq!(*component_type, SWA);
|
||||
assert_eq!(host_indices.len(), 1);
|
||||
assert!(host_indices[0].equal(&Tensor::from_slice(&[7i64])));
|
||||
@@ -5499,8 +5468,10 @@ fn build_load_back_spec_includes_the_swa_transfers() {
|
||||
);
|
||||
assert_eq!(actions.len(), 1);
|
||||
assert!(matches!(
|
||||
actions[0],
|
||||
CacheAction::RebuildFullToSwaMapping { .. }
|
||||
(actions[0])
|
||||
.as_any()
|
||||
.downcast_ref::<RebuildFullToSwaMapping>(),
|
||||
Some(RebuildFullToSwaMapping { .. })
|
||||
));
|
||||
}
|
||||
|
||||
@@ -5642,16 +5613,14 @@ fn check_swa_device_restored_during_full_load_back(pin_ancestor: bool) {
|
||||
// A is outside the loaded SWA window. A new request emits the normal
|
||||
// locked-Full repair action; resolve only its allocator/mapping boundary.
|
||||
let inserted = tc.insert(&insert_params_swa(&vec![1], &[90], 0, 0));
|
||||
let [
|
||||
CacheAction::RecoverSwaWithLockedFull {
|
||||
node_id,
|
||||
kept_full,
|
||||
incoming_full,
|
||||
},
|
||||
] = inserted.cache_actions.as_slice()
|
||||
else {
|
||||
let [action] = inserted.cache_actions.as_slice() else {
|
||||
panic!("expected one RecoverSwaWithLockedFull action");
|
||||
};
|
||||
let RecoverSwaWithLockedFull {
|
||||
node_id,
|
||||
kept_full,
|
||||
incoming_full,
|
||||
} = expect_action::<RecoverSwaWithLockedFull>(action.as_ref());
|
||||
assert_eq!(*node_id, a_id);
|
||||
assert!(kept_full.equal(&Tensor::from_slice(&[50i64])));
|
||||
assert!(incoming_full.equal(&Tensor::from_slice(&[90i64])));
|
||||
@@ -6009,9 +5978,11 @@ fn write_through_offloads_a_boundary_split_leaf() {
|
||||
let backups: Vec<_> = result
|
||||
.cache_actions
|
||||
.iter()
|
||||
.filter_map(|action| match action {
|
||||
CacheAction::BackupKV(backup) => Some(backup.node_ids.clone()),
|
||||
_ => None,
|
||||
.filter_map(|action| {
|
||||
action
|
||||
.as_any()
|
||||
.downcast_ref::<BackupKV>()
|
||||
.map(|backup| backup.node_ids.clone())
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(
|
||||
@@ -6083,10 +6054,10 @@ fn deep_request_ring_tree_survives_full_backup_evict_and_load_back_rounds() {
|
||||
let result = tc.insert(&insert_params_swa(&key, &kv, 0, swa_evicted_seqlen));
|
||||
// Apply the emitted rebuilds the way the cache would.
|
||||
for action in &result.cache_actions {
|
||||
if let CacheAction::SwaRebuild {
|
||||
if let Some(SwaRebuild {
|
||||
node_id,
|
||||
source_value,
|
||||
} = action
|
||||
}) = action.as_any().downcast_ref::<SwaRebuild>()
|
||||
{
|
||||
tc.set_component_device_value(*node_id, SWA, source_value.copy())
|
||||
.expect("live test node");
|
||||
@@ -6198,18 +6169,16 @@ fn recovered_swa_span_evicts_before_the_window_leaf() {
|
||||
let prefix = child_of(&tc, root, &[1]);
|
||||
let leaf = child_of(&tc, prefix, &[9]);
|
||||
assert_eq!(tc.arena.node(prefix).key, vec![1, 2, 3, 4, 5, 6, 7, 8]);
|
||||
let [
|
||||
CacheAction::SwaRebuild {
|
||||
node_id,
|
||||
source_value,
|
||||
},
|
||||
] = result.cache_actions.as_slice()
|
||||
else {
|
||||
let [action] = result.cache_actions.as_slice() else {
|
||||
panic!(
|
||||
"expected one SwaRebuild action, got {:?}",
|
||||
action_kinds(&result.cache_actions)
|
||||
);
|
||||
};
|
||||
let SwaRebuild {
|
||||
node_id,
|
||||
source_value,
|
||||
} = expect_action::<SwaRebuild>(action.as_ref());
|
||||
assert_eq!(*node_id, tc.arena.node(leaf).id);
|
||||
tc.set_component_device_value(*node_id, SWA, source_value.copy())
|
||||
.expect("live test node");
|
||||
@@ -6225,19 +6194,18 @@ fn recovered_swa_span_evicts_before_the_window_leaf() {
|
||||
/* swa_evicted_seqlen = */ 0,
|
||||
));
|
||||
assert!(step.result.is_none());
|
||||
let [
|
||||
CacheAction::FreeDeviceKVFullOnly(old_full),
|
||||
CacheAction::SwaRebuild {
|
||||
node_id,
|
||||
source_value,
|
||||
},
|
||||
] = step.actions.as_slice()
|
||||
else {
|
||||
let [action0, action1] = step.actions.as_slice() else {
|
||||
panic!(
|
||||
"expected FreeDeviceKVFullOnly then SwaRebuild, got {:?}",
|
||||
action_kinds(&step.actions)
|
||||
);
|
||||
};
|
||||
let FreeDeviceKVFullOnly { indices: old_full } =
|
||||
expect_action::<FreeDeviceKVFullOnly>(action0.as_ref());
|
||||
let SwaRebuild {
|
||||
node_id,
|
||||
source_value,
|
||||
} = expect_action::<SwaRebuild>(action1.as_ref());
|
||||
assert!(old_full[0].equal(&Tensor::from_slice(&[
|
||||
100i64, 101, 102, 103, 104, 105, 106, 107
|
||||
])));
|
||||
@@ -6613,7 +6581,7 @@ fn write_back_reinsert_defers_swa_backup_while_write_through_publishes() {
|
||||
!result
|
||||
.cache_actions
|
||||
.iter()
|
||||
.any(|action| matches!(action, CacheAction::BackupKV(_)))
|
||||
.any(|action| action.as_any().is::<BackupKV>())
|
||||
);
|
||||
|
||||
tc.is_write_back = false;
|
||||
@@ -6621,9 +6589,11 @@ fn write_back_reinsert_defers_swa_backup_while_write_through_publishes() {
|
||||
let backups: Vec<_> = result
|
||||
.cache_actions
|
||||
.iter()
|
||||
.filter_map(|action| match action {
|
||||
CacheAction::BackupKV(backup) => Some(backup.node_ids.clone()),
|
||||
_ => None,
|
||||
.filter_map(|action| {
|
||||
action
|
||||
.as_any()
|
||||
.downcast_ref::<BackupKV>()
|
||||
.map(|backup| backup.node_ids.clone())
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(backups, vec![vec![leaf]]);
|
||||
|
||||
@@ -35,21 +35,14 @@ pub(crate) fn accumulate_step(
|
||||
}
|
||||
}
|
||||
|
||||
/// Short variant names for diagnosing an action sequence's shape.
|
||||
pub(crate) fn action_kinds(actions: &[CacheAction]) -> Vec<&'static str> {
|
||||
actions
|
||||
.iter()
|
||||
.map(|action| match action {
|
||||
CacheAction::FreeDeviceKV(_) => "FreeDeviceKV",
|
||||
CacheAction::FreeDeviceKVFullOnly(_) => "FreeDeviceKVFullOnly",
|
||||
CacheAction::BackupKV(_) => "BackupKV",
|
||||
CacheAction::UpdatePendingBackupOnNodeSplit { .. } => "UpdatePendingBackupOnNodeSplit",
|
||||
CacheAction::MambaEvictExcessPathStates { .. } => "MambaEvictExcessPathStates",
|
||||
CacheAction::FreeComponentDeviceSlot { .. } => "FreeComponentDeviceSlot",
|
||||
CacheAction::FreeComponentHostSlot { .. } => "FreeComponentHostSlot",
|
||||
CacheAction::RebuildFullToSwaMapping { .. } => "RebuildFullToSwaMapping",
|
||||
CacheAction::RecoverSwaWithLockedFull { .. } => "RecoverSwaWithLockedFull",
|
||||
CacheAction::SwaRebuild { .. } => "SwaRebuild",
|
||||
})
|
||||
.collect()
|
||||
/// Tags for diagnosing an action sequence's shape.
|
||||
pub(crate) fn action_kinds(actions: &[Box<dyn CacheAction>]) -> Vec<&'static str> {
|
||||
actions.iter().map(|action| action.tag()).collect()
|
||||
}
|
||||
|
||||
pub(crate) fn expect_action<T: std::any::Any>(action: &dyn CacheAction) -> &T {
|
||||
action
|
||||
.as_any()
|
||||
.downcast_ref::<T>()
|
||||
.unwrap_or_else(|| panic!("expected {}, got {:?}", std::any::type_name::<T>(), action))
|
||||
}
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
use crate::cache_action::{
|
||||
BackupKV, FreeDeviceKV, RecoverSwaWithLockedFull, UpdatePendingBackupOnNodeSplit,
|
||||
};
|
||||
use std::sync::Mutex;
|
||||
|
||||
use tch::Tensor;
|
||||
@@ -5,7 +8,7 @@ use tch::Tensor;
|
||||
use super::*;
|
||||
use crate::components::{ComponentSet, FULL, MAMBA, SWA};
|
||||
use crate::node::{NodeAccessError, ValueSlotIdx};
|
||||
use crate::test_utils::{accumulate_step, action_kinds, matched_device_indices};
|
||||
use crate::test_utils::{accumulate_step, action_kinds, expect_action, matched_device_indices};
|
||||
|
||||
fn core() -> UnifiedTreeCore<Vec<i64>> {
|
||||
UnifiedTreeCore::new(CacheInitParams::default(), vec![FULL])
|
||||
@@ -314,7 +317,7 @@ impl TreeComponent<Vec<i64>> for RecordingComponentForTest {
|
||||
_node_id: NodeIdx_,
|
||||
_device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
_host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
_cache_actions: &mut Vec<CacheAction>,
|
||||
_cache_actions: &mut CacheActions,
|
||||
_target: EvictLayer,
|
||||
) -> (usize, usize) {
|
||||
unimplemented!()
|
||||
@@ -330,7 +333,7 @@ impl TreeComponent<Vec<i64>> for RecordingComponentForTest {
|
||||
_tracker: &mut HashMap<ComponentType, usize>,
|
||||
_device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
_host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
_cache_actions: &mut Vec<CacheAction>,
|
||||
_cache_actions: &mut CacheActions,
|
||||
) -> Option<NodeIdx_> {
|
||||
unimplemented!()
|
||||
}
|
||||
@@ -366,7 +369,7 @@ impl TreeComponent<Vec<i64>> for RecordingComponentForTest {
|
||||
tracker: &mut HashMap<ComponentType, usize>,
|
||||
_device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
_host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
_cache_actions: &mut Vec<CacheAction>,
|
||||
_cache_actions: &mut CacheActions,
|
||||
) {
|
||||
let tracked = tracker[&SWA];
|
||||
self.host_eviction_calls
|
||||
@@ -383,7 +386,7 @@ impl TreeComponent<Vec<i64>> for RecordingComponentForTest {
|
||||
tracker: &mut HashMap<ComponentType, usize>,
|
||||
_device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
_host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
_cache_actions: &mut Vec<CacheAction>,
|
||||
_cache_actions: &mut CacheActions,
|
||||
) {
|
||||
let tracked = tracker[&SWA];
|
||||
self.host_eviction_calls
|
||||
@@ -440,7 +443,7 @@ impl TreeComponent<Vec<i64>> for CountingComponentForTest {
|
||||
_node_id: NodeIdx_,
|
||||
_device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
_host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
_cache_actions: &mut Vec<CacheAction>,
|
||||
_cache_actions: &mut CacheActions,
|
||||
_target: EvictLayer,
|
||||
) -> (usize, usize) {
|
||||
unimplemented!()
|
||||
@@ -456,7 +459,7 @@ impl TreeComponent<Vec<i64>> for CountingComponentForTest {
|
||||
_tracker: &mut HashMap<ComponentType, usize>,
|
||||
_device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
_host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
_cache_actions: &mut Vec<CacheAction>,
|
||||
_cache_actions: &mut CacheActions,
|
||||
) -> Option<NodeIdx_> {
|
||||
unimplemented!()
|
||||
}
|
||||
@@ -522,7 +525,7 @@ impl TreeComponent<Vec<i64>> for LowPriorityComponentForTest {
|
||||
_node_id: NodeIdx_,
|
||||
_device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
_host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
_cache_actions: &mut Vec<CacheAction>,
|
||||
_cache_actions: &mut CacheActions,
|
||||
_target: EvictLayer,
|
||||
) -> (usize, usize) {
|
||||
unimplemented!()
|
||||
@@ -538,7 +541,7 @@ impl TreeComponent<Vec<i64>> for LowPriorityComponentForTest {
|
||||
_tracker: &mut HashMap<ComponentType, usize>,
|
||||
_device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
_host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
_cache_actions: &mut Vec<CacheAction>,
|
||||
_cache_actions: &mut CacheActions,
|
||||
) -> Option<NodeIdx_> {
|
||||
unimplemented!()
|
||||
}
|
||||
@@ -599,7 +602,7 @@ impl TreeComponent<Vec<i64>> for SwaComponentForTest {
|
||||
_node_id: NodeIdx_,
|
||||
_device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
_host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
_cache_actions: &mut Vec<CacheAction>,
|
||||
_cache_actions: &mut CacheActions,
|
||||
_target: EvictLayer,
|
||||
) -> (usize, usize) {
|
||||
unimplemented!()
|
||||
@@ -615,7 +618,7 @@ impl TreeComponent<Vec<i64>> for SwaComponentForTest {
|
||||
_tracker: &mut HashMap<ComponentType, usize>,
|
||||
_device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
_host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
_cache_actions: &mut Vec<CacheAction>,
|
||||
_cache_actions: &mut CacheActions,
|
||||
) -> Option<NodeIdx_> {
|
||||
unimplemented!()
|
||||
}
|
||||
@@ -682,7 +685,7 @@ impl TreeComponent<Vec<i64>> for SwaEvictionComponentForTest {
|
||||
node_id: NodeIdx_,
|
||||
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
_cache_actions: &mut Vec<CacheAction>,
|
||||
_cache_actions: &mut CacheActions,
|
||||
target: EvictLayer,
|
||||
) -> (usize, usize) {
|
||||
self.evictions.lock().unwrap().push((node_id, target));
|
||||
@@ -730,7 +733,7 @@ impl TreeComponent<Vec<i64>> for SwaEvictionComponentForTest {
|
||||
_tracker: &mut HashMap<ComponentType, usize>,
|
||||
_device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
_host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
_cache_actions: &mut Vec<CacheAction>,
|
||||
_cache_actions: &mut CacheActions,
|
||||
) -> Option<NodeIdx_> {
|
||||
unimplemented!()
|
||||
}
|
||||
@@ -2444,12 +2447,13 @@ fn insert_full_overlap_frees_the_duplicates() {
|
||||
let result = tc.insert(&insert_params(&vec![1, 2, 3], &[20, 21, 22]));
|
||||
assert_eq!(result.prefix_len, 3);
|
||||
// Nothing was consumed by any component: the whole overlap is duplicate.
|
||||
let [CacheAction::FreeDeviceKV(freed)] = result.cache_actions.as_slice() else {
|
||||
let [action] = result.cache_actions.as_slice() else {
|
||||
panic!(
|
||||
"expected one FreeDeviceKV action, got {:?}",
|
||||
action_kinds(&result.cache_actions)
|
||||
);
|
||||
};
|
||||
let FreeDeviceKV { indices: freed } = expect_action::<FreeDeviceKV>(action.as_ref());
|
||||
assert_eq!(freed.len(), 1);
|
||||
assert!(freed[0].equal(&Tensor::from_slice(&[20i64, 21, 22])));
|
||||
}
|
||||
@@ -2463,12 +2467,13 @@ fn insert_prev_prefix_len_narrows_the_dup_window() {
|
||||
prev_prefix_len: 2,
|
||||
..insert_params(&vec![1, 2, 3], &[20, 21, 22])
|
||||
});
|
||||
let [CacheAction::FreeDeviceKV(freed)] = result.cache_actions.as_slice() else {
|
||||
let [action] = result.cache_actions.as_slice() else {
|
||||
panic!(
|
||||
"expected one FreeDeviceKV action, got {:?}",
|
||||
action_kinds(&result.cache_actions)
|
||||
);
|
||||
};
|
||||
let FreeDeviceKV { indices: freed } = expect_action::<FreeDeviceKV>(action.as_ref());
|
||||
assert!(freed[0].equal(&Tensor::from_slice(&[22i64])));
|
||||
}
|
||||
|
||||
@@ -2479,12 +2484,13 @@ fn insert_extends_an_existing_prefix() {
|
||||
let result = tc.insert(&insert_params(&vec![1, 2, 3, 4, 5], &[20, 21, 22, 13, 14]));
|
||||
assert_eq!(result.prefix_len, 3);
|
||||
// The existing node's overlap is duplicate; the new suffix is kept.
|
||||
let [CacheAction::FreeDeviceKV(freed)] = result.cache_actions.as_slice() else {
|
||||
let [action] = result.cache_actions.as_slice() else {
|
||||
panic!(
|
||||
"expected one FreeDeviceKV action, got {:?}",
|
||||
action_kinds(&result.cache_actions)
|
||||
);
|
||||
};
|
||||
let FreeDeviceKV { indices: freed } = expect_action::<FreeDeviceKV>(action.as_ref());
|
||||
assert_eq!(freed.len(), 1);
|
||||
assert!(freed[0].equal(&Tensor::from_slice(&[20i64, 21, 22])));
|
||||
let matched = tc.match_prefix(&match_params(&vec![1, 2, 3, 4, 5]));
|
||||
@@ -2506,12 +2512,13 @@ fn insert_prev_prefix_len_spans_a_multi_node_walk() {
|
||||
..insert_params(&vec![1, 2, 3, 4, 5], &[30, 31, 32, 33, 34])
|
||||
});
|
||||
assert_eq!(result.prefix_len, 5);
|
||||
let [CacheAction::FreeDeviceKV(freed)] = result.cache_actions.as_slice() else {
|
||||
let [action] = result.cache_actions.as_slice() else {
|
||||
panic!(
|
||||
"expected one FreeDeviceKV action, got {:?}",
|
||||
action_kinds(&result.cache_actions)
|
||||
);
|
||||
};
|
||||
let FreeDeviceKV { indices: freed } = expect_action::<FreeDeviceKV>(action.as_ref());
|
||||
assert_eq!(freed.len(), 1);
|
||||
assert!(freed[0].equal(&Tensor::from_slice(&[33i64, 34])));
|
||||
}
|
||||
@@ -2528,12 +2535,13 @@ fn insert_prev_prefix_len_narrows_mid_node_on_a_multi_node_walk() {
|
||||
..insert_params(&vec![1, 2, 3, 4, 5], &[30, 31, 32, 33, 34])
|
||||
});
|
||||
assert_eq!(result.prefix_len, 5);
|
||||
let [CacheAction::FreeDeviceKV(freed)] = result.cache_actions.as_slice() else {
|
||||
let [action] = result.cache_actions.as_slice() else {
|
||||
panic!(
|
||||
"expected one FreeDeviceKV action, got {:?}",
|
||||
action_kinds(&result.cache_actions)
|
||||
);
|
||||
};
|
||||
let FreeDeviceKV { indices: freed } = expect_action::<FreeDeviceKV>(action.as_ref());
|
||||
assert_eq!(freed.len(), 1);
|
||||
assert!(freed[0].equal(&Tensor::from_slice(&[34i64])));
|
||||
}
|
||||
@@ -2680,9 +2688,11 @@ fn insert_threshold_crossing_emits_the_backup_kv_action() {
|
||||
let backups: Vec<_> = result
|
||||
.cache_actions
|
||||
.iter()
|
||||
.filter_map(|action| match action {
|
||||
CacheAction::BackupKV(backup) => Some(backup.node_ids.clone()),
|
||||
_ => None,
|
||||
.filter_map(|action| {
|
||||
action
|
||||
.as_any()
|
||||
.downcast_ref::<BackupKV>()
|
||||
.map(|backup| backup.node_ids.clone())
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(backups, vec![vec![leaf]]);
|
||||
@@ -2701,7 +2711,7 @@ fn external_linker_hashes_new_nodes_and_triggers_offload_action() {
|
||||
let result = tc.insert(&insert_params(&vec![1, 2], &[10, 11]));
|
||||
let leaf = result.last_device_node_id.unwrap();
|
||||
assert!(result.cache_actions.iter().any(|action| {
|
||||
matches!(action, CacheAction::BackupKV(backup) if backup.node_ids == vec![leaf])
|
||||
matches!(action.as_any().downcast_ref::<BackupKV>(), Some(backup) if backup.node_ids == vec![leaf])
|
||||
}));
|
||||
|
||||
let transfers = tc
|
||||
@@ -3090,10 +3100,7 @@ fn write_back_defers_swa_window_backup_until_eviction() {
|
||||
let backups: Vec<&BackupKV> = result
|
||||
.cache_actions
|
||||
.iter()
|
||||
.filter_map(|action| match action {
|
||||
CacheAction::BackupKV(backup) => Some(backup),
|
||||
_ => None,
|
||||
})
|
||||
.filter_map(|action| action.as_any().downcast_ref::<BackupKV>())
|
||||
.collect();
|
||||
assert!(backups.is_empty());
|
||||
}
|
||||
@@ -3168,16 +3175,15 @@ fn split_updates_pending_backup_with_or_without_a_tree_ack() {
|
||||
let prefix_id = result.best_match_node_id;
|
||||
assert_ne!(prefix_id, tail_id);
|
||||
if pending || buffer_only {
|
||||
assert!(matches!(
|
||||
result.cache_actions.as_slice(),
|
||||
[CacheAction::UpdatePendingBackupOnNodeSplit {
|
||||
assert!(
|
||||
matches!(result.cache_actions.as_slice(), [action] if matches!(action.as_any().downcast_ref::<UpdatePendingBackupOnNodeSplit>(), Some(UpdatePendingBackupOnNodeSplit {
|
||||
ack_id,
|
||||
old_node_id,
|
||||
new_node_id,
|
||||
new_child_node_id,
|
||||
}] if *ack_id == pending.then_some(tail_id) && *old_node_id == tail_id
|
||||
&& *new_node_id == prefix_id && *new_child_node_id == tail_id
|
||||
));
|
||||
}) if *ack_id == pending.then_some(tail_id) && *old_node_id == tail_id
|
||||
&& *new_node_id == prefix_id && *new_child_node_id == tail_id))
|
||||
);
|
||||
} else {
|
||||
assert!(result.cache_actions.is_empty());
|
||||
}
|
||||
@@ -4063,16 +4069,21 @@ fn finish_write_through_after_a_split_publishes_both_fragments() {
|
||||
let new_node_id = result
|
||||
.cache_actions
|
||||
.iter()
|
||||
.find_map(|action| match action {
|
||||
CacheAction::UpdatePendingBackupOnNodeSplit {
|
||||
ack_id,
|
||||
new_node_id,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(*ack_id, Some(leaf));
|
||||
Some(*new_node_id)
|
||||
.find_map(|action| {
|
||||
match action
|
||||
.as_any()
|
||||
.downcast_ref::<UpdatePendingBackupOnNodeSplit>()
|
||||
{
|
||||
Some(UpdatePendingBackupOnNodeSplit {
|
||||
ack_id,
|
||||
new_node_id,
|
||||
..
|
||||
}) => {
|
||||
assert_eq!(*ack_id, Some(leaf));
|
||||
Some(*new_node_id)
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.expect("the split relocates the pending write-through");
|
||||
// Nothing reaches the host tier before the ack.
|
||||
@@ -4531,10 +4542,9 @@ fn insert_host_refill_preserves_split_actions_and_lengths() {
|
||||
assert_eq!(result.total_len, 2);
|
||||
assert!(result.inserted_host_node.is_some());
|
||||
assert!(!result.host_insert_dropped);
|
||||
assert!(matches!(
|
||||
result.cache_actions.as_slice(),
|
||||
[CacheAction::UpdatePendingBackupOnNodeSplit { ack_id, .. }] if *ack_id == Some(leaf)
|
||||
));
|
||||
assert!(
|
||||
matches!(result.cache_actions.as_slice(), [action] if matches!(action.as_any().downcast_ref::<UpdatePendingBackupOnNodeSplit>(), Some(UpdatePendingBackupOnNodeSplit { ack_id, .. }) if *ack_id == Some(leaf)))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -5614,12 +5624,13 @@ fn insert_page_size_two_splits_mid_page_divergence_at_the_page_boundary() {
|
||||
// The keys share 3 atoms but pages quantize the overlap down to 2.
|
||||
let result = tc.insert(&insert_params(&vec![1, 2, 3, 9], &[20, 21, 22, 29]));
|
||||
assert_eq!(result.prefix_len, 2);
|
||||
let [CacheAction::FreeDeviceKV(freed)] = result.cache_actions.as_slice() else {
|
||||
let [action] = result.cache_actions.as_slice() else {
|
||||
panic!(
|
||||
"expected one FreeDeviceKV action, got {:?}",
|
||||
action_kinds(&result.cache_actions)
|
||||
);
|
||||
};
|
||||
let FreeDeviceKV { indices: freed } = expect_action::<FreeDeviceKV>(action.as_ref());
|
||||
assert_eq!(freed.len(), 1);
|
||||
assert!(freed[0].equal(&Tensor::from_slice(&[20i64, 21])));
|
||||
let prefix = tc
|
||||
@@ -8641,12 +8652,13 @@ fn insert_unevicts_a_tombstoned_deep_node() {
|
||||
.device_value(tc.arena.resolve(leaf).expect("live test node"), FULL)
|
||||
.equal(&Tensor::from_slice(&[33i64, 34]))
|
||||
);
|
||||
let [CacheAction::FreeDeviceKV(freed)] = result.cache_actions.as_slice() else {
|
||||
let [action] = result.cache_actions.as_slice() else {
|
||||
panic!(
|
||||
"expected one FreeDeviceKV action, got {:?}",
|
||||
action_kinds(&result.cache_actions)
|
||||
);
|
||||
};
|
||||
let FreeDeviceKV { indices: freed } = expect_action::<FreeDeviceKV>(action.as_ref());
|
||||
assert!(freed[0].equal(&Tensor::from_slice(&[30i64, 31, 32])));
|
||||
}
|
||||
|
||||
@@ -8658,12 +8670,13 @@ fn insert_ragged_key_onto_an_existing_prefix_page_size_two() {
|
||||
assert_eq!(result.prefix_len, 2);
|
||||
// The ragged tail never enters: no new leaf, the aligned span is duplicate.
|
||||
assert_eq!(tc.arena.len(), 2);
|
||||
let [CacheAction::FreeDeviceKV(freed)] = result.cache_actions.as_slice() else {
|
||||
let [action] = result.cache_actions.as_slice() else {
|
||||
panic!(
|
||||
"expected one FreeDeviceKV action, got {:?}",
|
||||
action_kinds(&result.cache_actions)
|
||||
);
|
||||
};
|
||||
let FreeDeviceKV { indices: freed } = expect_action::<FreeDeviceKV>(action.as_ref());
|
||||
assert!(freed[0].equal(&Tensor::from_slice(&[20i64, 21])));
|
||||
}
|
||||
|
||||
@@ -8675,16 +8688,18 @@ fn insert_ragged_key_traverses_a_node_before_the_tail_page_size_two() {
|
||||
let result = tc.insert(&insert_params(&vec![1, 2, 3, 4, 5], &[30, 31, 32, 33, 34]));
|
||||
assert_eq!(result.prefix_len, 4);
|
||||
assert_eq!(tc.arena.len(), 3);
|
||||
let [
|
||||
CacheAction::FreeDeviceKV(freed_head),
|
||||
CacheAction::FreeDeviceKV(freed_tail),
|
||||
] = result.cache_actions.as_slice()
|
||||
else {
|
||||
let [action0, action1] = result.cache_actions.as_slice() else {
|
||||
panic!(
|
||||
"expected one FreeDeviceKV per walked node, got {:?}",
|
||||
action_kinds(&result.cache_actions)
|
||||
);
|
||||
};
|
||||
let FreeDeviceKV {
|
||||
indices: freed_head,
|
||||
} = expect_action::<FreeDeviceKV>(action0.as_ref());
|
||||
let FreeDeviceKV {
|
||||
indices: freed_tail,
|
||||
} = expect_action::<FreeDeviceKV>(action1.as_ref());
|
||||
assert!(freed_head[0].equal(&Tensor::from_slice(&[30i64, 31])));
|
||||
assert!(freed_tail[0].equal(&Tensor::from_slice(&[32i64, 33])));
|
||||
}
|
||||
@@ -8714,7 +8729,7 @@ fn deferrable_dup_frees_ride_the_final_step_without_suspension() {
|
||||
let mut tc = core();
|
||||
tc.insert(&insert_params(&vec![1, 2, 3], &[10, 11, 12]));
|
||||
let step = tc.begin_insert(&insert_params(&vec![1, 2, 3, 4], &[20, 21, 22, 13]));
|
||||
assert_eq!(action_kinds(&step.actions), vec!["FreeDeviceKV"]);
|
||||
assert_eq!(action_kinds(&step.actions), vec!["free_device_kv"]);
|
||||
let result = step
|
||||
.result
|
||||
.expect("a deferrable-only walk completes in one step");
|
||||
@@ -8749,11 +8764,9 @@ fn walk_backup_crossing_suspends_then_resume_completes() {
|
||||
assert!(tc.has_ongoing_insert());
|
||||
assert_eq!(
|
||||
action_kinds(&step.actions),
|
||||
vec!["FreeDeviceKV", "BackupKV"]
|
||||
vec!["free_device_kv", "backup_kv"]
|
||||
);
|
||||
let CacheAction::BackupKV(backup) = &step.actions[1] else {
|
||||
unreachable!();
|
||||
};
|
||||
let backup = expect_action::<BackupKV>(step.actions[1].as_ref());
|
||||
assert_eq!(backup.node_ids, vec![tc.arena.node(a).id]);
|
||||
|
||||
let done = tc.resume_insert();
|
||||
@@ -8841,7 +8854,7 @@ fn resume_insert_completes_after_an_on_path_host_leaf_is_evicted() {
|
||||
assert!(step.result.is_none());
|
||||
assert_eq!(
|
||||
action_kinds(&step.actions),
|
||||
vec!["FreeDeviceKV", "BackupKV"]
|
||||
vec!["free_device_kv", "backup_kv"]
|
||||
);
|
||||
// The barrier's backup host-evicts the on-path H-leaf before committing.
|
||||
let mut tracker = HashMap::from([(FULL, 0)]);
|
||||
@@ -8889,9 +8902,11 @@ fn aborted_barrier_crossing_refires_on_the_next_insert() {
|
||||
let backups: Vec<_> = step
|
||||
.actions
|
||||
.iter()
|
||||
.filter_map(|action| match action {
|
||||
CacheAction::BackupKV(backup) => Some(backup.node_ids.clone()),
|
||||
_ => None,
|
||||
.filter_map(|action| {
|
||||
action
|
||||
.as_any()
|
||||
.downcast_ref::<BackupKV>()
|
||||
.map(|backup| backup.node_ids.clone())
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(backups, vec![vec![tc.arena.node(a).id]]);
|
||||
@@ -8959,9 +8974,11 @@ fn one_insert_walk_fires_two_crossings_around_a_backuped_middle() {
|
||||
let backups: Vec<_> = result
|
||||
.cache_actions
|
||||
.iter()
|
||||
.filter_map(|action| match action {
|
||||
CacheAction::BackupKV(backup) => Some(backup.node_ids.clone()),
|
||||
_ => None,
|
||||
.filter_map(|action| {
|
||||
action
|
||||
.as_any()
|
||||
.downcast_ref::<BackupKV>()
|
||||
.map(|backup| backup.node_ids.clone())
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(backups, vec![vec![top], vec![deep]]);
|
||||
@@ -9518,17 +9535,17 @@ fn caller_owned_overlap_drains_split_and_backup_before_duplicate_frees() {
|
||||
);
|
||||
assert_eq!(
|
||||
action_kinds(&walk.pending_actions),
|
||||
vec!["UpdatePendingBackupOnNodeSplit"]
|
||||
vec!["update_pending_backup_on_node_split"]
|
||||
);
|
||||
assert!(!walk.has_barrier_actions_since(0));
|
||||
walk.pending_actions.push(CacheAction::BackupKV(BackupKV {
|
||||
walk.pending_actions.push(Box::new(BackupKV {
|
||||
node_ids: vec![node.id],
|
||||
}));
|
||||
assert!(walk.has_barrier_actions_since(0));
|
||||
let actions = std::mem::take(&mut walk.pending_actions);
|
||||
assert_eq!(
|
||||
action_kinds(&actions),
|
||||
vec!["UpdatePendingBackupOnNodeSplit", "BackupKV"]
|
||||
vec!["update_pending_backup_on_node_split", "backup_kv"]
|
||||
);
|
||||
assert_eq!(tc.arena.node(head).parent(), tc.arena.root());
|
||||
assert_eq!(
|
||||
@@ -9550,10 +9567,8 @@ fn caller_owned_overlap_drains_split_and_backup_before_duplicate_frees() {
|
||||
));
|
||||
let step = walk.finish();
|
||||
assert_eq!(step.result.unwrap().prefix_len, 2);
|
||||
assert_eq!(action_kinds(&step.actions), vec!["FreeDeviceKV"]);
|
||||
let CacheAction::FreeDeviceKV(frees) = &step.actions[0] else {
|
||||
unreachable!()
|
||||
};
|
||||
assert_eq!(action_kinds(&step.actions), vec!["free_device_kv"]);
|
||||
let FreeDeviceKV { indices: frees } = expect_action::<FreeDeviceKV>(step.actions[0].as_ref());
|
||||
assert!(frees[0].equal(&Tensor::from_slice(&[20i64, 21])));
|
||||
assert!(!tc.has_ongoing_insert());
|
||||
tc.sanity_check(&[], &[]);
|
||||
@@ -9577,7 +9592,7 @@ fn caller_owned_walk_can_abort_after_split_without_claiming_donor() {
|
||||
drop(walk);
|
||||
assert_eq!(
|
||||
action_kinds(&actions),
|
||||
vec!["UpdatePendingBackupOnNodeSplit"]
|
||||
vec!["update_pending_backup_on_node_split"]
|
||||
);
|
||||
assert!(!tc.has_ongoing_insert());
|
||||
assert!(tc.end_insert().is_empty());
|
||||
@@ -9590,7 +9605,7 @@ fn caller_owned_walk_can_abort_after_split_without_claiming_donor() {
|
||||
assert!(matched_device_indices(&tc, &matched).equal(&Tensor::from_slice(&[10i64, 11, 12, 13])));
|
||||
let result = tc.insert(¶ms);
|
||||
assert_eq!(result.prefix_len, 2);
|
||||
assert_eq!(action_kinds(&result.cache_actions), vec!["FreeDeviceKV"]);
|
||||
assert_eq!(action_kinds(&result.cache_actions), vec!["free_device_kv"]);
|
||||
tc.sanity_check(&[], &[]);
|
||||
}
|
||||
|
||||
@@ -9675,15 +9690,12 @@ fn overlap_context_reaches_swa_before_it_replaces_unlocked_full_ids() {
|
||||
);
|
||||
assert!(walk.has_barrier_actions_since(start));
|
||||
let actions = std::mem::take(&mut walk.pending_actions);
|
||||
assert_eq!(action_kinds(&actions), vec!["RecoverSwaWithLockedFull"]);
|
||||
let CacheAction::RecoverSwaWithLockedFull {
|
||||
assert_eq!(action_kinds(&actions), vec!["recover_swa_with_locked_full"]);
|
||||
let RecoverSwaWithLockedFull {
|
||||
node_id: actual_node_id,
|
||||
kept_full,
|
||||
incoming_full,
|
||||
} = &actions[0]
|
||||
else {
|
||||
unreachable!()
|
||||
};
|
||||
} = expect_action::<RecoverSwaWithLockedFull>(actions[0].as_ref());
|
||||
assert_eq!(*actual_node_id, node_id);
|
||||
assert!(kept_full.equal(&kept));
|
||||
assert!(incoming_full.equal(&Tensor::from_slice(&[20i64, 21])));
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
//! The radix prefix tree of cached KV.
|
||||
#![allow(unused_variables)]
|
||||
|
||||
pub use crate::cache_action::{BackupKV, CacheAction, CacheActions};
|
||||
use crate::cache_action::{FreeDeviceKV, FreeDeviceKVFullOnly, UpdatePendingBackupOnNodeSplit};
|
||||
use std::cmp::Reverse;
|
||||
use std::collections::{BinaryHeap, HashMap, HashSet};
|
||||
use std::sync::Arc;
|
||||
@@ -160,7 +162,7 @@ pub struct MatchResult {
|
||||
/// of other components.
|
||||
pub full_kv_hit_length: usize,
|
||||
/// Actions for the controller to apply.
|
||||
pub cache_actions: Vec<CacheAction>,
|
||||
pub cache_actions: CacheActions,
|
||||
}
|
||||
|
||||
/// Params for a prefix match; the key is borrowed from the caller.
|
||||
@@ -229,7 +231,7 @@ pub struct InsertResult {
|
||||
/// Incoming ranges retained by each component, in key-relative atom offsets.
|
||||
pub adopted_ranges: Option<HashMap<ComponentType, Vec<(usize, usize)>>>,
|
||||
/// Actions for the controller to apply.
|
||||
pub cache_actions: Vec<CacheAction>,
|
||||
pub cache_actions: CacheActions,
|
||||
}
|
||||
|
||||
impl InsertResult {
|
||||
@@ -260,7 +262,7 @@ impl InsertResult {
|
||||
/// One step of a resumable insert: the Controller executes `actions`, then
|
||||
/// resumes while `result` is None; `result` is set on the final step.
|
||||
pub struct InsertStepResult {
|
||||
pub actions: Vec<CacheAction>,
|
||||
pub actions: CacheActions,
|
||||
pub result: Option<InsertResult>,
|
||||
}
|
||||
|
||||
@@ -305,7 +307,7 @@ pub(crate) struct InsertWalkState<K: ChildKeyType> {
|
||||
target_node_id: Option<NodeIdx_>,
|
||||
result: Option<InsertResult>,
|
||||
/// Emitted actions awaiting the next barrier flush (or the final step).
|
||||
pub(crate) pending_actions: Vec<CacheAction>,
|
||||
pub(crate) pending_actions: CacheActions,
|
||||
}
|
||||
|
||||
/// A new walk or the result of an empty or declined insert.
|
||||
@@ -331,14 +333,9 @@ impl<K: ChildKeyType> InsertWalkState<K> {
|
||||
|
||||
/// Free and split bookkeeping can wait; other actions pause the walk.
|
||||
pub(crate) fn has_barrier_actions_since(&self, start: usize) -> bool {
|
||||
self.pending_actions[start..].iter().any(|action| {
|
||||
!matches!(
|
||||
action,
|
||||
CacheAction::FreeDeviceKV(_)
|
||||
| CacheAction::FreeDeviceKVFullOnly(_)
|
||||
| CacheAction::UpdatePendingBackupOnNodeSplit { .. }
|
||||
)
|
||||
})
|
||||
self.pending_actions[start..]
|
||||
.iter()
|
||||
.any(|action| action.requires_insert_barrier())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -352,61 +349,6 @@ pub struct KvCanaryWalkResult {
|
||||
pub prev_slot_indices: Vec<i64>,
|
||||
}
|
||||
|
||||
/// A queued cache IO action.
|
||||
#[derive(Debug)]
|
||||
pub enum CacheAction {
|
||||
/// Duplicate device KV slices the cache frees after the insert.
|
||||
FreeDeviceKV(Vec<Tensor>),
|
||||
/// Free the full side only, for a tombstoned node whose SWA peers are gone;
|
||||
/// FreeDeviceKV would release the SWA side twice.
|
||||
FreeDeviceKVFullOnly(Vec<Tensor>),
|
||||
/// A device->host backup work item (the write-through threshold fired).
|
||||
BackupKV(BackupKV),
|
||||
/// Update cache-side write bookkeeping after a node split:
|
||||
///
|
||||
/// parent -> node => parent -> new_node -> new_child
|
||||
///
|
||||
/// old_node_id (the pre-split node) is replaced by new_node_id + new_child_node_id.
|
||||
UpdatePendingBackupOnNodeSplit {
|
||||
/// None when buffer-only bookkeeping has no tree-tracked write.
|
||||
ack_id: Option<usize>,
|
||||
old_node_id: NodeId,
|
||||
new_node_id: NodeId,
|
||||
new_child_node_id: NodeId,
|
||||
},
|
||||
/// Per-path Mamba state-cap eviction from the tail's root path; applied at
|
||||
/// the insert's commit barrier, after the walk-time backups whose
|
||||
/// write-through locks shield the backed-up chain.
|
||||
MambaEvictExcessPathStates { tail_node_id: NodeId },
|
||||
/// Free only the given component's device KV slots.
|
||||
FreeComponentDeviceSlot {
|
||||
component_type: ComponentType,
|
||||
indices: Vec<Tensor>,
|
||||
},
|
||||
/// Free the given component's host KV pages.
|
||||
FreeComponentHostSlot {
|
||||
component_type: ComponentType,
|
||||
host_indices: Vec<Tensor>,
|
||||
},
|
||||
/// Rebuild the SWA allocator's full->swa index mapping for loaded chunks.
|
||||
RebuildFullToSwaMapping {
|
||||
full_indices: Vec<Tensor>,
|
||||
swa_indices: Vec<Tensor>,
|
||||
},
|
||||
/// Recover an SWA tombstone whose full is locked: keep the locked full, remap
|
||||
/// it onto the incoming full's SWA translation, and free only the incoming full.
|
||||
RecoverSwaWithLockedFull {
|
||||
node_id: NodeId,
|
||||
kept_full: Tensor,
|
||||
incoming_full: Tensor,
|
||||
},
|
||||
/// Rebuild a node's SWA value by translating its source full value, then store it.
|
||||
SwaRebuild {
|
||||
node_id: NodeId,
|
||||
source_value: Tensor,
|
||||
},
|
||||
}
|
||||
|
||||
/// A HiCache pool transfer descriptor.
|
||||
#[derive(Default)]
|
||||
pub struct PoolTransfer {
|
||||
@@ -482,15 +424,6 @@ pub struct PoolTransferResult {
|
||||
pub extra_pool_hit_pages: HashMap<PoolName, usize>,
|
||||
}
|
||||
|
||||
/// A device->host backup work item for the cache to execute.
|
||||
#[derive(Default, Debug)]
|
||||
pub struct BackupKV {
|
||||
/// Backup these nodes device->host in order, stopping at the first failure; the
|
||||
/// caller orders them parent-before-child for write-through and child-first for
|
||||
/// write-back. External handles: the list crosses to the orchestrator.
|
||||
pub node_ids: Vec<NodeId>,
|
||||
}
|
||||
|
||||
/// A device->storage backup spec.
|
||||
#[derive(Default)]
|
||||
pub struct StorageBackupSpec {
|
||||
@@ -654,7 +587,7 @@ pub struct EvictionStepResult {
|
||||
pub tracker: HashMap<ComponentType, usize>,
|
||||
pub device_frees: HashMap<ComponentType, Vec<Tensor>>,
|
||||
pub host_frees: HashMap<ComponentType, Vec<Tensor>>,
|
||||
pub cache_actions: Vec<CacheAction>,
|
||||
pub cache_actions: CacheActions,
|
||||
pub backup_kv: Option<BackupKV>,
|
||||
/// Full device tokens freed without a host copy during this device step.
|
||||
pub unbacked_tokens: usize,
|
||||
@@ -1258,7 +1191,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
params: &DecLockRefParams,
|
||||
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
) -> Result<(), NodeAccessError> {
|
||||
let node_idx = self.arena.resolve(node_id)?;
|
||||
self.assert_receipt_anchor_(node_idx, params);
|
||||
@@ -1286,7 +1219,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
params: &DecLockRefParams,
|
||||
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
) -> Result<(), NodeAccessError> {
|
||||
let node_idx = self.arena.resolve(node_id)?;
|
||||
self.assert_receipt_anchor_(node_idx, params);
|
||||
@@ -1510,7 +1443,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
usize,
|
||||
usize,
|
||||
Option<NodeId>,
|
||||
Option<CacheAction>,
|
||||
Option<UpdatePendingBackupOnNodeSplit>,
|
||||
) {
|
||||
// Non-HiCache mode has only device-resident matches, so the scheduler
|
||||
// device anchor follows the best match. In HiCache mode, host-backed
|
||||
@@ -1524,7 +1457,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
let mut best_match_device_node_id = node_id;
|
||||
let mut best_match_device_value_len = 0;
|
||||
let mut full_kv_hit_length = 0;
|
||||
let mut action: Option<CacheAction> = None;
|
||||
let mut action: Option<UpdatePendingBackupOnNodeSplit> = None;
|
||||
let separate_device_match = self.enable_hicache;
|
||||
|
||||
fn update_best_if_valid<K: ChildKeyType>(
|
||||
@@ -1639,7 +1572,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
best_match_device_value_len: usize,
|
||||
full_kv_hit_length: usize,
|
||||
last_visited_node_id: Option<NodeId>,
|
||||
action: Option<CacheAction>,
|
||||
action: Option<UpdatePendingBackupOnNodeSplit>,
|
||||
) -> MatchResult {
|
||||
for i in 0..self.components.len() {
|
||||
// Full uses last_access ticks, not LRU.
|
||||
@@ -1705,7 +1638,10 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
best_match_device_value_len,
|
||||
);
|
||||
}
|
||||
result.cache_actions = action.into_iter().collect();
|
||||
result.cache_actions = action
|
||||
.into_iter()
|
||||
.map(|action| Box::new(action) as Box<dyn CacheAction>)
|
||||
.collect();
|
||||
result
|
||||
}
|
||||
|
||||
@@ -1808,7 +1744,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
) -> Result<InsertResult, TreeCoreRuntimeError> {
|
||||
// Single-shot pump over the resumable walk: run every step inline and
|
||||
// fold the step actions into the result for the caller to apply.
|
||||
let mut actions = Vec::new();
|
||||
let mut actions: CacheActions = Vec::new();
|
||||
let mut step = self.try_begin_insert(params)?;
|
||||
loop {
|
||||
actions.append(&mut step.actions);
|
||||
@@ -1988,7 +1924,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
}
|
||||
|
||||
/// Finish the insert (idempotent); returns still-pending actions to drain.
|
||||
pub fn end_insert(&mut self) -> Vec<CacheAction> {
|
||||
pub fn end_insert(&mut self) -> CacheActions {
|
||||
self.ongoing_insert_walk_state
|
||||
.take()
|
||||
.map(|state| state.pending_actions)
|
||||
@@ -2050,7 +1986,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
let (split_node_id, action) = self.split_node_(node_id, prefix_len);
|
||||
node_id = split_node_id;
|
||||
if let Some(action) = action {
|
||||
state.pending_actions.push(action);
|
||||
state.pending_actions.push(Box::new(action));
|
||||
}
|
||||
}
|
||||
{
|
||||
@@ -2166,22 +2102,22 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
.saturating_sub(cursor + dup_start)
|
||||
.min(dup_len);
|
||||
if swa_already_freed > 0 {
|
||||
state
|
||||
.pending_actions
|
||||
.push(CacheAction::FreeDeviceKVFullOnly(vec![value_slice.narrow(
|
||||
state.pending_actions.push(Box::new(FreeDeviceKVFullOnly {
|
||||
indices: vec![value_slice.narrow(
|
||||
0,
|
||||
dup_start as i64,
|
||||
swa_already_freed as i64,
|
||||
)]));
|
||||
)],
|
||||
}));
|
||||
}
|
||||
if swa_already_freed < dup_len {
|
||||
state
|
||||
.pending_actions
|
||||
.push(CacheAction::FreeDeviceKV(vec![value_slice.narrow(
|
||||
state.pending_actions.push(Box::new(FreeDeviceKV {
|
||||
indices: vec![value_slice.narrow(
|
||||
0,
|
||||
(dup_start + swa_already_freed) as i64,
|
||||
(dup_len - swa_already_freed) as i64,
|
||||
)]));
|
||||
)],
|
||||
}));
|
||||
}
|
||||
}
|
||||
state.overlap_node_id = None;
|
||||
@@ -2200,7 +2136,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
if node_end > state.inserted_len && self.inc_hit_count_and_check_(node_id) {
|
||||
let backup = self
|
||||
.build_backup_kv_action_(self.arena.node(node_id), /* write_back = */ false);
|
||||
state.pending_actions.push(CacheAction::BackupKV(backup));
|
||||
state.pending_actions.push(Box::new(backup));
|
||||
}
|
||||
state.node_id = node_id;
|
||||
state.total_prefix_length += prefix_len;
|
||||
@@ -2335,7 +2271,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
if self.should_backup_after_insert_(state, target_node_id) {
|
||||
let backup =
|
||||
self.build_backup_kv_action_(self.arena.node(target_node_id), self.is_write_back);
|
||||
state.pending_actions.push(CacheAction::BackupKV(backup));
|
||||
state.pending_actions.push(Box::new(backup));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2344,7 +2280,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
&mut self,
|
||||
child_id: NodeIdx_,
|
||||
split_len: usize,
|
||||
) -> (NodeIdx_, Option<CacheAction>) {
|
||||
) -> (NodeIdx_, Option<UpdatePendingBackupOnNodeSplit>) {
|
||||
assert!(
|
||||
split_len > 0 && split_len.is_multiple_of(self.page_size),
|
||||
"split_node_: split_len {split_len} must be a nonzero page multiple"
|
||||
@@ -2429,7 +2365,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
}
|
||||
// Buffer-only writes are tracked by the cache rather than a node ack.
|
||||
let action = if ack_id.is_some() || self.is_host_memory_buffer_only {
|
||||
Some(CacheAction::UpdatePendingBackupOnNodeSplit {
|
||||
Some(UpdatePendingBackupOnNodeSplit {
|
||||
ack_id,
|
||||
old_node_id: self.arena.node(child_id).id,
|
||||
new_node_id: self.arena.node(new_node_id).id,
|
||||
@@ -2951,7 +2887,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
tracker: &mut HashMap<ComponentType, usize>,
|
||||
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
) {
|
||||
self.record_remove_event_(node_id, medium);
|
||||
for i in 0..self.components.len() {
|
||||
@@ -2977,7 +2913,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
tracker: &mut HashMap<ComponentType, usize>,
|
||||
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
) {
|
||||
self.release_all_component_layers_(
|
||||
node_id,
|
||||
@@ -3066,7 +3002,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
tracker: &mut HashMap<ComponentType, usize>,
|
||||
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
) {
|
||||
assert!(
|
||||
self.can_reclaim_coexisting_host_value_(node_id, component_type),
|
||||
@@ -3102,7 +3038,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
tracker: &mut HashMap<ComponentType, usize>,
|
||||
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
) {
|
||||
assert!(
|
||||
self.is_evictable_host_leaf_(self.arena.node(node_id)),
|
||||
@@ -3170,7 +3106,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
tracker: &mut HashMap<ComponentType, usize>,
|
||||
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
) {
|
||||
{
|
||||
let node = self.arena.node(node_id);
|
||||
@@ -3215,7 +3151,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
tracker: &mut HashMap<ComponentType, usize>,
|
||||
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
target: EvictLayer,
|
||||
) {
|
||||
let is_leaf = match target {
|
||||
@@ -3347,7 +3283,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
component_type: ComponentType,
|
||||
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
target: EvictLayer,
|
||||
tracker: Option<&mut HashMap<ComponentType, usize>>,
|
||||
) -> (usize, usize) {
|
||||
@@ -3404,7 +3340,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
tracker: &mut HashMap<ComponentType, usize>,
|
||||
device_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
host_frees: &mut HashMap<ComponentType, Vec<Tensor>>,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
) {
|
||||
let ct = BASE_COMPONENT_TYPE;
|
||||
let mut cur = deleted_node_parent_id;
|
||||
@@ -3895,7 +3831,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
let mut host_prefix_len = 0;
|
||||
let mut refilled_unbacked_node = false;
|
||||
let mut inserted_host_node = None;
|
||||
let mut cache_actions: Vec<CacheAction> = Vec::new();
|
||||
let mut cache_actions: CacheActions = Vec::new();
|
||||
while matched_length < total_len {
|
||||
let Some(child_id) = self.arena.child_on_page_in_namespace(
|
||||
node_id,
|
||||
@@ -3916,7 +3852,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
let (split_node_id, action) = self.split_node_(node_id, prefix_len);
|
||||
node_id = split_node_id;
|
||||
if let Some(action) = action {
|
||||
cache_actions.push(action);
|
||||
cache_actions.push(Box::new(action));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4472,7 +4408,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
node_id: NodeId,
|
||||
phase: CacheTransferPhase,
|
||||
comp_xfers: HashMap<ComponentType, Vec<PoolTransfer>>,
|
||||
cache_actions: &mut Vec<CacheAction>,
|
||||
cache_actions: &mut CacheActions,
|
||||
mut insert_result: Option<&mut InsertResult>,
|
||||
pool_storage_result: Option<&PoolTransferResult>,
|
||||
) -> Result<(), NodeAccessError> {
|
||||
@@ -4510,7 +4446,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
) -> Result<(), NodeAccessError> {
|
||||
let node_id = self.arena.resolve(node_id)?;
|
||||
self.validate_pool_transfer_handles(comp_xfers.values().flatten())?;
|
||||
let mut cache_actions: Vec<CacheAction> = Vec::new();
|
||||
let mut cache_actions: CacheActions = Vec::new();
|
||||
if host_indices.numel() > 0 {
|
||||
let kv_xfer = PoolTransfer {
|
||||
name: PoolName::Kv,
|
||||
@@ -4555,7 +4491,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
device_indices: Tensor,
|
||||
mut kv_xfer: PoolTransfer,
|
||||
comp_xfers: HashMap<ComponentType, Vec<PoolTransfer>>,
|
||||
) -> Result<Vec<CacheAction>, NodeAccessError> {
|
||||
) -> Result<CacheActions, NodeAccessError> {
|
||||
let anchor_id = node_id;
|
||||
let node_id = self.arena.resolve(node_id)?;
|
||||
let loaded_node_indices = kv_xfer
|
||||
@@ -4565,7 +4501,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
.map(|&loaded_id| self.arena.resolve(loaded_id))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
self.validate_pool_transfer_handles(comp_xfers.values().flatten())?;
|
||||
let mut cache_actions: Vec<CacheAction> = Vec::new();
|
||||
let mut cache_actions: CacheActions = Vec::new();
|
||||
kv_xfer.device_indices = Some(device_indices);
|
||||
let nodes_to_load = kv_xfer.nodes_to_load.clone();
|
||||
if self.is_write_back {
|
||||
@@ -4894,7 +4830,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
window_start: usize,
|
||||
window_end: usize,
|
||||
swa_values: &Tensor,
|
||||
) -> Result<Vec<CacheAction>, String> {
|
||||
) -> Result<CacheActions, String> {
|
||||
self.validate_swa_span_(key, window_start, window_end)?;
|
||||
if swa_values.kind() != Kind::Int64 || swa_values.device() != self.device {
|
||||
return Err(format!(
|
||||
@@ -4940,7 +4876,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
));
|
||||
}
|
||||
|
||||
let mut actions = Vec::new();
|
||||
let mut actions: CacheActions = Vec::new();
|
||||
for (child_id, pos, prefix_len) in spans {
|
||||
let seg_end = window_end.min(pos + prefix_len);
|
||||
if seg_end <= window_start {
|
||||
@@ -4951,11 +4887,15 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
if seg_start > pos {
|
||||
// The original handle remains on the suffix, our target.
|
||||
let (_, action) = self.split_node_(target, seg_start - pos);
|
||||
actions.extend(action);
|
||||
if let Some(action) = action {
|
||||
actions.push(Box::new(action));
|
||||
}
|
||||
}
|
||||
if seg_start + self.arena.node(target).key.atom_len() > seg_end {
|
||||
let (fragment, action) = self.split_node_(target, seg_end - seg_start);
|
||||
actions.extend(action);
|
||||
if let Some(action) = action {
|
||||
actions.push(Box::new(action));
|
||||
}
|
||||
target = fragment;
|
||||
}
|
||||
let values = swa_values
|
||||
|
||||
@@ -5611,3 +5611,72 @@ def test_finished_request_prompt_priority_saturates_at_int64_max(priority):
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__]))
|
||||
|
||||
|
||||
def test_load_back_action_tuple_preserves_chunk_order_and_tensor_ownership():
|
||||
binding = mem_cache.RustUnifiedTreeCoreBinding(
|
||||
mem_cache.TreeCoreInitParamsBinding(
|
||||
page_size=1, swa_sliding_window_size=8, enable_hicache=True
|
||||
),
|
||||
[0, 1],
|
||||
)
|
||||
binding.set_has_swa_host_pool()
|
||||
key = array("q", range(5))
|
||||
prefix = binding.insert(
|
||||
mem_cache.InsertParamsBinding(key=key[:2], value=torch.tensor([10, 11]))
|
||||
).last_device_node
|
||||
binding.set_component_device_value(prefix, 1, torch.tensor([110, 111]))
|
||||
tail = binding.insert(
|
||||
mem_cache.InsertParamsBinding(
|
||||
key=key, value=torch.arange(10, 15), prev_prefix_len=2
|
||||
)
|
||||
).last_device_node
|
||||
binding.set_component_device_value(tail, 1, torch.tensor([112, 113, 114]))
|
||||
for node, host in (
|
||||
(prefix, torch.arange(310, 312)),
|
||||
(tail, torch.arange(312, 315)),
|
||||
):
|
||||
_, transfers = binding.build_backup_spec(node)
|
||||
transfer = transfers[1][0]
|
||||
transfers[1][0] = (transfer[0], transfer[2] + 100, *transfer[2:])
|
||||
binding.mark_write_through_pending([node], node)
|
||||
binding.commit_backup(node, host, transfers)
|
||||
binding.finish_write_through([node], node)
|
||||
binding.demote(tail)
|
||||
binding.demote(prefix)
|
||||
|
||||
kv_transfer, transfers = binding.build_load_back_spec(tail)
|
||||
transfer = transfers[1][0]
|
||||
transfers[1][0] = (*transfer[:2], torch.arange(50, 55), *transfer[3:])
|
||||
actions = binding.commit_load_back(
|
||||
tail, torch.arange(40, 45), kv_transfer, transfers
|
||||
)
|
||||
receipt = binding.inc_lock_ref(tail)
|
||||
assert isinstance(actions, list) and len(actions) == 1
|
||||
action = actions[0]
|
||||
assert isinstance(action, tuple) and len(action) == 3
|
||||
assert action[0] == "rebuild_full_to_swa_mapping"
|
||||
assert isinstance(action[1], list) and isinstance(action[2], list)
|
||||
assert [value.tolist() for value in action[1]] == [[40, 41], [42, 43, 44]]
|
||||
assert [value.tolist() for value in action[2]] == [[50, 51], [52, 53, 54]]
|
||||
binding.dec_lock_ref(
|
||||
tail,
|
||||
mem_cache.DecLockRefParamsBinding(
|
||||
node_id=receipt.node_id,
|
||||
skipped_lock_components=receipt.skipped_lock_components,
|
||||
component_lock_uuids=receipt.component_lock_uuids,
|
||||
component_host_lock_uuids=receipt.component_host_lock_uuids,
|
||||
),
|
||||
)
|
||||
binding.finish_load_back(tail)
|
||||
binding.sanity_check([], [])
|
||||
|
||||
chunk = action[2][1]
|
||||
storage_ref = chunk.untyped_storage()._weak_ref()
|
||||
try:
|
||||
binding.reset()
|
||||
assert chunk.tolist() == [52, 53, 54]
|
||||
del chunk, action, actions
|
||||
assert torch.UntypedStorage._expired(storage_ref)
|
||||
finally:
|
||||
torch.UntypedStorage._free_weak_ref(storage_ref)
|
||||
|
||||
Reference in New Issue
Block a user