diff --git a/rust/sglang-radix-tree/src/cache_action.rs b/rust/sglang-radix-tree/src/cache_action.rs new file mode 100644 index 0000000000..cb94ab5f3e --- /dev/null +++ b/rust/sglang-radix-tree/src/cache_action.rs @@ -0,0 +1,157 @@ +//! Cache actions returned to the cache for execution. + +use std::fmt::Debug; + +use tch::Tensor; + +use crate::components::ComponentType; +use crate::node::NodeId; + +pub trait CacheAction: Debug + Send { + fn tag(&self) -> &'static str; + + /// Whether insertion must pause for the cache to apply this action. + fn requires_insert_barrier(&self) -> bool { + true + } + + #[cfg(feature = "python-extension")] + fn into_py_tagged( + self: Box, + py: pyo3::Python<'_>, + ) -> pyo3::PyResult>; + + /// Rust tests inspect emitted payloads without a production downcast API. + #[cfg(test)] + fn as_any(&self) -> &dyn std::any::Any; +} + +pub type CacheActions = Vec>; + +macro_rules! impl_cache_action { + ($action:ty, $tag:literal $(, insert_barrier = $barrier:expr)? $(,)?) => { + impl $crate::cache_action::CacheAction for $action { + fn tag(&self) -> &'static str { + $tag + } + + $(fn requires_insert_barrier(&self) -> bool { + $barrier + })? + + #[cfg(feature = "python-extension")] + fn into_py_tagged( + self: Box, + py: pyo3::Python<'_>, + ) -> pyo3::PyResult> { + <$action>::into_py_tagged(*self, py) + } + + #[cfg(test)] + fn as_any(&self) -> &dyn std::any::Any { + self + } + } + }; +} +pub(crate) use impl_cache_action; + +/// Duplicate device KV slices the cache frees after the insert. +#[derive(Debug)] +pub struct FreeDeviceKV { + pub indices: Vec, +} + +/// Free the full side only, for a tombstoned node whose SWA peers are gone; +/// FreeDeviceKV would release the SWA side twice. +#[derive(Debug)] +pub struct FreeDeviceKVFullOnly { + pub indices: Vec, +} + +/// Update cache-side write bookkeeping after a node split: +/// +/// parent -> node => parent -> new_node -> new_child +/// +/// old_node_id (the pre-split node) is replaced by new_node_id + new_child_node_id. +#[derive(Debug)] +pub struct UpdatePendingBackupOnNodeSplit { + /// None when buffer-only bookkeeping has no tree-tracked write. + pub ack_id: Option, + pub old_node_id: NodeId, + pub new_node_id: NodeId, + pub new_child_node_id: NodeId, +} + +/// Per-path Mamba state-cap eviction from the tail's root path; applied at +/// the insert's commit barrier, after the walk-time backups whose +/// write-through locks shield the backed-up chain. +#[derive(Debug)] +pub struct MambaEvictExcessPathStates { + pub tail_node_id: NodeId, +} + +/// Free only the given component's device KV slots. +#[derive(Debug)] +pub struct FreeComponentDeviceSlot { + pub component_type: ComponentType, + pub indices: Vec, +} + +/// Free the given component's host KV pages. +#[derive(Debug)] +pub struct FreeComponentHostSlot { + pub component_type: ComponentType, + pub host_indices: Vec, +} + +/// Rebuild the SWA allocator's full->swa index mapping for loaded chunks. +#[derive(Debug)] +pub struct RebuildFullToSwaMapping { + pub full_indices: Vec, + pub swa_indices: Vec, +} + +/// Recover an SWA tombstone whose full is locked: keep the locked full, remap +/// it onto the incoming full's SWA translation, and free only the incoming full. +#[derive(Debug)] +pub struct RecoverSwaWithLockedFull { + pub node_id: NodeId, + pub kept_full: Tensor, + pub incoming_full: Tensor, +} + +/// Rebuild a node's SWA value by translating its source full value, then store it. +#[derive(Debug)] +pub struct SwaRebuild { + pub node_id: NodeId, + pub source_value: Tensor, +} + +/// A device->host backup work item for the cache to execute. +#[derive(Default, Debug)] +pub struct BackupKV { + /// Backup these nodes device->host in order, stopping at the first failure; the + /// caller orders them parent-before-child for write-through and child-first for + /// write-back. External handles: the list crosses to the orchestrator. + pub node_ids: Vec, +} + +self::impl_cache_action!(FreeDeviceKV, "free_device_kv", insert_barrier = false); +self::impl_cache_action!( + FreeDeviceKVFullOnly, + "free_device_kv_full_only", + insert_barrier = false +); +self::impl_cache_action!(BackupKV, "backup_kv"); +self::impl_cache_action!( + UpdatePendingBackupOnNodeSplit, + "update_pending_backup_on_node_split", + insert_barrier = false +); +self::impl_cache_action!(MambaEvictExcessPathStates, "mamba_evict_excess_path_states"); +self::impl_cache_action!(FreeComponentDeviceSlot, "free_component_device_slot"); +self::impl_cache_action!(FreeComponentHostSlot, "free_component_host_slot"); +self::impl_cache_action!(RebuildFullToSwaMapping, "rebuild_full_to_swa_mapping"); +self::impl_cache_action!(RecoverSwaWithLockedFull, "recover_swa_with_locked_full"); +self::impl_cache_action!(SwaRebuild, "swa_rebuild"); diff --git a/rust/sglang-radix-tree/src/components/full.rs b/rust/sglang-radix-tree/src/components/full.rs index f270076776..33b4aba039 100644 --- a/rust/sglang-radix-tree/src/components/full.rs +++ b/rust/sglang-radix-tree/src/components/full.rs @@ -13,7 +13,7 @@ use crate::node::Node; use crate::node::{NodeId, NodeIdx_, TreeCoreRuntimeError, ValueSlotIdx}; use crate::unified_lru_list::PriorityKey; use crate::unified_tree_core::{ - CacheAction, CacheTransferPhase, DecLockRefParams, EvictLayer, IncLockRefResult, InsertResult, + CacheActions, CacheTransferPhase, DecLockRefParams, EvictLayer, IncLockRefResult, InsertResult, MatchPrefixParams, MatchResult, PoolName, PoolTransfer, PoolTransferResult, UnifiedTreeCore, }; @@ -104,7 +104,7 @@ impl TreeComponent for FullComponent { node_id: NodeIdx_, device_frees: &mut HashMap>, host_frees: &mut HashMap>, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, target: EvictLayer, ) -> (usize, usize) { let node = tree_core.arena.node(node_id); @@ -156,7 +156,7 @@ impl TreeComponent for FullComponent { tracker: &mut HashMap, _device_frees: &mut HashMap>, _host_frees: &mut HashMap>, - _cache_actions: &mut Vec, + _cache_actions: &mut CacheActions, ) -> Option { let ct = FULL; assert!( @@ -205,7 +205,7 @@ impl TreeComponent for FullComponent { tracker: &mut HashMap, device_frees: &mut HashMap>, host_frees: &mut HashMap>, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, ) { for spare_imminent_demotes in [true, false] { if tracker[&FULL] >= num_tokens { @@ -248,7 +248,7 @@ impl TreeComponent for FullComponent { tracker: &mut HashMap, device_frees: &mut HashMap>, host_frees: &mut HashMap>, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, ) { let ct = FULL; let arena = &tree_core.arena; @@ -491,7 +491,7 @@ impl TreeComponent for FullComponent { node_id: NodeIdx_, phase: CacheTransferPhase, transfers: Vec, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, insert_result: Option<&mut InsertResult>, pool_storage_result: Option<&PoolTransferResult>, ) { diff --git a/rust/sglang-radix-tree/src/components/mamba.rs b/rust/sglang-radix-tree/src/components/mamba.rs index 1bd53935ac..a6f5e33be4 100644 --- a/rust/sglang-radix-tree/src/components/mamba.rs +++ b/rust/sglang-radix-tree/src/components/mamba.rs @@ -6,13 +6,14 @@ use std::collections::HashMap; use tch::Tensor; +use crate::cache_action::{FreeComponentHostSlot, MambaEvictExcessPathStates}; use crate::components::TreeComponent; use crate::components::{ComponentType, FULL, MAMBA}; use crate::node::ChildKeyType; use crate::node::Node; use crate::node::{NodeId, NodeIdx_, TreeCoreRuntimeError, ValueSlotIdx}; use crate::unified_tree_core::{ - CacheAction, CacheInitParams, CacheTransferPhase, DecLockRefParams, EvictLayer, + CacheActions, CacheInitParams, CacheTransferPhase, DecLockRefParams, EvictLayer, IncLockRefResult, InsertParams, InsertResult, LRURefreshPhase, MatchPrefixParams, MatchResult, PoolHitPolicy, PoolName, PoolTransfer, PoolTransferResult, UnifiedTreeCore, }; @@ -70,12 +71,12 @@ impl MambaComponent { fn emit_excess_path_states_eviction_( &self, tail_node_id: NodeId, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, ) { if self.mamba_max_states_per_path.is_none() { return; } - cache_actions.push(CacheAction::MambaEvictExcessPathStates { tail_node_id }); + cache_actions.push(Box::new(MambaEvictExcessPathStates { tail_node_id })); } } @@ -160,7 +161,7 @@ impl TreeComponent for MambaComponent { is_new_leaf: bool, params: &InsertParams<'_, K>, result: &mut InsertResult, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, ) { let mamba_value = params .mamba_value @@ -200,7 +201,7 @@ impl TreeComponent for MambaComponent { tail_node_id: NodeIdx_, device_frees: &mut HashMap>, host_frees: &mut HashMap>, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, ) { let Some(cap) = self.mamba_max_states_per_path else { return; @@ -282,7 +283,7 @@ impl TreeComponent for MambaComponent { node_id: NodeIdx_, device_frees: &mut HashMap>, host_frees: &mut HashMap>, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, target: EvictLayer, ) -> (usize, usize) { let ct = MAMBA; @@ -348,7 +349,7 @@ impl TreeComponent for MambaComponent { tracker: &mut HashMap, device_frees: &mut HashMap>, host_frees: &mut HashMap>, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, ) -> Option { let ct = MAMBA; assert!( @@ -636,7 +637,7 @@ impl TreeComponent for MambaComponent { node_id: NodeIdx_, phase: CacheTransferPhase, transfers: Vec, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, insert_result: Option<&mut InsertResult>, pool_storage_result: Option<&PoolTransferResult>, ) { @@ -696,12 +697,12 @@ impl TreeComponent for MambaComponent { let Some(target) = attach_target else { // The buffer cannot attach: free it and let the caller keep // its own donated slot bookkeeping. - cache_actions.push(CacheAction::FreeComponentHostSlot { + cache_actions.push(Box::new(FreeComponentHostSlot { component_type: MAMBA, host_indices: host_indices .map(|host| vec![host.shallow_clone()]) .unwrap_or_default(), - }); + })); if let Some(insert_result) = insert_result { insert_result.mamba_exist = true; } @@ -733,7 +734,7 @@ impl TreeComponent for MambaComponent { tracker: &mut HashMap, device_frees: &mut HashMap>, host_frees: &mut HashMap>, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, ) { let ct = MAMBA; let mut x = tree_core diff --git a/rust/sglang-radix-tree/src/components/mod.rs b/rust/sglang-radix-tree/src/components/mod.rs index 3ee0b76194..92da3c6444 100644 --- a/rust/sglang-radix-tree/src/components/mod.rs +++ b/rust/sglang-radix-tree/src/components/mod.rs @@ -7,7 +7,7 @@ use tch::Tensor; use crate::node::{ChildKeyType, Node, NodeArena, NodeId, NodeIdx_, TreeCoreRuntimeError}; use crate::unified_tree_core::{ - CacheAction, CacheTransferPhase, DecLockRefParams, EvictLayer, IncLockRefResult, InsertParams, + CacheActions, CacheTransferPhase, DecLockRefParams, EvictLayer, IncLockRefResult, InsertParams, InsertResult, LRURefreshPhase, MatchPrefixParams, MatchResult, PoolTransfer, PoolTransferResult, UnifiedTreeCore, }; @@ -199,7 +199,7 @@ pub trait TreeComponent { params: &InsertParams<'_, K>, preserve_full_ids: bool, result: &mut InsertResult, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, ) -> usize { prefix_len } @@ -216,7 +216,7 @@ pub trait TreeComponent { total_prefix_len: usize, params: &InsertParams<'_, K>, result: &mut InsertResult, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, ) { } @@ -239,7 +239,7 @@ pub trait TreeComponent { is_new_leaf: bool, params: &InsertParams<'_, K>, result: &mut InsertResult, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, ) { } @@ -251,7 +251,7 @@ pub trait TreeComponent { tail_node_id: NodeIdx_, device_frees: &mut HashMap>, host_frees: &mut HashMap>, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, ) { } @@ -307,7 +307,7 @@ pub trait TreeComponent { node_id: NodeIdx_, device_frees: &mut HashMap>, host_frees: &mut HashMap>, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, target: EvictLayer, ) -> (usize, usize); @@ -330,7 +330,7 @@ pub trait TreeComponent { tracker: &mut HashMap, device_frees: &mut HashMap>, host_frees: &mut HashMap>, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, ) -> Option; /// Clear this component's device-eviction walk state. @@ -406,7 +406,7 @@ pub trait TreeComponent { _params: &DecLockRefParams, _device_frees: &mut HashMap>, _host_frees: &mut HashMap>, - _cache_actions: &mut Vec, + _cache_actions: &mut CacheActions, ) { unimplemented!("release_window_lock is SWA-only") } @@ -460,7 +460,7 @@ pub trait TreeComponent { node_id: NodeIdx_, phase: CacheTransferPhase, transfers: Vec, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, insert_result: Option<&mut InsertResult>, pool_storage_result: Option<&PoolTransferResult>, ) { @@ -489,7 +489,7 @@ pub trait TreeComponent { _tracker: &mut HashMap, _device_frees: &mut HashMap>, _host_frees: &mut HashMap>, - _cache_actions: &mut Vec, + _cache_actions: &mut CacheActions, ) { } @@ -503,7 +503,7 @@ pub trait TreeComponent { _tracker: &mut HashMap, _device_frees: &mut HashMap>, _host_frees: &mut HashMap>, - _cache_actions: &mut Vec, + _cache_actions: &mut CacheActions, ) { } } diff --git a/rust/sglang-radix-tree/src/components/swa.rs b/rust/sglang-radix-tree/src/components/swa.rs index 27ed86472c..3305bfe496 100644 --- a/rust/sglang-radix-tree/src/components/swa.rs +++ b/rust/sglang-radix-tree/src/components/swa.rs @@ -7,13 +7,17 @@ use std::collections::HashMap; use tch::{Kind, Tensor}; +use crate::cache_action::{ + FreeComponentHostSlot, FreeDeviceKVFullOnly, RebuildFullToSwaMapping, RecoverSwaWithLockedFull, + SwaRebuild, +}; use crate::components::TreeComponent; use crate::components::{ComponentType, FULL, SWA}; use crate::node::ChildKeyType; use crate::node::Node; use crate::node::{NodeId, NodeIdx_, TreeCoreRuntimeError, ValueSlotIdx}; use crate::unified_tree_core::{ - CacheAction, CacheInitParams, CacheTransferPhase, DecLockRefParams, EvictLayer, + CacheActions, CacheInitParams, CacheTransferPhase, DecLockRefParams, EvictLayer, IncLockRefResult, InsertParams, InsertResult, LRURefreshPhase, MatchPrefixParams, MatchResult, PoolHitPolicy, PoolName, PoolTransfer, PoolTransferResult, UnifiedTreeCore, }; @@ -57,7 +61,7 @@ impl SwaComponent { &self, tree_core: &mut UnifiedTreeCore, leaf_id: NodeIdx_, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, ) -> Option { let leaf = tree_core.arena.node(leaf_id); if leaf.is_root() || leaf.device_lock_ref(SWA) > 0 { @@ -80,17 +84,11 @@ impl SwaComponent { let (new_parent, action) = tree_core.split_node_(leaf_id, split_at); assert!( - !matches!( - action, - Some(CacheAction::UpdatePendingBackupOnNodeSplit { - ack_id: Some(_), - .. - }) - ), + action.as_ref().is_none_or(|action| action.ack_id.is_none()), "fresh SWA leaf cannot be write-through-pending" ); if let Some(action) = action { - cache_actions.push(action); + cache_actions.push(Box::new(action)); } Some(new_parent) } @@ -193,12 +191,12 @@ impl SwaComponent { impl SwaComponent { /// Queue a free of the given SWA host slots; empty tensors are dropped. - fn release_swa_host_(&self, host_indices: Tensor, cache_actions: &mut Vec) { + fn release_swa_host_(&self, host_indices: Tensor, cache_actions: &mut CacheActions) { if host_indices.numel() > 0 { - cache_actions.push(CacheAction::FreeComponentHostSlot { + cache_actions.push(Box::new(FreeComponentHostSlot { component_type: SWA, host_indices: vec![host_indices], - }); + })); } } @@ -234,7 +232,7 @@ impl SwaComponent { tree_core: &mut UnifiedTreeCore, node_id: NodeIdx_, transfers: Vec, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, insert_result: Option<&InsertResult>, pool_storage_result: Option<&PoolTransferResult>, ) { @@ -307,7 +305,7 @@ impl SwaComponent { if fill_start > node_start { let (_, action) = tree_core.split_node_(cur, fill_start - node_start); if let Some(action) = action { - cache_actions.push(action); + cache_actions.push(Box::new(action)); } } self.attach_swa_host_value_(tree_core, cur, slice); @@ -441,7 +439,7 @@ impl TreeComponent for SwaComponent { params: &InsertParams<'_, K>, preserve_full_ids: bool, result: &mut InsertResult, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, ) -> usize { if params.prev_prefix_len >= total_prefix_len + prefix_len { return prefix_len; @@ -467,21 +465,23 @@ impl TreeComponent for SwaComponent { // Branch 1: entire value_slice is within SWA window — recover result.record_adopted_range(SWA, total_prefix_len, total_prefix_len + prefix_len); if node.device_lock_ref(FULL) > 0 || preserve_full_ids { - cache_actions.push(CacheAction::RecoverSwaWithLockedFull { + cache_actions.push(Box::new(RecoverSwaWithLockedFull { node_id: node.id, kept_full: node.device_value(FULL).shallow_clone(), incoming_full: value_slice, - }); + })); return 0; } result.record_adopted_range(FULL, total_prefix_len, total_prefix_len + prefix_len); let old_full = node.take_device_value(FULL); node.set_device_value(FULL, value_slice.copy()); - cache_actions.push(CacheAction::FreeDeviceKVFullOnly(vec![old_full])); - cache_actions.push(CacheAction::SwaRebuild { + cache_actions.push(Box::new(FreeDeviceKVFullOnly { + indices: vec![old_full], + })); + cache_actions.push(Box::new(SwaRebuild { node_id: node.id, source_value: value_slice, - }); + })); 0 } else if swa_evicted_seqlen < total_prefix_len + prefix_len { // Branch 2: value_slice[start_idx:] is within SWA window — partial recover @@ -495,26 +495,28 @@ impl TreeComponent for SwaComponent { .narrow(0, start_idx as i64, (full_len - start_idx) as i64); let (_, action) = tree_core.split_node_(node_id, start_idx); if let Some(action) = action { - cache_actions.push(action); + cache_actions.push(Box::new(action)); } let new_full = value_slice.narrow(0, start_idx as i64, (prefix_len - start_idx) as i64); if preserve_full { - cache_actions.push(CacheAction::RecoverSwaWithLockedFull { + cache_actions.push(Box::new(RecoverSwaWithLockedFull { node_id: node_ext_id, kept_full: old_full, incoming_full: new_full, - }); + })); return start_idx; } result.record_adopted_range(FULL, swa_evicted_seqlen, total_prefix_len + prefix_len); let node = tree_core.arena.node_mut(node_id); let _ = node.take_device_value(FULL); node.set_device_value(FULL, new_full.copy()); - cache_actions.push(CacheAction::FreeDeviceKVFullOnly(vec![old_full])); - cache_actions.push(CacheAction::SwaRebuild { + cache_actions.push(Box::new(FreeDeviceKVFullOnly { + indices: vec![old_full], + })); + cache_actions.push(Box::new(SwaRebuild { node_id: node_ext_id, source_value: new_full, - }); + })); start_idx } else { // Branch 3: entire value_slice is outside SWA window — not consumed @@ -530,7 +532,7 @@ impl TreeComponent for SwaComponent { total_prefix_len: usize, params: &InsertParams<'_, K>, result: &mut InsertResult, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, ) { // _unevict_node_on_insert already wrote the request's fresh KV slice // into the base value. We just need to rebuild SWA from that slice for @@ -552,7 +554,7 @@ impl TreeComponent for SwaComponent { let start_idx = swa_evicted_seqlen - total_prefix_len; let (_, action) = tree_core.split_node_(node_id, start_idx); if let Some(action) = action { - cache_actions.push(action); + cache_actions.push(Box::new(action)); } } else { return; @@ -562,10 +564,10 @@ impl TreeComponent for SwaComponent { total_prefix_len.max(swa_evicted_seqlen), total_prefix_len + prefix_len, ); - cache_actions.push(CacheAction::SwaRebuild { + cache_actions.push(Box::new(SwaRebuild { node_id: tree_core.arena.node(node_id).id, source_value: tree_core.arena.device_value(node_id, FULL).shallow_clone(), - }); + })); } fn commit_insert_component_data( @@ -575,7 +577,7 @@ impl TreeComponent for SwaComponent { is_new_leaf: bool, params: &InsertParams<'_, K>, result: &mut InsertResult, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, ) { if let Some(branching_seqlen) = params.swa_branching_seqlen { result.swa_branch_inserted = params.key.atom_len() >= branching_seqlen; @@ -599,7 +601,7 @@ impl TreeComponent for SwaComponent { // (tombstone) and an in-window child; `node` becomes the child. let (_, action) = tree_core.split_node_(node_id, split_pos); if let Some(action) = action { - cache_actions.push(action); + cache_actions.push(Box::new(action)); } } } @@ -609,18 +611,18 @@ impl TreeComponent for SwaComponent { // the in-window tail lands more-MRU. let capped_parent = self.maybe_split_leaf_for_swa_lock_(tree_core, node_id, cache_actions); if let Some(capped_parent) = capped_parent { - cache_actions.push(CacheAction::SwaRebuild { + cache_actions.push(Box::new(SwaRebuild { node_id: tree_core.arena.node(capped_parent).id, source_value: tree_core .arena .device_value(capped_parent, FULL) .shallow_clone(), - }); + })); } - cache_actions.push(CacheAction::SwaRebuild { + cache_actions.push(Box::new(SwaRebuild { node_id: tree_core.arena.node(node_id).id, source_value: tree_core.arena.device_value(node_id, FULL).shallow_clone(), - }); + })); } fn redistribute_on_node_split( @@ -671,7 +673,7 @@ impl TreeComponent for SwaComponent { node_id: NodeIdx_, device_frees: &mut HashMap>, host_frees: &mut HashMap>, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, target: EvictLayer, ) -> (usize, usize) { let ct = SWA; @@ -741,7 +743,7 @@ impl TreeComponent for SwaComponent { tracker: &mut HashMap, device_frees: &mut HashMap>, host_frees: &mut HashMap>, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, ) -> Option { let ct = SWA; assert!( @@ -848,7 +850,7 @@ impl TreeComponent for SwaComponent { tracker: &mut HashMap, device_frees: &mut HashMap>, host_frees: &mut HashMap>, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, ) { let ct = SWA; let mut x = tree_core @@ -1057,7 +1059,7 @@ impl TreeComponent for SwaComponent { node_id: NodeIdx_, phase: CacheTransferPhase, transfers: Vec, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, insert_result: Option<&mut InsertResult>, pool_storage_result: Option<&PoolTransferResult>, ) { @@ -1142,10 +1144,10 @@ impl TreeComponent for SwaComponent { assert_eq!(offset, host_indices.size()[0]); // Rebuild the full->swa mapping for the loaded chunks at the orchestration layer. if !full_chunks.is_empty() { - cache_actions.push(CacheAction::RebuildFullToSwaMapping { + cache_actions.push(Box::new(RebuildFullToSwaMapping { full_indices: full_chunks, swa_indices: swa_chunks, - }); + })); } } CacheTransferPhase::Prefetch => { @@ -1289,7 +1291,7 @@ impl TreeComponent for SwaComponent { params: &DecLockRefParams, device_frees: &mut HashMap>, host_frees: &mut HashMap>, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, ) { if tree_core.arena.node(node_id).is_root() { return; diff --git a/rust/sglang-radix-tree/src/lib.rs b/rust/sglang-radix-tree/src/lib.rs index 3f67a7d5e9..118de7578e 100644 --- a/rust/sglang-radix-tree/src/lib.rs +++ b/rust/sglang-radix-tree/src/lib.rs @@ -16,6 +16,7 @@ clippy::useless_conversion )] +mod cache_action; mod components; mod node; #[cfg(feature = "python-extension")] diff --git a/rust/sglang-radix-tree/src/python_bindings.rs b/rust/sglang-radix-tree/src/python_bindings.rs index 7e6d75d4eb..82ad6fa0e6 100644 --- a/rust/sglang-radix-tree/src/python_bindings.rs +++ b/rust/sglang-radix-tree/src/python_bindings.rs @@ -18,10 +18,10 @@ use crate::node::{KeyNamespaceRef, NodeAccessError, NodeId, TreeCoreRuntimeError use crate::unified_lru_list::{TlruFloatConfig, TlruPromptEstimate}; use crate::unified_tree_core::KvCacheEvent; use crate::unified_tree_core::{ - BufferBackupSnapshot, BufferBackupState, CacheAction, CacheInitParams, CacheTransferPhase, - DecLockRefParams, EvictLayer, EvictionStepResult, InsertParams, InsertResult, InsertStepResult, - MatchPrefixParams, MatchResult, PoolHitPolicy, PoolName, PoolTransfer, PoolTransferResult, Req, - UnifiedTreeCore, + BufferBackupSnapshot, BufferBackupState, CacheAction, CacheActions, CacheInitParams, + CacheTransferPhase, DecLockRefParams, EvictLayer, EvictionStepResult, InsertParams, + InsertResult, InsertStepResult, MatchPrefixParams, MatchResult, PoolHitPolicy, PoolName, + PoolTransfer, PoolTransferResult, Req, UnifiedTreeCore, }; /// Translate only Rust panics; ordinary Python errors pass through unchanged. @@ -173,83 +173,13 @@ fn py_array_to_vec_i64(py: Python<'_>, key: &Bound<'_, PyAny>) -> PyResult &'static str { - match action { - CacheAction::FreeDeviceKV(_) => "free_device_kv", - CacheAction::FreeDeviceKVFullOnly(_) => "free_device_kv_full_only", - CacheAction::BackupKV(_) => "backup_kv", - CacheAction::UpdatePendingBackupOnNodeSplit { .. } => "update_pending_backup_on_node_split", - CacheAction::MambaEvictExcessPathStates { .. } => "mamba_evict_excess_path_states", - CacheAction::FreeComponentDeviceSlot { .. } => "free_component_device_slot", - CacheAction::FreeComponentHostSlot { .. } => "free_component_host_slot", - CacheAction::RebuildFullToSwaMapping { .. } => "rebuild_full_to_swa_mapping", - CacheAction::RecoverSwaWithLockedFull { .. } => "recover_swa_with_locked_full", - CacheAction::SwaRebuild { .. } => "swa_rebuild", - } -} - -/// Convert a cache action into its Python tagged tuple. -fn cache_action_to_py(py: Python<'_>, action: CacheAction) -> PyResult> { - let tag = cache_action_tag(&action); - match action { - CacheAction::FreeDeviceKV(tensors) => { - let tensors = PyList::new(py, tensors.into_iter().map(PyTensor))?; - (tag, tensors).into_py_any(py) - } - CacheAction::FreeDeviceKVFullOnly(tensors) => { - let tensors = PyList::new(py, tensors.into_iter().map(PyTensor))?; - (tag, tensors).into_py_any(py) - } - CacheAction::BackupKV(backup) => (tag, backup.node_ids).into_py_any(py), - CacheAction::UpdatePendingBackupOnNodeSplit { - ack_id, - old_node_id, - new_node_id, - new_child_node_id, - } => (tag, ack_id, old_node_id, new_node_id, new_child_node_id).into_py_any(py), - CacheAction::MambaEvictExcessPathStates { tail_node_id } => { - (tag, tail_node_id).into_py_any(py) - } - CacheAction::FreeComponentDeviceSlot { - component_type, - indices, - } => { - let indices = PyList::new(py, indices.into_iter().map(PyTensor))?; - (tag, component_type_to_u8(component_type), indices).into_py_any(py) - } - CacheAction::FreeComponentHostSlot { - component_type, - host_indices, - } => { - let host_indices = PyList::new(py, host_indices.into_iter().map(PyTensor))?; - (tag, component_type_to_u8(component_type), host_indices).into_py_any(py) - } - CacheAction::RebuildFullToSwaMapping { - full_indices, - swa_indices, - } => { - let full_indices = PyList::new(py, full_indices.into_iter().map(PyTensor))?; - let swa_indices = PyList::new(py, swa_indices.into_iter().map(PyTensor))?; - (tag, full_indices, swa_indices).into_py_any(py) - } - CacheAction::RecoverSwaWithLockedFull { - node_id, - kept_full, - incoming_full, - } => (tag, node_id, PyTensor(kept_full), PyTensor(incoming_full)).into_py_any(py), - CacheAction::SwaRebuild { - node_id, - source_value, - } => (tag, node_id, PyTensor(source_value)).into_py_any(py), - } -} +mod cache_action; /// Convert cache actions into a Python list of tagged tuples. -fn cache_actions_to_py(py: Python<'_>, actions: Vec) -> PyResult> { +fn cache_actions_to_py(py: Python<'_>, actions: CacheActions) -> PyResult> { let list = PyList::empty(py); for action in actions { - list.append(cache_action_to_py(py, action)?)?; + list.append(action.into_py_tagged(py)?)?; } Ok(list.unbind()) } @@ -1587,7 +1517,7 @@ macro_rules! tree_core_methods { Ok(EvictDeviceLeafResultBinding { unbacked_tokens: result.unbacked_tokens, backup_kv: backup - .map(|backup| cache_action_to_py(py, CacheAction::BackupKV(backup))) + .map(|backup| backup.into_py_tagged(py)) .transpose()?, tracker: tracker_to_py(result.tracker), new_device_frees: frees_to_py(py, result.device_frees)?, @@ -1774,7 +1704,7 @@ macro_rules! tree_core_methods { let backup = py .detach(|| self.core().build_backup_kv_action(node_id)) .map_err(node_access_error)?; - cache_action_to_py(py, CacheAction::BackupKV(backup)) + backup.into_py_tagged(py) } /// Gather a node's device value plus per-component BACKUP_HOST transfers. diff --git a/rust/sglang-radix-tree/src/python_bindings/cache_action.rs b/rust/sglang-radix-tree/src/python_bindings/cache_action.rs new file mode 100644 index 0000000000..aa2df287f3 --- /dev/null +++ b/rust/sglang-radix-tree/src/python_bindings/cache_action.rs @@ -0,0 +1,97 @@ +//! Python encoding for the common cache actions. + +use super::*; +use crate::cache_action::{ + BackupKV, FreeComponentDeviceSlot, FreeComponentHostSlot, FreeDeviceKV, FreeDeviceKVFullOnly, + MambaEvictExcessPathStates, RebuildFullToSwaMapping, RecoverSwaWithLockedFull, SwaRebuild, + UpdatePendingBackupOnNodeSplit, +}; + +impl FreeDeviceKV { + pub(crate) fn into_py_tagged(self, py: Python<'_>) -> PyResult> { + let tag = self.tag(); + let indices = PyList::new(py, self.indices.into_iter().map(PyTensor))?; + (tag, indices).into_py_any(py) + } +} + +impl FreeDeviceKVFullOnly { + pub(crate) fn into_py_tagged(self, py: Python<'_>) -> PyResult> { + let tag = self.tag(); + let indices = PyList::new(py, self.indices.into_iter().map(PyTensor))?; + (tag, indices).into_py_any(py) + } +} + +impl BackupKV { + pub(crate) fn into_py_tagged(self, py: Python<'_>) -> PyResult> { + let tag = self.tag(); + (tag, self.node_ids).into_py_any(py) + } +} + +impl UpdatePendingBackupOnNodeSplit { + pub(crate) fn into_py_tagged(self, py: Python<'_>) -> PyResult> { + let tag = self.tag(); + ( + tag, + self.ack_id, + self.old_node_id, + self.new_node_id, + self.new_child_node_id, + ) + .into_py_any(py) + } +} + +impl MambaEvictExcessPathStates { + pub(crate) fn into_py_tagged(self, py: Python<'_>) -> PyResult> { + let tag = self.tag(); + (tag, self.tail_node_id).into_py_any(py) + } +} + +impl FreeComponentDeviceSlot { + pub(crate) fn into_py_tagged(self, py: Python<'_>) -> PyResult> { + let tag = self.tag(); + let indices = PyList::new(py, self.indices.into_iter().map(PyTensor))?; + (tag, component_type_to_u8(self.component_type), indices).into_py_any(py) + } +} + +impl FreeComponentHostSlot { + pub(crate) fn into_py_tagged(self, py: Python<'_>) -> PyResult> { + let tag = self.tag(); + let indices = PyList::new(py, self.host_indices.into_iter().map(PyTensor))?; + (tag, component_type_to_u8(self.component_type), indices).into_py_any(py) + } +} + +impl RebuildFullToSwaMapping { + pub(crate) fn into_py_tagged(self, py: Python<'_>) -> PyResult> { + let tag = self.tag(); + let full_indices = PyList::new(py, self.full_indices.into_iter().map(PyTensor))?; + let swa_indices = PyList::new(py, self.swa_indices.into_iter().map(PyTensor))?; + (tag, full_indices, swa_indices).into_py_any(py) + } +} + +impl RecoverSwaWithLockedFull { + pub(crate) fn into_py_tagged(self, py: Python<'_>) -> PyResult> { + let tag = self.tag(); + ( + tag, + self.node_id, + PyTensor(self.kept_full), + PyTensor(self.incoming_full), + ) + .into_py_any(py) + } +} + +impl SwaRebuild { + pub(crate) fn into_py_tagged(self, py: Python<'_>) -> PyResult> { + let tag = self.tag(); + (tag, self.node_id, PyTensor(self.source_value)).into_py_any(py) + } +} diff --git a/rust/sglang-radix-tree/src/tests/components/base.rs b/rust/sglang-radix-tree/src/tests/components/base.rs index 1e78fef850..940806cb2b 100644 --- a/rust/sglang-radix-tree/src/tests/components/base.rs +++ b/rust/sglang-radix-tree/src/tests/components/base.rs @@ -1,5 +1,7 @@ use super::*; +use crate::cache_action::{CacheAction, MambaEvictExcessPathStates}; use crate::components::{FULL, FullComponent, MAMBA, MambaComponent, SWA, SwaComponent}; +use crate::test_utils::expect_action; use crate::unified_tree_core::CacheInitParams; use std::sync::{Arc, Mutex}; @@ -56,12 +58,12 @@ impl TreeComponent> for DefaultComponentForTest { node_id: NodeIdx_, device_frees: &mut HashMap>, host_frees: &mut HashMap>, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, target: EvictLayer, ) -> (usize, usize) { - cache_actions.push(CacheAction::MambaEvictExcessPathStates { + cache_actions.push(Box::new(MambaEvictExcessPathStates { tail_node_id: tree_core.arena.node(node_id).id, - }); + })); FullComponent.evict_component( tree_core, node_id, @@ -82,7 +84,7 @@ impl TreeComponent> for DefaultComponentForTest { tracker: &mut HashMap, device_frees: &mut HashMap>, host_frees: &mut HashMap>, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, ) -> Option { unimplemented!() } @@ -234,13 +236,10 @@ fn action_node( node } -fn action_node_ids(actions: &[CacheAction]) -> Vec { +fn action_node_ids(actions: &[Box]) -> Vec { actions .iter() - .map(|action| match action { - CacheAction::MambaEvictExcessPathStates { tail_node_id } => *tail_node_id, - _ => panic!("unexpected component action"), - }) + .map(|action| expect_action::(action.as_ref()).tail_node_id) .collect() } diff --git a/rust/sglang-radix-tree/src/tests/components/full.rs b/rust/sglang-radix-tree/src/tests/components/full.rs index bec7644063..c07803b5ca 100644 --- a/rust/sglang-radix-tree/src/tests/components/full.rs +++ b/rust/sglang-radix-tree/src/tests/components/full.rs @@ -1,4 +1,5 @@ use super::*; +use crate::cache_action::UpdatePendingBackupOnNodeSplit; use crate::components::{ComponentSet, FULL}; use crate::node::NodeAccessError; use crate::test_utils::accumulate_step; @@ -496,7 +497,7 @@ fn host_reclaim_orders_pending_internal_split_fragments_by_ack() { // asynchronously under write_back; matching can split it before the ack. let (prefix_idx, action) = tc.split_node_(tc.arena.resolve(a).unwrap(), 2); let published_a = match action { - Some(CacheAction::UpdatePendingBackupOnNodeSplit { + Some(UpdatePendingBackupOnNodeSplit { ack_id, old_node_id, new_node_id, diff --git a/rust/sglang-radix-tree/src/tests/components/mamba.rs b/rust/sglang-radix-tree/src/tests/components/mamba.rs index a37ec593b5..086e9ca9a5 100644 --- a/rust/sglang-radix-tree/src/tests/components/mamba.rs +++ b/rust/sglang-radix-tree/src/tests/components/mamba.rs @@ -1,6 +1,7 @@ use super::*; +use crate::cache_action::{BackupKV, FreeComponentHostSlot, MambaEvictExcessPathStates}; use crate::components::{ComponentSet, FULL, MAMBA, SWA}; -use crate::test_utils::{accumulate_step, action_kinds}; +use crate::test_utils::{accumulate_step, action_kinds, expect_action}; use crate::unified_lru_list::UnifiedLRUList; fn mamba_core(page_size: usize) -> UnifiedTreeCore> { @@ -560,10 +561,7 @@ fn reinsert_full_backed_target_schedules_mamba_only_backup() { let backups = result .cache_actions .iter() - .filter_map(|action| match action { - CacheAction::BackupKV(backup) => Some(backup), - _ => None, - }) + .filter_map(|action| action.as_any().downcast_ref::()) .collect::>(); assert_eq!(backups.len(), 1); assert_eq!(backups[0].node_ids, vec![leaf]); @@ -587,7 +585,7 @@ fn reinsert_full_backed_target_schedules_mamba_only_backup() { !pending .cache_actions .iter() - .any(|action| matches!(action, CacheAction::BackupKV(_))) + .any(|action| action.as_any().is::()) ); } @@ -607,7 +605,7 @@ fn write_back_reinsert_defers_the_mamba_backup_to_eviction() { !result .cache_actions .iter() - .any(|action| matches!(action, CacheAction::BackupKV(_))) + .any(|action| action.as_any().is::()) ); } @@ -984,7 +982,7 @@ fn check_mamba_restored_during_expanded_swa_backup(pin_ancestor: bool) { !inserted .cache_actions .iter() - .any(|action| matches!(action, CacheAction::BackupKV(_))) + .any(|action| action.as_any().is::()) ); assert!( tc.arena @@ -2123,13 +2121,10 @@ fn prefetch_commit_frees_the_buffer_when_it_cannot_attach() { .has_host_value(MAMBA) ); assert!(insert_result.mamba_exist); - let CacheAction::FreeComponentHostSlot { + let FreeComponentHostSlot { component_type, host_indices, - } = &cache_actions[0] - else { - panic!("expected a FreeComponentHostSlot action"); - }; + } = expect_action::(cache_actions[0].as_ref()); assert_eq!(*component_type, MAMBA); assert!(host_indices[0].equal(&Tensor::from_slice(&[50i64]))); @@ -2260,14 +2255,14 @@ fn insert_commit_emits_the_path_cap_action_only_when_capped() { let leaf = tc .match_prefix(&match_params(&vec![1, 2])) .best_match_node_id; - let [CacheAction::MambaEvictExcessPathStates { tail_node_id }] = - result.cache_actions.as_slice() - else { + let [action] = result.cache_actions.as_slice() else { panic!( "expected one MambaEvictExcessPathStates, got {:?}", action_kinds(&result.cache_actions) ); }; + let MambaEvictExcessPathStates { tail_node_id } = + expect_action::(action.as_ref()); assert_eq!(*tail_node_id, leaf); let mut uncapped = mamba_core(/* page_size = */ 1); diff --git a/rust/sglang-radix-tree/src/tests/components/swa.rs b/rust/sglang-radix-tree/src/tests/components/swa.rs index 5d5b8abf9e..37cc5bdd27 100644 --- a/rust/sglang-radix-tree/src/tests/components/swa.rs +++ b/rust/sglang-radix-tree/src/tests/components/swa.rs @@ -1,6 +1,10 @@ use super::*; +use crate::cache_action::{ + BackupKV, FreeComponentHostSlot, FreeDeviceKV, FreeDeviceKVFullOnly, RebuildFullToSwaMapping, + RecoverSwaWithLockedFull, SwaRebuild, UpdatePendingBackupOnNodeSplit, +}; use crate::components::{ComponentSet, FULL, MAMBA, SWA}; -use crate::test_utils::{accumulate_step, action_kinds, matched_device_indices}; +use crate::test_utils::{accumulate_step, action_kinds, expect_action, matched_device_indices}; use crate::unified_tree_core::CacheInitParams; #[test] @@ -76,7 +80,7 @@ fn swa_window_attach_splits_both_edges_preserving_full_and_pending_actions() { assert_eq!(actions.len(), 2); for action in &actions { assert!( - matches!(action, CacheAction::UpdatePendingBackupOnNodeSplit { ack_id: Some(91), old_node_id, new_child_node_id, .. } if *old_node_id == leaf_id && *new_child_node_id == leaf_id) + matches!(action.as_any().downcast_ref::(), Some(UpdatePendingBackupOnNodeSplit { ack_id: Some(91), old_node_id, new_child_node_id, .. }) if *old_node_id == leaf_id && *new_child_node_id == leaf_id) ); } let head = child_of(&tc, tc.arena.root(), &[1, 2]); @@ -904,15 +908,13 @@ fn insert_new_leaf_in_window_emits_one_leaf_rebuild() { assert_eq!(result.prefix_len, 0); let root = tc.arena.root(); let leaf = child_of(&tc, root, &[1]); - let [ - CacheAction::SwaRebuild { - node_id, - source_value, - }, - ] = result.cache_actions.as_slice() - else { + let [action] = result.cache_actions.as_slice() else { panic!("expected one SwaRebuild action"); }; + let SwaRebuild { + node_id, + source_value, + } = expect_action::(action.as_ref()); assert_eq!(*node_id, tc.arena.node(leaf).id); assert!(source_value.equal(&Tensor::from_slice(&[10i64, 11, 12]))); // The rebuild is deferred to apply time: the leaf is still an SWA tombstone. @@ -940,15 +942,13 @@ fn insert_new_leaf_straddling_the_boundary_splits_and_rebuilds_the_tail() { .device_value(parent, FULL) .equal(&Tensor::from_slice(&[10i64, 11])) ); - let [ - CacheAction::SwaRebuild { - node_id, - source_value, - }, - ] = result.cache_actions.as_slice() - else { + let [action] = result.cache_actions.as_slice() else { panic!("expected one SwaRebuild action"); }; + let SwaRebuild { + node_id, + source_value, + } = expect_action::(action.as_ref()); assert_eq!(*node_id, tc.arena.node(child).id); assert!(source_value.equal(&Tensor::from_slice(&[12i64, 13]))); } @@ -984,19 +984,17 @@ fn insert_long_leaf_caps_the_window_and_rebuilds_the_older_prefix_first() { let tail = child_of(&tc, capped, &[5]); assert_eq!(tc.arena.node(capped).key, vec![1, 2, 3, 4]); assert_eq!(tc.arena.node(tail).key, vec![5, 6]); - let [ - CacheAction::SwaRebuild { - node_id: first_id, - source_value: first_value, - }, - CacheAction::SwaRebuild { - node_id: second_id, - source_value: second_value, - }, - ] = result.cache_actions.as_slice() - else { + let [action0, action1] = result.cache_actions.as_slice() else { panic!("expected two SwaRebuild actions"); }; + let SwaRebuild { + node_id: first_id, + source_value: first_value, + } = expect_action::(action0.as_ref()); + let SwaRebuild { + node_id: second_id, + source_value: second_value, + } = expect_action::(action1.as_ref()); assert_eq!(*first_id, tc.arena.node(capped).id); assert!(first_value.equal(&Tensor::from_slice(&[10i64, 11, 12, 13]))); assert_eq!(*second_id, tc.arena.node(tail).id); @@ -1035,9 +1033,10 @@ fn insert_overlap_with_live_swa_frees_the_whole_duplicate() { .expect("live test node"); let result = tc.insert(&insert_params_swa(&vec![1, 2, 3], &[20, 21, 22], 0, 0)); assert_eq!(result.prefix_len, 3); - let [CacheAction::FreeDeviceKV(freed)] = result.cache_actions.as_slice() else { + let [action] = result.cache_actions.as_slice() else { panic!("expected one FreeDeviceKV action"); }; + let FreeDeviceKV { indices: freed } = expect_action::(action.as_ref()); assert_eq!(freed.len(), 1); assert!(freed[0].equal(&Tensor::from_slice(&[20i64, 21, 22]))); // The node keeps its original Full and SWA values. @@ -1071,16 +1070,15 @@ fn insert_overlap_recovers_a_tombstone_inside_the_window() { .device_value(leaf, FULL) .equal(&Tensor::from_slice(&[20i64, 21, 22])) ); - let [ - CacheAction::FreeDeviceKVFullOnly(freed), - CacheAction::SwaRebuild { - node_id, - source_value, - }, - ] = result.cache_actions.as_slice() - else { + let [action0, action1] = result.cache_actions.as_slice() else { panic!("expected FreeDeviceKVFullOnly then SwaRebuild"); }; + let FreeDeviceKVFullOnly { indices: freed } = + expect_action::(action0.as_ref()); + let SwaRebuild { + node_id, + source_value, + } = expect_action::(action1.as_ref()); assert!(freed[0].equal(&Tensor::from_slice(&[10i64, 11, 12]))); assert_eq!(*node_id, tc.arena.node(leaf).id); assert!(source_value.equal(&Tensor::from_slice(&[20i64, 21, 22]))); @@ -1110,12 +1108,10 @@ fn insert_overlap_recovers_a_locked_swa_tombstone() { .device_value(leaf, FULL) .equal(&Tensor::from_slice(&[10i64, 11, 12])) ); - assert!( - result - .cache_actions - .iter() - .any(|action| matches!(action, CacheAction::RecoverSwaWithLockedFull { .. })) - ); + assert!(result.cache_actions.iter().any(|action| matches!( + action.as_any().downcast_ref::(), + Some(RecoverSwaWithLockedFull { .. }) + ))); } #[test] @@ -1147,16 +1143,14 @@ fn insert_overlap_can_preserve_unlocked_full_ids() { assert_eq!(consumed_from, window_start); assert_eq!(tc.arena.node(leaf).id, original_id); assert_eq!(tc.arena.node(leaf).device_lock_ref(FULL), 0); - let [ - CacheAction::RecoverSwaWithLockedFull { - node_id, - kept_full, - incoming_full, - }, - ] = actions.as_slice() - else { + let [action] = actions.as_slice() else { panic!("expected recovery with canonical FULL IDs"); }; + let RecoverSwaWithLockedFull { + node_id, + kept_full, + incoming_full, + } = expect_action::(action.as_ref()); assert_eq!(*node_id, original_id); assert!(kept_full.equal(&Tensor::from_slice(&[10i64, 11, 12, 13][window_start..]))); assert!(incoming_full.equal(&Tensor::from_slice(&[20i64, 21, 22, 23][window_start..]))); @@ -1187,16 +1181,14 @@ fn insert_overlap_with_a_locked_full_emits_the_recover_action() { .device_value(leaf, FULL) .equal(&Tensor::from_slice(&[10i64, 11, 12])) ); - let [ - CacheAction::RecoverSwaWithLockedFull { - node_id, - kept_full, - incoming_full, - }, - ] = result.cache_actions.as_slice() - else { + let [action] = result.cache_actions.as_slice() else { panic!("expected one RecoverSwaWithLockedFull action"); }; + let RecoverSwaWithLockedFull { + node_id, + kept_full, + incoming_full, + } = expect_action::(action.as_ref()); assert_eq!(*node_id, tc.arena.node(leaf).id); assert!(kept_full.equal(&Tensor::from_slice(&[10i64, 11, 12]))); assert!(incoming_full.equal(&Tensor::from_slice(&[20i64, 21, 22]))); @@ -1238,20 +1230,21 @@ fn insert_overlap_straddling_the_boundary_splits_and_recovers_the_tail() { .device_value(node, FULL) .equal(&Tensor::from_slice(&[22i64, 23])) ); - let [ - CacheAction::FreeDeviceKVFullOnly(old_tail), - CacheAction::SwaRebuild { - node_id, - source_value, - }, - CacheAction::FreeDeviceKVFullOnly(duplicates), - ] = result.cache_actions.as_slice() - else { + let [action0, action1, action2] = result.cache_actions.as_slice() else { panic!( "expected FreeDeviceKVFullOnly, SwaRebuild, FreeDeviceKVFullOnly, got {:?}", action_kinds(&result.cache_actions) ); }; + let FreeDeviceKVFullOnly { indices: old_tail } = + expect_action::(action0.as_ref()); + let SwaRebuild { + node_id, + source_value, + } = expect_action::(action1.as_ref()); + let FreeDeviceKVFullOnly { + indices: duplicates, + } = expect_action::(action2.as_ref()); assert!(old_tail[0].equal(&Tensor::from_slice(&[12i64, 13]))); assert_eq!(*node_id, tc.arena.node(node).id); assert!(source_value.equal(&Tensor::from_slice(&[22i64, 23]))); @@ -1285,17 +1278,17 @@ fn insert_overlap_straddling_with_a_locked_full_defers_the_tail() { .device_value(node, FULL) .equal(&Tensor::from_slice(&[12i64, 13])) ); - let [ - CacheAction::RecoverSwaWithLockedFull { - node_id, - kept_full, - incoming_full, - }, - CacheAction::FreeDeviceKVFullOnly(duplicates), - ] = result.cache_actions.as_slice() - else { + let [action0, action1] = result.cache_actions.as_slice() else { panic!("expected RecoverSwaWithLockedFull then FreeDeviceKVFullOnly"); }; + let RecoverSwaWithLockedFull { + node_id, + kept_full, + incoming_full, + } = expect_action::(action0.as_ref()); + let FreeDeviceKVFullOnly { + indices: duplicates, + } = expect_action::(action1.as_ref()); assert_eq!(*node_id, tc.arena.node(node).id); assert!(kept_full.equal(&Tensor::from_slice(&[12i64, 13]))); assert!(incoming_full.equal(&Tensor::from_slice(&[22i64, 23]))); @@ -1313,9 +1306,11 @@ fn insert_overlap_entirely_outside_the_window_is_all_duplicate() { /* prev_prefix_len = */ 0, /* swa_evicted_seqlen = */ 3, )); - let [CacheAction::FreeDeviceKVFullOnly(freed)] = result.cache_actions.as_slice() else { + let [action] = result.cache_actions.as_slice() else { panic!("expected one FreeDeviceKVFullOnly action"); }; + let FreeDeviceKVFullOnly { indices: freed } = + expect_action::(action.as_ref()); assert!(freed[0].equal(&Tensor::from_slice(&[20i64, 21, 22]))); } @@ -1365,20 +1360,21 @@ fn insert_overlap_boundary_at_the_node_start_recovers_the_whole_node() { .device_value(b, FULL) .equal(&Tensor::from_slice(&[23i64, 24])) ); - let [ - CacheAction::FreeDeviceKVFullOnly(duplicates), - CacheAction::FreeDeviceKVFullOnly(old_full), - CacheAction::SwaRebuild { - node_id, - source_value, - }, - ] = result.cache_actions.as_slice() - else { + let [action0, action1, action2] = result.cache_actions.as_slice() else { panic!( "expected FreeDeviceKVFullOnly, FreeDeviceKVFullOnly, SwaRebuild, got {:?}", action_kinds(&result.cache_actions) ); }; + let FreeDeviceKVFullOnly { + indices: duplicates, + } = expect_action::(action0.as_ref()); + let FreeDeviceKVFullOnly { indices: old_full } = + expect_action::(action1.as_ref()); + let SwaRebuild { + node_id, + source_value, + } = expect_action::(action2.as_ref()); assert!(duplicates[0].equal(&Tensor::from_slice(&[20i64, 21, 22]))); assert!(old_full[0].equal(&Tensor::from_slice(&[13i64, 14]))); assert_eq!(*node_id, tc.arena.node(b).id); @@ -1420,21 +1416,24 @@ fn insert_overlap_straddling_a_second_level_node_recovers_the_tail() { .device_value(b, FULL) .equal(&Tensor::from_slice(&[24i64])) ); - let [ - CacheAction::FreeDeviceKVFullOnly(duplicates_head), - CacheAction::FreeDeviceKVFullOnly(old_tail), - CacheAction::SwaRebuild { - node_id, - source_value, - }, - CacheAction::FreeDeviceKVFullOnly(duplicates_tail), - ] = result.cache_actions.as_slice() - else { + let [action0, action1, action2, action3] = result.cache_actions.as_slice() else { panic!( "expected FreeDeviceKVFullOnly, FreeDeviceKVFullOnly, SwaRebuild, FreeDeviceKVFullOnly, got {:?}", action_kinds(&result.cache_actions) ); }; + let FreeDeviceKVFullOnly { + indices: duplicates_head, + } = expect_action::(action0.as_ref()); + let FreeDeviceKVFullOnly { indices: old_tail } = + expect_action::(action1.as_ref()); + let SwaRebuild { + node_id, + source_value, + } = expect_action::(action2.as_ref()); + let FreeDeviceKVFullOnly { + indices: duplicates_tail, + } = expect_action::(action3.as_ref()); assert!(duplicates_head[0].equal(&Tensor::from_slice(&[20i64, 21, 22]))); assert!(old_tail[0].equal(&Tensor::from_slice(&[14i64]))); assert_eq!(*node_id, tc.arena.node(b).id); @@ -1473,21 +1472,23 @@ fn insert_overlap_straddling_a_second_level_locked_node_defers_the_tail() { .device_value(b, FULL) .equal(&Tensor::from_slice(&[14i64])) ); - let [ - CacheAction::FreeDeviceKVFullOnly(duplicates_head), - CacheAction::RecoverSwaWithLockedFull { - node_id, - kept_full, - incoming_full, - }, - CacheAction::FreeDeviceKVFullOnly(duplicates_tail), - ] = result.cache_actions.as_slice() - else { + let [action0, action1, action2] = result.cache_actions.as_slice() else { panic!( "expected FreeDeviceKVFullOnly, RecoverSwaWithLockedFull, FreeDeviceKVFullOnly, got {:?}", action_kinds(&result.cache_actions) ); }; + let FreeDeviceKVFullOnly { + indices: duplicates_head, + } = expect_action::(action0.as_ref()); + let RecoverSwaWithLockedFull { + node_id, + kept_full, + incoming_full, + } = expect_action::(action1.as_ref()); + let FreeDeviceKVFullOnly { + indices: duplicates_tail, + } = expect_action::(action2.as_ref()); assert!(duplicates_head[0].equal(&Tensor::from_slice(&[20i64, 21, 22]))); assert_eq!(*node_id, tc.arena.node(b).id); assert!(kept_full.equal(&Tensor::from_slice(&[14i64]))); @@ -1518,10 +1519,7 @@ fn insert_overlap_prev_prefix_strictly_inside_the_node_still_recovers() { .device_value(node, FULL) .equal(&Tensor::from_slice(&[20i64, 21, 22, 23])) ); - assert!(result.cache_actions.iter().any(|action| matches!( - action, - CacheAction::SwaRebuild { node_id, .. } if *node_id == tc.arena.node(node).id - ))); + assert!(result.cache_actions.iter().any(|action| matches!(action.as_any().downcast_ref::(), Some(SwaRebuild { node_id, .. }) if *node_id == tc.arena.node(node).id))); } #[test] @@ -1551,15 +1549,13 @@ fn reinsert_after_full_eviction_rebuilds_swa_from_the_fresh_kv() { .device_value(leaf, FULL) .equal(&Tensor::from_slice(&[20i64, 21, 22])) ); - let [ - CacheAction::SwaRebuild { - node_id, - source_value, - }, - ] = result.cache_actions.as_slice() - else { + let [action] = result.cache_actions.as_slice() else { panic!("expected one SwaRebuild action"); }; + let SwaRebuild { + node_id, + source_value, + } = expect_action::(action.as_ref()); assert_eq!(*node_id, tc.arena.node(leaf).id); assert!(source_value.equal(&Tensor::from_slice(&[20i64, 21, 22]))); } @@ -1592,15 +1588,13 @@ fn reinsert_straddling_the_boundary_splits_before_the_rebuild() { .equal(&Tensor::from_slice(&[20i64, 21])) ); assert!(!tc.arena.has_device_value(parent, SWA)); - let [ - CacheAction::SwaRebuild { - node_id, - source_value, - }, - ] = result.cache_actions.as_slice() - else { + let [action] = result.cache_actions.as_slice() else { panic!("expected one SwaRebuild action"); }; + let SwaRebuild { + node_id, + source_value, + } = expect_action::(action.as_ref()); assert_eq!(*node_id, tc.arena.node(node).id); assert!(source_value.equal(&Tensor::from_slice(&[22i64, 23]))); } @@ -1641,15 +1635,9 @@ fn reinsert_straddling_the_boundary_at_a_second_level_node_splits_before_the_reb .equal(&Tensor::from_slice(&[24i64])) ); assert!(!tc.arena.has_device_value(p, SWA)); - assert!(result.cache_actions.iter().any(|action| matches!( - action, - CacheAction::SwaRebuild { node_id, source_value } - if *node_id == tc.arena.node(b).id && source_value.equal(&Tensor::from_slice(&[24i64])) - ))); - assert!(!result.cache_actions.iter().any(|action| matches!( - action, - CacheAction::SwaRebuild { node_id, .. } if *node_id == tc.arena.node(p).id - ))); + assert!(result.cache_actions.iter().any(|action| matches!(action.as_any().downcast_ref::(), Some(SwaRebuild { node_id, source_value }) + if *node_id == tc.arena.node(b).id && source_value.equal(&Tensor::from_slice(&[24i64]))))); + assert!(!result.cache_actions.iter().any(|action| matches!(action.as_any().downcast_ref::(), Some(SwaRebuild { node_id, .. }) if *node_id == tc.arena.node(p).id))); } #[test] @@ -1715,19 +1703,19 @@ fn reinsert_boundary_at_the_node_start_rebuilds_the_whole_node() { /* swa_evicted_seqlen = */ 3, )); assert_eq!(tc.arena.node(b).key, vec![4, 5]); - let [ - CacheAction::FreeDeviceKVFullOnly(duplicates), - CacheAction::SwaRebuild { - node_id, - source_value, - }, - ] = result.cache_actions.as_slice() - else { + let [action0, action1] = result.cache_actions.as_slice() else { panic!( "expected FreeDeviceKVFullOnly then SwaRebuild, got {:?}", action_kinds(&result.cache_actions) ); }; + let FreeDeviceKVFullOnly { + indices: duplicates, + } = expect_action::(action0.as_ref()); + let SwaRebuild { + node_id, + source_value, + } = expect_action::(action1.as_ref()); assert!(duplicates[0].equal(&Tensor::from_slice(&[20i64, 21, 22]))); assert_eq!(*node_id, tc.arena.node(b).id); assert!(source_value.equal(&Tensor::from_slice(&[23i64, 24]))); @@ -1768,16 +1756,16 @@ fn walk_split_redistributes_the_live_swa_value() { ); assert!(tc.device_lru_list(SWA).in_list(Some(parent))); assert!(tc.device_lru_list(SWA).in_list(Some(node))); - let [ - CacheAction::FreeDeviceKV(duplicates), - CacheAction::SwaRebuild { - node_id, - source_value, - }, - ] = result.cache_actions.as_slice() - else { + let [action0, action1] = result.cache_actions.as_slice() else { panic!("expected FreeDeviceKV then SwaRebuild"); }; + let FreeDeviceKV { + indices: duplicates, + } = expect_action::(action0.as_ref()); + let SwaRebuild { + node_id, + source_value, + } = expect_action::(action1.as_ref()); assert!(duplicates[0].equal(&Tensor::from_slice(&[20i64, 21]))); assert_eq!(*node_id, tc.arena.node(leaf).id); assert!(source_value.equal(&Tensor::from_slice(&[29i64]))); @@ -3968,15 +3956,9 @@ fn new_leaf_after_a_cached_prefix_splits_at_the_leaf_relative_boundary() { .device_value(leaf, FULL) .equal(&Tensor::from_slice(&[24i64, 25])) ); - assert!(result.cache_actions.iter().any(|action| matches!( - action, - CacheAction::SwaRebuild { node_id, source_value } - if *node_id == tc.arena.node(leaf).id && source_value.equal(&Tensor::from_slice(&[24i64, 25])) - ))); - assert!(!result.cache_actions.iter().any(|action| matches!( - action, - CacheAction::SwaRebuild { node_id, .. } if *node_id == tc.arena.node(p).id - ))); + assert!(result.cache_actions.iter().any(|action| matches!(action.as_any().downcast_ref::(), Some(SwaRebuild { node_id, source_value }) + if *node_id == tc.arena.node(leaf).id && source_value.equal(&Tensor::from_slice(&[24i64, 25]))))); + assert!(!result.cache_actions.iter().any(|action| matches!(action.as_any().downcast_ref::(), Some(SwaRebuild { node_id, .. }) if *node_id == tc.arena.node(p).id))); } #[test] @@ -4006,41 +3988,42 @@ fn new_leaf_boundary_split_and_window_cap_compose_in_one_commit() { let rebuilds: Vec = result .cache_actions .iter() - .filter_map(|action| match action { - CacheAction::SwaRebuild { node_id, .. } => Some(*node_id), - _ => None, - }) + .filter_map( + |action| match action.as_any().downcast_ref::() { + Some(SwaRebuild { node_id, .. }) => Some(*node_id), + _ => None, + }, + ) .collect(); assert_eq!( rebuilds, vec![tc.arena.node(capped).id, tc.arena.node(leaf).id] ); - assert!(result.cache_actions.iter().any(|action| matches!( - action, - CacheAction::SwaRebuild { node_id, source_value } - if *node_id == tc.arena.node(capped).id && source_value.equal(&Tensor::from_slice(&[23i64, 24])) - ))); - assert!(result.cache_actions.iter().any(|action| matches!( - action, - CacheAction::SwaRebuild { node_id, source_value } - if *node_id == tc.arena.node(leaf).id && source_value.equal(&Tensor::from_slice(&[25i64, 26])) - ))); + assert!(result.cache_actions.iter().any(|action| matches!(action.as_any().downcast_ref::(), Some(SwaRebuild { node_id, source_value }) + if *node_id == tc.arena.node(capped).id && source_value.equal(&Tensor::from_slice(&[23i64, 24]))))); + assert!(result.cache_actions.iter().any(|action| matches!(action.as_any().downcast_ref::(), Some(SwaRebuild { node_id, source_value }) + if *node_id == tc.arena.node(leaf).id && source_value.equal(&Tensor::from_slice(&[25i64, 26]))))); let split_ids: Vec = result .cache_actions .iter() - .filter_map(|action| match action { - CacheAction::UpdatePendingBackupOnNodeSplit { - ack_id, - old_node_id, - new_node_id, - new_child_node_id, - } => { - assert_eq!(*ack_id, None); - assert_eq!(*old_node_id, tc.arena.node(leaf).id); - assert_eq!(*new_child_node_id, tc.arena.node(leaf).id); - Some(*new_node_id) + .filter_map(|action| { + match action + .as_any() + .downcast_ref::() + { + Some(UpdatePendingBackupOnNodeSplit { + ack_id, + old_node_id, + new_node_id, + new_child_node_id, + }) => { + assert_eq!(*ack_id, None); + assert_eq!(*old_node_id, tc.arena.node(leaf).id); + assert_eq!(*new_child_node_id, tc.arena.node(leaf).id); + Some(*new_node_id) + } + _ => None, } - _ => None, }) .collect(); if buffer_only { @@ -4051,13 +4034,23 @@ fn new_leaf_boundary_split_and_window_cap_compose_in_one_commit() { let first_rebuild = result .cache_actions .iter() - .position(|action| matches!(action, CacheAction::SwaRebuild { .. })) + .position(|action| { + matches!( + action.as_any().downcast_ref::(), + Some(SwaRebuild { .. }) + ) + }) .unwrap(); assert_eq!( result.cache_actions[..first_rebuild] .iter() .filter(|action| { - matches!(action, CacheAction::UpdatePendingBackupOnNodeSplit { .. }) + matches!( + action + .as_any() + .downcast_ref::(), + Some(UpdatePendingBackupOnNodeSplit { .. }) + ) }) .count(), 2, @@ -4199,11 +4192,8 @@ fn new_leaf_with_the_boundary_above_its_start_skips_the_split() { // in-window and stays unsplit. let leaf = child_of(&tc, a, &[4]); assert_eq!(tc.arena.node(leaf).key, vec![4, 5]); - assert!(result.cache_actions.iter().any(|action| matches!( - action, - CacheAction::SwaRebuild { node_id, source_value } - if *node_id == tc.arena.node(leaf).id && source_value.equal(&Tensor::from_slice(&[23i64, 24])) - ))); + assert!(result.cache_actions.iter().any(|action| matches!(action.as_any().downcast_ref::(), Some(SwaRebuild { node_id, source_value }) + if *node_id == tc.arena.node(leaf).id && source_value.equal(&Tensor::from_slice(&[23i64, 24]))))); } #[test] @@ -4237,20 +4227,21 @@ fn insert_overlap_straddling_with_a_partial_prev_prefix_recovers_the_tail() { .device_value(node, FULL) .equal(&Tensor::from_slice(&[22i64, 23])) ); - let [ - CacheAction::FreeDeviceKVFullOnly(old_tail), - CacheAction::SwaRebuild { - node_id, - source_value, - }, - CacheAction::FreeDeviceKVFullOnly(duplicates), - ] = result.cache_actions.as_slice() - else { + let [action0, action1, action2] = result.cache_actions.as_slice() else { panic!( "expected FreeDeviceKVFullOnly, SwaRebuild, FreeDeviceKVFullOnly, got {:?}", action_kinds(&result.cache_actions) ); }; + let FreeDeviceKVFullOnly { indices: old_tail } = + expect_action::(action0.as_ref()); + let SwaRebuild { + node_id, + source_value, + } = expect_action::(action1.as_ref()); + let FreeDeviceKVFullOnly { + indices: duplicates, + } = expect_action::(action2.as_ref()); assert!(old_tail[0].equal(&Tensor::from_slice(&[12i64, 13]))); assert_eq!(*node_id, tc.arena.node(node).id); assert!(source_value.equal(&Tensor::from_slice(&[22i64, 23]))); @@ -4284,11 +4275,8 @@ fn insert_overlap_straddling_recovers_across_page_two() { .device_value(node, FULL) .equal(&Tensor::from_slice(&[22i64, 23])) ); - assert!(result.cache_actions.iter().any(|action| matches!( - action, - CacheAction::SwaRebuild { node_id, source_value } - if *node_id == tc.arena.node(node).id && source_value.equal(&Tensor::from_slice(&[22i64, 23])) - ))); + assert!(result.cache_actions.iter().any(|action| matches!(action.as_any().downcast_ref::(), Some(SwaRebuild { node_id, source_value }) + if *node_id == tc.arena.node(node).id && source_value.equal(&Tensor::from_slice(&[22i64, 23]))))); } #[test] @@ -4322,15 +4310,9 @@ fn reinsert_straddling_recovers_across_page_two() { .device_value(node, FULL) .equal(&Tensor::from_slice(&[22i64, 23])) ); - assert!(result.cache_actions.iter().any(|action| matches!( - action, - CacheAction::SwaRebuild { node_id, source_value } - if *node_id == tc.arena.node(node).id && source_value.equal(&Tensor::from_slice(&[22i64, 23])) - ))); - assert!(!result.cache_actions.iter().any(|action| matches!( - action, - CacheAction::SwaRebuild { node_id, .. } if *node_id == tc.arena.node(parent).id - ))); + assert!(result.cache_actions.iter().any(|action| matches!(action.as_any().downcast_ref::(), Some(SwaRebuild { node_id, source_value }) + if *node_id == tc.arena.node(node).id && source_value.equal(&Tensor::from_slice(&[22i64, 23]))))); + assert!(!result.cache_actions.iter().any(|action| matches!(action.as_any().downcast_ref::(), Some(SwaRebuild { node_id, .. }) if *node_id == tc.arena.node(parent).id))); } #[test] @@ -4362,12 +4344,10 @@ fn reinsert_rebuilds_a_locked_tombstone() { .node_mut(node) .set_lock_ref_(ValueSlotIdx::device(SWA), 1); let result = tc.insert(&insert_params_swa(&vec![1, 2], &[20, 21], 0, 0)); - assert!( - result - .cache_actions - .iter() - .any(|action| matches!(action, CacheAction::SwaRebuild { .. })) - ); + assert!(result.cache_actions.iter().any(|action| matches!( + action.as_any().downcast_ref::(), + Some(SwaRebuild { .. }) + ))); } fn set_full_host(tc: &mut UnifiedTreeCore>, node: NodeIdx_) { @@ -4955,13 +4935,10 @@ fn load_back_commit_attaches_chunks_and_emits_the_rebuild_action() { assert!(tc.device_lru_list(SWA).in_list(Some(b))); assert_eq!(tc.swa_evictable_size(), 2); assert_eq!(cache_actions.len(), 1); - let CacheAction::RebuildFullToSwaMapping { + let RebuildFullToSwaMapping { full_indices, swa_indices, - } = &cache_actions[0] - else { - panic!("expected a RebuildFullToSwaMapping action"); - }; + } = expect_action::(cache_actions[0].as_ref()); assert_eq!(full_indices.len(), 2); assert!(full_indices[0].equal(&Tensor::from_slice(&[40i64]))); assert!(full_indices[1].equal(&Tensor::from_slice(&[41i64]))); @@ -5196,9 +5173,8 @@ fn prefetch_commit_drops_the_whole_window_when_underloaded() { ); assert!(!tc.arena.node(a).has_host_value(SWA)); assert_eq!(cache_actions.len(), 1); - let CacheAction::FreeComponentHostSlot { host_indices, .. } = &cache_actions[0] else { - panic!("expected a host free"); - }; + let FreeComponentHostSlot { host_indices, .. } = + expect_action::(cache_actions[0].as_ref()); assert!(host_indices[0].equal(&Tensor::from_slice(&[30i64, 31]))); } @@ -5227,9 +5203,8 @@ fn prefetch_commit_without_a_target_releases_the_whole_buffer() { Some(&storage_result), ); assert_eq!(cache_actions.len(), 1); - let CacheAction::FreeComponentHostSlot { host_indices, .. } = &cache_actions[0] else { - panic!("expected a host free"); - }; + let FreeComponentHostSlot { host_indices, .. } = + expect_action::(cache_actions[0].as_ref()); assert!(host_indices[0].equal(&Tensor::from_slice(&[30i64, 31]))); // An insert result without an inserted host node releases the same way. @@ -5253,9 +5228,8 @@ fn prefetch_commit_without_a_target_releases_the_whole_buffer() { Some(&storage_result), ); assert_eq!(cache_actions.len(), 1); - let CacheAction::FreeComponentHostSlot { host_indices, .. } = &cache_actions[0] else { - panic!("expected a host free"); - }; + let FreeComponentHostSlot { host_indices, .. } = + expect_action::(cache_actions[0].as_ref()); assert!(host_indices[0].equal(&Tensor::from_slice(&[30i64, 31]))); assert!(!tc.arena.node(a).has_host_value(SWA)); } @@ -5294,9 +5268,8 @@ fn prefetch_commit_releases_a_shortened_window_under_a_non_root_anchor() { assert!(!tc.arena.node(a).has_host_value(SWA)); assert!(!tc.host_lru_list(SWA).in_list(Some(c))); assert_eq!(cache_actions.len(), 1); - let CacheAction::FreeComponentHostSlot { host_indices, .. } = &cache_actions[0] else { - panic!("expected a host free"); - }; + let FreeComponentHostSlot { host_indices, .. } = + expect_action::(cache_actions[0].as_ref()); assert!(host_indices[0].equal(&Tensor::from_slice(&[30i64, 31]))); } @@ -5346,9 +5319,8 @@ fn prefetch_commit_fills_tombstoned_nodes_and_releases_covered_ones() { .equal(&Tensor::from_slice(&[40i64])) ); assert_eq!(cache_actions.len(), 1); - let CacheAction::FreeComponentHostSlot { host_indices, .. } = &cache_actions[0] else { - panic!("expected a host free"); - }; + let FreeComponentHostSlot { host_indices, .. } = + expect_action::(cache_actions[0].as_ref()); assert!(host_indices[0].equal(&Tensor::from_slice(&[31i64]))); } @@ -5405,13 +5377,10 @@ fn release_swa_host_queues_a_free_action_for_non_empty_indices() { let mut cache_actions = Vec::new(); swa_component(4).release_swa_host_(Tensor::from_slice(&[7i64]), &mut cache_actions); assert_eq!(cache_actions.len(), 1); - let CacheAction::FreeComponentHostSlot { + let FreeComponentHostSlot { component_type, host_indices, - } = &cache_actions[0] - else { - panic!("expected a FreeComponentHostSlot action"); - }; + } = expect_action::(cache_actions[0].as_ref()); assert_eq!(*component_type, SWA); assert_eq!(host_indices.len(), 1); assert!(host_indices[0].equal(&Tensor::from_slice(&[7i64]))); @@ -5499,8 +5468,10 @@ fn build_load_back_spec_includes_the_swa_transfers() { ); assert_eq!(actions.len(), 1); assert!(matches!( - actions[0], - CacheAction::RebuildFullToSwaMapping { .. } + (actions[0]) + .as_any() + .downcast_ref::(), + Some(RebuildFullToSwaMapping { .. }) )); } @@ -5642,16 +5613,14 @@ fn check_swa_device_restored_during_full_load_back(pin_ancestor: bool) { // A is outside the loaded SWA window. A new request emits the normal // locked-Full repair action; resolve only its allocator/mapping boundary. let inserted = tc.insert(&insert_params_swa(&vec![1], &[90], 0, 0)); - let [ - CacheAction::RecoverSwaWithLockedFull { - node_id, - kept_full, - incoming_full, - }, - ] = inserted.cache_actions.as_slice() - else { + let [action] = inserted.cache_actions.as_slice() else { panic!("expected one RecoverSwaWithLockedFull action"); }; + let RecoverSwaWithLockedFull { + node_id, + kept_full, + incoming_full, + } = expect_action::(action.as_ref()); assert_eq!(*node_id, a_id); assert!(kept_full.equal(&Tensor::from_slice(&[50i64]))); assert!(incoming_full.equal(&Tensor::from_slice(&[90i64]))); @@ -6009,9 +5978,11 @@ fn write_through_offloads_a_boundary_split_leaf() { let backups: Vec<_> = result .cache_actions .iter() - .filter_map(|action| match action { - CacheAction::BackupKV(backup) => Some(backup.node_ids.clone()), - _ => None, + .filter_map(|action| { + action + .as_any() + .downcast_ref::() + .map(|backup| backup.node_ids.clone()) }) .collect(); assert_eq!( @@ -6083,10 +6054,10 @@ fn deep_request_ring_tree_survives_full_backup_evict_and_load_back_rounds() { let result = tc.insert(&insert_params_swa(&key, &kv, 0, swa_evicted_seqlen)); // Apply the emitted rebuilds the way the cache would. for action in &result.cache_actions { - if let CacheAction::SwaRebuild { + if let Some(SwaRebuild { node_id, source_value, - } = action + }) = action.as_any().downcast_ref::() { tc.set_component_device_value(*node_id, SWA, source_value.copy()) .expect("live test node"); @@ -6198,18 +6169,16 @@ fn recovered_swa_span_evicts_before_the_window_leaf() { let prefix = child_of(&tc, root, &[1]); let leaf = child_of(&tc, prefix, &[9]); assert_eq!(tc.arena.node(prefix).key, vec![1, 2, 3, 4, 5, 6, 7, 8]); - let [ - CacheAction::SwaRebuild { - node_id, - source_value, - }, - ] = result.cache_actions.as_slice() - else { + let [action] = result.cache_actions.as_slice() else { panic!( "expected one SwaRebuild action, got {:?}", action_kinds(&result.cache_actions) ); }; + let SwaRebuild { + node_id, + source_value, + } = expect_action::(action.as_ref()); assert_eq!(*node_id, tc.arena.node(leaf).id); tc.set_component_device_value(*node_id, SWA, source_value.copy()) .expect("live test node"); @@ -6225,19 +6194,18 @@ fn recovered_swa_span_evicts_before_the_window_leaf() { /* swa_evicted_seqlen = */ 0, )); assert!(step.result.is_none()); - let [ - CacheAction::FreeDeviceKVFullOnly(old_full), - CacheAction::SwaRebuild { - node_id, - source_value, - }, - ] = step.actions.as_slice() - else { + let [action0, action1] = step.actions.as_slice() else { panic!( "expected FreeDeviceKVFullOnly then SwaRebuild, got {:?}", action_kinds(&step.actions) ); }; + let FreeDeviceKVFullOnly { indices: old_full } = + expect_action::(action0.as_ref()); + let SwaRebuild { + node_id, + source_value, + } = expect_action::(action1.as_ref()); assert!(old_full[0].equal(&Tensor::from_slice(&[ 100i64, 101, 102, 103, 104, 105, 106, 107 ]))); @@ -6613,7 +6581,7 @@ fn write_back_reinsert_defers_swa_backup_while_write_through_publishes() { !result .cache_actions .iter() - .any(|action| matches!(action, CacheAction::BackupKV(_))) + .any(|action| action.as_any().is::()) ); tc.is_write_back = false; @@ -6621,9 +6589,11 @@ fn write_back_reinsert_defers_swa_backup_while_write_through_publishes() { let backups: Vec<_> = result .cache_actions .iter() - .filter_map(|action| match action { - CacheAction::BackupKV(backup) => Some(backup.node_ids.clone()), - _ => None, + .filter_map(|action| { + action + .as_any() + .downcast_ref::() + .map(|backup| backup.node_ids.clone()) }) .collect(); assert_eq!(backups, vec![vec![leaf]]); diff --git a/rust/sglang-radix-tree/src/tests/test_utils.rs b/rust/sglang-radix-tree/src/tests/test_utils.rs index b62afb65f3..3e269c0436 100644 --- a/rust/sglang-radix-tree/src/tests/test_utils.rs +++ b/rust/sglang-radix-tree/src/tests/test_utils.rs @@ -35,21 +35,14 @@ pub(crate) fn accumulate_step( } } -/// Short variant names for diagnosing an action sequence's shape. -pub(crate) fn action_kinds(actions: &[CacheAction]) -> Vec<&'static str> { - actions - .iter() - .map(|action| match action { - CacheAction::FreeDeviceKV(_) => "FreeDeviceKV", - CacheAction::FreeDeviceKVFullOnly(_) => "FreeDeviceKVFullOnly", - CacheAction::BackupKV(_) => "BackupKV", - CacheAction::UpdatePendingBackupOnNodeSplit { .. } => "UpdatePendingBackupOnNodeSplit", - CacheAction::MambaEvictExcessPathStates { .. } => "MambaEvictExcessPathStates", - CacheAction::FreeComponentDeviceSlot { .. } => "FreeComponentDeviceSlot", - CacheAction::FreeComponentHostSlot { .. } => "FreeComponentHostSlot", - CacheAction::RebuildFullToSwaMapping { .. } => "RebuildFullToSwaMapping", - CacheAction::RecoverSwaWithLockedFull { .. } => "RecoverSwaWithLockedFull", - CacheAction::SwaRebuild { .. } => "SwaRebuild", - }) - .collect() +/// Tags for diagnosing an action sequence's shape. +pub(crate) fn action_kinds(actions: &[Box]) -> Vec<&'static str> { + actions.iter().map(|action| action.tag()).collect() +} + +pub(crate) fn expect_action(action: &dyn CacheAction) -> &T { + action + .as_any() + .downcast_ref::() + .unwrap_or_else(|| panic!("expected {}, got {:?}", std::any::type_name::(), action)) } diff --git a/rust/sglang-radix-tree/src/tests/unified_tree_core.rs b/rust/sglang-radix-tree/src/tests/unified_tree_core.rs index f451373b09..840c3f5e06 100644 --- a/rust/sglang-radix-tree/src/tests/unified_tree_core.rs +++ b/rust/sglang-radix-tree/src/tests/unified_tree_core.rs @@ -1,3 +1,6 @@ +use crate::cache_action::{ + BackupKV, FreeDeviceKV, RecoverSwaWithLockedFull, UpdatePendingBackupOnNodeSplit, +}; use std::sync::Mutex; use tch::Tensor; @@ -5,7 +8,7 @@ use tch::Tensor; use super::*; use crate::components::{ComponentSet, FULL, MAMBA, SWA}; use crate::node::{NodeAccessError, ValueSlotIdx}; -use crate::test_utils::{accumulate_step, action_kinds, matched_device_indices}; +use crate::test_utils::{accumulate_step, action_kinds, expect_action, matched_device_indices}; fn core() -> UnifiedTreeCore> { UnifiedTreeCore::new(CacheInitParams::default(), vec![FULL]) @@ -314,7 +317,7 @@ impl TreeComponent> for RecordingComponentForTest { _node_id: NodeIdx_, _device_frees: &mut HashMap>, _host_frees: &mut HashMap>, - _cache_actions: &mut Vec, + _cache_actions: &mut CacheActions, _target: EvictLayer, ) -> (usize, usize) { unimplemented!() @@ -330,7 +333,7 @@ impl TreeComponent> for RecordingComponentForTest { _tracker: &mut HashMap, _device_frees: &mut HashMap>, _host_frees: &mut HashMap>, - _cache_actions: &mut Vec, + _cache_actions: &mut CacheActions, ) -> Option { unimplemented!() } @@ -366,7 +369,7 @@ impl TreeComponent> for RecordingComponentForTest { tracker: &mut HashMap, _device_frees: &mut HashMap>, _host_frees: &mut HashMap>, - _cache_actions: &mut Vec, + _cache_actions: &mut CacheActions, ) { let tracked = tracker[&SWA]; self.host_eviction_calls @@ -383,7 +386,7 @@ impl TreeComponent> for RecordingComponentForTest { tracker: &mut HashMap, _device_frees: &mut HashMap>, _host_frees: &mut HashMap>, - _cache_actions: &mut Vec, + _cache_actions: &mut CacheActions, ) { let tracked = tracker[&SWA]; self.host_eviction_calls @@ -440,7 +443,7 @@ impl TreeComponent> for CountingComponentForTest { _node_id: NodeIdx_, _device_frees: &mut HashMap>, _host_frees: &mut HashMap>, - _cache_actions: &mut Vec, + _cache_actions: &mut CacheActions, _target: EvictLayer, ) -> (usize, usize) { unimplemented!() @@ -456,7 +459,7 @@ impl TreeComponent> for CountingComponentForTest { _tracker: &mut HashMap, _device_frees: &mut HashMap>, _host_frees: &mut HashMap>, - _cache_actions: &mut Vec, + _cache_actions: &mut CacheActions, ) -> Option { unimplemented!() } @@ -522,7 +525,7 @@ impl TreeComponent> for LowPriorityComponentForTest { _node_id: NodeIdx_, _device_frees: &mut HashMap>, _host_frees: &mut HashMap>, - _cache_actions: &mut Vec, + _cache_actions: &mut CacheActions, _target: EvictLayer, ) -> (usize, usize) { unimplemented!() @@ -538,7 +541,7 @@ impl TreeComponent> for LowPriorityComponentForTest { _tracker: &mut HashMap, _device_frees: &mut HashMap>, _host_frees: &mut HashMap>, - _cache_actions: &mut Vec, + _cache_actions: &mut CacheActions, ) -> Option { unimplemented!() } @@ -599,7 +602,7 @@ impl TreeComponent> for SwaComponentForTest { _node_id: NodeIdx_, _device_frees: &mut HashMap>, _host_frees: &mut HashMap>, - _cache_actions: &mut Vec, + _cache_actions: &mut CacheActions, _target: EvictLayer, ) -> (usize, usize) { unimplemented!() @@ -615,7 +618,7 @@ impl TreeComponent> for SwaComponentForTest { _tracker: &mut HashMap, _device_frees: &mut HashMap>, _host_frees: &mut HashMap>, - _cache_actions: &mut Vec, + _cache_actions: &mut CacheActions, ) -> Option { unimplemented!() } @@ -682,7 +685,7 @@ impl TreeComponent> for SwaEvictionComponentForTest { node_id: NodeIdx_, device_frees: &mut HashMap>, host_frees: &mut HashMap>, - _cache_actions: &mut Vec, + _cache_actions: &mut CacheActions, target: EvictLayer, ) -> (usize, usize) { self.evictions.lock().unwrap().push((node_id, target)); @@ -730,7 +733,7 @@ impl TreeComponent> for SwaEvictionComponentForTest { _tracker: &mut HashMap, _device_frees: &mut HashMap>, _host_frees: &mut HashMap>, - _cache_actions: &mut Vec, + _cache_actions: &mut CacheActions, ) -> Option { unimplemented!() } @@ -2444,12 +2447,13 @@ fn insert_full_overlap_frees_the_duplicates() { let result = tc.insert(&insert_params(&vec![1, 2, 3], &[20, 21, 22])); assert_eq!(result.prefix_len, 3); // Nothing was consumed by any component: the whole overlap is duplicate. - let [CacheAction::FreeDeviceKV(freed)] = result.cache_actions.as_slice() else { + let [action] = result.cache_actions.as_slice() else { panic!( "expected one FreeDeviceKV action, got {:?}", action_kinds(&result.cache_actions) ); }; + let FreeDeviceKV { indices: freed } = expect_action::(action.as_ref()); assert_eq!(freed.len(), 1); assert!(freed[0].equal(&Tensor::from_slice(&[20i64, 21, 22]))); } @@ -2463,12 +2467,13 @@ fn insert_prev_prefix_len_narrows_the_dup_window() { prev_prefix_len: 2, ..insert_params(&vec![1, 2, 3], &[20, 21, 22]) }); - let [CacheAction::FreeDeviceKV(freed)] = result.cache_actions.as_slice() else { + let [action] = result.cache_actions.as_slice() else { panic!( "expected one FreeDeviceKV action, got {:?}", action_kinds(&result.cache_actions) ); }; + let FreeDeviceKV { indices: freed } = expect_action::(action.as_ref()); assert!(freed[0].equal(&Tensor::from_slice(&[22i64]))); } @@ -2479,12 +2484,13 @@ fn insert_extends_an_existing_prefix() { let result = tc.insert(&insert_params(&vec![1, 2, 3, 4, 5], &[20, 21, 22, 13, 14])); assert_eq!(result.prefix_len, 3); // The existing node's overlap is duplicate; the new suffix is kept. - let [CacheAction::FreeDeviceKV(freed)] = result.cache_actions.as_slice() else { + let [action] = result.cache_actions.as_slice() else { panic!( "expected one FreeDeviceKV action, got {:?}", action_kinds(&result.cache_actions) ); }; + let FreeDeviceKV { indices: freed } = expect_action::(action.as_ref()); assert_eq!(freed.len(), 1); assert!(freed[0].equal(&Tensor::from_slice(&[20i64, 21, 22]))); let matched = tc.match_prefix(&match_params(&vec![1, 2, 3, 4, 5])); @@ -2506,12 +2512,13 @@ fn insert_prev_prefix_len_spans_a_multi_node_walk() { ..insert_params(&vec![1, 2, 3, 4, 5], &[30, 31, 32, 33, 34]) }); assert_eq!(result.prefix_len, 5); - let [CacheAction::FreeDeviceKV(freed)] = result.cache_actions.as_slice() else { + let [action] = result.cache_actions.as_slice() else { panic!( "expected one FreeDeviceKV action, got {:?}", action_kinds(&result.cache_actions) ); }; + let FreeDeviceKV { indices: freed } = expect_action::(action.as_ref()); assert_eq!(freed.len(), 1); assert!(freed[0].equal(&Tensor::from_slice(&[33i64, 34]))); } @@ -2528,12 +2535,13 @@ fn insert_prev_prefix_len_narrows_mid_node_on_a_multi_node_walk() { ..insert_params(&vec![1, 2, 3, 4, 5], &[30, 31, 32, 33, 34]) }); assert_eq!(result.prefix_len, 5); - let [CacheAction::FreeDeviceKV(freed)] = result.cache_actions.as_slice() else { + let [action] = result.cache_actions.as_slice() else { panic!( "expected one FreeDeviceKV action, got {:?}", action_kinds(&result.cache_actions) ); }; + let FreeDeviceKV { indices: freed } = expect_action::(action.as_ref()); assert_eq!(freed.len(), 1); assert!(freed[0].equal(&Tensor::from_slice(&[34i64]))); } @@ -2680,9 +2688,11 @@ fn insert_threshold_crossing_emits_the_backup_kv_action() { let backups: Vec<_> = result .cache_actions .iter() - .filter_map(|action| match action { - CacheAction::BackupKV(backup) => Some(backup.node_ids.clone()), - _ => None, + .filter_map(|action| { + action + .as_any() + .downcast_ref::() + .map(|backup| backup.node_ids.clone()) }) .collect(); assert_eq!(backups, vec![vec![leaf]]); @@ -2701,7 +2711,7 @@ fn external_linker_hashes_new_nodes_and_triggers_offload_action() { let result = tc.insert(&insert_params(&vec![1, 2], &[10, 11])); let leaf = result.last_device_node_id.unwrap(); assert!(result.cache_actions.iter().any(|action| { - matches!(action, CacheAction::BackupKV(backup) if backup.node_ids == vec![leaf]) + matches!(action.as_any().downcast_ref::(), Some(backup) if backup.node_ids == vec![leaf]) })); let transfers = tc @@ -3090,10 +3100,7 @@ fn write_back_defers_swa_window_backup_until_eviction() { let backups: Vec<&BackupKV> = result .cache_actions .iter() - .filter_map(|action| match action { - CacheAction::BackupKV(backup) => Some(backup), - _ => None, - }) + .filter_map(|action| action.as_any().downcast_ref::()) .collect(); assert!(backups.is_empty()); } @@ -3168,16 +3175,15 @@ fn split_updates_pending_backup_with_or_without_a_tree_ack() { let prefix_id = result.best_match_node_id; assert_ne!(prefix_id, tail_id); if pending || buffer_only { - assert!(matches!( - result.cache_actions.as_slice(), - [CacheAction::UpdatePendingBackupOnNodeSplit { + assert!( + matches!(result.cache_actions.as_slice(), [action] if matches!(action.as_any().downcast_ref::(), Some(UpdatePendingBackupOnNodeSplit { ack_id, old_node_id, new_node_id, new_child_node_id, - }] if *ack_id == pending.then_some(tail_id) && *old_node_id == tail_id - && *new_node_id == prefix_id && *new_child_node_id == tail_id - )); + }) if *ack_id == pending.then_some(tail_id) && *old_node_id == tail_id + && *new_node_id == prefix_id && *new_child_node_id == tail_id)) + ); } else { assert!(result.cache_actions.is_empty()); } @@ -4063,16 +4069,21 @@ fn finish_write_through_after_a_split_publishes_both_fragments() { let new_node_id = result .cache_actions .iter() - .find_map(|action| match action { - CacheAction::UpdatePendingBackupOnNodeSplit { - ack_id, - new_node_id, - .. - } => { - assert_eq!(*ack_id, Some(leaf)); - Some(*new_node_id) + .find_map(|action| { + match action + .as_any() + .downcast_ref::() + { + Some(UpdatePendingBackupOnNodeSplit { + ack_id, + new_node_id, + .. + }) => { + assert_eq!(*ack_id, Some(leaf)); + Some(*new_node_id) + } + _ => None, } - _ => None, }) .expect("the split relocates the pending write-through"); // Nothing reaches the host tier before the ack. @@ -4531,10 +4542,9 @@ fn insert_host_refill_preserves_split_actions_and_lengths() { assert_eq!(result.total_len, 2); assert!(result.inserted_host_node.is_some()); assert!(!result.host_insert_dropped); - assert!(matches!( - result.cache_actions.as_slice(), - [CacheAction::UpdatePendingBackupOnNodeSplit { ack_id, .. }] if *ack_id == Some(leaf) - )); + assert!( + matches!(result.cache_actions.as_slice(), [action] if matches!(action.as_any().downcast_ref::(), Some(UpdatePendingBackupOnNodeSplit { ack_id, .. }) if *ack_id == Some(leaf))) + ); } #[test] @@ -5614,12 +5624,13 @@ fn insert_page_size_two_splits_mid_page_divergence_at_the_page_boundary() { // The keys share 3 atoms but pages quantize the overlap down to 2. let result = tc.insert(&insert_params(&vec![1, 2, 3, 9], &[20, 21, 22, 29])); assert_eq!(result.prefix_len, 2); - let [CacheAction::FreeDeviceKV(freed)] = result.cache_actions.as_slice() else { + let [action] = result.cache_actions.as_slice() else { panic!( "expected one FreeDeviceKV action, got {:?}", action_kinds(&result.cache_actions) ); }; + let FreeDeviceKV { indices: freed } = expect_action::(action.as_ref()); assert_eq!(freed.len(), 1); assert!(freed[0].equal(&Tensor::from_slice(&[20i64, 21]))); let prefix = tc @@ -8641,12 +8652,13 @@ fn insert_unevicts_a_tombstoned_deep_node() { .device_value(tc.arena.resolve(leaf).expect("live test node"), FULL) .equal(&Tensor::from_slice(&[33i64, 34])) ); - let [CacheAction::FreeDeviceKV(freed)] = result.cache_actions.as_slice() else { + let [action] = result.cache_actions.as_slice() else { panic!( "expected one FreeDeviceKV action, got {:?}", action_kinds(&result.cache_actions) ); }; + let FreeDeviceKV { indices: freed } = expect_action::(action.as_ref()); assert!(freed[0].equal(&Tensor::from_slice(&[30i64, 31, 32]))); } @@ -8658,12 +8670,13 @@ fn insert_ragged_key_onto_an_existing_prefix_page_size_two() { assert_eq!(result.prefix_len, 2); // The ragged tail never enters: no new leaf, the aligned span is duplicate. assert_eq!(tc.arena.len(), 2); - let [CacheAction::FreeDeviceKV(freed)] = result.cache_actions.as_slice() else { + let [action] = result.cache_actions.as_slice() else { panic!( "expected one FreeDeviceKV action, got {:?}", action_kinds(&result.cache_actions) ); }; + let FreeDeviceKV { indices: freed } = expect_action::(action.as_ref()); assert!(freed[0].equal(&Tensor::from_slice(&[20i64, 21]))); } @@ -8675,16 +8688,18 @@ fn insert_ragged_key_traverses_a_node_before_the_tail_page_size_two() { let result = tc.insert(&insert_params(&vec![1, 2, 3, 4, 5], &[30, 31, 32, 33, 34])); assert_eq!(result.prefix_len, 4); assert_eq!(tc.arena.len(), 3); - let [ - CacheAction::FreeDeviceKV(freed_head), - CacheAction::FreeDeviceKV(freed_tail), - ] = result.cache_actions.as_slice() - else { + let [action0, action1] = result.cache_actions.as_slice() else { panic!( "expected one FreeDeviceKV per walked node, got {:?}", action_kinds(&result.cache_actions) ); }; + let FreeDeviceKV { + indices: freed_head, + } = expect_action::(action0.as_ref()); + let FreeDeviceKV { + indices: freed_tail, + } = expect_action::(action1.as_ref()); assert!(freed_head[0].equal(&Tensor::from_slice(&[30i64, 31]))); assert!(freed_tail[0].equal(&Tensor::from_slice(&[32i64, 33]))); } @@ -8714,7 +8729,7 @@ fn deferrable_dup_frees_ride_the_final_step_without_suspension() { let mut tc = core(); tc.insert(&insert_params(&vec![1, 2, 3], &[10, 11, 12])); let step = tc.begin_insert(&insert_params(&vec![1, 2, 3, 4], &[20, 21, 22, 13])); - assert_eq!(action_kinds(&step.actions), vec!["FreeDeviceKV"]); + assert_eq!(action_kinds(&step.actions), vec!["free_device_kv"]); let result = step .result .expect("a deferrable-only walk completes in one step"); @@ -8749,11 +8764,9 @@ fn walk_backup_crossing_suspends_then_resume_completes() { assert!(tc.has_ongoing_insert()); assert_eq!( action_kinds(&step.actions), - vec!["FreeDeviceKV", "BackupKV"] + vec!["free_device_kv", "backup_kv"] ); - let CacheAction::BackupKV(backup) = &step.actions[1] else { - unreachable!(); - }; + let backup = expect_action::(step.actions[1].as_ref()); assert_eq!(backup.node_ids, vec![tc.arena.node(a).id]); let done = tc.resume_insert(); @@ -8841,7 +8854,7 @@ fn resume_insert_completes_after_an_on_path_host_leaf_is_evicted() { assert!(step.result.is_none()); assert_eq!( action_kinds(&step.actions), - vec!["FreeDeviceKV", "BackupKV"] + vec!["free_device_kv", "backup_kv"] ); // The barrier's backup host-evicts the on-path H-leaf before committing. let mut tracker = HashMap::from([(FULL, 0)]); @@ -8889,9 +8902,11 @@ fn aborted_barrier_crossing_refires_on_the_next_insert() { let backups: Vec<_> = step .actions .iter() - .filter_map(|action| match action { - CacheAction::BackupKV(backup) => Some(backup.node_ids.clone()), - _ => None, + .filter_map(|action| { + action + .as_any() + .downcast_ref::() + .map(|backup| backup.node_ids.clone()) }) .collect(); assert_eq!(backups, vec![vec![tc.arena.node(a).id]]); @@ -8959,9 +8974,11 @@ fn one_insert_walk_fires_two_crossings_around_a_backuped_middle() { let backups: Vec<_> = result .cache_actions .iter() - .filter_map(|action| match action { - CacheAction::BackupKV(backup) => Some(backup.node_ids.clone()), - _ => None, + .filter_map(|action| { + action + .as_any() + .downcast_ref::() + .map(|backup| backup.node_ids.clone()) }) .collect(); assert_eq!(backups, vec![vec![top], vec![deep]]); @@ -9518,17 +9535,17 @@ fn caller_owned_overlap_drains_split_and_backup_before_duplicate_frees() { ); assert_eq!( action_kinds(&walk.pending_actions), - vec!["UpdatePendingBackupOnNodeSplit"] + vec!["update_pending_backup_on_node_split"] ); assert!(!walk.has_barrier_actions_since(0)); - walk.pending_actions.push(CacheAction::BackupKV(BackupKV { + walk.pending_actions.push(Box::new(BackupKV { node_ids: vec![node.id], })); assert!(walk.has_barrier_actions_since(0)); let actions = std::mem::take(&mut walk.pending_actions); assert_eq!( action_kinds(&actions), - vec!["UpdatePendingBackupOnNodeSplit", "BackupKV"] + vec!["update_pending_backup_on_node_split", "backup_kv"] ); assert_eq!(tc.arena.node(head).parent(), tc.arena.root()); assert_eq!( @@ -9550,10 +9567,8 @@ fn caller_owned_overlap_drains_split_and_backup_before_duplicate_frees() { )); let step = walk.finish(); assert_eq!(step.result.unwrap().prefix_len, 2); - assert_eq!(action_kinds(&step.actions), vec!["FreeDeviceKV"]); - let CacheAction::FreeDeviceKV(frees) = &step.actions[0] else { - unreachable!() - }; + assert_eq!(action_kinds(&step.actions), vec!["free_device_kv"]); + let FreeDeviceKV { indices: frees } = expect_action::(step.actions[0].as_ref()); assert!(frees[0].equal(&Tensor::from_slice(&[20i64, 21]))); assert!(!tc.has_ongoing_insert()); tc.sanity_check(&[], &[]); @@ -9577,7 +9592,7 @@ fn caller_owned_walk_can_abort_after_split_without_claiming_donor() { drop(walk); assert_eq!( action_kinds(&actions), - vec!["UpdatePendingBackupOnNodeSplit"] + vec!["update_pending_backup_on_node_split"] ); assert!(!tc.has_ongoing_insert()); assert!(tc.end_insert().is_empty()); @@ -9590,7 +9605,7 @@ fn caller_owned_walk_can_abort_after_split_without_claiming_donor() { assert!(matched_device_indices(&tc, &matched).equal(&Tensor::from_slice(&[10i64, 11, 12, 13]))); let result = tc.insert(¶ms); assert_eq!(result.prefix_len, 2); - assert_eq!(action_kinds(&result.cache_actions), vec!["FreeDeviceKV"]); + assert_eq!(action_kinds(&result.cache_actions), vec!["free_device_kv"]); tc.sanity_check(&[], &[]); } @@ -9675,15 +9690,12 @@ fn overlap_context_reaches_swa_before_it_replaces_unlocked_full_ids() { ); assert!(walk.has_barrier_actions_since(start)); let actions = std::mem::take(&mut walk.pending_actions); - assert_eq!(action_kinds(&actions), vec!["RecoverSwaWithLockedFull"]); - let CacheAction::RecoverSwaWithLockedFull { + assert_eq!(action_kinds(&actions), vec!["recover_swa_with_locked_full"]); + let RecoverSwaWithLockedFull { node_id: actual_node_id, kept_full, incoming_full, - } = &actions[0] - else { - unreachable!() - }; + } = expect_action::(actions[0].as_ref()); assert_eq!(*actual_node_id, node_id); assert!(kept_full.equal(&kept)); assert!(incoming_full.equal(&Tensor::from_slice(&[20i64, 21]))); diff --git a/rust/sglang-radix-tree/src/unified_tree_core.rs b/rust/sglang-radix-tree/src/unified_tree_core.rs index 4fbeea8390..47b5dcb660 100644 --- a/rust/sglang-radix-tree/src/unified_tree_core.rs +++ b/rust/sglang-radix-tree/src/unified_tree_core.rs @@ -1,6 +1,8 @@ //! The radix prefix tree of cached KV. #![allow(unused_variables)] +pub use crate::cache_action::{BackupKV, CacheAction, CacheActions}; +use crate::cache_action::{FreeDeviceKV, FreeDeviceKVFullOnly, UpdatePendingBackupOnNodeSplit}; use std::cmp::Reverse; use std::collections::{BinaryHeap, HashMap, HashSet}; use std::sync::Arc; @@ -160,7 +162,7 @@ pub struct MatchResult { /// of other components. pub full_kv_hit_length: usize, /// Actions for the controller to apply. - pub cache_actions: Vec, + pub cache_actions: CacheActions, } /// Params for a prefix match; the key is borrowed from the caller. @@ -229,7 +231,7 @@ pub struct InsertResult { /// Incoming ranges retained by each component, in key-relative atom offsets. pub adopted_ranges: Option>>, /// Actions for the controller to apply. - pub cache_actions: Vec, + pub cache_actions: CacheActions, } impl InsertResult { @@ -260,7 +262,7 @@ impl InsertResult { /// One step of a resumable insert: the Controller executes `actions`, then /// resumes while `result` is None; `result` is set on the final step. pub struct InsertStepResult { - pub actions: Vec, + pub actions: CacheActions, pub result: Option, } @@ -305,7 +307,7 @@ pub(crate) struct InsertWalkState { target_node_id: Option, result: Option, /// Emitted actions awaiting the next barrier flush (or the final step). - pub(crate) pending_actions: Vec, + pub(crate) pending_actions: CacheActions, } /// A new walk or the result of an empty or declined insert. @@ -331,14 +333,9 @@ impl InsertWalkState { /// Free and split bookkeeping can wait; other actions pause the walk. pub(crate) fn has_barrier_actions_since(&self, start: usize) -> bool { - self.pending_actions[start..].iter().any(|action| { - !matches!( - action, - CacheAction::FreeDeviceKV(_) - | CacheAction::FreeDeviceKVFullOnly(_) - | CacheAction::UpdatePendingBackupOnNodeSplit { .. } - ) - }) + self.pending_actions[start..] + .iter() + .any(|action| action.requires_insert_barrier()) } } @@ -352,61 +349,6 @@ pub struct KvCanaryWalkResult { pub prev_slot_indices: Vec, } -/// A queued cache IO action. -#[derive(Debug)] -pub enum CacheAction { - /// Duplicate device KV slices the cache frees after the insert. - FreeDeviceKV(Vec), - /// Free the full side only, for a tombstoned node whose SWA peers are gone; - /// FreeDeviceKV would release the SWA side twice. - FreeDeviceKVFullOnly(Vec), - /// A device->host backup work item (the write-through threshold fired). - BackupKV(BackupKV), - /// Update cache-side write bookkeeping after a node split: - /// - /// parent -> node => parent -> new_node -> new_child - /// - /// old_node_id (the pre-split node) is replaced by new_node_id + new_child_node_id. - UpdatePendingBackupOnNodeSplit { - /// None when buffer-only bookkeeping has no tree-tracked write. - ack_id: Option, - old_node_id: NodeId, - new_node_id: NodeId, - new_child_node_id: NodeId, - }, - /// Per-path Mamba state-cap eviction from the tail's root path; applied at - /// the insert's commit barrier, after the walk-time backups whose - /// write-through locks shield the backed-up chain. - MambaEvictExcessPathStates { tail_node_id: NodeId }, - /// Free only the given component's device KV slots. - FreeComponentDeviceSlot { - component_type: ComponentType, - indices: Vec, - }, - /// Free the given component's host KV pages. - FreeComponentHostSlot { - component_type: ComponentType, - host_indices: Vec, - }, - /// Rebuild the SWA allocator's full->swa index mapping for loaded chunks. - RebuildFullToSwaMapping { - full_indices: Vec, - swa_indices: Vec, - }, - /// Recover an SWA tombstone whose full is locked: keep the locked full, remap - /// it onto the incoming full's SWA translation, and free only the incoming full. - RecoverSwaWithLockedFull { - node_id: NodeId, - kept_full: Tensor, - incoming_full: Tensor, - }, - /// Rebuild a node's SWA value by translating its source full value, then store it. - SwaRebuild { - node_id: NodeId, - source_value: Tensor, - }, -} - /// A HiCache pool transfer descriptor. #[derive(Default)] pub struct PoolTransfer { @@ -482,15 +424,6 @@ pub struct PoolTransferResult { pub extra_pool_hit_pages: HashMap, } -/// A device->host backup work item for the cache to execute. -#[derive(Default, Debug)] -pub struct BackupKV { - /// Backup these nodes device->host in order, stopping at the first failure; the - /// caller orders them parent-before-child for write-through and child-first for - /// write-back. External handles: the list crosses to the orchestrator. - pub node_ids: Vec, -} - /// A device->storage backup spec. #[derive(Default)] pub struct StorageBackupSpec { @@ -654,7 +587,7 @@ pub struct EvictionStepResult { pub tracker: HashMap, pub device_frees: HashMap>, pub host_frees: HashMap>, - pub cache_actions: Vec, + pub cache_actions: CacheActions, pub backup_kv: Option, /// Full device tokens freed without a host copy during this device step. pub unbacked_tokens: usize, @@ -1258,7 +1191,7 @@ impl UnifiedTreeCore { params: &DecLockRefParams, device_frees: &mut HashMap>, host_frees: &mut HashMap>, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, ) -> Result<(), NodeAccessError> { let node_idx = self.arena.resolve(node_id)?; self.assert_receipt_anchor_(node_idx, params); @@ -1286,7 +1219,7 @@ impl UnifiedTreeCore { params: &DecLockRefParams, device_frees: &mut HashMap>, host_frees: &mut HashMap>, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, ) -> Result<(), NodeAccessError> { let node_idx = self.arena.resolve(node_id)?; self.assert_receipt_anchor_(node_idx, params); @@ -1510,7 +1443,7 @@ impl UnifiedTreeCore { usize, usize, Option, - Option, + Option, ) { // Non-HiCache mode has only device-resident matches, so the scheduler // device anchor follows the best match. In HiCache mode, host-backed @@ -1524,7 +1457,7 @@ impl UnifiedTreeCore { let mut best_match_device_node_id = node_id; let mut best_match_device_value_len = 0; let mut full_kv_hit_length = 0; - let mut action: Option = None; + let mut action: Option = None; let separate_device_match = self.enable_hicache; fn update_best_if_valid( @@ -1639,7 +1572,7 @@ impl UnifiedTreeCore { best_match_device_value_len: usize, full_kv_hit_length: usize, last_visited_node_id: Option, - action: Option, + action: Option, ) -> MatchResult { for i in 0..self.components.len() { // Full uses last_access ticks, not LRU. @@ -1705,7 +1638,10 @@ impl UnifiedTreeCore { best_match_device_value_len, ); } - result.cache_actions = action.into_iter().collect(); + result.cache_actions = action + .into_iter() + .map(|action| Box::new(action) as Box) + .collect(); result } @@ -1808,7 +1744,7 @@ impl UnifiedTreeCore { ) -> Result { // Single-shot pump over the resumable walk: run every step inline and // fold the step actions into the result for the caller to apply. - let mut actions = Vec::new(); + let mut actions: CacheActions = Vec::new(); let mut step = self.try_begin_insert(params)?; loop { actions.append(&mut step.actions); @@ -1988,7 +1924,7 @@ impl UnifiedTreeCore { } /// Finish the insert (idempotent); returns still-pending actions to drain. - pub fn end_insert(&mut self) -> Vec { + pub fn end_insert(&mut self) -> CacheActions { self.ongoing_insert_walk_state .take() .map(|state| state.pending_actions) @@ -2050,7 +1986,7 @@ impl UnifiedTreeCore { let (split_node_id, action) = self.split_node_(node_id, prefix_len); node_id = split_node_id; if let Some(action) = action { - state.pending_actions.push(action); + state.pending_actions.push(Box::new(action)); } } { @@ -2166,22 +2102,22 @@ impl UnifiedTreeCore { .saturating_sub(cursor + dup_start) .min(dup_len); if swa_already_freed > 0 { - state - .pending_actions - .push(CacheAction::FreeDeviceKVFullOnly(vec![value_slice.narrow( + state.pending_actions.push(Box::new(FreeDeviceKVFullOnly { + indices: vec![value_slice.narrow( 0, dup_start as i64, swa_already_freed as i64, - )])); + )], + })); } if swa_already_freed < dup_len { - state - .pending_actions - .push(CacheAction::FreeDeviceKV(vec![value_slice.narrow( + state.pending_actions.push(Box::new(FreeDeviceKV { + indices: vec![value_slice.narrow( 0, (dup_start + swa_already_freed) as i64, (dup_len - swa_already_freed) as i64, - )])); + )], + })); } } state.overlap_node_id = None; @@ -2200,7 +2136,7 @@ impl UnifiedTreeCore { if node_end > state.inserted_len && self.inc_hit_count_and_check_(node_id) { let backup = self .build_backup_kv_action_(self.arena.node(node_id), /* write_back = */ false); - state.pending_actions.push(CacheAction::BackupKV(backup)); + state.pending_actions.push(Box::new(backup)); } state.node_id = node_id; state.total_prefix_length += prefix_len; @@ -2335,7 +2271,7 @@ impl UnifiedTreeCore { if self.should_backup_after_insert_(state, target_node_id) { let backup = self.build_backup_kv_action_(self.arena.node(target_node_id), self.is_write_back); - state.pending_actions.push(CacheAction::BackupKV(backup)); + state.pending_actions.push(Box::new(backup)); } } @@ -2344,7 +2280,7 @@ impl UnifiedTreeCore { &mut self, child_id: NodeIdx_, split_len: usize, - ) -> (NodeIdx_, Option) { + ) -> (NodeIdx_, Option) { assert!( split_len > 0 && split_len.is_multiple_of(self.page_size), "split_node_: split_len {split_len} must be a nonzero page multiple" @@ -2429,7 +2365,7 @@ impl UnifiedTreeCore { } // Buffer-only writes are tracked by the cache rather than a node ack. let action = if ack_id.is_some() || self.is_host_memory_buffer_only { - Some(CacheAction::UpdatePendingBackupOnNodeSplit { + Some(UpdatePendingBackupOnNodeSplit { ack_id, old_node_id: self.arena.node(child_id).id, new_node_id: self.arena.node(new_node_id).id, @@ -2951,7 +2887,7 @@ impl UnifiedTreeCore { tracker: &mut HashMap, device_frees: &mut HashMap>, host_frees: &mut HashMap>, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, ) { self.record_remove_event_(node_id, medium); for i in 0..self.components.len() { @@ -2977,7 +2913,7 @@ impl UnifiedTreeCore { tracker: &mut HashMap, device_frees: &mut HashMap>, host_frees: &mut HashMap>, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, ) { self.release_all_component_layers_( node_id, @@ -3066,7 +3002,7 @@ impl UnifiedTreeCore { tracker: &mut HashMap, device_frees: &mut HashMap>, host_frees: &mut HashMap>, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, ) { assert!( self.can_reclaim_coexisting_host_value_(node_id, component_type), @@ -3102,7 +3038,7 @@ impl UnifiedTreeCore { tracker: &mut HashMap, device_frees: &mut HashMap>, host_frees: &mut HashMap>, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, ) { assert!( self.is_evictable_host_leaf_(self.arena.node(node_id)), @@ -3170,7 +3106,7 @@ impl UnifiedTreeCore { tracker: &mut HashMap, device_frees: &mut HashMap>, host_frees: &mut HashMap>, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, ) { { let node = self.arena.node(node_id); @@ -3215,7 +3151,7 @@ impl UnifiedTreeCore { tracker: &mut HashMap, device_frees: &mut HashMap>, host_frees: &mut HashMap>, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, target: EvictLayer, ) { let is_leaf = match target { @@ -3347,7 +3283,7 @@ impl UnifiedTreeCore { component_type: ComponentType, device_frees: &mut HashMap>, host_frees: &mut HashMap>, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, target: EvictLayer, tracker: Option<&mut HashMap>, ) -> (usize, usize) { @@ -3404,7 +3340,7 @@ impl UnifiedTreeCore { tracker: &mut HashMap, device_frees: &mut HashMap>, host_frees: &mut HashMap>, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, ) { let ct = BASE_COMPONENT_TYPE; let mut cur = deleted_node_parent_id; @@ -3895,7 +3831,7 @@ impl UnifiedTreeCore { let mut host_prefix_len = 0; let mut refilled_unbacked_node = false; let mut inserted_host_node = None; - let mut cache_actions: Vec = Vec::new(); + let mut cache_actions: CacheActions = Vec::new(); while matched_length < total_len { let Some(child_id) = self.arena.child_on_page_in_namespace( node_id, @@ -3916,7 +3852,7 @@ impl UnifiedTreeCore { let (split_node_id, action) = self.split_node_(node_id, prefix_len); node_id = split_node_id; if let Some(action) = action { - cache_actions.push(action); + cache_actions.push(Box::new(action)); } } @@ -4472,7 +4408,7 @@ impl UnifiedTreeCore { node_id: NodeId, phase: CacheTransferPhase, comp_xfers: HashMap>, - cache_actions: &mut Vec, + cache_actions: &mut CacheActions, mut insert_result: Option<&mut InsertResult>, pool_storage_result: Option<&PoolTransferResult>, ) -> Result<(), NodeAccessError> { @@ -4510,7 +4446,7 @@ impl UnifiedTreeCore { ) -> Result<(), NodeAccessError> { let node_id = self.arena.resolve(node_id)?; self.validate_pool_transfer_handles(comp_xfers.values().flatten())?; - let mut cache_actions: Vec = Vec::new(); + let mut cache_actions: CacheActions = Vec::new(); if host_indices.numel() > 0 { let kv_xfer = PoolTransfer { name: PoolName::Kv, @@ -4555,7 +4491,7 @@ impl UnifiedTreeCore { device_indices: Tensor, mut kv_xfer: PoolTransfer, comp_xfers: HashMap>, - ) -> Result, NodeAccessError> { + ) -> Result { let anchor_id = node_id; let node_id = self.arena.resolve(node_id)?; let loaded_node_indices = kv_xfer @@ -4565,7 +4501,7 @@ impl UnifiedTreeCore { .map(|&loaded_id| self.arena.resolve(loaded_id)) .collect::, _>>()?; self.validate_pool_transfer_handles(comp_xfers.values().flatten())?; - let mut cache_actions: Vec = Vec::new(); + let mut cache_actions: CacheActions = Vec::new(); kv_xfer.device_indices = Some(device_indices); let nodes_to_load = kv_xfer.nodes_to_load.clone(); if self.is_write_back { @@ -4894,7 +4830,7 @@ impl UnifiedTreeCore { window_start: usize, window_end: usize, swa_values: &Tensor, - ) -> Result, String> { + ) -> Result { self.validate_swa_span_(key, window_start, window_end)?; if swa_values.kind() != Kind::Int64 || swa_values.device() != self.device { return Err(format!( @@ -4940,7 +4876,7 @@ impl UnifiedTreeCore { )); } - let mut actions = Vec::new(); + let mut actions: CacheActions = Vec::new(); for (child_id, pos, prefix_len) in spans { let seg_end = window_end.min(pos + prefix_len); if seg_end <= window_start { @@ -4951,11 +4887,15 @@ impl UnifiedTreeCore { if seg_start > pos { // The original handle remains on the suffix, our target. let (_, action) = self.split_node_(target, seg_start - pos); - actions.extend(action); + if let Some(action) = action { + actions.push(Box::new(action)); + } } if seg_start + self.arena.node(target).key.atom_len() > seg_end { let (fragment, action) = self.split_node_(target, seg_end - seg_start); - actions.extend(action); + if let Some(action) = action { + actions.push(Box::new(action)); + } target = fragment; } let values = swa_values diff --git a/test/registered/unit/mem_cache/test_rust_tree_core_integration.py b/test/registered/unit/mem_cache/test_rust_tree_core_integration.py index 63ada14749..a5cda82ab5 100644 --- a/test/registered/unit/mem_cache/test_rust_tree_core_integration.py +++ b/test/registered/unit/mem_cache/test_rust_tree_core_integration.py @@ -5611,3 +5611,72 @@ def test_finished_request_prompt_priority_saturates_at_int64_max(priority): if __name__ == "__main__": sys.exit(pytest.main([__file__])) + + +def test_load_back_action_tuple_preserves_chunk_order_and_tensor_ownership(): + binding = mem_cache.RustUnifiedTreeCoreBinding( + mem_cache.TreeCoreInitParamsBinding( + page_size=1, swa_sliding_window_size=8, enable_hicache=True + ), + [0, 1], + ) + binding.set_has_swa_host_pool() + key = array("q", range(5)) + prefix = binding.insert( + mem_cache.InsertParamsBinding(key=key[:2], value=torch.tensor([10, 11])) + ).last_device_node + binding.set_component_device_value(prefix, 1, torch.tensor([110, 111])) + tail = binding.insert( + mem_cache.InsertParamsBinding( + key=key, value=torch.arange(10, 15), prev_prefix_len=2 + ) + ).last_device_node + binding.set_component_device_value(tail, 1, torch.tensor([112, 113, 114])) + for node, host in ( + (prefix, torch.arange(310, 312)), + (tail, torch.arange(312, 315)), + ): + _, transfers = binding.build_backup_spec(node) + transfer = transfers[1][0] + transfers[1][0] = (transfer[0], transfer[2] + 100, *transfer[2:]) + binding.mark_write_through_pending([node], node) + binding.commit_backup(node, host, transfers) + binding.finish_write_through([node], node) + binding.demote(tail) + binding.demote(prefix) + + kv_transfer, transfers = binding.build_load_back_spec(tail) + transfer = transfers[1][0] + transfers[1][0] = (*transfer[:2], torch.arange(50, 55), *transfer[3:]) + actions = binding.commit_load_back( + tail, torch.arange(40, 45), kv_transfer, transfers + ) + receipt = binding.inc_lock_ref(tail) + assert isinstance(actions, list) and len(actions) == 1 + action = actions[0] + assert isinstance(action, tuple) and len(action) == 3 + assert action[0] == "rebuild_full_to_swa_mapping" + assert isinstance(action[1], list) and isinstance(action[2], list) + assert [value.tolist() for value in action[1]] == [[40, 41], [42, 43, 44]] + assert [value.tolist() for value in action[2]] == [[50, 51], [52, 53, 54]] + binding.dec_lock_ref( + tail, + mem_cache.DecLockRefParamsBinding( + node_id=receipt.node_id, + skipped_lock_components=receipt.skipped_lock_components, + component_lock_uuids=receipt.component_lock_uuids, + component_host_lock_uuids=receipt.component_host_lock_uuids, + ), + ) + binding.finish_load_back(tail) + binding.sanity_check([], []) + + chunk = action[2][1] + storage_ref = chunk.untyped_storage()._weak_ref() + try: + binding.reset() + assert chunk.tolist() == [52, 53, 54] + del chunk, action, actions + assert torch.UntypedStorage._expired(storage_ref) + finally: + torch.UntypedStorage._free_weak_ref(storage_ref)