Files
sglang/rust/sglang-radix-tree/src/python_bindings.rs
T

4077 lines
150 KiB
Rust

//! Python bindings: the `mem_cache` extension module and its TreeCore adapter.
use std::borrow::Cow;
use std::collections::HashMap;
use std::panic::{AssertUnwindSafe, catch_unwind};
use std::sync::Mutex;
use pyo3::IntoPyObjectExt;
use pyo3::buffer::PyBuffer;
use pyo3::exceptions::{PyAssertionError, PyKeyError, PyRuntimeError, PyValueError};
use pyo3::prelude::*;
use pyo3::types::{PyBytes, PyDict, PyList};
use tch::{Device, Kind, Tensor};
use crate::components::{ComponentSet, ComponentType, FULL, MAMBA, SWA};
use crate::node::ChildKeyType;
use crate::node::{KeyNamespaceRef, NodeAccessError, NodeId, TreeCoreRuntimeError};
use crate::unified_lru_list::{TlruFloatConfig, TlruPromptEstimate};
use crate::unified_tree_core::KvCacheEvent;
use crate::unified_tree_core::{
BufferBackupSnapshot, BufferBackupState, CacheAction, CacheInitParams, CacheTransferPhase,
DecLockRefParams, EvictLayer, EvictionStepResult, InsertParams, InsertResult, InsertStepResult,
MatchPrefixParams, MatchResult, PoolHitPolicy, PoolName, PoolTransfer, PoolTransferResult, Req,
UnifiedTreeCore,
};
/// Translate only Rust panics; ordinary Python errors pass through unchanged.
/// Keep this boundary outside the whole binding call so its MutexGuard unwinds
/// and poisons the core before the panic is caught. A poisoned core stays unusable.
fn catch_native_panic<T>(operation: impl FnOnce() -> PyResult<T>) -> PyResult<T> {
match catch_unwind(AssertUnwindSafe(operation)) {
Ok(result) => result,
Err(payload) => {
let message = if let Some(message) = payload.downcast_ref::<String>() {
message.as_str()
} else if let Some(message) = payload.downcast_ref::<&str>() {
message
} else {
"unknown native panic"
};
Err(PyRuntimeError::new_err(format!(
"Rust TreeCore panicked: {message}"
)))
}
}
}
/// Parse a torch-style device string (e.g. "cpu", "cuda", "cuda:1"); a bare
/// "cuda" means index 0, so callers must resolve the index themselves.
fn parse_device(device: &str) -> PyResult<Device> {
let device = device.to_lowercase();
if device == "cpu" {
return Ok(Device::Cpu);
}
if device == "cuda" {
return Ok(Device::Cuda(0));
}
if let Some(index) = device.strip_prefix("cuda:")
&& let Ok(index) = index.parse::<usize>()
{
return Ok(Device::Cuda(index));
}
Err(PyValueError::new_err(format!(
"unsupported device string: {device}"
)))
}
/// Map a Python ComponentType value onto the Rust enum.
fn parse_component_type(component_type: u8) -> PyResult<ComponentType> {
match component_type {
0 => Ok(FULL),
1 => Ok(SWA),
2 => Ok(MAMBA),
other => Err(PyValueError::new_err(format!(
"unknown component type: {other}"
))),
}
}
/// Map the Python EvictLayer IntFlag value onto the Rust enum.
fn parse_evict_layer(target: u8) -> PyResult<EvictLayer> {
match target {
1 => Ok(EvictLayer::Device),
2 => Ok(EvictLayer::Host),
3 => Ok(EvictLayer::All),
other => Err(PyValueError::new_err(format!(
"unknown eviction layer: {other}"
))),
}
}
fn node_access_error(error: NodeAccessError) -> PyErr {
PyKeyError::new_err(error.node_id)
}
/// Convert an expected tree-core contract failure without unwinding through PyO3.
fn tree_core_runtime_error(error: TreeCoreRuntimeError) -> PyErr {
match error {
TreeCoreRuntimeError::NodeAccess(error) => node_access_error(error),
error @ (TreeCoreRuntimeError::ComponentNotEnabled { .. }
| TreeCoreRuntimeError::InvalidComponentSizeDelta { .. }) => {
PyValueError::new_err(error.to_string())
}
error => PyRuntimeError::new_err(error.to_string()),
}
}
fn tree_core_assertion_error(error: TreeCoreRuntimeError) -> PyErr {
match error {
TreeCoreRuntimeError::NodeAccess(error) => node_access_error(error),
error => PyAssertionError::new_err(error.to_string()),
}
}
/// Map the Rust enum back onto the Python ComponentType value.
fn component_type_to_u8(component_type: ComponentType) -> u8 {
component_type as u8
}
/// Newtype bridging `tch::Tensor` and Python `torch.Tensor` over raw THPVariable
/// pointers (inlined from pyo3-tch, MIT/Apache-2.0, by Laurent Mazare).
pub struct PyTensor(pub Tensor);
impl<'a, 'py> FromPyObject<'a, 'py> for PyTensor {
type Error = PyErr;
fn extract(ob: Borrowed<'a, 'py, PyAny>) -> PyResult<Self> {
let ptr = ob.as_ptr() as *mut tch::python::CPyObject;
match unsafe { Tensor::pyobject_unpack(ptr) } {
Ok(Some(tensor)) => Ok(PyTensor(tensor)),
Ok(None) => Err(pyo3::exceptions::PyTypeError::new_err(format!(
"expected a torch.Tensor, got {}",
ob.get_type()
))),
Err(err) => Err(PyValueError::new_err(format!("{err:?}"))),
}
}
}
impl<'py> IntoPyObject<'py> for PyTensor {
type Target = PyAny;
type Output = Bound<'py, PyAny>;
type Error = PyErr;
fn into_pyobject(self, py: Python<'py>) -> PyResult<Self::Output> {
tensor_to_py(py, self.0).map(|obj| obj.into_bound(py))
}
}
/// Convert a tensor into a Python-held torch.Tensor reference.
fn tensor_to_py(py: Python<'_>, tensor: Tensor) -> PyResult<Py<PyAny>> {
let ptr = tensor
.pyobject_wrap()
.map_err(|err| PyValueError::new_err(format!("{err:?}")))?;
Ok(unsafe { Bound::from_owned_ptr(py, ptr as *mut pyo3::ffi::PyObject) }.unbind())
}
/// Convert a Python int64 sequence to an owned `Vec<i64>`.
fn py_array_to_vec_i64(py: Python<'_>, key: &Bound<'_, PyAny>) -> PyResult<Vec<i64>> {
// Special handling for empty keys, as empty pyarray might use
// a random address to represent empty buffer which
// non-deterministically violates alignment check
if key.len().map(|n| n == 0).unwrap_or(false) {
return Ok(Vec::new());
}
let buffer = key.extract::<PyBuffer<i64>>()?;
if !buffer.is_c_contiguous() {
return Err(pyo3::exceptions::PyTypeError::new_err(
"Unexpected key received, expected a C-contiguous int64 buffer \
(e.g. array.array('q'))",
));
}
buffer.to_vec(py)
}
/// The tagged-tuple tag of a cache action.
fn cache_action_tag(action: &CacheAction) -> &'static str {
match action {
CacheAction::FreeDeviceKV(_) => "free_device_kv",
CacheAction::FreeDeviceKVFullOnly(_) => "free_device_kv_full_only",
CacheAction::BackupKV(_) => "backup_kv",
CacheAction::ReplaceWriteThroughOnNodeSplit { .. } => "replace_write_through_on_node_split",
CacheAction::MambaEvictExcessPathStates { .. } => "mamba_evict_excess_path_states",
CacheAction::FreeComponentDeviceSlot { .. } => "free_component_device_slot",
CacheAction::FreeComponentHostSlot { .. } => "free_component_host_slot",
CacheAction::RebuildFullToSwaMapping { .. } => "rebuild_full_to_swa_mapping",
CacheAction::RecoverSwaWithLockedFull { .. } => "recover_swa_with_locked_full",
CacheAction::SwaRebuild { .. } => "swa_rebuild",
}
}
/// Convert a cache action into its Python tagged tuple.
fn cache_action_to_py(py: Python<'_>, action: CacheAction) -> PyResult<Py<PyAny>> {
let tag = cache_action_tag(&action);
match action {
CacheAction::FreeDeviceKV(tensors) => {
let tensors = PyList::new(py, tensors.into_iter().map(PyTensor))?;
(tag, tensors).into_py_any(py)
}
CacheAction::FreeDeviceKVFullOnly(tensors) => {
let tensors = PyList::new(py, tensors.into_iter().map(PyTensor))?;
(tag, tensors).into_py_any(py)
}
CacheAction::BackupKV(backup) => (tag, backup.node_ids).into_py_any(py),
CacheAction::ReplaceWriteThroughOnNodeSplit {
ack_id,
old_node_id,
new_node_id,
new_child_node_id,
} => (tag, ack_id, old_node_id, new_node_id, new_child_node_id).into_py_any(py),
CacheAction::MambaEvictExcessPathStates { tail_node_id } => {
(tag, tail_node_id).into_py_any(py)
}
CacheAction::FreeComponentDeviceSlot {
component_type,
indices,
} => {
let indices = PyList::new(py, indices.into_iter().map(PyTensor))?;
(tag, component_type_to_u8(component_type), indices).into_py_any(py)
}
CacheAction::FreeComponentHostSlot {
component_type,
host_indices,
} => {
let host_indices = PyList::new(py, host_indices.into_iter().map(PyTensor))?;
(tag, component_type_to_u8(component_type), host_indices).into_py_any(py)
}
CacheAction::RebuildFullToSwaMapping {
full_indices,
swa_indices,
} => {
let full_indices = PyList::new(py, full_indices.into_iter().map(PyTensor))?;
let swa_indices = PyList::new(py, swa_indices.into_iter().map(PyTensor))?;
(tag, full_indices, swa_indices).into_py_any(py)
}
CacheAction::RecoverSwaWithLockedFull {
node_id,
kept_full,
incoming_full,
} => (tag, node_id, PyTensor(kept_full), PyTensor(incoming_full)).into_py_any(py),
CacheAction::SwaRebuild {
node_id,
source_value,
} => (tag, node_id, PyTensor(source_value)).into_py_any(py),
}
}
/// Convert cache actions into a Python list of tagged tuples.
fn cache_actions_to_py(py: Python<'_>, actions: Vec<CacheAction>) -> PyResult<Py<PyList>> {
let list = PyList::empty(py);
for action in actions {
list.append(cache_action_to_py(py, action)?)?;
}
Ok(list.unbind())
}
/// The python PoolName string for a pool.
fn pool_name_str(name: PoolName) -> &'static str {
match name {
PoolName::Kv => "kv",
PoolName::Mamba => "mamba",
PoolName::Swa => "swa",
PoolName::AuxiliarySwa => "auxiliary_swa",
PoolName::Indexer => "indexer",
PoolName::DeepseekV4C4 => "deepseek_v4_c4",
PoolName::DeepseekV4C4Indexer => "deepseek_v4_c4_indexer",
PoolName::DeepseekV4C4IndexerScale => "deepseek_v4_c4_indexer_scale",
PoolName::DeepseekV4C128 => "deepseek_v4_c128",
PoolName::DeepseekV4C1 => "deepseek_v4_c1",
PoolName::DeepseekV4C1Indexer => "deepseek_v4_c1_indexer",
PoolName::DeepseekV4C1IndexerScale => "deepseek_v4_c1_indexer_scale",
PoolName::DeepseekV4C2 => "deepseek_v4_c2",
PoolName::DeepseekV4C2Indexer => "deepseek_v4_c2_indexer",
PoolName::DeepseekV4C2IndexerScale => "deepseek_v4_c2_indexer_scale",
PoolName::DeepseekV4C4Rope => "deepseek_v4_c4_rope",
PoolName::DeepseekV4C128Rope => "deepseek_v4_c128_rope",
PoolName::DeepseekV4C4State => "deepseek_v4_c4_state",
PoolName::DeepseekV4C4IndexerState => "deepseek_v4_c4_indexer_state",
PoolName::DeepseekV4C128State => "deepseek_v4_c128_state",
PoolName::Draft => "draft",
PoolName::DraftIndexer => "draft_indexer",
PoolName::DraftSwa => "draft_swa",
}
}
/// Map a python PoolName string onto the Rust enum.
fn parse_pool_name(name: &str) -> PyResult<PoolName> {
match name {
"kv" => Ok(PoolName::Kv),
"mamba" => Ok(PoolName::Mamba),
"swa" => Ok(PoolName::Swa),
"auxiliary_swa" => Ok(PoolName::AuxiliarySwa),
"indexer" => Ok(PoolName::Indexer),
"deepseek_v4_c4" => Ok(PoolName::DeepseekV4C4),
"deepseek_v4_c4_indexer" => Ok(PoolName::DeepseekV4C4Indexer),
"deepseek_v4_c4_indexer_scale" => Ok(PoolName::DeepseekV4C4IndexerScale),
"deepseek_v4_c128" => Ok(PoolName::DeepseekV4C128),
"deepseek_v4_c1" => Ok(PoolName::DeepseekV4C1),
"deepseek_v4_c1_indexer" => Ok(PoolName::DeepseekV4C1Indexer),
"deepseek_v4_c1_indexer_scale" => Ok(PoolName::DeepseekV4C1IndexerScale),
"deepseek_v4_c2" => Ok(PoolName::DeepseekV4C2),
"deepseek_v4_c2_indexer" => Ok(PoolName::DeepseekV4C2Indexer),
"deepseek_v4_c2_indexer_scale" => Ok(PoolName::DeepseekV4C2IndexerScale),
"deepseek_v4_c4_rope" => Ok(PoolName::DeepseekV4C4Rope),
"deepseek_v4_c128_rope" => Ok(PoolName::DeepseekV4C128Rope),
"deepseek_v4_c4_state" => Ok(PoolName::DeepseekV4C4State),
"deepseek_v4_c4_indexer_state" => Ok(PoolName::DeepseekV4C4IndexerState),
"deepseek_v4_c128_state" => Ok(PoolName::DeepseekV4C128State),
"draft" => Ok(PoolName::Draft),
"draft_indexer" => Ok(PoolName::DraftIndexer),
"draft_swa" => Ok(PoolName::DraftSwa),
other => Err(PyValueError::new_err(format!("unknown pool name: {other}"))),
}
}
/// A pool transfer's boundary form:
/// (name, host_indices, device_indices, nodes_to_load, keys, hit_policy).
type TransferArgs = (
String,
Option<PyTensor>,
Option<PyTensor>,
Option<Vec<NodeId>>,
Option<Vec<String>>,
String,
);
/// Strongly typed, attribute-based input view of a Python MatchResult.
/// Cache actions are intentionally omitted: component finalizers only update
/// match metadata, while the test adapter preserves the original actions.
#[cfg(feature = "inspection")]
#[derive(FromPyObject)]
struct InspectionMatchResultInput {
#[pyo3(attribute)]
device_prefix_len: usize,
#[pyo3(attribute)]
last_device_node: NodeId,
#[pyo3(attribute)]
last_host_node: NodeId,
#[pyo3(attribute)]
best_match_node: NodeId,
#[pyo3(attribute)]
host_hit_length: usize,
#[pyo3(attribute)]
swa_host_hit_length: usize,
#[pyo3(attribute)]
swa_branching_seqlen: Option<usize>,
#[pyo3(attribute)]
mamba_host_hit_length: usize,
#[pyo3(attribute)]
mamba_branching_seqlen: Option<usize>,
#[pyo3(attribute)]
full_kv_hit_length: usize,
}
/// Map a python CacheTransferPhase value onto the Rust enum.
fn parse_transfer_phase(phase: &str) -> PyResult<CacheTransferPhase> {
match phase {
"backup_host" => Ok(CacheTransferPhase::BackupHost),
"load_back" => Ok(CacheTransferPhase::LoadBack),
"backup_storage" => Ok(CacheTransferPhase::BackupStorage),
"prefetch" => Ok(CacheTransferPhase::Prefetch),
other => Err(PyValueError::new_err(format!(
"unknown transfer phase: {other}"
))),
}
}
/// Map a python PoolHitPolicy value onto the Rust enum.
fn parse_hit_policy(hit_policy: &str) -> PyResult<PoolHitPolicy> {
match hit_policy {
"all_pages" => Ok(PoolHitPolicy::AllPages),
"trailing_pages" => Ok(PoolHitPolicy::TrailingPages),
other => Err(PyValueError::new_err(format!(
"unknown hit policy: {other}"
))),
}
}
/// Convert a pool transfer into its boundary tuple.
fn transfer_to_py(py: Python<'_>, transfer: PoolTransfer) -> PyResult<Py<PyAny>> {
(
pool_name_str(transfer.name),
transfer.host_indices.map(PyTensor),
transfer.device_indices.map(PyTensor),
transfer.nodes_to_load,
transfer.keys,
transfer.hit_policy.as_str(),
)
.into_py_any(py)
}
/// Build a pool transfer from its boundary tuple.
fn transfer_from_args(args: TransferArgs) -> PyResult<PoolTransfer> {
let (name, host_indices, device_indices, nodes_to_load, keys, hit_policy) = args;
Ok(PoolTransfer {
name: parse_pool_name(&name)?,
host_indices: host_indices.map(|t| t.0),
device_indices: device_indices.map(|t| t.0),
keys,
hit_policy: parse_hit_policy(&hit_policy)?,
nodes_to_load,
})
}
/// Convert per-component transfers into a Python dict of boundary tuples.
fn comp_xfers_to_py(
py: Python<'_>,
comp_xfers: HashMap<ComponentType, Vec<PoolTransfer>>,
) -> PyResult<Py<PyDict>> {
let dict = PyDict::new(py);
// Pool allocation can evict across components, so preserve Python's order.
let mut comp_xfers: Vec<_> = comp_xfers.into_iter().collect();
comp_xfers.sort_unstable_by_key(|(ct, _)| component_type_to_u8(*ct));
for (ct, transfers) in comp_xfers {
let list = PyList::empty(py);
for transfer in transfers {
list.append(transfer_to_py(py, transfer)?)?;
}
dict.set_item(component_type_to_u8(ct), list)?;
}
Ok(dict.unbind())
}
/// Build per-component transfers from a Python dict of boundary tuples.
fn comp_xfers_from_args(
comp_xfers: HashMap<u8, Vec<TransferArgs>>,
) -> PyResult<HashMap<ComponentType, Vec<PoolTransfer>>> {
comp_xfers
.into_iter()
.map(|(ct, transfers)| {
Ok((
parse_component_type(ct)?,
transfers
.into_iter()
.map(transfer_from_args)
.collect::<PyResult<Vec<_>>>()?,
))
})
.collect()
}
/// Convert per-component freed tensors into a Python dict keyed by component value.
fn frees_to_py(py: Python<'_>, frees: HashMap<ComponentType, Vec<Tensor>>) -> PyResult<Py<PyDict>> {
let frees: HashMap<u8, Vec<PyTensor>> = frees
.into_iter()
.map(|(ct, tensors)| {
(
component_type_to_u8(ct),
tensors.into_iter().map(PyTensor).collect(),
)
})
.collect();
let dict = PyDict::new(py);
for (ct, tensors) in frees {
dict.set_item(ct, tensors)?;
}
Ok(dict.unbind())
}
/// Floating-point T-LRU arithmetic with the estimate's numeric type preserved.
#[pyclass(name = "TlruFloatConfig", from_py_object)]
#[derive(Clone)]
pub struct TlruFloatConfigBinding {
config: TlruFloatConfig,
}
#[pymethods]
impl TlruFloatConfigBinding {
#[new]
#[pyo3(signature = (threshold, next_prompt_estimate, integer_estimate = None))]
fn new(threshold: f64, next_prompt_estimate: f64, integer_estimate: Option<i128>) -> Self {
let estimate = match integer_estimate {
Some(value) => TlruPromptEstimate::Integer(value),
None => TlruPromptEstimate::Float(next_prompt_estimate),
};
Self {
config: TlruFloatConfig {
threshold,
next_prompt_estimate: estimate,
},
}
}
#[cfg(feature = "inspection")]
fn inspect_is_tel_safe(&self, history: usize, cached_without_node: usize) -> PyResult<bool> {
catch_native_panic(|| Ok(self.config.is_tel_safe(history, cached_without_node)))
}
}
/// Python-visible tree-core init params; converts into CacheInitParams.
#[pyclass(get_all, set_all, from_py_object)]
#[derive(Clone)]
pub struct TreeCoreInitParamsBinding {
pub eviction_policy: String,
pub slru_protected_threshold: i64,
pub tlru_tail_budget: usize,
pub tlru_float_config: Option<TlruFloatConfigBinding>,
pub page_size: usize,
pub is_write_back: bool,
pub enable_hicache: bool,
pub write_through_threshold: i64,
pub device: String,
pub swa_sliding_window_size: Option<usize>,
pub swa_req_ring: bool,
pub enable_kv_cache_events: bool,
pub dynamo_kv_event_format: bool,
pub mamba_cache_chunk_size: Option<usize>,
pub mamba_max_states_per_path: Option<usize>,
}
impl TreeCoreInitParamsBinding {
/// Convert into the tree core's construction params.
fn to_cache_init_params(&self) -> PyResult<CacheInitParams> {
Ok(CacheInitParams {
eviction_policy: self.eviction_policy.clone(),
slru_protected_threshold: self.slru_protected_threshold,
tlru_tail_budget: self.tlru_tail_budget,
tlru_float_config: self.tlru_float_config.as_ref().map(|value| value.config),
page_size: self.page_size,
is_write_back: self.is_write_back,
enable_hicache: self.enable_hicache,
write_through_threshold: self.write_through_threshold,
device: parse_device(&self.device)?,
swa_sliding_window_size: self.swa_sliding_window_size,
swa_req_ring: self.swa_req_ring,
// Wired post-construction via set_has_swa_host_pool.
has_swa_host_pool: false,
enable_kv_cache_events: self.enable_kv_cache_events,
dynamo_kv_event_format: self.dynamo_kv_event_format,
mamba_cache_chunk_size: self.mamba_cache_chunk_size,
mamba_max_states_per_path: self.mamba_max_states_per_path,
})
}
}
#[pymethods]
impl TreeCoreInitParamsBinding {
#[new]
#[pyo3(signature = (eviction_policy = "lru".to_string(), page_size = 1, is_write_back = false, enable_hicache = false, write_through_threshold = 256, device = "cpu".to_string(), swa_sliding_window_size = None, enable_kv_cache_events = false, mamba_cache_chunk_size = None, mamba_max_states_per_path = None, slru_protected_threshold = 2, swa_req_ring = false, tlru_tail_budget = 0, tlru_float_config = None, dynamo_kv_event_format = false))]
fn new(
eviction_policy: String,
page_size: usize,
is_write_back: bool,
enable_hicache: bool,
write_through_threshold: i64,
device: String,
swa_sliding_window_size: Option<usize>,
enable_kv_cache_events: bool,
mamba_cache_chunk_size: Option<usize>,
mamba_max_states_per_path: Option<usize>,
slru_protected_threshold: i64,
swa_req_ring: bool,
tlru_tail_budget: usize,
tlru_float_config: Option<TlruFloatConfigBinding>,
dynamo_kv_event_format: bool,
) -> Self {
TreeCoreInitParamsBinding {
eviction_policy,
slru_protected_threshold,
tlru_tail_budget,
tlru_float_config,
page_size,
is_write_back,
enable_hicache,
write_through_threshold,
device,
swa_sliding_window_size,
swa_req_ring,
enable_kv_cache_events,
dynamo_kv_event_format,
mamba_cache_chunk_size,
mamba_max_states_per_path,
}
}
}
/// Python-visible match params; converts into MatchPrefixParams.
#[pyclass(get_all, set_all, from_py_object)]
#[derive(Clone)]
pub struct MatchParamsBinding {
pub key: Vec<i64>,
pub extra_key: Option<String>,
pub cache_salt: Option<String>,
}
#[pymethods]
impl MatchParamsBinding {
#[new]
#[pyo3(signature = (key, extra_key = None, cache_salt = None))]
fn new(
py: Python<'_>,
key: &Bound<'_, PyAny>,
extra_key: Option<String>,
cache_salt: Option<String>,
) -> PyResult<Self> {
Ok(MatchParamsBinding {
key: py_array_to_vec_i64(py, key)?,
extra_key,
cache_salt,
})
}
}
/// Python-visible insert params; converts into InsertParams. The value tensor
/// stays a Python-held reference until the insert call unwraps it.
#[pyclass(get_all, set_all)]
pub struct InsertParamsBinding {
pub rotation_base: Option<i64>,
pub key: Vec<i64>,
pub value: Py<PyAny>,
pub extra_key: Option<String>,
pub cache_salt: Option<String>,
pub session_id: Option<String>,
pub mamba_value: Option<Py<PyAny>>,
pub prev_prefix_len: usize,
pub swa_evicted_seqlen: usize,
pub swa_branching_seqlen: Option<usize>,
pub inserted_len: usize,
pub priority: i64,
pub track_adopted_ranges: bool,
}
#[pymethods]
impl InsertParamsBinding {
#[new]
#[pyo3(signature = (key, value, extra_key = None, cache_salt = None, session_id = None, prev_prefix_len = 0, swa_evicted_seqlen = 0, swa_branching_seqlen = None, inserted_len = 0, priority = 0, mamba_value = None, track_adopted_ranges = false, rotation_base = None))]
fn new(
py: Python<'_>,
key: &Bound<'_, PyAny>,
value: Py<PyAny>,
extra_key: Option<String>,
cache_salt: Option<String>,
session_id: Option<String>,
prev_prefix_len: usize,
swa_evicted_seqlen: usize,
swa_branching_seqlen: Option<usize>,
inserted_len: usize,
priority: i64,
mamba_value: Option<Py<PyAny>>,
track_adopted_ranges: bool,
rotation_base: Option<i64>,
) -> PyResult<Self> {
Ok(InsertParamsBinding {
rotation_base,
key: py_array_to_vec_i64(py, key)?,
value,
extra_key,
cache_salt,
session_id,
mamba_value,
prev_prefix_len,
swa_evicted_seqlen,
swa_branching_seqlen,
inserted_len,
priority,
track_adopted_ranges,
})
}
}
/// Python-visible match result; actions are Python-held.
#[pyclass(get_all)]
pub struct MatchResultBinding {
device_prefix_len: usize,
last_device_node_id: NodeId,
last_host_node_id: NodeId,
best_match_node_id: NodeId,
host_hit_length: usize,
swa_host_hit_length: usize,
swa_branching_seqlen: Option<usize>,
mamba_host_hit_length: usize,
mamba_branching_seqlen: Option<usize>,
full_kv_hit_length: usize,
cache_actions: Py<PyList>,
}
impl MatchResultBinding {
/// Move a core match result across the boundary.
fn from_match_result(py: Python<'_>, result: MatchResult) -> PyResult<Self> {
Ok(MatchResultBinding {
device_prefix_len: result.device_prefix_len,
last_device_node_id: result.last_device_node_id,
last_host_node_id: result.last_host_node_id,
best_match_node_id: result.best_match_node_id,
host_hit_length: result.host_hit_length,
swa_host_hit_length: result.swa_host_hit_length,
swa_branching_seqlen: result.swa_branching_seqlen,
mamba_host_hit_length: result.mamba_host_hit_length,
mamba_branching_seqlen: result.mamba_branching_seqlen,
full_kv_hit_length: result.full_kv_hit_length,
cache_actions: cache_actions_to_py(py, result.cache_actions)?,
})
}
}
/// Python-visible insert result; actions are Python-held.
#[pyclass(get_all)]
pub struct InsertResultBinding {
rotation_tail_declined: bool,
prefix_len: usize,
total_len: usize,
last_device_node: Option<NodeId>,
inserted_host_node: Option<NodeId>,
host_insert_dropped: bool,
mamba_exist: bool,
swa_branch_inserted: bool,
adopted_ranges: Option<HashMap<u8, Vec<(usize, usize)>>>,
cache_actions: Py<PyList>,
}
/// One step of the resumable insert: the actions to apply at this barrier
/// and the final result once the walk completes.
#[pyclass(get_all)]
pub struct InsertStepResultBinding {
actions: Py<PyList>,
result: Option<Py<InsertResultBinding>>,
}
impl InsertStepResultBinding {
/// Move a core insert step across the boundary.
fn from_insert_step(py: Python<'_>, step: InsertStepResult) -> PyResult<Self> {
let result = match step.result {
Some(result) => Some(Py::new(
py,
InsertResultBinding::from_insert_result(py, result)?,
)?),
None => None,
};
Ok(InsertStepResultBinding {
actions: cache_actions_to_py(py, step.actions)?,
result,
})
}
}
impl InsertResultBinding {
/// Move a core insert result across the boundary.
fn from_insert_result(py: Python<'_>, result: InsertResult) -> PyResult<Self> {
Ok(InsertResultBinding {
rotation_tail_declined: result.rotation_tail_declined,
prefix_len: result.prefix_len,
total_len: result.total_len,
last_device_node: result.last_device_node_id,
inserted_host_node: result.inserted_host_node,
host_insert_dropped: result.host_insert_dropped,
mamba_exist: result.mamba_exist,
swa_branch_inserted: result.swa_branch_inserted,
adopted_ranges: result.adopted_ranges.map(|ranges| {
ranges
.into_iter()
.map(|(component_type, ranges)| (component_type_to_u8(component_type), ranges))
.collect()
}),
cache_actions: cache_actions_to_py(py, result.cache_actions)?,
})
}
}
/// Python-visible dec-lock params; converts into DecLockRefParams.
#[pyclass(from_py_object)]
#[derive(Clone, Default)]
pub struct DecLockRefParamsBinding {
#[pyo3(get, set)]
pub node_id: Option<NodeId>,
pub skipped_lock_components: Vec<u8>,
#[pyo3(get, set)]
pub component_lock_uuids: HashMap<u8, Option<i64>>,
#[pyo3(get, set)]
pub component_host_lock_uuids: HashMap<u8, Option<i64>>,
}
#[pymethods]
impl DecLockRefParamsBinding {
/// A plain get would turn `Vec<u8>` into `bytes`; Python expects `list[int]`.
#[getter]
fn skipped_lock_components<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyList>> {
PyList::new(py, &self.skipped_lock_components)
}
#[setter]
fn set_skipped_lock_components(&mut self, value: Vec<u8>) {
self.skipped_lock_components = value;
}
#[new]
#[pyo3(signature = (node_id = None, skipped_lock_components = Vec::new(), *, component_lock_uuids = None, component_host_lock_uuids = None))]
fn new(
node_id: Option<NodeId>,
skipped_lock_components: Vec<u8>,
component_lock_uuids: Option<HashMap<u8, Option<i64>>>,
component_host_lock_uuids: Option<HashMap<u8, Option<i64>>>,
) -> Self {
DecLockRefParamsBinding {
node_id,
skipped_lock_components,
component_lock_uuids: component_lock_uuids.unwrap_or_default(),
component_host_lock_uuids: component_host_lock_uuids.unwrap_or_default(),
}
}
}
impl DecLockRefParamsBinding {
/// Convert into the tree core's dec-lock params.
fn to_dec_lock_ref_params(&self) -> PyResult<DecLockRefParams> {
Ok(DecLockRefParams {
node_id: self.node_id,
skipped_lock_components: component_set_from_py(&self.skipped_lock_components)?,
component_lock_uuids: self.component_lock_uuids.clone(),
component_host_lock_uuids: self.component_host_lock_uuids.clone(),
})
}
}
/// Python-visible inc_lock_ref result; the receipt (anchor node, boundary
/// uuids, skipped components) is handed back to the matching dec_lock_ref.
#[pyclass]
pub struct IncLockRefResultBinding {
#[pyo3(get)]
delta: Option<usize>,
#[pyo3(get)]
node_id: Option<NodeId>,
skipped_lock_components: Vec<u8>,
#[pyo3(get)]
component_lock_uuids: HashMap<u8, Option<i64>>,
#[pyo3(get)]
component_host_lock_uuids: HashMap<u8, Option<i64>>,
}
#[pymethods]
impl IncLockRefResultBinding {
/// A plain get would turn `Vec<u8>` into `bytes`; Python expects `list[int]`.
#[getter]
fn skipped_lock_components<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyList>> {
PyList::new(py, &self.skipped_lock_components)
}
}
impl IncLockRefResultBinding {
fn from_result(result: crate::unified_tree_core::IncLockRefResult) -> Self {
Self {
delta: result.delta,
node_id: result.node_id,
component_lock_uuids: result.component_lock_uuids,
component_host_lock_uuids: result.component_host_lock_uuids,
skipped_lock_components: result
.skipped_lock_components
.iter()
.map(|ct| ct.idx() as u8)
.collect(),
}
}
}
/// Parse Python component-type ids into a component set.
fn component_set_from_py(component_types: &[u8]) -> PyResult<ComponentSet> {
component_types
.iter()
.map(|ct| parse_component_type(*ct))
.collect()
}
/// Convert a Python component-keyed tracker into the core's counts.
fn tracker_from_py(tracker: HashMap<u8, usize>) -> PyResult<HashMap<ComponentType, usize>> {
tracker
.into_iter()
.map(|(ct, freed)| Ok((parse_component_type(ct)?, freed)))
.collect()
}
/// Convert the core's tracker back into Python component-keyed counts.
fn tracker_to_py(tracker: HashMap<ComponentType, usize>) -> HashMap<u8, usize> {
tracker
.into_iter()
.map(|(ct, freed)| (component_type_to_u8(ct), freed))
.collect()
}
/// Next-eviction-node step result: the node to evict, whether the walk made
/// progress, this step's per-component evicted counts, and this step's newly
/// freed tensors.
#[pyclass(get_all)]
pub struct EvictDeviceNextNodeResultBinding {
node_id: Option<NodeId>,
mamba_backup_node_id: Option<NodeId>,
swa_backup_node_id: Option<NodeId>,
swa_backup_num_tokens: usize,
made_progress: bool,
unbacked_tokens: usize,
tracker: HashMap<u8, usize>,
new_device_frees: Py<PyDict>,
new_host_frees: Py<PyDict>,
}
/// Leaf-eviction step result: the backup action for an unbacked
/// write-back leaf (else None), this step's per-component evicted counts,
/// and this step's newly freed tensors.
#[pyclass(get_all)]
pub struct EvictDeviceLeafResultBinding {
backup_kv: Option<Py<PyAny>>,
unbacked_tokens: usize,
tracker: HashMap<u8, usize>,
new_device_frees: Py<PyDict>,
new_host_frees: Py<PyDict>,
}
/// Drop-subtree result: whether the drop happened, this step's
/// per-component evicted counts, and the subtree's newly freed tensors.
#[pyclass(get_all)]
pub struct DropSubtreeResultBinding {
dropped: bool,
tracker: HashMap<u8, usize>,
new_device_frees: Py<PyDict>,
new_host_frees: Py<PyDict>,
}
/// Demote result: this step's per-component evicted counts and the
/// demoted node's newly freed tensors.
#[pyclass(get_all)]
pub struct DemoteResultBinding {
tracker: HashMap<u8, usize>,
new_device_frees: Py<PyDict>,
new_host_frees: Py<PyDict>,
}
/// Python-visible device->storage backup spec.
#[pyclass(get_all)]
pub struct StorageBackupSpecBinding {
host_value: Py<PyAny>,
/// Raw native-endian int64 bytes; python rebuilds them via array("q").frombytes.
token_ids: Py<PyBytes>,
hash_value: Option<Vec<String>>,
prefix_keys: Option<Vec<String>>,
comp_xfers: Py<PyDict>,
}
#[pyclass(get_all)]
pub struct BufferBackupSnapshotBinding {
node_id: NodeId,
parent_node_id: NodeId,
parent_is_root: bool,
parent_last_hash: Option<String>,
key_token_ids: Py<PyBytes>,
extra_key: Option<String>,
cache_salt: Option<String>,
is_bigram: bool,
hash_values: Vec<String>,
prefix_keys: Option<Vec<String>>,
}
impl BufferBackupSnapshotBinding {
fn from_snapshot(py: Python<'_>, snapshot: BufferBackupSnapshot) -> Self {
let mut token_bytes = Vec::with_capacity(snapshot.token_ids.len() * 8);
for token in snapshot.token_ids {
token_bytes.extend_from_slice(&token.to_ne_bytes());
}
Self {
node_id: snapshot.node_id,
parent_node_id: snapshot.parent_node_id,
parent_is_root: snapshot.parent_is_root,
parent_last_hash: snapshot.parent_last_hash,
key_token_ids: PyBytes::new(py, &token_bytes).unbind(),
extra_key: snapshot.extra_key,
cache_salt: snapshot.cache_salt,
is_bigram: snapshot.is_bigram,
hash_values: snapshot.hash_values,
prefix_keys: snapshot.prefix_keys,
}
}
}
#[pyclass(get_all)]
pub struct BufferBackupStateBinding {
parent_node_id: NodeId,
parent_is_root: bool,
parent_last_hash: Option<String>,
}
impl From<BufferBackupState> for BufferBackupStateBinding {
fn from(state: BufferBackupState) -> Self {
Self {
parent_node_id: state.parent_node_id,
parent_is_root: state.parent_is_root,
parent_last_hash: state.parent_last_hash,
}
}
}
/// Python-visible KV-canary walk rows: parallel int64 tensors of slots,
/// positions, and preceding slots.
#[pyclass(get_all)]
pub struct KvCanaryWalkResultBinding {
slot_indices: Py<PyAny>,
positions: Py<PyAny>,
prev_slot_indices: Py<PyAny>,
}
/// Host-eviction drive result: this step's per-component evicted counts
/// and the drive's newly freed tensors.
#[pyclass(get_all)]
pub struct HostEvictionResultBinding {
tracker: HashMap<u8, usize>,
new_device_frees: Py<PyDict>,
new_host_frees: Py<PyDict>,
}
impl HostEvictionResultBinding {
fn from_eviction_step(py: Python<'_>, result: EvictionStepResult) -> PyResult<Self> {
Ok(Self {
tracker: tracker_to_py(result.tracker),
new_device_frees: frees_to_py(py, result.device_frees)?,
new_host_frees: frees_to_py(py, result.host_frees)?,
})
}
}
type TreePathNodeInfoTuple = (NodeId, NodeId, usize, usize, Option<usize>, usize);
/// The generic UnifiedTreeCore adapter the per-key-type pyclasses delegate to;
/// the Mutex makes the Send-only core satisfy pyclass's Sync bound.
struct TreeCoreBinding<K: ChildKeyType> {
core: Mutex<UnifiedTreeCore<K>>,
/// The core's construction device, kept outside the Mutex for pre-lock validation.
device: Device,
/// The core's page size, kept outside the Mutex for pre-lock validation.
page_size: usize,
}
// Send + Sync lets detach release the GIL around core calls.
impl<K: ChildKeyType + Send + Sync> TreeCoreBinding<K> {
/// Build a tree core for the given component types from the cache's
/// init params.
fn new(init_params: &TreeCoreInitParamsBinding, component_types: Vec<u8>) -> PyResult<Self> {
let component_types = component_types
.into_iter()
.map(parse_component_type)
.collect::<PyResult<Vec<_>>>()?;
if component_types != [FULL]
&& component_types != [FULL, SWA]
&& component_types != [FULL, MAMBA]
&& component_types != [FULL, SWA, MAMBA]
{
return Err(PyValueError::new_err(
"only the [Full], [Full, Swa], [Full, Mamba], and [Full, Swa, Mamba] component sets are supported",
));
}
if component_types.contains(&SWA) && init_params.swa_sliding_window_size.is_none() {
return Err(PyValueError::new_err(
"the Swa component requires swa_sliding_window_size",
));
}
if component_types.contains(&MAMBA) && init_params.mamba_cache_chunk_size.is_none() {
return Err(PyValueError::new_err(
"the Mamba component requires mamba_cache_chunk_size",
));
}
if init_params.page_size == 0 {
return Err(PyValueError::new_err("page_size must be at least 1"));
}
let eviction_policy = init_params.eviction_policy.to_lowercase();
if !matches!(
eviction_policy.as_str(),
"lru" | "lfu" | "fifo" | "mru" | "filo" | "priority" | "slru" | "tlru"
) {
return Err(PyValueError::new_err(format!(
"Unknown eviction policy: {eviction_policy}. Supported policies: \
'lru', 'lfu', 'fifo', 'mru', 'filo', 'priority', 'slru', 'tlru'."
)));
}
let params = init_params.to_cache_init_params()?;
let device = params.device;
let page_size = params.page_size;
Ok(TreeCoreBinding {
core: Mutex::new(UnifiedTreeCore::new(params, component_types)),
device,
page_size,
})
}
/// Lock the core for one adapter call. A panic can leave a mutation half-applied,
/// so a poisoned core is never reused.
fn core(&self) -> std::sync::MutexGuard<'_, UnifiedTreeCore<K>> {
self.core.lock().unwrap_or_else(|_| {
panic!("Rust TreeCore mutex poisoned; refusing to reuse state after an earlier panic")
})
}
/// Reject an insert value whose dtype, device, or length cannot cover the key.
fn validate_insert_value(&self, value: &Tensor, key_atom_len: usize) -> PyResult<()> {
if value.kind() != Kind::Int64 {
return Err(PyValueError::new_err(format!(
"insert value must be an int64 tensor, got {:?}",
value.kind()
)));
}
if value.device() != self.device {
return Err(PyValueError::new_err(format!(
"insert value device {:?} does not match the tree core device {:?}",
value.device(),
self.device
)));
}
let aligned_key_len = key_atom_len / self.page_size * self.page_size;
let value_len = value.size().first().copied().unwrap_or(0);
if value_len < aligned_key_len as i64 {
return Err(PyValueError::new_err(format!(
"insert value length {value_len} is shorter than the aligned key length {aligned_key_len}"
)));
}
Ok(())
}
/// Drop the entire tree and reinitialize empty state.
fn reset(&self, py: Python<'_>) {
py.detach(|| self.core().reset());
}
/// Match a key against the tree.
fn match_prefix(
&self,
py: Python<'_>,
params: &MatchParamsBinding,
) -> PyResult<MatchResultBinding> {
let key = K::key_from(Cow::Borrowed(&params.key));
let key = key.as_ref();
let params = MatchPrefixParams {
key,
namespace: KeyNamespaceRef::new(
params.extra_key.as_deref(),
params.cache_salt.as_deref(),
),
};
let result = py.detach(|| self.core().match_prefix(&params));
MatchResultBinding::from_match_result(py, result)
}
/// Read-only FULL-device match, independent of auxiliary components.
fn match_full_device_prefix(
&self,
py: Python<'_>,
params: &MatchParamsBinding,
) -> (usize, NodeId, usize) {
let key = K::key_from(Cow::Borrowed(&params.key));
let key = key.as_ref();
let namespace =
KeyNamespaceRef::new(params.extra_key.as_deref(), params.cache_salt.as_deref());
py.detach(|| self.core().match_full_device_prefix(key, namespace))
}
fn swa_tombstone_ranges(
&self,
py: Python<'_>,
params: &MatchParamsBinding,
start: usize,
end: usize,
) -> PyResult<Vec<(usize, usize)>> {
let key = K::key_from(Cow::Borrowed(&params.key));
let namespace =
KeyNamespaceRef::new(params.extra_key.as_deref(), params.cache_salt.as_deref());
py.detach(|| {
self.core()
.swa_tombstone_ranges(key.as_ref(), namespace, start, end)
})
.map_err(PyAssertionError::new_err)
}
fn attach_swa_window(
&self,
py: Python<'_>,
params: &MatchParamsBinding,
window_start: usize,
window_end: usize,
swa_values: PyTensor,
) -> PyResult<Py<PyList>> {
let key = K::key_from(Cow::Borrowed(&params.key));
let namespace =
KeyNamespaceRef::new(params.extra_key.as_deref(), params.cache_salt.as_deref());
let swa_values = swa_values.0;
let actions = py
.detach(move || {
self.core().attach_swa_window(
key.as_ref(),
namespace,
window_start,
window_end,
&swa_values,
)
})
.map_err(PyAssertionError::new_err)?;
cache_actions_to_py(py, actions)
}
/// The empty match result anchored at the root.
fn empty_match_result(&self, py: Python<'_>) -> PyResult<MatchResultBinding> {
let result = py.detach(|| self.core().empty_match_result());
MatchResultBinding::from_match_result(py, result)
}
/// Insert device values into the tree per the provided key.
fn insert(
&self,
py: Python<'_>,
params: &InsertParamsBinding,
) -> PyResult<InsertResultBinding> {
let key = K::key_from(Cow::Borrowed(&params.key));
let key = key.as_ref();
let value: PyTensor = params.value.bind(py).extract()?;
// The value covers key atoms (bigram: raw len - 1), so validate the converted key.
self.validate_insert_value(&value.0, key.atom_len())?;
let mamba_value = match &params.mamba_value {
Some(mamba_value) => Some(mamba_value.bind(py).extract::<PyTensor>()?.0),
None => None,
};
let params = InsertParams {
rotation_base: params.rotation_base,
key,
namespace: KeyNamespaceRef::new(
params.extra_key.as_deref(),
params.cache_salt.as_deref(),
),
session_id: params.session_id.as_deref(),
value: value.0,
mamba_value,
prev_prefix_len: params.prev_prefix_len,
swa_evicted_seqlen: params.swa_evicted_seqlen,
swa_branching_seqlen: params.swa_branching_seqlen,
inserted_len: params.inserted_len,
priority: params.priority,
track_adopted_ranges: params.track_adopted_ranges,
};
let result = py
.detach(move || self.core().try_insert(&params))
.map_err(tree_core_runtime_error)?;
InsertResultBinding::from_insert_result(py, result)
}
/// Start the resumable insert, running to its first barrier or completion.
fn begin_insert(
&self,
py: Python<'_>,
params: &InsertParamsBinding,
) -> PyResult<InsertStepResultBinding> {
let key = K::key_from(Cow::Borrowed(&params.key));
let key = key.as_ref();
let value: PyTensor = params.value.bind(py).extract()?;
// The value covers key atoms (bigram: raw len - 1), so validate the converted key.
self.validate_insert_value(&value.0, key.atom_len())?;
let mamba_value = match &params.mamba_value {
Some(mamba_value) => Some(mamba_value.bind(py).extract::<PyTensor>()?.0),
None => None,
};
let params = InsertParams {
rotation_base: params.rotation_base,
key,
namespace: KeyNamespaceRef::new(
params.extra_key.as_deref(),
params.cache_salt.as_deref(),
),
session_id: params.session_id.as_deref(),
value: value.0,
mamba_value,
prev_prefix_len: params.prev_prefix_len,
swa_evicted_seqlen: params.swa_evicted_seqlen,
swa_branching_seqlen: params.swa_branching_seqlen,
inserted_len: params.inserted_len,
priority: params.priority,
track_adopted_ranges: params.track_adopted_ranges,
};
let step = py
.detach(move || self.core().try_begin_insert(&params))
.map_err(tree_core_runtime_error)?;
InsertStepResultBinding::from_insert_step(py, step)
}
/// Continue the suspended insert after its step actions were applied.
fn resume_insert(&self, py: Python<'_>) -> PyResult<InsertStepResultBinding> {
let step = py
.detach(|| self.core().try_resume_insert())
.map_err(tree_core_runtime_error)?;
InsertStepResultBinding::from_insert_step(py, step)
}
/// Whether an insert walk is suspended at a barrier.
fn has_ongoing_insert(&self, py: Python<'_>) -> bool {
py.detach(|| self.core().has_ongoing_insert())
}
/// Finish the insert (idempotent); returns still-pending actions to drain.
fn end_insert(&self, py: Python<'_>) -> PyResult<Py<PyList>> {
let actions = py.detach(|| self.core().end_insert());
cache_actions_to_py(py, actions)
}
/// Bump the reference count on a node's component locks; the listed
/// components are left untaken and recorded in the receipt.
fn inc_lock_ref(
&self,
py: Python<'_>,
node_id: NodeId,
skip_lock_components: Vec<u8>,
) -> PyResult<IncLockRefResultBinding> {
let skip = component_set_from_py(&skip_lock_components)?;
let result = py
.detach(|| self.core().inc_lock_ref(node_id, skip))
.map_err(node_access_error)?;
Ok(IncLockRefResultBinding::from_result(result))
}
/// Pin only the FULL device values on a node's root path.
fn inc_full_pin(&self, py: Python<'_>, node_id: NodeId) -> PyResult<()> {
py.detach(|| self.core().inc_full_pin(node_id))
.map_err(node_access_error)
}
/// Release a FULL-only root-path pin.
fn dec_full_pin(&self, py: Python<'_>, node_id: NodeId) -> PyResult<()> {
py.detach(|| self.core().dec_full_pin(node_id))
.map_err(node_access_error)
}
/// Decrease the reference count on a node's component locks.
fn dec_lock_ref(
&self,
py: Python<'_>,
node_id: NodeId,
params: &DecLockRefParamsBinding,
skip_swa: bool,
) -> PyResult<()> {
let params = params.to_dec_lock_ref_params()?;
py.detach(|| self.core().dec_lock_ref(node_id, &params, skip_swa))
.map_err(node_access_error)?;
Ok(())
}
/// Release one window's lock, retaining other component locks.
fn dec_window_lock_only(
&self,
py: Python<'_>,
node_id: NodeId,
component_type: u8,
params: &DecLockRefParamsBinding,
) -> PyResult<(Py<PyDict>, Py<PyDict>)> {
let component_type = parse_component_type(component_type)?;
let params = params.to_dec_lock_ref_params()?;
let (device_frees, host_frees) = py
.detach(|| {
let mut device_frees = HashMap::new();
let mut host_frees = HashMap::new();
self.core().dec_window_lock_only(
node_id,
component_type,
&params,
&mut device_frees,
&mut host_frees,
)?;
Ok((device_frees, host_frees))
})
.map_err(node_access_error)?;
Ok((frees_to_py(py, device_frees)?, frees_to_py(py, host_frees)?))
}
/// Early-release the SWA portion of a request's tree lock; returns this
/// release's per-component (device_frees, host_frees).
fn dec_swa_lock_only(
&self,
py: Python<'_>,
node_id: NodeId,
params: &DecLockRefParamsBinding,
) -> PyResult<(Py<PyDict>, Py<PyDict>)> {
let params = params.to_dec_lock_ref_params()?;
let (device_frees, host_frees) = py
.detach(|| {
let mut device_frees = HashMap::new();
let mut host_frees = HashMap::new();
self.core().dec_swa_lock_only(
node_id,
&params,
&mut device_frees,
&mut host_frees,
)?;
Ok((device_frees, host_frees))
})
.map_err(node_access_error)?;
Ok((frees_to_py(py, device_frees)?, frees_to_py(py, host_frees)?))
}
/// Adjust the component's reclaimable device size.
fn adjust_component_size(
&self,
py: Python<'_>,
node_id: NodeId,
component_type: u8,
delta: i64,
) -> PyResult<()> {
let component_type = parse_component_type(component_type)?;
py.detach(|| {
self.core()
.adjust_component_size(node_id, component_type, delta)
})
.map_err(tree_core_runtime_error)
}
/// Collect paths under one tree lock.
fn collect_path_info(
&self,
py: Python<'_>,
end_node_ids: Vec<NodeId>,
) -> PyResult<Vec<Vec<TreePathNodeInfoTuple>>> {
let paths = py
.detach(|| self.core().collect_path_info(&end_node_ids))
.map_err(tree_core_runtime_error)?;
Ok(paths
.into_iter()
.map(|path| {
path.into_iter()
.map(|node| {
(
node.node_id,
node.parent_id,
node.path_start,
node.path_end,
node.full_value_len,
node.num_children,
)
})
.collect()
})
.collect())
}
/// Store a component's device value on a node (the SWA rebuild write-back).
fn set_component_device_value(
&self,
py: Python<'_>,
node_id: NodeId,
component_type: u8,
value: PyTensor,
) -> PyResult<()> {
let component_type = parse_component_type(component_type)?;
let value = value.0;
if value.kind() != Kind::Int64 {
return Err(PyValueError::new_err(format!(
"component device value must be an int64 tensor, got {:?}",
value.kind()
)));
}
if value.device() != self.device {
return Err(PyValueError::new_err(format!(
"component device value device {:?} does not match the tree core device {:?}",
value.device(),
self.device
)));
}
py.detach(|| {
self.core()
.set_component_device_value(node_id, component_type, value)
})
.map_err(node_access_error)?;
Ok(())
}
/// Logical key lengths for the nodes in a load-back transfer.
fn get_node_key_lengths(&self, py: Python<'_>, node_ids: Vec<NodeId>) -> PyResult<Vec<usize>> {
py.detach(|| self.core().get_node_key_lengths(&node_ids))
.map_err(node_access_error)
}
/// A component's device value on a node, if set.
fn get_component_device_value(
&self,
py: Python<'_>,
node_id: NodeId,
component_type: u8,
) -> PyResult<Option<Py<PyAny>>> {
let component_type = parse_component_type(component_type)?;
let value = py
.detach(|| {
self.core()
.get_component_device_value(node_id, component_type)
.map(|value| value.map(|tensor| tensor.shallow_clone()))
})
.map_err(node_access_error)?;
value.map(|value| tensor_to_py(py, value)).transpose()
}
// TODO(jialino): batch a full no-backup eviction round in Rust (one crossing
// instead of 2N+2); step-wise stays for the write-back interleave.
/// Begin a component's device-eviction walk for up to request_cnt tokens.
fn evict_device_start(
&self,
py: Python<'_>,
component_type: u8,
request_cnt: usize,
) -> PyResult<()> {
let ct = parse_component_type(component_type)?;
py.detach(|| self.core().evict_device_start(ct, request_cnt));
Ok(())
}
/// Advance one component eviction step. Progress without a leaf can be
/// an internal tombstone or a component backup request awaiting its finish;
/// no progress means the walk is exhausted.
fn evict_device_next_node(
&self,
py: Python<'_>,
component_type: u8,
tracker: HashMap<u8, usize>,
) -> PyResult<EvictDeviceNextNodeResultBinding> {
let ct = parse_component_type(component_type)?;
let baseline = tracker_from_py(tracker)?;
let (node_id, result) =
py.detach(move || self.core().evict_device_next_node(ct, &baseline));
let made_progress = node_id.is_some()
|| result.mamba_backup_node_id.is_some()
|| result.swa_backup_node_id.is_some()
|| !result.tracker.is_empty()
|| result.device_frees.values().any(|frees| !frees.is_empty())
|| result.host_frees.values().any(|frees| !frees.is_empty());
Ok(EvictDeviceNextNodeResultBinding {
node_id,
mamba_backup_node_id: result.mamba_backup_node_id,
swa_backup_node_id: result.swa_backup_node_id,
swa_backup_num_tokens: result.swa_backup_num_tokens,
made_progress,
unbacked_tokens: result.unbacked_tokens,
tracker: tracker_to_py(result.tracker),
new_device_frees: frees_to_py(py, result.device_frees)?,
new_host_frees: frees_to_py(py, result.host_frees)?,
})
}
/// Resume a pending internal Mamba tombstone after the host backup attempt.
fn finish_mamba_state_eviction(
&self,
py: Python<'_>,
node_id: NodeId,
) -> PyResult<EvictDeviceNextNodeResultBinding> {
let result = py.detach(move || self.core().finish_mamba_state_eviction(node_id));
self.internal_eviction_finish_to_py(py, result)
}
/// Resume a pending internal SWA tombstone after the host backup attempt.
fn finish_swa_state_eviction(
&self,
py: Python<'_>,
node_id: NodeId,
) -> PyResult<EvictDeviceNextNodeResultBinding> {
let result = py.detach(move || self.core().finish_swa_state_eviction(node_id));
self.internal_eviction_finish_to_py(py, result)
}
fn internal_eviction_finish_to_py(
&self,
py: Python<'_>,
result: EvictionStepResult,
) -> PyResult<EvictDeviceNextNodeResultBinding> {
Ok(EvictDeviceNextNodeResultBinding {
node_id: None,
mamba_backup_node_id: None,
swa_backup_node_id: None,
swa_backup_num_tokens: 0,
// Consuming the pending request advances the finite walk even
// when intervening I/O pinned or removed this particular node.
made_progress: true,
unbacked_tokens: result.unbacked_tokens,
tracker: tracker_to_py(result.tracker),
new_device_frees: frees_to_py(py, result.device_frees)?,
new_host_frees: frees_to_py(py, result.host_frees)?,
})
}
/// Evict one device leaf; an unbacked write-back leaf returns its backup
/// action for the caller to execute before demoting.
fn evict_device_leaf(
&self,
py: Python<'_>,
node_id: NodeId,
) -> PyResult<EvictDeviceLeafResultBinding> {
let (backup, result) = py
.detach(move || {
let mut core = self.core();
let is_write_back = core.is_write_back;
core.evict_device_leaf(node_id, is_write_back)
})
.map_err(node_access_error)?;
Ok(EvictDeviceLeafResultBinding {
unbacked_tokens: result.unbacked_tokens,
backup_kv: backup
.map(|backup| cache_action_to_py(py, CacheAction::BackupKV(backup)))
.transpose()?,
tracker: tracker_to_py(result.tracker),
new_device_frees: frees_to_py(py, result.device_frees)?,
new_host_frees: frees_to_py(py, result.host_frees)?,
})
}
/// Finish a component's device-eviction walk.
fn evict_device_end(&self, py: Python<'_>, component_type: u8) -> PyResult<()> {
let ct = parse_component_type(component_type)?;
py.detach(|| self.core().evict_device_end(ct));
Ok(())
}
/// Verify tree-structure, leaf-set, LRU, size, and ongoing-op invariants;
/// ongoing_* args are (id, node_id) pairs.
fn sanity_check(
&self,
py: Python<'_>,
ongoing_write_through: Vec<(i64, NodeId)>,
ongoing_load_back: Vec<(i64, NodeId)>,
) -> PyResult<()> {
py.detach(|| {
self.core()
.try_sanity_check(&ongoing_write_through, &ongoing_load_back)
})
.map_err(PyAssertionError::new_err)
}
/// Concatenated FULL device values from from_node up to (exclusive) until_node.
fn collect_full_device_indices(
&self,
py: Python<'_>,
from_node_id: NodeId,
until_node_id: NodeId,
) -> PyResult<Py<PyAny>> {
let value = py
.detach(|| {
self.core()
.collect_full_device_indices(from_node_id, until_node_id)
})
.map_err(node_access_error)?;
tensor_to_py(py, value)
}
/// Every FULL device value in the tree, concatenated.
fn all_values_flatten(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
let value = py.detach(|| self.core().all_values_flatten());
tensor_to_py(py, value)
}
/// Every Mamba device value in the tree, concatenated.
fn all_mamba_values_flatten(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
let value = py.detach(|| self.core().all_mamba_values_flatten());
tensor_to_py(py, value)
}
/// Flatten every FULL device slot into (slot, position, prev-slot) rows for the KV-canary sweep.
fn walk_for_kv_canary(
&self,
py: Python<'_>,
unlocked_only: bool,
swa_resident_only: bool,
) -> PyResult<KvCanaryWalkResultBinding> {
let result = py.detach(|| {
self.core()
.walk_for_kv_canary(unlocked_only, swa_resident_only)
});
Ok(KvCanaryWalkResultBinding {
slot_indices: tensor_to_py(py, Tensor::from_slice(&result.slot_indices))?,
positions: tensor_to_py(py, Tensor::from_slice(&result.positions))?,
prev_slot_indices: tensor_to_py(py, Tensor::from_slice(&result.prev_slot_indices))?,
})
}
/// Evictable token count of the FULL (base) component.
fn evictable_size(&self, py: Python<'_>) -> usize {
py.detach(|| self.core().evictable_size())
}
/// Protected (locked) token count of the FULL (base) component.
fn protected_size(&self, py: Python<'_>) -> usize {
py.detach(|| self.core().protected_size())
}
/// FULL component evictable token count.
fn full_evictable_size(&self, py: Python<'_>) -> usize {
py.detach(|| self.core().full_evictable_size())
}
/// FULL component protected token count.
fn full_protected_size(&self, py: Python<'_>) -> usize {
py.detach(|| self.core().full_protected_size())
}
/// Evictable token count for one component (0 if the component is absent).
fn component_evictable_size(&self, py: Python<'_>, component_type: u8) -> PyResult<usize> {
let ct = parse_component_type(component_type)?;
Ok(py.detach(|| self.core().component_evictable_size(ct)))
}
/// Protected token count for one component (0 if the component is absent).
fn component_protected_size(&self, py: Python<'_>, component_type: u8) -> PyResult<usize> {
let ct = parse_component_type(component_type)?;
Ok(py.detach(|| self.core().component_protected_size(ct)))
}
/// (full_tokens, aux_tokens) summed across the whole tree.
fn total_size(&self, py: Python<'_>) -> (usize, usize) {
py.detach(|| self.core().total_size())
}
/// Whether the node's FULL device value has been evicted.
fn is_full_device_evicted(&self, py: Python<'_>, node_id: NodeId) -> PyResult<bool> {
py.detach(|| self.core().is_full_device_evicted(node_id))
.map_err(node_access_error)
}
/// Mark the host tier as buffer-only; wired after the host pools are built.
fn set_host_memory_buffer_only(&self, py: Python<'_>) {
py.detach(|| self.core().set_host_memory_buffer_only());
}
/// Whether the host tier runs as a storage staging buffer, not a cache.
fn is_host_memory_buffer_only(&self, py: Python<'_>) -> bool {
py.detach(|| self.core().is_host_memory_buffer_only)
}
/// Mark the host tier (HiCache) as wired.
fn set_hicache_enabled(&self, py: Python<'_>) {
py.detach(|| self.core().set_hicache_enabled());
}
/// Whether the host tier (HiCache) is wired.
fn enable_hicache(&self, py: Python<'_>) -> bool {
py.detach(|| self.core().enable_hicache)
}
/// Mark the SWA host pool as wired (HiCache).
fn set_has_swa_host_pool(&self, py: Python<'_>) {
py.detach(|| self.core().set_has_swa_host_pool());
}
/// Whether the SWA host pool is wired.
fn has_swa_host_pool(&self, py: Python<'_>) -> bool {
py.detach(|| self.core().has_swa_host_pool)
}
/// Insert a host-side (backuped) tree path descending from the given node.
fn insert_host(
&self,
py: Python<'_>,
node_id: NodeId,
extra_key: Option<String>,
key: &Bound<'_, PyAny>,
host_value: PyTensor,
hash_value: Vec<String>,
cache_salt: Option<String>,
) -> PyResult<InsertResultBinding> {
let key = K::key_from(Cow::Owned(py_array_to_vec_i64(py, key)?)).into_owned();
let host_value = host_value.0;
if host_value.kind() != Kind::Int64 {
return Err(PyValueError::new_err(format!(
"insert_host host_value must be an int64 tensor, got {:?}",
host_value.kind()
)));
}
let result = py
.detach(move || {
self.core().insert_host_in_namespace(
node_id,
KeyNamespaceRef::new(extra_key.as_deref(), cache_salt.as_deref()),
key,
host_value,
hash_value,
)
})
.map_err(tree_core_runtime_error)?;
InsertResultBinding::from_insert_result(py, result)
}
/// Gather a node's device value plus per-component BACKUP_HOST transfers.
fn build_backup_spec(
&self,
py: Python<'_>,
node_id: NodeId,
) -> PyResult<(Py<PyAny>, Py<PyDict>)> {
let (device_value, comp_xfers) = py
.detach(|| self.core().build_backup_spec(node_id))
.map_err(node_access_error)?;
Ok((
tensor_to_py(py, device_value)?,
comp_xfers_to_py(py, comp_xfers)?,
))
}
/// Gather a node's device->storage backup spec; None if the node is not backuped.
fn build_storage_backup_spec(
&self,
py: Python<'_>,
node_id: NodeId,
pass_prefix_keys: bool,
) -> PyResult<Option<StorageBackupSpecBinding>> {
let spec = py
.detach(|| {
self.core()
.build_storage_backup_spec(node_id, pass_prefix_keys)
})
.map_err(node_access_error)?;
let Some(spec) = spec else {
return Ok(None);
};
let mut token_bytes = Vec::with_capacity(spec.token_ids.len() * 8);
for token in &spec.token_ids {
token_bytes.extend_from_slice(&token.to_ne_bytes());
}
Ok(Some(StorageBackupSpecBinding {
host_value: tensor_to_py(py, spec.host_value)?,
token_ids: PyBytes::new(py, &token_bytes).unbind(),
hash_value: spec.hash_value,
prefix_keys: spec.prefix_keys,
comp_xfers: comp_xfers_to_py(py, spec.comp_xfers)?,
}))
}
/// Route a build_hicache_transfers call to the component for the given type.
#[allow(clippy::too_many_arguments)]
fn build_hicache_transfers(
&self,
py: Python<'_>,
component_type: u8,
node_id: NodeId,
phase: &str,
host_indices: Option<PyTensor>,
token_ids: Option<Vec<i64>>,
prefetch_tokens: usize,
staging_tokens: usize,
last_hash: Option<String>,
) -> PyResult<Option<Vec<Py<PyAny>>>> {
let component_type = parse_component_type(component_type)?;
let phase = parse_transfer_phase(phase)?;
let host_indices = host_indices.map(|t| t.0);
let transfers = py
.detach(|| {
self.core().build_hicache_transfers(
component_type,
node_id,
phase,
host_indices,
token_ids.as_deref(),
prefetch_tokens,
staging_tokens,
last_hash.as_deref(),
)
})
.map_err(tree_core_assertion_error)?;
transfers
.map(|transfers| {
transfers
.into_iter()
.map(|transfer| transfer_to_py(py, transfer))
.collect::<PyResult<Vec<_>>>()
})
.transpose()
}
/// The anchor node's namespace; None for root-like anchors.
fn prefetch_anchor_info(
&self,
py: Python<'_>,
node_id: NodeId,
) -> PyResult<(Option<String>, Option<String>)> {
py.detach(|| self.core().prefetch_anchor_info(node_id))
.map_err(node_access_error)
}
/// Whether the node's Full KV is present on host.
fn node_backuped(&self, py: Python<'_>, node_id: NodeId) -> PyResult<bool> {
py.detach(|| self.core().node_backuped(node_id))
.map_err(node_access_error)
}
/// Whether the node is a (default or named) root.
fn is_root(&self, py: Python<'_>, node_id: NodeId) -> PyResult<bool> {
py.detach(|| self.core().is_root(node_id))
.map_err(node_access_error)
}
/// The node's last page hash, or None when it was never hashed.
fn get_last_hash_value(&self, py: Python<'_>, node_id: NodeId) -> PyResult<Option<String>> {
py.detach(|| self.core().get_last_hash_value(node_id))
.map_err(node_access_error)
}
/// The hash chain of the node's ancestors, in root-to-parent order.
fn get_prefix_hash_values(&self, py: Python<'_>, node_id: NodeId) -> PyResult<Vec<String>> {
py.detach(|| self.core().get_prefix_hash_values(node_id))
.map_err(node_access_error)
}
fn get_hash_values(&self, py: Python<'_>, node_id: NodeId) -> PyResult<Vec<String>> {
py.detach(|| self.core().get_hash_values(node_id))
.map_err(node_access_error)
}
fn snapshot_buffer_backup(
&self,
py: Python<'_>,
node_id: NodeId,
pass_prefix_keys: bool,
) -> Option<BufferBackupSnapshotBinding> {
py.detach(|| {
self.core()
.snapshot_buffer_backup(node_id, pass_prefix_keys)
})
.map(|snapshot| BufferBackupSnapshotBinding::from_snapshot(py, snapshot))
}
fn validate_buffer_backup(
&self,
py: Python<'_>,
node_id: NodeId,
expected_key_length: usize,
) -> Option<BufferBackupStateBinding> {
py.detach(|| {
self.core()
.validate_buffer_backup(node_id, expected_key_length)
})
.map(BufferBackupStateBinding::from)
}
/// Hash every node built while storage was disabled.
fn backfill_missing_hash_values(&self, py: Python<'_>) -> usize {
py.detach(|| self.core().backfill_missing_hash_values())
}
fn root_node_handle(&self, py: Python<'_>, extra_key: Option<String>) -> NodeId {
py.detach(|| self.core().root_node_handle(extra_key.as_deref()))
}
fn rotation_base_of(&self, py: Python<'_>, node_id: NodeId) -> PyResult<Option<i64>> {
py.detach(|| self.core().rotation_base_of(node_id))
.map_err(node_access_error)
}
fn dfs_weight_order(&self, py: Python<'_>, node_ids: Vec<NodeId>) -> PyResult<Vec<usize>> {
py.detach(|| self.core().dfs_weight_order(&node_ids))
.map_err(node_access_error)
}
/// Commit each component's HiCache transfers; returns the new cache actions.
fn commit_hicache_transfers(
&self,
py: Python<'_>,
node_id: NodeId,
phase: &str,
comp_xfers: HashMap<u8, Vec<TransferArgs>>,
insert_result: Option<(usize, Option<NodeId>, bool)>,
pool_storage_result: Option<(usize, HashMap<String, usize>)>,
) -> PyResult<(Py<PyList>, Option<bool>)> {
let phase = parse_transfer_phase(phase)?;
let comp_xfers = comp_xfers_from_args(comp_xfers)?;
let insert_result =
insert_result.map(
|(total_len, inserted_host_node, mamba_exist)| InsertResult {
total_len,
inserted_host_node,
mamba_exist,
..InsertResult::default()
},
);
let pool_storage_result = pool_storage_result
.map(|(kv_hit_pages, extra_pool_hit_pages)| {
Ok::<_, PyErr>(PoolTransferResult {
kv_hit_pages,
extra_pool_hit_pages: extra_pool_hit_pages
.into_iter()
.map(|(name, pages)| Ok((parse_pool_name(&name)?, pages)))
.collect::<PyResult<HashMap<_, _>>>()?,
})
})
.transpose()?;
let (cache_actions, mamba_exist) = py
.detach(move || {
let mut cache_actions = Vec::new();
let mut insert_result = insert_result;
self.core().commit_hicache_transfers(
node_id,
phase,
comp_xfers,
&mut cache_actions,
insert_result.as_mut(),
pool_storage_result.as_ref(),
)?;
Ok((
cache_actions,
insert_result.map(|result| result.mamba_exist),
))
})
.map_err(node_access_error)?;
Ok((cache_actions_to_py(py, cache_actions)?, mamba_exist))
}
/// Commit a successful backup to the node.
fn commit_backup(
&self,
py: Python<'_>,
node_id: NodeId,
host_indices: PyTensor,
comp_xfers: HashMap<u8, Vec<TransferArgs>>,
) -> PyResult<()> {
let comp_xfers = comp_xfers_from_args(comp_xfers)?;
let host_indices = host_indices.0;
py.detach(move || self.core().commit_backup(node_id, host_indices, comp_xfers))
.map_err(node_access_error)?;
Ok(())
}
/// Build the H->D load-back KV transfer plus per-component aux transfers.
fn build_load_back_spec(
&self,
py: Python<'_>,
node_id: NodeId,
mamba_pool_idx: Option<PyTensor>,
) -> PyResult<(Py<PyAny>, Py<PyDict>)> {
let req = Req {
mamba_pool_idx: mamba_pool_idx.map(|t| t.0),
};
let (kv_xfer, comp_xfers) = py
.detach(move || self.core().build_load_back_spec(node_id, Some(&req)))
.map_err(tree_core_assertion_error)?;
Ok((
transfer_to_py(py, kv_xfer)?,
comp_xfers_to_py(py, comp_xfers)?,
))
}
/// Commit a successful H->D load-back onto the node; returns its actions.
fn commit_load_back(
&self,
py: Python<'_>,
node_id: NodeId,
device_indices: PyTensor,
kv_xfer: TransferArgs,
comp_xfers: HashMap<u8, Vec<TransferArgs>>,
) -> PyResult<Py<PyList>> {
let kv_xfer = transfer_from_args(kv_xfer)?;
let comp_xfers = comp_xfers_from_args(comp_xfers)?;
let device_indices = device_indices.0;
let actions = py
.detach(move || {
self.core()
.commit_load_back(node_id, device_indices, kv_xfer, comp_xfers)
})
.map_err(node_access_error)?;
cache_actions_to_py(py, actions)
}
/// Release a node's device KV once its host copy exists.
fn demote(&self, py: Python<'_>, node_id: NodeId) -> PyResult<DemoteResultBinding> {
let result = py
.detach(move || self.core().demote(node_id))
.map_err(tree_core_assertion_error)?;
Ok(DemoteResultBinding {
tracker: tracker_to_py(result.tracker),
new_device_frees: frees_to_py(py, result.device_frees)?,
new_host_frees: frees_to_py(py, result.host_frees)?,
})
}
/// Evict up to num_tokens of one component's host resources.
fn drive_host_eviction(
&self,
py: Python<'_>,
component_type: u8,
num_tokens: usize,
skip_full_duplicate_reclaim: bool,
) -> PyResult<HostEvictionResultBinding> {
let ct = parse_component_type(component_type)?;
let result = py.detach(move || {
self.core().drive_host_eviction_with_options(
ct,
num_tokens,
skip_full_duplicate_reclaim,
)
});
Ok(HostEvictionResultBinding {
tracker: tracker_to_py(result.tracker),
new_device_frees: frees_to_py(py, result.device_frees)?,
new_host_frees: frees_to_py(py, result.host_frees)?,
})
}
/// Evict shallow Mamba device checkpoints beyond the per-path cap on the
/// tail's root path; returns the step's freed tensors.
fn evict_excess_path_states(
&self,
py: Python<'_>,
tail_node_id: NodeId,
) -> PyResult<HostEvictionResultBinding> {
let result = py
.detach(move || self.core().evict_excess_path_states(tail_node_id))
.map_err(node_access_error)?;
Ok(HostEvictionResultBinding {
tracker: tracker_to_py(result.tracker),
new_device_frees: frees_to_py(py, result.device_frees)?,
new_host_frees: frees_to_py(py, result.host_frees)?,
})
}
/// Bump the reference count on a node's host-side component locks.
fn inc_host_lock_ref(
&self,
py: Python<'_>,
node_id: NodeId,
) -> PyResult<IncLockRefResultBinding> {
let result = py
.detach(|| self.core().inc_host_lock_ref(node_id))
.map_err(node_access_error)?;
Ok(IncLockRefResultBinding::from_result(result))
}
/// Decrease the reference count on a node's host-side component locks.
fn dec_host_lock_ref(
&self,
py: Python<'_>,
node_id: NodeId,
params: &DecLockRefParamsBinding,
) -> PyResult<()> {
let params = params.to_dec_lock_ref_params()?;
py.detach(|| self.core().dec_host_lock_ref(node_id, &params))
.map_err(node_access_error)?;
Ok(())
}
fn is_write_through_compatible(&self, py: Python<'_>) -> bool {
py.detach(|| self.core().is_write_through_compatible())
}
/// Set the write-back (vs write-through) policy; decided at HiCache init.
fn set_is_write_back(&self, py: Python<'_>, is_write_back: bool) {
py.detach(|| self.core().is_write_back = is_write_back);
}
/// The current write-back (vs write-through) policy.
fn is_write_back(&self, py: Python<'_>) -> bool {
py.detach(|| self.core().is_write_back)
}
/// Set the write-through backup hit threshold; decided at HiCache init.
fn set_write_through_threshold(&self, py: Python<'_>, threshold: i64) {
py.detach(|| self.core().write_through_threshold = threshold);
}
/// The current write-through backup hit threshold.
fn write_through_threshold(&self, py: Python<'_>) -> i64 {
py.detach(|| self.core().write_through_threshold)
}
/// Mark the storage tier (L3) wired; storage attaches after tree construction.
fn set_enable_storage(&self, py: Python<'_>, value: bool) {
py.detach(|| self.core().set_enable_storage(value));
}
/// Whether the storage tier (L3) is wired.
fn enable_storage(&self, py: Python<'_>) -> bool {
py.detach(|| self.core().enable_storage)
}
/// Enable or disable the direct external-cache linker.
fn set_enable_external_cache_linker(&self, py: Python<'_>, value: bool) -> PyResult<()> {
py.detach(|| self.core().set_enable_external_cache_linker(value))
.map_err(tree_core_assertion_error)
}
/// Whether the direct external-cache linker is wired.
fn enable_external_cache_linker(&self, py: Python<'_>) -> bool {
py.detach(|| self.core().enable_external_cache_linker)
}
/// Queue the all-cleared placement event.
fn record_all_cleared_event(&self, py: Python<'_>) {
py.detach(|| self.core().record_all_cleared_event());
}
/// Drain the queued placement events as tagged tuples.
fn take_events(&self, py: Python<'_>) -> PyResult<Py<PyList>>
where
for<'py> Vec<K::Atom>: IntoPyObject<'py>,
{
let events = py.detach(|| self.core().take_events());
let list = PyList::empty(py);
for event in events {
match event {
KvCacheEvent::BlockStored {
block_hashes,
parent_block_hash,
token_ids,
block_size,
medium,
cache_salt,
session_id,
extra_key,
} => {
let item: Py<PyAny> = (
"block_stored",
block_hashes,
parent_block_hash,
token_ids,
block_size,
medium.as_str(),
cache_salt.map(|salt| salt.to_string()),
session_id.map(|session_id| session_id.to_string()),
extra_key.map(|extra_key| extra_key.to_string()),
)
.into_py_any(py)?;
list.append(item)?;
}
KvCacheEvent::BlockRemoved {
block_hashes,
medium,
extra_key,
} => {
let item: Py<PyAny> = (
"block_removed",
block_hashes,
medium.as_str(),
extra_key.map(|extra_key| extra_key.to_string()),
)
.into_py_any(py)?;
list.append(item)?;
}
KvCacheEvent::AllBlocksCleared => {
let item: Py<PyAny> = ("all_blocks_cleared",).into_py_any(py)?;
list.append(item)?;
}
}
}
Ok(list.unbind())
}
/// Drop the subtree rooted at an unbacked D-leaf; not dropped when a lock
/// blocks it.
fn drop_subtree_no_host(
&self,
py: Python<'_>,
node_id: NodeId,
) -> PyResult<DropSubtreeResultBinding> {
let (dropped, result) = py
.detach(move || self.core().drop_subtree_no_host(node_id))
.map_err(node_access_error)?;
Ok(DropSubtreeResultBinding {
dropped,
tracker: tracker_to_py(result.tracker),
new_device_frees: frees_to_py(py, result.device_frees)?,
new_host_frees: frees_to_py(py, result.host_frees)?,
})
}
/// Mark the nodes one write-through backup covers; returns them ancestors first.
fn mark_write_through_pending(
&self,
py: Python<'_>,
node_ids: Vec<NodeId>,
ack_id: NodeId,
) -> PyResult<Vec<NodeId>> {
py.detach(move || self.core().mark_write_through_pending(node_ids, ack_id))
.map_err(node_access_error)
}
/// Clear the write-through-pending mark on the acked nodes.
fn finish_write_through(
&self,
py: Python<'_>,
node_ids: Vec<NodeId>,
ack_id: NodeId,
) -> PyResult<()> {
py.detach(move || self.core().finish_write_through(node_ids, ack_id))
.map_err(node_access_error)
}
/// Clear the in-flight H->D marks on the anchor's root path at ack time.
fn finish_load_back(&self, py: Python<'_>, anchor_node_id: NodeId) -> PyResult<()> {
py.detach(|| self.core().finish_load_back(anchor_node_id))
.map_err(node_access_error)
}
/// Build transfers for a node with no stored or pending external copy.
fn build_external_linker_offload_transfers(
&self,
py: Python<'_>,
node_id: NodeId,
) -> PyResult<Option<Vec<Py<PyAny>>>> {
let transfers = py
.detach(|| self.core().build_external_linker_offload_transfers(node_id))
.map_err(node_access_error)?;
transfers
.map(|transfers| {
transfers
.into_iter()
.map(|transfer| transfer_to_py(py, transfer))
.collect::<PyResult<Vec<_>>>()
})
.transpose()
}
/// Mark an externally restored path, excluding its existing anchor.
fn mark_external_cache_stored_path(
&self,
py: Python<'_>,
from_node_id: NodeId,
until_node_id: NodeId,
) -> PyResult<()> {
py.detach(|| {
self.core()
.mark_external_cache_stored_path(from_node_id, until_node_id)
})
.map_err(tree_core_runtime_error)
}
/// Publish an accepted external offload as pending.
fn mark_external_linker_offload_pending(
&self,
py: Python<'_>,
node_id: NodeId,
) -> PyResult<()> {
py.detach(|| self.core().mark_external_linker_offload_pending(node_id))
.map_err(tree_core_assertion_error)
}
/// Finalize external-store state for an offload and its split fragments.
fn finish_external_linker_offload(
&self,
py: Python<'_>,
node_ids: Vec<NodeId>,
ack_id: NodeId,
success: bool,
) -> PyResult<()> {
py.detach(|| {
self.core()
.finish_external_linker_offload(&node_ids, ack_id, success)
})
.map_err(tree_core_assertion_error)
}
/// Order-sensitive digest of reclaimed coexisting host values.
fn write_back_coexist_reclaim_digest(&self, py: Python<'_>) -> i64 {
py.detach(|| self.core().write_back_coexist_reclaim_digest)
}
/// Whether the component's data is device-evicted but host-backed.
fn component_has_host_value_only(
&self,
py: Python<'_>,
node_id: NodeId,
component_type: u8,
) -> PyResult<bool> {
let ct = parse_component_type(component_type)?;
py.detach(|| self.core().component_has_host_value_only(node_id, ct))
.map_err(node_access_error)
}
}
#[cfg(feature = "inspection")]
impl<K: ChildKeyType + Send + Sync> TreeCoreBinding<K> {
fn inspect_contains_node(&self, py: Python<'_>, node_id: NodeId) -> bool {
py.detach(|| self.core().inspect_contains_node(node_id))
}
fn inspect_get_parent_node_id(
&self,
py: Python<'_>,
node_id: NodeId,
) -> PyResult<Option<NodeId>> {
py.detach(|| self.core().inspect_get_parent_node_id(node_id))
.map_err(node_access_error)
}
fn inspect_get_child_node_ids(&self, py: Python<'_>, node_id: NodeId) -> PyResult<Vec<NodeId>> {
py.detach(|| self.core().inspect_get_child_node_ids(node_id))
.map_err(node_access_error)
}
fn inspect_get_node_key_length(&self, py: Python<'_>, node_id: NodeId) -> PyResult<usize> {
py.detach(|| self.core().inspect_get_node_key_length(node_id))
.map_err(node_access_error)
}
fn inspect_get_node_token_ids(&self, py: Python<'_>, node_id: NodeId) -> PyResult<Vec<i64>> {
py.detach(|| self.core().inspect_get_node_token_ids(node_id))
.map_err(node_access_error)
}
fn inspect_is_node_key_bigram(&self, py: Python<'_>, node_id: NodeId) -> PyResult<bool> {
py.detach(|| self.core().inspect_is_node_key_bigram(node_id))
.map_err(node_access_error)
}
fn inspect_get_component_host_value(
&self,
py: Python<'_>,
node_id: NodeId,
component_type: u8,
) -> PyResult<Option<Py<PyAny>>> {
let component_type = parse_component_type(component_type)?;
let value = py
.detach(|| {
self.core()
.inspect_get_component_host_value(node_id, component_type)
})
.map_err(node_access_error)?;
value.map(|value| tensor_to_py(py, value)).transpose()
}
fn inspect_get_component_device_lock_ref(
&self,
py: Python<'_>,
node_id: NodeId,
component_type: u8,
) -> PyResult<u32> {
let component_type = parse_component_type(component_type)?;
py.detach(|| {
self.core()
.inspect_get_component_device_lock_ref(node_id, component_type)
})
.map_err(node_access_error)
}
fn inspect_get_component_host_lock_ref(
&self,
py: Python<'_>,
node_id: NodeId,
component_type: u8,
) -> PyResult<u32> {
let component_type = parse_component_type(component_type)?;
py.detach(|| {
self.core()
.inspect_get_component_host_lock_ref(node_id, component_type)
})
.map_err(node_access_error)
}
fn inspect_get_node_hit_count(&self, py: Python<'_>, node_id: NodeId) -> PyResult<i64> {
py.detach(|| self.core().inspect_get_node_hit_count(node_id))
.map_err(node_access_error)
}
fn inspect_get_write_through_pending_id(
&self,
py: Python<'_>,
node_id: NodeId,
) -> PyResult<Option<usize>> {
py.detach(|| self.core().inspect_get_write_through_pending_id(node_id))
.map_err(node_access_error)
}
fn inspect_is_external_cache_stored(&self, py: Python<'_>, node_id: NodeId) -> PyResult<bool> {
py.detach(|| self.core().inspect_is_external_cache_stored(node_id))
.map_err(node_access_error)
}
fn inspect_is_node_in_device_lru(
&self,
py: Python<'_>,
node_id: NodeId,
component_type: u8,
) -> PyResult<bool> {
let component_type = parse_component_type(component_type)?;
py.detach(|| {
self.core()
.inspect_is_node_in_device_lru(node_id, component_type)
})
.map_err(node_access_error)
}
fn inspect_is_node_in_host_lru(
&self,
py: Python<'_>,
node_id: NodeId,
component_type: u8,
) -> PyResult<bool> {
let component_type = parse_component_type(component_type)?;
py.detach(|| {
self.core()
.inspect_is_node_in_host_lru(node_id, component_type)
})
.map_err(node_access_error)
}
fn inspect_get_component_device_lru_node_ids(
&self,
py: Python<'_>,
component_type: u8,
) -> PyResult<Vec<NodeId>> {
let component_type = parse_component_type(component_type)?;
Ok(py.detach(|| {
self.core()
.inspect_get_component_device_lru_node_ids(component_type)
}))
}
fn inspect_is_device_evictable_leaf(&self, py: Python<'_>, node_id: NodeId) -> bool {
py.detach(|| self.core().inspect_is_device_evictable_leaf(node_id))
}
fn inspect_is_host_evictable_leaf(&self, py: Python<'_>, node_id: NodeId) -> bool {
py.detach(|| self.core().inspect_is_host_evictable_leaf(node_id))
}
fn inspect_is_device_leaf(&self, py: Python<'_>, node_id: NodeId) -> PyResult<bool> {
py.detach(|| self.core().inspect_is_device_leaf(node_id))
.map_err(node_access_error)
}
fn inspect_get_all_node_ids(&self, py: Python<'_>) -> Vec<NodeId> {
py.detach(|| self.core().inspect_get_all_node_ids())
}
fn inspect_component_protected_size(
&self,
py: Python<'_>,
component_type: u8,
) -> PyResult<usize> {
let component_type = parse_component_type(component_type)?;
Ok(py.detach(|| self.core().inspect_component_protected_size(component_type)))
}
fn inspect_set_node_hash_values(
&self,
py: Python<'_>,
node_id: NodeId,
hash_values: Option<Vec<String>>,
) -> PyResult<()> {
py.detach(move || {
self.core()
.inspect_set_node_hash_values(node_id, hash_values)
})
.map_err(node_access_error)
}
fn inspect_set_component_device_value_raw(
&self,
py: Python<'_>,
node_id: NodeId,
component_type: u8,
value: Option<PyTensor>,
) -> PyResult<()> {
let component_type = parse_component_type(component_type)?;
let value = value.map(|value| value.0);
py.detach(move || {
self.core()
.inspect_set_component_device_value_raw(node_id, component_type, value)
})
.map_err(node_access_error)
}
fn inspect_set_component_host_value_raw(
&self,
py: Python<'_>,
node_id: NodeId,
component_type: u8,
value: Option<PyTensor>,
) -> PyResult<()> {
let component_type = parse_component_type(component_type)?;
let value = value.map(|value| value.0);
py.detach(move || {
self.core()
.inspect_set_component_host_value_raw(node_id, component_type, value)
})
.map_err(node_access_error)
}
fn inspect_set_component_device_lock_ref(
&self,
py: Python<'_>,
node_id: NodeId,
component_type: u8,
lock_ref: u32,
) -> PyResult<()> {
let component_type = parse_component_type(component_type)?;
py.detach(|| {
self.core()
.inspect_set_component_device_lock_ref(node_id, component_type, lock_ref)
})
.map_err(node_access_error)
}
fn inspect_remove_node_from_device_lru(
&self,
py: Python<'_>,
node_id: NodeId,
component_type: u8,
) -> PyResult<()> {
let component_type = parse_component_type(component_type)?;
py.detach(|| {
self.core()
.inspect_remove_node_from_device_lru(node_id, component_type)
})
.map_err(node_access_error)
}
fn inspect_insert_node_into_host_lru(
&self,
py: Python<'_>,
node_id: NodeId,
component_type: u8,
) -> PyResult<()> {
let component_type = parse_component_type(component_type)?;
py.detach(|| {
self.core()
.inspect_insert_node_into_host_lru(node_id, component_type)
})
.map_err(node_access_error)
}
fn inspect_set_component_evictable_size(
&self,
py: Python<'_>,
component_type: u8,
value: usize,
) -> PyResult<()> {
let component_type = parse_component_type(component_type)?;
py.detach(|| {
self.core()
.inspect_set_component_evictable_size(component_type, value)
});
Ok(())
}
fn inspect_set_component_protected_size(
&self,
py: Python<'_>,
component_type: u8,
value: usize,
) -> PyResult<()> {
let component_type = parse_component_type(component_type)?;
py.detach(|| {
self.core()
.inspect_set_component_protected_size(component_type, value)
});
Ok(())
}
fn inspect_update_duplicate_tracking(&self, py: Python<'_>, node_id: NodeId) -> PyResult<()> {
py.detach(|| self.core().inspect_update_duplicate_tracking(node_id))
.map_err(node_access_error)
}
fn inspect_advance_insert_walk_once(&self, py: Python<'_>) -> PyResult<()> {
py.detach(|| self.core().inspect_advance_insert_walk_once())
.map_err(PyRuntimeError::new_err)
}
fn inspect_evict_component(
&self,
py: Python<'_>,
node_id: NodeId,
component_type: u8,
target: u8,
) -> PyResult<HostEvictionResultBinding> {
let component_type = parse_component_type(component_type)?;
let target = parse_evict_layer(target)?;
let result = py
.detach(|| {
self.core()
.inspect_evict_component(node_id, component_type, target)
})
.map_err(node_access_error)?;
HostEvictionResultBinding::from_eviction_step(py, result)
}
fn inspect_validate_cascade_evict(
&self,
py: Python<'_>,
node_id: NodeId,
component_type: u8,
target: u8,
) -> PyResult<()> {
let component_type = parse_component_type(component_type)?;
let target = parse_evict_layer(target)?;
py.detach(|| {
self.core()
.inspect_validate_cascade_evict(node_id, component_type, target)
})
.map_err(tree_core_assertion_error)
}
fn inspect_cleanup_tombstone_ancestors(
&self,
py: Python<'_>,
node_id: NodeId,
) -> PyResult<HostEvictionResultBinding> {
let result = py
.detach(|| self.core().inspect_cleanup_tombstone_ancestors(node_id))
.map_err(node_access_error)?;
HostEvictionResultBinding::from_eviction_step(py, result)
}
#[allow(clippy::too_many_arguments)]
fn inspect_finalize_component_match_result(
&self,
py: Python<'_>,
component_type: u8,
result: InspectionMatchResultInput,
key: &Bound<'_, PyAny>,
extra_key: Option<String>,
cache_salt: Option<String>,
value_chunks: Vec<PyTensor>,
best_value_len: usize,
) -> PyResult<MatchResultBinding> {
let component_type = parse_component_type(component_type)?;
let key = K::key_from(Cow::Owned(py_array_to_vec_i64(py, key)?)).into_owned();
let InspectionMatchResultInput {
device_prefix_len,
last_device_node: last_device_node_id,
last_host_node: last_host_node_id,
best_match_node: best_match_node_id,
host_hit_length,
swa_host_hit_length,
swa_branching_seqlen,
mamba_host_hit_length,
mamba_branching_seqlen,
full_kv_hit_length,
} = result;
let result = MatchResult {
device_prefix_len,
last_device_node_id,
last_host_node_id,
best_match_node_id,
host_hit_length,
swa_host_hit_length,
swa_branching_seqlen,
mamba_host_hit_length,
mamba_branching_seqlen,
full_kv_hit_length,
cache_actions: Vec::new(),
};
let value_chunks = value_chunks
.into_iter()
.map(|value| value.0)
.collect::<Vec<_>>();
let result = py.detach(move || {
let params = MatchPrefixParams {
key: &key,
namespace: KeyNamespaceRef::new(extra_key.as_deref(), cache_salt.as_deref()),
};
self.core().inspect_finalize_component_match_result(
component_type,
result,
&params,
&value_chunks,
best_value_len,
)
});
let result = result.map_err(node_access_error)?;
MatchResultBinding::from_match_result(py, result)
}
fn inspect_build_backup_node_ids(
&self,
py: Python<'_>,
node_id: NodeId,
write_back: bool,
) -> PyResult<Vec<NodeId>> {
py.detach(|| {
self.core()
.inspect_build_backup_node_ids(node_id, write_back)
})
.map_err(node_access_error)
}
}
impl<K: ChildKeyType + Send + Sync> TreeCoreBinding<K> {
/// Print the tree structure for debugging.
fn pretty_print(&self, py: Python<'_>) {
py.detach(|| self.core().pretty_print());
}
}
// The delegate surface is identical for every key type; the macro stamps the
// pyclass + pymethods pair per concrete key.
macro_rules! tree_core_binding {
($(#[$doc:meta])* $name:ident, $key:ty) => {
$(#[$doc])*
#[pyclass]
pub struct $name {
inner: TreeCoreBinding<$key>,
}
#[pymethods]
impl $name {
/// Build a tree core for the given component types from the cache's
/// init params.
#[new]
fn new(init_params: &TreeCoreInitParamsBinding, component_types: Vec<u8>) -> PyResult<Self> {
catch_native_panic(|| {
Ok($name {
inner: TreeCoreBinding::new(init_params, component_types)?,
})
})
}
/// Drop the entire tree and reinitialize empty state.
fn reset(&self, py: Python<'_>) -> PyResult<()> {
catch_native_panic(|| {
self.inner.reset(py);
Ok(())
})
}
/// Match a key against the tree.
fn match_prefix(
&self,
py: Python<'_>,
params: &MatchParamsBinding,
) -> PyResult<MatchResultBinding> {
catch_native_panic(|| self.inner.match_prefix(py, params))
}
/// Read-only FULL-device match, independent of auxiliary components.
fn match_full_device_prefix(
&self,
py: Python<'_>,
params: &MatchParamsBinding,
) -> PyResult<(usize, NodeId, usize)> {
catch_native_panic(|| Ok(self.inner.match_full_device_prefix(py, params)))
}
fn swa_tombstone_ranges(
&self,
py: Python<'_>,
params: &MatchParamsBinding,
start: usize,
end: usize,
) -> PyResult<Vec<(usize, usize)>> {
catch_native_panic(|| self.inner.swa_tombstone_ranges(py, params, start, end))
}
fn attach_swa_window(
&self,
py: Python<'_>,
params: &MatchParamsBinding,
window_start: usize,
window_end: usize,
swa_values: PyTensor,
) -> PyResult<Py<PyList>> {
catch_native_panic(|| {
self.inner
.attach_swa_window(py, params, window_start, window_end, swa_values)
})
}
/// The empty match result anchored at the root.
fn empty_match_result(&self, py: Python<'_>) -> PyResult<MatchResultBinding> {
catch_native_panic(|| self.inner.empty_match_result(py))
}
/// Insert device values into the tree per the provided key.
fn insert(
&self,
py: Python<'_>,
params: &InsertParamsBinding,
) -> PyResult<InsertResultBinding> {
catch_native_panic(|| self.inner.insert(py, params))
}
/// Start the resumable insert, running to its first barrier or completion.
fn begin_insert(
&self,
py: Python<'_>,
params: &InsertParamsBinding,
) -> PyResult<InsertStepResultBinding> {
catch_native_panic(|| self.inner.begin_insert(py, params))
}
/// Continue the suspended insert after its step actions were applied.
fn resume_insert(&self, py: Python<'_>) -> PyResult<InsertStepResultBinding> {
catch_native_panic(|| self.inner.resume_insert(py))
}
/// Whether an insert walk is suspended at a barrier.
fn has_ongoing_insert(&self, py: Python<'_>) -> PyResult<bool> {
catch_native_panic(|| Ok(self.inner.has_ongoing_insert(py)))
}
/// Finish the insert (idempotent); returns still-pending actions to drain.
fn end_insert(&self, py: Python<'_>) -> PyResult<Py<PyList>> {
catch_native_panic(|| self.inner.end_insert(py))
}
/// Bump the reference count on a node's component locks.
#[pyo3(signature = (node_id, skip_lock_components = Vec::new()))]
fn inc_lock_ref(
&self,
py: Python<'_>,
node_id: NodeId,
skip_lock_components: Vec<u8>,
) -> PyResult<IncLockRefResultBinding> {
catch_native_panic(|| self.inner.inc_lock_ref(py, node_id, skip_lock_components))
}
/// Pin only the FULL device values on a node's root path.
fn inc_full_pin(&self, py: Python<'_>, node_id: NodeId) -> PyResult<()> {
catch_native_panic(|| self.inner.inc_full_pin(py, node_id))
}
/// Release a FULL-only root-path pin.
fn dec_full_pin(&self, py: Python<'_>, node_id: NodeId) -> PyResult<()> {
catch_native_panic(|| self.inner.dec_full_pin(py, node_id))
}
/// Decrease the reference count on a node's component locks. The
/// receipt is required: a release must replay its acquire's evidence.
#[pyo3(signature = (node_id, params, skip_swa = false))]
fn dec_lock_ref(
&self,
py: Python<'_>,
node_id: NodeId,
params: &DecLockRefParamsBinding,
skip_swa: bool,
) -> PyResult<()> {
catch_native_panic(|| self.inner.dec_lock_ref(py, node_id, params, skip_swa))
}
/// Release one window's lock, retaining other component locks.
fn dec_window_lock_only(
&self,
py: Python<'_>,
node_id: NodeId,
component_type: u8,
params: &DecLockRefParamsBinding,
) -> PyResult<(Py<PyDict>, Py<PyDict>)> {
catch_native_panic(|| {
self.inner
.dec_window_lock_only(py, node_id, component_type, params)
})
}
/// Early-release the SWA portion of a request's tree lock; returns this
/// release's per-component (device_frees, host_frees).
#[pyo3(signature = (node_id, params))]
fn dec_swa_lock_only(
&self,
py: Python<'_>,
node_id: NodeId,
params: &DecLockRefParamsBinding,
) -> PyResult<(Py<PyDict>, Py<PyDict>)> {
catch_native_panic(|| self.inner.dec_swa_lock_only(py, node_id, params))
}
/// Adjust the component's reclaimable device size.
fn adjust_component_size(
&self,
py: Python<'_>,
node_id: NodeId,
component_type: u8,
delta: i64,
) -> PyResult<()> {
catch_native_panic(|| {
self.inner.adjust_component_size(py, node_id, component_type, delta)
})
}
/// Collect paths under one tree lock.
fn collect_path_info(
&self,
py: Python<'_>,
end_node_ids: Vec<NodeId>,
) -> PyResult<Vec<Vec<TreePathNodeInfoTuple>>> {
catch_native_panic(|| self.inner.collect_path_info(py, end_node_ids))
}
/// Store a component's device value on a node (the SWA rebuild write-back).
fn set_component_device_value(
&self,
py: Python<'_>,
node_id: NodeId,
component_type: u8,
value: PyTensor,
) -> PyResult<()> {
catch_native_panic(|| {
self.inner
.set_component_device_value(py, node_id, component_type, value)
})
}
/// Logical key lengths for the nodes in a load-back transfer.
fn get_node_key_lengths(&self, py: Python<'_>, node_ids: Vec<NodeId>) -> PyResult<Vec<usize>> {
catch_native_panic(|| self.inner.get_node_key_lengths(py, node_ids))
}
/// A component's device value on a node, if set.
fn get_component_device_value(
&self,
py: Python<'_>,
node_id: NodeId,
component_type: u8,
) -> PyResult<Option<Py<PyAny>>> {
catch_native_panic(|| {
self.inner
.get_component_device_value(py, node_id, component_type)
})
}
/// Begin a component's device-eviction walk for up to request_cnt tokens.
fn evict_device_start(
&self,
py: Python<'_>,
component_type: u8,
request_cnt: usize,
) -> PyResult<()> {
catch_native_panic(|| {
self.inner
.evict_device_start(py, component_type, request_cnt)
})
}
/// The next device leaf to evict, or None when the walk is done; the
/// passed running tracker gates the budget, and the result carries
/// this step's deltas.
fn evict_device_next_node(
&self,
py: Python<'_>,
component_type: u8,
tracker: HashMap<u8, usize>,
) -> PyResult<EvictDeviceNextNodeResultBinding> {
catch_native_panic(|| {
self.inner
.evict_device_next_node(py, component_type, tracker)
})
}
/// Finish an internal Mamba eviction after its host backup attempt.
fn finish_mamba_state_eviction(
&self,
py: Python<'_>,
node_id: NodeId,
) -> PyResult<EvictDeviceNextNodeResultBinding> {
catch_native_panic(|| self.inner.finish_mamba_state_eviction(py, node_id))
}
/// Finish an internal SWA eviction after its host backup attempt.
fn finish_swa_state_eviction(
&self,
py: Python<'_>,
node_id: NodeId,
) -> PyResult<EvictDeviceNextNodeResultBinding> {
catch_native_panic(|| self.inner.finish_swa_state_eviction(py, node_id))
}
/// Evict one device leaf; an unbacked write-back leaf returns its backup
/// action for the caller to execute before demoting.
fn evict_device_leaf(
&self,
py: Python<'_>,
node_id: NodeId,
) -> PyResult<EvictDeviceLeafResultBinding> {
catch_native_panic(|| self.inner.evict_device_leaf(py, node_id))
}
/// Finish a component's device-eviction walk.
fn evict_device_end(&self, py: Python<'_>, component_type: u8) -> PyResult<()> {
catch_native_panic(|| self.inner.evict_device_end(py, component_type))
}
/// Verify tree-structure, leaf-set, LRU, size, and ongoing-op invariants;
/// ongoing_* args are (id, node_id) pairs.
fn sanity_check(
&self,
py: Python<'_>,
ongoing_write_through: Vec<(i64, NodeId)>,
ongoing_load_back: Vec<(i64, NodeId)>,
) -> PyResult<()> {
catch_native_panic(|| {
self.inner
.sanity_check(py, ongoing_write_through, ongoing_load_back)
})
}
/// Concatenated FULL device values from from_node up to (exclusive) until_node.
fn collect_full_device_indices(
&self,
py: Python<'_>,
from_node_id: NodeId,
until_node_id: NodeId,
) -> PyResult<Py<PyAny>> {
catch_native_panic(|| {
self.inner
.collect_full_device_indices(py, from_node_id, until_node_id)
})
}
/// Every FULL device value in the tree, concatenated.
fn all_values_flatten(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
catch_native_panic(|| self.inner.all_values_flatten(py))
}
/// Every Mamba device value in the tree, concatenated.
fn all_mamba_values_flatten(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
catch_native_panic(|| self.inner.all_mamba_values_flatten(py))
}
/// Flatten every FULL device slot into (slot, position, prev-slot) rows for the KV-canary sweep.
fn walk_for_kv_canary(
&self,
py: Python<'_>,
unlocked_only: bool,
swa_resident_only: bool,
) -> PyResult<KvCanaryWalkResultBinding> {
catch_native_panic(|| {
self.inner
.walk_for_kv_canary(py, unlocked_only, swa_resident_only)
})
}
/// Evictable token count of the FULL (base) component.
fn evictable_size(&self, py: Python<'_>) -> PyResult<usize> {
catch_native_panic(|| Ok(self.inner.evictable_size(py)))
}
/// Protected (locked) token count of the FULL (base) component.
fn protected_size(&self, py: Python<'_>) -> PyResult<usize> {
catch_native_panic(|| Ok(self.inner.protected_size(py)))
}
/// FULL component evictable token count.
fn full_evictable_size(&self, py: Python<'_>) -> PyResult<usize> {
catch_native_panic(|| Ok(self.inner.full_evictable_size(py)))
}
/// FULL component protected token count.
fn full_protected_size(&self, py: Python<'_>) -> PyResult<usize> {
catch_native_panic(|| Ok(self.inner.full_protected_size(py)))
}
/// Evictable token count for one component (0 if the component is absent).
fn component_evictable_size(&self, py: Python<'_>, component_type: u8) -> PyResult<usize> {
catch_native_panic(|| self.inner.component_evictable_size(py, component_type))
}
/// Protected token count for one component (0 if the component is absent).
fn component_protected_size(&self, py: Python<'_>, component_type: u8) -> PyResult<usize> {
catch_native_panic(|| self.inner.component_protected_size(py, component_type))
}
/// (full_tokens, aux_tokens) summed across the whole tree.
fn total_size(&self, py: Python<'_>) -> PyResult<(usize, usize)> {
catch_native_panic(|| Ok(self.inner.total_size(py)))
}
/// Whether the node's FULL device value has been evicted.
fn is_full_device_evicted(&self, py: Python<'_>, node_id: NodeId) -> PyResult<bool> {
catch_native_panic(|| self.inner.is_full_device_evicted(py, node_id))
}
/// Mark the host tier as buffer-only; wired after the host pools are built.
fn set_host_memory_buffer_only(&self, py: Python<'_>) -> PyResult<()> {
catch_native_panic(|| {
self.inner.set_host_memory_buffer_only(py);
Ok(())
})
}
/// Whether the host tier runs as a storage staging buffer, not a cache.
fn is_host_memory_buffer_only(&self, py: Python<'_>) -> PyResult<bool> {
catch_native_panic(|| Ok(self.inner.is_host_memory_buffer_only(py)))
}
/// Mark the host tier (HiCache) as wired.
fn set_hicache_enabled(&self, py: Python<'_>) -> PyResult<()> {
catch_native_panic(|| {
self.inner.set_hicache_enabled(py);
Ok(())
})
}
/// Whether the host tier (HiCache) is wired.
fn enable_hicache(&self, py: Python<'_>) -> PyResult<bool> {
catch_native_panic(|| Ok(self.inner.enable_hicache(py)))
}
/// Mark the SWA host pool as wired (HiCache).
fn set_has_swa_host_pool(&self, py: Python<'_>) -> PyResult<()> {
catch_native_panic(|| {
self.inner.set_has_swa_host_pool(py);
Ok(())
})
}
/// Whether the SWA host pool is wired.
fn has_swa_host_pool(&self, py: Python<'_>) -> PyResult<bool> {
catch_native_panic(|| Ok(self.inner.has_swa_host_pool(py)))
}
/// Insert a host-side (backuped) tree path descending from the given node.
#[pyo3(signature = (node_id, extra_key, key, host_value, hash_value, cache_salt = None))]
fn insert_host(
&self,
py: Python<'_>,
node_id: NodeId,
extra_key: Option<String>,
key: &Bound<'_, PyAny>,
host_value: PyTensor,
hash_value: Vec<String>,
cache_salt: Option<String>,
) -> PyResult<InsertResultBinding> {
catch_native_panic(|| {
self.inner.insert_host(
py, node_id, extra_key, key, host_value, hash_value, cache_salt,
)
})
}
/// Gather a node's device value plus per-component BACKUP_HOST transfers.
fn build_backup_spec(
&self,
py: Python<'_>,
node_id: NodeId,
) -> PyResult<(Py<PyAny>, Py<PyDict>)> {
catch_native_panic(|| self.inner.build_backup_spec(py, node_id))
}
/// Gather a node's device->storage backup spec; None if the node is not backuped.
fn build_storage_backup_spec(
&self,
py: Python<'_>,
node_id: NodeId,
pass_prefix_keys: bool,
) -> PyResult<Option<StorageBackupSpecBinding>> {
catch_native_panic(|| {
self.inner
.build_storage_backup_spec(py, node_id, pass_prefix_keys)
})
}
/// Route a build_hicache_transfers call to the component for the given type.
#[allow(clippy::too_many_arguments)]
#[pyo3(signature = (component_type, node_id, phase, host_indices = None, token_ids = None, prefetch_tokens = 0, staging_tokens = 0, last_hash = None))]
fn build_hicache_transfers(
&self,
py: Python<'_>,
component_type: u8,
node_id: NodeId,
phase: &str,
host_indices: Option<PyTensor>,
token_ids: Option<Vec<i64>>,
prefetch_tokens: usize,
staging_tokens: usize,
last_hash: Option<String>,
) -> PyResult<Option<Vec<Py<PyAny>>>> {
catch_native_panic(|| {
self.inner.build_hicache_transfers(
py,
component_type,
node_id,
phase,
host_indices,
token_ids,
prefetch_tokens,
staging_tokens,
last_hash,
)
})
}
/// The anchor node's caller-defined key and cache salt.
fn prefetch_anchor_info(
&self,
py: Python<'_>,
node_id: NodeId,
) -> PyResult<(Option<String>, Option<String>)> {
catch_native_panic(|| self.inner.prefetch_anchor_info(py, node_id))
}
/// Whether the node's Full KV is present on host.
fn node_backuped(&self, py: Python<'_>, node_id: NodeId) -> PyResult<bool> {
catch_native_panic(|| self.inner.node_backuped(py, node_id))
}
/// Whether the node is a (default or named) root.
fn is_root(&self, py: Python<'_>, node_id: NodeId) -> PyResult<bool> {
catch_native_panic(|| self.inner.is_root(py, node_id))
}
/// The node's last page hash, or None when it was never hashed.
fn get_last_hash_value(&self, py: Python<'_>, node_id: NodeId) -> PyResult<Option<String>> {
catch_native_panic(|| self.inner.get_last_hash_value(py, node_id))
}
/// The hash chain of the node's ancestors, in root-to-parent order.
fn get_prefix_hash_values(&self, py: Python<'_>, node_id: NodeId) -> PyResult<Vec<String>> {
catch_native_panic(|| self.inner.get_prefix_hash_values(py, node_id))
}
fn get_hash_values(&self, py: Python<'_>, node_id: NodeId) -> PyResult<Vec<String>> {
catch_native_panic(|| self.inner.get_hash_values(py, node_id))
}
fn snapshot_buffer_backup(
&self,
py: Python<'_>,
node_id: NodeId,
pass_prefix_keys: bool,
) -> PyResult<Option<BufferBackupSnapshotBinding>> {
catch_native_panic(|| {
Ok(self
.inner
.snapshot_buffer_backup(py, node_id, pass_prefix_keys))
})
}
fn validate_buffer_backup(
&self,
py: Python<'_>,
node_id: NodeId,
expected_key_length: usize,
) -> PyResult<Option<BufferBackupStateBinding>> {
catch_native_panic(|| {
Ok(self
.inner
.validate_buffer_backup(py, node_id, expected_key_length))
})
}
/// Hash every node built while storage was disabled.
fn backfill_missing_hash_values(&self, py: Python<'_>) -> PyResult<usize> {
catch_native_panic(|| Ok(self.inner.backfill_missing_hash_values(py)))
}
#[pyo3(signature = (extra_key = None))]
fn root_node_handle(&self, py: Python<'_>, extra_key: Option<String>) -> PyResult<NodeId> {
catch_native_panic(|| Ok(self.inner.root_node_handle(py, extra_key)))
}
fn rotation_base_of(&self, py: Python<'_>, node_id: NodeId) -> PyResult<Option<i64>> {
catch_native_panic(|| self.inner.rotation_base_of(py, node_id))
}
fn dfs_weight_order(&self, py: Python<'_>, node_ids: Vec<NodeId>) -> PyResult<Vec<usize>> {
catch_native_panic(|| self.inner.dfs_weight_order(py, node_ids))
}
/// Commit each component's HiCache transfers; returns the new cache actions.
#[pyo3(signature = (node_id, phase, comp_xfers, insert_result = None, pool_storage_result = None))]
fn commit_hicache_transfers(
&self,
py: Python<'_>,
node_id: NodeId,
phase: &str,
comp_xfers: HashMap<u8, Vec<TransferArgs>>,
insert_result: Option<(usize, Option<NodeId>, bool)>,
pool_storage_result: Option<(usize, HashMap<String, usize>)>,
) -> PyResult<(Py<PyList>, Option<bool>)> {
catch_native_panic(|| {
self.inner.commit_hicache_transfers(
py,
node_id,
phase,
comp_xfers,
insert_result,
pool_storage_result,
)
})
}
/// Commit a successful backup to the node.
fn commit_backup(
&self,
py: Python<'_>,
node_id: NodeId,
host_indices: PyTensor,
comp_xfers: HashMap<u8, Vec<TransferArgs>>,
) -> PyResult<()> {
catch_native_panic(|| {
self.inner
.commit_backup(py, node_id, host_indices, comp_xfers)
})
}
/// Build the H->D load-back KV transfer plus per-component aux transfers.
#[pyo3(signature = (node_id, mamba_pool_idx = None))]
fn build_load_back_spec(
&self,
py: Python<'_>,
node_id: NodeId,
mamba_pool_idx: Option<PyTensor>,
) -> PyResult<(Py<PyAny>, Py<PyDict>)> {
catch_native_panic(|| self.inner.build_load_back_spec(py, node_id, mamba_pool_idx))
}
/// Commit a successful H->D load-back onto the node; returns its actions.
fn commit_load_back(
&self,
py: Python<'_>,
node_id: NodeId,
device_indices: PyTensor,
kv_xfer: TransferArgs,
comp_xfers: HashMap<u8, Vec<TransferArgs>>,
) -> PyResult<Py<PyList>> {
catch_native_panic(|| {
self.inner
.commit_load_back(py, node_id, device_indices, kv_xfer, comp_xfers)
})
}
/// Release a node's device KV once its host copy exists.
fn demote(&self, py: Python<'_>, node_id: NodeId) -> PyResult<DemoteResultBinding> {
catch_native_panic(|| self.inner.demote(py, node_id))
}
/// Evict up to num_tokens of one component's host resources.
#[pyo3(signature = (component_type, num_tokens, skip_full_duplicate_reclaim = false))]
fn drive_host_eviction(
&self,
py: Python<'_>,
component_type: u8,
num_tokens: usize,
skip_full_duplicate_reclaim: bool,
) -> PyResult<HostEvictionResultBinding> {
catch_native_panic(|| {
self.inner.drive_host_eviction(
py,
component_type,
num_tokens,
skip_full_duplicate_reclaim,
)
})
}
/// Evict shallow Mamba device checkpoints beyond the per-path cap
/// on the tail's root path.
fn evict_excess_path_states(
&self,
py: Python<'_>,
tail_node_id: NodeId,
) -> PyResult<HostEvictionResultBinding> {
catch_native_panic(|| self.inner.evict_excess_path_states(py, tail_node_id))
}
/// Bump the reference count on a node's host-side component locks.
fn inc_host_lock_ref(
&self,
py: Python<'_>,
node_id: NodeId,
) -> PyResult<IncLockRefResultBinding> {
catch_native_panic(|| self.inner.inc_host_lock_ref(py, node_id))
}
/// Decrease the reference count on a node's host-side component locks.
/// The receipt is required, as for dec_lock_ref.
fn dec_host_lock_ref(
&self,
py: Python<'_>,
node_id: NodeId,
params: &DecLockRefParamsBinding,
) -> PyResult<()> {
catch_native_panic(|| self.inner.dec_host_lock_ref(py, node_id, params))
}
fn is_write_through_compatible(&self, py: Python<'_>) -> bool {
self.inner.is_write_through_compatible(py)
}
/// Set the write-back (vs write-through) policy; decided at HiCache init.
fn set_is_write_back(&self, py: Python<'_>, is_write_back: bool) -> PyResult<()> {
catch_native_panic(|| {
self.inner.set_is_write_back(py, is_write_back);
Ok(())
})
}
/// The current write-back (vs write-through) policy.
fn is_write_back(&self, py: Python<'_>) -> PyResult<bool> {
catch_native_panic(|| Ok(self.inner.is_write_back(py)))
}
/// Set the write-through backup hit threshold; decided at HiCache init.
fn set_write_through_threshold(&self, py: Python<'_>, threshold: i64) -> PyResult<()> {
catch_native_panic(|| {
self.inner.set_write_through_threshold(py, threshold);
Ok(())
})
}
/// The current write-through backup hit threshold.
fn write_through_threshold(&self, py: Python<'_>) -> PyResult<i64> {
catch_native_panic(|| Ok(self.inner.write_through_threshold(py)))
}
/// Mark the storage tier (L3) wired; storage attaches after tree construction.
fn set_enable_storage(&self, py: Python<'_>, value: bool) -> PyResult<()> {
catch_native_panic(|| {
self.inner.set_enable_storage(py, value);
Ok(())
})
}
/// Whether the storage tier (L3) is wired.
fn enable_storage(&self, py: Python<'_>) -> PyResult<bool> {
catch_native_panic(|| Ok(self.inner.enable_storage(py)))
}
/// Enable or disable the direct external-cache linker.
fn set_enable_external_cache_linker(&self, py: Python<'_>, value: bool) -> PyResult<()> {
catch_native_panic(|| self.inner.set_enable_external_cache_linker(py, value))
}
/// Whether the direct external-cache linker is wired.
fn enable_external_cache_linker(&self, py: Python<'_>) -> PyResult<bool> {
catch_native_panic(|| Ok(self.inner.enable_external_cache_linker(py)))
}
/// Queue the all-cleared placement event.
fn record_all_cleared_event(&self, py: Python<'_>) -> PyResult<()> {
catch_native_panic(|| {
self.inner.record_all_cleared_event(py);
Ok(())
})
}
/// Drain the queued placement events as tagged tuples.
fn take_events(&self, py: Python<'_>) -> PyResult<Py<PyList>> {
catch_native_panic(|| self.inner.take_events(py))
}
/// Drop the subtree rooted at an unbacked D-leaf; not dropped when a lock
/// blocks it.
fn drop_subtree_no_host(
&self,
py: Python<'_>,
node_id: NodeId,
) -> PyResult<DropSubtreeResultBinding> {
catch_native_panic(|| self.inner.drop_subtree_no_host(py, node_id))
}
/// Mark the nodes one write-through backup covers; returns them ancestors first.
fn mark_write_through_pending(
&self,
py: Python<'_>,
node_ids: Vec<NodeId>,
ack_id: NodeId,
) -> PyResult<Vec<NodeId>> {
catch_native_panic(|| self.inner.mark_write_through_pending(py, node_ids, ack_id))
}
/// Clear the write-through-pending mark on the acked nodes.
fn finish_write_through(
&self,
py: Python<'_>,
node_ids: Vec<NodeId>,
ack_id: NodeId,
) -> PyResult<()> {
catch_native_panic(|| self.inner.finish_write_through(py, node_ids, ack_id))
}
/// Clear the in-flight H->D marks on the anchor's root path at ack time.
fn finish_load_back(&self, py: Python<'_>, anchor_node_id: NodeId) -> PyResult<()> {
catch_native_panic(|| self.inner.finish_load_back(py, anchor_node_id))
}
/// Build transfers for a node with no stored or pending external copy.
fn build_external_linker_offload_transfers(
&self,
py: Python<'_>,
node_id: NodeId,
) -> PyResult<Option<Vec<Py<PyAny>>>> {
catch_native_panic(|| {
self.inner
.build_external_linker_offload_transfers(py, node_id)
})
}
/// Mark an externally restored path, excluding its existing anchor.
fn mark_external_cache_stored_path(
&self,
py: Python<'_>,
from_node_id: NodeId,
until_node_id: NodeId,
) -> PyResult<()> {
catch_native_panic(|| {
self.inner
.mark_external_cache_stored_path(py, from_node_id, until_node_id)
})
}
/// Publish an accepted external offload as pending.
fn mark_external_linker_offload_pending(
&self,
py: Python<'_>,
node_id: NodeId,
) -> PyResult<()> {
catch_native_panic(|| self.inner.mark_external_linker_offload_pending(py, node_id))
}
/// Finalize external-store state for an offload and its split fragments.
fn finish_external_linker_offload(
&self,
py: Python<'_>,
node_ids: Vec<NodeId>,
ack_id: NodeId,
success: bool,
) -> PyResult<()> {
catch_native_panic(|| {
self.inner
.finish_external_linker_offload(py, node_ids, ack_id, success)
})
}
/// Order-sensitive digest of reclaimed coexisting host values.
#[pyo3(name = "write_back_duplicate_reclaim_digest")]
fn write_back_coexist_reclaim_digest(&self, py: Python<'_>) -> PyResult<i64> {
catch_native_panic(|| Ok(self.inner.write_back_coexist_reclaim_digest(py)))
}
/// Whether the component's data is device-evicted but host-backed.
fn component_has_host_value_only(
&self,
py: Python<'_>,
node_id: NodeId,
component_type: u8,
) -> PyResult<bool> {
catch_native_panic(|| {
self.inner
.component_has_host_value_only(py, node_id, component_type)
})
}
// ==== Test-only inspection surface ====
#[cfg(feature = "inspection")]
fn inspect_contains_node(&self, py: Python<'_>, node_id: NodeId) -> PyResult<bool> {
catch_native_panic(|| Ok(self.inner.inspect_contains_node(py, node_id)))
}
#[cfg(feature = "inspection")]
fn inspect_get_parent_node_id(
&self,
py: Python<'_>,
node_id: NodeId,
) -> PyResult<Option<NodeId>> {
catch_native_panic(|| self.inner.inspect_get_parent_node_id(py, node_id))
}
#[cfg(feature = "inspection")]
fn inspect_get_child_node_ids(&self, py: Python<'_>, node_id: NodeId) -> PyResult<Vec<NodeId>> {
catch_native_panic(|| self.inner.inspect_get_child_node_ids(py, node_id))
}
#[cfg(feature = "inspection")]
fn inspect_get_node_key_length(&self, py: Python<'_>, node_id: NodeId) -> PyResult<usize> {
catch_native_panic(|| self.inner.inspect_get_node_key_length(py, node_id))
}
#[cfg(feature = "inspection")]
fn inspect_get_node_token_ids(&self, py: Python<'_>, node_id: NodeId) -> PyResult<Vec<i64>> {
catch_native_panic(|| self.inner.inspect_get_node_token_ids(py, node_id))
}
#[cfg(feature = "inspection")]
fn inspect_is_node_key_bigram(&self, py: Python<'_>, node_id: NodeId) -> PyResult<bool> {
catch_native_panic(|| self.inner.inspect_is_node_key_bigram(py, node_id))
}
#[cfg(feature = "inspection")]
fn inspect_get_component_host_value(
&self,
py: Python<'_>,
node_id: NodeId,
component_type: u8,
) -> PyResult<Option<Py<PyAny>>> {
catch_native_panic(|| {
self.inner
.inspect_get_component_host_value(py, node_id, component_type)
})
}
#[cfg(feature = "inspection")]
fn inspect_get_component_device_lock_ref(
&self,
py: Python<'_>,
node_id: NodeId,
component_type: u8,
) -> PyResult<u32> {
catch_native_panic(|| {
self.inner
.inspect_get_component_device_lock_ref(py, node_id, component_type)
})
}
#[cfg(feature = "inspection")]
fn inspect_get_component_host_lock_ref(
&self,
py: Python<'_>,
node_id: NodeId,
component_type: u8,
) -> PyResult<u32> {
catch_native_panic(|| {
self.inner
.inspect_get_component_host_lock_ref(py, node_id, component_type)
})
}
#[cfg(feature = "inspection")]
fn inspect_get_node_hit_count(&self, py: Python<'_>, node_id: NodeId) -> PyResult<i64> {
catch_native_panic(|| self.inner.inspect_get_node_hit_count(py, node_id))
}
#[cfg(feature = "inspection")]
fn inspect_get_write_through_pending_id(
&self,
py: Python<'_>,
node_id: NodeId,
) -> PyResult<Option<usize>> {
catch_native_panic(|| self.inner.inspect_get_write_through_pending_id(py, node_id))
}
#[cfg(feature = "inspection")]
fn inspect_is_external_cache_stored(&self, py: Python<'_>, node_id: NodeId) -> PyResult<bool> {
catch_native_panic(|| self.inner.inspect_is_external_cache_stored(py, node_id))
}
#[cfg(feature = "inspection")]
fn inspect_is_node_in_device_lru(
&self,
py: Python<'_>,
node_id: NodeId,
component_type: u8,
) -> PyResult<bool> {
catch_native_panic(|| {
self.inner
.inspect_is_node_in_device_lru(py, node_id, component_type)
})
}
#[cfg(feature = "inspection")]
fn inspect_is_node_in_host_lru(
&self,
py: Python<'_>,
node_id: NodeId,
component_type: u8,
) -> PyResult<bool> {
catch_native_panic(|| {
self.inner
.inspect_is_node_in_host_lru(py, node_id, component_type)
})
}
#[cfg(feature = "inspection")]
fn inspect_get_component_device_lru_node_ids(
&self,
py: Python<'_>,
component_type: u8,
) -> PyResult<Vec<NodeId>> {
catch_native_panic(|| {
self.inner
.inspect_get_component_device_lru_node_ids(py, component_type)
})
}
#[cfg(feature = "inspection")]
fn inspect_is_device_evictable_leaf(&self, py: Python<'_>, node_id: NodeId) -> PyResult<bool> {
catch_native_panic(|| Ok(self.inner.inspect_is_device_evictable_leaf(py, node_id)))
}
#[cfg(feature = "inspection")]
fn inspect_is_host_evictable_leaf(&self, py: Python<'_>, node_id: NodeId) -> PyResult<bool> {
catch_native_panic(|| Ok(self.inner.inspect_is_host_evictable_leaf(py, node_id)))
}
#[cfg(feature = "inspection")]
fn inspect_is_device_leaf(&self, py: Python<'_>, node_id: NodeId) -> PyResult<bool> {
catch_native_panic(|| self.inner.inspect_is_device_leaf(py, node_id))
}
#[cfg(feature = "inspection")]
fn inspect_get_all_node_ids(&self, py: Python<'_>) -> PyResult<Vec<NodeId>> {
catch_native_panic(|| Ok(self.inner.inspect_get_all_node_ids(py)))
}
#[cfg(feature = "inspection")]
fn inspect_component_protected_size(
&self,
py: Python<'_>,
component_type: u8,
) -> PyResult<usize> {
catch_native_panic(|| {
self.inner
.inspect_component_protected_size(py, component_type)
})
}
#[cfg(feature = "inspection")]
#[pyo3(signature = (node_id, hash_values = None))]
fn inspect_set_node_hash_values(
&self,
py: Python<'_>,
node_id: NodeId,
hash_values: Option<Vec<String>>,
) -> PyResult<()> {
catch_native_panic(|| {
self.inner
.inspect_set_node_hash_values(py, node_id, hash_values)
})
}
#[cfg(feature = "inspection")]
#[pyo3(signature = (node_id, component_type, value = None))]
fn inspect_set_component_device_value_raw(
&self,
py: Python<'_>,
node_id: NodeId,
component_type: u8,
value: Option<PyTensor>,
) -> PyResult<()> {
catch_native_panic(|| {
self.inner
.inspect_set_component_device_value_raw(py, node_id, component_type, value)
})
}
#[cfg(feature = "inspection")]
#[pyo3(signature = (node_id, component_type, value = None))]
fn inspect_set_component_host_value_raw(
&self,
py: Python<'_>,
node_id: NodeId,
component_type: u8,
value: Option<PyTensor>,
) -> PyResult<()> {
catch_native_panic(|| {
self.inner
.inspect_set_component_host_value_raw(py, node_id, component_type, value)
})
}
#[cfg(feature = "inspection")]
fn inspect_set_component_device_lock_ref(
&self,
py: Python<'_>,
node_id: NodeId,
component_type: u8,
lock_ref: u32,
) -> PyResult<()> {
catch_native_panic(|| {
self.inner
.inspect_set_component_device_lock_ref(py, node_id, component_type, lock_ref)
})
}
#[cfg(feature = "inspection")]
fn inspect_remove_node_from_device_lru(
&self,
py: Python<'_>,
node_id: NodeId,
component_type: u8,
) -> PyResult<()> {
catch_native_panic(|| {
self.inner
.inspect_remove_node_from_device_lru(py, node_id, component_type)
})
}
#[cfg(feature = "inspection")]
fn inspect_insert_node_into_host_lru(
&self,
py: Python<'_>,
node_id: NodeId,
component_type: u8,
) -> PyResult<()> {
catch_native_panic(|| {
self.inner
.inspect_insert_node_into_host_lru(py, node_id, component_type)
})
}
#[cfg(feature = "inspection")]
fn inspect_set_component_evictable_size(
&self,
py: Python<'_>,
component_type: u8,
value: usize,
) -> PyResult<()> {
catch_native_panic(|| {
self.inner
.inspect_set_component_evictable_size(py, component_type, value)
})
}
#[cfg(feature = "inspection")]
fn inspect_set_component_protected_size(
&self,
py: Python<'_>,
component_type: u8,
value: usize,
) -> PyResult<()> {
catch_native_panic(|| {
self.inner
.inspect_set_component_protected_size(py, component_type, value)
})
}
#[cfg(feature = "inspection")]
fn inspect_update_duplicate_tracking(&self, py: Python<'_>, node_id: NodeId) -> PyResult<()> {
catch_native_panic(|| self.inner.inspect_update_duplicate_tracking(py, node_id))
}
#[cfg(feature = "inspection")]
fn inspect_advance_insert_walk_once(&self, py: Python<'_>) -> PyResult<()> {
catch_native_panic(|| self.inner.inspect_advance_insert_walk_once(py))
}
#[cfg(feature = "inspection")]
fn inspect_evict_component(
&self,
py: Python<'_>,
node_id: NodeId,
component_type: u8,
target: u8,
) -> PyResult<HostEvictionResultBinding> {
catch_native_panic(|| {
self.inner
.inspect_evict_component(py, node_id, component_type, target)
})
}
#[cfg(feature = "inspection")]
fn inspect_validate_cascade_evict(
&self,
py: Python<'_>,
node_id: NodeId,
component_type: u8,
target: u8,
) -> PyResult<()> {
catch_native_panic(|| {
self.inner
.inspect_validate_cascade_evict(py, node_id, component_type, target)
})
}
#[cfg(feature = "inspection")]
fn inspect_cleanup_tombstone_ancestors(
&self,
py: Python<'_>,
node_id: NodeId,
) -> PyResult<HostEvictionResultBinding> {
catch_native_panic(|| self.inner.inspect_cleanup_tombstone_ancestors(py, node_id))
}
#[cfg(feature = "inspection")]
#[allow(clippy::too_many_arguments)]
#[pyo3(signature = (component_type, result, key, extra_key, cache_salt, value_chunks, best_value_len))]
fn inspect_finalize_component_match_result(
&self,
py: Python<'_>,
component_type: u8,
result: InspectionMatchResultInput,
key: &Bound<'_, PyAny>,
extra_key: Option<String>,
cache_salt: Option<String>,
value_chunks: Vec<PyTensor>,
best_value_len: usize,
) -> PyResult<MatchResultBinding> {
catch_native_panic(|| {
self.inner.inspect_finalize_component_match_result(
py,
component_type,
result,
key,
extra_key,
cache_salt,
value_chunks,
best_value_len,
)
})
}
#[cfg(feature = "inspection")]
#[pyo3(signature = (node_id, write_back = false))]
fn inspect_build_backup_node_ids(
&self,
py: Python<'_>,
node_id: NodeId,
write_back: bool,
) -> PyResult<Vec<NodeId>> {
catch_native_panic(|| {
self.inner
.inspect_build_backup_node_ids(py, node_id, write_back)
})
}
/// Print the tree structure for debugging.
fn pretty_print(&self, py: Python<'_>) -> PyResult<()> {
catch_native_panic(|| {
self.inner.pretty_print(py);
Ok(())
})
}
}
};
}
tree_core_binding!(
/// The UnifiedTreeCore Python adapter over single-token (unigram) child keys.
RustUnifiedTreeCoreBinding,
Vec<i64>
);
tree_core_binding!(
/// The UnifiedTreeCore Python adapter over bigram (EAGLE) child keys; keys
/// cross the boundary as raw token ids and pair up rust-side.
RustBigramUnifiedTreeCoreBinding,
Vec<(i64, i64)>
);
/// Per-page chained hashes over raw token ids.
#[pyfunction]
#[pyo3(signature = (token_ids, prior_hash, page_size, is_bigram = false))]
fn get_hash_str(
py: Python<'_>,
token_ids: &Bound<'_, PyAny>,
prior_hash: Option<String>,
page_size: usize,
is_bigram: bool,
) -> PyResult<Vec<String>> {
catch_native_panic(|| {
let raw = py_array_to_vec_i64(py, token_ids)?;
if page_size == 0 {
return Err(PyValueError::new_err("page_size must be positive"));
}
if let Some(prior_hash) = prior_hash.as_deref().filter(|hash| !hash.is_empty())
&& (prior_hash.len() != 64 || !prior_hash.bytes().all(|byte| byte.is_ascii_hexdigit()))
{
return Err(PyValueError::new_err(
"prior_hash must be a 64-character hexadecimal digest",
));
}
if let Some(token_id) = raw.iter().find(|token_id| **token_id < 0) {
return Err(PyValueError::new_err(format!(
"token id {token_id} must be a non-negative int64"
)));
}
Ok(py.detach(move || {
if is_bigram {
let key = <Vec<(i64, i64)> as ChildKeyType>::key_from(Cow::Owned(raw)).into_owned();
crate::node::get_hash_str::<Vec<(i64, i64)>>(&key, prior_hash.as_deref(), page_size)
} else {
crate::node::get_hash_str::<Vec<i64>>(&raw, prior_hash.as_deref(), page_size)
}
}))
})
}
fn register_mem_cache_module(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<TlruFloatConfigBinding>()?;
m.add_function(wrap_pyfunction!(get_hash_str, m)?)?;
m.add_class::<TreeCoreInitParamsBinding>()?;
m.add_class::<MatchParamsBinding>()?;
m.add_class::<InsertParamsBinding>()?;
m.add_class::<MatchResultBinding>()?;
m.add_class::<InsertResultBinding>()?;
m.add_class::<IncLockRefResultBinding>()?;
m.add_class::<DecLockRefParamsBinding>()?;
m.add_class::<EvictDeviceNextNodeResultBinding>()?;
m.add_class::<EvictDeviceLeafResultBinding>()?;
m.add_class::<DropSubtreeResultBinding>()?;
m.add_class::<DemoteResultBinding>()?;
m.add_class::<KvCanaryWalkResultBinding>()?;
m.add_class::<StorageBackupSpecBinding>()?;
m.add_class::<BufferBackupSnapshotBinding>()?;
m.add_class::<BufferBackupStateBinding>()?;
m.add_class::<HostEvictionResultBinding>()?;
m.add_class::<RustUnifiedTreeCoreBinding>()?;
m.add_class::<RustBigramUnifiedTreeCoreBinding>()?;
Ok(())
}
/// The production TreeCore extension module.
#[pymodule]
fn mem_cache(m: &Bound<'_, PyModule>) -> PyResult<()> {
register_mem_cache_module(m)
}
/// White-box variant used only by the shared test inspector.
#[cfg(feature = "inspection")]
#[pymodule]
fn mem_cache_inspection(m: &Bound<'_, PyModule>) -> PyResult<()> {
register_mem_cache_module(m)
}