Co-authored-by: metamergebot <metamergebot@users.noreply.github.com> Co-authored-by: Jialin <Jialin@users.noreply.github.com>
4077 lines
150 KiB
Rust
4077 lines
150 KiB
Rust
//! Python bindings: the `mem_cache` extension module and its TreeCore adapter.
|
|
|
|
use std::borrow::Cow;
|
|
use std::collections::HashMap;
|
|
use std::panic::{AssertUnwindSafe, catch_unwind};
|
|
use std::sync::Mutex;
|
|
|
|
use pyo3::IntoPyObjectExt;
|
|
use pyo3::buffer::PyBuffer;
|
|
use pyo3::exceptions::{PyAssertionError, PyKeyError, PyRuntimeError, PyValueError};
|
|
use pyo3::prelude::*;
|
|
use pyo3::types::{PyBytes, PyDict, PyList};
|
|
use tch::{Device, Kind, Tensor};
|
|
|
|
use crate::components::{ComponentSet, ComponentType, FULL, MAMBA, SWA};
|
|
use crate::node::ChildKeyType;
|
|
use crate::node::{KeyNamespaceRef, NodeAccessError, NodeId, TreeCoreRuntimeError};
|
|
use crate::unified_lru_list::{TlruFloatConfig, TlruPromptEstimate};
|
|
use crate::unified_tree_core::KvCacheEvent;
|
|
use crate::unified_tree_core::{
|
|
BufferBackupSnapshot, BufferBackupState, CacheAction, CacheInitParams, CacheTransferPhase,
|
|
DecLockRefParams, EvictLayer, EvictionStepResult, InsertParams, InsertResult, InsertStepResult,
|
|
MatchPrefixParams, MatchResult, PoolHitPolicy, PoolName, PoolTransfer, PoolTransferResult, Req,
|
|
UnifiedTreeCore,
|
|
};
|
|
|
|
/// Translate only Rust panics; ordinary Python errors pass through unchanged.
|
|
/// Keep this boundary outside the whole binding call so its MutexGuard unwinds
|
|
/// and poisons the core before the panic is caught. A poisoned core stays unusable.
|
|
fn catch_native_panic<T>(operation: impl FnOnce() -> PyResult<T>) -> PyResult<T> {
|
|
match catch_unwind(AssertUnwindSafe(operation)) {
|
|
Ok(result) => result,
|
|
Err(payload) => {
|
|
let message = if let Some(message) = payload.downcast_ref::<String>() {
|
|
message.as_str()
|
|
} else if let Some(message) = payload.downcast_ref::<&str>() {
|
|
message
|
|
} else {
|
|
"unknown native panic"
|
|
};
|
|
Err(PyRuntimeError::new_err(format!(
|
|
"Rust TreeCore panicked: {message}"
|
|
)))
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Parse a torch-style device string (e.g. "cpu", "cuda", "cuda:1"); a bare
|
|
/// "cuda" means index 0, so callers must resolve the index themselves.
|
|
fn parse_device(device: &str) -> PyResult<Device> {
|
|
let device = device.to_lowercase();
|
|
if device == "cpu" {
|
|
return Ok(Device::Cpu);
|
|
}
|
|
if device == "cuda" {
|
|
return Ok(Device::Cuda(0));
|
|
}
|
|
if let Some(index) = device.strip_prefix("cuda:")
|
|
&& let Ok(index) = index.parse::<usize>()
|
|
{
|
|
return Ok(Device::Cuda(index));
|
|
}
|
|
Err(PyValueError::new_err(format!(
|
|
"unsupported device string: {device}"
|
|
)))
|
|
}
|
|
|
|
/// Map a Python ComponentType value onto the Rust enum.
|
|
fn parse_component_type(component_type: u8) -> PyResult<ComponentType> {
|
|
match component_type {
|
|
0 => Ok(FULL),
|
|
1 => Ok(SWA),
|
|
2 => Ok(MAMBA),
|
|
other => Err(PyValueError::new_err(format!(
|
|
"unknown component type: {other}"
|
|
))),
|
|
}
|
|
}
|
|
|
|
/// Map the Python EvictLayer IntFlag value onto the Rust enum.
|
|
fn parse_evict_layer(target: u8) -> PyResult<EvictLayer> {
|
|
match target {
|
|
1 => Ok(EvictLayer::Device),
|
|
2 => Ok(EvictLayer::Host),
|
|
3 => Ok(EvictLayer::All),
|
|
other => Err(PyValueError::new_err(format!(
|
|
"unknown eviction layer: {other}"
|
|
))),
|
|
}
|
|
}
|
|
|
|
fn node_access_error(error: NodeAccessError) -> PyErr {
|
|
PyKeyError::new_err(error.node_id)
|
|
}
|
|
|
|
/// Convert an expected tree-core contract failure without unwinding through PyO3.
|
|
fn tree_core_runtime_error(error: TreeCoreRuntimeError) -> PyErr {
|
|
match error {
|
|
TreeCoreRuntimeError::NodeAccess(error) => node_access_error(error),
|
|
error @ (TreeCoreRuntimeError::ComponentNotEnabled { .. }
|
|
| TreeCoreRuntimeError::InvalidComponentSizeDelta { .. }) => {
|
|
PyValueError::new_err(error.to_string())
|
|
}
|
|
error => PyRuntimeError::new_err(error.to_string()),
|
|
}
|
|
}
|
|
|
|
fn tree_core_assertion_error(error: TreeCoreRuntimeError) -> PyErr {
|
|
match error {
|
|
TreeCoreRuntimeError::NodeAccess(error) => node_access_error(error),
|
|
error => PyAssertionError::new_err(error.to_string()),
|
|
}
|
|
}
|
|
|
|
/// Map the Rust enum back onto the Python ComponentType value.
|
|
fn component_type_to_u8(component_type: ComponentType) -> u8 {
|
|
component_type as u8
|
|
}
|
|
|
|
/// Newtype bridging `tch::Tensor` and Python `torch.Tensor` over raw THPVariable
|
|
/// pointers (inlined from pyo3-tch, MIT/Apache-2.0, by Laurent Mazare).
|
|
pub struct PyTensor(pub Tensor);
|
|
|
|
impl<'a, 'py> FromPyObject<'a, 'py> for PyTensor {
|
|
type Error = PyErr;
|
|
|
|
fn extract(ob: Borrowed<'a, 'py, PyAny>) -> PyResult<Self> {
|
|
let ptr = ob.as_ptr() as *mut tch::python::CPyObject;
|
|
match unsafe { Tensor::pyobject_unpack(ptr) } {
|
|
Ok(Some(tensor)) => Ok(PyTensor(tensor)),
|
|
Ok(None) => Err(pyo3::exceptions::PyTypeError::new_err(format!(
|
|
"expected a torch.Tensor, got {}",
|
|
ob.get_type()
|
|
))),
|
|
Err(err) => Err(PyValueError::new_err(format!("{err:?}"))),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl<'py> IntoPyObject<'py> for PyTensor {
|
|
type Target = PyAny;
|
|
type Output = Bound<'py, PyAny>;
|
|
type Error = PyErr;
|
|
|
|
fn into_pyobject(self, py: Python<'py>) -> PyResult<Self::Output> {
|
|
tensor_to_py(py, self.0).map(|obj| obj.into_bound(py))
|
|
}
|
|
}
|
|
|
|
/// Convert a tensor into a Python-held torch.Tensor reference.
|
|
fn tensor_to_py(py: Python<'_>, tensor: Tensor) -> PyResult<Py<PyAny>> {
|
|
let ptr = tensor
|
|
.pyobject_wrap()
|
|
.map_err(|err| PyValueError::new_err(format!("{err:?}")))?;
|
|
Ok(unsafe { Bound::from_owned_ptr(py, ptr as *mut pyo3::ffi::PyObject) }.unbind())
|
|
}
|
|
|
|
/// Convert a Python int64 sequence to an owned `Vec<i64>`.
|
|
fn py_array_to_vec_i64(py: Python<'_>, key: &Bound<'_, PyAny>) -> PyResult<Vec<i64>> {
|
|
// Special handling for empty keys, as empty pyarray might use
|
|
// a random address to represent empty buffer which
|
|
// non-deterministically violates alignment check
|
|
if key.len().map(|n| n == 0).unwrap_or(false) {
|
|
return Ok(Vec::new());
|
|
}
|
|
let buffer = key.extract::<PyBuffer<i64>>()?;
|
|
if !buffer.is_c_contiguous() {
|
|
return Err(pyo3::exceptions::PyTypeError::new_err(
|
|
"Unexpected key received, expected a C-contiguous int64 buffer \
|
|
(e.g. array.array('q'))",
|
|
));
|
|
}
|
|
buffer.to_vec(py)
|
|
}
|
|
|
|
/// The tagged-tuple tag of a cache action.
|
|
fn cache_action_tag(action: &CacheAction) -> &'static str {
|
|
match action {
|
|
CacheAction::FreeDeviceKV(_) => "free_device_kv",
|
|
CacheAction::FreeDeviceKVFullOnly(_) => "free_device_kv_full_only",
|
|
CacheAction::BackupKV(_) => "backup_kv",
|
|
CacheAction::ReplaceWriteThroughOnNodeSplit { .. } => "replace_write_through_on_node_split",
|
|
CacheAction::MambaEvictExcessPathStates { .. } => "mamba_evict_excess_path_states",
|
|
CacheAction::FreeComponentDeviceSlot { .. } => "free_component_device_slot",
|
|
CacheAction::FreeComponentHostSlot { .. } => "free_component_host_slot",
|
|
CacheAction::RebuildFullToSwaMapping { .. } => "rebuild_full_to_swa_mapping",
|
|
CacheAction::RecoverSwaWithLockedFull { .. } => "recover_swa_with_locked_full",
|
|
CacheAction::SwaRebuild { .. } => "swa_rebuild",
|
|
}
|
|
}
|
|
|
|
/// Convert a cache action into its Python tagged tuple.
|
|
fn cache_action_to_py(py: Python<'_>, action: CacheAction) -> PyResult<Py<PyAny>> {
|
|
let tag = cache_action_tag(&action);
|
|
match action {
|
|
CacheAction::FreeDeviceKV(tensors) => {
|
|
let tensors = PyList::new(py, tensors.into_iter().map(PyTensor))?;
|
|
(tag, tensors).into_py_any(py)
|
|
}
|
|
CacheAction::FreeDeviceKVFullOnly(tensors) => {
|
|
let tensors = PyList::new(py, tensors.into_iter().map(PyTensor))?;
|
|
(tag, tensors).into_py_any(py)
|
|
}
|
|
CacheAction::BackupKV(backup) => (tag, backup.node_ids).into_py_any(py),
|
|
CacheAction::ReplaceWriteThroughOnNodeSplit {
|
|
ack_id,
|
|
old_node_id,
|
|
new_node_id,
|
|
new_child_node_id,
|
|
} => (tag, ack_id, old_node_id, new_node_id, new_child_node_id).into_py_any(py),
|
|
CacheAction::MambaEvictExcessPathStates { tail_node_id } => {
|
|
(tag, tail_node_id).into_py_any(py)
|
|
}
|
|
CacheAction::FreeComponentDeviceSlot {
|
|
component_type,
|
|
indices,
|
|
} => {
|
|
let indices = PyList::new(py, indices.into_iter().map(PyTensor))?;
|
|
(tag, component_type_to_u8(component_type), indices).into_py_any(py)
|
|
}
|
|
CacheAction::FreeComponentHostSlot {
|
|
component_type,
|
|
host_indices,
|
|
} => {
|
|
let host_indices = PyList::new(py, host_indices.into_iter().map(PyTensor))?;
|
|
(tag, component_type_to_u8(component_type), host_indices).into_py_any(py)
|
|
}
|
|
CacheAction::RebuildFullToSwaMapping {
|
|
full_indices,
|
|
swa_indices,
|
|
} => {
|
|
let full_indices = PyList::new(py, full_indices.into_iter().map(PyTensor))?;
|
|
let swa_indices = PyList::new(py, swa_indices.into_iter().map(PyTensor))?;
|
|
(tag, full_indices, swa_indices).into_py_any(py)
|
|
}
|
|
CacheAction::RecoverSwaWithLockedFull {
|
|
node_id,
|
|
kept_full,
|
|
incoming_full,
|
|
} => (tag, node_id, PyTensor(kept_full), PyTensor(incoming_full)).into_py_any(py),
|
|
CacheAction::SwaRebuild {
|
|
node_id,
|
|
source_value,
|
|
} => (tag, node_id, PyTensor(source_value)).into_py_any(py),
|
|
}
|
|
}
|
|
|
|
/// Convert cache actions into a Python list of tagged tuples.
|
|
fn cache_actions_to_py(py: Python<'_>, actions: Vec<CacheAction>) -> PyResult<Py<PyList>> {
|
|
let list = PyList::empty(py);
|
|
for action in actions {
|
|
list.append(cache_action_to_py(py, action)?)?;
|
|
}
|
|
Ok(list.unbind())
|
|
}
|
|
|
|
/// The python PoolName string for a pool.
|
|
fn pool_name_str(name: PoolName) -> &'static str {
|
|
match name {
|
|
PoolName::Kv => "kv",
|
|
PoolName::Mamba => "mamba",
|
|
PoolName::Swa => "swa",
|
|
PoolName::AuxiliarySwa => "auxiliary_swa",
|
|
PoolName::Indexer => "indexer",
|
|
PoolName::DeepseekV4C4 => "deepseek_v4_c4",
|
|
PoolName::DeepseekV4C4Indexer => "deepseek_v4_c4_indexer",
|
|
PoolName::DeepseekV4C4IndexerScale => "deepseek_v4_c4_indexer_scale",
|
|
PoolName::DeepseekV4C128 => "deepseek_v4_c128",
|
|
PoolName::DeepseekV4C1 => "deepseek_v4_c1",
|
|
PoolName::DeepseekV4C1Indexer => "deepseek_v4_c1_indexer",
|
|
PoolName::DeepseekV4C1IndexerScale => "deepseek_v4_c1_indexer_scale",
|
|
PoolName::DeepseekV4C2 => "deepseek_v4_c2",
|
|
PoolName::DeepseekV4C2Indexer => "deepseek_v4_c2_indexer",
|
|
PoolName::DeepseekV4C2IndexerScale => "deepseek_v4_c2_indexer_scale",
|
|
PoolName::DeepseekV4C4Rope => "deepseek_v4_c4_rope",
|
|
PoolName::DeepseekV4C128Rope => "deepseek_v4_c128_rope",
|
|
PoolName::DeepseekV4C4State => "deepseek_v4_c4_state",
|
|
PoolName::DeepseekV4C4IndexerState => "deepseek_v4_c4_indexer_state",
|
|
PoolName::DeepseekV4C128State => "deepseek_v4_c128_state",
|
|
PoolName::Draft => "draft",
|
|
PoolName::DraftIndexer => "draft_indexer",
|
|
PoolName::DraftSwa => "draft_swa",
|
|
}
|
|
}
|
|
|
|
/// Map a python PoolName string onto the Rust enum.
|
|
fn parse_pool_name(name: &str) -> PyResult<PoolName> {
|
|
match name {
|
|
"kv" => Ok(PoolName::Kv),
|
|
"mamba" => Ok(PoolName::Mamba),
|
|
"swa" => Ok(PoolName::Swa),
|
|
"auxiliary_swa" => Ok(PoolName::AuxiliarySwa),
|
|
"indexer" => Ok(PoolName::Indexer),
|
|
"deepseek_v4_c4" => Ok(PoolName::DeepseekV4C4),
|
|
"deepseek_v4_c4_indexer" => Ok(PoolName::DeepseekV4C4Indexer),
|
|
"deepseek_v4_c4_indexer_scale" => Ok(PoolName::DeepseekV4C4IndexerScale),
|
|
"deepseek_v4_c128" => Ok(PoolName::DeepseekV4C128),
|
|
"deepseek_v4_c1" => Ok(PoolName::DeepseekV4C1),
|
|
"deepseek_v4_c1_indexer" => Ok(PoolName::DeepseekV4C1Indexer),
|
|
"deepseek_v4_c1_indexer_scale" => Ok(PoolName::DeepseekV4C1IndexerScale),
|
|
"deepseek_v4_c2" => Ok(PoolName::DeepseekV4C2),
|
|
"deepseek_v4_c2_indexer" => Ok(PoolName::DeepseekV4C2Indexer),
|
|
"deepseek_v4_c2_indexer_scale" => Ok(PoolName::DeepseekV4C2IndexerScale),
|
|
"deepseek_v4_c4_rope" => Ok(PoolName::DeepseekV4C4Rope),
|
|
"deepseek_v4_c128_rope" => Ok(PoolName::DeepseekV4C128Rope),
|
|
"deepseek_v4_c4_state" => Ok(PoolName::DeepseekV4C4State),
|
|
"deepseek_v4_c4_indexer_state" => Ok(PoolName::DeepseekV4C4IndexerState),
|
|
"deepseek_v4_c128_state" => Ok(PoolName::DeepseekV4C128State),
|
|
"draft" => Ok(PoolName::Draft),
|
|
"draft_indexer" => Ok(PoolName::DraftIndexer),
|
|
"draft_swa" => Ok(PoolName::DraftSwa),
|
|
other => Err(PyValueError::new_err(format!("unknown pool name: {other}"))),
|
|
}
|
|
}
|
|
|
|
/// A pool transfer's boundary form:
|
|
/// (name, host_indices, device_indices, nodes_to_load, keys, hit_policy).
|
|
type TransferArgs = (
|
|
String,
|
|
Option<PyTensor>,
|
|
Option<PyTensor>,
|
|
Option<Vec<NodeId>>,
|
|
Option<Vec<String>>,
|
|
String,
|
|
);
|
|
|
|
/// Strongly typed, attribute-based input view of a Python MatchResult.
|
|
/// Cache actions are intentionally omitted: component finalizers only update
|
|
/// match metadata, while the test adapter preserves the original actions.
|
|
#[cfg(feature = "inspection")]
|
|
#[derive(FromPyObject)]
|
|
struct InspectionMatchResultInput {
|
|
#[pyo3(attribute)]
|
|
device_prefix_len: usize,
|
|
#[pyo3(attribute)]
|
|
last_device_node: NodeId,
|
|
#[pyo3(attribute)]
|
|
last_host_node: NodeId,
|
|
#[pyo3(attribute)]
|
|
best_match_node: NodeId,
|
|
#[pyo3(attribute)]
|
|
host_hit_length: usize,
|
|
#[pyo3(attribute)]
|
|
swa_host_hit_length: usize,
|
|
#[pyo3(attribute)]
|
|
swa_branching_seqlen: Option<usize>,
|
|
#[pyo3(attribute)]
|
|
mamba_host_hit_length: usize,
|
|
#[pyo3(attribute)]
|
|
mamba_branching_seqlen: Option<usize>,
|
|
#[pyo3(attribute)]
|
|
full_kv_hit_length: usize,
|
|
}
|
|
|
|
/// Map a python CacheTransferPhase value onto the Rust enum.
|
|
fn parse_transfer_phase(phase: &str) -> PyResult<CacheTransferPhase> {
|
|
match phase {
|
|
"backup_host" => Ok(CacheTransferPhase::BackupHost),
|
|
"load_back" => Ok(CacheTransferPhase::LoadBack),
|
|
"backup_storage" => Ok(CacheTransferPhase::BackupStorage),
|
|
"prefetch" => Ok(CacheTransferPhase::Prefetch),
|
|
other => Err(PyValueError::new_err(format!(
|
|
"unknown transfer phase: {other}"
|
|
))),
|
|
}
|
|
}
|
|
|
|
/// Map a python PoolHitPolicy value onto the Rust enum.
|
|
fn parse_hit_policy(hit_policy: &str) -> PyResult<PoolHitPolicy> {
|
|
match hit_policy {
|
|
"all_pages" => Ok(PoolHitPolicy::AllPages),
|
|
"trailing_pages" => Ok(PoolHitPolicy::TrailingPages),
|
|
other => Err(PyValueError::new_err(format!(
|
|
"unknown hit policy: {other}"
|
|
))),
|
|
}
|
|
}
|
|
|
|
/// Convert a pool transfer into its boundary tuple.
|
|
fn transfer_to_py(py: Python<'_>, transfer: PoolTransfer) -> PyResult<Py<PyAny>> {
|
|
(
|
|
pool_name_str(transfer.name),
|
|
transfer.host_indices.map(PyTensor),
|
|
transfer.device_indices.map(PyTensor),
|
|
transfer.nodes_to_load,
|
|
transfer.keys,
|
|
transfer.hit_policy.as_str(),
|
|
)
|
|
.into_py_any(py)
|
|
}
|
|
|
|
/// Build a pool transfer from its boundary tuple.
|
|
fn transfer_from_args(args: TransferArgs) -> PyResult<PoolTransfer> {
|
|
let (name, host_indices, device_indices, nodes_to_load, keys, hit_policy) = args;
|
|
Ok(PoolTransfer {
|
|
name: parse_pool_name(&name)?,
|
|
host_indices: host_indices.map(|t| t.0),
|
|
device_indices: device_indices.map(|t| t.0),
|
|
keys,
|
|
hit_policy: parse_hit_policy(&hit_policy)?,
|
|
nodes_to_load,
|
|
})
|
|
}
|
|
|
|
/// Convert per-component transfers into a Python dict of boundary tuples.
|
|
fn comp_xfers_to_py(
|
|
py: Python<'_>,
|
|
comp_xfers: HashMap<ComponentType, Vec<PoolTransfer>>,
|
|
) -> PyResult<Py<PyDict>> {
|
|
let dict = PyDict::new(py);
|
|
// Pool allocation can evict across components, so preserve Python's order.
|
|
let mut comp_xfers: Vec<_> = comp_xfers.into_iter().collect();
|
|
comp_xfers.sort_unstable_by_key(|(ct, _)| component_type_to_u8(*ct));
|
|
for (ct, transfers) in comp_xfers {
|
|
let list = PyList::empty(py);
|
|
for transfer in transfers {
|
|
list.append(transfer_to_py(py, transfer)?)?;
|
|
}
|
|
dict.set_item(component_type_to_u8(ct), list)?;
|
|
}
|
|
Ok(dict.unbind())
|
|
}
|
|
|
|
/// Build per-component transfers from a Python dict of boundary tuples.
|
|
fn comp_xfers_from_args(
|
|
comp_xfers: HashMap<u8, Vec<TransferArgs>>,
|
|
) -> PyResult<HashMap<ComponentType, Vec<PoolTransfer>>> {
|
|
comp_xfers
|
|
.into_iter()
|
|
.map(|(ct, transfers)| {
|
|
Ok((
|
|
parse_component_type(ct)?,
|
|
transfers
|
|
.into_iter()
|
|
.map(transfer_from_args)
|
|
.collect::<PyResult<Vec<_>>>()?,
|
|
))
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
/// Convert per-component freed tensors into a Python dict keyed by component value.
|
|
fn frees_to_py(py: Python<'_>, frees: HashMap<ComponentType, Vec<Tensor>>) -> PyResult<Py<PyDict>> {
|
|
let frees: HashMap<u8, Vec<PyTensor>> = frees
|
|
.into_iter()
|
|
.map(|(ct, tensors)| {
|
|
(
|
|
component_type_to_u8(ct),
|
|
tensors.into_iter().map(PyTensor).collect(),
|
|
)
|
|
})
|
|
.collect();
|
|
let dict = PyDict::new(py);
|
|
for (ct, tensors) in frees {
|
|
dict.set_item(ct, tensors)?;
|
|
}
|
|
Ok(dict.unbind())
|
|
}
|
|
|
|
/// Floating-point T-LRU arithmetic with the estimate's numeric type preserved.
|
|
#[pyclass(name = "TlruFloatConfig", from_py_object)]
|
|
#[derive(Clone)]
|
|
pub struct TlruFloatConfigBinding {
|
|
config: TlruFloatConfig,
|
|
}
|
|
|
|
#[pymethods]
|
|
impl TlruFloatConfigBinding {
|
|
#[new]
|
|
#[pyo3(signature = (threshold, next_prompt_estimate, integer_estimate = None))]
|
|
fn new(threshold: f64, next_prompt_estimate: f64, integer_estimate: Option<i128>) -> Self {
|
|
let estimate = match integer_estimate {
|
|
Some(value) => TlruPromptEstimate::Integer(value),
|
|
None => TlruPromptEstimate::Float(next_prompt_estimate),
|
|
};
|
|
Self {
|
|
config: TlruFloatConfig {
|
|
threshold,
|
|
next_prompt_estimate: estimate,
|
|
},
|
|
}
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
fn inspect_is_tel_safe(&self, history: usize, cached_without_node: usize) -> PyResult<bool> {
|
|
catch_native_panic(|| Ok(self.config.is_tel_safe(history, cached_without_node)))
|
|
}
|
|
}
|
|
|
|
/// Python-visible tree-core init params; converts into CacheInitParams.
|
|
#[pyclass(get_all, set_all, from_py_object)]
|
|
#[derive(Clone)]
|
|
pub struct TreeCoreInitParamsBinding {
|
|
pub eviction_policy: String,
|
|
pub slru_protected_threshold: i64,
|
|
pub tlru_tail_budget: usize,
|
|
pub tlru_float_config: Option<TlruFloatConfigBinding>,
|
|
pub page_size: usize,
|
|
pub is_write_back: bool,
|
|
pub enable_hicache: bool,
|
|
pub write_through_threshold: i64,
|
|
pub device: String,
|
|
pub swa_sliding_window_size: Option<usize>,
|
|
pub swa_req_ring: bool,
|
|
pub enable_kv_cache_events: bool,
|
|
pub dynamo_kv_event_format: bool,
|
|
pub mamba_cache_chunk_size: Option<usize>,
|
|
pub mamba_max_states_per_path: Option<usize>,
|
|
}
|
|
|
|
impl TreeCoreInitParamsBinding {
|
|
/// Convert into the tree core's construction params.
|
|
fn to_cache_init_params(&self) -> PyResult<CacheInitParams> {
|
|
Ok(CacheInitParams {
|
|
eviction_policy: self.eviction_policy.clone(),
|
|
slru_protected_threshold: self.slru_protected_threshold,
|
|
tlru_tail_budget: self.tlru_tail_budget,
|
|
tlru_float_config: self.tlru_float_config.as_ref().map(|value| value.config),
|
|
page_size: self.page_size,
|
|
is_write_back: self.is_write_back,
|
|
enable_hicache: self.enable_hicache,
|
|
write_through_threshold: self.write_through_threshold,
|
|
device: parse_device(&self.device)?,
|
|
swa_sliding_window_size: self.swa_sliding_window_size,
|
|
swa_req_ring: self.swa_req_ring,
|
|
// Wired post-construction via set_has_swa_host_pool.
|
|
has_swa_host_pool: false,
|
|
enable_kv_cache_events: self.enable_kv_cache_events,
|
|
dynamo_kv_event_format: self.dynamo_kv_event_format,
|
|
mamba_cache_chunk_size: self.mamba_cache_chunk_size,
|
|
mamba_max_states_per_path: self.mamba_max_states_per_path,
|
|
})
|
|
}
|
|
}
|
|
|
|
#[pymethods]
|
|
impl TreeCoreInitParamsBinding {
|
|
#[new]
|
|
#[pyo3(signature = (eviction_policy = "lru".to_string(), page_size = 1, is_write_back = false, enable_hicache = false, write_through_threshold = 256, device = "cpu".to_string(), swa_sliding_window_size = None, enable_kv_cache_events = false, mamba_cache_chunk_size = None, mamba_max_states_per_path = None, slru_protected_threshold = 2, swa_req_ring = false, tlru_tail_budget = 0, tlru_float_config = None, dynamo_kv_event_format = false))]
|
|
fn new(
|
|
eviction_policy: String,
|
|
page_size: usize,
|
|
is_write_back: bool,
|
|
enable_hicache: bool,
|
|
write_through_threshold: i64,
|
|
device: String,
|
|
swa_sliding_window_size: Option<usize>,
|
|
enable_kv_cache_events: bool,
|
|
mamba_cache_chunk_size: Option<usize>,
|
|
mamba_max_states_per_path: Option<usize>,
|
|
slru_protected_threshold: i64,
|
|
swa_req_ring: bool,
|
|
tlru_tail_budget: usize,
|
|
tlru_float_config: Option<TlruFloatConfigBinding>,
|
|
dynamo_kv_event_format: bool,
|
|
) -> Self {
|
|
TreeCoreInitParamsBinding {
|
|
eviction_policy,
|
|
slru_protected_threshold,
|
|
tlru_tail_budget,
|
|
tlru_float_config,
|
|
page_size,
|
|
is_write_back,
|
|
enable_hicache,
|
|
write_through_threshold,
|
|
device,
|
|
swa_sliding_window_size,
|
|
swa_req_ring,
|
|
enable_kv_cache_events,
|
|
dynamo_kv_event_format,
|
|
mamba_cache_chunk_size,
|
|
mamba_max_states_per_path,
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Python-visible match params; converts into MatchPrefixParams.
|
|
#[pyclass(get_all, set_all, from_py_object)]
|
|
#[derive(Clone)]
|
|
pub struct MatchParamsBinding {
|
|
pub key: Vec<i64>,
|
|
pub extra_key: Option<String>,
|
|
pub cache_salt: Option<String>,
|
|
}
|
|
|
|
#[pymethods]
|
|
impl MatchParamsBinding {
|
|
#[new]
|
|
#[pyo3(signature = (key, extra_key = None, cache_salt = None))]
|
|
fn new(
|
|
py: Python<'_>,
|
|
key: &Bound<'_, PyAny>,
|
|
extra_key: Option<String>,
|
|
cache_salt: Option<String>,
|
|
) -> PyResult<Self> {
|
|
Ok(MatchParamsBinding {
|
|
key: py_array_to_vec_i64(py, key)?,
|
|
extra_key,
|
|
cache_salt,
|
|
})
|
|
}
|
|
}
|
|
|
|
/// Python-visible insert params; converts into InsertParams. The value tensor
|
|
/// stays a Python-held reference until the insert call unwraps it.
|
|
#[pyclass(get_all, set_all)]
|
|
pub struct InsertParamsBinding {
|
|
pub rotation_base: Option<i64>,
|
|
pub key: Vec<i64>,
|
|
pub value: Py<PyAny>,
|
|
pub extra_key: Option<String>,
|
|
pub cache_salt: Option<String>,
|
|
pub session_id: Option<String>,
|
|
pub mamba_value: Option<Py<PyAny>>,
|
|
pub prev_prefix_len: usize,
|
|
pub swa_evicted_seqlen: usize,
|
|
pub swa_branching_seqlen: Option<usize>,
|
|
pub inserted_len: usize,
|
|
pub priority: i64,
|
|
pub track_adopted_ranges: bool,
|
|
}
|
|
|
|
#[pymethods]
|
|
impl InsertParamsBinding {
|
|
#[new]
|
|
#[pyo3(signature = (key, value, extra_key = None, cache_salt = None, session_id = None, prev_prefix_len = 0, swa_evicted_seqlen = 0, swa_branching_seqlen = None, inserted_len = 0, priority = 0, mamba_value = None, track_adopted_ranges = false, rotation_base = None))]
|
|
fn new(
|
|
py: Python<'_>,
|
|
key: &Bound<'_, PyAny>,
|
|
value: Py<PyAny>,
|
|
extra_key: Option<String>,
|
|
cache_salt: Option<String>,
|
|
session_id: Option<String>,
|
|
prev_prefix_len: usize,
|
|
swa_evicted_seqlen: usize,
|
|
swa_branching_seqlen: Option<usize>,
|
|
inserted_len: usize,
|
|
priority: i64,
|
|
mamba_value: Option<Py<PyAny>>,
|
|
track_adopted_ranges: bool,
|
|
rotation_base: Option<i64>,
|
|
) -> PyResult<Self> {
|
|
Ok(InsertParamsBinding {
|
|
rotation_base,
|
|
key: py_array_to_vec_i64(py, key)?,
|
|
value,
|
|
extra_key,
|
|
cache_salt,
|
|
session_id,
|
|
mamba_value,
|
|
prev_prefix_len,
|
|
swa_evicted_seqlen,
|
|
swa_branching_seqlen,
|
|
inserted_len,
|
|
priority,
|
|
track_adopted_ranges,
|
|
})
|
|
}
|
|
}
|
|
|
|
/// Python-visible match result; actions are Python-held.
|
|
#[pyclass(get_all)]
|
|
pub struct MatchResultBinding {
|
|
device_prefix_len: usize,
|
|
last_device_node_id: NodeId,
|
|
last_host_node_id: NodeId,
|
|
best_match_node_id: NodeId,
|
|
host_hit_length: usize,
|
|
swa_host_hit_length: usize,
|
|
swa_branching_seqlen: Option<usize>,
|
|
mamba_host_hit_length: usize,
|
|
mamba_branching_seqlen: Option<usize>,
|
|
full_kv_hit_length: usize,
|
|
cache_actions: Py<PyList>,
|
|
}
|
|
|
|
impl MatchResultBinding {
|
|
/// Move a core match result across the boundary.
|
|
fn from_match_result(py: Python<'_>, result: MatchResult) -> PyResult<Self> {
|
|
Ok(MatchResultBinding {
|
|
device_prefix_len: result.device_prefix_len,
|
|
last_device_node_id: result.last_device_node_id,
|
|
last_host_node_id: result.last_host_node_id,
|
|
best_match_node_id: result.best_match_node_id,
|
|
host_hit_length: result.host_hit_length,
|
|
swa_host_hit_length: result.swa_host_hit_length,
|
|
swa_branching_seqlen: result.swa_branching_seqlen,
|
|
mamba_host_hit_length: result.mamba_host_hit_length,
|
|
mamba_branching_seqlen: result.mamba_branching_seqlen,
|
|
full_kv_hit_length: result.full_kv_hit_length,
|
|
cache_actions: cache_actions_to_py(py, result.cache_actions)?,
|
|
})
|
|
}
|
|
}
|
|
|
|
/// Python-visible insert result; actions are Python-held.
|
|
#[pyclass(get_all)]
|
|
pub struct InsertResultBinding {
|
|
rotation_tail_declined: bool,
|
|
prefix_len: usize,
|
|
total_len: usize,
|
|
last_device_node: Option<NodeId>,
|
|
inserted_host_node: Option<NodeId>,
|
|
host_insert_dropped: bool,
|
|
mamba_exist: bool,
|
|
swa_branch_inserted: bool,
|
|
adopted_ranges: Option<HashMap<u8, Vec<(usize, usize)>>>,
|
|
cache_actions: Py<PyList>,
|
|
}
|
|
|
|
/// One step of the resumable insert: the actions to apply at this barrier
|
|
/// and the final result once the walk completes.
|
|
#[pyclass(get_all)]
|
|
pub struct InsertStepResultBinding {
|
|
actions: Py<PyList>,
|
|
result: Option<Py<InsertResultBinding>>,
|
|
}
|
|
|
|
impl InsertStepResultBinding {
|
|
/// Move a core insert step across the boundary.
|
|
fn from_insert_step(py: Python<'_>, step: InsertStepResult) -> PyResult<Self> {
|
|
let result = match step.result {
|
|
Some(result) => Some(Py::new(
|
|
py,
|
|
InsertResultBinding::from_insert_result(py, result)?,
|
|
)?),
|
|
None => None,
|
|
};
|
|
Ok(InsertStepResultBinding {
|
|
actions: cache_actions_to_py(py, step.actions)?,
|
|
result,
|
|
})
|
|
}
|
|
}
|
|
|
|
impl InsertResultBinding {
|
|
/// Move a core insert result across the boundary.
|
|
fn from_insert_result(py: Python<'_>, result: InsertResult) -> PyResult<Self> {
|
|
Ok(InsertResultBinding {
|
|
rotation_tail_declined: result.rotation_tail_declined,
|
|
prefix_len: result.prefix_len,
|
|
total_len: result.total_len,
|
|
last_device_node: result.last_device_node_id,
|
|
inserted_host_node: result.inserted_host_node,
|
|
host_insert_dropped: result.host_insert_dropped,
|
|
mamba_exist: result.mamba_exist,
|
|
swa_branch_inserted: result.swa_branch_inserted,
|
|
adopted_ranges: result.adopted_ranges.map(|ranges| {
|
|
ranges
|
|
.into_iter()
|
|
.map(|(component_type, ranges)| (component_type_to_u8(component_type), ranges))
|
|
.collect()
|
|
}),
|
|
cache_actions: cache_actions_to_py(py, result.cache_actions)?,
|
|
})
|
|
}
|
|
}
|
|
|
|
/// Python-visible dec-lock params; converts into DecLockRefParams.
|
|
#[pyclass(from_py_object)]
|
|
#[derive(Clone, Default)]
|
|
pub struct DecLockRefParamsBinding {
|
|
#[pyo3(get, set)]
|
|
pub node_id: Option<NodeId>,
|
|
pub skipped_lock_components: Vec<u8>,
|
|
#[pyo3(get, set)]
|
|
pub component_lock_uuids: HashMap<u8, Option<i64>>,
|
|
#[pyo3(get, set)]
|
|
pub component_host_lock_uuids: HashMap<u8, Option<i64>>,
|
|
}
|
|
|
|
#[pymethods]
|
|
impl DecLockRefParamsBinding {
|
|
/// A plain get would turn `Vec<u8>` into `bytes`; Python expects `list[int]`.
|
|
#[getter]
|
|
fn skipped_lock_components<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyList>> {
|
|
PyList::new(py, &self.skipped_lock_components)
|
|
}
|
|
|
|
#[setter]
|
|
fn set_skipped_lock_components(&mut self, value: Vec<u8>) {
|
|
self.skipped_lock_components = value;
|
|
}
|
|
|
|
#[new]
|
|
#[pyo3(signature = (node_id = None, skipped_lock_components = Vec::new(), *, component_lock_uuids = None, component_host_lock_uuids = None))]
|
|
fn new(
|
|
node_id: Option<NodeId>,
|
|
skipped_lock_components: Vec<u8>,
|
|
component_lock_uuids: Option<HashMap<u8, Option<i64>>>,
|
|
component_host_lock_uuids: Option<HashMap<u8, Option<i64>>>,
|
|
) -> Self {
|
|
DecLockRefParamsBinding {
|
|
node_id,
|
|
skipped_lock_components,
|
|
component_lock_uuids: component_lock_uuids.unwrap_or_default(),
|
|
component_host_lock_uuids: component_host_lock_uuids.unwrap_or_default(),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl DecLockRefParamsBinding {
|
|
/// Convert into the tree core's dec-lock params.
|
|
fn to_dec_lock_ref_params(&self) -> PyResult<DecLockRefParams> {
|
|
Ok(DecLockRefParams {
|
|
node_id: self.node_id,
|
|
skipped_lock_components: component_set_from_py(&self.skipped_lock_components)?,
|
|
component_lock_uuids: self.component_lock_uuids.clone(),
|
|
component_host_lock_uuids: self.component_host_lock_uuids.clone(),
|
|
})
|
|
}
|
|
}
|
|
|
|
/// Python-visible inc_lock_ref result; the receipt (anchor node, boundary
|
|
/// uuids, skipped components) is handed back to the matching dec_lock_ref.
|
|
#[pyclass]
|
|
pub struct IncLockRefResultBinding {
|
|
#[pyo3(get)]
|
|
delta: Option<usize>,
|
|
#[pyo3(get)]
|
|
node_id: Option<NodeId>,
|
|
skipped_lock_components: Vec<u8>,
|
|
#[pyo3(get)]
|
|
component_lock_uuids: HashMap<u8, Option<i64>>,
|
|
#[pyo3(get)]
|
|
component_host_lock_uuids: HashMap<u8, Option<i64>>,
|
|
}
|
|
|
|
#[pymethods]
|
|
impl IncLockRefResultBinding {
|
|
/// A plain get would turn `Vec<u8>` into `bytes`; Python expects `list[int]`.
|
|
#[getter]
|
|
fn skipped_lock_components<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyList>> {
|
|
PyList::new(py, &self.skipped_lock_components)
|
|
}
|
|
}
|
|
|
|
impl IncLockRefResultBinding {
|
|
fn from_result(result: crate::unified_tree_core::IncLockRefResult) -> Self {
|
|
Self {
|
|
delta: result.delta,
|
|
node_id: result.node_id,
|
|
component_lock_uuids: result.component_lock_uuids,
|
|
component_host_lock_uuids: result.component_host_lock_uuids,
|
|
skipped_lock_components: result
|
|
.skipped_lock_components
|
|
.iter()
|
|
.map(|ct| ct.idx() as u8)
|
|
.collect(),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Parse Python component-type ids into a component set.
|
|
fn component_set_from_py(component_types: &[u8]) -> PyResult<ComponentSet> {
|
|
component_types
|
|
.iter()
|
|
.map(|ct| parse_component_type(*ct))
|
|
.collect()
|
|
}
|
|
|
|
/// Convert a Python component-keyed tracker into the core's counts.
|
|
fn tracker_from_py(tracker: HashMap<u8, usize>) -> PyResult<HashMap<ComponentType, usize>> {
|
|
tracker
|
|
.into_iter()
|
|
.map(|(ct, freed)| Ok((parse_component_type(ct)?, freed)))
|
|
.collect()
|
|
}
|
|
|
|
/// Convert the core's tracker back into Python component-keyed counts.
|
|
fn tracker_to_py(tracker: HashMap<ComponentType, usize>) -> HashMap<u8, usize> {
|
|
tracker
|
|
.into_iter()
|
|
.map(|(ct, freed)| (component_type_to_u8(ct), freed))
|
|
.collect()
|
|
}
|
|
|
|
/// Next-eviction-node step result: the node to evict, whether the walk made
|
|
/// progress, this step's per-component evicted counts, and this step's newly
|
|
/// freed tensors.
|
|
#[pyclass(get_all)]
|
|
pub struct EvictDeviceNextNodeResultBinding {
|
|
node_id: Option<NodeId>,
|
|
mamba_backup_node_id: Option<NodeId>,
|
|
swa_backup_node_id: Option<NodeId>,
|
|
swa_backup_num_tokens: usize,
|
|
made_progress: bool,
|
|
unbacked_tokens: usize,
|
|
tracker: HashMap<u8, usize>,
|
|
new_device_frees: Py<PyDict>,
|
|
new_host_frees: Py<PyDict>,
|
|
}
|
|
|
|
/// Leaf-eviction step result: the backup action for an unbacked
|
|
/// write-back leaf (else None), this step's per-component evicted counts,
|
|
/// and this step's newly freed tensors.
|
|
#[pyclass(get_all)]
|
|
pub struct EvictDeviceLeafResultBinding {
|
|
backup_kv: Option<Py<PyAny>>,
|
|
unbacked_tokens: usize,
|
|
tracker: HashMap<u8, usize>,
|
|
new_device_frees: Py<PyDict>,
|
|
new_host_frees: Py<PyDict>,
|
|
}
|
|
|
|
/// Drop-subtree result: whether the drop happened, this step's
|
|
/// per-component evicted counts, and the subtree's newly freed tensors.
|
|
#[pyclass(get_all)]
|
|
pub struct DropSubtreeResultBinding {
|
|
dropped: bool,
|
|
tracker: HashMap<u8, usize>,
|
|
new_device_frees: Py<PyDict>,
|
|
new_host_frees: Py<PyDict>,
|
|
}
|
|
|
|
/// Demote result: this step's per-component evicted counts and the
|
|
/// demoted node's newly freed tensors.
|
|
#[pyclass(get_all)]
|
|
pub struct DemoteResultBinding {
|
|
tracker: HashMap<u8, usize>,
|
|
new_device_frees: Py<PyDict>,
|
|
new_host_frees: Py<PyDict>,
|
|
}
|
|
|
|
/// Python-visible device->storage backup spec.
|
|
#[pyclass(get_all)]
|
|
pub struct StorageBackupSpecBinding {
|
|
host_value: Py<PyAny>,
|
|
/// Raw native-endian int64 bytes; python rebuilds them via array("q").frombytes.
|
|
token_ids: Py<PyBytes>,
|
|
hash_value: Option<Vec<String>>,
|
|
prefix_keys: Option<Vec<String>>,
|
|
comp_xfers: Py<PyDict>,
|
|
}
|
|
|
|
#[pyclass(get_all)]
|
|
pub struct BufferBackupSnapshotBinding {
|
|
node_id: NodeId,
|
|
parent_node_id: NodeId,
|
|
parent_is_root: bool,
|
|
parent_last_hash: Option<String>,
|
|
key_token_ids: Py<PyBytes>,
|
|
extra_key: Option<String>,
|
|
cache_salt: Option<String>,
|
|
is_bigram: bool,
|
|
hash_values: Vec<String>,
|
|
prefix_keys: Option<Vec<String>>,
|
|
}
|
|
|
|
impl BufferBackupSnapshotBinding {
|
|
fn from_snapshot(py: Python<'_>, snapshot: BufferBackupSnapshot) -> Self {
|
|
let mut token_bytes = Vec::with_capacity(snapshot.token_ids.len() * 8);
|
|
for token in snapshot.token_ids {
|
|
token_bytes.extend_from_slice(&token.to_ne_bytes());
|
|
}
|
|
Self {
|
|
node_id: snapshot.node_id,
|
|
parent_node_id: snapshot.parent_node_id,
|
|
parent_is_root: snapshot.parent_is_root,
|
|
parent_last_hash: snapshot.parent_last_hash,
|
|
key_token_ids: PyBytes::new(py, &token_bytes).unbind(),
|
|
extra_key: snapshot.extra_key,
|
|
cache_salt: snapshot.cache_salt,
|
|
is_bigram: snapshot.is_bigram,
|
|
hash_values: snapshot.hash_values,
|
|
prefix_keys: snapshot.prefix_keys,
|
|
}
|
|
}
|
|
}
|
|
|
|
#[pyclass(get_all)]
|
|
pub struct BufferBackupStateBinding {
|
|
parent_node_id: NodeId,
|
|
parent_is_root: bool,
|
|
parent_last_hash: Option<String>,
|
|
}
|
|
|
|
impl From<BufferBackupState> for BufferBackupStateBinding {
|
|
fn from(state: BufferBackupState) -> Self {
|
|
Self {
|
|
parent_node_id: state.parent_node_id,
|
|
parent_is_root: state.parent_is_root,
|
|
parent_last_hash: state.parent_last_hash,
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Python-visible KV-canary walk rows: parallel int64 tensors of slots,
|
|
/// positions, and preceding slots.
|
|
#[pyclass(get_all)]
|
|
pub struct KvCanaryWalkResultBinding {
|
|
slot_indices: Py<PyAny>,
|
|
positions: Py<PyAny>,
|
|
prev_slot_indices: Py<PyAny>,
|
|
}
|
|
|
|
/// Host-eviction drive result: this step's per-component evicted counts
|
|
/// and the drive's newly freed tensors.
|
|
#[pyclass(get_all)]
|
|
pub struct HostEvictionResultBinding {
|
|
tracker: HashMap<u8, usize>,
|
|
new_device_frees: Py<PyDict>,
|
|
new_host_frees: Py<PyDict>,
|
|
}
|
|
|
|
impl HostEvictionResultBinding {
|
|
fn from_eviction_step(py: Python<'_>, result: EvictionStepResult) -> PyResult<Self> {
|
|
Ok(Self {
|
|
tracker: tracker_to_py(result.tracker),
|
|
new_device_frees: frees_to_py(py, result.device_frees)?,
|
|
new_host_frees: frees_to_py(py, result.host_frees)?,
|
|
})
|
|
}
|
|
}
|
|
|
|
type TreePathNodeInfoTuple = (NodeId, NodeId, usize, usize, Option<usize>, usize);
|
|
|
|
/// The generic UnifiedTreeCore adapter the per-key-type pyclasses delegate to;
|
|
/// the Mutex makes the Send-only core satisfy pyclass's Sync bound.
|
|
struct TreeCoreBinding<K: ChildKeyType> {
|
|
core: Mutex<UnifiedTreeCore<K>>,
|
|
/// The core's construction device, kept outside the Mutex for pre-lock validation.
|
|
device: Device,
|
|
/// The core's page size, kept outside the Mutex for pre-lock validation.
|
|
page_size: usize,
|
|
}
|
|
|
|
// Send + Sync lets detach release the GIL around core calls.
|
|
impl<K: ChildKeyType + Send + Sync> TreeCoreBinding<K> {
|
|
/// Build a tree core for the given component types from the cache's
|
|
/// init params.
|
|
fn new(init_params: &TreeCoreInitParamsBinding, component_types: Vec<u8>) -> PyResult<Self> {
|
|
let component_types = component_types
|
|
.into_iter()
|
|
.map(parse_component_type)
|
|
.collect::<PyResult<Vec<_>>>()?;
|
|
if component_types != [FULL]
|
|
&& component_types != [FULL, SWA]
|
|
&& component_types != [FULL, MAMBA]
|
|
&& component_types != [FULL, SWA, MAMBA]
|
|
{
|
|
return Err(PyValueError::new_err(
|
|
"only the [Full], [Full, Swa], [Full, Mamba], and [Full, Swa, Mamba] component sets are supported",
|
|
));
|
|
}
|
|
if component_types.contains(&SWA) && init_params.swa_sliding_window_size.is_none() {
|
|
return Err(PyValueError::new_err(
|
|
"the Swa component requires swa_sliding_window_size",
|
|
));
|
|
}
|
|
if component_types.contains(&MAMBA) && init_params.mamba_cache_chunk_size.is_none() {
|
|
return Err(PyValueError::new_err(
|
|
"the Mamba component requires mamba_cache_chunk_size",
|
|
));
|
|
}
|
|
if init_params.page_size == 0 {
|
|
return Err(PyValueError::new_err("page_size must be at least 1"));
|
|
}
|
|
let eviction_policy = init_params.eviction_policy.to_lowercase();
|
|
if !matches!(
|
|
eviction_policy.as_str(),
|
|
"lru" | "lfu" | "fifo" | "mru" | "filo" | "priority" | "slru" | "tlru"
|
|
) {
|
|
return Err(PyValueError::new_err(format!(
|
|
"Unknown eviction policy: {eviction_policy}. Supported policies: \
|
|
'lru', 'lfu', 'fifo', 'mru', 'filo', 'priority', 'slru', 'tlru'."
|
|
)));
|
|
}
|
|
let params = init_params.to_cache_init_params()?;
|
|
let device = params.device;
|
|
let page_size = params.page_size;
|
|
Ok(TreeCoreBinding {
|
|
core: Mutex::new(UnifiedTreeCore::new(params, component_types)),
|
|
device,
|
|
page_size,
|
|
})
|
|
}
|
|
|
|
/// Lock the core for one adapter call. A panic can leave a mutation half-applied,
|
|
/// so a poisoned core is never reused.
|
|
fn core(&self) -> std::sync::MutexGuard<'_, UnifiedTreeCore<K>> {
|
|
self.core.lock().unwrap_or_else(|_| {
|
|
panic!("Rust TreeCore mutex poisoned; refusing to reuse state after an earlier panic")
|
|
})
|
|
}
|
|
|
|
/// Reject an insert value whose dtype, device, or length cannot cover the key.
|
|
fn validate_insert_value(&self, value: &Tensor, key_atom_len: usize) -> PyResult<()> {
|
|
if value.kind() != Kind::Int64 {
|
|
return Err(PyValueError::new_err(format!(
|
|
"insert value must be an int64 tensor, got {:?}",
|
|
value.kind()
|
|
)));
|
|
}
|
|
if value.device() != self.device {
|
|
return Err(PyValueError::new_err(format!(
|
|
"insert value device {:?} does not match the tree core device {:?}",
|
|
value.device(),
|
|
self.device
|
|
)));
|
|
}
|
|
let aligned_key_len = key_atom_len / self.page_size * self.page_size;
|
|
let value_len = value.size().first().copied().unwrap_or(0);
|
|
if value_len < aligned_key_len as i64 {
|
|
return Err(PyValueError::new_err(format!(
|
|
"insert value length {value_len} is shorter than the aligned key length {aligned_key_len}"
|
|
)));
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Drop the entire tree and reinitialize empty state.
|
|
fn reset(&self, py: Python<'_>) {
|
|
py.detach(|| self.core().reset());
|
|
}
|
|
|
|
/// Match a key against the tree.
|
|
fn match_prefix(
|
|
&self,
|
|
py: Python<'_>,
|
|
params: &MatchParamsBinding,
|
|
) -> PyResult<MatchResultBinding> {
|
|
let key = K::key_from(Cow::Borrowed(¶ms.key));
|
|
let key = key.as_ref();
|
|
let params = MatchPrefixParams {
|
|
key,
|
|
namespace: KeyNamespaceRef::new(
|
|
params.extra_key.as_deref(),
|
|
params.cache_salt.as_deref(),
|
|
),
|
|
};
|
|
let result = py.detach(|| self.core().match_prefix(¶ms));
|
|
MatchResultBinding::from_match_result(py, result)
|
|
}
|
|
|
|
/// Read-only FULL-device match, independent of auxiliary components.
|
|
fn match_full_device_prefix(
|
|
&self,
|
|
py: Python<'_>,
|
|
params: &MatchParamsBinding,
|
|
) -> (usize, NodeId, usize) {
|
|
let key = K::key_from(Cow::Borrowed(¶ms.key));
|
|
let key = key.as_ref();
|
|
let namespace =
|
|
KeyNamespaceRef::new(params.extra_key.as_deref(), params.cache_salt.as_deref());
|
|
py.detach(|| self.core().match_full_device_prefix(key, namespace))
|
|
}
|
|
|
|
fn swa_tombstone_ranges(
|
|
&self,
|
|
py: Python<'_>,
|
|
params: &MatchParamsBinding,
|
|
start: usize,
|
|
end: usize,
|
|
) -> PyResult<Vec<(usize, usize)>> {
|
|
let key = K::key_from(Cow::Borrowed(¶ms.key));
|
|
let namespace =
|
|
KeyNamespaceRef::new(params.extra_key.as_deref(), params.cache_salt.as_deref());
|
|
py.detach(|| {
|
|
self.core()
|
|
.swa_tombstone_ranges(key.as_ref(), namespace, start, end)
|
|
})
|
|
.map_err(PyAssertionError::new_err)
|
|
}
|
|
|
|
fn attach_swa_window(
|
|
&self,
|
|
py: Python<'_>,
|
|
params: &MatchParamsBinding,
|
|
window_start: usize,
|
|
window_end: usize,
|
|
swa_values: PyTensor,
|
|
) -> PyResult<Py<PyList>> {
|
|
let key = K::key_from(Cow::Borrowed(¶ms.key));
|
|
let namespace =
|
|
KeyNamespaceRef::new(params.extra_key.as_deref(), params.cache_salt.as_deref());
|
|
let swa_values = swa_values.0;
|
|
let actions = py
|
|
.detach(move || {
|
|
self.core().attach_swa_window(
|
|
key.as_ref(),
|
|
namespace,
|
|
window_start,
|
|
window_end,
|
|
&swa_values,
|
|
)
|
|
})
|
|
.map_err(PyAssertionError::new_err)?;
|
|
cache_actions_to_py(py, actions)
|
|
}
|
|
|
|
/// The empty match result anchored at the root.
|
|
fn empty_match_result(&self, py: Python<'_>) -> PyResult<MatchResultBinding> {
|
|
let result = py.detach(|| self.core().empty_match_result());
|
|
MatchResultBinding::from_match_result(py, result)
|
|
}
|
|
|
|
/// Insert device values into the tree per the provided key.
|
|
fn insert(
|
|
&self,
|
|
py: Python<'_>,
|
|
params: &InsertParamsBinding,
|
|
) -> PyResult<InsertResultBinding> {
|
|
let key = K::key_from(Cow::Borrowed(¶ms.key));
|
|
let key = key.as_ref();
|
|
let value: PyTensor = params.value.bind(py).extract()?;
|
|
// The value covers key atoms (bigram: raw len - 1), so validate the converted key.
|
|
self.validate_insert_value(&value.0, key.atom_len())?;
|
|
let mamba_value = match ¶ms.mamba_value {
|
|
Some(mamba_value) => Some(mamba_value.bind(py).extract::<PyTensor>()?.0),
|
|
None => None,
|
|
};
|
|
let params = InsertParams {
|
|
rotation_base: params.rotation_base,
|
|
key,
|
|
namespace: KeyNamespaceRef::new(
|
|
params.extra_key.as_deref(),
|
|
params.cache_salt.as_deref(),
|
|
),
|
|
session_id: params.session_id.as_deref(),
|
|
value: value.0,
|
|
mamba_value,
|
|
prev_prefix_len: params.prev_prefix_len,
|
|
swa_evicted_seqlen: params.swa_evicted_seqlen,
|
|
swa_branching_seqlen: params.swa_branching_seqlen,
|
|
inserted_len: params.inserted_len,
|
|
priority: params.priority,
|
|
track_adopted_ranges: params.track_adopted_ranges,
|
|
};
|
|
let result = py
|
|
.detach(move || self.core().try_insert(¶ms))
|
|
.map_err(tree_core_runtime_error)?;
|
|
InsertResultBinding::from_insert_result(py, result)
|
|
}
|
|
|
|
/// Start the resumable insert, running to its first barrier or completion.
|
|
fn begin_insert(
|
|
&self,
|
|
py: Python<'_>,
|
|
params: &InsertParamsBinding,
|
|
) -> PyResult<InsertStepResultBinding> {
|
|
let key = K::key_from(Cow::Borrowed(¶ms.key));
|
|
let key = key.as_ref();
|
|
let value: PyTensor = params.value.bind(py).extract()?;
|
|
// The value covers key atoms (bigram: raw len - 1), so validate the converted key.
|
|
self.validate_insert_value(&value.0, key.atom_len())?;
|
|
let mamba_value = match ¶ms.mamba_value {
|
|
Some(mamba_value) => Some(mamba_value.bind(py).extract::<PyTensor>()?.0),
|
|
None => None,
|
|
};
|
|
let params = InsertParams {
|
|
rotation_base: params.rotation_base,
|
|
key,
|
|
namespace: KeyNamespaceRef::new(
|
|
params.extra_key.as_deref(),
|
|
params.cache_salt.as_deref(),
|
|
),
|
|
session_id: params.session_id.as_deref(),
|
|
value: value.0,
|
|
mamba_value,
|
|
prev_prefix_len: params.prev_prefix_len,
|
|
swa_evicted_seqlen: params.swa_evicted_seqlen,
|
|
swa_branching_seqlen: params.swa_branching_seqlen,
|
|
inserted_len: params.inserted_len,
|
|
priority: params.priority,
|
|
track_adopted_ranges: params.track_adopted_ranges,
|
|
};
|
|
let step = py
|
|
.detach(move || self.core().try_begin_insert(¶ms))
|
|
.map_err(tree_core_runtime_error)?;
|
|
InsertStepResultBinding::from_insert_step(py, step)
|
|
}
|
|
|
|
/// Continue the suspended insert after its step actions were applied.
|
|
fn resume_insert(&self, py: Python<'_>) -> PyResult<InsertStepResultBinding> {
|
|
let step = py
|
|
.detach(|| self.core().try_resume_insert())
|
|
.map_err(tree_core_runtime_error)?;
|
|
InsertStepResultBinding::from_insert_step(py, step)
|
|
}
|
|
|
|
/// Whether an insert walk is suspended at a barrier.
|
|
fn has_ongoing_insert(&self, py: Python<'_>) -> bool {
|
|
py.detach(|| self.core().has_ongoing_insert())
|
|
}
|
|
|
|
/// Finish the insert (idempotent); returns still-pending actions to drain.
|
|
fn end_insert(&self, py: Python<'_>) -> PyResult<Py<PyList>> {
|
|
let actions = py.detach(|| self.core().end_insert());
|
|
cache_actions_to_py(py, actions)
|
|
}
|
|
|
|
/// Bump the reference count on a node's component locks; the listed
|
|
/// components are left untaken and recorded in the receipt.
|
|
fn inc_lock_ref(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
skip_lock_components: Vec<u8>,
|
|
) -> PyResult<IncLockRefResultBinding> {
|
|
let skip = component_set_from_py(&skip_lock_components)?;
|
|
let result = py
|
|
.detach(|| self.core().inc_lock_ref(node_id, skip))
|
|
.map_err(node_access_error)?;
|
|
Ok(IncLockRefResultBinding::from_result(result))
|
|
}
|
|
|
|
/// Pin only the FULL device values on a node's root path.
|
|
fn inc_full_pin(&self, py: Python<'_>, node_id: NodeId) -> PyResult<()> {
|
|
py.detach(|| self.core().inc_full_pin(node_id))
|
|
.map_err(node_access_error)
|
|
}
|
|
|
|
/// Release a FULL-only root-path pin.
|
|
fn dec_full_pin(&self, py: Python<'_>, node_id: NodeId) -> PyResult<()> {
|
|
py.detach(|| self.core().dec_full_pin(node_id))
|
|
.map_err(node_access_error)
|
|
}
|
|
|
|
/// Decrease the reference count on a node's component locks.
|
|
fn dec_lock_ref(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
params: &DecLockRefParamsBinding,
|
|
skip_swa: bool,
|
|
) -> PyResult<()> {
|
|
let params = params.to_dec_lock_ref_params()?;
|
|
py.detach(|| self.core().dec_lock_ref(node_id, ¶ms, skip_swa))
|
|
.map_err(node_access_error)?;
|
|
Ok(())
|
|
}
|
|
|
|
/// Release one window's lock, retaining other component locks.
|
|
fn dec_window_lock_only(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
component_type: u8,
|
|
params: &DecLockRefParamsBinding,
|
|
) -> PyResult<(Py<PyDict>, Py<PyDict>)> {
|
|
let component_type = parse_component_type(component_type)?;
|
|
let params = params.to_dec_lock_ref_params()?;
|
|
let (device_frees, host_frees) = py
|
|
.detach(|| {
|
|
let mut device_frees = HashMap::new();
|
|
let mut host_frees = HashMap::new();
|
|
self.core().dec_window_lock_only(
|
|
node_id,
|
|
component_type,
|
|
¶ms,
|
|
&mut device_frees,
|
|
&mut host_frees,
|
|
)?;
|
|
Ok((device_frees, host_frees))
|
|
})
|
|
.map_err(node_access_error)?;
|
|
Ok((frees_to_py(py, device_frees)?, frees_to_py(py, host_frees)?))
|
|
}
|
|
|
|
/// Early-release the SWA portion of a request's tree lock; returns this
|
|
/// release's per-component (device_frees, host_frees).
|
|
fn dec_swa_lock_only(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
params: &DecLockRefParamsBinding,
|
|
) -> PyResult<(Py<PyDict>, Py<PyDict>)> {
|
|
let params = params.to_dec_lock_ref_params()?;
|
|
let (device_frees, host_frees) = py
|
|
.detach(|| {
|
|
let mut device_frees = HashMap::new();
|
|
let mut host_frees = HashMap::new();
|
|
self.core().dec_swa_lock_only(
|
|
node_id,
|
|
¶ms,
|
|
&mut device_frees,
|
|
&mut host_frees,
|
|
)?;
|
|
Ok((device_frees, host_frees))
|
|
})
|
|
.map_err(node_access_error)?;
|
|
Ok((frees_to_py(py, device_frees)?, frees_to_py(py, host_frees)?))
|
|
}
|
|
|
|
/// Adjust the component's reclaimable device size.
|
|
fn adjust_component_size(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
component_type: u8,
|
|
delta: i64,
|
|
) -> PyResult<()> {
|
|
let component_type = parse_component_type(component_type)?;
|
|
py.detach(|| {
|
|
self.core()
|
|
.adjust_component_size(node_id, component_type, delta)
|
|
})
|
|
.map_err(tree_core_runtime_error)
|
|
}
|
|
|
|
/// Collect paths under one tree lock.
|
|
fn collect_path_info(
|
|
&self,
|
|
py: Python<'_>,
|
|
end_node_ids: Vec<NodeId>,
|
|
) -> PyResult<Vec<Vec<TreePathNodeInfoTuple>>> {
|
|
let paths = py
|
|
.detach(|| self.core().collect_path_info(&end_node_ids))
|
|
.map_err(tree_core_runtime_error)?;
|
|
Ok(paths
|
|
.into_iter()
|
|
.map(|path| {
|
|
path.into_iter()
|
|
.map(|node| {
|
|
(
|
|
node.node_id,
|
|
node.parent_id,
|
|
node.path_start,
|
|
node.path_end,
|
|
node.full_value_len,
|
|
node.num_children,
|
|
)
|
|
})
|
|
.collect()
|
|
})
|
|
.collect())
|
|
}
|
|
|
|
/// Store a component's device value on a node (the SWA rebuild write-back).
|
|
fn set_component_device_value(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
component_type: u8,
|
|
value: PyTensor,
|
|
) -> PyResult<()> {
|
|
let component_type = parse_component_type(component_type)?;
|
|
let value = value.0;
|
|
if value.kind() != Kind::Int64 {
|
|
return Err(PyValueError::new_err(format!(
|
|
"component device value must be an int64 tensor, got {:?}",
|
|
value.kind()
|
|
)));
|
|
}
|
|
if value.device() != self.device {
|
|
return Err(PyValueError::new_err(format!(
|
|
"component device value device {:?} does not match the tree core device {:?}",
|
|
value.device(),
|
|
self.device
|
|
)));
|
|
}
|
|
py.detach(|| {
|
|
self.core()
|
|
.set_component_device_value(node_id, component_type, value)
|
|
})
|
|
.map_err(node_access_error)?;
|
|
Ok(())
|
|
}
|
|
|
|
/// Logical key lengths for the nodes in a load-back transfer.
|
|
fn get_node_key_lengths(&self, py: Python<'_>, node_ids: Vec<NodeId>) -> PyResult<Vec<usize>> {
|
|
py.detach(|| self.core().get_node_key_lengths(&node_ids))
|
|
.map_err(node_access_error)
|
|
}
|
|
|
|
/// A component's device value on a node, if set.
|
|
fn get_component_device_value(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
component_type: u8,
|
|
) -> PyResult<Option<Py<PyAny>>> {
|
|
let component_type = parse_component_type(component_type)?;
|
|
let value = py
|
|
.detach(|| {
|
|
self.core()
|
|
.get_component_device_value(node_id, component_type)
|
|
.map(|value| value.map(|tensor| tensor.shallow_clone()))
|
|
})
|
|
.map_err(node_access_error)?;
|
|
value.map(|value| tensor_to_py(py, value)).transpose()
|
|
}
|
|
|
|
// TODO(jialino): batch a full no-backup eviction round in Rust (one crossing
|
|
// instead of 2N+2); step-wise stays for the write-back interleave.
|
|
/// Begin a component's device-eviction walk for up to request_cnt tokens.
|
|
fn evict_device_start(
|
|
&self,
|
|
py: Python<'_>,
|
|
component_type: u8,
|
|
request_cnt: usize,
|
|
) -> PyResult<()> {
|
|
let ct = parse_component_type(component_type)?;
|
|
py.detach(|| self.core().evict_device_start(ct, request_cnt));
|
|
Ok(())
|
|
}
|
|
|
|
/// Advance one component eviction step. Progress without a leaf can be
|
|
/// an internal tombstone or a component backup request awaiting its finish;
|
|
/// no progress means the walk is exhausted.
|
|
fn evict_device_next_node(
|
|
&self,
|
|
py: Python<'_>,
|
|
component_type: u8,
|
|
tracker: HashMap<u8, usize>,
|
|
) -> PyResult<EvictDeviceNextNodeResultBinding> {
|
|
let ct = parse_component_type(component_type)?;
|
|
let baseline = tracker_from_py(tracker)?;
|
|
let (node_id, result) =
|
|
py.detach(move || self.core().evict_device_next_node(ct, &baseline));
|
|
let made_progress = node_id.is_some()
|
|
|| result.mamba_backup_node_id.is_some()
|
|
|| result.swa_backup_node_id.is_some()
|
|
|| !result.tracker.is_empty()
|
|
|| result.device_frees.values().any(|frees| !frees.is_empty())
|
|
|| result.host_frees.values().any(|frees| !frees.is_empty());
|
|
Ok(EvictDeviceNextNodeResultBinding {
|
|
node_id,
|
|
mamba_backup_node_id: result.mamba_backup_node_id,
|
|
swa_backup_node_id: result.swa_backup_node_id,
|
|
swa_backup_num_tokens: result.swa_backup_num_tokens,
|
|
made_progress,
|
|
unbacked_tokens: result.unbacked_tokens,
|
|
tracker: tracker_to_py(result.tracker),
|
|
new_device_frees: frees_to_py(py, result.device_frees)?,
|
|
new_host_frees: frees_to_py(py, result.host_frees)?,
|
|
})
|
|
}
|
|
|
|
/// Resume a pending internal Mamba tombstone after the host backup attempt.
|
|
fn finish_mamba_state_eviction(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
) -> PyResult<EvictDeviceNextNodeResultBinding> {
|
|
let result = py.detach(move || self.core().finish_mamba_state_eviction(node_id));
|
|
self.internal_eviction_finish_to_py(py, result)
|
|
}
|
|
|
|
/// Resume a pending internal SWA tombstone after the host backup attempt.
|
|
fn finish_swa_state_eviction(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
) -> PyResult<EvictDeviceNextNodeResultBinding> {
|
|
let result = py.detach(move || self.core().finish_swa_state_eviction(node_id));
|
|
self.internal_eviction_finish_to_py(py, result)
|
|
}
|
|
|
|
fn internal_eviction_finish_to_py(
|
|
&self,
|
|
py: Python<'_>,
|
|
result: EvictionStepResult,
|
|
) -> PyResult<EvictDeviceNextNodeResultBinding> {
|
|
Ok(EvictDeviceNextNodeResultBinding {
|
|
node_id: None,
|
|
mamba_backup_node_id: None,
|
|
swa_backup_node_id: None,
|
|
swa_backup_num_tokens: 0,
|
|
// Consuming the pending request advances the finite walk even
|
|
// when intervening I/O pinned or removed this particular node.
|
|
made_progress: true,
|
|
unbacked_tokens: result.unbacked_tokens,
|
|
tracker: tracker_to_py(result.tracker),
|
|
new_device_frees: frees_to_py(py, result.device_frees)?,
|
|
new_host_frees: frees_to_py(py, result.host_frees)?,
|
|
})
|
|
}
|
|
|
|
/// Evict one device leaf; an unbacked write-back leaf returns its backup
|
|
/// action for the caller to execute before demoting.
|
|
fn evict_device_leaf(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
) -> PyResult<EvictDeviceLeafResultBinding> {
|
|
let (backup, result) = py
|
|
.detach(move || {
|
|
let mut core = self.core();
|
|
let is_write_back = core.is_write_back;
|
|
core.evict_device_leaf(node_id, is_write_back)
|
|
})
|
|
.map_err(node_access_error)?;
|
|
Ok(EvictDeviceLeafResultBinding {
|
|
unbacked_tokens: result.unbacked_tokens,
|
|
backup_kv: backup
|
|
.map(|backup| cache_action_to_py(py, CacheAction::BackupKV(backup)))
|
|
.transpose()?,
|
|
tracker: tracker_to_py(result.tracker),
|
|
new_device_frees: frees_to_py(py, result.device_frees)?,
|
|
new_host_frees: frees_to_py(py, result.host_frees)?,
|
|
})
|
|
}
|
|
|
|
/// Finish a component's device-eviction walk.
|
|
fn evict_device_end(&self, py: Python<'_>, component_type: u8) -> PyResult<()> {
|
|
let ct = parse_component_type(component_type)?;
|
|
py.detach(|| self.core().evict_device_end(ct));
|
|
Ok(())
|
|
}
|
|
|
|
/// Verify tree-structure, leaf-set, LRU, size, and ongoing-op invariants;
|
|
/// ongoing_* args are (id, node_id) pairs.
|
|
fn sanity_check(
|
|
&self,
|
|
py: Python<'_>,
|
|
ongoing_write_through: Vec<(i64, NodeId)>,
|
|
ongoing_load_back: Vec<(i64, NodeId)>,
|
|
) -> PyResult<()> {
|
|
py.detach(|| {
|
|
self.core()
|
|
.try_sanity_check(&ongoing_write_through, &ongoing_load_back)
|
|
})
|
|
.map_err(PyAssertionError::new_err)
|
|
}
|
|
|
|
/// Concatenated FULL device values from from_node up to (exclusive) until_node.
|
|
fn collect_full_device_indices(
|
|
&self,
|
|
py: Python<'_>,
|
|
from_node_id: NodeId,
|
|
until_node_id: NodeId,
|
|
) -> PyResult<Py<PyAny>> {
|
|
let value = py
|
|
.detach(|| {
|
|
self.core()
|
|
.collect_full_device_indices(from_node_id, until_node_id)
|
|
})
|
|
.map_err(node_access_error)?;
|
|
tensor_to_py(py, value)
|
|
}
|
|
|
|
/// Every FULL device value in the tree, concatenated.
|
|
fn all_values_flatten(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
|
|
let value = py.detach(|| self.core().all_values_flatten());
|
|
tensor_to_py(py, value)
|
|
}
|
|
|
|
/// Every Mamba device value in the tree, concatenated.
|
|
fn all_mamba_values_flatten(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
|
|
let value = py.detach(|| self.core().all_mamba_values_flatten());
|
|
tensor_to_py(py, value)
|
|
}
|
|
|
|
/// Flatten every FULL device slot into (slot, position, prev-slot) rows for the KV-canary sweep.
|
|
fn walk_for_kv_canary(
|
|
&self,
|
|
py: Python<'_>,
|
|
unlocked_only: bool,
|
|
swa_resident_only: bool,
|
|
) -> PyResult<KvCanaryWalkResultBinding> {
|
|
let result = py.detach(|| {
|
|
self.core()
|
|
.walk_for_kv_canary(unlocked_only, swa_resident_only)
|
|
});
|
|
Ok(KvCanaryWalkResultBinding {
|
|
slot_indices: tensor_to_py(py, Tensor::from_slice(&result.slot_indices))?,
|
|
positions: tensor_to_py(py, Tensor::from_slice(&result.positions))?,
|
|
prev_slot_indices: tensor_to_py(py, Tensor::from_slice(&result.prev_slot_indices))?,
|
|
})
|
|
}
|
|
|
|
/// Evictable token count of the FULL (base) component.
|
|
fn evictable_size(&self, py: Python<'_>) -> usize {
|
|
py.detach(|| self.core().evictable_size())
|
|
}
|
|
|
|
/// Protected (locked) token count of the FULL (base) component.
|
|
fn protected_size(&self, py: Python<'_>) -> usize {
|
|
py.detach(|| self.core().protected_size())
|
|
}
|
|
|
|
/// FULL component evictable token count.
|
|
fn full_evictable_size(&self, py: Python<'_>) -> usize {
|
|
py.detach(|| self.core().full_evictable_size())
|
|
}
|
|
|
|
/// FULL component protected token count.
|
|
fn full_protected_size(&self, py: Python<'_>) -> usize {
|
|
py.detach(|| self.core().full_protected_size())
|
|
}
|
|
|
|
/// Evictable token count for one component (0 if the component is absent).
|
|
fn component_evictable_size(&self, py: Python<'_>, component_type: u8) -> PyResult<usize> {
|
|
let ct = parse_component_type(component_type)?;
|
|
Ok(py.detach(|| self.core().component_evictable_size(ct)))
|
|
}
|
|
|
|
/// Protected token count for one component (0 if the component is absent).
|
|
fn component_protected_size(&self, py: Python<'_>, component_type: u8) -> PyResult<usize> {
|
|
let ct = parse_component_type(component_type)?;
|
|
Ok(py.detach(|| self.core().component_protected_size(ct)))
|
|
}
|
|
|
|
/// (full_tokens, aux_tokens) summed across the whole tree.
|
|
fn total_size(&self, py: Python<'_>) -> (usize, usize) {
|
|
py.detach(|| self.core().total_size())
|
|
}
|
|
|
|
/// Whether the node's FULL device value has been evicted.
|
|
fn is_full_device_evicted(&self, py: Python<'_>, node_id: NodeId) -> PyResult<bool> {
|
|
py.detach(|| self.core().is_full_device_evicted(node_id))
|
|
.map_err(node_access_error)
|
|
}
|
|
|
|
/// Mark the host tier as buffer-only; wired after the host pools are built.
|
|
fn set_host_memory_buffer_only(&self, py: Python<'_>) {
|
|
py.detach(|| self.core().set_host_memory_buffer_only());
|
|
}
|
|
|
|
/// Whether the host tier runs as a storage staging buffer, not a cache.
|
|
fn is_host_memory_buffer_only(&self, py: Python<'_>) -> bool {
|
|
py.detach(|| self.core().is_host_memory_buffer_only)
|
|
}
|
|
|
|
/// Mark the host tier (HiCache) as wired.
|
|
fn set_hicache_enabled(&self, py: Python<'_>) {
|
|
py.detach(|| self.core().set_hicache_enabled());
|
|
}
|
|
|
|
/// Whether the host tier (HiCache) is wired.
|
|
fn enable_hicache(&self, py: Python<'_>) -> bool {
|
|
py.detach(|| self.core().enable_hicache)
|
|
}
|
|
|
|
/// Mark the SWA host pool as wired (HiCache).
|
|
fn set_has_swa_host_pool(&self, py: Python<'_>) {
|
|
py.detach(|| self.core().set_has_swa_host_pool());
|
|
}
|
|
|
|
/// Whether the SWA host pool is wired.
|
|
fn has_swa_host_pool(&self, py: Python<'_>) -> bool {
|
|
py.detach(|| self.core().has_swa_host_pool)
|
|
}
|
|
|
|
/// Insert a host-side (backuped) tree path descending from the given node.
|
|
fn insert_host(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
extra_key: Option<String>,
|
|
key: &Bound<'_, PyAny>,
|
|
host_value: PyTensor,
|
|
hash_value: Vec<String>,
|
|
cache_salt: Option<String>,
|
|
) -> PyResult<InsertResultBinding> {
|
|
let key = K::key_from(Cow::Owned(py_array_to_vec_i64(py, key)?)).into_owned();
|
|
let host_value = host_value.0;
|
|
if host_value.kind() != Kind::Int64 {
|
|
return Err(PyValueError::new_err(format!(
|
|
"insert_host host_value must be an int64 tensor, got {:?}",
|
|
host_value.kind()
|
|
)));
|
|
}
|
|
let result = py
|
|
.detach(move || {
|
|
self.core().insert_host_in_namespace(
|
|
node_id,
|
|
KeyNamespaceRef::new(extra_key.as_deref(), cache_salt.as_deref()),
|
|
key,
|
|
host_value,
|
|
hash_value,
|
|
)
|
|
})
|
|
.map_err(tree_core_runtime_error)?;
|
|
InsertResultBinding::from_insert_result(py, result)
|
|
}
|
|
|
|
/// Gather a node's device value plus per-component BACKUP_HOST transfers.
|
|
fn build_backup_spec(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
) -> PyResult<(Py<PyAny>, Py<PyDict>)> {
|
|
let (device_value, comp_xfers) = py
|
|
.detach(|| self.core().build_backup_spec(node_id))
|
|
.map_err(node_access_error)?;
|
|
Ok((
|
|
tensor_to_py(py, device_value)?,
|
|
comp_xfers_to_py(py, comp_xfers)?,
|
|
))
|
|
}
|
|
|
|
/// Gather a node's device->storage backup spec; None if the node is not backuped.
|
|
fn build_storage_backup_spec(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
pass_prefix_keys: bool,
|
|
) -> PyResult<Option<StorageBackupSpecBinding>> {
|
|
let spec = py
|
|
.detach(|| {
|
|
self.core()
|
|
.build_storage_backup_spec(node_id, pass_prefix_keys)
|
|
})
|
|
.map_err(node_access_error)?;
|
|
let Some(spec) = spec else {
|
|
return Ok(None);
|
|
};
|
|
let mut token_bytes = Vec::with_capacity(spec.token_ids.len() * 8);
|
|
for token in &spec.token_ids {
|
|
token_bytes.extend_from_slice(&token.to_ne_bytes());
|
|
}
|
|
Ok(Some(StorageBackupSpecBinding {
|
|
host_value: tensor_to_py(py, spec.host_value)?,
|
|
token_ids: PyBytes::new(py, &token_bytes).unbind(),
|
|
hash_value: spec.hash_value,
|
|
prefix_keys: spec.prefix_keys,
|
|
comp_xfers: comp_xfers_to_py(py, spec.comp_xfers)?,
|
|
}))
|
|
}
|
|
|
|
/// Route a build_hicache_transfers call to the component for the given type.
|
|
#[allow(clippy::too_many_arguments)]
|
|
fn build_hicache_transfers(
|
|
&self,
|
|
py: Python<'_>,
|
|
component_type: u8,
|
|
node_id: NodeId,
|
|
phase: &str,
|
|
host_indices: Option<PyTensor>,
|
|
token_ids: Option<Vec<i64>>,
|
|
prefetch_tokens: usize,
|
|
staging_tokens: usize,
|
|
last_hash: Option<String>,
|
|
) -> PyResult<Option<Vec<Py<PyAny>>>> {
|
|
let component_type = parse_component_type(component_type)?;
|
|
let phase = parse_transfer_phase(phase)?;
|
|
let host_indices = host_indices.map(|t| t.0);
|
|
let transfers = py
|
|
.detach(|| {
|
|
self.core().build_hicache_transfers(
|
|
component_type,
|
|
node_id,
|
|
phase,
|
|
host_indices,
|
|
token_ids.as_deref(),
|
|
prefetch_tokens,
|
|
staging_tokens,
|
|
last_hash.as_deref(),
|
|
)
|
|
})
|
|
.map_err(tree_core_assertion_error)?;
|
|
transfers
|
|
.map(|transfers| {
|
|
transfers
|
|
.into_iter()
|
|
.map(|transfer| transfer_to_py(py, transfer))
|
|
.collect::<PyResult<Vec<_>>>()
|
|
})
|
|
.transpose()
|
|
}
|
|
|
|
/// The anchor node's namespace; None for root-like anchors.
|
|
fn prefetch_anchor_info(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
) -> PyResult<(Option<String>, Option<String>)> {
|
|
py.detach(|| self.core().prefetch_anchor_info(node_id))
|
|
.map_err(node_access_error)
|
|
}
|
|
|
|
/// Whether the node's Full KV is present on host.
|
|
fn node_backuped(&self, py: Python<'_>, node_id: NodeId) -> PyResult<bool> {
|
|
py.detach(|| self.core().node_backuped(node_id))
|
|
.map_err(node_access_error)
|
|
}
|
|
|
|
/// Whether the node is a (default or named) root.
|
|
fn is_root(&self, py: Python<'_>, node_id: NodeId) -> PyResult<bool> {
|
|
py.detach(|| self.core().is_root(node_id))
|
|
.map_err(node_access_error)
|
|
}
|
|
|
|
/// The node's last page hash, or None when it was never hashed.
|
|
fn get_last_hash_value(&self, py: Python<'_>, node_id: NodeId) -> PyResult<Option<String>> {
|
|
py.detach(|| self.core().get_last_hash_value(node_id))
|
|
.map_err(node_access_error)
|
|
}
|
|
|
|
/// The hash chain of the node's ancestors, in root-to-parent order.
|
|
fn get_prefix_hash_values(&self, py: Python<'_>, node_id: NodeId) -> PyResult<Vec<String>> {
|
|
py.detach(|| self.core().get_prefix_hash_values(node_id))
|
|
.map_err(node_access_error)
|
|
}
|
|
|
|
fn get_hash_values(&self, py: Python<'_>, node_id: NodeId) -> PyResult<Vec<String>> {
|
|
py.detach(|| self.core().get_hash_values(node_id))
|
|
.map_err(node_access_error)
|
|
}
|
|
|
|
fn snapshot_buffer_backup(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
pass_prefix_keys: bool,
|
|
) -> Option<BufferBackupSnapshotBinding> {
|
|
py.detach(|| {
|
|
self.core()
|
|
.snapshot_buffer_backup(node_id, pass_prefix_keys)
|
|
})
|
|
.map(|snapshot| BufferBackupSnapshotBinding::from_snapshot(py, snapshot))
|
|
}
|
|
|
|
fn validate_buffer_backup(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
expected_key_length: usize,
|
|
) -> Option<BufferBackupStateBinding> {
|
|
py.detach(|| {
|
|
self.core()
|
|
.validate_buffer_backup(node_id, expected_key_length)
|
|
})
|
|
.map(BufferBackupStateBinding::from)
|
|
}
|
|
|
|
/// Hash every node built while storage was disabled.
|
|
fn backfill_missing_hash_values(&self, py: Python<'_>) -> usize {
|
|
py.detach(|| self.core().backfill_missing_hash_values())
|
|
}
|
|
|
|
fn root_node_handle(&self, py: Python<'_>, extra_key: Option<String>) -> NodeId {
|
|
py.detach(|| self.core().root_node_handle(extra_key.as_deref()))
|
|
}
|
|
|
|
fn rotation_base_of(&self, py: Python<'_>, node_id: NodeId) -> PyResult<Option<i64>> {
|
|
py.detach(|| self.core().rotation_base_of(node_id))
|
|
.map_err(node_access_error)
|
|
}
|
|
|
|
fn dfs_weight_order(&self, py: Python<'_>, node_ids: Vec<NodeId>) -> PyResult<Vec<usize>> {
|
|
py.detach(|| self.core().dfs_weight_order(&node_ids))
|
|
.map_err(node_access_error)
|
|
}
|
|
|
|
/// Commit each component's HiCache transfers; returns the new cache actions.
|
|
fn commit_hicache_transfers(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
phase: &str,
|
|
comp_xfers: HashMap<u8, Vec<TransferArgs>>,
|
|
insert_result: Option<(usize, Option<NodeId>, bool)>,
|
|
pool_storage_result: Option<(usize, HashMap<String, usize>)>,
|
|
) -> PyResult<(Py<PyList>, Option<bool>)> {
|
|
let phase = parse_transfer_phase(phase)?;
|
|
let comp_xfers = comp_xfers_from_args(comp_xfers)?;
|
|
let insert_result =
|
|
insert_result.map(
|
|
|(total_len, inserted_host_node, mamba_exist)| InsertResult {
|
|
total_len,
|
|
inserted_host_node,
|
|
mamba_exist,
|
|
..InsertResult::default()
|
|
},
|
|
);
|
|
let pool_storage_result = pool_storage_result
|
|
.map(|(kv_hit_pages, extra_pool_hit_pages)| {
|
|
Ok::<_, PyErr>(PoolTransferResult {
|
|
kv_hit_pages,
|
|
extra_pool_hit_pages: extra_pool_hit_pages
|
|
.into_iter()
|
|
.map(|(name, pages)| Ok((parse_pool_name(&name)?, pages)))
|
|
.collect::<PyResult<HashMap<_, _>>>()?,
|
|
})
|
|
})
|
|
.transpose()?;
|
|
let (cache_actions, mamba_exist) = py
|
|
.detach(move || {
|
|
let mut cache_actions = Vec::new();
|
|
let mut insert_result = insert_result;
|
|
self.core().commit_hicache_transfers(
|
|
node_id,
|
|
phase,
|
|
comp_xfers,
|
|
&mut cache_actions,
|
|
insert_result.as_mut(),
|
|
pool_storage_result.as_ref(),
|
|
)?;
|
|
Ok((
|
|
cache_actions,
|
|
insert_result.map(|result| result.mamba_exist),
|
|
))
|
|
})
|
|
.map_err(node_access_error)?;
|
|
Ok((cache_actions_to_py(py, cache_actions)?, mamba_exist))
|
|
}
|
|
|
|
/// Commit a successful backup to the node.
|
|
fn commit_backup(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
host_indices: PyTensor,
|
|
comp_xfers: HashMap<u8, Vec<TransferArgs>>,
|
|
) -> PyResult<()> {
|
|
let comp_xfers = comp_xfers_from_args(comp_xfers)?;
|
|
let host_indices = host_indices.0;
|
|
py.detach(move || self.core().commit_backup(node_id, host_indices, comp_xfers))
|
|
.map_err(node_access_error)?;
|
|
Ok(())
|
|
}
|
|
|
|
/// Build the H->D load-back KV transfer plus per-component aux transfers.
|
|
fn build_load_back_spec(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
mamba_pool_idx: Option<PyTensor>,
|
|
) -> PyResult<(Py<PyAny>, Py<PyDict>)> {
|
|
let req = Req {
|
|
mamba_pool_idx: mamba_pool_idx.map(|t| t.0),
|
|
};
|
|
let (kv_xfer, comp_xfers) = py
|
|
.detach(move || self.core().build_load_back_spec(node_id, Some(&req)))
|
|
.map_err(tree_core_assertion_error)?;
|
|
Ok((
|
|
transfer_to_py(py, kv_xfer)?,
|
|
comp_xfers_to_py(py, comp_xfers)?,
|
|
))
|
|
}
|
|
|
|
/// Commit a successful H->D load-back onto the node; returns its actions.
|
|
fn commit_load_back(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
device_indices: PyTensor,
|
|
kv_xfer: TransferArgs,
|
|
comp_xfers: HashMap<u8, Vec<TransferArgs>>,
|
|
) -> PyResult<Py<PyList>> {
|
|
let kv_xfer = transfer_from_args(kv_xfer)?;
|
|
let comp_xfers = comp_xfers_from_args(comp_xfers)?;
|
|
let device_indices = device_indices.0;
|
|
let actions = py
|
|
.detach(move || {
|
|
self.core()
|
|
.commit_load_back(node_id, device_indices, kv_xfer, comp_xfers)
|
|
})
|
|
.map_err(node_access_error)?;
|
|
cache_actions_to_py(py, actions)
|
|
}
|
|
|
|
/// Release a node's device KV once its host copy exists.
|
|
fn demote(&self, py: Python<'_>, node_id: NodeId) -> PyResult<DemoteResultBinding> {
|
|
let result = py
|
|
.detach(move || self.core().demote(node_id))
|
|
.map_err(tree_core_assertion_error)?;
|
|
Ok(DemoteResultBinding {
|
|
tracker: tracker_to_py(result.tracker),
|
|
new_device_frees: frees_to_py(py, result.device_frees)?,
|
|
new_host_frees: frees_to_py(py, result.host_frees)?,
|
|
})
|
|
}
|
|
|
|
/// Evict up to num_tokens of one component's host resources.
|
|
fn drive_host_eviction(
|
|
&self,
|
|
py: Python<'_>,
|
|
component_type: u8,
|
|
num_tokens: usize,
|
|
skip_full_duplicate_reclaim: bool,
|
|
) -> PyResult<HostEvictionResultBinding> {
|
|
let ct = parse_component_type(component_type)?;
|
|
let result = py.detach(move || {
|
|
self.core().drive_host_eviction_with_options(
|
|
ct,
|
|
num_tokens,
|
|
skip_full_duplicate_reclaim,
|
|
)
|
|
});
|
|
Ok(HostEvictionResultBinding {
|
|
tracker: tracker_to_py(result.tracker),
|
|
new_device_frees: frees_to_py(py, result.device_frees)?,
|
|
new_host_frees: frees_to_py(py, result.host_frees)?,
|
|
})
|
|
}
|
|
|
|
/// Evict shallow Mamba device checkpoints beyond the per-path cap on the
|
|
/// tail's root path; returns the step's freed tensors.
|
|
fn evict_excess_path_states(
|
|
&self,
|
|
py: Python<'_>,
|
|
tail_node_id: NodeId,
|
|
) -> PyResult<HostEvictionResultBinding> {
|
|
let result = py
|
|
.detach(move || self.core().evict_excess_path_states(tail_node_id))
|
|
.map_err(node_access_error)?;
|
|
Ok(HostEvictionResultBinding {
|
|
tracker: tracker_to_py(result.tracker),
|
|
new_device_frees: frees_to_py(py, result.device_frees)?,
|
|
new_host_frees: frees_to_py(py, result.host_frees)?,
|
|
})
|
|
}
|
|
|
|
/// Bump the reference count on a node's host-side component locks.
|
|
fn inc_host_lock_ref(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
) -> PyResult<IncLockRefResultBinding> {
|
|
let result = py
|
|
.detach(|| self.core().inc_host_lock_ref(node_id))
|
|
.map_err(node_access_error)?;
|
|
Ok(IncLockRefResultBinding::from_result(result))
|
|
}
|
|
|
|
/// Decrease the reference count on a node's host-side component locks.
|
|
fn dec_host_lock_ref(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
params: &DecLockRefParamsBinding,
|
|
) -> PyResult<()> {
|
|
let params = params.to_dec_lock_ref_params()?;
|
|
py.detach(|| self.core().dec_host_lock_ref(node_id, ¶ms))
|
|
.map_err(node_access_error)?;
|
|
Ok(())
|
|
}
|
|
|
|
fn is_write_through_compatible(&self, py: Python<'_>) -> bool {
|
|
py.detach(|| self.core().is_write_through_compatible())
|
|
}
|
|
|
|
/// Set the write-back (vs write-through) policy; decided at HiCache init.
|
|
fn set_is_write_back(&self, py: Python<'_>, is_write_back: bool) {
|
|
py.detach(|| self.core().is_write_back = is_write_back);
|
|
}
|
|
|
|
/// The current write-back (vs write-through) policy.
|
|
fn is_write_back(&self, py: Python<'_>) -> bool {
|
|
py.detach(|| self.core().is_write_back)
|
|
}
|
|
|
|
/// Set the write-through backup hit threshold; decided at HiCache init.
|
|
fn set_write_through_threshold(&self, py: Python<'_>, threshold: i64) {
|
|
py.detach(|| self.core().write_through_threshold = threshold);
|
|
}
|
|
|
|
/// The current write-through backup hit threshold.
|
|
fn write_through_threshold(&self, py: Python<'_>) -> i64 {
|
|
py.detach(|| self.core().write_through_threshold)
|
|
}
|
|
|
|
/// Mark the storage tier (L3) wired; storage attaches after tree construction.
|
|
fn set_enable_storage(&self, py: Python<'_>, value: bool) {
|
|
py.detach(|| self.core().set_enable_storage(value));
|
|
}
|
|
|
|
/// Whether the storage tier (L3) is wired.
|
|
fn enable_storage(&self, py: Python<'_>) -> bool {
|
|
py.detach(|| self.core().enable_storage)
|
|
}
|
|
|
|
/// Enable or disable the direct external-cache linker.
|
|
fn set_enable_external_cache_linker(&self, py: Python<'_>, value: bool) -> PyResult<()> {
|
|
py.detach(|| self.core().set_enable_external_cache_linker(value))
|
|
.map_err(tree_core_assertion_error)
|
|
}
|
|
|
|
/// Whether the direct external-cache linker is wired.
|
|
fn enable_external_cache_linker(&self, py: Python<'_>) -> bool {
|
|
py.detach(|| self.core().enable_external_cache_linker)
|
|
}
|
|
|
|
/// Queue the all-cleared placement event.
|
|
fn record_all_cleared_event(&self, py: Python<'_>) {
|
|
py.detach(|| self.core().record_all_cleared_event());
|
|
}
|
|
|
|
/// Drain the queued placement events as tagged tuples.
|
|
fn take_events(&self, py: Python<'_>) -> PyResult<Py<PyList>>
|
|
where
|
|
for<'py> Vec<K::Atom>: IntoPyObject<'py>,
|
|
{
|
|
let events = py.detach(|| self.core().take_events());
|
|
let list = PyList::empty(py);
|
|
for event in events {
|
|
match event {
|
|
KvCacheEvent::BlockStored {
|
|
block_hashes,
|
|
parent_block_hash,
|
|
token_ids,
|
|
block_size,
|
|
medium,
|
|
cache_salt,
|
|
session_id,
|
|
extra_key,
|
|
} => {
|
|
let item: Py<PyAny> = (
|
|
"block_stored",
|
|
block_hashes,
|
|
parent_block_hash,
|
|
token_ids,
|
|
block_size,
|
|
medium.as_str(),
|
|
cache_salt.map(|salt| salt.to_string()),
|
|
session_id.map(|session_id| session_id.to_string()),
|
|
extra_key.map(|extra_key| extra_key.to_string()),
|
|
)
|
|
.into_py_any(py)?;
|
|
list.append(item)?;
|
|
}
|
|
KvCacheEvent::BlockRemoved {
|
|
block_hashes,
|
|
medium,
|
|
extra_key,
|
|
} => {
|
|
let item: Py<PyAny> = (
|
|
"block_removed",
|
|
block_hashes,
|
|
medium.as_str(),
|
|
extra_key.map(|extra_key| extra_key.to_string()),
|
|
)
|
|
.into_py_any(py)?;
|
|
list.append(item)?;
|
|
}
|
|
KvCacheEvent::AllBlocksCleared => {
|
|
let item: Py<PyAny> = ("all_blocks_cleared",).into_py_any(py)?;
|
|
list.append(item)?;
|
|
}
|
|
}
|
|
}
|
|
Ok(list.unbind())
|
|
}
|
|
|
|
/// Drop the subtree rooted at an unbacked D-leaf; not dropped when a lock
|
|
/// blocks it.
|
|
fn drop_subtree_no_host(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
) -> PyResult<DropSubtreeResultBinding> {
|
|
let (dropped, result) = py
|
|
.detach(move || self.core().drop_subtree_no_host(node_id))
|
|
.map_err(node_access_error)?;
|
|
Ok(DropSubtreeResultBinding {
|
|
dropped,
|
|
tracker: tracker_to_py(result.tracker),
|
|
new_device_frees: frees_to_py(py, result.device_frees)?,
|
|
new_host_frees: frees_to_py(py, result.host_frees)?,
|
|
})
|
|
}
|
|
|
|
/// Mark the nodes one write-through backup covers; returns them ancestors first.
|
|
fn mark_write_through_pending(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_ids: Vec<NodeId>,
|
|
ack_id: NodeId,
|
|
) -> PyResult<Vec<NodeId>> {
|
|
py.detach(move || self.core().mark_write_through_pending(node_ids, ack_id))
|
|
.map_err(node_access_error)
|
|
}
|
|
|
|
/// Clear the write-through-pending mark on the acked nodes.
|
|
fn finish_write_through(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_ids: Vec<NodeId>,
|
|
ack_id: NodeId,
|
|
) -> PyResult<()> {
|
|
py.detach(move || self.core().finish_write_through(node_ids, ack_id))
|
|
.map_err(node_access_error)
|
|
}
|
|
|
|
/// Clear the in-flight H->D marks on the anchor's root path at ack time.
|
|
fn finish_load_back(&self, py: Python<'_>, anchor_node_id: NodeId) -> PyResult<()> {
|
|
py.detach(|| self.core().finish_load_back(anchor_node_id))
|
|
.map_err(node_access_error)
|
|
}
|
|
|
|
/// Build transfers for a node with no stored or pending external copy.
|
|
fn build_external_linker_offload_transfers(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
) -> PyResult<Option<Vec<Py<PyAny>>>> {
|
|
let transfers = py
|
|
.detach(|| self.core().build_external_linker_offload_transfers(node_id))
|
|
.map_err(node_access_error)?;
|
|
transfers
|
|
.map(|transfers| {
|
|
transfers
|
|
.into_iter()
|
|
.map(|transfer| transfer_to_py(py, transfer))
|
|
.collect::<PyResult<Vec<_>>>()
|
|
})
|
|
.transpose()
|
|
}
|
|
|
|
/// Mark an externally restored path, excluding its existing anchor.
|
|
fn mark_external_cache_stored_path(
|
|
&self,
|
|
py: Python<'_>,
|
|
from_node_id: NodeId,
|
|
until_node_id: NodeId,
|
|
) -> PyResult<()> {
|
|
py.detach(|| {
|
|
self.core()
|
|
.mark_external_cache_stored_path(from_node_id, until_node_id)
|
|
})
|
|
.map_err(tree_core_runtime_error)
|
|
}
|
|
|
|
/// Publish an accepted external offload as pending.
|
|
fn mark_external_linker_offload_pending(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
) -> PyResult<()> {
|
|
py.detach(|| self.core().mark_external_linker_offload_pending(node_id))
|
|
.map_err(tree_core_assertion_error)
|
|
}
|
|
|
|
/// Finalize external-store state for an offload and its split fragments.
|
|
fn finish_external_linker_offload(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_ids: Vec<NodeId>,
|
|
ack_id: NodeId,
|
|
success: bool,
|
|
) -> PyResult<()> {
|
|
py.detach(|| {
|
|
self.core()
|
|
.finish_external_linker_offload(&node_ids, ack_id, success)
|
|
})
|
|
.map_err(tree_core_assertion_error)
|
|
}
|
|
|
|
/// Order-sensitive digest of reclaimed coexisting host values.
|
|
fn write_back_coexist_reclaim_digest(&self, py: Python<'_>) -> i64 {
|
|
py.detach(|| self.core().write_back_coexist_reclaim_digest)
|
|
}
|
|
|
|
/// Whether the component's data is device-evicted but host-backed.
|
|
fn component_has_host_value_only(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
component_type: u8,
|
|
) -> PyResult<bool> {
|
|
let ct = parse_component_type(component_type)?;
|
|
py.detach(|| self.core().component_has_host_value_only(node_id, ct))
|
|
.map_err(node_access_error)
|
|
}
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
impl<K: ChildKeyType + Send + Sync> TreeCoreBinding<K> {
|
|
fn inspect_contains_node(&self, py: Python<'_>, node_id: NodeId) -> bool {
|
|
py.detach(|| self.core().inspect_contains_node(node_id))
|
|
}
|
|
|
|
fn inspect_get_parent_node_id(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
) -> PyResult<Option<NodeId>> {
|
|
py.detach(|| self.core().inspect_get_parent_node_id(node_id))
|
|
.map_err(node_access_error)
|
|
}
|
|
|
|
fn inspect_get_child_node_ids(&self, py: Python<'_>, node_id: NodeId) -> PyResult<Vec<NodeId>> {
|
|
py.detach(|| self.core().inspect_get_child_node_ids(node_id))
|
|
.map_err(node_access_error)
|
|
}
|
|
|
|
fn inspect_get_node_key_length(&self, py: Python<'_>, node_id: NodeId) -> PyResult<usize> {
|
|
py.detach(|| self.core().inspect_get_node_key_length(node_id))
|
|
.map_err(node_access_error)
|
|
}
|
|
|
|
fn inspect_get_node_token_ids(&self, py: Python<'_>, node_id: NodeId) -> PyResult<Vec<i64>> {
|
|
py.detach(|| self.core().inspect_get_node_token_ids(node_id))
|
|
.map_err(node_access_error)
|
|
}
|
|
|
|
fn inspect_is_node_key_bigram(&self, py: Python<'_>, node_id: NodeId) -> PyResult<bool> {
|
|
py.detach(|| self.core().inspect_is_node_key_bigram(node_id))
|
|
.map_err(node_access_error)
|
|
}
|
|
|
|
fn inspect_get_component_host_value(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
component_type: u8,
|
|
) -> PyResult<Option<Py<PyAny>>> {
|
|
let component_type = parse_component_type(component_type)?;
|
|
let value = py
|
|
.detach(|| {
|
|
self.core()
|
|
.inspect_get_component_host_value(node_id, component_type)
|
|
})
|
|
.map_err(node_access_error)?;
|
|
value.map(|value| tensor_to_py(py, value)).transpose()
|
|
}
|
|
|
|
fn inspect_get_component_device_lock_ref(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
component_type: u8,
|
|
) -> PyResult<u32> {
|
|
let component_type = parse_component_type(component_type)?;
|
|
py.detach(|| {
|
|
self.core()
|
|
.inspect_get_component_device_lock_ref(node_id, component_type)
|
|
})
|
|
.map_err(node_access_error)
|
|
}
|
|
|
|
fn inspect_get_component_host_lock_ref(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
component_type: u8,
|
|
) -> PyResult<u32> {
|
|
let component_type = parse_component_type(component_type)?;
|
|
py.detach(|| {
|
|
self.core()
|
|
.inspect_get_component_host_lock_ref(node_id, component_type)
|
|
})
|
|
.map_err(node_access_error)
|
|
}
|
|
|
|
fn inspect_get_node_hit_count(&self, py: Python<'_>, node_id: NodeId) -> PyResult<i64> {
|
|
py.detach(|| self.core().inspect_get_node_hit_count(node_id))
|
|
.map_err(node_access_error)
|
|
}
|
|
|
|
fn inspect_get_write_through_pending_id(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
) -> PyResult<Option<usize>> {
|
|
py.detach(|| self.core().inspect_get_write_through_pending_id(node_id))
|
|
.map_err(node_access_error)
|
|
}
|
|
|
|
fn inspect_is_external_cache_stored(&self, py: Python<'_>, node_id: NodeId) -> PyResult<bool> {
|
|
py.detach(|| self.core().inspect_is_external_cache_stored(node_id))
|
|
.map_err(node_access_error)
|
|
}
|
|
|
|
fn inspect_is_node_in_device_lru(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
component_type: u8,
|
|
) -> PyResult<bool> {
|
|
let component_type = parse_component_type(component_type)?;
|
|
py.detach(|| {
|
|
self.core()
|
|
.inspect_is_node_in_device_lru(node_id, component_type)
|
|
})
|
|
.map_err(node_access_error)
|
|
}
|
|
|
|
fn inspect_is_node_in_host_lru(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
component_type: u8,
|
|
) -> PyResult<bool> {
|
|
let component_type = parse_component_type(component_type)?;
|
|
py.detach(|| {
|
|
self.core()
|
|
.inspect_is_node_in_host_lru(node_id, component_type)
|
|
})
|
|
.map_err(node_access_error)
|
|
}
|
|
|
|
fn inspect_get_component_device_lru_node_ids(
|
|
&self,
|
|
py: Python<'_>,
|
|
component_type: u8,
|
|
) -> PyResult<Vec<NodeId>> {
|
|
let component_type = parse_component_type(component_type)?;
|
|
Ok(py.detach(|| {
|
|
self.core()
|
|
.inspect_get_component_device_lru_node_ids(component_type)
|
|
}))
|
|
}
|
|
|
|
fn inspect_is_device_evictable_leaf(&self, py: Python<'_>, node_id: NodeId) -> bool {
|
|
py.detach(|| self.core().inspect_is_device_evictable_leaf(node_id))
|
|
}
|
|
|
|
fn inspect_is_host_evictable_leaf(&self, py: Python<'_>, node_id: NodeId) -> bool {
|
|
py.detach(|| self.core().inspect_is_host_evictable_leaf(node_id))
|
|
}
|
|
|
|
fn inspect_is_device_leaf(&self, py: Python<'_>, node_id: NodeId) -> PyResult<bool> {
|
|
py.detach(|| self.core().inspect_is_device_leaf(node_id))
|
|
.map_err(node_access_error)
|
|
}
|
|
|
|
fn inspect_get_all_node_ids(&self, py: Python<'_>) -> Vec<NodeId> {
|
|
py.detach(|| self.core().inspect_get_all_node_ids())
|
|
}
|
|
|
|
fn inspect_component_protected_size(
|
|
&self,
|
|
py: Python<'_>,
|
|
component_type: u8,
|
|
) -> PyResult<usize> {
|
|
let component_type = parse_component_type(component_type)?;
|
|
Ok(py.detach(|| self.core().inspect_component_protected_size(component_type)))
|
|
}
|
|
|
|
fn inspect_set_node_hash_values(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
hash_values: Option<Vec<String>>,
|
|
) -> PyResult<()> {
|
|
py.detach(move || {
|
|
self.core()
|
|
.inspect_set_node_hash_values(node_id, hash_values)
|
|
})
|
|
.map_err(node_access_error)
|
|
}
|
|
|
|
fn inspect_set_component_device_value_raw(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
component_type: u8,
|
|
value: Option<PyTensor>,
|
|
) -> PyResult<()> {
|
|
let component_type = parse_component_type(component_type)?;
|
|
let value = value.map(|value| value.0);
|
|
py.detach(move || {
|
|
self.core()
|
|
.inspect_set_component_device_value_raw(node_id, component_type, value)
|
|
})
|
|
.map_err(node_access_error)
|
|
}
|
|
|
|
fn inspect_set_component_host_value_raw(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
component_type: u8,
|
|
value: Option<PyTensor>,
|
|
) -> PyResult<()> {
|
|
let component_type = parse_component_type(component_type)?;
|
|
let value = value.map(|value| value.0);
|
|
py.detach(move || {
|
|
self.core()
|
|
.inspect_set_component_host_value_raw(node_id, component_type, value)
|
|
})
|
|
.map_err(node_access_error)
|
|
}
|
|
|
|
fn inspect_set_component_device_lock_ref(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
component_type: u8,
|
|
lock_ref: u32,
|
|
) -> PyResult<()> {
|
|
let component_type = parse_component_type(component_type)?;
|
|
py.detach(|| {
|
|
self.core()
|
|
.inspect_set_component_device_lock_ref(node_id, component_type, lock_ref)
|
|
})
|
|
.map_err(node_access_error)
|
|
}
|
|
|
|
fn inspect_remove_node_from_device_lru(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
component_type: u8,
|
|
) -> PyResult<()> {
|
|
let component_type = parse_component_type(component_type)?;
|
|
py.detach(|| {
|
|
self.core()
|
|
.inspect_remove_node_from_device_lru(node_id, component_type)
|
|
})
|
|
.map_err(node_access_error)
|
|
}
|
|
|
|
fn inspect_insert_node_into_host_lru(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
component_type: u8,
|
|
) -> PyResult<()> {
|
|
let component_type = parse_component_type(component_type)?;
|
|
py.detach(|| {
|
|
self.core()
|
|
.inspect_insert_node_into_host_lru(node_id, component_type)
|
|
})
|
|
.map_err(node_access_error)
|
|
}
|
|
|
|
fn inspect_set_component_evictable_size(
|
|
&self,
|
|
py: Python<'_>,
|
|
component_type: u8,
|
|
value: usize,
|
|
) -> PyResult<()> {
|
|
let component_type = parse_component_type(component_type)?;
|
|
py.detach(|| {
|
|
self.core()
|
|
.inspect_set_component_evictable_size(component_type, value)
|
|
});
|
|
Ok(())
|
|
}
|
|
|
|
fn inspect_set_component_protected_size(
|
|
&self,
|
|
py: Python<'_>,
|
|
component_type: u8,
|
|
value: usize,
|
|
) -> PyResult<()> {
|
|
let component_type = parse_component_type(component_type)?;
|
|
py.detach(|| {
|
|
self.core()
|
|
.inspect_set_component_protected_size(component_type, value)
|
|
});
|
|
Ok(())
|
|
}
|
|
|
|
fn inspect_update_duplicate_tracking(&self, py: Python<'_>, node_id: NodeId) -> PyResult<()> {
|
|
py.detach(|| self.core().inspect_update_duplicate_tracking(node_id))
|
|
.map_err(node_access_error)
|
|
}
|
|
|
|
fn inspect_advance_insert_walk_once(&self, py: Python<'_>) -> PyResult<()> {
|
|
py.detach(|| self.core().inspect_advance_insert_walk_once())
|
|
.map_err(PyRuntimeError::new_err)
|
|
}
|
|
|
|
fn inspect_evict_component(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
component_type: u8,
|
|
target: u8,
|
|
) -> PyResult<HostEvictionResultBinding> {
|
|
let component_type = parse_component_type(component_type)?;
|
|
let target = parse_evict_layer(target)?;
|
|
let result = py
|
|
.detach(|| {
|
|
self.core()
|
|
.inspect_evict_component(node_id, component_type, target)
|
|
})
|
|
.map_err(node_access_error)?;
|
|
HostEvictionResultBinding::from_eviction_step(py, result)
|
|
}
|
|
|
|
fn inspect_validate_cascade_evict(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
component_type: u8,
|
|
target: u8,
|
|
) -> PyResult<()> {
|
|
let component_type = parse_component_type(component_type)?;
|
|
let target = parse_evict_layer(target)?;
|
|
py.detach(|| {
|
|
self.core()
|
|
.inspect_validate_cascade_evict(node_id, component_type, target)
|
|
})
|
|
.map_err(tree_core_assertion_error)
|
|
}
|
|
|
|
fn inspect_cleanup_tombstone_ancestors(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
) -> PyResult<HostEvictionResultBinding> {
|
|
let result = py
|
|
.detach(|| self.core().inspect_cleanup_tombstone_ancestors(node_id))
|
|
.map_err(node_access_error)?;
|
|
HostEvictionResultBinding::from_eviction_step(py, result)
|
|
}
|
|
|
|
#[allow(clippy::too_many_arguments)]
|
|
fn inspect_finalize_component_match_result(
|
|
&self,
|
|
py: Python<'_>,
|
|
component_type: u8,
|
|
result: InspectionMatchResultInput,
|
|
key: &Bound<'_, PyAny>,
|
|
extra_key: Option<String>,
|
|
cache_salt: Option<String>,
|
|
value_chunks: Vec<PyTensor>,
|
|
best_value_len: usize,
|
|
) -> PyResult<MatchResultBinding> {
|
|
let component_type = parse_component_type(component_type)?;
|
|
let key = K::key_from(Cow::Owned(py_array_to_vec_i64(py, key)?)).into_owned();
|
|
let InspectionMatchResultInput {
|
|
device_prefix_len,
|
|
last_device_node: last_device_node_id,
|
|
last_host_node: last_host_node_id,
|
|
best_match_node: best_match_node_id,
|
|
host_hit_length,
|
|
swa_host_hit_length,
|
|
swa_branching_seqlen,
|
|
mamba_host_hit_length,
|
|
mamba_branching_seqlen,
|
|
full_kv_hit_length,
|
|
} = result;
|
|
let result = MatchResult {
|
|
device_prefix_len,
|
|
last_device_node_id,
|
|
last_host_node_id,
|
|
best_match_node_id,
|
|
host_hit_length,
|
|
swa_host_hit_length,
|
|
swa_branching_seqlen,
|
|
mamba_host_hit_length,
|
|
mamba_branching_seqlen,
|
|
full_kv_hit_length,
|
|
cache_actions: Vec::new(),
|
|
};
|
|
let value_chunks = value_chunks
|
|
.into_iter()
|
|
.map(|value| value.0)
|
|
.collect::<Vec<_>>();
|
|
let result = py.detach(move || {
|
|
let params = MatchPrefixParams {
|
|
key: &key,
|
|
namespace: KeyNamespaceRef::new(extra_key.as_deref(), cache_salt.as_deref()),
|
|
};
|
|
self.core().inspect_finalize_component_match_result(
|
|
component_type,
|
|
result,
|
|
¶ms,
|
|
&value_chunks,
|
|
best_value_len,
|
|
)
|
|
});
|
|
let result = result.map_err(node_access_error)?;
|
|
MatchResultBinding::from_match_result(py, result)
|
|
}
|
|
|
|
fn inspect_build_backup_node_ids(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
write_back: bool,
|
|
) -> PyResult<Vec<NodeId>> {
|
|
py.detach(|| {
|
|
self.core()
|
|
.inspect_build_backup_node_ids(node_id, write_back)
|
|
})
|
|
.map_err(node_access_error)
|
|
}
|
|
}
|
|
|
|
impl<K: ChildKeyType + Send + Sync> TreeCoreBinding<K> {
|
|
/// Print the tree structure for debugging.
|
|
fn pretty_print(&self, py: Python<'_>) {
|
|
py.detach(|| self.core().pretty_print());
|
|
}
|
|
}
|
|
|
|
// The delegate surface is identical for every key type; the macro stamps the
|
|
// pyclass + pymethods pair per concrete key.
|
|
macro_rules! tree_core_binding {
|
|
($(#[$doc:meta])* $name:ident, $key:ty) => {
|
|
$(#[$doc])*
|
|
#[pyclass]
|
|
pub struct $name {
|
|
inner: TreeCoreBinding<$key>,
|
|
}
|
|
|
|
#[pymethods]
|
|
impl $name {
|
|
/// Build a tree core for the given component types from the cache's
|
|
/// init params.
|
|
#[new]
|
|
fn new(init_params: &TreeCoreInitParamsBinding, component_types: Vec<u8>) -> PyResult<Self> {
|
|
catch_native_panic(|| {
|
|
Ok($name {
|
|
inner: TreeCoreBinding::new(init_params, component_types)?,
|
|
})
|
|
})
|
|
}
|
|
|
|
/// Drop the entire tree and reinitialize empty state.
|
|
fn reset(&self, py: Python<'_>) -> PyResult<()> {
|
|
catch_native_panic(|| {
|
|
self.inner.reset(py);
|
|
Ok(())
|
|
})
|
|
}
|
|
|
|
/// Match a key against the tree.
|
|
fn match_prefix(
|
|
&self,
|
|
py: Python<'_>,
|
|
params: &MatchParamsBinding,
|
|
) -> PyResult<MatchResultBinding> {
|
|
catch_native_panic(|| self.inner.match_prefix(py, params))
|
|
}
|
|
|
|
/// Read-only FULL-device match, independent of auxiliary components.
|
|
fn match_full_device_prefix(
|
|
&self,
|
|
py: Python<'_>,
|
|
params: &MatchParamsBinding,
|
|
) -> PyResult<(usize, NodeId, usize)> {
|
|
catch_native_panic(|| Ok(self.inner.match_full_device_prefix(py, params)))
|
|
}
|
|
|
|
fn swa_tombstone_ranges(
|
|
&self,
|
|
py: Python<'_>,
|
|
params: &MatchParamsBinding,
|
|
start: usize,
|
|
end: usize,
|
|
) -> PyResult<Vec<(usize, usize)>> {
|
|
catch_native_panic(|| self.inner.swa_tombstone_ranges(py, params, start, end))
|
|
}
|
|
|
|
fn attach_swa_window(
|
|
&self,
|
|
py: Python<'_>,
|
|
params: &MatchParamsBinding,
|
|
window_start: usize,
|
|
window_end: usize,
|
|
swa_values: PyTensor,
|
|
) -> PyResult<Py<PyList>> {
|
|
catch_native_panic(|| {
|
|
self.inner
|
|
.attach_swa_window(py, params, window_start, window_end, swa_values)
|
|
})
|
|
}
|
|
|
|
/// The empty match result anchored at the root.
|
|
fn empty_match_result(&self, py: Python<'_>) -> PyResult<MatchResultBinding> {
|
|
catch_native_panic(|| self.inner.empty_match_result(py))
|
|
}
|
|
|
|
/// Insert device values into the tree per the provided key.
|
|
fn insert(
|
|
&self,
|
|
py: Python<'_>,
|
|
params: &InsertParamsBinding,
|
|
) -> PyResult<InsertResultBinding> {
|
|
catch_native_panic(|| self.inner.insert(py, params))
|
|
}
|
|
|
|
/// Start the resumable insert, running to its first barrier or completion.
|
|
fn begin_insert(
|
|
&self,
|
|
py: Python<'_>,
|
|
params: &InsertParamsBinding,
|
|
) -> PyResult<InsertStepResultBinding> {
|
|
catch_native_panic(|| self.inner.begin_insert(py, params))
|
|
}
|
|
|
|
/// Continue the suspended insert after its step actions were applied.
|
|
fn resume_insert(&self, py: Python<'_>) -> PyResult<InsertStepResultBinding> {
|
|
catch_native_panic(|| self.inner.resume_insert(py))
|
|
}
|
|
|
|
/// Whether an insert walk is suspended at a barrier.
|
|
fn has_ongoing_insert(&self, py: Python<'_>) -> PyResult<bool> {
|
|
catch_native_panic(|| Ok(self.inner.has_ongoing_insert(py)))
|
|
}
|
|
|
|
/// Finish the insert (idempotent); returns still-pending actions to drain.
|
|
fn end_insert(&self, py: Python<'_>) -> PyResult<Py<PyList>> {
|
|
catch_native_panic(|| self.inner.end_insert(py))
|
|
}
|
|
|
|
/// Bump the reference count on a node's component locks.
|
|
#[pyo3(signature = (node_id, skip_lock_components = Vec::new()))]
|
|
fn inc_lock_ref(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
skip_lock_components: Vec<u8>,
|
|
) -> PyResult<IncLockRefResultBinding> {
|
|
catch_native_panic(|| self.inner.inc_lock_ref(py, node_id, skip_lock_components))
|
|
}
|
|
|
|
/// Pin only the FULL device values on a node's root path.
|
|
fn inc_full_pin(&self, py: Python<'_>, node_id: NodeId) -> PyResult<()> {
|
|
catch_native_panic(|| self.inner.inc_full_pin(py, node_id))
|
|
}
|
|
|
|
/// Release a FULL-only root-path pin.
|
|
fn dec_full_pin(&self, py: Python<'_>, node_id: NodeId) -> PyResult<()> {
|
|
catch_native_panic(|| self.inner.dec_full_pin(py, node_id))
|
|
}
|
|
|
|
/// Decrease the reference count on a node's component locks. The
|
|
/// receipt is required: a release must replay its acquire's evidence.
|
|
#[pyo3(signature = (node_id, params, skip_swa = false))]
|
|
fn dec_lock_ref(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
params: &DecLockRefParamsBinding,
|
|
skip_swa: bool,
|
|
) -> PyResult<()> {
|
|
catch_native_panic(|| self.inner.dec_lock_ref(py, node_id, params, skip_swa))
|
|
}
|
|
|
|
/// Release one window's lock, retaining other component locks.
|
|
fn dec_window_lock_only(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
component_type: u8,
|
|
params: &DecLockRefParamsBinding,
|
|
) -> PyResult<(Py<PyDict>, Py<PyDict>)> {
|
|
catch_native_panic(|| {
|
|
self.inner
|
|
.dec_window_lock_only(py, node_id, component_type, params)
|
|
})
|
|
}
|
|
|
|
/// Early-release the SWA portion of a request's tree lock; returns this
|
|
/// release's per-component (device_frees, host_frees).
|
|
#[pyo3(signature = (node_id, params))]
|
|
fn dec_swa_lock_only(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
params: &DecLockRefParamsBinding,
|
|
) -> PyResult<(Py<PyDict>, Py<PyDict>)> {
|
|
catch_native_panic(|| self.inner.dec_swa_lock_only(py, node_id, params))
|
|
}
|
|
|
|
/// Adjust the component's reclaimable device size.
|
|
fn adjust_component_size(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
component_type: u8,
|
|
delta: i64,
|
|
) -> PyResult<()> {
|
|
catch_native_panic(|| {
|
|
self.inner.adjust_component_size(py, node_id, component_type, delta)
|
|
})
|
|
}
|
|
|
|
/// Collect paths under one tree lock.
|
|
fn collect_path_info(
|
|
&self,
|
|
py: Python<'_>,
|
|
end_node_ids: Vec<NodeId>,
|
|
) -> PyResult<Vec<Vec<TreePathNodeInfoTuple>>> {
|
|
catch_native_panic(|| self.inner.collect_path_info(py, end_node_ids))
|
|
}
|
|
|
|
/// Store a component's device value on a node (the SWA rebuild write-back).
|
|
fn set_component_device_value(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
component_type: u8,
|
|
value: PyTensor,
|
|
) -> PyResult<()> {
|
|
catch_native_panic(|| {
|
|
self.inner
|
|
.set_component_device_value(py, node_id, component_type, value)
|
|
})
|
|
}
|
|
|
|
/// Logical key lengths for the nodes in a load-back transfer.
|
|
fn get_node_key_lengths(&self, py: Python<'_>, node_ids: Vec<NodeId>) -> PyResult<Vec<usize>> {
|
|
catch_native_panic(|| self.inner.get_node_key_lengths(py, node_ids))
|
|
}
|
|
|
|
/// A component's device value on a node, if set.
|
|
fn get_component_device_value(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
component_type: u8,
|
|
) -> PyResult<Option<Py<PyAny>>> {
|
|
catch_native_panic(|| {
|
|
self.inner
|
|
.get_component_device_value(py, node_id, component_type)
|
|
})
|
|
}
|
|
|
|
/// Begin a component's device-eviction walk for up to request_cnt tokens.
|
|
fn evict_device_start(
|
|
&self,
|
|
py: Python<'_>,
|
|
component_type: u8,
|
|
request_cnt: usize,
|
|
) -> PyResult<()> {
|
|
catch_native_panic(|| {
|
|
self.inner
|
|
.evict_device_start(py, component_type, request_cnt)
|
|
})
|
|
}
|
|
|
|
/// The next device leaf to evict, or None when the walk is done; the
|
|
/// passed running tracker gates the budget, and the result carries
|
|
/// this step's deltas.
|
|
fn evict_device_next_node(
|
|
&self,
|
|
py: Python<'_>,
|
|
component_type: u8,
|
|
tracker: HashMap<u8, usize>,
|
|
) -> PyResult<EvictDeviceNextNodeResultBinding> {
|
|
catch_native_panic(|| {
|
|
self.inner
|
|
.evict_device_next_node(py, component_type, tracker)
|
|
})
|
|
}
|
|
|
|
/// Finish an internal Mamba eviction after its host backup attempt.
|
|
fn finish_mamba_state_eviction(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
) -> PyResult<EvictDeviceNextNodeResultBinding> {
|
|
catch_native_panic(|| self.inner.finish_mamba_state_eviction(py, node_id))
|
|
}
|
|
|
|
/// Finish an internal SWA eviction after its host backup attempt.
|
|
fn finish_swa_state_eviction(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
) -> PyResult<EvictDeviceNextNodeResultBinding> {
|
|
catch_native_panic(|| self.inner.finish_swa_state_eviction(py, node_id))
|
|
}
|
|
|
|
/// Evict one device leaf; an unbacked write-back leaf returns its backup
|
|
/// action for the caller to execute before demoting.
|
|
fn evict_device_leaf(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
) -> PyResult<EvictDeviceLeafResultBinding> {
|
|
catch_native_panic(|| self.inner.evict_device_leaf(py, node_id))
|
|
}
|
|
|
|
/// Finish a component's device-eviction walk.
|
|
fn evict_device_end(&self, py: Python<'_>, component_type: u8) -> PyResult<()> {
|
|
catch_native_panic(|| self.inner.evict_device_end(py, component_type))
|
|
}
|
|
|
|
/// Verify tree-structure, leaf-set, LRU, size, and ongoing-op invariants;
|
|
/// ongoing_* args are (id, node_id) pairs.
|
|
fn sanity_check(
|
|
&self,
|
|
py: Python<'_>,
|
|
ongoing_write_through: Vec<(i64, NodeId)>,
|
|
ongoing_load_back: Vec<(i64, NodeId)>,
|
|
) -> PyResult<()> {
|
|
catch_native_panic(|| {
|
|
self.inner
|
|
.sanity_check(py, ongoing_write_through, ongoing_load_back)
|
|
})
|
|
}
|
|
|
|
/// Concatenated FULL device values from from_node up to (exclusive) until_node.
|
|
fn collect_full_device_indices(
|
|
&self,
|
|
py: Python<'_>,
|
|
from_node_id: NodeId,
|
|
until_node_id: NodeId,
|
|
) -> PyResult<Py<PyAny>> {
|
|
catch_native_panic(|| {
|
|
self.inner
|
|
.collect_full_device_indices(py, from_node_id, until_node_id)
|
|
})
|
|
}
|
|
|
|
/// Every FULL device value in the tree, concatenated.
|
|
fn all_values_flatten(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
|
|
catch_native_panic(|| self.inner.all_values_flatten(py))
|
|
}
|
|
|
|
/// Every Mamba device value in the tree, concatenated.
|
|
fn all_mamba_values_flatten(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
|
|
catch_native_panic(|| self.inner.all_mamba_values_flatten(py))
|
|
}
|
|
|
|
/// Flatten every FULL device slot into (slot, position, prev-slot) rows for the KV-canary sweep.
|
|
fn walk_for_kv_canary(
|
|
&self,
|
|
py: Python<'_>,
|
|
unlocked_only: bool,
|
|
swa_resident_only: bool,
|
|
) -> PyResult<KvCanaryWalkResultBinding> {
|
|
catch_native_panic(|| {
|
|
self.inner
|
|
.walk_for_kv_canary(py, unlocked_only, swa_resident_only)
|
|
})
|
|
}
|
|
|
|
/// Evictable token count of the FULL (base) component.
|
|
fn evictable_size(&self, py: Python<'_>) -> PyResult<usize> {
|
|
catch_native_panic(|| Ok(self.inner.evictable_size(py)))
|
|
}
|
|
|
|
/// Protected (locked) token count of the FULL (base) component.
|
|
fn protected_size(&self, py: Python<'_>) -> PyResult<usize> {
|
|
catch_native_panic(|| Ok(self.inner.protected_size(py)))
|
|
}
|
|
|
|
/// FULL component evictable token count.
|
|
fn full_evictable_size(&self, py: Python<'_>) -> PyResult<usize> {
|
|
catch_native_panic(|| Ok(self.inner.full_evictable_size(py)))
|
|
}
|
|
|
|
/// FULL component protected token count.
|
|
fn full_protected_size(&self, py: Python<'_>) -> PyResult<usize> {
|
|
catch_native_panic(|| Ok(self.inner.full_protected_size(py)))
|
|
}
|
|
|
|
/// Evictable token count for one component (0 if the component is absent).
|
|
fn component_evictable_size(&self, py: Python<'_>, component_type: u8) -> PyResult<usize> {
|
|
catch_native_panic(|| self.inner.component_evictable_size(py, component_type))
|
|
}
|
|
|
|
/// Protected token count for one component (0 if the component is absent).
|
|
fn component_protected_size(&self, py: Python<'_>, component_type: u8) -> PyResult<usize> {
|
|
catch_native_panic(|| self.inner.component_protected_size(py, component_type))
|
|
}
|
|
|
|
/// (full_tokens, aux_tokens) summed across the whole tree.
|
|
fn total_size(&self, py: Python<'_>) -> PyResult<(usize, usize)> {
|
|
catch_native_panic(|| Ok(self.inner.total_size(py)))
|
|
}
|
|
|
|
/// Whether the node's FULL device value has been evicted.
|
|
fn is_full_device_evicted(&self, py: Python<'_>, node_id: NodeId) -> PyResult<bool> {
|
|
catch_native_panic(|| self.inner.is_full_device_evicted(py, node_id))
|
|
}
|
|
|
|
/// Mark the host tier as buffer-only; wired after the host pools are built.
|
|
fn set_host_memory_buffer_only(&self, py: Python<'_>) -> PyResult<()> {
|
|
catch_native_panic(|| {
|
|
self.inner.set_host_memory_buffer_only(py);
|
|
Ok(())
|
|
})
|
|
}
|
|
|
|
/// Whether the host tier runs as a storage staging buffer, not a cache.
|
|
fn is_host_memory_buffer_only(&self, py: Python<'_>) -> PyResult<bool> {
|
|
catch_native_panic(|| Ok(self.inner.is_host_memory_buffer_only(py)))
|
|
}
|
|
|
|
/// Mark the host tier (HiCache) as wired.
|
|
fn set_hicache_enabled(&self, py: Python<'_>) -> PyResult<()> {
|
|
catch_native_panic(|| {
|
|
self.inner.set_hicache_enabled(py);
|
|
Ok(())
|
|
})
|
|
}
|
|
|
|
/// Whether the host tier (HiCache) is wired.
|
|
fn enable_hicache(&self, py: Python<'_>) -> PyResult<bool> {
|
|
catch_native_panic(|| Ok(self.inner.enable_hicache(py)))
|
|
}
|
|
|
|
/// Mark the SWA host pool as wired (HiCache).
|
|
fn set_has_swa_host_pool(&self, py: Python<'_>) -> PyResult<()> {
|
|
catch_native_panic(|| {
|
|
self.inner.set_has_swa_host_pool(py);
|
|
Ok(())
|
|
})
|
|
}
|
|
|
|
/// Whether the SWA host pool is wired.
|
|
fn has_swa_host_pool(&self, py: Python<'_>) -> PyResult<bool> {
|
|
catch_native_panic(|| Ok(self.inner.has_swa_host_pool(py)))
|
|
}
|
|
|
|
/// Insert a host-side (backuped) tree path descending from the given node.
|
|
#[pyo3(signature = (node_id, extra_key, key, host_value, hash_value, cache_salt = None))]
|
|
fn insert_host(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
extra_key: Option<String>,
|
|
key: &Bound<'_, PyAny>,
|
|
host_value: PyTensor,
|
|
hash_value: Vec<String>,
|
|
cache_salt: Option<String>,
|
|
) -> PyResult<InsertResultBinding> {
|
|
catch_native_panic(|| {
|
|
self.inner.insert_host(
|
|
py, node_id, extra_key, key, host_value, hash_value, cache_salt,
|
|
)
|
|
})
|
|
}
|
|
|
|
/// Gather a node's device value plus per-component BACKUP_HOST transfers.
|
|
fn build_backup_spec(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
) -> PyResult<(Py<PyAny>, Py<PyDict>)> {
|
|
catch_native_panic(|| self.inner.build_backup_spec(py, node_id))
|
|
}
|
|
|
|
/// Gather a node's device->storage backup spec; None if the node is not backuped.
|
|
fn build_storage_backup_spec(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
pass_prefix_keys: bool,
|
|
) -> PyResult<Option<StorageBackupSpecBinding>> {
|
|
catch_native_panic(|| {
|
|
self.inner
|
|
.build_storage_backup_spec(py, node_id, pass_prefix_keys)
|
|
})
|
|
}
|
|
|
|
/// Route a build_hicache_transfers call to the component for the given type.
|
|
#[allow(clippy::too_many_arguments)]
|
|
#[pyo3(signature = (component_type, node_id, phase, host_indices = None, token_ids = None, prefetch_tokens = 0, staging_tokens = 0, last_hash = None))]
|
|
fn build_hicache_transfers(
|
|
&self,
|
|
py: Python<'_>,
|
|
component_type: u8,
|
|
node_id: NodeId,
|
|
phase: &str,
|
|
host_indices: Option<PyTensor>,
|
|
token_ids: Option<Vec<i64>>,
|
|
prefetch_tokens: usize,
|
|
staging_tokens: usize,
|
|
last_hash: Option<String>,
|
|
) -> PyResult<Option<Vec<Py<PyAny>>>> {
|
|
catch_native_panic(|| {
|
|
self.inner.build_hicache_transfers(
|
|
py,
|
|
component_type,
|
|
node_id,
|
|
phase,
|
|
host_indices,
|
|
token_ids,
|
|
prefetch_tokens,
|
|
staging_tokens,
|
|
last_hash,
|
|
)
|
|
})
|
|
}
|
|
|
|
/// The anchor node's caller-defined key and cache salt.
|
|
fn prefetch_anchor_info(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
) -> PyResult<(Option<String>, Option<String>)> {
|
|
catch_native_panic(|| self.inner.prefetch_anchor_info(py, node_id))
|
|
}
|
|
|
|
/// Whether the node's Full KV is present on host.
|
|
fn node_backuped(&self, py: Python<'_>, node_id: NodeId) -> PyResult<bool> {
|
|
catch_native_panic(|| self.inner.node_backuped(py, node_id))
|
|
}
|
|
|
|
/// Whether the node is a (default or named) root.
|
|
fn is_root(&self, py: Python<'_>, node_id: NodeId) -> PyResult<bool> {
|
|
catch_native_panic(|| self.inner.is_root(py, node_id))
|
|
}
|
|
|
|
/// The node's last page hash, or None when it was never hashed.
|
|
fn get_last_hash_value(&self, py: Python<'_>, node_id: NodeId) -> PyResult<Option<String>> {
|
|
catch_native_panic(|| self.inner.get_last_hash_value(py, node_id))
|
|
}
|
|
|
|
/// The hash chain of the node's ancestors, in root-to-parent order.
|
|
fn get_prefix_hash_values(&self, py: Python<'_>, node_id: NodeId) -> PyResult<Vec<String>> {
|
|
catch_native_panic(|| self.inner.get_prefix_hash_values(py, node_id))
|
|
}
|
|
|
|
fn get_hash_values(&self, py: Python<'_>, node_id: NodeId) -> PyResult<Vec<String>> {
|
|
catch_native_panic(|| self.inner.get_hash_values(py, node_id))
|
|
}
|
|
|
|
fn snapshot_buffer_backup(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
pass_prefix_keys: bool,
|
|
) -> PyResult<Option<BufferBackupSnapshotBinding>> {
|
|
catch_native_panic(|| {
|
|
Ok(self
|
|
.inner
|
|
.snapshot_buffer_backup(py, node_id, pass_prefix_keys))
|
|
})
|
|
}
|
|
|
|
fn validate_buffer_backup(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
expected_key_length: usize,
|
|
) -> PyResult<Option<BufferBackupStateBinding>> {
|
|
catch_native_panic(|| {
|
|
Ok(self
|
|
.inner
|
|
.validate_buffer_backup(py, node_id, expected_key_length))
|
|
})
|
|
}
|
|
|
|
/// Hash every node built while storage was disabled.
|
|
fn backfill_missing_hash_values(&self, py: Python<'_>) -> PyResult<usize> {
|
|
catch_native_panic(|| Ok(self.inner.backfill_missing_hash_values(py)))
|
|
}
|
|
|
|
#[pyo3(signature = (extra_key = None))]
|
|
fn root_node_handle(&self, py: Python<'_>, extra_key: Option<String>) -> PyResult<NodeId> {
|
|
catch_native_panic(|| Ok(self.inner.root_node_handle(py, extra_key)))
|
|
}
|
|
|
|
fn rotation_base_of(&self, py: Python<'_>, node_id: NodeId) -> PyResult<Option<i64>> {
|
|
catch_native_panic(|| self.inner.rotation_base_of(py, node_id))
|
|
}
|
|
|
|
fn dfs_weight_order(&self, py: Python<'_>, node_ids: Vec<NodeId>) -> PyResult<Vec<usize>> {
|
|
catch_native_panic(|| self.inner.dfs_weight_order(py, node_ids))
|
|
}
|
|
|
|
/// Commit each component's HiCache transfers; returns the new cache actions.
|
|
#[pyo3(signature = (node_id, phase, comp_xfers, insert_result = None, pool_storage_result = None))]
|
|
fn commit_hicache_transfers(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
phase: &str,
|
|
comp_xfers: HashMap<u8, Vec<TransferArgs>>,
|
|
insert_result: Option<(usize, Option<NodeId>, bool)>,
|
|
pool_storage_result: Option<(usize, HashMap<String, usize>)>,
|
|
) -> PyResult<(Py<PyList>, Option<bool>)> {
|
|
catch_native_panic(|| {
|
|
self.inner.commit_hicache_transfers(
|
|
py,
|
|
node_id,
|
|
phase,
|
|
comp_xfers,
|
|
insert_result,
|
|
pool_storage_result,
|
|
)
|
|
})
|
|
}
|
|
|
|
/// Commit a successful backup to the node.
|
|
fn commit_backup(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
host_indices: PyTensor,
|
|
comp_xfers: HashMap<u8, Vec<TransferArgs>>,
|
|
) -> PyResult<()> {
|
|
catch_native_panic(|| {
|
|
self.inner
|
|
.commit_backup(py, node_id, host_indices, comp_xfers)
|
|
})
|
|
}
|
|
|
|
/// Build the H->D load-back KV transfer plus per-component aux transfers.
|
|
#[pyo3(signature = (node_id, mamba_pool_idx = None))]
|
|
fn build_load_back_spec(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
mamba_pool_idx: Option<PyTensor>,
|
|
) -> PyResult<(Py<PyAny>, Py<PyDict>)> {
|
|
catch_native_panic(|| self.inner.build_load_back_spec(py, node_id, mamba_pool_idx))
|
|
}
|
|
|
|
/// Commit a successful H->D load-back onto the node; returns its actions.
|
|
fn commit_load_back(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
device_indices: PyTensor,
|
|
kv_xfer: TransferArgs,
|
|
comp_xfers: HashMap<u8, Vec<TransferArgs>>,
|
|
) -> PyResult<Py<PyList>> {
|
|
catch_native_panic(|| {
|
|
self.inner
|
|
.commit_load_back(py, node_id, device_indices, kv_xfer, comp_xfers)
|
|
})
|
|
}
|
|
|
|
/// Release a node's device KV once its host copy exists.
|
|
fn demote(&self, py: Python<'_>, node_id: NodeId) -> PyResult<DemoteResultBinding> {
|
|
catch_native_panic(|| self.inner.demote(py, node_id))
|
|
}
|
|
|
|
/// Evict up to num_tokens of one component's host resources.
|
|
#[pyo3(signature = (component_type, num_tokens, skip_full_duplicate_reclaim = false))]
|
|
fn drive_host_eviction(
|
|
&self,
|
|
py: Python<'_>,
|
|
component_type: u8,
|
|
num_tokens: usize,
|
|
skip_full_duplicate_reclaim: bool,
|
|
) -> PyResult<HostEvictionResultBinding> {
|
|
catch_native_panic(|| {
|
|
self.inner.drive_host_eviction(
|
|
py,
|
|
component_type,
|
|
num_tokens,
|
|
skip_full_duplicate_reclaim,
|
|
)
|
|
})
|
|
}
|
|
|
|
/// Evict shallow Mamba device checkpoints beyond the per-path cap
|
|
/// on the tail's root path.
|
|
fn evict_excess_path_states(
|
|
&self,
|
|
py: Python<'_>,
|
|
tail_node_id: NodeId,
|
|
) -> PyResult<HostEvictionResultBinding> {
|
|
catch_native_panic(|| self.inner.evict_excess_path_states(py, tail_node_id))
|
|
}
|
|
|
|
/// Bump the reference count on a node's host-side component locks.
|
|
fn inc_host_lock_ref(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
) -> PyResult<IncLockRefResultBinding> {
|
|
catch_native_panic(|| self.inner.inc_host_lock_ref(py, node_id))
|
|
}
|
|
|
|
/// Decrease the reference count on a node's host-side component locks.
|
|
/// The receipt is required, as for dec_lock_ref.
|
|
fn dec_host_lock_ref(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
params: &DecLockRefParamsBinding,
|
|
) -> PyResult<()> {
|
|
catch_native_panic(|| self.inner.dec_host_lock_ref(py, node_id, params))
|
|
}
|
|
|
|
fn is_write_through_compatible(&self, py: Python<'_>) -> bool {
|
|
self.inner.is_write_through_compatible(py)
|
|
}
|
|
|
|
/// Set the write-back (vs write-through) policy; decided at HiCache init.
|
|
fn set_is_write_back(&self, py: Python<'_>, is_write_back: bool) -> PyResult<()> {
|
|
catch_native_panic(|| {
|
|
self.inner.set_is_write_back(py, is_write_back);
|
|
Ok(())
|
|
})
|
|
}
|
|
|
|
/// The current write-back (vs write-through) policy.
|
|
fn is_write_back(&self, py: Python<'_>) -> PyResult<bool> {
|
|
catch_native_panic(|| Ok(self.inner.is_write_back(py)))
|
|
}
|
|
|
|
/// Set the write-through backup hit threshold; decided at HiCache init.
|
|
fn set_write_through_threshold(&self, py: Python<'_>, threshold: i64) -> PyResult<()> {
|
|
catch_native_panic(|| {
|
|
self.inner.set_write_through_threshold(py, threshold);
|
|
Ok(())
|
|
})
|
|
}
|
|
|
|
/// The current write-through backup hit threshold.
|
|
fn write_through_threshold(&self, py: Python<'_>) -> PyResult<i64> {
|
|
catch_native_panic(|| Ok(self.inner.write_through_threshold(py)))
|
|
}
|
|
|
|
/// Mark the storage tier (L3) wired; storage attaches after tree construction.
|
|
fn set_enable_storage(&self, py: Python<'_>, value: bool) -> PyResult<()> {
|
|
catch_native_panic(|| {
|
|
self.inner.set_enable_storage(py, value);
|
|
Ok(())
|
|
})
|
|
}
|
|
|
|
/// Whether the storage tier (L3) is wired.
|
|
fn enable_storage(&self, py: Python<'_>) -> PyResult<bool> {
|
|
catch_native_panic(|| Ok(self.inner.enable_storage(py)))
|
|
}
|
|
|
|
/// Enable or disable the direct external-cache linker.
|
|
fn set_enable_external_cache_linker(&self, py: Python<'_>, value: bool) -> PyResult<()> {
|
|
catch_native_panic(|| self.inner.set_enable_external_cache_linker(py, value))
|
|
}
|
|
|
|
/// Whether the direct external-cache linker is wired.
|
|
fn enable_external_cache_linker(&self, py: Python<'_>) -> PyResult<bool> {
|
|
catch_native_panic(|| Ok(self.inner.enable_external_cache_linker(py)))
|
|
}
|
|
|
|
/// Queue the all-cleared placement event.
|
|
fn record_all_cleared_event(&self, py: Python<'_>) -> PyResult<()> {
|
|
catch_native_panic(|| {
|
|
self.inner.record_all_cleared_event(py);
|
|
Ok(())
|
|
})
|
|
}
|
|
|
|
/// Drain the queued placement events as tagged tuples.
|
|
fn take_events(&self, py: Python<'_>) -> PyResult<Py<PyList>> {
|
|
catch_native_panic(|| self.inner.take_events(py))
|
|
}
|
|
|
|
/// Drop the subtree rooted at an unbacked D-leaf; not dropped when a lock
|
|
/// blocks it.
|
|
fn drop_subtree_no_host(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
) -> PyResult<DropSubtreeResultBinding> {
|
|
catch_native_panic(|| self.inner.drop_subtree_no_host(py, node_id))
|
|
}
|
|
|
|
/// Mark the nodes one write-through backup covers; returns them ancestors first.
|
|
fn mark_write_through_pending(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_ids: Vec<NodeId>,
|
|
ack_id: NodeId,
|
|
) -> PyResult<Vec<NodeId>> {
|
|
catch_native_panic(|| self.inner.mark_write_through_pending(py, node_ids, ack_id))
|
|
}
|
|
|
|
/// Clear the write-through-pending mark on the acked nodes.
|
|
fn finish_write_through(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_ids: Vec<NodeId>,
|
|
ack_id: NodeId,
|
|
) -> PyResult<()> {
|
|
catch_native_panic(|| self.inner.finish_write_through(py, node_ids, ack_id))
|
|
}
|
|
|
|
/// Clear the in-flight H->D marks on the anchor's root path at ack time.
|
|
fn finish_load_back(&self, py: Python<'_>, anchor_node_id: NodeId) -> PyResult<()> {
|
|
catch_native_panic(|| self.inner.finish_load_back(py, anchor_node_id))
|
|
}
|
|
|
|
/// Build transfers for a node with no stored or pending external copy.
|
|
fn build_external_linker_offload_transfers(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
) -> PyResult<Option<Vec<Py<PyAny>>>> {
|
|
catch_native_panic(|| {
|
|
self.inner
|
|
.build_external_linker_offload_transfers(py, node_id)
|
|
})
|
|
}
|
|
|
|
/// Mark an externally restored path, excluding its existing anchor.
|
|
fn mark_external_cache_stored_path(
|
|
&self,
|
|
py: Python<'_>,
|
|
from_node_id: NodeId,
|
|
until_node_id: NodeId,
|
|
) -> PyResult<()> {
|
|
catch_native_panic(|| {
|
|
self.inner
|
|
.mark_external_cache_stored_path(py, from_node_id, until_node_id)
|
|
})
|
|
}
|
|
|
|
/// Publish an accepted external offload as pending.
|
|
fn mark_external_linker_offload_pending(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
) -> PyResult<()> {
|
|
catch_native_panic(|| self.inner.mark_external_linker_offload_pending(py, node_id))
|
|
}
|
|
|
|
/// Finalize external-store state for an offload and its split fragments.
|
|
fn finish_external_linker_offload(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_ids: Vec<NodeId>,
|
|
ack_id: NodeId,
|
|
success: bool,
|
|
) -> PyResult<()> {
|
|
catch_native_panic(|| {
|
|
self.inner
|
|
.finish_external_linker_offload(py, node_ids, ack_id, success)
|
|
})
|
|
}
|
|
|
|
/// Order-sensitive digest of reclaimed coexisting host values.
|
|
#[pyo3(name = "write_back_duplicate_reclaim_digest")]
|
|
fn write_back_coexist_reclaim_digest(&self, py: Python<'_>) -> PyResult<i64> {
|
|
catch_native_panic(|| Ok(self.inner.write_back_coexist_reclaim_digest(py)))
|
|
}
|
|
|
|
/// Whether the component's data is device-evicted but host-backed.
|
|
fn component_has_host_value_only(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
component_type: u8,
|
|
) -> PyResult<bool> {
|
|
catch_native_panic(|| {
|
|
self.inner
|
|
.component_has_host_value_only(py, node_id, component_type)
|
|
})
|
|
}
|
|
|
|
// ==== Test-only inspection surface ====
|
|
|
|
#[cfg(feature = "inspection")]
|
|
fn inspect_contains_node(&self, py: Python<'_>, node_id: NodeId) -> PyResult<bool> {
|
|
catch_native_panic(|| Ok(self.inner.inspect_contains_node(py, node_id)))
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
fn inspect_get_parent_node_id(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
) -> PyResult<Option<NodeId>> {
|
|
catch_native_panic(|| self.inner.inspect_get_parent_node_id(py, node_id))
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
fn inspect_get_child_node_ids(&self, py: Python<'_>, node_id: NodeId) -> PyResult<Vec<NodeId>> {
|
|
catch_native_panic(|| self.inner.inspect_get_child_node_ids(py, node_id))
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
fn inspect_get_node_key_length(&self, py: Python<'_>, node_id: NodeId) -> PyResult<usize> {
|
|
catch_native_panic(|| self.inner.inspect_get_node_key_length(py, node_id))
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
fn inspect_get_node_token_ids(&self, py: Python<'_>, node_id: NodeId) -> PyResult<Vec<i64>> {
|
|
catch_native_panic(|| self.inner.inspect_get_node_token_ids(py, node_id))
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
fn inspect_is_node_key_bigram(&self, py: Python<'_>, node_id: NodeId) -> PyResult<bool> {
|
|
catch_native_panic(|| self.inner.inspect_is_node_key_bigram(py, node_id))
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
fn inspect_get_component_host_value(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
component_type: u8,
|
|
) -> PyResult<Option<Py<PyAny>>> {
|
|
catch_native_panic(|| {
|
|
self.inner
|
|
.inspect_get_component_host_value(py, node_id, component_type)
|
|
})
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
fn inspect_get_component_device_lock_ref(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
component_type: u8,
|
|
) -> PyResult<u32> {
|
|
catch_native_panic(|| {
|
|
self.inner
|
|
.inspect_get_component_device_lock_ref(py, node_id, component_type)
|
|
})
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
fn inspect_get_component_host_lock_ref(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
component_type: u8,
|
|
) -> PyResult<u32> {
|
|
catch_native_panic(|| {
|
|
self.inner
|
|
.inspect_get_component_host_lock_ref(py, node_id, component_type)
|
|
})
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
fn inspect_get_node_hit_count(&self, py: Python<'_>, node_id: NodeId) -> PyResult<i64> {
|
|
catch_native_panic(|| self.inner.inspect_get_node_hit_count(py, node_id))
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
fn inspect_get_write_through_pending_id(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
) -> PyResult<Option<usize>> {
|
|
catch_native_panic(|| self.inner.inspect_get_write_through_pending_id(py, node_id))
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
fn inspect_is_external_cache_stored(&self, py: Python<'_>, node_id: NodeId) -> PyResult<bool> {
|
|
catch_native_panic(|| self.inner.inspect_is_external_cache_stored(py, node_id))
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
fn inspect_is_node_in_device_lru(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
component_type: u8,
|
|
) -> PyResult<bool> {
|
|
catch_native_panic(|| {
|
|
self.inner
|
|
.inspect_is_node_in_device_lru(py, node_id, component_type)
|
|
})
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
fn inspect_is_node_in_host_lru(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
component_type: u8,
|
|
) -> PyResult<bool> {
|
|
catch_native_panic(|| {
|
|
self.inner
|
|
.inspect_is_node_in_host_lru(py, node_id, component_type)
|
|
})
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
fn inspect_get_component_device_lru_node_ids(
|
|
&self,
|
|
py: Python<'_>,
|
|
component_type: u8,
|
|
) -> PyResult<Vec<NodeId>> {
|
|
catch_native_panic(|| {
|
|
self.inner
|
|
.inspect_get_component_device_lru_node_ids(py, component_type)
|
|
})
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
fn inspect_is_device_evictable_leaf(&self, py: Python<'_>, node_id: NodeId) -> PyResult<bool> {
|
|
catch_native_panic(|| Ok(self.inner.inspect_is_device_evictable_leaf(py, node_id)))
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
fn inspect_is_host_evictable_leaf(&self, py: Python<'_>, node_id: NodeId) -> PyResult<bool> {
|
|
catch_native_panic(|| Ok(self.inner.inspect_is_host_evictable_leaf(py, node_id)))
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
fn inspect_is_device_leaf(&self, py: Python<'_>, node_id: NodeId) -> PyResult<bool> {
|
|
catch_native_panic(|| self.inner.inspect_is_device_leaf(py, node_id))
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
fn inspect_get_all_node_ids(&self, py: Python<'_>) -> PyResult<Vec<NodeId>> {
|
|
catch_native_panic(|| Ok(self.inner.inspect_get_all_node_ids(py)))
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
fn inspect_component_protected_size(
|
|
&self,
|
|
py: Python<'_>,
|
|
component_type: u8,
|
|
) -> PyResult<usize> {
|
|
catch_native_panic(|| {
|
|
self.inner
|
|
.inspect_component_protected_size(py, component_type)
|
|
})
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
#[pyo3(signature = (node_id, hash_values = None))]
|
|
fn inspect_set_node_hash_values(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
hash_values: Option<Vec<String>>,
|
|
) -> PyResult<()> {
|
|
catch_native_panic(|| {
|
|
self.inner
|
|
.inspect_set_node_hash_values(py, node_id, hash_values)
|
|
})
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
#[pyo3(signature = (node_id, component_type, value = None))]
|
|
fn inspect_set_component_device_value_raw(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
component_type: u8,
|
|
value: Option<PyTensor>,
|
|
) -> PyResult<()> {
|
|
catch_native_panic(|| {
|
|
self.inner
|
|
.inspect_set_component_device_value_raw(py, node_id, component_type, value)
|
|
})
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
#[pyo3(signature = (node_id, component_type, value = None))]
|
|
fn inspect_set_component_host_value_raw(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
component_type: u8,
|
|
value: Option<PyTensor>,
|
|
) -> PyResult<()> {
|
|
catch_native_panic(|| {
|
|
self.inner
|
|
.inspect_set_component_host_value_raw(py, node_id, component_type, value)
|
|
})
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
fn inspect_set_component_device_lock_ref(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
component_type: u8,
|
|
lock_ref: u32,
|
|
) -> PyResult<()> {
|
|
catch_native_panic(|| {
|
|
self.inner
|
|
.inspect_set_component_device_lock_ref(py, node_id, component_type, lock_ref)
|
|
})
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
fn inspect_remove_node_from_device_lru(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
component_type: u8,
|
|
) -> PyResult<()> {
|
|
catch_native_panic(|| {
|
|
self.inner
|
|
.inspect_remove_node_from_device_lru(py, node_id, component_type)
|
|
})
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
fn inspect_insert_node_into_host_lru(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
component_type: u8,
|
|
) -> PyResult<()> {
|
|
catch_native_panic(|| {
|
|
self.inner
|
|
.inspect_insert_node_into_host_lru(py, node_id, component_type)
|
|
})
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
fn inspect_set_component_evictable_size(
|
|
&self,
|
|
py: Python<'_>,
|
|
component_type: u8,
|
|
value: usize,
|
|
) -> PyResult<()> {
|
|
catch_native_panic(|| {
|
|
self.inner
|
|
.inspect_set_component_evictable_size(py, component_type, value)
|
|
})
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
fn inspect_set_component_protected_size(
|
|
&self,
|
|
py: Python<'_>,
|
|
component_type: u8,
|
|
value: usize,
|
|
) -> PyResult<()> {
|
|
catch_native_panic(|| {
|
|
self.inner
|
|
.inspect_set_component_protected_size(py, component_type, value)
|
|
})
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
fn inspect_update_duplicate_tracking(&self, py: Python<'_>, node_id: NodeId) -> PyResult<()> {
|
|
catch_native_panic(|| self.inner.inspect_update_duplicate_tracking(py, node_id))
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
fn inspect_advance_insert_walk_once(&self, py: Python<'_>) -> PyResult<()> {
|
|
catch_native_panic(|| self.inner.inspect_advance_insert_walk_once(py))
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
fn inspect_evict_component(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
component_type: u8,
|
|
target: u8,
|
|
) -> PyResult<HostEvictionResultBinding> {
|
|
catch_native_panic(|| {
|
|
self.inner
|
|
.inspect_evict_component(py, node_id, component_type, target)
|
|
})
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
fn inspect_validate_cascade_evict(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
component_type: u8,
|
|
target: u8,
|
|
) -> PyResult<()> {
|
|
catch_native_panic(|| {
|
|
self.inner
|
|
.inspect_validate_cascade_evict(py, node_id, component_type, target)
|
|
})
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
fn inspect_cleanup_tombstone_ancestors(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
) -> PyResult<HostEvictionResultBinding> {
|
|
catch_native_panic(|| self.inner.inspect_cleanup_tombstone_ancestors(py, node_id))
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
#[allow(clippy::too_many_arguments)]
|
|
#[pyo3(signature = (component_type, result, key, extra_key, cache_salt, value_chunks, best_value_len))]
|
|
fn inspect_finalize_component_match_result(
|
|
&self,
|
|
py: Python<'_>,
|
|
component_type: u8,
|
|
result: InspectionMatchResultInput,
|
|
key: &Bound<'_, PyAny>,
|
|
extra_key: Option<String>,
|
|
cache_salt: Option<String>,
|
|
value_chunks: Vec<PyTensor>,
|
|
best_value_len: usize,
|
|
) -> PyResult<MatchResultBinding> {
|
|
catch_native_panic(|| {
|
|
self.inner.inspect_finalize_component_match_result(
|
|
py,
|
|
component_type,
|
|
result,
|
|
key,
|
|
extra_key,
|
|
cache_salt,
|
|
value_chunks,
|
|
best_value_len,
|
|
)
|
|
})
|
|
}
|
|
|
|
#[cfg(feature = "inspection")]
|
|
#[pyo3(signature = (node_id, write_back = false))]
|
|
fn inspect_build_backup_node_ids(
|
|
&self,
|
|
py: Python<'_>,
|
|
node_id: NodeId,
|
|
write_back: bool,
|
|
) -> PyResult<Vec<NodeId>> {
|
|
catch_native_panic(|| {
|
|
self.inner
|
|
.inspect_build_backup_node_ids(py, node_id, write_back)
|
|
})
|
|
}
|
|
|
|
/// Print the tree structure for debugging.
|
|
fn pretty_print(&self, py: Python<'_>) -> PyResult<()> {
|
|
catch_native_panic(|| {
|
|
self.inner.pretty_print(py);
|
|
Ok(())
|
|
})
|
|
}
|
|
}
|
|
};
|
|
}
|
|
|
|
tree_core_binding!(
|
|
/// The UnifiedTreeCore Python adapter over single-token (unigram) child keys.
|
|
RustUnifiedTreeCoreBinding,
|
|
Vec<i64>
|
|
);
|
|
|
|
tree_core_binding!(
|
|
/// The UnifiedTreeCore Python adapter over bigram (EAGLE) child keys; keys
|
|
/// cross the boundary as raw token ids and pair up rust-side.
|
|
RustBigramUnifiedTreeCoreBinding,
|
|
Vec<(i64, i64)>
|
|
);
|
|
|
|
/// Per-page chained hashes over raw token ids.
|
|
#[pyfunction]
|
|
#[pyo3(signature = (token_ids, prior_hash, page_size, is_bigram = false))]
|
|
fn get_hash_str(
|
|
py: Python<'_>,
|
|
token_ids: &Bound<'_, PyAny>,
|
|
prior_hash: Option<String>,
|
|
page_size: usize,
|
|
is_bigram: bool,
|
|
) -> PyResult<Vec<String>> {
|
|
catch_native_panic(|| {
|
|
let raw = py_array_to_vec_i64(py, token_ids)?;
|
|
if page_size == 0 {
|
|
return Err(PyValueError::new_err("page_size must be positive"));
|
|
}
|
|
if let Some(prior_hash) = prior_hash.as_deref().filter(|hash| !hash.is_empty())
|
|
&& (prior_hash.len() != 64 || !prior_hash.bytes().all(|byte| byte.is_ascii_hexdigit()))
|
|
{
|
|
return Err(PyValueError::new_err(
|
|
"prior_hash must be a 64-character hexadecimal digest",
|
|
));
|
|
}
|
|
if let Some(token_id) = raw.iter().find(|token_id| **token_id < 0) {
|
|
return Err(PyValueError::new_err(format!(
|
|
"token id {token_id} must be a non-negative int64"
|
|
)));
|
|
}
|
|
Ok(py.detach(move || {
|
|
if is_bigram {
|
|
let key = <Vec<(i64, i64)> as ChildKeyType>::key_from(Cow::Owned(raw)).into_owned();
|
|
crate::node::get_hash_str::<Vec<(i64, i64)>>(&key, prior_hash.as_deref(), page_size)
|
|
} else {
|
|
crate::node::get_hash_str::<Vec<i64>>(&raw, prior_hash.as_deref(), page_size)
|
|
}
|
|
}))
|
|
})
|
|
}
|
|
|
|
fn register_mem_cache_module(m: &Bound<'_, PyModule>) -> PyResult<()> {
|
|
m.add_class::<TlruFloatConfigBinding>()?;
|
|
m.add_function(wrap_pyfunction!(get_hash_str, m)?)?;
|
|
m.add_class::<TreeCoreInitParamsBinding>()?;
|
|
m.add_class::<MatchParamsBinding>()?;
|
|
m.add_class::<InsertParamsBinding>()?;
|
|
m.add_class::<MatchResultBinding>()?;
|
|
m.add_class::<InsertResultBinding>()?;
|
|
m.add_class::<IncLockRefResultBinding>()?;
|
|
m.add_class::<DecLockRefParamsBinding>()?;
|
|
m.add_class::<EvictDeviceNextNodeResultBinding>()?;
|
|
m.add_class::<EvictDeviceLeafResultBinding>()?;
|
|
m.add_class::<DropSubtreeResultBinding>()?;
|
|
m.add_class::<DemoteResultBinding>()?;
|
|
m.add_class::<KvCanaryWalkResultBinding>()?;
|
|
m.add_class::<StorageBackupSpecBinding>()?;
|
|
m.add_class::<BufferBackupSnapshotBinding>()?;
|
|
m.add_class::<BufferBackupStateBinding>()?;
|
|
m.add_class::<HostEvictionResultBinding>()?;
|
|
m.add_class::<RustUnifiedTreeCoreBinding>()?;
|
|
m.add_class::<RustBigramUnifiedTreeCoreBinding>()?;
|
|
Ok(())
|
|
}
|
|
|
|
/// The production TreeCore extension module.
|
|
#[pymodule]
|
|
fn mem_cache(m: &Bound<'_, PyModule>) -> PyResult<()> {
|
|
register_mem_cache_module(m)
|
|
}
|
|
|
|
/// White-box variant used only by the shared test inspector.
|
|
#[cfg(feature = "inspection")]
|
|
#[pymodule]
|
|
fn mem_cache_inspection(m: &Bound<'_, PyModule>) -> PyResult<()> {
|
|
register_mem_cache_module(m)
|
|
}
|