diff --git a/python/sglang/srt/hardware_backend/npu/dsv4/c128_sidecar_component.py b/python/sglang/srt/hardware_backend/npu/dsv4/c128_sidecar_component.py index a7942aef2c..4e6e5f891b 100644 --- a/python/sglang/srt/hardware_backend/npu/dsv4/c128_sidecar_component.py +++ b/python/sglang/srt/hardware_backend/npu/dsv4/c128_sidecar_component.py @@ -285,6 +285,7 @@ class C128SidecarComponent(TreeComponent): node: UnifiedTreeNode, device_frees: dict[ComponentType, list[torch.Tensor]], host_frees: dict[ComponentType, list[torch.Tensor]], + cache_actions: list[CacheAction | ComponentAction], target: EvictLayer = EvictLayer.DEVICE, ) -> tuple[int, int]: cd = node.component_data[self.component_type] @@ -352,7 +353,7 @@ class C128SidecarComponent(TreeComponent): def _evict_device_start(self, request_cnt): pass - def _evict_device_next_node(self, tracker, device_frees, host_frees): + def _evict_device_next_node(self, tracker, device_frees, host_frees, cache_actions): return None def _evict_device_end(self) -> None: diff --git a/python/sglang/srt/mem_cache/rust_tree_core/adapter.py b/python/sglang/srt/mem_cache/rust_tree_core/adapter.py index b6db72b362..227ba02e19 100644 --- a/python/sglang/srt/mem_cache/rust_tree_core/adapter.py +++ b/python/sglang/srt/mem_cache/rust_tree_core/adapter.py @@ -274,67 +274,11 @@ def _comp_xfers_from_binding( } -def _insert_step_from_binding(step) -> InsertStepResult: - """Build the interface step for the binding's step (result on the final one).""" - result = None - if step.result is not None: - # A stepped insert delivers all actions through steps, never the result. - assert not step.result.cache_actions - result = InsertResult( - prefix_len=step.result.prefix_len, - last_device_node=step.result.last_device_node, - mamba_exist=step.result.mamba_exist, - swa_branch_inserted=step.result.swa_branch_inserted, - host_insert_dropped=step.result.host_insert_dropped, - rotation_tail_declined=step.result.rotation_tail_declined, - adopted_ranges=( - { - ComponentType(component_type): list(ranges) - for component_type, ranges in step.result.adopted_ranges.items() - } - if step.result.adopted_ranges is not None - else None - ), - ) - return InsertStepResult( - actions=_cache_actions_from_tagged(step.actions), result=result - ) - - -def _match_result_from_binding(result) -> MatchResult: - """Build the Python MatchResult for the binding's match result.""" - return MatchResult( - device_prefix_len=result.device_prefix_len, - last_device_node=result.last_device_node_id, - last_host_node=result.last_host_node_id, - best_match_node=result.best_match_node_id, - host_hit_length=result.host_hit_length, - swa_host_hit_length=result.swa_host_hit_length, - swa_branching_seqlen=result.swa_branching_seqlen, - mamba_host_hit_length=result.mamba_host_hit_length, - mamba_branching_seqlen=result.mamba_branching_seqlen, - full_kv_hit_length=result.full_kv_hit_length, - cache_actions=_cache_actions_from_tagged(result.cache_actions), - ) - - def _tracker_to_binding(tracker: dict[ComponentType, int]) -> dict[int, int]: """Rekey a ComponentType tracker by the binding's component values.""" return {int(component): freed for component, freed in tracker.items()} -def _fill_evict_result(binding_result, result): - """Map a binding eviction step into an interface step result; both carry - this step's per-component deltas and freed tensors.""" - for component, delta in binding_result.tracker.items(): - result.tracker[ComponentType(component)] = delta - for component, tensors in binding_result.new_device_frees.items(): - result.device_frees[ComponentType(component)].extend(tensors) - for component, tensors in binding_result.new_host_frees.items(): - result.host_frees[ComponentType(component)].extend(tensors) - return result - - class _RustKVCacheEventRecorder: """Expose the Rust event queue through the Python recorder interface.""" @@ -367,6 +311,70 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): _bindings = bindings supports_rotation_base = True + def _cache_action_from_tagged(self, action: tuple) -> CacheAction | ComponentAction: + return _cache_action_from_tagged(action) + + def _cache_actions_from_tagged( + self, actions: Sequence[tuple] + ) -> list[CacheAction | ComponentAction]: + return [self._cache_action_from_tagged(action) for action in actions] + + def _insert_step_from_binding(self, step) -> InsertStepResult: + """Build the interface step for the binding's step (result on the final one).""" + result = None + if step.result is not None: + # A stepped insert delivers all actions through steps, never the result. + assert not step.result.cache_actions + result = InsertResult( + prefix_len=step.result.prefix_len, + last_device_node=step.result.last_device_node, + mamba_exist=step.result.mamba_exist, + swa_branch_inserted=step.result.swa_branch_inserted, + host_insert_dropped=step.result.host_insert_dropped, + rotation_tail_declined=step.result.rotation_tail_declined, + adopted_ranges=( + { + ComponentType(component_type): list(ranges) + for component_type, ranges in step.result.adopted_ranges.items() + } + if step.result.adopted_ranges is not None + else None + ), + ) + return InsertStepResult( + actions=self._cache_actions_from_tagged(step.actions), result=result + ) + + def _match_result_from_binding(self, result) -> MatchResult: + """Build the Python MatchResult for the binding's match result.""" + return MatchResult( + device_prefix_len=result.device_prefix_len, + last_device_node=result.last_device_node_id, + last_host_node=result.last_host_node_id, + best_match_node=result.best_match_node_id, + host_hit_length=result.host_hit_length, + swa_host_hit_length=result.swa_host_hit_length, + swa_branching_seqlen=result.swa_branching_seqlen, + mamba_host_hit_length=result.mamba_host_hit_length, + mamba_branching_seqlen=result.mamba_branching_seqlen, + full_kv_hit_length=result.full_kv_hit_length, + cache_actions=self._cache_actions_from_tagged(result.cache_actions), + ) + + def _fill_evict_result(self, binding_result, result): + """Map a binding eviction step into an interface step result; both carry + this step's deltas, frees, and actions.""" + for component, delta in binding_result.tracker.items(): + result.tracker[ComponentType(component)] = delta + for component, tensors in binding_result.new_device_frees.items(): + result.device_frees[ComponentType(component)].extend(tensors) + for component, tensors in binding_result.new_host_frees.items(): + result.host_frees[ComponentType(component)].extend(tensors) + result.cache_actions.extend( + self._cache_actions_from_tagged(binding_result.cache_actions) + ) + return result + def __init__(self, params: CacheInitParams): assert params.tree_components is not None self.tree_components = tuple(params.tree_components) @@ -492,7 +500,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): dynamo_format=params.dynamo_kv_event_format, ) # The default-root empty result, prebuilt once from the binding. - self._empty_match_result = _match_result_from_binding( + self._empty_match_result = self._match_result_from_binding( self._binding.empty_match_result() ) @@ -507,7 +515,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): def reset(self) -> None: self._binding.reset() # Node handles are never re-minted, so the fresh root gets a new one. - self._empty_match_result = _match_result_from_binding( + self._empty_match_result = self._match_result_from_binding( self._binding.empty_match_result() ) @@ -549,7 +557,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): ) -> list[CacheAction | ComponentAction]: if component_type != ComponentType.SWA: raise NotImplementedError("The Rust tree core supports one SWA window") - return _cache_actions_from_tagged( + return self._cache_actions_from_tagged( self._binding.attach_swa_window( self._bindings.MatchParamsBinding( key=_radix_key_buffer(key), @@ -590,7 +598,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): params: DecLockRefParams, ) -> DecSwaLockOnlyResult: result = DecSwaLockOnlyResult() - new_device_frees, new_host_frees = self._binding.dec_window_lock_only( + new_device_frees, new_host_frees, actions = self._binding.dec_window_lock_only( node_id, int(component_type), _dec_lock_ref_params_to_binding(self._bindings, params), @@ -599,6 +607,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): result.device_frees[ComponentType(component)].extend(tensors) for component, tensors in new_host_frees.items(): result.host_frees[ComponentType(component)].extend(tensors) + result.cache_actions.extend(self._cache_actions_from_tagged(actions)) return result def dec_swa_lock_only( @@ -607,13 +616,14 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): params: DecLockRefParams, ) -> DecSwaLockOnlyResult: result = DecSwaLockOnlyResult() - new_device_frees, new_host_frees = self._binding.dec_swa_lock_only( + new_device_frees, new_host_frees, actions = self._binding.dec_swa_lock_only( node_id, _dec_lock_ref_params_to_binding(self._bindings, params) ) for component, tensors in new_device_frees.items(): result.device_frees[ComponentType(component)].extend(tensors) for component, tensors in new_host_frees.items(): result.host_frees[ComponentType(component)].extend(tensors) + result.cache_actions.extend(self._cache_actions_from_tagged(actions)) return result # ==== Device eviction (driven step-wise by the Controller's evict()) ==== @@ -637,7 +647,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): swa_backup_node_id=binding_result.swa_backup_node_id, swa_backup_num_tokens=binding_result.swa_backup_num_tokens, ) - return _fill_evict_result(binding_result, result) + return self._fill_evict_result(binding_result, result) def finish_mamba_state_eviction(self, node_id: NodeId) -> EvictDeviceNextNodeResult: binding_result = self._binding.finish_mamba_state_eviction(node_id) @@ -646,7 +656,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): made_progress=binding_result.made_progress, unbacked_tokens=binding_result.unbacked_tokens, ) - return _fill_evict_result(binding_result, result) + return self._fill_evict_result(binding_result, result) def finish_swa_state_eviction(self, node_id: NodeId) -> EvictDeviceNextNodeResult: binding_result = self._binding.finish_swa_state_eviction(node_id) @@ -655,7 +665,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): made_progress=binding_result.made_progress, unbacked_tokens=binding_result.unbacked_tokens, ) - return _fill_evict_result(binding_result, result) + return self._fill_evict_result(binding_result, result) def evict_device_leaf( self, node_id: NodeId, is_write_back: bool @@ -668,13 +678,15 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): backup = binding_result.backup_kv result = EvictDeviceLeafResult( unbacked_tokens=binding_result.unbacked_tokens, - backup_kv=_cache_action_from_tagged(backup) if backup is not None else None, + backup_kv=self._cache_action_from_tagged(backup) + if backup is not None + else None, ) - return _fill_evict_result(binding_result, result) + return self._fill_evict_result(binding_result, result) def demote(self, node_id: NodeId) -> DemoteResult: binding_result = self._binding.demote(node_id) - return _fill_evict_result(binding_result, DemoteResult()) + return self._fill_evict_result(binding_result, DemoteResult()) def evict_device_end(self, component_type: ComponentType) -> None: self._binding.evict_device_end(int(component_type)) @@ -755,7 +767,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): cache_salt=key.cache_salt, ) ) - return _match_result_from_binding(result) + return self._match_result_from_binding(result) def match_full_device_prefix(self, key: RadixKey) -> tuple[int, NodeId, int]: return self._binding.match_full_device_prefix( @@ -809,16 +821,16 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): rotation_base=params.rotation_base, ) ) - return _insert_step_from_binding(step) + return self._insert_step_from_binding(step) def resume_insert(self) -> InsertStepResult: - return _insert_step_from_binding(self._binding.resume_insert()) + return self._insert_step_from_binding(self._binding.resume_insert()) def has_ongoing_insert(self) -> bool: return self._binding.has_ongoing_insert() def end_insert(self) -> list[CacheAction | ComponentAction]: - return _cache_actions_from_tagged(self._binding.end_insert()) + return self._cache_actions_from_tagged(self._binding.end_insert()) def drive_host_eviction( self, component_type: ComponentType, num_tokens: int @@ -828,19 +840,23 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): num_tokens, envs.SGLANG_HICACHE_SKIP_HOST_DUPLICATE_RECLAIM.get(), ) - return _fill_evict_result(binding_result, DriveHostEvictionResult()) + return self._fill_evict_result(binding_result, DriveHostEvictionResult()) def evict_excess_path_states( self, tail_node_id: NodeId, device_frees: dict[ComponentType, list[torch.Tensor]], host_frees: dict[ComponentType, list[torch.Tensor]], + cache_actions: list[CacheAction | ComponentAction], ) -> None: binding_result = self._binding.evict_excess_path_states(tail_node_id) for component, tensors in binding_result.new_device_frees.items(): device_frees[ComponentType(component)].extend(tensors) for component, tensors in binding_result.new_host_frees.items(): host_frees[ComponentType(component)].extend(tensors) + cache_actions.extend( + self._cache_actions_from_tagged(binding_result.cache_actions) + ) # ==== HiCache ==== @@ -931,9 +947,16 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): inserted_host_node=result.inserted_host_node, host_insert_dropped=result.host_insert_dropped, mamba_exist=result.mamba_exist, - cache_actions=_cache_actions_from_tagged(result.cache_actions), + cache_actions=self._cache_actions_from_tagged(result.cache_actions), ) + def build_backup_kv_action(self, node_id: NodeId) -> BackupKV: + action = self._cache_action_from_tagged( + self._binding.build_backup_kv_action(node_id) + ) + assert isinstance(action, BackupKV) + return action + def build_backup_spec( self, node_id: NodeId ) -> tuple[torch.Tensor, dict[ComponentType, list[PoolTransfer]]]: @@ -1123,7 +1146,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): ) if insert_result is not None and mamba_exist is not None: insert_result.mamba_exist = mamba_exist - cache_actions.extend(_cache_actions_from_tagged(actions)) + cache_actions.extend(self._cache_actions_from_tagged(actions)) def commit_backup( self, @@ -1148,12 +1171,12 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): _transfer_to_binding(kv_xfer), _comp_xfers_to_binding(comp_xfers), ) - return _cache_actions_from_tagged(actions) + return self._cache_actions_from_tagged(actions) def drop_subtree_no_host(self, node_id: NodeId) -> DropSubtreeNoHostResult: binding_result = self._binding.drop_subtree_no_host(node_id) result = DropSubtreeNoHostResult(is_dropped=binding_result.dropped) - return _fill_evict_result(binding_result, result) + return self._fill_evict_result(binding_result, result) def mark_write_through_pending( self, node_ids: list[NodeId], ack_id: NodeId diff --git a/python/sglang/srt/mem_cache/unified_cache/components/base.py b/python/sglang/srt/mem_cache/unified_cache/components/base.py index f0da805345..c54b35c91b 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/base.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/base.py @@ -528,6 +528,7 @@ class TreeComponent(ABC): node: UnifiedTreeNode, device_frees: dict[ComponentType, list[torch.Tensor]], host_frees: dict[ComponentType, list[torch.Tensor]], + cache_actions: list[CacheAction | ComponentAction], target: EvictLayer = EvictLayer.DEVICE, ) -> tuple[int, int]: """Free this component's KV resources on a node being evicted. @@ -541,7 +542,8 @@ class TreeComponent(ABC): No tombstone — caller will delete the node. Freed indices are collected into *device_frees*/*host_frees* for the - Controller to drain, never freed inline. + Controller to drain, never freed inline. Additional cache work is + appended to *cache_actions* in the same operation. Returns (device_freed, host_freed) token counts.""" ... @@ -586,6 +588,7 @@ class TreeComponent(ABC): tracker: dict[ComponentType, int], device_frees: dict[ComponentType, list[torch.Tensor]], host_frees: dict[ComponentType, list[torch.Tensor]], + cache_actions: list[CacheAction | ComponentAction], ) -> NodeId | InternalStateBackup | None: """Return a device leaf, an internal backup request, or no selection. @@ -596,7 +599,9 @@ class TreeComponent(ABC): assert self.is_evict_device_ongoing, ( f"{self.component_type} device eviction not started" ) - return self._evict_device_next_node(tracker, device_frees, host_frees) + return self._evict_device_next_node( + tracker, device_frees, host_frees, cache_actions=cache_actions + ) def evict_device_end(self) -> None: """Clear this component's device-eviction walk state.""" @@ -617,6 +622,7 @@ class TreeComponent(ABC): tracker: dict[ComponentType, int], device_frees: dict[ComponentType, list[torch.Tensor]], host_frees: dict[ComponentType, list[torch.Tensor]], + cache_actions: list[CacheAction | ComponentAction], ) -> NodeId | InternalStateBackup | None: """Select a leaf, request backup, or perform at most one internal mutation.""" ... @@ -842,6 +848,7 @@ class TreeComponent(ABC): tracker: dict[ComponentType, int], device_frees: dict[ComponentType, list[torch.Tensor]], host_frees: dict[ComponentType, list[torch.Tensor]], + cache_actions: list[CacheAction | ComponentAction], ) -> None: """Evict from this component's host-side resources, collecting freed values into *device_frees*/*host_frees* for the Controller to drain. diff --git a/python/sglang/srt/mem_cache/unified_cache/components/full.py b/python/sglang/srt/mem_cache/unified_cache/components/full.py index 1476db8928..0021124cda 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/full.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/full.py @@ -174,6 +174,7 @@ class FullComponent(TreeComponent): node: UnifiedTreeNode, device_frees: dict[ComponentType, list[torch.Tensor]], host_frees: dict[ComponentType, list[torch.Tensor]], + cache_actions: list[CacheAction | ComponentAction], target: EvictLayer = EvictLayer.DEVICE, ) -> tuple[int, int]: cd = node.component_data[self.component_type] @@ -220,6 +221,7 @@ class FullComponent(TreeComponent): tracker: dict[ComponentType, int], device_frees: dict[ComponentType, list[torch.Tensor]], host_frees: dict[ComponentType, list[torch.Tensor]], + cache_actions: list[CacheAction | ComponentAction], ) -> Optional[NodeId]: ct = self.component_type lv = self._evict_device_last_node @@ -251,6 +253,7 @@ class FullComponent(TreeComponent): tracker: dict[ComponentType, int], device_frees: dict[ComponentType, list[torch.Tensor]], host_frees: dict[ComponentType, list[torch.Tensor]], + cache_actions: list[CacheAction | ComponentAction], ) -> None: """Evict host leaves to free KV host pool space.""" self._ensure_eviction_strategy() @@ -264,7 +267,9 @@ class FullComponent(TreeComponent): _, x = heapq.heappop(heap) if x not in self.tree_core.evictable_host_leaves: continue - self.tree_core._evict_host_leaf(x, tracker, device_frees, host_frees) + self.tree_core._evict_host_leaf( + x, tracker, device_frees, host_frees, cache_actions=cache_actions + ) if ( x.parent is not None and x.parent in self.tree_core.evictable_host_leaves diff --git a/python/sglang/srt/mem_cache/unified_cache/components/mamba.py b/python/sglang/srt/mem_cache/unified_cache/components/mamba.py index df1419e6e1..a69394e6d7 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/mamba.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/mamba.py @@ -1,6 +1,5 @@ from __future__ import annotations -from collections import defaultdict from dataclasses import replace from typing import TYPE_CHECKING, Callable, Optional, Sequence @@ -41,6 +40,9 @@ from sglang.srt.mem_cache.unified_cache.components.base import ( TreeComponent, get_and_increase_time_counter, ) +from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import ( + BaseEvictionResult, +) from sglang.srt.runtime_context import ( get_exec, mamba_cache_chunk_size, @@ -265,6 +267,7 @@ class MambaComponent(TreeComponent): tail: UnifiedTreeNode, device_frees: dict[ComponentType, list[torch.Tensor]], host_frees: dict[ComponentType, list[torch.Tensor]], + cache_actions: list[CacheAction | ComponentAction], ) -> None: """Evict shallow eligible device checkpoints beyond the path cap. @@ -304,8 +307,16 @@ class MambaComponent(TreeComponent): host_frees, target=EvictLayer.DEVICE, tracker=tracker, + cache_actions=cache_actions, + ) + self.tree_core._cascade_evict( + node, + self, + tracker, + device_frees, + host_frees, + cache_actions=cache_actions, ) - self.tree_core._cascade_evict(node, self, tracker, device_frees, host_frees) excess -= 1 def redistribute_on_node_split( @@ -325,6 +336,7 @@ class MambaComponent(TreeComponent): node: UnifiedTreeNode, device_frees: dict[ComponentType, list[torch.Tensor]], host_frees: dict[ComponentType, list[torch.Tensor]], + cache_actions: list[CacheAction | ComponentAction], target: EvictLayer = EvictLayer.DEVICE, ) -> tuple[int, int]: cd = node.component_data[self.component_type] @@ -375,6 +387,7 @@ class MambaComponent(TreeComponent): tracker: dict[ComponentType, int], device_frees: dict[ComponentType, list[torch.Tensor]], host_frees: dict[ComponentType, list[torch.Tensor]], + cache_actions: list[CacheAction | ComponentAction], ) -> NodeId | InternalStateBackup | None: """Advance one device-eviction step and return a leaf, if selected. @@ -429,9 +442,15 @@ class MambaComponent(TreeComponent): tracker=tracker, device_frees=device_frees, host_frees=host_frees, + cache_actions=cache_actions, ) self.tree_core._cascade_evict( - x, self, tracker, device_frees=device_frees, host_frees=host_frees + x, + self, + tracker, + device_frees=device_frees, + host_frees=host_frees, + cache_actions=cache_actions, ) if enabled: self._evict_device_cursor = lru.cursor_next() @@ -996,6 +1015,7 @@ class MambaComponent(TreeComponent): tracker: dict[ComponentType, int], device_frees: dict[ComponentType, list[torch.Tensor]], host_frees: dict[ComponentType, list[torch.Tensor]], + cache_actions: list[CacheAction | ComponentAction], ) -> None: """Evict mamba host resources. Internal nodes: private tombstone (free host mamba only). @@ -1016,7 +1036,9 @@ class MambaComponent(TreeComponent): not enabled or self._can_evict_leaf_atomically(x) ): # Host leaf: atomic eviction (all components host + delete) - self.tree_core._evict_host_leaf(x, tracker, device_frees, host_frees) + self.tree_core._evict_host_leaf( + x, tracker, device_frees, host_frees, cache_actions=cache_actions + ) else: # Internal (or a leaf a session still pins): tombstone Mamba + cascade assert cd.host_value is not None @@ -1027,6 +1049,7 @@ class MambaComponent(TreeComponent): tracker=tracker, device_frees=device_frees, host_frees=host_frees, + cache_actions=cache_actions, ) self.tree_core._cascade_evict( x, @@ -1035,6 +1058,7 @@ class MambaComponent(TreeComponent): device_frees=device_frees, host_frees=host_frees, target=EvictLayer.HOST, + cache_actions=cache_actions, ) self.tree_core._update_evictable_leaf_sets(x) if enabled: @@ -1052,16 +1076,18 @@ class MambaComponent(TreeComponent): def apply_component_action(self, action: ComponentAction) -> None: if isinstance(action, MambaEvictExcessPathStates): - device_frees: dict[ComponentType, list[torch.Tensor]] = defaultdict(list) - host_frees: dict[ComponentType, list[torch.Tensor]] = defaultdict(list) + result = BaseEvictionResult() # Drain even if the walk raises so tombstoned slots are not leaked; # the walk runs behind the tree-core interface (Rust runs it natively). try: self.tree_core.evict_excess_path_states( - action.tail_node_id, device_frees, host_frees + action.tail_node_id, + result.device_frees, + result.host_frees, + result.cache_actions, ) finally: - self.cache._free_values(device_frees, host_frees) + self.cache._apply_eviction_result(result) return if isinstance(action, FreeComponentDeviceSlot): for indices in action.indices: diff --git a/python/sglang/srt/mem_cache/unified_cache/components/swa.py b/python/sglang/srt/mem_cache/unified_cache/components/swa.py index dc49f93339..e30a21c487 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/swa.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/swa.py @@ -696,6 +696,7 @@ class SWAComponent(TreeComponent): node: UnifiedTreeNode, device_frees: dict[ComponentType, list[torch.Tensor]], host_frees: dict[ComponentType, list[torch.Tensor]], + cache_actions: list[CacheAction | ComponentAction], target: EvictLayer = EvictLayer.DEVICE, ) -> tuple[int, int]: ct = self.component_type @@ -755,6 +756,7 @@ class SWAComponent(TreeComponent): tracker: dict[ComponentType, int], device_frees: dict[ComponentType, list[torch.Tensor]], host_frees: dict[ComponentType, list[torch.Tensor]], + cache_actions: list[CacheAction | ComponentAction], ) -> NodeId | InternalStateBackup | None: """Advance one device-eviction step and return a leaf, if selected. @@ -824,9 +826,15 @@ class SWAComponent(TreeComponent): tracker=tracker, device_frees=device_frees, host_frees=host_frees, + cache_actions=cache_actions, ) self.tree_core._cascade_evict( - x, self, tracker, device_frees=device_frees, host_frees=host_frees + x, + self, + tracker, + device_frees=device_frees, + host_frees=host_frees, + cache_actions=cache_actions, ) if enabled: self._evict_device_cursor = lru.cursor_next() @@ -947,6 +955,7 @@ class SWAComponent(TreeComponent): swa_uuid_for_lock: Optional[int], device_frees: dict[ComponentType, list[torch.Tensor]], host_frees: dict[ComponentType, list[torch.Tensor]], + cache_actions: list[CacheAction | ComponentAction], ) -> None: """Early-release the SWA lock along [node, swa_uuid_for_lock]; this method touches only SWA state. The wrapping ``dec_swa_lock_only`` also @@ -980,6 +989,7 @@ class SWAComponent(TreeComponent): target=EvictLayer.DEVICE, device_frees=device_frees, host_frees=host_frees, + cache_actions=cache_actions, ) if swa_uuid_for_lock and cd.metadata.get("uuid") == swa_uuid_for_lock: @@ -1488,6 +1498,7 @@ class SWAComponent(TreeComponent): tracker: dict[ComponentType, int], device_frees: dict[ComponentType, list[torch.Tensor]], host_frees: dict[ComponentType, list[torch.Tensor]], + cache_actions: list[CacheAction | ComponentAction], ) -> None: """Evict SWA host resources. Internal nodes: private tombstone (free SWA host only). @@ -1507,7 +1518,9 @@ class SWAComponent(TreeComponent): if x in self.tree_core.evictable_host_leaves and ( not enabled or self._can_evict_leaf_atomically(x) ): - self.tree_core._evict_host_leaf(x, tracker, device_frees, host_frees) + self.tree_core._evict_host_leaf( + x, tracker, device_frees, host_frees, cache_actions=cache_actions + ) else: assert cd.host_value is not None self.tree_core._evict_component_and_detach_lru( @@ -1517,6 +1530,7 @@ class SWAComponent(TreeComponent): tracker=tracker, device_frees=device_frees, host_frees=host_frees, + cache_actions=cache_actions, ) self.tree_core._cascade_evict( x, @@ -1525,6 +1539,7 @@ class SWAComponent(TreeComponent): device_frees=device_frees, host_frees=host_frees, target=EvictLayer.HOST, + cache_actions=cache_actions, ) if enabled: x = host_lru.cursor_next(host_lock=True) diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py index 97b4cd80fc..c921d323bb 100644 --- a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py +++ b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py @@ -811,6 +811,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): params.get_lock_uuid(component_type), result.device_frees, result.host_frees, + cache_actions=result.cache_actions, ) return result @@ -834,6 +835,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): params.get_lock_uuid(ComponentType.SWA), result.device_frees, result.host_frees, + cache_actions=result.cache_actions, ) # Drop strictly-lower-priority locks co-located on the node, skipping @@ -1653,7 +1655,10 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): self._begin_tracking_unbacked_tokens() try: step = component.evict_device_next_node( - updated_tracker, result.device_frees, result.host_frees + updated_tracker, + result.device_frees, + result.host_frees, + cache_actions=result.cache_actions, ) finally: result.unbacked_tokens = self._finish_tracking_unbacked_tokens() @@ -1676,6 +1681,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): or result.mamba_backup_node_id is not None or result.swa_backup_node_id is not None or bool(result.tracker) + or bool(result.cache_actions) or any(result.device_frees.values()) or any(result.host_frees.values()) ) @@ -1738,6 +1744,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): tracker=result.tracker, device_frees=result.device_frees, host_frees=result.host_frees, + cache_actions=result.cache_actions, ) self._cascade_evict( node, @@ -1745,6 +1752,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): result.tracker, device_frees=result.device_frees, host_frees=result.host_frees, + cache_actions=result.cache_actions, ) finally: result.unbacked_tokens = self._finish_tracking_unbacked_tokens() @@ -1780,6 +1788,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): result.tracker, device_frees=result.device_frees, host_frees=result.host_frees, + cache_actions=result.cache_actions, ) return result self._demote( @@ -1787,6 +1796,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): result.tracker, device_frees=result.device_frees, host_frees=result.host_frees, + cache_actions=result.cache_actions, ) return result finally: @@ -1832,6 +1842,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): result.tracker, result.device_frees, result.host_frees, + cache_actions=result.cache_actions, ) self._remove_leaf_from_parent(desc) self._delete_unbacked_device_leaf( @@ -1839,6 +1850,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): result.tracker, device_frees=result.device_frees, host_frees=result.host_frees, + cache_actions=result.cache_actions, ) result.is_dropped = True return result @@ -1850,6 +1862,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): tracker: dict[ComponentType, int], device_frees: dict[ComponentType, list[torch.Tensor]], host_frees: dict[ComponentType, list[torch.Tensor]], + cache_actions: list[CacheAction | ComponentAction], ) -> None: """Free every component layer on the node and detach it from the LRU lists and evictable leaf sets.""" @@ -1862,6 +1875,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): tracker=tracker, device_frees=device_frees, host_frees=host_frees, + cache_actions=cache_actions, ) self.evictable_device_leaves.discard(node) self.evictable_host_leaves.discard(node) @@ -1872,16 +1886,26 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): tracker: dict[ComponentType, int], device_frees: dict[ComponentType, list[torch.Tensor]], host_frees: dict[ComponentType, list[torch.Tensor]], + cache_actions: list[CacheAction | ComponentAction], ) -> None: """Delete a device leaf that has no host backup, freeing all layers.""" self._release_all_component_layers( - node, StorageMedium.GPU, tracker, device_frees, host_frees + node, + StorageMedium.GPU, + tracker, + device_frees, + host_frees, + cache_actions=cache_actions, ) parent = node.parent self._remove_leaf_from_parent(node) self._update_evictable_leaf_sets(parent) self._iteratively_delete_tombstone_leaf( - node, tracker, device_frees=device_frees, host_frees=host_frees + node, + tracker, + device_frees=device_frees, + host_frees=host_frees, + cache_actions=cache_actions, ) def drive_host_eviction( @@ -1903,12 +1927,14 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): result.tracker, result.device_frees, result.host_frees, + cache_actions=result.cache_actions, ) comp.drive_host_eviction( num_tokens, result.tracker, result.device_frees, result.host_frees, + cache_actions=result.cache_actions, ) return result @@ -1917,9 +1943,13 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): tail_node_id: NodeId, device_frees: dict[ComponentType, list[torch.Tensor]], host_frees: dict[ComponentType, list[torch.Tensor]], + cache_actions: list[CacheAction | ComponentAction], ) -> None: self.components_by_type[ComponentType.MAMBA]._evict_excess_path_states( - self.node_by_id(tail_node_id), device_frees, host_frees + self.node_by_id(tail_node_id), + device_frees, + host_frees, + cache_actions=cache_actions, ) def _reclaim_full_host_duplicates( @@ -1928,6 +1958,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): tracker: dict[ComponentType, int], device_frees: dict[ComponentType, list[torch.Tensor]], host_frees: dict[ComponentType, list[torch.Tensor]], + cache_actions: list[CacheAction | ComponentAction], ) -> None: """Reclaim Full host duplicates until num_tokens are freed; pass 1 spares evictable D-leaves (imminent free demotes), pass 2 takes them.""" @@ -1947,7 +1978,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): if not self._can_reclaim_full_host_duplicate(node): continue self._release_full_host_duplicate( - node, tracker, device_frees, host_frees + node, tracker, device_frees, host_frees, cache_actions=cache_actions ) swept_ids.append(node.id) # released -> no longer a duplicate # Sweep after the walk: the dict must not be mutated mid-iteration. @@ -1973,6 +2004,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): tracker: dict[ComponentType, int], device_frees: dict[ComponentType, list[torch.Tensor]], host_frees: dict[ComponentType, list[torch.Tensor]], + cache_actions: list[CacheAction | ComponentAction], ) -> None: """Free only the Full host layer; aux host slices stay under their own pools' LRU (a host-only aux slice may be a sole copy).""" @@ -1985,6 +2017,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): tracker=tracker, device_frees=device_frees, host_frees=host_frees, + cache_actions=cache_actions, ) self.write_back_duplicate_reclaim_digest = ( self.write_back_duplicate_reclaim_digest * 1000003 + node.id + 1 @@ -1996,6 +2029,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): tracker: dict[ComponentType, int], device_frees: dict[ComponentType, list[torch.Tensor]], host_frees: dict[ComponentType, list[torch.Tensor]], + cache_actions: list[CacheAction | ComponentAction], ) -> None: """Atomically evict all components on a host leaf. @@ -2011,11 +2045,14 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): tracker=None, device_frees=device_frees, host_frees=host_frees, + cache_actions=cache_actions, ) tracker[comp.component_type] += hf self.evictable_host_leaves.discard(node) self._remove_leaf_from_parent(node) - self._iteratively_delete_tombstone_leaf(node, tracker, device_frees, host_frees) + self._iteratively_delete_tombstone_leaf( + node, tracker, device_frees, host_frees, cache_actions=cache_actions + ) def demote(self, node_id: NodeId) -> DemoteResult: """Release a node's device KV once its host copy exists; the node stays in the @@ -2026,6 +2063,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): result.tracker, result.device_frees, result.host_frees, + cache_actions=result.cache_actions, ) return result @@ -2035,6 +2073,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): tracker: dict[ComponentType, int], device_frees: dict[ComponentType, list[torch.Tensor]], host_frees: dict[ComponentType, list[torch.Tensor]], + cache_actions: list[CacheAction | ComponentAction], ) -> None: assert not node.evicted and node.backuped trigger = self.components_by_type[BASE_COMPONENT_TYPE] @@ -2045,9 +2084,15 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): tracker=tracker, device_frees=device_frees, host_frees=host_frees, + cache_actions=cache_actions, ) self._cascade_evict( - node, trigger, tracker, device_frees=device_frees, host_frees=host_frees + node, + trigger, + tracker, + device_frees=device_frees, + host_frees=host_frees, + cache_actions=cache_actions, ) self.kv_events.record_remove(node, medium=StorageMedium.GPU) @@ -2064,6 +2109,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): tracker: dict[ComponentType, int], device_frees: dict[ComponentType, list[torch.Tensor]], host_frees: dict[ComponentType, list[torch.Tensor]], + cache_actions: list[CacheAction | ComponentAction], target: EvictLayer = EvictLayer.DEVICE, ): """Cascade eviction from trigger to lower-or-equal priority components.""" @@ -2082,6 +2128,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): tracker=tracker, device_frees=device_frees, host_frees=host_frees, + cache_actions=cache_actions, ) if comp.component_type == BASE_COMPONENT_TYPE: base_evicted = True @@ -2159,12 +2206,17 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): comp: TreeComponent, device_frees: dict[ComponentType, list[torch.Tensor]], host_frees: dict[ComponentType, list[torch.Tensor]], + cache_actions: list[CacheAction | ComponentAction], target: EvictLayer = EvictLayer.DEVICE, tracker: Optional[dict[ComponentType, int]] = None, ) -> tuple[int, int]: had_host_copy = node.component_data[comp.component_type].host_value is not None device_freed, host_freed = comp.evict_component( - node, target=target, device_frees=device_frees, host_frees=host_frees + node, + target=target, + device_frees=device_frees, + host_frees=host_frees, + cache_actions=cache_actions, ) if tracker is not None: if EvictLayer.DEVICE in target: @@ -2197,6 +2249,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): tracker: dict[ComponentType, int], device_frees: dict[ComponentType, list[torch.Tensor]], host_frees: dict[ComponentType, list[torch.Tensor]], + cache_actions: list[CacheAction | ComponentAction], ): """Walk up from *deleted_node* and cascade-delete childless ancestors. @@ -2206,7 +2259,11 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): - neither → evict all remaining data, delete, continue up """ self._iteratively_delete_tombstone_ancestors( - deleted_node.parent, tracker, device_frees, host_frees + deleted_node.parent, + tracker, + device_frees, + host_frees, + cache_actions=cache_actions, ) def _iteratively_delete_tombstone_ancestors( @@ -2215,6 +2272,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): tracker: dict[ComponentType, int], device_frees: dict[ComponentType, list[torch.Tensor]], host_frees: dict[ComponentType, list[torch.Tensor]], + cache_actions: list[CacheAction | ComponentAction], ) -> None: """Delete childless tombstone ancestors until a live or locked node is reached.""" ct = BASE_COMPONENT_TYPE @@ -2241,6 +2299,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): tracker=tracker, device_frees=device_frees, host_frees=host_frees, + cache_actions=cache_actions, ) if has_host: @@ -2257,6 +2316,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): tracker=tracker, device_frees=device_frees, host_frees=host_frees, + cache_actions=cache_actions, ) self.evictable_host_leaves.discard(cur) diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py index d25e2301b2..3bc4616419 100644 --- a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py +++ b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py @@ -23,7 +23,7 @@ class BaseEvictionResult(msgspec.Struct): """Base of the eviction step results: the component-keyed device/host values the step freed, for the Controller to drain right after the call returns, plus `tracker` -- the step's per-component evicted counts for the - Controller to accumulate.""" + Controller to accumulate. Free the values, then drain cache actions.""" device_frees: dict[ComponentType, list[torch.Tensor]] = msgspec.field( default_factory=lambda: defaultdict(list) @@ -34,12 +34,15 @@ class BaseEvictionResult(msgspec.Struct): tracker: dict[ComponentType, int] = msgspec.field( default_factory=lambda: defaultdict(int) ) + cache_actions: list[CacheAction | ComponentAction] = msgspec.field( + default_factory=list + ) def __del__(self) -> None: - # Drop tripwire: every returned value must be drained before disposal. - assert not self.device_frees and not self.host_frees, ( - "BaseEvictionResult dropped with undrained values" - ) + # Drop tripwire: every returned resource must be drained before disposal. + assert ( + not self.device_frees and not self.host_frees and not self.cache_actions + ), "BaseEvictionResult dropped with undrained resources" class EvictDeviceNextNodeResult(BaseEvictionResult): @@ -548,9 +551,11 @@ class UnifiedTreeCoreInterface(ABC): tail_node_id: NodeId, device_frees: dict[ComponentType, list[torch.Tensor]], host_frees: dict[ComponentType, list[torch.Tensor]], + cache_actions: list[CacheAction | ComponentAction], ) -> None: """Evict shallow Mamba device checkpoints beyond the per-path cap on the - tail's root path, collecting freed values into the caller's dicts.""" + tail's root path, collecting freed values and actions into the caller's + buffers. The caller frees values, then drains actions, even on error.""" ... # ==== HiCache ==== diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index c6b0181bd6..ab59618a5e 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -120,6 +120,9 @@ if TYPE_CHECKING: from sglang.srt.managers.cache_controller import HiCacheAck from sglang.srt.managers.schedule_batch import Req, ReqKvInfo from sglang.srt.mem_cache.cache_init_params import CacheInitParams + from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import ( + BaseEvictionResult, + ) from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import ( PrefetchOperation, ) @@ -825,6 +828,13 @@ class UnifiedRadixCache(BasePrefixCache): finally: self._drain_host_frees(host_frees) + def _apply_eviction_result(self, result: BaseEvictionResult) -> None: + """Drain a tree step before the next allocator-dependent decision.""" + try: + self._free_values(result.device_frees, result.host_frees) + finally: + self._apply_cache_actions(result.cache_actions) + def _accumulate_tracker( self, tracker: dict[ComponentType, int], @@ -842,7 +852,10 @@ class UnifiedRadixCache(BasePrefixCache): if result.mamba_backup_node_id is not None: assert component_type == ComponentType.MAMBA and result.node_id is None assert ( - not result.device_frees and not result.host_frees and not result.tracker + not result.device_frees + and not result.host_frees + and not result.tracker + and not result.cache_actions ) # Reserve a host state slot and wait for the backup acknowledgment # before freeing device state. If allocation fails, eviction still @@ -856,7 +869,10 @@ class UnifiedRadixCache(BasePrefixCache): elif result.swa_backup_node_id is not None: assert component_type == ComponentType.SWA and result.node_id is None assert ( - not result.device_frees and not result.host_frees and not result.tracker + not result.device_frees + and not result.host_frees + and not result.tracker + and not result.cache_actions ) # The backup can cover several unbacked SWA segments. Reserve host # space for the whole window before copying it, then resume eviction @@ -868,7 +884,7 @@ class UnifiedRadixCache(BasePrefixCache): self.evict_host(needed, ComponentType.SWA) self.backup_node_for_write_back(node_id) result = self.tree_core.finish_swa_state_eviction(node_id) - self._free_values(result.device_frees, result.host_frees) + self._apply_eviction_result(result) if self._tracks_write_through_unbacked_evictions(): self._record_dropped_tokens( result.unbacked_tokens, @@ -883,7 +899,7 @@ class UnifiedRadixCache(BasePrefixCache): """Evict one device leaf, consuming its step result; returns the deferred write-back BackupKV when one must run before the demote.""" result = self.tree_core.evict_device_leaf(node_id, self.is_write_back) - self._free_values(result.device_frees, result.host_frees) + self._apply_eviction_result(result) if self._tracks_write_through_unbacked_evictions(): self._record_dropped_tokens( result.unbacked_tokens, @@ -895,7 +911,7 @@ class UnifiedRadixCache(BasePrefixCache): def _demote(self, node_id: NodeId, tracker: dict[ComponentType, int]) -> None: """Demote a backed-up node, consuming its step result.""" result = self.tree_core.demote(node_id) - self._free_values(result.device_frees, result.host_frees) + self._apply_eviction_result(result) self._accumulate_tracker(tracker, result.tracker) def _drop_subtree_no_host( @@ -903,7 +919,7 @@ class UnifiedRadixCache(BasePrefixCache): ) -> bool: """Run the write-back drop fallback, consuming its step result.""" result = self.tree_core.drop_subtree_no_host(node_id) - self._free_values(result.device_frees, result.host_frees) + self._apply_eviction_result(result) if result.is_dropped: self._record_dropped_tokens( result.tracker.get(BASE_COMPONENT_TYPE, 0), reason="host_pressure" @@ -1072,7 +1088,7 @@ class UnifiedRadixCache(BasePrefixCache): if self.disable or lock is None or lock.swa_released: return result = self.tree_core.dec_swa_lock_only(lock.node, lock.receipt) - self._free_values(result.device_frees, result.host_frees) + self._apply_eviction_result(result) lock.swa_released = True def release_swa_prefix_lock(self, req: Req) -> None: @@ -1401,7 +1417,7 @@ class UnifiedRadixCache(BasePrefixCache): # is operation-owned (freed at each ack): nothing is evictable. return 0 result = self.tree_core.drive_host_eviction(component_type, num_tokens) - self._free_values(result.device_frees, result.host_frees) + self._apply_eviction_result(result) return result.tracker.get(component_type, 0) # ---- Decode retraction ---- diff --git a/python/sglang/test/kits/unified_radix_cache/core.py b/python/sglang/test/kits/unified_radix_cache/core.py index 38e8a43c39..7c62535431 100644 --- a/python/sglang/test/kits/unified_radix_cache/core.py +++ b/python/sglang/test/kits/unified_radix_cache/core.py @@ -51,7 +51,12 @@ class FakeFullComponent(TreeComponent): return None def evict_component( - self, node, device_frees, host_frees, target: EvictLayer = EvictLayer.DEVICE + self, + node, + device_frees, + host_frees, + cache_actions, + target: EvictLayer = EvictLayer.DEVICE, ) -> tuple[int, int]: return 0, 0 @@ -64,7 +69,7 @@ class FakeFullComponent(TreeComponent): def _evict_device_start(self, request_cnt) -> None: pass - def _evict_device_next_node(self, tracker, device_frees, host_frees): + def _evict_device_next_node(self, tracker, device_frees, host_frees, cache_actions): return None def _evict_device_end(self) -> None: diff --git a/python/sglang/test/kits/unified_radix_cache/rust_unified_tree_core_inspector.py b/python/sglang/test/kits/unified_radix_cache/rust_unified_tree_core_inspector.py index abe3b6fa5f..1c15549923 100644 --- a/python/sglang/test/kits/unified_radix_cache/rust_unified_tree_core_inspector.py +++ b/python/sglang/test/kits/unified_radix_cache/rust_unified_tree_core_inspector.py @@ -9,8 +9,6 @@ import torch from sglang.srt.mem_cache.base_prefix_cache import MatchPrefixParams, MatchResult from sglang.srt.mem_cache.rust_tree_core.adapter import ( RustUnifiedTreeCore, - _fill_evict_result, - _match_result_from_binding, _radix_key_buffer, ) from sglang.srt.mem_cache.rust_tree_core.extension import load_tree_core_extension @@ -189,7 +187,7 @@ class RustUnifiedTreeCoreInspector( binding_result = self._binding.inspect_evict_component( node_id, int(component_type), int(target) ) - return _fill_evict_result(binding_result, BaseEvictionResult()) + return self._fill_evict_result(binding_result, BaseEvictionResult()) def validate_cascade_evict( self, @@ -203,7 +201,7 @@ class RustUnifiedTreeCoreInspector( def cleanup_tombstone_ancestors(self, node_id: NodeId) -> BaseEvictionResult: binding_result = self._binding.inspect_cleanup_tombstone_ancestors(node_id) - return _fill_evict_result(binding_result, BaseEvictionResult()) + return self._fill_evict_result(binding_result, BaseEvictionResult()) def finalize_component_match_result( self, @@ -222,7 +220,7 @@ class RustUnifiedTreeCoreInspector( value_chunks, best_value_len, ) - return _match_result_from_binding(binding_result)._replace( + return self._match_result_from_binding(binding_result)._replace( cache_protected_len=result.cache_protected_len, cache_actions=result.cache_actions, ) diff --git a/python/sglang/test/kits/unified_radix_cache/unified_tree_core_inspector.py b/python/sglang/test/kits/unified_radix_cache/unified_tree_core_inspector.py index 147a3ad153..eb45304f1a 100644 --- a/python/sglang/test/kits/unified_radix_cache/unified_tree_core_inspector.py +++ b/python/sglang/test/kits/unified_radix_cache/unified_tree_core_inspector.py @@ -223,6 +223,7 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa self.components_by_type[component_type], result.device_frees, result.host_frees, + result.cache_actions, target, result.tracker, ) @@ -249,6 +250,7 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa result.tracker, result.device_frees, result.host_frees, + result.cache_actions, ) return result diff --git a/rust/sglang-radix-tree/src/components/full.rs b/rust/sglang-radix-tree/src/components/full.rs index acfe6c3501..f270076776 100644 --- a/rust/sglang-radix-tree/src/components/full.rs +++ b/rust/sglang-radix-tree/src/components/full.rs @@ -104,14 +104,15 @@ impl TreeComponent for FullComponent { node_id: NodeIdx_, device_frees: &mut HashMap>, host_frees: &mut HashMap>, + cache_actions: &mut Vec, target: EvictLayer, ) -> (usize, usize) { - let node = tree_core.arena.node_mut(node_id); + let node = tree_core.arena.node(node_id); let mut freed = 0; let mut host_freed = 0; if target.contains(EvictLayer::Device) && node.has_device_value(FULL) { let value = node.device_value(FULL); - freed = node.device_reclaimable_value_len(FULL); + freed = tree_core.component_reclaimable_value_len_(node_id, FULL); device_frees .entry(FULL) .or_default() @@ -124,7 +125,7 @@ impl TreeComponent for FullComponent { host_frees .entry(FULL) .or_default() - .push(node.take_host_value(FULL)); + .push(tree_core.arena.node_mut(node_id).take_host_value(FULL)); } if freed > 0 { tree_core.dec_evictable_size(FULL, freed); @@ -155,6 +156,7 @@ impl TreeComponent for FullComponent { tracker: &mut HashMap, _device_frees: &mut HashMap>, _host_frees: &mut HashMap>, + _cache_actions: &mut Vec, ) -> Option { let ct = FULL; assert!( @@ -203,6 +205,7 @@ impl TreeComponent for FullComponent { tracker: &mut HashMap, device_frees: &mut HashMap>, host_frees: &mut HashMap>, + cache_actions: &mut Vec, ) { for spare_imminent_demotes in [true, false] { if tracker[&FULL] >= num_tokens { @@ -230,6 +233,7 @@ impl TreeComponent for FullComponent { tracker, device_frees, host_frees, + cache_actions, ); tree_core.full_coexisting_host_nodes.discard(node_id); } @@ -244,6 +248,7 @@ impl TreeComponent for FullComponent { tracker: &mut HashMap, device_frees: &mut HashMap>, host_frees: &mut HashMap>, + cache_actions: &mut Vec, ) { let ct = FULL; let arena = &tree_core.arena; @@ -262,7 +267,7 @@ impl TreeComponent for FullComponent { } // The parent id is captured before the eviction frees the leaf. let parent = tree_core.arena.node(x).try_parent(); - tree_core.evict_host_leaf_(x, tracker, device_frees, host_frees); + tree_core.evict_host_leaf_(x, tracker, device_frees, host_frees, cache_actions); if let Some(parent) = parent && tree_core.evictable_host_leaves.contains(parent) { @@ -299,18 +304,19 @@ impl TreeComponent for FullComponent { // lock credits protected directly. let mut cur = node_id; loop { - let node = tree_core.arena.node_mut(cur); + let node = tree_core.arena.node(cur); if node.is_root() || node.has_device_value(FULL) { break; } - node.inc_device_lock_ref(FULL); - cur = node.parent(); + let parent = node.parent(); + tree_core.arena.node_mut(cur).inc_device_lock_ref(FULL); + cur = parent; } // Lock the device-on segment up to the root. let mut delta = 0; loop { - let node = tree_core.arena.node_mut(cur); + let node = tree_core.arena.node(cur); if node.is_root() { break; } @@ -320,11 +326,11 @@ impl TreeComponent for FullComponent { ); let parent = node.parent(); let newly_locked_len = if node.device_lock_ref(FULL) == 0 { - Some(node.device_reclaimable_value_len(FULL)) + Some(tree_core.component_reclaimable_value_len_(cur, FULL)) } else { None }; - node.inc_device_lock_ref(FULL); + tree_core.arena.node_mut(cur).inc_device_lock_ref(FULL); if let Some(key_len) = newly_locked_len { tree_core.dec_evictable_size(FULL, key_len); tree_core.inc_protected_size(FULL, key_len); @@ -387,7 +393,7 @@ impl TreeComponent for FullComponent { let mut cur = node_id; loop { - let node = tree_core.arena.node_mut(cur); + let node = tree_core.arena.node(cur); if node.is_root() { break; } @@ -398,11 +404,11 @@ impl TreeComponent for FullComponent { "FULL segment release hit lock_ref=0 on node {cur}" ); let newly_unlocked_len = if old_lock_ref == 1 { - Some(node.device_reclaimable_value_len(FULL)) + Some(tree_core.component_reclaimable_value_len_(cur, FULL)) } else { None }; - node.dec_device_lock_ref(FULL); + tree_core.arena.node_mut(cur).dec_device_lock_ref(FULL); if let Some(key_len) = newly_unlocked_len { tree_core.dec_protected_size(FULL, key_len); tree_core.inc_evictable_size(FULL, key_len); @@ -514,7 +520,7 @@ impl TreeComponent for FullComponent { loaded .set_device_value(FULL, device_indices.narrow(0, offset, n_len).copy()); let locked = loaded.device_lock_ref(FULL) > 0; - let tokens = loaded.device_reclaimable_value_len(FULL); + let tokens = tree_core.component_reclaimable_value_len_(loaded_idx, FULL); offset += n_len; // Full uses leaf sets, not LRU. A value materialized // under lock is protected; the last release moves it diff --git a/rust/sglang-radix-tree/src/components/mamba.rs b/rust/sglang-radix-tree/src/components/mamba.rs index 03ee89d7d0..1bd53935ac 100644 --- a/rust/sglang-radix-tree/src/components/mamba.rs +++ b/rust/sglang-radix-tree/src/components/mamba.rs @@ -172,10 +172,7 @@ impl TreeComponent for MambaComponent { .arena .set_device_value(node_id, MAMBA, mamba_value.shallow_clone()); tree_core.device_lru_list_mut(MAMBA).insert_mru(node_id); - let tokens = tree_core - .arena - .node(node_id) - .device_reclaimable_value_len(MAMBA); + let tokens = tree_core.component_reclaimable_value_len_(node_id, MAMBA); tree_core.inc_evictable_size(MAMBA, tokens); self.emit_excess_path_states_eviction_(tree_core.arena.node(node_id).id, cache_actions); return; @@ -203,6 +200,7 @@ impl TreeComponent for MambaComponent { tail_node_id: NodeIdx_, device_frees: &mut HashMap>, host_frees: &mut HashMap>, + cache_actions: &mut Vec, ) { let Some(cap) = self.mamba_max_states_per_path else { return; @@ -242,6 +240,7 @@ impl TreeComponent for MambaComponent { MAMBA, device_frees, host_frees, + cache_actions, EvictLayer::Device, Some(&mut tracker), ); @@ -251,6 +250,7 @@ impl TreeComponent for MambaComponent { &mut tracker, device_frees, host_frees, + cache_actions, EvictLayer::Device, ); excess -= 1; @@ -282,20 +282,21 @@ impl TreeComponent for MambaComponent { node_id: NodeIdx_, device_frees: &mut HashMap>, host_frees: &mut HashMap>, + cache_actions: &mut Vec, target: EvictLayer, ) -> (usize, usize) { let ct = MAMBA; - let node = tree_core.arena.node_mut(node_id); + let node = tree_core.arena.node(node_id); let mut freed = 0; let mut host_freed = 0; // Device layer if target.contains(EvictLayer::Device) && node.has_device_value(MAMBA) { - freed = node.device_reclaimable_value_len(MAMBA); + freed = tree_core.component_reclaimable_value_len_(node_id, MAMBA); device_frees .entry(ct) .or_default() - .push(node.take_device_value(MAMBA)); + .push(tree_core.arena.node_mut(node_id).take_device_value(MAMBA)); tree_core.dec_evictable_size(MAMBA, freed); } @@ -347,6 +348,7 @@ impl TreeComponent for MambaComponent { tracker: &mut HashMap, device_frees: &mut HashMap>, host_frees: &mut HashMap>, + cache_actions: &mut Vec, ) -> Option { let ct = MAMBA; assert!( @@ -412,10 +414,19 @@ impl TreeComponent for MambaComponent { ct, device_frees, host_frees, + cache_actions, EvictLayer::Device, Some(tracker), ); - tree_core.cascade_evict_(x, ct, tracker, device_frees, host_frees, EvictLayer::Device); + tree_core.cascade_evict_( + x, + ct, + tracker, + device_frees, + host_frees, + cache_actions, + EvictLayer::Device, + ); None }; tree_core.component_state_mut(MAMBA).evict_device_cursor = cursor; @@ -452,7 +463,7 @@ impl TreeComponent for MambaComponent { } tree_core.arena.inc_host_lock_ref(node_id, MAMBA); } else { - let value_len = node.device_reclaimable_value_len(MAMBA); + let value_len = tree_core.component_reclaimable_value_len_(node_id, MAMBA); if node.device_lock_ref(MAMBA) == 0 { tree_core.dec_evictable_size(MAMBA, value_len); tree_core.inc_protected_size(MAMBA, value_len); @@ -498,7 +509,7 @@ impl TreeComponent for MambaComponent { "Mamba release hit lock_ref=0 on node {node_id}" ); if device_lock_ref == 1 { - let value_len = node.device_reclaimable_value_len(MAMBA); + let value_len = tree_core.component_reclaimable_value_len_(node_id, MAMBA); tree_core.inc_evictable_size(MAMBA, value_len); tree_core.dec_protected_size(MAMBA, value_len); } @@ -722,6 +733,7 @@ impl TreeComponent for MambaComponent { tracker: &mut HashMap, device_frees: &mut HashMap>, host_frees: &mut HashMap>, + cache_actions: &mut Vec, ) { let ct = MAMBA; let mut x = tree_core @@ -744,7 +756,7 @@ impl TreeComponent for MambaComponent { // cover an ancestor whose Mamba state is not being transferred. if tree_core.evictable_host_leaves.contains(cur) { // Host leaf: atomic eviction (all components host + delete) - tree_core.evict_host_leaf_(cur, tracker, device_frees, host_frees); + tree_core.evict_host_leaf_(cur, tracker, device_frees, host_frees, cache_actions); } else { // Internal: tombstone Mamba + cascade assert!( @@ -756,6 +768,7 @@ impl TreeComponent for MambaComponent { ct, device_frees, host_frees, + cache_actions, EvictLayer::Host, Some(tracker), ); @@ -765,6 +778,7 @@ impl TreeComponent for MambaComponent { tracker, device_frees, host_frees, + cache_actions, EvictLayer::Host, ); } diff --git a/rust/sglang-radix-tree/src/components/mod.rs b/rust/sglang-radix-tree/src/components/mod.rs index dfeeb459eb..3ee0b76194 100644 --- a/rust/sglang-radix-tree/src/components/mod.rs +++ b/rust/sglang-radix-tree/src/components/mod.rs @@ -5,7 +5,7 @@ use std::collections::HashMap; use tch::Tensor; -use crate::node::{ChildKeyType, NodeArena, NodeIdx_, TreeCoreRuntimeError}; +use crate::node::{ChildKeyType, Node, NodeArena, NodeId, NodeIdx_, TreeCoreRuntimeError}; use crate::unified_tree_core::{ CacheAction, CacheTransferPhase, DecLockRefParams, EvictLayer, IncLockRefResult, InsertParams, InsertResult, LRURefreshPhase, MatchPrefixParams, MatchResult, PoolTransfer, @@ -60,6 +60,15 @@ pub trait TreeComponent { /// The component this driver serves. fn component_type(&self) -> ComponentType; + /// Allocator slots reclaimed from this component's device value. + /// Roots and absent device values contribute zero; splits preserve the total. + fn reclaimable_value_len(&self, node: &Node) -> usize { + node.device_reclaimable_value_len(self.component_type()) + } + + /// Drop component-owned metadata after the node is removed. + fn on_node_removed(&self, _node_id: NodeId) {} + /// Return the node's lock-segment UUID, minting and storing one when absent. fn get_or_fill_uuid( &self, @@ -174,6 +183,7 @@ pub trait TreeComponent { } /// Called per-node when an insert's key overlaps an existing node. + /// `preserve_full_ids` keeps canonical FULL IDs during auxiliary recovery. /// Returns the index within `value_slice` from which this component /// consumed (took ownership of) the underlying KV pool slots. /// Returns `prefix_len` if nothing was consumed (default). @@ -187,6 +197,7 @@ pub trait TreeComponent { total_prefix_len: usize, value_slice: Tensor, params: &InsertParams<'_, K>, + preserve_full_ids: bool, result: &mut InsertResult, cache_actions: &mut Vec, ) -> usize { @@ -240,6 +251,7 @@ pub trait TreeComponent { tail_node_id: NodeIdx_, device_frees: &mut HashMap>, host_frees: &mut HashMap>, + cache_actions: &mut Vec, ) { } @@ -295,6 +307,7 @@ pub trait TreeComponent { node_id: NodeIdx_, device_frees: &mut HashMap>, host_frees: &mut HashMap>, + cache_actions: &mut Vec, target: EvictLayer, ) -> (usize, usize); @@ -317,6 +330,7 @@ pub trait TreeComponent { tracker: &mut HashMap, device_frees: &mut HashMap>, host_frees: &mut HashMap>, + cache_actions: &mut Vec, ) -> Option; /// Clear this component's device-eviction walk state. @@ -392,6 +406,7 @@ pub trait TreeComponent { _params: &DecLockRefParams, _device_frees: &mut HashMap>, _host_frees: &mut HashMap>, + _cache_actions: &mut Vec, ) { unimplemented!("release_window_lock is SWA-only") } @@ -474,6 +489,7 @@ pub trait TreeComponent { _tracker: &mut HashMap, _device_frees: &mut HashMap>, _host_frees: &mut HashMap>, + _cache_actions: &mut Vec, ) { } @@ -487,6 +503,7 @@ pub trait TreeComponent { _tracker: &mut HashMap, _device_frees: &mut HashMap>, _host_frees: &mut HashMap>, + _cache_actions: &mut Vec, ) { } } diff --git a/rust/sglang-radix-tree/src/components/swa.rs b/rust/sglang-radix-tree/src/components/swa.rs index c2f34b7829..27ed86472c 100644 --- a/rust/sglang-radix-tree/src/components/swa.rs +++ b/rust/sglang-radix-tree/src/components/swa.rs @@ -439,6 +439,7 @@ impl TreeComponent for SwaComponent { total_prefix_len: usize, value_slice: Tensor, params: &InsertParams<'_, K>, + preserve_full_ids: bool, result: &mut InsertResult, cache_actions: &mut Vec, ) -> usize { @@ -465,7 +466,7 @@ impl TreeComponent for SwaComponent { if swa_evicted_seqlen <= total_prefix_len { // Branch 1: entire value_slice is within SWA window — recover result.record_adopted_range(SWA, total_prefix_len, total_prefix_len + prefix_len); - if node.device_lock_ref(FULL) > 0 { + if node.device_lock_ref(FULL) > 0 || preserve_full_ids { cache_actions.push(CacheAction::RecoverSwaWithLockedFull { node_id: node.id, kept_full: node.device_value(FULL).shallow_clone(), @@ -487,7 +488,7 @@ impl TreeComponent for SwaComponent { let start_idx = swa_evicted_seqlen - total_prefix_len; result.record_adopted_range(SWA, swa_evicted_seqlen, total_prefix_len + prefix_len); let node_ext_id = node.id; - let is_locked = node.device_lock_ref(FULL) > 0; + let preserve_full = node.device_lock_ref(FULL) > 0 || preserve_full_ids; let full_len = node.device_value_len(FULL); let old_full = node.device_value(FULL) @@ -497,7 +498,7 @@ impl TreeComponent for SwaComponent { cache_actions.push(action); } let new_full = value_slice.narrow(0, start_idx as i64, (prefix_len - start_idx) as i64); - if is_locked { + if preserve_full { cache_actions.push(CacheAction::RecoverSwaWithLockedFull { node_id: node_ext_id, kept_full: old_full, @@ -670,10 +671,11 @@ impl TreeComponent for SwaComponent { node_id: NodeIdx_, device_frees: &mut HashMap>, host_frees: &mut HashMap>, + cache_actions: &mut Vec, target: EvictLayer, ) -> (usize, usize) { let ct = SWA; - let node = tree_core.arena.node_mut(node_id); + let node = tree_core.arena.node(node_id); let mut freed = 0; let mut host_freed = 0; @@ -686,8 +688,8 @@ impl TreeComponent for SwaComponent { .entry(ct) .or_default() .push(node.device_value(FULL).shallow_clone()); - freed = node.device_reclaimable_value_len(SWA); - let _ = node.take_device_value(SWA); + freed = tree_core.component_reclaimable_value_len_(node_id, SWA); + let _ = tree_core.arena.node_mut(node_id).take_device_value(SWA); tree_core.dec_evictable_size(SWA, freed); } @@ -739,6 +741,7 @@ impl TreeComponent for SwaComponent { tracker: &mut HashMap, device_frees: &mut HashMap>, host_frees: &mut HashMap>, + cache_actions: &mut Vec, ) -> Option { let ct = SWA; assert!( @@ -812,10 +815,19 @@ impl TreeComponent for SwaComponent { ct, device_frees, host_frees, + cache_actions, EvictLayer::Device, Some(tracker), ); - tree_core.cascade_evict_(x, ct, tracker, device_frees, host_frees, EvictLayer::Device); + tree_core.cascade_evict_( + x, + ct, + tracker, + device_frees, + host_frees, + cache_actions, + EvictLayer::Device, + ); None }; tree_core.component_state_mut(SWA).evict_device_cursor = cursor; @@ -836,6 +848,7 @@ impl TreeComponent for SwaComponent { tracker: &mut HashMap, device_frees: &mut HashMap>, host_frees: &mut HashMap>, + cache_actions: &mut Vec, ) { let ct = SWA; let mut x = tree_core @@ -857,7 +870,7 @@ impl TreeComponent for SwaComponent { // The host LRU filters SWA's own locks. A Full load-back pin may // cover an ancestor outside the SWA transfer window. if tree_core.evictable_host_leaves.contains(cur) { - tree_core.evict_host_leaf_(cur, tracker, device_frees, host_frees); + tree_core.evict_host_leaf_(cur, tracker, device_frees, host_frees, cache_actions); } else { assert!( tree_core.arena.has_host_value(cur, SWA), @@ -868,6 +881,7 @@ impl TreeComponent for SwaComponent { ct, device_frees, host_frees, + cache_actions, EvictLayer::Host, Some(tracker), ); @@ -877,6 +891,7 @@ impl TreeComponent for SwaComponent { tracker, device_frees, host_frees, + cache_actions, EvictLayer::Host, ); } @@ -1169,16 +1184,16 @@ impl TreeComponent for SwaComponent { let mut cur = node_id; loop { - let node = tree_core.arena.node_mut(cur); + let node = tree_core.arena.node(cur); if node.is_root() || covered >= sliding_window_size { break; } let parent = node.parent(); let key_len = node.key.atom_len(); let has_value = Self::has_value(node, lock_host); - let value_len = node.device_reclaimable_value_len(SWA); + let value_len = tree_core.component_reclaimable_value_len_(cur, SWA); let newly_locked = Self::lock_ref(node, lock_host) == 0; - Self::inc_lock_ref(node, lock_host); + Self::inc_lock_ref(tree_core.arena.node_mut(cur), lock_host); if newly_locked && has_value { if lock_host { let host_lru = tree_core.host_lru_list_mut(SWA); @@ -1215,7 +1230,7 @@ impl TreeComponent for SwaComponent { let mut cur = node_id; loop { - let node = tree_core.arena.node_mut(cur); + let node = tree_core.arena.node(cur); if node.is_root() { break; } @@ -1229,7 +1244,7 @@ impl TreeComponent for SwaComponent { if lock_host { "host_" } else { "" } ); let has_value = Self::has_value(node, lock_host); - let value_len = node.device_reclaimable_value_len(SWA); + let value_len = tree_core.component_reclaimable_value_len_(cur, SWA); if lock_ref == 1 && has_value { if lock_host { if !node.has_device_value(SWA) { @@ -1274,6 +1289,7 @@ impl TreeComponent for SwaComponent { params: &DecLockRefParams, device_frees: &mut HashMap>, host_frees: &mut HashMap>, + cache_actions: &mut Vec, ) { if tree_core.arena.node(node_id).is_root() { return; @@ -1282,7 +1298,7 @@ impl TreeComponent for SwaComponent { let ct = SWA; let mut cur = node_id; loop { - let node = tree_core.arena.node_mut(cur); + let node = tree_core.arena.node(cur); if node.is_root() { break; } @@ -1292,9 +1308,9 @@ impl TreeComponent for SwaComponent { "SWA window release hit lock_ref=0 on node {cur}" ); let has_value = node.has_device_value(SWA); - let value_len = node.device_reclaimable_value_len(SWA); - node.dec_device_lock_ref(SWA); - let now_unlocked = node.device_lock_ref(SWA) == 0; + let value_len = tree_core.component_reclaimable_value_len_(cur, SWA); + let now_unlocked = node.device_lock_ref(SWA) == 1; + tree_core.arena.node_mut(cur).dec_device_lock_ref(SWA); if now_unlocked { tree_core.update_evictable_leaf_sets_(cur); } @@ -1307,6 +1323,7 @@ impl TreeComponent for SwaComponent { ct, device_frees, host_frees, + cache_actions, EvictLayer::Device, /* tracker = */ None, ); diff --git a/rust/sglang-radix-tree/src/node.rs b/rust/sglang-radix-tree/src/node.rs index ca2a20f1fb..82eec9627f 100644 --- a/rust/sglang-radix-tree/src/node.rs +++ b/rust/sglang-radix-tree/src/node.rs @@ -765,6 +765,9 @@ pub enum TreeCoreRuntimeError { /// `resume_insert` called without a suspended insert. #[error("no in-flight insert")] NoInFlightInsert, + /// Overlap preparation must complete before incoming values can be consumed. + #[error("invalid insert overlap: {0}")] + InvalidInsertOverlap(&'static str), /// A `NodeIdx_` beyond the arena's bounds — never allocated. `size` is the /// arena's current slot count, so valid ids are `0..size`. #[error("node access out of bounds: id {id} not in [0, {size})")] diff --git a/rust/sglang-radix-tree/src/python_bindings.rs b/rust/sglang-radix-tree/src/python_bindings.rs index 405cd94591..ca39548c5a 100644 --- a/rust/sglang-radix-tree/src/python_bindings.rs +++ b/rust/sglang-radix-tree/src/python_bindings.rs @@ -889,6 +889,7 @@ pub struct EvictDeviceNextNodeResultBinding { tracker: HashMap, new_device_frees: Py, new_host_frees: Py, + cache_actions: Py, } /// Leaf-eviction step result: the backup action for an unbacked @@ -901,6 +902,7 @@ pub struct EvictDeviceLeafResultBinding { tracker: HashMap, new_device_frees: Py, new_host_frees: Py, + cache_actions: Py, } /// Drop-subtree result: whether the drop happened, this step's @@ -911,6 +913,7 @@ pub struct DropSubtreeResultBinding { tracker: HashMap, new_device_frees: Py, new_host_frees: Py, + cache_actions: Py, } /// Demote result: this step's per-component evicted counts and the @@ -920,6 +923,7 @@ pub struct DemoteResultBinding { tracker: HashMap, new_device_frees: Py, new_host_frees: Py, + cache_actions: Py, } /// Python-visible device->storage backup spec. @@ -1001,6 +1005,7 @@ pub struct HostEvictionResultBinding { tracker: HashMap, new_device_frees: Py, new_host_frees: Py, + cache_actions: Py, } impl HostEvictionResultBinding { @@ -1009,6 +1014,7 @@ impl HostEvictionResultBinding { tracker: tracker_to_py(result.tracker), new_device_frees: frees_to_py(py, result.device_frees)?, new_host_frees: frees_to_py(py, result.host_frees)?, + cache_actions: cache_actions_to_py(py, result.cache_actions)?, }) } } @@ -1017,8 +1023,8 @@ type TreePathNodeInfoTuple = (NodeId, NodeId, usize, usize, Option, usize /// The generic UnifiedTreeCore adapter the per-key-type pyclasses delegate to; /// the Mutex makes the Send-only core satisfy pyclass's Sync bound. -struct TreeCoreBinding { - core: Mutex>, +struct TreeCoreBinding { + core: Mutex, /// The core's construction device, kept outside the Mutex for pre-lock validation. device: Device, /// The core's page size, kept outside the Mutex for pre-lock validation. @@ -1026,1753 +1032,1760 @@ struct TreeCoreBinding { } // Send + Sync lets detach release the GIL around core calls. -impl TreeCoreBinding { - /// Build a tree core for the given component types from the cache's - /// init params. - fn new(init_params: &TreeCoreInitParamsBinding, component_types: Vec) -> PyResult { - let component_types = component_types - .into_iter() - .map(parse_component_type) - .collect::>>()?; - if component_types != [FULL] - && component_types != [FULL, SWA] - && component_types != [FULL, MAMBA] - && component_types != [FULL, SWA, MAMBA] - { - return Err(PyValueError::new_err( - "only the [Full], [Full, Swa], [Full, Mamba], and [Full, Swa, Mamba] component sets are supported", - )); - } - if component_types.contains(&SWA) && init_params.swa_sliding_window_size.is_none() { - return Err(PyValueError::new_err( - "the Swa component requires swa_sliding_window_size", - )); - } - if component_types.contains(&MAMBA) && init_params.mamba_cache_chunk_size.is_none() { - return Err(PyValueError::new_err( - "the Mamba component requires mamba_cache_chunk_size", - )); - } - if init_params.page_size == 0 { - return Err(PyValueError::new_err("page_size must be at least 1")); - } - let eviction_policy = init_params.eviction_policy.to_lowercase(); - if !matches!( - eviction_policy.as_str(), - "lru" | "lfu" | "fifo" | "mru" | "filo" | "priority" | "slru" | "tlru" - ) { - return Err(PyValueError::new_err(format!( - "Unknown eviction policy: {eviction_policy}. Supported policies: \ - 'lru', 'lfu', 'fifo', 'mru', 'filo', 'priority', 'slru', 'tlru'." - ))); - } - let params = init_params.to_cache_init_params()?; - let device = params.device; - let page_size = params.page_size; - Ok(TreeCoreBinding { - core: Mutex::new(UnifiedTreeCore::new(params, component_types)), - device, - page_size, - }) - } +macro_rules! tree_core_methods { + ($core:ident) => { + impl TreeCoreBinding<$core> { + /// Build a tree core for the given component types from the cache's + /// init params. + fn new(init_params: &TreeCoreInitParamsBinding, component_types: Vec) -> PyResult { + let component_types = component_types + .into_iter() + .map(parse_component_type) + .collect::>>()?; + let params = init_params.to_cache_init_params()?; + $core::::validate_init_params(¶ms, &component_types) + .map_err(PyValueError::new_err)?; + let device = params.device; + let page_size = params.page_size; + Ok(TreeCoreBinding { + core: Mutex::new($core::new(params, component_types)), + device, + page_size, + }) + } - /// Lock the core for one adapter call. A panic can leave a mutation half-applied, - /// so a poisoned core is never reused. - fn core(&self) -> std::sync::MutexGuard<'_, UnifiedTreeCore> { - self.core.lock().unwrap_or_else(|_| { - panic!("Rust TreeCore mutex poisoned; refusing to reuse state after an earlier panic") - }) - } + /// Lock the core for one adapter call. A panic can leave a mutation half-applied, + /// so a poisoned core is never reused. + fn core(&self) -> std::sync::MutexGuard<'_, $core> { + self.core.lock().unwrap_or_else(|_| { + panic!("Rust TreeCore mutex poisoned; refusing to reuse state after an earlier panic") + }) + } - /// Reject an insert value whose dtype, device, or length cannot cover the key. - fn validate_insert_value(&self, value: &Tensor, key_atom_len: usize) -> PyResult<()> { - if value.kind() != Kind::Int64 { - return Err(PyValueError::new_err(format!( - "insert value must be an int64 tensor, got {:?}", - value.kind() - ))); - } - if value.device() != self.device { - return Err(PyValueError::new_err(format!( - "insert value device {:?} does not match the tree core device {:?}", - value.device(), - self.device - ))); - } - let aligned_key_len = key_atom_len / self.page_size * self.page_size; - let value_len = value.size().first().copied().unwrap_or(0); - if value_len < aligned_key_len as i64 { - return Err(PyValueError::new_err(format!( - "insert value length {value_len} is shorter than the aligned key length {aligned_key_len}" - ))); - } - Ok(()) - } + /// Reject an insert value whose dtype, device, or length cannot cover the key. + fn validate_insert_value(&self, value: &Tensor, key_atom_len: usize) -> PyResult<()> { + if value.kind() != Kind::Int64 { + return Err(PyValueError::new_err(format!( + "insert value must be an int64 tensor, got {:?}", + value.kind() + ))); + } + if value.device() != self.device { + return Err(PyValueError::new_err(format!( + "insert value device {:?} does not match the tree core device {:?}", + value.device(), + self.device + ))); + } + let aligned_key_len = key_atom_len / self.page_size * self.page_size; + let value_len = value.size().first().copied().unwrap_or(0); + if value_len < aligned_key_len as i64 { + return Err(PyValueError::new_err(format!( + "insert value length {value_len} is shorter than the aligned key length {aligned_key_len}" + ))); + } + Ok(()) + } - /// Drop the entire tree and reinitialize empty state. - fn reset(&self, py: Python<'_>) { - py.detach(|| self.core().reset()); - } + /// Drop the entire tree and reinitialize empty state. + fn reset(&self, py: Python<'_>) { + py.detach(|| self.core().reset()); + } - /// Match a key against the tree. - fn match_prefix( - &self, - py: Python<'_>, - params: &MatchParamsBinding, - ) -> PyResult { - let key = K::key_from(Cow::Borrowed(¶ms.key)); - let key = key.as_ref(); - let params = MatchPrefixParams { - key, - namespace: KeyNamespaceRef::new( - params.extra_key.as_deref(), - params.cache_salt.as_deref(), - ), - }; - let result = py.detach(|| self.core().match_prefix(¶ms)); - MatchResultBinding::from_match_result(py, result) - } + /// Match a key against the tree. + fn match_prefix( + &self, + py: Python<'_>, + params: &MatchParamsBinding, + ) -> PyResult { + let key = K::key_from(Cow::Borrowed(¶ms.key)); + let key = key.as_ref(); + let params = MatchPrefixParams { + key, + namespace: KeyNamespaceRef::new( + params.extra_key.as_deref(), + params.cache_salt.as_deref(), + ), + }; + let result = py.detach(|| self.core().match_prefix(¶ms)); + MatchResultBinding::from_match_result(py, result) + } - /// Read-only FULL-device match, independent of auxiliary components. - fn match_full_device_prefix( - &self, - py: Python<'_>, - params: &MatchParamsBinding, - ) -> (usize, NodeId, usize) { - let key = K::key_from(Cow::Borrowed(¶ms.key)); - let key = key.as_ref(); - let namespace = - KeyNamespaceRef::new(params.extra_key.as_deref(), params.cache_salt.as_deref()); - py.detach(|| self.core().match_full_device_prefix(key, namespace)) - } + /// Read-only FULL-device match, independent of auxiliary components. + fn match_full_device_prefix( + &self, + py: Python<'_>, + params: &MatchParamsBinding, + ) -> (usize, NodeId, usize) { + let key = K::key_from(Cow::Borrowed(¶ms.key)); + let key = key.as_ref(); + let namespace = + KeyNamespaceRef::new(params.extra_key.as_deref(), params.cache_salt.as_deref()); + py.detach(|| self.core().match_full_device_prefix(key, namespace)) + } - fn swa_tombstone_ranges( - &self, - py: Python<'_>, - params: &MatchParamsBinding, - start: usize, - end: usize, - ) -> PyResult> { - let key = K::key_from(Cow::Borrowed(¶ms.key)); - let namespace = - KeyNamespaceRef::new(params.extra_key.as_deref(), params.cache_salt.as_deref()); - py.detach(|| { - self.core() - .swa_tombstone_ranges(key.as_ref(), namespace, start, end) - }) - .map_err(PyAssertionError::new_err) - } + fn swa_tombstone_ranges( + &self, + py: Python<'_>, + params: &MatchParamsBinding, + start: usize, + end: usize, + ) -> PyResult> { + let key = K::key_from(Cow::Borrowed(¶ms.key)); + let namespace = + KeyNamespaceRef::new(params.extra_key.as_deref(), params.cache_salt.as_deref()); + py.detach(|| { + self.core() + .swa_tombstone_ranges(key.as_ref(), namespace, start, end) + }) + .map_err(PyAssertionError::new_err) + } - fn attach_swa_window( - &self, - py: Python<'_>, - params: &MatchParamsBinding, - window_start: usize, - window_end: usize, - swa_values: PyTensor, - ) -> PyResult> { - let key = K::key_from(Cow::Borrowed(¶ms.key)); - let namespace = - KeyNamespaceRef::new(params.extra_key.as_deref(), params.cache_salt.as_deref()); - let swa_values = swa_values.0; - let actions = py - .detach(move || { - self.core().attach_swa_window( - key.as_ref(), - namespace, - window_start, - window_end, - &swa_values, - ) - }) - .map_err(PyAssertionError::new_err)?; - cache_actions_to_py(py, actions) - } - - /// The empty match result anchored at the root. - fn empty_match_result(&self, py: Python<'_>) -> PyResult { - let result = py.detach(|| self.core().empty_match_result()); - MatchResultBinding::from_match_result(py, result) - } - - /// Insert device values into the tree per the provided key. - fn insert( - &self, - py: Python<'_>, - params: &InsertParamsBinding, - ) -> PyResult { - let key = K::key_from(Cow::Borrowed(¶ms.key)); - let key = key.as_ref(); - let value: PyTensor = params.value.bind(py).extract()?; - // The value covers key atoms (bigram: raw len - 1), so validate the converted key. - self.validate_insert_value(&value.0, key.atom_len())?; - let mamba_value = match ¶ms.mamba_value { - Some(mamba_value) => Some(mamba_value.bind(py).extract::()?.0), - None => None, - }; - let params = InsertParams { - rotation_base: params.rotation_base, - key, - namespace: KeyNamespaceRef::new( - params.extra_key.as_deref(), - params.cache_salt.as_deref(), - ), - session_id: params.session_id.as_deref(), - value: value.0, - mamba_value, - prev_prefix_len: params.prev_prefix_len, - swa_evicted_seqlen: params.swa_evicted_seqlen, - swa_branching_seqlen: params.swa_branching_seqlen, - inserted_len: params.inserted_len, - priority: params.priority, - track_adopted_ranges: params.track_adopted_ranges, - }; - let result = py - .detach(move || self.core().try_insert(¶ms)) - .map_err(tree_core_runtime_error)?; - InsertResultBinding::from_insert_result(py, result) - } - - /// Start the resumable insert, running to its first barrier or completion. - fn begin_insert( - &self, - py: Python<'_>, - params: &InsertParamsBinding, - ) -> PyResult { - let key = K::key_from(Cow::Borrowed(¶ms.key)); - let key = key.as_ref(); - let value: PyTensor = params.value.bind(py).extract()?; - // The value covers key atoms (bigram: raw len - 1), so validate the converted key. - self.validate_insert_value(&value.0, key.atom_len())?; - let mamba_value = match ¶ms.mamba_value { - Some(mamba_value) => Some(mamba_value.bind(py).extract::()?.0), - None => None, - }; - let params = InsertParams { - rotation_base: params.rotation_base, - key, - namespace: KeyNamespaceRef::new( - params.extra_key.as_deref(), - params.cache_salt.as_deref(), - ), - session_id: params.session_id.as_deref(), - value: value.0, - mamba_value, - prev_prefix_len: params.prev_prefix_len, - swa_evicted_seqlen: params.swa_evicted_seqlen, - swa_branching_seqlen: params.swa_branching_seqlen, - inserted_len: params.inserted_len, - priority: params.priority, - track_adopted_ranges: params.track_adopted_ranges, - }; - let step = py - .detach(move || self.core().try_begin_insert(¶ms)) - .map_err(tree_core_runtime_error)?; - InsertStepResultBinding::from_insert_step(py, step) - } - - /// Continue the suspended insert after its step actions were applied. - fn resume_insert(&self, py: Python<'_>) -> PyResult { - let step = py - .detach(|| self.core().try_resume_insert()) - .map_err(tree_core_runtime_error)?; - InsertStepResultBinding::from_insert_step(py, step) - } - - /// Whether an insert walk is suspended at a barrier. - fn has_ongoing_insert(&self, py: Python<'_>) -> bool { - py.detach(|| self.core().has_ongoing_insert()) - } - - /// Finish the insert (idempotent); returns still-pending actions to drain. - fn end_insert(&self, py: Python<'_>) -> PyResult> { - let actions = py.detach(|| self.core().end_insert()); - cache_actions_to_py(py, actions) - } - - /// Bump the reference count on a node's component locks; the listed - /// components are left untaken and recorded in the receipt. - fn inc_lock_ref( - &self, - py: Python<'_>, - node_id: NodeId, - skip_lock_components: Vec, - ) -> PyResult { - let skip = component_set_from_py(&skip_lock_components)?; - let result = py - .detach(|| self.core().inc_lock_ref(node_id, skip)) - .map_err(node_access_error)?; - Ok(IncLockRefResultBinding::from_result(result)) - } - - /// Pin only the FULL device values on a node's root path. - fn inc_full_pin(&self, py: Python<'_>, node_id: NodeId) -> PyResult<()> { - py.detach(|| self.core().inc_full_pin(node_id)) - .map_err(node_access_error) - } - - /// Release a FULL-only root-path pin. - fn dec_full_pin(&self, py: Python<'_>, node_id: NodeId) -> PyResult<()> { - py.detach(|| self.core().dec_full_pin(node_id)) - .map_err(node_access_error) - } - - /// Decrease the reference count on a node's component locks. - fn dec_lock_ref( - &self, - py: Python<'_>, - node_id: NodeId, - params: &DecLockRefParamsBinding, - skip_swa: bool, - ) -> PyResult<()> { - let params = params.to_dec_lock_ref_params()?; - py.detach(|| self.core().dec_lock_ref(node_id, ¶ms, skip_swa)) - .map_err(node_access_error)?; - Ok(()) - } - - /// Release one window's lock, retaining other component locks. - fn dec_window_lock_only( - &self, - py: Python<'_>, - node_id: NodeId, - component_type: u8, - params: &DecLockRefParamsBinding, - ) -> PyResult<(Py, Py)> { - let component_type = parse_component_type(component_type)?; - let params = params.to_dec_lock_ref_params()?; - let (device_frees, host_frees) = py - .detach(|| { - let mut device_frees = HashMap::new(); - let mut host_frees = HashMap::new(); - self.core().dec_window_lock_only( - node_id, - component_type, - ¶ms, - &mut device_frees, - &mut host_frees, - )?; - Ok((device_frees, host_frees)) - }) - .map_err(node_access_error)?; - Ok((frees_to_py(py, device_frees)?, frees_to_py(py, host_frees)?)) - } - - /// Early-release the SWA portion of a request's tree lock; returns this - /// release's per-component (device_frees, host_frees). - fn dec_swa_lock_only( - &self, - py: Python<'_>, - node_id: NodeId, - params: &DecLockRefParamsBinding, - ) -> PyResult<(Py, Py)> { - let params = params.to_dec_lock_ref_params()?; - let (device_frees, host_frees) = py - .detach(|| { - let mut device_frees = HashMap::new(); - let mut host_frees = HashMap::new(); - self.core().dec_swa_lock_only( - node_id, - ¶ms, - &mut device_frees, - &mut host_frees, - )?; - Ok((device_frees, host_frees)) - }) - .map_err(node_access_error)?; - Ok((frees_to_py(py, device_frees)?, frees_to_py(py, host_frees)?)) - } - - /// Adjust the component's reclaimable device size. - fn adjust_component_size( - &self, - py: Python<'_>, - node_id: NodeId, - component_type: u8, - delta: i64, - ) -> PyResult<()> { - let component_type = parse_component_type(component_type)?; - py.detach(|| { - self.core() - .adjust_component_size(node_id, component_type, delta) - }) - .map_err(tree_core_runtime_error) - } - - /// Collect paths under one tree lock. - fn collect_path_info( - &self, - py: Python<'_>, - end_node_ids: Vec, - ) -> PyResult>> { - let paths = py - .detach(|| self.core().collect_path_info(&end_node_ids)) - .map_err(tree_core_runtime_error)?; - Ok(paths - .into_iter() - .map(|path| { - path.into_iter() - .map(|node| { - ( - node.node_id, - node.parent_id, - node.path_start, - node.path_end, - node.full_value_len, - node.num_children, + fn attach_swa_window( + &self, + py: Python<'_>, + params: &MatchParamsBinding, + window_start: usize, + window_end: usize, + swa_values: PyTensor, + ) -> PyResult> { + let key = K::key_from(Cow::Borrowed(¶ms.key)); + let namespace = + KeyNamespaceRef::new(params.extra_key.as_deref(), params.cache_salt.as_deref()); + let swa_values = swa_values.0; + let actions = py + .detach(move || { + self.core().attach_swa_window( + key.as_ref(), + namespace, + window_start, + window_end, + &swa_values, ) }) - .collect() - }) - .collect()) - } + .map_err(PyAssertionError::new_err)?; + cache_actions_to_py(py, actions) + } - /// Store a component's device value on a node (the SWA rebuild write-back). - fn set_component_device_value( - &self, - py: Python<'_>, - node_id: NodeId, - component_type: u8, - value: PyTensor, - ) -> PyResult<()> { - let component_type = parse_component_type(component_type)?; - let value = value.0; - if value.kind() != Kind::Int64 { - return Err(PyValueError::new_err(format!( - "component device value must be an int64 tensor, got {:?}", - value.kind() - ))); - } - if value.device() != self.device { - return Err(PyValueError::new_err(format!( - "component device value device {:?} does not match the tree core device {:?}", - value.device(), - self.device - ))); - } - py.detach(|| { - self.core() - .set_component_device_value(node_id, component_type, value) - }) - .map_err(node_access_error)?; - Ok(()) - } + /// The empty match result anchored at the root. + fn empty_match_result(&self, py: Python<'_>) -> PyResult { + let result = py.detach(|| self.core().empty_match_result()); + MatchResultBinding::from_match_result(py, result) + } - /// Logical key lengths for the nodes in a load-back transfer. - fn get_node_key_lengths(&self, py: Python<'_>, node_ids: Vec) -> PyResult> { - py.detach(|| self.core().get_node_key_lengths(&node_ids)) - .map_err(node_access_error) - } - /// A component's device value on a node, if set. - fn get_component_device_value( - &self, - py: Python<'_>, - node_id: NodeId, - component_type: u8, - ) -> PyResult>> { - let component_type = parse_component_type(component_type)?; - let value = py - .detach(|| { - self.core() - .get_component_device_value(node_id, component_type) - .map(|value| value.map(|tensor| tensor.shallow_clone())) - }) - .map_err(node_access_error)?; - value.map(|value| tensor_to_py(py, value)).transpose() - } - - // TODO(jialino): batch a full no-backup eviction round in Rust (one crossing - // instead of 2N+2); step-wise stays for the write-back interleave. - /// Begin a component's device-eviction walk for up to request_cnt tokens. - fn evict_device_start( - &self, - py: Python<'_>, - component_type: u8, - request_cnt: usize, - ) -> PyResult<()> { - let ct = parse_component_type(component_type)?; - py.detach(|| self.core().evict_device_start(ct, request_cnt)); - Ok(()) - } - - /// Advance one component eviction step. Progress without a leaf can be - /// an internal tombstone or a component backup request awaiting its finish; - /// no progress means the walk is exhausted. - fn evict_device_next_node( - &self, - py: Python<'_>, - component_type: u8, - tracker: HashMap, - ) -> PyResult { - let ct = parse_component_type(component_type)?; - let baseline = tracker_from_py(tracker)?; - let (node_id, result) = - py.detach(move || self.core().evict_device_next_node(ct, &baseline)); - let made_progress = node_id.is_some() - || result.mamba_backup_node_id.is_some() - || result.swa_backup_node_id.is_some() - || !result.tracker.is_empty() - || result.device_frees.values().any(|frees| !frees.is_empty()) - || result.host_frees.values().any(|frees| !frees.is_empty()); - Ok(EvictDeviceNextNodeResultBinding { - node_id, - mamba_backup_node_id: result.mamba_backup_node_id, - swa_backup_node_id: result.swa_backup_node_id, - swa_backup_num_tokens: result.swa_backup_num_tokens, - made_progress, - unbacked_tokens: result.unbacked_tokens, - tracker: tracker_to_py(result.tracker), - new_device_frees: frees_to_py(py, result.device_frees)?, - new_host_frees: frees_to_py(py, result.host_frees)?, - }) - } - - /// Resume a pending internal Mamba tombstone after the host backup attempt. - fn finish_mamba_state_eviction( - &self, - py: Python<'_>, - node_id: NodeId, - ) -> PyResult { - let result = py.detach(move || self.core().finish_mamba_state_eviction(node_id)); - self.internal_eviction_finish_to_py(py, result) - } - - /// Resume a pending internal SWA tombstone after the host backup attempt. - fn finish_swa_state_eviction( - &self, - py: Python<'_>, - node_id: NodeId, - ) -> PyResult { - let result = py.detach(move || self.core().finish_swa_state_eviction(node_id)); - self.internal_eviction_finish_to_py(py, result) - } - - fn internal_eviction_finish_to_py( - &self, - py: Python<'_>, - result: EvictionStepResult, - ) -> PyResult { - Ok(EvictDeviceNextNodeResultBinding { - node_id: None, - mamba_backup_node_id: None, - swa_backup_node_id: None, - swa_backup_num_tokens: 0, - // Consuming the pending request advances the finite walk even - // when intervening I/O pinned or removed this particular node. - made_progress: true, - unbacked_tokens: result.unbacked_tokens, - tracker: tracker_to_py(result.tracker), - new_device_frees: frees_to_py(py, result.device_frees)?, - new_host_frees: frees_to_py(py, result.host_frees)?, - }) - } - - /// Evict one device leaf; an unbacked write-back leaf returns its backup - /// action for the caller to execute before demoting. - fn evict_device_leaf( - &self, - py: Python<'_>, - node_id: NodeId, - ) -> PyResult { - let (backup, result) = py - .detach(move || { - let mut core = self.core(); - let is_write_back = core.is_write_back; - core.evict_device_leaf(node_id, is_write_back) - }) - .map_err(node_access_error)?; - Ok(EvictDeviceLeafResultBinding { - unbacked_tokens: result.unbacked_tokens, - backup_kv: backup - .map(|backup| cache_action_to_py(py, CacheAction::BackupKV(backup))) - .transpose()?, - tracker: tracker_to_py(result.tracker), - new_device_frees: frees_to_py(py, result.device_frees)?, - new_host_frees: frees_to_py(py, result.host_frees)?, - }) - } - - /// Finish a component's device-eviction walk. - fn evict_device_end(&self, py: Python<'_>, component_type: u8) -> PyResult<()> { - let ct = parse_component_type(component_type)?; - py.detach(|| self.core().evict_device_end(ct)); - Ok(()) - } - - /// Verify tree-structure, leaf-set, LRU, size, and ongoing-op invariants; - /// ongoing_* args are (id, node_id) pairs. - fn sanity_check( - &self, - py: Python<'_>, - ongoing_write_through: Vec<(i64, NodeId)>, - ongoing_load_back: Vec<(i64, NodeId)>, - ) -> PyResult<()> { - py.detach(|| { - self.core() - .try_sanity_check(&ongoing_write_through, &ongoing_load_back) - }) - .map_err(PyAssertionError::new_err) - } - - /// Concatenated FULL device values from from_node up to (exclusive) until_node. - fn collect_full_device_indices( - &self, - py: Python<'_>, - from_node_id: NodeId, - until_node_id: NodeId, - ) -> PyResult> { - let value = py - .detach(|| { - self.core() - .collect_full_device_indices(from_node_id, until_node_id) - }) - .map_err(node_access_error)?; - tensor_to_py(py, value) - } - - /// Every FULL device value in the tree, concatenated. - fn all_values_flatten(&self, py: Python<'_>) -> PyResult> { - let value = py.detach(|| self.core().all_values_flatten()); - tensor_to_py(py, value) - } - - /// Every Mamba device value in the tree, concatenated. - fn all_mamba_values_flatten(&self, py: Python<'_>) -> PyResult> { - let value = py.detach(|| self.core().all_mamba_values_flatten()); - tensor_to_py(py, value) - } - - /// Flatten every FULL device slot into (slot, position, prev-slot) rows for the KV-canary sweep. - fn walk_for_kv_canary( - &self, - py: Python<'_>, - unlocked_only: bool, - swa_resident_only: bool, - ) -> PyResult { - let result = py.detach(|| { - self.core() - .walk_for_kv_canary(unlocked_only, swa_resident_only) - }); - Ok(KvCanaryWalkResultBinding { - slot_indices: tensor_to_py(py, Tensor::from_slice(&result.slot_indices))?, - positions: tensor_to_py(py, Tensor::from_slice(&result.positions))?, - prev_slot_indices: tensor_to_py(py, Tensor::from_slice(&result.prev_slot_indices))?, - }) - } - - /// Evictable token count of the FULL (base) component. - fn evictable_size(&self, py: Python<'_>) -> usize { - py.detach(|| self.core().evictable_size()) - } - - /// Protected (locked) token count of the FULL (base) component. - fn protected_size(&self, py: Python<'_>) -> usize { - py.detach(|| self.core().protected_size()) - } - - /// FULL component evictable token count. - fn full_evictable_size(&self, py: Python<'_>) -> usize { - py.detach(|| self.core().full_evictable_size()) - } - - /// FULL component protected token count. - fn full_protected_size(&self, py: Python<'_>) -> usize { - py.detach(|| self.core().full_protected_size()) - } - - /// Evictable token count for one component (0 if the component is absent). - fn component_evictable_size(&self, py: Python<'_>, component_type: u8) -> PyResult { - let ct = parse_component_type(component_type)?; - Ok(py.detach(|| self.core().component_evictable_size(ct))) - } - - /// Protected token count for one component (0 if the component is absent). - fn component_protected_size(&self, py: Python<'_>, component_type: u8) -> PyResult { - let ct = parse_component_type(component_type)?; - Ok(py.detach(|| self.core().component_protected_size(ct))) - } - - /// (full_tokens, aux_tokens) summed across the whole tree. - fn total_size(&self, py: Python<'_>) -> (usize, usize) { - py.detach(|| self.core().total_size()) - } - - /// Whether the node's FULL device value has been evicted. - fn is_full_device_evicted(&self, py: Python<'_>, node_id: NodeId) -> PyResult { - py.detach(|| self.core().is_full_device_evicted(node_id)) - .map_err(node_access_error) - } - - /// Mark the host tier as buffer-only; wired after the host pools are built. - fn set_host_memory_buffer_only(&self, py: Python<'_>) { - py.detach(|| self.core().set_host_memory_buffer_only()); - } - - /// Whether the host tier runs as a storage staging buffer, not a cache. - fn is_host_memory_buffer_only(&self, py: Python<'_>) -> bool { - py.detach(|| self.core().is_host_memory_buffer_only) - } - - /// Mark the host tier (HiCache) as wired. - fn set_hicache_enabled(&self, py: Python<'_>) { - py.detach(|| self.core().set_hicache_enabled()); - } - - /// Whether the host tier (HiCache) is wired. - fn enable_hicache(&self, py: Python<'_>) -> bool { - py.detach(|| self.core().enable_hicache) - } - - /// Mark the SWA host pool as wired (HiCache). - fn set_has_swa_host_pool(&self, py: Python<'_>) { - py.detach(|| self.core().set_has_swa_host_pool()); - } - - /// Whether the SWA host pool is wired. - fn has_swa_host_pool(&self, py: Python<'_>) -> bool { - py.detach(|| self.core().has_swa_host_pool) - } - - /// Insert a host-side (backuped) tree path descending from the given node. - fn insert_host( - &self, - py: Python<'_>, - node_id: NodeId, - extra_key: Option, - key: &Bound<'_, PyAny>, - host_value: PyTensor, - hash_value: Vec, - cache_salt: Option, - ) -> PyResult { - let key = K::key_from(Cow::Owned(py_array_to_vec_i64(py, key)?)).into_owned(); - let host_value = host_value.0; - if host_value.kind() != Kind::Int64 { - return Err(PyValueError::new_err(format!( - "insert_host host_value must be an int64 tensor, got {:?}", - host_value.kind() - ))); - } - let result = py - .detach(move || { - self.core().insert_host_in_namespace( - node_id, - KeyNamespaceRef::new(extra_key.as_deref(), cache_salt.as_deref()), + /// Insert device values into the tree per the provided key. + fn insert( + &self, + py: Python<'_>, + params: &InsertParamsBinding, + ) -> PyResult { + let key = K::key_from(Cow::Borrowed(¶ms.key)); + let key = key.as_ref(); + let value: PyTensor = params.value.bind(py).extract()?; + // The value covers key atoms (bigram: raw len - 1), so validate the converted key. + self.validate_insert_value(&value.0, key.atom_len())?; + let mamba_value = match ¶ms.mamba_value { + Some(mamba_value) => Some(mamba_value.bind(py).extract::()?.0), + None => None, + }; + let params = InsertParams { + rotation_base: params.rotation_base, key, - host_value, - hash_value, - ) - }) - .map_err(tree_core_runtime_error)?; - InsertResultBinding::from_insert_result(py, result) - } + namespace: KeyNamespaceRef::new( + params.extra_key.as_deref(), + params.cache_salt.as_deref(), + ), + session_id: params.session_id.as_deref(), + value: value.0, + mamba_value, + prev_prefix_len: params.prev_prefix_len, + swa_evicted_seqlen: params.swa_evicted_seqlen, + swa_branching_seqlen: params.swa_branching_seqlen, + inserted_len: params.inserted_len, + priority: params.priority, + track_adopted_ranges: params.track_adopted_ranges, + }; + let result = py + .detach(move || self.core().try_insert(¶ms)) + .map_err(tree_core_runtime_error)?; + InsertResultBinding::from_insert_result(py, result) + } - /// Gather a node's device value plus per-component BACKUP_HOST transfers. - fn build_backup_spec( - &self, - py: Python<'_>, - node_id: NodeId, - ) -> PyResult<(Py, Py)> { - let (device_value, comp_xfers) = py - .detach(|| self.core().build_backup_spec(node_id)) - .map_err(node_access_error)?; - Ok(( - tensor_to_py(py, device_value)?, - comp_xfers_to_py(py, comp_xfers)?, - )) - } + /// Start the resumable insert, running to its first barrier or completion. + fn begin_insert( + &self, + py: Python<'_>, + params: &InsertParamsBinding, + ) -> PyResult { + let key = K::key_from(Cow::Borrowed(¶ms.key)); + let key = key.as_ref(); + let value: PyTensor = params.value.bind(py).extract()?; + // The value covers key atoms (bigram: raw len - 1), so validate the converted key. + self.validate_insert_value(&value.0, key.atom_len())?; + let mamba_value = match ¶ms.mamba_value { + Some(mamba_value) => Some(mamba_value.bind(py).extract::()?.0), + None => None, + }; + let params = InsertParams { + rotation_base: params.rotation_base, + key, + namespace: KeyNamespaceRef::new( + params.extra_key.as_deref(), + params.cache_salt.as_deref(), + ), + session_id: params.session_id.as_deref(), + value: value.0, + mamba_value, + prev_prefix_len: params.prev_prefix_len, + swa_evicted_seqlen: params.swa_evicted_seqlen, + swa_branching_seqlen: params.swa_branching_seqlen, + inserted_len: params.inserted_len, + priority: params.priority, + track_adopted_ranges: params.track_adopted_ranges, + }; + let step = py + .detach(move || self.core().try_begin_insert(¶ms)) + .map_err(tree_core_runtime_error)?; + InsertStepResultBinding::from_insert_step(py, step) + } - /// Gather a node's device->storage backup spec; None if the node is not backuped. - fn build_storage_backup_spec( - &self, - py: Python<'_>, - node_id: NodeId, - pass_prefix_keys: bool, - ) -> PyResult> { - let spec = py - .detach(|| { - self.core() - .build_storage_backup_spec(node_id, pass_prefix_keys) - }) - .map_err(node_access_error)?; - let Some(spec) = spec else { - return Ok(None); - }; - let mut token_bytes = Vec::with_capacity(spec.token_ids.len() * 8); - for token in &spec.token_ids { - token_bytes.extend_from_slice(&token.to_ne_bytes()); - } - Ok(Some(StorageBackupSpecBinding { - host_value: tensor_to_py(py, spec.host_value)?, - token_ids: PyBytes::new(py, &token_bytes).unbind(), - hash_value: spec.hash_value, - prefix_keys: spec.prefix_keys, - comp_xfers: comp_xfers_to_py(py, spec.comp_xfers)?, - })) - } + /// Continue the suspended insert after its step actions were applied. + fn resume_insert(&self, py: Python<'_>) -> PyResult { + let step = py + .detach(|| self.core().try_resume_insert()) + .map_err(tree_core_runtime_error)?; + InsertStepResultBinding::from_insert_step(py, step) + } - /// Route a build_hicache_transfers call to the component for the given type. - #[allow(clippy::too_many_arguments)] - fn build_hicache_transfers( - &self, - py: Python<'_>, - component_type: u8, - node_id: NodeId, - phase: &str, - host_indices: Option, - token_ids: Option>, - prefetch_tokens: usize, - staging_tokens: usize, - last_hash: Option, - ) -> PyResult>>> { - let component_type = parse_component_type(component_type)?; - let phase = parse_transfer_phase(phase)?; - let host_indices = host_indices.map(|t| t.0); - let transfers = py - .detach(|| { - self.core().build_hicache_transfers( - component_type, - node_id, - phase, - host_indices, - token_ids.as_deref(), - prefetch_tokens, - staging_tokens, - last_hash.as_deref(), - ) - }) - .map_err(tree_core_assertion_error)?; - transfers - .map(|transfers| { - transfers - .into_iter() - .map(|transfer| transfer_to_py(py, transfer)) - .collect::>>() - }) - .transpose() - } + /// Whether an insert walk is suspended at a barrier. + fn has_ongoing_insert(&self, py: Python<'_>) -> bool { + py.detach(|| self.core().has_ongoing_insert()) + } - /// The anchor node's namespace; None for root-like anchors. - fn prefetch_anchor_info( - &self, - py: Python<'_>, - node_id: NodeId, - ) -> PyResult<(Option, Option)> { - py.detach(|| self.core().prefetch_anchor_info(node_id)) - .map_err(node_access_error) - } + /// Finish the insert (idempotent); returns still-pending actions to drain. + fn end_insert(&self, py: Python<'_>) -> PyResult> { + let actions = py.detach(|| self.core().end_insert()); + cache_actions_to_py(py, actions) + } - /// Whether the node's Full KV is present on host. - fn node_backuped(&self, py: Python<'_>, node_id: NodeId) -> PyResult { - py.detach(|| self.core().node_backuped(node_id)) - .map_err(node_access_error) - } + /// Bump the reference count on a node's component locks; the listed + /// components are left untaken and recorded in the receipt. + fn inc_lock_ref( + &self, + py: Python<'_>, + node_id: NodeId, + skip_lock_components: Vec, + ) -> PyResult { + let skip = component_set_from_py(&skip_lock_components)?; + let result = py + .detach(|| self.core().inc_lock_ref(node_id, skip)) + .map_err(node_access_error)?; + Ok(IncLockRefResultBinding::from_result(result)) + } - /// Whether the node is a (default or named) root. - fn is_root(&self, py: Python<'_>, node_id: NodeId) -> PyResult { - py.detach(|| self.core().is_root(node_id)) - .map_err(node_access_error) - } + /// Pin only the FULL device values on a node's root path. + fn inc_full_pin(&self, py: Python<'_>, node_id: NodeId) -> PyResult<()> { + py.detach(|| self.core().inc_full_pin(node_id)) + .map_err(node_access_error) + } - /// The node's last page hash, or None when it was never hashed. - fn get_last_hash_value(&self, py: Python<'_>, node_id: NodeId) -> PyResult> { - py.detach(|| self.core().get_last_hash_value(node_id)) - .map_err(node_access_error) - } + /// Release a FULL-only root-path pin. + fn dec_full_pin(&self, py: Python<'_>, node_id: NodeId) -> PyResult<()> { + py.detach(|| self.core().dec_full_pin(node_id)) + .map_err(node_access_error) + } - /// The hash chain of the node's ancestors, in root-to-parent order. - fn get_prefix_hash_values(&self, py: Python<'_>, node_id: NodeId) -> PyResult> { - py.detach(|| self.core().get_prefix_hash_values(node_id)) - .map_err(node_access_error) - } + /// Decrease the reference count on a node's component locks. + fn dec_lock_ref( + &self, + py: Python<'_>, + node_id: NodeId, + params: &DecLockRefParamsBinding, + skip_swa: bool, + ) -> PyResult<()> { + let params = params.to_dec_lock_ref_params()?; + py.detach(|| self.core().dec_lock_ref(node_id, ¶ms, skip_swa)) + .map_err(node_access_error)?; + Ok(()) + } - fn get_hash_values(&self, py: Python<'_>, node_id: NodeId) -> PyResult> { - py.detach(|| self.core().get_hash_values(node_id)) - .map_err(node_access_error) - } - - fn snapshot_buffer_backup( - &self, - py: Python<'_>, - node_id: NodeId, - pass_prefix_keys: bool, - ) -> Option { - py.detach(|| { - self.core() - .snapshot_buffer_backup(node_id, pass_prefix_keys) - }) - .map(|snapshot| BufferBackupSnapshotBinding::from_snapshot(py, snapshot)) - } - - fn validate_buffer_backup( - &self, - py: Python<'_>, - node_id: NodeId, - expected_key_length: usize, - ) -> Option { - py.detach(|| { - self.core() - .validate_buffer_backup(node_id, expected_key_length) - }) - .map(BufferBackupStateBinding::from) - } - - /// Hash every node built while storage was disabled. - fn backfill_missing_hash_values(&self, py: Python<'_>) -> usize { - py.detach(|| self.core().backfill_missing_hash_values()) - } - - fn root_node_handle(&self, py: Python<'_>, extra_key: Option) -> NodeId { - py.detach(|| self.core().root_node_handle(extra_key.as_deref())) - } - - fn rotation_base_of(&self, py: Python<'_>, node_id: NodeId) -> PyResult> { - py.detach(|| self.core().rotation_base_of(node_id)) - .map_err(node_access_error) - } - - fn dfs_weight_order(&self, py: Python<'_>, node_ids: Vec) -> PyResult> { - py.detach(|| self.core().dfs_weight_order(&node_ids)) - .map_err(node_access_error) - } - - /// Commit each component's HiCache transfers; returns the new cache actions. - fn commit_hicache_transfers( - &self, - py: Python<'_>, - node_id: NodeId, - phase: &str, - comp_xfers: HashMap>, - insert_result: Option<(usize, Option, bool)>, - pool_storage_result: Option<(usize, HashMap)>, - ) -> PyResult<(Py, Option)> { - let phase = parse_transfer_phase(phase)?; - let comp_xfers = comp_xfers_from_args(comp_xfers)?; - let insert_result = - insert_result.map( - |(total_len, inserted_host_node, mamba_exist)| InsertResult { - total_len, - inserted_host_node, - mamba_exist, - ..InsertResult::default() - }, - ); - let pool_storage_result = pool_storage_result - .map(|(kv_hit_pages, extra_pool_hit_pages)| { - Ok::<_, PyErr>(PoolTransferResult { - kv_hit_pages, - extra_pool_hit_pages: extra_pool_hit_pages - .into_iter() - .map(|(name, pages)| Ok((parse_pool_name(&name)?, pages))) - .collect::>>()?, - }) - }) - .transpose()?; - let (cache_actions, mamba_exist) = py - .detach(move || { - let mut cache_actions = Vec::new(); - let mut insert_result = insert_result; - self.core().commit_hicache_transfers( - node_id, - phase, - comp_xfers, - &mut cache_actions, - insert_result.as_mut(), - pool_storage_result.as_ref(), - )?; + /// Release one window's lock, retaining other component locks. + fn dec_window_lock_only( + &self, + py: Python<'_>, + node_id: NodeId, + component_type: u8, + params: &DecLockRefParamsBinding, + ) -> PyResult<(Py, Py, Py)> { + let component_type = parse_component_type(component_type)?; + let params = params.to_dec_lock_ref_params()?; + let (device_frees, host_frees, cache_actions) = py + .detach(|| { + let mut device_frees = HashMap::new(); + let mut host_frees = HashMap::new(); + let mut cache_actions = Vec::new(); + self.core().dec_window_lock_only( + node_id, + component_type, + ¶ms, + &mut device_frees, + &mut host_frees, + &mut cache_actions, + )?; + Ok((device_frees, host_frees, cache_actions)) + }) + .map_err(node_access_error)?; Ok(( - cache_actions, - insert_result.map(|result| result.mamba_exist), + frees_to_py(py, device_frees)?, + frees_to_py(py, host_frees)?, + cache_actions_to_py(py, cache_actions)?, )) - }) - .map_err(node_access_error)?; - Ok((cache_actions_to_py(py, cache_actions)?, mamba_exist)) - } + } - /// Commit a successful backup to the node. - fn commit_backup( - &self, - py: Python<'_>, - node_id: NodeId, - host_indices: PyTensor, - comp_xfers: HashMap>, - ) -> PyResult<()> { - let comp_xfers = comp_xfers_from_args(comp_xfers)?; - let host_indices = host_indices.0; - py.detach(move || self.core().commit_backup(node_id, host_indices, comp_xfers)) - .map_err(node_access_error)?; - Ok(()) - } + /// Early-release the SWA portion of a request's tree lock; returns this + /// release's per-component frees and cache actions. + fn dec_swa_lock_only( + &self, + py: Python<'_>, + node_id: NodeId, + params: &DecLockRefParamsBinding, + ) -> PyResult<(Py, Py, Py)> { + let params = params.to_dec_lock_ref_params()?; + let (device_frees, host_frees, cache_actions) = py + .detach(|| { + let mut device_frees = HashMap::new(); + let mut host_frees = HashMap::new(); + let mut cache_actions = Vec::new(); + self.core().dec_swa_lock_only( + node_id, + ¶ms, + &mut device_frees, + &mut host_frees, + &mut cache_actions, + )?; + Ok((device_frees, host_frees, cache_actions)) + }) + .map_err(node_access_error)?; + Ok(( + frees_to_py(py, device_frees)?, + frees_to_py(py, host_frees)?, + cache_actions_to_py(py, cache_actions)?, + )) + } - /// Build the H->D load-back KV transfer plus per-component aux transfers. - fn build_load_back_spec( - &self, - py: Python<'_>, - node_id: NodeId, - mamba_pool_idx: Option, - ) -> PyResult<(Py, Py)> { - let req = Req { - mamba_pool_idx: mamba_pool_idx.map(|t| t.0), - }; - let (kv_xfer, comp_xfers) = py - .detach(move || self.core().build_load_back_spec(node_id, Some(&req))) - .map_err(tree_core_assertion_error)?; - Ok(( - transfer_to_py(py, kv_xfer)?, - comp_xfers_to_py(py, comp_xfers)?, - )) - } + /// Adjust the component's reclaimable device size. + fn adjust_component_size( + &self, + py: Python<'_>, + node_id: NodeId, + component_type: u8, + delta: i64, + ) -> PyResult<()> { + let component_type = parse_component_type(component_type)?; + py.detach(|| { + self.core() + .adjust_component_size(node_id, component_type, delta) + }) + .map_err(tree_core_runtime_error) + } - /// Commit a successful H->D load-back onto the node; returns its actions. - fn commit_load_back( - &self, - py: Python<'_>, - node_id: NodeId, - device_indices: PyTensor, - kv_xfer: TransferArgs, - comp_xfers: HashMap>, - ) -> PyResult> { - let kv_xfer = transfer_from_args(kv_xfer)?; - let comp_xfers = comp_xfers_from_args(comp_xfers)?; - let device_indices = device_indices.0; - let actions = py - .detach(move || { - self.core() - .commit_load_back(node_id, device_indices, kv_xfer, comp_xfers) - }) - .map_err(node_access_error)?; - cache_actions_to_py(py, actions) - } + /// Collect paths under one tree lock. + fn collect_path_info( + &self, + py: Python<'_>, + end_node_ids: Vec, + ) -> PyResult>> { + let paths = py + .detach(|| self.core().collect_path_info(&end_node_ids)) + .map_err(tree_core_runtime_error)?; + Ok(paths + .into_iter() + .map(|path| { + path.into_iter() + .map(|node| { + ( + node.node_id, + node.parent_id, + node.path_start, + node.path_end, + node.full_value_len, + node.num_children, + ) + }) + .collect() + }) + .collect()) + } - /// Release a node's device KV once its host copy exists. - fn demote(&self, py: Python<'_>, node_id: NodeId) -> PyResult { - let result = py - .detach(move || self.core().demote(node_id)) - .map_err(tree_core_assertion_error)?; - Ok(DemoteResultBinding { - tracker: tracker_to_py(result.tracker), - new_device_frees: frees_to_py(py, result.device_frees)?, - new_host_frees: frees_to_py(py, result.host_frees)?, - }) - } + /// Store a component's device value on a node (the SWA rebuild write-back). + fn set_component_device_value( + &self, + py: Python<'_>, + node_id: NodeId, + component_type: u8, + value: PyTensor, + ) -> PyResult<()> { + let component_type = parse_component_type(component_type)?; + let value = value.0; + if value.kind() != Kind::Int64 { + return Err(PyValueError::new_err(format!( + "component device value must be an int64 tensor, got {:?}", + value.kind() + ))); + } + if value.device() != self.device { + return Err(PyValueError::new_err(format!( + "component device value device {:?} does not match the tree core device {:?}", + value.device(), + self.device + ))); + } + py.detach(|| { + self.core() + .set_component_device_value(node_id, component_type, value) + }) + .map_err(node_access_error)?; + Ok(()) + } - /// Evict up to num_tokens of one component's host resources. - fn drive_host_eviction( - &self, - py: Python<'_>, - component_type: u8, - num_tokens: usize, - skip_full_duplicate_reclaim: bool, - ) -> PyResult { - let ct = parse_component_type(component_type)?; - let result = py.detach(move || { - self.core().drive_host_eviction_with_options( - ct, - num_tokens, - skip_full_duplicate_reclaim, - ) - }); - Ok(HostEvictionResultBinding { - tracker: tracker_to_py(result.tracker), - new_device_frees: frees_to_py(py, result.device_frees)?, - new_host_frees: frees_to_py(py, result.host_frees)?, - }) - } + /// Logical key lengths for the nodes in a load-back transfer. + fn get_node_key_lengths(&self, py: Python<'_>, node_ids: Vec) -> PyResult> { + py.detach(|| self.core().get_node_key_lengths(&node_ids)) + .map_err(node_access_error) + } - /// Evict shallow Mamba device checkpoints beyond the per-path cap on the - /// tail's root path; returns the step's freed tensors. - fn evict_excess_path_states( - &self, - py: Python<'_>, - tail_node_id: NodeId, - ) -> PyResult { - let result = py - .detach(move || self.core().evict_excess_path_states(tail_node_id)) - .map_err(node_access_error)?; - Ok(HostEvictionResultBinding { - tracker: tracker_to_py(result.tracker), - new_device_frees: frees_to_py(py, result.device_frees)?, - new_host_frees: frees_to_py(py, result.host_frees)?, - }) - } + /// A component's device value on a node, if set. + fn get_component_device_value( + &self, + py: Python<'_>, + node_id: NodeId, + component_type: u8, + ) -> PyResult>> { + let component_type = parse_component_type(component_type)?; + let value = py + .detach(|| { + self.core() + .get_component_device_value(node_id, component_type) + .map(|value| value.map(|tensor| tensor.shallow_clone())) + }) + .map_err(node_access_error)?; + value.map(|value| tensor_to_py(py, value)).transpose() + } - /// Bump the reference count on a node's host-side component locks. - fn inc_host_lock_ref( - &self, - py: Python<'_>, - node_id: NodeId, - ) -> PyResult { - let result = py - .detach(|| self.core().inc_host_lock_ref(node_id)) - .map_err(node_access_error)?; - Ok(IncLockRefResultBinding::from_result(result)) - } + // TODO(jialino): batch a full no-backup eviction round in Rust (one crossing + // instead of 2N+2); step-wise stays for the write-back interleave. + /// Begin a component's device-eviction walk for up to request_cnt tokens. + fn evict_device_start( + &self, + py: Python<'_>, + component_type: u8, + request_cnt: usize, + ) -> PyResult<()> { + let ct = parse_component_type(component_type)?; + py.detach(|| self.core().evict_device_start(ct, request_cnt)); + Ok(()) + } - /// Decrease the reference count on a node's host-side component locks. - fn dec_host_lock_ref( - &self, - py: Python<'_>, - node_id: NodeId, - params: &DecLockRefParamsBinding, - ) -> PyResult<()> { - let params = params.to_dec_lock_ref_params()?; - py.detach(|| self.core().dec_host_lock_ref(node_id, ¶ms)) - .map_err(node_access_error)?; - Ok(()) - } + /// Advance one component eviction step. Progress without a leaf can be + /// an internal tombstone or a component backup request awaiting its finish; + /// no progress means the walk is exhausted. + fn evict_device_next_node( + &self, + py: Python<'_>, + component_type: u8, + tracker: HashMap, + ) -> PyResult { + let ct = parse_component_type(component_type)?; + let baseline = tracker_from_py(tracker)?; + let (node_id, result) = + py.detach(move || self.core().evict_device_next_node(ct, &baseline)); + let made_progress = node_id.is_some() + || result.mamba_backup_node_id.is_some() + || result.swa_backup_node_id.is_some() + || !result.tracker.is_empty() + || result.device_frees.values().any(|frees| !frees.is_empty()) + || result.host_frees.values().any(|frees| !frees.is_empty()) + || !result.cache_actions.is_empty(); + Ok(EvictDeviceNextNodeResultBinding { + node_id, + mamba_backup_node_id: result.mamba_backup_node_id, + swa_backup_node_id: result.swa_backup_node_id, + swa_backup_num_tokens: result.swa_backup_num_tokens, + made_progress, + unbacked_tokens: result.unbacked_tokens, + tracker: tracker_to_py(result.tracker), + new_device_frees: frees_to_py(py, result.device_frees)?, + new_host_frees: frees_to_py(py, result.host_frees)?, + cache_actions: cache_actions_to_py(py, result.cache_actions)?, + }) + } - fn is_write_through_compatible(&self, py: Python<'_>) -> bool { - py.detach(|| self.core().is_write_through_compatible()) - } + /// Resume a pending internal Mamba tombstone after the host backup attempt. + fn finish_mamba_state_eviction( + &self, + py: Python<'_>, + node_id: NodeId, + ) -> PyResult { + let result = py.detach(move || self.core().finish_mamba_state_eviction(node_id)); + self.internal_eviction_finish_to_py(py, result) + } - /// Set the write-back (vs write-through) policy; decided at HiCache init. - fn set_is_write_back(&self, py: Python<'_>, is_write_back: bool) { - py.detach(|| self.core().is_write_back = is_write_back); - } + /// Resume a pending internal SWA tombstone after the host backup attempt. + fn finish_swa_state_eviction( + &self, + py: Python<'_>, + node_id: NodeId, + ) -> PyResult { + let result = py.detach(move || self.core().finish_swa_state_eviction(node_id)); + self.internal_eviction_finish_to_py(py, result) + } - /// The current write-back (vs write-through) policy. - fn is_write_back(&self, py: Python<'_>) -> bool { - py.detach(|| self.core().is_write_back) - } + fn internal_eviction_finish_to_py( + &self, + py: Python<'_>, + result: EvictionStepResult, + ) -> PyResult { + Ok(EvictDeviceNextNodeResultBinding { + node_id: None, + mamba_backup_node_id: None, + swa_backup_node_id: None, + swa_backup_num_tokens: 0, + // Consuming the pending request advances the finite walk even + // when intervening I/O pinned or removed this particular node. + made_progress: true, + unbacked_tokens: result.unbacked_tokens, + tracker: tracker_to_py(result.tracker), + new_device_frees: frees_to_py(py, result.device_frees)?, + new_host_frees: frees_to_py(py, result.host_frees)?, + cache_actions: cache_actions_to_py(py, result.cache_actions)?, + }) + } - /// Set the write-through backup hit threshold; decided at HiCache init. - fn set_write_through_threshold(&self, py: Python<'_>, threshold: i64) { - py.detach(|| self.core().write_through_threshold = threshold); - } + /// Evict one device leaf; an unbacked write-back leaf returns its backup + /// action for the caller to execute before demoting. + fn evict_device_leaf( + &self, + py: Python<'_>, + node_id: NodeId, + ) -> PyResult { + let (backup, result) = py + .detach(move || { + let mut core = self.core(); + let is_write_back = core.is_write_back(); + core.evict_device_leaf(node_id, is_write_back) + }) + .map_err(node_access_error)?; + Ok(EvictDeviceLeafResultBinding { + unbacked_tokens: result.unbacked_tokens, + backup_kv: backup + .map(|backup| cache_action_to_py(py, CacheAction::BackupKV(backup))) + .transpose()?, + tracker: tracker_to_py(result.tracker), + new_device_frees: frees_to_py(py, result.device_frees)?, + new_host_frees: frees_to_py(py, result.host_frees)?, + cache_actions: cache_actions_to_py(py, result.cache_actions)?, + }) + } - /// The current write-through backup hit threshold. - fn write_through_threshold(&self, py: Python<'_>) -> i64 { - py.detach(|| self.core().write_through_threshold) - } + /// Finish a component's device-eviction walk. + fn evict_device_end(&self, py: Python<'_>, component_type: u8) -> PyResult<()> { + let ct = parse_component_type(component_type)?; + py.detach(|| self.core().evict_device_end(ct)); + Ok(()) + } - /// Mark the storage tier (L3) wired; storage attaches after tree construction. - fn set_enable_storage(&self, py: Python<'_>, value: bool) { - py.detach(|| self.core().set_enable_storage(value)); - } + /// Verify tree-structure, leaf-set, LRU, size, and ongoing-op invariants; + /// ongoing_* args are (id, node_id) pairs. + fn sanity_check( + &self, + py: Python<'_>, + ongoing_write_through: Vec<(i64, NodeId)>, + ongoing_load_back: Vec<(i64, NodeId)>, + ) -> PyResult<()> { + py.detach(|| { + self.core() + .try_sanity_check(&ongoing_write_through, &ongoing_load_back) + }) + .map_err(PyAssertionError::new_err) + } - /// Whether the storage tier (L3) is wired. - fn enable_storage(&self, py: Python<'_>) -> bool { - py.detach(|| self.core().enable_storage) - } + /// Concatenated FULL device values from from_node up to (exclusive) until_node. + fn collect_full_device_indices( + &self, + py: Python<'_>, + from_node_id: NodeId, + until_node_id: NodeId, + ) -> PyResult> { + let value = py + .detach(|| { + self.core() + .collect_full_device_indices(from_node_id, until_node_id) + }) + .map_err(node_access_error)?; + tensor_to_py(py, value) + } - /// Enable or disable the direct external-cache linker. - fn set_enable_external_cache_linker(&self, py: Python<'_>, value: bool) -> PyResult<()> { - py.detach(|| self.core().set_enable_external_cache_linker(value)) - .map_err(tree_core_assertion_error) - } + /// Every FULL device value in the tree, concatenated. + fn all_values_flatten(&self, py: Python<'_>) -> PyResult> { + let value = py.detach(|| self.core().all_values_flatten()); + tensor_to_py(py, value) + } - /// Whether the direct external-cache linker is wired. - fn enable_external_cache_linker(&self, py: Python<'_>) -> bool { - py.detach(|| self.core().enable_external_cache_linker) - } + /// Every Mamba device value in the tree, concatenated. + fn all_mamba_values_flatten(&self, py: Python<'_>) -> PyResult> { + let value = py.detach(|| self.core().all_mamba_values_flatten()); + tensor_to_py(py, value) + } - /// Queue the all-cleared placement event. - fn record_all_cleared_event(&self, py: Python<'_>) { - py.detach(|| self.core().record_all_cleared_event()); - } + /// Flatten every FULL device slot into (slot, position, prev-slot) rows for the KV-canary sweep. + fn walk_for_kv_canary( + &self, + py: Python<'_>, + unlocked_only: bool, + swa_resident_only: bool, + ) -> PyResult { + let result = py.detach(|| { + self.core() + .walk_for_kv_canary(unlocked_only, swa_resident_only) + }); + Ok(KvCanaryWalkResultBinding { + slot_indices: tensor_to_py(py, Tensor::from_slice(&result.slot_indices))?, + positions: tensor_to_py(py, Tensor::from_slice(&result.positions))?, + prev_slot_indices: tensor_to_py(py, Tensor::from_slice(&result.prev_slot_indices))?, + }) + } - /// Drain the queued placement events as tagged tuples. - fn take_events(&self, py: Python<'_>) -> PyResult> - where - for<'py> Vec: IntoPyObject<'py>, - { - let events = py.detach(|| self.core().take_events()); - let list = PyList::empty(py); - for event in events { - match event { - KvCacheEvent::BlockStored { - block_hashes, - parent_block_hash, - token_ids, - block_size, - medium, - cache_salt, - session_id, - extra_key, - } => { - let item: Py = ( - "block_stored", - block_hashes, - parent_block_hash, - token_ids, - block_size, - medium.as_str(), - cache_salt.map(|salt| salt.to_string()), - session_id.map(|session_id| session_id.to_string()), - extra_key.map(|extra_key| extra_key.to_string()), + /// Evictable token count of the FULL (base) component. + fn evictable_size(&self, py: Python<'_>) -> usize { + py.detach(|| self.core().evictable_size()) + } + + /// Protected (locked) token count of the FULL (base) component. + fn protected_size(&self, py: Python<'_>) -> usize { + py.detach(|| self.core().protected_size()) + } + + /// FULL component evictable token count. + fn full_evictable_size(&self, py: Python<'_>) -> usize { + py.detach(|| self.core().full_evictable_size()) + } + + /// FULL component protected token count. + fn full_protected_size(&self, py: Python<'_>) -> usize { + py.detach(|| self.core().full_protected_size()) + } + + /// Evictable token count for one component (0 if the component is absent). + fn component_evictable_size(&self, py: Python<'_>, component_type: u8) -> PyResult { + let ct = parse_component_type(component_type)?; + Ok(py.detach(|| self.core().component_evictable_size(ct))) + } + + /// Protected token count for one component (0 if the component is absent). + fn component_protected_size(&self, py: Python<'_>, component_type: u8) -> PyResult { + let ct = parse_component_type(component_type)?; + Ok(py.detach(|| self.core().component_protected_size(ct))) + } + + /// (full_tokens, aux_tokens) summed across the whole tree. + fn total_size(&self, py: Python<'_>) -> (usize, usize) { + py.detach(|| self.core().total_size()) + } + + /// Whether the node's FULL device value has been evicted. + fn is_full_device_evicted(&self, py: Python<'_>, node_id: NodeId) -> PyResult { + py.detach(|| self.core().is_full_device_evicted(node_id)) + .map_err(node_access_error) + } + + /// Mark the host tier as buffer-only; wired after the host pools are built. + fn set_host_memory_buffer_only(&self, py: Python<'_>) { + py.detach(|| self.core().set_host_memory_buffer_only()); + } + + /// Whether the host tier runs as a storage staging buffer, not a cache. + fn is_host_memory_buffer_only(&self, py: Python<'_>) -> bool { + py.detach(|| self.core().is_host_memory_buffer_only()) + } + + /// Mark the host tier (HiCache) as wired. + fn set_hicache_enabled(&self, py: Python<'_>) { + py.detach(|| self.core().set_hicache_enabled()); + } + + /// Whether the host tier (HiCache) is wired. + fn enable_hicache(&self, py: Python<'_>) -> bool { + py.detach(|| self.core().enable_hicache()) + } + + /// Mark the SWA host pool as wired (HiCache). + fn set_has_swa_host_pool(&self, py: Python<'_>) { + py.detach(|| self.core().set_has_swa_host_pool()); + } + + /// Whether the SWA host pool is wired. + fn has_swa_host_pool(&self, py: Python<'_>) -> bool { + py.detach(|| self.core().has_swa_host_pool()) + } + + /// Insert a host-side (backuped) tree path descending from the given node. + fn insert_host( + &self, + py: Python<'_>, + node_id: NodeId, + extra_key: Option, + key: &Bound<'_, PyAny>, + host_value: PyTensor, + hash_value: Vec, + cache_salt: Option, + ) -> PyResult { + let key = K::key_from(Cow::Owned(py_array_to_vec_i64(py, key)?)).into_owned(); + let host_value = host_value.0; + if host_value.kind() != Kind::Int64 { + return Err(PyValueError::new_err(format!( + "insert_host host_value must be an int64 tensor, got {:?}", + host_value.kind() + ))); + } + let result = py + .detach(move || { + self.core().insert_host_in_namespace( + node_id, + KeyNamespaceRef::new(extra_key.as_deref(), cache_salt.as_deref()), + key, + host_value, + hash_value, + ) + }) + .map_err(tree_core_runtime_error)?; + InsertResultBinding::from_insert_result(py, result) + } + + fn build_backup_kv_action(&self, py: Python<'_>, node_id: NodeId) -> PyResult> { + let backup = py + .detach(|| self.core().build_backup_kv_action(node_id)) + .map_err(node_access_error)?; + cache_action_to_py(py, CacheAction::BackupKV(backup)) + } + + /// Gather a node's device value plus per-component BACKUP_HOST transfers. + fn build_backup_spec( + &self, + py: Python<'_>, + node_id: NodeId, + ) -> PyResult<(Py, Py)> { + let (device_value, comp_xfers) = py + .detach(|| self.core().build_backup_spec(node_id)) + .map_err(node_access_error)?; + Ok(( + tensor_to_py(py, device_value)?, + comp_xfers_to_py(py, comp_xfers)?, + )) + } + + /// Gather a node's device->storage backup spec; None if the node is not backuped. + fn build_storage_backup_spec( + &self, + py: Python<'_>, + node_id: NodeId, + pass_prefix_keys: bool, + ) -> PyResult> { + let spec = py + .detach(|| { + self.core() + .build_storage_backup_spec(node_id, pass_prefix_keys) + }) + .map_err(node_access_error)?; + let Some(spec) = spec else { + return Ok(None); + }; + let mut token_bytes = Vec::with_capacity(spec.token_ids.len() * 8); + for token in &spec.token_ids { + token_bytes.extend_from_slice(&token.to_ne_bytes()); + } + Ok(Some(StorageBackupSpecBinding { + host_value: tensor_to_py(py, spec.host_value)?, + token_ids: PyBytes::new(py, &token_bytes).unbind(), + hash_value: spec.hash_value, + prefix_keys: spec.prefix_keys, + comp_xfers: comp_xfers_to_py(py, spec.comp_xfers)?, + })) + } + + /// Route a build_hicache_transfers call to the component for the given type. + #[allow(clippy::too_many_arguments)] + fn build_hicache_transfers( + &self, + py: Python<'_>, + component_type: u8, + node_id: NodeId, + phase: &str, + host_indices: Option, + token_ids: Option>, + prefetch_tokens: usize, + staging_tokens: usize, + last_hash: Option, + ) -> PyResult>>> { + let component_type = parse_component_type(component_type)?; + let phase = parse_transfer_phase(phase)?; + let host_indices = host_indices.map(|t| t.0); + let transfers = py + .detach(|| { + self.core().build_hicache_transfers( + component_type, + node_id, + phase, + host_indices, + token_ids.as_deref(), + prefetch_tokens, + staging_tokens, + last_hash.as_deref(), + ) + }) + .map_err(tree_core_assertion_error)?; + transfers + .map(|transfers| { + transfers + .into_iter() + .map(|transfer| transfer_to_py(py, transfer)) + .collect::>>() + }) + .transpose() + } + + /// The anchor node's namespace; None for root-like anchors. + fn prefetch_anchor_info( + &self, + py: Python<'_>, + node_id: NodeId, + ) -> PyResult<(Option, Option)> { + py.detach(|| self.core().prefetch_anchor_info(node_id)) + .map_err(node_access_error) + } + + /// Whether the node's Full KV is present on host. + fn node_backuped(&self, py: Python<'_>, node_id: NodeId) -> PyResult { + py.detach(|| self.core().node_backuped(node_id)) + .map_err(node_access_error) + } + + /// Whether the node is a (default or named) root. + fn is_root(&self, py: Python<'_>, node_id: NodeId) -> PyResult { + py.detach(|| self.core().is_root(node_id)) + .map_err(node_access_error) + } + + /// The node's last page hash, or None when it was never hashed. + fn get_last_hash_value(&self, py: Python<'_>, node_id: NodeId) -> PyResult> { + py.detach(|| self.core().get_last_hash_value(node_id)) + .map_err(node_access_error) + } + + /// The hash chain of the node's ancestors, in root-to-parent order. + fn get_prefix_hash_values(&self, py: Python<'_>, node_id: NodeId) -> PyResult> { + py.detach(|| self.core().get_prefix_hash_values(node_id)) + .map_err(node_access_error) + } + + fn get_hash_values(&self, py: Python<'_>, node_id: NodeId) -> PyResult> { + py.detach(|| self.core().get_hash_values(node_id)) + .map_err(node_access_error) + } + + fn snapshot_buffer_backup( + &self, + py: Python<'_>, + node_id: NodeId, + pass_prefix_keys: bool, + ) -> Option { + py.detach(|| { + self.core() + .snapshot_buffer_backup(node_id, pass_prefix_keys) + }) + .map(|snapshot| BufferBackupSnapshotBinding::from_snapshot(py, snapshot)) + } + + fn validate_buffer_backup( + &self, + py: Python<'_>, + node_id: NodeId, + expected_key_length: usize, + ) -> Option { + py.detach(|| { + self.core() + .validate_buffer_backup(node_id, expected_key_length) + }) + .map(BufferBackupStateBinding::from) + } + + /// Hash every node built while storage was disabled. + fn backfill_missing_hash_values(&self, py: Python<'_>) -> usize { + py.detach(|| self.core().backfill_missing_hash_values()) + } + + fn root_node_handle(&self, py: Python<'_>, extra_key: Option) -> NodeId { + py.detach(|| self.core().root_node_handle(extra_key.as_deref())) + } + + fn rotation_base_of(&self, py: Python<'_>, node_id: NodeId) -> PyResult> { + py.detach(|| self.core().rotation_base_of(node_id)) + .map_err(node_access_error) + } + + fn dfs_weight_order(&self, py: Python<'_>, node_ids: Vec) -> PyResult> { + py.detach(|| self.core().dfs_weight_order(&node_ids)) + .map_err(node_access_error) + } + + /// Commit each component's HiCache transfers; returns the new cache actions. + fn commit_hicache_transfers( + &self, + py: Python<'_>, + node_id: NodeId, + phase: &str, + comp_xfers: HashMap>, + insert_result: Option<(usize, Option, bool)>, + pool_storage_result: Option<(usize, HashMap)>, + ) -> PyResult<(Py, Option)> { + let phase = parse_transfer_phase(phase)?; + let comp_xfers = comp_xfers_from_args(comp_xfers)?; + let insert_result = + insert_result.map( + |(total_len, inserted_host_node, mamba_exist)| InsertResult { + total_len, + inserted_host_node, + mamba_exist, + ..InsertResult::default() + }, + ); + let pool_storage_result = pool_storage_result + .map(|(kv_hit_pages, extra_pool_hit_pages)| { + Ok::<_, PyErr>(PoolTransferResult { + kv_hit_pages, + extra_pool_hit_pages: extra_pool_hit_pages + .into_iter() + .map(|(name, pages)| Ok((parse_pool_name(&name)?, pages))) + .collect::>>()?, + }) + }) + .transpose()?; + let (cache_actions, mamba_exist) = py + .detach(move || { + let mut cache_actions = Vec::new(); + let mut insert_result = insert_result; + self.core().commit_hicache_transfers( + node_id, + phase, + comp_xfers, + &mut cache_actions, + insert_result.as_mut(), + pool_storage_result.as_ref(), + )?; + Ok(( + cache_actions, + insert_result.map(|result| result.mamba_exist), + )) + }) + .map_err(node_access_error)?; + Ok((cache_actions_to_py(py, cache_actions)?, mamba_exist)) + } + + /// Commit a successful backup to the node. + fn commit_backup( + &self, + py: Python<'_>, + node_id: NodeId, + host_indices: PyTensor, + comp_xfers: HashMap>, + ) -> PyResult<()> { + let comp_xfers = comp_xfers_from_args(comp_xfers)?; + let host_indices = host_indices.0; + py.detach(move || self.core().commit_backup(node_id, host_indices, comp_xfers)) + .map_err(node_access_error)?; + Ok(()) + } + + /// Build the H->D load-back KV transfer plus per-component aux transfers. + fn build_load_back_spec( + &self, + py: Python<'_>, + node_id: NodeId, + mamba_pool_idx: Option, + ) -> PyResult<(Py, Py)> { + let req = Req { + mamba_pool_idx: mamba_pool_idx.map(|t| t.0), + }; + let (kv_xfer, comp_xfers) = py + .detach(move || self.core().build_load_back_spec(node_id, Some(&req))) + .map_err(tree_core_assertion_error)?; + Ok(( + transfer_to_py(py, kv_xfer)?, + comp_xfers_to_py(py, comp_xfers)?, + )) + } + + /// Commit a successful H->D load-back onto the node; returns its actions. + fn commit_load_back( + &self, + py: Python<'_>, + node_id: NodeId, + device_indices: PyTensor, + kv_xfer: TransferArgs, + comp_xfers: HashMap>, + ) -> PyResult> { + let kv_xfer = transfer_from_args(kv_xfer)?; + let comp_xfers = comp_xfers_from_args(comp_xfers)?; + let device_indices = device_indices.0; + let actions = py + .detach(move || { + self.core() + .commit_load_back(node_id, device_indices, kv_xfer, comp_xfers) + }) + .map_err(node_access_error)?; + cache_actions_to_py(py, actions) + } + + /// Release a node's device KV once its host copy exists. + fn demote(&self, py: Python<'_>, node_id: NodeId) -> PyResult { + let result = py + .detach(move || self.core().demote(node_id)) + .map_err(tree_core_assertion_error)?; + Ok(DemoteResultBinding { + tracker: tracker_to_py(result.tracker), + new_device_frees: frees_to_py(py, result.device_frees)?, + new_host_frees: frees_to_py(py, result.host_frees)?, + cache_actions: cache_actions_to_py(py, result.cache_actions)?, + }) + } + + /// Evict up to num_tokens of one component's host resources. + fn drive_host_eviction( + &self, + py: Python<'_>, + component_type: u8, + num_tokens: usize, + skip_full_duplicate_reclaim: bool, + ) -> PyResult { + let ct = parse_component_type(component_type)?; + let result = py.detach(move || { + self.core().drive_host_eviction_with_options( + ct, + num_tokens, + skip_full_duplicate_reclaim, ) - .into_py_any(py)?; - list.append(item)?; - } - KvCacheEvent::BlockRemoved { - block_hashes, - medium, - extra_key, - } => { - let item: Py = ( - "block_removed", - block_hashes, - medium.as_str(), - extra_key.map(|extra_key| extra_key.to_string()), - ) - .into_py_any(py)?; - list.append(item)?; - } - KvCacheEvent::AllBlocksCleared => { - let item: Py = ("all_blocks_cleared",).into_py_any(py)?; - list.append(item)?; + }); + Ok(HostEvictionResultBinding { + tracker: tracker_to_py(result.tracker), + new_device_frees: frees_to_py(py, result.device_frees)?, + new_host_frees: frees_to_py(py, result.host_frees)?, + cache_actions: cache_actions_to_py(py, result.cache_actions)?, + }) + } + + /// Evict shallow Mamba device checkpoints beyond the per-path cap on the + /// tail's root path; returns the step's freed tensors. + fn evict_excess_path_states( + &self, + py: Python<'_>, + tail_node_id: NodeId, + ) -> PyResult { + let result = py + .detach(move || self.core().evict_excess_path_states(tail_node_id)) + .map_err(node_access_error)?; + Ok(HostEvictionResultBinding { + tracker: tracker_to_py(result.tracker), + new_device_frees: frees_to_py(py, result.device_frees)?, + new_host_frees: frees_to_py(py, result.host_frees)?, + cache_actions: cache_actions_to_py(py, result.cache_actions)?, + }) + } + + /// Bump the reference count on a node's host-side component locks. + fn inc_host_lock_ref( + &self, + py: Python<'_>, + node_id: NodeId, + ) -> PyResult { + let result = py + .detach(|| self.core().inc_host_lock_ref(node_id)) + .map_err(node_access_error)?; + Ok(IncLockRefResultBinding::from_result(result)) + } + + /// Decrease the reference count on a node's host-side component locks. + fn dec_host_lock_ref( + &self, + py: Python<'_>, + node_id: NodeId, + params: &DecLockRefParamsBinding, + ) -> PyResult<()> { + let params = params.to_dec_lock_ref_params()?; + py.detach(|| self.core().dec_host_lock_ref(node_id, ¶ms)) + .map_err(node_access_error)?; + Ok(()) + } + + fn is_write_through_compatible(&self, py: Python<'_>) -> bool { + py.detach(|| self.core().is_write_through_compatible()) + } + + /// Set the write-back (vs write-through) policy; decided at HiCache init. + fn set_is_write_back(&self, py: Python<'_>, is_write_back: bool) { + py.detach(|| self.core().set_is_write_back(is_write_back)); + } + + /// The current write-back (vs write-through) policy. + fn is_write_back(&self, py: Python<'_>) -> bool { + py.detach(|| self.core().is_write_back()) + } + + /// Set the write-through backup hit threshold; decided at HiCache init. + fn set_write_through_threshold(&self, py: Python<'_>, threshold: i64) { + py.detach(|| self.core().set_write_through_threshold(threshold)); + } + + /// The current write-through backup hit threshold. + fn write_through_threshold(&self, py: Python<'_>) -> i64 { + py.detach(|| self.core().write_through_threshold()) + } + + /// Mark the storage tier (L3) wired; storage attaches after tree construction. + fn set_enable_storage(&self, py: Python<'_>, value: bool) { + py.detach(|| self.core().set_enable_storage(value)); + } + + /// Whether the storage tier (L3) is wired. + fn enable_storage(&self, py: Python<'_>) -> bool { + py.detach(|| self.core().enable_storage()) + } + + /// Enable or disable the direct external-cache linker. + fn set_enable_external_cache_linker(&self, py: Python<'_>, value: bool) -> PyResult<()> { + py.detach(|| self.core().set_enable_external_cache_linker(value)) + .map_err(tree_core_assertion_error) + } + + /// Whether the direct external-cache linker is wired. + fn enable_external_cache_linker(&self, py: Python<'_>) -> bool { + py.detach(|| self.core().enable_external_cache_linker()) + } + + /// Queue the all-cleared placement event. + fn record_all_cleared_event(&self, py: Python<'_>) { + py.detach(|| self.core().record_all_cleared_event()); + } + + /// Drain the queued placement events as tagged tuples. + fn take_events(&self, py: Python<'_>) -> PyResult> + where + for<'py> Vec: IntoPyObject<'py>, + { + let events = py.detach(|| self.core().take_events()); + let list = PyList::empty(py); + for event in events { + match event { + KvCacheEvent::BlockStored { + block_hashes, + parent_block_hash, + token_ids, + block_size, + medium, + cache_salt, + session_id, + extra_key, + } => { + let item: Py = ( + "block_stored", + block_hashes, + parent_block_hash, + token_ids, + block_size, + medium.as_str(), + cache_salt.map(|salt| salt.to_string()), + session_id.map(|session_id| session_id.to_string()), + extra_key.map(|extra_key| extra_key.to_string()), + ) + .into_py_any(py)?; + list.append(item)?; + } + KvCacheEvent::BlockRemoved { + block_hashes, + medium, + extra_key, + } => { + let item: Py = ( + "block_removed", + block_hashes, + medium.as_str(), + extra_key.map(|extra_key| extra_key.to_string()), + ) + .into_py_any(py)?; + list.append(item)?; + } + KvCacheEvent::AllBlocksCleared => { + let item: Py = ("all_blocks_cleared",).into_py_any(py)?; + list.append(item)?; + } + } } + Ok(list.unbind()) + } + + /// Drop the subtree rooted at an unbacked D-leaf; not dropped when a lock + /// blocks it. + fn drop_subtree_no_host( + &self, + py: Python<'_>, + node_id: NodeId, + ) -> PyResult { + let (dropped, result) = py + .detach(move || self.core().drop_subtree_no_host(node_id)) + .map_err(node_access_error)?; + Ok(DropSubtreeResultBinding { + dropped, + tracker: tracker_to_py(result.tracker), + new_device_frees: frees_to_py(py, result.device_frees)?, + new_host_frees: frees_to_py(py, result.host_frees)?, + cache_actions: cache_actions_to_py(py, result.cache_actions)?, + }) + } + + /// Mark the nodes one write-through backup covers; returns them ancestors first. + fn mark_write_through_pending( + &self, + py: Python<'_>, + node_ids: Vec, + ack_id: NodeId, + ) -> PyResult> { + py.detach(move || self.core().mark_write_through_pending(node_ids, ack_id)) + .map_err(node_access_error) + } + + /// Clear the write-through-pending mark on the acked nodes. + fn finish_write_through( + &self, + py: Python<'_>, + node_ids: Vec, + ack_id: NodeId, + ) -> PyResult<()> { + py.detach(move || self.core().finish_write_through(node_ids, ack_id)) + .map_err(node_access_error) + } + + /// Clear the in-flight H->D marks on the anchor's root path at ack time. + fn finish_load_back(&self, py: Python<'_>, anchor_node_id: NodeId) -> PyResult<()> { + py.detach(|| self.core().finish_load_back(anchor_node_id)) + .map_err(node_access_error) + } + + /// Build transfers for a node with no stored or pending external copy. + fn build_external_linker_offload_transfers( + &self, + py: Python<'_>, + node_id: NodeId, + ) -> PyResult>>> { + let transfers = py + .detach(|| self.core().build_external_linker_offload_transfers(node_id)) + .map_err(node_access_error)?; + transfers + .map(|transfers| { + transfers + .into_iter() + .map(|transfer| transfer_to_py(py, transfer)) + .collect::>>() + }) + .transpose() + } + + /// Mark an externally restored path, excluding its existing anchor. + fn mark_external_cache_stored_path( + &self, + py: Python<'_>, + from_node_id: NodeId, + until_node_id: NodeId, + ) -> PyResult<()> { + py.detach(|| { + self.core() + .mark_external_cache_stored_path(from_node_id, until_node_id) + }) + .map_err(tree_core_runtime_error) + } + + /// Publish an accepted external offload as pending. + fn mark_external_linker_offload_pending( + &self, + py: Python<'_>, + node_id: NodeId, + ) -> PyResult<()> { + py.detach(|| self.core().mark_external_linker_offload_pending(node_id)) + .map_err(tree_core_assertion_error) + } + + /// Finalize external-store state for an offload and its split fragments. + fn finish_external_linker_offload( + &self, + py: Python<'_>, + node_ids: Vec, + ack_id: NodeId, + success: bool, + ) -> PyResult<()> { + py.detach(|| { + self.core() + .finish_external_linker_offload(&node_ids, ack_id, success) + }) + .map_err(tree_core_assertion_error) + } + + /// Order-sensitive digest of reclaimed coexisting host values. + fn write_back_coexist_reclaim_digest(&self, py: Python<'_>) -> i64 { + py.detach(|| self.core().write_back_coexist_reclaim_digest()) + } + + /// Whether the component's data is device-evicted but host-backed. + fn component_has_host_value_only( + &self, + py: Python<'_>, + node_id: NodeId, + component_type: u8, + ) -> PyResult { + let ct = parse_component_type(component_type)?; + py.detach(|| self.core().component_has_host_value_only(node_id, ct)) + .map_err(node_access_error) } } - Ok(list.unbind()) - } - /// Drop the subtree rooted at an unbacked D-leaf; not dropped when a lock - /// blocks it. - fn drop_subtree_no_host( - &self, - py: Python<'_>, - node_id: NodeId, - ) -> PyResult { - let (dropped, result) = py - .detach(move || self.core().drop_subtree_no_host(node_id)) - .map_err(node_access_error)?; - Ok(DropSubtreeResultBinding { - dropped, - tracker: tracker_to_py(result.tracker), - new_device_frees: frees_to_py(py, result.device_frees)?, - new_host_frees: frees_to_py(py, result.host_frees)?, - }) - } + #[cfg(feature = "inspection")] + impl TreeCoreBinding<$core> { + fn inspect_contains_node(&self, py: Python<'_>, node_id: NodeId) -> bool { + py.detach(|| self.core().inspect_contains_node(node_id)) + } - /// Mark the nodes one write-through backup covers; returns them ancestors first. - fn mark_write_through_pending( - &self, - py: Python<'_>, - node_ids: Vec, - ack_id: NodeId, - ) -> PyResult> { - py.detach(move || self.core().mark_write_through_pending(node_ids, ack_id)) - .map_err(node_access_error) - } + fn inspect_get_parent_node_id( + &self, + py: Python<'_>, + node_id: NodeId, + ) -> PyResult> { + py.detach(|| self.core().inspect_get_parent_node_id(node_id)) + .map_err(node_access_error) + } - /// Clear the write-through-pending mark on the acked nodes. - fn finish_write_through( - &self, - py: Python<'_>, - node_ids: Vec, - ack_id: NodeId, - ) -> PyResult<()> { - py.detach(move || self.core().finish_write_through(node_ids, ack_id)) - .map_err(node_access_error) - } + fn inspect_get_child_node_ids(&self, py: Python<'_>, node_id: NodeId) -> PyResult> { + py.detach(|| self.core().inspect_get_child_node_ids(node_id)) + .map_err(node_access_error) + } - /// Clear the in-flight H->D marks on the anchor's root path at ack time. - fn finish_load_back(&self, py: Python<'_>, anchor_node_id: NodeId) -> PyResult<()> { - py.detach(|| self.core().finish_load_back(anchor_node_id)) - .map_err(node_access_error) - } + fn inspect_get_node_key_length(&self, py: Python<'_>, node_id: NodeId) -> PyResult { + py.detach(|| self.core().inspect_get_node_key_length(node_id)) + .map_err(node_access_error) + } - /// Build transfers for a node with no stored or pending external copy. - fn build_external_linker_offload_transfers( - &self, - py: Python<'_>, - node_id: NodeId, - ) -> PyResult>>> { - let transfers = py - .detach(|| self.core().build_external_linker_offload_transfers(node_id)) - .map_err(node_access_error)?; - transfers - .map(|transfers| { - transfers + fn inspect_get_node_token_ids(&self, py: Python<'_>, node_id: NodeId) -> PyResult> { + py.detach(|| self.core().inspect_get_node_token_ids(node_id)) + .map_err(node_access_error) + } + + fn inspect_is_node_key_bigram(&self, py: Python<'_>, node_id: NodeId) -> PyResult { + py.detach(|| self.core().inspect_is_node_key_bigram(node_id)) + .map_err(node_access_error) + } + + fn inspect_get_component_host_value( + &self, + py: Python<'_>, + node_id: NodeId, + component_type: u8, + ) -> PyResult>> { + let component_type = parse_component_type(component_type)?; + let value = py + .detach(|| { + self.core() + .inspect_get_component_host_value(node_id, component_type) + }) + .map_err(node_access_error)?; + value.map(|value| tensor_to_py(py, value)).transpose() + } + + fn inspect_get_component_device_lock_ref( + &self, + py: Python<'_>, + node_id: NodeId, + component_type: u8, + ) -> PyResult { + let component_type = parse_component_type(component_type)?; + py.detach(|| { + self.core() + .inspect_get_component_device_lock_ref(node_id, component_type) + }) + .map_err(node_access_error) + } + + fn inspect_get_component_host_lock_ref( + &self, + py: Python<'_>, + node_id: NodeId, + component_type: u8, + ) -> PyResult { + let component_type = parse_component_type(component_type)?; + py.detach(|| { + self.core() + .inspect_get_component_host_lock_ref(node_id, component_type) + }) + .map_err(node_access_error) + } + + fn inspect_get_node_hit_count(&self, py: Python<'_>, node_id: NodeId) -> PyResult { + py.detach(|| self.core().inspect_get_node_hit_count(node_id)) + .map_err(node_access_error) + } + + fn inspect_get_write_through_pending_id( + &self, + py: Python<'_>, + node_id: NodeId, + ) -> PyResult> { + py.detach(|| self.core().inspect_get_write_through_pending_id(node_id)) + .map_err(node_access_error) + } + + fn inspect_is_external_cache_stored(&self, py: Python<'_>, node_id: NodeId) -> PyResult { + py.detach(|| self.core().inspect_is_external_cache_stored(node_id)) + .map_err(node_access_error) + } + + fn inspect_is_node_in_device_lru( + &self, + py: Python<'_>, + node_id: NodeId, + component_type: u8, + ) -> PyResult { + let component_type = parse_component_type(component_type)?; + py.detach(|| { + self.core() + .inspect_is_node_in_device_lru(node_id, component_type) + }) + .map_err(node_access_error) + } + + fn inspect_is_node_in_host_lru( + &self, + py: Python<'_>, + node_id: NodeId, + component_type: u8, + ) -> PyResult { + let component_type = parse_component_type(component_type)?; + py.detach(|| { + self.core() + .inspect_is_node_in_host_lru(node_id, component_type) + }) + .map_err(node_access_error) + } + + fn inspect_get_component_device_lru_node_ids( + &self, + py: Python<'_>, + component_type: u8, + ) -> PyResult> { + let component_type = parse_component_type(component_type)?; + Ok(py.detach(|| { + self.core() + .inspect_get_component_device_lru_node_ids(component_type) + })) + } + + fn inspect_is_device_evictable_leaf(&self, py: Python<'_>, node_id: NodeId) -> bool { + py.detach(|| self.core().inspect_is_device_evictable_leaf(node_id)) + } + + fn inspect_is_host_evictable_leaf(&self, py: Python<'_>, node_id: NodeId) -> bool { + py.detach(|| self.core().inspect_is_host_evictable_leaf(node_id)) + } + + fn inspect_is_device_leaf(&self, py: Python<'_>, node_id: NodeId) -> PyResult { + py.detach(|| self.core().inspect_is_device_leaf(node_id)) + .map_err(node_access_error) + } + + fn inspect_get_all_node_ids(&self, py: Python<'_>) -> Vec { + py.detach(|| self.core().inspect_get_all_node_ids()) + } + + fn inspect_component_protected_size( + &self, + py: Python<'_>, + component_type: u8, + ) -> PyResult { + let component_type = parse_component_type(component_type)?; + Ok(py.detach(|| self.core().inspect_component_protected_size(component_type))) + } + + fn inspect_set_node_hash_values( + &self, + py: Python<'_>, + node_id: NodeId, + hash_values: Option>, + ) -> PyResult<()> { + py.detach(move || { + self.core() + .inspect_set_node_hash_values(node_id, hash_values) + }) + .map_err(node_access_error) + } + + fn inspect_set_component_device_value_raw( + &self, + py: Python<'_>, + node_id: NodeId, + component_type: u8, + value: Option, + ) -> PyResult<()> { + let component_type = parse_component_type(component_type)?; + let value = value.map(|value| value.0); + py.detach(move || { + self.core() + .inspect_set_component_device_value_raw(node_id, component_type, value) + }) + .map_err(node_access_error) + } + + fn inspect_set_component_host_value_raw( + &self, + py: Python<'_>, + node_id: NodeId, + component_type: u8, + value: Option, + ) -> PyResult<()> { + let component_type = parse_component_type(component_type)?; + let value = value.map(|value| value.0); + py.detach(move || { + self.core() + .inspect_set_component_host_value_raw(node_id, component_type, value) + }) + .map_err(node_access_error) + } + + fn inspect_set_component_device_lock_ref( + &self, + py: Python<'_>, + node_id: NodeId, + component_type: u8, + lock_ref: u32, + ) -> PyResult<()> { + let component_type = parse_component_type(component_type)?; + py.detach(|| { + self.core() + .inspect_set_component_device_lock_ref(node_id, component_type, lock_ref) + }) + .map_err(node_access_error) + } + + fn inspect_remove_node_from_device_lru( + &self, + py: Python<'_>, + node_id: NodeId, + component_type: u8, + ) -> PyResult<()> { + let component_type = parse_component_type(component_type)?; + py.detach(|| { + self.core() + .inspect_remove_node_from_device_lru(node_id, component_type) + }) + .map_err(node_access_error) + } + + fn inspect_insert_node_into_host_lru( + &self, + py: Python<'_>, + node_id: NodeId, + component_type: u8, + ) -> PyResult<()> { + let component_type = parse_component_type(component_type)?; + py.detach(|| { + self.core() + .inspect_insert_node_into_host_lru(node_id, component_type) + }) + .map_err(node_access_error) + } + + fn inspect_set_component_evictable_size( + &self, + py: Python<'_>, + component_type: u8, + value: usize, + ) -> PyResult<()> { + let component_type = parse_component_type(component_type)?; + py.detach(|| { + self.core() + .inspect_set_component_evictable_size(component_type, value) + }); + Ok(()) + } + + fn inspect_set_component_protected_size( + &self, + py: Python<'_>, + component_type: u8, + value: usize, + ) -> PyResult<()> { + let component_type = parse_component_type(component_type)?; + py.detach(|| { + self.core() + .inspect_set_component_protected_size(component_type, value) + }); + Ok(()) + } + + fn inspect_update_duplicate_tracking(&self, py: Python<'_>, node_id: NodeId) -> PyResult<()> { + py.detach(|| self.core().inspect_update_duplicate_tracking(node_id)) + .map_err(node_access_error) + } + + fn inspect_advance_insert_walk_once(&self, py: Python<'_>) -> PyResult<()> { + py.detach(|| self.core().inspect_advance_insert_walk_once()) + .map_err(PyRuntimeError::new_err) + } + + fn inspect_evict_component( + &self, + py: Python<'_>, + node_id: NodeId, + component_type: u8, + target: u8, + ) -> PyResult { + let component_type = parse_component_type(component_type)?; + let target = parse_evict_layer(target)?; + let result = py + .detach(|| { + self.core() + .inspect_evict_component(node_id, component_type, target) + }) + .map_err(node_access_error)?; + HostEvictionResultBinding::from_eviction_step(py, result) + } + + fn inspect_validate_cascade_evict( + &self, + py: Python<'_>, + node_id: NodeId, + component_type: u8, + target: u8, + ) -> PyResult<()> { + let component_type = parse_component_type(component_type)?; + let target = parse_evict_layer(target)?; + py.detach(|| { + self.core() + .inspect_validate_cascade_evict(node_id, component_type, target) + }) + .map_err(tree_core_assertion_error) + } + + fn inspect_cleanup_tombstone_ancestors( + &self, + py: Python<'_>, + node_id: NodeId, + ) -> PyResult { + let result = py + .detach(|| self.core().inspect_cleanup_tombstone_ancestors(node_id)) + .map_err(node_access_error)?; + HostEvictionResultBinding::from_eviction_step(py, result) + } + + #[allow(clippy::too_many_arguments)] + fn inspect_finalize_component_match_result( + &self, + py: Python<'_>, + component_type: u8, + result: InspectionMatchResultInput, + key: &Bound<'_, PyAny>, + extra_key: Option, + cache_salt: Option, + value_chunks: Vec, + best_value_len: usize, + ) -> PyResult { + let component_type = parse_component_type(component_type)?; + let key = K::key_from(Cow::Owned(py_array_to_vec_i64(py, key)?)).into_owned(); + let InspectionMatchResultInput { + device_prefix_len, + last_device_node: last_device_node_id, + last_host_node: last_host_node_id, + best_match_node: best_match_node_id, + host_hit_length, + swa_host_hit_length, + swa_branching_seqlen, + mamba_host_hit_length, + mamba_branching_seqlen, + full_kv_hit_length, + } = result; + let result = MatchResult { + device_prefix_len, + last_device_node_id, + last_host_node_id, + best_match_node_id, + host_hit_length, + swa_host_hit_length, + swa_branching_seqlen, + mamba_host_hit_length, + mamba_branching_seqlen, + full_kv_hit_length, + cache_actions: Vec::new(), + }; + let value_chunks = value_chunks .into_iter() - .map(|transfer| transfer_to_py(py, transfer)) - .collect::>>() - }) - .transpose() - } + .map(|value| value.0) + .collect::>(); + let result = py.detach(move || { + let params = MatchPrefixParams { + key: &key, + namespace: KeyNamespaceRef::new(extra_key.as_deref(), cache_salt.as_deref()), + }; + self.core().inspect_finalize_component_match_result( + component_type, + result, + ¶ms, + &value_chunks, + best_value_len, + ) + }); + let result = result.map_err(node_access_error)?; + MatchResultBinding::from_match_result(py, result) + } - /// Mark an externally restored path, excluding its existing anchor. - fn mark_external_cache_stored_path( - &self, - py: Python<'_>, - from_node_id: NodeId, - until_node_id: NodeId, - ) -> PyResult<()> { - py.detach(|| { - self.core() - .mark_external_cache_stored_path(from_node_id, until_node_id) - }) - .map_err(tree_core_runtime_error) - } + fn inspect_build_backup_node_ids( + &self, + py: Python<'_>, + node_id: NodeId, + write_back: bool, + ) -> PyResult> { + py.detach(|| { + self.core() + .inspect_build_backup_node_ids(node_id, write_back) + }) + .map_err(node_access_error) + } + } - /// Publish an accepted external offload as pending. - fn mark_external_linker_offload_pending( - &self, - py: Python<'_>, - node_id: NodeId, - ) -> PyResult<()> { - py.detach(|| self.core().mark_external_linker_offload_pending(node_id)) - .map_err(tree_core_assertion_error) - } - - /// Finalize external-store state for an offload and its split fragments. - fn finish_external_linker_offload( - &self, - py: Python<'_>, - node_ids: Vec, - ack_id: NodeId, - success: bool, - ) -> PyResult<()> { - py.detach(|| { - self.core() - .finish_external_linker_offload(&node_ids, ack_id, success) - }) - .map_err(tree_core_assertion_error) - } - - /// Order-sensitive digest of reclaimed coexisting host values. - fn write_back_coexist_reclaim_digest(&self, py: Python<'_>) -> i64 { - py.detach(|| self.core().write_back_coexist_reclaim_digest) - } - - /// Whether the component's data is device-evicted but host-backed. - fn component_has_host_value_only( - &self, - py: Python<'_>, - node_id: NodeId, - component_type: u8, - ) -> PyResult { - let ct = parse_component_type(component_type)?; - py.detach(|| self.core().component_has_host_value_only(node_id, ct)) - .map_err(node_access_error) - } -} - -#[cfg(feature = "inspection")] -impl TreeCoreBinding { - fn inspect_contains_node(&self, py: Python<'_>, node_id: NodeId) -> bool { - py.detach(|| self.core().inspect_contains_node(node_id)) - } - - fn inspect_get_parent_node_id( - &self, - py: Python<'_>, - node_id: NodeId, - ) -> PyResult> { - py.detach(|| self.core().inspect_get_parent_node_id(node_id)) - .map_err(node_access_error) - } - - fn inspect_get_child_node_ids(&self, py: Python<'_>, node_id: NodeId) -> PyResult> { - py.detach(|| self.core().inspect_get_child_node_ids(node_id)) - .map_err(node_access_error) - } - - fn inspect_get_node_key_length(&self, py: Python<'_>, node_id: NodeId) -> PyResult { - py.detach(|| self.core().inspect_get_node_key_length(node_id)) - .map_err(node_access_error) - } - - fn inspect_get_node_token_ids(&self, py: Python<'_>, node_id: NodeId) -> PyResult> { - py.detach(|| self.core().inspect_get_node_token_ids(node_id)) - .map_err(node_access_error) - } - - fn inspect_is_node_key_bigram(&self, py: Python<'_>, node_id: NodeId) -> PyResult { - py.detach(|| self.core().inspect_is_node_key_bigram(node_id)) - .map_err(node_access_error) - } - - fn inspect_get_component_host_value( - &self, - py: Python<'_>, - node_id: NodeId, - component_type: u8, - ) -> PyResult>> { - let component_type = parse_component_type(component_type)?; - let value = py - .detach(|| { - self.core() - .inspect_get_component_host_value(node_id, component_type) - }) - .map_err(node_access_error)?; - value.map(|value| tensor_to_py(py, value)).transpose() - } - - fn inspect_get_component_device_lock_ref( - &self, - py: Python<'_>, - node_id: NodeId, - component_type: u8, - ) -> PyResult { - let component_type = parse_component_type(component_type)?; - py.detach(|| { - self.core() - .inspect_get_component_device_lock_ref(node_id, component_type) - }) - .map_err(node_access_error) - } - - fn inspect_get_component_host_lock_ref( - &self, - py: Python<'_>, - node_id: NodeId, - component_type: u8, - ) -> PyResult { - let component_type = parse_component_type(component_type)?; - py.detach(|| { - self.core() - .inspect_get_component_host_lock_ref(node_id, component_type) - }) - .map_err(node_access_error) - } - - fn inspect_get_node_hit_count(&self, py: Python<'_>, node_id: NodeId) -> PyResult { - py.detach(|| self.core().inspect_get_node_hit_count(node_id)) - .map_err(node_access_error) - } - - fn inspect_get_write_through_pending_id( - &self, - py: Python<'_>, - node_id: NodeId, - ) -> PyResult> { - py.detach(|| self.core().inspect_get_write_through_pending_id(node_id)) - .map_err(node_access_error) - } - - fn inspect_is_external_cache_stored(&self, py: Python<'_>, node_id: NodeId) -> PyResult { - py.detach(|| self.core().inspect_is_external_cache_stored(node_id)) - .map_err(node_access_error) - } - - fn inspect_is_node_in_device_lru( - &self, - py: Python<'_>, - node_id: NodeId, - component_type: u8, - ) -> PyResult { - let component_type = parse_component_type(component_type)?; - py.detach(|| { - self.core() - .inspect_is_node_in_device_lru(node_id, component_type) - }) - .map_err(node_access_error) - } - - fn inspect_is_node_in_host_lru( - &self, - py: Python<'_>, - node_id: NodeId, - component_type: u8, - ) -> PyResult { - let component_type = parse_component_type(component_type)?; - py.detach(|| { - self.core() - .inspect_is_node_in_host_lru(node_id, component_type) - }) - .map_err(node_access_error) - } - - fn inspect_get_component_device_lru_node_ids( - &self, - py: Python<'_>, - component_type: u8, - ) -> PyResult> { - let component_type = parse_component_type(component_type)?; - Ok(py.detach(|| { - self.core() - .inspect_get_component_device_lru_node_ids(component_type) - })) - } - - fn inspect_is_device_evictable_leaf(&self, py: Python<'_>, node_id: NodeId) -> bool { - py.detach(|| self.core().inspect_is_device_evictable_leaf(node_id)) - } - - fn inspect_is_host_evictable_leaf(&self, py: Python<'_>, node_id: NodeId) -> bool { - py.detach(|| self.core().inspect_is_host_evictable_leaf(node_id)) - } - - fn inspect_is_device_leaf(&self, py: Python<'_>, node_id: NodeId) -> PyResult { - py.detach(|| self.core().inspect_is_device_leaf(node_id)) - .map_err(node_access_error) - } - - fn inspect_get_all_node_ids(&self, py: Python<'_>) -> Vec { - py.detach(|| self.core().inspect_get_all_node_ids()) - } - - fn inspect_component_protected_size( - &self, - py: Python<'_>, - component_type: u8, - ) -> PyResult { - let component_type = parse_component_type(component_type)?; - Ok(py.detach(|| self.core().inspect_component_protected_size(component_type))) - } - - fn inspect_set_node_hash_values( - &self, - py: Python<'_>, - node_id: NodeId, - hash_values: Option>, - ) -> PyResult<()> { - py.detach(move || { - self.core() - .inspect_set_node_hash_values(node_id, hash_values) - }) - .map_err(node_access_error) - } - - fn inspect_set_component_device_value_raw( - &self, - py: Python<'_>, - node_id: NodeId, - component_type: u8, - value: Option, - ) -> PyResult<()> { - let component_type = parse_component_type(component_type)?; - let value = value.map(|value| value.0); - py.detach(move || { - self.core() - .inspect_set_component_device_value_raw(node_id, component_type, value) - }) - .map_err(node_access_error) - } - - fn inspect_set_component_host_value_raw( - &self, - py: Python<'_>, - node_id: NodeId, - component_type: u8, - value: Option, - ) -> PyResult<()> { - let component_type = parse_component_type(component_type)?; - let value = value.map(|value| value.0); - py.detach(move || { - self.core() - .inspect_set_component_host_value_raw(node_id, component_type, value) - }) - .map_err(node_access_error) - } - - fn inspect_set_component_device_lock_ref( - &self, - py: Python<'_>, - node_id: NodeId, - component_type: u8, - lock_ref: u32, - ) -> PyResult<()> { - let component_type = parse_component_type(component_type)?; - py.detach(|| { - self.core() - .inspect_set_component_device_lock_ref(node_id, component_type, lock_ref) - }) - .map_err(node_access_error) - } - - fn inspect_remove_node_from_device_lru( - &self, - py: Python<'_>, - node_id: NodeId, - component_type: u8, - ) -> PyResult<()> { - let component_type = parse_component_type(component_type)?; - py.detach(|| { - self.core() - .inspect_remove_node_from_device_lru(node_id, component_type) - }) - .map_err(node_access_error) - } - - fn inspect_insert_node_into_host_lru( - &self, - py: Python<'_>, - node_id: NodeId, - component_type: u8, - ) -> PyResult<()> { - let component_type = parse_component_type(component_type)?; - py.detach(|| { - self.core() - .inspect_insert_node_into_host_lru(node_id, component_type) - }) - .map_err(node_access_error) - } - - fn inspect_set_component_evictable_size( - &self, - py: Python<'_>, - component_type: u8, - value: usize, - ) -> PyResult<()> { - let component_type = parse_component_type(component_type)?; - py.detach(|| { - self.core() - .inspect_set_component_evictable_size(component_type, value) - }); - Ok(()) - } - - fn inspect_set_component_protected_size( - &self, - py: Python<'_>, - component_type: u8, - value: usize, - ) -> PyResult<()> { - let component_type = parse_component_type(component_type)?; - py.detach(|| { - self.core() - .inspect_set_component_protected_size(component_type, value) - }); - Ok(()) - } - - fn inspect_update_duplicate_tracking(&self, py: Python<'_>, node_id: NodeId) -> PyResult<()> { - py.detach(|| self.core().inspect_update_duplicate_tracking(node_id)) - .map_err(node_access_error) - } - - fn inspect_advance_insert_walk_once(&self, py: Python<'_>) -> PyResult<()> { - py.detach(|| self.core().inspect_advance_insert_walk_once()) - .map_err(PyRuntimeError::new_err) - } - - fn inspect_evict_component( - &self, - py: Python<'_>, - node_id: NodeId, - component_type: u8, - target: u8, - ) -> PyResult { - let component_type = parse_component_type(component_type)?; - let target = parse_evict_layer(target)?; - let result = py - .detach(|| { - self.core() - .inspect_evict_component(node_id, component_type, target) - }) - .map_err(node_access_error)?; - HostEvictionResultBinding::from_eviction_step(py, result) - } - - fn inspect_validate_cascade_evict( - &self, - py: Python<'_>, - node_id: NodeId, - component_type: u8, - target: u8, - ) -> PyResult<()> { - let component_type = parse_component_type(component_type)?; - let target = parse_evict_layer(target)?; - py.detach(|| { - self.core() - .inspect_validate_cascade_evict(node_id, component_type, target) - }) - .map_err(tree_core_assertion_error) - } - - fn inspect_cleanup_tombstone_ancestors( - &self, - py: Python<'_>, - node_id: NodeId, - ) -> PyResult { - let result = py - .detach(|| self.core().inspect_cleanup_tombstone_ancestors(node_id)) - .map_err(node_access_error)?; - HostEvictionResultBinding::from_eviction_step(py, result) - } - - #[allow(clippy::too_many_arguments)] - fn inspect_finalize_component_match_result( - &self, - py: Python<'_>, - component_type: u8, - result: InspectionMatchResultInput, - key: &Bound<'_, PyAny>, - extra_key: Option, - cache_salt: Option, - value_chunks: Vec, - best_value_len: usize, - ) -> PyResult { - let component_type = parse_component_type(component_type)?; - let key = K::key_from(Cow::Owned(py_array_to_vec_i64(py, key)?)).into_owned(); - let InspectionMatchResultInput { - device_prefix_len, - last_device_node: last_device_node_id, - last_host_node: last_host_node_id, - best_match_node: best_match_node_id, - host_hit_length, - swa_host_hit_length, - swa_branching_seqlen, - mamba_host_hit_length, - mamba_branching_seqlen, - full_kv_hit_length, - } = result; - let result = MatchResult { - device_prefix_len, - last_device_node_id, - last_host_node_id, - best_match_node_id, - host_hit_length, - swa_host_hit_length, - swa_branching_seqlen, - mamba_host_hit_length, - mamba_branching_seqlen, - full_kv_hit_length, - cache_actions: Vec::new(), - }; - let value_chunks = value_chunks - .into_iter() - .map(|value| value.0) - .collect::>(); - let result = py.detach(move || { - let params = MatchPrefixParams { - key: &key, - namespace: KeyNamespaceRef::new(extra_key.as_deref(), cache_salt.as_deref()), - }; - self.core().inspect_finalize_component_match_result( - component_type, - result, - ¶ms, - &value_chunks, - best_value_len, - ) - }); - let result = result.map_err(node_access_error)?; - MatchResultBinding::from_match_result(py, result) - } - - fn inspect_build_backup_node_ids( - &self, - py: Python<'_>, - node_id: NodeId, - write_back: bool, - ) -> PyResult> { - py.detach(|| { - self.core() - .inspect_build_backup_node_ids(node_id, write_back) - }) - .map_err(node_access_error) - } -} - -impl TreeCoreBinding { - /// Print the tree structure for debugging. - fn pretty_print(&self, py: Python<'_>) { - py.detach(|| self.core().pretty_print()); - } + impl TreeCoreBinding<$core> { + /// Print the tree structure for debugging. + fn pretty_print(&self, py: Python<'_>) { + py.detach(|| self.core().pretty_print()); + } + } + }; } // The delegate surface is identical for every key type; the macro stamps the // pyclass + pymethods pair per concrete key. macro_rules! tree_core_binding { - ($(#[$doc:meta])* $name:ident, $key:ty) => { + ( + core = $core:ident; + $($(#[$doc:meta])* $name:ident, $key:ty $(, { $($extra:item)* })?;)+ + ) => { + tree_core_methods!($core); + $( $(#[$doc])* #[pyclass] pub struct $name { - inner: TreeCoreBinding<$key>, + inner: TreeCoreBinding<$core<$key>>, } #[pymethods] @@ -2783,7 +2796,7 @@ macro_rules! tree_core_binding { fn new(init_params: &TreeCoreInitParamsBinding, component_types: Vec) -> PyResult { catch_native_panic(|| { Ok($name { - inner: TreeCoreBinding::new(init_params, component_types)?, + inner: TreeCoreBinding::<$core<$key>>::new(init_params, component_types)?, }) }) } @@ -2917,7 +2930,7 @@ macro_rules! tree_core_binding { node_id: NodeId, component_type: u8, params: &DecLockRefParamsBinding, - ) -> PyResult<(Py, Py)> { + ) -> PyResult<(Py, Py, Py)> { catch_native_panic(|| { self.inner .dec_window_lock_only(py, node_id, component_type, params) @@ -2925,14 +2938,14 @@ macro_rules! tree_core_binding { } /// Early-release the SWA portion of a request's tree lock; returns this - /// release's per-component (device_frees, host_frees). + /// release's per-component frees and cache actions. #[pyo3(signature = (node_id, params))] fn dec_swa_lock_only( &self, py: Python<'_>, node_id: NodeId, params: &DecLockRefParamsBinding, - ) -> PyResult<(Py, Py)> { + ) -> PyResult<(Py, Py, Py)> { catch_native_panic(|| self.inner.dec_swa_lock_only(py, node_id, params)) } @@ -3199,6 +3212,14 @@ macro_rules! tree_core_binding { }) } + fn build_backup_kv_action( + &self, + py: Python<'_>, + node_id: NodeId, + ) -> PyResult> { + catch_native_panic(|| self.inner.build_backup_kv_action(py, node_id)) + } + /// Gather a node's device value plus per-component BACKUP_HOST transfers. fn build_backup_spec( &self, @@ -3983,22 +4004,23 @@ macro_rules! tree_core_binding { Ok(()) }) } + $($($extra)*)? } + )+ }; } -tree_core_binding!( +tree_core_binding! { + core = UnifiedTreeCore; /// The UnifiedTreeCore Python adapter over single-token (unigram) child keys. RustUnifiedTreeCoreBinding, - Vec -); + Vec; -tree_core_binding!( /// The UnifiedTreeCore Python adapter over bigram (EAGLE) child keys; keys /// cross the boundary as raw token ids and pair up rust-side. RustBigramUnifiedTreeCoreBinding, - Vec<(i64, i64)> -); + Vec<(i64, i64)>; +} /// Per-page chained hashes over raw token ids. #[pyfunction] diff --git a/rust/sglang-radix-tree/src/tests/components/base.rs b/rust/sglang-radix-tree/src/tests/components/base.rs index b53f313fd5..1e78fef850 100644 --- a/rust/sglang-radix-tree/src/tests/components/base.rs +++ b/rust/sglang-radix-tree/src/tests/components/base.rs @@ -1,13 +1,36 @@ use super::*; -use crate::components::FULL; +use crate::components::{FULL, FullComponent, MAMBA, MambaComponent, SWA, SwaComponent}; use crate::unified_tree_core::CacheInitParams; +use std::sync::{Arc, Mutex}; -// Test-only component exercising the trait defaults; abstract hooks stay unimplemented. -struct DefaultComponentForTest; +// Exercise trait defaults and trace delegated FULL evictions. +struct DefaultComponentForTest { + component_type: ComponentType, + count_scale: usize, + removed: Mutex>, +} + +impl Default for DefaultComponentForTest { + fn default() -> Self { + Self { + component_type: FULL, + count_scale: 1, + removed: Mutex::default(), + } + } +} impl TreeComponent> for DefaultComponentForTest { fn component_type(&self) -> ComponentType { - FULL + self.component_type + } + + fn reclaimable_value_len(&self, node: &Node>) -> usize { + node.device_reclaimable_value_len(self.component_type) * self.count_scale + } + + fn on_node_removed(&self, node_id: NodeId) { + self.removed.lock().unwrap().push(node_id); } fn create_match_validator( @@ -33,9 +56,20 @@ impl TreeComponent> for DefaultComponentForTest { node_id: NodeIdx_, device_frees: &mut HashMap>, host_frees: &mut HashMap>, + cache_actions: &mut Vec, target: EvictLayer, ) -> (usize, usize) { - unimplemented!() + cache_actions.push(CacheAction::MambaEvictExcessPathStates { + tail_node_id: tree_core.arena.node(node_id).id, + }); + FullComponent.evict_component( + tree_core, + node_id, + device_frees, + host_frees, + cache_actions, + target, + ) } fn evict_device_start(&self, tree_core: &mut UnifiedTreeCore>, request_cnt: usize) { @@ -48,6 +82,7 @@ impl TreeComponent> for DefaultComponentForTest { tracker: &mut HashMap, device_frees: &mut HashMap>, host_frees: &mut HashMap>, + cache_actions: &mut Vec, ) -> Option { unimplemented!() } @@ -82,7 +117,7 @@ fn insert_overlap_default_consumes_nothing() { let mut tc: UnifiedTreeCore> = UnifiedTreeCore::new(CacheInitParams::default(), vec![FULL]); let root = tc.arena.root(); - let consumed_from = DefaultComponentForTest.update_component_on_insert_overlap( + let consumed_from = DefaultComponentForTest::default().update_component_on_insert_overlap( &mut tc, root, /* prefix_len = */ 3, @@ -102,6 +137,7 @@ fn insert_overlap_default_consumes_nothing() { session_id: None, track_adopted_ranges: false, }, + false, &mut InsertResult::default(), &mut Vec::new(), ); @@ -119,7 +155,7 @@ fn finalize_match_result_default_returns_result_unchanged() { host_hit_length: 11, ..tc.empty_match_result() }; - let out = DefaultComponentForTest.finalize_match_result_in_tree_core( + let out = DefaultComponentForTest::default().finalize_match_result_in_tree_core( &tc, result, tc.arena.root(), @@ -143,12 +179,13 @@ fn drive_host_eviction_default_is_a_noop() { let mut tracker = HashMap::from([(FULL, 5usize)]); let mut device_frees = HashMap::new(); let mut host_frees = HashMap::new(); - DefaultComponentForTest.drive_host_eviction( + DefaultComponentForTest::default().drive_host_eviction( &mut tc, /* num_tokens = */ 100, &mut tracker, &mut device_frees, &mut host_frees, + &mut Vec::new(), ); assert_eq!(tracker[&FULL], 5); assert!(device_frees.is_empty()); @@ -163,3 +200,228 @@ fn idx_matches_discriminants() { assert_eq!(ComponentType::Swa.idx(), 1); assert_eq!(ComponentType::Mamba.idx(), 2); } + +fn action_core() -> UnifiedTreeCore> { + UnifiedTreeCore::new_with_components_( + CacheInitParams { + enable_hicache: true, + ..Default::default() + }, + vec![Arc::new(DefaultComponentForTest::default())], + ) +} + +fn action_node( + core: &mut UnifiedTreeCore>, + parent: NodeIdx_, + key: i64, + device: bool, + host: bool, +) -> NodeIdx_ { + let node = core.arena.alloc_child(parent, vec![key], 0, None).unwrap(); + if device { + core.arena + .set_device_value(node, FULL, Tensor::from_slice(&[key])); + core.inc_evictable_size(FULL, 1); + } + if host { + core.arena + .set_host_value(node, FULL, Tensor::from_slice(&[key + 100])); + } + core.update_evictable_leaf_sets_(node); + core.update_evictable_leaf_sets_(parent); + core.update_full_coexisting_host_tracking_(node); + node +} + +fn action_node_ids(actions: &[CacheAction]) -> Vec { + actions + .iter() + .map(|action| match action { + CacheAction::MambaEvictExcessPathStates { tail_node_id } => *tail_node_id, + _ => panic!("unexpected component action"), + }) + .collect() +} + +#[test] +fn leaf_eviction_results_preserve_component_actions() { + for demote in [false, true] { + let mut core = action_core(); + let root = core.arena.root(); + let node = action_node(&mut core, root, 1, true, demote); + let node_id = core.arena.node(node).id; + let result = if demote { + core.demote(node_id).unwrap() + } else { + core.evict_device_leaf(node_id, false).unwrap().1 + }; + assert_eq!(action_node_ids(&result.cache_actions), [node_id]); + assert_eq!(result.device_frees[&FULL].len(), 1); + assert_eq!(result.tracker[&FULL], 1); + } +} + +#[test] +fn subtree_cleanup_returns_child_actions_before_parent_actions() { + let mut core = action_core(); + let root = core.arena.root(); + let parent = action_node(&mut core, root, 1, true, false); + let child = action_node(&mut core, parent, 2, false, true); + let parent_id = core.arena.node(parent).id; + let child_id = core.arena.node(child).id; + let (dropped, result) = core.drop_subtree_no_host(parent_id).unwrap(); + assert!(dropped); + assert_eq!( + action_node_ids(&result.cache_actions), + [child_id, parent_id] + ); + assert_eq!(result.device_frees[&FULL].len(), 1); + assert_eq!(result.host_frees[&FULL].len(), 1); +} + +#[test] +fn host_drivers_forward_component_actions_in_order() { + for coexisting in [false, true] { + let mut core = action_core(); + let root = core.arena.root(); + let nodes = [ + action_node(&mut core, root, 1, coexisting, true), + action_node(&mut core, root, 2, coexisting, true), + ]; + let ids = nodes.map(|node| core.arena.node(node).id); + let mut tracker = HashMap::from([(FULL, 0)]); + let mut device_frees = HashMap::new(); + let mut host_frees = HashMap::new(); + let mut actions = Vec::new(); + if coexisting { + FullComponent.reclaim_coexisting_host_values( + &mut core, + 2, + &mut tracker, + &mut device_frees, + &mut host_frees, + &mut actions, + ); + } else { + FullComponent.drive_host_eviction( + &mut core, + 2, + &mut tracker, + &mut device_frees, + &mut host_frees, + &mut actions, + ); + } + assert_eq!(action_node_ids(&actions), ids); + assert_eq!(tracker[&FULL], 2); + assert!(device_frees.is_empty()); + assert_eq!(host_frees[&FULL].len(), 2); + } +} + +#[test] +fn reused_drivers_count_the_configured_components_backing() { + for component_type in [FULL, SWA, MAMBA] { + let params = CacheInitParams { + swa_sliding_window_size: Some(4), + mamba_cache_chunk_size: Some(1), + ..Default::default() + }; + let counted = Arc::new(DefaultComponentForTest { + component_type, + count_scale: 2, + ..Default::default() + }); + let mut components: Vec> + Send + Sync>> = Vec::new(); + if component_type != FULL { + components.push(Arc::new(FullComponent)); + } + components.push(counted.clone()); + let driver: Box>> = if component_type == FULL { + Box::new(FullComponent) + } else if component_type == SWA { + Box::new(SwaComponent::new(¶ms)) + } else { + Box::new(MambaComponent::new(¶ms)) + }; + let mut core = UnifiedTreeCore::new_with_components_(params, components); + let root = core.arena.root(); + let node = core.add_new_node_( + root, + vec![1, 2, 3, 4], + &Tensor::from_slice(&[10i64, 11, 12, 13]), + 0, + None, + ); + let node_id = core.arena.node(node).id; + let logical_len = if component_type == MAMBA { 1 } else { 4 }; + if component_type != FULL { + assert_eq!(counted.reclaimable_value_len(core.arena.node(node)), 0); + core.set_component_device_value( + node_id, + component_type, + Tensor::from_slice(&vec![20i64; logical_len]), + ) + .unwrap(); + } + let counted_len = logical_len * 2; + assert_eq!( + core.arena + .node(node) + .device_reclaimable_value_len(component_type), + logical_len + ); + assert_eq!(counted.reclaimable_value_len(core.arena.node(root)), 0); + assert_eq!(core.component_evictable_size(component_type), counted_len); + core.try_sanity_check(&[], &[]).unwrap(); + + // These ordinary drivers must consult their configured wrapper's count. + let receipt = + driver.acquire_component_lock(&mut core, node, IncLockRefResult::default(), false); + driver.acquire_component_lock(&mut core, node, IncLockRefResult::default(), false); + assert_eq!(core.component_evictable_size(component_type), 0); + assert_eq!(core.component_protected_size(component_type), counted_len); + driver.release_component_lock(&mut core, node, &receipt.to_dec_params(), false); + assert_eq!(core.component_protected_size(component_type), counted_len); + driver.release_component_lock(&mut core, node, &receipt.to_dec_params(), false); + assert_eq!(core.component_evictable_size(component_type), counted_len); + assert_eq!(core.component_protected_size(component_type), 0); + core.try_sanity_check(&[], &[]).unwrap(); + let mut device_frees = HashMap::new(); + let (freed, _) = driver.evict_component( + &mut core, + node, + &mut device_frees, + &mut HashMap::new(), + &mut Vec::new(), + EvictLayer::Device, + ); + assert_eq!(freed, counted_len); + assert_eq!(core.component_evictable_size(component_type), 0); + assert_eq!(device_frees[&component_type].len(), 1); + } +} + +#[test] +fn removal_notifies_components_only_after_success() { + let observer = Arc::new(DefaultComponentForTest::default()); + let mut core = + UnifiedTreeCore::new_with_components_(CacheInitParams::default(), vec![observer.clone()]); + let root = core.arena.root(); + let parent = core.arena.alloc_child(root, vec![1], 0, None).unwrap(); + let child = core.arena.alloc_child(parent, vec![2], 0, None).unwrap(); + let parent_id = core.arena.node(parent).id; + let child_id = core.arena.node(child).id; + let rejected = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + core.remove_leaf_from_parent_(parent) + })); + assert!(rejected.is_err()); + assert!(observer.removed.lock().unwrap().is_empty()); + assert!(core.arena.resolve(parent_id).is_ok()); + core.remove_leaf_from_parent_(child); + core.remove_leaf_from_parent_(parent); + assert_eq!(*observer.removed.lock().unwrap(), [child_id, parent_id]); + assert!(core.arena.resolve(child_id).is_err()); + assert!(core.arena.resolve(parent_id).is_err()); +} diff --git a/rust/sglang-radix-tree/src/tests/components/full.rs b/rust/sglang-radix-tree/src/tests/components/full.rs index f621eac0a6..bec7644063 100644 --- a/rust/sglang-radix-tree/src/tests/components/full.rs +++ b/rust/sglang-radix-tree/src/tests/components/full.rs @@ -2499,6 +2499,7 @@ fn evict_device_pushes_value_and_decrements_size() { node, &mut device_frees, &mut host_frees, + &mut Vec::new(), EvictLayer::Device, ); assert_eq!(freed, 3); @@ -2532,6 +2533,7 @@ fn evict_device_on_valueless_node_is_noop() { node, &mut device_frees, &mut host_frees, + &mut Vec::new(), EvictLayer::Device, ); assert_eq!(freed, 0); @@ -2567,6 +2569,7 @@ fn evict_device_pushes_a_present_but_empty_value() { node, &mut device_frees, &mut host_frees, + &mut Vec::new(), EvictLayer::Device, ); assert_eq!(freed, 0); @@ -2598,6 +2601,7 @@ fn evict_host_pushes_host_value_and_tombstones() { node, &mut device_frees, &mut host_frees, + &mut Vec::new(), EvictLayer::Host, ); assert_eq!(freed, 0); @@ -2633,6 +2637,7 @@ fn evict_host_on_node_without_host_value_is_noop() { node, &mut device_frees, &mut host_frees, + &mut Vec::new(), EvictLayer::Host, ); assert_eq!(freed, 0); @@ -2667,6 +2672,7 @@ fn evict_host_pushes_a_present_but_empty_host_value() { node, &mut device_frees, &mut host_frees, + &mut Vec::new(), EvictLayer::Host, ); assert_eq!(host_freed, 0); @@ -2707,6 +2713,7 @@ fn evict_host_accumulates_into_shared_host_frees() { a, &mut device_frees, &mut host_frees, + &mut Vec::new(), EvictLayer::Host, ); let freed_b = FullComponent.evict_component( @@ -2714,6 +2721,7 @@ fn evict_host_accumulates_into_shared_host_frees() { b, &mut device_frees, &mut host_frees, + &mut Vec::new(), EvictLayer::Host, ); assert_eq!(freed_a, (0, 2)); @@ -2749,6 +2757,7 @@ fn evict_device_leaves_host_value_untouched() { node, &mut device_frees, &mut host_frees, + &mut Vec::new(), EvictLayer::Device, ); assert_eq!(freed, 1); @@ -2783,6 +2792,7 @@ fn evict_all_frees_both_layers() { node, &mut device_frees, &mut host_frees, + &mut Vec::new(), EvictLayer::All, ); assert_eq!(freed, 3); @@ -2830,6 +2840,7 @@ fn evict_device_accumulates_into_shared_device_frees() { a, &mut device_frees, &mut host_frees, + &mut Vec::new(), EvictLayer::Device, ); FullComponent.evict_component( @@ -2837,6 +2848,7 @@ fn evict_device_accumulates_into_shared_device_frees() { b, &mut device_frees, &mut host_frees, + &mut Vec::new(), EvictLayer::Device, ); let pushed = &device_frees[&FULL]; @@ -2869,6 +2881,7 @@ fn evict_device_pushed_entry_aliases_the_node_value() { node, &mut device_frees, &mut host_frees, + &mut Vec::new(), EvictLayer::Device, ); // The pushed entry shares storage with the node's value (Python appends cd.value). @@ -2891,6 +2904,7 @@ fn evict_panics_on_missing_node() { NodeIdx_(999), &mut device_frees, &mut host_frees, + &mut Vec::new(), EvictLayer::Device, ); } @@ -2919,6 +2933,7 @@ fn evict_panics_when_size_would_underflow() { node, &mut device_frees, &mut host_frees, + &mut Vec::new(), EvictLayer::Device, ); } diff --git a/rust/sglang-radix-tree/src/tests/components/mamba.rs b/rust/sglang-radix-tree/src/tests/components/mamba.rs index 86f5df84af..a37ec593b5 100644 --- a/rust/sglang-radix-tree/src/tests/components/mamba.rs +++ b/rust/sglang-radix-tree/src/tests/components/mamba.rs @@ -381,6 +381,7 @@ fn swa_only_release_spares_another_holders_mamba_lock() { &holder_params, &mut device_frees, &mut host_frees, + &mut Vec::new(), ) .expect("live test node"); @@ -740,6 +741,7 @@ fn evict_component_device_frees_and_tombstones_the_slot() { a, &mut device_frees, &mut host_frees, + &mut Vec::new(), EvictLayer::Device, ); assert_eq!((freed, host_freed), (1, 0)); @@ -764,6 +766,7 @@ fn evict_component_device_moves_a_host_backed_node_into_the_host_lru() { a, &mut device_frees, &mut host_frees, + &mut Vec::new(), EvictLayer::Device, ); assert!(tc.arena.node(a).has_host_value(MAMBA)); @@ -783,6 +786,7 @@ fn evict_component_host_frees_and_leaves_the_host_lru() { a, &mut device_frees, &mut host_frees, + &mut Vec::new(), EvictLayer::Host, ); assert_eq!((freed, host_freed), (0, 1)); @@ -804,6 +808,7 @@ fn evict_component_all_frees_both_tiers_without_the_host_lru() { a, &mut device_frees, &mut host_frees, + &mut Vec::new(), EvictLayer::All, ); assert_eq!((freed, host_freed), (1, 1)); @@ -1295,6 +1300,7 @@ fn host_eviction_tombstones_internal_host_values() { &mut tracker, &mut device_frees, &mut host_frees, + &mut Vec::new(), ); assert!(!tc.arena.node(a).has_host_value(MAMBA)); assert!(!tc.host_lru_list(MAMBA).in_list(Some(a))); @@ -1334,6 +1340,7 @@ fn host_eviction_takes_a_host_leaf_atomically() { &mut tracker, &mut device_frees, &mut host_frees, + &mut Vec::new(), ); // The atomic host-leaf eviction frees both components and the node. assert_eq!(tracker[&MAMBA], 1); @@ -1358,6 +1365,7 @@ fn host_drive_panics_on_an_lru_member_without_a_mamba_host_value() { &mut tracker, &mut HashMap::new(), &mut HashMap::new(), + &mut Vec::new(), ); } @@ -1384,6 +1392,7 @@ fn swa_triggered_cascade_takes_the_mamba_slot() { &mut tracker, &mut device_frees, &mut host_frees, + &mut Vec::new(), EvictLayer::Device, ); assert!(!tc.arena.node(a).has_device_value(MAMBA)); @@ -1415,6 +1424,7 @@ fn mamba_triggered_cascade_spares_the_higher_tiers() { &mut tracker, &mut device_frees, &mut host_frees, + &mut Vec::new(), EvictLayer::Device, ); // FULL (2) and SWA (1) both outrank the mamba trigger (0). diff --git a/rust/sglang-radix-tree/src/tests/components/swa.rs b/rust/sglang-radix-tree/src/tests/components/swa.rs index 851f7caeb5..5d5b8abf9e 100644 --- a/rust/sglang-radix-tree/src/tests/components/swa.rs +++ b/rust/sglang-radix-tree/src/tests/components/swa.rs @@ -1118,6 +1118,55 @@ fn insert_overlap_recovers_a_locked_swa_tombstone() { ); } +#[test] +fn insert_overlap_can_preserve_unlocked_full_ids() { + for window_start in [0, 2] { + let mut tc = swa_core(/* window = */ 8, /* page_size = */ 1); + let key = vec![1, 2, 3, 4]; + tc.insert(&insert_params_swa(&key, &[10, 11, 12, 13], 0, 0)); + let root = tc.arena.root(); + let leaf = child_of(&tc, root, &[1]); + let original_id = tc.arena.node(leaf).id; + let params = insert_params_swa(&key, &[20, 21, 22, 23], 0, window_start); + let mut result = InsertResult { + adopted_ranges: Some(HashMap::new()), + ..Default::default() + }; + let mut actions = Vec::new(); + let consumed_from = swa_component(8).update_component_on_insert_overlap( + &mut tc, + leaf, + 4, + 0, + params.value.shallow_clone(), + ¶ms, + true, + &mut result, + &mut actions, + ); + assert_eq!(consumed_from, window_start); + assert_eq!(tc.arena.node(leaf).id, original_id); + assert_eq!(tc.arena.node(leaf).device_lock_ref(FULL), 0); + let [ + CacheAction::RecoverSwaWithLockedFull { + node_id, + kept_full, + incoming_full, + }, + ] = actions.as_slice() + else { + panic!("expected recovery with canonical FULL IDs"); + }; + assert_eq!(*node_id, original_id); + assert!(kept_full.equal(&Tensor::from_slice(&[10i64, 11, 12, 13][window_start..]))); + assert!(incoming_full.equal(&Tensor::from_slice(&[20i64, 21, 22, 23][window_start..]))); + assert!(tc.arena.device_value(leaf, FULL).equal(kept_full)); + let adopted = result.adopted_ranges.unwrap(); + assert_eq!(adopted[&SWA], [(window_start, 4)]); + assert!(!adopted.contains_key(&FULL)); + } +} + #[test] fn insert_overlap_with_a_locked_full_emits_the_recover_action() { let mut tc = swa_core(/* window = */ 8, /* page_size = */ 1); @@ -2060,6 +2109,7 @@ fn unanchored_receipts_are_rejected_before_any_release() { let before = lock_state(&tc); let mut device_frees = HashMap::new(); let mut host_frees = HashMap::new(); + let mut cache_actions = Vec::new(); assert!( catch_unwind(AssertUnwindSafe(|| { @@ -2080,6 +2130,7 @@ fn unanchored_receipts_are_rejected_before_any_release() { &unanchored(&device_lock), &mut device_frees, &mut host_frees, + &mut cache_actions, ) })) .is_err() @@ -2092,12 +2143,14 @@ fn unanchored_receipts_are_rejected_before_any_release() { &unanchored(&device_lock), &mut device_frees, &mut host_frees, + &mut cache_actions, ) })) .is_err() ); assert_eq!(lock_state(&tc), before); assert!(device_frees.is_empty() && host_frees.is_empty()); + assert!(cache_actions.is_empty()); // The anchored receipts still release exactly what was acquired. tc.dec_lock_ref( @@ -2376,6 +2429,7 @@ fn evict_component_device_frees_the_full_indices_and_tombstones_swa() { node, &mut device_frees, &mut host_frees, + &mut Vec::new(), EvictLayer::Device, ); assert_eq!((freed, host_freed), (2, 0)); @@ -2411,6 +2465,7 @@ fn evict_component_device_parks_a_remaining_host_value() { node, &mut device_frees, &mut host_frees, + &mut Vec::new(), EvictLayer::Device, ); assert!(tc.arena.has_host_value(node, SWA)); @@ -2433,6 +2488,7 @@ fn evict_component_host_frees_and_delists_the_host_value() { node, &mut device_frees, &mut host_frees, + &mut Vec::new(), EvictLayer::Host, ); assert_eq!((freed, host_freed), (0, 2)); @@ -2651,8 +2707,15 @@ fn dec_window_lock_only_preserves_full_and_mamba_locks() { let mut device_frees = HashMap::new(); let mut host_frees = HashMap::new(); assert_eq!(tc.component_protected_size(SWA), 2); - tc.dec_window_lock_only(leaf, SWA, ¶ms, &mut device_frees, &mut host_frees) - .unwrap(); + tc.dec_window_lock_only( + leaf, + SWA, + ¶ms, + &mut device_frees, + &mut host_frees, + &mut Vec::new(), + ) + .unwrap(); assert_eq!(tc.arena.device_lock_ref(idx, SWA), 0); assert_eq!(tc.arena.device_lock_ref(idx, FULL), 1); assert_eq!(tc.arena.device_lock_ref(idx, MAMBA), 1); @@ -2685,13 +2748,21 @@ fn dec_window_lock_only_ignores_root_and_skipped_window() { }, &mut device_frees, &mut host_frees, + &mut Vec::new(), ) .unwrap(); let receipt = tc.inc_lock_ref(leaf, ComponentSet::of(SWA)).unwrap(); let params = receipt.to_dec_params(); assert!(!params.component_lock_uuids.contains_key(&(SWA.idx() as u8))); - tc.dec_window_lock_only(leaf, SWA, ¶ms, &mut device_frees, &mut host_frees) - .unwrap(); + tc.dec_window_lock_only( + leaf, + SWA, + ¶ms, + &mut device_frees, + &mut host_frees, + &mut Vec::new(), + ) + .unwrap(); assert_eq!(tc.component_protected_size(FULL), 3); assert_eq!(tc.component_protected_size(MAMBA), 1); assert_eq!(tc.component_protected_size(SWA), 0); @@ -2712,6 +2783,7 @@ fn dec_window_lock_only_checks_receipt_anchor_before_skipping() { &receipt.to_dec_params(), &mut HashMap::new(), &mut HashMap::new(), + &mut Vec::new(), ) .unwrap(); } @@ -2738,6 +2810,7 @@ fn dec_swa_lock_only_releases_swa_while_full_stays_locked() { &result.to_dec_params(), &mut device_frees, &mut host_frees, + &mut Vec::new(), ) .expect("live test node"); // SWA is early-released; the FULL locks on the path stay. @@ -2779,6 +2852,7 @@ fn dec_swa_lock_only_evicts_a_fully_unlocked_device_leaf() { &result.to_dec_params(), &mut device_frees, &mut host_frees, + &mut Vec::new(), ) .expect("live test node"); // The fully unlocked leaf c is device-evicted on release; b keeps its @@ -2806,6 +2880,7 @@ fn dec_swa_lock_only_is_a_noop_without_the_swa_component() { }, &mut device_frees, &mut host_frees, + &mut Vec::new(), ) .expect("live test node"); assert!(device_frees.is_empty()); @@ -2838,6 +2913,7 @@ fn release_window_lock_breaks_on_a_tombstone_carrying_the_uuid() { }, &mut device_frees, &mut host_frees, + &mut Vec::new(), ); assert_eq!(tc.arena.device_lock_ref(c, SWA), 0); assert_eq!(tc.arena.device_lock_ref(a, SWA), 1); @@ -2858,6 +2934,7 @@ fn release_window_lock_panics_on_a_non_swa_component() { &DecLockRefParams::default(), &mut device_frees, &mut host_frees, + &mut Vec::new(), ); } @@ -2947,6 +3024,7 @@ fn dec_swa_lock_only_releases_the_window_exactly_once() { &first.to_dec_params(), &mut device_frees, &mut host_frees, + &mut Vec::new(), ) .expect("live test node"); // The second window still holds the lock: refs drop to 1, sizes stay. @@ -2959,6 +3037,7 @@ fn dec_swa_lock_only_releases_the_window_exactly_once() { &first.to_dec_params(), &mut device_frees, &mut host_frees, + &mut Vec::new(), ) .expect("live test node"); assert_eq!(tc.arena.device_lock_ref(c, SWA), 0); @@ -2999,6 +3078,7 @@ fn dec_swa_lock_only_leaves_out_of_window_swa_locks_alone() { &result.to_dec_params(), &mut device_frees, &mut host_frees, + &mut Vec::new(), ) .expect("live test node"); // Only the SWA window is released; a's out-of-window lock survives. @@ -3036,6 +3116,7 @@ fn release_window_lock_without_the_uuid_dies_past_the_boundary() { }, &mut device_frees, &mut host_frees, + &mut Vec::new(), ); } @@ -3062,6 +3143,7 @@ fn release_window_lock_passes_over_a_mid_chain_tombstone_without_a_uuid() { &receipt.to_dec_params(), &mut device_frees, &mut host_frees, + &mut Vec::new(), ); assert_eq!(tc.arena.device_lock_ref(c, SWA), 0); assert_eq!(tc.arena.device_lock_ref(b, SWA), 0); diff --git a/rust/sglang-radix-tree/src/tests/unified_tree_core.rs b/rust/sglang-radix-tree/src/tests/unified_tree_core.rs index d9b1111450..85bdedffc3 100644 --- a/rust/sglang-radix-tree/src/tests/unified_tree_core.rs +++ b/rust/sglang-radix-tree/src/tests/unified_tree_core.rs @@ -314,6 +314,7 @@ impl TreeComponent> for RecordingComponentForTest { _node_id: NodeIdx_, _device_frees: &mut HashMap>, _host_frees: &mut HashMap>, + _cache_actions: &mut Vec, _target: EvictLayer, ) -> (usize, usize) { unimplemented!() @@ -329,6 +330,7 @@ impl TreeComponent> for RecordingComponentForTest { _tracker: &mut HashMap, _device_frees: &mut HashMap>, _host_frees: &mut HashMap>, + _cache_actions: &mut Vec, ) -> Option { unimplemented!() } @@ -364,6 +366,7 @@ impl TreeComponent> for RecordingComponentForTest { tracker: &mut HashMap, _device_frees: &mut HashMap>, _host_frees: &mut HashMap>, + _cache_actions: &mut Vec, ) { let tracked = tracker[&SWA]; self.host_eviction_calls @@ -380,6 +383,7 @@ impl TreeComponent> for RecordingComponentForTest { tracker: &mut HashMap, _device_frees: &mut HashMap>, _host_frees: &mut HashMap>, + _cache_actions: &mut Vec, ) { let tracked = tracker[&SWA]; self.host_eviction_calls @@ -436,6 +440,7 @@ impl TreeComponent> for CountingComponentForTest { _node_id: NodeIdx_, _device_frees: &mut HashMap>, _host_frees: &mut HashMap>, + _cache_actions: &mut Vec, _target: EvictLayer, ) -> (usize, usize) { unimplemented!() @@ -451,6 +456,7 @@ impl TreeComponent> for CountingComponentForTest { _tracker: &mut HashMap, _device_frees: &mut HashMap>, _host_frees: &mut HashMap>, + _cache_actions: &mut Vec, ) -> Option { unimplemented!() } @@ -516,6 +522,7 @@ impl TreeComponent> for LowPriorityComponentForTest { _node_id: NodeIdx_, _device_frees: &mut HashMap>, _host_frees: &mut HashMap>, + _cache_actions: &mut Vec, _target: EvictLayer, ) -> (usize, usize) { unimplemented!() @@ -531,6 +538,7 @@ impl TreeComponent> for LowPriorityComponentForTest { _tracker: &mut HashMap, _device_frees: &mut HashMap>, _host_frees: &mut HashMap>, + _cache_actions: &mut Vec, ) -> Option { unimplemented!() } @@ -591,6 +599,7 @@ impl TreeComponent> for SwaComponentForTest { _node_id: NodeIdx_, _device_frees: &mut HashMap>, _host_frees: &mut HashMap>, + _cache_actions: &mut Vec, _target: EvictLayer, ) -> (usize, usize) { unimplemented!() @@ -606,6 +615,7 @@ impl TreeComponent> for SwaComponentForTest { _tracker: &mut HashMap, _device_frees: &mut HashMap>, _host_frees: &mut HashMap>, + _cache_actions: &mut Vec, ) -> Option { unimplemented!() } @@ -672,6 +682,7 @@ impl TreeComponent> for SwaEvictionComponentForTest { node_id: NodeIdx_, device_frees: &mut HashMap>, host_frees: &mut HashMap>, + _cache_actions: &mut Vec, target: EvictLayer, ) -> (usize, usize) { self.evictions.lock().unwrap().push((node_id, target)); @@ -719,6 +730,7 @@ impl TreeComponent> for SwaEvictionComponentForTest { _tracker: &mut HashMap, _device_frees: &mut HashMap>, _host_frees: &mut HashMap>, + _cache_actions: &mut Vec, ) -> Option { unimplemented!() } @@ -908,6 +920,7 @@ fn dec_swa_lock_only_dispatches_lower_priority_releases() { }, &mut device_frees, &mut host_frees, + &mut Vec::new(), ); } @@ -953,6 +966,7 @@ fn dec_swa_lock_only_returns_device_frees_in_the_device_dict() { &result.to_dec_params(), &mut device_frees, &mut host_frees, + &mut Vec::new(), ) .expect("live test node"); // The fully unlocked D-leaf's SWA value is device-evicted on release; @@ -2993,22 +3007,30 @@ fn backup_kv_action_chains_unbacked_ancestors_first() { let c = tc .match_prefix(&match_params(&vec![1, 2, 3])) .best_match_node_id; + assert_eq!( + tc.build_backup_kv_action(c).unwrap().node_ids, + vec![a, b, c] + ); + let root_id = tc.arena.node(tc.arena.root()).id; + assert_eq!( + tc.build_backup_kv_action(root_id).unwrap().node_ids, + vec![root_id] + ); // a is backuped: the chain stops there and orders ancestors first. tc.arena.set_host_value( tc.arena.resolve(a).expect("live test node"), FULL, Tensor::from_slice(&[20i64]), ); - let action = tc.build_backup_kv_action_( - tc.arena.node(tc.arena.resolve(c).expect("live test node")), - /* write_back = */ false, - ); + let action = tc.build_backup_kv_action(c).unwrap(); assert_eq!(action.node_ids, vec![b, c]); let action = tc.build_backup_kv_action_( tc.arena.node(tc.arena.resolve(c).expect("live test node")), /* write_back = */ true, ); assert_eq!(action.node_ids, vec![c]); + tc.reset(); + assert!(tc.build_backup_kv_action(c).is_err()); } #[test] @@ -3106,18 +3128,12 @@ fn backup_kv_action_stops_at_an_externally_stored_or_pending_ancestor() { let a_idx = tc.arena.resolve(a).expect("live test node"); tc.arena.node_mut(a_idx).external_cache_stored = true; - let action = tc.build_backup_kv_action_( - tc.arena.node(tc.arena.resolve(c).expect("live test node")), - /* write_back = */ false, - ); + let action = tc.build_backup_kv_action(c).unwrap(); assert_eq!(action.node_ids, vec![b, c]); tc.arena.node_mut(a_idx).external_cache_stored = false; tc.mark_external_linker_offload_pending(a).unwrap(); - let action = tc.build_backup_kv_action_( - tc.arena.node(tc.arena.resolve(c).expect("live test node")), - /* write_back = */ false, - ); + let action = tc.build_backup_kv_action(c).unwrap(); assert_eq!(action.node_ids, vec![b, c]); } @@ -5690,6 +5706,7 @@ fn evict_and_detach_frees_the_device_value_and_tracks() { FULL, &mut df, &mut hf, + &mut Vec::new(), EvictLayer::Device, Some(&mut tracker), ); @@ -5721,6 +5738,7 @@ fn evict_and_detach_host_targets_the_host_tier() { FULL, &mut df, &mut hf, + &mut Vec::new(), EvictLayer::Host, Some(&mut tracker), ); @@ -5802,7 +5820,7 @@ fn tombstone_chain(tc: &mut UnifiedTreeCore>) -> NodeIdx_ { fn delete_walk(tc: &mut UnifiedTreeCore>, from: NodeIdx_) { let mut tracker = HashMap::from([(FULL, 0)]); let (mut df, mut hf) = (HashMap::new(), HashMap::new()); - tc.iteratively_delete_tombstone_leaf_(from, &mut tracker, &mut df, &mut hf); + tc.iteratively_delete_tombstone_leaf_(from, &mut tracker, &mut df, &mut hf, &mut Vec::new()); } #[test] @@ -5909,7 +5927,7 @@ fn tombstone_walk_sweeps_orphaned_aux_device_data() { tc.device_lru_list_mut(SWA).insert_mru(a); let mut tracker = HashMap::new(); let (mut df, mut hf) = (HashMap::new(), HashMap::new()); - tc.iteratively_delete_tombstone_leaf_(a, &mut tracker, &mut df, &mut hf); + tc.iteratively_delete_tombstone_leaf_(a, &mut tracker, &mut df, &mut hf, &mut Vec::new()); // The orphaned Swa device data is evicted before the node is deleted. assert_eq!(tc.arena.len(), 1); assert_eq!( @@ -5937,7 +5955,7 @@ fn tombstone_walk_sweeps_orphaned_aux_host_data() { tc.host_lru_list_mut(SWA).insert_mru(a); let mut tracker = HashMap::new(); let (mut df, mut hf) = (HashMap::new(), HashMap::new()); - tc.iteratively_delete_tombstone_leaf_(a, &mut tracker, &mut df, &mut hf); + tc.iteratively_delete_tombstone_leaf_(a, &mut tracker, &mut df, &mut hf, &mut Vec::new()); // The orphaned Swa host data is evicted before the node is deleted. assert_eq!(tc.arena.len(), 1); assert_eq!( @@ -5971,6 +5989,7 @@ fn cascade_tombstones_full_after_the_component_sweep() { &mut tracker, &mut df, &mut hf, + &mut Vec::new(), EvictLayer::Device, ); assert!(!tc.arena.has_device_value(a, FULL)); @@ -6000,6 +6019,7 @@ fn cascade_rejects_the_all_layer() { &mut tracker, &mut df, &mut hf, + &mut Vec::new(), EvictLayer::All, ); } @@ -6025,6 +6045,7 @@ fn cascade_host_target_keeps_the_device_value() { &mut tracker, &mut df, &mut hf, + &mut Vec::new(), EvictLayer::Host, ); // No tombstone on a host-target cascade; the D-leaf survives. @@ -6053,6 +6074,7 @@ fn cascade_moves_a_host_backed_node_into_the_h_leaf_set() { &mut tracker, &mut df, &mut hf, + &mut Vec::new(), EvictLayer::Device, ); // Evicted but backuped: the node leaves the D-set and joins the H-set. @@ -6091,6 +6113,7 @@ fn cascade_sweeps_an_equal_priority_aux_component() { &mut tracker, &mut df, &mut hf, + &mut Vec::new(), EvictLayer::Device, ); assert_eq!( @@ -6119,6 +6142,7 @@ fn cascade_sweeps_a_lower_priority_aux_component() { &mut tracker, &mut df, &mut hf, + &mut Vec::new(), EvictLayer::Device, ); assert_eq!( @@ -6145,6 +6169,7 @@ fn cascade_spares_a_higher_priority_aux_component() { &mut tracker, &mut df, &mut hf, + &mut Vec::new(), EvictLayer::Device, ); assert!(recorder.evictions.lock().unwrap().is_empty()); @@ -6172,6 +6197,7 @@ fn cascade_spares_a_locked_component_of_equal_internal_priority() { &mut tracker, &mut df, &mut hf, + &mut Vec::new(), EvictLayer::Device, ); assert!(recorder.evictions.lock().unwrap().is_empty()); @@ -6197,6 +6223,7 @@ fn cascade_panics_on_a_locked_lower_internal_priority_component() { &mut tracker, &mut df, &mut hf, + &mut Vec::new(), EvictLayer::Device, ); } @@ -6238,6 +6265,7 @@ fn cascade_host_spares_a_locked_component_of_equal_internal_priority() { &mut tracker, &mut df, &mut hf, + &mut Vec::new(), EvictLayer::Host, ); assert!(recorder.evictions.lock().unwrap().is_empty()); @@ -6265,6 +6293,7 @@ fn cascade_host_panics_on_a_locked_lower_internal_priority_component() { &mut tracker, &mut df, &mut hf, + &mut Vec::new(), EvictLayer::Host, ); } @@ -6345,6 +6374,7 @@ fn release_all_component_layers_scrubs_a_host_leaf_from_both_leaf_sets() { &mut tracker, &mut device_frees, &mut host_frees, + &mut Vec::new(), ); assert!(!tc.evictable_host_leaves.contains(b)); assert!(!tc.evictable_device_leaves.contains(b)); @@ -7125,7 +7155,7 @@ fn evict_host_leaf_frees_host_values_and_credits_the_tracker() { tc.evictable_host_leaves.add(n1); let mut tr = HashMap::from([(FULL, 0)]); let (mut df, mut hf) = (HashMap::new(), HashMap::new()); - tc.evict_host_leaf_(n1, &mut tr, &mut df, &mut hf); + tc.evict_host_leaf_(n1, &mut tr, &mut df, &mut hf, &mut Vec::new()); assert_eq!(tr[&FULL], 2); assert_eq!(hf[&FULL].len(), 1); assert!(df.is_empty()); @@ -7152,7 +7182,7 @@ fn evict_host_leaf_panics_on_a_device_valued_node() { .set_device_value(n1, FULL, Tensor::from_slice(&[10i64])); let mut tr = HashMap::new(); let (mut df, mut hf) = (HashMap::new(), HashMap::new()); - tc.evict_host_leaf_(n1, &mut tr, &mut df, &mut hf); + tc.evict_host_leaf_(n1, &mut tr, &mut df, &mut hf, &mut Vec::new()); } #[test] @@ -7182,7 +7212,7 @@ fn evict_host_leaf_cascades_tombstone_ancestors() { tc.evictable_host_leaves.add(h); let mut tr = HashMap::from([(FULL, 0)]); let (mut df, mut hf) = (HashMap::new(), HashMap::new()); - tc.evict_host_leaf_(h, &mut tr, &mut df, &mut hf); + tc.evict_host_leaf_(h, &mut tr, &mut df, &mut hf, &mut Vec::new()); // The valueless ancestor t is deleted by the tombstone walk. assert_eq!(tc.arena.len(), 1); assert_eq!(tr[&FULL], 1); @@ -8810,6 +8840,7 @@ fn resume_insert_completes_after_an_on_path_host_leaf_is_evicted() { &mut tracker, &mut df, &mut hf, + &mut Vec::new(), ); assert_eq!(tracker[&FULL], 4); tc.commit_backup( @@ -9402,3 +9433,255 @@ fn a_zero_length_match_anchors_at_the_root() { assert!(tc.arena.resolve(anchor).is_ok()); tc.sanity_check(&[], &[]); } + +#[test] +fn match_with_borrowed_validators_retains_walk_state_and_common_postprocessing() { + for enable_hicache in [false, true] { + let mut tc = UnifiedTreeCore::new( + CacheInitParams { + enable_hicache, + ..Default::default() + }, + vec![FULL], + ); + let root = tc.arena.root(); + let head = add_leaf(&mut tc, root, vec![1, 2]); + let tail = add_leaf(&mut tc, head, vec![3, 4]); + let head_id = tc.arena.node(head).id; + let tail_id = tc.arena.node(tail).id; + let key = vec![1, 2, 3, 4]; + let mut visited = Vec::new(); + let result = { + let (mut validators, mut device_validators) = tc.create_match_validators_(); + let validator: MatchValidator<'_, Vec> = Box::new(|tree, node_id| { + visited.push(tree.arena.node(node_id).id); + visited.len() == 1 + }); + if enable_hicache { + device_validators[0] = validator; + } else { + validators[0] = validator; + } + tc.match_prefix_with_validators_( + &match_params(&key), + &mut validators, + &mut device_validators, + ) + }; + assert_eq!(visited, vec![head_id, tail_id]); + assert_eq!(result.last_device_node_id, head_id); + assert_eq!( + result.best_match_node_id, + if enable_hicache { tail_id } else { head_id } + ); + assert_eq!(result.full_kv_hit_length, 4); + assert_eq!(result.device_prefix_len, 2); + tc.sanity_check(&[], &[]); + } +} + +#[test] +fn caller_owned_overlap_drains_split_and_backup_before_duplicate_frees() { + let mut tc = core(); + tc.is_host_memory_buffer_only = true; + let original = tc.insert(&insert_params(&vec![1, 2, 3, 4], &[10, 11, 12, 13])); + let tail_id = original.last_device_node_id.unwrap(); + let key = vec![1, 2]; + let params = insert_params(&key, &[20, 21]); + let InsertWalkStart::Walk(mut walk) = tc.start_insert_walk_(¶ms) else { + panic!("nonempty insert needs a walk"); + }; + assert!(matches!( + tc.advance_insert_walk_(&mut walk, true), + InsertWalkProgress::Overlap + )); + assert_eq!(walk.total_prefix_length, 0); + assert_eq!(walk.overlap_prefix_len, 2); + let head = walk.overlap_node_id.unwrap(); + let node = tc.arena.node(head); + assert!( + node.device_value(FULL) + .equal(&Tensor::from_slice(&[10i64, 11])) + ); + assert_eq!( + action_kinds(&walk.pending_actions), + vec!["UpdatePendingBackupOnNodeSplit"] + ); + assert!(!walk.has_barrier_actions_since(0)); + walk.pending_actions.push(CacheAction::BackupKV(BackupKV { + node_ids: vec![node.id], + })); + assert!(walk.has_barrier_actions_since(0)); + let actions = std::mem::take(&mut walk.pending_actions); + assert_eq!( + action_kinds(&actions), + vec!["UpdatePendingBackupOnNodeSplit", "BackupKV"] + ); + assert_eq!(tc.arena.node(head).parent(), tc.arena.root()); + assert_eq!( + tc.arena.node(tc.arena.resolve(tail_id).unwrap()).parent(), + head + ); + // Advancing cannot claim the donor until the caller executes overlap. + assert!(matches!( + tc.advance_insert_walk_(&mut walk, true), + InsertWalkProgress::Overlap + )); + assert_eq!(walk.total_prefix_length, 0); + assert!(walk.pending_actions.is_empty()); + tc.insert_overlap_step_(&mut walk, InsertOverlapContext::default()); + assert!(!walk.has_barrier_actions_since(0)); + assert!(matches!( + tc.advance_insert_walk_(&mut walk, true), + InsertWalkProgress::Complete + )); + let step = walk.finish(); + assert_eq!(step.result.unwrap().prefix_len, 2); + assert_eq!(action_kinds(&step.actions), vec!["FreeDeviceKV"]); + let CacheAction::FreeDeviceKV(frees) = &step.actions[0] else { + unreachable!() + }; + assert!(frees[0].equal(&Tensor::from_slice(&[20i64, 21]))); + assert!(!tc.has_ongoing_insert()); + tc.sanity_check(&[], &[]); +} + +#[test] +fn caller_owned_walk_can_abort_after_split_without_claiming_donor() { + let mut tc = core(); + tc.is_host_memory_buffer_only = true; + let original = tc.insert(&insert_params(&vec![1, 2, 3, 4], &[10, 11, 12, 13])); + let key = vec![1, 2]; + let params = insert_params(&key, &[20, 21]); + let InsertWalkStart::Walk(mut walk) = tc.start_insert_walk_(¶ms) else { + panic!("nonempty insert needs a walk"); + }; + assert!(matches!( + tc.advance_insert_walk_(&mut walk, true), + InsertWalkProgress::Overlap + )); + let actions = std::mem::take(&mut walk.pending_actions); + drop(walk); + assert_eq!( + action_kinds(&actions), + vec!["UpdatePendingBackupOnNodeSplit"] + ); + assert!(!tc.has_ongoing_insert()); + assert!(tc.end_insert().is_empty()); + assert_eq!(tc.evictable_size_(FULL), 4); + let matched = tc.match_prefix(&match_params(&vec![1, 2, 3, 4])); + assert_eq!( + Some(matched.last_device_node_id), + original.last_device_node_id + ); + assert!(matched_device_indices(&tc, &matched).equal(&Tensor::from_slice(&[10i64, 11, 12, 13]))); + let result = tc.insert(¶ms); + assert_eq!(result.prefix_len, 2); + assert_eq!(action_kinds(&result.cache_actions), vec!["FreeDeviceKV"]); + tc.sanity_check(&[], &[]); +} + +#[test] +fn fixed_components_and_caller_owned_overlap_match_the_stock_insert() { + let params = CacheInitParams::default(); + let mut custom = UnifiedTreeCore::new_with_components_(params, vec![Arc::new(FullComponent)]); + let mut stock = core(); + for key in [vec![1, 2, 3, 4], vec![1, 2, 5, 6], vec![1, 2, 3, 4]] { + let params = insert_params(&key, &[10, 11, 12, 13]); + let expected = stock.insert(¶ms); + let InsertWalkStart::Walk(mut walk) = custom.start_insert_walk_(¶ms) else { + panic!("nonempty insert needs a walk"); + }; + let mut visited = 0; + let mut actions = Vec::new(); + let final_step = loop { + match custom.advance_insert_walk_(&mut walk, true) { + InsertWalkProgress::Overlap => { + visited += 1; + let start = walk.pending_actions.len(); + custom.insert_overlap_step_(&mut walk, InsertOverlapContext::default()); + if walk.has_barrier_actions_since(start) { + actions.append(&mut walk.pending_actions); + } + } + InsertWalkProgress::Actions => actions.append(&mut walk.pending_actions), + InsertWalkProgress::Complete => break walk.finish(), + } + }; + actions.extend(final_step.actions); + let result = final_step.result.unwrap(); + assert_eq!(result.prefix_len, expected.prefix_len); + assert_eq!(result.total_len, expected.total_len); + assert_eq!( + action_kinds(&actions), + action_kinds(&expected.cache_actions) + ); + assert_eq!(visited > 0, result.prefix_len > 0); + let stock_match = stock.match_prefix(&match_params(&key)); + let custom_match = custom.match_prefix(&match_params(&key)); + assert_eq!( + stock_match.device_prefix_len, + custom_match.device_prefix_len + ); + assert!( + matched_device_indices(&stock, &stock_match) + .equal(&matched_device_indices(&custom, &custom_match)) + ); + custom.sanity_check(&[], &[]); + } +} + +#[test] +fn overlap_context_reaches_swa_before_it_replaces_unlocked_full_ids() { + let mut tc = UnifiedTreeCore::new( + CacheInitParams { + swa_sliding_window_size: Some(2), + ..Default::default() + }, + vec![FULL, SWA], + ); + let root = tc.arena.root(); + let node = add_leaf(&mut tc, root, vec![1, 2]); + let node_id = tc.arena.node(node).id; + let kept = tc.arena.node(node).device_value(FULL).shallow_clone(); + let key = vec![1, 2]; + let InsertWalkStart::Walk(mut walk) = tc.start_insert_walk_(&insert_params(&key, &[20, 21])) + else { + panic!("nonempty insert needs a walk"); + }; + assert!(matches!( + tc.advance_insert_walk_(&mut walk, true), + InsertWalkProgress::Overlap + )); + let start = walk.pending_actions.len(); + tc.insert_overlap_step_( + &mut walk, + InsertOverlapContext { + preserve_full_ids: true, + }, + ); + assert!(walk.has_barrier_actions_since(start)); + let actions = std::mem::take(&mut walk.pending_actions); + assert_eq!(action_kinds(&actions), vec!["RecoverSwaWithLockedFull"]); + let CacheAction::RecoverSwaWithLockedFull { + node_id: actual_node_id, + kept_full, + incoming_full, + } = &actions[0] + else { + unreachable!() + }; + assert_eq!(*actual_node_id, node_id); + assert!(kept_full.equal(&kept)); + assert!(incoming_full.equal(&Tensor::from_slice(&[20i64, 21]))); + assert!(tc.arena.node(node).device_value(FULL).equal(&kept)); + assert_eq!(tc.arena.node(node).device_lock_ref(FULL), 0); + assert!(matches!( + tc.advance_insert_walk_(&mut walk, true), + InsertWalkProgress::Complete + )); + let step = walk.finish(); + assert_eq!(step.result.unwrap().prefix_len, 2); + assert!(step.actions.is_empty()); + tc.sanity_check(&[], &[]); +} diff --git a/rust/sglang-radix-tree/src/unified_tree_core.rs b/rust/sglang-radix-tree/src/unified_tree_core.rs index e10826ce41..358495fc78 100644 --- a/rust/sglang-radix-tree/src/unified_tree_core.rs +++ b/rust/sglang-radix-tree/src/unified_tree_core.rs @@ -165,6 +165,8 @@ pub struct MatchPrefixParams<'k, K: ChildKeyType> { pub namespace: KeyNamespaceRef<'k>, } +pub(crate) type MatchValidator<'a, K> = Box, NodeIdx_) -> bool + 'a>; + /// Params for an insert; the key is borrowed from the caller. pub struct InsertParams<'k, K: ChildKeyType> { /// Logical-page owner rotation; None keeps unsharded insert behavior. @@ -256,38 +258,82 @@ pub struct InsertStepResult { pub result: Option, } -// WALK (one node per step) -> COMMIT (leaf + commit hooks) -> TAIL (refresh + backup). -pub enum InsertPhase { +// WALK -> OVERLAP (existing device values) -> COMMIT -> TAIL. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum InsertPhase { Walk, + Overlap, Commit, Tail, } +/// Tree-local decisions made before components claim overlapping values. +#[derive(Default)] +pub(crate) struct InsertOverlapContext { + pub preserve_full_ids: bool, +} + /// In-flight resumable-insert state persisted across step barriers; owns its /// key/value/params snapshot so the walk survives across boundary calls. -pub struct InsertWalkState { - phase: InsertPhase, - node_id: NodeIdx_, +pub(crate) struct InsertWalkState { + pub(crate) phase: InsertPhase, + pub(crate) node_id: NodeIdx_, + pub(crate) overlap_node_id: Option, + pub(crate) overlap_prefix_len: usize, /// The full page-aligned insert key; `total_prefix_length` is the walk cursor. key: K, aligned_key_len: usize, - value: Tensor, + pub(crate) value: Tensor, namespace: KeyNamespace, session_id: Option>, rotation_base: Option, - prev_prefix_len: usize, + pub(crate) prev_prefix_len: usize, swa_evicted_seqlen: usize, swa_branching_seqlen: Option, mamba_value: Option, inserted_len: usize, priority: i64, track_adopted_ranges: bool, - total_prefix_length: usize, + pub(crate) total_prefix_length: usize, is_new_leaf: bool, target_node_id: Option, result: Option, /// Emitted actions awaiting the next barrier flush (or the final step). - pending_actions: Vec, + pub(crate) pending_actions: Vec, +} + +/// A new walk or the result of an empty or declined insert. +#[allow(clippy::large_enum_variant)] +pub(crate) enum InsertWalkStart { + Walk(InsertWalkState), + Complete(InsertResult), +} + +pub(crate) enum InsertWalkProgress { + Overlap, + Actions, + Complete, +} + +impl InsertWalkState { + pub(crate) fn finish(self) -> InsertStepResult { + InsertStepResult { + actions: self.pending_actions, + result: self.result, + } + } + + /// Free and split bookkeeping can wait; other actions pause the walk. + pub(crate) fn has_barrier_actions_since(&self, start: usize) -> bool { + self.pending_actions[start..].iter().any(|action| { + !matches!( + action, + CacheAction::FreeDeviceKV(_) + | CacheAction::FreeDeviceKVFullOnly(_) + | CacheAction::UpdatePendingBackupOnNodeSplit { .. } + ) + }) + } } /// Result of a KV-canary walk: parallel per-slot rows over the tree's FULL device slots. @@ -301,6 +347,7 @@ pub struct KvCanaryWalkResult { } /// A queued cache IO action. +#[derive(Debug)] pub enum CacheAction { /// Duplicate device KV slices the cache frees after the insert. FreeDeviceKV(Vec), @@ -601,6 +648,8 @@ pub struct EvictionStepResult { pub tracker: HashMap, pub device_frees: HashMap>, pub host_frees: HashMap>, + pub cache_actions: Vec, + pub backup_kv: Option, /// Full device tokens freed without a host copy during this device step. pub unbacked_tokens: usize, /// Back up this internal Mamba state before resuming its device tombstone. @@ -898,7 +947,7 @@ impl UnifiedTreeCore { }); } - pub fn new(params: CacheInitParams, component_types: Vec) -> Self { + fn validate_components_(component_types: &[ComponentType], page_size: usize) { assert!( !component_types.is_empty(), "at least one component type is required" @@ -907,7 +956,64 @@ impl UnifiedTreeCore { component_types.contains(&BASE_COMPONENT_TYPE), "the base (Full) component is required" ); - assert!(params.page_size >= 1, "page_size must be at least 1"); + assert!(page_size >= 1, "page_size must be at least 1"); + } + + /// Validate the component configuration used by the public bindings. + pub fn validate_init_params( + params: &CacheInitParams, + component_types: &[ComponentType], + ) -> Result<(), String> { + if component_types != [FULL] + && component_types != [FULL, SWA] + && component_types != [FULL, MAMBA] + && component_types != [FULL, SWA, MAMBA] + { + return Err("only the [Full], [Full, Swa], [Full, Mamba], and [Full, Swa, Mamba] component sets are supported".to_owned()); + } + if component_types.contains(&SWA) && params.swa_sliding_window_size.is_none() { + return Err("the Swa component requires swa_sliding_window_size".to_owned()); + } + if component_types.contains(&MAMBA) && params.mamba_cache_chunk_size.is_none() { + return Err("the Mamba component requires mamba_cache_chunk_size".to_owned()); + } + if params.page_size == 0 { + return Err("page_size must be at least 1".to_owned()); + } + let eviction_policy = params.eviction_policy.to_lowercase(); + if !matches!( + eviction_policy.as_str(), + "lru" | "lfu" | "fifo" | "mru" | "filo" | "priority" | "slru" | "tlru" + ) { + return Err(format!( + "Unknown eviction policy: {eviction_policy}. Supported policies: \ + 'lru', 'lfu', 'fifo', 'mru', 'filo', 'priority', 'slru', 'tlru'." + )); + } + Ok(()) + } + + pub fn new(params: CacheInitParams, component_types: Vec) -> Self { + Self::validate_components_(&component_types, params.page_size); + let mut components = Vec::with_capacity(component_types.len()); + for ct in &component_types { + let component: Arc + Send + Sync> = match ct { + ComponentType::Full => Arc::new(FullComponent), + ComponentType::Swa => Arc::new(SwaComponent::new(¶ms)), + ComponentType::Mamba => Arc::new(MambaComponent::new(¶ms)), + }; + components.push(component); + } + Self::new_with_components_(params, components) + } + + /// Construct a core from a fixed set of native component drivers. + pub(crate) fn new_with_components_( + params: CacheInitParams, + components: Vec + Send + Sync>>, + ) -> Self { + let component_types: Vec<_> = components.iter().map(|c| c.component_type()).collect(); + Self::validate_components_(&component_types, params.page_size); let arena = NodeArena::new(component_types.clone(), params.page_size); let mut tree_core = UnifiedTreeCore { arena, @@ -946,12 +1052,7 @@ impl UnifiedTreeCore { empty_device_indices: Tensor::empty([0], (Kind::Int64, params.device)), ongoing_insert_walk_state: None, }; - for ct in &component_types { - let component: Arc + Send + Sync> = match ct { - ComponentType::Full => Arc::new(FullComponent), - ComponentType::Swa => Arc::new(SwaComponent::new(¶ms)), - ComponentType::Mamba => Arc::new(MambaComponent::new(¶ms)), - }; + for component in components { tree_core.register_component_(component); } tree_core @@ -1151,6 +1252,7 @@ impl UnifiedTreeCore { params: &DecLockRefParams, device_frees: &mut HashMap>, host_frees: &mut HashMap>, + cache_actions: &mut Vec, ) -> Result<(), NodeAccessError> { let node_idx = self.arena.resolve(node_id)?; self.assert_receipt_anchor_(node_idx, params); @@ -1165,6 +1267,7 @@ impl UnifiedTreeCore { params, device_frees, host_frees, + cache_actions, ); Ok(()) } @@ -1177,13 +1280,21 @@ impl UnifiedTreeCore { params: &DecLockRefParams, device_frees: &mut HashMap>, host_frees: &mut HashMap>, + cache_actions: &mut Vec, ) -> Result<(), NodeAccessError> { let node_idx = self.arena.resolve(node_id)?; self.assert_receipt_anchor_(node_idx, params); let Some(swa) = self.try_component_by_type_(SWA) else { return Ok(()); }; - swa.release_window_lock(self, node_idx, params, device_frees, host_frees); + swa.release_window_lock( + self, + node_idx, + params, + device_frees, + host_frees, + cache_actions, + ); // Drop strictly-lower-priority locks co-located on the node, skipping // any the paired inc never took. @@ -1218,6 +1329,7 @@ impl UnifiedTreeCore { tail_node_id, &mut result.device_frees, &mut result.host_frees, + &mut result.cache_actions, ); Ok(result) } @@ -1257,6 +1369,20 @@ impl UnifiedTreeCore { /// Match a key; returns the device prefix length + boundary NodeIds. pub fn match_prefix(&mut self, params: &MatchPrefixParams<'_, K>) -> MatchResult { + if params.key.atom_len() < self.page_size { + return self.empty_match_result(); + } + let (mut validators, mut device_validators) = self.create_match_validators_(); + self.match_prefix_with_validators_(params, &mut validators, &mut device_validators) + } + + /// Run a scan with tree-local validators that may borrow per-match state. + pub(crate) fn match_prefix_with_validators_( + &mut self, + params: &MatchPrefixParams<'_, K>, + validators: &mut [MatchValidator<'_, K>], + device_validators: &mut [MatchValidator<'_, K>], + ) -> MatchResult { // Bigram view conversion happens at the boundary; the key arrives typed. let aligned_key_len = params.key.atom_len() / self.page_size * self.page_size; if aligned_key_len == 0 { @@ -1273,7 +1399,14 @@ impl UnifiedTreeCore { best_match_device_value_len, full_kv_hit_length, action, - ) = self.match_prefix_helper_(root_id, params.namespace, key, aligned_key_len); + ) = self.match_prefix_helper_( + root_id, + params.namespace, + key, + aligned_key_len, + validators, + device_validators, + ); self.match_post_processor_( params, root_id, @@ -1324,14 +1457,44 @@ impl UnifiedTreeCore { (offset, self.arena.node(node_id).id, pinned_len) } + /// Build default validators in component order, without borrowing the core. + pub(crate) fn create_match_validators_<'a>( + &self, + ) -> (Vec>, Vec>) { + let separate_device_match = self.enable_hicache; + let mut validators: Vec> = Vec::with_capacity(self.components.len()); + let mut device_validators: Vec> = if separate_device_match { + Vec::with_capacity(self.components.len()) + } else { + Vec::new() + }; + for i in 0..self.components.len() { + let component = Arc::clone(&self.components[i]); + if separate_device_match { + validators.push( + component.create_match_validator(self, /* match_device_only = */ false), + ); + device_validators + .push(component.create_match_validator(self, /* match_device_only = */ true)); + } else { + validators + .push(component.create_match_validator(self, /* match_device_only = */ true)); + } + } + + (validators, device_validators) + } + /// Walk the tree for `key`; returns matched value chunks, the best match, /// the best device-resident match, its device value length, and any split action. - pub fn match_prefix_helper_( + pub(crate) fn match_prefix_helper_( &mut self, root_id: NodeIdx_, namespace: KeyNamespaceRef<'_>, key: &K, aligned_key_len: usize, + validators: &mut [MatchValidator<'_, K>], + device_validators: &mut [MatchValidator<'_, K>], ) -> ( Vec, NodeIdx_, @@ -1354,33 +1517,14 @@ impl UnifiedTreeCore { let mut full_kv_hit_length = 0; let mut action: Option = None; let separate_device_match = self.enable_hicache; - let mut validators = Vec::with_capacity(self.components.len()); - let mut device_validators = if separate_device_match { - Vec::with_capacity(self.components.len()) - } else { - Vec::new() - }; - for i in 0..self.components.len() { - let component = Arc::clone(&self.components[i]); - if separate_device_match { - validators.push( - component.create_match_validator(self, /* match_device_only = */ false), - ); - device_validators - .push(component.create_match_validator(self, /* match_device_only = */ true)); - } else { - validators - .push(component.create_match_validator(self, /* match_device_only = */ true)); - } - } fn update_best_if_valid( tree: &UnifiedTreeCore, node_id: NodeIdx_, value_len: usize, separate_device_match: bool, - validators: &mut [Box, NodeIdx_) -> bool>], - device_validators: &mut [Box, NodeIdx_) -> bool>], + validators: &mut [MatchValidator<'_, K>], + device_validators: &mut [MatchValidator<'_, K>], best_match_node_id: &mut NodeIdx_, best_match_device_node_id: &mut NodeIdx_, best_match_device_value_len: &mut usize, @@ -1436,8 +1580,8 @@ impl UnifiedTreeCore { node_id, value.len(), separate_device_match, - &mut validators, - &mut device_validators, + validators, + device_validators, &mut best_match_node_id, &mut best_match_device_node_id, &mut best_match_device_value_len, @@ -1454,8 +1598,8 @@ impl UnifiedTreeCore { node_id, value.len(), separate_device_match, - &mut validators, - &mut device_validators, + validators, + device_validators, &mut best_match_node_id, &mut best_match_device_node_id, &mut best_match_device_value_len, @@ -1671,10 +1815,26 @@ impl UnifiedTreeCore { &mut self, params: &InsertParams<'_, K>, ) -> Result { - // Insert walks are single-flight; a live walk means re-entrancy. if self.ongoing_insert_walk_state.is_some() { return Err(TreeCoreRuntimeError::ConcurrentInsertWalk); } + match self.start_insert_walk_(params) { + InsertWalkStart::Complete(result) => Ok(InsertStepResult { + actions: Vec::new(), + result: Some(result), + }), + InsertWalkStart::Walk(state) => { + self.ongoing_insert_walk_state = Some(state); + self.try_resume_insert() + } + } + } + + /// Initialize a caller-owned walk; its owner enforces single-flight inserts. + pub(crate) fn start_insert_walk_( + &mut self, + params: &InsertParams<'_, K>, + ) -> InsertWalkStart { // Bigram view conversion happens at the boundary; the key arrives typed. let aligned_key_len = params.key.atom_len() / self.page_size * self.page_size; if aligned_key_len == 0 { @@ -1685,20 +1845,17 @@ impl UnifiedTreeCore { let node = self.arena.node_mut(root_id); node.priority = node.priority.max(params.priority); } - return Ok(InsertStepResult { - actions: Vec::new(), - result: Some(InsertResult { - prefix_len: 0, - total_len: 0, - last_device_node_id: Some(self.arena.node(root_id).id), - inserted_host_node: None, - host_insert_dropped: false, - rotation_tail_declined: false, - mamba_exist: true, - swa_branch_inserted: false, - adopted_ranges: None, - cache_actions: Vec::new(), - }), + return InsertWalkStart::Complete(InsertResult { + prefix_len: 0, + total_len: 0, + last_device_node_id: Some(self.arena.node(root_id).id), + inserted_host_node: None, + host_insert_dropped: false, + rotation_tail_declined: false, + mamba_exist: true, + swa_branch_inserted: false, + adopted_ranges: None, + cache_actions: Vec::new(), }); } let root_id = self.arena.root(); @@ -1711,21 +1868,20 @@ impl UnifiedTreeCore { && let Some((prefix_len, node_id)) = self.rotation_conflict_(params, aligned_key_len, rotation_base) { - return Ok(InsertStepResult { - actions: Vec::new(), - result: Some(InsertResult { - prefix_len, - last_device_node_id: Some(self.arena.node(node_id).id), - rotation_tail_declined: true, - adopted_ranges: params.track_adopted_ranges.then(HashMap::new), - ..InsertResult::default() - }), + return InsertWalkStart::Complete(InsertResult { + prefix_len, + last_device_node_id: Some(self.arena.node(node_id).id), + rotation_tail_declined: true, + adopted_ranges: params.track_adopted_ranges.then(HashMap::new), + ..InsertResult::default() }); } // The walk reads only [0, aligned_key_len); the ragged tail never enters. - self.ongoing_insert_walk_state = Some(InsertWalkState { + InsertWalkStart::Walk(InsertWalkState { phase: InsertPhase::Walk, node_id: root_id, + overlap_node_id: None, + overlap_prefix_len: 0, key: K::from(params.key.as_ref()[..aligned_key_len].to_vec()), aligned_key_len, value: params.value.narrow(0, 0, aligned_key_len as i64), @@ -1747,8 +1903,7 @@ impl UnifiedTreeCore { ..InsertResult::default() }), pending_actions: Vec::new(), - }); - Ok(self.advance_insert_()) + }) } /// Check the matched chain before insert can restore, repoint, split, or @@ -1793,10 +1948,22 @@ impl UnifiedTreeCore { /// Fallible variant of [`Self::resume_insert`]. pub fn try_resume_insert(&mut self) -> Result { - if self.ongoing_insert_walk_state.is_none() { - return Err(TreeCoreRuntimeError::NoInFlightInsert); + let mut state = self + .ongoing_insert_walk_state + .take() + .ok_or(TreeCoreRuntimeError::NoInFlightInsert)?; + match self.advance_insert_walk_(&mut state, false) { + InsertWalkProgress::Complete => Ok(state.finish()), + InsertWalkProgress::Actions => { + let actions = std::mem::take(&mut state.pending_actions); + self.ongoing_insert_walk_state = Some(state); + Ok(InsertStepResult { + actions, + result: None, + }) + } + InsertWalkProgress::Overlap => unreachable!("ordinary overlap runs inline"), } - Ok(self.advance_insert_()) } /// Whether an insert walk is suspended at a barrier. @@ -1812,49 +1979,36 @@ impl UnifiedTreeCore { .unwrap_or_default() } - /// Run the in-flight insert to its next barrier or to completion. - fn advance_insert_(&mut self) -> InsertStepResult { - // The state moves out of self while steps run (they borrow the tree mutably). - let mut state = self - .ongoing_insert_walk_state - .take() - .expect("no in-flight insert"); + /// Advance a supplied walk, optionally stopping before overlap claims. + pub(crate) fn advance_insert_walk_( + &mut self, + state: &mut InsertWalkState, + pause_at_overlap: bool, + ) -> InsertWalkProgress { loop { let flushed_len = state.pending_actions.len(); match state.phase { - InsertPhase::Walk => self.insert_walk_step_(&mut state), - InsertPhase::Commit => self.insert_commit_step_(&mut state), + InsertPhase::Walk => self.insert_walk_step_(state), + InsertPhase::Overlap => {} + InsertPhase::Commit => self.insert_commit_step_(state), InsertPhase::Tail => { - self.insert_tail_step_(&mut state); - return InsertStepResult { - actions: state.pending_actions, - result: state.result, - }; + self.insert_tail_step_(state); + return InsertWalkProgress::Complete; } } - let new_actions = &state.pending_actions[flushed_len..]; - // Suspend only when a step emitted a non-deferrable action. - if !new_actions.is_empty() && !new_actions.iter().all(Self::is_deferrable_action_) { - let flushed = std::mem::take(&mut state.pending_actions); - self.ongoing_insert_walk_state = Some(state); - return InsertStepResult { - actions: flushed, - result: None, - }; + // The ordinary path completes overlap in the same iteration as WALK. + if state.phase == InsertPhase::Overlap { + if pause_at_overlap { + return InsertWalkProgress::Overlap; + } + self.insert_overlap_step_(state, InsertOverlapContext::default()); + } + if state.has_barrier_actions_since(flushed_len) { + return InsertWalkProgress::Actions; } } } - /// Fire-and-forget actions safe to batch until the next barrier. - fn is_deferrable_action_(action: &CacheAction) -> bool { - matches!( - action, - CacheAction::FreeDeviceKV(_) - | CacheAction::FreeDeviceKVFullOnly(_) - | CacheAction::UpdatePendingBackupOnNodeSplit { .. } - ) - } - /// Process one walked node, appending its barrier actions to the state. fn insert_walk_step_(&mut self, state: &mut InsertWalkState) { // Walk cursor: atoms of `key` already matched (also the running prefix length). @@ -1888,6 +2042,12 @@ impl UnifiedTreeCore { node.priority = node.priority.max(state.priority); } + if !self.arena.node(node_id).evicted() { + state.overlap_node_id = Some(node_id); + state.overlap_prefix_len = prefix_len; + state.phase = InsertPhase::Overlap; + return; + } let params = InsertParams { rotation_base: state.rotation_base, key: &state.key, @@ -1902,90 +2062,123 @@ impl UnifiedTreeCore { priority: state.priority, track_adopted_ranges: state.track_adopted_ranges, }; - if self.arena.node(node_id).evicted() { - self.unevict_node_on_insert_( + self.unevict_node_on_insert_( + node_id, + &state.value.narrow(0, cursor as i64, prefix_len as i64), + state.session_id.as_deref(), + ); + state + .result + .as_mut() + .expect("insert result exists during the walk") + .record_adopted_range(BASE_COMPONENT_TYPE, cursor, cursor + prefix_len); + // FULL was restored from the request's fresh KV. Aux + // components (e.g. SWA) may still hold tombstones and need + // to rebuild their value from the same slice. + for i in 0..self.components.len() { + if self.components[i].component_type() == BASE_COMPONENT_TYPE { + continue; + } + let component = Arc::clone(&self.components[i]); + component.recover_after_unevict( + self, node_id, - &state.value.narrow(0, cursor as i64, prefix_len as i64), - state.session_id.as_deref(), + prefix_len, + cursor, + ¶ms, + state + .result + .as_mut() + .expect("insert result exists during the walk"), + &mut state.pending_actions, ); - state - .result - .as_mut() - .expect("insert result exists during the walk") - .record_adopted_range(BASE_COMPONENT_TYPE, cursor, cursor + prefix_len); - // FULL was restored from the request's fresh KV. Aux - // components (e.g. SWA) may still hold tombstones and need - // to rebuild their value from the same slice. - for i in 0..self.components.len() { - if self.components[i].component_type() == BASE_COMPONENT_TYPE { - continue; - } - let component = Arc::clone(&self.components[i]); - component.recover_after_unevict( - self, - node_id, - prefix_len, - cursor, - ¶ms, - state - .result - .as_mut() - .expect("insert result exists during the walk"), - &mut state.pending_actions, - ); - } - } else { - let value_slice = state.value.narrow(0, cursor as i64, prefix_len as i64); - let mut consumed_from = prefix_len; - // Let each component claim ownership of overlapping KV slots. - for i in 0..self.components.len() { - let component = Arc::clone(&self.components[i]); - let comp_consumed_from = component.update_component_on_insert_overlap( - self, - node_id, - prefix_len, - cursor, - value_slice.shallow_clone(), - ¶ms, - state - .result - .as_mut() - .expect("insert result exists during the walk"), - &mut state.pending_actions, - ); - consumed_from = consumed_from.min(comp_consumed_from); - } + } + self.finish_insert_walk_node_(state, node_id, prefix_len); + } - let dup_start = state.prev_prefix_len.saturating_sub(cursor); - if dup_start < consumed_from { - // The duplicate slice may straddle this request's own eviction - // floor; below it only the full side is still ours to release. - let dup_len = consumed_from - dup_start; - let swa_already_freed = state - .swa_evicted_seqlen - .saturating_sub(cursor + dup_start) - .min(dup_len); - if swa_already_freed > 0 { - state - .pending_actions - .push(CacheAction::FreeDeviceKVFullOnly(vec![value_slice.narrow( - 0, - dup_start as i64, - swa_already_freed as i64, - )])); - } - if swa_already_freed < dup_len { - state.pending_actions.push(CacheAction::FreeDeviceKV(vec![ - value_slice.narrow( - 0, - (dup_start + swa_already_freed) as i64, - (dup_len - swa_already_freed) as i64, - ), - ])); - } - } + pub(crate) fn insert_overlap_step_( + &mut self, + state: &mut InsertWalkState, + context: InsertOverlapContext, + ) { + let node_id = state.overlap_node_id.expect("overlap node exists"); + let prefix_len = state.overlap_prefix_len; + let cursor = state.total_prefix_length; + let params = InsertParams { + rotation_base: state.rotation_base, + key: &state.key, + namespace: state.namespace.as_ref(), + session_id: state.session_id.as_deref(), + value: state.value.shallow_clone(), + prev_prefix_len: state.prev_prefix_len, + swa_evicted_seqlen: state.swa_evicted_seqlen, + swa_branching_seqlen: state.swa_branching_seqlen, + mamba_value: state.mamba_value.as_ref().map(Tensor::shallow_clone), + inserted_len: state.inserted_len, + priority: state.priority, + track_adopted_ranges: state.track_adopted_ranges, + }; + let value_slice = state.value.narrow(0, cursor as i64, prefix_len as i64); + let mut consumed_from = prefix_len; + // Let each component claim ownership of overlapping KV slots. + for i in 0..self.components.len() { + let component = Arc::clone(&self.components[i]); + let comp_consumed_from = component.update_component_on_insert_overlap( + self, + node_id, + prefix_len, + cursor, + value_slice.shallow_clone(), + ¶ms, + context.preserve_full_ids, + state + .result + .as_mut() + .expect("insert result exists during the walk"), + &mut state.pending_actions, + ); + consumed_from = consumed_from.min(comp_consumed_from); } + let dup_start = state.prev_prefix_len.saturating_sub(cursor); + if dup_start < consumed_from { + // The duplicate slice may straddle this request's own eviction + // floor; below it only the full side is still ours to release. + let dup_len = consumed_from - dup_start; + let swa_already_freed = state + .swa_evicted_seqlen + .saturating_sub(cursor + dup_start) + .min(dup_len); + if swa_already_freed > 0 { + state + .pending_actions + .push(CacheAction::FreeDeviceKVFullOnly(vec![value_slice.narrow( + 0, + dup_start as i64, + swa_already_freed as i64, + )])); + } + if swa_already_freed < dup_len { + state + .pending_actions + .push(CacheAction::FreeDeviceKV(vec![value_slice.narrow( + 0, + (dup_start + swa_already_freed) as i64, + (dup_len - swa_already_freed) as i64, + )])); + } + } + state.overlap_node_id = None; + state.phase = InsertPhase::Walk; + self.finish_insert_walk_node_(state, node_id, prefix_len); + } + + fn finish_insert_walk_node_( + &mut self, + state: &mut InsertWalkState, + node_id: NodeIdx_, + prefix_len: usize, + ) { // Nodes this request already inserted were counted back then. let node_end = state.total_prefix_length + prefix_len; if node_end > state.inserted_len && self.inc_hit_count_and_check_(node_id) { @@ -2315,10 +2508,7 @@ impl UnifiedTreeCore { displaced.is_none(), "add_new_node_: parent {parent_id} already has a child on the new node's page" ); - let tokens = self - .arena - .node(new_node_id) - .device_reclaimable_value_len(FULL); + let tokens = self.component_reclaimable_value_len_(new_node_id, FULL); self.inc_evictable_size(FULL, tokens); if self.enable_storage || self.enable_external_cache_linker { let hash_values = self.arena.compute_node_hash_values(new_node_id, page_size); @@ -2341,7 +2531,7 @@ impl UnifiedTreeCore { ) { self.arena .set_device_value(node_id, FULL, fresh_value.copy()); - let tokens = self.arena.node(node_id).device_reclaimable_value_len(FULL); + let tokens = self.component_reclaimable_value_len_(node_id, FULL); // A value materialized under lock is protected; the last release // moves it to evictable. if self.arena.device_lock_ref(node_id, FULL) > 0 { @@ -2457,6 +2647,18 @@ impl UnifiedTreeCore { .unwrap_or_else(|| panic!("{component_type:?} component is not enabled")) } + /// Consult the configured driver, including when a wrapper reuses a base component. + pub(crate) fn component_reclaimable_value_len_( + &self, + node_id: NodeIdx_, + component_type: ComponentType, + ) -> usize { + self.components_by_type[component_type.idx()] + .as_ref() + .unwrap_or_else(|| panic!("{component_type:?} component is not enabled")) + .reclaimable_value_len(self.arena.node(node_id)) + } + /// The component driver for `component_type`, or None when not enabled. fn try_component_by_type_( &self, @@ -2497,6 +2699,7 @@ impl UnifiedTreeCore { &mut tracker, &mut result.device_frees, &mut result.host_frees, + &mut result.cache_actions, ); result.unbacked_tokens = self.tracked_unbacked_tokens.take().unwrap(); let state = self.component_state(component_type); @@ -2577,6 +2780,7 @@ impl UnifiedTreeCore { component_type, &mut result.device_frees, &mut result.host_frees, + &mut result.cache_actions, EvictLayer::Device, Some(&mut result.tracker), ); @@ -2586,6 +2790,7 @@ impl UnifiedTreeCore { &mut result.tracker, &mut result.device_frees, &mut result.host_frees, + &mut result.cache_actions, EvictLayer::Device, ); result.unbacked_tokens = self.tracked_unbacked_tokens.take().unwrap(); @@ -2623,6 +2828,7 @@ impl UnifiedTreeCore { &mut result.tracker, &mut result.device_frees, &mut result.host_frees, + &mut result.cache_actions, ); result.unbacked_tokens = self.tracked_unbacked_tokens.take().unwrap(); return Ok((None, result)); @@ -2640,6 +2846,7 @@ impl UnifiedTreeCore { &mut result.tracker, &mut result.device_frees, &mut result.host_frees, + &mut result.cache_actions, ); result.unbacked_tokens = self.tracked_unbacked_tokens.take().unwrap(); Ok((None, result)) @@ -2705,6 +2912,7 @@ impl UnifiedTreeCore { &mut result.tracker, &mut result.device_frees, &mut result.host_frees, + &mut result.cache_actions, ); self.remove_leaf_from_parent_(desc_id); } @@ -2713,6 +2921,7 @@ impl UnifiedTreeCore { &mut result.tracker, &mut result.device_frees, &mut result.host_frees, + &mut result.cache_actions, ); Ok((true, result)) } @@ -2726,6 +2935,7 @@ impl UnifiedTreeCore { tracker: &mut HashMap, device_frees: &mut HashMap>, host_frees: &mut HashMap>, + cache_actions: &mut Vec, ) { self.record_remove_event_(node_id, medium); for i in 0..self.components.len() { @@ -2735,6 +2945,7 @@ impl UnifiedTreeCore { component_type, device_frees, host_frees, + cache_actions, EvictLayer::All, Some(tracker), ); @@ -2750,6 +2961,7 @@ impl UnifiedTreeCore { tracker: &mut HashMap, device_frees: &mut HashMap>, host_frees: &mut HashMap>, + cache_actions: &mut Vec, ) { self.release_all_component_layers_( node_id, @@ -2757,11 +2969,18 @@ impl UnifiedTreeCore { tracker, device_frees, host_frees, + cache_actions, ); let parent = self.arena.node(node_id).parent(); self.remove_leaf_from_parent_(node_id); self.update_evictable_leaf_sets_(parent); - self.iteratively_delete_tombstone_leaf_(parent, tracker, device_frees, host_frees); + self.iteratively_delete_tombstone_leaf_( + parent, + tracker, + device_frees, + host_frees, + cache_actions, + ); } /// Evict a component's host-side resources. @@ -2794,6 +3013,7 @@ impl UnifiedTreeCore { &mut result.tracker, &mut result.device_frees, &mut result.host_frees, + &mut result.cache_actions, ); } component.drive_host_eviction( @@ -2802,6 +3022,7 @@ impl UnifiedTreeCore { &mut result.tracker, &mut result.device_frees, &mut result.host_frees, + &mut result.cache_actions, ); } result @@ -2829,6 +3050,7 @@ impl UnifiedTreeCore { tracker: &mut HashMap, device_frees: &mut HashMap>, host_frees: &mut HashMap>, + cache_actions: &mut Vec, ) { assert!( self.can_reclaim_coexisting_host_value_(node_id, component_type), @@ -2843,6 +3065,7 @@ impl UnifiedTreeCore { component_type, device_frees, host_frees, + cache_actions, EvictLayer::Host, Some(tracker), ); @@ -2863,6 +3086,7 @@ impl UnifiedTreeCore { tracker: &mut HashMap, device_frees: &mut HashMap>, host_frees: &mut HashMap>, + cache_actions: &mut Vec, ) { assert!( self.is_evictable_host_leaf_(self.arena.node(node_id)), @@ -2876,6 +3100,7 @@ impl UnifiedTreeCore { component_type, device_frees, host_frees, + cache_actions, EvictLayer::All, None, ); @@ -2884,7 +3109,13 @@ impl UnifiedTreeCore { self.evictable_host_leaves.discard(node_id); let parent = self.arena.node(node_id).parent(); self.remove_leaf_from_parent_(node_id); - self.iteratively_delete_tombstone_leaf_(parent, tracker, device_frees, host_frees); + self.iteratively_delete_tombstone_leaf_( + parent, + tracker, + device_frees, + host_frees, + cache_actions, + ); } /// Release a node's device KV once its host copy exists; the node stays in the @@ -2911,6 +3142,7 @@ impl UnifiedTreeCore { &mut result.tracker, &mut result.device_frees, &mut result.host_frees, + &mut result.cache_actions, ); Ok(result) } @@ -2922,6 +3154,7 @@ impl UnifiedTreeCore { tracker: &mut HashMap, device_frees: &mut HashMap>, host_frees: &mut HashMap>, + cache_actions: &mut Vec, ) { { let node = self.arena.node(node_id); @@ -2932,6 +3165,7 @@ impl UnifiedTreeCore { BASE_COMPONENT_TYPE, device_frees, host_frees, + cache_actions, EvictLayer::Device, Some(tracker), ); @@ -2941,6 +3175,7 @@ impl UnifiedTreeCore { tracker, device_frees, host_frees, + cache_actions, EvictLayer::Device, ); self.record_remove_event_(node_id, StorageMedium::Gpu); @@ -2964,6 +3199,7 @@ impl UnifiedTreeCore { tracker: &mut HashMap, device_frees: &mut HashMap>, host_frees: &mut HashMap>, + cache_actions: &mut Vec, target: EvictLayer, ) { let is_leaf = match target { @@ -2999,6 +3235,7 @@ impl UnifiedTreeCore { ct, device_frees, host_frees, + cache_actions, target, Some(tracker), ); @@ -3073,6 +3310,7 @@ impl UnifiedTreeCore { /// Unlink a leaf from its parent. pub fn remove_leaf_from_parent_(&mut self, node_id: NodeIdx_) { + let node_handle = self.arena.node(node_id).id; // Arena slots are reused, so discard tracking before freeing the node. self.full_coexisting_host_nodes.discard(node_id); self.namespaced_event_hashes @@ -3081,6 +3319,9 @@ impl UnifiedTreeCore { self.arena .free_leaf(node_id) .expect("remove_leaf_from_parent_: a deletable leaf"); + for component in &self.components { + component.on_node_removed(node_handle); + } } /// Evict one component on the node and detach its LRU entries. @@ -3090,13 +3331,20 @@ impl UnifiedTreeCore { component_type: ComponentType, device_frees: &mut HashMap>, host_frees: &mut HashMap>, + cache_actions: &mut Vec, target: EvictLayer, tracker: Option<&mut HashMap>, ) -> (usize, usize) { let had_host_copy = self.arena.has_host_value(node_id, component_type); let component = self.component_by_type_(component_type); - let (device_freed, host_freed) = - component.evict_component(self, node_id, device_frees, host_frees, target); + let (device_freed, host_freed) = component.evict_component( + self, + node_id, + device_frees, + host_frees, + cache_actions, + target, + ); if let Some(tracker) = tracker { let freed = if target.contains(EvictLayer::Device) { device_freed @@ -3140,6 +3388,7 @@ impl UnifiedTreeCore { tracker: &mut HashMap, device_frees: &mut HashMap>, host_frees: &mut HashMap>, + cache_actions: &mut Vec, ) { let ct = BASE_COMPONENT_TYPE; let mut cur = deleted_node_parent_id; @@ -3168,6 +3417,7 @@ impl UnifiedTreeCore { component.component_type(), device_frees, host_frees, + cache_actions, EvictLayer::Device, Some(tracker), ); @@ -3188,6 +3438,7 @@ impl UnifiedTreeCore { component.component_type(), device_frees, host_frees, + cache_actions, EvictLayer::Host, Some(tracker), ); @@ -3248,6 +3499,46 @@ impl UnifiedTreeCore { true } + pub fn is_host_memory_buffer_only(&self) -> bool { + self.is_host_memory_buffer_only + } + + pub fn enable_hicache(&self) -> bool { + self.enable_hicache + } + + pub fn has_swa_host_pool(&self) -> bool { + self.has_swa_host_pool + } + + pub fn is_write_back(&self) -> bool { + self.is_write_back + } + + pub fn set_is_write_back(&mut self, value: bool) { + self.is_write_back = value; + } + + pub fn write_through_threshold(&self) -> i64 { + self.write_through_threshold + } + + pub fn set_write_through_threshold(&mut self, value: i64) { + self.write_through_threshold = value; + } + + pub fn enable_storage(&self) -> bool { + self.enable_storage + } + + pub fn enable_external_cache_linker(&self) -> bool { + self.enable_external_cache_linker + } + + pub fn write_back_coexist_reclaim_digest(&self) -> i64 { + self.write_back_coexist_reclaim_digest + } + /// Mark the host tier (HiCache) as wired. pub fn set_hicache_enabled(&mut self) { self.enable_hicache = true; @@ -4129,6 +4420,11 @@ impl UnifiedTreeCore { Ok(order) } + pub fn build_backup_kv_action(&self, node_id: NodeId) -> Result { + let node_id = self.arena.resolve(node_id)?; + Ok(self.build_backup_kv_action_(self.arena.node(node_id), false)) + } + /// Build the backup action for a node; write-through also chains its /// unbacked ancestors. pub fn build_backup_kv_action_(&self, node: &Node, write_back: bool) -> BackupKV { @@ -4684,10 +4980,7 @@ impl UnifiedTreeCore { "set_component_device_value: auxiliary components only" ); self.arena.set_device_value(node_id, component_type, value); - let tokens = self - .arena - .node(node_id) - .device_reclaimable_value_len(component_type); + let tokens = self.component_reclaimable_value_len_(node_id, component_type); let host_lru = self.host_lru_list_mut(component_type); if host_lru.in_list(Some(node_id)) { host_lru.remove_node(node_id); @@ -5148,7 +5441,7 @@ impl UnifiedTreeCore { let mut protected = 0usize; for &node_id in &all_nodes { let node = self.arena.node(node_id); - let tokens = node.device_reclaimable_value_len(ct); + let tokens = self.components[i].reclaimable_value_len(node); if node.device_lock_ref(ct) > 0 { protected += tokens; } else { @@ -5560,6 +5853,9 @@ impl UnifiedTreeCore { return Err("in-flight insert is not in walk phase"); } self.insert_walk_step_(&mut state); + if state.phase == InsertPhase::Overlap { + self.insert_overlap_step_(&mut state, InsertOverlapContext::default()); + } self.ongoing_insert_walk_state = Some(state); Ok(()) } @@ -5579,6 +5875,7 @@ impl UnifiedTreeCore { component_type, &mut result.device_frees, &mut result.host_frees, + &mut result.cache_actions, target, Some(&mut result.tracker), ); @@ -5630,6 +5927,7 @@ impl UnifiedTreeCore { &mut result.tracker, &mut result.device_frees, &mut result.host_frees, + &mut result.cache_actions, ); Ok(result) } diff --git a/test/registered/unit/mem_cache/test_dsv4_hicache_l2.py b/test/registered/unit/mem_cache/test_dsv4_hicache_l2.py index 14163cd2cf..48b4b28c9f 100644 --- a/test/registered/unit/mem_cache/test_dsv4_hicache_l2.py +++ b/test/registered/unit/mem_cache/test_dsv4_hicache_l2.py @@ -178,6 +178,42 @@ class TestDSV4HostViews(CustomTestCase): class TestC128L2Ownership(CustomTestCase): + def test_eviction_hooks_accept_cache_actions(self): + pages = torch.tensor([7, 8]) + data = SimpleNamespace(value=pages, host_value=None) + node = SimpleNamespace(component_data={ComponentType.C128: data}) + component = C128SidecarComponent.__new__(C128SidecarComponent) + component.tree_core = SimpleNamespace( + component_evictable_size_={ComponentType.C128: 2} + ) + device_frees = {ComponentType.C128: []} + host_frees = {ComponentType.C128: []} + actions = [] + + self.assertEqual( + component.evict_component( + node, device_frees, host_frees, cache_actions=actions + ), + (0, 0), + ) + self.assertIsNone(data.value) + self.assertIs(device_frees[ComponentType.C128][0], pages) + self.assertEqual( + component.tree_core.component_evictable_size_[ComponentType.C128], 0 + ) + + component.is_evict_device_ongoing = False + component.evict_device_start(1) + try: + self.assertIsNone( + component.evict_device_next_node( + {}, device_frees, host_frees, cache_actions=actions + ) + ) + finally: + component.evict_device_end() + self.assertEqual(actions, []) + def test_load_back_retains_and_attaches_each_c128_page(self): allocator = SimpleNamespace( c128_attn_allocator=SimpleNamespace(page_size=4), diff --git a/test/registered/unit/mem_cache/test_mamba_donated_alloc_ratio.py b/test/registered/unit/mem_cache/test_mamba_donated_alloc_ratio.py index 99ba65a547..c9c87adc90 100644 --- a/test/registered/unit/mem_cache/test_mamba_donated_alloc_ratio.py +++ b/test/registered/unit/mem_cache/test_mamba_donated_alloc_ratio.py @@ -217,7 +217,7 @@ class _RecordingComp: self.released.append(params) def release_window_lock( # SWA only - self, node, swa_uuid_for_lock, device_frees, host_frees + self, node, swa_uuid_for_lock, device_frees, host_frees, cache_actions ): pass diff --git a/test/registered/unit/mem_cache/test_mamba_path_state_cap.py b/test/registered/unit/mem_cache/test_mamba_path_state_cap.py index a5440de85a..7042f0e9d3 100644 --- a/test/registered/unit/mem_cache/test_mamba_path_state_cap.py +++ b/test/registered/unit/mem_cache/test_mamba_path_state_cap.py @@ -60,7 +60,9 @@ class _FakeTreeCore: self, node, component, *args, **kwargs ) - def _cascade_evict(self, node, component, tracker, device_frees, host_frees): + def _cascade_evict( + self, node, component, tracker, device_frees, host_frees, cache_actions + ): self.cascaded.append(node) @@ -109,7 +111,11 @@ class TestMambaPathStateCap(unittest.TestCase): device_frees = defaultdict(list) host_frees = defaultdict(list) - component._evict_excess_path_states(nodes[-1], device_frees, host_frees) + cache_actions = [] + component._evict_excess_path_states( + nodes[-1], device_frees, host_frees, cache_actions + ) + self.assertEqual(cache_actions, []) self.assertEqual(core.evicted, [nodes[0]]) self.assertEqual(core.cascaded, [nodes[0]]) @@ -137,7 +143,11 @@ class TestMambaPathStateCap(unittest.TestCase): device_frees = defaultdict(list) host_frees = defaultdict(list) - component._evict_excess_path_states(nodes[-1], device_frees, host_frees) + cache_actions = [] + component._evict_excess_path_states( + nodes[-1], device_frees, host_frees, cache_actions + ) + self.assertEqual(cache_actions, []) self.assertEqual(core.evicted, [nodes[2]]) self.assertIsNotNone(nodes[0].component_data[ComponentType.MAMBA].value) @@ -152,7 +162,11 @@ class TestMambaPathStateCap(unittest.TestCase): device_frees = defaultdict(list) host_frees = defaultdict(list) - component._evict_excess_path_states(nodes[-1], device_frees, host_frees) + cache_actions = [] + component._evict_excess_path_states( + nodes[-1], device_frees, host_frees, cache_actions + ) + self.assertEqual(cache_actions, []) self.assertIsNone(mamba_data.value) self.assertIsNotNone(mamba_data.host_value) @@ -163,7 +177,11 @@ class TestMambaPathStateCap(unittest.TestCase): device_frees = defaultdict(list) host_frees = defaultdict(list) - component._evict_excess_path_states(nodes[-1], device_frees, host_frees) + cache_actions = [] + component._evict_excess_path_states( + nodes[-1], device_frees, host_frees, cache_actions + ) + self.assertEqual(cache_actions, []) self.assertEqual(dict(device_frees), {}) self.assertEqual(core.evicted, []) @@ -298,9 +316,8 @@ class TestMambaPathCapWriteThroughOrdering(UnifiedRadixCacheFixtures, CustomTest self.assertIsNotNone(ancestor_cd.value) self.assertIsNone(ancestor_cd.host_value) - def test_cap_walk_failure_still_drains_collected_frees(self): - """A cap walk that raises mid-eviction must still free the tombstoned - slots it already collected (the pre-split inline frees could not leak).""" + def test_cap_walk_failure_still_drains_frees_and_actions(self): + """A failed cap walk drains collected frees before its actions.""" cache, allocator, req_to_token_pool = self._build_hicache_fixture() mamba_comp = cache.components[ComponentType.MAMBA] @@ -312,12 +329,36 @@ class TestMambaPathCapWriteThroughOrdering(UnifiedRadixCacheFixtures, CustomTest mamba_comp.mamba_max_states_per_path = 1 available = req_to_token_pool.mamba_allocator.available_size() - with mock.patch.object( - cache.tree_core, "_cascade_evict", side_effect=RuntimeError("boom") + action = mock.sentinel.eviction_action + + def fail_after_action(*args, **kwargs): + kwargs["cache_actions"].append(action) + raise RuntimeError("boom") + + applied_actions = [] + apply_cache_action = cache._apply_cache_action + + def check_action_after_free(received): + if received is action: + self.assertEqual( + req_to_token_pool.mamba_allocator.available_size(), available + 1 + ) + applied_actions.append(received) + else: + apply_cache_action(received) + + with ( + mock.patch.object( + cache.tree_core, "_cascade_evict", side_effect=fail_after_action + ), + mock.patch.object( + cache, "_apply_cache_action", side_effect=check_action_after_free + ), ): - with self.assertRaises(RuntimeError): + with self.assertRaisesRegex(RuntimeError, "boom"): mamba_comp.apply_component_action(MambaEvictExcessPathStates(leaf.id)) + self.assertEqual(applied_actions, [action]) self.assertIsNone(ancestor.component_data[ComponentType.MAMBA].value) self.assertEqual( req_to_token_pool.mamba_allocator.available_size(), available + 1 diff --git a/test/registered/unit/mem_cache/test_rust_tree_core_integration.py b/test/registered/unit/mem_cache/test_rust_tree_core_integration.py index ec2e0320f3..451c1da07b 100644 --- a/test/registered/unit/mem_cache/test_rust_tree_core_integration.py +++ b/test/registered/unit/mem_cache/test_rust_tree_core_integration.py @@ -876,7 +876,7 @@ def test_stale_handle_operations_raise_key_error_without_poisoning_the_core(): [live_root, stale_root], live_root, True ), "evict_excess_path_states": lambda: core.evict_excess_path_states( - stale_root, {}, {} + stale_root, {}, {}, [] ), "inc_host_lock_ref": lambda: core.inc_host_lock_ref(stale_root), "dec_host_lock_ref": lambda: core.dec_host_lock_ref( @@ -3334,14 +3334,20 @@ def test_python_component_returns_immutable_internal_backup_request(component): for ct in cache.tree_components } tracker, device_frees, host_frees = {component: 0}, {}, {} + cache_actions = [] core.evict_device_start(component, num_tokens) try: request = cache.components[component].evict_device_next_node( - tracker, device_frees, host_frees + tracker, device_frees, host_frees, cache_actions ) assert isinstance(request, InternalStateBackup) assert (request.node_id, request.num_tokens) == (victim, num_tokens) - assert not case.events and not device_frees and not host_frees + assert ( + not case.events + and not device_frees + and not host_frees + and not cache_actions + ) assert tracker == {component: 0} for ct, value in original.items(): assert torch.equal(core.get_component_device_value(victim, ct), value) @@ -3984,6 +3990,9 @@ def test_buffer_backup_snapshot_round_trips_and_detects_a_split(): ) leaf = inserted.last_device_node + action = core.build_backup_kv_action(leaf) + assert isinstance(action, BackupKV) + assert action.node_ids == [leaf] snapshot = core.snapshot_buffer_backup(leaf, pass_prefix_keys=True) assert snapshot.node_id == leaf assert snapshot.parent_is_root @@ -4008,6 +4017,8 @@ def test_buffer_backup_snapshot_round_trips_and_detects_a_split(): core.reset() assert core.snapshot_buffer_backup(leaf, pass_prefix_keys=True) is None assert core.validate_buffer_backup(leaf, len(snapshot.key)) is None + with pytest.raises(KeyError): + core.build_backup_kv_action(leaf) def test_buffer_backup_snapshot_preserves_bigram_keys(): @@ -4594,7 +4605,11 @@ def test_mamba_path_cap_evicts_excess_states_through_the_adapter(): a for a in result.cache_actions if isinstance(a, MambaEvictExcessPathStates) ] device_frees, host_frees = defaultdict(list), defaultdict(list) - core.evict_excess_path_states(action.tail_node_id, device_frees, host_frees) + cache_actions = [] + core.evict_excess_path_states( + action.tail_node_id, device_frees, host_frees, cache_actions + ) + assert cache_actions == [] # The two shallow states free; the tail's survives the soft cap. assert sorted(t.item() for t in device_frees[ComponentType.MAMBA]) == [7, 8] assert host_frees == {} diff --git a/test/registered/unit/mem_cache/test_tree_core_registry.py b/test/registered/unit/mem_cache/test_tree_core_registry.py index d8396155c3..c1501a4f7e 100644 --- a/test/registered/unit/mem_cache/test_tree_core_registry.py +++ b/test/registered/unit/mem_cache/test_tree_core_registry.py @@ -54,7 +54,12 @@ class _StubFullComponent(TreeComponent): return None def evict_component( - self, node, device_frees, host_frees, target: EvictLayer = EvictLayer.DEVICE + self, + node, + device_frees, + host_frees, + cache_actions, + target: EvictLayer = EvictLayer.DEVICE, ) -> tuple[int, int]: return 0, 0 @@ -67,7 +72,7 @@ class _StubFullComponent(TreeComponent): def _evict_device_start(self, request_cnt) -> None: pass - def _evict_device_next_node(self, tracker, device_frees, host_frees): + def _evict_device_next_node(self, tracker, device_frees, host_frees, cache_actions): return None def _evict_device_end(self) -> None: diff --git a/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_core.py b/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_core.py index 7f4fdb9d49..894cdf6c3f 100644 --- a/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_core.py +++ b/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_core.py @@ -37,6 +37,9 @@ from sglang.srt.mem_cache.unified_cache.components.base import ( ComponentType, EvictLayer, ) +from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import ( + EvictDeviceLeafResult, +) from sglang.srt.mem_cache.unified_radix_cache import ( COMPONENT_REGISTRY, UnifiedLRUList, @@ -2097,6 +2100,37 @@ class TestUnifiedRadixCacheActionRouting(CustomTestCase): [mock.call(first), mock.call(second)] ) + def test_eviction_free_failure_still_drains_actions_in_order(self): + cache = object.__new__(UnifiedRadixCache) + first = UpdatePendingBackupOnNodeSplit( + ack_id=1, old_node_id=1, new_node_id=2, new_child_node_id=1 + ) + second = UpdatePendingBackupOnNodeSplit( + ack_id=2, old_node_id=3, new_node_id=4, new_child_node_id=3 + ) + result = EvictDeviceLeafResult() + result.cache_actions.extend([first, second]) + error = RuntimeError("allocator free failed") + calls = [] + + def fail_free(device_frees, host_frees): + calls.append("free") + raise error + + cache._free_values = mock.Mock(side_effect=fail_free) + cache._apply_cache_action = mock.Mock(side_effect=calls.append) + with self.assertRaises(RuntimeError) as raised: + cache._apply_eviction_result(result) + + self.assertIs(raised.exception, error) + cache._free_values.assert_called_once_with( + result.device_frees, result.host_frees + ) + self.assertEqual(calls, ["free", first, second]) + self.assertFalse(result.cache_actions) + cache._apply_cache_actions(result.cache_actions) + self.assertEqual(cache._apply_cache_action.call_count, 2) + def test_chained_replace_write_through_requires_list_order(self): # A pending node split twice in one walk emits two chained Replaces: # the second one's old_node_id only enters the publish list when the diff --git a/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_insert_walk.py b/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_insert_walk.py index ae8503ac21..199425b0d8 100644 --- a/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_insert_walk.py +++ b/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_insert_walk.py @@ -24,6 +24,7 @@ from sglang.srt.mem_cache.radix_cache import RadixKey from sglang.srt.mem_cache.unified_cache.cache_action import ( FreeDeviceKV, FreeDeviceKVFullOnly, + UpdatePendingBackupOnNodeSplit, ) from sglang.srt.mem_cache.unified_cache.components.base import ( ComponentType, @@ -522,6 +523,34 @@ class TestReturnedValuesDrain(UnifiedRadixCacheFixtures, CustomTestCase): cfg = CacheConfig() + def test_action_only_eviction_step_makes_progress(self): + # The patched hook is the Python component's; the Rust core runs its own. + cache, _, _ = build_fixture(self.cfg, tree_core_backend="python") + core = cache.tree_core + component = cache.components[ComponentType.FULL] + root = cache.root_node_handle() + action = UpdatePendingBackupOnNodeSplit( + ack_id=None, old_node_id=root, new_node_id=root, new_child_node_id=root + ) + + def emit_action(tracker, device_frees, host_frees, cache_actions): + cache_actions.append(action) + return None + + core.evict_device_start(ComponentType.FULL, 1) + try: + with ( + mock.patch.object(component, "_evict_device_next_node", emit_action), + mock.patch.object(cache, "_apply_cache_action") as apply, + ): + self.assertEqual( + cache._evict_device_next_node(ComponentType.FULL, {}), + (None, True), + ) + apply.assert_called_once_with(action) + finally: + core.evict_device_end(ComponentType.FULL) + def test_each_eviction_step_result_is_drained(self): """Every step wrapper hands its result's frees to _free_values and passes the payload through; a wrapper that forgets strands pool slots."""