From 77091cea68d0701d6cc71edfb45f2815c4f8d791 Mon Sep 17 00:00:00 2001 From: Jialin Ouyang Date: Mon, 28 Sep 2026 19:18:57 -0700 Subject: [PATCH] [Radix Cache] Sync Rust TreeCore and make it the default (#39627) Co-authored-by: Ke Bao Co-authored-by: Zhangheng Co-authored-by: alphabetc1 --- .../radix_eviction_policy.mdx | 4 - python/sglang/srt/environ.py | 13 +- .../srt/mem_cache/rust_tree_core/adapter.py | 199 +- .../unified_cache/components/__init__.py | 2 + .../unified_cache/components/base.py | 18 +- .../unified_cache/components/mamba.py | 50 +- .../mem_cache/unified_cache/components/swa.py | 75 +- .../unified_cache/tree_core_registry.py | 116 +- .../unified_cache/unified_tree_core.py | 106 +- .../unified_tree_core_interface.py | 49 +- .../srt/mem_cache/unified_radix_cache.py | 43 +- .../sglang/test/scripted_runtime/__init__.py | 11 + .../context/lock_ref_exhauster.py | 35 +- .../test/scripted_runtime/context/radix.py | 80 +- .../test/scripted_runtime/req_handle.py | 11 +- .../scripted_runtime/tree_core_inspection.py | 34 + rust/sglang-radix-tree/README.md | 41 +- .../sglang-radix-tree/src/components/mamba.rs | 52 +- rust/sglang-radix-tree/src/components/swa.rs | 117 +- rust/sglang-radix-tree/src/node.rs | 156 +- rust/sglang-radix-tree/src/python_bindings.rs | 1055 ++++--- .../src/tests/components/base.rs | 1 + .../src/tests/components/full.rs | 126 + .../src/tests/components/mamba.rs | 589 +++- .../src/tests/components/swa.rs | 1063 ++++++- rust/sglang-radix-tree/src/tests/node.rs | 258 +- .../src/tests/unified_lru_list.rs | 274 +- .../src/tests/unified_tree_core.rs | 273 ++ .../sglang-radix-tree/src/unified_lru_list.rs | 67 +- .../src/unified_tree_core.rs | 514 ++- .../test_scripted_core_1gpu.py | 17 +- .../test_disaggregation_optimistic_prefill.py | 4 + .../test_unified_radix_cache_kl_full.py | 2 - .../mem_cache/test_page_interleave_shard.py | 60 +- .../unit/mem_cache/test_rust_tree_core.py | 31 +- .../mem_cache/test_rust_tree_core_errors.py | 142 + .../test_rust_tree_core_integration.py | 2781 ++++++++++++++++- .../test_rust_unified_radix_cache_unittest.py | 4 +- .../test_session_unified_radix_cache.py | 16 + .../unit/mem_cache/test_tree_core_registry.py | 356 ++- .../test_unified_radix_cache_unittest.py | 116 +- .../test_tree_core_inspection.py | 124 + 42 files changed, 8065 insertions(+), 1020 deletions(-) create mode 100644 python/sglang/test/scripted_runtime/tree_core_inspection.py create mode 100644 test/registered/unit/mem_cache/test_rust_tree_core_errors.py create mode 100644 test/registered/unit/scripted_runtime/test_tree_core_inspection.py diff --git a/docs/docs/advanced_features/radix_eviction_policy.mdx b/docs/docs/advanced_features/radix_eviction_policy.mdx index 787430be3f..91e3cfd4d8 100644 --- a/docs/docs/advanced_features/radix_eviction_policy.mdx +++ b/docs/docs/advanced_features/radix_eviction_policy.mdx @@ -50,10 +50,6 @@ Omit the flag to accept every default. An unrecognized key fails at startup rath TypeError: SLRUStrategy.__init__() got an unexpected keyword argument 'protected_treshold' ``` - -`--radix-eviction-policy-config` is not supported by the experimental Rust tree core (`SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND=rust`), which builds its strategy from the policy name alone. Passing both fails at startup. - - ### Policy parameters `slru` and `tlru` take parameters. `lru`, `lfu`, and `priority` take none, so `--radix-eviction-policy-config` has no effect with them and any key is an error. diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 9af31cae81..6566413a75 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -677,8 +677,17 @@ class Envs: SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS = EnvBool(True) # Decode batches between SWA out-of-window evictions. SGLANG_SWA_EVICTION_INTERVAL = EnvInt(128) - # Registered TreeCore backend serving the unified radix cache. - SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND = EnvStr("python") + # The tree-core registry falls back to Python for: + # - Session-aware caching. + # - C128 or other unsupported components. + # - Custom component overrides. + # - Non-Linux platforms. + # - Unsupported PyTorch versions. + # - Devices other than CPU or CUDA. + # - Installs with neither the Rust extension nor its sources. + # - Source builds with a missing or unusable Rust toolchain. + # This also applies when Rust is explicitly selected. + SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND = EnvStr("rust") SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW = EnvBool(False) # =================================================================== diff --git a/python/sglang/srt/mem_cache/rust_tree_core/adapter.py b/python/sglang/srt/mem_cache/rust_tree_core/adapter.py index 95b52960b6..22671f7dbd 100644 --- a/python/sglang/srt/mem_cache/rust_tree_core/adapter.py +++ b/python/sglang/srt/mem_cache/rust_tree_core/adapter.py @@ -3,6 +3,7 @@ from __future__ import annotations +import sys from array import array from typing import TYPE_CHECKING, Optional, Sequence @@ -14,6 +15,8 @@ from sglang.srt.disaggregation.kv_events import ( BlockStored, StorageMedium, ) +from sglang.srt.environ import envs +from sglang.srt.mem_cache.allocator.swa import is_swa_req_ring from sglang.srt.mem_cache.base_prefix_cache import ( DecLockRefParams, DecLockRefResult, @@ -39,6 +42,7 @@ from sglang.srt.mem_cache.unified_cache.cache_action import ( SWARebuild, ) from sglang.srt.mem_cache.unified_cache.component_type import ComponentType +from sglang.srt.mem_cache.unified_cache.components import CacheTransferPhase from sglang.srt.mem_cache.unified_cache.unified_tree_core import StorageBackupSpec from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import ( BufferBackupSnapshot, @@ -54,6 +58,7 @@ from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import ( RadixCacheWalkResult, UnifiedTreeCoreInterface, ) +from sglang.srt.mem_cache.utils import get_eviction_strategy from sglang.srt.runtime_context import get_exec, mamba_cache_chunk_size from sglang.srt.utils import assert_int64_array @@ -65,10 +70,23 @@ if TYPE_CHECKING: CacheAction, ComponentAction, ) - from sglang.srt.mem_cache.unified_cache.components import CacheTransferPhase from sglang.srt.mem_cache.unified_cache.unified_tree_core import UnifiedTreeNode +def _tlru_float_config(native_bindings, threshold, next_prompt_estimate): + """Pass numeric values without changing Python's integer/float distinction.""" + if not all( + isinstance(value, (int, float)) for value in (threshold, next_prompt_estimate) + ): + raise TypeError("T-LRU parameters must be integer or floating-point numbers") + threshold = float(threshold) + if isinstance(next_prompt_estimate, int): + return native_bindings.TlruFloatConfig( + threshold, 0.0, integer_estimate=next_prompt_estimate + ) + return native_bindings.TlruFloatConfig(threshold, float(next_prompt_estimate)) + + def _radix_key_buffer(key: RadixKey) -> array: """The key's token ids honoring `limit`; view-independent since the binding derives its own atoms.""" @@ -231,6 +249,7 @@ def _insert_step_from_binding(step) -> InsertStepResult: mamba_exist=step.result.mamba_exist, swa_branch_inserted=step.result.swa_branch_inserted, host_insert_dropped=step.result.host_insert_dropped, + rotation_tail_declined=step.result.rotation_tail_declined, adopted_ranges=( { ComponentType(component_type): list(ranges) @@ -297,6 +316,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): """A TreeCore backed by the Rust extension binding.""" _bindings = bindings + supports_rotation_base = True def __init__(self, params: CacheInitParams): assert params.tree_components is not None @@ -324,12 +344,25 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): raise ValueError( "Rust TreeCore does not support component_registry_override" ) - # The Rust core builds its own eviction strategy from the policy name - # alone, so a config would be dropped rather than applied. - if params.eviction_policy_config: - raise ValueError( - "Rust TreeCore does not support --radix-eviction-policy-config" - ) + # Validate the same constructor options as Python before passing the + # configured eviction parameters to the native strategy. + eviction_strategy = get_eviction_strategy( + params.eviction_policy, params.eviction_policy_config + ) + tlru_tail_budget = 0 + tlru_float_config = None + if params.eviction_policy.lower() == "tlru": + threshold = eviction_strategy.threshold + next_prompt_estimate = eviction_strategy.next_prompt_estimate + if isinstance(threshold, int) and isinstance(next_prompt_estimate, int): + # Subtract before clamping to preserve arbitrary-size integers. + tlru_tail_budget = min( + max(threshold - next_prompt_estimate, 0), 2 * sys.maxsize + 1 + ) + else: + tlru_float_config = _tlru_float_config( + self._bindings, threshold, next_prompt_estimate + ) if ComponentType.SWA in self.tree_components and ( params.sliding_window_size is None or params.sliding_window_size <= 0 ): @@ -342,8 +375,9 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): params.is_eagle and ComponentType.MAMBA not in self.tree_components ) - # ``device`` is derived from the construction-time allocator; the - # allocator/pool themselves are owned by the cache, not the tree. + # The cache owns allocation; keep its allocator for resolving current + # unified SWA addresses when building a host backup. + self._allocator = params.token_to_kv_pool_allocator if params.token_to_kv_pool_allocator: device = torch.device(params.token_to_kv_pool_allocator.device) # A bare "cuda" means the process's current device, not cuda:0. @@ -362,12 +396,18 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): self._binding = self._binding_class()( self._bindings.TreeCoreInitParamsBinding( eviction_policy=params.eviction_policy, + slru_protected_threshold=getattr( + eviction_strategy, "protected_threshold", 2 + ), + tlru_tail_budget=tlru_tail_budget, + tlru_float_config=tlru_float_config, page_size=params.page_size, is_write_back=False, enable_hicache=False, write_through_threshold=256, device=str(self.device), swa_sliding_window_size=params.sliding_window_size, + swa_req_ring=is_swa_req_ring(self._allocator), enable_kv_cache_events=params.enable_kv_cache_events, mamba_cache_chunk_size=( mamba_cache_chunk_size() if has_mamba else None @@ -415,9 +455,14 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): def swa_tombstone_ranges( self, key: RadixKey, start: int, end: int ) -> list[tuple[int, int]]: - raise NotImplementedError( - "swa_tombstone_ranges: buffer-mode SWA window repair is not yet " - "ported to the Rust tree core" + return self._binding.swa_tombstone_ranges( + self._bindings.MatchParamsBinding( + key=_radix_key_buffer(key), + extra_key=key.extra_key, + cache_salt=key.cache_salt, + ), + start, + end, ) def attach_swa_window( @@ -426,10 +471,18 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): window_start: int, window_end: int, swa_values: torch.Tensor, - ) -> list: - raise NotImplementedError( - "attach_swa_window: buffer-mode SWA window repair is not yet " - "ported to the Rust tree core" + ) -> list[CacheAction | ComponentAction]: + return _cache_actions_from_tagged( + self._binding.attach_swa_window( + self._bindings.MatchParamsBinding( + key=_radix_key_buffer(key), + extra_key=key.extra_key, + cache_salt=key.cache_salt, + ), + window_start, + window_end, + swa_values, + ) ) def inc_lock_ref( @@ -453,6 +506,24 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): ) return DecLockRefResult() + def dec_window_lock_only( + self, + node_id: NodeId, + component_type: ComponentType, + params: DecLockRefParams, + ) -> DecSwaLockOnlyResult: + result = DecSwaLockOnlyResult() + new_device_frees, new_host_frees = self._binding.dec_window_lock_only( + node_id, + int(component_type), + _dec_lock_ref_params_to_binding(self._bindings, params), + ) + for component, tensors in new_device_frees.items(): + result.device_frees[ComponentType(component)].extend(tensors) + for component, tensors in new_host_frees.items(): + result.host_frees[ComponentType(component)].extend(tensors) + return result + def dec_swa_lock_only( self, node_id: NodeId, @@ -484,20 +555,43 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): result = EvictDeviceNextNodeResult( node_id=binding_result.node_id, made_progress=binding_result.made_progress, + unbacked_tokens=binding_result.unbacked_tokens, + mamba_backup_node_id=binding_result.mamba_backup_node_id, + swa_backup_node_id=binding_result.swa_backup_node_id, + swa_backup_num_tokens=binding_result.swa_backup_num_tokens, + ) + return _fill_evict_result(binding_result, result) + + def finish_mamba_state_eviction(self, node_id: NodeId) -> EvictDeviceNextNodeResult: + binding_result = self._binding.finish_mamba_state_eviction(node_id) + result = EvictDeviceNextNodeResult( + node_id=binding_result.node_id, + made_progress=binding_result.made_progress, + unbacked_tokens=binding_result.unbacked_tokens, + ) + return _fill_evict_result(binding_result, result) + + def finish_swa_state_eviction(self, node_id: NodeId) -> EvictDeviceNextNodeResult: + binding_result = self._binding.finish_swa_state_eviction(node_id) + result = EvictDeviceNextNodeResult( + node_id=binding_result.node_id, + made_progress=binding_result.made_progress, + unbacked_tokens=binding_result.unbacked_tokens, ) return _fill_evict_result(binding_result, result) def evict_device_leaf( self, node_id: NodeId, is_write_back: bool ) -> EvictDeviceLeafResult: - # The binding reads is_write_back from the core's construction config. + # The binding reads is_write_back from the core's current config. assert is_write_back == self.is_write_back, ( - "is_write_back must match the core's construction config" + "is_write_back must match the core's current config" ) binding_result = self._binding.evict_device_leaf(node_id) backup = binding_result.backup_kv result = EvictDeviceLeafResult( - backup_kv=_cache_action_from_tagged(backup) if backup is not None else None + unbacked_tokens=binding_result.unbacked_tokens, + backup_kv=_cache_action_from_tagged(backup) if backup is not None else None, ) return _fill_evict_result(binding_result, result) @@ -529,6 +623,9 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): def component_evictable_size(self, component_type: ComponentType) -> int: return self._binding.component_evictable_size(int(component_type)) + def component_protected_size(self, component_type: ComponentType) -> int: + return self._binding.component_protected_size(int(component_type)) + def full_evictable_size(self) -> int: return self._binding.full_evictable_size() @@ -632,6 +729,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): chunked=params.chunked, priority=0 if params.priority is None else params.priority, track_adopted_ranges=params.track_adopted_ranges, + rotation_base=params.rotation_base, ) ) return _insert_step_from_binding(step) @@ -649,7 +747,9 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): self, component_type: ComponentType, num_tokens: int ) -> DriveHostEvictionResult: binding_result = self._binding.drive_host_eviction( - int(component_type), num_tokens + int(component_type), + num_tokens, + envs.SGLANG_HICACHE_SKIP_HOST_DUPLICATE_RECLAIM.get(), ) return _fill_evict_result(binding_result, DriveHostEvictionResult()) @@ -761,7 +861,31 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): self, node_id: NodeId ) -> tuple[torch.Tensor, dict[ComponentType, list[PoolTransfer]]]: device_value, comp_xfers = self._binding.build_backup_spec(node_id) - return device_value, _comp_xfers_from_binding(comp_xfers) + comp_xfers = _comp_xfers_from_binding(comp_xfers) + self._refresh_swa_backup_indices(comp_xfers.get(ComponentType.SWA, ())) + return device_value, comp_xfers + + def _refresh_swa_backup_indices(self, transfers: Sequence[PoolTransfer]) -> None: + if not transfers: + return + from sglang.srt.mem_cache.allocator.unified_hybrid_swa import ( + UnifiedSWAAllocatorBase, + ) + + if not isinstance(self._allocator, UnifiedSWAAllocatorBase): + return + for transfer in transfers: + if transfer.name != PoolName.SWA or not transfer.nodes_to_load: + continue + # Compaction can relocate SWA rows after the tree records them. + # FULL virtual IDs retain their meaning across that relocation. + full_values = [ + self.get_component_device_value(node_id, ComponentType.FULL) + for node_id in transfer.nodes_to_load + ] + transfer.device_indices = self._allocator.translate_loc_from_full_to_swa( + torch.cat(full_values) + ).to(torch.int64) def build_storage_backup_spec( self, node_id: NodeId, pass_prefix_keys: bool @@ -805,7 +929,10 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): ) if transfers is None: return None - return [_transfer_from_binding(transfer) for transfer in transfers] + transfers = [_transfer_from_binding(transfer) for transfer in transfers] + if phase == CacheTransferPhase.BACKUP_HOST: + self._refresh_swa_backup_indices(transfers) + return transfers def build_load_back_spec( self, node_id: NodeId, req: Optional[Req] = None @@ -815,7 +942,30 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): kv_xfer, comp_xfers = self._binding.build_load_back_spec( node_id, mamba_pool_idx ) - return _transfer_from_binding(kv_xfer), _comp_xfers_from_binding(comp_xfers) + kv_xfer = _transfer_from_binding(kv_xfer) + comp_xfers = _comp_xfers_from_binding(comp_xfers) + swa_xfers = comp_xfers.get(ComponentType.SWA, ()) + if swa_xfers: + full_node_ids = kv_xfer.nodes_to_load or [] + full_load_slices = {} + offset = 0 + for full_node_id, count in zip( + full_node_ids, self._binding.get_node_key_lengths(full_node_ids) + ): + full_load_slices[full_node_id] = slice(offset, offset + count) + offset += count + for transfer in swa_xfers: + # SWA may have holes between resident nodes, or reload while + # FULL stays resident. Preserve the SWA transfer's node order. + transfer.anchor_index_parts = [ + ( + full_load_slices[nid] + if nid in full_load_slices + else self.get_component_device_value(nid, ComponentType.FULL) + ) + for nid in transfer.nodes_to_load or () + ] + return kv_xfer, comp_xfers def prefetch_anchor_info( self, node_id: NodeId @@ -878,6 +1028,9 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): def root_node_handle(self, extra_key: Optional[str] = None) -> NodeId: return self._binding.root_node_handle(extra_key) + def rotation_base_of(self, node_id: NodeId) -> Optional[int]: + return self._binding.rotation_base_of(node_id) + def dfs_weight_order(self, node_ids: Sequence[NodeId]) -> list[int]: return self._binding.dfs_weight_order(list(node_ids)) diff --git a/python/sglang/srt/mem_cache/unified_cache/components/__init__.py b/python/sglang/srt/mem_cache/unified_cache/components/__init__.py index 147f57205e..794431788a 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/__init__.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/__init__.py @@ -6,6 +6,7 @@ from sglang.srt.mem_cache.unified_cache.components.base import ( ComponentType, EvictLayer, ExternalLinkerLoadPhase, + InternalStateBackup, LinkerTransferPhase, LRURefreshPhase, PrepareLoadBackResult, @@ -23,6 +24,7 @@ __all__ = [ "ComponentData", "ComponentType", "ExternalLinkerLoadPhase", + "InternalStateBackup", "LinkerTransferPhase", "EvictLayer", "FullComponent", diff --git a/python/sglang/srt/mem_cache/unified_cache/components/base.py b/python/sglang/srt/mem_cache/unified_cache/components/base.py index d29099d424..e9c3996056 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/base.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/base.py @@ -63,6 +63,15 @@ class EvictLayer(IntFlag): ALL = DEVICE | HOST +@dataclasses.dataclass(frozen=True) +class InternalStateBackup: + """Pause eviction for a host backup before freeing internal device state.""" + + node_id: NodeId + # Host capacity required by the backup, in this component's pool units. + num_tokens: int + + @dataclasses.dataclass(frozen=True) class PrepareLoadBackResult: """Outcome of prepare_load_back; default = nothing to prepare.""" @@ -522,11 +531,12 @@ class TreeComponent(ABC): tracker: dict[ComponentType, int], device_frees: dict[ComponentType, list[torch.Tensor]], host_frees: dict[ComponentType, list[torch.Tensor]], - ) -> Optional[NodeId]: - """Advance one eviction step and return a device leaf, if selected. + ) -> NodeId | InternalStateBackup | None: + """Return a device leaf, an internal backup request, or no selection. Implementations must return after one allocator-relevant internal mutation so the caller can drain pending frees before continuing. + Backup requests leave device state intact until the core resumes eviction. """ assert self.is_evict_device_ongoing, ( f"{self.component_type} device eviction not started" @@ -552,8 +562,8 @@ class TreeComponent(ABC): tracker: dict[ComponentType, int], device_frees: dict[ComponentType, list[torch.Tensor]], host_frees: dict[ComponentType, list[torch.Tensor]], - ) -> Optional[NodeId]: - """Advance the walk by at most one allocator-relevant mutation.""" + ) -> NodeId | InternalStateBackup | None: + """Select a leaf, request backup, or perform at most one internal mutation.""" ... @abstractmethod diff --git a/python/sglang/srt/mem_cache/unified_cache/components/mamba.py b/python/sglang/srt/mem_cache/unified_cache/components/mamba.py index 16c113f598..c55ddab016 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/mamba.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/mamba.py @@ -30,6 +30,7 @@ from sglang.srt.mem_cache.unified_cache.components.base import ( CacheTransferPhase, ComponentType, EvictLayer, + InternalStateBackup, LinkerTransferPhase, LRURefreshPhase, PrepareLoadBackResult, @@ -369,7 +370,7 @@ class MambaComponent(TreeComponent): tracker: dict[ComponentType, int], device_frees: dict[ComponentType, list[torch.Tensor]], host_frees: dict[ComponentType, list[torch.Tensor]], - ) -> Optional[NodeId]: + ) -> NodeId | InternalStateBackup | None: """Advance one device-eviction step and return a leaf, if selected. An internal tombstone is one complete step so the caller can apply its @@ -404,9 +405,18 @@ class MambaComponent(TreeComponent): ) return x.id if not enabled: - x_next = lru.get_prev_no_lock(x) - # write_back: demote the state to host before the internal tombstone. - self._maybe_backup_node_before_state_tombstone(x) + self._evict_device_cursor = lru.get_prev_no_lock(x) + cd = x.component_data[ct] + if ( + self.tree_core.enable_hicache + and self.tree_core.is_write_back + and cd.host_value is None + and not x.backuped + and x.component_data[BASE_COMPONENT_TYPE].value is not None + ): + # Keep the state live until the controller has attempted its host + # backup. Session cursors advance after the resumed tombstone. + return InternalStateBackup(node_id=x.id, num_tokens=1) self.tree_core._evict_component_and_detach_lru( x, self, @@ -418,38 +428,10 @@ class MambaComponent(TreeComponent): self.tree_core._cascade_evict( x, self, tracker, device_frees=device_frees, host_frees=host_frees ) - self._evict_device_cursor = lru.cursor_next() if enabled else x_next + if enabled: + self._evict_device_cursor = lru.cursor_next() return None - def _maybe_backup_node_before_state_tombstone(self, node: UnifiedTreeNode) -> None: - """Demote an internal node's mamba state to host before its tombstone - (write_back only), mirroring the leaf deferred-demote path. - - The match validator passes only nodes holding the state on some - layer, so a dropped internal state caps the match frontier at this - node forever, leaving the subtree's still-resident KV unservable. - Best-effort: this walk must make progress (it satisfies an imminent - slot allocation), so any failure falls back to the legacy drop. - """ - cache = self.cache - cd = node.component_data[self.component_type] - if ( - cache.cache_controller is None - or not cache.is_write_back - or cd.host_value is not None - or node.backuped - or node.component_data[BASE_COMPONENT_TYPE].value is None - ): - return - # The backup executor pre-evicts only the KV host pool; make room - # for the state slot the way the PREFETCH hook does. - if ( - self._mamba_pool_host is not None - and self._mamba_pool_host.available_size() < 1 - ): - cache.evict_host(1, self.component_type) - cache.backup_node_for_write_back(node.id) - def _evict_device_end(self) -> None: """Clear the device-eviction walk cursor state.""" if self.tree_core.enable_session_radix_cache: diff --git a/python/sglang/srt/mem_cache/unified_cache/components/swa.py b/python/sglang/srt/mem_cache/unified_cache/components/swa.py index 2d2550e45f..e5263d91eb 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/swa.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/swa.py @@ -36,6 +36,7 @@ from sglang.srt.mem_cache.unified_cache.components.base import ( ComponentType, EvictLayer, ExternalLinkerLoadPhase, + InternalStateBackup, LinkerTransferPhase, LRURefreshPhase, PreparePrefetchResult, @@ -711,7 +712,7 @@ class SWAComponent(TreeComponent): tracker: dict[ComponentType, int], device_frees: dict[ComponentType, list[torch.Tensor]], host_frees: dict[ComponentType, list[torch.Tensor]], - ) -> Optional[NodeId]: + ) -> NodeId | InternalStateBackup | None: """Advance one device-eviction step and return a leaf, if selected. An internal tombstone is one complete step so the caller can apply its @@ -746,9 +747,33 @@ class SWAComponent(TreeComponent): ) return x.id if not enabled: - x_next = lru.get_prev_no_lock(x) - # write_back: demote the SWA KV to host before the internal tombstone. - self._maybe_backup_node_before_swa_tombstone(x) + self._evict_device_cursor = lru.get_prev_no_lock(x) + cd = x.component_data[ct] + if ( + self.tree_core.enable_hicache + and self.tree_core.is_write_back + and self.tree_core.has_swa_host_pool + and cd.host_value is None + and not x.backuped + and x.component_data[BASE_COMPONENT_TYPE].value is not None + ): + # Reserve the whole unbacked window, not just this victim, before + # its best-effort host backup and resumed internal tombstone. + needed = sum( + len(node.component_data[ct].value) + for node in self._collect_unbacked_swa_nodes(x) + ) + if needed: + if ct == ComponentType.SWA: + return InternalStateBackup(node_id=x.id, num_tokens=needed) + # Custom SWA component types use Python's inherited inline + # path; the shared finish hook is for built-in SWA only. + if ( + self._swa_kv_pool_host is not None + and self._swa_kv_pool_host.available_size() < needed + ): + self.cache.evict_host(needed, ct) + self.cache.backup_node_for_write_back(x.id) self.tree_core._evict_component_and_detach_lru( x, self, @@ -760,48 +785,10 @@ class SWAComponent(TreeComponent): self.tree_core._cascade_evict( x, self, tracker, device_frees=device_frees, host_frees=host_frees ) - self._evict_device_cursor = lru.cursor_next() if enabled else x_next + if enabled: + self._evict_device_cursor = lru.cursor_next() return None - def _maybe_backup_node_before_swa_tombstone(self, node: UnifiedTreeNode) -> None: - """Demote an internal node's SWA KV to host before its tombstone - (write_back only), mirroring the leaf deferred-demote path. - - The match validator treats an unbacked tombstone as a window reset, - so a dropped internal SWA segment caps the match frontier until a - full sliding window re-accumulates below it, leaving up to one - window of still-resident KV unservable. The leaf backup walk covers - ancestors only within one window of the evicted leaf, so an - internal node whose child spans the window arrives here unbacked. - Best-effort: this walk must make progress (it satisfies an imminent - allocation), so any failure falls back to the legacy drop. - """ - cache = self.cache - cd = node.component_data[self.component_type] - if ( - cache.cache_controller is None - or not cache.is_write_back - or not self.tree_core.has_swa_host_pool - or cd.host_value is not None - or node.backuped - or node.component_data[BASE_COMPONENT_TYPE].value is None - ): - return - # The backup executor pre-evicts only the KV host pool; make room - # in the SWA host pool the way the PREFETCH hook does. - needed = sum( - len(n.component_data[self.component_type].value) - for n in self._collect_unbacked_swa_nodes(node) - ) - if needed == 0: - return - if ( - self._swa_kv_pool_host is not None - and self._swa_kv_pool_host.available_size() < needed - ): - cache.evict_host(needed, self.component_type) - cache.backup_node_for_write_back(node.id) - def _evict_device_end(self) -> None: """Clear the device-eviction walk cursor state.""" if self.tree_core.enable_session_radix_cache: diff --git a/python/sglang/srt/mem_cache/unified_cache/tree_core_registry.py b/python/sglang/srt/mem_cache/unified_cache/tree_core_registry.py index 47bd6e1b1e..bed9e74b4a 100644 --- a/python/sglang/srt/mem_cache/unified_cache/tree_core_registry.py +++ b/python/sglang/srt/mem_cache/unified_cache/tree_core_registry.py @@ -1,18 +1,28 @@ """Registry for pluggable TreeCore implementations. The unified cache constructs its TreeCore through `create_tree_core`, selected -by SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND (default "python"). To plug in a custom -implementation, register it under a string name via -`register_tree_core_backend(name, factory)`. +by its per-cache override or SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND (default +"rust"). Rust selections use centralized compatibility fallbacks. To plug in a +custom implementation, register it via `register_tree_core_backend(name, factory)`. """ from __future__ import annotations +import importlib.util +import logging +import re +import subprocess +import sys +from pathlib import Path from typing import TYPE_CHECKING, Callable, Optional +import torch + +from sglang.srt.environ import envs +from sglang.srt.mem_cache.unified_cache.component_type import ComponentType + if TYPE_CHECKING: from sglang.srt.mem_cache.cache_init_params import CacheInitParams - from sglang.srt.mem_cache.unified_cache.component_type import ComponentType from sglang.srt.mem_cache.unified_cache.components import TreeComponent from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import ( UnifiedTreeCoreInterface, @@ -24,6 +34,101 @@ TreeCoreFactory = Callable[ ] _TREE_CORE_REGISTRY: dict[str, TreeCoreFactory] = {} +_RUST_TREE_CORE_MODULE = "sglang.srt.mem_cache.rust_tree_core.mem_cache" +_RUST_TREE_CORE_MANIFEST = ( + Path(__file__).resolve().parents[5] / "rust" / "sglang-radix-tree" / "Cargo.toml" +) +logger = logging.getLogger(__name__) + + +def _rust_fallback_reason(params: CacheInitParams) -> Optional[str]: + if params.enable_session_radix_cache: + return "session-aware caching requires the Python TreeCore" + if params.tree_components is not None and set(params.tree_components) - { + ComponentType.FULL, + ComponentType.SWA, + ComponentType.MAMBA, + }: + return "the configured components require the Python TreeCore" + if params.component_registry_override: + return "custom components require the Python TreeCore" + if sys.platform != "linux": + return "the Rust TreeCore supports Linux only" + from sglang.srt.rust_extensions.torch_build import ( + _MAX_SUPPORTED_TORCH, + _MIN_SUPPORTED_TORCH, + ) + + match = re.match(r"^(\d+)\.(\d+)", str(torch.__version__)) + if match is None or not ( + _MIN_SUPPORTED_TORCH + <= (int(match.group(1)), int(match.group(2))) + <= _MAX_SUPPORTED_TORCH + ): + return f"PyTorch {torch.__version__} is outside the Rust TreeCore support range" + allocator = params.token_to_kv_pool_allocator + device = ( + torch.device(allocator.device) if allocator is not None else torch.device("cpu") + ) + if device.type not in ("cpu", "cuda"): + return f"the Rust TreeCore does not support device {device.type}" + mode = envs.SGLANG_RUST_BUILD_MODE.get() + if sys.modules.get(_RUST_TREE_CORE_MODULE) is not None or mode not in ( + "auto", + "never", + "force", + ): + # Preserve the loader's invalid-mode and force-after-import errors. + return None + workspace = _RUST_TREE_CORE_MANIFEST.parent.parent + bundled = importlib.util.find_spec(_RUST_TREE_CORE_MODULE) is not None + source_checkout = (workspace / "Cargo.toml").is_file() + if mode != "force" and (mode == "never" or not source_checkout) and bundled: + return None + if not _RUST_TREE_CORE_MANIFEST.is_file(): + if bundled: + return "the requested Rust TreeCore source build has no sources" + return "this installation contains neither the Rust TreeCore extension nor its sources" + # Even a cached source build needs these version queries for its fingerprint. + # Match the loader's commands and cwd so rustup workspace overrides apply. + for command in (("cargo", "--version", "--verbose"), ("rustc", "-vV")): + try: + subprocess.run( + command, + cwd=workspace, + check=True, + capture_output=True, + text=True, + timeout=10, + ) + except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired): + return f"the Rust TreeCore toolchain is unavailable ({' '.join(command)})" + return None + + +def resolve_tree_core_backend(name: str, params: CacheInitParams) -> str: + """Resolve known Rust capability gaps before loading a backend. + + Explicit Rust selections use the same compatibility policy as the default. + Build, import, and runtime failures remain errors when the toolchain is usable. + """ + if name != "rust": + return name + reason = _rust_fallback_reason(params) + if reason is not None: + logger.info("Using the Python TreeCore: %s", reason) + return "python" + return name + + +def select_tree_core_backend(params: CacheInitParams) -> str: + """Resolve the instance override or default through the shared fallback policy.""" + name = ( + params.tree_core_backend + if params.tree_core_backend is not None + else envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.get() + ) + return resolve_tree_core_backend(name, params) def register_tree_core_backend(name: str, factory: TreeCoreFactory) -> None: @@ -72,7 +177,8 @@ def create_tree_core( params: CacheInitParams, components: dict[ComponentType, TreeComponent], ) -> UnifiedTreeCoreInterface: - """Construct the TreeCore registered under `name`.""" + """Resolve compatibility and construct the registered TreeCore.""" + name = resolve_tree_core_backend(name, params) factory = get_tree_core_factory(name) if factory is None: raise ValueError( diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py index 4027b80354..70482bf879 100644 --- a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py +++ b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py @@ -60,6 +60,7 @@ from sglang.srt.mem_cache.unified_cache.components import ( ComponentData, ComponentType, EvictLayer, + InternalStateBackup, LinkerTransferPhase, LRURefreshPhase, TreeComponent, @@ -474,6 +475,13 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): def reset(self) -> None: """Rebuild the root, LRUs, sizes, evictable-leaf sets, and the empty match result.""" + # End suspended walks while their session-cursor sentinels still + # belong to the old LRUs, clearing any pending internal victim. + for component in self.components: + if component.is_evict_device_ongoing: + component.evict_device_end() + # Internal victims awaiting the controller's backup and finish call. + self._pending_internal_evictions: dict[ComponentType, NodeId] = {} # Maintains the NodeId -> active tree node mapping. self._node_arena: dict[NodeId, UnifiedTreeNode] = {} @@ -1586,12 +1594,15 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): result = EvictDeviceNextNodeResult() # The walk reads running totals for its doneness check; the result # carries only this step's delta. + component = self.components_by_type[component_type] + assert component_type not in self._pending_internal_evictions, ( + f"finish the pending internal {component_type.name} eviction " + "before advancing" + ) updated_tracker = defaultdict(int, tracker) self._begin_tracking_unbacked_tokens() try: - result.node_id = self.components_by_type[ - component_type - ].evict_device_next_node( + step = component.evict_device_next_node( updated_tracker, result.device_frees, result.host_frees ) finally: @@ -1600,12 +1611,99 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): delta = n - tracker.get(ct, 0) if delta: result.tracker[ct] = delta - result.made_progress = result.node_id is not None or bool(result.tracker) + if isinstance(step, InternalStateBackup): + assert component_type in (ComponentType.MAMBA, ComponentType.SWA) + self._pending_internal_evictions[component_type] = step.node_id + if component_type == ComponentType.MAMBA: + result.mamba_backup_node_id = step.node_id + else: + result.swa_backup_node_id = step.node_id + result.swa_backup_num_tokens = step.num_tokens + else: + result.node_id = step + result.made_progress = ( + result.node_id is not None + or result.mamba_backup_node_id is not None + or result.swa_backup_node_id is not None + or bool(result.tracker) + ) + return result + + def finish_mamba_state_eviction(self, node_id: NodeId) -> EvictDeviceNextNodeResult: + return self._finish_internal_component_eviction(ComponentType.MAMBA, node_id) + + def finish_swa_state_eviction(self, node_id: NodeId) -> EvictDeviceNextNodeResult: + return self._finish_internal_component_eviction(ComponentType.SWA, node_id) + + def _finish_internal_component_eviction( + self, component_type: ComponentType, node_id: NodeId + ) -> EvictDeviceNextNodeResult: + """Resume an internal tombstone after the controller's backup attempt.""" + component = self.components_by_type[component_type] + assert component.is_evict_device_ongoing, ( + f"{component_type.name} device eviction not started" + ) + assert self._pending_internal_evictions.get(component_type) == node_id, ( + f"no matching pending internal {component_type.name} eviction" + ) + del self._pending_internal_evictions[component_type] + # Consuming the pending request advances the walk even if I/O changed + # this victim's eligibility. No frees occur before backup completion. + result = EvictDeviceNextNodeResult(made_progress=True) + node = self._node_arena.get(node_id) + lru = self.lru_lists[component_type] + enabled = self.enable_session_radix_cache + if ( + node is None + or node.component_data[component_type].value is None + or node.component_data[component_type].lock_ref > 0 + or ( + component_type == ComponentType.MAMBA + and node.load_back_pending_id is not None + ) + or not lru.in_list(node) + ): + if enabled: + component._evict_device_cursor = lru.cursor_next() + return result + if ( + node in self.evictable_device_leaves + and (not enabled or component._can_evict_leaf_atomically(node)) + ) or ( + component_type == ComponentType.MAMBA + and node.component_data[BASE_COMPONENT_TYPE].value is None + ): + # Re-select changed leaves through the normal walk; internal + # finish must not accidentally perform an atomic Full eviction. + component._evict_device_cursor = node + return result + self._begin_tracking_unbacked_tokens() + try: + self._evict_component_and_detach_lru( + node, + component, + target=EvictLayer.DEVICE, + tracker=result.tracker, + device_frees=result.device_frees, + host_frees=result.host_frees, + ) + self._cascade_evict( + node, + component, + result.tracker, + device_frees=result.device_frees, + host_frees=result.host_frees, + ) + finally: + result.unbacked_tokens = self._finish_tracking_unbacked_tokens() + if enabled: + component._evict_device_cursor = lru.cursor_next() return result def evict_device_end(self, component_type: ComponentType) -> None: """Finish a component's device-eviction walk.""" self.components_by_type[component_type].evict_device_end() + self._pending_internal_evictions.pop(component_type, None) def evict_device_leaf( self, node_id: NodeId, is_write_back: bool diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py index 4987b100f3..4698fc105e 100644 --- a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py +++ b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py @@ -45,12 +45,18 @@ class EvictDeviceNextNodeResult(BaseEvictionResult): ``node_id`` selects a leaf for the Controller to evict. ``made_progress`` also covers an internal tombstone that returned no leaf, distinguishing it - from true walk exhaustion. + from true walk exhaustion. ``mamba_backup_node_id`` and + ``swa_backup_node_id`` pause an internal eviction until the Controller + finishes its best-effort host backup. ``swa_backup_num_tokens`` includes + all unbacked SWA segments in the backup window, not just the victim. """ node_id: Optional[NodeId] = None made_progress: bool = False unbacked_tokens: int = 0 + mamba_backup_node_id: Optional[NodeId] = None + swa_backup_node_id: Optional[NodeId] = None + swa_backup_num_tokens: int = 0 class EvictDeviceLeafResult(BaseEvictionResult): @@ -319,6 +325,24 @@ class UnifiedTreeCoreInterface(ABC): D->H backup must run (write_back) before the node can be demoted.""" ... + @abstractmethod + def finish_mamba_state_eviction(self, node_id: NodeId) -> EvictDeviceNextNodeResult: + """Resume the pending internal Mamba eviction after its backup attempt. + + The controller must finish any submitted D->H transfer before calling. + A failed allocation still permits eviction to make device space. + """ + ... + + @abstractmethod + def finish_swa_state_eviction(self, node_id: NodeId) -> EvictDeviceNextNodeResult: + """Resume the pending internal SWA eviction after its backup attempt. + + The controller must finish any submitted D->H transfer before calling. + A failed allocation still permits eviction to make device space. + """ + ... + @abstractmethod def drop_subtree_no_host(self, node_id: NodeId) -> DropSubtreeNoHostResult: """Drop an unbacked D-leaf's subtree when its write-back backup failed @@ -642,6 +666,29 @@ class UnifiedTreeCoreInterface(ABC): host store event for each acked node.""" ... + @abstractmethod + def swa_tombstone_ranges( + self, key: RadixKey, start: int, end: int + ) -> list[tuple[int, int]]: + """Return maximal missing SWA ranges within the matched [start, end) span.""" + ... + + @abstractmethod + def attach_swa_window( + self, + key: RadixKey, + window_start: int, + window_end: int, + swa_values: torch.Tensor, + ) -> list[CacheAction | ComponentAction]: + """Attach a loaded SWA window to tombstoned spans, returning split actions. + + The shared pipeline supplies page-aligned logical token offsets and + int64 values on the core's device. The entire span must be matched and + tombstoned before publication. + """ + ... + @abstractmethod def set_component_device_value( self, node_id: NodeId, component_type: ComponentType, value: torch.Tensor diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index f9c0fcc04f..421a215722 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -74,7 +74,10 @@ from sglang.srt.mem_cache.unified_cache.session_ref_tracker import ( UnifiedSessionRefTracker, ) from sglang.srt.mem_cache.unified_cache.storage_attachment import StorageAttachment -from sglang.srt.mem_cache.unified_cache.tree_core_registry import create_tree_core +from sglang.srt.mem_cache.unified_cache.tree_core_registry import ( + create_tree_core, + select_tree_core_backend, +) from sglang.srt.mem_cache.unified_cache.unified_cache_linker import ( UnifiedCacheLinker, UnifiedCacheLinkerWrapper, @@ -204,11 +207,7 @@ class UnifiedRadixCache(BasePrefixCache): ) # The TreeCore owns the tree member-var state (structure, LRUs, sizes, # evictable leaves) and drives the components' tree-level hooks. - self._tree_core_backend = ( - params.tree_core_backend - if params.tree_core_backend is not None - else envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.get() - ) + self._tree_core_backend = select_tree_core_backend(params) self.tree_core = create_tree_core( name=self._tree_core_backend, params=params, @@ -756,6 +755,35 @@ class UnifiedRadixCache(BasePrefixCache): ) -> tuple[Optional[NodeId], bool]: """Advance the eviction walk one node, consuming its step result.""" result = self.tree_core.evict_device_next_node(component_type, tracker) + if result.mamba_backup_node_id is not None: + assert component_type == ComponentType.MAMBA and result.node_id is None + assert ( + not result.device_frees and not result.host_frees and not result.tracker + ) + # Reserve a host state slot and wait for the backup acknowledgment + # before freeing device state. If allocation fails, eviction still + # proceeds to make room on the device. + node_id = result.mamba_backup_node_id + mamba_host_pool = self.host_pool_group.get_pool(PoolName.MAMBA) + if mamba_host_pool is not None and mamba_host_pool.available_size() < 1: + self.evict_host(1, ComponentType.MAMBA) + self.backup_node_for_write_back(node_id) + result = self.tree_core.finish_mamba_state_eviction(node_id) + elif result.swa_backup_node_id is not None: + assert component_type == ComponentType.SWA and result.node_id is None + assert ( + not result.device_frees and not result.host_frees and not result.tracker + ) + # The backup can cover several unbacked SWA segments. Reserve host + # space for the whole window before copying it, then resume eviction + # even if host allocation fails. + node_id = result.swa_backup_node_id + needed = result.swa_backup_num_tokens + swa_host_pool = self.host_pool_group.get_pool(PoolName.SWA) + if swa_host_pool is not None and swa_host_pool.available_size() < needed: + self.evict_host(needed, ComponentType.SWA) + self.backup_node_for_write_back(node_id) + result = self.tree_core.finish_swa_state_eviction(node_id) self._free_values(result.device_frees, result.host_frees) if self._tracks_write_through_unbacked_evictions(): self._record_dropped_tokens( @@ -887,8 +915,7 @@ class UnifiedRadixCache(BasePrefixCache): def _tracks_write_through_unbacked_evictions(self) -> bool: return ( - isinstance(self.tree_core, UnifiedTreeCore) - and self.host_memory_mode == "cache" + self.host_memory_mode == "cache" and self.cache_controller is not None and self.cache_controller.write_policy == "write_through" ) diff --git a/python/sglang/test/scripted_runtime/__init__.py b/python/sglang/test/scripted_runtime/__init__.py index e69de29bb2..b0f1bf9908 100644 --- a/python/sglang/test/scripted_runtime/__init__.py +++ b/python/sglang/test/scripted_runtime/__init__.py @@ -0,0 +1,11 @@ +from sglang.srt.environ import envs + +# Spawned server and scheduler processes import the scripted-runtime package +# before constructing their cache. Keep inspection in test processes while +# preserving the selected backend and its compatibility fallback. +if envs.SGLANG_TEST_SCRIPTED_RUNTIME.get(): + from sglang.test.scripted_runtime.tree_core_inspection import ( + install_tree_core_inspectors, + ) + + install_tree_core_inspectors() diff --git a/python/sglang/test/scripted_runtime/context/lock_ref_exhauster.py b/python/sglang/test/scripted_runtime/context/lock_ref_exhauster.py index dac4fbdba9..bb6508b876 100644 --- a/python/sglang/test/scripted_runtime/context/lock_ref_exhauster.py +++ b/python/sglang/test/scripted_runtime/context/lock_ref_exhauster.py @@ -1,8 +1,11 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, List +from typing import TYPE_CHECKING, Any -from sglang.test.scripted_runtime.context.radix import _node_lock_ref, to_node_handle +from sglang.test.scripted_runtime.context.radix import ( + get_node_lock_ref, + iter_node_handles, +) if TYPE_CHECKING: from sglang.srt.managers.scheduler import Scheduler @@ -12,9 +15,9 @@ class ScriptedLockRefExhauster: def __init__(self, scheduler: Scheduler) -> None: self.scheduler = scheduler # (node, dec receipt) pairs; the receipt bounds the release walk. - self._locked: List[tuple[Any, Any]] = [] + self._locked: list[tuple[Any, Any]] = [] - def exhaust(self, *, leave_refs: int) -> None: + def exhaust(self, leave_refs: int) -> None: tree_cache = self.scheduler.tree_cache if tree_cache.disable: return @@ -25,9 +28,11 @@ class ScriptedLockRefExhauster: return target = evictable[0] - result = tree_cache.inc_lock_ref(to_node_handle(tree_cache, target)) + result = tree_cache.inc_lock_ref(target) - newly_locked = [node for node in evictable if _node_lock_ref(node) > 0] + newly_locked = [ + node for node in evictable if get_node_lock_ref(tree_cache, node) > 0 + ] if not newly_locked: return self._locked.append((target, result.to_dec_params())) @@ -35,15 +40,13 @@ class ScriptedLockRefExhauster: def release(self) -> None: tree_cache = self.scheduler.tree_cache for node, dec_params in self._locked: - tree_cache.dec_lock_ref(to_node_handle(tree_cache, node), dec_params) + tree_cache.dec_lock_ref(node, dec_params) self._locked.clear() - def _evictable_nodes(self) -> List[Any]: - evictable: List[Any] = [] - stack = list(self.scheduler.tree_cache.root_node.children.values()) - while stack: - node = stack.pop() - if _node_lock_ref(node) == 0: - evictable.append(node) - stack.extend(node.children.values()) - return evictable + def _evictable_nodes(self) -> list[Any]: + tree_cache = self.scheduler.tree_cache + return [ + node + for node in iter_node_handles(tree_cache) + if get_node_lock_ref(tree_cache, node) == 0 + ] diff --git a/python/sglang/test/scripted_runtime/context/radix.py b/python/sglang/test/scripted_runtime/context/radix.py index 91403b2c0d..0bee706077 100644 --- a/python/sglang/test/scripted_runtime/context/radix.py +++ b/python/sglang/test/scripted_runtime/context/radix.py @@ -1,6 +1,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Callable, Dict +from collections.abc import Iterator +from typing import TYPE_CHECKING, Any from sglang.srt.mem_cache.unified_radix_cache import UnifiedTreeNode @@ -8,12 +9,24 @@ if TYPE_CHECKING: from sglang.test.scripted_runtime.context.api import ScriptedContext -def get_all_node_hit_counts(ctx: ScriptedContext) -> Dict[int, int]: - return _collect_node_attr(ctx, lambda node: node.hit_count) +def get_all_node_hit_counts(ctx: ScriptedContext) -> dict[int, int]: + tree_cache = ctx.scheduler.tree_cache + core = getattr(tree_cache, "tree_core", None) + return { + node if core is not None else node.id: ( + core.get_node_hit_count(node) if core is not None else node.hit_count + ) + for node in iter_node_handles(tree_cache) + } -def get_all_node_lock_refs(ctx: ScriptedContext) -> Dict[int, int]: - return _collect_node_attr(ctx, _node_lock_ref) +def get_all_node_lock_refs(ctx: ScriptedContext) -> dict[int, int]: + tree_cache = ctx.scheduler.tree_cache + core = getattr(tree_cache, "tree_core", None) + return { + node if core is not None else node.id: get_node_lock_ref(tree_cache, node) + for node in iter_node_handles(tree_cache) + } def _node_lock_ref(node: Any) -> int: @@ -22,37 +35,34 @@ def _node_lock_ref(node: Any) -> int: return node.lock_ref -def resolve_node(tree_cache: Any, node_handle: Any) -> Any: - """Resolve whatever a req or match result carries into a tree node. - - ``UnifiedRadixCache`` hands out NodeIds (ints); every other cache hands out - the node object itself. Returns None when the handle no longer maps to a - live node, which only happens once the node has been freed -- and a freed - node holds no locks. - """ - resolve = getattr(tree_cache, "resolve_node_handle", None) - if resolve is None: - return node_handle - try: - return resolve(node_handle) - except KeyError: - return None +def get_node_lock_ref(tree_cache: Any, node_handle: Any) -> int: + """Read locks through the cache's native handle, including stale handles.""" + if node_handle is None: + return 0 + core = getattr(tree_cache, "tree_core", None) + if core is not None: + if not core.contains_node(node_handle): + return 0 + return sum( + core.get_component_device_lock_ref(node_handle, component) + for component in tree_cache.tree_components + ) + return _node_lock_ref(node_handle) -def to_node_handle(tree_cache: Any, node: Any) -> Any: - """Inverse of `resolve_node`: what the tree_cache lock APIs accept.""" - if isinstance(node, UnifiedTreeNode): - return node.id - return node - - -def _collect_node_attr( - ctx: ScriptedContext, get_value: Callable[[Any], int] -) -> Dict[int, int]: - values: Dict[int, int] = {} - stack = list(ctx.scheduler.tree_cache.root_node.children.values()) +def iter_node_handles(tree_cache: Any) -> Iterator[Any]: + """Walk non-root nodes as IDs for unified caches or native legacy nodes.""" + core = getattr(tree_cache, "tree_core", None) + stack = ( + list(core.get_child_node_ids(core.root_node_handle())) + if core is not None + else list(tree_cache.root_node.children.values()) + ) while stack: node = stack.pop() - values[node.id] = get_value(node) - stack.extend(node.children.values()) - return values + yield node + stack.extend( + core.get_child_node_ids(node) + if core is not None + else node.children.values() + ) diff --git a/python/sglang/test/scripted_runtime/req_handle.py b/python/sglang/test/scripted_runtime/req_handle.py index 339f8a5ff2..ae44e83d20 100644 --- a/python/sglang/test/scripted_runtime/req_handle.py +++ b/python/sglang/test/scripted_runtime/req_handle.py @@ -1,9 +1,9 @@ from __future__ import annotations from dataclasses import dataclass -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING -from sglang.test.scripted_runtime.context.radix import _node_lock_ref, resolve_node +from sglang.test.scripted_runtime.context.radix import get_node_lock_ref if TYPE_CHECKING: from sglang.srt.managers.schedule_batch import Req @@ -16,7 +16,7 @@ class ScriptedReqHandle: context: ScriptedContext @property - def req(self) -> Optional[Req]: + def req(self) -> Req | None: return self.context.find_req_by_rid(self.rid) @property @@ -52,7 +52,4 @@ class ScriptedReqHandle: req = self.req if req is None: return 0 - node = resolve_node(self.context.scheduler.tree_cache, req.last_node) - if node is None: - return 0 - return _node_lock_ref(node) + return get_node_lock_ref(self.context.scheduler.tree_cache, req.last_node) diff --git a/python/sglang/test/scripted_runtime/tree_core_inspection.py b/python/sglang/test/scripted_runtime/tree_core_inspection.py new file mode 100644 index 0000000000..6a27cd0923 --- /dev/null +++ b/python/sglang/test/scripted_runtime/tree_core_inspection.py @@ -0,0 +1,34 @@ +"""Use the shared TreeCore inspectors in scripted-runtime test processes.""" + +import sys +from pathlib import Path + + +def install_tree_core_inspectors(): + from sglang.srt.mem_cache.unified_cache.tree_core_registry import ( + _TREE_CORE_REGISTRY, + ) + + inspector_dir = str( + Path(__file__).resolve().parents[4] + / "test" + / "registered" + / "unit" + / "mem_cache" + ) + if inspector_dir not in sys.path: + sys.path.insert(0, inspector_dir) + + def python_inspector_factory(params, components): + from unified_tree_core_inspector import UnifiedTreeCoreInspector + + return UnifiedTreeCoreInspector(params, components) + + def rust_inspector_factory(params, _components): + from rust_unified_tree_core_inspector import RustUnifiedTreeCoreInspector + + return RustUnifiedTreeCoreInspector(params) + + _TREE_CORE_REGISTRY.update( + python=python_inspector_factory, rust=rust_inspector_factory + ) diff --git a/rust/sglang-radix-tree/README.md b/rust/sglang-radix-tree/README.md index 2a5ec00be6..82f209fa2d 100644 --- a/rust/sglang-radix-tree/README.md +++ b/rust/sglang-radix-tree/README.md @@ -4,20 +4,57 @@ Rust tree core for the Unified Radix Cache, covering Full attention, sliding win ## Usage -Select the backend with: +Rust is the default tree core. The centralized tree-core registry falls back to +Python in these cases: + +- Session-aware caching. +- C128 or other unsupported components. +- Custom component overrides. +- Non-Linux platforms. +- PyTorch versions outside 2.11 through 2.13. +- Devices other than CPU or CUDA. +- Installations containing neither the Rust extension nor its sources. +- Source builds with a missing or unusable Rust toolchain. + +This policy also applies when Rust is explicitly selected. +Build, import, and runtime failures in supported configurations remain errors. +The Rust bindings report unexpected native panics as `RuntimeError` so Python's +crash handlers can report them and coordinate shutdown. A panic during a core +operation poisons its mutex, and subsequent calls refuse to reuse that state. + +T-LRU accepts integer and floating-point `threshold` and `next_prompt_estimate` +parameters. In mixed integer/float configurations, the integer estimate must fit +in i128; larger values raise `OverflowError` at initialization. Native integer +addition is checked, and overflow is reported as `RuntimeError`. + +Select a backend explicitly with: ```bash SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND=rust +# Use the Python implementation instead: +SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND=python ``` -SGLang wheels bundle the production extension. A source checkout falls back to +Standard SGLang wheels bundle the production extension; some platform +distributions omit it. A source checkout falls back to the shared fingerprinted Rust-extension cache; it never writes a shared object into the Python package. LibTorch and the Python headers come from the running interpreter's PyTorch install. PyTorch 2.11 through 2.13 are accepted explicitly, and `torch_2_13_compat.h` covers two alignment APIs removed in PyTorch 2.13. +Source checkouts need working `cargo` and `rustc` to use Rust, including for +fingerprinted cache lookup. If either tool is unavailable, the registry selects +Python. Trusted bundled extensions do not require a Rust compiler. ## Development +Alternative backends implement `UnifiedTreeCoreInterface` and register through +`register_tree_core_backend`. For built-in Mamba and SWA internal-state write-back, +the core returns a backup request and the controller performs transfers and waits +for acknowledgment. +The controller then calls `finish_mamba_state_eviction` or +`finish_swa_state_eviction` to resume eviction. This keeps I/O outside the Rust +tree's mutex; both tree cores implement the same contract. + ```bash # Build (libtorch from the installed torch package): cd rust/sglang-radix-tree diff --git a/rust/sglang-radix-tree/src/components/mamba.rs b/rust/sglang-radix-tree/src/components/mamba.rs index 7d3e6cb50d..9dae4cfa52 100644 --- a/rust/sglang-radix-tree/src/components/mamba.rs +++ b/rust/sglang-radix-tree/src/components/mamba.rs @@ -7,7 +7,7 @@ use std::collections::HashMap; use tch::Tensor; use crate::components::TreeComponent; -use crate::components::{ComponentType, MAMBA}; +use crate::components::{ComponentType, FULL, MAMBA}; use crate::node::ChildKeyType; use crate::node::Node; use crate::node::{NodeId, NodeIdx_, TreeCoreRuntimeError, ValueSlotIdx}; @@ -350,6 +350,13 @@ impl TreeComponent for MambaComponent { tree_core.component_state(MAMBA).is_evict_device_ongoing, "Mamba device eviction not started" ); + assert!( + tree_core + .component_state(MAMBA) + .evict_device_pending_node + .is_none(), + "finish the pending internal Mamba eviction before advancing" + ); let mut cursor = tree_core.component_state(MAMBA).evict_device_cursor; // The cursor is re-validated (reset to LRU head) if the previous // node's eviction removed it. @@ -358,15 +365,15 @@ impl TreeComponent for MambaComponent { .device_lru_list(MAMBA) .get_lru_no_lock(&tree_core.arena); } - let next = loop { + let next = 'step: { if tracker[&ct] >= tree_core.component_state(MAMBA).evict_device_request_cnt { - break None; + break 'step None; } let Some(x) = cursor else { - break None; + break 'step None; }; if !tree_core.device_lru_list(MAMBA).in_list(Some(x)) { - break None; + break 'step None; } assert!( tree_core.arena.has_device_value(x, MAMBA), @@ -375,14 +382,28 @@ impl TreeComponent for MambaComponent { cursor = tree_core .device_lru_list(MAMBA) .get_prev_no_lock(x, &tree_core.arena); - // A load-back pin means an in-flight DMA targets this node's slices. - if tree_core.arena.node(x).is_load_back_pending() { - continue; - } + // The Mamba LRU filters its own transfer/request locks. Full load + // pins and expanded SWA backups can cover unrelated Mamba state + // restored by a later request, which remains independently evictable. if tree_core.evictable_device_leaves.contains(x) { - break Some(x); + break 'step Some(x); } - // Internal nodes are tombstoned inline (no IO). + let node = tree_core.arena.node(x); + if tree_core.enable_hicache + && tree_core.is_write_back + && !node.has_host_value(MAMBA) + && !node.backuped() + && node.has_device_value(FULL) + { + // Keep the state live until the controller finishes D->H I/O. + // Native code never calls Python while holding the tree lock. + let node_id = node.id; + tree_core + .component_state_mut(MAMBA) + .evict_device_pending_node = Some(node_id); + break 'step None; + } + // Other policies and already-backed states need no I/O. tree_core.evict_component_and_detach_lru_( x, ct, @@ -392,7 +413,7 @@ impl TreeComponent for MambaComponent { Some(tracker), ); tree_core.cascade_evict_(x, ct, tracker, device_frees, host_frees, EvictLayer::Device); - break None; + None }; tree_core.component_state_mut(MAMBA).evict_device_cursor = cursor; next @@ -698,11 +719,8 @@ impl TreeComponent for MambaComponent { let x_next = tree_core .host_lru_list(MAMBA) .get_prev_no_lock(cur, &tree_core.arena); - // A load-back pin means an in-flight DMA reads this node's host slices. - if tree_core.arena.node(cur).is_load_back_pending() { - x = x_next; - continue; - } + // The host LRU filters Mamba's own locks. A Full load-back pin may + // cover an ancestor whose Mamba state is not being transferred. if tree_core.evictable_host_leaves.contains(cur) { // Host leaf: atomic eviction (all components host + delete) tree_core.evict_host_leaf_(cur, tracker, device_frees, host_frees); diff --git a/rust/sglang-radix-tree/src/components/swa.rs b/rust/sglang-radix-tree/src/components/swa.rs index acd7c19d91..cd840002d8 100644 --- a/rust/sglang-radix-tree/src/components/swa.rs +++ b/rust/sglang-radix-tree/src/components/swa.rs @@ -22,6 +22,8 @@ use crate::unified_tree_core::{ pub struct SwaComponent { /// Sliding window size in tokens. sliding_window_size: usize, + /// Per-request rings are rebuilt by the request and do not gate tree reuse. + swa_req_ring: bool, } impl SwaComponent { @@ -43,6 +45,7 @@ impl SwaComponent { ); SwaComponent { sliding_window_size, + swa_req_ring: params.swa_req_ring, } } @@ -205,18 +208,6 @@ impl SwaComponent { } unbacked } - - fn next_host_unlocked_device_lru_node( - tree_core: &UnifiedTreeCore, - from: Option, - ) -> Option { - let lru = tree_core.device_lru_list(SWA); - let unlocked = |id: NodeIdx_| tree_core.arena.node(id).host_lock_ref(SWA) == 0; - match from { - Some(node_id) => lru.get_prev_where(node_id, unlocked), - None => lru.get_lru_where(unlocked), - } - } } impl SwaComponent { @@ -392,13 +383,13 @@ impl TreeComponent for SwaComponent { fn create_match_validator( &self, - tree_core: &UnifiedTreeCore, + _tree_core: &UnifiedTreeCore, match_device_only: bool, ) -> Box, NodeIdx_) -> bool> { let sliding_window_size = self.sliding_window_size; - // unified_kv never caches the SWA ring (per-request, not - // content-stable), so SWA bookkeeping must not gate the match here. - let swa_device_only_hicache = !tree_core.has_swa_host_pool && tree_core.enable_hicache; + // A per-request SWA ring is not stored in tree nodes. Its layout, not + // the presence of a host tier, determines whether tombstones gate reuse. + let swa_req_ring = self.swa_req_ring; let mut contiguous_len = usize::MAX; Box::new(move |tree_core: &UnifiedTreeCore, node_id: NodeIdx_| { let node = tree_core.arena.node(node_id); @@ -406,7 +397,7 @@ impl TreeComponent for SwaComponent { // — load_back will restore SWA from host before use. if !node.has_device_value(SWA) && (match_device_only || !node.has_host_value(SWA)) { contiguous_len = 0; - return swa_device_only_hicache && (node.backuped() || !node.evicted()); + return swa_req_ring && (node.backuped() || !node.evicted()); } contiguous_len = contiguous_len.saturating_add(node.key.atom_len()); contiguous_len >= sliding_window_size @@ -771,6 +762,13 @@ impl TreeComponent for SwaComponent { tree_core.component_state(SWA).is_evict_device_ongoing, "Swa device eviction not started" ); + assert!( + tree_core + .component_state(SWA) + .evict_device_pending_node + .is_none(), + "finish the pending internal SWA eviction before advancing" + ); let mut cursor = tree_core.component_state(SWA).evict_device_cursor; // The cursor is re-validated (reset to LRU head) if the previous // node's eviction removed it. @@ -779,15 +777,15 @@ impl TreeComponent for SwaComponent { .device_lru_list(SWA) .get_lru_no_lock(&tree_core.arena); } - let next = loop { + let next = 'step: { if tracker[&ct] >= tree_core.component_state(SWA).evict_device_request_cnt { - break None; + break 'step None; } let Some(x) = cursor else { - break None; + break 'step None; }; if !tree_core.device_lru_list(SWA).in_list(Some(x)) { - break None; + break 'step None; } assert!( tree_core.arena.has_device_value(x, SWA), @@ -796,14 +794,35 @@ impl TreeComponent for SwaComponent { cursor = tree_core .device_lru_list(SWA) .get_prev_no_lock(x, &tree_core.arena); - // A load-back pin means an in-flight DMA targets this node's slices. - if tree_core.arena.node(x).is_load_back_pending() { - continue; - } + // The SWA LRU filters its own transfer/request locks. A Full load + // pin can also cover an ancestor outside the loaded SWA window, + // whose separately restored SWA rows remain independently evictable. if tree_core.evictable_device_leaves.contains(x) { - break Some(x); + break 'step Some(x); + } + let node = tree_core.arena.node(x); + if tree_core.enable_hicache + && tree_core.is_write_back + && tree_core.has_swa_host_pool + && !node.has_host_value(SWA) + && !node.has_host_value(FULL) + && node.has_device_value(FULL) + { + // Backup may include several unbacked ancestors. Let the + // controller reserve that whole window outside the core lock. + let needed = self + .collect_unbacked_swa_nodes_(tree_core, x) + .iter() + .map(|&idx| tree_core.arena.device_value(idx, SWA).numel()) + .sum(); + if needed > 0 { + let node_id = node.id; + let state = tree_core.component_state_mut(SWA); + state.evict_device_pending_node = Some(node_id); + state.evict_device_pending_num_tokens = needed; + break 'step None; + } } - // Internal nodes are tombstoned inline (no IO). tree_core.evict_component_and_detach_lru_( x, ct, @@ -813,7 +832,7 @@ impl TreeComponent for SwaComponent { Some(tracker), ); tree_core.cascade_evict_(x, ct, tracker, device_frees, host_frees, EvictLayer::Device); - break None; + None }; tree_core.component_state_mut(SWA).evict_device_cursor = cursor; next @@ -823,41 +842,6 @@ impl TreeComponent for SwaComponent { tree_core.set_evict_device_end(SWA); } - fn reclaim_coexisting_host_values( - &self, - tree_core: &mut UnifiedTreeCore, - num_tokens: usize, - tracker: &mut HashMap, - device_frees: &mut HashMap>, - host_frees: &mut HashMap>, - ) { - for spare_imminent_demotes in [true, false] { - if tracker[&SWA] >= num_tokens { - break; - } - let mut next = Self::next_host_unlocked_device_lru_node(tree_core, None); - while let Some(node_id) = next { - if tracker[&SWA] >= num_tokens { - break; - } - next = Self::next_host_unlocked_device_lru_node(tree_core, Some(node_id)); - if spare_imminent_demotes && tree_core.evictable_device_leaves.contains(node_id) { - continue; - } - if !tree_core.can_reclaim_coexisting_host_value_(node_id, SWA) { - continue; - } - tree_core.release_coexisting_host_value_( - node_id, - SWA, - tracker, - device_frees, - host_frees, - ); - } - } - } - /// Evict SWA host resources. /// Internal nodes: private tombstone (free SWA host only). /// Host leaves: atomic eviction via _evict_host_leaf. @@ -886,11 +870,8 @@ impl TreeComponent for SwaComponent { let x_next = tree_core .host_lru_list(SWA) .get_prev_no_lock(cur, &tree_core.arena); - // A load-back pin means an in-flight DMA reads this node's host slices. - if tree_core.arena.node(cur).is_load_back_pending() { - x = x_next; - continue; - } + // The host LRU filters SWA's own locks. A Full load-back pin may + // cover an ancestor outside the SWA transfer window. if tree_core.evictable_host_leaves.contains(cur) { tree_core.evict_host_leaf_(cur, tracker, device_frees, host_frees); } else { diff --git a/rust/sglang-radix-tree/src/node.rs b/rust/sglang-radix-tree/src/node.rs index c157d326f5..cc53e62276 100644 --- a/rust/sglang-radix-tree/src/node.rs +++ b/rust/sglang-radix-tree/src/node.rs @@ -177,9 +177,12 @@ pub struct Node { pub hash_value: Option>, /// Whether this node is available through the direct external-cache linker. pub external_cache_stored: bool, + /// Chain-wide logical-page owner rotation, retained across node splits. + pub rotation_base: Option, /// The in-flight write-through backup's ack id. pub write_through_pending_id: Option, - /// Load-back anchor currently reading this node's host slots. + /// Load-back anchor currently reading this node's Full host slots. + /// Auxiliary transfers are protected by their component locks. pub load_back_pending_id: Option, /// Monotonic access tick for LRU ordering (exact; not wall-clock). pub last_access_counter: i64, @@ -187,6 +190,10 @@ pub struct Node { pub creation_counter: i64, /// Match hits accumulated for write-through and LFU decisions. pub hit_count: i64, + /// Logical root-path length; maintained only for T-LRU. + pub tlru_cached_prefix_len: usize, + /// Greatest subtree depth reached, retained when cached tails are evicted. + pub tlru_history_len: usize, /// Eviction priority; the root uses `i64::MIN` and is never a leaf. pub priority: i64, /// This node's external handle; minted once, never recycled. @@ -380,7 +387,7 @@ impl Node { .any(|state| state.lock_ref > 0) } - /// Whether an in-flight load-back currently pins this node. + /// Whether an in-flight load-back currently pins this node's Full slots. pub fn is_load_back_pending(&self) -> bool { self.load_back_pending_id.is_some() } @@ -399,11 +406,14 @@ impl Node { swa_host_uuid: None, hash_value: Some(Vec::new()), external_cache_stored: false, + rotation_base: None, write_through_pending_id: None, load_back_pending_id: None, last_access_counter: 0, creation_counter: 0, hit_count: 0, + tlru_cached_prefix_len: 0, + tlru_history_len: 0, priority: i64::MIN, id, idx: NodeIdx_(id), @@ -422,11 +432,14 @@ impl Node { swa_host_uuid: None, hash_value: None, external_cache_stored: false, + rotation_base: None, write_through_pending_id: None, load_back_pending_id: None, last_access_counter: 0, creation_counter: 0, hit_count: 0, + tlru_cached_prefix_len: 0, + tlru_history_len: 0, priority, id, idx: NodeIdx_(id), @@ -1557,68 +1570,135 @@ impl NodeArena { } } -// Eviction-eligible node set. +// Shared node membership for eviction leaves and Full host duplicates. -/// Set of `NodeIdx_`s with O(1) membership ops and dense iteration. -#[derive(Default)] -pub struct EvictableNodeSet { - /// Dense member list; order is unspecified (swap-remove). - nodes: Vec, - /// Each member's position in `nodes`, indexed by `NodeIdx_`. - slots: Vec>, +/// An insertion-ordered set of arena slots. Full host duplicate reclamation +/// consumes this order directly; leaf eviction ranks members in a policy heap. +/// Links are indexed by arena slot: membership/removal are O(1), insertion is +/// amortized O(1), and iteration visits only current members. +pub struct NodeSet { + /// Per arena slot: `Some((previous, next))` in insertion order, or `None` if absent. + /// `END` marks a missing neighbor at either end. + links: Vec>, + /// First member in insertion order, or `END` when empty. + head: NodeIdx_, + /// Last member in insertion order; new members append here. `END` when empty. + tail: NodeIdx_, + /// Number of current members, excluding absent slots in `links`. + len: usize, } -impl EvictableNodeSet { +impl Default for NodeSet { + fn default() -> Self { + Self::new() + } +} + +impl NodeSet { + const END: NodeIdx_ = NodeIdx_(usize::MAX); + pub fn new() -> Self { - Default::default() - } - - /// Whether `node_id` is a member. - pub fn contains(&self, node_id: NodeIdx_) -> bool { - self.slots.get(node_id.0).copied().flatten().is_some() - } - - /// Insert `node_id`; no-op when already a member. - pub fn add(&mut self, node_id: NodeIdx_) { - if node_id.0 >= self.slots.len() { - self.slots.resize(node_id.0 + 1, None); + Self { + links: Vec::new(), + head: Self::END, + tail: Self::END, + len: 0, } - if self.slots[node_id.0].is_some() { + } + + pub fn contains(&self, node_id: NodeIdx_) -> bool { + self.links.get(node_id.0).is_some_and(Option::is_some) + } + + /// Existing membership keeps its position; a removed/reinserted slot is + /// appended, matching assignment to Python's insertion-ordered dict. + pub fn add(&mut self, node_id: NodeIdx_) { + assert_ne!( + node_id, + Self::END, + "node slot is reserved for the set sentinel" + ); + if node_id.0 >= self.links.len() { + self.links.resize(node_id.0 + 1, None); + } + if self.links[node_id.0].is_some() { return; } - self.slots[node_id.0] = Some(self.nodes.len()); - self.nodes.push(node_id); + self.links[node_id.0] = Some((self.tail, Self::END)); + if self.tail == Self::END { + self.head = node_id; + } else { + self.links[self.tail.0].as_mut().unwrap().1 = node_id; + } + self.tail = node_id; + self.len += 1; } - /// Remove `node_id`; no-op when not a member. pub fn discard(&mut self, node_id: NodeIdx_) { - let Some(slot) = self.slots.get(node_id.0).copied().flatten() else { + let Some((prev, next)) = self.links.get_mut(node_id.0).and_then(Option::take) else { return; }; - self.slots[node_id.0] = None; - self.nodes.swap_remove(slot); - // The swapped-in tail member (if any) now lives at `slot`. - if let Some(&moved) = self.nodes.get(slot) { - self.slots[moved.0] = Some(slot); + if prev == Self::END { + self.head = next; + } else { + self.links[prev.0].as_mut().unwrap().1 = next; } + if next == Self::END { + self.tail = prev; + } else { + self.links[next.0].as_mut().unwrap().0 = prev; + } + self.len -= 1; } - /// The members, in unspecified order. - pub fn iter(&self) -> impl Iterator + '_ { - self.nodes.iter().copied() + /// Visit members from `head` to `tail` by following each entry's `next` link. + pub fn iter(&self) -> impl ExactSizeIterator + '_ { + // Keep an exact size hint so eviction heaps can reserve once. + NodeSetIter { + links: &self.links, + current: self.head, + remaining: self.len, + } } // Test-only conveniences: production callers use add/discard/contains/iter. #[cfg(test)] pub fn len(&self) -> usize { - self.nodes.len() + self.len } #[cfg(test)] pub fn is_empty(&self) -> bool { - self.nodes.is_empty() + self.head == Self::END } } + +struct NodeSetIter<'a> { + links: &'a [Option<(NodeIdx_, NodeIdx_)>], + current: NodeIdx_, + remaining: usize, +} + +impl Iterator for NodeSetIter<'_> { + type Item = NodeIdx_; + + fn next(&mut self) -> Option { + if self.remaining == 0 { + return None; + } + let node_id = self.current; + self.current = self.links[node_id.0].unwrap().1; + self.remaining -= 1; + Some(node_id) + } + + fn size_hint(&self) -> (usize, Option) { + (self.remaining, Some(self.remaining)) + } +} + +impl ExactSizeIterator for NodeSetIter<'_> {} + #[cfg(test)] #[path = "tests/node.rs"] mod tests; diff --git a/rust/sglang-radix-tree/src/python_bindings.rs b/rust/sglang-radix-tree/src/python_bindings.rs index 3be75b21eb..3b9308569b 100644 --- a/rust/sglang-radix-tree/src/python_bindings.rs +++ b/rust/sglang-radix-tree/src/python_bindings.rs @@ -2,6 +2,7 @@ use std::borrow::Cow; use std::collections::HashMap; +use std::panic::{AssertUnwindSafe, catch_unwind}; use std::sync::Mutex; use pyo3::buffer::PyBuffer; @@ -13,6 +14,7 @@ use tch::{Device, Kind, Tensor}; use crate::components::{ComponentSet, ComponentType, FULL, MAMBA, SWA}; use crate::node::ChildKeyType; use crate::node::{KeyNamespaceRef, NodeAccessError, NodeId, TreeCoreRuntimeError}; +use crate::unified_lru_list::{TlruFloatConfig, TlruPromptEstimate}; use crate::unified_tree_core::KvCacheEvent; use crate::unified_tree_core::{ BufferBackupSnapshot, BufferBackupState, CacheAction, CacheInitParams, CacheTransferPhase, @@ -21,6 +23,27 @@ use crate::unified_tree_core::{ UnifiedTreeCore, }; +/// Translate only Rust panics; ordinary Python errors pass through unchanged. +/// Keep this boundary outside the whole binding call so its MutexGuard unwinds +/// and poisons the core before the panic is caught. A poisoned core stays unusable. +fn catch_native_panic(operation: impl FnOnce() -> PyResult) -> PyResult { + match catch_unwind(AssertUnwindSafe(operation)) { + Ok(result) => result, + Err(payload) => { + let message = if let Some(message) = payload.downcast_ref::() { + message.as_str() + } else if let Some(message) = payload.downcast_ref::<&str>() { + message + } else { + "unknown native panic" + }; + Err(PyRuntimeError::new_err(format!( + "Rust TreeCore panicked: {message}" + ))) + } + } +} + /// Parse a torch-style device string (e.g. "cpu", "cuda", "cuda:1"); a bare /// "cuda" means index 0, so callers must resolve the index themselves. fn parse_device(device: &str) -> PyResult { @@ -382,6 +405,9 @@ fn comp_xfers_to_py( comp_xfers: HashMap>, ) -> PyResult> { let dict = PyDict::new_bound(py); + // Pool allocation can evict across components, so preserve Python's order. + let mut comp_xfers: Vec<_> = comp_xfers.into_iter().collect(); + comp_xfers.sort_unstable_by_key(|(ct, _)| component_type_to_u8(*ct)); for (ct, transfers) in comp_xfers { let list = PyList::empty_bound(py); for transfer in transfers { @@ -428,17 +454,51 @@ fn frees_to_py(py: Python<'_>, frees: HashMap>) -> Py Ok(dict.unbind()) } +/// Floating-point T-LRU arithmetic with the estimate's numeric type preserved. +#[pyclass(name = "TlruFloatConfig")] +#[derive(Clone)] +pub struct TlruFloatConfigBinding { + config: TlruFloatConfig, +} + +#[pymethods] +impl TlruFloatConfigBinding { + #[new] + #[pyo3(signature = (threshold, next_prompt_estimate, integer_estimate = None))] + fn new(threshold: f64, next_prompt_estimate: f64, integer_estimate: Option) -> Self { + let estimate = match integer_estimate { + Some(value) => TlruPromptEstimate::Integer(value), + None => TlruPromptEstimate::Float(next_prompt_estimate), + }; + Self { + config: TlruFloatConfig { + threshold, + next_prompt_estimate: estimate, + }, + } + } + + #[cfg(feature = "inspection")] + fn inspect_is_tel_safe(&self, history: usize, cached_without_node: usize) -> PyResult { + catch_native_panic(|| Ok(self.config.is_tel_safe(history, cached_without_node))) + } +} + /// Python-visible tree-core init params; converts into CacheInitParams. #[pyclass(get_all, set_all)] #[derive(Clone)] pub struct TreeCoreInitParamsBinding { pub eviction_policy: String, + pub slru_protected_threshold: i64, + pub tlru_tail_budget: usize, + pub tlru_float_config: Option, pub page_size: usize, pub is_write_back: bool, pub enable_hicache: bool, pub write_through_threshold: i64, pub device: String, pub swa_sliding_window_size: Option, + pub swa_req_ring: bool, pub enable_kv_cache_events: bool, pub mamba_cache_chunk_size: Option, pub mamba_max_states_per_path: Option, @@ -449,12 +509,16 @@ impl TreeCoreInitParamsBinding { fn to_cache_init_params(&self) -> PyResult { Ok(CacheInitParams { eviction_policy: self.eviction_policy.clone(), + slru_protected_threshold: self.slru_protected_threshold, + tlru_tail_budget: self.tlru_tail_budget, + tlru_float_config: self.tlru_float_config.as_ref().map(|value| value.config), page_size: self.page_size, is_write_back: self.is_write_back, enable_hicache: self.enable_hicache, write_through_threshold: self.write_through_threshold, device: parse_device(&self.device)?, swa_sliding_window_size: self.swa_sliding_window_size, + swa_req_ring: self.swa_req_ring, // Wired post-construction via set_has_swa_host_pool. has_swa_host_pool: false, enable_kv_cache_events: self.enable_kv_cache_events, @@ -467,7 +531,7 @@ impl TreeCoreInitParamsBinding { #[pymethods] impl TreeCoreInitParamsBinding { #[new] - #[pyo3(signature = (eviction_policy = "lru".to_string(), page_size = 1, is_write_back = false, enable_hicache = false, write_through_threshold = 256, device = "cpu".to_string(), swa_sliding_window_size = None, enable_kv_cache_events = false, mamba_cache_chunk_size = None, mamba_max_states_per_path = None))] + #[pyo3(signature = (eviction_policy = "lru".to_string(), page_size = 1, is_write_back = false, enable_hicache = false, write_through_threshold = 256, device = "cpu".to_string(), swa_sliding_window_size = None, enable_kv_cache_events = false, mamba_cache_chunk_size = None, mamba_max_states_per_path = None, slru_protected_threshold = 2, swa_req_ring = false, tlru_tail_budget = 0, tlru_float_config = None))] fn new( eviction_policy: String, page_size: usize, @@ -479,15 +543,23 @@ impl TreeCoreInitParamsBinding { enable_kv_cache_events: bool, mamba_cache_chunk_size: Option, mamba_max_states_per_path: Option, + slru_protected_threshold: i64, + swa_req_ring: bool, + tlru_tail_budget: usize, + tlru_float_config: Option, ) -> Self { TreeCoreInitParamsBinding { eviction_policy, + slru_protected_threshold, + tlru_tail_budget, + tlru_float_config, page_size, is_write_back, enable_hicache, write_through_threshold, device, swa_sliding_window_size, + swa_req_ring, enable_kv_cache_events, mamba_cache_chunk_size, mamba_max_states_per_path, @@ -526,6 +598,7 @@ impl MatchParamsBinding { /// stays a Python-held reference until the insert call unwraps it. #[pyclass(get_all, set_all)] pub struct InsertParamsBinding { + pub rotation_base: Option, pub key: Vec, pub value: Py, pub extra_key: Option, @@ -543,7 +616,7 @@ pub struct InsertParamsBinding { #[pymethods] impl InsertParamsBinding { #[new] - #[pyo3(signature = (key, value, extra_key = None, cache_salt = None, session_id = None, prev_prefix_len = 0, swa_evicted_seqlen = 0, swa_branching_seqlen = None, chunked = false, priority = 0, mamba_value = None, track_adopted_ranges = false))] + #[pyo3(signature = (key, value, extra_key = None, cache_salt = None, session_id = None, prev_prefix_len = 0, swa_evicted_seqlen = 0, swa_branching_seqlen = None, chunked = false, priority = 0, mamba_value = None, track_adopted_ranges = false, rotation_base = None))] fn new( py: Python<'_>, key: &Bound<'_, PyAny>, @@ -558,8 +631,10 @@ impl InsertParamsBinding { priority: i64, mamba_value: Option>, track_adopted_ranges: bool, + rotation_base: Option, ) -> PyResult { Ok(InsertParamsBinding { + rotation_base, key: py_array_to_vec_i64(py, key)?, value, extra_key, @@ -614,6 +689,7 @@ impl MatchResultBinding { /// Python-visible insert result; actions are Python-held. #[pyclass(get_all)] pub struct InsertResultBinding { + rotation_tail_declined: bool, prefix_len: usize, total_len: usize, last_device_node: Option, @@ -654,6 +730,7 @@ impl InsertResultBinding { /// Move a core insert result across the boundary. fn from_insert_result(py: Python<'_>, result: InsertResult) -> PyResult { Ok(InsertResultBinding { + rotation_tail_declined: result.rotation_tail_declined, prefix_len: result.prefix_len, total_len: result.total_len, last_device_node: result.last_device_node_id, @@ -770,7 +847,11 @@ fn tracker_to_py(tracker: HashMap) -> HashMap { #[pyclass(get_all)] pub struct EvictDeviceNextNodeResultBinding { node_id: Option, + mamba_backup_node_id: Option, + swa_backup_node_id: Option, + swa_backup_num_tokens: usize, made_progress: bool, + unbacked_tokens: usize, tracker: HashMap, new_device_frees: Py, new_host_frees: Py, @@ -782,6 +863,7 @@ pub struct EvictDeviceNextNodeResultBinding { #[pyclass(get_all)] pub struct EvictDeviceLeafResultBinding { backup_kv: Option>, + unbacked_tokens: usize, tracker: HashMap, new_device_frees: Py, new_host_frees: Py, @@ -941,11 +1023,11 @@ impl TreeCoreBinding { let eviction_policy = init_params.eviction_policy.to_lowercase(); if !matches!( eviction_policy.as_str(), - "lru" | "lfu" | "fifo" | "mru" | "filo" | "priority" | "slru" + "lru" | "lfu" | "fifo" | "mru" | "filo" | "priority" | "slru" | "tlru" ) { return Err(PyValueError::new_err(format!( "Unknown eviction policy: {eviction_policy}. Supported policies: \ - 'lru', 'lfu', 'fifo', 'mru', 'filo', 'priority', 'slru'." + 'lru', 'lfu', 'fifo', 'mru', 'filo', 'priority', 'slru', 'tlru'." ))); } let params = init_params.to_cache_init_params()?; @@ -1028,6 +1110,49 @@ impl TreeCoreBinding { py.allow_threads(|| self.core().match_full_device_prefix(key, namespace)) } + fn swa_tombstone_ranges( + &self, + py: Python<'_>, + params: &MatchParamsBinding, + start: usize, + end: usize, + ) -> PyResult> { + let key = K::key_from(Cow::Borrowed(¶ms.key)); + let namespace = + KeyNamespaceRef::new(params.extra_key.as_deref(), params.cache_salt.as_deref()); + py.allow_threads(|| { + self.core() + .swa_tombstone_ranges(key.as_ref(), namespace, start, end) + }) + .map_err(PyAssertionError::new_err) + } + + fn attach_swa_window( + &self, + py: Python<'_>, + params: &MatchParamsBinding, + window_start: usize, + window_end: usize, + swa_values: PyTensor, + ) -> PyResult> { + let key = K::key_from(Cow::Borrowed(¶ms.key)); + let namespace = + KeyNamespaceRef::new(params.extra_key.as_deref(), params.cache_salt.as_deref()); + let swa_values = swa_values.0; + let actions = py + .allow_threads(move || { + self.core().attach_swa_window( + key.as_ref(), + namespace, + window_start, + window_end, + &swa_values, + ) + }) + .map_err(PyAssertionError::new_err)?; + cache_actions_to_py(py, actions) + } + /// The empty match result anchored at the root. fn empty_match_result(&self, py: Python<'_>) -> PyResult { let result = py.allow_threads(|| self.core().empty_match_result()); @@ -1050,6 +1175,7 @@ impl TreeCoreBinding { None => None, }; let params = InsertParams { + rotation_base: params.rotation_base, key, namespace: KeyNamespaceRef::new( params.extra_key.as_deref(), @@ -1087,6 +1213,7 @@ impl TreeCoreBinding { None => None, }; let params = InsertParams { + rotation_base: params.rotation_base, key, namespace: KeyNamespaceRef::new( params.extra_key.as_deref(), @@ -1168,6 +1295,33 @@ impl TreeCoreBinding { Ok(()) } + /// Release one window's lock, retaining other component locks. + fn dec_window_lock_only( + &self, + py: Python<'_>, + node_id: NodeId, + component_type: u8, + params: &DecLockRefParamsBinding, + ) -> PyResult<(Py, Py)> { + let component_type = parse_component_type(component_type)?; + let params = params.to_dec_lock_ref_params()?; + let (device_frees, host_frees) = py + .allow_threads(|| { + let mut device_frees = HashMap::new(); + let mut host_frees = HashMap::new(); + self.core().dec_window_lock_only( + node_id, + component_type, + ¶ms, + &mut device_frees, + &mut host_frees, + )?; + Ok((device_frees, host_frees)) + }) + .map_err(node_access_error)?; + Ok((frees_to_py(py, device_frees)?, frees_to_py(py, host_frees)?)) + } + /// Early-release the SWA portion of a request's tree lock; returns this /// release's per-component (device_frees, host_frees). fn dec_swa_lock_only( @@ -1224,13 +1378,19 @@ impl TreeCoreBinding { Ok(()) } + /// Logical key lengths for the nodes in a load-back transfer. + fn get_node_key_lengths(&self, py: Python<'_>, node_ids: Vec) -> PyResult> { + py.allow_threads(|| self.core().get_node_key_lengths(&node_ids)) + .map_err(node_access_error) + } + /// A component's device value on a node, if set. fn get_component_device_value( &self, py: Python<'_>, node_id: NodeId, component_type: u8, - ) -> PyResult> { + ) -> PyResult>> { let component_type = parse_component_type(component_type)?; let value = py .allow_threads(|| { @@ -1239,7 +1399,7 @@ impl TreeCoreBinding { .map(|value| value.map(|tensor| tensor.shallow_clone())) }) .map_err(node_access_error)?; - Ok(value.map(PyTensor)) + value.map(|value| tensor_to_py(py, value)).transpose() } // TODO(jialino): batch a full no-backup eviction round in Rust (one crossing @@ -1256,9 +1416,9 @@ impl TreeCoreBinding { Ok(()) } - /// Advance one component eviction step. A missing node with - /// `made_progress` set means an internal tombstone completed the step; - /// otherwise a missing node means the walk is exhausted. + /// Advance one component eviction step. Progress without a leaf can be + /// an internal tombstone or a component backup request awaiting its finish; + /// no progress means the walk is exhausted. fn evict_device_next_node( &self, py: Python<'_>, @@ -1269,10 +1429,57 @@ impl TreeCoreBinding { let baseline = tracker_from_py(tracker)?; let (node_id, result) = py.allow_threads(move || self.core().evict_device_next_node(ct, &baseline)); - let made_progress = node_id.is_some() || !result.tracker.is_empty(); + let made_progress = node_id.is_some() + || result.mamba_backup_node_id.is_some() + || result.swa_backup_node_id.is_some() + || !result.tracker.is_empty(); Ok(EvictDeviceNextNodeResultBinding { node_id, + mamba_backup_node_id: result.mamba_backup_node_id, + swa_backup_node_id: result.swa_backup_node_id, + swa_backup_num_tokens: result.swa_backup_num_tokens, made_progress, + unbacked_tokens: result.unbacked_tokens, + tracker: tracker_to_py(result.tracker), + new_device_frees: frees_to_py(py, result.device_frees)?, + new_host_frees: frees_to_py(py, result.host_frees)?, + }) + } + + /// Resume a pending internal Mamba tombstone after the host backup attempt. + fn finish_mamba_state_eviction( + &self, + py: Python<'_>, + node_id: NodeId, + ) -> PyResult { + let result = py.allow_threads(move || self.core().finish_mamba_state_eviction(node_id)); + self.internal_eviction_finish_to_py(py, result) + } + + /// Resume a pending internal SWA tombstone after the host backup attempt. + fn finish_swa_state_eviction( + &self, + py: Python<'_>, + node_id: NodeId, + ) -> PyResult { + let result = py.allow_threads(move || self.core().finish_swa_state_eviction(node_id)); + self.internal_eviction_finish_to_py(py, result) + } + + fn internal_eviction_finish_to_py( + &self, + py: Python<'_>, + result: EvictionStepResult, + ) -> PyResult { + Ok(EvictDeviceNextNodeResultBinding { + node_id: None, + mamba_backup_node_id: None, + swa_backup_node_id: None, + swa_backup_num_tokens: 0, + // Consuming the pending request advances the finite walk even + // when intervening I/O pinned or removed this particular node. + made_progress: true, + unbacked_tokens: result.unbacked_tokens, tracker: tracker_to_py(result.tracker), new_device_frees: frees_to_py(py, result.device_frees)?, new_host_frees: frees_to_py(py, result.host_frees)?, @@ -1294,6 +1501,7 @@ impl TreeCoreBinding { }) .map_err(node_access_error)?; Ok(EvictDeviceLeafResultBinding { + unbacked_tokens: result.unbacked_tokens, backup_kv: backup .map(|backup| cache_action_to_py(py, CacheAction::BackupKV(backup))) .transpose()?, @@ -1331,24 +1539,26 @@ impl TreeCoreBinding { py: Python<'_>, from_node_id: NodeId, until_node_id: NodeId, - ) -> PyResult { + ) -> PyResult> { let value = py .allow_threads(|| { self.core() .collect_full_device_indices(from_node_id, until_node_id) }) .map_err(node_access_error)?; - Ok(PyTensor(value)) + tensor_to_py(py, value) } /// Every FULL device value in the tree, concatenated. - fn all_values_flatten(&self, py: Python<'_>) -> PyTensor { - PyTensor(py.allow_threads(|| self.core().all_values_flatten())) + fn all_values_flatten(&self, py: Python<'_>) -> PyResult> { + let value = py.allow_threads(|| self.core().all_values_flatten()); + tensor_to_py(py, value) } /// Every Mamba device value in the tree, concatenated. - fn all_mamba_values_flatten(&self, py: Python<'_>) -> PyTensor { - PyTensor(py.allow_threads(|| self.core().all_mamba_values_flatten())) + fn all_mamba_values_flatten(&self, py: Python<'_>) -> PyResult> { + let value = py.allow_threads(|| self.core().all_mamba_values_flatten()); + tensor_to_py(py, value) } /// Flatten every FULL device slot into (slot, position, prev-slot) rows for the KV-canary sweep. @@ -1480,11 +1690,14 @@ impl TreeCoreBinding { &self, py: Python<'_>, node_id: NodeId, - ) -> PyResult<(PyTensor, Py)> { + ) -> PyResult<(Py, Py)> { let (device_value, comp_xfers) = py .allow_threads(|| self.core().build_backup_spec(node_id)) .map_err(node_access_error)?; - Ok((PyTensor(device_value), comp_xfers_to_py(py, comp_xfers)?)) + Ok(( + tensor_to_py(py, device_value)?, + comp_xfers_to_py(py, comp_xfers)?, + )) } /// Gather a node's device->storage backup spec; None if the node is not backuped. @@ -1631,6 +1844,11 @@ impl TreeCoreBinding { py.allow_threads(|| self.core().root_node_handle(extra_key.as_deref())) } + fn rotation_base_of(&self, py: Python<'_>, node_id: NodeId) -> PyResult> { + py.allow_threads(|| self.core().rotation_base_of(node_id)) + .map_err(node_access_error) + } + fn dfs_weight_order(&self, py: Python<'_>, node_ids: Vec) -> PyResult> { py.allow_threads(|| self.core().dfs_weight_order(&node_ids)) .map_err(node_access_error) @@ -1762,9 +1980,16 @@ impl TreeCoreBinding { py: Python<'_>, component_type: u8, num_tokens: usize, + skip_full_duplicate_reclaim: bool, ) -> PyResult { let ct = parse_component_type(component_type)?; - let result = py.allow_threads(move || self.core().drive_host_eviction(ct, num_tokens)); + let result = py.allow_threads(move || { + self.core().drive_host_eviction_with_options( + ct, + num_tokens, + skip_full_duplicate_reclaim, + ) + }); Ok(HostEvictionResultBinding { tracker: tracker_to_py(result.tracker), new_device_frees: frees_to_py(py, result.device_frees)?, @@ -2070,14 +2295,15 @@ impl TreeCoreBinding { py: Python<'_>, node_id: NodeId, component_type: u8, - ) -> PyResult> { + ) -> PyResult>> { let component_type = parse_component_type(component_type)?; - py.allow_threads(|| { - self.core() - .inspect_get_component_host_value(node_id, component_type) - }) - .map(|value| value.map(PyTensor)) - .map_err(node_access_error) + let value = py + .allow_threads(|| { + self.core() + .inspect_get_component_host_value(node_id, component_type) + }) + .map_err(node_access_error)?; + value.map(|value| tensor_to_py(py, value)).transpose() } fn inspect_get_component_device_lock_ref( @@ -2447,14 +2673,19 @@ macro_rules! tree_core_binding { /// init params. #[new] fn new(init_params: &TreeCoreInitParamsBinding, component_types: Vec) -> PyResult { - Ok($name { - inner: TreeCoreBinding::new(init_params, component_types)?, + catch_native_panic(|| { + Ok($name { + inner: TreeCoreBinding::new(init_params, component_types)?, + }) }) } /// Drop the entire tree and reinitialize empty state. - fn reset(&self, py: Python<'_>) { - self.inner.reset(py) + fn reset(&self, py: Python<'_>) -> PyResult<()> { + catch_native_panic(|| { + self.inner.reset(py); + Ok(()) + }) } /// Match a key against the tree. @@ -2463,7 +2694,7 @@ macro_rules! tree_core_binding { py: Python<'_>, params: &MatchParamsBinding, ) -> PyResult { - self.inner.match_prefix(py, params) + catch_native_panic(|| self.inner.match_prefix(py, params)) } /// Read-only FULL-device match, independent of auxiliary components. @@ -2471,13 +2702,37 @@ macro_rules! tree_core_binding { &self, py: Python<'_>, params: &MatchParamsBinding, - ) -> (usize, NodeId, usize) { - self.inner.match_full_device_prefix(py, params) + ) -> PyResult<(usize, NodeId, usize)> { + catch_native_panic(|| Ok(self.inner.match_full_device_prefix(py, params))) + } + + fn swa_tombstone_ranges( + &self, + py: Python<'_>, + params: &MatchParamsBinding, + start: usize, + end: usize, + ) -> PyResult> { + catch_native_panic(|| self.inner.swa_tombstone_ranges(py, params, start, end)) + } + + fn attach_swa_window( + &self, + py: Python<'_>, + params: &MatchParamsBinding, + window_start: usize, + window_end: usize, + swa_values: PyTensor, + ) -> PyResult> { + catch_native_panic(|| { + self.inner + .attach_swa_window(py, params, window_start, window_end, swa_values) + }) } /// The empty match result anchored at the root. fn empty_match_result(&self, py: Python<'_>) -> PyResult { - self.inner.empty_match_result(py) + catch_native_panic(|| self.inner.empty_match_result(py)) } /// Insert device values into the tree per the provided key. @@ -2486,7 +2741,7 @@ macro_rules! tree_core_binding { py: Python<'_>, params: &InsertParamsBinding, ) -> PyResult { - self.inner.insert(py, params) + catch_native_panic(|| self.inner.insert(py, params)) } /// Start the resumable insert, running to its first barrier or completion. @@ -2495,22 +2750,22 @@ macro_rules! tree_core_binding { py: Python<'_>, params: &InsertParamsBinding, ) -> PyResult { - self.inner.begin_insert(py, params) + catch_native_panic(|| self.inner.begin_insert(py, params)) } /// Continue the suspended insert after its step actions were applied. fn resume_insert(&self, py: Python<'_>) -> PyResult { - self.inner.resume_insert(py) + catch_native_panic(|| self.inner.resume_insert(py)) } /// Whether an insert walk is suspended at a barrier. - fn has_ongoing_insert(&self, py: Python<'_>) -> bool { - self.inner.has_ongoing_insert(py) + fn has_ongoing_insert(&self, py: Python<'_>) -> PyResult { + catch_native_panic(|| Ok(self.inner.has_ongoing_insert(py))) } /// Finish the insert (idempotent); returns still-pending actions to drain. fn end_insert(&self, py: Python<'_>) -> PyResult> { - self.inner.end_insert(py) + catch_native_panic(|| self.inner.end_insert(py)) } /// Bump the reference count on a node's component locks. @@ -2521,17 +2776,17 @@ macro_rules! tree_core_binding { node_id: NodeId, skip_lock_components: Vec, ) -> PyResult { - self.inner.inc_lock_ref(py, node_id, skip_lock_components) + catch_native_panic(|| self.inner.inc_lock_ref(py, node_id, skip_lock_components)) } /// Pin only the FULL device values on a node's root path. fn inc_full_pin(&self, py: Python<'_>, node_id: NodeId) -> PyResult<()> { - self.inner.inc_full_pin(py, node_id) + catch_native_panic(|| self.inner.inc_full_pin(py, node_id)) } /// Release a FULL-only root-path pin. fn dec_full_pin(&self, py: Python<'_>, node_id: NodeId) -> PyResult<()> { - self.inner.dec_full_pin(py, node_id) + catch_native_panic(|| self.inner.dec_full_pin(py, node_id)) } /// Decrease the reference count on a node's component locks. The @@ -2544,7 +2799,21 @@ macro_rules! tree_core_binding { params: &DecLockRefParamsBinding, skip_swa: bool, ) -> PyResult<()> { - self.inner.dec_lock_ref(py, node_id, params, skip_swa) + catch_native_panic(|| self.inner.dec_lock_ref(py, node_id, params, skip_swa)) + } + + /// Release one window's lock, retaining other component locks. + fn dec_window_lock_only( + &self, + py: Python<'_>, + node_id: NodeId, + component_type: u8, + params: &DecLockRefParamsBinding, + ) -> PyResult<(Py, Py)> { + catch_native_panic(|| { + self.inner + .dec_window_lock_only(py, node_id, component_type, params) + }) } /// Early-release the SWA portion of a request's tree lock; returns this @@ -2556,7 +2825,7 @@ macro_rules! tree_core_binding { node_id: NodeId, params: &DecLockRefParamsBinding, ) -> PyResult<(Py, Py)> { - self.inner.dec_swa_lock_only(py, node_id, params) + catch_native_panic(|| self.inner.dec_swa_lock_only(py, node_id, params)) } /// Store a component's device value on a node (the SWA rebuild write-back). @@ -2567,8 +2836,15 @@ macro_rules! tree_core_binding { component_type: u8, value: PyTensor, ) -> PyResult<()> { - self.inner - .set_component_device_value(py, node_id, component_type, value) + catch_native_panic(|| { + self.inner + .set_component_device_value(py, node_id, component_type, value) + }) + } + + /// Logical key lengths for the nodes in a load-back transfer. + fn get_node_key_lengths(&self, py: Python<'_>, node_ids: Vec) -> PyResult> { + catch_native_panic(|| self.inner.get_node_key_lengths(py, node_ids)) } /// A component's device value on a node, if set. @@ -2577,9 +2853,11 @@ macro_rules! tree_core_binding { py: Python<'_>, node_id: NodeId, component_type: u8, - ) -> PyResult> { - self.inner - .get_component_device_value(py, node_id, component_type) + ) -> PyResult>> { + catch_native_panic(|| { + self.inner + .get_component_device_value(py, node_id, component_type) + }) } /// Begin a component's device-eviction walk for up to request_cnt tokens. @@ -2589,8 +2867,10 @@ macro_rules! tree_core_binding { component_type: u8, request_cnt: usize, ) -> PyResult<()> { - self.inner - .evict_device_start(py, component_type, request_cnt) + catch_native_panic(|| { + self.inner + .evict_device_start(py, component_type, request_cnt) + }) } /// The next device leaf to evict, or None when the walk is done; the @@ -2602,8 +2882,28 @@ macro_rules! tree_core_binding { component_type: u8, tracker: HashMap, ) -> PyResult { - self.inner - .evict_device_next_node(py, component_type, tracker) + catch_native_panic(|| { + self.inner + .evict_device_next_node(py, component_type, tracker) + }) + } + + /// Finish an internal Mamba eviction after its host backup attempt. + fn finish_mamba_state_eviction( + &self, + py: Python<'_>, + node_id: NodeId, + ) -> PyResult { + catch_native_panic(|| self.inner.finish_mamba_state_eviction(py, node_id)) + } + + /// Finish an internal SWA eviction after its host backup attempt. + fn finish_swa_state_eviction( + &self, + py: Python<'_>, + node_id: NodeId, + ) -> PyResult { + catch_native_panic(|| self.inner.finish_swa_state_eviction(py, node_id)) } /// Evict one device leaf; an unbacked write-back leaf returns its backup @@ -2613,12 +2913,12 @@ macro_rules! tree_core_binding { py: Python<'_>, node_id: NodeId, ) -> PyResult { - self.inner.evict_device_leaf(py, node_id) + catch_native_panic(|| self.inner.evict_device_leaf(py, node_id)) } /// Finish a component's device-eviction walk. fn evict_device_end(&self, py: Python<'_>, component_type: u8) -> PyResult<()> { - self.inner.evict_device_end(py, component_type) + catch_native_panic(|| self.inner.evict_device_end(py, component_type)) } /// Verify tree-structure, leaf-set, LRU, size, and ongoing-op invariants; @@ -2629,8 +2929,10 @@ macro_rules! tree_core_binding { ongoing_write_through: Vec<(i64, NodeId)>, ongoing_load_back: Vec<(i64, NodeId)>, ) -> PyResult<()> { - self.inner - .sanity_check(py, ongoing_write_through, ongoing_load_back) + catch_native_panic(|| { + self.inner + .sanity_check(py, ongoing_write_through, ongoing_load_back) + }) } /// Concatenated FULL device values from from_node up to (exclusive) until_node. @@ -2639,19 +2941,21 @@ macro_rules! tree_core_binding { py: Python<'_>, from_node_id: NodeId, until_node_id: NodeId, - ) -> PyResult { - self.inner - .collect_full_device_indices(py, from_node_id, until_node_id) + ) -> PyResult> { + catch_native_panic(|| { + self.inner + .collect_full_device_indices(py, from_node_id, until_node_id) + }) } /// Every FULL device value in the tree, concatenated. - fn all_values_flatten(&self, py: Python<'_>) -> PyTensor { - self.inner.all_values_flatten(py) + fn all_values_flatten(&self, py: Python<'_>) -> PyResult> { + catch_native_panic(|| self.inner.all_values_flatten(py)) } /// Every Mamba device value in the tree, concatenated. - fn all_mamba_values_flatten(&self, py: Python<'_>) -> PyTensor { - self.inner.all_mamba_values_flatten(py) + fn all_mamba_values_flatten(&self, py: Python<'_>) -> PyResult> { + catch_native_panic(|| self.inner.all_mamba_values_flatten(py)) } /// Flatten every FULL device slot into (slot, position, prev-slot) rows for the KV-canary sweep. @@ -2661,78 +2965,89 @@ macro_rules! tree_core_binding { unlocked_only: bool, swa_resident_only: bool, ) -> PyResult { - self.inner - .walk_for_kv_canary(py, unlocked_only, swa_resident_only) + catch_native_panic(|| { + self.inner + .walk_for_kv_canary(py, unlocked_only, swa_resident_only) + }) } /// Evictable token count of the FULL (base) component. - fn evictable_size(&self, py: Python<'_>) -> usize { - self.inner.evictable_size(py) + fn evictable_size(&self, py: Python<'_>) -> PyResult { + catch_native_panic(|| Ok(self.inner.evictable_size(py))) } /// Protected (locked) token count of the FULL (base) component. - fn protected_size(&self, py: Python<'_>) -> usize { - self.inner.protected_size(py) + fn protected_size(&self, py: Python<'_>) -> PyResult { + catch_native_panic(|| Ok(self.inner.protected_size(py))) } /// FULL component evictable token count. - fn full_evictable_size(&self, py: Python<'_>) -> usize { - self.inner.full_evictable_size(py) + fn full_evictable_size(&self, py: Python<'_>) -> PyResult { + catch_native_panic(|| Ok(self.inner.full_evictable_size(py))) } /// FULL component protected token count. - fn full_protected_size(&self, py: Python<'_>) -> usize { - self.inner.full_protected_size(py) + fn full_protected_size(&self, py: Python<'_>) -> PyResult { + catch_native_panic(|| Ok(self.inner.full_protected_size(py))) } /// Evictable token count for one component (0 if the component is absent). fn component_evictable_size(&self, py: Python<'_>, component_type: u8) -> PyResult { - self.inner.component_evictable_size(py, component_type) + catch_native_panic(|| self.inner.component_evictable_size(py, component_type)) } /// Protected token count for one component (0 if the component is absent). fn component_protected_size(&self, py: Python<'_>, component_type: u8) -> PyResult { - self.inner.component_protected_size(py, component_type) + catch_native_panic(|| self.inner.component_protected_size(py, component_type)) } /// (full_tokens, aux_tokens) summed across the whole tree. - fn total_size(&self, py: Python<'_>) -> (usize, usize) { - self.inner.total_size(py) + fn total_size(&self, py: Python<'_>) -> PyResult<(usize, usize)> { + catch_native_panic(|| Ok(self.inner.total_size(py))) } /// Whether the node's FULL device value has been evicted. fn is_full_device_evicted(&self, py: Python<'_>, node_id: NodeId) -> PyResult { - self.inner.is_full_device_evicted(py, node_id) + catch_native_panic(|| self.inner.is_full_device_evicted(py, node_id)) } /// Mark the host tier as buffer-only; wired after the host pools are built. - fn set_host_memory_buffer_only(&self, py: Python<'_>) { - self.inner.set_host_memory_buffer_only(py) + fn set_host_memory_buffer_only(&self, py: Python<'_>) -> PyResult<()> { + catch_native_panic(|| { + self.inner.set_host_memory_buffer_only(py); + Ok(()) + }) } /// Whether the host tier runs as a storage staging buffer, not a cache. - fn is_host_memory_buffer_only(&self, py: Python<'_>) -> bool { - self.inner.is_host_memory_buffer_only(py) + fn is_host_memory_buffer_only(&self, py: Python<'_>) -> PyResult { + catch_native_panic(|| Ok(self.inner.is_host_memory_buffer_only(py))) } /// Mark the host tier (HiCache) as wired. - fn set_hicache_enabled(&self, py: Python<'_>) { - self.inner.set_hicache_enabled(py) + fn set_hicache_enabled(&self, py: Python<'_>) -> PyResult<()> { + catch_native_panic(|| { + self.inner.set_hicache_enabled(py); + Ok(()) + }) } /// Whether the host tier (HiCache) is wired. - fn enable_hicache(&self, py: Python<'_>) -> bool { - self.inner.enable_hicache(py) + fn enable_hicache(&self, py: Python<'_>) -> PyResult { + catch_native_panic(|| Ok(self.inner.enable_hicache(py))) } /// Mark the SWA host pool as wired (HiCache). - fn set_has_swa_host_pool(&self, py: Python<'_>) { - self.inner.set_has_swa_host_pool(py) + fn set_has_swa_host_pool(&self, py: Python<'_>) -> PyResult<()> { + catch_native_panic(|| { + self.inner.set_has_swa_host_pool(py); + Ok(()) + }) } /// Whether the SWA host pool is wired. - fn has_swa_host_pool(&self, py: Python<'_>) -> bool { - self.inner.has_swa_host_pool(py) + fn has_swa_host_pool(&self, py: Python<'_>) -> PyResult { + catch_native_panic(|| Ok(self.inner.has_swa_host_pool(py))) } /// Insert a host-side (backuped) tree path descending from the given node. @@ -2747,15 +3062,11 @@ macro_rules! tree_core_binding { hash_value: Vec, cache_salt: Option, ) -> PyResult { - self.inner.insert_host( - py, - node_id, - extra_key, - key, - host_value, - hash_value, - cache_salt, - ) + catch_native_panic(|| { + self.inner.insert_host( + py, node_id, extra_key, key, host_value, hash_value, cache_salt, + ) + }) } /// Gather a node's device value plus per-component BACKUP_HOST transfers. @@ -2763,8 +3074,8 @@ macro_rules! tree_core_binding { &self, py: Python<'_>, node_id: NodeId, - ) -> PyResult<(PyTensor, Py)> { - self.inner.build_backup_spec(py, node_id) + ) -> PyResult<(Py, Py)> { + catch_native_panic(|| self.inner.build_backup_spec(py, node_id)) } /// Gather a node's device->storage backup spec; None if the node is not backuped. @@ -2774,8 +3085,10 @@ macro_rules! tree_core_binding { node_id: NodeId, pass_prefix_keys: bool, ) -> PyResult> { - self.inner - .build_storage_backup_spec(py, node_id, pass_prefix_keys) + catch_native_panic(|| { + self.inner + .build_storage_backup_spec(py, node_id, pass_prefix_keys) + }) } /// Route a build_hicache_transfers call to the component for the given type. @@ -2793,17 +3106,19 @@ macro_rules! tree_core_binding { staging_tokens: usize, last_hash: Option, ) -> PyResult>>> { - self.inner.build_hicache_transfers( - py, - component_type, - node_id, - phase, - host_indices, - token_ids, - prefetch_tokens, - staging_tokens, - last_hash, - ) + catch_native_panic(|| { + self.inner.build_hicache_transfers( + py, + component_type, + node_id, + phase, + host_indices, + token_ids, + prefetch_tokens, + staging_tokens, + last_hash, + ) + }) } /// The anchor node's caller-defined key and cache salt. @@ -2812,39 +3127,31 @@ macro_rules! tree_core_binding { py: Python<'_>, node_id: NodeId, ) -> PyResult<(Option, Option)> { - self.inner.prefetch_anchor_info(py, node_id) + catch_native_panic(|| self.inner.prefetch_anchor_info(py, node_id)) } /// Whether the node's Full KV is present on host. fn node_backuped(&self, py: Python<'_>, node_id: NodeId) -> PyResult { - self.inner.node_backuped(py, node_id) + catch_native_panic(|| self.inner.node_backuped(py, node_id)) } /// Whether the node is a (default or named) root. fn is_root(&self, py: Python<'_>, node_id: NodeId) -> PyResult { - self.inner.is_root(py, node_id) + catch_native_panic(|| self.inner.is_root(py, node_id)) } /// The node's last page hash, or None when it was never hashed. - fn get_last_hash_value( - &self, - py: Python<'_>, - node_id: NodeId, - ) -> PyResult> { - self.inner.get_last_hash_value(py, node_id) + fn get_last_hash_value(&self, py: Python<'_>, node_id: NodeId) -> PyResult> { + catch_native_panic(|| self.inner.get_last_hash_value(py, node_id)) } /// The hash chain of the node's ancestors, in root-to-parent order. - fn get_prefix_hash_values( - &self, - py: Python<'_>, - node_id: NodeId, - ) -> PyResult> { - self.inner.get_prefix_hash_values(py, node_id) + fn get_prefix_hash_values(&self, py: Python<'_>, node_id: NodeId) -> PyResult> { + catch_native_panic(|| self.inner.get_prefix_hash_values(py, node_id)) } fn get_hash_values(&self, py: Python<'_>, node_id: NodeId) -> PyResult> { - self.inner.get_hash_values(py, node_id) + catch_native_panic(|| self.inner.get_hash_values(py, node_id)) } fn snapshot_buffer_backup( @@ -2852,9 +3159,12 @@ macro_rules! tree_core_binding { py: Python<'_>, node_id: NodeId, pass_prefix_keys: bool, - ) -> Option { - self.inner - .snapshot_buffer_backup(py, node_id, pass_prefix_keys) + ) -> PyResult> { + catch_native_panic(|| { + Ok(self + .inner + .snapshot_buffer_backup(py, node_id, pass_prefix_keys)) + }) } fn validate_buffer_backup( @@ -2862,27 +3172,30 @@ macro_rules! tree_core_binding { py: Python<'_>, node_id: NodeId, expected_key_length: usize, - ) -> Option { - self.inner - .validate_buffer_backup(py, node_id, expected_key_length) + ) -> PyResult> { + catch_native_panic(|| { + Ok(self + .inner + .validate_buffer_backup(py, node_id, expected_key_length)) + }) } /// Hash every node built while storage was disabled. - fn backfill_missing_hash_values(&self, py: Python<'_>) -> usize { - self.inner.backfill_missing_hash_values(py) + fn backfill_missing_hash_values(&self, py: Python<'_>) -> PyResult { + catch_native_panic(|| Ok(self.inner.backfill_missing_hash_values(py))) } #[pyo3(signature = (extra_key = None))] - fn root_node_handle(&self, py: Python<'_>, extra_key: Option) -> NodeId { - self.inner.root_node_handle(py, extra_key) + fn root_node_handle(&self, py: Python<'_>, extra_key: Option) -> PyResult { + catch_native_panic(|| Ok(self.inner.root_node_handle(py, extra_key))) } - fn dfs_weight_order( - &self, - py: Python<'_>, - node_ids: Vec, - ) -> PyResult> { - self.inner.dfs_weight_order(py, node_ids) + fn rotation_base_of(&self, py: Python<'_>, node_id: NodeId) -> PyResult> { + catch_native_panic(|| self.inner.rotation_base_of(py, node_id)) + } + + fn dfs_weight_order(&self, py: Python<'_>, node_ids: Vec) -> PyResult> { + catch_native_panic(|| self.inner.dfs_weight_order(py, node_ids)) } /// Commit each component's HiCache transfers; returns the new cache actions. @@ -2896,14 +3209,16 @@ macro_rules! tree_core_binding { insert_result: Option<(usize, Option, bool)>, pool_storage_result: Option<(usize, HashMap)>, ) -> PyResult<(Py, Option)> { - self.inner.commit_hicache_transfers( - py, - node_id, - phase, - comp_xfers, - insert_result, - pool_storage_result, - ) + catch_native_panic(|| { + self.inner.commit_hicache_transfers( + py, + node_id, + phase, + comp_xfers, + insert_result, + pool_storage_result, + ) + }) } /// Commit a successful backup to the node. @@ -2914,8 +3229,10 @@ macro_rules! tree_core_binding { host_indices: PyTensor, comp_xfers: HashMap>, ) -> PyResult<()> { - self.inner - .commit_backup(py, node_id, host_indices, comp_xfers) + catch_native_panic(|| { + self.inner + .commit_backup(py, node_id, host_indices, comp_xfers) + }) } /// Build the H->D load-back KV transfer plus per-component aux transfers. @@ -2926,7 +3243,7 @@ macro_rules! tree_core_binding { node_id: NodeId, mamba_pool_idx: Option, ) -> PyResult<(Py, Py)> { - self.inner.build_load_back_spec(py, node_id, mamba_pool_idx) + catch_native_panic(|| self.inner.build_load_back_spec(py, node_id, mamba_pool_idx)) } /// Commit a successful H->D load-back onto the node; returns its actions. @@ -2938,23 +3255,34 @@ macro_rules! tree_core_binding { kv_xfer: TransferArgs, comp_xfers: HashMap>, ) -> PyResult> { - self.inner - .commit_load_back(py, node_id, device_indices, kv_xfer, comp_xfers) + catch_native_panic(|| { + self.inner + .commit_load_back(py, node_id, device_indices, kv_xfer, comp_xfers) + }) } /// Release a node's device KV once its host copy exists. fn demote(&self, py: Python<'_>, node_id: NodeId) -> PyResult { - self.inner.demote(py, node_id) + catch_native_panic(|| self.inner.demote(py, node_id)) } /// Evict up to num_tokens of one component's host resources. + #[pyo3(signature = (component_type, num_tokens, skip_full_duplicate_reclaim = false))] fn drive_host_eviction( &self, py: Python<'_>, component_type: u8, num_tokens: usize, + skip_full_duplicate_reclaim: bool, ) -> PyResult { - self.inner.drive_host_eviction(py, component_type, num_tokens) + catch_native_panic(|| { + self.inner.drive_host_eviction( + py, + component_type, + num_tokens, + skip_full_duplicate_reclaim, + ) + }) } /// Evict shallow Mamba device checkpoints beyond the per-path cap @@ -2964,7 +3292,7 @@ macro_rules! tree_core_binding { py: Python<'_>, tail_node_id: NodeId, ) -> PyResult { - self.inner.evict_excess_path_states(py, tail_node_id) + catch_native_panic(|| self.inner.evict_excess_path_states(py, tail_node_id)) } /// Bump the reference count on a node's host-side component locks. @@ -2973,7 +3301,7 @@ macro_rules! tree_core_binding { py: Python<'_>, node_id: NodeId, ) -> PyResult { - self.inner.inc_host_lock_ref(py, node_id) + catch_native_panic(|| self.inner.inc_host_lock_ref(py, node_id)) } /// Decrease the reference count on a node's host-side component locks. @@ -2984,61 +3312,69 @@ macro_rules! tree_core_binding { node_id: NodeId, params: &DecLockRefParamsBinding, ) -> PyResult<()> { - self.inner.dec_host_lock_ref(py, node_id, params) + catch_native_panic(|| self.inner.dec_host_lock_ref(py, node_id, params)) } /// Set the write-back (vs write-through) policy; decided at HiCache init. - fn set_is_write_back(&self, py: Python<'_>, is_write_back: bool) { - self.inner.set_is_write_back(py, is_write_back) + fn set_is_write_back(&self, py: Python<'_>, is_write_back: bool) -> PyResult<()> { + catch_native_panic(|| { + self.inner.set_is_write_back(py, is_write_back); + Ok(()) + }) } /// The current write-back (vs write-through) policy. - fn is_write_back(&self, py: Python<'_>) -> bool { - self.inner.is_write_back(py) + fn is_write_back(&self, py: Python<'_>) -> PyResult { + catch_native_panic(|| Ok(self.inner.is_write_back(py))) } /// Set the write-through backup hit threshold; decided at HiCache init. - fn set_write_through_threshold(&self, py: Python<'_>, threshold: i64) { - self.inner.set_write_through_threshold(py, threshold) + fn set_write_through_threshold(&self, py: Python<'_>, threshold: i64) -> PyResult<()> { + catch_native_panic(|| { + self.inner.set_write_through_threshold(py, threshold); + Ok(()) + }) } /// The current write-through backup hit threshold. - fn write_through_threshold(&self, py: Python<'_>) -> i64 { - self.inner.write_through_threshold(py) + fn write_through_threshold(&self, py: Python<'_>) -> PyResult { + catch_native_panic(|| Ok(self.inner.write_through_threshold(py))) } /// Mark the storage tier (L3) wired; storage attaches after tree construction. - fn set_enable_storage(&self, py: Python<'_>, value: bool) { - self.inner.set_enable_storage(py, value) + fn set_enable_storage(&self, py: Python<'_>, value: bool) -> PyResult<()> { + catch_native_panic(|| { + self.inner.set_enable_storage(py, value); + Ok(()) + }) } /// Whether the storage tier (L3) is wired. - fn enable_storage(&self, py: Python<'_>) -> bool { - self.inner.enable_storage(py) + fn enable_storage(&self, py: Python<'_>) -> PyResult { + catch_native_panic(|| Ok(self.inner.enable_storage(py))) } /// Enable or disable the direct external-cache linker. - fn set_enable_external_cache_linker( - &self, - py: Python<'_>, - value: bool, - ) -> PyResult<()> { - self.inner.set_enable_external_cache_linker(py, value) + fn set_enable_external_cache_linker(&self, py: Python<'_>, value: bool) -> PyResult<()> { + catch_native_panic(|| self.inner.set_enable_external_cache_linker(py, value)) } /// Whether the direct external-cache linker is wired. - fn enable_external_cache_linker(&self, py: Python<'_>) -> bool { - self.inner.enable_external_cache_linker(py) + fn enable_external_cache_linker(&self, py: Python<'_>) -> PyResult { + catch_native_panic(|| Ok(self.inner.enable_external_cache_linker(py))) } /// Queue the all-cleared placement event. - fn record_all_cleared_event(&self, py: Python<'_>) { - self.inner.record_all_cleared_event(py) + fn record_all_cleared_event(&self, py: Python<'_>) -> PyResult<()> { + catch_native_panic(|| { + self.inner.record_all_cleared_event(py); + Ok(()) + }) } /// Drain the queued placement events as tagged tuples. fn take_events(&self, py: Python<'_>) -> PyResult> { - self.inner.take_events(py) + catch_native_panic(|| self.inner.take_events(py)) } /// Drop the subtree rooted at an unbacked D-leaf; not dropped when a lock @@ -3048,7 +3384,7 @@ macro_rules! tree_core_binding { py: Python<'_>, node_id: NodeId, ) -> PyResult { - self.inner.drop_subtree_no_host(py, node_id) + catch_native_panic(|| self.inner.drop_subtree_no_host(py, node_id)) } /// Mark the nodes one write-through backup covers; returns them ancestors first. @@ -3058,7 +3394,7 @@ macro_rules! tree_core_binding { node_ids: Vec, ack_id: NodeId, ) -> PyResult> { - self.inner.mark_write_through_pending(py, node_ids, ack_id) + catch_native_panic(|| self.inner.mark_write_through_pending(py, node_ids, ack_id)) } /// Clear the write-through-pending mark on the acked nodes. @@ -3068,12 +3404,12 @@ macro_rules! tree_core_binding { node_ids: Vec, ack_id: NodeId, ) -> PyResult<()> { - self.inner.finish_write_through(py, node_ids, ack_id) + catch_native_panic(|| self.inner.finish_write_through(py, node_ids, ack_id)) } /// Clear the in-flight H->D marks on the anchor's root path at ack time. fn finish_load_back(&self, py: Python<'_>, anchor_node_id: NodeId) -> PyResult<()> { - self.inner.finish_load_back(py, anchor_node_id) + catch_native_panic(|| self.inner.finish_load_back(py, anchor_node_id)) } /// Build transfers for a node with no stored or pending external copy. @@ -3082,8 +3418,10 @@ macro_rules! tree_core_binding { py: Python<'_>, node_id: NodeId, ) -> PyResult>>> { - self.inner - .build_external_linker_offload_transfers(py, node_id) + catch_native_panic(|| { + self.inner + .build_external_linker_offload_transfers(py, node_id) + }) } /// Mark an externally restored path, excluding its existing anchor. @@ -3093,11 +3431,10 @@ macro_rules! tree_core_binding { from_node_id: NodeId, until_node_id: NodeId, ) -> PyResult<()> { - self.inner.mark_external_cache_stored_path( - py, - from_node_id, - until_node_id, - ) + catch_native_panic(|| { + self.inner + .mark_external_cache_stored_path(py, from_node_id, until_node_id) + }) } /// Publish an accepted external offload as pending. @@ -3106,8 +3443,7 @@ macro_rules! tree_core_binding { py: Python<'_>, node_id: NodeId, ) -> PyResult<()> { - self.inner - .mark_external_linker_offload_pending(py, node_id) + catch_native_panic(|| self.inner.mark_external_linker_offload_pending(py, node_id)) } /// Finalize external-store state for an offload and its split fragments. @@ -3118,15 +3454,16 @@ macro_rules! tree_core_binding { ack_id: NodeId, success: bool, ) -> PyResult<()> { - self.inner.finish_external_linker_offload( - py, node_ids, ack_id, success, - ) + catch_native_panic(|| { + self.inner + .finish_external_linker_offload(py, node_ids, ack_id, success) + }) } /// Order-sensitive digest of reclaimed coexisting host values. #[pyo3(name = "write_back_duplicate_reclaim_digest")] - fn write_back_coexist_reclaim_digest(&self, py: Python<'_>) -> i64 { - self.inner.write_back_coexist_reclaim_digest(py) + fn write_back_coexist_reclaim_digest(&self, py: Python<'_>) -> PyResult { + catch_native_panic(|| Ok(self.inner.write_back_coexist_reclaim_digest(py))) } /// Whether the component's data is device-evicted but host-backed. @@ -3136,15 +3473,17 @@ macro_rules! tree_core_binding { node_id: NodeId, component_type: u8, ) -> PyResult { - self.inner - .component_has_host_value_only(py, node_id, component_type) + catch_native_panic(|| { + self.inner + .component_has_host_value_only(py, node_id, component_type) + }) } // ==== Test-only inspection surface ==== #[cfg(feature = "inspection")] - fn inspect_contains_node(&self, py: Python<'_>, node_id: NodeId) -> bool { - self.inner.inspect_contains_node(py, node_id) + fn inspect_contains_node(&self, py: Python<'_>, node_id: NodeId) -> PyResult { + catch_native_panic(|| Ok(self.inner.inspect_contains_node(py, node_id))) } #[cfg(feature = "inspection")] @@ -3153,43 +3492,27 @@ macro_rules! tree_core_binding { py: Python<'_>, node_id: NodeId, ) -> PyResult> { - self.inner.inspect_get_parent_node_id(py, node_id) + catch_native_panic(|| self.inner.inspect_get_parent_node_id(py, node_id)) } #[cfg(feature = "inspection")] - fn inspect_get_child_node_ids( - &self, - py: Python<'_>, - node_id: NodeId, - ) -> PyResult> { - self.inner.inspect_get_child_node_ids(py, node_id) + fn inspect_get_child_node_ids(&self, py: Python<'_>, node_id: NodeId) -> PyResult> { + catch_native_panic(|| self.inner.inspect_get_child_node_ids(py, node_id)) } #[cfg(feature = "inspection")] - fn inspect_get_node_key_length( - &self, - py: Python<'_>, - node_id: NodeId, - ) -> PyResult { - self.inner.inspect_get_node_key_length(py, node_id) + fn inspect_get_node_key_length(&self, py: Python<'_>, node_id: NodeId) -> PyResult { + catch_native_panic(|| self.inner.inspect_get_node_key_length(py, node_id)) } #[cfg(feature = "inspection")] - fn inspect_get_node_token_ids( - &self, - py: Python<'_>, - node_id: NodeId, - ) -> PyResult> { - self.inner.inspect_get_node_token_ids(py, node_id) + fn inspect_get_node_token_ids(&self, py: Python<'_>, node_id: NodeId) -> PyResult> { + catch_native_panic(|| self.inner.inspect_get_node_token_ids(py, node_id)) } #[cfg(feature = "inspection")] - fn inspect_is_node_key_bigram( - &self, - py: Python<'_>, - node_id: NodeId, - ) -> PyResult { - self.inner.inspect_is_node_key_bigram(py, node_id) + fn inspect_is_node_key_bigram(&self, py: Python<'_>, node_id: NodeId) -> PyResult { + catch_native_panic(|| self.inner.inspect_is_node_key_bigram(py, node_id)) } #[cfg(feature = "inspection")] @@ -3198,9 +3521,11 @@ macro_rules! tree_core_binding { py: Python<'_>, node_id: NodeId, component_type: u8, - ) -> PyResult> { - self.inner - .inspect_get_component_host_value(py, node_id, component_type) + ) -> PyResult>> { + catch_native_panic(|| { + self.inner + .inspect_get_component_host_value(py, node_id, component_type) + }) } #[cfg(feature = "inspection")] @@ -3210,17 +3535,15 @@ macro_rules! tree_core_binding { node_id: NodeId, component_type: u8, ) -> PyResult { - self.inner - .inspect_get_component_device_lock_ref(py, node_id, component_type) + catch_native_panic(|| { + self.inner + .inspect_get_component_device_lock_ref(py, node_id, component_type) + }) } #[cfg(feature = "inspection")] - fn inspect_get_node_hit_count( - &self, - py: Python<'_>, - node_id: NodeId, - ) -> PyResult { - self.inner.inspect_get_node_hit_count(py, node_id) + fn inspect_get_node_hit_count(&self, py: Python<'_>, node_id: NodeId) -> PyResult { + catch_native_panic(|| self.inner.inspect_get_node_hit_count(py, node_id)) } #[cfg(feature = "inspection")] @@ -3229,17 +3552,12 @@ macro_rules! tree_core_binding { py: Python<'_>, node_id: NodeId, ) -> PyResult> { - self.inner - .inspect_get_write_through_pending_id(py, node_id) + catch_native_panic(|| self.inner.inspect_get_write_through_pending_id(py, node_id)) } #[cfg(feature = "inspection")] - fn inspect_is_external_cache_stored( - &self, - py: Python<'_>, - node_id: NodeId, - ) -> PyResult { - self.inner.inspect_is_external_cache_stored(py, node_id) + fn inspect_is_external_cache_stored(&self, py: Python<'_>, node_id: NodeId) -> PyResult { + catch_native_panic(|| self.inner.inspect_is_external_cache_stored(py, node_id)) } #[cfg(feature = "inspection")] @@ -3249,8 +3567,10 @@ macro_rules! tree_core_binding { node_id: NodeId, component_type: u8, ) -> PyResult { - self.inner - .inspect_is_node_in_device_lru(py, node_id, component_type) + catch_native_panic(|| { + self.inner + .inspect_is_node_in_device_lru(py, node_id, component_type) + }) } #[cfg(feature = "inspection")] @@ -3260,8 +3580,10 @@ macro_rules! tree_core_binding { node_id: NodeId, component_type: u8, ) -> PyResult { - self.inner - .inspect_is_node_in_host_lru(py, node_id, component_type) + catch_native_panic(|| { + self.inner + .inspect_is_node_in_host_lru(py, node_id, component_type) + }) } #[cfg(feature = "inspection")] @@ -3270,40 +3592,30 @@ macro_rules! tree_core_binding { py: Python<'_>, component_type: u8, ) -> PyResult> { - self.inner - .inspect_get_component_device_lru_node_ids(py, component_type) + catch_native_panic(|| { + self.inner + .inspect_get_component_device_lru_node_ids(py, component_type) + }) } #[cfg(feature = "inspection")] - fn inspect_is_device_evictable_leaf( - &self, - py: Python<'_>, - node_id: NodeId, - ) -> bool { - self.inner.inspect_is_device_evictable_leaf(py, node_id) + fn inspect_is_device_evictable_leaf(&self, py: Python<'_>, node_id: NodeId) -> PyResult { + catch_native_panic(|| Ok(self.inner.inspect_is_device_evictable_leaf(py, node_id))) } #[cfg(feature = "inspection")] - fn inspect_is_host_evictable_leaf( - &self, - py: Python<'_>, - node_id: NodeId, - ) -> bool { - self.inner.inspect_is_host_evictable_leaf(py, node_id) + fn inspect_is_host_evictable_leaf(&self, py: Python<'_>, node_id: NodeId) -> PyResult { + catch_native_panic(|| Ok(self.inner.inspect_is_host_evictable_leaf(py, node_id))) } #[cfg(feature = "inspection")] - fn inspect_is_device_leaf( - &self, - py: Python<'_>, - node_id: NodeId, - ) -> PyResult { - self.inner.inspect_is_device_leaf(py, node_id) + fn inspect_is_device_leaf(&self, py: Python<'_>, node_id: NodeId) -> PyResult { + catch_native_panic(|| self.inner.inspect_is_device_leaf(py, node_id)) } #[cfg(feature = "inspection")] - fn inspect_get_all_node_ids(&self, py: Python<'_>) -> Vec { - self.inner.inspect_get_all_node_ids(py) + fn inspect_get_all_node_ids(&self, py: Python<'_>) -> PyResult> { + catch_native_panic(|| Ok(self.inner.inspect_get_all_node_ids(py))) } #[cfg(feature = "inspection")] @@ -3312,8 +3624,10 @@ macro_rules! tree_core_binding { py: Python<'_>, component_type: u8, ) -> PyResult { - self.inner - .inspect_component_protected_size(py, component_type) + catch_native_panic(|| { + self.inner + .inspect_component_protected_size(py, component_type) + }) } #[cfg(feature = "inspection")] @@ -3324,8 +3638,10 @@ macro_rules! tree_core_binding { node_id: NodeId, hash_values: Option>, ) -> PyResult<()> { - self.inner - .inspect_set_node_hash_values(py, node_id, hash_values) + catch_native_panic(|| { + self.inner + .inspect_set_node_hash_values(py, node_id, hash_values) + }) } #[cfg(feature = "inspection")] @@ -3337,12 +3653,10 @@ macro_rules! tree_core_binding { component_type: u8, value: Option, ) -> PyResult<()> { - self.inner.inspect_set_component_device_value_raw( - py, - node_id, - component_type, - value, - ) + catch_native_panic(|| { + self.inner + .inspect_set_component_device_value_raw(py, node_id, component_type, value) + }) } #[cfg(feature = "inspection")] @@ -3354,12 +3668,10 @@ macro_rules! tree_core_binding { component_type: u8, value: Option, ) -> PyResult<()> { - self.inner.inspect_set_component_host_value_raw( - py, - node_id, - component_type, - value, - ) + catch_native_panic(|| { + self.inner + .inspect_set_component_host_value_raw(py, node_id, component_type, value) + }) } #[cfg(feature = "inspection")] @@ -3370,12 +3682,10 @@ macro_rules! tree_core_binding { component_type: u8, lock_ref: u32, ) -> PyResult<()> { - self.inner.inspect_set_component_device_lock_ref( - py, - node_id, - component_type, - lock_ref, - ) + catch_native_panic(|| { + self.inner + .inspect_set_component_device_lock_ref(py, node_id, component_type, lock_ref) + }) } #[cfg(feature = "inspection")] @@ -3385,8 +3695,10 @@ macro_rules! tree_core_binding { node_id: NodeId, component_type: u8, ) -> PyResult<()> { - self.inner - .inspect_remove_node_from_device_lru(py, node_id, component_type) + catch_native_panic(|| { + self.inner + .inspect_remove_node_from_device_lru(py, node_id, component_type) + }) } #[cfg(feature = "inspection")] @@ -3396,8 +3708,10 @@ macro_rules! tree_core_binding { node_id: NodeId, component_type: u8, ) -> PyResult<()> { - self.inner - .inspect_insert_node_into_host_lru(py, node_id, component_type) + catch_native_panic(|| { + self.inner + .inspect_insert_node_into_host_lru(py, node_id, component_type) + }) } #[cfg(feature = "inspection")] @@ -3407,8 +3721,10 @@ macro_rules! tree_core_binding { component_type: u8, value: usize, ) -> PyResult<()> { - self.inner - .inspect_set_component_evictable_size(py, component_type, value) + catch_native_panic(|| { + self.inner + .inspect_set_component_evictable_size(py, component_type, value) + }) } #[cfg(feature = "inspection")] @@ -3418,22 +3734,20 @@ macro_rules! tree_core_binding { component_type: u8, value: usize, ) -> PyResult<()> { - self.inner - .inspect_set_component_protected_size(py, component_type, value) + catch_native_panic(|| { + self.inner + .inspect_set_component_protected_size(py, component_type, value) + }) } #[cfg(feature = "inspection")] - fn inspect_update_duplicate_tracking( - &self, - py: Python<'_>, - node_id: NodeId, - ) -> PyResult<()> { - self.inner.inspect_update_duplicate_tracking(py, node_id) + fn inspect_update_duplicate_tracking(&self, py: Python<'_>, node_id: NodeId) -> PyResult<()> { + catch_native_panic(|| self.inner.inspect_update_duplicate_tracking(py, node_id)) } #[cfg(feature = "inspection")] fn inspect_advance_insert_walk_once(&self, py: Python<'_>) -> PyResult<()> { - self.inner.inspect_advance_insert_walk_once(py) + catch_native_panic(|| self.inner.inspect_advance_insert_walk_once(py)) } #[cfg(feature = "inspection")] @@ -3444,8 +3758,10 @@ macro_rules! tree_core_binding { component_type: u8, target: u8, ) -> PyResult { - self.inner - .inspect_evict_component(py, node_id, component_type, target) + catch_native_panic(|| { + self.inner + .inspect_evict_component(py, node_id, component_type, target) + }) } #[cfg(feature = "inspection")] @@ -3456,8 +3772,10 @@ macro_rules! tree_core_binding { component_type: u8, target: u8, ) -> PyResult<()> { - self.inner - .inspect_validate_cascade_evict(py, node_id, component_type, target) + catch_native_panic(|| { + self.inner + .inspect_validate_cascade_evict(py, node_id, component_type, target) + }) } #[cfg(feature = "inspection")] @@ -3466,8 +3784,7 @@ macro_rules! tree_core_binding { py: Python<'_>, node_id: NodeId, ) -> PyResult { - self.inner - .inspect_cleanup_tombstone_ancestors(py, node_id) + catch_native_panic(|| self.inner.inspect_cleanup_tombstone_ancestors(py, node_id)) } #[cfg(feature = "inspection")] @@ -3484,16 +3801,18 @@ macro_rules! tree_core_binding { value_chunks: Vec, best_value_len: usize, ) -> PyResult { - self.inner.inspect_finalize_component_match_result( - py, - component_type, - result, - key, - extra_key, - cache_salt, - value_chunks, - best_value_len, - ) + catch_native_panic(|| { + self.inner.inspect_finalize_component_match_result( + py, + component_type, + result, + key, + extra_key, + cache_salt, + value_chunks, + best_value_len, + ) + }) } #[cfg(feature = "inspection")] @@ -3504,13 +3823,18 @@ macro_rules! tree_core_binding { node_id: NodeId, write_back: bool, ) -> PyResult> { - self.inner - .inspect_build_backup_node_ids(py, node_id, write_back) + catch_native_panic(|| { + self.inner + .inspect_build_backup_node_ids(py, node_id, write_back) + }) } /// Print the tree structure for debugging. - fn pretty_print(&self, py: Python<'_>) { - self.inner.pretty_print(py) + fn pretty_print(&self, py: Python<'_>) -> PyResult<()> { + catch_native_panic(|| { + self.inner.pretty_print(py); + Ok(()) + }) } } }; @@ -3539,36 +3863,39 @@ fn get_hash_str( page_size: usize, is_bigram: bool, ) -> PyResult> { - let raw = py_array_to_vec_i64(py, token_ids)?; - if page_size == 0 { - return Err(PyValueError::new_err("page_size must be positive")); - } - if let Some(prior_hash) = prior_hash.as_deref().filter(|hash| !hash.is_empty()) - && (prior_hash.len() != 64 || !prior_hash.bytes().all(|byte| byte.is_ascii_hexdigit())) - { - return Err(PyValueError::new_err( - "prior_hash must be a 64-character hexadecimal digest", - )); - } - if let Some(token_id) = raw - .iter() - .find(|token_id| u32::try_from(**token_id).is_err()) - { - return Err(PyValueError::new_err(format!( - "token id {token_id} does not fit in uint32" - ))); - } - Ok(py.allow_threads(move || { - if is_bigram { - let key = as ChildKeyType>::key_from(Cow::Owned(raw)).into_owned(); - crate::node::get_hash_str::>(&key, prior_hash.as_deref(), page_size) - } else { - crate::node::get_hash_str::>(&raw, prior_hash.as_deref(), page_size) + catch_native_panic(|| { + let raw = py_array_to_vec_i64(py, token_ids)?; + if page_size == 0 { + return Err(PyValueError::new_err("page_size must be positive")); } - })) + if let Some(prior_hash) = prior_hash.as_deref().filter(|hash| !hash.is_empty()) + && (prior_hash.len() != 64 || !prior_hash.bytes().all(|byte| byte.is_ascii_hexdigit())) + { + return Err(PyValueError::new_err( + "prior_hash must be a 64-character hexadecimal digest", + )); + } + if let Some(token_id) = raw + .iter() + .find(|token_id| u32::try_from(**token_id).is_err()) + { + return Err(PyValueError::new_err(format!( + "token id {token_id} does not fit in uint32" + ))); + } + Ok(py.allow_threads(move || { + if is_bigram { + let key = as ChildKeyType>::key_from(Cow::Owned(raw)).into_owned(); + crate::node::get_hash_str::>(&key, prior_hash.as_deref(), page_size) + } else { + crate::node::get_hash_str::>(&raw, prior_hash.as_deref(), page_size) + } + })) + }) } fn register_mem_cache_module(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_class::()?; m.add_function(wrap_pyfunction!(get_hash_str, m)?)?; m.add_class::()?; m.add_class::()?; diff --git a/rust/sglang-radix-tree/src/tests/components/base.rs b/rust/sglang-radix-tree/src/tests/components/base.rs index c835597c2e..cddae0bb04 100644 --- a/rust/sglang-radix-tree/src/tests/components/base.rs +++ b/rust/sglang-radix-tree/src/tests/components/base.rs @@ -89,6 +89,7 @@ fn insert_overlap_default_consumes_nothing() { /* total_prefix_len = */ 0, Tensor::from_slice(&[0i64, 1, 2]), &InsertParams { + rotation_base: None, key: &vec![0, 1, 2], namespace: Default::default(), value: Tensor::from_slice(&[0i64, 1, 2]), diff --git a/rust/sglang-radix-tree/src/tests/components/full.rs b/rust/sglang-radix-tree/src/tests/components/full.rs index 956b97808e..1f4e51967f 100644 --- a/rust/sglang-radix-tree/src/tests/components/full.rs +++ b/rust/sglang-radix-tree/src/tests/components/full.rs @@ -66,6 +66,7 @@ fn match_params(key: &Vec) -> MatchPrefixParams<'_, Vec> { fn insert(tc: &mut UnifiedTreeCore>, key: &Vec, value: &[i64]) { tc.insert(&crate::unified_tree_core::InsertParams { + rotation_base: None, key, namespace: Default::default(), value: Tensor::from_slice(value), @@ -405,6 +406,10 @@ fn host_drive_reclaims_coexisting_host_values_while_sparing_the_device_leaf() { .expect("live test node"); tc.commit_backup(leaf_handle, Tensor::from_slice(&[22i64]), HashMap::new()) .expect("live test node"); + for handle in [tc.arena.node(parent).id, leaf_handle] { + tc.mark_write_through_pending(vec![handle], handle).unwrap(); + tc.finish_write_through(vec![handle], handle).unwrap(); + } assert!(tc.evictable_host_leaves.is_empty()); let (mut tr, mut df, mut hf) = (tracker(), frees(), frees()); @@ -422,6 +427,118 @@ fn host_drive_reclaims_coexisting_host_values_while_sparing_the_device_leaf() { tc.sanity_check(&[], &[]); } +#[test] +fn host_reclaim_keeps_insertion_order_across_calls_and_rebackup() { + let mut tc = write_back_core(); + let mut handles = Vec::new(); + for token in 1..=3i64 { + let key = vec![token]; + insert(&mut tc, &key, &[token + 100]); + let handle = tc.match_prefix(&match_params(&key)).best_match_node_id; + tc.commit_backup(handle, Tensor::from_slice(&[token + 1000]), HashMap::new()) + .expect("live test node"); + tc.mark_write_through_pending(vec![handle], handle).unwrap(); + tc.finish_write_through(vec![handle], handle).unwrap(); + handles.push(handle); + } + + for expected_host_slot in [1001, 1002, 1003, 2001] { + let step = tc.drive_host_eviction(FULL, 1); + assert_eq!(step.tracker[&FULL], 1); + assert!(step.device_frees.is_empty()); + assert_eq!(step.host_frees[&FULL].len(), 1); + assert_eq!( + step.host_frees[&FULL][0].int64_value(&[0]), + expected_host_slot + ); + if expected_host_slot == 1001 { + // A new host copy of the reclaimed slot joins behind the surviving + // duplicates. Refreshing another settled member must not move it. + tc.commit_backup(handles[0], Tensor::from_slice(&[2001i64]), HashMap::new()) + .unwrap(); + tc.mark_write_through_pending(vec![handles[0]], handles[0]) + .unwrap(); + tc.finish_write_through(vec![handles[0]], handles[0]) + .unwrap(); + tc.mark_write_through_pending(vec![handles[1]], handles[1]) + .unwrap(); + tc.finish_write_through(vec![handles[1]], handles[1]) + .unwrap(); + } + tc.sanity_check(&[], &[]); + } + assert!(tc.full_coexisting_host_nodes.iter().next().is_none()); +} + +#[test] +fn host_reclaim_orders_pending_internal_split_fragments_by_ack() { + let mut tc = write_back_core(); + insert(&mut tc, &vec![1, 2, 3, 4], &[10, 11, 12, 13]); + insert(&mut tc, &vec![1, 2, 3, 4, 5], &[10, 11, 12, 13, 14]); + insert(&mut tc, &vec![7, 8], &[17, 18]); + let a = tc + .match_prefix(&match_params(&vec![1, 2, 3, 4])) + .best_match_node_id; + let b = tc + .match_prefix(&match_params(&vec![7, 8])) + .best_match_node_id; + for (handle, slots) in [(a, vec![20i64, 21, 22, 23]), (b, vec![27, 28])] { + tc.commit_backup(handle, Tensor::from_slice(&slots), HashMap::new()) + .unwrap(); + tc.inc_lock_ref(handle, ComponentSet::EMPTY).unwrap(); + tc.mark_write_through_pending(vec![handle], handle).unwrap(); + } + // Incremental SWA backup can submit an unbacked Full ancestor + // asynchronously under write_back; matching can split it before the ack. + let (prefix_idx, action) = tc.split_node_(tc.arena.resolve(a).unwrap(), 2); + let published_a = match action { + Some(CacheAction::ReplaceWriteThroughOnNodeSplit { + ack_id, + old_node_id, + new_node_id, + new_child_node_id, + }) => { + assert_eq!((ack_id, old_node_id, new_child_node_id), (a, a, a)); + assert_eq!(new_node_id, tc.arena.node(prefix_idx).id); + vec![new_node_id, new_child_node_id] + } + _ => panic!("expected pending-backup split publication"), + }; + tc.sanity_check(&[(a as i64, a), (b as i64, b)], &[]); + + // The controller publishes each FIFO ack's fragments ancestors first, + // then releases the original node's lock (which also covers its prefix). + for (ack, published) in [(a, published_a), (b, vec![b])] { + tc.finish_write_through(published, ack).unwrap(); + tc.dec_lock_ref( + ack, + &DecLockRefParams { + node_id: Some(ack), + ..Default::default() + }, + false, + ) + .unwrap(); + } + assert!(!tc.evictable_device_leaves.contains(prefix_idx)); + assert!( + !tc.arena + .node(tc.arena.resolve(a).unwrap()) + .children + .is_empty() + ); + for expected in [[20i64, 21], [22, 23], [27, 28]] { + let step = tc.drive_host_eviction(FULL, 1); + assert_eq!(step.tracker[&FULL], 2); + assert!(step.device_frees.is_empty()); + assert_eq!(step.host_frees[&FULL].len(), 1); + let slots = &step.host_frees[&FULL][0]; + assert_eq!([slots.int64_value(&[0]), slots.int64_value(&[1])], expected); + tc.sanity_check(&[], &[]); + } + assert!(tc.full_coexisting_host_nodes.iter().next().is_none()); +} + #[test] fn host_drive_spares_coexisting_host_values_under_an_in_flight_transfer() { let mut tc = write_back_core(); @@ -431,8 +548,12 @@ fn host_drive_spares_coexisting_host_values_under_an_in_flight_transfer() { .best_match_node_id; tc.commit_backup(handle, Tensor::from_slice(&[20i64, 21]), HashMap::new()) .expect("live test node"); + // commit_backup precedes the controller's pending marker; membership is + // born only when the transfer is acknowledged, not in this short gap. + assert!(tc.full_coexisting_host_nodes.iter().next().is_none()); tc.mark_write_through_pending(vec![handle], /* ack_id = */ handle) .expect("live test node"); + assert!(tc.full_coexisting_host_nodes.iter().next().is_none()); let (mut tr, mut df, mut hf) = (tracker(), frees(), frees()); accumulate_step( @@ -450,6 +571,10 @@ fn host_drive_spares_coexisting_host_values_under_an_in_flight_transfer() { tc.finish_write_through(vec![handle], handle) .expect("live test node"); + assert!( + tc.full_coexisting_host_nodes + .contains(tc.arena.resolve(handle).unwrap()) + ); accumulate_step( tc.drive_host_eviction(FULL, /* num_tokens = */ 2), &mut tr, @@ -469,6 +594,7 @@ fn host_drive_spares_coexisting_host_values_under_an_in_flight_transfer() { fn host_drive_is_a_noop_without_host_leaves() { let mut tc = core(); tc.insert(&crate::unified_tree_core::InsertParams { + rotation_base: None, key: &vec![1, 2], namespace: Default::default(), value: Tensor::from_slice(&[10i64, 11]), diff --git a/rust/sglang-radix-tree/src/tests/components/mamba.rs b/rust/sglang-radix-tree/src/tests/components/mamba.rs index 6e59aebd3c..66b87ec6b9 100644 --- a/rust/sglang-radix-tree/src/tests/components/mamba.rs +++ b/rust/sglang-radix-tree/src/tests/components/mamba.rs @@ -59,6 +59,7 @@ fn insert_params_mamba<'k>( mamba_slot: Option, ) -> InsertParams<'k, Vec> { InsertParams { + rotation_base: None, key, namespace: Default::default(), value: Tensor::from_slice(value), @@ -882,6 +883,374 @@ fn device_walk_skips_locked_nodes() { tc.evict_device_end(MAMBA); } +fn internal_write_back_fixture() -> (UnifiedTreeCore>, NodeId, NodeId) { + let mut tc = mamba_core(/* page_size = */ 1); + tc.set_hicache_enabled(); + tc.is_write_back = true; + tc.insert(&insert_params_mamba(&vec![1], &[10], Some(7))); + tc.insert(&insert_params_mamba(&vec![1, 2], &[10, 11], Some(8))); + let parent = tc.match_prefix(&match_params(&vec![1])).best_match_node_id; + let leaf = tc + .match_prefix(&match_params(&vec![1, 2])) + .best_match_node_id; + (tc, parent, leaf) +} + +fn request_internal_mamba_backup(tc: &mut UnifiedTreeCore>, parent: NodeId) { + tc.evict_device_start(MAMBA, 2); + let (leaf, step) = tc.evict_device_next_node(MAMBA, &HashMap::new()); + assert_eq!(leaf, None); + assert_eq!(step.mamba_backup_node_id, Some(parent)); + assert!(step.tracker.is_empty()); + assert!(step.device_frees.is_empty()); + assert!(step.host_frees.is_empty()); + assert_eq!(step.unbacked_tokens, 0); + let node = tc.arena.node(tc.arena.resolve(parent).unwrap()); + assert!(node.device_value(FULL).equal(&Tensor::from_slice(&[10i64]))); + assert!(node.device_value(MAMBA).equal(&Tensor::from_slice(&[7i64]))); + assert_eq!(tc.mamba_evictable_size(), 2); +} + +fn check_mamba_restored_during_expanded_swa_backup(pin_ancestor: bool) { + let mut tc = UnifiedTreeCore::new( + CacheInitParams { + page_size: 1, + swa_sliding_window_size: Some(2), + mamba_cache_chunk_size: Some(256), + enable_hicache: true, + has_swa_host_pool: true, + is_write_back: true, + ..Default::default() + }, + vec![FULL, SWA, MAMBA], + ); + let [a, b, c] = chain::<3>(&mut tc); + let [a_id, b_id, c_id] = [a, b, c].map(|idx| tc.arena.node(idx).id); + // State after SWA eviction and repair: Full backups survived, SWA A/B + // needs an expanded backup, and the old Mamba A was cascade-evicted. + for (idx, full, swa) in [(a, 10i64, 20i64), (b, 11, 21), (c, 12, 22)] { + let id = tc.arena.node(idx).id; + tc.arena + .set_device_value(idx, FULL, Tensor::from_slice(&[full])); + tc.inc_evictable_size(FULL, 1); + tc.set_component_device_value(id, SWA, Tensor::from_slice(&[swa])) + .unwrap(); + set_full_host(&mut tc, idx, full + 1000); + tc.mark_write_through_pending(vec![id], id).unwrap(); + tc.finish_write_through(vec![id], id).unwrap(); + tc.update_evictable_leaf_sets_(idx); + } + set_mamba_device(&mut tc, b, 202); + set_mamba_device(&mut tc, c, 203); + let (full, mut transfers) = tc.build_backup_spec(b_id).unwrap(); + assert_eq!(full.numel(), 0); + assert_eq!(transfers[&SWA][0].nodes_to_load, Some(vec![a_id, b_id])); + assert!( + transfers[&MAMBA][0] + .device_indices + .as_ref() + .unwrap() + .equal(&Tensor::from_slice(&[202i64])) + ); + for xfers in transfers.values_mut() { + for transfer in xfers { + transfer.host_indices = Some(transfer.device_indices.as_ref().unwrap() + 1000); + } + } + tc.commit_backup(b_id, full, transfers).unwrap(); + let release_params = |receipt: IncLockRefResult| receipt.to_dec_params(); + let backup_lock = release_params(tc.inc_lock_ref(b_id, ComponentSet::EMPTY).unwrap()); + tc.mark_write_through_pending(vec![a_id, b_id], b_id) + .unwrap(); + assert!(!tc.arena.has_device_value(a, MAMBA)); + + // A separate request finishes A while B's transfer is pending. Its new + // Mamba checkpoint is not in the already-submitted DMA source tensor. + let inserted = tc.insert(&insert_params_mamba(&vec![1], &[90], Some(201))); + assert!( + !inserted + .cache_actions + .iter() + .any(|action| matches!(action, CacheAction::BackupKV(_))) + ); + assert!( + tc.arena + .device_value(a, MAMBA) + .equal(&Tensor::from_slice(&[201i64])) + ); + assert_eq!(tc.arena.node(a).write_through_pending_id, Some(b_id)); + assert_eq!(tc.arena.device_lock_ref(a, MAMBA), 0); + assert_eq!(tc.arena.device_lock_ref(b, MAMBA), 1); + assert_eq!(tc.arena.device_lock_ref(a, SWA), 1); + + let child_lock = release_params(tc.inc_lock_ref(c_id, ComponentSet::EMPTY).unwrap()); + let ancestor_lock = + pin_ancestor.then(|| release_params(tc.inc_lock_ref(a_id, ComponentSet::EMPTY).unwrap())); + let mut active = vec![(b_id as i64, b_id), (c_id as i64, c_id)]; + if pin_ancestor { + active.push((a_id as i64, a_id)); + } + tc.sanity_check(&active, &[]); + + tc.evict_device_start(MAMBA, 1); + let (next, step) = tc.evict_device_next_node(MAMBA, &HashMap::new()); + tc.evict_device_end(MAMBA); + assert_eq!(next, None); + assert_eq!(step.mamba_backup_node_id, None); + assert_eq!( + step.tracker.get(&MAMBA).copied().unwrap_or(0), + usize::from(!pin_ancestor) + ); + assert!(step.host_frees.is_empty()); + if pin_ancestor { + assert!(step.device_frees.is_empty()); + } else { + assert_eq!(step.device_frees.len(), 1); + assert_eq!(step.device_frees[&MAMBA].len(), 1); + assert!(step.device_frees[&MAMBA][0].equal(&Tensor::from_slice(&[201i64]))); + } + assert!( + tc.arena + .device_value(b, MAMBA) + .equal(&Tensor::from_slice(&[202i64])) + ); + tc.sanity_check(&active, &[]); + + tc.finish_write_through(vec![a_id, b_id], b_id).unwrap(); + tc.dec_lock_ref(b_id, &backup_lock, false).unwrap(); + tc.evict_device_start(MAMBA, 1); + let (next, step) = tc.evict_device_next_node(MAMBA, &HashMap::new()); + tc.evict_device_end(MAMBA); + assert_eq!(next, None); + assert_eq!(step.mamba_backup_node_id, None); + assert_eq!(step.tracker[&MAMBA], 1); + assert_eq!(step.device_frees.len(), 1); + assert_eq!(step.device_frees[&MAMBA].len(), 1); + assert!(step.device_frees[&MAMBA][0].equal(&Tensor::from_slice(&[202i64]))); + assert!(step.host_frees.is_empty()); + for (idx, full, swa) in [(a, 10i64, 20i64), (b, 11, 21), (c, 12, 22)] { + assert!( + tc.arena + .device_value(idx, FULL) + .equal(&Tensor::from_slice(&[full])) + ); + assert!( + tc.arena + .device_value(idx, SWA) + .equal(&Tensor::from_slice(&[swa])) + ); + } + tc.dec_lock_ref(c_id, &child_lock, false).unwrap(); + if let Some(params) = ancestor_lock { + tc.dec_lock_ref(a_id, ¶ms, false).unwrap(); + } + tc.sanity_check(&[], &[]); +} + +#[test] +fn mamba_device_eviction_reclaims_state_restored_during_swa_backup() { + check_mamba_restored_during_expanded_swa_backup(false); +} + +#[test] +fn mamba_device_eviction_preserves_pins_during_expanded_swa_backup() { + check_mamba_restored_during_expanded_swa_backup(true); +} + +fn commit_internal_mamba_backup(tc: &mut UnifiedTreeCore>, parent: NodeId) { + let (full, mut transfers) = tc.build_backup_spec(parent).unwrap(); + assert!(full.equal(&Tensor::from_slice(&[10i64]))); + transfers.get_mut(&MAMBA).unwrap()[0].host_indices = Some(Tensor::from_slice(&[70i64])); + tc.commit_backup(parent, Tensor::from_slice(&[100i64]), transfers) + .unwrap(); + tc.mark_write_through_pending(vec![parent], parent).unwrap(); + tc.finish_write_through(vec![parent], parent).unwrap(); +} + +#[test] +fn internal_write_back_defers_mamba_free_until_backup_then_keeps_full_and_host_state() { + let (mut tc, parent, leaf) = internal_write_back_fixture(); + request_internal_mamba_backup(&mut tc, parent); + commit_internal_mamba_backup(&mut tc, parent); + let step = tc.finish_mamba_state_eviction(parent); + assert_eq!(step.tracker, HashMap::from([(MAMBA, 1)])); + assert_eq!(step.device_frees.len(), 1); + assert!(step.device_frees[&MAMBA][0].equal(&Tensor::from_slice(&[7i64]))); + assert!(step.host_frees.is_empty()); + assert_eq!(step.unbacked_tokens, 0); + let node = tc.arena.node(tc.arena.resolve(parent).unwrap()); + assert!(node.device_value(FULL).equal(&Tensor::from_slice(&[10i64]))); + assert!(node.host_value(FULL).equal(&Tensor::from_slice(&[100i64]))); + assert!(node.host_value(MAMBA).equal(&Tensor::from_slice(&[70i64]))); + assert!(!node.has_device_value(MAMBA)); + let matched = tc.match_prefix(&match_params(&vec![1])); + assert_eq!(matched.best_match_node_id, parent); + assert_eq!(matched.mamba_host_hit_length, 1); + let (next, step) = tc.evict_device_next_node(MAMBA, &HashMap::from([(MAMBA, 1)])); + assert_eq!(next, Some(leaf)); + assert!(step.mamba_backup_node_id.is_none()); + tc.evict_device_end(MAMBA); + tc.sanity_check(&[], &[]); +} + +#[test] +fn internal_write_back_failed_backup_drops_only_mamba_and_advances() { + let (mut tc, parent, leaf) = internal_write_back_fixture(); + request_internal_mamba_backup(&mut tc, parent); + let step = tc.finish_mamba_state_eviction(parent); + assert_eq!(step.tracker, HashMap::from([(MAMBA, 1)])); + assert_eq!(step.device_frees.len(), 1); + let node = tc.arena.node(tc.arena.resolve(parent).unwrap()); + assert!(node.has_device_value(FULL)); + assert!(!node.has_device_value(MAMBA)); + assert!(!node.has_host_value(MAMBA)); + let (next, _) = tc.evict_device_next_node(MAMBA, &HashMap::from([(MAMBA, 1)])); + assert_eq!(next, Some(leaf)); + tc.evict_device_end(MAMBA); + tc.sanity_check(&[], &[]); +} + +#[test] +fn internal_mamba_backup_is_limited_to_unbacked_hicache_write_back() { + for mode in ["write_through", "no_hicache", "already_backed"] { + let (mut tc, parent, _) = internal_write_back_fixture(); + match mode { + "write_through" => tc.is_write_back = false, + "no_hicache" => tc.enable_hicache = false, + "already_backed" => commit_internal_mamba_backup(&mut tc, parent), + _ => unreachable!(), + } + tc.evict_device_start(MAMBA, 1); + let (leaf, step) = tc.evict_device_next_node(MAMBA, &HashMap::new()); + assert_eq!(leaf, None); + assert_eq!(step.mamba_backup_node_id, None); + assert_eq!(step.tracker, HashMap::from([(MAMBA, 1)])); + let node = tc.arena.node(tc.arena.resolve(parent).unwrap()); + assert!(node.has_device_value(FULL)); + assert!(!node.has_device_value(MAMBA)); + tc.evict_device_end(MAMBA); + } +} + +#[test] +fn internal_mamba_resume_preserves_a_new_lock_or_inflight_transfer() { + for guard in ["lock", "load_back", "backup"] { + let (mut tc, parent, leaf) = internal_write_back_fixture(); + request_internal_mamba_backup(&mut tc, parent); + let idx = tc.arena.resolve(parent).unwrap(); + match guard { + "lock" | "backup" => { + mamba_component().acquire_component_lock( + &mut tc, + idx, + IncLockRefResult::default(), + false, + ); + if guard == "backup" { + // Actual Mamba DMA sources carry their own component lock. + tc.arena.node_mut(idx).write_through_pending_id = Some(parent); + } + } + "load_back" => tc.arena.node_mut(idx).load_back_pending_id = Some(parent), + _ => unreachable!(), + } + let step = tc.finish_mamba_state_eviction(parent); + assert!(step.tracker.is_empty()); + assert!(step.device_frees.is_empty()); + assert!(tc.arena.has_device_value(idx, MAMBA)); + assert!(tc.arena.has_device_value(idx, FULL)); + let (next, _) = tc.evict_device_next_node(MAMBA, &HashMap::new()); + assert_eq!(next, Some(leaf)); + tc.evict_device_end(MAMBA); + } +} + +#[test] +fn internal_mamba_resume_ignores_an_unrelated_backup_marker() { + let (mut tc, parent, leaf) = internal_write_back_fixture(); + request_internal_mamba_backup(&mut tc, parent); + let idx = tc.arena.resolve(parent).unwrap(); + tc.arena.node_mut(idx).write_through_pending_id = Some(leaf); + let step = tc.finish_mamba_state_eviction(parent); + assert_eq!(step.tracker, HashMap::from([(MAMBA, 1)])); + assert!(step.device_frees[&MAMBA][0].equal(&Tensor::from_slice(&[7i64]))); + assert!(!tc.arena.has_device_value(idx, MAMBA)); + assert!(tc.arena.has_device_value(idx, FULL)); + assert_eq!(tc.arena.node(idx).write_through_pending_id, Some(leaf)); + tc.evict_device_end(MAMBA); + tc.arena.node_mut(idx).write_through_pending_id = None; + tc.sanity_check(&[], &[]); +} + +#[test] +fn internal_mamba_abandonment_and_repeated_calls_never_free_twice() { + use std::panic::{AssertUnwindSafe, catch_unwind}; + + let (mut tc, parent, leaf) = internal_write_back_fixture(); + request_internal_mamba_backup(&mut tc, parent); + assert!( + catch_unwind(AssertUnwindSafe(|| { + tc.evict_device_next_node(MAMBA, &HashMap::new()); + })) + .is_err() + ); + assert!( + catch_unwind(AssertUnwindSafe(|| { + tc.finish_mamba_state_eviction(leaf); + })) + .is_err() + ); + tc.evict_device_end(MAMBA); + assert!( + tc.arena + .has_device_value(tc.arena.resolve(parent).unwrap(), MAMBA) + ); + request_internal_mamba_backup(&mut tc, parent); + assert_eq!(tc.finish_mamba_state_eviction(parent).tracker[&MAMBA], 1); + assert!( + catch_unwind(AssertUnwindSafe(|| { + tc.finish_mamba_state_eviction(parent); + })) + .is_err() + ); + tc.evict_device_end(MAMBA); + tc.sanity_check(&[], &[]); +} + +#[test] +fn internal_mamba_resume_reselects_a_node_that_became_a_leaf() { + let (mut tc, parent, leaf) = internal_write_back_fixture(); + request_internal_mamba_backup(&mut tc, parent); + tc.evict_device_leaf(leaf, false).unwrap(); + let step = tc.finish_mamba_state_eviction(parent); + assert!(step.tracker.is_empty()); + assert!(step.device_frees.is_empty()); + let (next, _) = tc.evict_device_next_node(MAMBA, &HashMap::new()); + assert_eq!(next, Some(parent)); + assert!( + tc.arena + .has_device_value(tc.arena.resolve(parent).unwrap(), FULL) + ); + tc.evict_device_end(MAMBA); + tc.sanity_check(&[], &[]); +} + +#[test] +fn internal_mamba_resume_ignores_a_removed_node_and_repairs_the_cursor() { + let (mut tc, parent, leaf) = internal_write_back_fixture(); + request_internal_mamba_backup(&mut tc, parent); + tc.evict_device_leaf(leaf, false).unwrap(); + tc.evict_device_leaf(parent, false).unwrap(); + let step = tc.finish_mamba_state_eviction(parent); + assert!(step.tracker.is_empty()); + assert!(step.device_frees.is_empty()); + let (next, step) = tc.evict_device_next_node(MAMBA, &HashMap::new()); + assert_eq!(next, None); + assert!(step.tracker.is_empty()); + tc.evict_device_end(MAMBA); + tc.sanity_check(&[], &[]); +} + #[test] #[should_panic(expected = "Mamba device eviction not started")] fn device_walk_requires_a_start() { @@ -1275,7 +1644,7 @@ fn load_back_commit_moves_the_node_onto_the_device_tier() { } #[test] -fn mamba_device_eviction_skips_a_load_back_pinned_node() { +fn mamba_device_eviction_preserves_a_locked_load_back_destination() { let mut tc = mamba_core(/* page_size = */ 1); tc.is_write_back = true; let [n] = chain::<1>(&mut tc); @@ -1292,6 +1661,9 @@ fn mamba_device_eviction_skips_a_load_back_pinned_node() { comp_xfers, ) .expect("live test node"); + let receipt = tc + .inc_lock_ref(tc.arena.node(n).id, ComponentSet::EMPTY) + .expect("live test node"); tc.evict_device_start(MAMBA, /* request_cnt = */ 1); let (next, _) = tc.evict_device_next_node(MAMBA, &HashMap::new()); @@ -1299,6 +1671,8 @@ fn mamba_device_eviction_skips_a_load_back_pinned_node() { tc.evict_device_end(MAMBA); assert!(tc.arena.has_device_value(n, MAMBA)); + tc.dec_lock_ref(tc.arena.node(n).id, &receipt.to_dec_params(), false) + .expect("live test node"); tc.finish_load_back(tc.arena.node(n).id) .expect("live test node"); tc.evict_device_start(MAMBA, /* request_cnt = */ 1); @@ -1307,41 +1681,214 @@ fn mamba_device_eviction_skips_a_load_back_pinned_node() { tc.evict_device_end(MAMBA); } -#[test] -fn mamba_host_eviction_skips_a_load_back_pinned_node() { +fn check_mamba_device_restored_during_full_load_back(pin_ancestor: bool) { let mut tc = mamba_core(/* page_size = */ 1); tc.is_write_back = true; + tc.enable_hicache = true; let [a, b] = chain::<2>(&mut tc); + let [a_id, b_id] = [a, b].map(|idx| tc.arena.node(idx).id); + for (idx, full, mamba) in [(a, 10, 20), (b, 11, 21)] { + set_full_host(&mut tc, idx, full); + set_mamba_host(&mut tc, idx, mamba); + tc.host_lru_list_mut(MAMBA).insert_mru(idx); + tc.update_evictable_leaf_sets_(idx); + } + let release_params = |receipt: IncLockRefResult| receipt.to_dec_params(); + let host_lock = release_params(tc.inc_host_lock_ref(b_id).unwrap()); + let temporary_lock = release_params(tc.inc_lock_ref(b_id, ComponentSet::EMPTY).unwrap()); + let (full, mut transfers) = tc.build_load_back_spec(b_id, None).unwrap(); + assert_eq!(full.nodes_to_load, Some(vec![a_id, b_id])); + assert_eq!(transfers[&MAMBA][0].nodes_to_load, Some(vec![b_id])); + transfers.get_mut(&MAMBA).unwrap()[0].device_indices = Some(Tensor::from_slice(&[40i64])); + tc.dec_lock_ref(b_id, &temporary_lock, false).unwrap(); + tc.commit_load_back(b_id, Tensor::from_slice(&[30i64, 31]), full, transfers) + .unwrap(); + let pending_lock = release_params(tc.inc_lock_ref(b_id, ComponentSet::EMPTY).unwrap()); + assert!(!tc.arena.has_device_value(a, MAMBA)); + + // A separate request restores A after the transfer selected only B's + // Mamba destination. A's Full row is still part of that pending load. + tc.insert(&insert_params_mamba(&vec![1], &[90], Some(41))); + assert!( + tc.arena + .device_value(a, MAMBA) + .equal(&Tensor::from_slice(&[41i64])) + ); + assert!(tc.arena.node(a).is_load_back_pending()); + assert_eq!(tc.arena.device_lock_ref(a, MAMBA), 0); + assert_eq!(tc.arena.device_lock_ref(b, MAMBA), 1); + let ancestor_lock = + pin_ancestor.then(|| release_params(tc.inc_lock_ref(a_id, ComponentSet::EMPTY).unwrap())); + let active: Vec<_> = ancestor_lock.iter().map(|_| (a_id as i64, a_id)).collect(); + tc.sanity_check(&active, &[(b_id as i64, b_id)]); + + tc.evict_device_start(MAMBA, 1); + let (next, step) = tc.evict_device_next_node(MAMBA, &HashMap::new()); + tc.evict_device_end(MAMBA); + assert_eq!(next, None); + assert_eq!(step.mamba_backup_node_id, None); + assert_eq!( + step.tracker.get(&MAMBA).copied().unwrap_or(0), + usize::from(!pin_ancestor) + ); + assert!(step.host_frees.is_empty()); + if pin_ancestor { + assert!(step.device_frees.is_empty()); + } else { + assert_eq!(step.device_frees.len(), 1); + assert_eq!(step.device_frees[&MAMBA].len(), 1); + assert!(step.device_frees[&MAMBA][0].equal(&Tensor::from_slice(&[41i64]))); + } + assert_eq!(tc.arena.has_device_value(a, MAMBA), pin_ancestor); + assert!( + tc.arena + .device_value(b, MAMBA) + .equal(&Tensor::from_slice(&[40i64])) + ); + assert!( + tc.arena + .device_value(a, FULL) + .equal(&Tensor::from_slice(&[30i64])) + ); + assert!( + tc.arena + .device_value(b, FULL) + .equal(&Tensor::from_slice(&[31i64])) + ); + tc.sanity_check(&active, &[(b_id as i64, b_id)]); + + // The controller synchronizes the DMA before releasing this receipt. + tc.dec_lock_ref(b_id, &pending_lock, false).unwrap(); + tc.dec_host_lock_ref(b_id, &host_lock).unwrap(); + tc.finish_load_back(b_id).unwrap(); + tc.evict_device_start(MAMBA, 1); + let (next, _) = tc.evict_device_next_node(MAMBA, &HashMap::new()); + tc.evict_device_end(MAMBA); + assert_eq!(next, Some(b_id)); + if let Some(params) = ancestor_lock { + tc.dec_lock_ref(a_id, ¶ms, false).unwrap(); + } + tc.sanity_check(&[], &[]); +} + +#[test] +fn mamba_device_eviction_reclaims_state_restored_during_full_load_back() { + check_mamba_device_restored_during_full_load_back(false); +} + +#[test] +fn mamba_device_eviction_preserves_pins_during_full_load_back() { + check_mamba_device_restored_during_full_load_back(true); +} + +fn check_mamba_host_eviction_during_full_load_back(lock_ancestor_host: bool) { + let mut tc = mamba_core(/* page_size = */ 1); + tc.is_write_back = true; + tc.enable_hicache = true; + let [a, b] = chain::<2>(&mut tc); + let a_id = tc.arena.node(a).id; + let b_id = tc.arena.node(b).id; set_full_host(&mut tc, a, 10); set_full_host(&mut tc, b, 11); set_mamba_host(&mut tc, a, 20); + set_mamba_host(&mut tc, b, 21); tc.host_lru_list_mut(MAMBA).insert_mru(a); - let (kv_xfer, comp_xfers) = tc - .build_load_back_spec(tc.arena.node(b).id, /* req = */ None) - .expect("live test node"); - assert!(comp_xfers.is_empty()); - tc.commit_load_back( - tc.arena.node(b).id, - Tensor::from_slice(&[30i64, 31]), - kv_xfer, - comp_xfers, - ) - .expect("live test node"); + tc.host_lru_list_mut(MAMBA).insert_mru(b); + tc.update_evictable_leaf_sets_(a); + tc.update_evictable_leaf_sets_(b); + let release_params = |receipt: IncLockRefResult| receipt.to_dec_params(); + let ancestor_host_lock = lock_ancestor_host + .then(|| release_params(tc.inc_host_lock_ref(a_id).expect("live ancestor"))); + // Mirror the controller: protect host sources, build under a temporary + // device lock, then replace that lock after committing the allocated rows. + let host_lock = release_params(tc.inc_host_lock_ref(b_id).expect("live anchor")); + let before_lock = release_params( + tc.inc_lock_ref(b_id, ComponentSet::EMPTY) + .expect("live anchor"), + ); + let (kv_xfer, mut comp_xfers) = tc + .build_load_back_spec(b_id, /* req = */ None) + .expect("live anchor"); + assert_eq!(kv_xfer.nodes_to_load, Some(vec![a_id, b_id])); + let mamba_xfer = &mut comp_xfers.get_mut(&MAMBA).unwrap()[0]; + assert_eq!(mamba_xfer.nodes_to_load, Some(vec![b_id])); + assert!( + mamba_xfer + .host_indices + .as_ref() + .unwrap() + .equal(&Tensor::from_slice(&[21i64])) + ); + mamba_xfer.device_indices = Some(Tensor::from_slice(&[40i64])); + tc.dec_lock_ref(b_id, &before_lock, false) + .expect("live anchor"); + tc.commit_load_back(b_id, Tensor::from_slice(&[30i64, 31]), kv_xfer, comp_xfers) + .expect("live anchor"); + let pending_lock = release_params( + tc.inc_lock_ref(b_id, ComponentSet::EMPTY) + .expect("live anchor"), + ); + assert!(tc.arena.node(a).is_load_back_pending()); + assert!(tc.arena.node(b).is_load_back_pending()); + assert_eq!( + tc.arena.host_lock_ref(a, MAMBA), + u32::from(lock_ancestor_host) + ); + assert_eq!(tc.arena.host_lock_ref(b, MAMBA), 1); + tc.sanity_check(&[], &[(b_id as i64, b_id)]); let result = tc.drive_host_eviction(MAMBA, /* num_tokens = */ 1); - assert_eq!(result.tracker[&MAMBA], 0); - assert!(result.host_frees.is_empty()); - assert!(tc.arena.has_host_value(a, MAMBA)); + assert_eq!(result.tracker[&MAMBA], usize::from(!lock_ancestor_host)); + assert!(result.device_frees.is_empty()); + if lock_ancestor_host { + assert!(result.host_frees.is_empty()); + } else { + assert_eq!(result.host_frees.len(), 1); + assert!(result.host_frees[&MAMBA][0].equal(&Tensor::from_slice(&[20i64]))); + } + assert_eq!(tc.arena.has_host_value(a, MAMBA), lock_ancestor_host); + assert!(tc.arena.has_host_value(b, MAMBA)); + // The Full transfer still owns both sources even when A's unused Mamba + // host slot can be reclaimed independently. + let full_pressure = tc.drive_host_eviction(FULL, 2); + assert_eq!(full_pressure.tracker[&FULL], 0); + assert!(full_pressure.host_frees.is_empty()); + assert!(tc.arena.has_host_value(a, FULL)); + assert!(tc.arena.has_host_value(b, FULL)); + tc.sanity_check(&[], &[(b_id as i64, b_id)]); - tc.finish_load_back(tc.arena.node(b).id) - .expect("live test node"); + tc.dec_lock_ref(b_id, &pending_lock, false) + .expect("live anchor"); + tc.dec_host_lock_ref(b_id, &host_lock).expect("live anchor"); + tc.finish_load_back(b_id).expect("live anchor"); + if let Some(params) = ancestor_host_lock { + tc.dec_host_lock_ref(a_id, ¶ms).expect("live ancestor"); + } let result = tc.drive_host_eviction(MAMBA, /* num_tokens = */ 1); - assert_eq!(result.tracker[&MAMBA], 1); - assert_eq!(result.host_frees[&MAMBA].len(), 1); + assert_eq!(result.tracker[&MAMBA], usize::from(lock_ancestor_host)); + if lock_ancestor_host { + assert_eq!(result.host_frees[&MAMBA].len(), 1); + assert!(result.host_frees[&MAMBA][0].equal(&Tensor::from_slice(&[20i64]))); + } else { + assert!(result.host_frees.is_empty()); + } assert!(!tc.arena.has_host_value(a, MAMBA)); + assert!(tc.arena.has_host_value(b, MAMBA)); + assert!(!tc.arena.node(a).is_load_back_pending()); + assert!(!tc.arena.node(b).is_load_back_pending()); tc.sanity_check(&[], &[]); } +#[test] +fn mamba_host_eviction_reclaims_unused_ancestor_during_full_load_back() { + check_mamba_host_eviction_during_full_load_back(false); +} + +#[test] +fn mamba_host_eviction_preserves_locked_ancestor_during_full_load_back() { + check_mamba_host_eviction_during_full_load_back(true); +} + #[test] fn backup_storage_commit_is_a_noop() { let mut tc = mamba_core(/* page_size = */ 1); diff --git a/rust/sglang-radix-tree/src/tests/components/swa.rs b/rust/sglang-radix-tree/src/tests/components/swa.rs index 2c3738b7ae..fd4ca8475d 100644 --- a/rust/sglang-radix-tree/src/tests/components/swa.rs +++ b/rust/sglang-radix-tree/src/tests/components/swa.rs @@ -3,6 +3,285 @@ use crate::components::{ComponentSet, FULL, MAMBA, SWA}; use crate::test_utils::{accumulate_step, action_kinds}; use crate::unified_tree_core::CacheInitParams; +#[test] +fn swa_window_ranges_coalesce_clip_and_stop_only_without_full() { + let mut tc = swa_core(8, 1); + let nodes: [_; 6] = chain(&mut tc); + let key = vec![1, 2, 3, 4, 5, 6]; + for node in nodes { + tc.arena + .set_device_value(node, FULL, Tensor::from_slice(&[10i64])); + } + set_swa_device(&mut tc, nodes[2]); + set_swa_device(&mut tc, nodes[5]); + set_swa_host(&mut tc, nodes[1]); + assert_eq!( + tc.swa_tombstone_ranges(&key, Default::default(), 1, 6) + .unwrap(), + vec![(1, 2), (3, 5)] + ); + let before = tc.arena.node(nodes[0]).last_access_counter; + assert_eq!( + tc.swa_tombstone_ranges(&key, Default::default(), 0, 5) + .unwrap(), + vec![(0, 2), (3, 5)] + ); + assert_eq!(tc.arena.node(nodes[0]).last_access_counter, before); + assert_eq!(tc.arena.len(), 7); + let _ = tc.arena.take_device_value(nodes[4], FULL); + assert_eq!( + tc.swa_tombstone_ranges(&key, Default::default(), 0, 6) + .unwrap(), + vec![(0, 2), (3, 4)] + ); + tc.arena + .set_host_value(nodes[4], FULL, Tensor::from_slice(&[10i64])); + assert_eq!( + tc.swa_tombstone_ranges(&key, Default::default(), 0, 6) + .unwrap(), + vec![(0, 2), (3, 5)] + ); + assert!( + tc.swa_tombstone_ranges(&key, Default::default(), 2, 2) + .unwrap() + .is_empty() + ); + assert!( + tc.swa_tombstone_ranges(&vec![9], Default::default(), 0, 1) + .unwrap() + .is_empty() + ); +} + +#[test] +fn swa_window_attach_splits_both_edges_preserving_full_and_pending_actions() { + let mut tc = swa_core(8, 2); + let key = vec![1, 2, 3, 4, 5, 6, 7, 8]; + tc.insert(&insert_params_swa( + &key, + &[10, 11, 12, 13, 14, 15, 16, 17], + 0, + 8, + )); + let leaf = child_of(&tc, tc.arena.root(), &[1, 2]); + let leaf_id = tc.arena.node(leaf).id; + tc.arena.node_mut(leaf).write_through_pending_id = Some(91); + tc.arena.node_mut(leaf).load_back_pending_id = Some(92); + tc.arena.node_mut(leaf).rotation_base = Some(7); + tc.arena.node_mut(leaf).external_cache_stored = true; + let mut values = Tensor::from_slice(&[50i64, 51, 52, 53]); + let actions = tc + .attach_swa_window(&key, Default::default(), 2, 6, &values) + .unwrap(); + assert_eq!(actions.len(), 2); + for action in &actions { + assert!( + matches!(action, CacheAction::ReplaceWriteThroughOnNodeSplit { ack_id: 91, old_node_id, new_child_node_id, .. } if *old_node_id == leaf_id && *new_child_node_id == leaf_id) + ); + } + let head = child_of(&tc, tc.arena.root(), &[1, 2]); + let middle = child_of(&tc, head, &[3, 4]); + assert_eq!(child_of(&tc, middle, &[7, 8]), leaf); + for node in [head, middle, leaf] { + assert_eq!(tc.arena.node(node).rotation_base, Some(7)); + assert_eq!(tc.arena.node(node).load_back_pending_id, Some(92)); + assert_eq!(tc.arena.node(node).write_through_pending_id, Some(91)); + assert!(tc.arena.node(node).external_cache_stored); + } + let full = Tensor::cat( + &[ + tc.arena.device_value(head, FULL), + tc.arena.device_value(middle, FULL), + tc.arena.device_value(leaf, FULL), + ], + 0, + ); + assert!(full.equal(&Tensor::from_slice(&[10i64, 11, 12, 13, 14, 15, 16, 17]))); + let _ = values.fill_(99); + assert!( + tc.arena + .device_value(middle, SWA) + .equal(&Tensor::from_slice(&[50i64, 51, 52, 53])) + ); + assert_eq!(tc.swa_evictable_size(), 4); + assert_eq!(tc.swa_protected_size(), 0); + assert_eq!( + tc.swa_tombstone_ranges(&key, Default::default(), 0, 8) + .unwrap(), + vec![(0, 2), (6, 8)] + ); +} + +#[test] +fn swa_window_attach_under_lock_restamps_lru_and_releases_accounting() { + let mut tc = swa_core(8, 1); + let key = vec![1, 2, 3, 4]; + tc.insert(&insert_params_swa(&key, &[10, 11, 12, 13], 0, 4)); + let leaf = child_of(&tc, tc.arena.root(), &[1]); + let leaf_id = tc.arena.node(leaf).id; + tc.arena + .set_host_value(leaf, SWA, Tensor::from_slice(&[20i64, 21, 22, 23])); + tc.host_lru_list_mut(SWA).insert_mru(leaf); + let receipt = tc.inc_lock_ref(leaf_id, ComponentSet::EMPTY).unwrap(); + tc.attach_swa_window( + &key, + Default::default(), + 1, + 3, + &Tensor::from_slice(&[51i64, 52]), + ) + .unwrap(); + let head = child_of(&tc, tc.arena.root(), &[1]); + let middle = child_of(&tc, head, &[2]); + assert_eq!(tc.swa_protected_size(), 2); + assert_eq!(tc.swa_evictable_size(), 0); + assert!(tc.device_lru_list(SWA).in_list(Some(middle))); + assert!(!tc.host_lru_list(SWA).in_list(Some(middle))); + assert!(tc.host_lru_list(SWA).in_list(Some(head))); + assert!(tc.host_lru_list(SWA).in_list(Some(leaf))); + tc.dec_lock_ref(leaf_id, &receipt.to_dec_params(), false) + .unwrap(); + assert_eq!(tc.swa_protected_size(), 0); + assert_eq!(tc.swa_evictable_size(), 2); + for node in [head, middle, leaf] { + assert_eq!(tc.arena.device_lock_ref(node, SWA), 0); + assert_eq!(tc.arena.device_lock_ref(node, FULL), 0); + } +} + +#[test] +fn swa_window_attach_rejects_incomplete_or_live_spans_before_mutation() { + let mut tc = swa_core(8, 1); + let key = vec![1, 2, 3, 4]; + tc.insert(&insert_params_swa(&key, &[10, 11, 12, 13], 0, 4)); + let leaf = child_of(&tc, tc.arena.root(), &[1]); + let (head, _) = tc.split_node_(leaf, 2); + tc.set_component_device_value_(leaf, SWA, Tensor::from_slice(&[52i64, 53])); + let count = tc.arena.len(); + let error = tc + .attach_swa_window( + &key, + Default::default(), + 1, + 4, + &Tensor::from_slice(&[51i64, 52, 53]), + ) + .err() + .unwrap(); + assert!(error.contains("over live SWA")); + assert_eq!(tc.arena.len(), count); + assert!(!tc.arena.has_device_value(head, SWA)); + assert_eq!(tc.swa_evictable_size(), 2); + let error = tc + .attach_swa_window( + &vec![1, 2, 9, 10], + Default::default(), + 1, + 4, + &Tensor::from_slice(&[51i64, 52, 53]), + ) + .err() + .unwrap(); + assert!(error.contains("covered 2")); + assert_eq!(tc.arena.len(), count); + assert!(!tc.arena.has_device_value(head, SWA)); +} + +#[test] +fn swa_window_attach_validates_tensor_and_page_bounds_before_mutation() { + let mut tc = swa_core(8, 2); + let key = vec![1, 2, 3, 4]; + tc.insert(&insert_params_swa(&key, &[10, 11, 12, 13], 0, 4)); + for (start, end, values) in [ + (1, 3, Tensor::from_slice(&[51i64, 52])), + (0, 4, Tensor::from_slice(&[51i64, 52])), + ( + 0, + 4, + Tensor::from_slice(&[51i64, 52, 53, 54]).reshape([2, 2]), + ), + (0, 4, Tensor::from_slice(&[51f32, 52., 53., 54.])), + (4, 2, Tensor::from_slice(&[51i64, 52])), + (2, 6, Tensor::from_slice(&[51i64, 52, 53, 54])), + ] { + assert!( + tc.attach_swa_window(&key, Default::default(), start, end, &values) + .is_err() + ); + assert_eq!(tc.arena.len(), 2); + assert_eq!(tc.swa_evictable_size(), 0); + } + assert!( + tc.attach_swa_window( + &key, + Default::default(), + 0, + 0, + &Tensor::from_slice::(&[]) + ) + .unwrap() + .is_empty() + ); + let mut full_only: UnifiedTreeCore> = + UnifiedTreeCore::new(CacheInitParams::default(), vec![FULL]); + assert!( + full_only + .swa_tombstone_ranges(&key, Default::default(), 0, 4) + .is_err() + ); + assert!( + full_only + .attach_swa_window( + &key, + Default::default(), + 0, + 4, + &Tensor::from_slice(&[51i64, 52, 53, 54]) + ) + .is_err() + ); +} + +#[test] +fn swa_window_bigram_namespaces_preserve_atom_positions() { + let mut tc: UnifiedTreeCore> = UnifiedTreeCore::new( + CacheInitParams { + page_size: 2, + ..swa_params_with_window(8) + }, + vec![FULL, SWA], + ); + let key = vec![(1, 2), (2, 3), (3, 4), (4, 5)]; + let namespace = crate::node::KeyNamespaceRef::new(Some("adapter"), Some("tenant")); + let leaf = tc + .arena + .alloc_child_in_namespace(tc.arena.root(), key.clone(), 0, namespace) + .unwrap(); + tc.arena + .set_device_value(leaf, FULL, Tensor::from_slice(&[10i64, 11, 12, 13])); + assert!( + tc.swa_tombstone_ranges(&key, Default::default(), 0, 4) + .unwrap() + .is_empty() + ); + assert_eq!( + tc.swa_tombstone_ranges(&key, namespace, 1, 3).unwrap(), + vec![(1, 3)] + ); + tc.attach_swa_window(&key, namespace, 2, 4, &Tensor::from_slice(&[52i64, 53])) + .unwrap(); + assert_eq!( + tc.swa_tombstone_ranges(&key, namespace, 0, 4).unwrap(), + vec![(0, 2)] + ); + assert!( + tc.arena + .device_value(leaf, SWA) + .equal(&Tensor::from_slice(&[52i64, 53])) + ); + assert_eq!(tc.arena.node(leaf).key, vec![(3, 4), (4, 5)]); +} + #[test] fn component_type_is_swa() { let swa = SwaComponent::new(&swa_params()); @@ -252,19 +531,27 @@ fn match_validator_host_backed_nodes_extend_the_window() { } #[test] -fn match_validator_hicache_accepts_live_or_backuped_swa_tombstones() { - let mut tc = swa_core(/* window = */ 2, /* page_size = */ 1); - tc.set_hicache_enabled(); - let [live, backuped, dead] = chain(&mut tc); - tc.arena - .set_device_value(live, FULL, Tensor::from_slice(&[0i64])); - tc.arena - .set_host_value(backuped, FULL, Tensor::from_slice(&[0i64])); - let mut validator = - swa_component(2).create_match_validator(&tc, /* match_device_only = */ true); - assert!(validator(&tc, live)); - assert!(validator(&tc, backuped)); - assert!(!validator(&tc, dead)); +fn match_validator_request_ring_accepts_live_or_backuped_tombstones_without_a_tier_condition() { + for enable_hicache in [false, true] { + let params = CacheInitParams { + swa_req_ring: true, + enable_hicache, + ..swa_params_with_window(2) + }; + let swa = SwaComponent::new(¶ms); + let mut tc = UnifiedTreeCore::new(params, vec![FULL, SWA]); + let [live, backuped, dead] = chain(&mut tc); + tc.arena + .set_device_value(live, FULL, Tensor::from_slice(&[0i64])); + tc.arena + .set_host_value(backuped, FULL, Tensor::from_slice(&[0i64])); + for device_only in [false, true] { + let mut validator = swa.create_match_validator(&tc, device_only); + assert!(validator(&tc, live)); + assert!(validator(&tc, backuped)); + assert!(!validator(&tc, dead)); + } + } } #[test] @@ -289,11 +576,9 @@ fn match_validator_hicache_with_a_host_pool_rejects_swa_tombstones() { let [live] = chain(&mut tc); tc.arena .set_device_value(live, FULL, Tensor::from_slice(&[0i64])); - // A wired host SWA pool means tombstones must gate the match again. + // A host pool cannot supply an SWA window whose nodes hold no host values. tc.set_has_swa_host_pool(); - let swa = SwaComponent { - sliding_window_size: 2, - }; + let swa = swa_component(2); let mut validator = >>::create_match_validator( &swa, &tc, /* match_device_only = */ true, ); @@ -301,21 +586,56 @@ fn match_validator_hicache_with_a_host_pool_rejects_swa_tombstones() { } #[test] -fn match_validator_hicache_tombstone_acceptance_still_resets_the_window() { +fn match_validator_request_ring_tombstone_acceptance_still_resets_the_window() { let mut tc = swa_core(/* window = */ 2, /* page_size = */ 1); tc.set_hicache_enabled(); let [live, c] = chain(&mut tc); tc.arena .set_device_value(live, FULL, Tensor::from_slice(&[0i64])); set_swa_device(&mut tc, c); - let mut validator = - swa_component(2).create_match_validator(&tc, /* match_device_only = */ true); + let swa = SwaComponent::new(&CacheInitParams { + swa_req_ring: true, + ..swa_params_with_window(2) + }); + let mut validator = swa.create_match_validator(&tc, /* match_device_only = */ true); // The accepted tombstone still zeroes the run: the next valued node // sits below the window. assert!(validator(&tc, live)); assert!(!validator(&tc, c)); } +#[test] +fn match_prefix_uses_request_ring_layout_instead_of_hicache_pool_presence() { + let key: Vec = (0..12).collect(); + let values: Vec = (10..22).collect(); + let prefix = key[..8].to_vec(); + for (swa_req_ring, enable_hicache, has_swa_host_pool) in [ + (false, false, false), + (true, false, false), + (false, true, false), + (false, true, true), + (true, true, false), + ] { + let mut tc = UnifiedTreeCore::new( + CacheInitParams { + swa_req_ring, + enable_hicache, + has_swa_host_pool, + ..swa_params_with_window(8) + }, + vec![FULL, SWA], + ); + tc.insert(&insert_params_swa(&key, &values, 0, 8)); + let result = tc.match_prefix(&match_params(&prefix)); + assert_eq!(result.full_kv_hit_length, 8); + let expected = if swa_req_ring { &values[..8] } else { &[] }; + assert!( + result.device_indices.equal(&Tensor::from_slice(expected)), + "ring={swa_req_ring}, hicache={enable_hicache}, swa_host={has_swa_host_pool}" + ); + } +} + // The SWA LRU order, MRU to LRU. fn swa_lru_order(tc: &UnifiedTreeCore>) -> Vec { tc.device_lru_list(SWA).iter().collect() @@ -543,6 +863,7 @@ fn insert_params_swa<'k>( swa_evicted_seqlen: usize, ) -> InsertParams<'k, Vec> { InsertParams { + rotation_base: None, key, namespace: Default::default(), value: Tensor::from_slice(value), @@ -739,6 +1060,7 @@ fn insert_overlap_recovers_a_tombstone_inside_the_window() { let root = tc.arena.root(); let leaf = child_of(&tc, root, &[1]); let result = tc.insert(&InsertParams { + rotation_base: None, track_adopted_ranges: true, ..insert_params_swa(&vec![1, 2, 3], &[20, 21, 22], 0, 0) }); @@ -806,6 +1128,7 @@ fn insert_overlap_with_a_locked_full_emits_the_recover_action() { .node_mut(leaf) .set_lock_ref_(ValueSlotIdx::device(FULL), 1); let result = tc.insert(&InsertParams { + rotation_base: None, track_adopted_ranges: true, ..insert_params_swa(&vec![1, 2, 3], &[20, 21, 22], 0, 0) }); @@ -2209,6 +2532,77 @@ fn inc_then_dec_lock_ref_roundtrips_with_dec_params() { assert_eq!(tc.swa_protected_size(), 0); } +#[test] +fn dec_window_lock_only_preserves_full_and_mamba_locks() { + let (mut tc, [_, _, leaf]) = internal_swa_write_back_fixture(2, 1, true); + let receipt = tc.inc_lock_ref(leaf, ComponentSet::EMPTY).unwrap(); + let idx = tc.arena.resolve(leaf).unwrap(); + let mut params = receipt.to_dec_params(); + let mut device_frees = HashMap::new(); + let mut host_frees = HashMap::new(); + assert_eq!(tc.component_protected_size(SWA), 2); + tc.dec_window_lock_only(leaf, SWA, ¶ms, &mut device_frees, &mut host_frees) + .unwrap(); + assert_eq!(tc.arena.device_lock_ref(idx, SWA), 0); + assert_eq!(tc.arena.device_lock_ref(idx, FULL), 1); + assert_eq!(tc.arena.device_lock_ref(idx, MAMBA), 1); + assert_eq!(tc.component_protected_size(SWA), 0); + assert_eq!(tc.component_protected_size(FULL), 3); + assert_eq!(tc.component_protected_size(MAMBA), 1); + assert!(device_frees.is_empty()); + assert!(host_frees.is_empty()); + + // The later receipt skips only the window already released; Mamba must + // still be released together with Full. + params.skipped_lock_components.insert(SWA); + tc.dec_lock_ref(leaf, ¶ms, false).unwrap(); + assert_eq!(tc.component_protected_size(FULL), 0); + assert_eq!(tc.component_protected_size(MAMBA), 0); + tc.sanity_check(&[], &[]); +} + +#[test] +fn dec_window_lock_only_ignores_root_and_skipped_window() { + let (mut tc, [_, _, leaf]) = internal_swa_write_back_fixture(2, 1, true); + let mut device_frees = HashMap::new(); + let mut host_frees = HashMap::new(); + tc.dec_window_lock_only( + tc.arena.node(tc.arena.root()).id, + SWA, + &DecLockRefParams::default(), + &mut device_frees, + &mut host_frees, + ) + .unwrap(); + let receipt = tc.inc_lock_ref(leaf, ComponentSet::of(SWA)).unwrap(); + let params = receipt.to_dec_params(); + assert!(!params.component_lock_uuids.contains_key(&(SWA.idx() as u8))); + tc.dec_window_lock_only(leaf, SWA, ¶ms, &mut device_frees, &mut host_frees) + .unwrap(); + assert_eq!(tc.component_protected_size(FULL), 3); + assert_eq!(tc.component_protected_size(MAMBA), 1); + assert_eq!(tc.component_protected_size(SWA), 0); + assert!(device_frees.is_empty()); + assert!(host_frees.is_empty()); + tc.dec_lock_ref(leaf, ¶ms, false).unwrap(); + tc.sanity_check(&[], &[]); +} + +#[test] +#[should_panic(expected = "lock receipt anchored on node")] +fn dec_window_lock_only_checks_receipt_anchor_before_skipping() { + let (mut tc, [other, _, leaf]) = internal_swa_write_back_fixture(2, 1, true); + let receipt = tc.inc_lock_ref(leaf, ComponentSet::of(SWA)).unwrap(); + tc.dec_window_lock_only( + other, + SWA, + &receipt.to_dec_params(), + &mut HashMap::new(), + &mut HashMap::new(), + ) + .unwrap(); +} + #[test] fn dec_swa_lock_only_releases_swa_while_full_stays_locked() { let mut tc = swa_core(/* window = */ 2, /* page_size = */ 1); @@ -2635,6 +3029,362 @@ fn swa_tracker() -> HashMap { HashMap::from([(FULL, 0), (SWA, 0)]) } +fn internal_swa_write_back_fixture( + window: usize, + page_size: usize, + with_mamba: bool, +) -> (UnifiedTreeCore>, [NodeId; 3]) { + let mut tc = UnifiedTreeCore::new( + CacheInitParams { + page_size, + swa_sliding_window_size: Some(window), + mamba_cache_chunk_size: with_mamba.then_some(256), + enable_hicache: true, + has_swa_host_pool: true, + is_write_back: true, + ..Default::default() + }, + if with_mamba { + vec![FULL, SWA, MAMBA] + } else { + vec![FULL, SWA] + }, + ); + let mut ids = [0; 3]; + for (i, id) in ids.iter_mut().enumerate() { + let len = (i + 1) * page_size; + let key: Vec<_> = (1..=len as i64).collect(); + let values: Vec<_> = (10..10 + len as i64).collect(); + let mut params = insert_params_swa(&key, &values, 0, 0); + params.mamba_value = with_mamba.then(|| Tensor::from_slice(&[70 + i as i64])); + *id = tc.insert(¶ms).last_device_node_id.unwrap(); + let idx = tc.arena.resolve(*id).unwrap(); + store_swa_device(&mut tc, idx); + } + (tc, ids) +} + +fn commit_internal_swa_backup(tc: &mut UnifiedTreeCore>, node_id: NodeId) { + let (full, mut transfers) = tc.build_backup_spec(node_id).unwrap(); + let nodes = transfers[&SWA][0].nodes_to_load.clone().unwrap(); + for transfer in transfers.values_mut().flatten() { + transfer.host_indices = Some(transfer.device_indices.as_ref().unwrap() + 1000); + } + tc.commit_backup(node_id, full + 1000, transfers).unwrap(); + let nodes = tc.mark_write_through_pending(nodes, node_id).unwrap(); + tc.finish_write_through(nodes, node_id).unwrap(); +} + +fn request_internal_swa_backup(tc: &mut UnifiedTreeCore>, node: NodeId, needed: usize) { + let (leaf, step) = tc.evict_device_next_node(SWA, &HashMap::new()); + assert_eq!(leaf, None); + assert_eq!(step.swa_backup_node_id, Some(node)); + assert_eq!(step.swa_backup_num_tokens, needed); + assert_eq!(step.mamba_backup_node_id, None); + assert!(step.tracker.is_empty()); + assert!(step.device_frees.is_empty()); + assert!(step.host_frees.is_empty()); + assert_eq!(step.unbacked_tokens, 0); + assert!( + tc.arena + .has_device_value(tc.arena.resolve(node).unwrap(), SWA) + ); +} + +#[test] +fn internal_swa_write_back_keeps_full_and_host_window_after_backup() { + let (mut tc, [a, b, c]) = internal_swa_write_back_fixture(2, 1, false); + tc.evict_device_start(SWA, 3); + request_internal_swa_backup(&mut tc, a, 1); + commit_internal_swa_backup(&mut tc, a); + let step = tc.finish_swa_state_eviction(a); + assert_eq!(step.tracker, HashMap::from([(SWA, 1)])); + assert!(step.device_frees[&SWA][0].equal(&Tensor::from_slice(&[10i64]))); + assert_eq!(step.device_frees.len(), 1); + assert!(step.host_frees.is_empty()); + assert_eq!(step.unbacked_tokens, 0); + let idx = tc.arena.resolve(a).unwrap(); + let node = tc.arena.node(idx); + assert!(node.device_value(FULL).equal(&Tensor::from_slice(&[10i64]))); + assert!(node.host_value(FULL).equal(&Tensor::from_slice(&[1010i64]))); + assert!(node.host_value(SWA).equal(&Tensor::from_slice(&[1000i64]))); + assert!(!node.has_device_value(SWA)); + assert!(tc.host_lru_list(SWA).in_list(Some(idx))); + let matched = tc.match_prefix(&match_params(&vec![1])); + assert_eq!(matched.best_match_node_id, a); + assert_eq!(matched.swa_host_hit_length, 1); + + // The next backup counts only B: A's SWA copy is already on host. + request_internal_swa_backup(&mut tc, b, 1); + tc.finish_swa_state_eviction(b); + let (leaf, step) = tc.evict_device_next_node(SWA, &HashMap::from([(SWA, 2)])); + assert_eq!(leaf, Some(c)); + assert_eq!(step.swa_backup_node_id, None); + assert_eq!(step.swa_backup_num_tokens, 0); + tc.evict_device_end(SWA); + tc.sanity_check(&[], &[]); +} + +#[test] +fn internal_swa_failed_backup_still_frees_and_obeys_the_budget() { + let (mut tc, [a, b, _]) = internal_swa_write_back_fixture(2, 1, false); + tc.evict_device_start(SWA, 1); + request_internal_swa_backup(&mut tc, a, 1); + let step = tc.finish_swa_state_eviction(a); + assert_eq!(step.tracker, HashMap::from([(SWA, 1)])); + let node = tc.arena.node(tc.arena.resolve(a).unwrap()); + assert!(node.has_device_value(FULL)); + assert!(!node.has_device_value(SWA)); + assert!(!node.has_host_value(SWA)); + let (next, exhausted) = tc.evict_device_next_node(SWA, &step.tracker); + assert_eq!(next, None); + assert!(exhausted.tracker.is_empty()); + assert_eq!(exhausted.swa_backup_node_id, None); + assert_eq!(exhausted.swa_backup_num_tokens, 0); + assert!(tc.arena.has_device_value(tc.arena.resolve(b).unwrap(), SWA)); + tc.evict_device_end(SWA); + tc.sanity_check(&[], &[]); +} + +#[test] +fn internal_swa_backup_counts_the_collected_window_including_page_overshoot() { + for buffer_only in [false, true] { + // Two 2-token nodes cover a 3-token window. Buffer mode stages B alone. + let (mut tc, [a, b, c]) = internal_swa_write_back_fixture(3, 2, false); + tc.is_host_memory_buffer_only = buffer_only; + let a_idx = tc.arena.resolve(a).unwrap(); + tc.device_lru_list_mut(SWA).reset_node_mru(a_idx); + tc.evict_device_start(SWA, 6); + let needed = if buffer_only { 2 } else { 4 }; + request_internal_swa_backup(&mut tc, b, needed); + let (_, transfers) = tc.build_backup_spec(b).unwrap(); + assert_eq!( + transfers[&SWA][0].device_indices.as_ref().unwrap().numel(), + needed + ); + commit_internal_swa_backup(&mut tc, b); + assert_eq!( + tc.finish_swa_state_eviction(b).tracker, + HashMap::from([(SWA, 2)]) + ); + let a_node = tc.arena.node(a_idx); + assert!(a_node.has_device_value(SWA)); + assert_eq!(a_node.has_host_value(SWA), !buffer_only); + let (next, _) = tc.evict_device_next_node(SWA, &HashMap::from([(SWA, 2)])); + assert_eq!(next, Some(c)); + tc.evict_device_end(SWA); + tc.sanity_check(&[], &[]); + } +} + +#[test] +fn internal_swa_backup_guards_keep_the_existing_inline_tombstone() { + for guard in [ + "no_hicache", + "write_through", + "no_swa_pool", + "full_host", + "swa_host", + "pending", + ] { + let (mut tc, [a, _, _]) = internal_swa_write_back_fixture(2, 1, false); + let idx = tc.arena.resolve(a).unwrap(); + match guard { + "no_hicache" => tc.enable_hicache = false, + "write_through" => tc.is_write_back = false, + "no_swa_pool" => tc.has_swa_host_pool = false, + "full_host" => { + tc.commit_backup(a, Tensor::from_slice(&[100i64]), HashMap::new()) + .unwrap(); + tc.mark_write_through_pending(vec![a], a).unwrap(); + tc.finish_write_through(vec![a], a).unwrap(); + } + "swa_host" => { + let (_, mut transfers) = tc.build_backup_spec(a).unwrap(); + transfers.get_mut(&SWA).unwrap()[0].host_indices = + Some(Tensor::from_slice(&[100i64])); + tc.commit_backup(a, Tensor::from_slice(&[] as &[i64]), transfers) + .unwrap(); + } + "pending" => tc.arena.node_mut(idx).write_through_pending_id = Some(a), + _ => unreachable!(), + } + tc.evict_device_start(SWA, 1); + let (next, step) = tc.evict_device_next_node(SWA, &HashMap::new()); + assert_eq!(next, None, "{guard}"); + assert_eq!(step.swa_backup_node_id, None, "{guard}"); + assert_eq!(step.swa_backup_num_tokens, 0, "{guard}"); + assert_eq!(step.tracker, HashMap::from([(SWA, 1)]), "{guard}"); + assert!(tc.arena.has_device_value(idx, FULL)); + assert!(!tc.arena.has_device_value(idx, SWA)); + tc.arena.node_mut(idx).write_through_pending_id = None; + tc.evict_device_end(SWA); + tc.sanity_check(&[], &[]); + } +} + +#[test] +fn internal_swa_resume_preserves_new_request_and_dma_locks() { + for pending_backup in [false, true] { + let (mut tc, [a, b, _]) = internal_swa_write_back_fixture(2, 1, false); + tc.evict_device_start(SWA, 3); + request_internal_swa_backup(&mut tc, a, 1); + let idx = tc.arena.resolve(a).unwrap(); + let swa = swa_component(2); + let lock = swa.acquire_component_lock(&mut tc, idx, IncLockRefResult::default(), false); + if pending_backup { + tc.arena.node_mut(idx).write_through_pending_id = Some(a); + } + let step = tc.finish_swa_state_eviction(a); + assert!(step.tracker.is_empty()); + assert!(step.device_frees.is_empty()); + assert!(tc.arena.has_device_value(idx, SWA)); + tc.arena.node_mut(idx).write_through_pending_id = None; + swa.release_component_lock(&mut tc, idx, &lock.to_dec_params(), false); + request_internal_swa_backup(&mut tc, b, 2); + tc.evict_device_end(SWA); + tc.sanity_check(&[], &[]); + } +} + +#[test] +fn internal_swa_resume_ignores_other_components_pending_markers() { + for load_back in [false, true] { + let (mut tc, [a, _, c]) = internal_swa_write_back_fixture(2, 1, false); + tc.evict_device_start(SWA, 1); + request_internal_swa_backup(&mut tc, a, 1); + let idx = tc.arena.resolve(a).unwrap(); + if load_back { + tc.arena.node_mut(idx).load_back_pending_id = Some(c); + } else { + tc.arena.node_mut(idx).write_through_pending_id = Some(c); + } + let step = tc.finish_swa_state_eviction(a); + assert_eq!(step.tracker, HashMap::from([(SWA, 1)])); + assert!(tc.arena.has_device_value(idx, FULL)); + assert!(!tc.arena.has_device_value(idx, SWA)); + let node = tc.arena.node_mut(idx); + assert_eq!( + if load_back { + node.load_back_pending_id + } else { + node.write_through_pending_id + }, + Some(c) + ); + node.load_back_pending_id = None; + node.write_through_pending_id = None; + tc.evict_device_end(SWA); + tc.sanity_check(&[], &[]); + } +} + +#[test] +fn internal_swa_resume_reselects_only_an_atomically_evictable_leaf() { + use crate::components::full::FullComponent; + for full_locked in [false, true] { + let (mut tc, [a, b, c]) = internal_swa_write_back_fixture(2, 1, false); + tc.evict_device_start(SWA, 1); + request_internal_swa_backup(&mut tc, a, 1); + tc.evict_device_leaf(c, false).unwrap(); + tc.evict_device_leaf(b, false).unwrap(); + let idx = tc.arena.resolve(a).unwrap(); + if full_locked { + FullComponent.acquire_component_lock(&mut tc, idx, IncLockRefResult::default(), false); + } + let step = tc.finish_swa_state_eviction(a); + let (next, exhausted) = tc.evict_device_next_node(SWA, &step.tracker); + if full_locked { + // Full alone is pinned; SWA finishes rather than endlessly requeueing. + assert_eq!(step.tracker, HashMap::from([(SWA, 1)])); + assert_eq!(next, None); + assert!(!tc.arena.has_device_value(idx, SWA)); + FullComponent.release_component_lock(&mut tc, idx, &DecLockRefParams::default(), false); + } else { + assert!(step.tracker.is_empty()); + assert!(step.device_frees.is_empty()); + assert_eq!(next, Some(a)); + assert!(tc.arena.has_device_value(idx, SWA)); + } + assert_eq!(exhausted.swa_backup_node_id, None); + assert!(tc.arena.has_device_value(idx, FULL)); + tc.evict_device_end(SWA); + tc.sanity_check(&[], &[]); + } +} + +#[test] +fn internal_swa_resume_does_not_touch_a_recycled_node() { + let (mut tc, [a, b, c]) = internal_swa_write_back_fixture(2, 1, false); + tc.evict_device_start(SWA, 1); + request_internal_swa_backup(&mut tc, a, 1); + for id in [c, b, a] { + tc.evict_device_leaf(id, false).unwrap(); + } + let replacement = tc + .insert(&insert_params_swa(&vec![9], &[90], 0, 0)) + .last_device_node_id + .unwrap(); + let idx = tc.arena.resolve(replacement).unwrap(); + store_swa_device(&mut tc, idx); + assert!(tc.arena.resolve(a).is_err()); + let step = tc.finish_swa_state_eviction(a); + assert!(step.tracker.is_empty()); + assert!(step.device_frees.is_empty()); + let (next, _) = tc.evict_device_next_node(SWA, &HashMap::new()); + assert_eq!(next, Some(replacement)); + assert!(tc.arena.has_device_value(idx, SWA)); + tc.evict_device_end(SWA); + tc.sanity_check(&[], &[]); +} + +#[test] +fn internal_swa_pending_protocol_prevents_double_free_and_allows_abandonment() { + use std::panic::{AssertUnwindSafe, catch_unwind}; + let (mut tc, [a, b, _]) = internal_swa_write_back_fixture(2, 1, false); + tc.evict_device_start(SWA, 1); + request_internal_swa_backup(&mut tc, a, 1); + assert!( + catch_unwind(AssertUnwindSafe( + || tc.evict_device_next_node(SWA, &HashMap::new()) + )) + .is_err() + ); + assert!(catch_unwind(AssertUnwindSafe(|| tc.finish_swa_state_eviction(b))).is_err()); + tc.evict_device_end(SWA); + assert!(tc.arena.has_device_value(tc.arena.resolve(a).unwrap(), SWA)); + tc.evict_device_start(SWA, 1); + request_internal_swa_backup(&mut tc, a, 1); + assert_eq!(tc.finish_swa_state_eviction(a).tracker[&SWA], 1); + assert!(catch_unwind(AssertUnwindSafe(|| tc.finish_swa_state_eviction(a))).is_err()); + tc.evict_device_end(SWA); + tc.sanity_check(&[], &[]); +} + +#[test] +fn internal_swa_write_back_preserves_host_mamba_before_device_cascade() { + for backup_succeeds in [false, true] { + let (mut tc, [a, _, _]) = internal_swa_write_back_fixture(2, 1, true); + tc.evict_device_start(SWA, 1); + request_internal_swa_backup(&mut tc, a, 1); + if backup_succeeds { + commit_internal_swa_backup(&mut tc, a); + } + let step = tc.finish_swa_state_eviction(a); + assert_eq!(step.tracker, HashMap::from([(SWA, 1), (MAMBA, 1)])); + assert!(step.device_frees[&MAMBA][0].equal(&Tensor::from_slice(&[70i64]))); + assert!(!step.device_frees.contains_key(&FULL)); + let node = tc.arena.node(tc.arena.resolve(a).unwrap()); + assert!(node.has_device_value(FULL)); + for ct in [SWA, MAMBA] { + assert!(!node.has_device_value(ct)); + assert_eq!(node.has_host_value(ct), backup_succeeds); + } + tc.evict_device_end(SWA); + tc.sanity_check(&[], &[]); + } +} + #[test] fn evict_walk_advances_one_allocator_mutation_per_call() { let mut tc = swa_core(/* window = */ 2, /* page_size = */ 1); @@ -4507,7 +5257,7 @@ fn auxiliary_load_does_not_reuse_a_full_pending_pin() { } #[test] -fn swa_device_eviction_skips_a_load_back_pinned_node() { +fn swa_device_eviction_preserves_a_locked_load_back_destination() { let mut tc: UnifiedTreeCore> = UnifiedTreeCore::new( CacheInitParams { is_write_back: true, @@ -4531,12 +5281,17 @@ fn swa_device_eviction_skips_a_load_back_pinned_node() { comp_xfers, ) .expect("live test node"); - // The pin alone keeps the in-flight SWA slice out of every eviction branch. + let receipt = tc + .inc_lock_ref(tc.arena.node(n).id, ComponentSet::EMPTY) + .expect("live test node"); + // The selected SWA destination is protected by its component lock. tc.evict_device_start(SWA, 4); let (next, _) = tc.evict_device_next_node(SWA, &HashMap::new()); assert_eq!(next, None); tc.evict_device_end(SWA); assert!(tc.arena.has_device_value(n, SWA)); + tc.dec_lock_ref(tc.arena.node(n).id, &receipt.to_dec_params(), false) + .expect("live test node"); tc.finish_load_back(tc.arena.node(n).id) .expect("live test node"); tc.evict_device_start(SWA, 4); @@ -4545,8 +5300,7 @@ fn swa_device_eviction_skips_a_load_back_pinned_node() { tc.evict_device_end(SWA); } -#[test] -fn swa_host_eviction_skips_a_load_back_pinned_node() { +fn check_swa_device_restored_during_full_load_back(pin_ancestor: bool) { let mut tc: UnifiedTreeCore> = UnifiedTreeCore::new( CacheInitParams { is_write_back: true, @@ -4557,43 +5311,224 @@ fn swa_host_eviction_skips_a_load_back_pinned_node() { vec![FULL, SWA], ); let [a, b] = chain::<2>(&mut tc); - set_full_host(&mut tc, a); - set_swa_host(&mut tc, a); - set_full_host(&mut tc, b); - set_swa_host(&mut tc, b); - tc.host_lru_list_mut(SWA).insert_mru(a); - let (kv_xfer, mut comp_xfers) = tc - .build_load_back_spec(tc.arena.node(b).id, /* req = */ None) - .expect("live test node"); + let [a_id, b_id] = [a, b].map(|idx| tc.arena.node(idx).id); + for (idx, swa) in [(a, 20i64), (b, 21)] { + set_full_host(&mut tc, idx); + tc.arena + .set_host_value(idx, SWA, Tensor::from_slice(&[swa])); + tc.host_lru_list_mut(SWA).insert_mru(idx); + tc.update_evictable_leaf_sets_(idx); + } + let release_params = |receipt: IncLockRefResult| receipt.to_dec_params(); + let host_lock = release_params(tc.inc_host_lock_ref(b_id).unwrap()); + let temporary_lock = release_params(tc.inc_lock_ref(b_id, ComponentSet::EMPTY).unwrap()); + let (full, mut transfers) = tc.build_load_back_spec(b_id, None).unwrap(); + assert_eq!(full.nodes_to_load, Some(vec![a_id, b_id])); + assert_eq!(transfers[&SWA][0].nodes_to_load, Some(vec![b_id])); + transfers.get_mut(&SWA).unwrap()[0].device_indices = Some(Tensor::from_slice(&[60i64])); + tc.dec_lock_ref(b_id, &temporary_lock, false).unwrap(); + tc.commit_load_back(b_id, Tensor::from_slice(&[50i64, 51]), full, transfers) + .unwrap(); + let pending_lock = release_params(tc.inc_lock_ref(b_id, ComponentSet::EMPTY).unwrap()); + assert!(!tc.arena.has_device_value(a, SWA)); + + // A is outside the loaded SWA window. A new request emits the normal + // locked-Full repair action; resolve only its allocator/mapping boundary. + let inserted = tc.insert(&insert_params_swa(&vec![1], &[90], 0, 0)); + let [ + CacheAction::RecoverSwaWithLockedFull { + node_id, + kept_full, + incoming_full, + }, + ] = inserted.cache_actions.as_slice() + else { + panic!("expected one RecoverSwaWithLockedFull action"); + }; + assert_eq!(*node_id, a_id); + assert!(kept_full.equal(&Tensor::from_slice(&[50i64]))); + assert!(incoming_full.equal(&Tensor::from_slice(&[90i64]))); + tc.set_component_device_value(a_id, SWA, Tensor::from_slice(&[70i64])) + .unwrap(); + assert!(tc.arena.node(a).is_load_back_pending()); + assert_eq!(tc.arena.device_lock_ref(a, SWA), 0); + assert_eq!(tc.arena.device_lock_ref(b, SWA), 1); + let ancestor_lock = + pin_ancestor.then(|| release_params(tc.inc_lock_ref(a_id, ComponentSet::EMPTY).unwrap())); + let active: Vec<_> = ancestor_lock.iter().map(|_| (a_id as i64, a_id)).collect(); + tc.sanity_check(&active, &[(b_id as i64, b_id)]); + + tc.evict_device_start(SWA, 1); + let (next, step) = tc.evict_device_next_node(SWA, &HashMap::new()); + tc.evict_device_end(SWA); + assert_eq!(next, None); assert_eq!( - comp_xfers.get(&SWA).unwrap()[0].nodes_to_load, - Some(vec![tc.arena.node(b).id]) + step.tracker.get(&SWA).copied().unwrap_or(0), + usize::from(!pin_ancestor) ); - comp_xfers.get_mut(&SWA).unwrap()[0].device_indices = Some(Tensor::from_slice(&[60i64])); - tc.commit_load_back( - tc.arena.node(b).id, - Tensor::from_slice(&[50i64, 51]), - kv_xfer, - comp_xfers, - ) - .expect("live test node"); + assert!(step.host_frees.is_empty()); + if pin_ancestor { + assert!(step.device_frees.is_empty()); + } else { + assert_eq!(step.device_frees.len(), 1); + assert_eq!(step.device_frees[&SWA].len(), 1); + // free_swa resolves physical rows through the retained Full indices. + assert!(step.device_frees[&SWA][0].equal(&Tensor::from_slice(&[50i64]))); + } + assert_eq!(tc.arena.has_device_value(a, SWA), pin_ancestor); + assert!( + tc.arena + .device_value(b, SWA) + .equal(&Tensor::from_slice(&[60i64])) + ); + assert!( + tc.arena + .device_value(a, FULL) + .equal(&Tensor::from_slice(&[50i64])) + ); + assert!( + tc.arena + .device_value(b, FULL) + .equal(&Tensor::from_slice(&[51i64])) + ); + tc.sanity_check(&active, &[(b_id as i64, b_id)]); - let result = tc.drive_host_eviction(SWA, /* num_tokens = */ 1); - assert_eq!(result.tracker[&SWA], 0); - assert!(result.host_frees.is_empty()); - assert!(tc.arena.has_host_value(a, SWA)); - - tc.finish_load_back(tc.arena.node(b).id) - .expect("live test node"); - let result = tc.drive_host_eviction(SWA, /* num_tokens = */ 1); - assert_eq!(result.tracker[&SWA], 1); - assert_eq!(result.host_frees[&SWA].len(), 1); - // Write-back reclaims the loaded node's coexisting host duplicate first. - assert!(tc.arena.has_host_value(a, SWA)); - assert!(!tc.arena.has_host_value(b, SWA)); + tc.dec_lock_ref(b_id, &pending_lock, false).unwrap(); + tc.dec_host_lock_ref(b_id, &host_lock).unwrap(); + tc.finish_load_back(b_id).unwrap(); + tc.evict_device_start(SWA, 1); + let (next, _) = tc.evict_device_next_node(SWA, &HashMap::new()); + tc.evict_device_end(SWA); + assert_eq!(next, Some(b_id)); + if let Some(params) = ancestor_lock { + tc.dec_lock_ref(a_id, ¶ms, false).unwrap(); + } tc.sanity_check(&[], &[]); } +#[test] +fn swa_device_eviction_reclaims_state_restored_during_full_load_back() { + check_swa_device_restored_during_full_load_back(false); +} + +#[test] +fn swa_device_eviction_preserves_pins_during_full_load_back() { + check_swa_device_restored_during_full_load_back(true); +} + +fn check_swa_host_eviction_during_full_load_back(lock_ancestor_host: bool) { + let mut tc: UnifiedTreeCore> = UnifiedTreeCore::new( + CacheInitParams { + is_write_back: true, + enable_hicache: true, + has_swa_host_pool: true, + ..swa_params_with_window(1) + }, + vec![FULL, SWA], + ); + let [a, b] = chain::<2>(&mut tc); + let a_id = tc.arena.node(a).id; + let b_id = tc.arena.node(b).id; + set_full_host(&mut tc, a); + set_full_host(&mut tc, b); + tc.arena + .set_host_value(a, SWA, Tensor::from_slice(&[20i64])); + tc.arena + .set_host_value(b, SWA, Tensor::from_slice(&[21i64])); + tc.host_lru_list_mut(SWA).insert_mru(a); + tc.host_lru_list_mut(SWA).insert_mru(b); + tc.update_evictable_leaf_sets_(a); + tc.update_evictable_leaf_sets_(b); + let release_params = |receipt: IncLockRefResult| receipt.to_dec_params(); + let ancestor_host_lock = lock_ancestor_host + .then(|| release_params(tc.inc_host_lock_ref(a_id).expect("live ancestor"))); + let host_lock = release_params(tc.inc_host_lock_ref(b_id).expect("live anchor")); + let before_lock = release_params( + tc.inc_lock_ref(b_id, ComponentSet::EMPTY) + .expect("live anchor"), + ); + let (kv_xfer, mut comp_xfers) = tc + .build_load_back_spec(b_id, /* req = */ None) + .expect("live anchor"); + assert_eq!(kv_xfer.nodes_to_load, Some(vec![a_id, b_id])); + let swa_xfer = &mut comp_xfers.get_mut(&SWA).unwrap()[0]; + // A contributes Full rows but lies outside B's trailing SWA window. + assert_eq!(swa_xfer.nodes_to_load, Some(vec![b_id])); + assert!( + swa_xfer + .host_indices + .as_ref() + .unwrap() + .equal(&Tensor::from_slice(&[21i64])) + ); + swa_xfer.device_indices = Some(Tensor::from_slice(&[60i64])); + tc.dec_lock_ref(b_id, &before_lock, false) + .expect("live anchor"); + tc.commit_load_back(b_id, Tensor::from_slice(&[50i64, 51]), kv_xfer, comp_xfers) + .expect("live anchor"); + let pending_lock = release_params( + tc.inc_lock_ref(b_id, ComponentSet::EMPTY) + .expect("live anchor"), + ); + assert!(tc.arena.node(a).is_load_back_pending()); + assert!(tc.arena.node(b).is_load_back_pending()); + assert_eq!( + tc.arena.host_lock_ref(a, SWA), + u32::from(lock_ancestor_host) + ); + assert_eq!(tc.arena.host_lock_ref(b, SWA), 1); + tc.sanity_check(&[], &[(b_id as i64, b_id)]); + + let result = tc.drive_host_eviction(SWA, /* num_tokens = */ 1); + assert_eq!(result.tracker[&SWA], usize::from(!lock_ancestor_host)); + assert!(result.device_frees.is_empty()); + if lock_ancestor_host { + assert!(result.host_frees.is_empty()); + } else { + assert_eq!(result.host_frees.len(), 1); + assert!(result.host_frees[&SWA][0].equal(&Tensor::from_slice(&[20i64]))); + } + assert_eq!(tc.arena.has_host_value(a, SWA), lock_ancestor_host); + assert!(tc.arena.has_host_value(b, SWA)); + let full_pressure = tc.drive_host_eviction(FULL, 2); + assert_eq!(full_pressure.tracker[&FULL], 0); + assert!(full_pressure.host_frees.is_empty()); + assert!(tc.arena.has_host_value(a, FULL)); + assert!(tc.arena.has_host_value(b, FULL)); + tc.sanity_check(&[], &[(b_id as i64, b_id)]); + + tc.dec_lock_ref(b_id, &pending_lock, false) + .expect("live anchor"); + tc.dec_host_lock_ref(b_id, &host_lock).expect("live anchor"); + tc.finish_load_back(b_id).expect("live anchor"); + if let Some(params) = ancestor_host_lock { + tc.dec_host_lock_ref(a_id, ¶ms).expect("live ancestor"); + } + let result = tc.drive_host_eviction(SWA, /* num_tokens = */ 1); + assert_eq!(result.tracker[&SWA], usize::from(lock_ancestor_host)); + if lock_ancestor_host { + assert_eq!(result.host_frees[&SWA].len(), 1); + assert!(result.host_frees[&SWA][0].equal(&Tensor::from_slice(&[20i64]))); + } else { + assert!(result.host_frees.is_empty()); + } + assert!(!tc.arena.has_host_value(a, SWA)); + assert!(tc.arena.has_host_value(b, SWA)); + assert!(!tc.arena.node(a).is_load_back_pending()); + assert!(!tc.arena.node(b).is_load_back_pending()); + tc.sanity_check(&[], &[]); +} + +#[test] +fn swa_host_eviction_reclaims_outside_window_during_full_load_back() { + check_swa_host_eviction_during_full_load_back(false); +} + +#[test] +fn swa_host_eviction_preserves_locked_ancestor_during_full_load_back() { + check_swa_host_eviction_during_full_load_back(true); +} + #[test] fn build_load_back_spec_degrades_to_empty_on_a_foreign_pin() { let mut tc: UnifiedTreeCore> = UnifiedTreeCore::new( @@ -4641,7 +5576,7 @@ fn build_load_back_spec_degrades_to_empty_on_a_foreign_pin() { } #[test] -fn host_drive_reclaims_swa_coexisting_host_values_when_the_host_lru_is_empty() { +fn host_drive_preserves_swa_coexisting_host_values_when_the_host_lru_is_empty() { let mut tc: UnifiedTreeCore> = UnifiedTreeCore::new( CacheInitParams { is_write_back: true, @@ -4673,6 +5608,8 @@ fn host_drive_reclaims_swa_coexisting_host_values_when_the_host_lru_is_empty() { HashMap::from([(SWA, vec![swa_xfer])]), ) .expect("live test node"); + tc.mark_write_through_pending(vec![handle], handle).unwrap(); + tc.finish_write_through(vec![handle], handle).unwrap(); } assert_eq!(tc.host_lru_list(SWA).len(), 0); @@ -4684,8 +5621,10 @@ fn host_drive_reclaims_swa_coexisting_host_values_when_the_host_lru_is_empty() { &mut df, &mut hf, ); - assert_eq!(tracker[&SWA], 2); - assert!(!tc.arena.node(parent_idx).has_host_value(SWA)); + assert_eq!(tracker[&SWA], 0); + assert!(hf.is_empty()); + assert!(df.is_empty()); + assert!(tc.arena.node(parent_idx).has_host_value(SWA)); assert!(tc.arena.node(parent_idx).has_device_value(SWA)); assert!(tc.arena.node(parent_idx).has_host_value(FULL)); assert!(tc.arena.node(leaf_idx).has_host_value(SWA)); @@ -4806,10 +5745,12 @@ fn write_through_offloads_a_boundary_split_leaf() { } #[test] -fn deep_swa_tree_survives_backup_evict_and_load_back_rounds() { +fn deep_request_ring_tree_survives_full_backup_evict_and_load_back_rounds() { let mut tc: UnifiedTreeCore> = UnifiedTreeCore::new( CacheInitParams { enable_hicache: true, + // Only Full is backed up; the request rebuilds its SWA ring. + swa_req_ring: true, ..swa_params_with_window(4) }, vec![FULL, SWA], @@ -4860,6 +5801,9 @@ fn deep_swa_tree_survives_backup_evict_and_load_back_rounds() { HashMap::new(), ) .expect("live test node"); + let handle = tc.arena.node(node).id; + tc.mark_write_through_pending(vec![handle], handle).unwrap(); + tc.finish_write_through(vec![handle], handle).unwrap(); } tc.sanity_check(&[], &[]); // Stepwise eviction rounds: half the Full budget, then the whole SWA budget. @@ -5436,6 +6380,7 @@ fn insert_reports_whether_it_reached_the_branch_boundary() { for (branching_seqlen, expected) in [(Some(3), true), (Some(4), false), (None, false)] { let mut tc = swa_hicache_core(/* window = */ 4, /* page_size = */ 1); let result = tc.insert(&InsertParams { + rotation_base: None, swa_branching_seqlen: branching_seqlen, ..insert_params_swa(&vec![1, 2, 3], &[10, 11, 12], 0, 0) }); diff --git a/rust/sglang-radix-tree/src/tests/node.rs b/rust/sglang-radix-tree/src/tests/node.rs index 4b9dc4518b..23cb4a7812 100644 --- a/rust/sglang-radix-tree/src/tests/node.rs +++ b/rust/sglang-radix-tree/src/tests/node.rs @@ -1839,111 +1839,183 @@ fn id_map_stays_consistent_across_free_and_realloc() -> Result<(), TreeCoreRunti Ok(()) } -// Eviction-eligible node set. +// One insertion-ordered membership collection for leaves and host duplicates. #[test] -fn add_then_contains_and_len() { - let mut set = EvictableNodeSet::new(); - assert!(!set.contains(NodeIdx_(3))); - assert!(set.is_empty()); - set.add(NodeIdx_(3)); - assert!(set.contains(NodeIdx_(3))); - assert_eq!(set.len(), 1); - assert!(!set.is_empty()); +fn node_set_empty_singleton_and_reinsertion() { + for mut set in [NodeSet::new(), NodeSet::default()] { + assert!(set.is_empty()); + assert_eq!(set.len(), 0); + assert_eq!(set.iter().next(), None); + assert!(!set.contains(NodeIdx_(3))); + set.discard(NodeIdx_(3)); + for _ in 0..2 { + set.add(NodeIdx_(3)); + set.add(NodeIdx_(3)); + assert!(set.contains(NodeIdx_(3))); + assert!(!set.is_empty()); + assert_eq!(set.len(), 1); + assert_eq!(set.iter().collect::>(), [NodeIdx_(3)]); + set.discard(NodeIdx_(3)); + set.discard(NodeIdx_(3)); + assert!(!set.contains(NodeIdx_(3))); + assert!(set.is_empty()); + assert_eq!(set.iter().next(), None); + } + } } #[test] -fn add_is_idempotent() { - let mut set = EvictableNodeSet::new(); +fn node_set_preserves_order_across_sequential_and_sparse_slots() { + let mut set = NodeSet::new(); + for slot in [0, 1, 2, 4096, 17] { + set.add(NodeIdx_(slot)); + assert!(set.contains(NodeIdx_(slot))); + } + assert_eq!(set.len(), 5); + assert_eq!( + set.iter().collect::>(), + [0, 1, 2, 4096, 17].map(NodeIdx_) + ); + for absent in [3, 4095, 8192] { + assert!(!set.contains(NodeIdx_(absent))); + set.discard(NodeIdx_(absent)); + } + set.discard(NodeIdx_(1)); + set.add(NodeIdx_(1)); + assert_eq!( + set.iter().collect::>(), + [0, 2, 4096, 17, 1].map(NodeIdx_) + ); + assert_eq!(set.len(), 5); +} + +#[test] +fn node_set_preserves_survivors_through_head_middle_tail_removal() { + let mut set = NodeSet::new(); + for slot in [10, 20, 30, 40, 20] { + set.add(NodeIdx_(slot)); + } + assert_eq!( + set.iter().collect::>(), + [10, 20, 30, 40].map(NodeIdx_) + ); + set.discard(NodeIdx_(10)); + set.discard(NodeIdx_(30)); + set.discard(NodeIdx_(99)); + set.add(NodeIdx_(10)); + assert_eq!(set.iter().collect::>(), [20, 40, 10].map(NodeIdx_)); + assert!(!set.contains(NodeIdx_(30))); + for slot in [10, 20, 40] { + assert!(set.contains(NodeIdx_(slot))); + set.discard(NodeIdx_(slot)); + assert!(!set.contains(NodeIdx_(slot))); + } + assert_eq!(set.iter().next(), None); set.add(NodeIdx_(5)); - set.add(NodeIdx_(5)); - assert_eq!(set.len(), 1); assert_eq!(set.iter().collect::>(), vec![NodeIdx_(5)]); } #[test] -fn discard_removes_the_member() { - let mut set = EvictableNodeSet::new(); - set.add(NodeIdx_(2)); - set.discard(NodeIdx_(2)); - assert!(!set.contains(NodeIdx_(2))); - assert_eq!(set.len(), 0); - assert!(set.is_empty()); -} - -#[test] -fn add_sequential_ids_grows_the_slot_table_one_by_one() { - // Arena NodeIds are sequential, so add(node_id == slots.len()) is the common case. - let mut set = EvictableNodeSet::new(); - set.add(NodeIdx_(0)); - set.add(NodeIdx_(1)); - set.add(NodeIdx_(2)); - assert!(set.contains(NodeIdx_(0))); - assert!(set.contains(NodeIdx_(1))); - assert!(set.contains(NodeIdx_(2))); - assert_eq!(set.len(), 3); -} - -#[test] -fn discard_absent_is_noop() { - let mut set = EvictableNodeSet::new(); - set.add(NodeIdx_(1)); - set.discard(NodeIdx_(9)); - set.discard(NodeIdx_(0)); - assert!(set.contains(NodeIdx_(1))); - assert_eq!(set.len(), 1); -} - -#[test] -fn discard_fixes_up_the_swapped_member_slot() { - let mut set = EvictableNodeSet::new(); - set.add(NodeIdx_(10)); - set.add(NodeIdx_(20)); - set.add(NodeIdx_(30)); - // Removing the first member swap-moves the tail (30) into its slot. - set.discard(NodeIdx_(10)); - assert!(!set.contains(NodeIdx_(10))); - assert!(set.contains(NodeIdx_(20))); - assert!(set.contains(NodeIdx_(30))); - assert_eq!(set.len(), 2); - // The moved member's slot stays consistent for a follow-up discard. - set.discard(NodeIdx_(30)); - assert!(!set.contains(NodeIdx_(30))); - assert!(set.contains(NodeIdx_(20))); - assert_eq!(set.len(), 1); -} - -#[test] -fn discard_the_tail_member() { - let mut set = EvictableNodeSet::new(); - set.add(NodeIdx_(10)); - set.add(NodeIdx_(20)); +fn node_set_iterator_preserves_order_and_remaining_length_after_skips() { + let mut set = NodeSet::new(); + for slot in [10, 20, 30, 40] { + set.add(NodeIdx_(slot)); + } set.discard(NodeIdx_(20)); - assert!(set.contains(NodeIdx_(10))); - assert!(!set.contains(NodeIdx_(20))); - assert_eq!(set.len(), 1); -} - -#[test] -fn re_add_after_discard() { - let mut set = EvictableNodeSet::new(); - set.add(NodeIdx_(4)); - set.discard(NodeIdx_(4)); - set.add(NodeIdx_(4)); - assert!(set.contains(NodeIdx_(4))); - assert_eq!(set.len(), 1); -} - -#[test] -fn iter_yields_all_members() { - let mut set = EvictableNodeSet::new(); - set.add(NodeIdx_(10)); set.add(NodeIdx_(20)); - set.add(NodeIdx_(30)); - set.discard(NodeIdx_(20)); - let mut members = set.iter().collect::>(); - members.sort_unstable(); - assert_eq!(members, vec![NodeIdx_(10), NodeIdx_(30)]); + assert_eq!(set.iter().last(), Some(NodeIdx_(20))); + assert_eq!( + set.iter().skip(2).collect::>(), + [NodeIdx_(40), NodeIdx_(20)] + ); + + let mut iter = set.iter(); + assert_eq!(iter.size_hint(), (4, Some(4))); + assert_eq!(iter.nth(1), Some(NodeIdx_(30))); + assert_eq!(iter.len(), 2); + assert_eq!(iter.size_hint(), (2, Some(2))); + assert_eq!(iter.next(), Some(NodeIdx_(40))); + assert_eq!(iter.next(), Some(NodeIdx_(20))); + assert_eq!(iter.size_hint(), (0, Some(0))); + assert_eq!(iter.next(), None); + assert_eq!(iter.next(), None); +} + +fn assert_node_set_matches_reference(set: &NodeSet, expected: &[NodeIdx_], universe: &[NodeIdx_]) { + // Bound the iterator read so a broken cycle reports an order failure + // rather than hanging this test in collect() or the test-only len(). + assert_eq!( + set.iter().take(expected.len() + 1).collect::>(), + expected + ); + assert_eq!(set.len(), expected.len()); + assert_eq!(set.is_empty(), expected.is_empty()); + for &slot in universe { + assert_eq!( + set.contains(slot), + expected.contains(&slot), + "slot {slot:?}" + ); + } +} + +#[test] +fn node_set_matches_vec_reference_through_deterministic_mixed_updates() { + let universe: Vec = (0usize..64) + .map(|i| NodeIdx_(if i.is_multiple_of(9) { i * 97 } else { i })) + .collect(); + for mut state in [0x5eed_u64, 0x1234_5678_abcd_u64] { + let mut set = NodeSet::new(); + let mut expected = Vec::new(); + for &slot in universe.iter().rev() { + set.add(slot); + expected.push(slot); + assert_node_set_matches_reference(&set, &expected, &universe); + } + for step in 0..2048 { + state = state.wrapping_mul(6364136223846793005).wrapping_add(1); + let chosen = universe[((state >> 32) as usize) % universe.len()]; + let operation = step % 8; + let slot = match operation { + 3 if !expected.is_empty() => expected[0], + 4 if !expected.is_empty() => expected[expected.len() - 1], + 5 if !expected.is_empty() => expected[expected.len() / 2], + 6 if !expected.is_empty() => expected[((state >> 16) as usize) % expected.len()], + _ => chosen, + }; + if matches!(operation, 0..=2 | 6) { + set.add(slot); + if !expected.contains(&slot) { + expected.push(slot); + } + } else { + set.discard(slot); + expected.retain(|&member| member != slot); + } + assert_node_set_matches_reference(&set, &expected, &universe); + if operation == 7 { + // Reuse the same arena slot immediately, after checking the + // intermediate removal. It must append behind all survivors. + set.add(slot); + expected.push(slot); + assert_node_set_matches_reference(&set, &expected, &universe); + } + } + while !expected.is_empty() { + let index = if expected.len().is_multiple_of(2) { + 0 + } else { + expected.len() - 1 + }; + let removed = expected.remove(index); + set.discard(removed); + assert_node_set_matches_reference(&set, &expected, &universe); + } + set.add(universe[63]); + expected.push(universe[63]); + assert_node_set_matches_reference(&set, &expected, &universe); + } } // Pin Python/Rust storage hashes across a parent-child boundary. diff --git a/rust/sglang-radix-tree/src/tests/unified_lru_list.rs b/rust/sglang-radix-tree/src/tests/unified_lru_list.rs index 94a471fa49..c9c21d70cb 100644 --- a/rust/sglang-radix-tree/src/tests/unified_lru_list.rs +++ b/rust/sglang-radix-tree/src/tests/unified_lru_list.rs @@ -601,6 +601,8 @@ fn arena_with_node() -> (NodeArena>, NodeIdx_) { node.last_access_counter = 5; node.creation_counter = 7; node.hit_count = 3; + node.tlru_cached_prefix_len = 1; + node.tlru_history_len = 1; (arena, NodeIdx_(a.0)) } @@ -640,6 +642,253 @@ fn slru_segments_on_the_protected_threshold() { ); } +#[test] +fn tlru_evicts_a_whole_node_only_when_its_uncached_history_fits_the_budget() { + let (mut arena, a) = arena_with_node(); + let node = arena.node_mut(a); + // This one-token node ends at depth 6; its branch once reached depth 8. + // Removing it leaves three history tokens uncached, not just its own one. + node.tlru_cached_prefix_len = 6; + node.tlru_history_len = 8; + for (tail_budget, expected_class) in [(0, 0), (2, 0), (3, -1), (4, -1)] { + assert_eq!( + TlruStrategy { + tail_budget, + float_config: None, + } + .get_priority(node), + PriorityKey(expected_class, 5), + "tail budget {tail_budget}" + ); + } +} + +#[test] +fn tlru_safe_nodes_precede_older_unsafe_nodes_but_keep_recency_within_each_phase() { + let (mut arena, a) = arena_with_node(); + let strategy = TlruStrategy { + tail_budget: 2, + float_config: None, + }; + let node = arena.node_mut(a); + node.tlru_cached_prefix_len = 4; + node.tlru_history_len = 8; + node.last_access_counter = 1; + let old_unsafe = strategy.get_priority(node); + node.last_access_counter = 2; + let new_unsafe = strategy.get_priority(node); + node.tlru_history_len = 4; + node.last_access_counter = 100; + let old_safe = strategy.get_priority(node); + node.last_access_counter = 101; + let new_safe = strategy.get_priority(node); + assert!(old_safe < new_safe); + assert!(new_safe < old_unsafe); + assert!(old_unsafe < new_unsafe); +} + +#[test] +fn tlru_handles_full_usize_depths_and_budgets_without_overflow() { + let (mut arena, a) = arena_with_node(); + let node = arena.node_mut(a); + node.tlru_history_len = usize::MAX; + // Valid cumulative metadata can span the entire unsigned domain even + // though this leaf itself has only one atom. Check exact boundaries. + for (prefix, budget, expected_class) in [ + (usize::MAX, 0, 0), + (usize::MAX, 1, -1), + (usize::MAX - 5, 5, 0), + (usize::MAX - 5, 6, -1), + (1, usize::MAX - 1, 0), + (1, usize::MAX, -1), + ] { + node.tlru_cached_prefix_len = prefix; + assert_eq!( + TlruStrategy { + tail_budget: budget, + float_config: None, + } + .get_priority(node), + PriorityKey(expected_class, 5), + "prefix {prefix}, budget {budget}" + ); + } +} + +fn tlru_float_strategy(threshold: f64, next_prompt_estimate: TlruPromptEstimate) -> TlruStrategy { + TlruStrategy { + // A nonzero integer budget makes accidental use of the integer path + // visible in the float cases that must remain unsafe. + tail_budget: usize::MAX, + float_config: Some(TlruFloatConfig { + threshold, + next_prompt_estimate, + }), + } +} + +#[test] +fn tlru_float_counts_preserve_fractional_eviction_boundaries() { + let (mut arena, a) = arena_with_node(); + let node = arena.node_mut(a); + node.tlru_history_len = 8; + let strategy = tlru_float_strategy(2.0, TlruPromptEstimate::Float(0.25)); + // Python's budget is 8 + 0.25 - 2 == 6.25: six cached tokens are + // insufficient, while seven suffice. Truncating the budget changes this. + node.tlru_cached_prefix_len = 7; + assert_eq!(strategy.get_priority(node), PriorityKey(0, 5)); + node.tlru_cached_prefix_len = 8; + assert_eq!(strategy.get_priority(node), PriorityKey(-1, 5)); +} + +#[test] +fn tlru_float_counts_add_history_before_subtracting_threshold() { + let (mut arena, a) = arena_with_node(); + let node = arena.node_mut(a); + let boundary = 2.0_f64.powi(53); + let strategy = tlru_float_strategy(boundary, TlruPromptEstimate::Float(boundary)); + // Python rounds 1 + 2**53 down to 2**53, then subtracts the threshold. + // Folding threshold - estimate first would classify this leaf as unsafe. + assert_eq!(strategy.get_priority(node), PriorityKey(-1, 5)); + node.tlru_history_len = 2; + node.tlru_cached_prefix_len = 2; + assert_eq!(strategy.get_priority(node), PriorityKey(0, 5)); +} + +#[test] +fn tlru_mixed_counts_add_integer_history_exactly_before_float_conversion() { + let (mut arena, a) = arena_with_node(); + let node = arena.node_mut(a); + let boundary = 2.0_f64.powi(53); + let strategy = tlru_float_strategy(boundary, TlruPromptEstimate::Integer((1_i128 << 53) + 1)); + // (1 + (2**53 + 1)) - float(2**53) == 2. Converting the estimate + // before the integer sum would round twice and incorrectly produce zero. + assert_eq!(strategy.get_priority(node), PriorityKey(0, 5)); + let strategy = tlru_float_strategy( + -boundary, + TlruPromptEstimate::Integer(-((1_i128 << 53) + 1)), + ); + assert_eq!(strategy.get_priority(node), PriorityKey(-1, 5)); + + // A midpoint much wider than usize still changes its rounded value when + // integer history crosses the tie; casting the estimate first loses it. + let strategy = tlru_float_strategy( + 2.0_f64.powi(100), + TlruPromptEstimate::Integer((1_i128 << 100) + (1_i128 << 47)), + ); + assert_eq!(strategy.get_priority(node), PriorityKey(0, 5)); +} + +#[test] +fn tlru_mixed_counts_cover_both_ends_of_the_exact_i128_sum_range() { + let (mut arena, a) = arena_with_node(); + let node = arena.node_mut(a); + node.tlru_history_len = usize::MAX; + node.tlru_cached_prefix_len = usize::MAX; + for estimate in [i128::MIN, i128::MAX - usize::MAX as i128] { + let strategy = tlru_float_strategy( + (estimate + usize::MAX as i128) as f64, + TlruPromptEstimate::Integer(estimate), + ); + assert_eq!(strategy.get_priority(node), PriorityKey(-1, 5)); + } + // A large estimate remains valid while this node's actual history fits; + // it need not reserve space for every possible native history length. + node.tlru_history_len = 1; + node.tlru_cached_prefix_len = 1; + let strategy = + tlru_float_strategy(i128::MAX as f64, TlruPromptEstimate::Integer(i128::MAX - 1)); + assert_eq!(strategy.get_priority(node), PriorityKey(-1, 5)); +} + +#[test] +fn tlru_float_counts_preserve_nan_infinity_and_zero_behavior() { + let (arena, a) = arena_with_node(); + let node = arena.node(a); + for (threshold, estimate, expected_class) in [ + (f64::NAN, 0.0, 0), + (0.0, f64::NAN, 0), + (f64::INFINITY, f64::INFINITY, 0), + (f64::NEG_INFINITY, f64::NEG_INFINITY, 0), + (f64::INFINITY, 0.0, -1), + (f64::NEG_INFINITY, 0.0, 0), + (0.0, f64::INFINITY, 0), + (0.0, f64::NEG_INFINITY, -1), + (-f64::MAX, f64::MAX, 0), + (f64::MAX, -f64::MAX, -1), + (1.0, -0.0, -1), + (-0.0, 0.0, 0), + ] { + let strategy = tlru_float_strategy(threshold, TlruPromptEstimate::Float(estimate)); + assert_eq!( + strategy.get_priority(node), + PriorityKey(expected_class, 5), + "threshold {threshold}, estimate {estimate}" + ); + } +} + +#[test] +#[cfg(target_pointer_width = "64")] +fn tlru_float_budget_compares_cached_integer_lengths_without_rounding_them() { + let (mut arena, a) = arena_with_node(); + let node = arena.node_mut(a); + node.tlru_history_len = (1_usize << 53) + 4; + node.tlru_cached_prefix_len = node.tlru_history_len; + // Cached length 2**53 + 3 rounds upward to the budget 2**53 + 4 as + // f64, but the exact Python integer/float comparison must remain false. + let strategy = tlru_float_strategy(0.0, TlruPromptEstimate::Float(0.0)); + assert_eq!(strategy.get_priority(node), PriorityKey(0, 5)); + let strategy = tlru_float_strategy(2.0, TlruPromptEstimate::Float(0.0)); + node.tlru_cached_prefix_len = node.tlru_history_len - 2; + assert_eq!(strategy.get_priority(node), PriorityKey(0, 5)); + node.tlru_cached_prefix_len += 1; + assert_eq!(strategy.get_priority(node), PriorityKey(-1, 5)); + + // At the unsigned limit, converting history produces 2**64, which + // cannot be met by any usize cached length. Saturating that conversion + // or rounding the cached length first would misclassify the last leaf. + node.tlru_history_len = usize::MAX; + node.tlru_cached_prefix_len = usize::MAX; + let strategy = tlru_float_strategy(0.0, TlruPromptEstimate::Float(0.0)); + assert_eq!(strategy.get_priority(node), PriorityKey(0, 5)); + assert!( + !strategy + .float_config + .as_ref() + .unwrap() + .is_tel_safe(usize::MAX, usize::MAX) + ); + let strategy = tlru_float_strategy(2048.0, TlruPromptEstimate::Float(0.0)); + node.tlru_cached_prefix_len = usize::MAX - 2047; + assert_eq!(strategy.get_priority(node), PriorityKey(0, 5)); + node.tlru_cached_prefix_len += 1; + assert_eq!(strategy.get_priority(node), PriorityKey(-1, 5)); +} + +#[test] +#[should_panic(expected = "T-LRU history + next_prompt_estimate overflowed i128")] +fn tlru_integer_history_overflow_panics_before_float_conversion() { + let (arena, a) = arena_with_node(); + let strategy = tlru_float_strategy(i128::MAX as f64, TlruPromptEstimate::Integer(i128::MAX)); + strategy.get_priority(arena.node(a)); +} + +#[test] +fn tlru_factory_forwards_float_config_instead_of_the_integer_budget() { + let (arena, a) = arena_with_node(); + let strategy = get_eviction_strategy::>( + "TLrU", + 999, + 0, + Some(TlruFloatConfig { + threshold: 2.0, + next_prompt_estimate: TlruPromptEstimate::Float(0.5), + }), + ); + assert_eq!(strategy.get_priority(arena.node(a)), PriorityKey(-1, 5)); +} + #[test] fn get_eviction_strategy_resolves_each_policy_name() { let (arena, a) = arena_with_node(); @@ -653,10 +902,11 @@ fn get_eviction_strategy_resolves_each_policy_name() { ("filo", PriorityKey(-7, 0)), ("priority", PriorityKey(9, 5)), ("slru", PriorityKey(1, 5)), + ("tlru", PriorityKey(0, 5)), ]; for (policy, expected) in cases { assert_eq!( - get_eviction_strategy::>(policy).get_priority(node), + get_eviction_strategy::>(policy, 2, 0, None).get_priority(node), expected, "policy {policy}" ); @@ -669,11 +919,11 @@ fn eviction_policy_names_are_case_insensitive() { let node = arena.node(NodeIdx_(a.0)); // Mixed-case names resolve to the same strategies as their lowercase forms. assert_eq!( - get_eviction_strategy::>("LRU").get_priority(node), + get_eviction_strategy::>("LRU", 2, 0, None).get_priority(node), PriorityKey(5, 0) ); assert_eq!( - get_eviction_strategy::>("Priority").get_priority(node), + get_eviction_strategy::>("Priority", 2, 0, None).get_priority(node), PriorityKey(9, 5) ); } @@ -681,7 +931,7 @@ fn eviction_policy_names_are_case_insensitive() { #[test] fn get_eviction_strategy_slru_default_threshold_is_two() { let (mut arena, a) = arena_with_node(); - let slru = get_eviction_strategy::>("slru"); + let slru = get_eviction_strategy::>("slru", 2, 0, None); // Exactly 2 hits is protected under the factory default; 1 is not. arena.node_mut(NodeIdx_(a.0)).hit_count = 2; assert_eq!( @@ -695,10 +945,24 @@ fn get_eviction_strategy_slru_default_threshold_is_two() { ); } +#[test] +fn tlru_factory_uses_the_tail_budget_and_accepts_mixed_case_names() { + let (arena, a) = arena_with_node(); + let node = arena.node(a); + assert_eq!( + get_eviction_strategy::>("TLrU", 999, 0, None).get_priority(node), + PriorityKey(0, 5) + ); + assert_eq!( + get_eviction_strategy::>("TLrU", 999, 1, None).get_priority(node), + PriorityKey(-1, 5) + ); +} + #[test] #[should_panic(expected = "Unknown eviction policy: random. Supported policies:")] fn get_eviction_strategy_panics_on_an_unknown_policy() { - get_eviction_strategy::>("Random"); + get_eviction_strategy::>("Random", 2, 0, None); } #[test] diff --git a/rust/sglang-radix-tree/src/tests/unified_tree_core.rs b/rust/sglang-radix-tree/src/tests/unified_tree_core.rs index 82ecc752e7..d683888d9d 100644 --- a/rust/sglang-radix-tree/src/tests/unified_tree_core.rs +++ b/rust/sglang-radix-tree/src/tests/unified_tree_core.rs @@ -1375,6 +1375,7 @@ fn insert_first_write_creates_the_namespace() { let mut tc = core(); matched_chain(&mut tc); tc.insert(&InsertParams { + rotation_base: None, namespace: KeyNamespaceRef::new(Some("lora-1"), None), ..insert_params(&vec![7, 8], &[40, 41]) }); @@ -1876,6 +1877,7 @@ fn swa_host_backed_node_advances_best_match_but_keeps_the_device_anchor() { fn insert_params<'k>(key: &'k Vec, value: &[i64]) -> InsertParams<'k, Vec> { InsertParams { + rotation_base: None, key, namespace: Default::default(), value: Tensor::from_slice(value), @@ -1892,11 +1894,228 @@ fn insert_params<'k>(key: &'k Vec, value: &[i64]) -> InsertParams<'k, Vec(key: &'k Vec, value: &[i64]) -> InsertParams<'k, Vec> { InsertParams { + rotation_base: None, track_adopted_ranges: true, ..insert_params(key, value) } } +fn tlru_core(tail_budget: usize) -> UnifiedTreeCore> { + UnifiedTreeCore::new( + CacheInitParams { + eviction_policy: "tlru".to_string(), + tlru_tail_budget: tail_budget, + ..Default::default() + }, + vec![FULL], + ) +} + +fn tlru_lens(tc: &UnifiedTreeCore>, node_id: NodeId) -> (usize, usize) { + let node = tc.arena.node(tc.arena.resolve(node_id).expect("live node")); + (node.tlru_cached_prefix_len, node.tlru_history_len) +} + +#[test] +fn tlru_device_splits_inherit_branch_history_without_changing_suffix_depth() { + let mut tc = tlru_core(2); + let a = tc + .insert(&insert_params( + &vec![1, 2, 3, 4, 5, 6], + &[10, 11, 12, 13, 14, 15], + )) + .last_device_node_id + .expect("inserted device node"); + let b = tc + .insert(&insert_params( + &vec![1, 2, 3, 4, 5, 6, 7, 8], + &[10, 11, 12, 13, 14, 15, 16, 17], + )) + .last_device_node_id + .expect("inserted device node"); + assert_eq!(tlru_lens(&tc, a), (6, 8)); + assert_eq!(tlru_lens(&tc, b), (8, 8)); + let sibling = tc + .insert(&insert_params(&vec![1, 2, 9, 10], &[20, 21, 22, 23])) + .last_device_node_id + .expect("inserted device node"); + let prefix = tc.arena.node(tc.arena.resolve(a).unwrap()).parent(); + let prefix_id = tc.arena.node(prefix).id; + assert_eq!(tlru_lens(&tc, prefix_id), (2, 8)); + assert_eq!(tlru_lens(&tc, a), (6, 8)); + assert_eq!(tlru_lens(&tc, b), (8, 8)); + assert_eq!(tlru_lens(&tc, sibling), (4, 4)); + + // Tail removal must not turn the ancestor's historical depth into its + // shorter current residency. Otherwise each subsequent pass over-trims. + tc.evict_device_leaf(b, false).expect("unlocked leaf"); + assert!(tc.arena.resolve(b).is_err()); + assert_eq!(tlru_lens(&tc, a), (6, 8)); + assert_eq!(tlru_lens(&tc, prefix_id), (2, 8)); + + // A longer extension raises only its own ancestry, not the other branch. + let extended = tc + .insert(&insert_params( + &vec![1, 2, 9, 10, 11, 12, 13, 14, 15, 16], + &[20, 21, 22, 23, 24, 25, 26, 27, 28, 29], + )) + .last_device_node_id + .expect("inserted device node"); + assert_eq!(tlru_lens(&tc, extended), (10, 10)); + assert_eq!(tlru_lens(&tc, sibling), (4, 10)); + assert_eq!(tlru_lens(&tc, prefix_id), (2, 10)); + assert_eq!(tlru_lens(&tc, a), (6, 8)); + tc.sanity_check(&[], &[]); +} + +#[test] +fn tlru_repeated_small_evictions_stop_trimming_at_the_historical_tail_budget() { + let mut tc = tlru_core(2); + let older = tc + .insert(&insert_params(&vec![90, 91, 92, 93], &[90, 91, 92, 93])) + .last_device_node_id + .expect("inserted device node"); + let mut chain = Vec::new(); + for size in [2, 4, 6, 8] { + let key: Vec = (1..=size).collect(); + let values: Vec = (10..10 + size).collect(); + chain.push( + tc.insert(&insert_params(&key, &values)) + .last_device_node_id + .expect("inserted device node"), + ); + } + // The newest two-token tail is safe; the old four-token branch is not. + // After the tail is gone, history remains eight and ordinary recency + // chooses the old branch rather than stripping another two tail tokens. + for expected in [chain[3], older] { + tc.evict_device_start(FULL, 1); + let (candidate, step) = tc.evict_device_next_node(FULL, &HashMap::new()); + assert!(step.device_frees.is_empty()); + assert_eq!(candidate, Some(expected)); + let (_, freed) = tc + .evict_device_leaf(expected, false) + .expect("unlocked leaf"); + assert!(freed.tracker[&FULL] > 0); + tc.evict_device_end(FULL); + assert_eq!(tlru_lens(&tc, chain[2]), (6, 8)); + tc.sanity_check(&[], &[]); + } +} + +#[test] +fn tlru_host_insert_and_split_keep_history_after_host_tail_reclamation() { + let mut tc = tlru_core(2); + let root_id = tc.arena.node(tc.arena.root()).id; + let a = tc + .insert_host( + root_id, + None, + vec![1, 2, 3, 4, 5, 6], + Tensor::from_slice(&[100i64, 101, 102, 103, 104, 105]), + (0..6).map(|i| format!("h{i}")).collect(), + ) + .unwrap() + .inserted_host_node + .unwrap(); + let b = tc + .insert_host( + a, + None, + vec![7, 8], + Tensor::from_slice(&[106i64, 107]), + vec!["h6".to_string(), "h7".to_string()], + ) + .unwrap() + .inserted_host_node + .unwrap(); + assert_eq!(tlru_lens(&tc, a), (6, 8)); + assert_eq!(tlru_lens(&tc, b), (8, 8)); + let reclaimed = tc.drive_host_eviction(FULL, 1); + assert_eq!(reclaimed.tracker[&FULL], 2); + assert!(tc.arena.resolve(b).is_err()); + assert_eq!(tlru_lens(&tc, a), (6, 8)); + let branch = tc + .insert_host( + root_id, + None, + vec![1, 2, 9, 10], + Tensor::from_slice(&[200i64, 201, 202, 203]), + (0..4).map(|i| format!("g{i}")).collect(), + ) + .unwrap() + .inserted_host_node + .unwrap(); + let prefix = tc.arena.node(tc.arena.resolve(a).unwrap()).parent(); + assert_eq!(tlru_lens(&tc, tc.arena.node(prefix).id), (2, 8)); + assert_eq!(tlru_lens(&tc, a), (6, 8)); + assert_eq!(tlru_lens(&tc, branch), (4, 4)); + assert_eq!(tlru_lens(&tc, root_id), (0, 8)); + tc.sanity_check(&[], &[]); +} + +#[test] +fn tlru_reused_node_slots_and_reset_do_not_inherit_old_branch_history() { + let mut tc = tlru_core(2); + let old = tc + .insert(&insert_params( + &vec![1, 2, 3, 4, 5, 6], + &[10, 11, 12, 13, 14, 15], + )) + .last_device_node_id + .expect("inserted device node"); + let old_slot = tc.arena.resolve(old).unwrap(); + tc.evict_device_leaf(old, false).expect("unlocked leaf"); + let fresh = tc + .insert(&insert_params(&vec![9, 8], &[90, 80])) + .last_device_node_id + .expect("inserted device node"); + assert_eq!(tc.arena.resolve(fresh).unwrap(), old_slot); + assert!(tc.arena.resolve(old).is_err()); + assert_eq!(tlru_lens(&tc, fresh), (2, 2)); + assert_eq!(tlru_lens(&tc, tc.arena.node(tc.arena.root()).id), (0, 6)); + tc.reset(); + assert!(tc.arena.resolve(fresh).is_err()); + assert_eq!(tlru_lens(&tc, tc.arena.node(tc.arena.root()).id), (0, 0)); + let after_reset = tc + .insert(&insert_params(&vec![7], &[70])) + .last_device_node_id + .expect("inserted device node"); + assert_eq!(tlru_lens(&tc, after_reset), (1, 1)); + assert_eq!(tlru_lens(&tc, tc.arena.node(tc.arena.root()).id), (0, 1)); + tc.sanity_check(&[], &[]); +} + +#[test] +fn non_tlru_policies_leave_history_bookkeeping_disabled_for_both_tiers() { + for policy in ["lru", "slru", "priority"] { + let mut tc = UnifiedTreeCore::new( + CacheInitParams { + eviction_policy: policy.to_string(), + tlru_tail_budget: usize::MAX, + ..Default::default() + }, + vec![FULL], + ); + tc.insert(&insert_params(&vec![1, 2, 3, 4], &[10, 11, 12, 13])); + tc.insert(&insert_params(&vec![1, 2, 9], &[20, 21, 29])); + tc.insert_host( + tc.arena.node(tc.arena.root()).id, + None, + vec![8, 9], + Tensor::from_slice(&[80i64, 90]), + vec!["h0".to_string(), "h1".to_string()], + ) + .unwrap(); + for node in tc.collect_all_nodes_() { + let node = tc.arena.node(node); + assert_eq!(node.tlru_cached_prefix_len, 0, "{policy}"); + assert_eq!(node.tlru_history_len, 0, "{policy}"); + } + tc.sanity_check(&[], &[]); + } +} + #[test] fn adopted_ranges_are_opt_in_and_coalesce() { let mut untracked = InsertResult::default(); @@ -1944,6 +2163,7 @@ fn insert_params_in_namespace<'a>( cache_salt: Option<&'a str>, ) -> InsertParams<'a, Vec> { InsertParams { + rotation_base: None, namespace: KeyNamespaceRef::new(extra_key, cache_salt), ..insert_params(key, value) } @@ -1988,6 +2208,7 @@ fn insert_prev_prefix_len_narrows_the_dup_window() { let mut tc = core(); tc.insert(&insert_params(&vec![1, 2, 3], &[10, 11, 12])); let result = tc.insert(&InsertParams { + rotation_base: None, prev_prefix_len: 2, ..insert_params(&vec![1, 2, 3], &[20, 21, 22]) }); @@ -2031,6 +2252,7 @@ fn insert_prev_prefix_len_spans_a_multi_node_walk() { // The request already matched [1,2,3]: only the second node's overlap // is duplicate. let result = tc.insert(&InsertParams { + rotation_base: None, prev_prefix_len: 3, ..insert_params(&vec![1, 2, 3, 4, 5], &[30, 31, 32, 33, 34]) }); @@ -2052,6 +2274,7 @@ fn insert_prev_prefix_len_narrows_mid_node_on_a_multi_node_walk() { tc.insert(&insert_params(&vec![1, 2, 3, 4, 5], &[20, 21, 22, 13, 14])); // prev_prefix_len 4 lands mid second node: only its last token is duplicate. let result = tc.insert(&InsertParams { + rotation_base: None, prev_prefix_len: 4, ..insert_params(&vec![1, 2, 3, 4, 5], &[30, 31, 32, 33, 34]) }); @@ -2124,6 +2347,7 @@ fn insert_priority_floor_applies_along_the_path() { .match_prefix(&match_params(&vec![1, 2])) .best_match_node_id; tc.insert(&InsertParams { + rotation_base: None, priority: 5, ..insert_params(&vec![1, 2], &[20, 21]) }); @@ -2147,6 +2371,7 @@ fn insert_chunked_skips_the_hit_count() { .node(tc.arena.resolve(a).expect("live test node")) .hit_count; tc.insert(&InsertParams { + rotation_base: None, chunked: true, ..insert_params(&vec![1, 2], &[20, 21]) }); @@ -3100,6 +3325,7 @@ fn bigram_insert_events_carry_pair_token_payloads() { ); let key: Vec<(i64, i64)> = vec![(1, 2), (2, 3)]; tc.insert(&InsertParams { + rotation_base: None, key: &key, namespace: Default::default(), value: Tensor::from_slice(&[10i64, 11]), @@ -3419,6 +3645,7 @@ fn prefetch_anchor_info_maps_the_namespace() { let mut tc = core(); tc.insert(&insert_params(&vec![1, 2], &[10, 11])); tc.insert(&InsertParams { + rotation_base: None, namespace: KeyNamespaceRef::new(Some("chat"), Some("tenant-a")), ..insert_params(&vec![7, 8], &[20, 21]) }); @@ -3448,6 +3675,7 @@ fn prefetch_anchor_info_maps_the_namespace() { ); // A node minted by a split inherits the namespace. tc.insert(&InsertParams { + rotation_base: None, namespace: KeyNamespaceRef::new(Some("chat"), Some("tenant-a")), ..insert_params(&vec![7], &[30]) }); @@ -4082,6 +4310,12 @@ fn backuped_chain(tc: &mut UnifiedTreeCore>) -> (NodeIdx_, NodeIdx_) { .expect("live test node"); tc.commit_backup(child, Tensor::from_slice(&[22i64, 23]), HashMap::new()) .expect("live test node"); + for node_id in [parent, child] { + tc.mark_write_through_pending(vec![node_id], node_id) + .expect("live test node"); + tc.finish_write_through(vec![node_id], node_id) + .expect("live test node"); + } ( tc.arena.resolve(parent).expect("live test node"), tc.arena.resolve(child).expect("live test node"), @@ -4487,6 +4721,12 @@ fn mixed_backup_evict_insert_keeps_the_leaf_sets_disjoint() { .best_match_node_id; tc.commit_backup(third, Tensor::from_slice(&[104i64, 105]), HashMap::new()) .expect("live test node"); + for node_id in [first, second, third] { + tc.mark_write_through_pending(vec![node_id], node_id) + .expect("live test node"); + tc.finish_write_through(vec![node_id], node_id) + .expect("live test node"); + } let mut tracker = HashMap::from([(FULL, 0)]); let (mut df, mut hf) = (HashMap::new(), HashMap::new()); tc.evict_device_start(FULL, /* request_cnt = */ 4); @@ -4713,6 +4953,7 @@ fn insert_empty_key_is_a_noop_and_mints_no_namespace_root() { assert_eq!(tc.arena.len(), 1); // A namespaced empty insert never creates the namespace root. let result = tc.insert(&InsertParams { + rotation_base: None, namespace: KeyNamespaceRef::new(Some("ghost"), None), ..insert_params(&vec![], &[]) }); @@ -4730,6 +4971,7 @@ fn root_node_handle_is_namespace_independent() { assert_eq!(tc.root_node_handle(Some("ghost")), root_handle); assert!(!tc.arena.namespace_exists(Some("ghost"))); tc.insert(&InsertParams { + rotation_base: None, namespace: KeyNamespaceRef::new(Some("chat"), None), ..insert_params(&vec![1], &[10]) }); @@ -4801,12 +5043,14 @@ fn get_hash_values_reads_the_nodes_own_hashes() { fn insert_empty_key_still_touches_the_existing_root() { let mut tc = core(); tc.insert(&InsertParams { + rotation_base: None, priority: 7, ..insert_params(&vec![], &[]) }); assert_eq!(tc.arena.node(tc.arena.root()).priority, 7); // A namespaced empty insert touches the same single root. tc.insert(&InsertParams { + rotation_base: None, namespace: KeyNamespaceRef::new(Some("chat"), None), priority: 9, ..insert_params(&vec![], &[]) @@ -4818,6 +5062,7 @@ fn insert_empty_key_still_touches_the_existing_root() { fn insert_into_a_named_namespace_is_isolated() { let mut tc = core(); tc.insert(&InsertParams { + rotation_base: None, namespace: KeyNamespaceRef::new(Some("lora-1"), None), ..insert_params(&vec![1, 2], &[10, 11]) }); @@ -4849,6 +5094,7 @@ fn insert_sub_page_key_is_a_noop_and_mints_no_namespace_root() { assert!(result.mamba_exist); assert_eq!(tc.arena.len(), 1); let result = tc.insert(&InsertParams { + rotation_base: None, namespace: KeyNamespaceRef::new(Some("ghost"), None), ..insert_params(&vec![1], &[10]) }); @@ -6623,6 +6869,7 @@ fn reset_restores_a_fresh_tree() { let mut tc = core(); tc.insert(&insert_params(&vec![1, 2, 3], &[10, 11, 12])); tc.insert(&InsertParams { + rotation_base: None, namespace: KeyNamespaceRef::new(Some("chat"), None), ..insert_params(&vec![7, 8], &[20, 21]) }); @@ -6703,6 +6950,7 @@ fn total_size_spans_namespaces_and_aux_values() { let mut tc = core(); tc.insert(&insert_params(&vec![1, 2, 3], &[10, 11, 12])); tc.insert(&InsertParams { + rotation_base: None, namespace: KeyNamespaceRef::new(Some("chat"), None), ..insert_params(&vec![7, 8], &[20, 21]) }); @@ -6750,6 +6998,7 @@ fn walk_for_kv_canary_chains_slots_across_namespaces() { let mut tc = core(); tc.insert(&insert_params(&vec![1, 2, 3], &[10, 11, 12])); tc.insert(&InsertParams { + rotation_base: None, namespace: KeyNamespaceRef::new(Some("chat"), None), ..insert_params(&vec![7, 8], &[20, 21]) }); @@ -7049,6 +7298,7 @@ fn all_values_flatten_spans_namespaces() { tc.insert(&insert_params(&vec![1, 2, 3], &[10, 11, 12])); tc.insert(&insert_params(&vec![1, 2, 3, 4, 5], &[20, 21, 22, 13, 14])); tc.insert(&InsertParams { + rotation_base: None, namespace: KeyNamespaceRef::new(Some("chat"), None), ..insert_params(&vec![7, 8], &[20, 21]) }); @@ -7061,6 +7311,7 @@ fn collect_all_nodes_visits_every_root_subtree() { let mut tc = core(); tc.insert(&insert_params(&vec![1, 2, 3], &[10, 11, 12])); tc.insert(&InsertParams { + rotation_base: None, namespace: KeyNamespaceRef::new(Some("chat"), None), ..insert_params(&vec![7, 8], &[20, 21]) }); @@ -7076,6 +7327,7 @@ fn pretty_format_renders_every_namespace_and_component() { tc.insert(&insert_params(&vec![1, 2, 3], &[10, 11, 12])); tc.insert(&insert_params(&vec![1, 2, 3, 4, 5], &[20, 21, 22, 13, 14])); tc.insert(&InsertParams { + rotation_base: None, namespace: KeyNamespaceRef::new(Some("chat"), None), ..insert_params(&vec![7], &[30]) }); @@ -7109,6 +7361,7 @@ fn sane_tree() -> UnifiedTreeCore> { tc.insert(&insert_params(&vec![1, 2, 3, 4, 5], &[20, 21, 22, 13, 14])); tc.insert(&insert_params(&vec![1, 2, 9], &[30, 31, 39])); tc.insert(&InsertParams { + rotation_base: None, namespace: KeyNamespaceRef::new(Some("chat"), None), ..insert_params(&vec![7, 8], &[40, 41]) }); @@ -7657,11 +7910,13 @@ fn refresh_dispatches_fire_per_walk_phase_in_a_namespace() { let recorder = Arc::new(RecordingComponentForTest::default()); tc.register_component_(recorder.clone()); tc.insert(&InsertParams { + rotation_base: None, namespace: KeyNamespaceRef::new(Some("chat"), None), ..insert_params(&vec![7, 8], &[40, 41]) }); // The deeper insert walks down through the existing [7,8] node. tc.insert(&InsertParams { + rotation_base: None, namespace: KeyNamespaceRef::new(Some("chat"), None), ..insert_params(&vec![7, 8, 9], &[40, 41, 42]) }); @@ -8107,6 +8362,10 @@ fn resume_insert_completes_after_an_on_path_host_leaf_is_evicted() { HashMap::new(), ) .expect("live test node"); + tc.mark_write_through_pending(vec![h_leaf], h_leaf) + .expect("live test node"); + tc.finish_write_through(vec![h_leaf], h_leaf) + .expect("live test node"); demote_node(&mut tc, h_leaf_idx); let step = tc.begin_insert(&insert_params( &vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], @@ -8133,6 +8392,10 @@ fn resume_insert_completes_after_an_on_path_host_leaf_is_evicted() { HashMap::new(), ) .expect("live test node"); + tc.mark_write_through_pending(vec![top], top) + .expect("live test node"); + tc.finish_write_through(vec![top], top) + .expect("live test node"); let done = tc.resume_insert(); let result = done.result.expect("the resumed walk completes"); assert_eq!(result.prefix_len, 4); @@ -8170,6 +8433,11 @@ fn aborted_barrier_crossing_refires_on_the_next_insert() { HashMap::new(), ) .expect("live test node"); + let node_id = tc.arena.node(a).id; + tc.mark_write_through_pending(vec![node_id], node_id) + .expect("live test node"); + tc.finish_write_through(vec![node_id], node_id) + .expect("live test node"); let done = tc.resume_insert(); assert_eq!( done.result.expect("the resumed walk completes").prefix_len, @@ -8475,6 +8743,7 @@ fn sequence_insert_params<'k>( Tensor::from_slice(&[*mamba_next]) }); InsertParams { + rotation_base: None, key, namespace: Default::default(), value: Tensor::from_slice(&kv), @@ -8636,6 +8905,7 @@ fn drain_full_device(tc: &mut UnifiedTreeCore>) { fn an_emptied_namespace_leaves_nothing_behind() { let mut tc = core(); tc.insert(&InsertParams { + rotation_base: None, namespace: KeyNamespaceRef::new(Some("salted"), None), ..insert_params(&vec![1, 2], &[10, 11]) }); @@ -8653,6 +8923,7 @@ fn an_emptied_namespace_leaves_nothing_behind() { tc.sanity_check(&[], &[]); // A later insert respins the namespace from scratch. tc.insert(&InsertParams { + rotation_base: None, namespace: KeyNamespaceRef::new(Some("salted"), None), ..insert_params(&vec![1, 2], &[10, 11]) }); @@ -8666,6 +8937,7 @@ fn namespaces_do_not_accumulate_across_salts() { for salt in 0..64 { let salt = format!("session-{salt}"); tc.insert(&InsertParams { + rotation_base: None, namespace: KeyNamespaceRef::new(Some(&salt), None), ..insert_params(&vec![1, 2], &[10, 11]) }); @@ -8680,6 +8952,7 @@ fn namespaces_do_not_accumulate_across_salts() { fn a_zero_length_match_anchors_at_the_root() { let mut tc = core(); tc.insert(&InsertParams { + rotation_base: None, namespace: KeyNamespaceRef::new(Some("salted"), None), ..insert_params(&vec![1, 2], &[10, 11]) }); diff --git a/rust/sglang-radix-tree/src/unified_lru_list.rs b/rust/sglang-radix-tree/src/unified_lru_list.rs index 454a426614..18c90ab800 100644 --- a/rust/sglang-radix-tree/src/unified_lru_list.rs +++ b/rust/sglang-radix-tree/src/unified_lru_list.rs @@ -483,8 +483,65 @@ impl EvictionStrategy for SlruStrategy { } } +/// Tail-optimized LRU: reclaim whole nodes within the conversation's tail +/// budget before other nodes, using recency within each group. +pub struct TlruStrategy { + /// max(threshold - next_prompt_estimate, 0), in logical tokens. + pub tail_budget: usize, + /// Preserve Python's operation order when either parameter is a float. + pub float_config: Option, +} + +#[derive(Clone, Copy)] +pub struct TlruFloatConfig { + pub threshold: f64, + pub next_prompt_estimate: TlruPromptEstimate, +} + +/// How Python evaluates history + next_prompt_estimate before subtraction. +#[derive(Clone, Copy)] +pub enum TlruPromptEstimate { + Float(f64), + /// Add the actual history before converting to float, as Python does. + Integer(i128), +} + +impl TlruFloatConfig { + pub(crate) fn is_tel_safe(&self, history: usize, cached_without_node: usize) -> bool { + let next_prompt = match self.next_prompt_estimate { + TlruPromptEstimate::Float(estimate) => history as f64 + estimate, + TlruPromptEstimate::Integer(estimate) => estimate + .checked_add(history as i128) + .expect("T-LRU history + next_prompt_estimate overflowed i128") + as f64, + }; + let budget = next_prompt - self.threshold; + // Match Python's max(budget, 0) and exact int >= float comparison. + // NaN stays unsafe; casting the integer to f64 could round it upward. + budget <= 0.0 + || (budget < (usize::MAX as u128 + 1) as f64 + && cached_without_node >= budget.ceil() as usize) + } +} + +impl EvictionStrategy for TlruStrategy { + fn get_priority(&self, node: &Node) -> PriorityKey { + let cached_without_node = node.tlru_cached_prefix_len - node.key.atom_len(); + let tel_safe = match self.float_config { + Some(config) => config.is_tel_safe(node.tlru_history_len, cached_without_node), + None => node.tlru_history_len - cached_without_node <= self.tail_budget, + }; + PriorityKey(if tel_safe { -1 } else { 0 }, node.last_access_counter) + } +} + /// The strategy for an eviction-policy name. -pub fn get_eviction_strategy(policy: &str) -> Box + Send> { +pub fn get_eviction_strategy( + policy: &str, + slru_protected_threshold: i64, + tlru_tail_budget: usize, + tlru_float_config: Option, +) -> Box + Send> { match policy.to_lowercase().as_str() { "lru" => Box::new(LruStrategy), "lfu" => Box::new(LfuStrategy), @@ -493,11 +550,15 @@ pub fn get_eviction_strategy(policy: &str) -> Box Box::new(FiloStrategy), "priority" => Box::new(PriorityStrategy), "slru" => Box::new(SlruStrategy { - protected_threshold: 2, + protected_threshold: slru_protected_threshold, + }), + "tlru" => Box::new(TlruStrategy { + tail_budget: tlru_tail_budget, + float_config: tlru_float_config, }), other => panic!( "Unknown eviction policy: {other}. Supported policies: \ - 'lru', 'lfu', 'fifo', 'mru', 'filo', 'priority', 'slru'." + 'lru', 'lfu', 'fifo', 'mru', 'filo', 'priority', 'slru', 'tlru'." ), } } diff --git a/rust/sglang-radix-tree/src/unified_tree_core.rs b/rust/sglang-radix-tree/src/unified_tree_core.rs index 74f9938a36..e39fd3743a 100644 --- a/rust/sglang-radix-tree/src/unified_tree_core.rs +++ b/rust/sglang-radix-tree/src/unified_tree_core.rs @@ -14,15 +14,17 @@ use crate::components::{ use crate::components::{ BASE_COMPONENT_TYPE, ComponentType, FULL, MAMBA, NUM_COMPONENT_TYPES, SWA, }; -use crate::node::EvictableNodeSet; use crate::node::Node; use crate::node::NodeArena; +use crate::node::NodeSet; use crate::node::{ChildKeyType, HashDigest, KeyNamespace, KeyNamespaceRef}; use crate::node::{ NUM_VALUE_SLOTS, NodeAccessError, NodeId, NodeIdx_, TreeCoreRuntimeError, ValueSlotIdx, }; use crate::unified_lru_list::UnifiedLRUList; -use crate::unified_lru_list::{EvictionStrategy, PriorityKey, get_eviction_strategy}; +use crate::unified_lru_list::{ + EvictionStrategy, PriorityKey, TlruFloatConfig, get_eviction_strategy, +}; // A 42-bit mask keeps digest multiplication by 1_000_003 within i64. const COEXIST_RECLAIM_DIGEST_MULTIPLIER: i64 = 1_000_003; @@ -134,6 +136,8 @@ pub struct MatchPrefixParams<'k, K: ChildKeyType> { /// Params for an insert; the key is borrowed from the caller. pub struct InsertParams<'k, K: ChildKeyType> { + /// Logical-page owner rotation; None keeps unsharded insert behavior. + pub rotation_base: Option, /// The insert key (already page-typed; bigram conversion happens at the boundary). pub key: &'k K, /// Namespace of the insert; picks the matching subtree root. @@ -163,6 +167,8 @@ pub struct InsertParams<'k, K: ChildKeyType> { /// Result of an insert. #[derive(Default)] pub struct InsertResult { + /// The incoming pages use another chain rotation and were not adopted. + pub rotation_tail_declined: bool, /// Tokens of the insert key that overlapped existing nodes. pub prefix_len: usize, /// The inserted key's full (page-aligned) length. @@ -234,6 +240,7 @@ pub struct InsertWalkState { value: Tensor, namespace: KeyNamespace, session_id: Option>, + rotation_base: Option, prev_prefix_len: usize, swa_evicted_seqlen: usize, swa_branching_seqlen: Option, @@ -482,6 +489,11 @@ pub struct ComponentState { /// leaf may be freed: the leaf's parent for Full, the LRU predecessor for /// SWA and Mamba. pub(crate) evict_device_cursor: Option, + /// Internal component victim waiting for the controller's host backup attempt. + /// A generation-checked handle survives host eviction during that I/O. + pub(crate) evict_device_pending_node: Option, + /// SWA host capacity needed for the pending victim's complete backup window. + pub(crate) evict_device_pending_num_tokens: usize, /// Token budget for the current eviction walk. pub(crate) evict_device_request_cnt: usize, } @@ -490,6 +502,12 @@ pub struct ComponentState { pub struct CacheInitParams { /// Eviction-policy name resolved into the tree's strategy. pub eviction_policy: String, + /// Hit count at which SLRU promotes a node to the protected segment. + pub slru_protected_threshold: i64, + /// Nonnegative T-LRU threshold minus the next-prompt estimate, in tokens. + pub tlru_tail_budget: usize, + /// Original arithmetic for floating-point T-LRU parameters, when configured. + pub tlru_float_config: Option, /// Atoms per radix page; children are keyed by their key's first page. pub page_size: usize, /// Whether the cache runs the write-back (vs write-through) policy. @@ -502,6 +520,8 @@ pub struct CacheInitParams { pub device: Device, /// SWA sliding window size in tokens; None when SWA is disabled. pub swa_sliding_window_size: Option, + /// Whether SWA lives in a per-request ring rather than cached paged slots. + pub swa_req_ring: bool, /// Whether the cache wired a host SWA pool (HiCache). pub has_swa_host_pool: bool, /// Whether tree mutations emit BlockStored/BlockRemoved events. @@ -516,12 +536,16 @@ impl Default for CacheInitParams { fn default() -> Self { CacheInitParams { eviction_policy: "lru".to_string(), + slru_protected_threshold: 2, + tlru_tail_budget: 0, + tlru_float_config: None, page_size: 1, is_write_back: false, enable_hicache: false, write_through_threshold: 256, device: Device::Cpu, swa_sliding_window_size: None, + swa_req_ring: false, has_swa_host_pool: false, enable_kv_cache_events: false, mamba_cache_chunk_size: None, @@ -538,6 +562,14 @@ pub struct EvictionStepResult { pub tracker: HashMap, pub device_frees: HashMap>, pub host_frees: HashMap>, + /// Full device tokens freed without a host copy during this device step. + pub unbacked_tokens: usize, + /// Back up this internal Mamba state before resuming its device tombstone. + pub mamba_backup_node_id: Option, + /// Back up this internal SWA window before resuming its device tombstone. + pub swa_backup_node_id: Option, + /// Required SWA host capacity, including unbacked ancestors in the window. + pub swa_backup_num_tokens: usize, } /// The radix tree mechanism: owns the tree structure, per-node values, the @@ -552,18 +584,22 @@ pub struct UnifiedTreeCore { /// Per-component bookkeeping, indexed by `ComponentType::idx`. pub(crate) component_states: [ComponentState; NUM_COMPONENT_TYPES], /// Nodes currently eligible for device eviction (D-leaves). - pub(crate) evictable_device_leaves: EvictableNodeSet, + pub(crate) evictable_device_leaves: NodeSet, /// Nodes currently eligible for host eviction (H-leaves). - pub(crate) evictable_host_leaves: EvictableNodeSet, - /// Full has no device LRU, so track nodes whose device and host values coexist. - pub(crate) full_coexisting_host_nodes: EvictableNodeSet, + pub(crate) evictable_host_leaves: NodeSet, + /// Full has no device LRU; track host/device duplicates in insertion order. + pub(crate) full_coexisting_host_nodes: NodeSet, pub(crate) write_back_coexist_reclaim_digest: i64, + /// Present only during a device-eviction step, including its cascades. + tracked_unbacked_tokens: Option, /// Per-slot LRU lists, indexed by `ValueSlotIdx::idx`. pub(crate) lru_lists: [UnifiedLRUList; NUM_VALUE_SLOTS], /// Device-eviction candidates; the lowest priority is popped first. pub(crate) full_evict_device_heap: BinaryHeap>, /// Eviction-priority strategy; lower priority evicts first. pub(crate) eviction_strategy: Box + Send>, + /// Keep path depths and branch high-water marks only for T-LRU. + pub(crate) tlru_bookkeeping: bool, /// Atoms per radix page; children are keyed by their key's first page. pub(crate) page_size: usize, /// Whether the cache runs the write-back (vs write-through) policy. @@ -691,6 +727,8 @@ impl UnifiedTreeCore { state.is_evict_device_ongoing = true; state.evict_device_request_cnt = request_cnt; state.evict_device_cursor = None; + state.evict_device_pending_node = None; + state.evict_device_pending_num_tokens = 0; } /// Finish the component's device-eviction bookkeeping; panics if no walk @@ -703,6 +741,8 @@ impl UnifiedTreeCore { ); state.is_evict_device_ongoing = false; state.evict_device_cursor = None; + state.evict_device_pending_node = None; + state.evict_device_pending_num_tokens = 0; } /// Add newly evictable device tokens to the component's evictable size. @@ -747,14 +787,21 @@ impl UnifiedTreeCore { components: Vec::new(), components_by_type: Default::default(), component_states: Default::default(), - evictable_device_leaves: EvictableNodeSet::new(), - evictable_host_leaves: EvictableNodeSet::new(), - full_coexisting_host_nodes: EvictableNodeSet::new(), + evictable_device_leaves: NodeSet::new(), + evictable_host_leaves: NodeSet::new(), + full_coexisting_host_nodes: NodeSet::new(), write_back_coexist_reclaim_digest: 0, + tracked_unbacked_tokens: None, // Disabled components keep harmless empty lists, like component_states. lru_lists: Self::new_lru_lists(), full_evict_device_heap: BinaryHeap::new(), - eviction_strategy: get_eviction_strategy(¶ms.eviction_policy), + eviction_strategy: get_eviction_strategy( + ¶ms.eviction_policy, + params.slru_protected_threshold, + params.tlru_tail_budget, + params.tlru_float_config, + ), + tlru_bookkeeping: params.eviction_policy.eq_ignore_ascii_case("tlru"), page_size: params.page_size, is_write_back: params.is_write_back, enable_hicache: params.enable_hicache, @@ -787,10 +834,11 @@ impl UnifiedTreeCore { pub fn reset(&mut self) { self.arena.reset(); self.component_states = Default::default(); - self.evictable_device_leaves = EvictableNodeSet::new(); - self.evictable_host_leaves = EvictableNodeSet::new(); - self.full_coexisting_host_nodes = EvictableNodeSet::new(); + self.evictable_device_leaves = NodeSet::new(); + self.evictable_host_leaves = NodeSet::new(); + self.full_coexisting_host_nodes = NodeSet::new(); self.write_back_coexist_reclaim_digest = 0; + self.tracked_unbacked_tokens = None; self.lru_lists = Self::new_lru_lists(); self.full_evict_device_heap.clear(); self.namespaced_event_hashes.clear(); @@ -967,6 +1015,32 @@ impl UnifiedTreeCore { Ok(DecLockRefResult::default()) } + /// Release one window's lock without releasing other components. + pub fn dec_window_lock_only( + &mut self, + node_id: NodeId, + component_type: ComponentType, + params: &DecLockRefParams, + device_frees: &mut HashMap>, + host_frees: &mut HashMap>, + ) -> Result<(), NodeAccessError> { + let node_idx = self.arena.resolve(node_id)?; + self.assert_receipt_anchor_(node_idx, params); + if self.arena.node(node_idx).is_root() + || params.skipped_lock_components.contains(component_type) + { + return Ok(()); + } + self.component_by_type_(component_type).release_window_lock( + self, + node_idx, + params, + device_frees, + host_frees, + ); + Ok(()) + } + /// Early-release the SWA portion of a request's tree lock, plus any /// strictly-lower-priority locks (e.g. Mamba) co-located on the node. pub fn dec_swa_lock_only( @@ -1492,6 +1566,7 @@ impl UnifiedTreeCore { last_device_node_id: Some(self.arena.node(root_id).id), inserted_host_node: None, host_insert_dropped: false, + rotation_tail_declined: false, mamba_exist: true, swa_branch_inserted: false, adopted_ranges: None, @@ -1505,6 +1580,21 @@ impl UnifiedTreeCore { let node = self.arena.node_mut(root_id); node.priority = node.priority.max(params.priority); } + if let Some(rotation_base) = params.rotation_base + && let Some((prefix_len, node_id)) = + self.rotation_conflict_(params, aligned_key_len, rotation_base) + { + return Ok(InsertStepResult { + actions: Vec::new(), + result: Some(InsertResult { + prefix_len, + last_device_node_id: Some(self.arena.node(node_id).id), + rotation_tail_declined: true, + adopted_ranges: params.track_adopted_ranges.then(HashMap::new), + ..InsertResult::default() + }), + }); + } // The walk reads only [0, aligned_key_len); the ragged tail never enters. self.ongoing_insert_walk_state = Some(InsertWalkState { phase: InsertPhase::Walk, @@ -1514,6 +1604,7 @@ impl UnifiedTreeCore { value: params.value.narrow(0, 0, aligned_key_len as i64), namespace: params.namespace.to_owned(), session_id: params.session_id.map(Arc::from), + rotation_base: params.rotation_base, prev_prefix_len: params.prev_prefix_len, swa_evicted_seqlen: params.swa_evicted_seqlen, swa_branching_seqlen: params.swa_branching_seqlen, @@ -1533,6 +1624,40 @@ impl UnifiedTreeCore { Ok(self.advance_insert_()) } + /// Check the matched chain before insert can restore, repoint, split, or + /// free any incoming pages. A mixed rotation would invalidate the cyclic + /// page-owner mapping used by logical-page KV sharding. + fn rotation_conflict_( + &self, + params: &InsertParams<'_, K>, + aligned_key_len: usize, + rotation_base: i64, + ) -> Option<(usize, NodeIdx_)> { + let mut node_id = self.arena.root(); + let mut matched = 0; + while matched < aligned_key_len { + let Some(child_id) = self.arena.child_on_page_in_namespace( + node_id, + params.namespace, + params.key.page_at(matched, self.page_size), + ) else { + break; + }; + let child = self.arena.node(child_id); + let prefix_len = params + .key + .match_len(matched, &child.key, self.page_size) + .min(aligned_key_len - matched); + node_id = child_id; + matched += prefix_len; + if prefix_len < child.key.atom_len() { + break; + } + } + let node = self.arena.node(node_id); + (!node.is_root() && node.rotation_base != Some(rotation_base)).then_some((matched, node_id)) + } + /// Continue the suspended insert after its step actions were executed. pub fn resume_insert(&mut self) -> InsertStepResult { self.try_resume_insert() @@ -1637,6 +1762,7 @@ impl UnifiedTreeCore { } let params = InsertParams { + rotation_base: state.rotation_base, key: &state.key, namespace: state.namespace.as_ref(), session_id: state.session_id.as_deref(), @@ -1765,7 +1891,7 @@ impl UnifiedTreeCore { state.total_prefix_length as i64, (state.aligned_key_len - state.total_prefix_length) as i64, ); - self.add_new_node_in_namespace_( + let new_node_id = self.add_new_node_in_namespace_( state.node_id, K::from( state.key.as_ref()[state.total_prefix_length..state.aligned_key_len].to_vec(), @@ -1774,7 +1900,9 @@ impl UnifiedTreeCore { state.priority, state.namespace.as_ref(), state.session_id.as_deref(), - ) + ); + self.arena.node_mut(new_node_id).rotation_base = state.rotation_base; + new_node_id } else { state.node_id }; @@ -1791,6 +1919,7 @@ impl UnifiedTreeCore { result.prefix_len = state.total_prefix_length; result.last_device_node_id = Some(self.arena.node(target_node_id).id); let params = InsertParams { + rotation_base: state.rotation_base, key: &state.key, namespace: state.namespace.as_ref(), session_id: state.session_id.as_deref(), @@ -1888,6 +2017,8 @@ impl UnifiedTreeCore { let parent_id = child.parent(); let child_namespace = child.namespace.clone(); let child_external_cache_stored = child.external_cache_stored; + let child_rotation_base = child.rotation_base; + let child_tlru_history_len = child.tlru_history_len; let (key_head, key_tail) = child.key.split_at(split_len); // key_head keeps the original key's first page, which keys the parent's child map. let parent_map_key = key_head.child_key(page_size); @@ -1904,6 +2035,19 @@ impl UnifiedTreeCore { child_id, ); self.arena.node_mut(new_node_id).external_cache_stored = child_external_cache_stored; + self.arena.node_mut(new_node_id).rotation_base = child_rotation_base; + if self.tlru_bookkeeping { + // Splitting preserves the suffix's depth and branch history. + let prefix_len = self + .arena + .node(parent_id) + .tlru_cached_prefix_len + .checked_add(split_len) + .expect("T-LRU path length overflow"); + let prefix = self.arena.node_mut(new_node_id); + prefix.tlru_cached_prefix_len = prefix_len; + prefix.tlru_history_len = child_tlru_history_len; + } let child = self.arena.node_mut(child_id); child.parent = Some(new_node_id); @@ -1969,6 +2113,32 @@ impl UnifiedTreeCore { (new_node_id, action) } + /// Record a new tail's depth and raise ancestor history without shrinking + /// it on eviction. Only T-LRU needs this O(path nodes) worst-case walk. + fn set_tlru_lens_and_raise_history_(&mut self, node_id: NodeIdx_, parent_id: NodeIdx_) { + if !self.tlru_bookkeeping { + return; + } + let depth = self + .arena + .node(parent_id) + .tlru_cached_prefix_len + .checked_add(self.arena.node(node_id).key.atom_len()) + .expect("T-LRU path length overflow"); + let node = self.arena.node_mut(node_id); + node.tlru_cached_prefix_len = depth; + node.tlru_history_len = node.tlru_history_len.max(depth); + let mut ancestor = Some(parent_id); + while let Some(ancestor_id) = ancestor { + let node = self.arena.node_mut(ancestor_id); + if node.tlru_history_len >= depth { + break; + } + node.tlru_history_len = depth; + ancestor = node.try_parent(); + } + } + /// Create a leaf holding `value` under `parent`. pub fn add_new_node_( &mut self, @@ -2004,6 +2174,7 @@ impl UnifiedTreeCore { namespace, ); self.arena.set_device_value(new_node_id, FULL, value.copy()); + self.set_tlru_lens_and_raise_history_(new_node_id, parent_id); let displaced = self .arena .insert_child_edge(parent_id, child_map_key, new_node_id); @@ -2170,10 +2341,18 @@ impl UnifiedTreeCore { component_type: ComponentType, baseline: &HashMap, ) -> (Option, EvictionStepResult) { + assert!( + self.component_state(component_type) + .evict_device_pending_node + .is_none(), + "finish the pending internal {component_type:?} eviction before advancing" + ); let mut tracker = baseline.clone(); // The walk gates on the walked component's entry, so seed it. tracker.entry(component_type).or_insert(0); let mut result = EvictionStepResult::default(); + assert!(self.tracked_unbacked_tokens.is_none()); + self.tracked_unbacked_tokens = Some(0); let node_id = self .component_by_type_(component_type) .evict_device_next_node( @@ -2182,6 +2361,14 @@ impl UnifiedTreeCore { &mut result.device_frees, &mut result.host_frees, ); + result.unbacked_tokens = self.tracked_unbacked_tokens.take().unwrap(); + let state = self.component_state(component_type); + if component_type == MAMBA { + result.mamba_backup_node_id = state.evict_device_pending_node; + } else if component_type == SWA { + result.swa_backup_node_id = state.evict_device_pending_node; + result.swa_backup_num_tokens = state.evict_device_pending_num_tokens; + } for (ct, total) in tracker { let delta = total - baseline.get(&ct).copied().unwrap_or(0); if delta > 0 { @@ -2191,6 +2378,83 @@ impl UnifiedTreeCore { (node_id.map(|idx| self.arena.node(idx).id), result) } + /// Finish an internal Mamba eviction after the controller has attempted + /// and acknowledged its host backup. A failed backup still drops the + /// state so an imminent device allocation can make progress. + pub fn finish_mamba_state_eviction(&mut self, node_id: NodeId) -> EvictionStepResult { + self.finish_internal_component_eviction_(MAMBA, node_id) + } + + /// Finish an internal SWA eviction after its best-effort host backup. + pub fn finish_swa_state_eviction(&mut self, node_id: NodeId) -> EvictionStepResult { + self.finish_internal_component_eviction_(SWA, node_id) + } + + fn finish_internal_component_eviction_( + &mut self, + component_type: ComponentType, + node_id: NodeId, + ) -> EvictionStepResult { + let state = self.component_state_mut(component_type); + assert!( + state.is_evict_device_ongoing, + "{component_type:?} device eviction not started" + ); + assert_eq!( + state.evict_device_pending_node, + Some(node_id), + "no matching pending internal {component_type:?} eviction" + ); + state.evict_device_pending_node = None; + state.evict_device_pending_num_tokens = 0; + let mut result = EvictionStepResult::default(); + // Host eviction or a caller's intervening mutation may have removed + // the node, pinned its state, or changed the component's LRU. + let Ok(node_idx) = self.arena.resolve(node_id) else { + return result; + }; + let node = self.arena.node(node_idx); + // Own-component locks protect DMA sources; another component's + // transfer may mark this node without using its restored state. + // Match the Mamba walk's load-back guard. SWA can be independently + // evicted outside a pending Full load-back's restored SWA window. + if !node.has_device_value(component_type) + || self.arena.device_lock_ref(node_idx, component_type) > 0 + || (component_type == MAMBA && node.is_load_back_pending()) + || !self.device_lru_list(component_type).in_list(Some(node_idx)) + { + return result; + } + if self.evictable_device_leaves.contains(node_idx) + || (component_type == MAMBA && !node.has_device_value(FULL)) + { + // Re-select through the ordinary walk if the victim's shape has + // changed; an internal-state finish must never demote Full KV. + self.component_state_mut(component_type).evict_device_cursor = Some(node_idx); + return result; + } + assert!(self.tracked_unbacked_tokens.is_none()); + self.tracked_unbacked_tokens = Some(0); + self.evict_component_and_detach_lru_( + node_idx, + component_type, + &mut result.device_frees, + &mut result.host_frees, + EvictLayer::Device, + Some(&mut result.tracker), + ); + self.cascade_evict_( + node_idx, + component_type, + &mut result.tracker, + &mut result.device_frees, + &mut result.host_frees, + EvictLayer::Device, + ); + result.unbacked_tokens = self.tracked_unbacked_tokens.take().unwrap(); + result + } + /// Finish a component's device-eviction walk. pub fn evict_device_end(&mut self, component_type: ComponentType) { self.component_by_type_(component_type) @@ -2214,6 +2478,8 @@ impl UnifiedTreeCore { "node {node_id} is not a D-leaf" ); } + assert!(self.tracked_unbacked_tokens.is_none()); + self.tracked_unbacked_tokens = Some(0); if self.arena.node(node_id).backuped() { self.demote_( node_id, @@ -2221,11 +2487,13 @@ impl UnifiedTreeCore { &mut result.device_frees, &mut result.host_frees, ); + result.unbacked_tokens = self.tracked_unbacked_tokens.take().unwrap(); return Ok((None, result)); } if is_write_back { let backup = self .build_backup_kv_action_(self.arena.node(node_id), /* write_back = */ true); + result.unbacked_tokens = self.tracked_unbacked_tokens.take().unwrap(); return Ok((Some(backup), result)); } @@ -2236,6 +2504,7 @@ impl UnifiedTreeCore { &mut result.device_frees, &mut result.host_frees, ); + result.unbacked_tokens = self.tracked_unbacked_tokens.take().unwrap(); Ok((None, result)) } @@ -2359,12 +2628,25 @@ impl UnifiedTreeCore { &mut self, component_type: ComponentType, num_tokens: usize, + ) -> EvictionStepResult { + self.drive_host_eviction_with_options(component_type, num_tokens, false) + } + + /// Skip Full duplicate reclaim when requested by the controller, while + /// preserving the normal host-leaf eviction walk. + pub fn drive_host_eviction_with_options( + &mut self, + component_type: ComponentType, + num_tokens: usize, + skip_full_duplicate_reclaim: bool, ) -> EvictionStepResult { let mut result = EvictionStepResult::default(); if let Some(component) = self.try_component_by_type_(component_type) { // The drive gates on the driven component's entry, so seed it. result.tracker.insert(component_type, 0); - if self.is_write_back { + if self.is_write_back + && !(skip_full_duplicate_reclaim && component_type == BASE_COMPONENT_TYPE) + { component.reclaim_coexisting_host_values( self, num_tokens, @@ -2670,6 +2952,7 @@ impl UnifiedTreeCore { target: EvictLayer, tracker: Option<&mut HashMap>, ) -> (usize, usize) { + let had_host_copy = self.arena.has_host_value(node_id, component_type); let component = self.component_by_type_(component_type); let (device_freed, host_freed) = component.evict_component(self, node_id, device_frees, host_frees, target); @@ -2681,6 +2964,12 @@ impl UnifiedTreeCore { }; *tracker.entry(component_type).or_insert(0) += freed; } + if component_type == BASE_COMPONENT_TYPE + && !had_host_copy + && let Some(unbacked_tokens) = self.tracked_unbacked_tokens.as_mut() + { + *unbacked_tokens += device_freed; + } // Detach from the targeted LRU list(s). if target.contains(EvictLayer::Device) { @@ -3120,6 +3409,7 @@ impl UnifiedTreeCore { last_device_node_id: None, inserted_host_node: None, host_insert_dropped: false, + rotation_tail_declined: false, mamba_exist: true, swa_branch_inserted: false, adopted_ranges: None, @@ -3160,6 +3450,7 @@ impl UnifiedTreeCore { last_device_node_id: None, inserted_host_node: None, host_insert_dropped: false, + rotation_tail_declined: false, mamba_exist: false, swa_branch_inserted: false, adopted_ranges: None, @@ -3192,6 +3483,7 @@ impl UnifiedTreeCore { /* creation_counter = */ None, namespace, ); + self.set_tlru_lens_and_raise_history_(new_node_id, node_id); { // The suffix moves into a right-sized list; the matched head drops. let mut hash_value = hash_value; @@ -3570,6 +3862,12 @@ impl UnifiedTreeCore { self.arena.node(self.arena.root()).id } + /// The chain's logical-page rotation, or None on the root/unsharded nodes. + pub fn rotation_base_of(&self, node_id: NodeId) -> Result, NodeAccessError> { + let node_id = self.arena.resolve(node_id)?; + Ok(self.arena.node(node_id).rotation_base) + } + /// Return input indices in depth-first, subtree-weight order. pub fn dfs_weight_order(&self, node_ids: &[NodeId]) -> Result, NodeAccessError> { let mut node_to_indices: HashMap> = HashMap::new(); @@ -3723,7 +4021,9 @@ impl UnifiedTreeCore { ); } assert!(cache_actions.is_empty()); // BACKUP_HOST emits no actions - self.update_full_coexisting_host_tracking_(node_id); + // The controller marks the transfer pending after this commit. Register + // the duplicate in finish_write_through, once its acknowledgment also + // establishes the ancestor-first order of any intervening split. Ok(()) } @@ -3981,6 +4281,175 @@ impl UnifiedTreeCore { Ok(()) } + /// Snapshot matched nodes and their spans without splitting or refreshing them. + fn collect_matched_spans_( + &self, + key: &K, + namespace: KeyNamespaceRef<'_>, + end: usize, + ) -> Vec<(NodeIdx_, usize, usize)> { + let mut spans = Vec::new(); + let mut node_id = self.arena.root(); + let mut pos = 0; + while pos < end && key.atom_len() - pos >= self.page_size { + let Some(child_id) = self.arena.child_on_page_in_namespace( + node_id, + namespace, + key.page_at(pos, self.page_size), + ) else { + break; + }; + let child = self.arena.node(child_id); + if !child.has_device_value(FULL) && !child.has_host_value(FULL) { + break; + } + let prefix_len = key.match_len(pos, &child.key, self.page_size); + if prefix_len == 0 { + break; + } + spans.push((child_id, pos, prefix_len)); + if prefix_len < child.key.atom_len() { + break; + } + node_id = child_id; + pos += prefix_len; + } + spans + } + + fn validate_swa_span_(&self, key: &K, start: usize, end: usize) -> Result<(), String> { + if self.components_by_type[SWA.idx()].is_none() { + return Err("SWA component is not enabled".into()); + } + if start > end || end > key.atom_len() { + return Err(format!( + "invalid SWA span [{start}, {end}) for key length {}", + key.atom_len() + )); + } + Ok(()) + } + + /// Maximal SWA-device-tombstoned ranges within the requested key span. + /// Host-backed FULL nodes remain traversable; an absent FULL copy stops the walk. + pub fn swa_tombstone_ranges( + &self, + key: &K, + namespace: KeyNamespaceRef<'_>, + start: usize, + end: usize, + ) -> Result, String> { + self.validate_swa_span_(key, start, end)?; + let mut ranges: Vec<(usize, usize)> = Vec::new(); + if start == end { + return Ok(ranges); + } + for (child_id, pos, prefix_len) in self.collect_matched_spans_(key, namespace, end) { + let seg_end = pos + prefix_len; + if seg_end <= start || self.arena.has_device_value(child_id, SWA) { + continue; + } + let lo = start.max(pos); + let hi = end.min(seg_end); + if let Some(last) = ranges.last_mut() + && last.1 == lo + { + last.1 = hi; + } else { + ranges.push((lo, hi)); + } + if hi >= end { + break; + } + } + Ok(ranges) + } + + /// Publish loaded SWA slots into an existing tombstoned span. + /// Validate the entire operation before splitting or publishing any values. + pub fn attach_swa_window( + &mut self, + key: &K, + namespace: KeyNamespaceRef<'_>, + window_start: usize, + window_end: usize, + swa_values: &Tensor, + ) -> Result, String> { + self.validate_swa_span_(key, window_start, window_end)?; + if swa_values.kind() != Kind::Int64 || swa_values.device() != self.device { + return Err(format!( + "attach_swa_window requires int64 values on {:?}", + self.device + )); + } + if swa_values.size() != [((window_end - window_start) as i64)] { + return Err(format!( + "attach_swa_window size mismatch: got {:?} for [{window_start}, {window_end})", + swa_values.size() + )); + } + if window_start == window_end { + return Ok(Vec::new()); + } + // Buffer-mode staging rounds the SWA window to complete tree pages; + // matched prefixes and storage-hit lengths use those same logical pages. + // Splitting at an unaligned offset would break the radix page keys. + if !window_start.is_multiple_of(self.page_size) + || !window_end.is_multiple_of(self.page_size) + { + return Err("attach_swa_window boundaries must be page aligned".into()); + } + let spans = self.collect_matched_spans_(key, namespace, window_end); + let mut covered = window_start; + for &(child_id, pos, prefix_len) in &spans { + if pos + prefix_len <= window_start { + continue; + } + if self.arena.has_device_value(child_id, SWA) { + return Err(format!( + "attach_swa_window over live SWA at [{}, {})", + window_start.max(pos), + window_end.min(pos + prefix_len) + )); + } + covered = window_end.min(pos + prefix_len); + } + if covered != window_end { + return Err(format!( + "attach_swa_window covered {covered} of [{window_start}, {window_end})" + )); + } + + let mut actions = Vec::new(); + for (child_id, pos, prefix_len) in spans { + let seg_end = window_end.min(pos + prefix_len); + if seg_end <= window_start { + continue; + } + let seg_start = window_start.max(pos); + let mut target = child_id; + if seg_start > pos { + // The original handle remains on the suffix, our target. + let (_, action) = self.split_node_(target, seg_start - pos); + actions.extend(action); + } + if seg_start + self.arena.node(target).key.atom_len() > seg_end { + let (fragment, action) = self.split_node_(target, seg_end - seg_start); + actions.extend(action); + target = fragment; + } + let values = swa_values + .narrow( + 0, + (seg_start - window_start) as i64, + (seg_end - seg_start) as i64, + ) + .copy(); + self.set_component_device_value_(target, SWA, values); + } + Ok(actions) + } + /// Store an auxiliary component's device value onto a node and restamp /// its LRU. pub fn set_component_device_value( @@ -4022,6 +4491,17 @@ impl UnifiedTreeCore { } } + /// Logical radix-key lengths for transfer nodes, preserving caller order. + pub fn get_node_key_lengths(&self, node_ids: &[NodeId]) -> Result, NodeAccessError> { + node_ids + .iter() + .map(|&node_id| { + let node_id = self.arena.resolve(node_id)?; + Ok(self.arena.node(node_id).key.atom_len()) + }) + .collect() + } + /// The component's device value on the node, or None if evicted. pub fn get_component_device_value( &self, diff --git a/test/registered/chunked_prefill/test_scripted_core_1gpu.py b/test/registered/chunked_prefill/test_scripted_core_1gpu.py index e37010d756..db4cbf9f44 100644 --- a/test/registered/chunked_prefill/test_scripted_core_1gpu.py +++ b/test/registered/chunked_prefill/test_scripted_core_1gpu.py @@ -159,17 +159,20 @@ class TestScriptedCore(ScriptedTestCase): _assert_prefill_twice_decode_once(t, prompt_len=prompt_len) -def _assert_prefill_twice_decode_once(t: ScriptedContext, *, prompt_len: int) -> None: - root = t.scheduler.tree_cache.root_node +def _assert_prefill_twice_decode_once(t: ScriptedContext, prompt_len: int) -> None: + core = t.scheduler.tree_cache.tree_core prefill_hits: list[int] = [] decode_hits: list[int] = [] - stack = [(child, len(child.key)) for child in root.children.values()] + stack = [ + (node_id, core.get_node_key_length(node_id)) + for node_id in core.get_child_node_ids(core.root_node_handle()) + ] while stack: - node, end_index = stack.pop() + node_id, end_index = stack.pop() bucket = prefill_hits if end_index <= prompt_len else decode_hits - bucket.append(node.hit_count) - for child in node.children.values(): - stack.append((child, end_index + len(child.key))) + bucket.append(core.get_node_hit_count(node_id)) + for child_id in core.get_child_node_ids(node_id): + stack.append((child_id, end_index + core.get_node_key_length(child_id))) assert prefill_hits and decode_hits, ( f"expected both prefill and decode radix nodes; " diff --git a/test/registered/disaggregation/test_disaggregation_optimistic_prefill.py b/test/registered/disaggregation/test_disaggregation_optimistic_prefill.py index b8be096f45..1982ee9bcd 100644 --- a/test/registered/disaggregation/test_disaggregation_optimistic_prefill.py +++ b/test/registered/disaggregation/test_disaggregation_optimistic_prefill.py @@ -245,6 +245,10 @@ class TestOptimisticPrefillL3BufferWriteThrough( retries wait for decode, so retries that recover the prefix from L3 run under real load; the gsm8k score is the correctness check.""" + # 32 layers x K/V x 16 fragmented pages produce at most 1024 transfer blocks, + # keeping each synchronous batch within the TCP fallback admission budget. + extra_prefill_env = {"SGLANG_MOONCAKE_MAX_TRANSFER_BATCH_INDICES": "16"} + @classmethod def setUpClass(cls): cls.hicache_dir = tempfile.mkdtemp(prefix="sglang-hicache-") diff --git a/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_full.py b/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_full.py index b7588ac6fd..ff50de3528 100644 --- a/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_full.py +++ b/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_full.py @@ -7,7 +7,6 @@ from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, - is_in_amd_ci, popen_launch_server, terminate_and_kill_process_tree, unified_radix_tree_server_env, @@ -50,7 +49,6 @@ class TestUnifiedFullRadixCache(UnifiedRadixTreeTestMixin, CustomTestCase): terminate_and_kill_process_tree(cls.process, wait_timeout=60) -@unittest.skipIf(is_in_amd_ci(), "Rust TreeCore is not packaged in AMD CI") class TestRustUnifiedFullRadixCache(TestUnifiedFullRadixCache): tree_core_backend = "rust" diff --git a/test/registered/unit/mem_cache/test_page_interleave_shard.py b/test/registered/unit/mem_cache/test_page_interleave_shard.py index 275edb238a..e714aed944 100644 --- a/test/registered/unit/mem_cache/test_page_interleave_shard.py +++ b/test/registered/unit/mem_cache/test_page_interleave_shard.py @@ -40,6 +40,7 @@ is every run of the CPU suite this file is registered to. """ import os +import shutil import unittest import unittest.mock from array import array @@ -48,11 +49,13 @@ from types import SimpleNamespace import torch import torch.multiprocessing as mp +from parameterized import parameterized_class from sglang.srt.distributed import ( init_distributed_environment, initialize_model_parallel, ) +from sglang.srt.environ import envs from sglang.srt.managers.schedule_batch import ReqKvInfo from sglang.srt.mem_cache import page_interleave from sglang.srt.mem_cache.allocator.page_interleave import ( @@ -83,7 +86,6 @@ from sglang.srt.mem_cache.page_interleave_pool import ( ) from sglang.srt.mem_cache.radix_cache import RadixKey from sglang.srt.mem_cache.unified_cache.components import ComponentType -from sglang.srt.mem_cache.unified_cache.unified_tree_core import UnifiedTreeCore from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache from sglang.srt.runtime_context import get_parallel, publish from sglang.srt.server_args import ServerArgs @@ -649,16 +651,25 @@ def _insert(tree, tokens, rotation_base=None, value=None): ) -def _node(tree, node_id): - return tree.tree_core.node_by_id(node_id) - - def _match_len(tree, tokens): res = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", tokens)))) return len(res.device_indices) -class TestUnifiedRotationBase(CustomTestCase): +class _TreeCoreBackendCase(CustomTestCase): + def setUp(self): + super().setUp() + if self.tree_core_backend == "rust" and shutil.which("cargo") is None: + self.skipTest("the Rust backend builds with cargo") + override = envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.override( + self.tree_core_backend + ) + override.__enter__() + self.addCleanup(override.__exit__, None, None, None) + + +@parameterized_class(("tree_core_backend",), [("python",), ("rust",)]) +class TestUnifiedRotationBase(_TreeCoreBackendCase): """The host rotation base on UnifiedTreeNode: the one new piece of metadata. The Full component's value is a device tensor, so the base must survive inserts and splits purely host-side or the alloc path gains a D2H @@ -673,12 +684,12 @@ class TestUnifiedRotationBase(CustomTestCase): probe = list(range(8)) + [99, 98, 97, 96] _insert(tree, probe, rotation_base=2) res = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", probe)))) - tail = _node(tree, res.last_device_node) - self.assertEqual(tail.rotation_base, 2) - parent = tail.parent - self.assertEqual(parent.rotation_base, 2) - for child in parent.children.values(): - self.assertEqual(child.rotation_base, 2) + self.assertEqual(tree.rotation_base_of(res.last_device_node), 2) + for tokens in (list(range(8)), list(range(12))): + matched = tree.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", tokens))) + ) + self.assertEqual(tree.rotation_base_of(matched.last_device_node), 2) def test_new_chain_gets_its_own_base(self): tree = _unified_tree() @@ -688,8 +699,8 @@ class TestUnifiedRotationBase(CustomTestCase): r2 = tree.match_prefix( MatchPrefixParams(key=RadixKey(array("q", range(100, 108)))) ) - self.assertEqual(_node(tree, r1.last_device_node).rotation_base, 1) - self.assertEqual(_node(tree, r2.last_device_node).rotation_base, 3) + self.assertEqual(tree.rotation_base_of(r1.last_device_node), 1) + self.assertEqual(tree.rotation_base_of(r2.last_device_node), 3) def test_extension_tail_node_stamped_from_request(self): tree = _unified_tree() @@ -698,13 +709,13 @@ class TestUnifiedRotationBase(CustomTestCase): # tail node with the (same, chain-constant) base. _insert(tree, list(range(16)), rotation_base=1) res = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", range(16))))) - self.assertEqual(_node(tree, res.last_device_node).rotation_base, 1) + self.assertEqual(tree.rotation_base_of(res.last_device_node), 1) def test_unsharded_inserts_keep_none(self): tree = _unified_tree() _insert(tree, list(range(8))) res = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", range(8))))) - self.assertIsNone(_node(tree, res.last_device_node).rotation_base) + self.assertIsNone(tree.rotation_base_of(res.last_device_node)) def test_rotation_base_of_reads_through_the_cache_boundary(self): """The alloc path holds a NodeId, not a node: the base must be @@ -719,12 +730,13 @@ class TestUnifiedRotationBase(CustomTestCase): self.assertIsNone(tree.rotation_base_of(tree.tree_core.root_node_handle())) -class TestShardedCoreGate(CustomTestCase): +@parameterized_class(("tree_core_backend",), [("python",), ("rust",)]) +class TestShardedCoreGate(_TreeCoreBackendCase): """A tree core that does not model rotation_base would never decline a cross-base graft. Pairing one with a sharded allocator must fail at construction, not produce wrong-owner gathers at serve time.""" - def test_python_core_supports_rotation_base(self): + def test_core_supports_rotation_base(self): tree = _unified_tree() self.assertTrue(tree.tree_core.supports_rotation_base) @@ -739,24 +751,25 @@ class TestShardedCoreGate(CustomTestCase): tree_components=(ComponentType.FULL,), ) with unittest.mock.patch.object( - UnifiedTreeCore, "supports_rotation_base", False + type(tree.tree_core), "supports_rotation_base", False ): with self.assertRaisesRegex(ValueError, "rotation bases"): UnifiedRadixCache(params) def test_unsharded_allocator_accepts_any_core(self): + tree = _unified_tree() params = CacheInitParams( disable=False, req_to_token_pool=ReqToTokenPool( size=8, max_context_len=128, device="cpu", enable_memory_saver=False ), - token_to_kv_pool_allocator=_unified_tree().token_to_kv_pool_allocator, + token_to_kv_pool_allocator=tree.token_to_kv_pool_allocator, page_size=4, eviction_policy="lru", tree_components=(ComponentType.FULL,), ) with unittest.mock.patch.object( - UnifiedTreeCore, "supports_rotation_base", False + type(tree.tree_core), "supports_rotation_base", False ): UnifiedRadixCache(params) # no raise: sharding is off @@ -785,7 +798,8 @@ class _GraftReq: return array("q", self.fill_ids) -class TestRotationGraftDecline(CustomTestCase): +@parameterized_class(("tree_core_backend",), [("python",), ("rust",)]) +class TestRotationGraftDecline(_TreeCoreBackendCase): """The overlap disagg-prefill loop plans batch t+1 before batch t's radix insert lands, so two requests sharing a prefix can allocate under different rotation bases. Grafting the second one's tail under the first @@ -817,7 +831,7 @@ class TestRotationGraftDecline(CustomTestCase): freed.extend(torch.as_tensor(seg).clone() for seg, _start in segments) return real_free_segments(segments) - def spy_segment(free_index, *, start_pos): + def spy_segment(free_index, start_pos): freed.append(torch.as_tensor(free_index).clone()) return real_free_segment(free_index, start_pos=start_pos) diff --git a/test/registered/unit/mem_cache/test_rust_tree_core.py b/test/registered/unit/mem_cache/test_rust_tree_core.py index ad46007d03..b7b0968459 100644 --- a/test/registered/unit/mem_cache/test_rust_tree_core.py +++ b/test/registered/unit/mem_cache/test_rust_tree_core.py @@ -3,25 +3,24 @@ Requires a Rust toolchain: the extension builds with cargo on first use. """ -import shutil +import os from array import array +from unittest import mock import pytest import torch from unified_tree_core_inspection_interface import UnifiedTreeCoreInspectionInterface -from sglang.test.ci.ci_register import register_cpu_ci - -register_cpu_ci(est_time=17, suite="base-a-test-cpu") - -if shutil.which("cargo") is None: - pytest.skip("the rust backend builds with cargo", allow_module_level=True) - +from sglang.srt.environ import envs from sglang.srt.mem_cache.base_prefix_cache import InsertParams, MatchPrefixParams from sglang.srt.mem_cache.cache_init_params import CacheInitParams from sglang.srt.mem_cache.radix_cache import RadixKey from sglang.srt.mem_cache.unified_cache.component_type import ComponentType from sglang.srt.mem_cache.unified_cache.tree_core_registry import create_tree_core +from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=17, suite="base-a-test-cpu") def _tree_core(): @@ -57,6 +56,22 @@ def test_registry_resolves_the_rust_backend_lazily(): assert not any(name.startswith("inspect_") for name in dir(core._binding)) +def test_unset_backend_constructs_the_rust_cache(): + with mock.patch.dict(os.environ): + envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.clear() + cache = UnifiedRadixCache( + CacheInitParams( + disable=True, + req_to_token_pool=None, + token_to_kv_pool_allocator=None, + page_size=1, + tree_components=(ComponentType.FULL,), + ) + ) + assert type(cache.tree_core).__name__ == "RustUnifiedTreeCore" + assert not isinstance(cache.tree_core, UnifiedTreeCoreInspectionInterface) + + def test_insert_then_match_round_trips(): core = _tree_core() _pump_insert( diff --git a/test/registered/unit/mem_cache/test_rust_tree_core_errors.py b/test/registered/unit/mem_cache/test_rust_tree_core_errors.py new file mode 100644 index 0000000000..c2bcee2712 --- /dev/null +++ b/test/registered/unit/mem_cache/test_rust_tree_core_errors.py @@ -0,0 +1,142 @@ +"""Native bindings report panics as Python errors and reject poisoned reuse.""" + +import sys + +import pytest + +from sglang.srt.mem_cache.cache_init_params import CacheInitParams +from sglang.srt.mem_cache.rust_tree_core.adapter import RustUnifiedTreeCore +from sglang.srt.mem_cache.unified_cache.component_type import ComponentType +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +@pytest.fixture(params=["production", "inspection"]) +def core_class(request): + if request.param == "inspection": + from rust_unified_tree_core_inspector import RustUnifiedTreeCoreInspector + + return RustUnifiedTreeCoreInspector + return RustUnifiedTreeCore + + +def _params(**overrides): + params = dict( + disable=False, + req_to_token_pool=None, + token_to_kv_pool_allocator=None, + page_size=1, + tree_components=(ComponentType.FULL,), + ) + params.update(overrides) + return CacheInitParams(**params) + + +def _raw_binding(core_class, is_eagle=False, component_types=(0,), **overrides): + native = core_class._bindings + binding_class = ( + native.RustBigramUnifiedTreeCoreBinding + if is_eagle + else native.RustUnifiedTreeCoreBinding + ) + return binding_class(native.TreeCoreInitParamsBinding(**overrides), component_types) + + +@pytest.mark.parametrize("is_eagle", [False, True]) +@pytest.mark.parametrize("raw_binding", [False, True]) +def test_native_panic_is_catchable_and_poisoned_core_stays_unusable( + core_class, is_eagle, raw_binding +): + core = ( + _raw_binding(core_class, is_eagle) + if raw_binding + else core_class(_params(is_eagle=is_eagle)) + ) + root = core.root_node_handle() + + # Catch Exception just as the scheduler's terminal error handler does. + # This trips a real native invariant while the binding owns the mutex. + with pytest.raises(Exception, match="Rust TreeCore panicked") as caught: + core.build_backup_spec(root) + assert type(caught.value) is RuntimeError + assert caught.value.__cause__ is None + + for operation in (core.root_node_handle, core.reset): + with pytest.raises(RuntimeError, match="Rust TreeCore mutex poisoned"): + operation() + with pytest.raises(RuntimeError, match="Rust TreeCore mutex poisoned"): + if raw_binding: + core.set_is_write_back(True) + else: + core.is_write_back = True + + +def test_native_configuration_and_introspection_use_the_concrete_binding(core_class): + core = core_class(_params()) + assert type(core._binding) is core._binding_class() + assert "root_node_handle" in dir(core._binding) + assert core.is_write_back is False + core.is_write_back = True + assert core.is_write_back is True + assert core.enable_hicache is False + core.enable_storage = True + assert core.enable_storage is True + core.enable_external_cache_linker = True + assert core.enable_external_cache_linker is True + + +@pytest.mark.parametrize("is_eagle", [False, True]) +def test_actual_native_constructor_panic_is_runtime_error(core_class, is_eagle): + # The raw binding validates window presence; the native SWA constructor + # asserts positivity. Exercise that actual panic without a Python mock. + with pytest.raises(RuntimeError, match="swa_sliding_window_size must be positive"): + _raw_binding( + core_class, is_eagle, component_types=(0, 1), swa_sliding_window_size=0 + ) + assert _raw_binding(core_class, is_eagle).root_node_handle() == 0 + + +def test_typed_native_constructor_error_is_preserved(core_class): + with pytest.raises(ValueError, match="page_size must be at least 1") as caught: + core_class(_params(page_size=0)) + assert caught.value.__cause__ is None + + +def test_typed_native_method_error_does_not_poison_the_core(core_class): + binding = _raw_binding(core_class) + with pytest.raises(KeyError): + binding.get_node_key_lengths([999]) + assert binding.root_node_handle() == 0 + binding.set_is_write_back(True) + assert binding.is_write_back() is True + + +@pytest.mark.parametrize("error_type", [ValueError, KeyboardInterrupt, SystemExit]) +def test_python_argument_errors_pass_through_unchanged(core_class, error_type): + original = error_type("leave unchanged") + + class FailingIndex: + def __index__(self): + raise original + + binding = _raw_binding(core_class) + with pytest.raises(error_type) as caught: + binding.set_write_through_threshold(FailingIndex()) + assert caught.value is original + assert binding.root_node_handle() == 0 + + +def test_tlru_inspection_panic_is_runtime_error(): + from rust_unified_tree_core_inspector import RustUnifiedTreeCoreInspector + + config = RustUnifiedTreeCoreInspector._bindings.TlruFloatConfig( + 1.0, 0.0, integer_estimate=(1 << 127) - 1 + ) + with pytest.raises(RuntimeError, match="overflowed i128"): + config.inspect_is_tel_safe(1, 0) + assert config.inspect_is_tel_safe(0, 0) is False + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/unit/mem_cache/test_rust_tree_core_integration.py b/test/registered/unit/mem_cache/test_rust_tree_core_integration.py index 8748e56c97..043edae520 100644 --- a/test/registered/unit/mem_cache/test_rust_tree_core_integration.py +++ b/test/registered/unit/mem_cache/test_rust_tree_core_integration.py @@ -1,10 +1,12 @@ """Integration tests driving the real compiled Rust mem_cache extension.""" import hashlib -import shutil +import math import sys from array import array +from itertools import pairwise from types import SimpleNamespace +from unittest.mock import Mock import pytest import torch @@ -13,9 +15,6 @@ from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci(est_time=17, suite="base-a-test-cpu") -if shutil.which("cargo") is None: - pytest.skip("the rust backend builds with cargo", allow_module_level=True) - from sglang.srt.disaggregation.kv_events import ( AllBlocksCleared, BlockRemoved, @@ -25,12 +24,14 @@ from sglang.srt.disaggregation.kv_events import ( from sglang.srt.environ import envs from sglang.srt.mem_cache.base_prefix_cache import ( DecLockRefParams, + EvictParams, InsertParams, InsertResult, MatchPrefixParams, MatchResult, ) from sglang.srt.mem_cache.cache_init_params import CacheInitParams +from sglang.srt.mem_cache.evict_policy import TLRUStrategy from sglang.srt.mem_cache.hicache_storage import ( PoolHitPolicy, PoolName, @@ -38,8 +39,12 @@ from sglang.srt.mem_cache.hicache_storage import ( PoolTransferResult, ) from sglang.srt.mem_cache.radix_cache import RadixKey -from sglang.srt.mem_cache.rust_tree_core.adapter import RustUnifiedTreeCore +from sglang.srt.mem_cache.rust_tree_core.adapter import ( + RustUnifiedTreeCore, + _tlru_float_config, +) from sglang.srt.mem_cache.rust_tree_core.extension import bindings as mem_cache +from sglang.srt.mem_cache.rust_tree_core.extension import load_tree_core_extension from sglang.srt.mem_cache.unified_cache.cache_action import ( BackupKV, FreeComponentHostSlot, @@ -100,6 +105,8 @@ def _pump_insert(core: RustUnifiedTreeCore, params: InsertParams) -> InsertResul last_device_node=step.result.last_device_node, mamba_exist=step.result.mamba_exist, swa_branch_inserted=step.result.swa_branch_inserted, + rotation_tail_declined=step.result.rotation_tail_declined, + adopted_ranges=step.result.adopted_ranges, cache_actions=actions, ) @@ -123,6 +130,32 @@ def test_match_on_the_empty_tree_returns_no_indices(): assert result.device_indices.numel() == 0 +@pytest.mark.parametrize("instance_backend", [None, "rust"]) +def test_default_backend_constructs_real_rust_cpu_cache(monkeypatch, instance_backend): + from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache + + if instance_backend is None: + monkeypatch.delenv("SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND", raising=False) + else: + monkeypatch.setenv("SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND", "python") + cache = UnifiedRadixCache( + CacheInitParams( + disable=False, + req_to_token_pool=None, + token_to_kv_pool_allocator=None, + page_size=1, + tree_components=(ComponentType.FULL,), + tree_core_backend=instance_backend, + ) + ) + assert cache._tree_core_backend == "rust" + assert isinstance(cache.tree_core, RustUnifiedTreeCore) + cache.insert(InsertParams(key=_key([1, 2, 3]), value=torch.tensor([11, 12, 13]))) + matched = cache.match_prefix(MatchPrefixParams(key=_key([1, 2, 3]))) + assert matched.device_indices.tolist() == [11, 12, 13] + cache.tree_core.sanity_check([], []) + + def test_insert_then_match_back_returns_the_exact_indices(): core = _tree_core() result = _insert(core, [1, 2, 3], [10, 11, 12]) @@ -153,6 +186,490 @@ def test_root_node_handle_is_namespace_independent(): assert missed.best_match_node == root +@pytest.mark.parametrize("is_eagle", [False, True]) +def test_rotation_decline_precedes_host_restore_and_respects_namespaces(is_eagle): + core = _tree_core(page_size=2, is_eagle=is_eagle, enable_kv_cache_events=True) + core.set_hicache_enabled() + tokens = array("q", range(9 if is_eagle else 8)) + key = RadixKey(tokens, extra_key="adapter", cache_salt="tenant-a") + inserted = _pump_insert( + core, + InsertParams( + key=key, + value=torch.arange(10, 18), + rotation_base=1, + ), + ) + leaf = inserted.last_device_node + core.commit_backup(leaf, torch.arange(100, 108), {}) + _accumulate_step(core.demote(leaf), {}, {}, {}) + core.take_events() + + step = core.begin_insert( + InsertParams( + key=key, + value=torch.arange(20, 28), + rotation_base=3, + track_adopted_ranges=True, + ) + ) + assert step.result is not None + assert step.result.rotation_tail_declined + assert step.result.prefix_len == 8 + assert step.result.last_device_node == leaf + assert step.result.adopted_ranges == {} + assert step.actions == [] + assert not core.has_ongoing_insert() + assert core.end_insert() == [] + assert core.is_full_device_evicted(leaf) + assert core.rotation_base_of(leaf) == 1 + assert core.take_events() == [] + + other = _pump_insert( + core, + InsertParams( + key=RadixKey(tokens, extra_key="adapter", cache_salt="tenant-b"), + value=torch.arange(20, 28), + rotation_base=3, + ), + ) + assert not other.rotation_tail_declined + assert core.rotation_base_of(other.last_device_node) == 3 + assert core.rotation_base_of(core.root_node_handle()) is None + core.sanity_check([], []) + + +@pytest.mark.parametrize( + "config,evict_recent", + [ + (None, True), + ({"protected_threshold": 4}, False), + ({"protected_threshold": 0}, False), + ], +) +def test_slru_config_changes_which_leaf_is_evicted(config, evict_recent): + core = _tree_core(eviction_policy="SLRU", eviction_policy_config=config) + for _ in range(3): + old = _insert(core, [1, 2], [10, 11]).last_device_node + recent = _insert(core, [3, 4], [12, 13]).last_device_node + + core.evict_device_start(ComponentType.FULL, 2) + try: + step = core.evict_device_next_node(ComponentType.FULL, {}) + assert step.node_id == (recent if evict_recent else old) + finally: + core.evict_device_end(ComponentType.FULL) + + +def _tlru_tree_core(backend, config=None, page_size=1, is_eagle=False): + from sglang.srt.mem_cache.unified_cache.components.full import FullComponent + from sglang.srt.mem_cache.unified_cache.unified_tree_core import UnifiedTreeCore + + params = CacheInitParams( + disable=False, + req_to_token_pool=None, + token_to_kv_pool_allocator=None, + page_size=page_size, + is_eagle=is_eagle, + tree_components=(ComponentType.FULL,), + eviction_policy="tlru", + eviction_policy_config=config, + ) + if backend == "rust": + return RustUnifiedTreeCore(params) + cache = SimpleNamespace(enable_session_radix_cache=False) + return UnifiedTreeCore(params, {ComponentType.FULL: FullComponent(cache, params)}) + + +def _tlru_insert(core, token_ids, indices, tier): + key = RadixKey(array("q", token_ids), is_bigram=core.is_eagle) + if tier == "device": + return _pump_insert( + core, InsertParams(key=key, value=torch.tensor(indices, dtype=torch.int64)) + ).last_device_node + core.set_hicache_enabled() + return core.insert_host( + core.root_node_handle(), + key, + torch.tensor(indices, dtype=torch.int64), + [f"{index:064x}" for index in range(len(indices) // core.page_size)], + ).inserted_host_node + + +def _tlru_evict_one_leaf(core, tier): + tracker, device_frees, host_frees = {}, {}, {} + if tier == "host": + _accumulate_step( + core.drive_host_eviction(ComponentType.FULL, 1), + tracker, + device_frees, + host_frees, + ) + return torch.cat(host_frees[ComponentType.FULL]).tolist() + + core.evict_device_start(ComponentType.FULL, 1) + try: + step = core.evict_device_next_node(ComponentType.FULL, tracker) + node = step.node_id + _accumulate_step(step, tracker, device_frees, host_frees) + assert node is not None + _accumulate_step( + core.evict_device_leaf(node, is_write_back=False), + tracker, + device_frees, + host_frees, + ) + assert core.evict_device_next_node(ComponentType.FULL, tracker).node_id is None + finally: + core.evict_device_end(ComponentType.FULL) + return torch.cat(device_frees[ComponentType.FULL]).tolist() + + +@pytest.mark.parametrize("backend", ["python", "rust"]) +@pytest.mark.parametrize( + "config,evict_recent", + [ + (None, False), + ({"threshold": 4, "next_prompt_estimate": 0}, True), + ({"threshold": 4, "next_prompt_estimate": 2}, True), + ({"threshold": 4, "next_prompt_estimate": 4}, False), + ({"threshold": 4, "next_prompt_estimate": 6}, False), + ({"threshold": -2, "next_prompt_estimate": 0}, False), + ({"threshold": 2, "next_prompt_estimate": -2}, True), + ({"threshold": 2**100, "next_prompt_estimate": 2**100 - 2}, True), + ({"threshold": 2**100, "next_prompt_estimate": 2**100 + 2}, False), + ({"threshold": 2**100, "next_prompt_estimate": -(2**100)}, False), + ({"threshold": -(2**100), "next_prompt_estimate": 2**100}, False), + ], +) +def test_tlru_config_prioritizes_safe_tails_then_recency(backend, config, evict_recent): + core = _tlru_tree_core(backend, config) + # The older conversation occupies one oversized node, so trimming it would + # exceed the tail budget. The recent short conversation can fit entirely. + _insert(core, list(range(100, 108)), list(range(200, 208))) + _insert(core, [1, 2], [10, 11]) + + assert _tlru_evict_one_leaf(core, "device") == ( + [10, 11] if evict_recent else list(range(200, 208)) + ) + core.sanity_check([], []) + + +@pytest.mark.parametrize("backend", ["python", "rust"]) +@pytest.mark.parametrize("tier", ["device", "host"]) +@pytest.mark.parametrize( + "threshold,next_prompt_estimate,evict_recent", + [ + pytest.param(4096.0, 4095.0, True, id="integral-floats"), + pytest.param(1.5, 0.25, True, id="fractional-safe"), + pytest.param(1.5, 0.75, False, id="fractional-protected"), + pytest.param(1.5, 0, True, id="float-threshold"), + pytest.param(2, 0.5, True, id="float-estimate"), + pytest.param(1.2, 0.2, False, id="addition-before-subtraction"), + pytest.param(1.2000000000000002, 0.2, True, id="adjacent-float-boundary"), + pytest.param( + float(2**53 + 2), + 2**53 + 1, + False, + id="integer-addition-before-float-conversion", + ), + pytest.param(2**53 + 1, float(2**53), False, id="large-integer-threshold"), + pytest.param( + -float(2**53), + -(2**53 + 1), + True, + id="negative-integer-addition-before-float-conversion", + ), + pytest.param(2**100 + 1, 2**100, True, id="arbitrary-integer-safe"), + pytest.param(2**100, 2**100, False, id="arbitrary-integer-protected"), + pytest.param(-0.5, -2, True, id="negative-threshold"), + pytest.param(0, -1.5, True, id="negative-estimate"), + pytest.param(True, False, True, id="boolean-counts"), + pytest.param(float("nan"), 0, False, id="nan-threshold"), + pytest.param(0, float("nan"), False, id="nan-estimate"), + pytest.param(float("inf"), 0, False, id="positive-infinite-threshold"), + pytest.param(float("-inf"), 0, False, id="negative-infinite-threshold"), + pytest.param(0, float("inf"), False, id="positive-infinite-estimate"), + pytest.param(0, float("-inf"), False, id="negative-infinite-estimate"), + pytest.param(float("inf"), float("inf"), False, id="infinite-cancellation"), + ], +) +def test_tlru_numeric_configuration_matches_python_eviction( + backend, tier, threshold, next_prompt_estimate, evict_recent +): + core = _tlru_tree_core( + backend, + {"threshold": threshold, "next_prompt_estimate": next_prompt_estimate}, + ) + _tlru_insert(core, list(range(100, 108)), list(range(200, 208)), tier) + _tlru_insert(core, [0], [10], tier) + _tlru_insert(core, [0, 1], [10, 11], tier) + + # Removing the recent leaf leaves one token cached from a two-token history. + # In particular, 2 + 0.2 - 1.2 exceeds 1, while rearranging the expression to + # 1.2 - 0.2 would incorrectly mark that leaf safe. Non-finite configurations + # put both leaves in the same priority class and retain recency ordering. + assert _tlru_evict_one_leaf(core, tier) == ( + [11] if evict_recent else list(range(200, 208)) + ) + core.sanity_check([], []) + + +def _assert_tlru_float_priorities_match_python( + bindings, threshold, next_prompt_estimate, histories, cached_counts +): + native = _tlru_float_config(bindings, threshold, next_prompt_estimate) + strategy = TLRUStrategy(threshold, next_prompt_estimate) + for history in histories: + for cached in cached_counts: + if cached > history: + continue + node = SimpleNamespace( + _tlru_history_len=history, + _tlru_cached_prefix_len=cached, + key=(), + last_access_time=0.0, + ) + expected = strategy.get_priority(node)[0] == -1 + assert native.inspect_is_tel_safe(history, cached) == expected, ( + threshold, + next_prompt_estimate, + history, + cached, + ) + + +def test_tlru_float_priorities_match_python_at_integer_comparison_boundaries(): + bindings = load_tree_core_extension(inspection=True) + max_history = 2 * sys.maxsize + 1 + counts = [ + 0, + 1, + 2, + 3, + 8, + 2**53 - 1, + 2**53, + 2**53 + 1, + 2**53 + 2, + 2**53 + 3, + 2**53 + 4, + max_history - 1, + max_history, + ] + for threshold, estimate in ( + (0.0, 0.0), + (1.0, 0.0), + (1.2, 0.2), + (1.2000000000000002, 0.2), + (-0.5, -2.0), + (4096.0, 1024.25), + (float(2**53), 0.0), + (2**53 + 1, float(2**53)), + (0.0, math.nextafter(float(max_history + 1), 0.0)), + (0.0, float(max_history + 1)), + (-sys.float_info.max, sys.float_info.max), + (sys.float_info.max, -sys.float_info.max), + (float("nan"), 0.0), + (0.0, float("nan")), + (float("inf"), 0.0), + (float("-inf"), 0.0), + (0.0, float("inf")), + (0.0, float("-inf")), + (float("inf"), float("inf")), + (float("-inf"), float("-inf")), + ): + _assert_tlru_float_priorities_match_python( + bindings, threshold, estimate, counts, counts + ) + + +def test_tlru_integer_estimate_priorities_match_python_at_i128_endpoints(): + bindings = load_tree_core_extension(inspection=True) + max_history = 2 * sys.maxsize + 1 + max_estimate = 2**127 - 1 + max_safe_estimate = max_estimate - max_history + history_candidates = [0, 1, 2, 2**53 + 1, max_history - 1, max_history] + for estimate in ( + -(2**127), + -(2**127) + 1, + max_safe_estimate - 1, + max_safe_estimate, + max_safe_estimate + 1, + max_estimate - 2, + max_estimate, + ): + # Only histories whose actual addition fits i128 belong in the parity + # oracle; native arithmetic overflow is covered by the Rust panic test. + histories = [ + history + for history in history_candidates + if history <= max_estimate - estimate + ] + for threshold in (float(estimate), math.nextafter(float(estimate), math.inf)): + _assert_tlru_float_priorities_match_python( + bindings, threshold, estimate, histories, histories + ) + + +@pytest.mark.parametrize("exponent", [53, 100, 126]) +def test_tlru_integer_estimate_priorities_match_python_at_rounding_ties(exponent): + bindings = load_tree_core_extension(inspection=True) + max_history = 2 * sys.maxsize + 1 + for sign in (-1, 1): + lower = sign * math.ldexp(1.0, exponent) + # Consecutive float pairs exercise both directions of ties-to-even. + for _ in range(2): + upper = math.nextafter(lower, math.inf) + midpoint = (int(lower) + int(upper)) // 2 + for tie_history in (0, 1, 2, 2**53 + 1, sys.maxsize, max_history): + estimate = midpoint - tie_history + histories = sorted( + { + 0, + 1, + 2, + 2**53 + 1, + max_history - 1, + max_history, + max(tie_history - 1, 0), + tie_history, + min(tie_history + 1, max_history), + } + ) + _assert_tlru_float_priorities_match_python( + bindings, lower, estimate, histories, [0, 1, max_history] + ) + lower = upper + + +@pytest.mark.parametrize("estimate", [-(2**127) - 1, 2**127]) +def test_tlru_mixed_integer_estimate_outside_i128_is_rejected(estimate): + with pytest.raises(OverflowError): + _tlru_tree_core("rust", {"threshold": 0.0, "next_prompt_estimate": estimate}) + + +@pytest.mark.parametrize("backend", ["python", "rust"]) +def test_tlru_near_i128_max_estimate_accepts_small_history(backend): + core = _tlru_tree_core( + backend, + {"threshold": float(2**127 - 1), "next_prompt_estimate": 2**127 - 3}, + ) + _insert(core, [0, 1], [10, 11]) + assert _tlru_evict_one_leaf(core, "device") == [10, 11] + core.sanity_check([], []) + + +@pytest.mark.parametrize( + "threshold,next_prompt_estimate", + [ + pytest.param("1.0", 0, id="string-threshold"), + pytest.param(0, "1.0", id="string-estimate"), + pytest.param(None, 0, id="null-threshold"), + pytest.param(0, None, id="null-estimate"), + pytest.param([1], 0, id="list-threshold"), + pytest.param(0, {}, id="object-estimate"), + ], +) +def test_tlru_non_numeric_configuration_raises_type_error( + threshold, next_prompt_estimate +): + config = {"threshold": threshold, "next_prompt_estimate": next_prompt_estimate} + with pytest.raises(TypeError): + _tlru_tree_core("rust", config) + node = SimpleNamespace( + _tlru_history_len=2, + _tlru_cached_prefix_len=2, + key=(0,), + last_access_time=0.0, + ) + with pytest.raises(TypeError): + TLRUStrategy(**config).get_priority(node) + + +@pytest.mark.parametrize("backend", ["python", "rust"]) +@pytest.mark.parametrize("tier", ["device", "host"]) +@pytest.mark.parametrize("page_size", [1, 2]) +@pytest.mark.parametrize("is_eagle", [False, True]) +def test_tlru_repeated_tail_eviction_preserves_history( + backend, tier, page_size, is_eagle +): + core = _tlru_tree_core( + backend, {"threshold": 6, "next_prompt_estimate": 2}, page_size, is_eagle + ) + _tlru_insert( + core, + list(range(100, 108 + is_eagle)), + list(range(200, 208)), + tier, + ) + for length in (2, 4, 6, 8): + _tlru_insert( + core, list(range(length + is_eagle)), list(range(10, 10 + length)), tier + ) + + # Four tokens are safe. A fresh eviction walk must retain the original + # eight-token high-water mark and protect the shortened conversation. + assert _tlru_evict_one_leaf(core, tier) == [16, 17] + assert _tlru_evict_one_leaf(core, tier) == [14, 15] + assert _tlru_evict_one_leaf(core, tier) == list(range(200, 208)) + core.sanity_check([], []) + + +@pytest.mark.parametrize("backend", ["python", "rust"]) +@pytest.mark.parametrize("tier", ["device", "host"]) +@pytest.mark.parametrize("page_size", [1, 2]) +def test_tlru_split_and_compacted_branch_preserve_history(backend, tier, page_size): + core = _tlru_tree_core( + backend, {"threshold": 6, "next_prompt_estimate": 2}, page_size + ) + _tlru_insert(core, list(range(100, 108)), list(range(200, 208)), tier) + _tlru_insert(core, list(range(8)), list(range(10, 18)), tier) + # Matching splits the long turn at depth four; inserting a compacted turn + # then splits that prefix again at depth two. Neither split shortens history. + core.match_prefix(MatchPrefixParams(key=_key(list(range(4))))) + _tlru_insert(core, [0, 1, 90, 91], [10, 11, 30, 31], tier) + + assert _tlru_evict_one_leaf(core, tier) == [14, 15, 16, 17] + assert _tlru_evict_one_leaf(core, tier) == [30, 31] + assert _tlru_evict_one_leaf(core, tier) == list(range(200, 208)) + core.sanity_check([], []) + + +@pytest.mark.parametrize("backend", ["python", "rust"]) +def test_tlru_host_refill_raises_device_ancestor_history(backend): + core = _tlru_tree_core(backend, {"threshold": 6, "next_prompt_estimate": 2}) + core.set_hicache_enabled() + old = _insert(core, list(range(100, 108)), list(range(200, 208))).last_device_node + prefix = _insert(core, [0, 1], [10, 11]).last_device_node + _complete_backup(core, prefix) + inserted = core.insert_host( + prefix, + _key(list(range(2, 8))), + torch.arange(1012, 1018), + [f"{index:064x}" for index in range(6)], + ) + assert inserted.inserted_host_node is not None + + # Prefetch creates no new device KV, but its depth must protect the short + # device ancestor from T-LRU's first phase. + core.evict_device_start(ComponentType.FULL, 1) + try: + assert core.evict_device_next_node(ComponentType.FULL, {}).node_id == old + finally: + core.evict_device_end(ComponentType.FULL) + core.sanity_check([], []) + + +@pytest.mark.parametrize( + "policy,config", + [("lru", {"protected_threshold": 4}), ("slru", {"unknown_option": 4})], +) +def test_eviction_config_rejects_unknown_constructor_options(policy, config): + with pytest.raises(TypeError): + _tree_core(eviction_policy=policy, eviction_policy_config=config) + + def test_stale_handle_reads_raise_key_error_without_poisoning_the_core(): core = _tree_core() stale_root = core.root_node_handle() @@ -612,16 +1129,13 @@ def test_poisoned_binding_refuses_to_reuse_the_core(): # Reading a backup spec from the value-less root deliberately trips a native # invariant while the binding owns the mutex. - with pytest.raises(BaseException) as initial_panic: + with pytest.raises(RuntimeError, match="Rust TreeCore panicked"): binding.build_backup_spec(root) - assert initial_panic.type.__name__ == "PanicException" - # The guard must fail closed instead of handing potentially partial state to - # the next operation through PoisonError::into_inner(). - with pytest.raises(BaseException) as poisoned: + # The binding must fail closed instead of handing potentially partial state + # to the next operation through PoisonError::into_inner(). + with pytest.raises(RuntimeError, match="Rust TreeCore mutex poisoned"): binding.root_node_handle() - assert poisoned.type.__name__ == "PanicException" - assert "Rust TreeCore mutex poisoned" in str(poisoned.value) def test_extra_key_isolates_namespaces(): @@ -906,8 +1420,8 @@ def test_insert_host_reports_a_dropped_write_through_suffix(): def test_host_lock_refs_round_trip(swa, missing_receipt): core = _swa_tree_core() if swa else _tree_core() core.set_hicache_enabled() - _insert(core, [1], [10]) - leaf = core.match_prefix(MatchPrefixParams(key=_key([1]))).best_match_node + # The Full-only insertion is not a resumable SWA match until SWA is set below. + leaf = _insert(core, [1], [10]).last_device_node component_transfers = {} if swa: core.has_swa_host_pool = True @@ -923,6 +1437,8 @@ def test_host_lock_refs_round_trip(swa, missing_receipt): core.commit_backup( leaf, torch.tensor([100], dtype=torch.int64), component_transfers ) + core.mark_write_through_pending([leaf], leaf) + core.finish_write_through([leaf], leaf) device_lock = core.inc_lock_ref(leaf) host_lock = core.inc_host_lock_ref(leaf) assert not host_lock.component_lock_uuids @@ -932,9 +1448,9 @@ def test_host_lock_refs_round_trip(swa, missing_receipt): params = host_lock.to_dec_params() if missing_receipt: params.component_host_lock_uuids.clear() - with pytest.raises(BaseException, match="no entry found for key") as error: + with pytest.raises(RuntimeError, match="no entry found for key") as error: core.dec_host_lock_ref(leaf, params) - assert error.type.__name__ == "PanicException" + assert error.value.__cause__ is None return # A Rust ownership violation poisons the core. core.dec_host_lock_ref(leaf, params) if swa: @@ -966,6 +1482,110 @@ def test_drive_host_eviction_frees_the_demoted_leaf(): core.sanity_check([], []) +def test_host_duplicate_reclaim_override_preserves_normal_host_eviction(): + core = _tree_core() + core.set_hicache_enabled() + core.is_write_back = True + duplicate = _insert(core, [1], [10]).last_device_node + core.commit_backup(duplicate, torch.tensor([100], dtype=torch.int64), {}) + core.mark_write_through_pending([duplicate], duplicate) + core.finish_write_through([duplicate], duplicate) + core.insert_host( + core.root_node_handle(), + _key([2]), + torch.tensor([200], dtype=torch.int64), + ["h0"], + ) + + host_frees = {} + with envs.SGLANG_HICACHE_SKIP_HOST_DUPLICATE_RECLAIM.override(True): + _accumulate_step( + core.drive_host_eviction(ComponentType.FULL, 1), {}, {}, host_frees + ) + assert torch.cat(host_frees[ComponentType.FULL]).tolist() == [200] + assert core.is_backuped(duplicate) + core.sanity_check([], []) + + # Read the override on every eviction call, including after construction. + host_frees = {} + with envs.SGLANG_HICACHE_SKIP_HOST_DUPLICATE_RECLAIM.override(False): + _accumulate_step( + core.drive_host_eviction(ComponentType.FULL, 1), {}, {}, host_frees + ) + assert torch.cat(host_frees[ComponentType.FULL]).tolist() == [100] + assert not core.is_backuped(duplicate) + core.sanity_check([], []) + + +@pytest.mark.parametrize("backend", ["python", "rust"]) +@pytest.mark.parametrize("backup_again", [False, True]) +def test_full_host_duplicates_preserve_backup_order(backend, backup_again): + from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache + + with envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.override(backend): + cache = UnifiedRadixCache( + CacheInitParams( + disable=True, + req_to_token_pool=None, + token_to_kv_pool_allocator=None, + page_size=1, + tree_components=(ComponentType.FULL,), + ) + ) + core = cache.tree_core + core.set_hicache_enabled() + core.is_write_back = True + nodes = [] + for index in range(3): + node = _insert(core, [index + 1], [index + 101]).last_device_node + core.commit_backup(node, torch.tensor([index + 1001]), {}) + core.mark_write_through_pending([node], node) + core.finish_write_through([node], node) + nodes.append(node) + + expected = [1001, 1002, 1003] + if backup_again: + expected.append(2001) + for index, expected_slot in enumerate(expected): + host_frees = {} + _accumulate_step( + core.drive_host_eviction(ComponentType.FULL, 1), {}, {}, host_frees + ) + assert torch.cat(host_frees[ComponentType.FULL]).tolist() == [expected_slot] + if index == 0 and backup_again: + # A new backup rejoins after existing duplicates; it must not take + # the removed entry's old position or reorder the surviving nodes. + core.commit_backup(nodes[0], torch.tensor([2001]), {}) + core.mark_write_through_pending([nodes[0]], nodes[0]) + core.finish_write_through([nodes[0]], nodes[0]) + assert all(not core.is_backuped(node) for node in nodes) + core.sanity_check([], []) + + +@pytest.mark.parametrize("backed_up", [False, True]) +def test_device_eviction_counts_only_full_tokens_without_a_host_copy(backed_up): + core = _mamba_tree_core() + core.set_hicache_enabled() + leaf = _mamba_insert(core, [1, 2, 3], [10, 11, 12], 7).last_device_node + if backed_up: + core.commit_backup(leaf, torch.tensor([100, 101, 102]), {}) + + step = core.evict_device_leaf(leaf, is_write_back=False) + unbacked_tokens = step.unbacked_tokens + tracker = {} + _accumulate_step(step, tracker, {}, {}) + assert tracker == {ComponentType.FULL: 3, ComponentType.MAMBA: 1} + assert unbacked_tokens == (0 if backed_up else 3) + + # Counters belong to a single step, never to the next eviction walk. + core.evict_device_start(ComponentType.FULL, 1) + step = core.evict_device_next_node(ComponentType.FULL, {}) + assert step.unbacked_tokens == 0 + _accumulate_step(step, {}, {}, {}) + core.evict_device_end(ComponentType.FULL) + core.sanity_check([], []) + + def test_events_disabled_take_events_is_empty(): core = _tree_core() _insert(core, [1, 2], [10, 11]) @@ -1323,6 +1943,392 @@ def _swa_tree_core(window: int = 8, **params_overrides) -> RustUnifiedTreeCore: ) +def _swa_transfer_core( + backend, unified=False, mamba=False, page_size=1, window=16, is_eagle=False +): + from sglang.srt.mem_cache.allocator.swa import SWATokenToKVPoolAllocator + from sglang.srt.mem_cache.allocator.unified_hybrid_swa import ( + UnifiedSWAAllocatorBase, + ) + from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool + from sglang.srt.mem_cache.unified_cache.components.full import FullComponent + from sglang.srt.mem_cache.unified_cache.components.mamba import MambaComponent + from sglang.srt.mem_cache.unified_cache.components.swa import SWAComponent + from sglang.srt.mem_cache.unified_cache.unified_tree_core import UnifiedTreeCore + + allocator = Mock( + spec=UnifiedSWAAllocatorBase if unified else SWATokenToKVPoolAllocator + ) + allocator.device = torch.device("cpu") + allocator.swa_req_ring = False + params = CacheInitParams( + disable=False, + req_to_token_pool=Mock(spec=HybridReqToTokenPool) if mamba else None, + token_to_kv_pool_allocator=allocator, + page_size=page_size, + is_eagle=is_eagle, + tree_components=(ComponentType.FULL, ComponentType.SWA) + + ((ComponentType.MAMBA,) if mamba else ()), + sliding_window_size=window, + ) + with get_context().override_server_args( + _mamba_cache_chunk_size=256, mamba_max_states_per_path=-1 + ): + if backend == "rust": + core = RustUnifiedTreeCore(params) + else: + cache = SimpleNamespace( + token_to_kv_pool_allocator=allocator, enable_session_radix_cache=False + ) + components = { + ComponentType.FULL: FullComponent(cache, params), + ComponentType.SWA: SWAComponent(cache, params), + } + if mamba: + components[ComponentType.MAMBA] = MambaComponent(cache, params) + core = UnifiedTreeCore(params, components) + cache.tree_core = core + core.set_hicache_enabled() + core.has_swa_host_pool = True + return core, allocator + + +def _hybrid_transfer_order_fixture(backend): + from sglang.srt.mem_cache.unified_cache.components import CacheTransferPhase + + core, _ = _swa_transfer_core(backend, mamba=True) + root = core.root_node_handle() + a = core.insert_host(root, _key([10, 11]), torch.tensor([100, 101]), ["a0", "a1"]) + b = core.insert_host(root, _key([20, 21]), torch.tensor([200, 201]), ["b0", "b1"]) + # The pools can have different LRU order: SWA oldest=A, Mamba oldest=B. + for result, component, pool, indices in ( + (a, ComponentType.SWA, PoolName.SWA, [1, 2]), + (b, ComponentType.SWA, PoolName.SWA, [3, 4]), + (b, ComponentType.MAMBA, PoolName.MAMBA, [12]), + (a, ComponentType.MAMBA, PoolName.MAMBA, [11]), + ): + actions = [] + core.commit_hicache_transfers( + root, + CacheTransferPhase.PREFETCH, + {component: [PoolTransfer(name=pool, host_indices=torch.tensor(indices))]}, + cache_actions=actions, + insert_result=result, + pool_storage_result=PoolTransferResult( + kv_hit_pages=2, extra_pool_hit_pages={pool: len(indices)} + ), + ) + assert actions == [] + c = _mamba_insert(core, [30, 31], [300, 301], 13) + for action in c.cache_actions: + if isinstance(action, SWARebuild): + core.set_component_device_value( + action.node_id, ComponentType.SWA, action.source_value + ) + core.sanity_check([], []) + return core, a.inserted_host_node, c.last_device_node + + +@pytest.mark.parametrize("backend", ["python", "rust"]) +@pytest.mark.parametrize("planner", ["backup", "load_back", "storage"]) +def test_hybrid_transfer_planners_preserve_component_order(backend, planner): + core, host_node, device_node = _hybrid_transfer_order_fixture(backend) + # Each native plan creates a fresh map. Its randomized iteration must not + # leak into allocation/reclamation order across calls or TP ranks. + for _ in range(64): + if planner == "backup": + _, transfers = core.build_backup_spec(device_node) + elif planner == "load_back": + _, transfers = core.build_load_back_spec(host_node) + else: + transfers = core.build_storage_backup_spec(host_node, False).comp_xfers + assert list(transfers) == [ComponentType.SWA, ComponentType.MAMBA] + + +@pytest.mark.parametrize("backend", ["python", "rust"]) +def test_hybrid_backup_pool_pressure_preserves_host_victim(backend): + from sglang.srt.mem_cache.pool_host.group import HostPoolGroup, PoolEntry + + core, _, device_node = _hybrid_transfer_order_fixture(backend) + _, transfers = core.build_backup_spec(device_node) + free_slots = { + ct: [] for ct in (ComponentType.FULL, ComponentType.SWA, ComponentType.MAMBA) + } + freed_full = [] + + def allocate(count, component): + slots = free_slots[component] + if len(slots) < count: + return None + indices = torch.tensor(slots[:count], dtype=torch.int64) + del slots[:count] + return indices + + def release(indices, component): + free_slots[component].extend(indices.tolist()) + return len(indices) + + def reclaim(count, component): + result = core.drive_host_eviction(component, count) + assert not result.device_frees + for ct, tensors in result.host_frees.items(): + for indices in tensors: + release(indices, ct) + if ct == ComponentType.FULL: + freed_full.extend(indices.tolist()) + result.host_frees.clear() + + entries = [] + for ct, name in ( + (ComponentType.FULL, PoolName.KV), + (ComponentType.SWA, PoolName.SWA), + (ComponentType.MAMBA, PoolName.MAMBA), + ): + # Simulate only physical host allocation; tree eviction and the host + # group's allocation/reclaim/rollback flow are the production code. + pool = Mock(can_use_write_back_jit=False) + pool.alloc.side_effect = lambda count, ct=ct: allocate(count, ct) + pool.free.side_effect = lambda indices, ct=ct: release(indices, ct) + entries.append( + PoolEntry( + name=name, + host_pool=pool, + device_pool=None, + layer_mapper=lambda _: 0, + host_evict_fn=lambda count, ct=ct: reclaim(count, ct), + ) + ) + resolved = HostPoolGroup(entries).resolve_host_transfers( + [transfer for xfers in transfers.values() for transfer in xfers] + ) + assert resolved is not None + # Host allocation follows pool-name order, so Mamba pressure frees B's host + # leaf first, satisfying both side pools regardless of transfer-map order. + assert freed_full == [200, 201] + core.sanity_check([], []) + + +@pytest.mark.parametrize("backend", ["python", "rust"]) +@pytest.mark.parametrize("guard", ["none", "host_lock", "pending_dma"]) +def test_swa_host_pressure_retains_device_resident_backups(backend, guard): + core, allocator = _swa_transfer_core(backend) + allocator.translate_loc_from_full_to_swa.side_effect = lambda values: values + 100 + core.is_write_back = True + nodes = [] + for tokens, values in (([1, 2], [10, 11]), ([1, 2, 3], [20, 21, 12])): + result = _insert(core, tokens, values) + for action in result.cache_actions: + if isinstance(action, SWARebuild): + core.set_component_device_value( + action.node_id, + ComponentType.SWA, + allocator.translate_loc_from_full_to_swa(action.source_value), + ) + else: + assert isinstance(action, (FreeDeviceKV, FreeDeviceKVFullOnly)) + nodes.append(result.last_device_node) + for node in nodes: + full, transfers = core.build_backup_spec(node) + for component_transfers in transfers.values(): + for transfer in component_transfers: + transfer.host_indices = transfer.device_indices + 1000 + core.commit_backup(node, full + 1000, transfers) + core.mark_write_through_pending([node], node) + core.finish_write_through([node], node) + + if guard == "host_lock": + lock = core.inc_host_lock_ref(nodes[-1]) + elif guard == "pending_dma": + core.mark_write_through_pending(nodes, nodes[-1]) + tracker, device_frees, host_frees = {}, {}, {} + _accumulate_step( + core.drive_host_eviction(ComponentType.SWA, 2), + tracker, + device_frees, + host_frees, + ) + assert tracker.get(ComponentType.SWA, 0) == 0 + assert not device_frees and not host_frees + if guard == "host_lock": + core.dec_host_lock_ref(nodes[-1], lock.to_dec_params()) + elif guard == "pending_dma": + core.finish_write_through(nodes, nodes[-1]) + + # Demotion makes the preserved SWA host copy the only copy. Ordinary host + # eviction must still reclaim that host-only tree under pressure. + for node in reversed(nodes): + _accumulate_step(core.demote(node), {}, {}, {}) + assert core.component_has_host_value_only(node, ComponentType.SWA) + host_frees = {} + _accumulate_step(core.drive_host_eviction(ComponentType.SWA, 3), {}, {}, host_frees) + assert sum(t.numel() for t in host_frees[ComponentType.SWA]) == 3 + core.sanity_check([], []) + + +@pytest.mark.parametrize("backend", ["python", "rust"]) +@pytest.mark.parametrize("resident_full", [(), (0,), (0, 1, 2, 3)]) +def test_swa_load_back_preserves_full_anchors_across_holes(backend, resident_full): + from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import ( + HybridCacheController, + ) + from sglang.srt.mem_cache.pool_host.group import PoolEntry + + core, _ = _swa_transfer_core(backend) + nodes = [] + boundaries = (0, 2, 5, 6, 8) + for start, end in pairwise(boundaries): + node = _insert( + core, list(range(end)), list(range(10, 10 + end)) + ).last_device_node + nodes.append(node) + core.set_component_device_value( + node, ComponentType.SWA, torch.arange(50 + start, 50 + end) + ) + core.commit_backup( + node, + torch.arange(100 + start, 100 + end), + { + ComponentType.SWA: [ + PoolTransfer( + name=PoolName.SWA, + host_indices=torch.arange(300 + start, 300 + end), + nodes_to_load=[node], + ) + ] + }, + ) + for node in reversed(nodes): + _accumulate_step(core.demote(node), {}, {}, {}) + # SWA is resident in the three-token middle node, leaving a hole in its + # load list even when that node still needs a Full load. + core.set_component_device_value(nodes[1], ComponentType.SWA, torch.arange(52, 55)) + for index in resident_full: + start, end = boundaries[index : index + 2] + core.commit_load_back( + nodes[index], + torch.arange(10 + start, 10 + end), + PoolTransfer( + name=PoolName.KV, + host_indices=torch.arange(100 + start, 100 + end), + nodes_to_load=[nodes[index]], + ), + {}, + ) + core.finish_load_back(nodes[index]) + + kv, auxiliary = core.build_load_back_spec(nodes[-1]) + (swa,) = auxiliary[ComponentType.SWA] + assert swa.nodes_to_load == [nodes[0], nodes[2], nodes[3]] + assert swa.host_indices.tolist() == [300, 301, 305, 306, 307] + expected = { + (): [slice(0, 2), slice(5, 6), slice(6, 8)], + (0,): [torch.tensor([10, 11]), slice(3, 4), slice(4, 6)], + (0, 1, 2, 3): [ + torch.tensor([10, 11]), + torch.tensor([15]), + torch.tensor([16, 17]), + ], + }[resident_full] + assert swa.anchor_index_parts is not None + assert len(swa.anchor_index_parts) == len(expected) + for actual, wanted in zip(swa.anchor_index_parts, expected): + if isinstance(wanted, slice): + assert actual == wanted + else: + torch.testing.assert_close(actual, wanted) + + # Exercise the consumer too: new Full rows and resident virtual IDs must + # bind the same five SWA rows, without consuming the resident-SWA hole. + bind = Mock(side_effect=lambda indices: indices + 1000) + controller = object.__new__(HybridCacheController) + controller.mem_pool_host = SimpleNamespace( + entry_map={ + PoolName.SWA: PoolEntry( + name=PoolName.SWA, + host_pool=SimpleNamespace(), + device_pool=SimpleNamespace(), + layer_mapper=lambda i: i, + device_indices_from_anchor_fn=bind, + device_free_fn=Mock(), + ) + } + ) + loaded = torch.arange(200, 200 + len(kv.host_indices)) + assert controller._resolve_device_transfers([swa], loaded) is not None + expected_bound = { + (): [1200, 1201, 1205, 1206, 1207], + (0,): [1010, 1011, 1203, 1204, 1205], + (0, 1, 2, 3): [1010, 1011, 1015, 1016, 1017], + }[resident_full] + assert swa.device_indices.tolist() == expected_bound + assert bind.call_count == 1 + + +@pytest.mark.parametrize("backend", ["python", "rust"]) +@pytest.mark.parametrize("unified", [False, True]) +@pytest.mark.parametrize("entrypoint", ["backup_spec", "component_transfer"]) +def test_swa_backup_resolves_relocated_full_virtual_ids(backend, unified, entrypoint): + from sglang.srt.mem_cache.unified_cache.components import CacheTransferPhase + + core, allocator = _swa_transfer_core(backend, unified=unified) + mapping = torch.arange(64) + 100 + allocator.translate_loc_from_full_to_swa.side_effect = lambda indices: mapping[ + indices + ] + values = [11, 7, 20, 4, 9, 13] + nodes = [] + for start, end in ((0, 2), (2, 5), (5, 6)): + node = _insert(core, list(range(end)), values[:end]).last_device_node + nodes.append(node) + core.set_component_device_value( + node, ComponentType.SWA, mapping[torch.tensor(values[start:end])].clone() + ) + # A hosted middle node is excluded from the next SWA backup. + core.commit_backup( + nodes[1], + torch.arange(302, 305), + { + ComponentType.SWA: [ + PoolTransfer( + name=PoolName.SWA, + host_indices=torch.arange(402, 405), + nodes_to_load=[nodes[1]], + ) + ] + }, + ) + # Relocation changes kernel-facing SWA addresses while tree-owned Full + # virtual IDs and cached SWA physical snapshots stay unchanged. + mapping[torch.tensor([11, 7, 13])] = torch.tensor([511, 407, 613]) + allocator.translate_loc_from_full_to_swa.reset_mock() + if entrypoint == "backup_spec": + full, auxiliary = core.build_backup_spec(nodes[-1]) + assert full.tolist() == [13] + (swa,) = auxiliary[ComponentType.SWA] + else: + (swa,) = core.build_hicache_transfers( + ComponentType.SWA, nodes[-1], CacheTransferPhase.BACKUP_HOST + ) + assert swa.nodes_to_load == [nodes[0], nodes[2]] + assert swa.device_indices.dtype == torch.int64 + assert swa.device_indices.tolist() == ( + [511, 407, 613] if unified else [111, 107, 113] + ) + if unified: + allocator.translate_loc_from_full_to_swa.assert_called_once() + assert allocator.translate_loc_from_full_to_swa.call_args.args[0].tolist() == [ + 11, + 7, + 13, + ] + else: + allocator.translate_loc_from_full_to_swa.assert_not_called() + assert core.get_component_device_value(nodes[0], ComponentType.SWA).tolist() == [ + 111, + 107, + ] + + def test_swa_core_rejects_a_missing_or_non_positive_window(): """A zero window can never fill, so no boundary uuid would ever be stamped; the adapter refuses it up front instead of letting the core misbehave later.""" @@ -1406,8 +2412,8 @@ def test_write_back_load_back_ignores_auxiliary_nodes_for_pending_ownership(): core.sanity_check([], []) -def _swa_cache(window: int = 8, page_size: int = 1): - """A real UnifiedRadixCache on the Rust tree core with a real SWA allocator.""" +def _swa_cache(window: int = 8, page_size: int = 1, backend="rust", ring=False): + """A real UnifiedRadixCache and SWA allocator, defaulting to the Rust core.""" from sglang.srt.mem_cache.allocator.swa import SWATokenToKVPoolAllocator from sglang.srt.mem_cache.memory_pool import ReqToTokenPool from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool @@ -1431,6 +2437,8 @@ def _swa_cache(window: int = 8, page_size: int = 1): full_attention_layer_ids=[1], device="cpu", ) + if ring: + kv_pool.swa_req_ring_size = window allocator = SWATokenToKVPoolAllocator( size=64, size_swa=64, @@ -1439,8 +2447,9 @@ def _swa_cache(window: int = 8, page_size: int = 1): device="cpu", kvcache=kv_pool, need_sort=False, + req_to_token_pool=req_to_token_pool, ) - with envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.override("rust"): + with envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.override(backend): cache = UnifiedRadixCache( params=CacheInitParams( req_to_token_pool=req_to_token_pool, @@ -1454,6 +2463,1340 @@ def _swa_cache(window: int = 8, page_size: int = 1): return cache, allocator +def _complete_backup(core, node, full_offset=1000, aux_offset=1000): + full, aux = core.build_backup_spec(node) + for transfers in aux.values(): + for transfer in transfers: + transfer.host_indices = transfer.device_indices + aux_offset + core.commit_backup(node, full + full_offset, aux) + core.mark_write_through_pending([node], node) + core.finish_write_through([node], node) + return full + full_offset, aux + + +def _swa_mamba_cache(backend, window, enable_session=False): + from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool + from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache + + _, allocator = _swa_cache(window=window, backend=backend) + request_pool = Mock(spec=HybridReqToTokenPool) + request_pool.mamba_allocator = Mock() + with ( + get_context().override_server_args( + _mamba_cache_chunk_size=256, mamba_max_states_per_path=-1 + ), + envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.override(backend), + ): + cache = UnifiedRadixCache( + CacheInitParams( + disable=False, + req_to_token_pool=request_pool, + token_to_kv_pool_allocator=allocator, + page_size=1, + sliding_window_size=window, + enable_session_radix_cache=enable_session, + tree_components=( + ComponentType.FULL, + ComponentType.SWA, + ComponentType.MAMBA, + ), + ) + ) + return cache, allocator + + +def _internal_swa_write_back_case( + backend, + page_size=1, + window=8, + with_mamba=False, + prefix_nodes=1, + enable_session=False, +): + cache, allocator = ( + _swa_mamba_cache(backend, window, enable_session) + if with_mamba + else _swa_cache(window=window, page_size=page_size, backend=backend) + ) + assert cache._tree_core_backend == backend + segment = 2 * page_size + child_length = math.ceil(window / page_size) * page_size + prefix_length = prefix_nodes * segment + total_length = prefix_length + child_length + if page_size == 1: + values = allocator.alloc(total_length) + else: + # Bulk page allocation is CPU-safe; alloc_extend runs a GPU kernel. + values = allocator.full_attn_allocator.alloc(total_length) + swa_values = allocator.swa_attn_allocator.alloc(total_length) + allocator.set_full_to_swa_mapping(values, swa_values) + sizes = list(range(segment, prefix_length + 1, segment)) + [len(values)] + nodes = [] + previous = 0 + for index, size in enumerate(sizes): + nodes.append( + cache.insert( + InsertParams( + key=_key(list(range(size))), + value=values[:size], + prev_prefix_len=previous, + mamba_value=torch.tensor([201 + index]) if with_mamba else None, + ) + ).last_device_node + ) + previous = size + cache.tree_core.set_hicache_enabled() + cache.tree_core.has_swa_host_pool = True + cache.is_write_back = True + cache.cache_controller = SimpleNamespace(write_policy="write_back") + cache._build_backup_sidecar = Mock(return_value=[]) + # The child spans a complete window, so its request lock leaves the older + # internal SWA segments eligible while preventing a whole-leaf eviction. + leaf_lock = cache.inc_lock_ref(nodes[-1]).to_dec_params() + return SimpleNamespace( + cache=cache, + core=cache.tree_core, + nodes=nodes, + sizes=sizes, + segment=segment, + leaf_lock=leaf_lock, + full={ + node: cache.tree_core.get_component_device_value( + node, ComponentType.FULL + ).clone() + for node in nodes + }, + swa={ + node: cache.tree_core.get_component_device_value( + node, ComponentType.SWA + ).clone() + for node in nodes + }, + ) + + +def _mock_swa_write_back_io(case, backup="success"): + cache, core = case.cache, case.core + case.events, case.submissions, case.freed = [], [], [] + host_pool = Mock() + host_pool.available_size.return_value = 0 if backup == "host_pressure" else 64 + cache.components[ComponentType.SWA]._swa_kv_pool_host = host_pool + cache.host_pool_group = Mock() + cache.host_pool_group.get_pool.return_value = host_pool + + def assert_resident(node, transfers): + assert torch.equal( + core.get_component_device_value(node, ComponentType.FULL), + case.full[node], + ) + for transfer in transfers[ComponentType.SWA]: + assert torch.equal( + torch.cat( + [ + core.get_component_device_value(source, ComponentType.SWA) + for source in transfer.nodes_to_load + ] + ), + transfer.device_indices, + ) + + def evict_host(count, component): + assert component == ComponentType.SWA + case.events.append(("host_evict", count)) + host_pool.available_size.return_value = count + return count + + def write(node, full, transfers, sidecars): + assert sidecars == [] + assert_resident(node, transfers) + case.events.append(("write", node)) + case.submissions.append((node, full.clone(), transfers)) + if backup == "raised": + raise RuntimeError("DMA submission failed") + if backup == "failed": + return None + for component_transfers in transfers.values(): + for transfer in component_transfers: + transfer.host_indices = transfer.device_indices + 20000 + return full + 10000 + + def acknowledge(write_back=False): + assert write_back + for node, _, transfers in case.submissions: + if node in cache.ongoing_write_through: + assert_resident(node, transfers) + case.events.append(("ack", node)) + if backup == "ack_raised": + raise RuntimeError("DMA completion failed") + for ack_id in list(cache.ongoing_write_through): + cache._finish_write_through_ack(ack_id) + + original_free = cache._free_values + + def free_values(device_frees, host_frees): + assert not host_frees + for component, tensors in device_frees.items(): + if tensors: + case.freed.append((component, torch.cat(tensors).tolist())) + case.events.append(("free", component)) + original_free(device_frees, host_frees) + + cache.evict_host = evict_host + cache._execute_kv_backup = write + cache.writing_check = acknowledge + cache._free_values = free_values + + +@pytest.mark.parametrize("backend", ["python", "rust"]) +@pytest.mark.parametrize( + "page_size,window,with_mamba", [(1, 8, False), (4, 5, False), (1, 8, True)] +) +@pytest.mark.parametrize( + "backup", ["success", "host_pressure", "failed", "raised", "ack_raised"] +) +def test_internal_swa_write_back_preserves_window_until_ack( + backend, page_size, window, with_mamba, backup +): + from sglang.srt.mem_cache.unified_cache.components import CacheTransferPhase + + # Real allocator, tree, backup planning, publication and eviction driver. + # The physical allocation/DMA boundary is simulated for CPU coverage. + case = _internal_swa_write_back_case(backend, page_size, window, with_mamba) + _mock_swa_write_back_io(case, backup) + cache, core = case.cache, case.core + parent, leaf = case.nodes + params = EvictParams(num_tokens=0, swa_num_tokens=case.segment) + if backup in ("raised", "ack_raised"): + with pytest.raises(RuntimeError, match="DMA .* failed"): + cache.evict(params) + assert [event[0] for event in case.events] == ( + ["write"] if backup == "raised" else ["write", "ack"] + ) + assert not case.freed + assert torch.equal( + core.get_component_device_value(parent, ComponentType.SWA), + case.swa[parent], + ) + if backup == "ack_raised": + cache._finish_write_through_ack(parent) + # The driver's finally block must discard the paused walk even when + # submission or completion fails, so a new eviction can make progress. + _mock_swa_write_back_io(case) + assert cache.evict(params).swa_num_tokens_evicted == case.segment + assert core.get_component_device_value(parent, ComponentType.SWA) is None + cache.dec_lock_ref(leaf, case.leaf_lock) + core.sanity_check([], []) + return + + result = cache.evict(params) + assert result.swa_num_tokens_evicted == case.segment + assert result.num_tokens_evicted == 0 + assert result.mamba_num_evicted == int(with_mamba) + if with_mamba: + assert core.get_component_device_value(parent, ComponentType.MAMBA) is None + assert (ComponentType.SWA, case.swa[parent].tolist()) in case.freed + assert not any(ct == ComponentType.FULL for ct, _ in case.freed) + assert core.get_component_device_value(parent, ComponentType.SWA) is None + assert torch.equal( + core.get_component_device_value(parent, ComponentType.FULL), case.full[parent] + ) + assert torch.equal( + core.get_component_device_value(leaf, ComponentType.SWA), case.swa[leaf] + ) + operations = [event[0] for event in case.events] + if backup == "failed": + assert operations == ["write"] + ["free"] * len(case.freed) + assert not core.component_has_host_value_only(parent, ComponentType.SWA) + else: + prefix = ["host_evict"] if backup == "host_pressure" else [] + assert operations == prefix + ["write", "ack"] + ["free"] * len(case.freed) + assert core.component_has_host_value_only(parent, ComponentType.SWA) + matched = cache.match_prefix( + MatchPrefixParams(key=_key(list(range(case.segment)))) + ) + assert matched.best_match_node == parent + assert matched.device_indices.numel() == 0 + assert matched.full_kv_hit_length == case.segment + assert matched.swa_host_hit_length == case.segment + assert matched.mamba_host_hit_length == int(with_mamba) + (transfer,) = core.build_hicache_transfers( + ComponentType.SWA, parent, CacheTransferPhase.LOAD_BACK + ) + assert transfer.nodes_to_load == [parent] + assert torch.equal(transfer.host_indices, case.swa[parent] + 20000) + cache.dec_lock_ref(leaf, case.leaf_lock) + core.sanity_check([], []) + + +@pytest.mark.parametrize("backend", ["python", "rust"]) +def test_internal_swa_write_back_reserves_entire_unbacked_window(backend): + case = _internal_swa_write_back_case(backend, prefix_nodes=2) + cache, core = case.cache, case.core + first, victim, leaf = case.nodes + # Refresh only the first segment; the next internal segment now heads the + # SWA LRU, and its backup window includes both unbacked prefix segments. + cache.match_prefix(MatchPrefixParams(key=_key(list(range(case.segment))))) + _mock_swa_write_back_io(case, "host_pressure") + result = cache.evict(EvictParams(num_tokens=0, swa_num_tokens=case.segment)) + assert result.swa_num_tokens_evicted == case.segment + assert case.events[:2] == [("host_evict", 2 * case.segment), ("write", victim)] + (submission,) = case.submissions + (transfer,) = submission[2][ComponentType.SWA] + assert transfer.nodes_to_load == [first, victim] + assert transfer.device_indices.numel() == 2 * case.segment + assert case.freed == [(ComponentType.SWA, case.swa[victim].tolist())] + assert torch.equal( + core.get_component_device_value(first, ComponentType.SWA), case.swa[first] + ) + assert core.component_has_host_value_only(victim, ComponentType.SWA) + cache.dec_lock_ref(leaf, case.leaf_lock) + core.sanity_check([], []) + + +@pytest.mark.parametrize("backend", ["python", "rust"]) +@pytest.mark.parametrize("guard", ["request", "pending_dma"]) +def test_internal_swa_write_back_respects_active_window_locks(backend, guard): + case = _internal_swa_write_back_case(backend) + _mock_swa_write_back_io(case) + cache, core = case.cache, case.core + parent, leaf = case.nodes + if guard == "request": + lock = cache.inc_lock_ref(parent).to_dec_params() + else: + # A normal asynchronous backup takes real component locks and leaves + # its source pinned until publication of the physical completion ACK. + assert ( + cache._execute_and_commit_kv_backup( + BackupKV(node_ids=[parent]), write_back=False + ) + == case.segment + ) + assert parent in cache.ongoing_write_through + submissions = len(case.submissions) + tracker = {ct: 0 for ct in cache.tree_components} + cache._evict_components( + { + ct: case.segment if ct == ComponentType.SWA else 0 + for ct in cache.tree_components + }, + tracker, + ) + assert tracker[ComponentType.SWA] == 0 and not case.freed + assert len(case.submissions) == submissions + assert torch.equal( + core.get_component_device_value(parent, ComponentType.SWA), case.swa[parent] + ) + if guard == "request": + cache.dec_lock_ref(parent, lock) + else: + cache._finish_write_through_ack(parent) + assert ( + cache.evict( + EvictParams(num_tokens=0, swa_num_tokens=case.segment) + ).swa_num_tokens_evicted + == case.segment + ) + assert core.component_has_host_value_only(parent, ComponentType.SWA) + cache.dec_lock_ref(leaf, case.leaf_lock) + core.sanity_check([], []) + + +@pytest.mark.parametrize("backend", ["python", "rust"]) +def test_internal_swa_write_back_follows_runtime_policy_update(backend): + from sglang.srt.mem_cache.unified_cache.storage_attachment import StorageAttachment + + case = _internal_swa_write_back_case(backend) + cache, core = case.cache, case.core + _mock_swa_write_back_io(case) + cache.is_write_back = False + cache.cache_controller.write_policy = "write_through" + cache.cache_controller.storage_backend_type = "file" + cache.enable_storage = True + cache.write_backup_storage = Mock() + success, _ = StorageAttachment(cache).attach( + "file", hicache_write_policy="write_back" + ) + assert success and cache.is_write_back + assert cache._tree_core_backend == backend + assert ( + cache.evict( + EvictParams(num_tokens=0, swa_num_tokens=case.segment) + ).swa_num_tokens_evicted + == case.segment + ) + assert core.component_has_host_value_only(case.nodes[0], ComponentType.SWA) + cache.write_backup_storage.assert_called_once_with(case.nodes[0]) + cache.dec_lock_ref(case.nodes[-1], case.leaf_lock) + core.sanity_check([], []) + + +def _internal_eviction_phase_case(backend, component, enable_session=False): + case = _internal_swa_write_back_case( + backend, with_mamba=True, enable_session=enable_session + ) + _mock_swa_write_back_io(case) + case.cache.components[ComponentType.MAMBA]._mamba_pool_host = Mock() + case.request = case.segment if component == ComponentType.SWA else 1 + case.finish = ( + case.core.finish_swa_state_eviction + if component == ComponentType.SWA + else case.core.finish_mamba_state_eviction + ) + return case + + +def _assert_internal_eviction_pending(step, component, node, swa_tokens=0): + assert step.node_id is None and step.made_progress + assert step.mamba_backup_node_id == ( + node if component == ComponentType.MAMBA else None + ) + assert step.swa_backup_node_id == (node if component == ComponentType.SWA else None) + assert step.swa_backup_num_tokens == swa_tokens + assert not step.tracker and not step.device_frees and not step.host_frees + assert step.unbacked_tokens == 0 + + +@pytest.mark.parametrize("component", [ComponentType.MAMBA, ComponentType.SWA]) +def test_python_component_returns_immutable_internal_backup_request(component): + from dataclasses import FrozenInstanceError + + from sglang.srt.mem_cache.unified_cache.components import InternalStateBackup + + case = _internal_swa_write_back_case("python", with_mamba=True, prefix_nodes=2) + _mock_swa_write_back_io(case) + cache, core = case.cache, case.core + first, second, leaf = case.nodes + if component == ComponentType.SWA: + cache.match_prefix(MatchPrefixParams(key=_key(list(range(case.segment))))) + victim, num_tokens = second, 2 * case.segment + else: + victim, num_tokens = first, 1 + original = { + ct: core.get_component_device_value(victim, ct).clone() + for ct in cache.tree_components + } + tracker, device_frees, host_frees = {component: 0}, {}, {} + core.evict_device_start(component, num_tokens) + try: + request = cache.components[component].evict_device_next_node( + tracker, device_frees, host_frees + ) + assert isinstance(request, InternalStateBackup) + assert (request.node_id, request.num_tokens) == (victim, num_tokens) + assert not case.events and not device_frees and not host_frees + assert tracker == {component: 0} + for ct, value in original.items(): + assert torch.equal(core.get_component_device_value(victim, ct), value) + with pytest.raises(FrozenInstanceError): + request.num_tokens = 0 + finally: + # The component returned directly to this caller; the core has not + # consumed the request, so only terminate the walk here. + core.evict_device_end(component) + cache.dec_lock_ref(leaf, case.leaf_lock) + core.sanity_check([], []) + + +@pytest.mark.parametrize("backend", ["python", "rust"]) +@pytest.mark.parametrize("component", [ComponentType.MAMBA, ComponentType.SWA]) +def test_internal_eviction_returns_backup_before_mutating_state(backend, component): + case = _internal_eviction_phase_case(backend, component) + cache, core = case.cache, case.core + parent, leaf = case.nodes + original = { + ct: core.get_component_device_value(parent, ct).clone() + for ct in cache.tree_components + } + before = {ct: core.component_evictable_size(ct) for ct in cache.tree_components} + tracker, freed, host_freed = {}, {}, {} + core.evict_device_start(component, case.request) + try: + step = core.evict_device_next_node(component, {}) + _assert_internal_eviction_pending( + step, + component, + parent, + case.segment if component == ComponentType.SWA else 0, + ) + assert not case.events + for ct, value in original.items(): + assert torch.equal(core.get_component_device_value(parent, ct), value) + assert core.component_evictable_size(ct) == before[ct] + + # Publish a completed backup through the same planner and ACK methods + # used by the controller, then consume exactly one mutation's deltas. + _complete_backup(core, parent) + result = case.finish(parent) + assert result.node_id is None and result.made_progress + assert result.mamba_backup_node_id is None + assert result.swa_backup_node_id is None and result.swa_backup_num_tokens == 0 + assert result.unbacked_tokens == 0 + _accumulate_step(result, tracker, freed, host_freed) + expected = {ComponentType.MAMBA: 1} + if component == ComponentType.SWA: + expected[ComponentType.SWA] = case.segment + assert tracker == expected and not host_freed + for ct, count in expected.items(): + assert torch.equal(torch.cat(freed[ct]), original[ct]) + assert core.get_component_device_value(parent, ct) is None + assert core.component_evictable_size(ct) == before[ct] - count + assert core.component_has_host_value_only(parent, ct) + assert torch.equal( + core.get_component_device_value(parent, ComponentType.FULL), + original[ComponentType.FULL], + ) + exhausted = core.evict_device_next_node(component, tracker) + assert not exhausted.made_progress and exhausted.node_id is None + assert not exhausted.tracker and not exhausted.device_frees + finally: + core.evict_device_end(component) + cache.dec_lock_ref(leaf, case.leaf_lock) + core.sanity_check([], []) + + +@pytest.mark.parametrize("backend", ["python", "rust"]) +@pytest.mark.parametrize("component", [ComponentType.MAMBA, ComponentType.SWA]) +@pytest.mark.parametrize("violation", ["advance", "wrong_node"]) +def test_internal_eviction_rejects_unfinished_or_mismatched_steps( + backend, component, violation +): + case = _internal_eviction_phase_case(backend, component) + parent, leaf = case.nodes + case.core.evict_device_start(component, case.request) + step = case.core.evict_device_next_node(component, {}) + _assert_internal_eviction_pending( + step, + component, + parent, + case.segment if component == ComponentType.SWA else 0, + ) + # A native protocol violation poisons this fresh core. Both implementations + # must expose an ordinary exception before attempting another mutation. + with pytest.raises((AssertionError, RuntimeError), match="pending internal"): + if violation == "advance": + case.core.evict_device_next_node(component, {}) + else: + case.finish(leaf) + + +@pytest.mark.parametrize("backend", ["python", "rust"]) +@pytest.mark.parametrize("component", [ComponentType.MAMBA, ComponentType.SWA]) +def test_internal_eviction_finish_rechecks_a_new_request_lock(backend, component): + case = _internal_eviction_phase_case(backend, component) + cache, core = case.cache, case.core + parent, leaf = case.nodes + original = core.get_component_device_value(parent, component).clone() + core.evict_device_start(component, case.request) + step = core.evict_device_next_node(component, {}) + _assert_internal_eviction_pending( + step, + component, + parent, + case.segment if component == ComponentType.SWA else 0, + ) + parent_lock = cache.inc_lock_ref(parent).to_dec_params() + result = case.finish(parent) + assert result.made_progress and result.node_id is None + assert not result.tracker and not result.device_frees and not result.host_frees + assert torch.equal(core.get_component_device_value(parent, component), original) + cache.dec_lock_ref(parent, parent_lock) + core.evict_device_end(component) + + core.evict_device_start(component, case.request) + try: + step = core.evict_device_next_node(component, {}) + _assert_internal_eviction_pending( + step, + component, + parent, + case.segment if component == ComponentType.SWA else 0, + ) + tracker, freed = {}, {} + _accumulate_step(case.finish(parent), tracker, freed, {}) + assert tracker[component] == case.request + assert torch.equal(torch.cat(freed[component]), original) + finally: + core.evict_device_end(component) + cache.dec_lock_ref(leaf, case.leaf_lock) + core.sanity_check([], []) + + +@pytest.mark.parametrize("backend", ["python", "rust"]) +@pytest.mark.parametrize("component", [ComponentType.MAMBA, ComponentType.SWA]) +@pytest.mark.parametrize("cleanup", ["end", "reset"]) +def test_internal_eviction_cleanup_discards_the_pending_request( + backend, component, cleanup +): + case = _internal_eviction_phase_case(backend, component) + cache, core = case.cache, case.core + parent, leaf = case.nodes + core.evict_device_start(component, case.request) + step = core.evict_device_next_node(component, {}) + _assert_internal_eviction_pending( + step, + component, + parent, + case.segment if component == ComponentType.SWA else 0, + ) + if cleanup == "reset": + core.reset() + else: + core.evict_device_end(component) + assert torch.equal( + core.get_component_device_value(parent, ComponentType.SWA), + case.swa[parent], + ) + core.evict_device_start(component, case.request) + try: + step = core.evict_device_next_node(component, {}) + if cleanup == "reset": + assert not step.made_progress and step.node_id is None + assert step.mamba_backup_node_id is None and step.swa_backup_node_id is None + assert all( + core.component_evictable_size(ct) == 0 for ct in cache.tree_components + ) + else: + _assert_internal_eviction_pending( + step, + component, + parent, + case.segment if component == ComponentType.SWA else 0, + ) + tracker = {} + _accumulate_step(case.finish(parent), tracker, {}, {}) + assert tracker[component] == case.request + finally: + core.evict_device_end(component) + if cleanup != "reset": + cache.dec_lock_ref(leaf, case.leaf_lock) + core.sanity_check([], []) + + +@pytest.mark.parametrize("component", [ComponentType.MAMBA, ComponentType.SWA]) +def test_python_session_internal_eviction_resumes_in_partition_order(component): + case = _internal_swa_write_back_case( + "python", with_mamba=True, prefix_nodes=3, enable_session=True + ) + cache, core = case.cache, case.core + assert cache._tree_core_backend == "python" and core.enable_session_radix_cache + first, second, third, leaf = case.nodes + comp = cache.components[component] + comp.register_session_leaf("retained", core.node_by_id(first)) + assert comp.session_ref(core.node_by_id(first)) == 1 + finish = ( + core.finish_mamba_state_eviction + if component == ComponentType.MAMBA + else core.finish_swa_state_eviction + ) + tracker = {} + request = 3 if component == ComponentType.MAMBA else 3 * case.segment + core.evict_device_start(component, request) + try: + # The referenced oldest prefix comes after both unreferenced prefixes. + # Each finish removes the current node before the sentinel moves on. + for victim, window_tokens in ((second, 4), (third, 2), (first, 2)): + step = core.evict_device_next_node(component, tracker) + _assert_internal_eviction_pending( + step, + component, + victim, + window_tokens if component == ComponentType.SWA else 0, + ) + _accumulate_step(finish(victim), tracker, {}, {}) + assert tracker[component] == request + assert not core.evict_device_next_node(component, tracker).made_progress + finally: + core.evict_device_end(component) + # The Full node still exists, so its session frontier remains registered + # across an auxiliary tombstone until the session itself is released. + assert comp.session_ref(core.node_by_id(first)) == 1 + assert comp.release_session("retained") == 1 + cache.dec_lock_ref(leaf, case.leaf_lock) + core.sanity_check([], []) + + +def test_custom_swa_component_keeps_its_inline_backup_contract(): + from sglang.srt.mem_cache.unified_cache.components.swa import SWAComponent + from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache + + class AuxiliarySWAComponent(SWAComponent): + component_type = ComponentType.AUXILIARY_SWA + + def _publish_window(self, node, cache_actions): + cache_actions.append( + SWARebuild( + node.id, + node.component_data[ComponentType.FULL].value, + component_type=self.component_type, + ) + ) + + reference, allocator = _swa_cache(window=4, backend="python") + cache = UnifiedRadixCache( + CacheInitParams( + disable=False, + req_to_token_pool=reference.req_to_token_pool, + token_to_kv_pool_allocator=allocator, + page_size=1, + sliding_window_size=4, + tree_components=(ComponentType.FULL, ComponentType.AUXILIARY_SWA), + component_registry_override={ + ComponentType.AUXILIARY_SWA: AuxiliarySWAComponent + }, + tree_core_backend="python", + ) + ) + core = cache.tree_core + values = allocator.alloc(6) + parent = cache.insert( + InsertParams(key=_key([0, 1]), value=values[:2]) + ).last_device_node + leaf = cache.insert( + InsertParams(key=_key(list(range(6))), value=values, prev_prefix_len=2) + ).last_device_node + lock = cache.inc_lock_ref(leaf).to_dec_params() + component = ComponentType.AUXILIARY_SWA + original = core.get_component_device_value(parent, component).clone() + core.set_hicache_enabled() + core.has_swa_host_pool = True + cache.is_write_back = True + cache.cache_controller = SimpleNamespace(write_policy="write_back") + events = [] + + def backup(node): + assert node == parent + assert torch.equal(core.get_component_device_value(node, component), original) + events.append(node) + return False # Best-effort allocation failure must still make progress. + + cache.backup_node_for_write_back = backup + core.evict_device_start(component, 2) + try: + result = core.evict_device_next_node(component, {}) + assert result.node_id is None and result.made_progress + assert result.swa_backup_node_id is None and result.mamba_backup_node_id is None + tracker, freed = {}, {} + _accumulate_step(result, tracker, freed, {}) + assert events == [parent] and tracker[component] == 2 + assert torch.equal(torch.cat(freed[component]), original) + assert core.get_component_device_value(parent, component) is None + assert not core.evict_device_next_node(component, tracker).made_progress + finally: + core.evict_device_end(component) + cache.dec_lock_ref(leaf, lock) + core.sanity_check([], []) + + +@pytest.mark.parametrize("backend", ["python", "rust"]) +@pytest.mark.parametrize("component", [ComponentType.MAMBA, ComponentType.SWA]) +@pytest.mark.parametrize("pin_ancestor", [False, True]) +def test_aux_host_reclaim_uses_its_own_load_back_sources( + backend, component, pin_ancestor +): + from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool + from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache + + if component == ComponentType.MAMBA: + with ( + get_context().override_server_args( + _mamba_cache_chunk_size=256, mamba_max_states_per_path=-1 + ), + envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.override(backend), + ): + cache = UnifiedRadixCache( + CacheInitParams( + disable=True, + req_to_token_pool=Mock(spec=HybridReqToTokenPool), + token_to_kv_pool_allocator=None, + page_size=1, + tree_components=(ComponentType.FULL, ComponentType.MAMBA), + ) + ) + core = cache.tree_core + parent = _mamba_insert(core, [1, 2], [101, 102], 201).last_device_node + child = _mamba_insert( + core, [1, 2, 3, 4], [101, 102, 103, 104], 202 + ).last_device_node + loaded_full = torch.tensor([301, 302, 303, 304]) + amount = 1 + else: + cache, allocator = _swa_cache(window=4, backend=backend) + core = cache.tree_core + loaded_full = allocator.alloc(8) + parent = cache.insert( + InsertParams(key=_key(list(range(4))), value=loaded_full[:4]) + ).last_device_node + child = cache.insert( + InsertParams(key=_key(list(range(8))), value=loaded_full, prev_prefix_len=4) + ).last_device_node + core.has_swa_host_pool = True + amount = 4 + + core.set_hicache_enabled() + core.is_write_back = True + _, parent_aux = _complete_backup(core, parent) + expected_free = parent_aux[component][0].host_indices.tolist() + _complete_backup(core, child) + for node in (child, parent): + _accumulate_step(core.demote(node), {}, {}, {}) + assert core.component_has_host_value_only(parent, component) + + # Loading the child reads Full from both nodes but auxiliary data only + # from the child. Follow the controller's lock/commit/relock sequence. + host_lock = core.inc_host_lock_ref(child).to_dec_params() + before_lock = core.inc_lock_ref(child).to_dec_params() + kv, aux = core.build_load_back_spec(child) + assert kv.nodes_to_load == [parent, child] + assert aux[component][0].nodes_to_load == [child] + for transfers in aux.values(): + for transfer in transfers: + transfer.device_indices = transfer.host_indices + 2000 + core.dec_lock_ref(child, before_lock) + core.commit_load_back(child, loaded_full, kv, aux) + pending_lock = core.inc_lock_ref(child).to_dec_params() + extra_lock = ( + core.inc_host_lock_ref(parent).to_dec_params() if pin_ancestor else None + ) + core.sanity_check([], [(child, child)]) + + tracker, host_frees = {}, {} + _accumulate_step( + core.drive_host_eviction(component, amount), tracker, {}, host_frees + ) + assert tracker.get(component, 0) == (0 if pin_ancestor else amount) + assert [value.tolist() for value in host_frees.get(component, [])] == ( + [] if pin_ancestor else [expected_free] + ) + assert core.is_backuped(parent) and core.is_backuped(child) + core.sanity_check([], [(child, child)]) + + core.dec_lock_ref(child, pending_lock) + core.dec_host_lock_ref(child, host_lock) + core.finish_load_back(child) + if extra_lock is not None: + core.dec_host_lock_ref(parent, extra_lock) + tracker, host_frees = {}, {} + _accumulate_step( + core.drive_host_eviction(component, amount), tracker, {}, host_frees + ) + assert tracker.get(component, 0) == (amount if pin_ancestor else 0) + assert [value.tolist() for value in host_frees.get(component, [])] == ( + [expected_free] if pin_ancestor else [] + ) + core.sanity_check([], []) + + +@pytest.mark.parametrize("backend", ["python", "rust"]) +@pytest.mark.parametrize("component", [ComponentType.MAMBA, ComponentType.SWA]) +@pytest.mark.parametrize("pin_ancestor", [False, True]) +def test_full_load_back_does_not_pin_restored_ancestor_aux( + backend, component, pin_ancestor +): + from sglang.srt.mem_cache.allocator.token import TokenToKVPoolAllocator + from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool + from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache + + if component == ComponentType.SWA: + cache, allocator = _swa_cache(window=4, backend=backend) + else: + allocator = TokenToKVPoolAllocator(64, torch.bfloat16, "cpu", None, False) + request_pool = Mock(spec=HybridReqToTokenPool) + request_pool.mamba_allocator = Mock() + with ( + get_context().override_server_args( + _mamba_cache_chunk_size=256, mamba_max_states_per_path=-1 + ), + envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.override(backend), + ): + cache = UnifiedRadixCache( + CacheInitParams( + disable=False, + req_to_token_pool=request_pool, + token_to_kv_pool_allocator=allocator, + page_size=1, + tree_components=(ComponentType.FULL, ComponentType.MAMBA), + ) + ) + core = cache.tree_core + values = allocator.alloc(8) + nodes = [ + cache.insert( + InsertParams( + key=_key(list(range(size))), + value=values[:size], + prev_prefix_len=size - 4, + mamba_value=torch.tensor([slot]) + if component == ComponentType.MAMBA + else None, + ) + ).last_device_node + for size, slot in ((4, 201), (8, 202)) + ] + parent, child = nodes + core.set_hicache_enabled() + core.has_swa_host_pool = component == ComponentType.SWA + core.is_write_back = True + full_host = [ + _complete_backup(core, node, full_offset=1000, aux_offset=2000)[0] + for node in nodes + ] + for node in (child, parent): + step = core.demote(node) + cache._free_values(step.device_frees, step.host_frees) + assert core.component_has_host_value_only(parent, component) + submissions = [] + + def load(host_indices, node_id, extra_pools): + assert node_id == child + assert host_indices.tolist() == torch.cat(full_host).tolist() + assert len(extra_pools) == 1 and extra_pools[0].nodes_to_load == [child] + full = allocator.alloc(host_indices.numel()) + for transfer in extra_pools: + transfer.device_indices = ( + allocator.alloc(transfer.host_indices.numel()) + if component == ComponentType.SWA + else transfer.host_indices + 1000 + ) + submissions.append(node_id) + return full + + cache.cache_controller = SimpleNamespace(load=load, write_policy="write_back") + cache.load_back_threshold = 10 + cache._build_sidecar_transfers = Mock(return_value=[]) + assert cache.load_back(child, req=None) + assert submissions == [child] + assert core.get_component_device_value(parent, component) is None + selected = core.get_component_device_value(child, component).tolist() + core.sanity_check([], [(child, child)]) + + # A separate completed request restores state outside the transfer's + # auxiliary span after load_back has committed and re-locked the path. + cache.insert( + InsertParams( + key=_key(list(range(4))), + value=allocator.alloc(4), + mamba_value=torch.tensor([211]) + if component == ComponentType.MAMBA + else None, + ) + ) + restored = core.get_component_device_value(parent, component).tolist() + extra_lock = cache.inc_lock_ref(parent).to_dec_params() if pin_ancestor else None + active = [(parent, parent)] if pin_ancestor else [] + core.sanity_check(active, [(child, child)]) + full_before = { + node: core.get_component_device_value(node, ComponentType.FULL).tolist() + for node in nodes + } + # SWA frees are Full-side IDs, which the allocator resolves through its + # current Full-to-SWA mapping; Mamba frees are its own state slots. + expected_free = full_before[parent] if component == ComponentType.SWA else restored + tracker, frees = {component: 0}, {} + core.evict_device_start(component, len(restored)) + try: + step = core.evict_device_next_node(component, tracker) + assert step.node_id is None and step.mamba_backup_node_id is None + _accumulate_step(step, tracker, frees, {}) + finally: + core.evict_device_end(component) + assert [value.tolist() for value in frees.get(component, [])] == ( + [] if pin_ancestor else [expected_free] + ) + assert core.get_component_device_value(child, component).tolist() == selected + for node in nodes: + assert ( + core.get_component_device_value(node, ComponentType.FULL).tolist() + == full_before[node] + ) + core.sanity_check(active, [(child, child)]) + + event = Mock() + cache.cache_controller.ack_load_queue = [ + SimpleNamespace( + finish_event=event, + node_ids=[child], + num_tokens_by_pool={}, + num_bytes=0, + timing_enabled=False, + ) + ] + cache.loading_check(finish_count=1) + event.synchronize.assert_called_once_with() + core.sanity_check(active, []) + core.evict_device_start(component, 1) + try: + step = core.evict_device_next_node(component, {component: 0}) + assert step.node_id == child + assert not step.device_frees + _accumulate_step(step, {}, {}, {}) + finally: + core.evict_device_end(component) + if extra_lock is not None: + cache.dec_lock_ref(parent, extra_lock) + core.sanity_check([], []) + + +@pytest.mark.parametrize("backend", ["python", "rust"]) +def test_full_host_duplicates_follow_ack_order_after_pending_swa_split(backend): + cache, allocator = _swa_cache(window=4, backend=backend) + core = cache.tree_core + values = allocator.alloc(12) + nodes = [ + cache.insert( + InsertParams( + key=_key(list(range(size))), + value=values[:size], + prev_prefix_len=size - 4, + ) + ).last_device_node + for size in (4, 8, 12) + ] + parent, child, leaf = nodes + core.set_hicache_enabled() + core.has_swa_host_pool = True + core.is_write_back = True + for node in nodes: + _complete_backup(core, node, full_offset=10000, aux_offset=20000) + + # Repairing SWA later must issue an async backup even in write-back mode. + # Reclaim the two internal SWA states and the parent's Full host copy. + tracker = {} + core.evict_device_start(ComponentType.SWA, 8) + try: + for _ in range(2): + step = core.evict_device_next_node(ComponentType.SWA, tracker) + assert step.node_id is None + for component, count in step.tracker.items(): + tracker[component] = tracker.get(component, 0) + count + cache._free_values(step.device_frees, step.host_frees) + finally: + core.evict_device_end(ComponentType.SWA) + assert tracker[ComponentType.SWA] == 8 + for component, amount in ((ComponentType.SWA, 8), (ComponentType.FULL, 4)): + step = core.drive_host_eviction(component, amount) + assert step.tracker[component] == amount + _accumulate_step(step, {}, {}, {}) + assert not core.is_backuped(parent) and core.is_backuped(child) + + # Mock DMA submission only; insertion, backup publication, the pending + # split and FIFO acknowledgment all use the real shared controller. + cache.cache_controller = SimpleNamespace(write_policy="write_back") + cache._build_backup_sidecar = Mock(return_value=[]) + submissions = [] + + def write(node, device, aux, sidecars): + submissions.append((node, device.numel())) + for transfers in aux.values(): + for transfer in transfers: + transfer.host_indices = transfer.device_indices + 30000 + return device + 40000 + + cache._execute_kv_backup = write + cache.insert( + InsertParams( + key=_key(list(range(8))), + value=allocator.alloc(8), + component_evicted_seqlens={ComponentType.SWA: 4}, + ) + ) + assert submissions == [(parent, 4), (child, 0)] + assert set(cache.ongoing_write_through) == {parent, child} + core.sanity_check([(node, node) for node in cache.ongoing_write_through], []) + cache.match_prefix(MatchPrefixParams(key=_key([0, 1]))) + prefix, suffix = cache.ongoing_write_through[parent].publish_node_ids + assert suffix == parent and prefix != parent + core.sanity_check([(node, node) for node in cache.ongoing_write_through], []) + for node in (parent, child): + cache._finish_write_through_ack(node) + core.sanity_check([], []) + + # The existing child keeps its place. New Full copies join at ack in + # prefix/suffix order, including when the pending node was internal. + expected = [ + (values[4:8] + 10000).tolist(), + (values[:2] + 40000).tolist(), + (values[2:4] + 40000).tolist(), + (values[8:] + 10000).tolist(), + ] + for host_indices in expected: + host_frees = {} + _accumulate_step( + core.drive_host_eviction(ComponentType.FULL, 1), {}, {}, host_frees + ) + assert [value.tolist() for value in host_frees[ComponentType.FULL]] == [ + host_indices + ] + assert all(not core.is_backuped(node) for node in (prefix, parent, child, leaf)) + core.sanity_check([], []) + + +@pytest.mark.parametrize("backend", ["python", "rust"]) +@pytest.mark.parametrize("pin_ancestor", [False, True]) +def test_pending_swa_backup_does_not_pin_restored_ancestor_mamba(backend, pin_ancestor): + cache, allocator = _swa_mamba_cache(backend, window=8) + core = cache.tree_core + values = allocator.alloc(12) + nodes = [ + cache.insert( + InsertParams( + key=_key(list(range(size))), + value=values[:size], + prev_prefix_len=size - 4, + mamba_value=torch.tensor([slot]), + ) + ).last_device_node + for size, slot in ((4, 201), (8, 202), (12, 203)) + ] + parent, child, leaf = nodes + core.set_hicache_enabled() + core.has_swa_host_pool = True + core.is_write_back = True + for node in nodes: + _complete_backup(core, node, full_offset=10000, aux_offset=20000) + + # SWA eviction also removes ancestor Mamba states. Repair SWA first; + # a separate request below restores the ancestor's Mamba checkpoint. + tracker = {} + core.evict_device_start(ComponentType.SWA, 8) + try: + for _ in range(2): + step = core.evict_device_next_node(ComponentType.SWA, tracker) + assert step.node_id is None + for component, count in step.tracker.items(): + tracker[component] = tracker.get(component, 0) + count + cache._free_values(step.device_frees, step.host_frees) + finally: + core.evict_device_end(ComponentType.SWA) + assert tracker[ComponentType.SWA] == 8 + step = core.drive_host_eviction(ComponentType.SWA, 8) + assert step.tracker[ComponentType.SWA] == 8 + _accumulate_step(step, {}, {}, {}) + assert all(core.is_backuped(node) for node in nodes) + cache.cache_controller = SimpleNamespace(write_policy="write_back") + cache._build_backup_sidecar = Mock(return_value=[]) + submissions = [] + + def write(node, full, aux, sidecars): + submissions.append((node, full.numel(), aux)) + for transfers in aux.values(): + for transfer in transfers: + transfer.host_indices = transfer.device_indices + 30000 + return full + 40000 + + cache._execute_kv_backup = write + cache.insert( + InsertParams( + key=_key(list(range(8))), + value=allocator.alloc(8), + component_evicted_seqlens={ComponentType.SWA: 0}, + mamba_value=torch.tensor([202]), + ) + ) + assert len(submissions) == 1 + node, full_rows, aux = submissions[0] + assert node == child and full_rows == 0 + assert aux[ComponentType.SWA][0].nodes_to_load == [parent, child] + assert aux[ComponentType.MAMBA][0].device_indices.tolist() == [202] + assert list(cache.ongoing_write_through) == [child] + assert cache.ongoing_write_through[child].publish_node_ids == [parent, child] + assert core.get_component_device_value(parent, ComponentType.MAMBA) is None + + cache.insert( + InsertParams( + key=_key(list(range(4))), + value=allocator.alloc(4), + mamba_value=torch.tensor([211]), + ) + ) + assert len(submissions) == 1 + assert core.get_component_device_value(parent, ComponentType.MAMBA).tolist() == [ + 211 + ] + leaf_lock = cache.inc_lock_ref(leaf).to_dec_params() + extra_lock = cache.inc_lock_ref(parent).to_dec_params() if pin_ancestor else None + active = [(leaf, leaf)] + ([(parent, parent)] if pin_ancestor else []) + core.sanity_check([(child, child)] + active, []) + retained = { + (node, component): core.get_component_device_value(node, component).tolist() + for node in nodes + for component in (ComponentType.FULL, ComponentType.SWA) + } + + def evict_one_mamba_state(): + tracker, frees = {ComponentType.MAMBA: 0}, {} + core.evict_device_start(ComponentType.MAMBA, 1) + try: + step = core.evict_device_next_node(ComponentType.MAMBA, tracker) + assert step.node_id is None and step.mamba_backup_node_id is None + _accumulate_step(step, tracker, frees, {}) + finally: + core.evict_device_end(ComponentType.MAMBA) + return [value.tolist() for value in frees.get(ComponentType.MAMBA, [])] + + assert evict_one_mamba_state() == ([] if pin_ancestor else [[211]]) + # The actual Mamba DMA source remains protected by its own device lock. + assert core.get_component_device_value(child, ComponentType.MAMBA).tolist() == [202] + core.sanity_check([(child, child)] + active, []) + for (node, component), indices in retained.items(): + assert core.get_component_device_value(node, component).tolist() == indices + cache._finish_write_through_ack(child) + assert evict_one_mamba_state() == [[202]] + core.sanity_check(active, []) + cache.dec_lock_ref(leaf, leaf_lock) + if extra_lock is not None: + cache.dec_lock_ref(parent, extra_lock) + core.sanity_check([], []) + + +@pytest.mark.parametrize("backend", ["python", "rust"]) +def test_failed_mamba_backup_drops_state_under_unrelated_pending_swa(backend): + cache, allocator = _swa_mamba_cache(backend, window=12) + core = cache.tree_core + values = allocator.alloc(16) + nodes = [ + cache.insert( + InsertParams( + key=_key(list(range(size))), + value=values[:size], + prev_prefix_len=size - 4, + mamba_value=torch.tensor([slot]), + ) + ).last_device_node + for size, slot in ((4, 201), (8, 202), (12, 203), (16, 204)) + ] + grandparent, parent, anchor, leaf = nodes + core.set_hicache_enabled() + core.has_swa_host_pool = True + core.is_write_back = True + cache.cache_controller = SimpleNamespace(write_policy="write_back") + cache._build_backup_sidecar = Mock(return_value=[]) + host_pool = Mock() + host_pool.available_size.return_value = 1 + cache.host_pool_group = Mock() + cache.host_pool_group.get_pool.return_value = host_pool + cache.components[ComponentType.MAMBA]._mamba_pool_host = host_pool + submissions = [] + failed_node = None + + def write(node, full, aux, sidecars): + submissions.append((node, full.numel(), aux)) + if node == failed_node: + return None # Full host allocation failed before submitting DMA. + for transfers in aux.values(): + for transfer in transfers: + transfer.host_indices = transfer.device_indices + 20000 + return full + 10000 + + def complete_physical_writes(write_back=False): + for ack_id in list(cache.ongoing_write_through): + cache._finish_write_through_ack(ack_id) + + cache._execute_kv_backup = write + cache.writing_check = complete_physical_writes + + def evict_component(component, count): + tracker = {ct: 0 for ct in cache.tree_components} + requested = { + ct: count if ct == component else 0 for ct in cache.tree_components + } + cache._evict_components(requested, tracker) + return tracker[component] + + # Reach the host topology through actual state backups and ACKs. Full + # duplicate reclaim then leaves an unbacked grandparent above a backed + # parent; no synthetic node flags or direct backup commits are needed. + assert evict_component(ComponentType.MAMBA, 3) == 3 + assert [(node, count) for node, count, _ in submissions] == [ + (node, 4) for node in nodes[:3] + ] + assert cache.evict_host(4, ComponentType.FULL) == 4 + assert [core.is_backuped(node) for node in nodes] == [False, True, True, False] + core.sanity_check([], []) + + assert evict_component(ComponentType.SWA, 12) == 12 + assert cache.evict_host(12, ComponentType.SWA) == 12 + cache.insert( + InsertParams( + key=_key(list(range(12))), + value=allocator.alloc(12), + mamba_value=torch.tensor([203]), + ) + ) + # Backup ancestry stops at the backed parent, while the expanded SWA + # window reaches the unbacked grandparent. Its pending mark belongs to SWA. + assert list(cache.ongoing_write_through) == [anchor] + assert cache.ongoing_write_through[anchor].publish_node_ids == nodes[:3] + assert submissions[-1][2][ComponentType.SWA][0].nodes_to_load == nodes[:3] + cache.insert( + InsertParams( + key=_key(list(range(4))), + value=allocator.alloc(4), + mamba_value=torch.tensor([211]), + ) + ) + assert not core.is_backuped(grandparent) + assert ComponentType.MAMBA in core.build_backup_spec(grandparent)[1] + leaf_lock = cache.inc_lock_ref(leaf).to_dec_params() + active = [(anchor, anchor), (leaf, leaf)] + core.sanity_check(active, []) + retained = { + (node, ct): core.get_component_device_value(node, ct).clone() + for node in nodes + for ct in (ComponentType.FULL, ComponentType.SWA) + } + + failed_node = grandparent + assert evict_component(ComponentType.MAMBA, 1) == 1 + assert submissions[-1][:2] == (grandparent, 4) + assert core.get_component_device_value(grandparent, ComponentType.MAMBA) is None + # The completed backup and active request retain their other device values. + for node, slot in ((anchor, 203), (leaf, 204)): + assert core.get_component_device_value(node, ComponentType.MAMBA).tolist() == [ + slot + ] + for (node, ct), value in retained.items(): + assert torch.equal(core.get_component_device_value(node, ct), value) + # The earlier SWA ACK is drained before the failed backup attempt. + assert not cache.ongoing_write_through + core.sanity_check([(leaf, leaf)], []) + cache.dec_lock_ref(leaf, leaf_lock) + core.sanity_check([], []) + + +@pytest.mark.parametrize("backend", ["python", "rust"]) +@pytest.mark.parametrize( + "ring,hicache,has_swa_host_pool", + [ + (False, False, False), + (True, False, False), + (False, True, False), + (False, True, True), + (True, True, False), + ], +) +def test_swa_match_uses_allocator_layout(backend, ring, hicache, has_swa_host_pool): + # A request ring rebuilds its SWA window outside the tree, while paged SWA + # requires resident state even if HiCache has no SWA host pool. + cache, allocator = _swa_cache(backend=backend, ring=ring) + assert type(cache.tree_core).__name__ == ( + "RustUnifiedTreeCore" if backend == "rust" else "UnifiedTreeCore" + ) + indices = allocator.alloc(12) + cache.insert( + InsertParams( + key=_key(list(range(12))), + value=indices, + component_evicted_seqlens={ComponentType.SWA: 8}, + ) + ) + if hicache: + cache.tree_core.set_hicache_enabled() + cache.tree_core.has_swa_host_pool = has_swa_host_pool + matched = cache.match_prefix(MatchPrefixParams(key=_key(list(range(8))))) + assert matched.full_kv_hit_length == 8 + assert matched.device_indices.tolist() == (indices[:8].tolist() if ring else []) + + def test_buffer_backup_snapshot_round_trips_and_detects_a_split(): core = _tree_core() core.enable_storage = True @@ -1529,6 +3872,148 @@ def test_swa_load_back_missing_value_raises_assertion_error(): core.build_load_back_spec(node) +@pytest.mark.parametrize("backend", ["python", "rust"]) +@pytest.mark.parametrize("is_bigram", [False, True]) +def test_swa_window_repair_with_page_rounded_window(backend, is_bigram): + core, _ = _swa_transfer_core(backend, page_size=4, window=5, is_eagle=is_bigram) + key = RadixKey( + array("q", range(16 + is_bigram)), + extra_key="adapter-a", + cache_salt="tenant-a", + is_bigram=is_bigram, + ) + full_values = torch.arange(10, 26, dtype=torch.int64) + _pump_insert( + core, + InsertParams( + key=key, + value=full_values, + component_evicted_seqlens={ComponentType.SWA: 16}, + ), + ) + # A five-token window stages two four-token pages. Bigrams add one raw + # boundary token but preserve the logical repair span [8, 16). + ranges = core.swa_tombstone_ranges(key, 8, 16) + assert ranges == [(8, 16)] + assert core.attach_swa_window(key, 8, 16, torch.arange(50, 58)) == [] + matched_len, leaf, _ = core.match_full_device_prefix(key) + assert matched_len == 16 + assert torch.equal( + core.collect_full_device_indices(leaf, core.root_node_handle()), full_values + ) + assert core.get_component_device_value(leaf, ComponentType.SWA).tolist() == list( + range(50, 58) + ) + assert core.swa_tombstone_ranges(key, 0, 16) == [(0, 8)] + assert core.full_evictable_size() == 16 + assert core.swa_evictable_size() == 8 + core.sanity_check([], []) + + +@pytest.mark.parametrize("page_size", [1, 2]) +@pytest.mark.parametrize("is_bigram", [False, True]) +def test_swa_window_repair_preserves_full_slots_and_key_namespace(page_size, is_bigram): + core = _swa_tree_core(page_size=page_size, is_eagle=is_bigram) + key = RadixKey( + array("q", range(8 + is_bigram)), + extra_key="adapter-a", + cache_salt="tenant-a", + is_bigram=is_bigram, + ) + _pump_insert( + core, + InsertParams( + key=key, + value=torch.arange(10, 18, dtype=torch.int64), + component_evicted_seqlens={ComponentType.SWA: 8}, + ), + ) + assert core.swa_tombstone_ranges(key, 0, 8) == [(0, 8)] + values = torch.arange(50, 54, dtype=torch.int64) + assert core.attach_swa_window(key, 2, 6, values) == [] + values.fill_(-1) + + # Slicing and namespace conversion must retain logical bigram positions. + full_values = [] + for end in (2, 6, 8): + _, node, _ = core.match_full_device_prefix(key[:end]) + full_values.extend( + core.get_component_device_value(node, ComponentType.FULL).tolist() + ) + swa = core.get_component_device_value(node, ComponentType.SWA) + if end == 6: + assert swa.tolist() == [50, 51, 52, 53] + else: + assert swa is None + assert full_values == list(range(10, 18)) + assert core.swa_tombstone_ranges(key, 0, 8) == [(0, 2), (6, 8)] + assert core.swa_tombstone_ranges(key, 1, 7) == [(1, 2), (6, 7)] + for extra_key, cache_salt in (("adapter-b", "tenant-a"), ("adapter-a", "tenant-b")): + other_key = RadixKey( + key.token_ids, + extra_key=extra_key, + cache_salt=cache_salt, + is_bigram=is_bigram, + ) + assert core.swa_tombstone_ranges(other_key, 0, 8) == [] + assert core.full_evictable_size() == 8 + assert core.swa_evictable_size() == 4 + core.sanity_check([], []) + + +def test_swa_window_repair_returns_split_actions_and_balances_existing_locks(): + core = _swa_tree_core() + core.set_hicache_enabled() + key = _key(list(range(8))) + _pump_insert( + core, + InsertParams( + key=key, + value=torch.arange(10, 18, dtype=torch.int64), + component_evicted_seqlens={ComponentType.SWA: 8}, + ), + ) + _, leaf, _ = core.match_full_device_prefix(key) + receipt = core.inc_lock_ref(leaf) + core.mark_write_through_pending([leaf], ack_id=leaf) + actions = core.attach_swa_window(key, 2, 6, torch.arange(50, 54)) + assert len(actions) == 2 + for action in actions: + assert isinstance(action, ReplaceWriteThroughOnNodeSplit) + assert action.ack_id == leaf + assert action.old_node_id == leaf + assert action.new_child_node_id == leaf + assert actions[0].new_node_id != actions[1].new_node_id + assert core.swa_protected_size() == 4 + assert core.swa_evictable_size() == 0 + core.finish_write_through([action.new_node_id for action in actions] + [leaf], leaf) + core.dec_lock_ref(leaf, receipt.to_dec_params()) + assert core.swa_protected_size() == 0 + assert core.swa_evictable_size() == 4 + core.sanity_check([], []) + + +def test_swa_window_repair_rejects_invalid_publication_without_partial_changes(): + core = _swa_tree_core() + key = _key(list(range(8))) + _pump_insert( + core, + InsertParams( + key=key, + value=torch.arange(10, 18, dtype=torch.int64), + component_evicted_seqlens={ComponentType.SWA: 8}, + ), + ) + core.attach_swa_window(key, 2, 6, torch.arange(50, 54)) + with pytest.raises(AssertionError, match="over live SWA"): + core.attach_swa_window(key, 0, 6, torch.arange(60, 66)) + with pytest.raises(AssertionError, match="int64"): + core.attach_swa_window(key, 0, 2, torch.zeros(2, dtype=torch.int32)) + assert core.swa_tombstone_ranges(key, 0, 8) == [(0, 2), (6, 8)] + assert core.swa_evictable_size() == 4 + core.sanity_check([], []) + + def test_swa_straddling_insert_crosses_the_boundary_actions(): core = _swa_tree_core(window=8) _insert(core, [1, 2, 3, 4], [10, 11, 12, 13]) @@ -1734,6 +4219,7 @@ def test_mamba_eviction_walk_frees_slots_through_the_adapter(): step = core.evict_device_next_node(ComponentType.MAMBA, tracker) assert step.node_id is None assert step.made_progress + assert step.unbacked_tokens == 0 # Only an auxiliary state was dropped. _accumulate_step(step, tracker, device_frees, host_frees) step = core.evict_device_next_node(ComponentType.MAMBA, tracker) @@ -1752,6 +4238,170 @@ def test_mamba_eviction_walk_frees_slots_through_the_adapter(): core.sanity_check([], []) +@pytest.mark.parametrize("backend", ["python", "rust"]) +@pytest.mark.parametrize( + "backup", + ["success", "host_pressure", "missing_pool", "failed", "raised", "ack_raised"], +) +def test_internal_mamba_write_back_preserves_state_until_ack(backend, backup): + from collections import defaultdict + + from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool + from sglang.srt.mem_cache.unified_cache.components import CacheTransferPhase + from sglang.srt.mem_cache.unified_cache.components.full import FullComponent + from sglang.srt.mem_cache.unified_cache.components.mamba import MambaComponent + from sglang.srt.mem_cache.unified_cache.unified_tree_core import UnifiedTreeCore + from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache + + # Real components, tree, backup planner/commit and controller eviction. Only + # allocation and DMA completion are simulated so this also runs on CPU. + cache = UnifiedRadixCache.__new__(UnifiedRadixCache) + cache.enable_session_radix_cache = False + cache.token_to_kv_pool_allocator = None + params = CacheInitParams( + disable=False, + req_to_token_pool=Mock(spec=HybridReqToTokenPool), + token_to_kv_pool_allocator=None, + page_size=1, + tree_components=(ComponentType.FULL, ComponentType.MAMBA), + ) + with get_context().override_server_args( + _mamba_cache_chunk_size=256, mamba_max_states_per_path=-1 + ): + cache.components = { + ComponentType.FULL: FullComponent(cache, params), + ComponentType.MAMBA: MambaComponent(cache, params), + } + core = ( + RustUnifiedTreeCore(params) + if backend == "rust" + else UnifiedTreeCore(params, cache.components) + ) + cache.tree_core = core + for component in cache.components.values(): + component.tree_core = core + core.set_hicache_enabled() + core.is_write_back = True + cache.host_memory_mode = "cache" + cache.buffer_pipeline = None + cache.cache_controller = SimpleNamespace(write_policy="write_back") + cache.ongoing_write_through = {} + cache._build_backup_sidecar = Mock(return_value=[]) + host_pool = Mock() + host_pool.available_size.return_value = 0 if backup == "host_pressure" else 1 + cache.components[ComponentType.MAMBA]._mamba_pool_host = ( + None if backup == "missing_pool" else host_pool + ) + cache.host_pool_group = Mock() + cache.host_pool_group.get_pool.return_value = ( + None if backup == "missing_pool" else host_pool + ) + _mamba_insert(core, [1], [10], 7) + node = core.match_prefix(MatchPrefixParams(key=_key([1]))).best_match_node + _mamba_insert(core, [1, 2], [10, 11], 8) + events = [] + + def assert_state_resident(): + assert core.get_component_device_value(node, ComponentType.MAMBA).tolist() == [ + 7 + ] + assert core.get_component_device_value(node, ComponentType.FULL).tolist() == [ + 10 + ] + + def evict_host(count, component): + assert (count, component) == (1, ComponentType.MAMBA) + assert_state_resident() + events.append("host_evict") + host_pool.available_size.return_value = 1 + return 1 + + def write(node_id, values, transfers, sidecars): + assert node_id == node and values.tolist() == [10] and sidecars == [] + assert_state_resident() + events.append("write") + if backup == "raised": + raise RuntimeError("DMA submission failed") + if backup == "failed": + return None + (state,) = transfers[ComponentType.MAMBA] + assert state.device_indices.tolist() == [7] + state.host_indices = torch.tensor([70], dtype=torch.int64) + return torch.tensor([100], dtype=torch.int64) + + def acknowledge(write_back): + assert write_back + assert_state_resident() + assert node in cache.ongoing_write_through + events.append("ack") + if backup == "ack_raised": + raise RuntimeError("DMA completion failed") + core.finish_write_through([node], ack_id=node) + cache.ongoing_write_through.clear() + + def free_values(device_frees, host_frees): + assert not host_frees + assert [t.tolist() for t in device_frees.pop(ComponentType.MAMBA)] == [[7]] + assert not device_frees + events.append("free") + + cache.evict_host = evict_host + cache._execute_kv_backup = write + cache.writing_check = acknowledge + cache._free_values = free_values + tracker = defaultdict(int) + core.evict_device_start(ComponentType.MAMBA, 1) + try: + if backup in ("raised", "ack_raised"): + with pytest.raises(RuntimeError, match="DMA .* failed"): + cache._evict_device_next_node(ComponentType.MAMBA, tracker) + assert events == (["write"] if backup == "raised" else ["write", "ack"]) + assert not tracker[ComponentType.MAMBA] + assert_state_resident() + else: + assert cache._evict_device_next_node(ComponentType.MAMBA, tracker) == ( + None, + True, + ) + finally: + core.evict_device_end(ComponentType.MAMBA) + if backup in ("raised", "ack_raised"): + if backup == "ack_raised": + core.finish_write_through([node], ack_id=node) + cache.ongoing_write_through.clear() + core.evict_device_start(ComponentType.MAMBA, 1) + try: + step = core.evict_device_next_node(ComponentType.MAMBA, tracker) + if step.mamba_backup_node_id is not None: + step = core.finish_mamba_state_eviction(node) + freed = {} + _accumulate_step(step, tracker, freed, {}) + assert tracker[ComponentType.MAMBA] == 1 + assert torch.cat(freed[ComponentType.MAMBA]).tolist() == [7] + finally: + core.evict_device_end(ComponentType.MAMBA) + core.sanity_check([], []) + return + assert tracker[ComponentType.MAMBA] == 1 and tracker[ComponentType.FULL] == 0 + assert core.get_component_device_value(node, ComponentType.FULL).tolist() == [10] + assert core.get_component_device_value(node, ComponentType.MAMBA) is None + if backup == "failed": + assert events == ["write", "free"] + assert not core.component_has_host_value_only(node, ComponentType.MAMBA) + else: + expected = ["write", "ack", "free"] + assert events == ( + ["host_evict"] + expected if backup == "host_pressure" else expected + ) + matched = core.match_prefix(MatchPrefixParams(key=_key([1]))) + assert matched.best_match_node == node and matched.mamba_host_hit_length == 1 + (state,) = core.build_hicache_transfers( + ComponentType.MAMBA, node, CacheTransferPhase.LOAD_BACK + ) + assert state.host_indices.tolist() == [70] and state.nodes_to_load == [node] + core.sanity_check([], []) + + def test_mamba_path_cap_evicts_excess_states_through_the_adapter(): from collections import defaultdict @@ -2031,9 +4681,9 @@ def test_swa_tombstones_cross_the_binding_and_release_balanced( if swa_only: # Early release always walks SWA, even if the receipt marks it skipped. missing.skipped_lock_components = (ComponentType.SWA,) - with pytest.raises(BaseException, match="no entry found for key") as error: + with pytest.raises(RuntimeError, match="no entry found for key") as error: release(leaf, missing) - assert error.type.__name__ == "PanicException" + assert error.value.__cause__ is None return # A Rust ownership violation poisons the core. release(leaf, result.to_dec_params()) assert core.swa_protected_size() == 0 @@ -2042,6 +4692,95 @@ def test_swa_tombstones_cross_the_binding_and_release_balanced( core.sanity_check([], []) +def _window_lock_api_case(backend, mamba=True): + cache, allocator = ( + _swa_mamba_cache(backend, window=2) + if mamba + else _swa_cache(window=2, backend=backend) + ) + assert cache._tree_core_backend == backend + node = cache.insert( + InsertParams( + key=_key([1, 2, 3]), + value=allocator.alloc(3), + mamba_value=torch.tensor([201]) if mamba else None, + ) + ).last_device_node + return cache.tree_core, node + + +@pytest.mark.parametrize("backend", ["python", "rust"]) +@pytest.mark.parametrize("mamba", [False, True]) +def test_component_window_release_preserves_other_locks(backend, mamba): + core, node = _window_lock_api_case(backend, mamba) + components = (ComponentType.FULL, ComponentType.SWA, ComponentType.MAMBA) + assert [core.component_protected_size(ct) for ct in components] == [0, 0, 0] + receipt = core.inc_lock_ref(node).to_dec_params() + assert [core.component_protected_size(ct) for ct in components] == [ + 3, + 2, + int(mamba), + ] + + result = core.dec_window_lock_only(node, ComponentType.SWA, receipt) + + assert not result.device_frees and not result.host_frees + assert [core.component_protected_size(ct) for ct in components] == [ + 3, + 0, + int(mamba), + ] + assert core.get_component_device_value(node, ComponentType.SWA) is not None + # Only this component was released; the final receipt still releases Mamba. + receipt.skipped_lock_components = (ComponentType.SWA,) + core.dec_lock_ref(node, receipt) + assert [core.component_protected_size(ct) for ct in components] == [0, 0, 0] + core.sanity_check([], []) + + +@pytest.mark.parametrize("backend", ["python", "rust"]) +@pytest.mark.parametrize("noop", ["root", "skipped"]) +def test_component_window_release_noop_needs_no_uuid(backend, noop): + core, node = _window_lock_api_case(backend) + skipped = (ComponentType.SWA,) if noop == "skipped" else () + receipt = core.inc_lock_ref(node, skip_lock_components=skipped).to_dec_params() + protected = [ + core.component_protected_size(ct) + for ct in (ComponentType.FULL, ComponentType.SWA, ComponentType.MAMBA) + ] + if noop == "root": + target = core.root_node_handle() + release = DecLockRefParams(node_id=target) + else: + target, release = node, receipt + assert ComponentType.SWA not in release.component_lock_uuids + + result = core.dec_window_lock_only(target, ComponentType.SWA, release) + + assert not result.device_frees and not result.host_frees + assert [ + core.component_protected_size(ct) + for ct in (ComponentType.FULL, ComponentType.SWA, ComponentType.MAMBA) + ] == protected + core.dec_lock_ref(node, receipt) + core.sanity_check([], []) + + +@pytest.mark.parametrize("backend", ["python", "rust"]) +@pytest.mark.parametrize("skipped", [False, True]) +def test_component_window_release_checks_anchor_before_noop(backend, skipped): + core, node = _window_lock_api_case(backend) + receipt = core.inc_lock_ref( + node, skip_lock_components=(ComponentType.SWA,) if skipped else () + ).to_dec_params() + error = AssertionError if backend == "python" else RuntimeError + with pytest.raises(error, match="lock receipt anchored"): + core.dec_window_lock_only(core.root_node_handle(), ComponentType.SWA, receipt) + if backend == "python": + core.dec_lock_ref(node, receipt) + core.sanity_check([], []) + + def test_dec_swa_lock_only_frees_flow_after_the_full_release(): core = _swa_tree_core(window=2) first = _insert(core, [1, 2], [10, 11]) diff --git a/test/registered/unit/mem_cache/test_rust_unified_radix_cache_unittest.py b/test/registered/unit/mem_cache/test_rust_unified_radix_cache_unittest.py index 06227b4f7d..22f943282b 100644 --- a/test/registered/unit/mem_cache/test_rust_unified_radix_cache_unittest.py +++ b/test/registered/unit/mem_cache/test_rust_unified_radix_cache_unittest.py @@ -1,4 +1,4 @@ -"""Run the shared UnifiedRadixCache unit suite with the Rust TreeCore.""" +"""Run the shared suite requesting Rust, including centralized Python fallbacks.""" import unittest @@ -22,7 +22,7 @@ class RustBackendSuite(unittest.TestSuite): def load_tests(loader, standard_tests, pattern): - """Reuse the exact cache-level suite while swapping only its test factory.""" + """Reuse the cache suite and its production backend compatibility resolver.""" return RustBackendSuite(loader.loadTestsFromModule(shared_suite)) diff --git a/test/registered/unit/mem_cache/test_session_unified_radix_cache.py b/test/registered/unit/mem_cache/test_session_unified_radix_cache.py index 41af9ece2d..719ab577ca 100644 --- a/test/registered/unit/mem_cache/test_session_unified_radix_cache.py +++ b/test/registered/unit/mem_cache/test_session_unified_radix_cache.py @@ -10,6 +10,7 @@ from types import SimpleNamespace import torch +from sglang.srt.environ import envs from sglang.srt.mem_cache.allocator import TokenToKVPoolAllocator from sglang.srt.mem_cache.base_prefix_cache import ( EvictParams, @@ -20,6 +21,7 @@ from sglang.srt.mem_cache.cache_init_params import CacheInitParams from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool, ReqToTokenPool from sglang.srt.mem_cache.radix_cache import RadixKey from sglang.srt.mem_cache.unified_cache.components import ComponentType +from sglang.srt.mem_cache.unified_cache.unified_tree_core import UnifiedTreeCore from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache from sglang.test.test_utils import CustomTestCase @@ -104,6 +106,20 @@ class TestSessionUnifiedRadixCache(CustomTestCase): self.cache = UnifiedRadixCache(make_params(enable_session=True)) self.full = self.cache.components[ComponentType.FULL] + def test_explicit_rust_selection_uses_python_session_semantics(self): + with envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.override("rust"): + cache = UnifiedRadixCache(make_params(enable_session=True)) + self.assertEqual(cache._tree_core_backend, "python") + self.assertIsInstance(cache.tree_core, UnifiedTreeCore) + leaf = insert(cache, [1, 2, 3, 4]) + generation = cache.open_radix_session("fallback") + register(cache, [1, 2, 3, 4], "fallback", generation) + full = cache.components[ComponentType.FULL] + self.assertEqual(full.session_ref(leaf), 1) + cache.release_radix_session("fallback") + self.assertEqual(full.session_ref(leaf), 0) + cache.sanity_check() + def test_register_and_release_update_full_component_reference(self): leaf = insert(self.cache, [1, 2, 3, 4]) generation = self.cache.open_radix_session("s1") diff --git a/test/registered/unit/mem_cache/test_tree_core_registry.py b/test/registered/unit/mem_cache/test_tree_core_registry.py index ccb9319dcf..ad7cb03055 100644 --- a/test/registered/unit/mem_cache/test_tree_core_registry.py +++ b/test/registered/unit/mem_cache/test_tree_core_registry.py @@ -1,6 +1,11 @@ """Unit tests for the tree-core backend registry.""" +import os +import subprocess +import tempfile import unittest +from pathlib import Path +from types import SimpleNamespace from unittest import mock from unified_tree_core_inspection_interface import UnifiedTreeCoreInspectionInterface @@ -8,16 +13,16 @@ from unified_tree_core_inspection_interface import UnifiedTreeCoreInspectionInte from sglang.srt.environ import envs from sglang.srt.mem_cache.cache_init_params import CacheInitParams from sglang.srt.mem_cache.memory_pool import ReqToTokenPool +from sglang.srt.mem_cache.unified_cache import tree_core_registry from sglang.srt.mem_cache.unified_cache.component_type import ComponentType -from sglang.srt.mem_cache.unified_cache.components.base import ( - EvictLayer, - TreeComponent, -) +from sglang.srt.mem_cache.unified_cache.components.base import EvictLayer, TreeComponent from sglang.srt.mem_cache.unified_cache.tree_core_registry import ( _TREE_CORE_REGISTRY, create_tree_core, register_tree_core_backend, registered_tree_core_backends, + resolve_tree_core_backend, + select_tree_core_backend, ) from sglang.srt.mem_cache.unified_cache.unified_tree_core import UnifiedTreeCore from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache @@ -181,12 +186,18 @@ class UnifiedRadixCacheTreeCoreSelectionTest(CustomTestCase): cache = UnifiedRadixCache(params) self.assertTrue(cache.tree_core.is_eagle) - def test_default_backend_builds_the_python_tree_core(self): + def test_custom_components_default_to_the_python_tree_core(self): cache = UnifiedRadixCache(params=self._cache_params()) self.assertIsInstance(cache.tree_core, UnifiedTreeCore) component = cache.components[ComponentType.FULL] self.assertIs(component.tree_core, cache.tree_core) + def test_explicit_rust_cache_records_the_python_fallback(self): + with envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.override("rust"): + cache = UnifiedRadixCache(params=self._cache_params()) + self.assertIsInstance(cache.tree_core, UnifiedTreeCore) + self.assertEqual(cache._tree_core_backend, "python") + def test_env_var_routes_construction_to_the_selected_backend(self): """SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND selects the registered factory the cache constructs its tree through.""" @@ -245,6 +256,341 @@ class UnifiedRadixCacheTreeCoreSelectionTest(CustomTestCase): cache.components[ComponentType.AUXILIARY_SWA].tree_core, cache.tree_core ) + def test_supported_default_routes_to_rust(self): + core = mock.MagicMock() + factory = mock.MagicMock(return_value=core) + params = self._cache_params(component_registry_override=None) + with ( + mock.patch.dict(os.environ), + mock.patch.object( + tree_core_registry, + "_rust_fallback_reason", + return_value=None, + ), + mock.patch.dict(_TREE_CORE_REGISTRY, {"rust": factory}), + ): + envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.clear() + cache = UnifiedRadixCache(params) + factory.assert_called_once_with(params, cache.components) + self.assertIs(cache.tree_core, core) + + def test_default_rust_construction_errors_propagate(self): + factory = mock.MagicMock(side_effect=RuntimeError("extension build failed")) + with ( + mock.patch.dict(os.environ), + mock.patch.dict(tree_core_registry.sys.modules), + mock.patch.object(tree_core_registry.sys, "platform", "linux"), + mock.patch.object(tree_core_registry.torch, "__version__", "2.13.0"), + mock.patch.object( + tree_core_registry.importlib.util, + "find_spec", + return_value=None, + ), + mock.patch.object( + tree_core_registry.subprocess, + "run", + return_value=subprocess.CompletedProcess([], 0, stdout="version"), + ) as run, + mock.patch.dict(_TREE_CORE_REGISTRY, {"rust": factory}), + envs.SGLANG_RUST_BUILD_MODE.override("auto"), + ): + tree_core_registry.sys.modules.pop( + tree_core_registry._RUST_TREE_CORE_MODULE, None + ) + envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.clear() + with self.assertRaisesRegex(RuntimeError, "extension build failed"): + UnifiedRadixCache(self._cache_params(component_registry_override=None)) + self.assertTrue(run.called) + + def test_default_cache_falls_back_for_unavailable_toolchain(self): + params = self._cache_params(component_registry_override=None) + for unavailable in ("missing", "cargo", "rustc"): + with ( + self.subTest(unavailable=unavailable), + tempfile.TemporaryDirectory() as tmp, + ): + if unavailable != "missing": + for command in ("cargo", "rustc"): + executable = Path(tmp) / command + executable.write_text( + "#!/bin/sh\n" + + ("exit 1\n" if command == unavailable else "exit 0\n") + ) + executable.chmod(0o755) + rust_factory = mock.MagicMock( + side_effect=AssertionError( + "unavailable Rust toolchain was selected" + ) + ) + with ( + mock.patch.dict(os.environ, {"PATH": tmp}), + mock.patch.dict(tree_core_registry.sys.modules), + mock.patch.object(tree_core_registry.sys, "platform", "linux"), + mock.patch.object( + tree_core_registry.torch, "__version__", "2.13.0" + ), + mock.patch.object( + tree_core_registry.importlib.util, + "find_spec", + return_value=None, + ), + mock.patch.dict(_TREE_CORE_REGISTRY, {"rust": rust_factory}), + envs.SGLANG_RUST_BUILD_MODE.override("auto"), + ): + tree_core_registry.sys.modules.pop( + tree_core_registry._RUST_TREE_CORE_MODULE, None + ) + envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.clear() + cache = UnifiedRadixCache(params) + self.assertIsInstance(cache.tree_core, UnifiedTreeCore) + self.assertEqual(cache._tree_core_backend, "python") + rust_factory.assert_not_called() + + +class TreeCoreDefaultCompatibilityTest(CustomTestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.workspace = Path(temporary.name) + (self.workspace / "Cargo.toml").touch() + patchers = ( + mock.patch.dict(os.environ, {"SGLANG_RUST_BUILD_MODE": "auto"}), + mock.patch.dict(tree_core_registry.sys.modules), + mock.patch.object(tree_core_registry.sys, "platform", "linux"), + mock.patch.object(tree_core_registry.torch, "__version__", "2.13.0"), + mock.patch.object( + tree_core_registry.importlib.util, "find_spec", return_value=None + ), + mock.patch.object(tree_core_registry, "_RUST_TREE_CORE_MANIFEST"), + mock.patch.object( + tree_core_registry.subprocess, + "run", + return_value=subprocess.CompletedProcess([], 0, stdout="version"), + ), + ) + for patcher in patchers: + patcher.start() + self.addCleanup(patcher.stop) + envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.clear() + tree_core_registry.sys.modules.pop( + tree_core_registry._RUST_TREE_CORE_MODULE, None + ) + tree_core_registry._RUST_TREE_CORE_MANIFEST.is_file.return_value = True + tree_core_registry._RUST_TREE_CORE_MANIFEST.parent.parent = self.workspace + + def test_source_install_defaults_to_rust_on_cpu_and_cuda(self): + for device in ("cpu", "cuda:1"): + with self.subTest(device=device): + params = _cache_init_params() + params.token_to_kv_pool_allocator = SimpleNamespace(device=device) + self.assertEqual(select_tree_core_backend(params), "rust") + + def test_bundled_extension_needs_neither_sources_nor_toolchain(self): + (self.workspace / "Cargo.toml").unlink() + tree_core_registry._RUST_TREE_CORE_MANIFEST.is_file.return_value = False + tree_core_registry.importlib.util.find_spec.return_value = object() + self.assertEqual(select_tree_core_backend(_cache_init_params()), "rust") + tree_core_registry.subprocess.run.assert_not_called() + + def test_loaded_extension_needs_no_toolchain(self): + tree_core_registry.sys.modules[tree_core_registry._RUST_TREE_CORE_MODULE] = ( + SimpleNamespace() + ) + self.assertEqual(select_tree_core_backend(_cache_init_params()), "rust") + tree_core_registry.subprocess.run.assert_not_called() + + def test_never_mode_trusts_bundled_extension_in_a_source_checkout(self): + tree_core_registry.importlib.util.find_spec.return_value = object() + with envs.SGLANG_RUST_BUILD_MODE.override("never"): + self.assertEqual(select_tree_core_backend(_cache_init_params()), "rust") + tree_core_registry.subprocess.run.assert_not_called() + + def test_source_build_modes_probe_tools_even_with_a_bundled_extension(self): + tree_core_registry.importlib.util.find_spec.return_value = object() + for mode in ("auto", "force"): + with self.subTest(mode=mode), envs.SGLANG_RUST_BUILD_MODE.override(mode): + self.assertEqual(select_tree_core_backend(_cache_init_params()), "rust") + self.assertEqual( + tree_core_registry.subprocess.run.call_args_list, + [ + mock.call( + command, + cwd=self.workspace, + check=True, + capture_output=True, + text=True, + timeout=10, + ) + for command in ( + ("cargo", "--version", "--verbose"), + ("rustc", "-vV"), + ) + ], + ) + tree_core_registry.subprocess.run.reset_mock() + + def test_toolchain_probe_failure_uses_python_for_all_source_build_modes(self): + for mode in ("auto", "never", "force"): + for error in ( + FileNotFoundError("cargo"), + subprocess.CalledProcessError(1, "cargo"), + subprocess.TimeoutExpired("rustc", 10), + ): + with ( + self.subTest(mode=mode, error=type(error).__name__), + envs.SGLANG_RUST_BUILD_MODE.override(mode), + ): + tree_core_registry.subprocess.run.side_effect = error + self.assertEqual( + select_tree_core_backend(_cache_init_params()), "python" + ) + self.assertEqual( + resolve_tree_core_backend("rust", _cache_init_params()), + "python", + ) + + def test_invalid_mode_and_force_after_import_remain_loader_errors(self): + from sglang.srt.rust_extensions.loader import load_rust_extension + + module = tree_core_registry._RUST_TREE_CORE_MODULE + for mode, loaded, error, message in ( + ("invalid", False, ValueError, "invalid Rust extension build mode"), + ("force", True, RuntimeError, "cannot force-build"), + ): + with ( + self.subTest(mode=mode), + envs.SGLANG_RUST_BUILD_MODE.override(mode), + mock.patch.dict( + _TREE_CORE_REGISTRY, + {"rust": lambda params, components: load_rust_extension(module)}, + ), + ): + if loaded: + tree_core_registry.sys.modules[module] = SimpleNamespace() + with self.assertRaisesRegex(error, message): + create_tree_core("rust", _cache_init_params(), {}) + tree_core_registry.subprocess.run.assert_not_called() + + def test_platform_distribution_without_tree_core_uses_python(self): + tree_core_registry._RUST_TREE_CORE_MANIFEST.is_file.return_value = False + self.assertEqual(select_tree_core_backend(_cache_init_params()), "python") + + def test_session_and_custom_components_use_python(self): + for overrides in ( + {"enable_session_radix_cache": True}, + {"component_registry_override": {ComponentType.FULL: _StubFullComponent}}, + ): + with self.subTest(overrides=overrides): + self.assertEqual( + select_tree_core_backend(_cache_init_params(**overrides)), "python" + ) + params = _cache_init_params() + params.tree_components = (ComponentType.FULL, ComponentType.C128) + self.assertEqual(select_tree_core_backend(params), "python") + + def test_unsupported_platform_and_torch_use_python(self): + for platform in ("darwin", "win32"): + with ( + self.subTest(platform=platform), + mock.patch.object(tree_core_registry.sys, "platform", platform), + ): + self.assertEqual( + select_tree_core_backend(_cache_init_params()), "python" + ) + for version in ("2.10.0", "2.14.0", "unknown"): + with ( + self.subTest(version=version), + mock.patch.object(tree_core_registry.torch, "__version__", version), + ): + self.assertEqual( + select_tree_core_backend(_cache_init_params()), "python" + ) + + def test_unsupported_device_uses_python(self): + params = _cache_init_params() + params.token_to_kv_pool_allocator = SimpleNamespace(device="xpu:0") + self.assertEqual(select_tree_core_backend(params), "python") + + def test_explicit_rust_uses_the_same_compatibility_fallback(self): + params = _cache_init_params(enable_session_radix_cache=True) + with envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.override("rust"): + self.assertEqual(select_tree_core_backend(params), "python") + self.assertEqual(resolve_tree_core_backend("rust", params), "python") + + def test_instance_rust_override_uses_the_same_compatibility_fallback(self): + for session_enabled, expected in ((False, "rust"), (True, "python")): + with ( + self.subTest(session_enabled=session_enabled), + envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.override("python"), + ): + params = _cache_init_params( + tree_core_backend="rust", + enable_session_radix_cache=session_enabled, + ) + self.assertEqual(select_tree_core_backend(params), expected) + self.assertEqual( + envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.get(), "python" + ) + + def test_numeric_tlru_uses_rust_and_preserves_policy_configuration(self): + for config in ( + {"threshold": 4096, "next_prompt_estimate": 1024}, + {"threshold": 2**100 + 4, "next_prompt_estimate": 2**100}, + {"threshold": 4096.0, "next_prompt_estimate": 1024.0}, + {"threshold": 4096.5, "next_prompt_estimate": 1024.25}, + {"threshold": 1.2, "next_prompt_estimate": 0.2}, + {"threshold": float(2**53 + 2), "next_prompt_estimate": 2**53 + 1}, + {"threshold": -0.5, "next_prompt_estimate": -2}, + {"threshold": float("inf"), "next_prompt_estimate": 0}, + {"threshold": 0, "next_prompt_estimate": float("nan")}, + ): + with self.subTest(config=config): + params = _cache_init_params( + eviction_policy="TLRU", eviction_policy_config=config + ) + self.assertEqual(select_tree_core_backend(params), "rust") + with envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.override("rust"): + self.assertEqual(select_tree_core_backend(params), "rust") + components = {ComponentType.FULL: mock.MagicMock()} + factory = mock.MagicMock() + with mock.patch.dict(_TREE_CORE_REGISTRY, {"rust": factory}): + core = create_tree_core("rust", params, components) + factory.assert_called_once_with(params, components) + self.assertIs(core, factory.return_value) + self.assertEqual(params.eviction_policy_config, config) + + def test_python_and_custom_backend_selections_are_unchanged(self): + params = _cache_init_params(enable_session_radix_cache=True) + for backend in ("python", "custom_backend"): + with ( + self.subTest(backend=backend), + envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.override(backend), + ): + self.assertEqual(select_tree_core_backend(params), backend) + + def test_factory_resolves_rust_fallback_before_loading_the_extension(self): + params = _cache_init_params(enable_session_radix_cache=True) + components = {ComponentType.FULL: mock.MagicMock()} + python_factory = mock.MagicMock() + rust_factory = mock.MagicMock(side_effect=AssertionError("Rust was loaded")) + with mock.patch.dict( + _TREE_CORE_REGISTRY, {"python": python_factory, "rust": rust_factory} + ): + result = create_tree_core("rust", params, components) + python_factory.assert_called_once_with(params, components) + rust_factory.assert_not_called() + self.assertIs(result, python_factory.return_value) + + def test_supported_explicit_rust_factory_errors_propagate(self): + rust_factory = mock.MagicMock(side_effect=RuntimeError("extension load failed")) + python_factory = mock.MagicMock() + with mock.patch.dict( + _TREE_CORE_REGISTRY, {"python": python_factory, "rust": rust_factory} + ): + with self.assertRaisesRegex(RuntimeError, "extension load failed"): + create_tree_core("rust", _cache_init_params(), {}) + python_factory.assert_not_called() + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py index 3a22143790..a046833edb 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py +++ b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py @@ -77,7 +77,10 @@ from sglang.srt.mem_cache.unified_cache.components.base import ( TreeComponent, ) from sglang.srt.mem_cache.unified_cache.storage_attachment import StorageAttachment -from sglang.srt.mem_cache.unified_cache.tree_core_registry import _TREE_CORE_REGISTRY +from sglang.srt.mem_cache.unified_cache.tree_core_registry import ( + _TREE_CORE_REGISTRY, + resolve_tree_core_backend, +) from sglang.srt.mem_cache.unified_cache.unified_tree_core import UnifiedTreeCore from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import ( DecSwaLockOnlyResult, @@ -116,7 +119,7 @@ register_amd_ci(est_time=50, suite="stage-b-test-1-gpu-small-amd") # A dedicated test entry point overrides this without changing the process-wide # production backend selection. Direct Python-core tests in this module remain # Python-only; every fixture-backed cache test is shared by both inspectors. -_TREE_CORE_TEST_BACKEND: Optional[str] = None +_TREE_CORE_TEST_BACKEND: Optional[str] = "python" def _selected_tree_core_test_backend() -> str: @@ -124,9 +127,6 @@ def _selected_tree_core_test_backend() -> str: def _session_radix_cache_test_values() -> tuple[bool, ...]: - # TODO(Jialin): Restore the session-enabled case after porting #29173 to Rust. - if _selected_tree_core_test_backend() == "rust": - return (False,) return False, True @@ -455,11 +455,11 @@ def _aux_storage_key_transfers(cache, node_id): def build_fixture( cfg: CacheConfig, - *, enable_kv_cache_events: bool = False, enable_session_radix_cache: bool = False, tree_page_size: Optional[int] = None, mamba_cache_chunk_size: Optional[int] = None, + tree_core_backend: Optional[str] = None, ): """Create (tree, allocator, req_to_token_pool) from a CacheConfig. @@ -594,29 +594,26 @@ def build_fixture( eviction_policy=cfg.eviction_policy, is_eagle=cfg.is_eagle, ) - selected_backend = _selected_tree_core_test_backend() - if selected_backend == "python": + requested_backend = tree_core_backend or _selected_tree_core_test_backend() + selected_backend = resolve_tree_core_backend(requested_backend, cache_init_params) - def inspector_factory(params, components): - return UnifiedTreeCoreInspector(params, components) + def python_inspector_factory(params, components): + return UnifiedTreeCoreInspector(params, components) - elif selected_backend == "rust": + def rust_inspector_factory(params, _components): from rust_unified_tree_core_inspector import RustUnifiedTreeCoreInspector - def inspector_factory(params, _components): - return RustUnifiedTreeCoreInspector(params) + return RustUnifiedTreeCoreInspector(params) - else: - inspector_factory = None - - if inspector_factory is not None: - with ( - envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.override(selected_backend), - mock.patch.dict(_TREE_CORE_REGISTRY, {selected_backend: inspector_factory}), - ): - cache = UnifiedRadixCache(params=cache_init_params) - else: + with ( + envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.override(requested_backend), + mock.patch.dict( + _TREE_CORE_REGISTRY, + {"python": python_inspector_factory, "rust": rust_inspector_factory}, + ), + ): cache = UnifiedRadixCache(params=cache_init_params) + assert cache._tree_core_backend == selected_backend assert isinstance(cache.tree_core, UnifiedTreeCoreInspectionInterface), ( "The shared unified radix-cache unit suite requires a TreeCore backend " "that implements UnifiedTreeCoreInspectionInterface" @@ -4641,7 +4638,6 @@ class UnifiedRadixCacheSuite: def test_buffer_only_load_back_uses_full_behind_swa_tombstone(self): """FULL-only rematch keeps resident FULL and loads the complete SWA window.""" self._skip_unsupported_hicache_test() - self._skip_swa_window_repair_on_rust() if not self.cfg.has_swa: self.skipTest("masked overlap requires an SWA component") storage_dir = tempfile.mkdtemp() @@ -4782,7 +4778,6 @@ class UnifiedRadixCacheSuite: def test_buffer_only_load_back_reuses_partial_masked_full(self): """FULL-only rematch reuses a resident head and fetches only its tail.""" self._skip_unsupported_hicache_test() - self._skip_swa_window_repair_on_rust() if not self.cfg.has_swa: self.skipTest("masked overlap requires an SWA component") page_size = self.cfg.page_size @@ -4995,7 +4990,6 @@ class UnifiedRadixCacheSuite: A window-sized fetch still passes the global threshold, but after sibling growth its FULL splice is shorter than its SWA transfer.""" self._skip_unsupported_hicache_test() - self._skip_swa_window_repair_on_rust() # Buffer-mode plan/commit logic is layout-independent, and each # hicache fixture retains its pools for the whole file run. Pin to # one config so the matrix does not exhaust a small CI GPU. @@ -5582,13 +5576,6 @@ class UnifiedRadixCacheSuite: self.skipTest("HiCache unit fixture does not support SWA + Mamba stacks") return False - def _skip_swa_window_repair_on_rust(self): - # Buffer-mode consumption repairs SWA tombstones under the loaded - # window through swa_tombstone_ranges/attach_swa_window, which the - # Rust tree core does not implement yet. - if _selected_tree_core_test_backend() == "rust": - self.skipTest("buffer-mode SWA window repair is Python-core only") - def _simulate_backup(self, cache, node): """Simulate D->H backup over the whole root->node path (parent-first).""" for ancestor in self._path_chain(cache, node): @@ -5683,8 +5670,8 @@ class UnifiedRadixCacheSuite: # Background prefetch/backup threads are daemon; stop them per-test. self.addCleanup(cache.cache_controller._stop_storage_threads) - def _build_hicache_fixture(self): - fixture = build_fixture(self.cfg) + def _build_hicache_fixture(self, tree_core_backend: Optional[str] = None): + fixture = build_fixture(self.cfg, tree_core_backend=tree_core_backend) cache, _, _ = fixture self._init_hicache(cache) return fixture @@ -6186,12 +6173,6 @@ class UnifiedRadixCacheSuite: it stays servable.""" if not self.cfg.has_mamba: self.skipTest("requires Mamba component") - if self.cfg.has_swa: - self.skipTest("no hicache strategy covers FULL+SWA+MAMBA") - # TODO(ShangmingCai): port the internal-node demote to the Rust core; - # its eviction walk still tombstones the state without a host backup. - if _selected_tree_core_test_backend() == "rust": - self.skipTest("internal-node state demote is Python-core only") cache, req_to_token_pool, seq_a = self._build_internal_mamba_fixture( "write_back" ) @@ -6214,11 +6195,13 @@ class UnifiedRadixCacheSuite: _host_value(cache, node, ComponentType.MAMBA), "state demoted to host, not dropped", ) + # SWA can split off device-only Full ancestors without Mamba states. + # They count toward the Full hit but not the host-hit budget; the + # scheduler collects their resident KV after reviving the state below. self.assertEqual( - len(m.device_indices) + m.host_hit_length, + m.full_kv_hit_length, len(seq_a), - "the full prefix stays servable (device KV is claimed once the " - "state is revived)", + "the full prefix remains cached", ) self.assertGreaterEqual( m.mamba_host_hit_length, 1, "load-back armed for the host state" @@ -6248,12 +6231,11 @@ class UnifiedRadixCacheSuite: def test_hicache_internal_mamba_backup_waits_for_pending_swa(self): if not (self.cfg.has_swa and self.cfg.has_mamba): self.skipTest("requires Full, SWA and Mamba components") - if _selected_tree_core_test_backend() == "rust": - self.skipTest("internal-node state demote is Python-core only") page = self.cfg.page_size cache, allocator, req_pool = build_fixture( replace(self.cfg, sliding_window_size=3 * page) ) + self.assertEqual(cache._tree_core_backend, _selected_tree_core_test_backend()) def insert(length): return self._insert( @@ -6294,8 +6276,6 @@ class UnifiedRadixCacheSuite: """Non-write_back policies keep the legacy tombstone-and-drop.""" if not self.cfg.has_mamba: self.skipTest("requires Mamba component") - if self.cfg.has_swa: - self.skipTest("no hicache strategy covers FULL+SWA+MAMBA") cache, _, seq_a = self._build_internal_mamba_fixture("write_through") cache.evict(EvictParams(num_tokens=0, mamba_num=10)) @@ -6327,15 +6307,10 @@ class UnifiedRadixCacheSuite: stays servable instead of losing a window of matchable prefix.""" if not self.cfg.has_swa: self.skipTest("requires SWA component") - if self.cfg.has_mamba: - self.skipTest("no hicache strategy covers FULL+SWA+MAMBA") - # TODO(ShangmingCai): port the internal-node demote to the Rust core; - # its eviction walk still tombstones the SWA KV without a host backup. - if _selected_tree_core_test_backend() == "rust": - self.skipTest("internal-node SWA demote is Python-core only") cache, req_to_token_pool, seq_a, seq_b = self._build_internal_swa_fixture( "write_back" ) + self.assertEqual(cache._tree_core_backend, _selected_tree_core_test_backend()) result = cache.evict(EvictParams(num_tokens=0, swa_num_tokens=len(seq_b))) self.assertGreaterEqual(result.swa_num_tokens_evicted, len(seq_a)) @@ -6389,8 +6364,6 @@ class UnifiedRadixCacheSuite: """Non-write_back policies keep the legacy tombstone-and-drop.""" if not self.cfg.has_swa: self.skipTest("requires SWA component") - if self.cfg.has_mamba: - self.skipTest("no hicache strategy covers FULL+SWA+MAMBA") cache, _, seq_a, seq_b = self._build_internal_swa_fixture("write_through") cache.evict(EvictParams(num_tokens=0, swa_num_tokens=len(seq_b))) @@ -6511,11 +6484,10 @@ class UnifiedRadixCacheSuite: self.skipTest("requires SWA-only") if self.cfg.sliding_window_size <= self.cfg.page_size: self.skipTest("the window must reach past the leaf's own page") - if _selected_tree_core_test_backend() == "rust": - # needs_incremental_backup is a component method on Python nodes; - # the Rust core pins the same contract in its own unit suite. - self.skipTest("component-level check is Python-core only") - cache, allocator, req_to_token_pool = self._build_hicache_fixture() + # This test exercises the Python component's collector directly. + cache, allocator, req_to_token_pool = self._build_hicache_fixture( + tree_core_backend="python" + ) chain = self._build_chain_pages(cache, allocator, req_to_token_pool, 2) if len(chain) < 2: self.skipTest("chain too short") @@ -6566,8 +6538,6 @@ class UnifiedRadixCacheSuite: cache.dec_lock_ref(zeroed.best_match_node, lock.to_dec_params()) def test_evict_host_drains_freed_host_values_to_the_pools(self): - if self.cfg.has_swa and self.cfg.has_mamba: - self.skipTest("no hicache strategy covers FULL+SWA+MAMBA") cache, allocator, req_to_token_pool = self._build_hicache_fixture() chain = self._build_chain_pages(cache, allocator, req_to_token_pool, 3) if len(chain) < 3: @@ -10194,12 +10164,6 @@ class TestUnifiedRadixPrefetchCorruption(CustomTestCase): cache.sanity_check() def test_write_through_eviction_counts_unbacked_tokens(self): - if _selected_tree_core_test_backend() == "rust": - # The unbacked-eviction tracker is a Python tree-core feature; - # UnifiedRadixCache only enables it for that backend. - self.skipTest( - "write-through unbacked-eviction tracking is Python-core only" - ) cache, allocator, _ = build_fixture(self.cfg) self._init_hicache(cache) cache.metrics_collector = mock.Mock() @@ -10623,14 +10587,10 @@ class TestSegmentLockProtocol(_InsertWalkSuite): node_id = children[0] def _assert_protocol_violation(self, fn, fragment): - """The Python core asserts; the Rust core panics (a BaseException - subclass at the PyO3 boundary). Either way the message names the - violation and the operation never completes silently.""" + """Both cores report ownership violations as ordinary Python exceptions.""" try: fn() - except (KeyboardInterrupt, SystemExit): - raise - except BaseException as exc: # pyo3 PanicException derives from BaseException + except (AssertionError, RuntimeError) as exc: self.assertIn(fragment, str(exc)) else: self.fail(f"protocol violation went unreported: {fragment}") @@ -10816,9 +10776,9 @@ class TestSegmentLockProtocol(_InsertWalkSuite): unlocks, so a leaf whose last lock is an auxiliary one is readmitted even when Full released first. Component-level replay of the Python core; the Rust crate covers its own order in its unit tests.""" - if _selected_tree_core_test_backend() != "python": - self.skipTest("drives Python component objects directly") - cache, allocator, req_to_token_pool = build_fixture(self.cfg) + cache, allocator, req_to_token_pool = build_fixture( + self.cfg, tree_core_backend="python" + ) seq = self._make_seq(1, self.cfg.sliding_window_size) self._insert(cache, allocator, req_to_token_pool, seq) leaf = self._match_leaf(cache, seq) diff --git a/test/registered/unit/scripted_runtime/test_tree_core_inspection.py b/test/registered/unit/scripted_runtime/test_tree_core_inspection.py new file mode 100644 index 0000000000..dddc1485fb --- /dev/null +++ b/test/registered/unit/scripted_runtime/test_tree_core_inspection.py @@ -0,0 +1,124 @@ +import os +import subprocess +import sys +from array import array +from types import SimpleNamespace +from unittest import mock + +import pytest +import torch + +from sglang.srt.environ import envs +from sglang.srt.mem_cache.base_prefix_cache import InsertParams +from sglang.srt.mem_cache.cache_init_params import CacheInitParams +from sglang.srt.mem_cache.radix_cache import RadixKey +from sglang.srt.mem_cache.unified_cache.component_type import ComponentType +from sglang.srt.mem_cache.unified_cache.tree_core_registry import _TREE_CORE_REGISTRY +from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.scripted_runtime.context.lock_ref_exhauster import ( + ScriptedLockRefExhauster, +) +from sglang.test.scripted_runtime.context.radix import ( + get_all_node_hit_counts, + get_all_node_lock_refs, + get_node_lock_ref, +) +from sglang.test.scripted_runtime.tree_core_inspection import ( + install_tree_core_inspectors, +) + +register_cpu_ci(est_time=20, suite="base-a-test-cpu") + + +def _cache(backend, enable_session=False): + with ( + mock.patch.dict(_TREE_CORE_REGISTRY), + envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.override(backend), + ): + install_tree_core_inspectors() + return UnifiedRadixCache( + CacheInitParams( + disable=False, + req_to_token_pool=None, + token_to_kv_pool_allocator=None, + page_size=1, + tree_components=(ComponentType.FULL,), + enable_session_radix_cache=enable_session, + ) + ) + + +def _insert(core, token_count): + step = core.begin_insert( + InsertParams( + key=RadixKey(array("q", range(token_count))), + value=torch.arange(token_count, dtype=torch.int64), + ) + ) + while step.result is None: + step = core.resume_insert() + core.end_insert() + return step.result.last_device_node + + +@pytest.mark.parametrize("backend", ["python", "rust"]) +def test_scripted_inspection_tracks_hits_locks_and_stale_handles(backend): + cache = _cache(backend) + assert cache._tree_core_backend == backend + context = SimpleNamespace(scheduler=SimpleNamespace(tree_cache=cache)) + prefix = _insert(cache.tree_core, 44) + decode = _insert(cache.tree_core, 45) + + assert get_all_node_hit_counts(context) == {prefix: 2, decode: 1} + assert get_all_node_lock_refs(context) == {prefix: 0, decode: 0} + exhauster = ScriptedLockRefExhauster(context.scheduler) + exhauster.exhaust(leave_refs=0) + assert all(value > 0 for value in get_all_node_lock_refs(context).values()) + assert get_node_lock_ref(cache, decode) > 0 + exhauster.release() + assert get_all_node_lock_refs(context) == {prefix: 0, decode: 0} + + cache.tree_core.reset() + assert get_all_node_hit_counts(context) == {} + assert get_node_lock_ref(cache, decode) == 0 + assert get_node_lock_ref(cache, None) == 0 + + +def test_scripted_inspection_preserves_session_fallback(): + cache = _cache("rust", enable_session=True) + assert cache._tree_core_backend == "python" + assert type(cache.tree_core).__name__ == "UnifiedTreeCoreInspector" + + +@pytest.mark.parametrize("scripted", [False, True]) +def test_inspectors_install_only_in_scripted_subprocesses(scripted): + process_env = dict(os.environ) + process_env["SGLANG_TEST_SCRIPTED_RUNTIME"] = str(scripted) + result = subprocess.run( + [ + sys.executable, + "-c", + ( + "import sglang.test.scripted_runtime; " + "from sglang.srt.mem_cache.unified_cache.tree_core_registry " + "import get_tree_core_factory; " + "print(get_tree_core_factory('rust').__module__)" + ), + ], + env=process_env, + capture_output=True, + text=True, + check=True, + timeout=60, + ) + expected_module = ( + "sglang.test.scripted_runtime.tree_core_inspection" + if scripted + else "sglang.srt.mem_cache.unified_cache.tree_core_registry" + ) + assert result.stdout.strip() == expected_module + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__]))