diff --git a/docs/docs/advanced_features/radix_eviction_policy.mdx b/docs/docs/advanced_features/radix_eviction_policy.mdx
index 787430be3f..91e3cfd4d8 100644
--- a/docs/docs/advanced_features/radix_eviction_policy.mdx
+++ b/docs/docs/advanced_features/radix_eviction_policy.mdx
@@ -50,10 +50,6 @@ Omit the flag to accept every default. An unrecognized key fails at startup rath
TypeError: SLRUStrategy.__init__() got an unexpected keyword argument 'protected_treshold'
```
-
-`--radix-eviction-policy-config` is not supported by the experimental Rust tree core (`SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND=rust`), which builds its strategy from the policy name alone. Passing both fails at startup.
-
-
### Policy parameters
`slru` and `tlru` take parameters. `lru`, `lfu`, and `priority` take none, so `--radix-eviction-policy-config` has no effect with them and any key is an error.
diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py
index 9af31cae81..6566413a75 100644
--- a/python/sglang/srt/environ.py
+++ b/python/sglang/srt/environ.py
@@ -677,8 +677,17 @@ class Envs:
SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS = EnvBool(True)
# Decode batches between SWA out-of-window evictions.
SGLANG_SWA_EVICTION_INTERVAL = EnvInt(128)
- # Registered TreeCore backend serving the unified radix cache.
- SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND = EnvStr("python")
+ # The tree-core registry falls back to Python for:
+ # - Session-aware caching.
+ # - C128 or other unsupported components.
+ # - Custom component overrides.
+ # - Non-Linux platforms.
+ # - Unsupported PyTorch versions.
+ # - Devices other than CPU or CUDA.
+ # - Installs with neither the Rust extension nor its sources.
+ # - Source builds with a missing or unusable Rust toolchain.
+ # This also applies when Rust is explicitly selected.
+ SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND = EnvStr("rust")
SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW = EnvBool(False)
# ===================================================================
diff --git a/python/sglang/srt/mem_cache/rust_tree_core/adapter.py b/python/sglang/srt/mem_cache/rust_tree_core/adapter.py
index 95b52960b6..22671f7dbd 100644
--- a/python/sglang/srt/mem_cache/rust_tree_core/adapter.py
+++ b/python/sglang/srt/mem_cache/rust_tree_core/adapter.py
@@ -3,6 +3,7 @@
from __future__ import annotations
+import sys
from array import array
from typing import TYPE_CHECKING, Optional, Sequence
@@ -14,6 +15,8 @@ from sglang.srt.disaggregation.kv_events import (
BlockStored,
StorageMedium,
)
+from sglang.srt.environ import envs
+from sglang.srt.mem_cache.allocator.swa import is_swa_req_ring
from sglang.srt.mem_cache.base_prefix_cache import (
DecLockRefParams,
DecLockRefResult,
@@ -39,6 +42,7 @@ from sglang.srt.mem_cache.unified_cache.cache_action import (
SWARebuild,
)
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
+from sglang.srt.mem_cache.unified_cache.components import CacheTransferPhase
from sglang.srt.mem_cache.unified_cache.unified_tree_core import StorageBackupSpec
from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import (
BufferBackupSnapshot,
@@ -54,6 +58,7 @@ from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import (
RadixCacheWalkResult,
UnifiedTreeCoreInterface,
)
+from sglang.srt.mem_cache.utils import get_eviction_strategy
from sglang.srt.runtime_context import get_exec, mamba_cache_chunk_size
from sglang.srt.utils import assert_int64_array
@@ -65,10 +70,23 @@ if TYPE_CHECKING:
CacheAction,
ComponentAction,
)
- from sglang.srt.mem_cache.unified_cache.components import CacheTransferPhase
from sglang.srt.mem_cache.unified_cache.unified_tree_core import UnifiedTreeNode
+def _tlru_float_config(native_bindings, threshold, next_prompt_estimate):
+ """Pass numeric values without changing Python's integer/float distinction."""
+ if not all(
+ isinstance(value, (int, float)) for value in (threshold, next_prompt_estimate)
+ ):
+ raise TypeError("T-LRU parameters must be integer or floating-point numbers")
+ threshold = float(threshold)
+ if isinstance(next_prompt_estimate, int):
+ return native_bindings.TlruFloatConfig(
+ threshold, 0.0, integer_estimate=next_prompt_estimate
+ )
+ return native_bindings.TlruFloatConfig(threshold, float(next_prompt_estimate))
+
+
def _radix_key_buffer(key: RadixKey) -> array:
"""The key's token ids honoring `limit`; view-independent since the
binding derives its own atoms."""
@@ -231,6 +249,7 @@ def _insert_step_from_binding(step) -> InsertStepResult:
mamba_exist=step.result.mamba_exist,
swa_branch_inserted=step.result.swa_branch_inserted,
host_insert_dropped=step.result.host_insert_dropped,
+ rotation_tail_declined=step.result.rotation_tail_declined,
adopted_ranges=(
{
ComponentType(component_type): list(ranges)
@@ -297,6 +316,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
"""A TreeCore backed by the Rust extension binding."""
_bindings = bindings
+ supports_rotation_base = True
def __init__(self, params: CacheInitParams):
assert params.tree_components is not None
@@ -324,12 +344,25 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
raise ValueError(
"Rust TreeCore does not support component_registry_override"
)
- # The Rust core builds its own eviction strategy from the policy name
- # alone, so a config would be dropped rather than applied.
- if params.eviction_policy_config:
- raise ValueError(
- "Rust TreeCore does not support --radix-eviction-policy-config"
- )
+ # Validate the same constructor options as Python before passing the
+ # configured eviction parameters to the native strategy.
+ eviction_strategy = get_eviction_strategy(
+ params.eviction_policy, params.eviction_policy_config
+ )
+ tlru_tail_budget = 0
+ tlru_float_config = None
+ if params.eviction_policy.lower() == "tlru":
+ threshold = eviction_strategy.threshold
+ next_prompt_estimate = eviction_strategy.next_prompt_estimate
+ if isinstance(threshold, int) and isinstance(next_prompt_estimate, int):
+ # Subtract before clamping to preserve arbitrary-size integers.
+ tlru_tail_budget = min(
+ max(threshold - next_prompt_estimate, 0), 2 * sys.maxsize + 1
+ )
+ else:
+ tlru_float_config = _tlru_float_config(
+ self._bindings, threshold, next_prompt_estimate
+ )
if ComponentType.SWA in self.tree_components and (
params.sliding_window_size is None or params.sliding_window_size <= 0
):
@@ -342,8 +375,9 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
params.is_eagle and ComponentType.MAMBA not in self.tree_components
)
- # ``device`` is derived from the construction-time allocator; the
- # allocator/pool themselves are owned by the cache, not the tree.
+ # The cache owns allocation; keep its allocator for resolving current
+ # unified SWA addresses when building a host backup.
+ self._allocator = params.token_to_kv_pool_allocator
if params.token_to_kv_pool_allocator:
device = torch.device(params.token_to_kv_pool_allocator.device)
# A bare "cuda" means the process's current device, not cuda:0.
@@ -362,12 +396,18 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
self._binding = self._binding_class()(
self._bindings.TreeCoreInitParamsBinding(
eviction_policy=params.eviction_policy,
+ slru_protected_threshold=getattr(
+ eviction_strategy, "protected_threshold", 2
+ ),
+ tlru_tail_budget=tlru_tail_budget,
+ tlru_float_config=tlru_float_config,
page_size=params.page_size,
is_write_back=False,
enable_hicache=False,
write_through_threshold=256,
device=str(self.device),
swa_sliding_window_size=params.sliding_window_size,
+ swa_req_ring=is_swa_req_ring(self._allocator),
enable_kv_cache_events=params.enable_kv_cache_events,
mamba_cache_chunk_size=(
mamba_cache_chunk_size() if has_mamba else None
@@ -415,9 +455,14 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
def swa_tombstone_ranges(
self, key: RadixKey, start: int, end: int
) -> list[tuple[int, int]]:
- raise NotImplementedError(
- "swa_tombstone_ranges: buffer-mode SWA window repair is not yet "
- "ported to the Rust tree core"
+ return self._binding.swa_tombstone_ranges(
+ self._bindings.MatchParamsBinding(
+ key=_radix_key_buffer(key),
+ extra_key=key.extra_key,
+ cache_salt=key.cache_salt,
+ ),
+ start,
+ end,
)
def attach_swa_window(
@@ -426,10 +471,18 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
window_start: int,
window_end: int,
swa_values: torch.Tensor,
- ) -> list:
- raise NotImplementedError(
- "attach_swa_window: buffer-mode SWA window repair is not yet "
- "ported to the Rust tree core"
+ ) -> list[CacheAction | ComponentAction]:
+ return _cache_actions_from_tagged(
+ self._binding.attach_swa_window(
+ self._bindings.MatchParamsBinding(
+ key=_radix_key_buffer(key),
+ extra_key=key.extra_key,
+ cache_salt=key.cache_salt,
+ ),
+ window_start,
+ window_end,
+ swa_values,
+ )
)
def inc_lock_ref(
@@ -453,6 +506,24 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
)
return DecLockRefResult()
+ def dec_window_lock_only(
+ self,
+ node_id: NodeId,
+ component_type: ComponentType,
+ params: DecLockRefParams,
+ ) -> DecSwaLockOnlyResult:
+ result = DecSwaLockOnlyResult()
+ new_device_frees, new_host_frees = self._binding.dec_window_lock_only(
+ node_id,
+ int(component_type),
+ _dec_lock_ref_params_to_binding(self._bindings, params),
+ )
+ for component, tensors in new_device_frees.items():
+ result.device_frees[ComponentType(component)].extend(tensors)
+ for component, tensors in new_host_frees.items():
+ result.host_frees[ComponentType(component)].extend(tensors)
+ return result
+
def dec_swa_lock_only(
self,
node_id: NodeId,
@@ -484,20 +555,43 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
result = EvictDeviceNextNodeResult(
node_id=binding_result.node_id,
made_progress=binding_result.made_progress,
+ unbacked_tokens=binding_result.unbacked_tokens,
+ mamba_backup_node_id=binding_result.mamba_backup_node_id,
+ swa_backup_node_id=binding_result.swa_backup_node_id,
+ swa_backup_num_tokens=binding_result.swa_backup_num_tokens,
+ )
+ return _fill_evict_result(binding_result, result)
+
+ def finish_mamba_state_eviction(self, node_id: NodeId) -> EvictDeviceNextNodeResult:
+ binding_result = self._binding.finish_mamba_state_eviction(node_id)
+ result = EvictDeviceNextNodeResult(
+ node_id=binding_result.node_id,
+ made_progress=binding_result.made_progress,
+ unbacked_tokens=binding_result.unbacked_tokens,
+ )
+ return _fill_evict_result(binding_result, result)
+
+ def finish_swa_state_eviction(self, node_id: NodeId) -> EvictDeviceNextNodeResult:
+ binding_result = self._binding.finish_swa_state_eviction(node_id)
+ result = EvictDeviceNextNodeResult(
+ node_id=binding_result.node_id,
+ made_progress=binding_result.made_progress,
+ unbacked_tokens=binding_result.unbacked_tokens,
)
return _fill_evict_result(binding_result, result)
def evict_device_leaf(
self, node_id: NodeId, is_write_back: bool
) -> EvictDeviceLeafResult:
- # The binding reads is_write_back from the core's construction config.
+ # The binding reads is_write_back from the core's current config.
assert is_write_back == self.is_write_back, (
- "is_write_back must match the core's construction config"
+ "is_write_back must match the core's current config"
)
binding_result = self._binding.evict_device_leaf(node_id)
backup = binding_result.backup_kv
result = EvictDeviceLeafResult(
- backup_kv=_cache_action_from_tagged(backup) if backup is not None else None
+ unbacked_tokens=binding_result.unbacked_tokens,
+ backup_kv=_cache_action_from_tagged(backup) if backup is not None else None,
)
return _fill_evict_result(binding_result, result)
@@ -529,6 +623,9 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
def component_evictable_size(self, component_type: ComponentType) -> int:
return self._binding.component_evictable_size(int(component_type))
+ def component_protected_size(self, component_type: ComponentType) -> int:
+ return self._binding.component_protected_size(int(component_type))
+
def full_evictable_size(self) -> int:
return self._binding.full_evictable_size()
@@ -632,6 +729,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
chunked=params.chunked,
priority=0 if params.priority is None else params.priority,
track_adopted_ranges=params.track_adopted_ranges,
+ rotation_base=params.rotation_base,
)
)
return _insert_step_from_binding(step)
@@ -649,7 +747,9 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
self, component_type: ComponentType, num_tokens: int
) -> DriveHostEvictionResult:
binding_result = self._binding.drive_host_eviction(
- int(component_type), num_tokens
+ int(component_type),
+ num_tokens,
+ envs.SGLANG_HICACHE_SKIP_HOST_DUPLICATE_RECLAIM.get(),
)
return _fill_evict_result(binding_result, DriveHostEvictionResult())
@@ -761,7 +861,31 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
self, node_id: NodeId
) -> tuple[torch.Tensor, dict[ComponentType, list[PoolTransfer]]]:
device_value, comp_xfers = self._binding.build_backup_spec(node_id)
- return device_value, _comp_xfers_from_binding(comp_xfers)
+ comp_xfers = _comp_xfers_from_binding(comp_xfers)
+ self._refresh_swa_backup_indices(comp_xfers.get(ComponentType.SWA, ()))
+ return device_value, comp_xfers
+
+ def _refresh_swa_backup_indices(self, transfers: Sequence[PoolTransfer]) -> None:
+ if not transfers:
+ return
+ from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
+ UnifiedSWAAllocatorBase,
+ )
+
+ if not isinstance(self._allocator, UnifiedSWAAllocatorBase):
+ return
+ for transfer in transfers:
+ if transfer.name != PoolName.SWA or not transfer.nodes_to_load:
+ continue
+ # Compaction can relocate SWA rows after the tree records them.
+ # FULL virtual IDs retain their meaning across that relocation.
+ full_values = [
+ self.get_component_device_value(node_id, ComponentType.FULL)
+ for node_id in transfer.nodes_to_load
+ ]
+ transfer.device_indices = self._allocator.translate_loc_from_full_to_swa(
+ torch.cat(full_values)
+ ).to(torch.int64)
def build_storage_backup_spec(
self, node_id: NodeId, pass_prefix_keys: bool
@@ -805,7 +929,10 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
)
if transfers is None:
return None
- return [_transfer_from_binding(transfer) for transfer in transfers]
+ transfers = [_transfer_from_binding(transfer) for transfer in transfers]
+ if phase == CacheTransferPhase.BACKUP_HOST:
+ self._refresh_swa_backup_indices(transfers)
+ return transfers
def build_load_back_spec(
self, node_id: NodeId, req: Optional[Req] = None
@@ -815,7 +942,30 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
kv_xfer, comp_xfers = self._binding.build_load_back_spec(
node_id, mamba_pool_idx
)
- return _transfer_from_binding(kv_xfer), _comp_xfers_from_binding(comp_xfers)
+ kv_xfer = _transfer_from_binding(kv_xfer)
+ comp_xfers = _comp_xfers_from_binding(comp_xfers)
+ swa_xfers = comp_xfers.get(ComponentType.SWA, ())
+ if swa_xfers:
+ full_node_ids = kv_xfer.nodes_to_load or []
+ full_load_slices = {}
+ offset = 0
+ for full_node_id, count in zip(
+ full_node_ids, self._binding.get_node_key_lengths(full_node_ids)
+ ):
+ full_load_slices[full_node_id] = slice(offset, offset + count)
+ offset += count
+ for transfer in swa_xfers:
+ # SWA may have holes between resident nodes, or reload while
+ # FULL stays resident. Preserve the SWA transfer's node order.
+ transfer.anchor_index_parts = [
+ (
+ full_load_slices[nid]
+ if nid in full_load_slices
+ else self.get_component_device_value(nid, ComponentType.FULL)
+ )
+ for nid in transfer.nodes_to_load or ()
+ ]
+ return kv_xfer, comp_xfers
def prefetch_anchor_info(
self, node_id: NodeId
@@ -878,6 +1028,9 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
def root_node_handle(self, extra_key: Optional[str] = None) -> NodeId:
return self._binding.root_node_handle(extra_key)
+ def rotation_base_of(self, node_id: NodeId) -> Optional[int]:
+ return self._binding.rotation_base_of(node_id)
+
def dfs_weight_order(self, node_ids: Sequence[NodeId]) -> list[int]:
return self._binding.dfs_weight_order(list(node_ids))
diff --git a/python/sglang/srt/mem_cache/unified_cache/components/__init__.py b/python/sglang/srt/mem_cache/unified_cache/components/__init__.py
index 147f57205e..794431788a 100644
--- a/python/sglang/srt/mem_cache/unified_cache/components/__init__.py
+++ b/python/sglang/srt/mem_cache/unified_cache/components/__init__.py
@@ -6,6 +6,7 @@ from sglang.srt.mem_cache.unified_cache.components.base import (
ComponentType,
EvictLayer,
ExternalLinkerLoadPhase,
+ InternalStateBackup,
LinkerTransferPhase,
LRURefreshPhase,
PrepareLoadBackResult,
@@ -23,6 +24,7 @@ __all__ = [
"ComponentData",
"ComponentType",
"ExternalLinkerLoadPhase",
+ "InternalStateBackup",
"LinkerTransferPhase",
"EvictLayer",
"FullComponent",
diff --git a/python/sglang/srt/mem_cache/unified_cache/components/base.py b/python/sglang/srt/mem_cache/unified_cache/components/base.py
index d29099d424..e9c3996056 100644
--- a/python/sglang/srt/mem_cache/unified_cache/components/base.py
+++ b/python/sglang/srt/mem_cache/unified_cache/components/base.py
@@ -63,6 +63,15 @@ class EvictLayer(IntFlag):
ALL = DEVICE | HOST
+@dataclasses.dataclass(frozen=True)
+class InternalStateBackup:
+ """Pause eviction for a host backup before freeing internal device state."""
+
+ node_id: NodeId
+ # Host capacity required by the backup, in this component's pool units.
+ num_tokens: int
+
+
@dataclasses.dataclass(frozen=True)
class PrepareLoadBackResult:
"""Outcome of prepare_load_back; default = nothing to prepare."""
@@ -522,11 +531,12 @@ class TreeComponent(ABC):
tracker: dict[ComponentType, int],
device_frees: dict[ComponentType, list[torch.Tensor]],
host_frees: dict[ComponentType, list[torch.Tensor]],
- ) -> Optional[NodeId]:
- """Advance one eviction step and return a device leaf, if selected.
+ ) -> NodeId | InternalStateBackup | None:
+ """Return a device leaf, an internal backup request, or no selection.
Implementations must return after one allocator-relevant internal
mutation so the caller can drain pending frees before continuing.
+ Backup requests leave device state intact until the core resumes eviction.
"""
assert self.is_evict_device_ongoing, (
f"{self.component_type} device eviction not started"
@@ -552,8 +562,8 @@ class TreeComponent(ABC):
tracker: dict[ComponentType, int],
device_frees: dict[ComponentType, list[torch.Tensor]],
host_frees: dict[ComponentType, list[torch.Tensor]],
- ) -> Optional[NodeId]:
- """Advance the walk by at most one allocator-relevant mutation."""
+ ) -> NodeId | InternalStateBackup | None:
+ """Select a leaf, request backup, or perform at most one internal mutation."""
...
@abstractmethod
diff --git a/python/sglang/srt/mem_cache/unified_cache/components/mamba.py b/python/sglang/srt/mem_cache/unified_cache/components/mamba.py
index 16c113f598..c55ddab016 100644
--- a/python/sglang/srt/mem_cache/unified_cache/components/mamba.py
+++ b/python/sglang/srt/mem_cache/unified_cache/components/mamba.py
@@ -30,6 +30,7 @@ from sglang.srt.mem_cache.unified_cache.components.base import (
CacheTransferPhase,
ComponentType,
EvictLayer,
+ InternalStateBackup,
LinkerTransferPhase,
LRURefreshPhase,
PrepareLoadBackResult,
@@ -369,7 +370,7 @@ class MambaComponent(TreeComponent):
tracker: dict[ComponentType, int],
device_frees: dict[ComponentType, list[torch.Tensor]],
host_frees: dict[ComponentType, list[torch.Tensor]],
- ) -> Optional[NodeId]:
+ ) -> NodeId | InternalStateBackup | None:
"""Advance one device-eviction step and return a leaf, if selected.
An internal tombstone is one complete step so the caller can apply its
@@ -404,9 +405,18 @@ class MambaComponent(TreeComponent):
)
return x.id
if not enabled:
- x_next = lru.get_prev_no_lock(x)
- # write_back: demote the state to host before the internal tombstone.
- self._maybe_backup_node_before_state_tombstone(x)
+ self._evict_device_cursor = lru.get_prev_no_lock(x)
+ cd = x.component_data[ct]
+ if (
+ self.tree_core.enable_hicache
+ and self.tree_core.is_write_back
+ and cd.host_value is None
+ and not x.backuped
+ and x.component_data[BASE_COMPONENT_TYPE].value is not None
+ ):
+ # Keep the state live until the controller has attempted its host
+ # backup. Session cursors advance after the resumed tombstone.
+ return InternalStateBackup(node_id=x.id, num_tokens=1)
self.tree_core._evict_component_and_detach_lru(
x,
self,
@@ -418,38 +428,10 @@ class MambaComponent(TreeComponent):
self.tree_core._cascade_evict(
x, self, tracker, device_frees=device_frees, host_frees=host_frees
)
- self._evict_device_cursor = lru.cursor_next() if enabled else x_next
+ if enabled:
+ self._evict_device_cursor = lru.cursor_next()
return None
- def _maybe_backup_node_before_state_tombstone(self, node: UnifiedTreeNode) -> None:
- """Demote an internal node's mamba state to host before its tombstone
- (write_back only), mirroring the leaf deferred-demote path.
-
- The match validator passes only nodes holding the state on some
- layer, so a dropped internal state caps the match frontier at this
- node forever, leaving the subtree's still-resident KV unservable.
- Best-effort: this walk must make progress (it satisfies an imminent
- slot allocation), so any failure falls back to the legacy drop.
- """
- cache = self.cache
- cd = node.component_data[self.component_type]
- if (
- cache.cache_controller is None
- or not cache.is_write_back
- or cd.host_value is not None
- or node.backuped
- or node.component_data[BASE_COMPONENT_TYPE].value is None
- ):
- return
- # The backup executor pre-evicts only the KV host pool; make room
- # for the state slot the way the PREFETCH hook does.
- if (
- self._mamba_pool_host is not None
- and self._mamba_pool_host.available_size() < 1
- ):
- cache.evict_host(1, self.component_type)
- cache.backup_node_for_write_back(node.id)
-
def _evict_device_end(self) -> None:
"""Clear the device-eviction walk cursor state."""
if self.tree_core.enable_session_radix_cache:
diff --git a/python/sglang/srt/mem_cache/unified_cache/components/swa.py b/python/sglang/srt/mem_cache/unified_cache/components/swa.py
index 2d2550e45f..e5263d91eb 100644
--- a/python/sglang/srt/mem_cache/unified_cache/components/swa.py
+++ b/python/sglang/srt/mem_cache/unified_cache/components/swa.py
@@ -36,6 +36,7 @@ from sglang.srt.mem_cache.unified_cache.components.base import (
ComponentType,
EvictLayer,
ExternalLinkerLoadPhase,
+ InternalStateBackup,
LinkerTransferPhase,
LRURefreshPhase,
PreparePrefetchResult,
@@ -711,7 +712,7 @@ class SWAComponent(TreeComponent):
tracker: dict[ComponentType, int],
device_frees: dict[ComponentType, list[torch.Tensor]],
host_frees: dict[ComponentType, list[torch.Tensor]],
- ) -> Optional[NodeId]:
+ ) -> NodeId | InternalStateBackup | None:
"""Advance one device-eviction step and return a leaf, if selected.
An internal tombstone is one complete step so the caller can apply its
@@ -746,9 +747,33 @@ class SWAComponent(TreeComponent):
)
return x.id
if not enabled:
- x_next = lru.get_prev_no_lock(x)
- # write_back: demote the SWA KV to host before the internal tombstone.
- self._maybe_backup_node_before_swa_tombstone(x)
+ self._evict_device_cursor = lru.get_prev_no_lock(x)
+ cd = x.component_data[ct]
+ if (
+ self.tree_core.enable_hicache
+ and self.tree_core.is_write_back
+ and self.tree_core.has_swa_host_pool
+ and cd.host_value is None
+ and not x.backuped
+ and x.component_data[BASE_COMPONENT_TYPE].value is not None
+ ):
+ # Reserve the whole unbacked window, not just this victim, before
+ # its best-effort host backup and resumed internal tombstone.
+ needed = sum(
+ len(node.component_data[ct].value)
+ for node in self._collect_unbacked_swa_nodes(x)
+ )
+ if needed:
+ if ct == ComponentType.SWA:
+ return InternalStateBackup(node_id=x.id, num_tokens=needed)
+ # Custom SWA component types use Python's inherited inline
+ # path; the shared finish hook is for built-in SWA only.
+ if (
+ self._swa_kv_pool_host is not None
+ and self._swa_kv_pool_host.available_size() < needed
+ ):
+ self.cache.evict_host(needed, ct)
+ self.cache.backup_node_for_write_back(x.id)
self.tree_core._evict_component_and_detach_lru(
x,
self,
@@ -760,48 +785,10 @@ class SWAComponent(TreeComponent):
self.tree_core._cascade_evict(
x, self, tracker, device_frees=device_frees, host_frees=host_frees
)
- self._evict_device_cursor = lru.cursor_next() if enabled else x_next
+ if enabled:
+ self._evict_device_cursor = lru.cursor_next()
return None
- def _maybe_backup_node_before_swa_tombstone(self, node: UnifiedTreeNode) -> None:
- """Demote an internal node's SWA KV to host before its tombstone
- (write_back only), mirroring the leaf deferred-demote path.
-
- The match validator treats an unbacked tombstone as a window reset,
- so a dropped internal SWA segment caps the match frontier until a
- full sliding window re-accumulates below it, leaving up to one
- window of still-resident KV unservable. The leaf backup walk covers
- ancestors only within one window of the evicted leaf, so an
- internal node whose child spans the window arrives here unbacked.
- Best-effort: this walk must make progress (it satisfies an imminent
- allocation), so any failure falls back to the legacy drop.
- """
- cache = self.cache
- cd = node.component_data[self.component_type]
- if (
- cache.cache_controller is None
- or not cache.is_write_back
- or not self.tree_core.has_swa_host_pool
- or cd.host_value is not None
- or node.backuped
- or node.component_data[BASE_COMPONENT_TYPE].value is None
- ):
- return
- # The backup executor pre-evicts only the KV host pool; make room
- # in the SWA host pool the way the PREFETCH hook does.
- needed = sum(
- len(n.component_data[self.component_type].value)
- for n in self._collect_unbacked_swa_nodes(node)
- )
- if needed == 0:
- return
- if (
- self._swa_kv_pool_host is not None
- and self._swa_kv_pool_host.available_size() < needed
- ):
- cache.evict_host(needed, self.component_type)
- cache.backup_node_for_write_back(node.id)
-
def _evict_device_end(self) -> None:
"""Clear the device-eviction walk cursor state."""
if self.tree_core.enable_session_radix_cache:
diff --git a/python/sglang/srt/mem_cache/unified_cache/tree_core_registry.py b/python/sglang/srt/mem_cache/unified_cache/tree_core_registry.py
index 47bd6e1b1e..bed9e74b4a 100644
--- a/python/sglang/srt/mem_cache/unified_cache/tree_core_registry.py
+++ b/python/sglang/srt/mem_cache/unified_cache/tree_core_registry.py
@@ -1,18 +1,28 @@
"""Registry for pluggable TreeCore implementations.
The unified cache constructs its TreeCore through `create_tree_core`, selected
-by SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND (default "python"). To plug in a custom
-implementation, register it under a string name via
-`register_tree_core_backend(name, factory)`.
+by its per-cache override or SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND (default
+"rust"). Rust selections use centralized compatibility fallbacks. To plug in a
+custom implementation, register it via `register_tree_core_backend(name, factory)`.
"""
from __future__ import annotations
+import importlib.util
+import logging
+import re
+import subprocess
+import sys
+from pathlib import Path
from typing import TYPE_CHECKING, Callable, Optional
+import torch
+
+from sglang.srt.environ import envs
+from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
+
if TYPE_CHECKING:
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
- from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
from sglang.srt.mem_cache.unified_cache.components import TreeComponent
from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import (
UnifiedTreeCoreInterface,
@@ -24,6 +34,101 @@ TreeCoreFactory = Callable[
]
_TREE_CORE_REGISTRY: dict[str, TreeCoreFactory] = {}
+_RUST_TREE_CORE_MODULE = "sglang.srt.mem_cache.rust_tree_core.mem_cache"
+_RUST_TREE_CORE_MANIFEST = (
+ Path(__file__).resolve().parents[5] / "rust" / "sglang-radix-tree" / "Cargo.toml"
+)
+logger = logging.getLogger(__name__)
+
+
+def _rust_fallback_reason(params: CacheInitParams) -> Optional[str]:
+ if params.enable_session_radix_cache:
+ return "session-aware caching requires the Python TreeCore"
+ if params.tree_components is not None and set(params.tree_components) - {
+ ComponentType.FULL,
+ ComponentType.SWA,
+ ComponentType.MAMBA,
+ }:
+ return "the configured components require the Python TreeCore"
+ if params.component_registry_override:
+ return "custom components require the Python TreeCore"
+ if sys.platform != "linux":
+ return "the Rust TreeCore supports Linux only"
+ from sglang.srt.rust_extensions.torch_build import (
+ _MAX_SUPPORTED_TORCH,
+ _MIN_SUPPORTED_TORCH,
+ )
+
+ match = re.match(r"^(\d+)\.(\d+)", str(torch.__version__))
+ if match is None or not (
+ _MIN_SUPPORTED_TORCH
+ <= (int(match.group(1)), int(match.group(2)))
+ <= _MAX_SUPPORTED_TORCH
+ ):
+ return f"PyTorch {torch.__version__} is outside the Rust TreeCore support range"
+ allocator = params.token_to_kv_pool_allocator
+ device = (
+ torch.device(allocator.device) if allocator is not None else torch.device("cpu")
+ )
+ if device.type not in ("cpu", "cuda"):
+ return f"the Rust TreeCore does not support device {device.type}"
+ mode = envs.SGLANG_RUST_BUILD_MODE.get()
+ if sys.modules.get(_RUST_TREE_CORE_MODULE) is not None or mode not in (
+ "auto",
+ "never",
+ "force",
+ ):
+ # Preserve the loader's invalid-mode and force-after-import errors.
+ return None
+ workspace = _RUST_TREE_CORE_MANIFEST.parent.parent
+ bundled = importlib.util.find_spec(_RUST_TREE_CORE_MODULE) is not None
+ source_checkout = (workspace / "Cargo.toml").is_file()
+ if mode != "force" and (mode == "never" or not source_checkout) and bundled:
+ return None
+ if not _RUST_TREE_CORE_MANIFEST.is_file():
+ if bundled:
+ return "the requested Rust TreeCore source build has no sources"
+ return "this installation contains neither the Rust TreeCore extension nor its sources"
+ # Even a cached source build needs these version queries for its fingerprint.
+ # Match the loader's commands and cwd so rustup workspace overrides apply.
+ for command in (("cargo", "--version", "--verbose"), ("rustc", "-vV")):
+ try:
+ subprocess.run(
+ command,
+ cwd=workspace,
+ check=True,
+ capture_output=True,
+ text=True,
+ timeout=10,
+ )
+ except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired):
+ return f"the Rust TreeCore toolchain is unavailable ({' '.join(command)})"
+ return None
+
+
+def resolve_tree_core_backend(name: str, params: CacheInitParams) -> str:
+ """Resolve known Rust capability gaps before loading a backend.
+
+ Explicit Rust selections use the same compatibility policy as the default.
+ Build, import, and runtime failures remain errors when the toolchain is usable.
+ """
+ if name != "rust":
+ return name
+ reason = _rust_fallback_reason(params)
+ if reason is not None:
+ logger.info("Using the Python TreeCore: %s", reason)
+ return "python"
+ return name
+
+
+def select_tree_core_backend(params: CacheInitParams) -> str:
+ """Resolve the instance override or default through the shared fallback policy."""
+ name = (
+ params.tree_core_backend
+ if params.tree_core_backend is not None
+ else envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.get()
+ )
+ return resolve_tree_core_backend(name, params)
def register_tree_core_backend(name: str, factory: TreeCoreFactory) -> None:
@@ -72,7 +177,8 @@ def create_tree_core(
params: CacheInitParams,
components: dict[ComponentType, TreeComponent],
) -> UnifiedTreeCoreInterface:
- """Construct the TreeCore registered under `name`."""
+ """Resolve compatibility and construct the registered TreeCore."""
+ name = resolve_tree_core_backend(name, params)
factory = get_tree_core_factory(name)
if factory is None:
raise ValueError(
diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py
index 4027b80354..70482bf879 100644
--- a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py
+++ b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py
@@ -60,6 +60,7 @@ from sglang.srt.mem_cache.unified_cache.components import (
ComponentData,
ComponentType,
EvictLayer,
+ InternalStateBackup,
LinkerTransferPhase,
LRURefreshPhase,
TreeComponent,
@@ -474,6 +475,13 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
def reset(self) -> None:
"""Rebuild the root, LRUs, sizes, evictable-leaf sets, and the empty
match result."""
+ # End suspended walks while their session-cursor sentinels still
+ # belong to the old LRUs, clearing any pending internal victim.
+ for component in self.components:
+ if component.is_evict_device_ongoing:
+ component.evict_device_end()
+ # Internal victims awaiting the controller's backup and finish call.
+ self._pending_internal_evictions: dict[ComponentType, NodeId] = {}
# Maintains the NodeId -> active tree node mapping.
self._node_arena: dict[NodeId, UnifiedTreeNode] = {}
@@ -1586,12 +1594,15 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
result = EvictDeviceNextNodeResult()
# The walk reads running totals for its doneness check; the result
# carries only this step's delta.
+ component = self.components_by_type[component_type]
+ assert component_type not in self._pending_internal_evictions, (
+ f"finish the pending internal {component_type.name} eviction "
+ "before advancing"
+ )
updated_tracker = defaultdict(int, tracker)
self._begin_tracking_unbacked_tokens()
try:
- result.node_id = self.components_by_type[
- component_type
- ].evict_device_next_node(
+ step = component.evict_device_next_node(
updated_tracker, result.device_frees, result.host_frees
)
finally:
@@ -1600,12 +1611,99 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
delta = n - tracker.get(ct, 0)
if delta:
result.tracker[ct] = delta
- result.made_progress = result.node_id is not None or bool(result.tracker)
+ if isinstance(step, InternalStateBackup):
+ assert component_type in (ComponentType.MAMBA, ComponentType.SWA)
+ self._pending_internal_evictions[component_type] = step.node_id
+ if component_type == ComponentType.MAMBA:
+ result.mamba_backup_node_id = step.node_id
+ else:
+ result.swa_backup_node_id = step.node_id
+ result.swa_backup_num_tokens = step.num_tokens
+ else:
+ result.node_id = step
+ result.made_progress = (
+ result.node_id is not None
+ or result.mamba_backup_node_id is not None
+ or result.swa_backup_node_id is not None
+ or bool(result.tracker)
+ )
+ return result
+
+ def finish_mamba_state_eviction(self, node_id: NodeId) -> EvictDeviceNextNodeResult:
+ return self._finish_internal_component_eviction(ComponentType.MAMBA, node_id)
+
+ def finish_swa_state_eviction(self, node_id: NodeId) -> EvictDeviceNextNodeResult:
+ return self._finish_internal_component_eviction(ComponentType.SWA, node_id)
+
+ def _finish_internal_component_eviction(
+ self, component_type: ComponentType, node_id: NodeId
+ ) -> EvictDeviceNextNodeResult:
+ """Resume an internal tombstone after the controller's backup attempt."""
+ component = self.components_by_type[component_type]
+ assert component.is_evict_device_ongoing, (
+ f"{component_type.name} device eviction not started"
+ )
+ assert self._pending_internal_evictions.get(component_type) == node_id, (
+ f"no matching pending internal {component_type.name} eviction"
+ )
+ del self._pending_internal_evictions[component_type]
+ # Consuming the pending request advances the walk even if I/O changed
+ # this victim's eligibility. No frees occur before backup completion.
+ result = EvictDeviceNextNodeResult(made_progress=True)
+ node = self._node_arena.get(node_id)
+ lru = self.lru_lists[component_type]
+ enabled = self.enable_session_radix_cache
+ if (
+ node is None
+ or node.component_data[component_type].value is None
+ or node.component_data[component_type].lock_ref > 0
+ or (
+ component_type == ComponentType.MAMBA
+ and node.load_back_pending_id is not None
+ )
+ or not lru.in_list(node)
+ ):
+ if enabled:
+ component._evict_device_cursor = lru.cursor_next()
+ return result
+ if (
+ node in self.evictable_device_leaves
+ and (not enabled or component._can_evict_leaf_atomically(node))
+ ) or (
+ component_type == ComponentType.MAMBA
+ and node.component_data[BASE_COMPONENT_TYPE].value is None
+ ):
+ # Re-select changed leaves through the normal walk; internal
+ # finish must not accidentally perform an atomic Full eviction.
+ component._evict_device_cursor = node
+ return result
+ self._begin_tracking_unbacked_tokens()
+ try:
+ self._evict_component_and_detach_lru(
+ node,
+ component,
+ target=EvictLayer.DEVICE,
+ tracker=result.tracker,
+ device_frees=result.device_frees,
+ host_frees=result.host_frees,
+ )
+ self._cascade_evict(
+ node,
+ component,
+ result.tracker,
+ device_frees=result.device_frees,
+ host_frees=result.host_frees,
+ )
+ finally:
+ result.unbacked_tokens = self._finish_tracking_unbacked_tokens()
+ if enabled:
+ component._evict_device_cursor = lru.cursor_next()
return result
def evict_device_end(self, component_type: ComponentType) -> None:
"""Finish a component's device-eviction walk."""
self.components_by_type[component_type].evict_device_end()
+ self._pending_internal_evictions.pop(component_type, None)
def evict_device_leaf(
self, node_id: NodeId, is_write_back: bool
diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py
index 4987b100f3..4698fc105e 100644
--- a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py
+++ b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py
@@ -45,12 +45,18 @@ class EvictDeviceNextNodeResult(BaseEvictionResult):
``node_id`` selects a leaf for the Controller to evict. ``made_progress``
also covers an internal tombstone that returned no leaf, distinguishing it
- from true walk exhaustion.
+ from true walk exhaustion. ``mamba_backup_node_id`` and
+ ``swa_backup_node_id`` pause an internal eviction until the Controller
+ finishes its best-effort host backup. ``swa_backup_num_tokens`` includes
+ all unbacked SWA segments in the backup window, not just the victim.
"""
node_id: Optional[NodeId] = None
made_progress: bool = False
unbacked_tokens: int = 0
+ mamba_backup_node_id: Optional[NodeId] = None
+ swa_backup_node_id: Optional[NodeId] = None
+ swa_backup_num_tokens: int = 0
class EvictDeviceLeafResult(BaseEvictionResult):
@@ -319,6 +325,24 @@ class UnifiedTreeCoreInterface(ABC):
D->H backup must run (write_back) before the node can be demoted."""
...
+ @abstractmethod
+ def finish_mamba_state_eviction(self, node_id: NodeId) -> EvictDeviceNextNodeResult:
+ """Resume the pending internal Mamba eviction after its backup attempt.
+
+ The controller must finish any submitted D->H transfer before calling.
+ A failed allocation still permits eviction to make device space.
+ """
+ ...
+
+ @abstractmethod
+ def finish_swa_state_eviction(self, node_id: NodeId) -> EvictDeviceNextNodeResult:
+ """Resume the pending internal SWA eviction after its backup attempt.
+
+ The controller must finish any submitted D->H transfer before calling.
+ A failed allocation still permits eviction to make device space.
+ """
+ ...
+
@abstractmethod
def drop_subtree_no_host(self, node_id: NodeId) -> DropSubtreeNoHostResult:
"""Drop an unbacked D-leaf's subtree when its write-back backup failed
@@ -642,6 +666,29 @@ class UnifiedTreeCoreInterface(ABC):
host store event for each acked node."""
...
+ @abstractmethod
+ def swa_tombstone_ranges(
+ self, key: RadixKey, start: int, end: int
+ ) -> list[tuple[int, int]]:
+ """Return maximal missing SWA ranges within the matched [start, end) span."""
+ ...
+
+ @abstractmethod
+ def attach_swa_window(
+ self,
+ key: RadixKey,
+ window_start: int,
+ window_end: int,
+ swa_values: torch.Tensor,
+ ) -> list[CacheAction | ComponentAction]:
+ """Attach a loaded SWA window to tombstoned spans, returning split actions.
+
+ The shared pipeline supplies page-aligned logical token offsets and
+ int64 values on the core's device. The entire span must be matched and
+ tombstoned before publication.
+ """
+ ...
+
@abstractmethod
def set_component_device_value(
self, node_id: NodeId, component_type: ComponentType, value: torch.Tensor
diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py
index f9c0fcc04f..421a215722 100644
--- a/python/sglang/srt/mem_cache/unified_radix_cache.py
+++ b/python/sglang/srt/mem_cache/unified_radix_cache.py
@@ -74,7 +74,10 @@ from sglang.srt.mem_cache.unified_cache.session_ref_tracker import (
UnifiedSessionRefTracker,
)
from sglang.srt.mem_cache.unified_cache.storage_attachment import StorageAttachment
-from sglang.srt.mem_cache.unified_cache.tree_core_registry import create_tree_core
+from sglang.srt.mem_cache.unified_cache.tree_core_registry import (
+ create_tree_core,
+ select_tree_core_backend,
+)
from sglang.srt.mem_cache.unified_cache.unified_cache_linker import (
UnifiedCacheLinker,
UnifiedCacheLinkerWrapper,
@@ -204,11 +207,7 @@ class UnifiedRadixCache(BasePrefixCache):
)
# The TreeCore owns the tree member-var state (structure, LRUs, sizes,
# evictable leaves) and drives the components' tree-level hooks.
- self._tree_core_backend = (
- params.tree_core_backend
- if params.tree_core_backend is not None
- else envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.get()
- )
+ self._tree_core_backend = select_tree_core_backend(params)
self.tree_core = create_tree_core(
name=self._tree_core_backend,
params=params,
@@ -756,6 +755,35 @@ class UnifiedRadixCache(BasePrefixCache):
) -> tuple[Optional[NodeId], bool]:
"""Advance the eviction walk one node, consuming its step result."""
result = self.tree_core.evict_device_next_node(component_type, tracker)
+ if result.mamba_backup_node_id is not None:
+ assert component_type == ComponentType.MAMBA and result.node_id is None
+ assert (
+ not result.device_frees and not result.host_frees and not result.tracker
+ )
+ # Reserve a host state slot and wait for the backup acknowledgment
+ # before freeing device state. If allocation fails, eviction still
+ # proceeds to make room on the device.
+ node_id = result.mamba_backup_node_id
+ mamba_host_pool = self.host_pool_group.get_pool(PoolName.MAMBA)
+ if mamba_host_pool is not None and mamba_host_pool.available_size() < 1:
+ self.evict_host(1, ComponentType.MAMBA)
+ self.backup_node_for_write_back(node_id)
+ result = self.tree_core.finish_mamba_state_eviction(node_id)
+ elif result.swa_backup_node_id is not None:
+ assert component_type == ComponentType.SWA and result.node_id is None
+ assert (
+ not result.device_frees and not result.host_frees and not result.tracker
+ )
+ # The backup can cover several unbacked SWA segments. Reserve host
+ # space for the whole window before copying it, then resume eviction
+ # even if host allocation fails.
+ node_id = result.swa_backup_node_id
+ needed = result.swa_backup_num_tokens
+ swa_host_pool = self.host_pool_group.get_pool(PoolName.SWA)
+ if swa_host_pool is not None and swa_host_pool.available_size() < needed:
+ self.evict_host(needed, ComponentType.SWA)
+ self.backup_node_for_write_back(node_id)
+ result = self.tree_core.finish_swa_state_eviction(node_id)
self._free_values(result.device_frees, result.host_frees)
if self._tracks_write_through_unbacked_evictions():
self._record_dropped_tokens(
@@ -887,8 +915,7 @@ class UnifiedRadixCache(BasePrefixCache):
def _tracks_write_through_unbacked_evictions(self) -> bool:
return (
- isinstance(self.tree_core, UnifiedTreeCore)
- and self.host_memory_mode == "cache"
+ self.host_memory_mode == "cache"
and self.cache_controller is not None
and self.cache_controller.write_policy == "write_through"
)
diff --git a/python/sglang/test/scripted_runtime/__init__.py b/python/sglang/test/scripted_runtime/__init__.py
index e69de29bb2..b0f1bf9908 100644
--- a/python/sglang/test/scripted_runtime/__init__.py
+++ b/python/sglang/test/scripted_runtime/__init__.py
@@ -0,0 +1,11 @@
+from sglang.srt.environ import envs
+
+# Spawned server and scheduler processes import the scripted-runtime package
+# before constructing their cache. Keep inspection in test processes while
+# preserving the selected backend and its compatibility fallback.
+if envs.SGLANG_TEST_SCRIPTED_RUNTIME.get():
+ from sglang.test.scripted_runtime.tree_core_inspection import (
+ install_tree_core_inspectors,
+ )
+
+ install_tree_core_inspectors()
diff --git a/python/sglang/test/scripted_runtime/context/lock_ref_exhauster.py b/python/sglang/test/scripted_runtime/context/lock_ref_exhauster.py
index dac4fbdba9..bb6508b876 100644
--- a/python/sglang/test/scripted_runtime/context/lock_ref_exhauster.py
+++ b/python/sglang/test/scripted_runtime/context/lock_ref_exhauster.py
@@ -1,8 +1,11 @@
from __future__ import annotations
-from typing import TYPE_CHECKING, Any, List
+from typing import TYPE_CHECKING, Any
-from sglang.test.scripted_runtime.context.radix import _node_lock_ref, to_node_handle
+from sglang.test.scripted_runtime.context.radix import (
+ get_node_lock_ref,
+ iter_node_handles,
+)
if TYPE_CHECKING:
from sglang.srt.managers.scheduler import Scheduler
@@ -12,9 +15,9 @@ class ScriptedLockRefExhauster:
def __init__(self, scheduler: Scheduler) -> None:
self.scheduler = scheduler
# (node, dec receipt) pairs; the receipt bounds the release walk.
- self._locked: List[tuple[Any, Any]] = []
+ self._locked: list[tuple[Any, Any]] = []
- def exhaust(self, *, leave_refs: int) -> None:
+ def exhaust(self, leave_refs: int) -> None:
tree_cache = self.scheduler.tree_cache
if tree_cache.disable:
return
@@ -25,9 +28,11 @@ class ScriptedLockRefExhauster:
return
target = evictable[0]
- result = tree_cache.inc_lock_ref(to_node_handle(tree_cache, target))
+ result = tree_cache.inc_lock_ref(target)
- newly_locked = [node for node in evictable if _node_lock_ref(node) > 0]
+ newly_locked = [
+ node for node in evictable if get_node_lock_ref(tree_cache, node) > 0
+ ]
if not newly_locked:
return
self._locked.append((target, result.to_dec_params()))
@@ -35,15 +40,13 @@ class ScriptedLockRefExhauster:
def release(self) -> None:
tree_cache = self.scheduler.tree_cache
for node, dec_params in self._locked:
- tree_cache.dec_lock_ref(to_node_handle(tree_cache, node), dec_params)
+ tree_cache.dec_lock_ref(node, dec_params)
self._locked.clear()
- def _evictable_nodes(self) -> List[Any]:
- evictable: List[Any] = []
- stack = list(self.scheduler.tree_cache.root_node.children.values())
- while stack:
- node = stack.pop()
- if _node_lock_ref(node) == 0:
- evictable.append(node)
- stack.extend(node.children.values())
- return evictable
+ def _evictable_nodes(self) -> list[Any]:
+ tree_cache = self.scheduler.tree_cache
+ return [
+ node
+ for node in iter_node_handles(tree_cache)
+ if get_node_lock_ref(tree_cache, node) == 0
+ ]
diff --git a/python/sglang/test/scripted_runtime/context/radix.py b/python/sglang/test/scripted_runtime/context/radix.py
index 91403b2c0d..0bee706077 100644
--- a/python/sglang/test/scripted_runtime/context/radix.py
+++ b/python/sglang/test/scripted_runtime/context/radix.py
@@ -1,6 +1,7 @@
from __future__ import annotations
-from typing import TYPE_CHECKING, Any, Callable, Dict
+from collections.abc import Iterator
+from typing import TYPE_CHECKING, Any
from sglang.srt.mem_cache.unified_radix_cache import UnifiedTreeNode
@@ -8,12 +9,24 @@ if TYPE_CHECKING:
from sglang.test.scripted_runtime.context.api import ScriptedContext
-def get_all_node_hit_counts(ctx: ScriptedContext) -> Dict[int, int]:
- return _collect_node_attr(ctx, lambda node: node.hit_count)
+def get_all_node_hit_counts(ctx: ScriptedContext) -> dict[int, int]:
+ tree_cache = ctx.scheduler.tree_cache
+ core = getattr(tree_cache, "tree_core", None)
+ return {
+ node if core is not None else node.id: (
+ core.get_node_hit_count(node) if core is not None else node.hit_count
+ )
+ for node in iter_node_handles(tree_cache)
+ }
-def get_all_node_lock_refs(ctx: ScriptedContext) -> Dict[int, int]:
- return _collect_node_attr(ctx, _node_lock_ref)
+def get_all_node_lock_refs(ctx: ScriptedContext) -> dict[int, int]:
+ tree_cache = ctx.scheduler.tree_cache
+ core = getattr(tree_cache, "tree_core", None)
+ return {
+ node if core is not None else node.id: get_node_lock_ref(tree_cache, node)
+ for node in iter_node_handles(tree_cache)
+ }
def _node_lock_ref(node: Any) -> int:
@@ -22,37 +35,34 @@ def _node_lock_ref(node: Any) -> int:
return node.lock_ref
-def resolve_node(tree_cache: Any, node_handle: Any) -> Any:
- """Resolve whatever a req or match result carries into a tree node.
-
- ``UnifiedRadixCache`` hands out NodeIds (ints); every other cache hands out
- the node object itself. Returns None when the handle no longer maps to a
- live node, which only happens once the node has been freed -- and a freed
- node holds no locks.
- """
- resolve = getattr(tree_cache, "resolve_node_handle", None)
- if resolve is None:
- return node_handle
- try:
- return resolve(node_handle)
- except KeyError:
- return None
+def get_node_lock_ref(tree_cache: Any, node_handle: Any) -> int:
+ """Read locks through the cache's native handle, including stale handles."""
+ if node_handle is None:
+ return 0
+ core = getattr(tree_cache, "tree_core", None)
+ if core is not None:
+ if not core.contains_node(node_handle):
+ return 0
+ return sum(
+ core.get_component_device_lock_ref(node_handle, component)
+ for component in tree_cache.tree_components
+ )
+ return _node_lock_ref(node_handle)
-def to_node_handle(tree_cache: Any, node: Any) -> Any:
- """Inverse of `resolve_node`: what the tree_cache lock APIs accept."""
- if isinstance(node, UnifiedTreeNode):
- return node.id
- return node
-
-
-def _collect_node_attr(
- ctx: ScriptedContext, get_value: Callable[[Any], int]
-) -> Dict[int, int]:
- values: Dict[int, int] = {}
- stack = list(ctx.scheduler.tree_cache.root_node.children.values())
+def iter_node_handles(tree_cache: Any) -> Iterator[Any]:
+ """Walk non-root nodes as IDs for unified caches or native legacy nodes."""
+ core = getattr(tree_cache, "tree_core", None)
+ stack = (
+ list(core.get_child_node_ids(core.root_node_handle()))
+ if core is not None
+ else list(tree_cache.root_node.children.values())
+ )
while stack:
node = stack.pop()
- values[node.id] = get_value(node)
- stack.extend(node.children.values())
- return values
+ yield node
+ stack.extend(
+ core.get_child_node_ids(node)
+ if core is not None
+ else node.children.values()
+ )
diff --git a/python/sglang/test/scripted_runtime/req_handle.py b/python/sglang/test/scripted_runtime/req_handle.py
index 339f8a5ff2..ae44e83d20 100644
--- a/python/sglang/test/scripted_runtime/req_handle.py
+++ b/python/sglang/test/scripted_runtime/req_handle.py
@@ -1,9 +1,9 @@
from __future__ import annotations
from dataclasses import dataclass
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING
-from sglang.test.scripted_runtime.context.radix import _node_lock_ref, resolve_node
+from sglang.test.scripted_runtime.context.radix import get_node_lock_ref
if TYPE_CHECKING:
from sglang.srt.managers.schedule_batch import Req
@@ -16,7 +16,7 @@ class ScriptedReqHandle:
context: ScriptedContext
@property
- def req(self) -> Optional[Req]:
+ def req(self) -> Req | None:
return self.context.find_req_by_rid(self.rid)
@property
@@ -52,7 +52,4 @@ class ScriptedReqHandle:
req = self.req
if req is None:
return 0
- node = resolve_node(self.context.scheduler.tree_cache, req.last_node)
- if node is None:
- return 0
- return _node_lock_ref(node)
+ return get_node_lock_ref(self.context.scheduler.tree_cache, req.last_node)
diff --git a/python/sglang/test/scripted_runtime/tree_core_inspection.py b/python/sglang/test/scripted_runtime/tree_core_inspection.py
new file mode 100644
index 0000000000..6a27cd0923
--- /dev/null
+++ b/python/sglang/test/scripted_runtime/tree_core_inspection.py
@@ -0,0 +1,34 @@
+"""Use the shared TreeCore inspectors in scripted-runtime test processes."""
+
+import sys
+from pathlib import Path
+
+
+def install_tree_core_inspectors():
+ from sglang.srt.mem_cache.unified_cache.tree_core_registry import (
+ _TREE_CORE_REGISTRY,
+ )
+
+ inspector_dir = str(
+ Path(__file__).resolve().parents[4]
+ / "test"
+ / "registered"
+ / "unit"
+ / "mem_cache"
+ )
+ if inspector_dir not in sys.path:
+ sys.path.insert(0, inspector_dir)
+
+ def python_inspector_factory(params, components):
+ from unified_tree_core_inspector import UnifiedTreeCoreInspector
+
+ return UnifiedTreeCoreInspector(params, components)
+
+ def rust_inspector_factory(params, _components):
+ from rust_unified_tree_core_inspector import RustUnifiedTreeCoreInspector
+
+ return RustUnifiedTreeCoreInspector(params)
+
+ _TREE_CORE_REGISTRY.update(
+ python=python_inspector_factory, rust=rust_inspector_factory
+ )
diff --git a/rust/sglang-radix-tree/README.md b/rust/sglang-radix-tree/README.md
index 2a5ec00be6..82f209fa2d 100644
--- a/rust/sglang-radix-tree/README.md
+++ b/rust/sglang-radix-tree/README.md
@@ -4,20 +4,57 @@ Rust tree core for the Unified Radix Cache, covering Full attention, sliding win
## Usage
-Select the backend with:
+Rust is the default tree core. The centralized tree-core registry falls back to
+Python in these cases:
+
+- Session-aware caching.
+- C128 or other unsupported components.
+- Custom component overrides.
+- Non-Linux platforms.
+- PyTorch versions outside 2.11 through 2.13.
+- Devices other than CPU or CUDA.
+- Installations containing neither the Rust extension nor its sources.
+- Source builds with a missing or unusable Rust toolchain.
+
+This policy also applies when Rust is explicitly selected.
+Build, import, and runtime failures in supported configurations remain errors.
+The Rust bindings report unexpected native panics as `RuntimeError` so Python's
+crash handlers can report them and coordinate shutdown. A panic during a core
+operation poisons its mutex, and subsequent calls refuse to reuse that state.
+
+T-LRU accepts integer and floating-point `threshold` and `next_prompt_estimate`
+parameters. In mixed integer/float configurations, the integer estimate must fit
+in i128; larger values raise `OverflowError` at initialization. Native integer
+addition is checked, and overflow is reported as `RuntimeError`.
+
+Select a backend explicitly with:
```bash
SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND=rust
+# Use the Python implementation instead:
+SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND=python
```
-SGLang wheels bundle the production extension. A source checkout falls back to
+Standard SGLang wheels bundle the production extension; some platform
+distributions omit it. A source checkout falls back to
the shared fingerprinted Rust-extension cache; it never writes a shared object
into the Python package. LibTorch and the Python headers come from the running
interpreter's PyTorch install. PyTorch 2.11 through 2.13 are accepted explicitly,
and `torch_2_13_compat.h` covers two alignment APIs removed in PyTorch 2.13.
+Source checkouts need working `cargo` and `rustc` to use Rust, including for
+fingerprinted cache lookup. If either tool is unavailable, the registry selects
+Python. Trusted bundled extensions do not require a Rust compiler.
## Development
+Alternative backends implement `UnifiedTreeCoreInterface` and register through
+`register_tree_core_backend`. For built-in Mamba and SWA internal-state write-back,
+the core returns a backup request and the controller performs transfers and waits
+for acknowledgment.
+The controller then calls `finish_mamba_state_eviction` or
+`finish_swa_state_eviction` to resume eviction. This keeps I/O outside the Rust
+tree's mutex; both tree cores implement the same contract.
+
```bash
# Build (libtorch from the installed torch package):
cd rust/sglang-radix-tree
diff --git a/rust/sglang-radix-tree/src/components/mamba.rs b/rust/sglang-radix-tree/src/components/mamba.rs
index 7d3e6cb50d..9dae4cfa52 100644
--- a/rust/sglang-radix-tree/src/components/mamba.rs
+++ b/rust/sglang-radix-tree/src/components/mamba.rs
@@ -7,7 +7,7 @@ use std::collections::HashMap;
use tch::Tensor;
use crate::components::TreeComponent;
-use crate::components::{ComponentType, MAMBA};
+use crate::components::{ComponentType, FULL, MAMBA};
use crate::node::ChildKeyType;
use crate::node::Node;
use crate::node::{NodeId, NodeIdx_, TreeCoreRuntimeError, ValueSlotIdx};
@@ -350,6 +350,13 @@ impl TreeComponent for MambaComponent {
tree_core.component_state(MAMBA).is_evict_device_ongoing,
"Mamba device eviction not started"
);
+ assert!(
+ tree_core
+ .component_state(MAMBA)
+ .evict_device_pending_node
+ .is_none(),
+ "finish the pending internal Mamba eviction before advancing"
+ );
let mut cursor = tree_core.component_state(MAMBA).evict_device_cursor;
// The cursor is re-validated (reset to LRU head) if the previous
// node's eviction removed it.
@@ -358,15 +365,15 @@ impl TreeComponent for MambaComponent {
.device_lru_list(MAMBA)
.get_lru_no_lock(&tree_core.arena);
}
- let next = loop {
+ let next = 'step: {
if tracker[&ct] >= tree_core.component_state(MAMBA).evict_device_request_cnt {
- break None;
+ break 'step None;
}
let Some(x) = cursor else {
- break None;
+ break 'step None;
};
if !tree_core.device_lru_list(MAMBA).in_list(Some(x)) {
- break None;
+ break 'step None;
}
assert!(
tree_core.arena.has_device_value(x, MAMBA),
@@ -375,14 +382,28 @@ impl TreeComponent for MambaComponent {
cursor = tree_core
.device_lru_list(MAMBA)
.get_prev_no_lock(x, &tree_core.arena);
- // A load-back pin means an in-flight DMA targets this node's slices.
- if tree_core.arena.node(x).is_load_back_pending() {
- continue;
- }
+ // The Mamba LRU filters its own transfer/request locks. Full load
+ // pins and expanded SWA backups can cover unrelated Mamba state
+ // restored by a later request, which remains independently evictable.
if tree_core.evictable_device_leaves.contains(x) {
- break Some(x);
+ break 'step Some(x);
}
- // Internal nodes are tombstoned inline (no IO).
+ let node = tree_core.arena.node(x);
+ if tree_core.enable_hicache
+ && tree_core.is_write_back
+ && !node.has_host_value(MAMBA)
+ && !node.backuped()
+ && node.has_device_value(FULL)
+ {
+ // Keep the state live until the controller finishes D->H I/O.
+ // Native code never calls Python while holding the tree lock.
+ let node_id = node.id;
+ tree_core
+ .component_state_mut(MAMBA)
+ .evict_device_pending_node = Some(node_id);
+ break 'step None;
+ }
+ // Other policies and already-backed states need no I/O.
tree_core.evict_component_and_detach_lru_(
x,
ct,
@@ -392,7 +413,7 @@ impl TreeComponent for MambaComponent {
Some(tracker),
);
tree_core.cascade_evict_(x, ct, tracker, device_frees, host_frees, EvictLayer::Device);
- break None;
+ None
};
tree_core.component_state_mut(MAMBA).evict_device_cursor = cursor;
next
@@ -698,11 +719,8 @@ impl TreeComponent for MambaComponent {
let x_next = tree_core
.host_lru_list(MAMBA)
.get_prev_no_lock(cur, &tree_core.arena);
- // A load-back pin means an in-flight DMA reads this node's host slices.
- if tree_core.arena.node(cur).is_load_back_pending() {
- x = x_next;
- continue;
- }
+ // The host LRU filters Mamba's own locks. A Full load-back pin may
+ // cover an ancestor whose Mamba state is not being transferred.
if tree_core.evictable_host_leaves.contains(cur) {
// Host leaf: atomic eviction (all components host + delete)
tree_core.evict_host_leaf_(cur, tracker, device_frees, host_frees);
diff --git a/rust/sglang-radix-tree/src/components/swa.rs b/rust/sglang-radix-tree/src/components/swa.rs
index acd7c19d91..cd840002d8 100644
--- a/rust/sglang-radix-tree/src/components/swa.rs
+++ b/rust/sglang-radix-tree/src/components/swa.rs
@@ -22,6 +22,8 @@ use crate::unified_tree_core::{
pub struct SwaComponent {
/// Sliding window size in tokens.
sliding_window_size: usize,
+ /// Per-request rings are rebuilt by the request and do not gate tree reuse.
+ swa_req_ring: bool,
}
impl SwaComponent {
@@ -43,6 +45,7 @@ impl SwaComponent {
);
SwaComponent {
sliding_window_size,
+ swa_req_ring: params.swa_req_ring,
}
}
@@ -205,18 +208,6 @@ impl SwaComponent {
}
unbacked
}
-
- fn next_host_unlocked_device_lru_node(
- tree_core: &UnifiedTreeCore,
- from: Option,
- ) -> Option {
- let lru = tree_core.device_lru_list(SWA);
- let unlocked = |id: NodeIdx_| tree_core.arena.node(id).host_lock_ref(SWA) == 0;
- match from {
- Some(node_id) => lru.get_prev_where(node_id, unlocked),
- None => lru.get_lru_where(unlocked),
- }
- }
}
impl SwaComponent {
@@ -392,13 +383,13 @@ impl TreeComponent for SwaComponent {
fn create_match_validator(
&self,
- tree_core: &UnifiedTreeCore,
+ _tree_core: &UnifiedTreeCore,
match_device_only: bool,
) -> Box, NodeIdx_) -> bool> {
let sliding_window_size = self.sliding_window_size;
- // unified_kv never caches the SWA ring (per-request, not
- // content-stable), so SWA bookkeeping must not gate the match here.
- let swa_device_only_hicache = !tree_core.has_swa_host_pool && tree_core.enable_hicache;
+ // A per-request SWA ring is not stored in tree nodes. Its layout, not
+ // the presence of a host tier, determines whether tombstones gate reuse.
+ let swa_req_ring = self.swa_req_ring;
let mut contiguous_len = usize::MAX;
Box::new(move |tree_core: &UnifiedTreeCore, node_id: NodeIdx_| {
let node = tree_core.arena.node(node_id);
@@ -406,7 +397,7 @@ impl TreeComponent for SwaComponent {
// — load_back will restore SWA from host before use.
if !node.has_device_value(SWA) && (match_device_only || !node.has_host_value(SWA)) {
contiguous_len = 0;
- return swa_device_only_hicache && (node.backuped() || !node.evicted());
+ return swa_req_ring && (node.backuped() || !node.evicted());
}
contiguous_len = contiguous_len.saturating_add(node.key.atom_len());
contiguous_len >= sliding_window_size
@@ -771,6 +762,13 @@ impl TreeComponent for SwaComponent {
tree_core.component_state(SWA).is_evict_device_ongoing,
"Swa device eviction not started"
);
+ assert!(
+ tree_core
+ .component_state(SWA)
+ .evict_device_pending_node
+ .is_none(),
+ "finish the pending internal SWA eviction before advancing"
+ );
let mut cursor = tree_core.component_state(SWA).evict_device_cursor;
// The cursor is re-validated (reset to LRU head) if the previous
// node's eviction removed it.
@@ -779,15 +777,15 @@ impl TreeComponent for SwaComponent {
.device_lru_list(SWA)
.get_lru_no_lock(&tree_core.arena);
}
- let next = loop {
+ let next = 'step: {
if tracker[&ct] >= tree_core.component_state(SWA).evict_device_request_cnt {
- break None;
+ break 'step None;
}
let Some(x) = cursor else {
- break None;
+ break 'step None;
};
if !tree_core.device_lru_list(SWA).in_list(Some(x)) {
- break None;
+ break 'step None;
}
assert!(
tree_core.arena.has_device_value(x, SWA),
@@ -796,14 +794,35 @@ impl TreeComponent for SwaComponent {
cursor = tree_core
.device_lru_list(SWA)
.get_prev_no_lock(x, &tree_core.arena);
- // A load-back pin means an in-flight DMA targets this node's slices.
- if tree_core.arena.node(x).is_load_back_pending() {
- continue;
- }
+ // The SWA LRU filters its own transfer/request locks. A Full load
+ // pin can also cover an ancestor outside the loaded SWA window,
+ // whose separately restored SWA rows remain independently evictable.
if tree_core.evictable_device_leaves.contains(x) {
- break Some(x);
+ break 'step Some(x);
+ }
+ let node = tree_core.arena.node(x);
+ if tree_core.enable_hicache
+ && tree_core.is_write_back
+ && tree_core.has_swa_host_pool
+ && !node.has_host_value(SWA)
+ && !node.has_host_value(FULL)
+ && node.has_device_value(FULL)
+ {
+ // Backup may include several unbacked ancestors. Let the
+ // controller reserve that whole window outside the core lock.
+ let needed = self
+ .collect_unbacked_swa_nodes_(tree_core, x)
+ .iter()
+ .map(|&idx| tree_core.arena.device_value(idx, SWA).numel())
+ .sum();
+ if needed > 0 {
+ let node_id = node.id;
+ let state = tree_core.component_state_mut(SWA);
+ state.evict_device_pending_node = Some(node_id);
+ state.evict_device_pending_num_tokens = needed;
+ break 'step None;
+ }
}
- // Internal nodes are tombstoned inline (no IO).
tree_core.evict_component_and_detach_lru_(
x,
ct,
@@ -813,7 +832,7 @@ impl TreeComponent for SwaComponent {
Some(tracker),
);
tree_core.cascade_evict_(x, ct, tracker, device_frees, host_frees, EvictLayer::Device);
- break None;
+ None
};
tree_core.component_state_mut(SWA).evict_device_cursor = cursor;
next
@@ -823,41 +842,6 @@ impl TreeComponent for SwaComponent {
tree_core.set_evict_device_end(SWA);
}
- fn reclaim_coexisting_host_values(
- &self,
- tree_core: &mut UnifiedTreeCore,
- num_tokens: usize,
- tracker: &mut HashMap,
- device_frees: &mut HashMap>,
- host_frees: &mut HashMap>,
- ) {
- for spare_imminent_demotes in [true, false] {
- if tracker[&SWA] >= num_tokens {
- break;
- }
- let mut next = Self::next_host_unlocked_device_lru_node(tree_core, None);
- while let Some(node_id) = next {
- if tracker[&SWA] >= num_tokens {
- break;
- }
- next = Self::next_host_unlocked_device_lru_node(tree_core, Some(node_id));
- if spare_imminent_demotes && tree_core.evictable_device_leaves.contains(node_id) {
- continue;
- }
- if !tree_core.can_reclaim_coexisting_host_value_(node_id, SWA) {
- continue;
- }
- tree_core.release_coexisting_host_value_(
- node_id,
- SWA,
- tracker,
- device_frees,
- host_frees,
- );
- }
- }
- }
-
/// Evict SWA host resources.
/// Internal nodes: private tombstone (free SWA host only).
/// Host leaves: atomic eviction via _evict_host_leaf.
@@ -886,11 +870,8 @@ impl TreeComponent for SwaComponent {
let x_next = tree_core
.host_lru_list(SWA)
.get_prev_no_lock(cur, &tree_core.arena);
- // A load-back pin means an in-flight DMA reads this node's host slices.
- if tree_core.arena.node(cur).is_load_back_pending() {
- x = x_next;
- continue;
- }
+ // The host LRU filters SWA's own locks. A Full load-back pin may
+ // cover an ancestor outside the SWA transfer window.
if tree_core.evictable_host_leaves.contains(cur) {
tree_core.evict_host_leaf_(cur, tracker, device_frees, host_frees);
} else {
diff --git a/rust/sglang-radix-tree/src/node.rs b/rust/sglang-radix-tree/src/node.rs
index c157d326f5..cc53e62276 100644
--- a/rust/sglang-radix-tree/src/node.rs
+++ b/rust/sglang-radix-tree/src/node.rs
@@ -177,9 +177,12 @@ pub struct Node {
pub hash_value: Option>,
/// Whether this node is available through the direct external-cache linker.
pub external_cache_stored: bool,
+ /// Chain-wide logical-page owner rotation, retained across node splits.
+ pub rotation_base: Option,
/// The in-flight write-through backup's ack id.
pub write_through_pending_id: Option,
- /// Load-back anchor currently reading this node's host slots.
+ /// Load-back anchor currently reading this node's Full host slots.
+ /// Auxiliary transfers are protected by their component locks.
pub load_back_pending_id: Option,
/// Monotonic access tick for LRU ordering (exact; not wall-clock).
pub last_access_counter: i64,
@@ -187,6 +190,10 @@ pub struct Node {
pub creation_counter: i64,
/// Match hits accumulated for write-through and LFU decisions.
pub hit_count: i64,
+ /// Logical root-path length; maintained only for T-LRU.
+ pub tlru_cached_prefix_len: usize,
+ /// Greatest subtree depth reached, retained when cached tails are evicted.
+ pub tlru_history_len: usize,
/// Eviction priority; the root uses `i64::MIN` and is never a leaf.
pub priority: i64,
/// This node's external handle; minted once, never recycled.
@@ -380,7 +387,7 @@ impl Node {
.any(|state| state.lock_ref > 0)
}
- /// Whether an in-flight load-back currently pins this node.
+ /// Whether an in-flight load-back currently pins this node's Full slots.
pub fn is_load_back_pending(&self) -> bool {
self.load_back_pending_id.is_some()
}
@@ -399,11 +406,14 @@ impl Node {
swa_host_uuid: None,
hash_value: Some(Vec::new()),
external_cache_stored: false,
+ rotation_base: None,
write_through_pending_id: None,
load_back_pending_id: None,
last_access_counter: 0,
creation_counter: 0,
hit_count: 0,
+ tlru_cached_prefix_len: 0,
+ tlru_history_len: 0,
priority: i64::MIN,
id,
idx: NodeIdx_(id),
@@ -422,11 +432,14 @@ impl Node {
swa_host_uuid: None,
hash_value: None,
external_cache_stored: false,
+ rotation_base: None,
write_through_pending_id: None,
load_back_pending_id: None,
last_access_counter: 0,
creation_counter: 0,
hit_count: 0,
+ tlru_cached_prefix_len: 0,
+ tlru_history_len: 0,
priority,
id,
idx: NodeIdx_(id),
@@ -1557,68 +1570,135 @@ impl NodeArena {
}
}
-// Eviction-eligible node set.
+// Shared node membership for eviction leaves and Full host duplicates.
-/// Set of `NodeIdx_`s with O(1) membership ops and dense iteration.
-#[derive(Default)]
-pub struct EvictableNodeSet {
- /// Dense member list; order is unspecified (swap-remove).
- nodes: Vec,
- /// Each member's position in `nodes`, indexed by `NodeIdx_`.
- slots: Vec