[Radix Cache] Sync Rust TreeCore and make it the default (#39627)

Co-authored-by: Ke Bao <ispobaoke@gmail.com>
Co-authored-by: Zhangheng <hzh0425@apache.org>
Co-authored-by: alphabetc1 <alphabetc1@users.noreply.github.com>
This commit is contained in:
Jialin Ouyang
2026-09-29 10:18:57 +08:00
committed by GitHub
co-authored by Ke Bao Zhangheng alphabetc1
parent 49d26ff0c2
commit 77091cea68
42 changed files with 8065 additions and 1020 deletions
+11 -2
View File
@@ -677,8 +677,17 @@ class Envs:
SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS = EnvBool(True)
# Decode batches between SWA out-of-window evictions.
SGLANG_SWA_EVICTION_INTERVAL = EnvInt(128)
# Registered TreeCore backend serving the unified radix cache.
SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND = EnvStr("python")
# The tree-core registry falls back to Python for:
# - Session-aware caching.
# - C128 or other unsupported components.
# - Custom component overrides.
# - Non-Linux platforms.
# - Unsupported PyTorch versions.
# - Devices other than CPU or CUDA.
# - Installs with neither the Rust extension nor its sources.
# - Source builds with a missing or unusable Rust toolchain.
# This also applies when Rust is explicitly selected.
SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND = EnvStr("rust")
SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW = EnvBool(False)
# ===================================================================
@@ -3,6 +3,7 @@
from __future__ import annotations
import sys
from array import array
from typing import TYPE_CHECKING, Optional, Sequence
@@ -14,6 +15,8 @@ from sglang.srt.disaggregation.kv_events import (
BlockStored,
StorageMedium,
)
from sglang.srt.environ import envs
from sglang.srt.mem_cache.allocator.swa import is_swa_req_ring
from sglang.srt.mem_cache.base_prefix_cache import (
DecLockRefParams,
DecLockRefResult,
@@ -39,6 +42,7 @@ from sglang.srt.mem_cache.unified_cache.cache_action import (
SWARebuild,
)
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
from sglang.srt.mem_cache.unified_cache.components import CacheTransferPhase
from sglang.srt.mem_cache.unified_cache.unified_tree_core import StorageBackupSpec
from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import (
BufferBackupSnapshot,
@@ -54,6 +58,7 @@ from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import (
RadixCacheWalkResult,
UnifiedTreeCoreInterface,
)
from sglang.srt.mem_cache.utils import get_eviction_strategy
from sglang.srt.runtime_context import get_exec, mamba_cache_chunk_size
from sglang.srt.utils import assert_int64_array
@@ -65,10 +70,23 @@ if TYPE_CHECKING:
CacheAction,
ComponentAction,
)
from sglang.srt.mem_cache.unified_cache.components import CacheTransferPhase
from sglang.srt.mem_cache.unified_cache.unified_tree_core import UnifiedTreeNode
def _tlru_float_config(native_bindings, threshold, next_prompt_estimate):
"""Pass numeric values without changing Python's integer/float distinction."""
if not all(
isinstance(value, (int, float)) for value in (threshold, next_prompt_estimate)
):
raise TypeError("T-LRU parameters must be integer or floating-point numbers")
threshold = float(threshold)
if isinstance(next_prompt_estimate, int):
return native_bindings.TlruFloatConfig(
threshold, 0.0, integer_estimate=next_prompt_estimate
)
return native_bindings.TlruFloatConfig(threshold, float(next_prompt_estimate))
def _radix_key_buffer(key: RadixKey) -> array:
"""The key's token ids honoring `limit`; view-independent since the
binding derives its own atoms."""
@@ -231,6 +249,7 @@ def _insert_step_from_binding(step) -> InsertStepResult:
mamba_exist=step.result.mamba_exist,
swa_branch_inserted=step.result.swa_branch_inserted,
host_insert_dropped=step.result.host_insert_dropped,
rotation_tail_declined=step.result.rotation_tail_declined,
adopted_ranges=(
{
ComponentType(component_type): list(ranges)
@@ -297,6 +316,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
"""A TreeCore backed by the Rust extension binding."""
_bindings = bindings
supports_rotation_base = True
def __init__(self, params: CacheInitParams):
assert params.tree_components is not None
@@ -324,12 +344,25 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
raise ValueError(
"Rust TreeCore does not support component_registry_override"
)
# The Rust core builds its own eviction strategy from the policy name
# alone, so a config would be dropped rather than applied.
if params.eviction_policy_config:
raise ValueError(
"Rust TreeCore does not support --radix-eviction-policy-config"
)
# Validate the same constructor options as Python before passing the
# configured eviction parameters to the native strategy.
eviction_strategy = get_eviction_strategy(
params.eviction_policy, params.eviction_policy_config
)
tlru_tail_budget = 0
tlru_float_config = None
if params.eviction_policy.lower() == "tlru":
threshold = eviction_strategy.threshold
next_prompt_estimate = eviction_strategy.next_prompt_estimate
if isinstance(threshold, int) and isinstance(next_prompt_estimate, int):
# Subtract before clamping to preserve arbitrary-size integers.
tlru_tail_budget = min(
max(threshold - next_prompt_estimate, 0), 2 * sys.maxsize + 1
)
else:
tlru_float_config = _tlru_float_config(
self._bindings, threshold, next_prompt_estimate
)
if ComponentType.SWA in self.tree_components and (
params.sliding_window_size is None or params.sliding_window_size <= 0
):
@@ -342,8 +375,9 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
params.is_eagle and ComponentType.MAMBA not in self.tree_components
)
# ``device`` is derived from the construction-time allocator; the
# allocator/pool themselves are owned by the cache, not the tree.
# The cache owns allocation; keep its allocator for resolving current
# unified SWA addresses when building a host backup.
self._allocator = params.token_to_kv_pool_allocator
if params.token_to_kv_pool_allocator:
device = torch.device(params.token_to_kv_pool_allocator.device)
# A bare "cuda" means the process's current device, not cuda:0.
@@ -362,12 +396,18 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
self._binding = self._binding_class()(
self._bindings.TreeCoreInitParamsBinding(
eviction_policy=params.eviction_policy,
slru_protected_threshold=getattr(
eviction_strategy, "protected_threshold", 2
),
tlru_tail_budget=tlru_tail_budget,
tlru_float_config=tlru_float_config,
page_size=params.page_size,
is_write_back=False,
enable_hicache=False,
write_through_threshold=256,
device=str(self.device),
swa_sliding_window_size=params.sliding_window_size,
swa_req_ring=is_swa_req_ring(self._allocator),
enable_kv_cache_events=params.enable_kv_cache_events,
mamba_cache_chunk_size=(
mamba_cache_chunk_size() if has_mamba else None
@@ -415,9 +455,14 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
def swa_tombstone_ranges(
self, key: RadixKey, start: int, end: int
) -> list[tuple[int, int]]:
raise NotImplementedError(
"swa_tombstone_ranges: buffer-mode SWA window repair is not yet "
"ported to the Rust tree core"
return self._binding.swa_tombstone_ranges(
self._bindings.MatchParamsBinding(
key=_radix_key_buffer(key),
extra_key=key.extra_key,
cache_salt=key.cache_salt,
),
start,
end,
)
def attach_swa_window(
@@ -426,10 +471,18 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
window_start: int,
window_end: int,
swa_values: torch.Tensor,
) -> list:
raise NotImplementedError(
"attach_swa_window: buffer-mode SWA window repair is not yet "
"ported to the Rust tree core"
) -> list[CacheAction | ComponentAction]:
return _cache_actions_from_tagged(
self._binding.attach_swa_window(
self._bindings.MatchParamsBinding(
key=_radix_key_buffer(key),
extra_key=key.extra_key,
cache_salt=key.cache_salt,
),
window_start,
window_end,
swa_values,
)
)
def inc_lock_ref(
@@ -453,6 +506,24 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
)
return DecLockRefResult()
def dec_window_lock_only(
self,
node_id: NodeId,
component_type: ComponentType,
params: DecLockRefParams,
) -> DecSwaLockOnlyResult:
result = DecSwaLockOnlyResult()
new_device_frees, new_host_frees = self._binding.dec_window_lock_only(
node_id,
int(component_type),
_dec_lock_ref_params_to_binding(self._bindings, params),
)
for component, tensors in new_device_frees.items():
result.device_frees[ComponentType(component)].extend(tensors)
for component, tensors in new_host_frees.items():
result.host_frees[ComponentType(component)].extend(tensors)
return result
def dec_swa_lock_only(
self,
node_id: NodeId,
@@ -484,20 +555,43 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
result = EvictDeviceNextNodeResult(
node_id=binding_result.node_id,
made_progress=binding_result.made_progress,
unbacked_tokens=binding_result.unbacked_tokens,
mamba_backup_node_id=binding_result.mamba_backup_node_id,
swa_backup_node_id=binding_result.swa_backup_node_id,
swa_backup_num_tokens=binding_result.swa_backup_num_tokens,
)
return _fill_evict_result(binding_result, result)
def finish_mamba_state_eviction(self, node_id: NodeId) -> EvictDeviceNextNodeResult:
binding_result = self._binding.finish_mamba_state_eviction(node_id)
result = EvictDeviceNextNodeResult(
node_id=binding_result.node_id,
made_progress=binding_result.made_progress,
unbacked_tokens=binding_result.unbacked_tokens,
)
return _fill_evict_result(binding_result, result)
def finish_swa_state_eviction(self, node_id: NodeId) -> EvictDeviceNextNodeResult:
binding_result = self._binding.finish_swa_state_eviction(node_id)
result = EvictDeviceNextNodeResult(
node_id=binding_result.node_id,
made_progress=binding_result.made_progress,
unbacked_tokens=binding_result.unbacked_tokens,
)
return _fill_evict_result(binding_result, result)
def evict_device_leaf(
self, node_id: NodeId, is_write_back: bool
) -> EvictDeviceLeafResult:
# The binding reads is_write_back from the core's construction config.
# The binding reads is_write_back from the core's current config.
assert is_write_back == self.is_write_back, (
"is_write_back must match the core's construction config"
"is_write_back must match the core's current config"
)
binding_result = self._binding.evict_device_leaf(node_id)
backup = binding_result.backup_kv
result = EvictDeviceLeafResult(
backup_kv=_cache_action_from_tagged(backup) if backup is not None else None
unbacked_tokens=binding_result.unbacked_tokens,
backup_kv=_cache_action_from_tagged(backup) if backup is not None else None,
)
return _fill_evict_result(binding_result, result)
@@ -529,6 +623,9 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
def component_evictable_size(self, component_type: ComponentType) -> int:
return self._binding.component_evictable_size(int(component_type))
def component_protected_size(self, component_type: ComponentType) -> int:
return self._binding.component_protected_size(int(component_type))
def full_evictable_size(self) -> int:
return self._binding.full_evictable_size()
@@ -632,6 +729,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
chunked=params.chunked,
priority=0 if params.priority is None else params.priority,
track_adopted_ranges=params.track_adopted_ranges,
rotation_base=params.rotation_base,
)
)
return _insert_step_from_binding(step)
@@ -649,7 +747,9 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
self, component_type: ComponentType, num_tokens: int
) -> DriveHostEvictionResult:
binding_result = self._binding.drive_host_eviction(
int(component_type), num_tokens
int(component_type),
num_tokens,
envs.SGLANG_HICACHE_SKIP_HOST_DUPLICATE_RECLAIM.get(),
)
return _fill_evict_result(binding_result, DriveHostEvictionResult())
@@ -761,7 +861,31 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
self, node_id: NodeId
) -> tuple[torch.Tensor, dict[ComponentType, list[PoolTransfer]]]:
device_value, comp_xfers = self._binding.build_backup_spec(node_id)
return device_value, _comp_xfers_from_binding(comp_xfers)
comp_xfers = _comp_xfers_from_binding(comp_xfers)
self._refresh_swa_backup_indices(comp_xfers.get(ComponentType.SWA, ()))
return device_value, comp_xfers
def _refresh_swa_backup_indices(self, transfers: Sequence[PoolTransfer]) -> None:
if not transfers:
return
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
UnifiedSWAAllocatorBase,
)
if not isinstance(self._allocator, UnifiedSWAAllocatorBase):
return
for transfer in transfers:
if transfer.name != PoolName.SWA or not transfer.nodes_to_load:
continue
# Compaction can relocate SWA rows after the tree records them.
# FULL virtual IDs retain their meaning across that relocation.
full_values = [
self.get_component_device_value(node_id, ComponentType.FULL)
for node_id in transfer.nodes_to_load
]
transfer.device_indices = self._allocator.translate_loc_from_full_to_swa(
torch.cat(full_values)
).to(torch.int64)
def build_storage_backup_spec(
self, node_id: NodeId, pass_prefix_keys: bool
@@ -805,7 +929,10 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
)
if transfers is None:
return None
return [_transfer_from_binding(transfer) for transfer in transfers]
transfers = [_transfer_from_binding(transfer) for transfer in transfers]
if phase == CacheTransferPhase.BACKUP_HOST:
self._refresh_swa_backup_indices(transfers)
return transfers
def build_load_back_spec(
self, node_id: NodeId, req: Optional[Req] = None
@@ -815,7 +942,30 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
kv_xfer, comp_xfers = self._binding.build_load_back_spec(
node_id, mamba_pool_idx
)
return _transfer_from_binding(kv_xfer), _comp_xfers_from_binding(comp_xfers)
kv_xfer = _transfer_from_binding(kv_xfer)
comp_xfers = _comp_xfers_from_binding(comp_xfers)
swa_xfers = comp_xfers.get(ComponentType.SWA, ())
if swa_xfers:
full_node_ids = kv_xfer.nodes_to_load or []
full_load_slices = {}
offset = 0
for full_node_id, count in zip(
full_node_ids, self._binding.get_node_key_lengths(full_node_ids)
):
full_load_slices[full_node_id] = slice(offset, offset + count)
offset += count
for transfer in swa_xfers:
# SWA may have holes between resident nodes, or reload while
# FULL stays resident. Preserve the SWA transfer's node order.
transfer.anchor_index_parts = [
(
full_load_slices[nid]
if nid in full_load_slices
else self.get_component_device_value(nid, ComponentType.FULL)
)
for nid in transfer.nodes_to_load or ()
]
return kv_xfer, comp_xfers
def prefetch_anchor_info(
self, node_id: NodeId
@@ -878,6 +1028,9 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
def root_node_handle(self, extra_key: Optional[str] = None) -> NodeId:
return self._binding.root_node_handle(extra_key)
def rotation_base_of(self, node_id: NodeId) -> Optional[int]:
return self._binding.rotation_base_of(node_id)
def dfs_weight_order(self, node_ids: Sequence[NodeId]) -> list[int]:
return self._binding.dfs_weight_order(list(node_ids))
@@ -6,6 +6,7 @@ from sglang.srt.mem_cache.unified_cache.components.base import (
ComponentType,
EvictLayer,
ExternalLinkerLoadPhase,
InternalStateBackup,
LinkerTransferPhase,
LRURefreshPhase,
PrepareLoadBackResult,
@@ -23,6 +24,7 @@ __all__ = [
"ComponentData",
"ComponentType",
"ExternalLinkerLoadPhase",
"InternalStateBackup",
"LinkerTransferPhase",
"EvictLayer",
"FullComponent",
@@ -63,6 +63,15 @@ class EvictLayer(IntFlag):
ALL = DEVICE | HOST
@dataclasses.dataclass(frozen=True)
class InternalStateBackup:
"""Pause eviction for a host backup before freeing internal device state."""
node_id: NodeId
# Host capacity required by the backup, in this component's pool units.
num_tokens: int
@dataclasses.dataclass(frozen=True)
class PrepareLoadBackResult:
"""Outcome of prepare_load_back; default = nothing to prepare."""
@@ -522,11 +531,12 @@ class TreeComponent(ABC):
tracker: dict[ComponentType, int],
device_frees: dict[ComponentType, list[torch.Tensor]],
host_frees: dict[ComponentType, list[torch.Tensor]],
) -> Optional[NodeId]:
"""Advance one eviction step and return a device leaf, if selected.
) -> NodeId | InternalStateBackup | None:
"""Return a device leaf, an internal backup request, or no selection.
Implementations must return after one allocator-relevant internal
mutation so the caller can drain pending frees before continuing.
Backup requests leave device state intact until the core resumes eviction.
"""
assert self.is_evict_device_ongoing, (
f"{self.component_type} device eviction not started"
@@ -552,8 +562,8 @@ class TreeComponent(ABC):
tracker: dict[ComponentType, int],
device_frees: dict[ComponentType, list[torch.Tensor]],
host_frees: dict[ComponentType, list[torch.Tensor]],
) -> Optional[NodeId]:
"""Advance the walk by at most one allocator-relevant mutation."""
) -> NodeId | InternalStateBackup | None:
"""Select a leaf, request backup, or perform at most one internal mutation."""
...
@abstractmethod
@@ -30,6 +30,7 @@ from sglang.srt.mem_cache.unified_cache.components.base import (
CacheTransferPhase,
ComponentType,
EvictLayer,
InternalStateBackup,
LinkerTransferPhase,
LRURefreshPhase,
PrepareLoadBackResult,
@@ -369,7 +370,7 @@ class MambaComponent(TreeComponent):
tracker: dict[ComponentType, int],
device_frees: dict[ComponentType, list[torch.Tensor]],
host_frees: dict[ComponentType, list[torch.Tensor]],
) -> Optional[NodeId]:
) -> NodeId | InternalStateBackup | None:
"""Advance one device-eviction step and return a leaf, if selected.
An internal tombstone is one complete step so the caller can apply its
@@ -404,9 +405,18 @@ class MambaComponent(TreeComponent):
)
return x.id
if not enabled:
x_next = lru.get_prev_no_lock(x)
# write_back: demote the state to host before the internal tombstone.
self._maybe_backup_node_before_state_tombstone(x)
self._evict_device_cursor = lru.get_prev_no_lock(x)
cd = x.component_data[ct]
if (
self.tree_core.enable_hicache
and self.tree_core.is_write_back
and cd.host_value is None
and not x.backuped
and x.component_data[BASE_COMPONENT_TYPE].value is not None
):
# Keep the state live until the controller has attempted its host
# backup. Session cursors advance after the resumed tombstone.
return InternalStateBackup(node_id=x.id, num_tokens=1)
self.tree_core._evict_component_and_detach_lru(
x,
self,
@@ -418,38 +428,10 @@ class MambaComponent(TreeComponent):
self.tree_core._cascade_evict(
x, self, tracker, device_frees=device_frees, host_frees=host_frees
)
self._evict_device_cursor = lru.cursor_next() if enabled else x_next
if enabled:
self._evict_device_cursor = lru.cursor_next()
return None
def _maybe_backup_node_before_state_tombstone(self, node: UnifiedTreeNode) -> None:
"""Demote an internal node's mamba state to host before its tombstone
(write_back only), mirroring the leaf deferred-demote path.
The match validator passes only nodes holding the state on some
layer, so a dropped internal state caps the match frontier at this
node forever, leaving the subtree's still-resident KV unservable.
Best-effort: this walk must make progress (it satisfies an imminent
slot allocation), so any failure falls back to the legacy drop.
"""
cache = self.cache
cd = node.component_data[self.component_type]
if (
cache.cache_controller is None
or not cache.is_write_back
or cd.host_value is not None
or node.backuped
or node.component_data[BASE_COMPONENT_TYPE].value is None
):
return
# The backup executor pre-evicts only the KV host pool; make room
# for the state slot the way the PREFETCH hook does.
if (
self._mamba_pool_host is not None
and self._mamba_pool_host.available_size() < 1
):
cache.evict_host(1, self.component_type)
cache.backup_node_for_write_back(node.id)
def _evict_device_end(self) -> None:
"""Clear the device-eviction walk cursor state."""
if self.tree_core.enable_session_radix_cache:
@@ -36,6 +36,7 @@ from sglang.srt.mem_cache.unified_cache.components.base import (
ComponentType,
EvictLayer,
ExternalLinkerLoadPhase,
InternalStateBackup,
LinkerTransferPhase,
LRURefreshPhase,
PreparePrefetchResult,
@@ -711,7 +712,7 @@ class SWAComponent(TreeComponent):
tracker: dict[ComponentType, int],
device_frees: dict[ComponentType, list[torch.Tensor]],
host_frees: dict[ComponentType, list[torch.Tensor]],
) -> Optional[NodeId]:
) -> NodeId | InternalStateBackup | None:
"""Advance one device-eviction step and return a leaf, if selected.
An internal tombstone is one complete step so the caller can apply its
@@ -746,9 +747,33 @@ class SWAComponent(TreeComponent):
)
return x.id
if not enabled:
x_next = lru.get_prev_no_lock(x)
# write_back: demote the SWA KV to host before the internal tombstone.
self._maybe_backup_node_before_swa_tombstone(x)
self._evict_device_cursor = lru.get_prev_no_lock(x)
cd = x.component_data[ct]
if (
self.tree_core.enable_hicache
and self.tree_core.is_write_back
and self.tree_core.has_swa_host_pool
and cd.host_value is None
and not x.backuped
and x.component_data[BASE_COMPONENT_TYPE].value is not None
):
# Reserve the whole unbacked window, not just this victim, before
# its best-effort host backup and resumed internal tombstone.
needed = sum(
len(node.component_data[ct].value)
for node in self._collect_unbacked_swa_nodes(x)
)
if needed:
if ct == ComponentType.SWA:
return InternalStateBackup(node_id=x.id, num_tokens=needed)
# Custom SWA component types use Python's inherited inline
# path; the shared finish hook is for built-in SWA only.
if (
self._swa_kv_pool_host is not None
and self._swa_kv_pool_host.available_size() < needed
):
self.cache.evict_host(needed, ct)
self.cache.backup_node_for_write_back(x.id)
self.tree_core._evict_component_and_detach_lru(
x,
self,
@@ -760,48 +785,10 @@ class SWAComponent(TreeComponent):
self.tree_core._cascade_evict(
x, self, tracker, device_frees=device_frees, host_frees=host_frees
)
self._evict_device_cursor = lru.cursor_next() if enabled else x_next
if enabled:
self._evict_device_cursor = lru.cursor_next()
return None
def _maybe_backup_node_before_swa_tombstone(self, node: UnifiedTreeNode) -> None:
"""Demote an internal node's SWA KV to host before its tombstone
(write_back only), mirroring the leaf deferred-demote path.
The match validator treats an unbacked tombstone as a window reset,
so a dropped internal SWA segment caps the match frontier until a
full sliding window re-accumulates below it, leaving up to one
window of still-resident KV unservable. The leaf backup walk covers
ancestors only within one window of the evicted leaf, so an
internal node whose child spans the window arrives here unbacked.
Best-effort: this walk must make progress (it satisfies an imminent
allocation), so any failure falls back to the legacy drop.
"""
cache = self.cache
cd = node.component_data[self.component_type]
if (
cache.cache_controller is None
or not cache.is_write_back
or not self.tree_core.has_swa_host_pool
or cd.host_value is not None
or node.backuped
or node.component_data[BASE_COMPONENT_TYPE].value is None
):
return
# The backup executor pre-evicts only the KV host pool; make room
# in the SWA host pool the way the PREFETCH hook does.
needed = sum(
len(n.component_data[self.component_type].value)
for n in self._collect_unbacked_swa_nodes(node)
)
if needed == 0:
return
if (
self._swa_kv_pool_host is not None
and self._swa_kv_pool_host.available_size() < needed
):
cache.evict_host(needed, self.component_type)
cache.backup_node_for_write_back(node.id)
def _evict_device_end(self) -> None:
"""Clear the device-eviction walk cursor state."""
if self.tree_core.enable_session_radix_cache:
@@ -1,18 +1,28 @@
"""Registry for pluggable TreeCore implementations.
The unified cache constructs its TreeCore through `create_tree_core`, selected
by SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND (default "python"). To plug in a custom
implementation, register it under a string name via
`register_tree_core_backend(name, factory)`.
by its per-cache override or SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND (default
"rust"). Rust selections use centralized compatibility fallbacks. To plug in a
custom implementation, register it via `register_tree_core_backend(name, factory)`.
"""
from __future__ import annotations
import importlib.util
import logging
import re
import subprocess
import sys
from pathlib import Path
from typing import TYPE_CHECKING, Callable, Optional
import torch
from sglang.srt.environ import envs
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
if TYPE_CHECKING:
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
from sglang.srt.mem_cache.unified_cache.components import TreeComponent
from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import (
UnifiedTreeCoreInterface,
@@ -24,6 +34,101 @@ TreeCoreFactory = Callable[
]
_TREE_CORE_REGISTRY: dict[str, TreeCoreFactory] = {}
_RUST_TREE_CORE_MODULE = "sglang.srt.mem_cache.rust_tree_core.mem_cache"
_RUST_TREE_CORE_MANIFEST = (
Path(__file__).resolve().parents[5] / "rust" / "sglang-radix-tree" / "Cargo.toml"
)
logger = logging.getLogger(__name__)
def _rust_fallback_reason(params: CacheInitParams) -> Optional[str]:
if params.enable_session_radix_cache:
return "session-aware caching requires the Python TreeCore"
if params.tree_components is not None and set(params.tree_components) - {
ComponentType.FULL,
ComponentType.SWA,
ComponentType.MAMBA,
}:
return "the configured components require the Python TreeCore"
if params.component_registry_override:
return "custom components require the Python TreeCore"
if sys.platform != "linux":
return "the Rust TreeCore supports Linux only"
from sglang.srt.rust_extensions.torch_build import (
_MAX_SUPPORTED_TORCH,
_MIN_SUPPORTED_TORCH,
)
match = re.match(r"^(\d+)\.(\d+)", str(torch.__version__))
if match is None or not (
_MIN_SUPPORTED_TORCH
<= (int(match.group(1)), int(match.group(2)))
<= _MAX_SUPPORTED_TORCH
):
return f"PyTorch {torch.__version__} is outside the Rust TreeCore support range"
allocator = params.token_to_kv_pool_allocator
device = (
torch.device(allocator.device) if allocator is not None else torch.device("cpu")
)
if device.type not in ("cpu", "cuda"):
return f"the Rust TreeCore does not support device {device.type}"
mode = envs.SGLANG_RUST_BUILD_MODE.get()
if sys.modules.get(_RUST_TREE_CORE_MODULE) is not None or mode not in (
"auto",
"never",
"force",
):
# Preserve the loader's invalid-mode and force-after-import errors.
return None
workspace = _RUST_TREE_CORE_MANIFEST.parent.parent
bundled = importlib.util.find_spec(_RUST_TREE_CORE_MODULE) is not None
source_checkout = (workspace / "Cargo.toml").is_file()
if mode != "force" and (mode == "never" or not source_checkout) and bundled:
return None
if not _RUST_TREE_CORE_MANIFEST.is_file():
if bundled:
return "the requested Rust TreeCore source build has no sources"
return "this installation contains neither the Rust TreeCore extension nor its sources"
# Even a cached source build needs these version queries for its fingerprint.
# Match the loader's commands and cwd so rustup workspace overrides apply.
for command in (("cargo", "--version", "--verbose"), ("rustc", "-vV")):
try:
subprocess.run(
command,
cwd=workspace,
check=True,
capture_output=True,
text=True,
timeout=10,
)
except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired):
return f"the Rust TreeCore toolchain is unavailable ({' '.join(command)})"
return None
def resolve_tree_core_backend(name: str, params: CacheInitParams) -> str:
"""Resolve known Rust capability gaps before loading a backend.
Explicit Rust selections use the same compatibility policy as the default.
Build, import, and runtime failures remain errors when the toolchain is usable.
"""
if name != "rust":
return name
reason = _rust_fallback_reason(params)
if reason is not None:
logger.info("Using the Python TreeCore: %s", reason)
return "python"
return name
def select_tree_core_backend(params: CacheInitParams) -> str:
"""Resolve the instance override or default through the shared fallback policy."""
name = (
params.tree_core_backend
if params.tree_core_backend is not None
else envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.get()
)
return resolve_tree_core_backend(name, params)
def register_tree_core_backend(name: str, factory: TreeCoreFactory) -> None:
@@ -72,7 +177,8 @@ def create_tree_core(
params: CacheInitParams,
components: dict[ComponentType, TreeComponent],
) -> UnifiedTreeCoreInterface:
"""Construct the TreeCore registered under `name`."""
"""Resolve compatibility and construct the registered TreeCore."""
name = resolve_tree_core_backend(name, params)
factory = get_tree_core_factory(name)
if factory is None:
raise ValueError(
@@ -60,6 +60,7 @@ from sglang.srt.mem_cache.unified_cache.components import (
ComponentData,
ComponentType,
EvictLayer,
InternalStateBackup,
LinkerTransferPhase,
LRURefreshPhase,
TreeComponent,
@@ -474,6 +475,13 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
def reset(self) -> None:
"""Rebuild the root, LRUs, sizes, evictable-leaf sets, and the empty
match result."""
# End suspended walks while their session-cursor sentinels still
# belong to the old LRUs, clearing any pending internal victim.
for component in self.components:
if component.is_evict_device_ongoing:
component.evict_device_end()
# Internal victims awaiting the controller's backup and finish call.
self._pending_internal_evictions: dict[ComponentType, NodeId] = {}
# Maintains the NodeId -> active tree node mapping.
self._node_arena: dict[NodeId, UnifiedTreeNode] = {}
@@ -1586,12 +1594,15 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
result = EvictDeviceNextNodeResult()
# The walk reads running totals for its doneness check; the result
# carries only this step's delta.
component = self.components_by_type[component_type]
assert component_type not in self._pending_internal_evictions, (
f"finish the pending internal {component_type.name} eviction "
"before advancing"
)
updated_tracker = defaultdict(int, tracker)
self._begin_tracking_unbacked_tokens()
try:
result.node_id = self.components_by_type[
component_type
].evict_device_next_node(
step = component.evict_device_next_node(
updated_tracker, result.device_frees, result.host_frees
)
finally:
@@ -1600,12 +1611,99 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
delta = n - tracker.get(ct, 0)
if delta:
result.tracker[ct] = delta
result.made_progress = result.node_id is not None or bool(result.tracker)
if isinstance(step, InternalStateBackup):
assert component_type in (ComponentType.MAMBA, ComponentType.SWA)
self._pending_internal_evictions[component_type] = step.node_id
if component_type == ComponentType.MAMBA:
result.mamba_backup_node_id = step.node_id
else:
result.swa_backup_node_id = step.node_id
result.swa_backup_num_tokens = step.num_tokens
else:
result.node_id = step
result.made_progress = (
result.node_id is not None
or result.mamba_backup_node_id is not None
or result.swa_backup_node_id is not None
or bool(result.tracker)
)
return result
def finish_mamba_state_eviction(self, node_id: NodeId) -> EvictDeviceNextNodeResult:
return self._finish_internal_component_eviction(ComponentType.MAMBA, node_id)
def finish_swa_state_eviction(self, node_id: NodeId) -> EvictDeviceNextNodeResult:
return self._finish_internal_component_eviction(ComponentType.SWA, node_id)
def _finish_internal_component_eviction(
self, component_type: ComponentType, node_id: NodeId
) -> EvictDeviceNextNodeResult:
"""Resume an internal tombstone after the controller's backup attempt."""
component = self.components_by_type[component_type]
assert component.is_evict_device_ongoing, (
f"{component_type.name} device eviction not started"
)
assert self._pending_internal_evictions.get(component_type) == node_id, (
f"no matching pending internal {component_type.name} eviction"
)
del self._pending_internal_evictions[component_type]
# Consuming the pending request advances the walk even if I/O changed
# this victim's eligibility. No frees occur before backup completion.
result = EvictDeviceNextNodeResult(made_progress=True)
node = self._node_arena.get(node_id)
lru = self.lru_lists[component_type]
enabled = self.enable_session_radix_cache
if (
node is None
or node.component_data[component_type].value is None
or node.component_data[component_type].lock_ref > 0
or (
component_type == ComponentType.MAMBA
and node.load_back_pending_id is not None
)
or not lru.in_list(node)
):
if enabled:
component._evict_device_cursor = lru.cursor_next()
return result
if (
node in self.evictable_device_leaves
and (not enabled or component._can_evict_leaf_atomically(node))
) or (
component_type == ComponentType.MAMBA
and node.component_data[BASE_COMPONENT_TYPE].value is None
):
# Re-select changed leaves through the normal walk; internal
# finish must not accidentally perform an atomic Full eviction.
component._evict_device_cursor = node
return result
self._begin_tracking_unbacked_tokens()
try:
self._evict_component_and_detach_lru(
node,
component,
target=EvictLayer.DEVICE,
tracker=result.tracker,
device_frees=result.device_frees,
host_frees=result.host_frees,
)
self._cascade_evict(
node,
component,
result.tracker,
device_frees=result.device_frees,
host_frees=result.host_frees,
)
finally:
result.unbacked_tokens = self._finish_tracking_unbacked_tokens()
if enabled:
component._evict_device_cursor = lru.cursor_next()
return result
def evict_device_end(self, component_type: ComponentType) -> None:
"""Finish a component's device-eviction walk."""
self.components_by_type[component_type].evict_device_end()
self._pending_internal_evictions.pop(component_type, None)
def evict_device_leaf(
self, node_id: NodeId, is_write_back: bool
@@ -45,12 +45,18 @@ class EvictDeviceNextNodeResult(BaseEvictionResult):
``node_id`` selects a leaf for the Controller to evict. ``made_progress``
also covers an internal tombstone that returned no leaf, distinguishing it
from true walk exhaustion.
from true walk exhaustion. ``mamba_backup_node_id`` and
``swa_backup_node_id`` pause an internal eviction until the Controller
finishes its best-effort host backup. ``swa_backup_num_tokens`` includes
all unbacked SWA segments in the backup window, not just the victim.
"""
node_id: Optional[NodeId] = None
made_progress: bool = False
unbacked_tokens: int = 0
mamba_backup_node_id: Optional[NodeId] = None
swa_backup_node_id: Optional[NodeId] = None
swa_backup_num_tokens: int = 0
class EvictDeviceLeafResult(BaseEvictionResult):
@@ -319,6 +325,24 @@ class UnifiedTreeCoreInterface(ABC):
D->H backup must run (write_back) before the node can be demoted."""
...
@abstractmethod
def finish_mamba_state_eviction(self, node_id: NodeId) -> EvictDeviceNextNodeResult:
"""Resume the pending internal Mamba eviction after its backup attempt.
The controller must finish any submitted D->H transfer before calling.
A failed allocation still permits eviction to make device space.
"""
...
@abstractmethod
def finish_swa_state_eviction(self, node_id: NodeId) -> EvictDeviceNextNodeResult:
"""Resume the pending internal SWA eviction after its backup attempt.
The controller must finish any submitted D->H transfer before calling.
A failed allocation still permits eviction to make device space.
"""
...
@abstractmethod
def drop_subtree_no_host(self, node_id: NodeId) -> DropSubtreeNoHostResult:
"""Drop an unbacked D-leaf's subtree when its write-back backup failed
@@ -642,6 +666,29 @@ class UnifiedTreeCoreInterface(ABC):
host store event for each acked node."""
...
@abstractmethod
def swa_tombstone_ranges(
self, key: RadixKey, start: int, end: int
) -> list[tuple[int, int]]:
"""Return maximal missing SWA ranges within the matched [start, end) span."""
...
@abstractmethod
def attach_swa_window(
self,
key: RadixKey,
window_start: int,
window_end: int,
swa_values: torch.Tensor,
) -> list[CacheAction | ComponentAction]:
"""Attach a loaded SWA window to tombstoned spans, returning split actions.
The shared pipeline supplies page-aligned logical token offsets and
int64 values on the core's device. The entire span must be matched and
tombstoned before publication.
"""
...
@abstractmethod
def set_component_device_value(
self, node_id: NodeId, component_type: ComponentType, value: torch.Tensor
@@ -74,7 +74,10 @@ from sglang.srt.mem_cache.unified_cache.session_ref_tracker import (
UnifiedSessionRefTracker,
)
from sglang.srt.mem_cache.unified_cache.storage_attachment import StorageAttachment
from sglang.srt.mem_cache.unified_cache.tree_core_registry import create_tree_core
from sglang.srt.mem_cache.unified_cache.tree_core_registry import (
create_tree_core,
select_tree_core_backend,
)
from sglang.srt.mem_cache.unified_cache.unified_cache_linker import (
UnifiedCacheLinker,
UnifiedCacheLinkerWrapper,
@@ -204,11 +207,7 @@ class UnifiedRadixCache(BasePrefixCache):
)
# The TreeCore owns the tree member-var state (structure, LRUs, sizes,
# evictable leaves) and drives the components' tree-level hooks.
self._tree_core_backend = (
params.tree_core_backend
if params.tree_core_backend is not None
else envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.get()
)
self._tree_core_backend = select_tree_core_backend(params)
self.tree_core = create_tree_core(
name=self._tree_core_backend,
params=params,
@@ -756,6 +755,35 @@ class UnifiedRadixCache(BasePrefixCache):
) -> tuple[Optional[NodeId], bool]:
"""Advance the eviction walk one node, consuming its step result."""
result = self.tree_core.evict_device_next_node(component_type, tracker)
if result.mamba_backup_node_id is not None:
assert component_type == ComponentType.MAMBA and result.node_id is None
assert (
not result.device_frees and not result.host_frees and not result.tracker
)
# Reserve a host state slot and wait for the backup acknowledgment
# before freeing device state. If allocation fails, eviction still
# proceeds to make room on the device.
node_id = result.mamba_backup_node_id
mamba_host_pool = self.host_pool_group.get_pool(PoolName.MAMBA)
if mamba_host_pool is not None and mamba_host_pool.available_size() < 1:
self.evict_host(1, ComponentType.MAMBA)
self.backup_node_for_write_back(node_id)
result = self.tree_core.finish_mamba_state_eviction(node_id)
elif result.swa_backup_node_id is not None:
assert component_type == ComponentType.SWA and result.node_id is None
assert (
not result.device_frees and not result.host_frees and not result.tracker
)
# The backup can cover several unbacked SWA segments. Reserve host
# space for the whole window before copying it, then resume eviction
# even if host allocation fails.
node_id = result.swa_backup_node_id
needed = result.swa_backup_num_tokens
swa_host_pool = self.host_pool_group.get_pool(PoolName.SWA)
if swa_host_pool is not None and swa_host_pool.available_size() < needed:
self.evict_host(needed, ComponentType.SWA)
self.backup_node_for_write_back(node_id)
result = self.tree_core.finish_swa_state_eviction(node_id)
self._free_values(result.device_frees, result.host_frees)
if self._tracks_write_through_unbacked_evictions():
self._record_dropped_tokens(
@@ -887,8 +915,7 @@ class UnifiedRadixCache(BasePrefixCache):
def _tracks_write_through_unbacked_evictions(self) -> bool:
return (
isinstance(self.tree_core, UnifiedTreeCore)
and self.host_memory_mode == "cache"
self.host_memory_mode == "cache"
and self.cache_controller is not None
and self.cache_controller.write_policy == "write_through"
)
@@ -0,0 +1,11 @@
from sglang.srt.environ import envs
# Spawned server and scheduler processes import the scripted-runtime package
# before constructing their cache. Keep inspection in test processes while
# preserving the selected backend and its compatibility fallback.
if envs.SGLANG_TEST_SCRIPTED_RUNTIME.get():
from sglang.test.scripted_runtime.tree_core_inspection import (
install_tree_core_inspectors,
)
install_tree_core_inspectors()
@@ -1,8 +1,11 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Any, List
from typing import TYPE_CHECKING, Any
from sglang.test.scripted_runtime.context.radix import _node_lock_ref, to_node_handle
from sglang.test.scripted_runtime.context.radix import (
get_node_lock_ref,
iter_node_handles,
)
if TYPE_CHECKING:
from sglang.srt.managers.scheduler import Scheduler
@@ -12,9 +15,9 @@ class ScriptedLockRefExhauster:
def __init__(self, scheduler: Scheduler) -> None:
self.scheduler = scheduler
# (node, dec receipt) pairs; the receipt bounds the release walk.
self._locked: List[tuple[Any, Any]] = []
self._locked: list[tuple[Any, Any]] = []
def exhaust(self, *, leave_refs: int) -> None:
def exhaust(self, leave_refs: int) -> None:
tree_cache = self.scheduler.tree_cache
if tree_cache.disable:
return
@@ -25,9 +28,11 @@ class ScriptedLockRefExhauster:
return
target = evictable[0]
result = tree_cache.inc_lock_ref(to_node_handle(tree_cache, target))
result = tree_cache.inc_lock_ref(target)
newly_locked = [node for node in evictable if _node_lock_ref(node) > 0]
newly_locked = [
node for node in evictable if get_node_lock_ref(tree_cache, node) > 0
]
if not newly_locked:
return
self._locked.append((target, result.to_dec_params()))
@@ -35,15 +40,13 @@ class ScriptedLockRefExhauster:
def release(self) -> None:
tree_cache = self.scheduler.tree_cache
for node, dec_params in self._locked:
tree_cache.dec_lock_ref(to_node_handle(tree_cache, node), dec_params)
tree_cache.dec_lock_ref(node, dec_params)
self._locked.clear()
def _evictable_nodes(self) -> List[Any]:
evictable: List[Any] = []
stack = list(self.scheduler.tree_cache.root_node.children.values())
while stack:
node = stack.pop()
if _node_lock_ref(node) == 0:
evictable.append(node)
stack.extend(node.children.values())
return evictable
def _evictable_nodes(self) -> list[Any]:
tree_cache = self.scheduler.tree_cache
return [
node
for node in iter_node_handles(tree_cache)
if get_node_lock_ref(tree_cache, node) == 0
]
@@ -1,6 +1,7 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Any, Callable, Dict
from collections.abc import Iterator
from typing import TYPE_CHECKING, Any
from sglang.srt.mem_cache.unified_radix_cache import UnifiedTreeNode
@@ -8,12 +9,24 @@ if TYPE_CHECKING:
from sglang.test.scripted_runtime.context.api import ScriptedContext
def get_all_node_hit_counts(ctx: ScriptedContext) -> Dict[int, int]:
return _collect_node_attr(ctx, lambda node: node.hit_count)
def get_all_node_hit_counts(ctx: ScriptedContext) -> dict[int, int]:
tree_cache = ctx.scheduler.tree_cache
core = getattr(tree_cache, "tree_core", None)
return {
node if core is not None else node.id: (
core.get_node_hit_count(node) if core is not None else node.hit_count
)
for node in iter_node_handles(tree_cache)
}
def get_all_node_lock_refs(ctx: ScriptedContext) -> Dict[int, int]:
return _collect_node_attr(ctx, _node_lock_ref)
def get_all_node_lock_refs(ctx: ScriptedContext) -> dict[int, int]:
tree_cache = ctx.scheduler.tree_cache
core = getattr(tree_cache, "tree_core", None)
return {
node if core is not None else node.id: get_node_lock_ref(tree_cache, node)
for node in iter_node_handles(tree_cache)
}
def _node_lock_ref(node: Any) -> int:
@@ -22,37 +35,34 @@ def _node_lock_ref(node: Any) -> int:
return node.lock_ref
def resolve_node(tree_cache: Any, node_handle: Any) -> Any:
"""Resolve whatever a req or match result carries into a tree node.
``UnifiedRadixCache`` hands out NodeIds (ints); every other cache hands out
the node object itself. Returns None when the handle no longer maps to a
live node, which only happens once the node has been freed -- and a freed
node holds no locks.
"""
resolve = getattr(tree_cache, "resolve_node_handle", None)
if resolve is None:
return node_handle
try:
return resolve(node_handle)
except KeyError:
return None
def get_node_lock_ref(tree_cache: Any, node_handle: Any) -> int:
"""Read locks through the cache's native handle, including stale handles."""
if node_handle is None:
return 0
core = getattr(tree_cache, "tree_core", None)
if core is not None:
if not core.contains_node(node_handle):
return 0
return sum(
core.get_component_device_lock_ref(node_handle, component)
for component in tree_cache.tree_components
)
return _node_lock_ref(node_handle)
def to_node_handle(tree_cache: Any, node: Any) -> Any:
"""Inverse of `resolve_node`: what the tree_cache lock APIs accept."""
if isinstance(node, UnifiedTreeNode):
return node.id
return node
def _collect_node_attr(
ctx: ScriptedContext, get_value: Callable[[Any], int]
) -> Dict[int, int]:
values: Dict[int, int] = {}
stack = list(ctx.scheduler.tree_cache.root_node.children.values())
def iter_node_handles(tree_cache: Any) -> Iterator[Any]:
"""Walk non-root nodes as IDs for unified caches or native legacy nodes."""
core = getattr(tree_cache, "tree_core", None)
stack = (
list(core.get_child_node_ids(core.root_node_handle()))
if core is not None
else list(tree_cache.root_node.children.values())
)
while stack:
node = stack.pop()
values[node.id] = get_value(node)
stack.extend(node.children.values())
return values
yield node
stack.extend(
core.get_child_node_ids(node)
if core is not None
else node.children.values()
)
@@ -1,9 +1,9 @@
from __future__ import annotations
from dataclasses import dataclass
from typing import TYPE_CHECKING, Optional
from typing import TYPE_CHECKING
from sglang.test.scripted_runtime.context.radix import _node_lock_ref, resolve_node
from sglang.test.scripted_runtime.context.radix import get_node_lock_ref
if TYPE_CHECKING:
from sglang.srt.managers.schedule_batch import Req
@@ -16,7 +16,7 @@ class ScriptedReqHandle:
context: ScriptedContext
@property
def req(self) -> Optional[Req]:
def req(self) -> Req | None:
return self.context.find_req_by_rid(self.rid)
@property
@@ -52,7 +52,4 @@ class ScriptedReqHandle:
req = self.req
if req is None:
return 0
node = resolve_node(self.context.scheduler.tree_cache, req.last_node)
if node is None:
return 0
return _node_lock_ref(node)
return get_node_lock_ref(self.context.scheduler.tree_cache, req.last_node)
@@ -0,0 +1,34 @@
"""Use the shared TreeCore inspectors in scripted-runtime test processes."""
import sys
from pathlib import Path
def install_tree_core_inspectors():
from sglang.srt.mem_cache.unified_cache.tree_core_registry import (
_TREE_CORE_REGISTRY,
)
inspector_dir = str(
Path(__file__).resolve().parents[4]
/ "test"
/ "registered"
/ "unit"
/ "mem_cache"
)
if inspector_dir not in sys.path:
sys.path.insert(0, inspector_dir)
def python_inspector_factory(params, components):
from unified_tree_core_inspector import UnifiedTreeCoreInspector
return UnifiedTreeCoreInspector(params, components)
def rust_inspector_factory(params, _components):
from rust_unified_tree_core_inspector import RustUnifiedTreeCoreInspector
return RustUnifiedTreeCoreInspector(params)
_TREE_CORE_REGISTRY.update(
python=python_inspector_factory, rust=rust_inspector_factory
)