[Radix Cache] Sync Rust TreeCore and make it the default (#39627)
Co-authored-by: Ke Bao <ispobaoke@gmail.com> Co-authored-by: Zhangheng <hzh0425@apache.org> Co-authored-by: alphabetc1 <alphabetc1@users.noreply.github.com>
This commit is contained in:
co-authored by
Ke Bao
Zhangheng
alphabetc1
parent
49d26ff0c2
commit
77091cea68
@@ -677,8 +677,17 @@ class Envs:
|
||||
SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS = EnvBool(True)
|
||||
# Decode batches between SWA out-of-window evictions.
|
||||
SGLANG_SWA_EVICTION_INTERVAL = EnvInt(128)
|
||||
# Registered TreeCore backend serving the unified radix cache.
|
||||
SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND = EnvStr("python")
|
||||
# The tree-core registry falls back to Python for:
|
||||
# - Session-aware caching.
|
||||
# - C128 or other unsupported components.
|
||||
# - Custom component overrides.
|
||||
# - Non-Linux platforms.
|
||||
# - Unsupported PyTorch versions.
|
||||
# - Devices other than CPU or CUDA.
|
||||
# - Installs with neither the Rust extension nor its sources.
|
||||
# - Source builds with a missing or unusable Rust toolchain.
|
||||
# This also applies when Rust is explicitly selected.
|
||||
SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND = EnvStr("rust")
|
||||
SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW = EnvBool(False)
|
||||
|
||||
# ===================================================================
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from array import array
|
||||
from typing import TYPE_CHECKING, Optional, Sequence
|
||||
|
||||
@@ -14,6 +15,8 @@ from sglang.srt.disaggregation.kv_events import (
|
||||
BlockStored,
|
||||
StorageMedium,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.mem_cache.allocator.swa import is_swa_req_ring
|
||||
from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
DecLockRefParams,
|
||||
DecLockRefResult,
|
||||
@@ -39,6 +42,7 @@ from sglang.srt.mem_cache.unified_cache.cache_action import (
|
||||
SWARebuild,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
|
||||
from sglang.srt.mem_cache.unified_cache.components import CacheTransferPhase
|
||||
from sglang.srt.mem_cache.unified_cache.unified_tree_core import StorageBackupSpec
|
||||
from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import (
|
||||
BufferBackupSnapshot,
|
||||
@@ -54,6 +58,7 @@ from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import (
|
||||
RadixCacheWalkResult,
|
||||
UnifiedTreeCoreInterface,
|
||||
)
|
||||
from sglang.srt.mem_cache.utils import get_eviction_strategy
|
||||
from sglang.srt.runtime_context import get_exec, mamba_cache_chunk_size
|
||||
from sglang.srt.utils import assert_int64_array
|
||||
|
||||
@@ -65,10 +70,23 @@ if TYPE_CHECKING:
|
||||
CacheAction,
|
||||
ComponentAction,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_cache.components import CacheTransferPhase
|
||||
from sglang.srt.mem_cache.unified_cache.unified_tree_core import UnifiedTreeNode
|
||||
|
||||
|
||||
def _tlru_float_config(native_bindings, threshold, next_prompt_estimate):
|
||||
"""Pass numeric values without changing Python's integer/float distinction."""
|
||||
if not all(
|
||||
isinstance(value, (int, float)) for value in (threshold, next_prompt_estimate)
|
||||
):
|
||||
raise TypeError("T-LRU parameters must be integer or floating-point numbers")
|
||||
threshold = float(threshold)
|
||||
if isinstance(next_prompt_estimate, int):
|
||||
return native_bindings.TlruFloatConfig(
|
||||
threshold, 0.0, integer_estimate=next_prompt_estimate
|
||||
)
|
||||
return native_bindings.TlruFloatConfig(threshold, float(next_prompt_estimate))
|
||||
|
||||
|
||||
def _radix_key_buffer(key: RadixKey) -> array:
|
||||
"""The key's token ids honoring `limit`; view-independent since the
|
||||
binding derives its own atoms."""
|
||||
@@ -231,6 +249,7 @@ def _insert_step_from_binding(step) -> InsertStepResult:
|
||||
mamba_exist=step.result.mamba_exist,
|
||||
swa_branch_inserted=step.result.swa_branch_inserted,
|
||||
host_insert_dropped=step.result.host_insert_dropped,
|
||||
rotation_tail_declined=step.result.rotation_tail_declined,
|
||||
adopted_ranges=(
|
||||
{
|
||||
ComponentType(component_type): list(ranges)
|
||||
@@ -297,6 +316,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
"""A TreeCore backed by the Rust extension binding."""
|
||||
|
||||
_bindings = bindings
|
||||
supports_rotation_base = True
|
||||
|
||||
def __init__(self, params: CacheInitParams):
|
||||
assert params.tree_components is not None
|
||||
@@ -324,12 +344,25 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
raise ValueError(
|
||||
"Rust TreeCore does not support component_registry_override"
|
||||
)
|
||||
# The Rust core builds its own eviction strategy from the policy name
|
||||
# alone, so a config would be dropped rather than applied.
|
||||
if params.eviction_policy_config:
|
||||
raise ValueError(
|
||||
"Rust TreeCore does not support --radix-eviction-policy-config"
|
||||
)
|
||||
# Validate the same constructor options as Python before passing the
|
||||
# configured eviction parameters to the native strategy.
|
||||
eviction_strategy = get_eviction_strategy(
|
||||
params.eviction_policy, params.eviction_policy_config
|
||||
)
|
||||
tlru_tail_budget = 0
|
||||
tlru_float_config = None
|
||||
if params.eviction_policy.lower() == "tlru":
|
||||
threshold = eviction_strategy.threshold
|
||||
next_prompt_estimate = eviction_strategy.next_prompt_estimate
|
||||
if isinstance(threshold, int) and isinstance(next_prompt_estimate, int):
|
||||
# Subtract before clamping to preserve arbitrary-size integers.
|
||||
tlru_tail_budget = min(
|
||||
max(threshold - next_prompt_estimate, 0), 2 * sys.maxsize + 1
|
||||
)
|
||||
else:
|
||||
tlru_float_config = _tlru_float_config(
|
||||
self._bindings, threshold, next_prompt_estimate
|
||||
)
|
||||
if ComponentType.SWA in self.tree_components and (
|
||||
params.sliding_window_size is None or params.sliding_window_size <= 0
|
||||
):
|
||||
@@ -342,8 +375,9 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
params.is_eagle and ComponentType.MAMBA not in self.tree_components
|
||||
)
|
||||
|
||||
# ``device`` is derived from the construction-time allocator; the
|
||||
# allocator/pool themselves are owned by the cache, not the tree.
|
||||
# The cache owns allocation; keep its allocator for resolving current
|
||||
# unified SWA addresses when building a host backup.
|
||||
self._allocator = params.token_to_kv_pool_allocator
|
||||
if params.token_to_kv_pool_allocator:
|
||||
device = torch.device(params.token_to_kv_pool_allocator.device)
|
||||
# A bare "cuda" means the process's current device, not cuda:0.
|
||||
@@ -362,12 +396,18 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
self._binding = self._binding_class()(
|
||||
self._bindings.TreeCoreInitParamsBinding(
|
||||
eviction_policy=params.eviction_policy,
|
||||
slru_protected_threshold=getattr(
|
||||
eviction_strategy, "protected_threshold", 2
|
||||
),
|
||||
tlru_tail_budget=tlru_tail_budget,
|
||||
tlru_float_config=tlru_float_config,
|
||||
page_size=params.page_size,
|
||||
is_write_back=False,
|
||||
enable_hicache=False,
|
||||
write_through_threshold=256,
|
||||
device=str(self.device),
|
||||
swa_sliding_window_size=params.sliding_window_size,
|
||||
swa_req_ring=is_swa_req_ring(self._allocator),
|
||||
enable_kv_cache_events=params.enable_kv_cache_events,
|
||||
mamba_cache_chunk_size=(
|
||||
mamba_cache_chunk_size() if has_mamba else None
|
||||
@@ -415,9 +455,14 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
def swa_tombstone_ranges(
|
||||
self, key: RadixKey, start: int, end: int
|
||||
) -> list[tuple[int, int]]:
|
||||
raise NotImplementedError(
|
||||
"swa_tombstone_ranges: buffer-mode SWA window repair is not yet "
|
||||
"ported to the Rust tree core"
|
||||
return self._binding.swa_tombstone_ranges(
|
||||
self._bindings.MatchParamsBinding(
|
||||
key=_radix_key_buffer(key),
|
||||
extra_key=key.extra_key,
|
||||
cache_salt=key.cache_salt,
|
||||
),
|
||||
start,
|
||||
end,
|
||||
)
|
||||
|
||||
def attach_swa_window(
|
||||
@@ -426,10 +471,18 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
window_start: int,
|
||||
window_end: int,
|
||||
swa_values: torch.Tensor,
|
||||
) -> list:
|
||||
raise NotImplementedError(
|
||||
"attach_swa_window: buffer-mode SWA window repair is not yet "
|
||||
"ported to the Rust tree core"
|
||||
) -> list[CacheAction | ComponentAction]:
|
||||
return _cache_actions_from_tagged(
|
||||
self._binding.attach_swa_window(
|
||||
self._bindings.MatchParamsBinding(
|
||||
key=_radix_key_buffer(key),
|
||||
extra_key=key.extra_key,
|
||||
cache_salt=key.cache_salt,
|
||||
),
|
||||
window_start,
|
||||
window_end,
|
||||
swa_values,
|
||||
)
|
||||
)
|
||||
|
||||
def inc_lock_ref(
|
||||
@@ -453,6 +506,24 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
)
|
||||
return DecLockRefResult()
|
||||
|
||||
def dec_window_lock_only(
|
||||
self,
|
||||
node_id: NodeId,
|
||||
component_type: ComponentType,
|
||||
params: DecLockRefParams,
|
||||
) -> DecSwaLockOnlyResult:
|
||||
result = DecSwaLockOnlyResult()
|
||||
new_device_frees, new_host_frees = self._binding.dec_window_lock_only(
|
||||
node_id,
|
||||
int(component_type),
|
||||
_dec_lock_ref_params_to_binding(self._bindings, params),
|
||||
)
|
||||
for component, tensors in new_device_frees.items():
|
||||
result.device_frees[ComponentType(component)].extend(tensors)
|
||||
for component, tensors in new_host_frees.items():
|
||||
result.host_frees[ComponentType(component)].extend(tensors)
|
||||
return result
|
||||
|
||||
def dec_swa_lock_only(
|
||||
self,
|
||||
node_id: NodeId,
|
||||
@@ -484,20 +555,43 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
result = EvictDeviceNextNodeResult(
|
||||
node_id=binding_result.node_id,
|
||||
made_progress=binding_result.made_progress,
|
||||
unbacked_tokens=binding_result.unbacked_tokens,
|
||||
mamba_backup_node_id=binding_result.mamba_backup_node_id,
|
||||
swa_backup_node_id=binding_result.swa_backup_node_id,
|
||||
swa_backup_num_tokens=binding_result.swa_backup_num_tokens,
|
||||
)
|
||||
return _fill_evict_result(binding_result, result)
|
||||
|
||||
def finish_mamba_state_eviction(self, node_id: NodeId) -> EvictDeviceNextNodeResult:
|
||||
binding_result = self._binding.finish_mamba_state_eviction(node_id)
|
||||
result = EvictDeviceNextNodeResult(
|
||||
node_id=binding_result.node_id,
|
||||
made_progress=binding_result.made_progress,
|
||||
unbacked_tokens=binding_result.unbacked_tokens,
|
||||
)
|
||||
return _fill_evict_result(binding_result, result)
|
||||
|
||||
def finish_swa_state_eviction(self, node_id: NodeId) -> EvictDeviceNextNodeResult:
|
||||
binding_result = self._binding.finish_swa_state_eviction(node_id)
|
||||
result = EvictDeviceNextNodeResult(
|
||||
node_id=binding_result.node_id,
|
||||
made_progress=binding_result.made_progress,
|
||||
unbacked_tokens=binding_result.unbacked_tokens,
|
||||
)
|
||||
return _fill_evict_result(binding_result, result)
|
||||
|
||||
def evict_device_leaf(
|
||||
self, node_id: NodeId, is_write_back: bool
|
||||
) -> EvictDeviceLeafResult:
|
||||
# The binding reads is_write_back from the core's construction config.
|
||||
# The binding reads is_write_back from the core's current config.
|
||||
assert is_write_back == self.is_write_back, (
|
||||
"is_write_back must match the core's construction config"
|
||||
"is_write_back must match the core's current config"
|
||||
)
|
||||
binding_result = self._binding.evict_device_leaf(node_id)
|
||||
backup = binding_result.backup_kv
|
||||
result = EvictDeviceLeafResult(
|
||||
backup_kv=_cache_action_from_tagged(backup) if backup is not None else None
|
||||
unbacked_tokens=binding_result.unbacked_tokens,
|
||||
backup_kv=_cache_action_from_tagged(backup) if backup is not None else None,
|
||||
)
|
||||
return _fill_evict_result(binding_result, result)
|
||||
|
||||
@@ -529,6 +623,9 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
def component_evictable_size(self, component_type: ComponentType) -> int:
|
||||
return self._binding.component_evictable_size(int(component_type))
|
||||
|
||||
def component_protected_size(self, component_type: ComponentType) -> int:
|
||||
return self._binding.component_protected_size(int(component_type))
|
||||
|
||||
def full_evictable_size(self) -> int:
|
||||
return self._binding.full_evictable_size()
|
||||
|
||||
@@ -632,6 +729,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
chunked=params.chunked,
|
||||
priority=0 if params.priority is None else params.priority,
|
||||
track_adopted_ranges=params.track_adopted_ranges,
|
||||
rotation_base=params.rotation_base,
|
||||
)
|
||||
)
|
||||
return _insert_step_from_binding(step)
|
||||
@@ -649,7 +747,9 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
self, component_type: ComponentType, num_tokens: int
|
||||
) -> DriveHostEvictionResult:
|
||||
binding_result = self._binding.drive_host_eviction(
|
||||
int(component_type), num_tokens
|
||||
int(component_type),
|
||||
num_tokens,
|
||||
envs.SGLANG_HICACHE_SKIP_HOST_DUPLICATE_RECLAIM.get(),
|
||||
)
|
||||
return _fill_evict_result(binding_result, DriveHostEvictionResult())
|
||||
|
||||
@@ -761,7 +861,31 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
self, node_id: NodeId
|
||||
) -> tuple[torch.Tensor, dict[ComponentType, list[PoolTransfer]]]:
|
||||
device_value, comp_xfers = self._binding.build_backup_spec(node_id)
|
||||
return device_value, _comp_xfers_from_binding(comp_xfers)
|
||||
comp_xfers = _comp_xfers_from_binding(comp_xfers)
|
||||
self._refresh_swa_backup_indices(comp_xfers.get(ComponentType.SWA, ()))
|
||||
return device_value, comp_xfers
|
||||
|
||||
def _refresh_swa_backup_indices(self, transfers: Sequence[PoolTransfer]) -> None:
|
||||
if not transfers:
|
||||
return
|
||||
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
|
||||
UnifiedSWAAllocatorBase,
|
||||
)
|
||||
|
||||
if not isinstance(self._allocator, UnifiedSWAAllocatorBase):
|
||||
return
|
||||
for transfer in transfers:
|
||||
if transfer.name != PoolName.SWA or not transfer.nodes_to_load:
|
||||
continue
|
||||
# Compaction can relocate SWA rows after the tree records them.
|
||||
# FULL virtual IDs retain their meaning across that relocation.
|
||||
full_values = [
|
||||
self.get_component_device_value(node_id, ComponentType.FULL)
|
||||
for node_id in transfer.nodes_to_load
|
||||
]
|
||||
transfer.device_indices = self._allocator.translate_loc_from_full_to_swa(
|
||||
torch.cat(full_values)
|
||||
).to(torch.int64)
|
||||
|
||||
def build_storage_backup_spec(
|
||||
self, node_id: NodeId, pass_prefix_keys: bool
|
||||
@@ -805,7 +929,10 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
)
|
||||
if transfers is None:
|
||||
return None
|
||||
return [_transfer_from_binding(transfer) for transfer in transfers]
|
||||
transfers = [_transfer_from_binding(transfer) for transfer in transfers]
|
||||
if phase == CacheTransferPhase.BACKUP_HOST:
|
||||
self._refresh_swa_backup_indices(transfers)
|
||||
return transfers
|
||||
|
||||
def build_load_back_spec(
|
||||
self, node_id: NodeId, req: Optional[Req] = None
|
||||
@@ -815,7 +942,30 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
kv_xfer, comp_xfers = self._binding.build_load_back_spec(
|
||||
node_id, mamba_pool_idx
|
||||
)
|
||||
return _transfer_from_binding(kv_xfer), _comp_xfers_from_binding(comp_xfers)
|
||||
kv_xfer = _transfer_from_binding(kv_xfer)
|
||||
comp_xfers = _comp_xfers_from_binding(comp_xfers)
|
||||
swa_xfers = comp_xfers.get(ComponentType.SWA, ())
|
||||
if swa_xfers:
|
||||
full_node_ids = kv_xfer.nodes_to_load or []
|
||||
full_load_slices = {}
|
||||
offset = 0
|
||||
for full_node_id, count in zip(
|
||||
full_node_ids, self._binding.get_node_key_lengths(full_node_ids)
|
||||
):
|
||||
full_load_slices[full_node_id] = slice(offset, offset + count)
|
||||
offset += count
|
||||
for transfer in swa_xfers:
|
||||
# SWA may have holes between resident nodes, or reload while
|
||||
# FULL stays resident. Preserve the SWA transfer's node order.
|
||||
transfer.anchor_index_parts = [
|
||||
(
|
||||
full_load_slices[nid]
|
||||
if nid in full_load_slices
|
||||
else self.get_component_device_value(nid, ComponentType.FULL)
|
||||
)
|
||||
for nid in transfer.nodes_to_load or ()
|
||||
]
|
||||
return kv_xfer, comp_xfers
|
||||
|
||||
def prefetch_anchor_info(
|
||||
self, node_id: NodeId
|
||||
@@ -878,6 +1028,9 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
def root_node_handle(self, extra_key: Optional[str] = None) -> NodeId:
|
||||
return self._binding.root_node_handle(extra_key)
|
||||
|
||||
def rotation_base_of(self, node_id: NodeId) -> Optional[int]:
|
||||
return self._binding.rotation_base_of(node_id)
|
||||
|
||||
def dfs_weight_order(self, node_ids: Sequence[NodeId]) -> list[int]:
|
||||
return self._binding.dfs_weight_order(list(node_ids))
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ from sglang.srt.mem_cache.unified_cache.components.base import (
|
||||
ComponentType,
|
||||
EvictLayer,
|
||||
ExternalLinkerLoadPhase,
|
||||
InternalStateBackup,
|
||||
LinkerTransferPhase,
|
||||
LRURefreshPhase,
|
||||
PrepareLoadBackResult,
|
||||
@@ -23,6 +24,7 @@ __all__ = [
|
||||
"ComponentData",
|
||||
"ComponentType",
|
||||
"ExternalLinkerLoadPhase",
|
||||
"InternalStateBackup",
|
||||
"LinkerTransferPhase",
|
||||
"EvictLayer",
|
||||
"FullComponent",
|
||||
|
||||
@@ -63,6 +63,15 @@ class EvictLayer(IntFlag):
|
||||
ALL = DEVICE | HOST
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class InternalStateBackup:
|
||||
"""Pause eviction for a host backup before freeing internal device state."""
|
||||
|
||||
node_id: NodeId
|
||||
# Host capacity required by the backup, in this component's pool units.
|
||||
num_tokens: int
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class PrepareLoadBackResult:
|
||||
"""Outcome of prepare_load_back; default = nothing to prepare."""
|
||||
@@ -522,11 +531,12 @@ class TreeComponent(ABC):
|
||||
tracker: dict[ComponentType, int],
|
||||
device_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
host_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
) -> Optional[NodeId]:
|
||||
"""Advance one eviction step and return a device leaf, if selected.
|
||||
) -> NodeId | InternalStateBackup | None:
|
||||
"""Return a device leaf, an internal backup request, or no selection.
|
||||
|
||||
Implementations must return after one allocator-relevant internal
|
||||
mutation so the caller can drain pending frees before continuing.
|
||||
Backup requests leave device state intact until the core resumes eviction.
|
||||
"""
|
||||
assert self.is_evict_device_ongoing, (
|
||||
f"{self.component_type} device eviction not started"
|
||||
@@ -552,8 +562,8 @@ class TreeComponent(ABC):
|
||||
tracker: dict[ComponentType, int],
|
||||
device_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
host_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
) -> Optional[NodeId]:
|
||||
"""Advance the walk by at most one allocator-relevant mutation."""
|
||||
) -> NodeId | InternalStateBackup | None:
|
||||
"""Select a leaf, request backup, or perform at most one internal mutation."""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
|
||||
@@ -30,6 +30,7 @@ from sglang.srt.mem_cache.unified_cache.components.base import (
|
||||
CacheTransferPhase,
|
||||
ComponentType,
|
||||
EvictLayer,
|
||||
InternalStateBackup,
|
||||
LinkerTransferPhase,
|
||||
LRURefreshPhase,
|
||||
PrepareLoadBackResult,
|
||||
@@ -369,7 +370,7 @@ class MambaComponent(TreeComponent):
|
||||
tracker: dict[ComponentType, int],
|
||||
device_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
host_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
) -> Optional[NodeId]:
|
||||
) -> NodeId | InternalStateBackup | None:
|
||||
"""Advance one device-eviction step and return a leaf, if selected.
|
||||
|
||||
An internal tombstone is one complete step so the caller can apply its
|
||||
@@ -404,9 +405,18 @@ class MambaComponent(TreeComponent):
|
||||
)
|
||||
return x.id
|
||||
if not enabled:
|
||||
x_next = lru.get_prev_no_lock(x)
|
||||
# write_back: demote the state to host before the internal tombstone.
|
||||
self._maybe_backup_node_before_state_tombstone(x)
|
||||
self._evict_device_cursor = lru.get_prev_no_lock(x)
|
||||
cd = x.component_data[ct]
|
||||
if (
|
||||
self.tree_core.enable_hicache
|
||||
and self.tree_core.is_write_back
|
||||
and cd.host_value is None
|
||||
and not x.backuped
|
||||
and x.component_data[BASE_COMPONENT_TYPE].value is not None
|
||||
):
|
||||
# Keep the state live until the controller has attempted its host
|
||||
# backup. Session cursors advance after the resumed tombstone.
|
||||
return InternalStateBackup(node_id=x.id, num_tokens=1)
|
||||
self.tree_core._evict_component_and_detach_lru(
|
||||
x,
|
||||
self,
|
||||
@@ -418,38 +428,10 @@ class MambaComponent(TreeComponent):
|
||||
self.tree_core._cascade_evict(
|
||||
x, self, tracker, device_frees=device_frees, host_frees=host_frees
|
||||
)
|
||||
self._evict_device_cursor = lru.cursor_next() if enabled else x_next
|
||||
if enabled:
|
||||
self._evict_device_cursor = lru.cursor_next()
|
||||
return None
|
||||
|
||||
def _maybe_backup_node_before_state_tombstone(self, node: UnifiedTreeNode) -> None:
|
||||
"""Demote an internal node's mamba state to host before its tombstone
|
||||
(write_back only), mirroring the leaf deferred-demote path.
|
||||
|
||||
The match validator passes only nodes holding the state on some
|
||||
layer, so a dropped internal state caps the match frontier at this
|
||||
node forever, leaving the subtree's still-resident KV unservable.
|
||||
Best-effort: this walk must make progress (it satisfies an imminent
|
||||
slot allocation), so any failure falls back to the legacy drop.
|
||||
"""
|
||||
cache = self.cache
|
||||
cd = node.component_data[self.component_type]
|
||||
if (
|
||||
cache.cache_controller is None
|
||||
or not cache.is_write_back
|
||||
or cd.host_value is not None
|
||||
or node.backuped
|
||||
or node.component_data[BASE_COMPONENT_TYPE].value is None
|
||||
):
|
||||
return
|
||||
# The backup executor pre-evicts only the KV host pool; make room
|
||||
# for the state slot the way the PREFETCH hook does.
|
||||
if (
|
||||
self._mamba_pool_host is not None
|
||||
and self._mamba_pool_host.available_size() < 1
|
||||
):
|
||||
cache.evict_host(1, self.component_type)
|
||||
cache.backup_node_for_write_back(node.id)
|
||||
|
||||
def _evict_device_end(self) -> None:
|
||||
"""Clear the device-eviction walk cursor state."""
|
||||
if self.tree_core.enable_session_radix_cache:
|
||||
|
||||
@@ -36,6 +36,7 @@ from sglang.srt.mem_cache.unified_cache.components.base import (
|
||||
ComponentType,
|
||||
EvictLayer,
|
||||
ExternalLinkerLoadPhase,
|
||||
InternalStateBackup,
|
||||
LinkerTransferPhase,
|
||||
LRURefreshPhase,
|
||||
PreparePrefetchResult,
|
||||
@@ -711,7 +712,7 @@ class SWAComponent(TreeComponent):
|
||||
tracker: dict[ComponentType, int],
|
||||
device_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
host_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
) -> Optional[NodeId]:
|
||||
) -> NodeId | InternalStateBackup | None:
|
||||
"""Advance one device-eviction step and return a leaf, if selected.
|
||||
|
||||
An internal tombstone is one complete step so the caller can apply its
|
||||
@@ -746,9 +747,33 @@ class SWAComponent(TreeComponent):
|
||||
)
|
||||
return x.id
|
||||
if not enabled:
|
||||
x_next = lru.get_prev_no_lock(x)
|
||||
# write_back: demote the SWA KV to host before the internal tombstone.
|
||||
self._maybe_backup_node_before_swa_tombstone(x)
|
||||
self._evict_device_cursor = lru.get_prev_no_lock(x)
|
||||
cd = x.component_data[ct]
|
||||
if (
|
||||
self.tree_core.enable_hicache
|
||||
and self.tree_core.is_write_back
|
||||
and self.tree_core.has_swa_host_pool
|
||||
and cd.host_value is None
|
||||
and not x.backuped
|
||||
and x.component_data[BASE_COMPONENT_TYPE].value is not None
|
||||
):
|
||||
# Reserve the whole unbacked window, not just this victim, before
|
||||
# its best-effort host backup and resumed internal tombstone.
|
||||
needed = sum(
|
||||
len(node.component_data[ct].value)
|
||||
for node in self._collect_unbacked_swa_nodes(x)
|
||||
)
|
||||
if needed:
|
||||
if ct == ComponentType.SWA:
|
||||
return InternalStateBackup(node_id=x.id, num_tokens=needed)
|
||||
# Custom SWA component types use Python's inherited inline
|
||||
# path; the shared finish hook is for built-in SWA only.
|
||||
if (
|
||||
self._swa_kv_pool_host is not None
|
||||
and self._swa_kv_pool_host.available_size() < needed
|
||||
):
|
||||
self.cache.evict_host(needed, ct)
|
||||
self.cache.backup_node_for_write_back(x.id)
|
||||
self.tree_core._evict_component_and_detach_lru(
|
||||
x,
|
||||
self,
|
||||
@@ -760,48 +785,10 @@ class SWAComponent(TreeComponent):
|
||||
self.tree_core._cascade_evict(
|
||||
x, self, tracker, device_frees=device_frees, host_frees=host_frees
|
||||
)
|
||||
self._evict_device_cursor = lru.cursor_next() if enabled else x_next
|
||||
if enabled:
|
||||
self._evict_device_cursor = lru.cursor_next()
|
||||
return None
|
||||
|
||||
def _maybe_backup_node_before_swa_tombstone(self, node: UnifiedTreeNode) -> None:
|
||||
"""Demote an internal node's SWA KV to host before its tombstone
|
||||
(write_back only), mirroring the leaf deferred-demote path.
|
||||
|
||||
The match validator treats an unbacked tombstone as a window reset,
|
||||
so a dropped internal SWA segment caps the match frontier until a
|
||||
full sliding window re-accumulates below it, leaving up to one
|
||||
window of still-resident KV unservable. The leaf backup walk covers
|
||||
ancestors only within one window of the evicted leaf, so an
|
||||
internal node whose child spans the window arrives here unbacked.
|
||||
Best-effort: this walk must make progress (it satisfies an imminent
|
||||
allocation), so any failure falls back to the legacy drop.
|
||||
"""
|
||||
cache = self.cache
|
||||
cd = node.component_data[self.component_type]
|
||||
if (
|
||||
cache.cache_controller is None
|
||||
or not cache.is_write_back
|
||||
or not self.tree_core.has_swa_host_pool
|
||||
or cd.host_value is not None
|
||||
or node.backuped
|
||||
or node.component_data[BASE_COMPONENT_TYPE].value is None
|
||||
):
|
||||
return
|
||||
# The backup executor pre-evicts only the KV host pool; make room
|
||||
# in the SWA host pool the way the PREFETCH hook does.
|
||||
needed = sum(
|
||||
len(n.component_data[self.component_type].value)
|
||||
for n in self._collect_unbacked_swa_nodes(node)
|
||||
)
|
||||
if needed == 0:
|
||||
return
|
||||
if (
|
||||
self._swa_kv_pool_host is not None
|
||||
and self._swa_kv_pool_host.available_size() < needed
|
||||
):
|
||||
cache.evict_host(needed, self.component_type)
|
||||
cache.backup_node_for_write_back(node.id)
|
||||
|
||||
def _evict_device_end(self) -> None:
|
||||
"""Clear the device-eviction walk cursor state."""
|
||||
if self.tree_core.enable_session_radix_cache:
|
||||
|
||||
@@ -1,18 +1,28 @@
|
||||
"""Registry for pluggable TreeCore implementations.
|
||||
|
||||
The unified cache constructs its TreeCore through `create_tree_core`, selected
|
||||
by SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND (default "python"). To plug in a custom
|
||||
implementation, register it under a string name via
|
||||
`register_tree_core_backend(name, factory)`.
|
||||
by its per-cache override or SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND (default
|
||||
"rust"). Rust selections use centralized compatibility fallbacks. To plug in a
|
||||
custom implementation, register it via `register_tree_core_backend(name, factory)`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import logging
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Callable, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
|
||||
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
|
||||
from sglang.srt.mem_cache.unified_cache.components import TreeComponent
|
||||
from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import (
|
||||
UnifiedTreeCoreInterface,
|
||||
@@ -24,6 +34,101 @@ TreeCoreFactory = Callable[
|
||||
]
|
||||
|
||||
_TREE_CORE_REGISTRY: dict[str, TreeCoreFactory] = {}
|
||||
_RUST_TREE_CORE_MODULE = "sglang.srt.mem_cache.rust_tree_core.mem_cache"
|
||||
_RUST_TREE_CORE_MANIFEST = (
|
||||
Path(__file__).resolve().parents[5] / "rust" / "sglang-radix-tree" / "Cargo.toml"
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _rust_fallback_reason(params: CacheInitParams) -> Optional[str]:
|
||||
if params.enable_session_radix_cache:
|
||||
return "session-aware caching requires the Python TreeCore"
|
||||
if params.tree_components is not None and set(params.tree_components) - {
|
||||
ComponentType.FULL,
|
||||
ComponentType.SWA,
|
||||
ComponentType.MAMBA,
|
||||
}:
|
||||
return "the configured components require the Python TreeCore"
|
||||
if params.component_registry_override:
|
||||
return "custom components require the Python TreeCore"
|
||||
if sys.platform != "linux":
|
||||
return "the Rust TreeCore supports Linux only"
|
||||
from sglang.srt.rust_extensions.torch_build import (
|
||||
_MAX_SUPPORTED_TORCH,
|
||||
_MIN_SUPPORTED_TORCH,
|
||||
)
|
||||
|
||||
match = re.match(r"^(\d+)\.(\d+)", str(torch.__version__))
|
||||
if match is None or not (
|
||||
_MIN_SUPPORTED_TORCH
|
||||
<= (int(match.group(1)), int(match.group(2)))
|
||||
<= _MAX_SUPPORTED_TORCH
|
||||
):
|
||||
return f"PyTorch {torch.__version__} is outside the Rust TreeCore support range"
|
||||
allocator = params.token_to_kv_pool_allocator
|
||||
device = (
|
||||
torch.device(allocator.device) if allocator is not None else torch.device("cpu")
|
||||
)
|
||||
if device.type not in ("cpu", "cuda"):
|
||||
return f"the Rust TreeCore does not support device {device.type}"
|
||||
mode = envs.SGLANG_RUST_BUILD_MODE.get()
|
||||
if sys.modules.get(_RUST_TREE_CORE_MODULE) is not None or mode not in (
|
||||
"auto",
|
||||
"never",
|
||||
"force",
|
||||
):
|
||||
# Preserve the loader's invalid-mode and force-after-import errors.
|
||||
return None
|
||||
workspace = _RUST_TREE_CORE_MANIFEST.parent.parent
|
||||
bundled = importlib.util.find_spec(_RUST_TREE_CORE_MODULE) is not None
|
||||
source_checkout = (workspace / "Cargo.toml").is_file()
|
||||
if mode != "force" and (mode == "never" or not source_checkout) and bundled:
|
||||
return None
|
||||
if not _RUST_TREE_CORE_MANIFEST.is_file():
|
||||
if bundled:
|
||||
return "the requested Rust TreeCore source build has no sources"
|
||||
return "this installation contains neither the Rust TreeCore extension nor its sources"
|
||||
# Even a cached source build needs these version queries for its fingerprint.
|
||||
# Match the loader's commands and cwd so rustup workspace overrides apply.
|
||||
for command in (("cargo", "--version", "--verbose"), ("rustc", "-vV")):
|
||||
try:
|
||||
subprocess.run(
|
||||
command,
|
||||
cwd=workspace,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired):
|
||||
return f"the Rust TreeCore toolchain is unavailable ({' '.join(command)})"
|
||||
return None
|
||||
|
||||
|
||||
def resolve_tree_core_backend(name: str, params: CacheInitParams) -> str:
|
||||
"""Resolve known Rust capability gaps before loading a backend.
|
||||
|
||||
Explicit Rust selections use the same compatibility policy as the default.
|
||||
Build, import, and runtime failures remain errors when the toolchain is usable.
|
||||
"""
|
||||
if name != "rust":
|
||||
return name
|
||||
reason = _rust_fallback_reason(params)
|
||||
if reason is not None:
|
||||
logger.info("Using the Python TreeCore: %s", reason)
|
||||
return "python"
|
||||
return name
|
||||
|
||||
|
||||
def select_tree_core_backend(params: CacheInitParams) -> str:
|
||||
"""Resolve the instance override or default through the shared fallback policy."""
|
||||
name = (
|
||||
params.tree_core_backend
|
||||
if params.tree_core_backend is not None
|
||||
else envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.get()
|
||||
)
|
||||
return resolve_tree_core_backend(name, params)
|
||||
|
||||
|
||||
def register_tree_core_backend(name: str, factory: TreeCoreFactory) -> None:
|
||||
@@ -72,7 +177,8 @@ def create_tree_core(
|
||||
params: CacheInitParams,
|
||||
components: dict[ComponentType, TreeComponent],
|
||||
) -> UnifiedTreeCoreInterface:
|
||||
"""Construct the TreeCore registered under `name`."""
|
||||
"""Resolve compatibility and construct the registered TreeCore."""
|
||||
name = resolve_tree_core_backend(name, params)
|
||||
factory = get_tree_core_factory(name)
|
||||
if factory is None:
|
||||
raise ValueError(
|
||||
|
||||
@@ -60,6 +60,7 @@ from sglang.srt.mem_cache.unified_cache.components import (
|
||||
ComponentData,
|
||||
ComponentType,
|
||||
EvictLayer,
|
||||
InternalStateBackup,
|
||||
LinkerTransferPhase,
|
||||
LRURefreshPhase,
|
||||
TreeComponent,
|
||||
@@ -474,6 +475,13 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
def reset(self) -> None:
|
||||
"""Rebuild the root, LRUs, sizes, evictable-leaf sets, and the empty
|
||||
match result."""
|
||||
# End suspended walks while their session-cursor sentinels still
|
||||
# belong to the old LRUs, clearing any pending internal victim.
|
||||
for component in self.components:
|
||||
if component.is_evict_device_ongoing:
|
||||
component.evict_device_end()
|
||||
# Internal victims awaiting the controller's backup and finish call.
|
||||
self._pending_internal_evictions: dict[ComponentType, NodeId] = {}
|
||||
# Maintains the NodeId -> active tree node mapping.
|
||||
self._node_arena: dict[NodeId, UnifiedTreeNode] = {}
|
||||
|
||||
@@ -1586,12 +1594,15 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
result = EvictDeviceNextNodeResult()
|
||||
# The walk reads running totals for its doneness check; the result
|
||||
# carries only this step's delta.
|
||||
component = self.components_by_type[component_type]
|
||||
assert component_type not in self._pending_internal_evictions, (
|
||||
f"finish the pending internal {component_type.name} eviction "
|
||||
"before advancing"
|
||||
)
|
||||
updated_tracker = defaultdict(int, tracker)
|
||||
self._begin_tracking_unbacked_tokens()
|
||||
try:
|
||||
result.node_id = self.components_by_type[
|
||||
component_type
|
||||
].evict_device_next_node(
|
||||
step = component.evict_device_next_node(
|
||||
updated_tracker, result.device_frees, result.host_frees
|
||||
)
|
||||
finally:
|
||||
@@ -1600,12 +1611,99 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
delta = n - tracker.get(ct, 0)
|
||||
if delta:
|
||||
result.tracker[ct] = delta
|
||||
result.made_progress = result.node_id is not None or bool(result.tracker)
|
||||
if isinstance(step, InternalStateBackup):
|
||||
assert component_type in (ComponentType.MAMBA, ComponentType.SWA)
|
||||
self._pending_internal_evictions[component_type] = step.node_id
|
||||
if component_type == ComponentType.MAMBA:
|
||||
result.mamba_backup_node_id = step.node_id
|
||||
else:
|
||||
result.swa_backup_node_id = step.node_id
|
||||
result.swa_backup_num_tokens = step.num_tokens
|
||||
else:
|
||||
result.node_id = step
|
||||
result.made_progress = (
|
||||
result.node_id is not None
|
||||
or result.mamba_backup_node_id is not None
|
||||
or result.swa_backup_node_id is not None
|
||||
or bool(result.tracker)
|
||||
)
|
||||
return result
|
||||
|
||||
def finish_mamba_state_eviction(self, node_id: NodeId) -> EvictDeviceNextNodeResult:
|
||||
return self._finish_internal_component_eviction(ComponentType.MAMBA, node_id)
|
||||
|
||||
def finish_swa_state_eviction(self, node_id: NodeId) -> EvictDeviceNextNodeResult:
|
||||
return self._finish_internal_component_eviction(ComponentType.SWA, node_id)
|
||||
|
||||
def _finish_internal_component_eviction(
|
||||
self, component_type: ComponentType, node_id: NodeId
|
||||
) -> EvictDeviceNextNodeResult:
|
||||
"""Resume an internal tombstone after the controller's backup attempt."""
|
||||
component = self.components_by_type[component_type]
|
||||
assert component.is_evict_device_ongoing, (
|
||||
f"{component_type.name} device eviction not started"
|
||||
)
|
||||
assert self._pending_internal_evictions.get(component_type) == node_id, (
|
||||
f"no matching pending internal {component_type.name} eviction"
|
||||
)
|
||||
del self._pending_internal_evictions[component_type]
|
||||
# Consuming the pending request advances the walk even if I/O changed
|
||||
# this victim's eligibility. No frees occur before backup completion.
|
||||
result = EvictDeviceNextNodeResult(made_progress=True)
|
||||
node = self._node_arena.get(node_id)
|
||||
lru = self.lru_lists[component_type]
|
||||
enabled = self.enable_session_radix_cache
|
||||
if (
|
||||
node is None
|
||||
or node.component_data[component_type].value is None
|
||||
or node.component_data[component_type].lock_ref > 0
|
||||
or (
|
||||
component_type == ComponentType.MAMBA
|
||||
and node.load_back_pending_id is not None
|
||||
)
|
||||
or not lru.in_list(node)
|
||||
):
|
||||
if enabled:
|
||||
component._evict_device_cursor = lru.cursor_next()
|
||||
return result
|
||||
if (
|
||||
node in self.evictable_device_leaves
|
||||
and (not enabled or component._can_evict_leaf_atomically(node))
|
||||
) or (
|
||||
component_type == ComponentType.MAMBA
|
||||
and node.component_data[BASE_COMPONENT_TYPE].value is None
|
||||
):
|
||||
# Re-select changed leaves through the normal walk; internal
|
||||
# finish must not accidentally perform an atomic Full eviction.
|
||||
component._evict_device_cursor = node
|
||||
return result
|
||||
self._begin_tracking_unbacked_tokens()
|
||||
try:
|
||||
self._evict_component_and_detach_lru(
|
||||
node,
|
||||
component,
|
||||
target=EvictLayer.DEVICE,
|
||||
tracker=result.tracker,
|
||||
device_frees=result.device_frees,
|
||||
host_frees=result.host_frees,
|
||||
)
|
||||
self._cascade_evict(
|
||||
node,
|
||||
component,
|
||||
result.tracker,
|
||||
device_frees=result.device_frees,
|
||||
host_frees=result.host_frees,
|
||||
)
|
||||
finally:
|
||||
result.unbacked_tokens = self._finish_tracking_unbacked_tokens()
|
||||
if enabled:
|
||||
component._evict_device_cursor = lru.cursor_next()
|
||||
return result
|
||||
|
||||
def evict_device_end(self, component_type: ComponentType) -> None:
|
||||
"""Finish a component's device-eviction walk."""
|
||||
self.components_by_type[component_type].evict_device_end()
|
||||
self._pending_internal_evictions.pop(component_type, None)
|
||||
|
||||
def evict_device_leaf(
|
||||
self, node_id: NodeId, is_write_back: bool
|
||||
|
||||
@@ -45,12 +45,18 @@ class EvictDeviceNextNodeResult(BaseEvictionResult):
|
||||
|
||||
``node_id`` selects a leaf for the Controller to evict. ``made_progress``
|
||||
also covers an internal tombstone that returned no leaf, distinguishing it
|
||||
from true walk exhaustion.
|
||||
from true walk exhaustion. ``mamba_backup_node_id`` and
|
||||
``swa_backup_node_id`` pause an internal eviction until the Controller
|
||||
finishes its best-effort host backup. ``swa_backup_num_tokens`` includes
|
||||
all unbacked SWA segments in the backup window, not just the victim.
|
||||
"""
|
||||
|
||||
node_id: Optional[NodeId] = None
|
||||
made_progress: bool = False
|
||||
unbacked_tokens: int = 0
|
||||
mamba_backup_node_id: Optional[NodeId] = None
|
||||
swa_backup_node_id: Optional[NodeId] = None
|
||||
swa_backup_num_tokens: int = 0
|
||||
|
||||
|
||||
class EvictDeviceLeafResult(BaseEvictionResult):
|
||||
@@ -319,6 +325,24 @@ class UnifiedTreeCoreInterface(ABC):
|
||||
D->H backup must run (write_back) before the node can be demoted."""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def finish_mamba_state_eviction(self, node_id: NodeId) -> EvictDeviceNextNodeResult:
|
||||
"""Resume the pending internal Mamba eviction after its backup attempt.
|
||||
|
||||
The controller must finish any submitted D->H transfer before calling.
|
||||
A failed allocation still permits eviction to make device space.
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def finish_swa_state_eviction(self, node_id: NodeId) -> EvictDeviceNextNodeResult:
|
||||
"""Resume the pending internal SWA eviction after its backup attempt.
|
||||
|
||||
The controller must finish any submitted D->H transfer before calling.
|
||||
A failed allocation still permits eviction to make device space.
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def drop_subtree_no_host(self, node_id: NodeId) -> DropSubtreeNoHostResult:
|
||||
"""Drop an unbacked D-leaf's subtree when its write-back backup failed
|
||||
@@ -642,6 +666,29 @@ class UnifiedTreeCoreInterface(ABC):
|
||||
host store event for each acked node."""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def swa_tombstone_ranges(
|
||||
self, key: RadixKey, start: int, end: int
|
||||
) -> list[tuple[int, int]]:
|
||||
"""Return maximal missing SWA ranges within the matched [start, end) span."""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def attach_swa_window(
|
||||
self,
|
||||
key: RadixKey,
|
||||
window_start: int,
|
||||
window_end: int,
|
||||
swa_values: torch.Tensor,
|
||||
) -> list[CacheAction | ComponentAction]:
|
||||
"""Attach a loaded SWA window to tombstoned spans, returning split actions.
|
||||
|
||||
The shared pipeline supplies page-aligned logical token offsets and
|
||||
int64 values on the core's device. The entire span must be matched and
|
||||
tombstoned before publication.
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def set_component_device_value(
|
||||
self, node_id: NodeId, component_type: ComponentType, value: torch.Tensor
|
||||
|
||||
@@ -74,7 +74,10 @@ from sglang.srt.mem_cache.unified_cache.session_ref_tracker import (
|
||||
UnifiedSessionRefTracker,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_cache.storage_attachment import StorageAttachment
|
||||
from sglang.srt.mem_cache.unified_cache.tree_core_registry import create_tree_core
|
||||
from sglang.srt.mem_cache.unified_cache.tree_core_registry import (
|
||||
create_tree_core,
|
||||
select_tree_core_backend,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_cache.unified_cache_linker import (
|
||||
UnifiedCacheLinker,
|
||||
UnifiedCacheLinkerWrapper,
|
||||
@@ -204,11 +207,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
)
|
||||
# The TreeCore owns the tree member-var state (structure, LRUs, sizes,
|
||||
# evictable leaves) and drives the components' tree-level hooks.
|
||||
self._tree_core_backend = (
|
||||
params.tree_core_backend
|
||||
if params.tree_core_backend is not None
|
||||
else envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.get()
|
||||
)
|
||||
self._tree_core_backend = select_tree_core_backend(params)
|
||||
self.tree_core = create_tree_core(
|
||||
name=self._tree_core_backend,
|
||||
params=params,
|
||||
@@ -756,6 +755,35 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
) -> tuple[Optional[NodeId], bool]:
|
||||
"""Advance the eviction walk one node, consuming its step result."""
|
||||
result = self.tree_core.evict_device_next_node(component_type, tracker)
|
||||
if result.mamba_backup_node_id is not None:
|
||||
assert component_type == ComponentType.MAMBA and result.node_id is None
|
||||
assert (
|
||||
not result.device_frees and not result.host_frees and not result.tracker
|
||||
)
|
||||
# Reserve a host state slot and wait for the backup acknowledgment
|
||||
# before freeing device state. If allocation fails, eviction still
|
||||
# proceeds to make room on the device.
|
||||
node_id = result.mamba_backup_node_id
|
||||
mamba_host_pool = self.host_pool_group.get_pool(PoolName.MAMBA)
|
||||
if mamba_host_pool is not None and mamba_host_pool.available_size() < 1:
|
||||
self.evict_host(1, ComponentType.MAMBA)
|
||||
self.backup_node_for_write_back(node_id)
|
||||
result = self.tree_core.finish_mamba_state_eviction(node_id)
|
||||
elif result.swa_backup_node_id is not None:
|
||||
assert component_type == ComponentType.SWA and result.node_id is None
|
||||
assert (
|
||||
not result.device_frees and not result.host_frees and not result.tracker
|
||||
)
|
||||
# The backup can cover several unbacked SWA segments. Reserve host
|
||||
# space for the whole window before copying it, then resume eviction
|
||||
# even if host allocation fails.
|
||||
node_id = result.swa_backup_node_id
|
||||
needed = result.swa_backup_num_tokens
|
||||
swa_host_pool = self.host_pool_group.get_pool(PoolName.SWA)
|
||||
if swa_host_pool is not None and swa_host_pool.available_size() < needed:
|
||||
self.evict_host(needed, ComponentType.SWA)
|
||||
self.backup_node_for_write_back(node_id)
|
||||
result = self.tree_core.finish_swa_state_eviction(node_id)
|
||||
self._free_values(result.device_frees, result.host_frees)
|
||||
if self._tracks_write_through_unbacked_evictions():
|
||||
self._record_dropped_tokens(
|
||||
@@ -887,8 +915,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
|
||||
def _tracks_write_through_unbacked_evictions(self) -> bool:
|
||||
return (
|
||||
isinstance(self.tree_core, UnifiedTreeCore)
|
||||
and self.host_memory_mode == "cache"
|
||||
self.host_memory_mode == "cache"
|
||||
and self.cache_controller is not None
|
||||
and self.cache_controller.write_policy == "write_through"
|
||||
)
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
from sglang.srt.environ import envs
|
||||
|
||||
# Spawned server and scheduler processes import the scripted-runtime package
|
||||
# before constructing their cache. Keep inspection in test processes while
|
||||
# preserving the selected backend and its compatibility fallback.
|
||||
if envs.SGLANG_TEST_SCRIPTED_RUNTIME.get():
|
||||
from sglang.test.scripted_runtime.tree_core_inspection import (
|
||||
install_tree_core_inspectors,
|
||||
)
|
||||
|
||||
install_tree_core_inspectors()
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any, List
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from sglang.test.scripted_runtime.context.radix import _node_lock_ref, to_node_handle
|
||||
from sglang.test.scripted_runtime.context.radix import (
|
||||
get_node_lock_ref,
|
||||
iter_node_handles,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.scheduler import Scheduler
|
||||
@@ -12,9 +15,9 @@ class ScriptedLockRefExhauster:
|
||||
def __init__(self, scheduler: Scheduler) -> None:
|
||||
self.scheduler = scheduler
|
||||
# (node, dec receipt) pairs; the receipt bounds the release walk.
|
||||
self._locked: List[tuple[Any, Any]] = []
|
||||
self._locked: list[tuple[Any, Any]] = []
|
||||
|
||||
def exhaust(self, *, leave_refs: int) -> None:
|
||||
def exhaust(self, leave_refs: int) -> None:
|
||||
tree_cache = self.scheduler.tree_cache
|
||||
if tree_cache.disable:
|
||||
return
|
||||
@@ -25,9 +28,11 @@ class ScriptedLockRefExhauster:
|
||||
return
|
||||
|
||||
target = evictable[0]
|
||||
result = tree_cache.inc_lock_ref(to_node_handle(tree_cache, target))
|
||||
result = tree_cache.inc_lock_ref(target)
|
||||
|
||||
newly_locked = [node for node in evictable if _node_lock_ref(node) > 0]
|
||||
newly_locked = [
|
||||
node for node in evictable if get_node_lock_ref(tree_cache, node) > 0
|
||||
]
|
||||
if not newly_locked:
|
||||
return
|
||||
self._locked.append((target, result.to_dec_params()))
|
||||
@@ -35,15 +40,13 @@ class ScriptedLockRefExhauster:
|
||||
def release(self) -> None:
|
||||
tree_cache = self.scheduler.tree_cache
|
||||
for node, dec_params in self._locked:
|
||||
tree_cache.dec_lock_ref(to_node_handle(tree_cache, node), dec_params)
|
||||
tree_cache.dec_lock_ref(node, dec_params)
|
||||
self._locked.clear()
|
||||
|
||||
def _evictable_nodes(self) -> List[Any]:
|
||||
evictable: List[Any] = []
|
||||
stack = list(self.scheduler.tree_cache.root_node.children.values())
|
||||
while stack:
|
||||
node = stack.pop()
|
||||
if _node_lock_ref(node) == 0:
|
||||
evictable.append(node)
|
||||
stack.extend(node.children.values())
|
||||
return evictable
|
||||
def _evictable_nodes(self) -> list[Any]:
|
||||
tree_cache = self.scheduler.tree_cache
|
||||
return [
|
||||
node
|
||||
for node in iter_node_handles(tree_cache)
|
||||
if get_node_lock_ref(tree_cache, node) == 0
|
||||
]
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any, Callable, Dict
|
||||
from collections.abc import Iterator
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from sglang.srt.mem_cache.unified_radix_cache import UnifiedTreeNode
|
||||
|
||||
@@ -8,12 +9,24 @@ if TYPE_CHECKING:
|
||||
from sglang.test.scripted_runtime.context.api import ScriptedContext
|
||||
|
||||
|
||||
def get_all_node_hit_counts(ctx: ScriptedContext) -> Dict[int, int]:
|
||||
return _collect_node_attr(ctx, lambda node: node.hit_count)
|
||||
def get_all_node_hit_counts(ctx: ScriptedContext) -> dict[int, int]:
|
||||
tree_cache = ctx.scheduler.tree_cache
|
||||
core = getattr(tree_cache, "tree_core", None)
|
||||
return {
|
||||
node if core is not None else node.id: (
|
||||
core.get_node_hit_count(node) if core is not None else node.hit_count
|
||||
)
|
||||
for node in iter_node_handles(tree_cache)
|
||||
}
|
||||
|
||||
|
||||
def get_all_node_lock_refs(ctx: ScriptedContext) -> Dict[int, int]:
|
||||
return _collect_node_attr(ctx, _node_lock_ref)
|
||||
def get_all_node_lock_refs(ctx: ScriptedContext) -> dict[int, int]:
|
||||
tree_cache = ctx.scheduler.tree_cache
|
||||
core = getattr(tree_cache, "tree_core", None)
|
||||
return {
|
||||
node if core is not None else node.id: get_node_lock_ref(tree_cache, node)
|
||||
for node in iter_node_handles(tree_cache)
|
||||
}
|
||||
|
||||
|
||||
def _node_lock_ref(node: Any) -> int:
|
||||
@@ -22,37 +35,34 @@ def _node_lock_ref(node: Any) -> int:
|
||||
return node.lock_ref
|
||||
|
||||
|
||||
def resolve_node(tree_cache: Any, node_handle: Any) -> Any:
|
||||
"""Resolve whatever a req or match result carries into a tree node.
|
||||
|
||||
``UnifiedRadixCache`` hands out NodeIds (ints); every other cache hands out
|
||||
the node object itself. Returns None when the handle no longer maps to a
|
||||
live node, which only happens once the node has been freed -- and a freed
|
||||
node holds no locks.
|
||||
"""
|
||||
resolve = getattr(tree_cache, "resolve_node_handle", None)
|
||||
if resolve is None:
|
||||
return node_handle
|
||||
try:
|
||||
return resolve(node_handle)
|
||||
except KeyError:
|
||||
return None
|
||||
def get_node_lock_ref(tree_cache: Any, node_handle: Any) -> int:
|
||||
"""Read locks through the cache's native handle, including stale handles."""
|
||||
if node_handle is None:
|
||||
return 0
|
||||
core = getattr(tree_cache, "tree_core", None)
|
||||
if core is not None:
|
||||
if not core.contains_node(node_handle):
|
||||
return 0
|
||||
return sum(
|
||||
core.get_component_device_lock_ref(node_handle, component)
|
||||
for component in tree_cache.tree_components
|
||||
)
|
||||
return _node_lock_ref(node_handle)
|
||||
|
||||
|
||||
def to_node_handle(tree_cache: Any, node: Any) -> Any:
|
||||
"""Inverse of `resolve_node`: what the tree_cache lock APIs accept."""
|
||||
if isinstance(node, UnifiedTreeNode):
|
||||
return node.id
|
||||
return node
|
||||
|
||||
|
||||
def _collect_node_attr(
|
||||
ctx: ScriptedContext, get_value: Callable[[Any], int]
|
||||
) -> Dict[int, int]:
|
||||
values: Dict[int, int] = {}
|
||||
stack = list(ctx.scheduler.tree_cache.root_node.children.values())
|
||||
def iter_node_handles(tree_cache: Any) -> Iterator[Any]:
|
||||
"""Walk non-root nodes as IDs for unified caches or native legacy nodes."""
|
||||
core = getattr(tree_cache, "tree_core", None)
|
||||
stack = (
|
||||
list(core.get_child_node_ids(core.root_node_handle()))
|
||||
if core is not None
|
||||
else list(tree_cache.root_node.children.values())
|
||||
)
|
||||
while stack:
|
||||
node = stack.pop()
|
||||
values[node.id] = get_value(node)
|
||||
stack.extend(node.children.values())
|
||||
return values
|
||||
yield node
|
||||
stack.extend(
|
||||
core.get_child_node_ids(node)
|
||||
if core is not None
|
||||
else node.children.values()
|
||||
)
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sglang.test.scripted_runtime.context.radix import _node_lock_ref, resolve_node
|
||||
from sglang.test.scripted_runtime.context.radix import get_node_lock_ref
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.schedule_batch import Req
|
||||
@@ -16,7 +16,7 @@ class ScriptedReqHandle:
|
||||
context: ScriptedContext
|
||||
|
||||
@property
|
||||
def req(self) -> Optional[Req]:
|
||||
def req(self) -> Req | None:
|
||||
return self.context.find_req_by_rid(self.rid)
|
||||
|
||||
@property
|
||||
@@ -52,7 +52,4 @@ class ScriptedReqHandle:
|
||||
req = self.req
|
||||
if req is None:
|
||||
return 0
|
||||
node = resolve_node(self.context.scheduler.tree_cache, req.last_node)
|
||||
if node is None:
|
||||
return 0
|
||||
return _node_lock_ref(node)
|
||||
return get_node_lock_ref(self.context.scheduler.tree_cache, req.last_node)
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Use the shared TreeCore inspectors in scripted-runtime test processes."""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def install_tree_core_inspectors():
|
||||
from sglang.srt.mem_cache.unified_cache.tree_core_registry import (
|
||||
_TREE_CORE_REGISTRY,
|
||||
)
|
||||
|
||||
inspector_dir = str(
|
||||
Path(__file__).resolve().parents[4]
|
||||
/ "test"
|
||||
/ "registered"
|
||||
/ "unit"
|
||||
/ "mem_cache"
|
||||
)
|
||||
if inspector_dir not in sys.path:
|
||||
sys.path.insert(0, inspector_dir)
|
||||
|
||||
def python_inspector_factory(params, components):
|
||||
from unified_tree_core_inspector import UnifiedTreeCoreInspector
|
||||
|
||||
return UnifiedTreeCoreInspector(params, components)
|
||||
|
||||
def rust_inspector_factory(params, _components):
|
||||
from rust_unified_tree_core_inspector import RustUnifiedTreeCoreInspector
|
||||
|
||||
return RustUnifiedTreeCoreInspector(params)
|
||||
|
||||
_TREE_CORE_REGISTRY.update(
|
||||
python=python_inspector_factory, rust=rust_inspector_factory
|
||||
)
|
||||
Reference in New Issue
Block a user