[Refactor] radix-tree: Reuse the insert walk, component hooks and bindings for custom tree cores (#45439)
Co-authored-by: metamergebot <metamergebot@users.noreply.github.com> Co-authored-by: Jialin <Jialin@users.noreply.github.com>
This commit is contained in:
co-authored by
metamergebot
Jialin
parent
b57b5e2286
commit
5e79333c95
@@ -285,6 +285,7 @@ class C128SidecarComponent(TreeComponent):
|
||||
node: UnifiedTreeNode,
|
||||
device_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
host_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
cache_actions: list[CacheAction | ComponentAction],
|
||||
target: EvictLayer = EvictLayer.DEVICE,
|
||||
) -> tuple[int, int]:
|
||||
cd = node.component_data[self.component_type]
|
||||
@@ -352,7 +353,7 @@ class C128SidecarComponent(TreeComponent):
|
||||
def _evict_device_start(self, request_cnt):
|
||||
pass
|
||||
|
||||
def _evict_device_next_node(self, tracker, device_frees, host_frees):
|
||||
def _evict_device_next_node(self, tracker, device_frees, host_frees, cache_actions):
|
||||
return None
|
||||
|
||||
def _evict_device_end(self) -> None:
|
||||
|
||||
@@ -274,67 +274,11 @@ def _comp_xfers_from_binding(
|
||||
}
|
||||
|
||||
|
||||
def _insert_step_from_binding(step) -> InsertStepResult:
|
||||
"""Build the interface step for the binding's step (result on the final one)."""
|
||||
result = None
|
||||
if step.result is not None:
|
||||
# A stepped insert delivers all actions through steps, never the result.
|
||||
assert not step.result.cache_actions
|
||||
result = InsertResult(
|
||||
prefix_len=step.result.prefix_len,
|
||||
last_device_node=step.result.last_device_node,
|
||||
mamba_exist=step.result.mamba_exist,
|
||||
swa_branch_inserted=step.result.swa_branch_inserted,
|
||||
host_insert_dropped=step.result.host_insert_dropped,
|
||||
rotation_tail_declined=step.result.rotation_tail_declined,
|
||||
adopted_ranges=(
|
||||
{
|
||||
ComponentType(component_type): list(ranges)
|
||||
for component_type, ranges in step.result.adopted_ranges.items()
|
||||
}
|
||||
if step.result.adopted_ranges is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
return InsertStepResult(
|
||||
actions=_cache_actions_from_tagged(step.actions), result=result
|
||||
)
|
||||
|
||||
|
||||
def _match_result_from_binding(result) -> MatchResult:
|
||||
"""Build the Python MatchResult for the binding's match result."""
|
||||
return MatchResult(
|
||||
device_prefix_len=result.device_prefix_len,
|
||||
last_device_node=result.last_device_node_id,
|
||||
last_host_node=result.last_host_node_id,
|
||||
best_match_node=result.best_match_node_id,
|
||||
host_hit_length=result.host_hit_length,
|
||||
swa_host_hit_length=result.swa_host_hit_length,
|
||||
swa_branching_seqlen=result.swa_branching_seqlen,
|
||||
mamba_host_hit_length=result.mamba_host_hit_length,
|
||||
mamba_branching_seqlen=result.mamba_branching_seqlen,
|
||||
full_kv_hit_length=result.full_kv_hit_length,
|
||||
cache_actions=_cache_actions_from_tagged(result.cache_actions),
|
||||
)
|
||||
|
||||
|
||||
def _tracker_to_binding(tracker: dict[ComponentType, int]) -> dict[int, int]:
|
||||
"""Rekey a ComponentType tracker by the binding's component values."""
|
||||
return {int(component): freed for component, freed in tracker.items()}
|
||||
|
||||
|
||||
def _fill_evict_result(binding_result, result):
|
||||
"""Map a binding eviction step into an interface step result; both carry
|
||||
this step's per-component deltas and freed tensors."""
|
||||
for component, delta in binding_result.tracker.items():
|
||||
result.tracker[ComponentType(component)] = delta
|
||||
for component, tensors in binding_result.new_device_frees.items():
|
||||
result.device_frees[ComponentType(component)].extend(tensors)
|
||||
for component, tensors in binding_result.new_host_frees.items():
|
||||
result.host_frees[ComponentType(component)].extend(tensors)
|
||||
return result
|
||||
|
||||
|
||||
class _RustKVCacheEventRecorder:
|
||||
"""Expose the Rust event queue through the Python recorder interface."""
|
||||
|
||||
@@ -367,6 +311,70 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
_bindings = bindings
|
||||
supports_rotation_base = True
|
||||
|
||||
def _cache_action_from_tagged(self, action: tuple) -> CacheAction | ComponentAction:
|
||||
return _cache_action_from_tagged(action)
|
||||
|
||||
def _cache_actions_from_tagged(
|
||||
self, actions: Sequence[tuple]
|
||||
) -> list[CacheAction | ComponentAction]:
|
||||
return [self._cache_action_from_tagged(action) for action in actions]
|
||||
|
||||
def _insert_step_from_binding(self, step) -> InsertStepResult:
|
||||
"""Build the interface step for the binding's step (result on the final one)."""
|
||||
result = None
|
||||
if step.result is not None:
|
||||
# A stepped insert delivers all actions through steps, never the result.
|
||||
assert not step.result.cache_actions
|
||||
result = InsertResult(
|
||||
prefix_len=step.result.prefix_len,
|
||||
last_device_node=step.result.last_device_node,
|
||||
mamba_exist=step.result.mamba_exist,
|
||||
swa_branch_inserted=step.result.swa_branch_inserted,
|
||||
host_insert_dropped=step.result.host_insert_dropped,
|
||||
rotation_tail_declined=step.result.rotation_tail_declined,
|
||||
adopted_ranges=(
|
||||
{
|
||||
ComponentType(component_type): list(ranges)
|
||||
for component_type, ranges in step.result.adopted_ranges.items()
|
||||
}
|
||||
if step.result.adopted_ranges is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
return InsertStepResult(
|
||||
actions=self._cache_actions_from_tagged(step.actions), result=result
|
||||
)
|
||||
|
||||
def _match_result_from_binding(self, result) -> MatchResult:
|
||||
"""Build the Python MatchResult for the binding's match result."""
|
||||
return MatchResult(
|
||||
device_prefix_len=result.device_prefix_len,
|
||||
last_device_node=result.last_device_node_id,
|
||||
last_host_node=result.last_host_node_id,
|
||||
best_match_node=result.best_match_node_id,
|
||||
host_hit_length=result.host_hit_length,
|
||||
swa_host_hit_length=result.swa_host_hit_length,
|
||||
swa_branching_seqlen=result.swa_branching_seqlen,
|
||||
mamba_host_hit_length=result.mamba_host_hit_length,
|
||||
mamba_branching_seqlen=result.mamba_branching_seqlen,
|
||||
full_kv_hit_length=result.full_kv_hit_length,
|
||||
cache_actions=self._cache_actions_from_tagged(result.cache_actions),
|
||||
)
|
||||
|
||||
def _fill_evict_result(self, binding_result, result):
|
||||
"""Map a binding eviction step into an interface step result; both carry
|
||||
this step's deltas, frees, and actions."""
|
||||
for component, delta in binding_result.tracker.items():
|
||||
result.tracker[ComponentType(component)] = delta
|
||||
for component, tensors in binding_result.new_device_frees.items():
|
||||
result.device_frees[ComponentType(component)].extend(tensors)
|
||||
for component, tensors in binding_result.new_host_frees.items():
|
||||
result.host_frees[ComponentType(component)].extend(tensors)
|
||||
result.cache_actions.extend(
|
||||
self._cache_actions_from_tagged(binding_result.cache_actions)
|
||||
)
|
||||
return result
|
||||
|
||||
def __init__(self, params: CacheInitParams):
|
||||
assert params.tree_components is not None
|
||||
self.tree_components = tuple(params.tree_components)
|
||||
@@ -492,7 +500,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
dynamo_format=params.dynamo_kv_event_format,
|
||||
)
|
||||
# The default-root empty result, prebuilt once from the binding.
|
||||
self._empty_match_result = _match_result_from_binding(
|
||||
self._empty_match_result = self._match_result_from_binding(
|
||||
self._binding.empty_match_result()
|
||||
)
|
||||
|
||||
@@ -507,7 +515,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
def reset(self) -> None:
|
||||
self._binding.reset()
|
||||
# Node handles are never re-minted, so the fresh root gets a new one.
|
||||
self._empty_match_result = _match_result_from_binding(
|
||||
self._empty_match_result = self._match_result_from_binding(
|
||||
self._binding.empty_match_result()
|
||||
)
|
||||
|
||||
@@ -549,7 +557,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
) -> list[CacheAction | ComponentAction]:
|
||||
if component_type != ComponentType.SWA:
|
||||
raise NotImplementedError("The Rust tree core supports one SWA window")
|
||||
return _cache_actions_from_tagged(
|
||||
return self._cache_actions_from_tagged(
|
||||
self._binding.attach_swa_window(
|
||||
self._bindings.MatchParamsBinding(
|
||||
key=_radix_key_buffer(key),
|
||||
@@ -590,7 +598,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
params: DecLockRefParams,
|
||||
) -> DecSwaLockOnlyResult:
|
||||
result = DecSwaLockOnlyResult()
|
||||
new_device_frees, new_host_frees = self._binding.dec_window_lock_only(
|
||||
new_device_frees, new_host_frees, actions = self._binding.dec_window_lock_only(
|
||||
node_id,
|
||||
int(component_type),
|
||||
_dec_lock_ref_params_to_binding(self._bindings, params),
|
||||
@@ -599,6 +607,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
result.device_frees[ComponentType(component)].extend(tensors)
|
||||
for component, tensors in new_host_frees.items():
|
||||
result.host_frees[ComponentType(component)].extend(tensors)
|
||||
result.cache_actions.extend(self._cache_actions_from_tagged(actions))
|
||||
return result
|
||||
|
||||
def dec_swa_lock_only(
|
||||
@@ -607,13 +616,14 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
params: DecLockRefParams,
|
||||
) -> DecSwaLockOnlyResult:
|
||||
result = DecSwaLockOnlyResult()
|
||||
new_device_frees, new_host_frees = self._binding.dec_swa_lock_only(
|
||||
new_device_frees, new_host_frees, actions = self._binding.dec_swa_lock_only(
|
||||
node_id, _dec_lock_ref_params_to_binding(self._bindings, params)
|
||||
)
|
||||
for component, tensors in new_device_frees.items():
|
||||
result.device_frees[ComponentType(component)].extend(tensors)
|
||||
for component, tensors in new_host_frees.items():
|
||||
result.host_frees[ComponentType(component)].extend(tensors)
|
||||
result.cache_actions.extend(self._cache_actions_from_tagged(actions))
|
||||
return result
|
||||
|
||||
# ==== Device eviction (driven step-wise by the Controller's evict()) ====
|
||||
@@ -637,7 +647,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
swa_backup_node_id=binding_result.swa_backup_node_id,
|
||||
swa_backup_num_tokens=binding_result.swa_backup_num_tokens,
|
||||
)
|
||||
return _fill_evict_result(binding_result, result)
|
||||
return self._fill_evict_result(binding_result, result)
|
||||
|
||||
def finish_mamba_state_eviction(self, node_id: NodeId) -> EvictDeviceNextNodeResult:
|
||||
binding_result = self._binding.finish_mamba_state_eviction(node_id)
|
||||
@@ -646,7 +656,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
made_progress=binding_result.made_progress,
|
||||
unbacked_tokens=binding_result.unbacked_tokens,
|
||||
)
|
||||
return _fill_evict_result(binding_result, result)
|
||||
return self._fill_evict_result(binding_result, result)
|
||||
|
||||
def finish_swa_state_eviction(self, node_id: NodeId) -> EvictDeviceNextNodeResult:
|
||||
binding_result = self._binding.finish_swa_state_eviction(node_id)
|
||||
@@ -655,7 +665,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
made_progress=binding_result.made_progress,
|
||||
unbacked_tokens=binding_result.unbacked_tokens,
|
||||
)
|
||||
return _fill_evict_result(binding_result, result)
|
||||
return self._fill_evict_result(binding_result, result)
|
||||
|
||||
def evict_device_leaf(
|
||||
self, node_id: NodeId, is_write_back: bool
|
||||
@@ -668,13 +678,15 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
backup = binding_result.backup_kv
|
||||
result = EvictDeviceLeafResult(
|
||||
unbacked_tokens=binding_result.unbacked_tokens,
|
||||
backup_kv=_cache_action_from_tagged(backup) if backup is not None else None,
|
||||
backup_kv=self._cache_action_from_tagged(backup)
|
||||
if backup is not None
|
||||
else None,
|
||||
)
|
||||
return _fill_evict_result(binding_result, result)
|
||||
return self._fill_evict_result(binding_result, result)
|
||||
|
||||
def demote(self, node_id: NodeId) -> DemoteResult:
|
||||
binding_result = self._binding.demote(node_id)
|
||||
return _fill_evict_result(binding_result, DemoteResult())
|
||||
return self._fill_evict_result(binding_result, DemoteResult())
|
||||
|
||||
def evict_device_end(self, component_type: ComponentType) -> None:
|
||||
self._binding.evict_device_end(int(component_type))
|
||||
@@ -755,7 +767,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
cache_salt=key.cache_salt,
|
||||
)
|
||||
)
|
||||
return _match_result_from_binding(result)
|
||||
return self._match_result_from_binding(result)
|
||||
|
||||
def match_full_device_prefix(self, key: RadixKey) -> tuple[int, NodeId, int]:
|
||||
return self._binding.match_full_device_prefix(
|
||||
@@ -809,16 +821,16 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
rotation_base=params.rotation_base,
|
||||
)
|
||||
)
|
||||
return _insert_step_from_binding(step)
|
||||
return self._insert_step_from_binding(step)
|
||||
|
||||
def resume_insert(self) -> InsertStepResult:
|
||||
return _insert_step_from_binding(self._binding.resume_insert())
|
||||
return self._insert_step_from_binding(self._binding.resume_insert())
|
||||
|
||||
def has_ongoing_insert(self) -> bool:
|
||||
return self._binding.has_ongoing_insert()
|
||||
|
||||
def end_insert(self) -> list[CacheAction | ComponentAction]:
|
||||
return _cache_actions_from_tagged(self._binding.end_insert())
|
||||
return self._cache_actions_from_tagged(self._binding.end_insert())
|
||||
|
||||
def drive_host_eviction(
|
||||
self, component_type: ComponentType, num_tokens: int
|
||||
@@ -828,19 +840,23 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
num_tokens,
|
||||
envs.SGLANG_HICACHE_SKIP_HOST_DUPLICATE_RECLAIM.get(),
|
||||
)
|
||||
return _fill_evict_result(binding_result, DriveHostEvictionResult())
|
||||
return self._fill_evict_result(binding_result, DriveHostEvictionResult())
|
||||
|
||||
def evict_excess_path_states(
|
||||
self,
|
||||
tail_node_id: NodeId,
|
||||
device_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
host_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
cache_actions: list[CacheAction | ComponentAction],
|
||||
) -> None:
|
||||
binding_result = self._binding.evict_excess_path_states(tail_node_id)
|
||||
for component, tensors in binding_result.new_device_frees.items():
|
||||
device_frees[ComponentType(component)].extend(tensors)
|
||||
for component, tensors in binding_result.new_host_frees.items():
|
||||
host_frees[ComponentType(component)].extend(tensors)
|
||||
cache_actions.extend(
|
||||
self._cache_actions_from_tagged(binding_result.cache_actions)
|
||||
)
|
||||
|
||||
# ==== HiCache ====
|
||||
|
||||
@@ -931,9 +947,16 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
inserted_host_node=result.inserted_host_node,
|
||||
host_insert_dropped=result.host_insert_dropped,
|
||||
mamba_exist=result.mamba_exist,
|
||||
cache_actions=_cache_actions_from_tagged(result.cache_actions),
|
||||
cache_actions=self._cache_actions_from_tagged(result.cache_actions),
|
||||
)
|
||||
|
||||
def build_backup_kv_action(self, node_id: NodeId) -> BackupKV:
|
||||
action = self._cache_action_from_tagged(
|
||||
self._binding.build_backup_kv_action(node_id)
|
||||
)
|
||||
assert isinstance(action, BackupKV)
|
||||
return action
|
||||
|
||||
def build_backup_spec(
|
||||
self, node_id: NodeId
|
||||
) -> tuple[torch.Tensor, dict[ComponentType, list[PoolTransfer]]]:
|
||||
@@ -1123,7 +1146,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
)
|
||||
if insert_result is not None and mamba_exist is not None:
|
||||
insert_result.mamba_exist = mamba_exist
|
||||
cache_actions.extend(_cache_actions_from_tagged(actions))
|
||||
cache_actions.extend(self._cache_actions_from_tagged(actions))
|
||||
|
||||
def commit_backup(
|
||||
self,
|
||||
@@ -1148,12 +1171,12 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
_transfer_to_binding(kv_xfer),
|
||||
_comp_xfers_to_binding(comp_xfers),
|
||||
)
|
||||
return _cache_actions_from_tagged(actions)
|
||||
return self._cache_actions_from_tagged(actions)
|
||||
|
||||
def drop_subtree_no_host(self, node_id: NodeId) -> DropSubtreeNoHostResult:
|
||||
binding_result = self._binding.drop_subtree_no_host(node_id)
|
||||
result = DropSubtreeNoHostResult(is_dropped=binding_result.dropped)
|
||||
return _fill_evict_result(binding_result, result)
|
||||
return self._fill_evict_result(binding_result, result)
|
||||
|
||||
def mark_write_through_pending(
|
||||
self, node_ids: list[NodeId], ack_id: NodeId
|
||||
|
||||
@@ -528,6 +528,7 @@ class TreeComponent(ABC):
|
||||
node: UnifiedTreeNode,
|
||||
device_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
host_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
cache_actions: list[CacheAction | ComponentAction],
|
||||
target: EvictLayer = EvictLayer.DEVICE,
|
||||
) -> tuple[int, int]:
|
||||
"""Free this component's KV resources on a node being evicted.
|
||||
@@ -541,7 +542,8 @@ class TreeComponent(ABC):
|
||||
No tombstone — caller will delete the node.
|
||||
|
||||
Freed indices are collected into *device_frees*/*host_frees* for the
|
||||
Controller to drain, never freed inline.
|
||||
Controller to drain, never freed inline. Additional cache work is
|
||||
appended to *cache_actions* in the same operation.
|
||||
|
||||
Returns (device_freed, host_freed) token counts."""
|
||||
...
|
||||
@@ -586,6 +588,7 @@ class TreeComponent(ABC):
|
||||
tracker: dict[ComponentType, int],
|
||||
device_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
host_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
cache_actions: list[CacheAction | ComponentAction],
|
||||
) -> NodeId | InternalStateBackup | None:
|
||||
"""Return a device leaf, an internal backup request, or no selection.
|
||||
|
||||
@@ -596,7 +599,9 @@ class TreeComponent(ABC):
|
||||
assert self.is_evict_device_ongoing, (
|
||||
f"{self.component_type} device eviction not started"
|
||||
)
|
||||
return self._evict_device_next_node(tracker, device_frees, host_frees)
|
||||
return self._evict_device_next_node(
|
||||
tracker, device_frees, host_frees, cache_actions=cache_actions
|
||||
)
|
||||
|
||||
def evict_device_end(self) -> None:
|
||||
"""Clear this component's device-eviction walk state."""
|
||||
@@ -617,6 +622,7 @@ class TreeComponent(ABC):
|
||||
tracker: dict[ComponentType, int],
|
||||
device_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
host_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
cache_actions: list[CacheAction | ComponentAction],
|
||||
) -> NodeId | InternalStateBackup | None:
|
||||
"""Select a leaf, request backup, or perform at most one internal mutation."""
|
||||
...
|
||||
@@ -842,6 +848,7 @@ class TreeComponent(ABC):
|
||||
tracker: dict[ComponentType, int],
|
||||
device_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
host_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
cache_actions: list[CacheAction | ComponentAction],
|
||||
) -> None:
|
||||
"""Evict from this component's host-side resources, collecting freed
|
||||
values into *device_frees*/*host_frees* for the Controller to drain.
|
||||
|
||||
@@ -174,6 +174,7 @@ class FullComponent(TreeComponent):
|
||||
node: UnifiedTreeNode,
|
||||
device_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
host_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
cache_actions: list[CacheAction | ComponentAction],
|
||||
target: EvictLayer = EvictLayer.DEVICE,
|
||||
) -> tuple[int, int]:
|
||||
cd = node.component_data[self.component_type]
|
||||
@@ -220,6 +221,7 @@ class FullComponent(TreeComponent):
|
||||
tracker: dict[ComponentType, int],
|
||||
device_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
host_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
cache_actions: list[CacheAction | ComponentAction],
|
||||
) -> Optional[NodeId]:
|
||||
ct = self.component_type
|
||||
lv = self._evict_device_last_node
|
||||
@@ -251,6 +253,7 @@ class FullComponent(TreeComponent):
|
||||
tracker: dict[ComponentType, int],
|
||||
device_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
host_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
cache_actions: list[CacheAction | ComponentAction],
|
||||
) -> None:
|
||||
"""Evict host leaves to free KV host pool space."""
|
||||
self._ensure_eviction_strategy()
|
||||
@@ -264,7 +267,9 @@ class FullComponent(TreeComponent):
|
||||
_, x = heapq.heappop(heap)
|
||||
if x not in self.tree_core.evictable_host_leaves:
|
||||
continue
|
||||
self.tree_core._evict_host_leaf(x, tracker, device_frees, host_frees)
|
||||
self.tree_core._evict_host_leaf(
|
||||
x, tracker, device_frees, host_frees, cache_actions=cache_actions
|
||||
)
|
||||
if (
|
||||
x.parent is not None
|
||||
and x.parent in self.tree_core.evictable_host_leaves
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict
|
||||
from dataclasses import replace
|
||||
from typing import TYPE_CHECKING, Callable, Optional, Sequence
|
||||
|
||||
@@ -41,6 +40,9 @@ from sglang.srt.mem_cache.unified_cache.components.base import (
|
||||
TreeComponent,
|
||||
get_and_increase_time_counter,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import (
|
||||
BaseEvictionResult,
|
||||
)
|
||||
from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
mamba_cache_chunk_size,
|
||||
@@ -265,6 +267,7 @@ class MambaComponent(TreeComponent):
|
||||
tail: UnifiedTreeNode,
|
||||
device_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
host_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
cache_actions: list[CacheAction | ComponentAction],
|
||||
) -> None:
|
||||
"""Evict shallow eligible device checkpoints beyond the path cap.
|
||||
|
||||
@@ -304,8 +307,16 @@ class MambaComponent(TreeComponent):
|
||||
host_frees,
|
||||
target=EvictLayer.DEVICE,
|
||||
tracker=tracker,
|
||||
cache_actions=cache_actions,
|
||||
)
|
||||
self.tree_core._cascade_evict(
|
||||
node,
|
||||
self,
|
||||
tracker,
|
||||
device_frees,
|
||||
host_frees,
|
||||
cache_actions=cache_actions,
|
||||
)
|
||||
self.tree_core._cascade_evict(node, self, tracker, device_frees, host_frees)
|
||||
excess -= 1
|
||||
|
||||
def redistribute_on_node_split(
|
||||
@@ -325,6 +336,7 @@ class MambaComponent(TreeComponent):
|
||||
node: UnifiedTreeNode,
|
||||
device_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
host_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
cache_actions: list[CacheAction | ComponentAction],
|
||||
target: EvictLayer = EvictLayer.DEVICE,
|
||||
) -> tuple[int, int]:
|
||||
cd = node.component_data[self.component_type]
|
||||
@@ -375,6 +387,7 @@ class MambaComponent(TreeComponent):
|
||||
tracker: dict[ComponentType, int],
|
||||
device_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
host_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
cache_actions: list[CacheAction | ComponentAction],
|
||||
) -> NodeId | InternalStateBackup | None:
|
||||
"""Advance one device-eviction step and return a leaf, if selected.
|
||||
|
||||
@@ -429,9 +442,15 @@ class MambaComponent(TreeComponent):
|
||||
tracker=tracker,
|
||||
device_frees=device_frees,
|
||||
host_frees=host_frees,
|
||||
cache_actions=cache_actions,
|
||||
)
|
||||
self.tree_core._cascade_evict(
|
||||
x, self, tracker, device_frees=device_frees, host_frees=host_frees
|
||||
x,
|
||||
self,
|
||||
tracker,
|
||||
device_frees=device_frees,
|
||||
host_frees=host_frees,
|
||||
cache_actions=cache_actions,
|
||||
)
|
||||
if enabled:
|
||||
self._evict_device_cursor = lru.cursor_next()
|
||||
@@ -996,6 +1015,7 @@ class MambaComponent(TreeComponent):
|
||||
tracker: dict[ComponentType, int],
|
||||
device_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
host_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
cache_actions: list[CacheAction | ComponentAction],
|
||||
) -> None:
|
||||
"""Evict mamba host resources.
|
||||
Internal nodes: private tombstone (free host mamba only).
|
||||
@@ -1016,7 +1036,9 @@ class MambaComponent(TreeComponent):
|
||||
not enabled or self._can_evict_leaf_atomically(x)
|
||||
):
|
||||
# Host leaf: atomic eviction (all components host + delete)
|
||||
self.tree_core._evict_host_leaf(x, tracker, device_frees, host_frees)
|
||||
self.tree_core._evict_host_leaf(
|
||||
x, tracker, device_frees, host_frees, cache_actions=cache_actions
|
||||
)
|
||||
else:
|
||||
# Internal (or a leaf a session still pins): tombstone Mamba + cascade
|
||||
assert cd.host_value is not None
|
||||
@@ -1027,6 +1049,7 @@ class MambaComponent(TreeComponent):
|
||||
tracker=tracker,
|
||||
device_frees=device_frees,
|
||||
host_frees=host_frees,
|
||||
cache_actions=cache_actions,
|
||||
)
|
||||
self.tree_core._cascade_evict(
|
||||
x,
|
||||
@@ -1035,6 +1058,7 @@ class MambaComponent(TreeComponent):
|
||||
device_frees=device_frees,
|
||||
host_frees=host_frees,
|
||||
target=EvictLayer.HOST,
|
||||
cache_actions=cache_actions,
|
||||
)
|
||||
self.tree_core._update_evictable_leaf_sets(x)
|
||||
if enabled:
|
||||
@@ -1052,16 +1076,18 @@ class MambaComponent(TreeComponent):
|
||||
|
||||
def apply_component_action(self, action: ComponentAction) -> None:
|
||||
if isinstance(action, MambaEvictExcessPathStates):
|
||||
device_frees: dict[ComponentType, list[torch.Tensor]] = defaultdict(list)
|
||||
host_frees: dict[ComponentType, list[torch.Tensor]] = defaultdict(list)
|
||||
result = BaseEvictionResult()
|
||||
# Drain even if the walk raises so tombstoned slots are not leaked;
|
||||
# the walk runs behind the tree-core interface (Rust runs it natively).
|
||||
try:
|
||||
self.tree_core.evict_excess_path_states(
|
||||
action.tail_node_id, device_frees, host_frees
|
||||
action.tail_node_id,
|
||||
result.device_frees,
|
||||
result.host_frees,
|
||||
result.cache_actions,
|
||||
)
|
||||
finally:
|
||||
self.cache._free_values(device_frees, host_frees)
|
||||
self.cache._apply_eviction_result(result)
|
||||
return
|
||||
if isinstance(action, FreeComponentDeviceSlot):
|
||||
for indices in action.indices:
|
||||
|
||||
@@ -696,6 +696,7 @@ class SWAComponent(TreeComponent):
|
||||
node: UnifiedTreeNode,
|
||||
device_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
host_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
cache_actions: list[CacheAction | ComponentAction],
|
||||
target: EvictLayer = EvictLayer.DEVICE,
|
||||
) -> tuple[int, int]:
|
||||
ct = self.component_type
|
||||
@@ -755,6 +756,7 @@ class SWAComponent(TreeComponent):
|
||||
tracker: dict[ComponentType, int],
|
||||
device_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
host_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
cache_actions: list[CacheAction | ComponentAction],
|
||||
) -> NodeId | InternalStateBackup | None:
|
||||
"""Advance one device-eviction step and return a leaf, if selected.
|
||||
|
||||
@@ -824,9 +826,15 @@ class SWAComponent(TreeComponent):
|
||||
tracker=tracker,
|
||||
device_frees=device_frees,
|
||||
host_frees=host_frees,
|
||||
cache_actions=cache_actions,
|
||||
)
|
||||
self.tree_core._cascade_evict(
|
||||
x, self, tracker, device_frees=device_frees, host_frees=host_frees
|
||||
x,
|
||||
self,
|
||||
tracker,
|
||||
device_frees=device_frees,
|
||||
host_frees=host_frees,
|
||||
cache_actions=cache_actions,
|
||||
)
|
||||
if enabled:
|
||||
self._evict_device_cursor = lru.cursor_next()
|
||||
@@ -947,6 +955,7 @@ class SWAComponent(TreeComponent):
|
||||
swa_uuid_for_lock: Optional[int],
|
||||
device_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
host_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
cache_actions: list[CacheAction | ComponentAction],
|
||||
) -> None:
|
||||
"""Early-release the SWA lock along [node, swa_uuid_for_lock]; this
|
||||
method touches only SWA state. The wrapping ``dec_swa_lock_only`` also
|
||||
@@ -980,6 +989,7 @@ class SWAComponent(TreeComponent):
|
||||
target=EvictLayer.DEVICE,
|
||||
device_frees=device_frees,
|
||||
host_frees=host_frees,
|
||||
cache_actions=cache_actions,
|
||||
)
|
||||
|
||||
if swa_uuid_for_lock and cd.metadata.get("uuid") == swa_uuid_for_lock:
|
||||
@@ -1488,6 +1498,7 @@ class SWAComponent(TreeComponent):
|
||||
tracker: dict[ComponentType, int],
|
||||
device_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
host_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
cache_actions: list[CacheAction | ComponentAction],
|
||||
) -> None:
|
||||
"""Evict SWA host resources.
|
||||
Internal nodes: private tombstone (free SWA host only).
|
||||
@@ -1507,7 +1518,9 @@ class SWAComponent(TreeComponent):
|
||||
if x in self.tree_core.evictable_host_leaves and (
|
||||
not enabled or self._can_evict_leaf_atomically(x)
|
||||
):
|
||||
self.tree_core._evict_host_leaf(x, tracker, device_frees, host_frees)
|
||||
self.tree_core._evict_host_leaf(
|
||||
x, tracker, device_frees, host_frees, cache_actions=cache_actions
|
||||
)
|
||||
else:
|
||||
assert cd.host_value is not None
|
||||
self.tree_core._evict_component_and_detach_lru(
|
||||
@@ -1517,6 +1530,7 @@ class SWAComponent(TreeComponent):
|
||||
tracker=tracker,
|
||||
device_frees=device_frees,
|
||||
host_frees=host_frees,
|
||||
cache_actions=cache_actions,
|
||||
)
|
||||
self.tree_core._cascade_evict(
|
||||
x,
|
||||
@@ -1525,6 +1539,7 @@ class SWAComponent(TreeComponent):
|
||||
device_frees=device_frees,
|
||||
host_frees=host_frees,
|
||||
target=EvictLayer.HOST,
|
||||
cache_actions=cache_actions,
|
||||
)
|
||||
if enabled:
|
||||
x = host_lru.cursor_next(host_lock=True)
|
||||
|
||||
@@ -811,6 +811,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
params.get_lock_uuid(component_type),
|
||||
result.device_frees,
|
||||
result.host_frees,
|
||||
cache_actions=result.cache_actions,
|
||||
)
|
||||
return result
|
||||
|
||||
@@ -834,6 +835,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
params.get_lock_uuid(ComponentType.SWA),
|
||||
result.device_frees,
|
||||
result.host_frees,
|
||||
cache_actions=result.cache_actions,
|
||||
)
|
||||
|
||||
# Drop strictly-lower-priority locks co-located on the node, skipping
|
||||
@@ -1653,7 +1655,10 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
self._begin_tracking_unbacked_tokens()
|
||||
try:
|
||||
step = component.evict_device_next_node(
|
||||
updated_tracker, result.device_frees, result.host_frees
|
||||
updated_tracker,
|
||||
result.device_frees,
|
||||
result.host_frees,
|
||||
cache_actions=result.cache_actions,
|
||||
)
|
||||
finally:
|
||||
result.unbacked_tokens = self._finish_tracking_unbacked_tokens()
|
||||
@@ -1676,6 +1681,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
or result.mamba_backup_node_id is not None
|
||||
or result.swa_backup_node_id is not None
|
||||
or bool(result.tracker)
|
||||
or bool(result.cache_actions)
|
||||
or any(result.device_frees.values())
|
||||
or any(result.host_frees.values())
|
||||
)
|
||||
@@ -1738,6 +1744,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
tracker=result.tracker,
|
||||
device_frees=result.device_frees,
|
||||
host_frees=result.host_frees,
|
||||
cache_actions=result.cache_actions,
|
||||
)
|
||||
self._cascade_evict(
|
||||
node,
|
||||
@@ -1745,6 +1752,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
result.tracker,
|
||||
device_frees=result.device_frees,
|
||||
host_frees=result.host_frees,
|
||||
cache_actions=result.cache_actions,
|
||||
)
|
||||
finally:
|
||||
result.unbacked_tokens = self._finish_tracking_unbacked_tokens()
|
||||
@@ -1780,6 +1788,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
result.tracker,
|
||||
device_frees=result.device_frees,
|
||||
host_frees=result.host_frees,
|
||||
cache_actions=result.cache_actions,
|
||||
)
|
||||
return result
|
||||
self._demote(
|
||||
@@ -1787,6 +1796,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
result.tracker,
|
||||
device_frees=result.device_frees,
|
||||
host_frees=result.host_frees,
|
||||
cache_actions=result.cache_actions,
|
||||
)
|
||||
return result
|
||||
finally:
|
||||
@@ -1832,6 +1842,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
result.tracker,
|
||||
result.device_frees,
|
||||
result.host_frees,
|
||||
cache_actions=result.cache_actions,
|
||||
)
|
||||
self._remove_leaf_from_parent(desc)
|
||||
self._delete_unbacked_device_leaf(
|
||||
@@ -1839,6 +1850,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
result.tracker,
|
||||
device_frees=result.device_frees,
|
||||
host_frees=result.host_frees,
|
||||
cache_actions=result.cache_actions,
|
||||
)
|
||||
result.is_dropped = True
|
||||
return result
|
||||
@@ -1850,6 +1862,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
tracker: dict[ComponentType, int],
|
||||
device_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
host_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
cache_actions: list[CacheAction | ComponentAction],
|
||||
) -> None:
|
||||
"""Free every component layer on the node and detach it from the LRU
|
||||
lists and evictable leaf sets."""
|
||||
@@ -1862,6 +1875,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
tracker=tracker,
|
||||
device_frees=device_frees,
|
||||
host_frees=host_frees,
|
||||
cache_actions=cache_actions,
|
||||
)
|
||||
self.evictable_device_leaves.discard(node)
|
||||
self.evictable_host_leaves.discard(node)
|
||||
@@ -1872,16 +1886,26 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
tracker: dict[ComponentType, int],
|
||||
device_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
host_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
cache_actions: list[CacheAction | ComponentAction],
|
||||
) -> None:
|
||||
"""Delete a device leaf that has no host backup, freeing all layers."""
|
||||
self._release_all_component_layers(
|
||||
node, StorageMedium.GPU, tracker, device_frees, host_frees
|
||||
node,
|
||||
StorageMedium.GPU,
|
||||
tracker,
|
||||
device_frees,
|
||||
host_frees,
|
||||
cache_actions=cache_actions,
|
||||
)
|
||||
parent = node.parent
|
||||
self._remove_leaf_from_parent(node)
|
||||
self._update_evictable_leaf_sets(parent)
|
||||
self._iteratively_delete_tombstone_leaf(
|
||||
node, tracker, device_frees=device_frees, host_frees=host_frees
|
||||
node,
|
||||
tracker,
|
||||
device_frees=device_frees,
|
||||
host_frees=host_frees,
|
||||
cache_actions=cache_actions,
|
||||
)
|
||||
|
||||
def drive_host_eviction(
|
||||
@@ -1903,12 +1927,14 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
result.tracker,
|
||||
result.device_frees,
|
||||
result.host_frees,
|
||||
cache_actions=result.cache_actions,
|
||||
)
|
||||
comp.drive_host_eviction(
|
||||
num_tokens,
|
||||
result.tracker,
|
||||
result.device_frees,
|
||||
result.host_frees,
|
||||
cache_actions=result.cache_actions,
|
||||
)
|
||||
return result
|
||||
|
||||
@@ -1917,9 +1943,13 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
tail_node_id: NodeId,
|
||||
device_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
host_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
cache_actions: list[CacheAction | ComponentAction],
|
||||
) -> None:
|
||||
self.components_by_type[ComponentType.MAMBA]._evict_excess_path_states(
|
||||
self.node_by_id(tail_node_id), device_frees, host_frees
|
||||
self.node_by_id(tail_node_id),
|
||||
device_frees,
|
||||
host_frees,
|
||||
cache_actions=cache_actions,
|
||||
)
|
||||
|
||||
def _reclaim_full_host_duplicates(
|
||||
@@ -1928,6 +1958,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
tracker: dict[ComponentType, int],
|
||||
device_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
host_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
cache_actions: list[CacheAction | ComponentAction],
|
||||
) -> None:
|
||||
"""Reclaim Full host duplicates until num_tokens are freed; pass 1
|
||||
spares evictable D-leaves (imminent free demotes), pass 2 takes them."""
|
||||
@@ -1947,7 +1978,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
if not self._can_reclaim_full_host_duplicate(node):
|
||||
continue
|
||||
self._release_full_host_duplicate(
|
||||
node, tracker, device_frees, host_frees
|
||||
node, tracker, device_frees, host_frees, cache_actions=cache_actions
|
||||
)
|
||||
swept_ids.append(node.id) # released -> no longer a duplicate
|
||||
# Sweep after the walk: the dict must not be mutated mid-iteration.
|
||||
@@ -1973,6 +2004,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
tracker: dict[ComponentType, int],
|
||||
device_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
host_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
cache_actions: list[CacheAction | ComponentAction],
|
||||
) -> None:
|
||||
"""Free only the Full host layer; aux host slices stay under their own
|
||||
pools' LRU (a host-only aux slice may be a sole copy)."""
|
||||
@@ -1985,6 +2017,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
tracker=tracker,
|
||||
device_frees=device_frees,
|
||||
host_frees=host_frees,
|
||||
cache_actions=cache_actions,
|
||||
)
|
||||
self.write_back_duplicate_reclaim_digest = (
|
||||
self.write_back_duplicate_reclaim_digest * 1000003 + node.id + 1
|
||||
@@ -1996,6 +2029,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
tracker: dict[ComponentType, int],
|
||||
device_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
host_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
cache_actions: list[CacheAction | ComponentAction],
|
||||
) -> None:
|
||||
"""Atomically evict all components on a host leaf.
|
||||
|
||||
@@ -2011,11 +2045,14 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
tracker=None,
|
||||
device_frees=device_frees,
|
||||
host_frees=host_frees,
|
||||
cache_actions=cache_actions,
|
||||
)
|
||||
tracker[comp.component_type] += hf
|
||||
self.evictable_host_leaves.discard(node)
|
||||
self._remove_leaf_from_parent(node)
|
||||
self._iteratively_delete_tombstone_leaf(node, tracker, device_frees, host_frees)
|
||||
self._iteratively_delete_tombstone_leaf(
|
||||
node, tracker, device_frees, host_frees, cache_actions=cache_actions
|
||||
)
|
||||
|
||||
def demote(self, node_id: NodeId) -> DemoteResult:
|
||||
"""Release a node's device KV once its host copy exists; the node stays in the
|
||||
@@ -2026,6 +2063,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
result.tracker,
|
||||
result.device_frees,
|
||||
result.host_frees,
|
||||
cache_actions=result.cache_actions,
|
||||
)
|
||||
return result
|
||||
|
||||
@@ -2035,6 +2073,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
tracker: dict[ComponentType, int],
|
||||
device_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
host_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
cache_actions: list[CacheAction | ComponentAction],
|
||||
) -> None:
|
||||
assert not node.evicted and node.backuped
|
||||
trigger = self.components_by_type[BASE_COMPONENT_TYPE]
|
||||
@@ -2045,9 +2084,15 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
tracker=tracker,
|
||||
device_frees=device_frees,
|
||||
host_frees=host_frees,
|
||||
cache_actions=cache_actions,
|
||||
)
|
||||
self._cascade_evict(
|
||||
node, trigger, tracker, device_frees=device_frees, host_frees=host_frees
|
||||
node,
|
||||
trigger,
|
||||
tracker,
|
||||
device_frees=device_frees,
|
||||
host_frees=host_frees,
|
||||
cache_actions=cache_actions,
|
||||
)
|
||||
self.kv_events.record_remove(node, medium=StorageMedium.GPU)
|
||||
|
||||
@@ -2064,6 +2109,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
tracker: dict[ComponentType, int],
|
||||
device_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
host_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
cache_actions: list[CacheAction | ComponentAction],
|
||||
target: EvictLayer = EvictLayer.DEVICE,
|
||||
):
|
||||
"""Cascade eviction from trigger to lower-or-equal priority components."""
|
||||
@@ -2082,6 +2128,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
tracker=tracker,
|
||||
device_frees=device_frees,
|
||||
host_frees=host_frees,
|
||||
cache_actions=cache_actions,
|
||||
)
|
||||
if comp.component_type == BASE_COMPONENT_TYPE:
|
||||
base_evicted = True
|
||||
@@ -2159,12 +2206,17 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
comp: TreeComponent,
|
||||
device_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
host_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
cache_actions: list[CacheAction | ComponentAction],
|
||||
target: EvictLayer = EvictLayer.DEVICE,
|
||||
tracker: Optional[dict[ComponentType, int]] = None,
|
||||
) -> tuple[int, int]:
|
||||
had_host_copy = node.component_data[comp.component_type].host_value is not None
|
||||
device_freed, host_freed = comp.evict_component(
|
||||
node, target=target, device_frees=device_frees, host_frees=host_frees
|
||||
node,
|
||||
target=target,
|
||||
device_frees=device_frees,
|
||||
host_frees=host_frees,
|
||||
cache_actions=cache_actions,
|
||||
)
|
||||
if tracker is not None:
|
||||
if EvictLayer.DEVICE in target:
|
||||
@@ -2197,6 +2249,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
tracker: dict[ComponentType, int],
|
||||
device_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
host_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
cache_actions: list[CacheAction | ComponentAction],
|
||||
):
|
||||
"""Walk up from *deleted_node* and cascade-delete childless ancestors.
|
||||
|
||||
@@ -2206,7 +2259,11 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
- neither → evict all remaining data, delete, continue up
|
||||
"""
|
||||
self._iteratively_delete_tombstone_ancestors(
|
||||
deleted_node.parent, tracker, device_frees, host_frees
|
||||
deleted_node.parent,
|
||||
tracker,
|
||||
device_frees,
|
||||
host_frees,
|
||||
cache_actions=cache_actions,
|
||||
)
|
||||
|
||||
def _iteratively_delete_tombstone_ancestors(
|
||||
@@ -2215,6 +2272,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
tracker: dict[ComponentType, int],
|
||||
device_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
host_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
cache_actions: list[CacheAction | ComponentAction],
|
||||
) -> None:
|
||||
"""Delete childless tombstone ancestors until a live or locked node is reached."""
|
||||
ct = BASE_COMPONENT_TYPE
|
||||
@@ -2241,6 +2299,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
tracker=tracker,
|
||||
device_frees=device_frees,
|
||||
host_frees=host_frees,
|
||||
cache_actions=cache_actions,
|
||||
)
|
||||
|
||||
if has_host:
|
||||
@@ -2257,6 +2316,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
tracker=tracker,
|
||||
device_frees=device_frees,
|
||||
host_frees=host_frees,
|
||||
cache_actions=cache_actions,
|
||||
)
|
||||
|
||||
self.evictable_host_leaves.discard(cur)
|
||||
|
||||
@@ -23,7 +23,7 @@ class BaseEvictionResult(msgspec.Struct):
|
||||
"""Base of the eviction step results: the component-keyed device/host
|
||||
values the step freed, for the Controller to drain right after the call
|
||||
returns, plus `tracker` -- the step's per-component evicted counts for the
|
||||
Controller to accumulate."""
|
||||
Controller to accumulate. Free the values, then drain cache actions."""
|
||||
|
||||
device_frees: dict[ComponentType, list[torch.Tensor]] = msgspec.field(
|
||||
default_factory=lambda: defaultdict(list)
|
||||
@@ -34,12 +34,15 @@ class BaseEvictionResult(msgspec.Struct):
|
||||
tracker: dict[ComponentType, int] = msgspec.field(
|
||||
default_factory=lambda: defaultdict(int)
|
||||
)
|
||||
cache_actions: list[CacheAction | ComponentAction] = msgspec.field(
|
||||
default_factory=list
|
||||
)
|
||||
|
||||
def __del__(self) -> None:
|
||||
# Drop tripwire: every returned value must be drained before disposal.
|
||||
assert not self.device_frees and not self.host_frees, (
|
||||
"BaseEvictionResult dropped with undrained values"
|
||||
)
|
||||
# Drop tripwire: every returned resource must be drained before disposal.
|
||||
assert (
|
||||
not self.device_frees and not self.host_frees and not self.cache_actions
|
||||
), "BaseEvictionResult dropped with undrained resources"
|
||||
|
||||
|
||||
class EvictDeviceNextNodeResult(BaseEvictionResult):
|
||||
@@ -548,9 +551,11 @@ class UnifiedTreeCoreInterface(ABC):
|
||||
tail_node_id: NodeId,
|
||||
device_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
host_frees: dict[ComponentType, list[torch.Tensor]],
|
||||
cache_actions: list[CacheAction | ComponentAction],
|
||||
) -> None:
|
||||
"""Evict shallow Mamba device checkpoints beyond the per-path cap on the
|
||||
tail's root path, collecting freed values into the caller's dicts."""
|
||||
tail's root path, collecting freed values and actions into the caller's
|
||||
buffers. The caller frees values, then drains actions, even on error."""
|
||||
...
|
||||
|
||||
# ==== HiCache ====
|
||||
|
||||
@@ -120,6 +120,9 @@ if TYPE_CHECKING:
|
||||
from sglang.srt.managers.cache_controller import HiCacheAck
|
||||
from sglang.srt.managers.schedule_batch import Req, ReqKvInfo
|
||||
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
|
||||
from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import (
|
||||
BaseEvictionResult,
|
||||
)
|
||||
from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import (
|
||||
PrefetchOperation,
|
||||
)
|
||||
@@ -825,6 +828,13 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
finally:
|
||||
self._drain_host_frees(host_frees)
|
||||
|
||||
def _apply_eviction_result(self, result: BaseEvictionResult) -> None:
|
||||
"""Drain a tree step before the next allocator-dependent decision."""
|
||||
try:
|
||||
self._free_values(result.device_frees, result.host_frees)
|
||||
finally:
|
||||
self._apply_cache_actions(result.cache_actions)
|
||||
|
||||
def _accumulate_tracker(
|
||||
self,
|
||||
tracker: dict[ComponentType, int],
|
||||
@@ -842,7 +852,10 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
if result.mamba_backup_node_id is not None:
|
||||
assert component_type == ComponentType.MAMBA and result.node_id is None
|
||||
assert (
|
||||
not result.device_frees and not result.host_frees and not result.tracker
|
||||
not result.device_frees
|
||||
and not result.host_frees
|
||||
and not result.tracker
|
||||
and not result.cache_actions
|
||||
)
|
||||
# Reserve a host state slot and wait for the backup acknowledgment
|
||||
# before freeing device state. If allocation fails, eviction still
|
||||
@@ -856,7 +869,10 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
elif result.swa_backup_node_id is not None:
|
||||
assert component_type == ComponentType.SWA and result.node_id is None
|
||||
assert (
|
||||
not result.device_frees and not result.host_frees and not result.tracker
|
||||
not result.device_frees
|
||||
and not result.host_frees
|
||||
and not result.tracker
|
||||
and not result.cache_actions
|
||||
)
|
||||
# The backup can cover several unbacked SWA segments. Reserve host
|
||||
# space for the whole window before copying it, then resume eviction
|
||||
@@ -868,7 +884,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
self.evict_host(needed, ComponentType.SWA)
|
||||
self.backup_node_for_write_back(node_id)
|
||||
result = self.tree_core.finish_swa_state_eviction(node_id)
|
||||
self._free_values(result.device_frees, result.host_frees)
|
||||
self._apply_eviction_result(result)
|
||||
if self._tracks_write_through_unbacked_evictions():
|
||||
self._record_dropped_tokens(
|
||||
result.unbacked_tokens,
|
||||
@@ -883,7 +899,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
"""Evict one device leaf, consuming its step result; returns the
|
||||
deferred write-back BackupKV when one must run before the demote."""
|
||||
result = self.tree_core.evict_device_leaf(node_id, self.is_write_back)
|
||||
self._free_values(result.device_frees, result.host_frees)
|
||||
self._apply_eviction_result(result)
|
||||
if self._tracks_write_through_unbacked_evictions():
|
||||
self._record_dropped_tokens(
|
||||
result.unbacked_tokens,
|
||||
@@ -895,7 +911,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
def _demote(self, node_id: NodeId, tracker: dict[ComponentType, int]) -> None:
|
||||
"""Demote a backed-up node, consuming its step result."""
|
||||
result = self.tree_core.demote(node_id)
|
||||
self._free_values(result.device_frees, result.host_frees)
|
||||
self._apply_eviction_result(result)
|
||||
self._accumulate_tracker(tracker, result.tracker)
|
||||
|
||||
def _drop_subtree_no_host(
|
||||
@@ -903,7 +919,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
) -> bool:
|
||||
"""Run the write-back drop fallback, consuming its step result."""
|
||||
result = self.tree_core.drop_subtree_no_host(node_id)
|
||||
self._free_values(result.device_frees, result.host_frees)
|
||||
self._apply_eviction_result(result)
|
||||
if result.is_dropped:
|
||||
self._record_dropped_tokens(
|
||||
result.tracker.get(BASE_COMPONENT_TYPE, 0), reason="host_pressure"
|
||||
@@ -1072,7 +1088,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
if self.disable or lock is None or lock.swa_released:
|
||||
return
|
||||
result = self.tree_core.dec_swa_lock_only(lock.node, lock.receipt)
|
||||
self._free_values(result.device_frees, result.host_frees)
|
||||
self._apply_eviction_result(result)
|
||||
lock.swa_released = True
|
||||
|
||||
def release_swa_prefix_lock(self, req: Req) -> None:
|
||||
@@ -1401,7 +1417,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
# is operation-owned (freed at each ack): nothing is evictable.
|
||||
return 0
|
||||
result = self.tree_core.drive_host_eviction(component_type, num_tokens)
|
||||
self._free_values(result.device_frees, result.host_frees)
|
||||
self._apply_eviction_result(result)
|
||||
return result.tracker.get(component_type, 0)
|
||||
|
||||
# ---- Decode retraction ----
|
||||
|
||||
@@ -51,7 +51,12 @@ class FakeFullComponent(TreeComponent):
|
||||
return None
|
||||
|
||||
def evict_component(
|
||||
self, node, device_frees, host_frees, target: EvictLayer = EvictLayer.DEVICE
|
||||
self,
|
||||
node,
|
||||
device_frees,
|
||||
host_frees,
|
||||
cache_actions,
|
||||
target: EvictLayer = EvictLayer.DEVICE,
|
||||
) -> tuple[int, int]:
|
||||
return 0, 0
|
||||
|
||||
@@ -64,7 +69,7 @@ class FakeFullComponent(TreeComponent):
|
||||
def _evict_device_start(self, request_cnt) -> None:
|
||||
pass
|
||||
|
||||
def _evict_device_next_node(self, tracker, device_frees, host_frees):
|
||||
def _evict_device_next_node(self, tracker, device_frees, host_frees, cache_actions):
|
||||
return None
|
||||
|
||||
def _evict_device_end(self) -> None:
|
||||
|
||||
@@ -9,8 +9,6 @@ import torch
|
||||
from sglang.srt.mem_cache.base_prefix_cache import MatchPrefixParams, MatchResult
|
||||
from sglang.srt.mem_cache.rust_tree_core.adapter import (
|
||||
RustUnifiedTreeCore,
|
||||
_fill_evict_result,
|
||||
_match_result_from_binding,
|
||||
_radix_key_buffer,
|
||||
)
|
||||
from sglang.srt.mem_cache.rust_tree_core.extension import load_tree_core_extension
|
||||
@@ -189,7 +187,7 @@ class RustUnifiedTreeCoreInspector(
|
||||
binding_result = self._binding.inspect_evict_component(
|
||||
node_id, int(component_type), int(target)
|
||||
)
|
||||
return _fill_evict_result(binding_result, BaseEvictionResult())
|
||||
return self._fill_evict_result(binding_result, BaseEvictionResult())
|
||||
|
||||
def validate_cascade_evict(
|
||||
self,
|
||||
@@ -203,7 +201,7 @@ class RustUnifiedTreeCoreInspector(
|
||||
|
||||
def cleanup_tombstone_ancestors(self, node_id: NodeId) -> BaseEvictionResult:
|
||||
binding_result = self._binding.inspect_cleanup_tombstone_ancestors(node_id)
|
||||
return _fill_evict_result(binding_result, BaseEvictionResult())
|
||||
return self._fill_evict_result(binding_result, BaseEvictionResult())
|
||||
|
||||
def finalize_component_match_result(
|
||||
self,
|
||||
@@ -222,7 +220,7 @@ class RustUnifiedTreeCoreInspector(
|
||||
value_chunks,
|
||||
best_value_len,
|
||||
)
|
||||
return _match_result_from_binding(binding_result)._replace(
|
||||
return self._match_result_from_binding(binding_result)._replace(
|
||||
cache_protected_len=result.cache_protected_len,
|
||||
cache_actions=result.cache_actions,
|
||||
)
|
||||
|
||||
@@ -223,6 +223,7 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa
|
||||
self.components_by_type[component_type],
|
||||
result.device_frees,
|
||||
result.host_frees,
|
||||
result.cache_actions,
|
||||
target,
|
||||
result.tracker,
|
||||
)
|
||||
@@ -249,6 +250,7 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa
|
||||
result.tracker,
|
||||
result.device_frees,
|
||||
result.host_frees,
|
||||
result.cache_actions,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
Reference in New Issue
Block a user