[HiCache] Demote internal-node mamba states on write_back eviction (#40680)
This commit is contained in:
@@ -26,6 +26,7 @@ from sglang.srt.mem_cache.unified_cache.cache_action import (
|
||||
MambaEvictExcessPathStates,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_cache.components.base import (
|
||||
BASE_COMPONENT_TYPE,
|
||||
CacheTransferPhase,
|
||||
ComponentType,
|
||||
EvictLayer,
|
||||
@@ -404,6 +405,8 @@ class MambaComponent(TreeComponent):
|
||||
return x.id
|
||||
if not enabled:
|
||||
x_next = lru.get_prev_no_lock(x)
|
||||
# write_back: demote the state to host before the internal tombstone.
|
||||
self._maybe_backup_node_before_state_tombstone(x)
|
||||
self.tree_core._evict_component_and_detach_lru(
|
||||
x,
|
||||
self,
|
||||
@@ -418,6 +421,35 @@ class MambaComponent(TreeComponent):
|
||||
self._evict_device_cursor = lru.cursor_next() if enabled else x_next
|
||||
return None
|
||||
|
||||
def _maybe_backup_node_before_state_tombstone(self, node: UnifiedTreeNode) -> None:
|
||||
"""Demote an internal node's mamba state to host before its tombstone
|
||||
(write_back only), mirroring the leaf deferred-demote path.
|
||||
|
||||
The match validator passes only nodes holding the state on some
|
||||
layer, so a dropped internal state caps the match frontier at this
|
||||
node forever, leaving the subtree's still-resident KV unservable.
|
||||
Best-effort: this walk must make progress (it satisfies an imminent
|
||||
slot allocation), so any failure falls back to the legacy drop.
|
||||
"""
|
||||
cache = self.cache
|
||||
cd = node.component_data[self.component_type]
|
||||
if (
|
||||
cache.cache_controller is None
|
||||
or not cache.is_write_back
|
||||
or cd.host_value is not None
|
||||
or node.backuped
|
||||
or node.component_data[BASE_COMPONENT_TYPE].value is None
|
||||
):
|
||||
return
|
||||
# The backup executor pre-evicts only the KV host pool; make room
|
||||
# for the state slot the way the PREFETCH hook does.
|
||||
if (
|
||||
self._mamba_pool_host is not None
|
||||
and self._mamba_pool_host.available_size() < 1
|
||||
):
|
||||
cache.evict_host(1, self.component_type)
|
||||
cache.backup_node_for_write_back(node.id)
|
||||
|
||||
def _evict_device_end(self) -> None:
|
||||
"""Clear the device-eviction walk cursor state."""
|
||||
if self.tree_core.enable_session_radix_cache:
|
||||
|
||||
@@ -1538,6 +1538,20 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
|
||||
# ---- HiCache: Backup / LoadBack ----
|
||||
|
||||
def backup_node_for_write_back(self, node_id: NodeId) -> bool:
|
||||
"""Synchronously back up one node (Full KV plus any unbacked component
|
||||
state) and drain the ack -- the deferred-demote shape the eviction
|
||||
loop runs for leaves, reusable by component evictors ahead of an
|
||||
internal-state tombstone. Returns True once the backup is committed.
|
||||
"""
|
||||
written = self._execute_and_commit_kv_backup(
|
||||
BackupKV(node_ids=[node_id]), write_back=True
|
||||
)
|
||||
if written == 0:
|
||||
return False
|
||||
self.writing_check(write_back=True)
|
||||
return True
|
||||
|
||||
def _execute_and_commit_kv_backup(
|
||||
self, action: BackupKV, write_back: bool = False
|
||||
) -> int:
|
||||
|
||||
Reference in New Issue
Block a user