[HiCache] Demote internal-node mamba states on write_back eviction (#40680)

This commit is contained in:
Shangming Cai
2026-09-23 02:18:58 +08:00
committed by GitHub
parent 91c329cc86
commit 6f4c2b9b91
3 changed files with 140 additions and 0 deletions
@@ -26,6 +26,7 @@ from sglang.srt.mem_cache.unified_cache.cache_action import (
MambaEvictExcessPathStates,
)
from sglang.srt.mem_cache.unified_cache.components.base import (
BASE_COMPONENT_TYPE,
CacheTransferPhase,
ComponentType,
EvictLayer,
@@ -404,6 +405,8 @@ class MambaComponent(TreeComponent):
return x.id
if not enabled:
x_next = lru.get_prev_no_lock(x)
# write_back: demote the state to host before the internal tombstone.
self._maybe_backup_node_before_state_tombstone(x)
self.tree_core._evict_component_and_detach_lru(
x,
self,
@@ -418,6 +421,35 @@ class MambaComponent(TreeComponent):
self._evict_device_cursor = lru.cursor_next() if enabled else x_next
return None
def _maybe_backup_node_before_state_tombstone(self, node: UnifiedTreeNode) -> None:
"""Demote an internal node's mamba state to host before its tombstone
(write_back only), mirroring the leaf deferred-demote path.
The match validator passes only nodes holding the state on some
layer, so a dropped internal state caps the match frontier at this
node forever, leaving the subtree's still-resident KV unservable.
Best-effort: this walk must make progress (it satisfies an imminent
slot allocation), so any failure falls back to the legacy drop.
"""
cache = self.cache
cd = node.component_data[self.component_type]
if (
cache.cache_controller is None
or not cache.is_write_back
or cd.host_value is not None
or node.backuped
or node.component_data[BASE_COMPONENT_TYPE].value is None
):
return
# The backup executor pre-evicts only the KV host pool; make room
# for the state slot the way the PREFETCH hook does.
if (
self._mamba_pool_host is not None
and self._mamba_pool_host.available_size() < 1
):
cache.evict_host(1, self.component_type)
cache.backup_node_for_write_back(node.id)
def _evict_device_end(self) -> None:
"""Clear the device-eviction walk cursor state."""
if self.tree_core.enable_session_radix_cache:
@@ -1538,6 +1538,20 @@ class UnifiedRadixCache(BasePrefixCache):
# ---- HiCache: Backup / LoadBack ----
def backup_node_for_write_back(self, node_id: NodeId) -> bool:
"""Synchronously back up one node (Full KV plus any unbacked component
state) and drain the ack -- the deferred-demote shape the eviction
loop runs for leaves, reusable by component evictors ahead of an
internal-state tombstone. Returns True once the backup is committed.
"""
written = self._execute_and_commit_kv_backup(
BackupKV(node_ids=[node_id]), write_back=True
)
if written == 0:
return False
self.writing_check(write_back=True)
return True
def _execute_and_commit_kv_backup(
self, action: BackupKV, write_back: bool = False
) -> int: