[Refactor] radix-cache: Route pending-backup split bookkeeping through one component action (#45242)

Co-authored-by: metamergebot <metamergebot@users.noreply.github.com>
Co-authored-by: Jialin <Jialin@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-20 13:24:49 -07:00
committed by GitHub
co-authored by metamergebot Jialin
parent 0b536ca112
commit cc7ae372ac
16 changed files with 387 additions and 164 deletions
@@ -40,8 +40,8 @@ from sglang.srt.mem_cache.unified_cache.cache_action import (
MambaEvictExcessPathStates,
RebuildFullToSWAMapping,
RecoverSWAWithLockedFull,
ReplaceWriteThroughOnNodeSplit,
SWARebuild,
UpdatePendingBackupOnNodeSplit,
)
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
from sglang.srt.mem_cache.unified_cache.components import CacheTransferPhase
@@ -164,8 +164,8 @@ def _cache_action_from_tagged(action: tuple) -> CacheAction:
return BackupKV(node_ids=list(action[1]))
if tag == "mamba_evict_excess_path_states":
return MambaEvictExcessPathStates(tail_node_id=action[1])
if tag == "replace_write_through_on_node_split":
return ReplaceWriteThroughOnNodeSplit(
if tag == "update_pending_backup_on_node_split":
return UpdatePendingBackupOnNodeSplit(
ack_id=action[1],
old_node_id=action[2],
new_node_id=action[3],
@@ -16,20 +16,6 @@ if TYPE_CHECKING:
from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import NodeId
class ReplaceWriteThroughOnNodeSplit(msgspec.Struct, frozen=True):
"""Replace the pending write-through node on a node split:
parent -> node => parent -> new_node -> new_child
old_node_id (the pre-split node) is replaced by new_node_id + new_child_node_id.
"""
ack_id: int
old_node_id: NodeId
new_node_id: NodeId
new_child_node_id: NodeId
class FreeDeviceKV(msgspec.Struct, frozen=True):
"""Free unreferenced device KV slots (a SWA-aware combined free)."""
@@ -49,6 +35,22 @@ class ComponentAction(msgspec.Struct, frozen=True):
declares its ``component_type`` field."""
class UpdatePendingBackupOnNodeSplit(ComponentAction, frozen=True):
"""Update pending backups for the prefix and child produced by a split.
The cache first relocates write-through/linker publication lists when
``ack_id`` is present, then routes this same action once to FULL for
component-owned buffer-write dependencies. Buffer-only splits may have
no tree acknowledgment.
"""
ack_id: int | None
old_node_id: NodeId
new_node_id: NodeId
new_child_node_id: NodeId
component_type: ComponentType = ComponentType.FULL
class FreeComponentDeviceSlot(ComponentAction, frozen=True):
"""Free only the given ``component_type``'s device KV slots."""
@@ -115,7 +117,7 @@ class SWARebuild(ComponentAction, frozen=True):
component_type: ComponentType = ComponentType.SWA
# Cache-owned actions, applied by UnifiedRadixCache itself.
# The cache handles these directly; split updates also reach their component.
CacheAction = (
ReplaceWriteThroughOnNodeSplit | FreeDeviceKV | FreeDeviceKVFullOnly | BackupKV
UpdatePendingBackupOnNodeSplit | FreeDeviceKV | FreeDeviceKVFullOnly | BackupKV
)
@@ -18,7 +18,10 @@ from sglang.srt.mem_cache.hicache_storage import (
PoolTransfer,
PoolTransferResult,
)
from sglang.srt.mem_cache.unified_cache.cache_action import FreeComponentDeviceSlot
from sglang.srt.mem_cache.unified_cache.cache_action import (
FreeComponentDeviceSlot,
UpdatePendingBackupOnNodeSplit,
)
from sglang.srt.mem_cache.unified_cache.components.base import (
CacheTransferPhase,
ComponentType,
@@ -527,7 +530,10 @@ class FullComponent(TreeComponent):
self.cache.host_pool_group.free(host_value, pool=PoolName.KV)
def apply_component_action(self, action: ComponentAction) -> None:
if isinstance(action, FreeComponentDeviceSlot):
if isinstance(action, UpdatePendingBackupOnNodeSplit):
# Custom FULL components track buffer-write dependencies.
pass
elif isinstance(action, FreeComponentDeviceSlot):
alloc = self.cache.token_to_kv_pool_allocator
for indices in action.indices:
# tree values are page-aligned copies of a kv row: page-exact segments
@@ -535,7 +541,7 @@ class FullComponent(TreeComponent):
alloc.full_attn_allocator.free_segment(indices, start_pos=0)
else:
alloc.free_segment(indices, start_pos=0)
return
raise AssertionError(
f"FullComponent: unhandled ComponentAction {type(action).__name__}"
)
else:
raise AssertionError(
f"FullComponent: unhandled ComponentAction {type(action).__name__}"
)
@@ -525,7 +525,11 @@ class SWAComponent(TreeComponent):
)
if live_start is None:
continue
parent = self._maybe_split_leaf_for_swa_lock(fragment) if cap_leaf else None
parent = (
self._maybe_split_leaf_for_swa_lock(fragment, cache_actions)
if cap_leaf
else None
)
# Older fragments publish first so the tail remains more-MRU.
if parent is not None:
self._publish_window(node=parent, cache_actions=cache_actions)
@@ -587,7 +591,9 @@ class SWAComponent(TreeComponent):
)
def _maybe_split_leaf_for_swa_lock(
self, leaf: UnifiedTreeNode
self,
leaf: UnifiedTreeNode,
cache_actions: list[CacheAction | ComponentAction],
) -> Optional[UnifiedTreeNode]:
"""Cap a fresh in-window SWA leaf at one page-aligned window so locking it pins
only one window of SWA pool, not the whole (long chunked-prefill) leaf; return
@@ -610,7 +616,11 @@ class SWAComponent(TreeComponent):
return None
new_parent, action = self.tree_core._split_node(leaf.key, leaf, split_at)
assert action is None, "fresh SWA leaf cannot be write-through-pending"
assert action is None or action.ack_id is None, (
"fresh SWA leaf cannot be write-through-pending"
)
if action is not None:
cache_actions.append(action)
return new_parent
def redistribute_on_node_split(
@@ -51,7 +51,7 @@ from sglang.srt.mem_cache.unified_cache.cache_action import (
ComponentAction,
FreeDeviceKV,
FreeDeviceKVFullOnly,
ReplaceWriteThroughOnNodeSplit,
UpdatePendingBackupOnNodeSplit,
)
from sglang.srt.mem_cache.unified_cache.components import (
_NUM_COMPONENT_TYPES,
@@ -1270,7 +1270,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
"""Fire-and-forget actions safe to batch until the next barrier."""
return isinstance(
action,
(FreeDeviceKV, FreeDeviceKVFullOnly, ReplaceWriteThroughOnNodeSplit),
(FreeDeviceKV, FreeDeviceKVFullOnly, UpdatePendingBackupOnNodeSplit),
)
def _insert_walk_step(self, state: _InsertWalkState) -> None:
@@ -1479,12 +1479,13 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
component.redistribute_on_node_split(new_parent=new_node, child=child)
new_node.parent.children[key.child_key(self.page_size)] = new_node
# A split of a backuped node tells the cache to fix its publish list.
# Buffer-mode dependencies are tracked by the cache, without an ack here.
action: Optional[CacheAction | ComponentAction] = None
if child.write_through_pending_id is not None:
ack_id = child.write_through_pending_id
ack_id = child.write_through_pending_id
if ack_id is not None:
new_node.write_through_pending_id = ack_id
action = ReplaceWriteThroughOnNodeSplit(
if ack_id is not None or self.is_host_memory_buffer_only:
action = UpdatePendingBackupOnNodeSplit(
ack_id=ack_id,
old_node_id=child.id,
new_node_id=new_node.id,
@@ -74,7 +74,7 @@ from sglang.srt.mem_cache.unified_cache.cache_action import (
FreeComponentDeviceSlot,
FreeDeviceKV,
FreeDeviceKVFullOnly,
ReplaceWriteThroughOnNodeSplit,
UpdatePendingBackupOnNodeSplit,
)
# UnifiedTreeNode / UnifiedLRUList live on the tree core; re-exported here
@@ -1348,19 +1348,19 @@ class UnifiedRadixCache(BasePrefixCache):
# Component actions route to their component class; the rest are
# cache-owned and handled here by type.
if isinstance(action, ComponentAction):
self.components[action.component_type].apply_component_action(action)
elif isinstance(action, ReplaceWriteThroughOnNodeSplit):
self._replace_pending_write_through_node(
action.ack_id,
action.old_node_id,
[action.new_node_id, action.new_child_node_id],
)
if self.linker is not None:
self.linker.replace_pending_offload_node(
action.ack_id,
action.old_node_id,
[action.new_node_id, action.new_child_node_id],
if (
isinstance(action, UpdatePendingBackupOnNodeSplit)
and action.ack_id is not None
):
node_ids = [action.new_node_id, action.new_child_node_id]
self._replace_pending_write_through_node(
action.ack_id, action.old_node_id, node_ids
)
if self.linker is not None:
self.linker.replace_pending_offload_node(
action.ack_id, action.old_node_id, node_ids
)
self.components[action.component_type].apply_component_action(action)
elif isinstance(action, FreeDeviceKV):
# tree values are page-aligned copies of a kv row: page-exact segments
for indices in action.indices: