[Refactor] radix-cache: Route pending-backup split bookkeeping through one component action (#45242)
Co-authored-by: metamergebot <metamergebot@users.noreply.github.com> Co-authored-by: Jialin <Jialin@users.noreply.github.com>
This commit is contained in:
co-authored by
metamergebot
Jialin
parent
0b536ca112
commit
cc7ae372ac
@@ -40,8 +40,8 @@ from sglang.srt.mem_cache.unified_cache.cache_action import (
|
||||
MambaEvictExcessPathStates,
|
||||
RebuildFullToSWAMapping,
|
||||
RecoverSWAWithLockedFull,
|
||||
ReplaceWriteThroughOnNodeSplit,
|
||||
SWARebuild,
|
||||
UpdatePendingBackupOnNodeSplit,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
|
||||
from sglang.srt.mem_cache.unified_cache.components import CacheTransferPhase
|
||||
@@ -164,8 +164,8 @@ def _cache_action_from_tagged(action: tuple) -> CacheAction:
|
||||
return BackupKV(node_ids=list(action[1]))
|
||||
if tag == "mamba_evict_excess_path_states":
|
||||
return MambaEvictExcessPathStates(tail_node_id=action[1])
|
||||
if tag == "replace_write_through_on_node_split":
|
||||
return ReplaceWriteThroughOnNodeSplit(
|
||||
if tag == "update_pending_backup_on_node_split":
|
||||
return UpdatePendingBackupOnNodeSplit(
|
||||
ack_id=action[1],
|
||||
old_node_id=action[2],
|
||||
new_node_id=action[3],
|
||||
|
||||
@@ -16,20 +16,6 @@ if TYPE_CHECKING:
|
||||
from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import NodeId
|
||||
|
||||
|
||||
class ReplaceWriteThroughOnNodeSplit(msgspec.Struct, frozen=True):
|
||||
"""Replace the pending write-through node on a node split:
|
||||
|
||||
parent -> node => parent -> new_node -> new_child
|
||||
|
||||
old_node_id (the pre-split node) is replaced by new_node_id + new_child_node_id.
|
||||
"""
|
||||
|
||||
ack_id: int
|
||||
old_node_id: NodeId
|
||||
new_node_id: NodeId
|
||||
new_child_node_id: NodeId
|
||||
|
||||
|
||||
class FreeDeviceKV(msgspec.Struct, frozen=True):
|
||||
"""Free unreferenced device KV slots (a SWA-aware combined free)."""
|
||||
|
||||
@@ -49,6 +35,22 @@ class ComponentAction(msgspec.Struct, frozen=True):
|
||||
declares its ``component_type`` field."""
|
||||
|
||||
|
||||
class UpdatePendingBackupOnNodeSplit(ComponentAction, frozen=True):
|
||||
"""Update pending backups for the prefix and child produced by a split.
|
||||
|
||||
The cache first relocates write-through/linker publication lists when
|
||||
``ack_id`` is present, then routes this same action once to FULL for
|
||||
component-owned buffer-write dependencies. Buffer-only splits may have
|
||||
no tree acknowledgment.
|
||||
"""
|
||||
|
||||
ack_id: int | None
|
||||
old_node_id: NodeId
|
||||
new_node_id: NodeId
|
||||
new_child_node_id: NodeId
|
||||
component_type: ComponentType = ComponentType.FULL
|
||||
|
||||
|
||||
class FreeComponentDeviceSlot(ComponentAction, frozen=True):
|
||||
"""Free only the given ``component_type``'s device KV slots."""
|
||||
|
||||
@@ -115,7 +117,7 @@ class SWARebuild(ComponentAction, frozen=True):
|
||||
component_type: ComponentType = ComponentType.SWA
|
||||
|
||||
|
||||
# Cache-owned actions, applied by UnifiedRadixCache itself.
|
||||
# The cache handles these directly; split updates also reach their component.
|
||||
CacheAction = (
|
||||
ReplaceWriteThroughOnNodeSplit | FreeDeviceKV | FreeDeviceKVFullOnly | BackupKV
|
||||
UpdatePendingBackupOnNodeSplit | FreeDeviceKV | FreeDeviceKVFullOnly | BackupKV
|
||||
)
|
||||
|
||||
@@ -18,7 +18,10 @@ from sglang.srt.mem_cache.hicache_storage import (
|
||||
PoolTransfer,
|
||||
PoolTransferResult,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_cache.cache_action import FreeComponentDeviceSlot
|
||||
from sglang.srt.mem_cache.unified_cache.cache_action import (
|
||||
FreeComponentDeviceSlot,
|
||||
UpdatePendingBackupOnNodeSplit,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_cache.components.base import (
|
||||
CacheTransferPhase,
|
||||
ComponentType,
|
||||
@@ -527,7 +530,10 @@ class FullComponent(TreeComponent):
|
||||
self.cache.host_pool_group.free(host_value, pool=PoolName.KV)
|
||||
|
||||
def apply_component_action(self, action: ComponentAction) -> None:
|
||||
if isinstance(action, FreeComponentDeviceSlot):
|
||||
if isinstance(action, UpdatePendingBackupOnNodeSplit):
|
||||
# Custom FULL components track buffer-write dependencies.
|
||||
pass
|
||||
elif isinstance(action, FreeComponentDeviceSlot):
|
||||
alloc = self.cache.token_to_kv_pool_allocator
|
||||
for indices in action.indices:
|
||||
# tree values are page-aligned copies of a kv row: page-exact segments
|
||||
@@ -535,7 +541,7 @@ class FullComponent(TreeComponent):
|
||||
alloc.full_attn_allocator.free_segment(indices, start_pos=0)
|
||||
else:
|
||||
alloc.free_segment(indices, start_pos=0)
|
||||
return
|
||||
raise AssertionError(
|
||||
f"FullComponent: unhandled ComponentAction {type(action).__name__}"
|
||||
)
|
||||
else:
|
||||
raise AssertionError(
|
||||
f"FullComponent: unhandled ComponentAction {type(action).__name__}"
|
||||
)
|
||||
|
||||
@@ -525,7 +525,11 @@ class SWAComponent(TreeComponent):
|
||||
)
|
||||
if live_start is None:
|
||||
continue
|
||||
parent = self._maybe_split_leaf_for_swa_lock(fragment) if cap_leaf else None
|
||||
parent = (
|
||||
self._maybe_split_leaf_for_swa_lock(fragment, cache_actions)
|
||||
if cap_leaf
|
||||
else None
|
||||
)
|
||||
# Older fragments publish first so the tail remains more-MRU.
|
||||
if parent is not None:
|
||||
self._publish_window(node=parent, cache_actions=cache_actions)
|
||||
@@ -587,7 +591,9 @@ class SWAComponent(TreeComponent):
|
||||
)
|
||||
|
||||
def _maybe_split_leaf_for_swa_lock(
|
||||
self, leaf: UnifiedTreeNode
|
||||
self,
|
||||
leaf: UnifiedTreeNode,
|
||||
cache_actions: list[CacheAction | ComponentAction],
|
||||
) -> Optional[UnifiedTreeNode]:
|
||||
"""Cap a fresh in-window SWA leaf at one page-aligned window so locking it pins
|
||||
only one window of SWA pool, not the whole (long chunked-prefill) leaf; return
|
||||
@@ -610,7 +616,11 @@ class SWAComponent(TreeComponent):
|
||||
return None
|
||||
|
||||
new_parent, action = self.tree_core._split_node(leaf.key, leaf, split_at)
|
||||
assert action is None, "fresh SWA leaf cannot be write-through-pending"
|
||||
assert action is None or action.ack_id is None, (
|
||||
"fresh SWA leaf cannot be write-through-pending"
|
||||
)
|
||||
if action is not None:
|
||||
cache_actions.append(action)
|
||||
return new_parent
|
||||
|
||||
def redistribute_on_node_split(
|
||||
|
||||
@@ -51,7 +51,7 @@ from sglang.srt.mem_cache.unified_cache.cache_action import (
|
||||
ComponentAction,
|
||||
FreeDeviceKV,
|
||||
FreeDeviceKVFullOnly,
|
||||
ReplaceWriteThroughOnNodeSplit,
|
||||
UpdatePendingBackupOnNodeSplit,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_cache.components import (
|
||||
_NUM_COMPONENT_TYPES,
|
||||
@@ -1270,7 +1270,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
"""Fire-and-forget actions safe to batch until the next barrier."""
|
||||
return isinstance(
|
||||
action,
|
||||
(FreeDeviceKV, FreeDeviceKVFullOnly, ReplaceWriteThroughOnNodeSplit),
|
||||
(FreeDeviceKV, FreeDeviceKVFullOnly, UpdatePendingBackupOnNodeSplit),
|
||||
)
|
||||
|
||||
def _insert_walk_step(self, state: _InsertWalkState) -> None:
|
||||
@@ -1479,12 +1479,13 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
component.redistribute_on_node_split(new_parent=new_node, child=child)
|
||||
new_node.parent.children[key.child_key(self.page_size)] = new_node
|
||||
|
||||
# A split of a backuped node tells the cache to fix its publish list.
|
||||
# Buffer-mode dependencies are tracked by the cache, without an ack here.
|
||||
action: Optional[CacheAction | ComponentAction] = None
|
||||
if child.write_through_pending_id is not None:
|
||||
ack_id = child.write_through_pending_id
|
||||
ack_id = child.write_through_pending_id
|
||||
if ack_id is not None:
|
||||
new_node.write_through_pending_id = ack_id
|
||||
action = ReplaceWriteThroughOnNodeSplit(
|
||||
if ack_id is not None or self.is_host_memory_buffer_only:
|
||||
action = UpdatePendingBackupOnNodeSplit(
|
||||
ack_id=ack_id,
|
||||
old_node_id=child.id,
|
||||
new_node_id=new_node.id,
|
||||
|
||||
@@ -74,7 +74,7 @@ from sglang.srt.mem_cache.unified_cache.cache_action import (
|
||||
FreeComponentDeviceSlot,
|
||||
FreeDeviceKV,
|
||||
FreeDeviceKVFullOnly,
|
||||
ReplaceWriteThroughOnNodeSplit,
|
||||
UpdatePendingBackupOnNodeSplit,
|
||||
)
|
||||
|
||||
# UnifiedTreeNode / UnifiedLRUList live on the tree core; re-exported here
|
||||
@@ -1348,19 +1348,19 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
# Component actions route to their component class; the rest are
|
||||
# cache-owned and handled here by type.
|
||||
if isinstance(action, ComponentAction):
|
||||
self.components[action.component_type].apply_component_action(action)
|
||||
elif isinstance(action, ReplaceWriteThroughOnNodeSplit):
|
||||
self._replace_pending_write_through_node(
|
||||
action.ack_id,
|
||||
action.old_node_id,
|
||||
[action.new_node_id, action.new_child_node_id],
|
||||
)
|
||||
if self.linker is not None:
|
||||
self.linker.replace_pending_offload_node(
|
||||
action.ack_id,
|
||||
action.old_node_id,
|
||||
[action.new_node_id, action.new_child_node_id],
|
||||
if (
|
||||
isinstance(action, UpdatePendingBackupOnNodeSplit)
|
||||
and action.ack_id is not None
|
||||
):
|
||||
node_ids = [action.new_node_id, action.new_child_node_id]
|
||||
self._replace_pending_write_through_node(
|
||||
action.ack_id, action.old_node_id, node_ids
|
||||
)
|
||||
if self.linker is not None:
|
||||
self.linker.replace_pending_offload_node(
|
||||
action.ack_id, action.old_node_id, node_ids
|
||||
)
|
||||
self.components[action.component_type].apply_component_action(action)
|
||||
elif isinstance(action, FreeDeviceKV):
|
||||
# tree values are page-aligned copies of a kv row: page-exact segments
|
||||
for indices in action.indices:
|
||||
|
||||
Reference in New Issue
Block a user