[Refactor] radix-cache: Route pending-backup split bookkeeping through one component action (#45242)

Co-authored-by: metamergebot <metamergebot@users.noreply.github.com>
Co-authored-by: Jialin <Jialin@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-20 13:24:49 -07:00
committed by GitHub
co-authored by metamergebot Jialin
parent 0b536ca112
commit cc7ae372ac
16 changed files with 387 additions and 164 deletions
@@ -40,8 +40,8 @@ from sglang.srt.mem_cache.unified_cache.cache_action import (
MambaEvictExcessPathStates,
RebuildFullToSWAMapping,
RecoverSWAWithLockedFull,
ReplaceWriteThroughOnNodeSplit,
SWARebuild,
UpdatePendingBackupOnNodeSplit,
)
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
from sglang.srt.mem_cache.unified_cache.components import CacheTransferPhase
@@ -164,8 +164,8 @@ def _cache_action_from_tagged(action: tuple) -> CacheAction:
return BackupKV(node_ids=list(action[1]))
if tag == "mamba_evict_excess_path_states":
return MambaEvictExcessPathStates(tail_node_id=action[1])
if tag == "replace_write_through_on_node_split":
return ReplaceWriteThroughOnNodeSplit(
if tag == "update_pending_backup_on_node_split":
return UpdatePendingBackupOnNodeSplit(
ack_id=action[1],
old_node_id=action[2],
new_node_id=action[3],
@@ -16,20 +16,6 @@ if TYPE_CHECKING:
from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import NodeId
class ReplaceWriteThroughOnNodeSplit(msgspec.Struct, frozen=True):
"""Replace the pending write-through node on a node split:
parent -> node => parent -> new_node -> new_child
old_node_id (the pre-split node) is replaced by new_node_id + new_child_node_id.
"""
ack_id: int
old_node_id: NodeId
new_node_id: NodeId
new_child_node_id: NodeId
class FreeDeviceKV(msgspec.Struct, frozen=True):
"""Free unreferenced device KV slots (a SWA-aware combined free)."""
@@ -49,6 +35,22 @@ class ComponentAction(msgspec.Struct, frozen=True):
declares its ``component_type`` field."""
class UpdatePendingBackupOnNodeSplit(ComponentAction, frozen=True):
"""Update pending backups for the prefix and child produced by a split.
The cache first relocates write-through/linker publication lists when
``ack_id`` is present, then routes this same action once to FULL for
component-owned buffer-write dependencies. Buffer-only splits may have
no tree acknowledgment.
"""
ack_id: int | None
old_node_id: NodeId
new_node_id: NodeId
new_child_node_id: NodeId
component_type: ComponentType = ComponentType.FULL
class FreeComponentDeviceSlot(ComponentAction, frozen=True):
"""Free only the given ``component_type``'s device KV slots."""
@@ -115,7 +117,7 @@ class SWARebuild(ComponentAction, frozen=True):
component_type: ComponentType = ComponentType.SWA
# Cache-owned actions, applied by UnifiedRadixCache itself.
# The cache handles these directly; split updates also reach their component.
CacheAction = (
ReplaceWriteThroughOnNodeSplit | FreeDeviceKV | FreeDeviceKVFullOnly | BackupKV
UpdatePendingBackupOnNodeSplit | FreeDeviceKV | FreeDeviceKVFullOnly | BackupKV
)
@@ -18,7 +18,10 @@ from sglang.srt.mem_cache.hicache_storage import (
PoolTransfer,
PoolTransferResult,
)
from sglang.srt.mem_cache.unified_cache.cache_action import FreeComponentDeviceSlot
from sglang.srt.mem_cache.unified_cache.cache_action import (
FreeComponentDeviceSlot,
UpdatePendingBackupOnNodeSplit,
)
from sglang.srt.mem_cache.unified_cache.components.base import (
CacheTransferPhase,
ComponentType,
@@ -527,7 +530,10 @@ class FullComponent(TreeComponent):
self.cache.host_pool_group.free(host_value, pool=PoolName.KV)
def apply_component_action(self, action: ComponentAction) -> None:
if isinstance(action, FreeComponentDeviceSlot):
if isinstance(action, UpdatePendingBackupOnNodeSplit):
# Custom FULL components track buffer-write dependencies.
pass
elif isinstance(action, FreeComponentDeviceSlot):
alloc = self.cache.token_to_kv_pool_allocator
for indices in action.indices:
# tree values are page-aligned copies of a kv row: page-exact segments
@@ -535,7 +541,7 @@ class FullComponent(TreeComponent):
alloc.full_attn_allocator.free_segment(indices, start_pos=0)
else:
alloc.free_segment(indices, start_pos=0)
return
raise AssertionError(
f"FullComponent: unhandled ComponentAction {type(action).__name__}"
)
else:
raise AssertionError(
f"FullComponent: unhandled ComponentAction {type(action).__name__}"
)
@@ -525,7 +525,11 @@ class SWAComponent(TreeComponent):
)
if live_start is None:
continue
parent = self._maybe_split_leaf_for_swa_lock(fragment) if cap_leaf else None
parent = (
self._maybe_split_leaf_for_swa_lock(fragment, cache_actions)
if cap_leaf
else None
)
# Older fragments publish first so the tail remains more-MRU.
if parent is not None:
self._publish_window(node=parent, cache_actions=cache_actions)
@@ -587,7 +591,9 @@ class SWAComponent(TreeComponent):
)
def _maybe_split_leaf_for_swa_lock(
self, leaf: UnifiedTreeNode
self,
leaf: UnifiedTreeNode,
cache_actions: list[CacheAction | ComponentAction],
) -> Optional[UnifiedTreeNode]:
"""Cap a fresh in-window SWA leaf at one page-aligned window so locking it pins
only one window of SWA pool, not the whole (long chunked-prefill) leaf; return
@@ -610,7 +616,11 @@ class SWAComponent(TreeComponent):
return None
new_parent, action = self.tree_core._split_node(leaf.key, leaf, split_at)
assert action is None, "fresh SWA leaf cannot be write-through-pending"
assert action is None or action.ack_id is None, (
"fresh SWA leaf cannot be write-through-pending"
)
if action is not None:
cache_actions.append(action)
return new_parent
def redistribute_on_node_split(
@@ -51,7 +51,7 @@ from sglang.srt.mem_cache.unified_cache.cache_action import (
ComponentAction,
FreeDeviceKV,
FreeDeviceKVFullOnly,
ReplaceWriteThroughOnNodeSplit,
UpdatePendingBackupOnNodeSplit,
)
from sglang.srt.mem_cache.unified_cache.components import (
_NUM_COMPONENT_TYPES,
@@ -1270,7 +1270,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
"""Fire-and-forget actions safe to batch until the next barrier."""
return isinstance(
action,
(FreeDeviceKV, FreeDeviceKVFullOnly, ReplaceWriteThroughOnNodeSplit),
(FreeDeviceKV, FreeDeviceKVFullOnly, UpdatePendingBackupOnNodeSplit),
)
def _insert_walk_step(self, state: _InsertWalkState) -> None:
@@ -1479,12 +1479,13 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
component.redistribute_on_node_split(new_parent=new_node, child=child)
new_node.parent.children[key.child_key(self.page_size)] = new_node
# A split of a backuped node tells the cache to fix its publish list.
# Buffer-mode dependencies are tracked by the cache, without an ack here.
action: Optional[CacheAction | ComponentAction] = None
if child.write_through_pending_id is not None:
ack_id = child.write_through_pending_id
ack_id = child.write_through_pending_id
if ack_id is not None:
new_node.write_through_pending_id = ack_id
action = ReplaceWriteThroughOnNodeSplit(
if ack_id is not None or self.is_host_memory_buffer_only:
action = UpdatePendingBackupOnNodeSplit(
ack_id=ack_id,
old_node_id=child.id,
new_node_id=new_node.id,
@@ -74,7 +74,7 @@ from sglang.srt.mem_cache.unified_cache.cache_action import (
FreeComponentDeviceSlot,
FreeDeviceKV,
FreeDeviceKVFullOnly,
ReplaceWriteThroughOnNodeSplit,
UpdatePendingBackupOnNodeSplit,
)
# UnifiedTreeNode / UnifiedLRUList live on the tree core; re-exported here
@@ -1348,19 +1348,19 @@ class UnifiedRadixCache(BasePrefixCache):
# Component actions route to their component class; the rest are
# cache-owned and handled here by type.
if isinstance(action, ComponentAction):
self.components[action.component_type].apply_component_action(action)
elif isinstance(action, ReplaceWriteThroughOnNodeSplit):
self._replace_pending_write_through_node(
action.ack_id,
action.old_node_id,
[action.new_node_id, action.new_child_node_id],
)
if self.linker is not None:
self.linker.replace_pending_offload_node(
action.ack_id,
action.old_node_id,
[action.new_node_id, action.new_child_node_id],
if (
isinstance(action, UpdatePendingBackupOnNodeSplit)
and action.ack_id is not None
):
node_ids = [action.new_node_id, action.new_child_node_id]
self._replace_pending_write_through_node(
action.ack_id, action.old_node_id, node_ids
)
if self.linker is not None:
self.linker.replace_pending_offload_node(
action.ack_id, action.old_node_id, node_ids
)
self.components[action.component_type].apply_component_action(action)
elif isinstance(action, FreeDeviceKV):
# tree values are page-aligned copies of a kv row: page-exact segments
for indices in action.indices:
+15 -3
View File
@@ -57,6 +57,7 @@ impl SwaComponent {
&self,
tree_core: &mut UnifiedTreeCore<K>,
leaf_id: NodeIdx_,
cache_actions: &mut Vec<CacheAction>,
) -> Option<NodeIdx_> {
let leaf = tree_core.arena.node(leaf_id);
if leaf.is_root() || leaf.device_lock_ref(SWA) > 0 {
@@ -79,9 +80,18 @@ impl SwaComponent {
let (new_parent, action) = tree_core.split_node_(leaf_id, split_at);
assert!(
action.is_none(),
!matches!(
action,
Some(CacheAction::UpdatePendingBackupOnNodeSplit {
ack_id: Some(_),
..
})
),
"fresh SWA leaf cannot be write-through-pending"
);
if let Some(action) = action {
cache_actions.push(action);
}
Some(new_parent)
}
@@ -587,14 +597,16 @@ impl<K: ChildKeyType> TreeComponent<K> for SwaComponent {
// Node straddles the boundary: split into an out-of-window parent
// (tombstone) and an in-window child; `node` becomes the child.
let (_, action) = tree_core.split_node_(node_id, split_pos);
assert!(action.is_none(), "new leaf cannot be write-through-pending");
if let Some(action) = action {
cache_actions.push(action);
}
}
}
result.record_adopted_range(SWA, node_start.max(params.swa_evicted_seqlen), node_end);
// Cap the in-window leaf at one window for lock granularity, then rebuild SWA
// onto the in-window node(s) at apply time; rebuild the older prefix first so
// the in-window tail lands more-MRU.
let capped_parent = self.maybe_split_leaf_for_swa_lock_(tree_core, node_id);
let capped_parent = self.maybe_split_leaf_for_swa_lock_(tree_core, node_id, cache_actions);
if let Some(capped_parent) = capped_parent {
cache_actions.push(CacheAction::SwaRebuild {
node_id: tree_core.arena.node(capped_parent).id,
@@ -179,7 +179,7 @@ fn cache_action_tag(action: &CacheAction) -> &'static str {
CacheAction::FreeDeviceKV(_) => "free_device_kv",
CacheAction::FreeDeviceKVFullOnly(_) => "free_device_kv_full_only",
CacheAction::BackupKV(_) => "backup_kv",
CacheAction::ReplaceWriteThroughOnNodeSplit { .. } => "replace_write_through_on_node_split",
CacheAction::UpdatePendingBackupOnNodeSplit { .. } => "update_pending_backup_on_node_split",
CacheAction::MambaEvictExcessPathStates { .. } => "mamba_evict_excess_path_states",
CacheAction::FreeComponentDeviceSlot { .. } => "free_component_device_slot",
CacheAction::FreeComponentHostSlot { .. } => "free_component_host_slot",
@@ -202,7 +202,7 @@ fn cache_action_to_py(py: Python<'_>, action: CacheAction) -> PyResult<Py<PyAny>
(tag, tensors).into_py_any(py)
}
CacheAction::BackupKV(backup) => (tag, backup.node_ids).into_py_any(py),
CacheAction::ReplaceWriteThroughOnNodeSplit {
CacheAction::UpdatePendingBackupOnNodeSplit {
ack_id,
old_node_id,
new_node_id,
@@ -496,13 +496,13 @@ fn host_reclaim_orders_pending_internal_split_fragments_by_ack() {
// asynchronously under write_back; matching can split it before the ack.
let (prefix_idx, action) = tc.split_node_(tc.arena.resolve(a).unwrap(), 2);
let published_a = match action {
Some(CacheAction::ReplaceWriteThroughOnNodeSplit {
Some(CacheAction::UpdatePendingBackupOnNodeSplit {
ack_id,
old_node_id,
new_node_id,
new_child_node_id,
}) => {
assert_eq!((ack_id, old_node_id, new_child_node_id), (a, a, a));
assert_eq!((ack_id, old_node_id, new_child_node_id), (Some(a), a, a));
assert_eq!(new_node_id, tc.arena.node(prefix_idx).id);
vec![new_node_id, new_child_node_id]
}
@@ -76,7 +76,7 @@ fn swa_window_attach_splits_both_edges_preserving_full_and_pending_actions() {
assert_eq!(actions.len(), 2);
for action in &actions {
assert!(
matches!(action, CacheAction::ReplaceWriteThroughOnNodeSplit { ack_id: 91, old_node_id, new_child_node_id, .. } if *old_node_id == leaf_id && *new_child_node_id == leaf_id)
matches!(action, CacheAction::UpdatePendingBackupOnNodeSplit { ack_id: Some(91), old_node_id, new_child_node_id, .. } if *old_node_id == leaf_id && *new_child_node_id == leaf_id)
);
}
let head = child_of(&tc, tc.arena.root(), &[1, 2]);
@@ -3899,46 +3899,94 @@ fn new_leaf_after_a_cached_prefix_splits_at_the_leaf_relative_boundary() {
#[test]
fn new_leaf_boundary_split_and_window_cap_compose_in_one_commit() {
let mut tc = swa_core(/* window = */ 2, /* page_size = */ 1);
tc.insert(&insert_params_swa(&vec![1, 2], &[10, 11], 0, 0));
let root = tc.arena.root();
let a = child_of(&tc, root, &[1]);
let result = tc.insert(&insert_params_swa(
&vec![1, 2, 3, 4, 5, 6, 7],
&[20, 21, 22, 23, 24, 25, 26],
/* prev_prefix_len = */ 2,
/* swa_evicted_seqlen = */ 3,
));
// Boundary split first ([3] tombstone), then the window cap splits the
// in-window run into [4, 5] + [6, 7], rebuilt older-prefix-first.
let p = child_of(&tc, a, &[3]);
let capped = child_of(&tc, p, &[4]);
let leaf = child_of(&tc, capped, &[6]);
assert_eq!(tc.arena.node(p).key, vec![3]);
assert_eq!(tc.arena.node(capped).key, vec![4, 5]);
assert_eq!(tc.arena.node(leaf).key, vec![6, 7]);
let rebuilds: Vec<NodeId> = result
.cache_actions
.iter()
.filter_map(|action| match action {
CacheAction::SwaRebuild { node_id, .. } => Some(*node_id),
_ => None,
})
.collect();
assert_eq!(
rebuilds,
vec![tc.arena.node(capped).id, tc.arena.node(leaf).id]
);
assert!(result.cache_actions.iter().any(|action| matches!(
action,
CacheAction::SwaRebuild { node_id, source_value }
if *node_id == tc.arena.node(capped).id && source_value.equal(&Tensor::from_slice(&[23i64, 24]))
)));
assert!(result.cache_actions.iter().any(|action| matches!(
action,
CacheAction::SwaRebuild { node_id, source_value }
if *node_id == tc.arena.node(leaf).id && source_value.equal(&Tensor::from_slice(&[25i64, 26]))
)));
for buffer_only in [false, true] {
let mut tc = swa_core(/* window = */ 2, /* page_size = */ 1);
if buffer_only {
tc.set_host_memory_buffer_only();
}
tc.insert(&insert_params_swa(&vec![1, 2], &[10, 11], 0, 0));
let root = tc.arena.root();
let a = child_of(&tc, root, &[1]);
let result = tc.insert(&insert_params_swa(
&vec![1, 2, 3, 4, 5, 6, 7],
&[20, 21, 22, 23, 24, 25, 26],
/* prev_prefix_len = */ 2,
/* swa_evicted_seqlen = */ 3,
));
// Boundary split first ([3] tombstone), then the window cap splits the
// in-window run into [4, 5] + [6, 7], rebuilt older-prefix-first.
let p = child_of(&tc, a, &[3]);
let capped = child_of(&tc, p, &[4]);
let leaf = child_of(&tc, capped, &[6]);
assert_eq!(tc.arena.node(p).key, vec![3]);
assert_eq!(tc.arena.node(capped).key, vec![4, 5]);
assert_eq!(tc.arena.node(leaf).key, vec![6, 7]);
let rebuilds: Vec<NodeId> = result
.cache_actions
.iter()
.filter_map(|action| match action {
CacheAction::SwaRebuild { node_id, .. } => Some(*node_id),
_ => None,
})
.collect();
assert_eq!(
rebuilds,
vec![tc.arena.node(capped).id, tc.arena.node(leaf).id]
);
assert!(result.cache_actions.iter().any(|action| matches!(
action,
CacheAction::SwaRebuild { node_id, source_value }
if *node_id == tc.arena.node(capped).id && source_value.equal(&Tensor::from_slice(&[23i64, 24]))
)));
assert!(result.cache_actions.iter().any(|action| matches!(
action,
CacheAction::SwaRebuild { node_id, source_value }
if *node_id == tc.arena.node(leaf).id && source_value.equal(&Tensor::from_slice(&[25i64, 26]))
)));
let split_ids: Vec<NodeId> = result
.cache_actions
.iter()
.filter_map(|action| match action {
CacheAction::UpdatePendingBackupOnNodeSplit {
ack_id,
old_node_id,
new_node_id,
new_child_node_id,
} => {
assert_eq!(*ack_id, None);
assert_eq!(*old_node_id, tc.arena.node(leaf).id);
assert_eq!(*new_child_node_id, tc.arena.node(leaf).id);
Some(*new_node_id)
}
_ => None,
})
.collect();
if buffer_only {
assert_eq!(
split_ids,
vec![tc.arena.node(p).id, tc.arena.node(capped).id]
);
let first_rebuild = result
.cache_actions
.iter()
.position(|action| matches!(action, CacheAction::SwaRebuild { .. }))
.unwrap();
assert_eq!(
result.cache_actions[..first_rebuild]
.iter()
.filter(|action| {
matches!(action, CacheAction::UpdatePendingBackupOnNodeSplit { .. })
})
.count(),
2,
);
} else {
assert!(split_ids.is_empty());
}
for node in [p, capped, leaf] {
assert_eq!(tc.arena.node(node).write_through_pending_id, None);
}
}
}
#[test]
@@ -4030,12 +4078,28 @@ fn window_cap_skips_a_page_misaligned_leaf() {
)
.unwrap();
// split_at = 5 - 2 = 3 is not page-aligned: the cap is skipped.
let capped = swa_component(2).maybe_split_leaf_for_swa_lock_(&mut tc, leaf);
let mut actions = Vec::new();
let capped = swa_component(2).maybe_split_leaf_for_swa_lock_(&mut tc, leaf, &mut actions);
assert!(actions.is_empty());
assert_eq!(capped, None);
assert_eq!(tc.arena.node(leaf).key, vec![1, 2, 3, 4, 5]);
assert_eq!(tc.arena.node(leaf).parent(), root);
}
#[test]
#[should_panic(expected = "fresh SWA leaf cannot be write-through-pending")]
fn window_cap_rejects_a_write_through_ack() {
let mut tc = swa_core(/* window = */ 2, /* page_size = */ 1);
tc.set_host_memory_buffer_only();
let root = tc.arena.root();
let leaf = tc
.arena
.alloc_child(root, vec![1, 2, 3, 4], 0, None)
.unwrap();
tc.arena.node_mut(leaf).write_through_pending_id = Some(91);
swa_component(2).maybe_split_leaf_for_swa_lock_(&mut tc, leaf, &mut Vec::new());
}
#[test]
fn new_leaf_with_the_boundary_above_its_start_skips_the_split() {
let mut tc = swa_core(/* window = */ 8, /* page_size = */ 1);
@@ -43,7 +43,7 @@ pub(crate) fn action_kinds(actions: &[CacheAction]) -> Vec<&'static str> {
CacheAction::FreeDeviceKV(_) => "FreeDeviceKV",
CacheAction::FreeDeviceKVFullOnly(_) => "FreeDeviceKVFullOnly",
CacheAction::BackupKV(_) => "BackupKV",
CacheAction::ReplaceWriteThroughOnNodeSplit { .. } => "ReplaceWriteThroughOnNodeSplit",
CacheAction::UpdatePendingBackupOnNodeSplit { .. } => "UpdatePendingBackupOnNodeSplit",
CacheAction::MambaEvictExcessPathStates { .. } => "MambaEvictExcessPathStates",
CacheAction::FreeComponentDeviceSlot { .. } => "FreeComponentDeviceSlot",
CacheAction::FreeComponentHostSlot { .. } => "FreeComponentHostSlot",
@@ -3122,38 +3122,46 @@ fn backup_kv_action_stops_at_an_externally_stored_or_pending_ancestor() {
}
#[test]
fn split_of_a_pending_node_transfers_the_ack_and_emits_the_replace_action() {
let mut tc = core();
tc.insert(&insert_params(&vec![1, 2, 3], &[10, 11, 12]));
let node = tc
.match_prefix(&match_params(&vec![1, 2, 3]))
.best_match_node_id;
tc.mark_write_through_pending(vec![node], /* ack_id = */ node)
.expect("live test node");
let (new_node, action) = tc.split_node_(
tc.arena.resolve(node).expect("live test node"),
/* split_len = */ 1,
);
assert_eq!(tc.arena.node(new_node).write_through_pending_id, Some(node));
assert_eq!(
tc.arena
.node(tc.arena.resolve(node).expect("live test node"))
.write_through_pending_id,
Some(node)
);
match action {
Some(CacheAction::ReplaceWriteThroughOnNodeSplit {
ack_id,
old_node_id,
new_node_id,
new_child_node_id,
}) => {
assert_eq!(ack_id, node);
assert_eq!(old_node_id, node);
assert_eq!(new_node_id, tc.arena.node(new_node).id);
assert_eq!(new_child_node_id, node);
fn split_updates_pending_backup_with_or_without_a_tree_ack() {
for buffer_only in [false, true] {
for pending in [false, true] {
let mut tc = core();
if buffer_only {
tc.set_host_memory_buffer_only();
}
tc.insert(&insert_params(&vec![1, 2, 3], &[10, 11, 12]));
let tail_id = tc
.match_prefix(&match_params(&vec![1, 2, 3]))
.best_match_node_id;
if pending {
tc.mark_write_through_pending(vec![tail_id], tail_id)
.unwrap();
}
let result = tc.match_prefix(&match_params(&vec![1]));
let prefix_id = result.best_match_node_id;
assert_ne!(prefix_id, tail_id);
if pending || buffer_only {
assert!(matches!(
result.cache_actions.as_slice(),
[CacheAction::UpdatePendingBackupOnNodeSplit {
ack_id,
old_node_id,
new_node_id,
new_child_node_id,
}] if *ack_id == pending.then_some(tail_id) && *old_node_id == tail_id
&& *new_node_id == prefix_id && *new_child_node_id == tail_id
));
} else {
assert!(result.cache_actions.is_empty());
}
for node_id in [prefix_id, tail_id] {
let node = tc.arena.node(tc.arena.resolve(node_id).unwrap());
assert_eq!(node.write_through_pending_id, pending.then_some(tail_id));
}
if !pending {
tc.sanity_check(&[], &[]);
}
}
other => panic!("expected the replace action, got {:?}", other.is_some()),
}
}
@@ -4029,12 +4037,12 @@ fn finish_write_through_after_a_split_publishes_both_fragments() {
.cache_actions
.iter()
.find_map(|action| match action {
CacheAction::ReplaceWriteThroughOnNodeSplit {
CacheAction::UpdatePendingBackupOnNodeSplit {
ack_id,
new_node_id,
..
} => {
assert_eq!(*ack_id, leaf);
assert_eq!(*ack_id, Some(leaf));
Some(*new_node_id)
}
_ => None,
@@ -4498,7 +4506,7 @@ fn insert_host_refill_preserves_split_actions_and_lengths() {
assert!(!result.host_insert_dropped);
assert!(matches!(
result.cache_actions.as_slice(),
[CacheAction::ReplaceWriteThroughOnNodeSplit { ack_id, .. }] if *ack_id == leaf
[CacheAction::UpdatePendingBackupOnNodeSplit { ack_id, .. }] if *ack_id == Some(leaf)
));
}
@@ -309,13 +309,14 @@ pub enum CacheAction {
FreeDeviceKVFullOnly(Vec<Tensor>),
/// A device->host backup work item (the write-through threshold fired).
BackupKV(BackupKV),
/// Replace the pending write-through node on a node split:
/// Update cache-side write bookkeeping after a node split:
///
/// parent -> node => parent -> new_node -> new_child
///
/// old_node_id (the pre-split node) is replaced by new_node_id + new_child_node_id.
ReplaceWriteThroughOnNodeSplit {
ack_id: usize,
UpdatePendingBackupOnNodeSplit {
/// None when buffer-only bookkeeping has no tree-tracked write.
ack_id: Option<usize>,
old_node_id: NodeId,
new_node_id: NodeId,
new_child_node_id: NodeId,
@@ -1850,7 +1851,7 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
action,
CacheAction::FreeDeviceKV(_)
| CacheAction::FreeDeviceKVFullOnly(_)
| CacheAction::ReplaceWriteThroughOnNodeSplit { .. }
| CacheAction::UpdatePendingBackupOnNodeSplit { .. }
)
}
@@ -2213,10 +2214,13 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
self.arena.node_mut(new_node_id).load_back_pending_id =
self.arena.node(child_id).load_back_pending_id;
// A split of a backuped node tells the cache to fix its publish list.
let action = if let Some(ack_id) = self.arena.node(child_id).write_through_pending_id {
let ack_id = self.arena.node(child_id).write_through_pending_id;
if let Some(ack_id) = ack_id {
self.arena.node_mut(new_node_id).write_through_pending_id = Some(ack_id);
Some(CacheAction::ReplaceWriteThroughOnNodeSplit {
}
// Buffer-only writes are tracked by the cache rather than a node ack.
let action = if ack_id.is_some() || self.is_host_memory_buffer_only {
Some(CacheAction::UpdatePendingBackupOnNodeSplit {
ack_id,
old_node_id: self.arena.node(child_id).id,
new_node_id: self.arena.node(new_node_id).id,
@@ -61,12 +61,13 @@ from sglang.srt.mem_cache.rust_tree_core.extension import bindings as mem_cache
from sglang.srt.mem_cache.rust_tree_core.extension import load_tree_core_extension
from sglang.srt.mem_cache.unified_cache.cache_action import (
BackupKV,
ComponentAction,
FreeComponentHostSlot,
FreeDeviceKV,
FreeDeviceKVFullOnly,
RecoverSWAWithLockedFull,
ReplaceWriteThroughOnNodeSplit,
SWARebuild,
UpdatePendingBackupOnNodeSplit,
)
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import (
@@ -4155,7 +4156,7 @@ def test_swa_window_repair_returns_split_actions_and_balances_existing_locks():
actions = core.attach_swa_window(key, 2, 6, torch.arange(50, 54))
assert len(actions) == 2
for action in actions:
assert isinstance(action, ReplaceWriteThroughOnNodeSplit)
assert isinstance(action, UpdatePendingBackupOnNodeSplit)
assert action.ack_id == leaf
assert action.old_node_id == leaf
assert action.new_child_node_id == leaf
@@ -4660,23 +4661,104 @@ def root_child(core):
return core.match_prefix(MatchPrefixParams(key=_key([1]))).last_host_node
def test_split_of_a_write_through_pending_node_crosses_the_replace_action():
core = _tree_core()
@pytest.mark.parametrize("backend", ["python", "rust"])
@pytest.mark.parametrize("mode", ["pending", "buffer", "both", "ordinary"])
@pytest.mark.parametrize("split_by", ["insert", "match"])
def test_split_pending_backup_action(backend, mode, split_by):
core = _tree_core(backend)
core.set_hicache_enabled()
has_ack = mode in ("pending", "both")
has_buffer = mode in ("buffer", "both")
if has_buffer:
core.set_host_memory_buffer_only()
_insert(core, [1, 2, 3, 4], [10, 11, 12, 13])
leaf = core.match_prefix(MatchPrefixParams(key=_key([1, 2, 3, 4]))).best_match_node
core.mark_write_through_pending([leaf], ack_id=leaf)
# A divergent prefix splits the pending node; the publish list must follow.
result = _insert(core, [1, 2], [10, 11])
(replace,) = [
if has_ack:
core.mark_write_through_pending([leaf], ack_id=leaf)
# Both insertion and matching can split the existing node.
if split_by == "insert":
result = _insert(core, [1, 2], [10, 11])
else:
result = core.match_prefix(MatchPrefixParams(key=_key([1, 2])))
split_actions = [
action
for action in result.cache_actions
if isinstance(action, ReplaceWriteThroughOnNodeSplit)
if isinstance(action, UpdatePendingBackupOnNodeSplit)
]
assert replace.ack_id == leaf
assert replace.old_node_id == leaf
assert replace.new_child_node_id == leaf
assert replace.new_node_id != leaf
assert len(split_actions) == int(has_ack or has_buffer)
(path,) = core.collect_path_info([leaf])
prefix, suffix = [node.node_id for node in path]
assert suffix == leaf and prefix != leaf
if split_actions:
(split_action,) = split_actions
assert split_action.ack_id == (leaf if has_ack else None)
assert split_action.old_node_id == leaf
assert split_action.new_node_id == prefix
assert split_action.new_child_node_id == leaf
assert isinstance(split_action, ComponentAction)
assert split_action.component_type == ComponentType.FULL
if backend == "python":
assert [
core.node_by_id(node_id).write_through_pending_id
for node_id in (prefix, leaf)
] == [leaf if has_ack else None] * 2
@pytest.mark.parametrize("backend", ["python", "rust"])
@pytest.mark.parametrize("window", [1, 8])
def test_fresh_swa_leaf_forwards_buffer_split_action(backend, window):
core, _ = _swa_transfer_core(backend, window=window)
core.set_host_memory_buffer_only()
result = _pump_insert(
core,
InsertParams(
key=_key([1, 2, 3, 4]),
value=torch.tensor([10, 11, 12, 13]),
component_evicted_seqlens={ComponentType.SWA: 2},
),
)
splits = [
action
for action in result.cache_actions
if isinstance(action, UpdatePendingBackupOnNodeSplit)
]
assert len(splits) == (2 if window == 1 else 1)
for split in splits:
assert split.ack_id is None
assert split.old_node_id == split.new_child_node_id == result.last_device_node
(path,) = core.collect_path_info([result.last_device_node])
assert [node.node_id for node in path] == [
*[split.new_node_id for split in splits],
result.last_device_node,
]
assert [(node.path_start, node.path_end) for node in path] == (
[(0, 2), (2, 3), (3, 4)] if window == 1 else [(0, 2), (2, 4)]
)
rebuilds = [a for a in result.cache_actions if isinstance(a, SWARebuild)]
assert [rebuild.node_id for rebuild in rebuilds] == [
part.node_id for part in path[1:]
]
assert torch.cat([rebuild.source_value for rebuild in rebuilds]).tolist() == [
12,
13,
]
assert result.cache_actions.index(splits[-1]) < result.cache_actions.index(
rebuilds[0]
)
def test_python_fresh_swa_leaf_cap_rejects_write_through_ack():
core, _ = _swa_transfer_core("python", window=2)
core.set_host_memory_buffer_only()
leaf = core._add_new_node(
core.root_node, _key([1, 2, 3, 4]), torch.tensor([10, 11, 12, 13])
)
core.mark_write_through_pending([leaf.id], ack_id=leaf.id)
swa = core.components_by_type[ComponentType.SWA]
with pytest.raises(
AssertionError, match="fresh SWA leaf cannot be write-through-pending"
):
swa._maybe_split_leaf_for_swa_lock(leaf, [])
def test_write_through_threshold_assignment_reaches_the_core():
@@ -25,7 +25,7 @@ from sglang.srt.mem_cache.hicache_storage import (
)
from sglang.srt.mem_cache.radix_cache import RadixKey
from sglang.srt.mem_cache.unified_cache.cache_action import (
ReplaceWriteThroughOnNodeSplit,
UpdatePendingBackupOnNodeSplit,
)
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
from sglang.srt.mem_cache.unified_cache.components.base import (
@@ -1198,7 +1198,12 @@ def test_split_action_retargets_pending_external_offload():
cache._replace_pending_write_through_node = lambda *args: calls.append(
("hicache", *args)
)
action = ReplaceWriteThroughOnNodeSplit(
cache.components = {
ComponentType.FULL: SimpleNamespace(
apply_component_action=lambda action: calls.append(("full", action))
)
}
action = UpdatePendingBackupOnNodeSplit(
ack_id=7,
old_node_id=7,
new_node_id=8,
@@ -1210,6 +1215,7 @@ def test_split_action_retargets_pending_external_offload():
assert calls == [
("hicache", 7, 7, [8, 7]),
("linker", 7, 7, [8, 7]),
("full", action),
]
@@ -31,7 +31,7 @@ from sglang.srt.mem_cache.memory_pool import (
from sglang.srt.mem_cache.radix_cache import RadixKey
from sglang.srt.mem_cache.unified_cache.cache_action import (
RecoverSWAWithLockedFull,
ReplaceWriteThroughOnNodeSplit,
UpdatePendingBackupOnNodeSplit,
)
from sglang.srt.mem_cache.unified_cache.components.base import (
ComponentType,
@@ -1997,6 +1997,14 @@ class TestUnifiedRadixCacheInt8MambaCheckpoint(
self.assertEqual(cache.mamba_evictable_size(), 0)
def _component_with_cache(component_type, cache):
"""A registry component instance bound to a (mock) cache and its tree_core."""
component = object.__new__(COMPONENT_REGISTRY[component_type])
component.cache = cache
component.tree_core = cache.tree_core
return component
class TestUnifiedRadixCacheActionRouting(CustomTestCase):
"""CacheAction routing: each type forwards to the right Controller API."""
@@ -2078,10 +2086,10 @@ class TestUnifiedRadixCacheActionRouting(CustomTestCase):
def test_apply_cache_actions_applies_each_in_order(self):
cache = mock.MagicMock()
first = ReplaceWriteThroughOnNodeSplit(
first = UpdatePendingBackupOnNodeSplit(
ack_id=1, old_node_id=1, new_node_id=2, new_child_node_id=1
)
second = ReplaceWriteThroughOnNodeSplit(
second = UpdatePendingBackupOnNodeSplit(
ack_id=2, old_node_id=3, new_node_id=4, new_child_node_id=3
)
UnifiedRadixCache._apply_cache_actions(cache, [first, second])
@@ -2096,21 +2104,20 @@ class TestUnifiedRadixCacheActionRouting(CustomTestCase):
def make_cache():
cache = mock.MagicMock()
cache.ongoing_write_through = {7: _OngoingWriteThrough(10, None, [5, 10])}
cache._replace_pending_write_through_node = lambda *args: (
UnifiedRadixCache._replace_pending_write_through_node(cache, *args)
)
cache.components = {ComponentType.FULL: mock.Mock()}
return cache
def apply(cache, action):
UnifiedRadixCache._replace_pending_write_through_node(
cache,
action.ack_id,
action.old_node_id,
[action.new_node_id, action.new_child_node_id],
)
UnifiedRadixCache._apply_cache_action(cache, action)
# split X(10) -> [A(11), X(10)], then fragment A(11) -> [B(12), A(11)]
first = ReplaceWriteThroughOnNodeSplit(
first = UpdatePendingBackupOnNodeSplit(
ack_id=7, old_node_id=10, new_node_id=11, new_child_node_id=10
)
second = ReplaceWriteThroughOnNodeSplit(
second = UpdatePendingBackupOnNodeSplit(
ack_id=7, old_node_id=11, new_node_id=12, new_child_node_id=11
)
@@ -2121,6 +2128,12 @@ class TestUnifiedRadixCacheActionRouting(CustomTestCase):
self.assertEqual(
cache.ongoing_write_through[7].publish_node_ids, [5, 12, 11, 10]
)
cache.linker.replace_pending_offload_node.assert_has_calls(
[mock.call(7, 10, [11, 10]), mock.call(7, 11, [12, 11])]
)
receiver = cache.components[ComponentType.FULL].apply_component_action
self.assertEqual(receiver.call_args_list, [mock.call(first), mock.call(second)])
self.assertEqual(cache.linker.replace_pending_offload_node.call_count, 2)
# reversed order silently drops the second fragment - documents why
# emission order must be preserved end to end
@@ -2129,6 +2142,21 @@ class TestUnifiedRadixCacheActionRouting(CustomTestCase):
apply(cache, first)
self.assertEqual(cache.ongoing_write_through[7].publish_node_ids, [5, 11, 10])
def test_split_action_without_ack_still_reaches_full(self):
cache = mock.MagicMock()
component = _component_with_cache(ComponentType.FULL, cache)
component.apply_component_action = mock.Mock(
wraps=component.apply_component_action
)
cache.components = {ComponentType.FULL: component}
action = UpdatePendingBackupOnNodeSplit(
ack_id=None, old_node_id=10, new_node_id=11, new_child_node_id=10
)
UnifiedRadixCache._apply_cache_action(cache, action)
component.apply_component_action.assert_called_once_with(action)
cache._replace_pending_write_through_node.assert_not_called()
cache.linker.replace_pending_offload_node.assert_not_called()
class TestSWADraftHistory(SWADraftHistoryFixtures, CustomTestCase):
"""A widened draft needs SWA history before the target window. A running