diff --git a/python/sglang/srt/mem_cache/rust_tree_core/adapter.py b/python/sglang/srt/mem_cache/rust_tree_core/adapter.py index 2f44ea4486..b6db72b362 100644 --- a/python/sglang/srt/mem_cache/rust_tree_core/adapter.py +++ b/python/sglang/srt/mem_cache/rust_tree_core/adapter.py @@ -40,8 +40,8 @@ from sglang.srt.mem_cache.unified_cache.cache_action import ( MambaEvictExcessPathStates, RebuildFullToSWAMapping, RecoverSWAWithLockedFull, - ReplaceWriteThroughOnNodeSplit, SWARebuild, + UpdatePendingBackupOnNodeSplit, ) from sglang.srt.mem_cache.unified_cache.component_type import ComponentType from sglang.srt.mem_cache.unified_cache.components import CacheTransferPhase @@ -164,8 +164,8 @@ def _cache_action_from_tagged(action: tuple) -> CacheAction: return BackupKV(node_ids=list(action[1])) if tag == "mamba_evict_excess_path_states": return MambaEvictExcessPathStates(tail_node_id=action[1]) - if tag == "replace_write_through_on_node_split": - return ReplaceWriteThroughOnNodeSplit( + if tag == "update_pending_backup_on_node_split": + return UpdatePendingBackupOnNodeSplit( ack_id=action[1], old_node_id=action[2], new_node_id=action[3], diff --git a/python/sglang/srt/mem_cache/unified_cache/cache_action.py b/python/sglang/srt/mem_cache/unified_cache/cache_action.py index cb7066744d..680747f2da 100644 --- a/python/sglang/srt/mem_cache/unified_cache/cache_action.py +++ b/python/sglang/srt/mem_cache/unified_cache/cache_action.py @@ -16,20 +16,6 @@ if TYPE_CHECKING: from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import NodeId -class ReplaceWriteThroughOnNodeSplit(msgspec.Struct, frozen=True): - """Replace the pending write-through node on a node split: - - parent -> node => parent -> new_node -> new_child - - old_node_id (the pre-split node) is replaced by new_node_id + new_child_node_id. - """ - - ack_id: int - old_node_id: NodeId - new_node_id: NodeId - new_child_node_id: NodeId - - class FreeDeviceKV(msgspec.Struct, frozen=True): """Free unreferenced device KV slots (a SWA-aware combined free).""" @@ -49,6 +35,22 @@ class ComponentAction(msgspec.Struct, frozen=True): declares its ``component_type`` field.""" +class UpdatePendingBackupOnNodeSplit(ComponentAction, frozen=True): + """Update pending backups for the prefix and child produced by a split. + + The cache first relocates write-through/linker publication lists when + ``ack_id`` is present, then routes this same action once to FULL for + component-owned buffer-write dependencies. Buffer-only splits may have + no tree acknowledgment. + """ + + ack_id: int | None + old_node_id: NodeId + new_node_id: NodeId + new_child_node_id: NodeId + component_type: ComponentType = ComponentType.FULL + + class FreeComponentDeviceSlot(ComponentAction, frozen=True): """Free only the given ``component_type``'s device KV slots.""" @@ -115,7 +117,7 @@ class SWARebuild(ComponentAction, frozen=True): component_type: ComponentType = ComponentType.SWA -# Cache-owned actions, applied by UnifiedRadixCache itself. +# The cache handles these directly; split updates also reach their component. CacheAction = ( - ReplaceWriteThroughOnNodeSplit | FreeDeviceKV | FreeDeviceKVFullOnly | BackupKV + UpdatePendingBackupOnNodeSplit | FreeDeviceKV | FreeDeviceKVFullOnly | BackupKV ) diff --git a/python/sglang/srt/mem_cache/unified_cache/components/full.py b/python/sglang/srt/mem_cache/unified_cache/components/full.py index a071c26a07..1476db8928 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/full.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/full.py @@ -18,7 +18,10 @@ from sglang.srt.mem_cache.hicache_storage import ( PoolTransfer, PoolTransferResult, ) -from sglang.srt.mem_cache.unified_cache.cache_action import FreeComponentDeviceSlot +from sglang.srt.mem_cache.unified_cache.cache_action import ( + FreeComponentDeviceSlot, + UpdatePendingBackupOnNodeSplit, +) from sglang.srt.mem_cache.unified_cache.components.base import ( CacheTransferPhase, ComponentType, @@ -527,7 +530,10 @@ class FullComponent(TreeComponent): self.cache.host_pool_group.free(host_value, pool=PoolName.KV) def apply_component_action(self, action: ComponentAction) -> None: - if isinstance(action, FreeComponentDeviceSlot): + if isinstance(action, UpdatePendingBackupOnNodeSplit): + # Custom FULL components track buffer-write dependencies. + pass + elif isinstance(action, FreeComponentDeviceSlot): alloc = self.cache.token_to_kv_pool_allocator for indices in action.indices: # tree values are page-aligned copies of a kv row: page-exact segments @@ -535,7 +541,7 @@ class FullComponent(TreeComponent): alloc.full_attn_allocator.free_segment(indices, start_pos=0) else: alloc.free_segment(indices, start_pos=0) - return - raise AssertionError( - f"FullComponent: unhandled ComponentAction {type(action).__name__}" - ) + else: + raise AssertionError( + f"FullComponent: unhandled ComponentAction {type(action).__name__}" + ) diff --git a/python/sglang/srt/mem_cache/unified_cache/components/swa.py b/python/sglang/srt/mem_cache/unified_cache/components/swa.py index ef6fa890b8..dc49f93339 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/swa.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/swa.py @@ -525,7 +525,11 @@ class SWAComponent(TreeComponent): ) if live_start is None: continue - parent = self._maybe_split_leaf_for_swa_lock(fragment) if cap_leaf else None + parent = ( + self._maybe_split_leaf_for_swa_lock(fragment, cache_actions) + if cap_leaf + else None + ) # Older fragments publish first so the tail remains more-MRU. if parent is not None: self._publish_window(node=parent, cache_actions=cache_actions) @@ -587,7 +591,9 @@ class SWAComponent(TreeComponent): ) def _maybe_split_leaf_for_swa_lock( - self, leaf: UnifiedTreeNode + self, + leaf: UnifiedTreeNode, + cache_actions: list[CacheAction | ComponentAction], ) -> Optional[UnifiedTreeNode]: """Cap a fresh in-window SWA leaf at one page-aligned window so locking it pins only one window of SWA pool, not the whole (long chunked-prefill) leaf; return @@ -610,7 +616,11 @@ class SWAComponent(TreeComponent): return None new_parent, action = self.tree_core._split_node(leaf.key, leaf, split_at) - assert action is None, "fresh SWA leaf cannot be write-through-pending" + assert action is None or action.ack_id is None, ( + "fresh SWA leaf cannot be write-through-pending" + ) + if action is not None: + cache_actions.append(action) return new_parent def redistribute_on_node_split( diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py index 7a4203f409..97b4cd80fc 100644 --- a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py +++ b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py @@ -51,7 +51,7 @@ from sglang.srt.mem_cache.unified_cache.cache_action import ( ComponentAction, FreeDeviceKV, FreeDeviceKVFullOnly, - ReplaceWriteThroughOnNodeSplit, + UpdatePendingBackupOnNodeSplit, ) from sglang.srt.mem_cache.unified_cache.components import ( _NUM_COMPONENT_TYPES, @@ -1270,7 +1270,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): """Fire-and-forget actions safe to batch until the next barrier.""" return isinstance( action, - (FreeDeviceKV, FreeDeviceKVFullOnly, ReplaceWriteThroughOnNodeSplit), + (FreeDeviceKV, FreeDeviceKVFullOnly, UpdatePendingBackupOnNodeSplit), ) def _insert_walk_step(self, state: _InsertWalkState) -> None: @@ -1479,12 +1479,13 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): component.redistribute_on_node_split(new_parent=new_node, child=child) new_node.parent.children[key.child_key(self.page_size)] = new_node - # A split of a backuped node tells the cache to fix its publish list. + # Buffer-mode dependencies are tracked by the cache, without an ack here. action: Optional[CacheAction | ComponentAction] = None - if child.write_through_pending_id is not None: - ack_id = child.write_through_pending_id + ack_id = child.write_through_pending_id + if ack_id is not None: new_node.write_through_pending_id = ack_id - action = ReplaceWriteThroughOnNodeSplit( + if ack_id is not None or self.is_host_memory_buffer_only: + action = UpdatePendingBackupOnNodeSplit( ack_id=ack_id, old_node_id=child.id, new_node_id=new_node.id, diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index b9f886839d..c6b0181bd6 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -74,7 +74,7 @@ from sglang.srt.mem_cache.unified_cache.cache_action import ( FreeComponentDeviceSlot, FreeDeviceKV, FreeDeviceKVFullOnly, - ReplaceWriteThroughOnNodeSplit, + UpdatePendingBackupOnNodeSplit, ) # UnifiedTreeNode / UnifiedLRUList live on the tree core; re-exported here @@ -1348,19 +1348,19 @@ class UnifiedRadixCache(BasePrefixCache): # Component actions route to their component class; the rest are # cache-owned and handled here by type. if isinstance(action, ComponentAction): - self.components[action.component_type].apply_component_action(action) - elif isinstance(action, ReplaceWriteThroughOnNodeSplit): - self._replace_pending_write_through_node( - action.ack_id, - action.old_node_id, - [action.new_node_id, action.new_child_node_id], - ) - if self.linker is not None: - self.linker.replace_pending_offload_node( - action.ack_id, - action.old_node_id, - [action.new_node_id, action.new_child_node_id], + if ( + isinstance(action, UpdatePendingBackupOnNodeSplit) + and action.ack_id is not None + ): + node_ids = [action.new_node_id, action.new_child_node_id] + self._replace_pending_write_through_node( + action.ack_id, action.old_node_id, node_ids ) + if self.linker is not None: + self.linker.replace_pending_offload_node( + action.ack_id, action.old_node_id, node_ids + ) + self.components[action.component_type].apply_component_action(action) elif isinstance(action, FreeDeviceKV): # tree values are page-aligned copies of a kv row: page-exact segments for indices in action.indices: diff --git a/rust/sglang-radix-tree/src/components/swa.rs b/rust/sglang-radix-tree/src/components/swa.rs index 003dc447eb..c2f34b7829 100644 --- a/rust/sglang-radix-tree/src/components/swa.rs +++ b/rust/sglang-radix-tree/src/components/swa.rs @@ -57,6 +57,7 @@ impl SwaComponent { &self, tree_core: &mut UnifiedTreeCore, leaf_id: NodeIdx_, + cache_actions: &mut Vec, ) -> Option { let leaf = tree_core.arena.node(leaf_id); if leaf.is_root() || leaf.device_lock_ref(SWA) > 0 { @@ -79,9 +80,18 @@ impl SwaComponent { let (new_parent, action) = tree_core.split_node_(leaf_id, split_at); assert!( - action.is_none(), + !matches!( + action, + Some(CacheAction::UpdatePendingBackupOnNodeSplit { + ack_id: Some(_), + .. + }) + ), "fresh SWA leaf cannot be write-through-pending" ); + if let Some(action) = action { + cache_actions.push(action); + } Some(new_parent) } @@ -587,14 +597,16 @@ impl TreeComponent for SwaComponent { // Node straddles the boundary: split into an out-of-window parent // (tombstone) and an in-window child; `node` becomes the child. let (_, action) = tree_core.split_node_(node_id, split_pos); - assert!(action.is_none(), "new leaf cannot be write-through-pending"); + if let Some(action) = action { + cache_actions.push(action); + } } } result.record_adopted_range(SWA, node_start.max(params.swa_evicted_seqlen), node_end); // Cap the in-window leaf at one window for lock granularity, then rebuild SWA // onto the in-window node(s) at apply time; rebuild the older prefix first so // the in-window tail lands more-MRU. - let capped_parent = self.maybe_split_leaf_for_swa_lock_(tree_core, node_id); + let capped_parent = self.maybe_split_leaf_for_swa_lock_(tree_core, node_id, cache_actions); if let Some(capped_parent) = capped_parent { cache_actions.push(CacheAction::SwaRebuild { node_id: tree_core.arena.node(capped_parent).id, diff --git a/rust/sglang-radix-tree/src/python_bindings.rs b/rust/sglang-radix-tree/src/python_bindings.rs index 935d6c71e4..405cd94591 100644 --- a/rust/sglang-radix-tree/src/python_bindings.rs +++ b/rust/sglang-radix-tree/src/python_bindings.rs @@ -179,7 +179,7 @@ fn cache_action_tag(action: &CacheAction) -> &'static str { CacheAction::FreeDeviceKV(_) => "free_device_kv", CacheAction::FreeDeviceKVFullOnly(_) => "free_device_kv_full_only", CacheAction::BackupKV(_) => "backup_kv", - CacheAction::ReplaceWriteThroughOnNodeSplit { .. } => "replace_write_through_on_node_split", + CacheAction::UpdatePendingBackupOnNodeSplit { .. } => "update_pending_backup_on_node_split", CacheAction::MambaEvictExcessPathStates { .. } => "mamba_evict_excess_path_states", CacheAction::FreeComponentDeviceSlot { .. } => "free_component_device_slot", CacheAction::FreeComponentHostSlot { .. } => "free_component_host_slot", @@ -202,7 +202,7 @@ fn cache_action_to_py(py: Python<'_>, action: CacheAction) -> PyResult (tag, tensors).into_py_any(py) } CacheAction::BackupKV(backup) => (tag, backup.node_ids).into_py_any(py), - CacheAction::ReplaceWriteThroughOnNodeSplit { + CacheAction::UpdatePendingBackupOnNodeSplit { ack_id, old_node_id, new_node_id, diff --git a/rust/sglang-radix-tree/src/tests/components/full.rs b/rust/sglang-radix-tree/src/tests/components/full.rs index 58da8ade48..f621eac0a6 100644 --- a/rust/sglang-radix-tree/src/tests/components/full.rs +++ b/rust/sglang-radix-tree/src/tests/components/full.rs @@ -496,13 +496,13 @@ fn host_reclaim_orders_pending_internal_split_fragments_by_ack() { // asynchronously under write_back; matching can split it before the ack. let (prefix_idx, action) = tc.split_node_(tc.arena.resolve(a).unwrap(), 2); let published_a = match action { - Some(CacheAction::ReplaceWriteThroughOnNodeSplit { + Some(CacheAction::UpdatePendingBackupOnNodeSplit { ack_id, old_node_id, new_node_id, new_child_node_id, }) => { - assert_eq!((ack_id, old_node_id, new_child_node_id), (a, a, a)); + assert_eq!((ack_id, old_node_id, new_child_node_id), (Some(a), a, a)); assert_eq!(new_node_id, tc.arena.node(prefix_idx).id); vec![new_node_id, new_child_node_id] } diff --git a/rust/sglang-radix-tree/src/tests/components/swa.rs b/rust/sglang-radix-tree/src/tests/components/swa.rs index eac6384bc8..851f7caeb5 100644 --- a/rust/sglang-radix-tree/src/tests/components/swa.rs +++ b/rust/sglang-radix-tree/src/tests/components/swa.rs @@ -76,7 +76,7 @@ fn swa_window_attach_splits_both_edges_preserving_full_and_pending_actions() { assert_eq!(actions.len(), 2); for action in &actions { assert!( - matches!(action, CacheAction::ReplaceWriteThroughOnNodeSplit { ack_id: 91, old_node_id, new_child_node_id, .. } if *old_node_id == leaf_id && *new_child_node_id == leaf_id) + matches!(action, CacheAction::UpdatePendingBackupOnNodeSplit { ack_id: Some(91), old_node_id, new_child_node_id, .. } if *old_node_id == leaf_id && *new_child_node_id == leaf_id) ); } let head = child_of(&tc, tc.arena.root(), &[1, 2]); @@ -3899,46 +3899,94 @@ fn new_leaf_after_a_cached_prefix_splits_at_the_leaf_relative_boundary() { #[test] fn new_leaf_boundary_split_and_window_cap_compose_in_one_commit() { - let mut tc = swa_core(/* window = */ 2, /* page_size = */ 1); - tc.insert(&insert_params_swa(&vec![1, 2], &[10, 11], 0, 0)); - let root = tc.arena.root(); - let a = child_of(&tc, root, &[1]); - let result = tc.insert(&insert_params_swa( - &vec![1, 2, 3, 4, 5, 6, 7], - &[20, 21, 22, 23, 24, 25, 26], - /* prev_prefix_len = */ 2, - /* swa_evicted_seqlen = */ 3, - )); - // Boundary split first ([3] tombstone), then the window cap splits the - // in-window run into [4, 5] + [6, 7], rebuilt older-prefix-first. - let p = child_of(&tc, a, &[3]); - let capped = child_of(&tc, p, &[4]); - let leaf = child_of(&tc, capped, &[6]); - assert_eq!(tc.arena.node(p).key, vec![3]); - assert_eq!(tc.arena.node(capped).key, vec![4, 5]); - assert_eq!(tc.arena.node(leaf).key, vec![6, 7]); - let rebuilds: Vec = result - .cache_actions - .iter() - .filter_map(|action| match action { - CacheAction::SwaRebuild { node_id, .. } => Some(*node_id), - _ => None, - }) - .collect(); - assert_eq!( - rebuilds, - vec![tc.arena.node(capped).id, tc.arena.node(leaf).id] - ); - assert!(result.cache_actions.iter().any(|action| matches!( - action, - CacheAction::SwaRebuild { node_id, source_value } - if *node_id == tc.arena.node(capped).id && source_value.equal(&Tensor::from_slice(&[23i64, 24])) - ))); - assert!(result.cache_actions.iter().any(|action| matches!( - action, - CacheAction::SwaRebuild { node_id, source_value } - if *node_id == tc.arena.node(leaf).id && source_value.equal(&Tensor::from_slice(&[25i64, 26])) - ))); + for buffer_only in [false, true] { + let mut tc = swa_core(/* window = */ 2, /* page_size = */ 1); + if buffer_only { + tc.set_host_memory_buffer_only(); + } + tc.insert(&insert_params_swa(&vec![1, 2], &[10, 11], 0, 0)); + let root = tc.arena.root(); + let a = child_of(&tc, root, &[1]); + let result = tc.insert(&insert_params_swa( + &vec![1, 2, 3, 4, 5, 6, 7], + &[20, 21, 22, 23, 24, 25, 26], + /* prev_prefix_len = */ 2, + /* swa_evicted_seqlen = */ 3, + )); + // Boundary split first ([3] tombstone), then the window cap splits the + // in-window run into [4, 5] + [6, 7], rebuilt older-prefix-first. + let p = child_of(&tc, a, &[3]); + let capped = child_of(&tc, p, &[4]); + let leaf = child_of(&tc, capped, &[6]); + assert_eq!(tc.arena.node(p).key, vec![3]); + assert_eq!(tc.arena.node(capped).key, vec![4, 5]); + assert_eq!(tc.arena.node(leaf).key, vec![6, 7]); + let rebuilds: Vec = result + .cache_actions + .iter() + .filter_map(|action| match action { + CacheAction::SwaRebuild { node_id, .. } => Some(*node_id), + _ => None, + }) + .collect(); + assert_eq!( + rebuilds, + vec![tc.arena.node(capped).id, tc.arena.node(leaf).id] + ); + assert!(result.cache_actions.iter().any(|action| matches!( + action, + CacheAction::SwaRebuild { node_id, source_value } + if *node_id == tc.arena.node(capped).id && source_value.equal(&Tensor::from_slice(&[23i64, 24])) + ))); + assert!(result.cache_actions.iter().any(|action| matches!( + action, + CacheAction::SwaRebuild { node_id, source_value } + if *node_id == tc.arena.node(leaf).id && source_value.equal(&Tensor::from_slice(&[25i64, 26])) + ))); + let split_ids: Vec = result + .cache_actions + .iter() + .filter_map(|action| match action { + CacheAction::UpdatePendingBackupOnNodeSplit { + ack_id, + old_node_id, + new_node_id, + new_child_node_id, + } => { + assert_eq!(*ack_id, None); + assert_eq!(*old_node_id, tc.arena.node(leaf).id); + assert_eq!(*new_child_node_id, tc.arena.node(leaf).id); + Some(*new_node_id) + } + _ => None, + }) + .collect(); + if buffer_only { + assert_eq!( + split_ids, + vec![tc.arena.node(p).id, tc.arena.node(capped).id] + ); + let first_rebuild = result + .cache_actions + .iter() + .position(|action| matches!(action, CacheAction::SwaRebuild { .. })) + .unwrap(); + assert_eq!( + result.cache_actions[..first_rebuild] + .iter() + .filter(|action| { + matches!(action, CacheAction::UpdatePendingBackupOnNodeSplit { .. }) + }) + .count(), + 2, + ); + } else { + assert!(split_ids.is_empty()); + } + for node in [p, capped, leaf] { + assert_eq!(tc.arena.node(node).write_through_pending_id, None); + } + } } #[test] @@ -4030,12 +4078,28 @@ fn window_cap_skips_a_page_misaligned_leaf() { ) .unwrap(); // split_at = 5 - 2 = 3 is not page-aligned: the cap is skipped. - let capped = swa_component(2).maybe_split_leaf_for_swa_lock_(&mut tc, leaf); + let mut actions = Vec::new(); + let capped = swa_component(2).maybe_split_leaf_for_swa_lock_(&mut tc, leaf, &mut actions); + assert!(actions.is_empty()); assert_eq!(capped, None); assert_eq!(tc.arena.node(leaf).key, vec![1, 2, 3, 4, 5]); assert_eq!(tc.arena.node(leaf).parent(), root); } +#[test] +#[should_panic(expected = "fresh SWA leaf cannot be write-through-pending")] +fn window_cap_rejects_a_write_through_ack() { + let mut tc = swa_core(/* window = */ 2, /* page_size = */ 1); + tc.set_host_memory_buffer_only(); + let root = tc.arena.root(); + let leaf = tc + .arena + .alloc_child(root, vec![1, 2, 3, 4], 0, None) + .unwrap(); + tc.arena.node_mut(leaf).write_through_pending_id = Some(91); + swa_component(2).maybe_split_leaf_for_swa_lock_(&mut tc, leaf, &mut Vec::new()); +} + #[test] fn new_leaf_with_the_boundary_above_its_start_skips_the_split() { let mut tc = swa_core(/* window = */ 8, /* page_size = */ 1); diff --git a/rust/sglang-radix-tree/src/tests/test_utils.rs b/rust/sglang-radix-tree/src/tests/test_utils.rs index d82dbe86cf..b62afb65f3 100644 --- a/rust/sglang-radix-tree/src/tests/test_utils.rs +++ b/rust/sglang-radix-tree/src/tests/test_utils.rs @@ -43,7 +43,7 @@ pub(crate) fn action_kinds(actions: &[CacheAction]) -> Vec<&'static str> { CacheAction::FreeDeviceKV(_) => "FreeDeviceKV", CacheAction::FreeDeviceKVFullOnly(_) => "FreeDeviceKVFullOnly", CacheAction::BackupKV(_) => "BackupKV", - CacheAction::ReplaceWriteThroughOnNodeSplit { .. } => "ReplaceWriteThroughOnNodeSplit", + CacheAction::UpdatePendingBackupOnNodeSplit { .. } => "UpdatePendingBackupOnNodeSplit", CacheAction::MambaEvictExcessPathStates { .. } => "MambaEvictExcessPathStates", CacheAction::FreeComponentDeviceSlot { .. } => "FreeComponentDeviceSlot", CacheAction::FreeComponentHostSlot { .. } => "FreeComponentHostSlot", diff --git a/rust/sglang-radix-tree/src/tests/unified_tree_core.rs b/rust/sglang-radix-tree/src/tests/unified_tree_core.rs index 6b11485fb9..d9b1111450 100644 --- a/rust/sglang-radix-tree/src/tests/unified_tree_core.rs +++ b/rust/sglang-radix-tree/src/tests/unified_tree_core.rs @@ -3122,38 +3122,46 @@ fn backup_kv_action_stops_at_an_externally_stored_or_pending_ancestor() { } #[test] -fn split_of_a_pending_node_transfers_the_ack_and_emits_the_replace_action() { - let mut tc = core(); - tc.insert(&insert_params(&vec![1, 2, 3], &[10, 11, 12])); - let node = tc - .match_prefix(&match_params(&vec![1, 2, 3])) - .best_match_node_id; - tc.mark_write_through_pending(vec![node], /* ack_id = */ node) - .expect("live test node"); - let (new_node, action) = tc.split_node_( - tc.arena.resolve(node).expect("live test node"), - /* split_len = */ 1, - ); - assert_eq!(tc.arena.node(new_node).write_through_pending_id, Some(node)); - assert_eq!( - tc.arena - .node(tc.arena.resolve(node).expect("live test node")) - .write_through_pending_id, - Some(node) - ); - match action { - Some(CacheAction::ReplaceWriteThroughOnNodeSplit { - ack_id, - old_node_id, - new_node_id, - new_child_node_id, - }) => { - assert_eq!(ack_id, node); - assert_eq!(old_node_id, node); - assert_eq!(new_node_id, tc.arena.node(new_node).id); - assert_eq!(new_child_node_id, node); +fn split_updates_pending_backup_with_or_without_a_tree_ack() { + for buffer_only in [false, true] { + for pending in [false, true] { + let mut tc = core(); + if buffer_only { + tc.set_host_memory_buffer_only(); + } + tc.insert(&insert_params(&vec![1, 2, 3], &[10, 11, 12])); + let tail_id = tc + .match_prefix(&match_params(&vec![1, 2, 3])) + .best_match_node_id; + if pending { + tc.mark_write_through_pending(vec![tail_id], tail_id) + .unwrap(); + } + let result = tc.match_prefix(&match_params(&vec![1])); + let prefix_id = result.best_match_node_id; + assert_ne!(prefix_id, tail_id); + if pending || buffer_only { + assert!(matches!( + result.cache_actions.as_slice(), + [CacheAction::UpdatePendingBackupOnNodeSplit { + ack_id, + old_node_id, + new_node_id, + new_child_node_id, + }] if *ack_id == pending.then_some(tail_id) && *old_node_id == tail_id + && *new_node_id == prefix_id && *new_child_node_id == tail_id + )); + } else { + assert!(result.cache_actions.is_empty()); + } + for node_id in [prefix_id, tail_id] { + let node = tc.arena.node(tc.arena.resolve(node_id).unwrap()); + assert_eq!(node.write_through_pending_id, pending.then_some(tail_id)); + } + if !pending { + tc.sanity_check(&[], &[]); + } } - other => panic!("expected the replace action, got {:?}", other.is_some()), } } @@ -4029,12 +4037,12 @@ fn finish_write_through_after_a_split_publishes_both_fragments() { .cache_actions .iter() .find_map(|action| match action { - CacheAction::ReplaceWriteThroughOnNodeSplit { + CacheAction::UpdatePendingBackupOnNodeSplit { ack_id, new_node_id, .. } => { - assert_eq!(*ack_id, leaf); + assert_eq!(*ack_id, Some(leaf)); Some(*new_node_id) } _ => None, @@ -4498,7 +4506,7 @@ fn insert_host_refill_preserves_split_actions_and_lengths() { assert!(!result.host_insert_dropped); assert!(matches!( result.cache_actions.as_slice(), - [CacheAction::ReplaceWriteThroughOnNodeSplit { ack_id, .. }] if *ack_id == leaf + [CacheAction::UpdatePendingBackupOnNodeSplit { ack_id, .. }] if *ack_id == Some(leaf) )); } diff --git a/rust/sglang-radix-tree/src/unified_tree_core.rs b/rust/sglang-radix-tree/src/unified_tree_core.rs index 0ebfa57f65..e10826ce41 100644 --- a/rust/sglang-radix-tree/src/unified_tree_core.rs +++ b/rust/sglang-radix-tree/src/unified_tree_core.rs @@ -309,13 +309,14 @@ pub enum CacheAction { FreeDeviceKVFullOnly(Vec), /// A device->host backup work item (the write-through threshold fired). BackupKV(BackupKV), - /// Replace the pending write-through node on a node split: + /// Update cache-side write bookkeeping after a node split: /// /// parent -> node => parent -> new_node -> new_child /// /// old_node_id (the pre-split node) is replaced by new_node_id + new_child_node_id. - ReplaceWriteThroughOnNodeSplit { - ack_id: usize, + UpdatePendingBackupOnNodeSplit { + /// None when buffer-only bookkeeping has no tree-tracked write. + ack_id: Option, old_node_id: NodeId, new_node_id: NodeId, new_child_node_id: NodeId, @@ -1850,7 +1851,7 @@ impl UnifiedTreeCore { action, CacheAction::FreeDeviceKV(_) | CacheAction::FreeDeviceKVFullOnly(_) - | CacheAction::ReplaceWriteThroughOnNodeSplit { .. } + | CacheAction::UpdatePendingBackupOnNodeSplit { .. } ) } @@ -2213,10 +2214,13 @@ impl UnifiedTreeCore { self.arena.node_mut(new_node_id).load_back_pending_id = self.arena.node(child_id).load_back_pending_id; - // A split of a backuped node tells the cache to fix its publish list. - let action = if let Some(ack_id) = self.arena.node(child_id).write_through_pending_id { + let ack_id = self.arena.node(child_id).write_through_pending_id; + if let Some(ack_id) = ack_id { self.arena.node_mut(new_node_id).write_through_pending_id = Some(ack_id); - Some(CacheAction::ReplaceWriteThroughOnNodeSplit { + } + // Buffer-only writes are tracked by the cache rather than a node ack. + let action = if ack_id.is_some() || self.is_host_memory_buffer_only { + Some(CacheAction::UpdatePendingBackupOnNodeSplit { ack_id, old_node_id: self.arena.node(child_id).id, new_node_id: self.arena.node(new_node_id).id, diff --git a/test/registered/unit/mem_cache/test_rust_tree_core_integration.py b/test/registered/unit/mem_cache/test_rust_tree_core_integration.py index a96dd8611c..ec2e0320f3 100644 --- a/test/registered/unit/mem_cache/test_rust_tree_core_integration.py +++ b/test/registered/unit/mem_cache/test_rust_tree_core_integration.py @@ -61,12 +61,13 @@ from sglang.srt.mem_cache.rust_tree_core.extension import bindings as mem_cache from sglang.srt.mem_cache.rust_tree_core.extension import load_tree_core_extension from sglang.srt.mem_cache.unified_cache.cache_action import ( BackupKV, + ComponentAction, FreeComponentHostSlot, FreeDeviceKV, FreeDeviceKVFullOnly, RecoverSWAWithLockedFull, - ReplaceWriteThroughOnNodeSplit, SWARebuild, + UpdatePendingBackupOnNodeSplit, ) from sglang.srt.mem_cache.unified_cache.component_type import ComponentType from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import ( @@ -4155,7 +4156,7 @@ def test_swa_window_repair_returns_split_actions_and_balances_existing_locks(): actions = core.attach_swa_window(key, 2, 6, torch.arange(50, 54)) assert len(actions) == 2 for action in actions: - assert isinstance(action, ReplaceWriteThroughOnNodeSplit) + assert isinstance(action, UpdatePendingBackupOnNodeSplit) assert action.ack_id == leaf assert action.old_node_id == leaf assert action.new_child_node_id == leaf @@ -4660,23 +4661,104 @@ def root_child(core): return core.match_prefix(MatchPrefixParams(key=_key([1]))).last_host_node -def test_split_of_a_write_through_pending_node_crosses_the_replace_action(): - core = _tree_core() +@pytest.mark.parametrize("backend", ["python", "rust"]) +@pytest.mark.parametrize("mode", ["pending", "buffer", "both", "ordinary"]) +@pytest.mark.parametrize("split_by", ["insert", "match"]) +def test_split_pending_backup_action(backend, mode, split_by): + core = _tree_core(backend) core.set_hicache_enabled() + has_ack = mode in ("pending", "both") + has_buffer = mode in ("buffer", "both") + if has_buffer: + core.set_host_memory_buffer_only() _insert(core, [1, 2, 3, 4], [10, 11, 12, 13]) leaf = core.match_prefix(MatchPrefixParams(key=_key([1, 2, 3, 4]))).best_match_node - core.mark_write_through_pending([leaf], ack_id=leaf) - # A divergent prefix splits the pending node; the publish list must follow. - result = _insert(core, [1, 2], [10, 11]) - (replace,) = [ + if has_ack: + core.mark_write_through_pending([leaf], ack_id=leaf) + # Both insertion and matching can split the existing node. + if split_by == "insert": + result = _insert(core, [1, 2], [10, 11]) + else: + result = core.match_prefix(MatchPrefixParams(key=_key([1, 2]))) + split_actions = [ action for action in result.cache_actions - if isinstance(action, ReplaceWriteThroughOnNodeSplit) + if isinstance(action, UpdatePendingBackupOnNodeSplit) ] - assert replace.ack_id == leaf - assert replace.old_node_id == leaf - assert replace.new_child_node_id == leaf - assert replace.new_node_id != leaf + assert len(split_actions) == int(has_ack or has_buffer) + (path,) = core.collect_path_info([leaf]) + prefix, suffix = [node.node_id for node in path] + assert suffix == leaf and prefix != leaf + if split_actions: + (split_action,) = split_actions + assert split_action.ack_id == (leaf if has_ack else None) + assert split_action.old_node_id == leaf + assert split_action.new_node_id == prefix + assert split_action.new_child_node_id == leaf + assert isinstance(split_action, ComponentAction) + assert split_action.component_type == ComponentType.FULL + if backend == "python": + assert [ + core.node_by_id(node_id).write_through_pending_id + for node_id in (prefix, leaf) + ] == [leaf if has_ack else None] * 2 + + +@pytest.mark.parametrize("backend", ["python", "rust"]) +@pytest.mark.parametrize("window", [1, 8]) +def test_fresh_swa_leaf_forwards_buffer_split_action(backend, window): + core, _ = _swa_transfer_core(backend, window=window) + core.set_host_memory_buffer_only() + result = _pump_insert( + core, + InsertParams( + key=_key([1, 2, 3, 4]), + value=torch.tensor([10, 11, 12, 13]), + component_evicted_seqlens={ComponentType.SWA: 2}, + ), + ) + splits = [ + action + for action in result.cache_actions + if isinstance(action, UpdatePendingBackupOnNodeSplit) + ] + assert len(splits) == (2 if window == 1 else 1) + for split in splits: + assert split.ack_id is None + assert split.old_node_id == split.new_child_node_id == result.last_device_node + (path,) = core.collect_path_info([result.last_device_node]) + assert [node.node_id for node in path] == [ + *[split.new_node_id for split in splits], + result.last_device_node, + ] + assert [(node.path_start, node.path_end) for node in path] == ( + [(0, 2), (2, 3), (3, 4)] if window == 1 else [(0, 2), (2, 4)] + ) + rebuilds = [a for a in result.cache_actions if isinstance(a, SWARebuild)] + assert [rebuild.node_id for rebuild in rebuilds] == [ + part.node_id for part in path[1:] + ] + assert torch.cat([rebuild.source_value for rebuild in rebuilds]).tolist() == [ + 12, + 13, + ] + assert result.cache_actions.index(splits[-1]) < result.cache_actions.index( + rebuilds[0] + ) + + +def test_python_fresh_swa_leaf_cap_rejects_write_through_ack(): + core, _ = _swa_transfer_core("python", window=2) + core.set_host_memory_buffer_only() + leaf = core._add_new_node( + core.root_node, _key([1, 2, 3, 4]), torch.tensor([10, 11, 12, 13]) + ) + core.mark_write_through_pending([leaf.id], ack_id=leaf.id) + swa = core.components_by_type[ComponentType.SWA] + with pytest.raises( + AssertionError, match="fresh SWA leaf cannot be write-through-pending" + ): + swa._maybe_split_leaf_for_swa_lock(leaf, []) def test_write_through_threshold_assignment_reaches_the_core(): diff --git a/test/registered/unit/mem_cache/test_unified_cache_linker.py b/test/registered/unit/mem_cache/test_unified_cache_linker.py index 69b42bca9d..ea39200f21 100644 --- a/test/registered/unit/mem_cache/test_unified_cache_linker.py +++ b/test/registered/unit/mem_cache/test_unified_cache_linker.py @@ -25,7 +25,7 @@ from sglang.srt.mem_cache.hicache_storage import ( ) from sglang.srt.mem_cache.radix_cache import RadixKey from sglang.srt.mem_cache.unified_cache.cache_action import ( - ReplaceWriteThroughOnNodeSplit, + UpdatePendingBackupOnNodeSplit, ) from sglang.srt.mem_cache.unified_cache.component_type import ComponentType from sglang.srt.mem_cache.unified_cache.components.base import ( @@ -1198,7 +1198,12 @@ def test_split_action_retargets_pending_external_offload(): cache._replace_pending_write_through_node = lambda *args: calls.append( ("hicache", *args) ) - action = ReplaceWriteThroughOnNodeSplit( + cache.components = { + ComponentType.FULL: SimpleNamespace( + apply_component_action=lambda action: calls.append(("full", action)) + ) + } + action = UpdatePendingBackupOnNodeSplit( ack_id=7, old_node_id=7, new_node_id=8, @@ -1210,6 +1215,7 @@ def test_split_action_retargets_pending_external_offload(): assert calls == [ ("hicache", 7, 7, [8, 7]), ("linker", 7, 7, [8, 7]), + ("full", action), ] diff --git a/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_core.py b/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_core.py index 759b3030d8..7f4fdb9d49 100644 --- a/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_core.py +++ b/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_core.py @@ -31,7 +31,7 @@ from sglang.srt.mem_cache.memory_pool import ( from sglang.srt.mem_cache.radix_cache import RadixKey from sglang.srt.mem_cache.unified_cache.cache_action import ( RecoverSWAWithLockedFull, - ReplaceWriteThroughOnNodeSplit, + UpdatePendingBackupOnNodeSplit, ) from sglang.srt.mem_cache.unified_cache.components.base import ( ComponentType, @@ -1997,6 +1997,14 @@ class TestUnifiedRadixCacheInt8MambaCheckpoint( self.assertEqual(cache.mamba_evictable_size(), 0) +def _component_with_cache(component_type, cache): + """A registry component instance bound to a (mock) cache and its tree_core.""" + component = object.__new__(COMPONENT_REGISTRY[component_type]) + component.cache = cache + component.tree_core = cache.tree_core + return component + + class TestUnifiedRadixCacheActionRouting(CustomTestCase): """CacheAction routing: each type forwards to the right Controller API.""" @@ -2078,10 +2086,10 @@ class TestUnifiedRadixCacheActionRouting(CustomTestCase): def test_apply_cache_actions_applies_each_in_order(self): cache = mock.MagicMock() - first = ReplaceWriteThroughOnNodeSplit( + first = UpdatePendingBackupOnNodeSplit( ack_id=1, old_node_id=1, new_node_id=2, new_child_node_id=1 ) - second = ReplaceWriteThroughOnNodeSplit( + second = UpdatePendingBackupOnNodeSplit( ack_id=2, old_node_id=3, new_node_id=4, new_child_node_id=3 ) UnifiedRadixCache._apply_cache_actions(cache, [first, second]) @@ -2096,21 +2104,20 @@ class TestUnifiedRadixCacheActionRouting(CustomTestCase): def make_cache(): cache = mock.MagicMock() cache.ongoing_write_through = {7: _OngoingWriteThrough(10, None, [5, 10])} + cache._replace_pending_write_through_node = lambda *args: ( + UnifiedRadixCache._replace_pending_write_through_node(cache, *args) + ) + cache.components = {ComponentType.FULL: mock.Mock()} return cache def apply(cache, action): - UnifiedRadixCache._replace_pending_write_through_node( - cache, - action.ack_id, - action.old_node_id, - [action.new_node_id, action.new_child_node_id], - ) + UnifiedRadixCache._apply_cache_action(cache, action) # split X(10) -> [A(11), X(10)], then fragment A(11) -> [B(12), A(11)] - first = ReplaceWriteThroughOnNodeSplit( + first = UpdatePendingBackupOnNodeSplit( ack_id=7, old_node_id=10, new_node_id=11, new_child_node_id=10 ) - second = ReplaceWriteThroughOnNodeSplit( + second = UpdatePendingBackupOnNodeSplit( ack_id=7, old_node_id=11, new_node_id=12, new_child_node_id=11 ) @@ -2121,6 +2128,12 @@ class TestUnifiedRadixCacheActionRouting(CustomTestCase): self.assertEqual( cache.ongoing_write_through[7].publish_node_ids, [5, 12, 11, 10] ) + cache.linker.replace_pending_offload_node.assert_has_calls( + [mock.call(7, 10, [11, 10]), mock.call(7, 11, [12, 11])] + ) + receiver = cache.components[ComponentType.FULL].apply_component_action + self.assertEqual(receiver.call_args_list, [mock.call(first), mock.call(second)]) + self.assertEqual(cache.linker.replace_pending_offload_node.call_count, 2) # reversed order silently drops the second fragment - documents why # emission order must be preserved end to end @@ -2129,6 +2142,21 @@ class TestUnifiedRadixCacheActionRouting(CustomTestCase): apply(cache, first) self.assertEqual(cache.ongoing_write_through[7].publish_node_ids, [5, 11, 10]) + def test_split_action_without_ack_still_reaches_full(self): + cache = mock.MagicMock() + component = _component_with_cache(ComponentType.FULL, cache) + component.apply_component_action = mock.Mock( + wraps=component.apply_component_action + ) + cache.components = {ComponentType.FULL: component} + action = UpdatePendingBackupOnNodeSplit( + ack_id=None, old_node_id=10, new_node_id=11, new_child_node_id=10 + ) + UnifiedRadixCache._apply_cache_action(cache, action) + component.apply_component_action.assert_called_once_with(action) + cache._replace_pending_write_through_node.assert_not_called() + cache.linker.replace_pending_offload_node.assert_not_called() + class TestSWADraftHistory(SWADraftHistoryFixtures, CustomTestCase): """A widened draft needs SWA history before the target window. A running