From 4611ea365e1a10125f1882ae67b46d6832b4408a Mon Sep 17 00:00:00 2001 From: Lukas Humbel Date: Sun, 4 Oct 2026 05:16:04 +0200 Subject: [PATCH] [HiCache]: Fix host lock ownership across radix splits (#38480) Signed-off-by: Lukas Humbel Co-authored-by: Jialin Ouyang Co-authored-by: Shuwen Wang <47200617+alphabetc1@users.noreply.github.com> --- .../unified_cache/components/README.md | 9 +- .../unified_cache/components/base.py | 25 +++- .../unified_cache/components/full.py | 37 ++++- .../mem_cache/unified_cache/components/swa.py | 2 +- rust/sglang-radix-tree/src/components/full.rs | 54 ++++++-- rust/sglang-radix-tree/src/components/mod.rs | 39 +++++- rust/sglang-radix-tree/src/components/swa.rs | 23 +--- rust/sglang-radix-tree/src/node.rs | 4 + rust/sglang-radix-tree/src/python_bindings.rs | 27 ++++ .../src/tests/components/full.rs | 80 +++++++---- .../src/tests/components/mamba.rs | 8 ++ .../src/tests/components/swa.rs | 68 ++++++++-- .../src/tests/unified_tree_core.rs | 9 +- .../src/unified_tree_core.rs | 42 +++++- .../rust_unified_tree_core_inspector.py | 7 + .../mem_cache/test_component_uuid_counter.py | 43 ++++++ .../test_rust_tree_core_integration.py | 13 +- .../test_unified_radix_cache_unittest.py | 126 +++++++++++++++++- .../unified_tree_core_inspection_interface.py | 7 + .../mem_cache/unified_tree_core_inspector.py | 6 + 20 files changed, 526 insertions(+), 103 deletions(-) create mode 100644 test/registered/unit/mem_cache/test_component_uuid_counter.py diff --git a/python/sglang/srt/mem_cache/unified_cache/components/README.md b/python/sglang/srt/mem_cache/unified_cache/components/README.md index d5bc53e642..1005f03616 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/README.md +++ b/python/sglang/srt/mem_cache/unified_cache/components/README.md @@ -215,7 +215,10 @@ so the order is not load-bearing for the leaf sets. Full walks to root; SWA stops at the receipt boundary; components in `skipped_lock_components` are left alone. `skip_swa=True` also skips lower-priority components already released by `dec_swa_lock_only`. The host-side `dec_host_lock_ref` takes the -same required receipt. +same required receipt. A Full host lock begins as a one-node UUID-bounded +segment. A later radix split copies the anchor's `host_lock_ref` to the inserted +prefix and migrates the UUID to that prefix; release walks every fragment until +it finds the receipt's UUID. --- @@ -289,8 +292,8 @@ Each component implements these hooks. See `base.py` for the ABC and docstrings. | Hook | Purpose | Called By | Default | |------|---------|-----------|----------| -| `acquire_component_lock(lock_host=False)` | Increment device or host lock refs; moves device tokens from evictable to protected. Full: path-lock for device, single-node host lock. SWA: window-lock with UUID boundary. Mamba: single-node lock. | `inc_lock_ref`, `inc_host_lock_ref` | *abstract* | -| `release_component_lock(lock_host=False)` | Decrement device or host lock refs; moves device tokens from protected to evictable when `lock_ref` → 0. Full path-unlocks device; SWA walks up to UUID boundary; Mamba unlocks a single node. | `dec_lock_ref`, `dec_host_lock_ref` | *abstract* | +| `acquire_component_lock(lock_host=False)` | Increment device or host lock refs; moves device tokens from evictable to protected. Full: path-lock for device; its host lock starts as a one-node UUID-bounded segment so later split fragments remain protected. SWA: window-lock with UUID boundary. Mamba: single-node lock. | `inc_lock_ref`, `inc_host_lock_ref` | *abstract* | +| `release_component_lock(lock_host=False)` | Decrement device or host lock refs; moves device tokens from protected to evictable when `lock_ref` -> 0. Full path-unlocks device and releases all host split fragments to the receipt boundary; SWA walks up to UUID boundary; Mamba unlocks a single node. | `dec_lock_ref`, `dec_host_lock_ref` | *abstract* | ### Caching Phase diff --git a/python/sglang/srt/mem_cache/unified_cache/components/base.py b/python/sglang/srt/mem_cache/unified_cache/components/base.py index e9c3996056..36739f744d 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/base.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/base.py @@ -41,7 +41,18 @@ if TYPE_CHECKING: _NUM_COMPONENT_TYPES = len(ComponentType) _LAST_ACCESS_TIME_COUNTER_FLOAT = float64(1.0) -_COMPONENT_UUID_COUNTER = 1 +_COMPONENT_UUID_RANGE_SIZE = 100_000_000_000_000 +_COMPONENT_UUID_RANGE_INDEX = { + ComponentType.SWA: 0, + ComponentType.FULL: 1, + ComponentType.MAMBA: 2, + ComponentType.C128: 3, + ComponentType.AUXILIARY_SWA: 4, +} +_COMPONENT_UUID_COUNTERS = { + component_type: (range_index + 1) * _COMPONENT_UUID_RANGE_SIZE + for component_type, range_index in _COMPONENT_UUID_RANGE_INDEX.items() +} @dataclasses.dataclass @@ -121,10 +132,9 @@ def get_and_increase_time_counter() -> float64: return ret -def next_component_uuid() -> int: - global _COMPONENT_UUID_COUNTER - _COMPONENT_UUID_COUNTER += 1 - return _COMPONENT_UUID_COUNTER +def next_component_uuid(component_type: ComponentType) -> int: + _COMPONENT_UUID_COUNTERS[component_type] += 1 + return _COMPONENT_UUID_COUNTERS[component_type] class TreeComponent(ABC): @@ -589,7 +599,7 @@ class TreeComponent(ABC): node itself (mamba state is per-leaf, not per-path). When ``lock_host`` is True, the lock applies to host-side state: - - Full: single-node host lock. + - Full: a one-node UUID-bounded segment that expands across splits. - SWA: host window-lock with a dedicated host UUID boundary. - Mamba: single-node host lock with host LRU detach.""" ... @@ -610,7 +620,8 @@ class TreeComponent(ABC): - Mamba: single-node unlock — only decrements lock_ref on the node itself. - When ``lock_host`` is True, the inverse host-side semantics apply.""" + When ``lock_host`` is True, Full and SWA replay their host boundary + UUIDs while Mamba retains its single-node semantics.""" ... def prepare_for_caching_req( diff --git a/python/sglang/srt/mem_cache/unified_cache/components/full.py b/python/sglang/srt/mem_cache/unified_cache/components/full.py index 1d4b255fad..f9dc55c82f 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/full.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/full.py @@ -26,6 +26,7 @@ from sglang.srt.mem_cache.unified_cache.components.base import ( ExternalLinkerLoadPhase, LinkerTransferPhase, TreeComponent, + next_component_uuid, ) if TYPE_CHECKING: @@ -146,6 +147,13 @@ class FullComponent(TreeComponent): new_parent.component_data[ct].lock_ref = child.component_data[ct].lock_ref new_parent.component_data[ct].session_ref = child.component_data[ct].session_ref child_cd = child.component_data[ct] + new_parent.component_data[ct].host_lock_ref = child_cd.host_lock_ref + # A Full host lock starts as a one-node segment. Splitting that node + # extends the segment through the inserted prefix, so its boundary + # moves to the prefix's older edge just like the SWA segment boundary. + host_uuid = child_cd.metadata.pop("host_uuid", None) + if host_uuid is not None: + new_parent.component_data[ct].metadata["host_uuid"] = host_uuid assert new_parent.component_data[ct].session_ids is None split_len = len(new_parent.key) if child_cd.value is not None: @@ -274,6 +282,9 @@ class FullComponent(TreeComponent): # write_back mode: the anchor may be device-only (no host_value); pin it anyway. if cd.host_value is None and not self.tree_core.is_write_back: return result + if cd.metadata.get("host_uuid") is None: + cd.metadata["host_uuid"] = next_component_uuid(ct) + result.set_lock_uuid(ct, cd.metadata["host_uuid"], lock_host=True) cd.host_lock_ref += 1 self.tree_core._update_evictable_leaf_sets(node) return result @@ -314,13 +325,27 @@ class FullComponent(TreeComponent): ) -> None: ct = self.component_type if lock_host: - cd = node.component_data[ct] - if cd.host_lock_ref == 0: + if ct not in params.component_host_lock_uuids: + # This receipt did not acquire the Full host component. return - if cd.host_value is None and not self.tree_core.is_write_back: - return - cd.host_lock_ref -= 1 - self.tree_core._update_evictable_leaf_sets(node) + boundary_uuid = params.get_lock_uuid(ct, lock_host=True) + assert boundary_uuid is not None, "Full host lock receipt has no boundary" + while True: + cd = node.component_data[ct] + if cd.host_value is None and not self.tree_core.is_write_back: + return + assert cd.host_lock_ref > 0, ( + f"Full host segment release hit host_lock_ref=0 on node {node.id}" + ) + cd.host_lock_ref -= 1 + self.tree_core._update_evictable_leaf_sets(node) + if cd.metadata.get("host_uuid") == boundary_uuid: + break + assert node.parent is not None, ( + f"Full host lock boundary {boundary_uuid} is not an ancestor " + f"of receipt anchor {params.node_id}" + ) + node = node.parent return root = self.tree_core.root_node diff --git a/python/sglang/srt/mem_cache/unified_cache/components/swa.py b/python/sglang/srt/mem_cache/unified_cache/components/swa.py index e5263d91eb..6198a0a9ab 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/swa.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/swa.py @@ -843,7 +843,7 @@ class SWAComponent(TreeComponent): covered += len(cur.key) if covered >= sliding_window_size: if comp.metadata.get(uuid_key) is None: - comp.metadata[uuid_key] = next_component_uuid() + comp.metadata[uuid_key] = next_component_uuid(ct) swa_uuid = comp.metadata[uuid_key] cur = cur.parent diff --git a/rust/sglang-radix-tree/src/components/full.rs b/rust/sglang-radix-tree/src/components/full.rs index 20f567b2c5..1eb72901e3 100644 --- a/rust/sglang-radix-tree/src/components/full.rs +++ b/rust/sglang-radix-tree/src/components/full.rs @@ -86,6 +86,10 @@ impl TreeComponent for FullComponent { let (new_parent, child) = tree_core.arena.node_pair_mut(new_parent_id, child_id); let split_len = new_parent.key.atom_len() as i64; new_parent.copy_device_lock_ref(FULL, child); + new_parent.set_lock_ref_(FullComponent::HOST, child.host_lock_ref(FULL)); + // The boundary marks the older edge of the host-lock segment. A split + // inserts the prefix at that edge, so move the boundary with it. + new_parent.full_host_uuid = child.full_host_uuid.take(); if child.has_device_value(FULL) { Node::redistribute_child_device_value(new_parent, child, FULL, split_len); } @@ -279,12 +283,13 @@ impl TreeComponent for FullComponent { ) -> IncLockRefResult { // Only the last host node needs to be protected. if lock_host { - let node = tree_core.arena.node_mut(node_id); // write_back mode: the anchor may be device-only (no host_value); pin it anyway. - if !node.has_host_value(FULL) && !tree_core.is_write_back { + if !tree_core.arena.node(node_id).has_host_value(FULL) && !tree_core.is_write_back { return result; } - node.inc_host_lock_ref(FULL); + let boundary_uuid = self.get_or_fill_uuid(tree_core, node_id, lock_host); + result.set_lock_uuid(FULL.idx() as u8, Some(boundary_uuid), lock_host); + tree_core.arena.node_mut(node_id).inc_host_lock_ref(FULL); tree_core.update_evictable_leaf_sets_(node_id); return result; } @@ -336,20 +341,47 @@ impl TreeComponent for FullComponent { &self, tree_core: &mut UnifiedTreeCore, node_id: NodeIdx_, - _params: &DecLockRefParams, + params: &DecLockRefParams, lock_host: bool, ) { if lock_host { - let node = tree_core.arena.node_mut(node_id); - if node.host_lock_ref(FULL) == 0 { + if !params + .component_host_lock_uuids + .contains_key(&(FULL.idx() as u8)) + { return; } - // Mirror of `acquire`. write_back uses a pure counter. - if !node.has_host_value(FULL) && !tree_core.is_write_back { - return; + let boundary_uuid = params + .get_lock_uuid(FULL.idx() as u8, lock_host) + .expect("Full host lock receipt has no boundary"); + let mut cur = node_id; + loop { + let node = tree_core.arena.node(cur); + if !node.has_host_value(FULL) && !tree_core.is_write_back { + return; + } + assert!( + node.host_lock_ref(FULL) > 0, + "Full host segment release hit host_lock_ref=0 on node {cur}" + ); + let at_boundary = node.full_host_uuid == Some(boundary_uuid); + let parent = if at_boundary { + None + } else { + Some(node.try_parent().unwrap_or_else(|| { + panic!( + "Full host lock boundary {boundary_uuid} is not an ancestor of receipt anchor {:?}", + params.node_id + ) + })) + }; + tree_core.arena.node_mut(cur).dec_host_lock_ref(FULL); + tree_core.update_evictable_leaf_sets_(cur); + if at_boundary { + break; + } + cur = parent.unwrap(); } - node.dec_host_lock_ref(FULL); - tree_core.update_evictable_leaf_sets_(node_id); return; } diff --git a/rust/sglang-radix-tree/src/components/mod.rs b/rust/sglang-radix-tree/src/components/mod.rs index 1d8c165c82..dfeeb459eb 100644 --- a/rust/sglang-radix-tree/src/components/mod.rs +++ b/rust/sglang-radix-tree/src/components/mod.rs @@ -60,6 +60,40 @@ pub trait TreeComponent { /// The component this driver serves. fn component_type(&self) -> ComponentType; + /// Return the node's lock-segment UUID, minting and storing one when absent. + fn get_or_fill_uuid( + &self, + tree_core: &mut UnifiedTreeCore, + node_id: NodeIdx_, + host: bool, + ) -> i64 { + let component_type = self.component_type(); + let existing = match (component_type, host) { + (ComponentType::Full, true) => tree_core.arena.node(node_id).full_host_uuid, + (ComponentType::Swa, true) => tree_core.arena.node(node_id).swa_host_uuid, + (ComponentType::Swa, false) => tree_core.arena.node(node_id).swa_uuid, + (ComponentType::Full, false) => { + panic!("get_or_fill_uuid is unsupported for Full device locks") + } + (ComponentType::Mamba, _) => { + panic!("get_or_fill_uuid is unsupported for Mamba") + } + }; + if let Some(uuid) = existing { + return uuid; + } + + let uuid = tree_core.next_component_uuid_(component_type); + let node = tree_core.arena.node_mut(node_id); + match (component_type, host) { + (ComponentType::Full, true) => node.full_host_uuid = Some(uuid), + (ComponentType::Swa, true) => node.swa_host_uuid = Some(uuid), + (ComponentType::Swa, false) => node.swa_uuid = Some(uuid), + (ComponentType::Full, false) | (ComponentType::Mamba, _) => unreachable!(), + } + uuid + } + /// Whether this component has device data that still needs a host backup. fn needs_incremental_backup( &self, @@ -308,7 +342,7 @@ pub trait TreeComponent { // node itself (mamba state is per-leaf, not per-path). // // When ``lock_host`` is True, the lock applies to host-side state: - // - Full: single-node host lock. + // - Full: a one-node UUID-bounded segment that expands across splits. // - SWA: host window-lock with a dedicated host UUID boundary. // - Mamba: single-node host lock with host LRU detach.""" // ... @@ -338,7 +372,8 @@ pub trait TreeComponent { // - Mamba: single-node unlock — only decrements lock_ref on the // node itself. // - // When ``lock_host`` is True, the inverse host-side semantics apply.""" + // When ``lock_host`` is True, Full and SWA replay their host boundary + // UUIDs while Mamba retains its single-node semantics.""" // ... fn release_component_lock( &self, diff --git a/rust/sglang-radix-tree/src/components/swa.rs b/rust/sglang-radix-tree/src/components/swa.rs index cd840002d8..bfa547549f 100644 --- a/rust/sglang-radix-tree/src/components/swa.rs +++ b/rust/sglang-radix-tree/src/components/swa.rs @@ -134,27 +134,6 @@ impl SwaComponent { } } - /// The node's SWA lock-window uuid for the tier, stamping a fresh one if absent. - fn ensure_swa_uuid( - tree_core: &mut UnifiedTreeCore, - node_id: NodeIdx_, - host: bool, - ) -> i64 { - match Self::swa_uuid(tree_core.arena.node(node_id), host) { - Some(uuid) => uuid, - None => { - let minted = tree_core.next_swa_uuid_(); - let node = tree_core.arena.node_mut(node_id); - if host { - node.swa_host_uuid = Some(minted); - } else { - node.swa_uuid = Some(minted); - } - minted - } - } - } - /// Nodes whose SWA data needs a host backup, deepest first. Buffer mode /// stages one node per FIFO backup intent; cache mode backs up every /// device-only node within one sliding window of `node_id`. @@ -1208,7 +1187,7 @@ impl TreeComponent for SwaComponent { } covered += key_len; if covered >= sliding_window_size { - swa_uuid = Some(Self::ensure_swa_uuid(tree_core, cur, lock_host)); + swa_uuid = Some(self.get_or_fill_uuid(tree_core, cur, lock_host)); } cur = parent; } diff --git a/rust/sglang-radix-tree/src/node.rs b/rust/sglang-radix-tree/src/node.rs index cc53e62276..492d83f4c1 100644 --- a/rust/sglang-radix-tree/src/node.rs +++ b/rust/sglang-radix-tree/src/node.rs @@ -172,6 +172,8 @@ pub struct Node { pub swa_uuid: Option, /// SWA lock-window uuid for host locks; stamped where a host lock walk fills the window. pub swa_host_uuid: Option, + /// Full host-lock segment boundary uuid. A split migrates it to the new prefix. + pub full_host_uuid: Option, /// Per-page hash chain; None when the node was never hashed. /// TODO: Store raw digests and hex-encode only at the Python or storage boundary. pub hash_value: Option>, @@ -404,6 +406,7 @@ impl Node { values: Default::default(), swa_uuid: None, swa_host_uuid: None, + full_host_uuid: None, hash_value: Some(Vec::new()), external_cache_stored: false, rotation_base: None, @@ -430,6 +433,7 @@ impl Node { values: Default::default(), swa_uuid: None, swa_host_uuid: None, + full_host_uuid: None, hash_value: None, external_cache_stored: false, rotation_base: None, diff --git a/rust/sglang-radix-tree/src/python_bindings.rs b/rust/sglang-radix-tree/src/python_bindings.rs index c7a9bdfbd2..fa6c04c07c 100644 --- a/rust/sglang-radix-tree/src/python_bindings.rs +++ b/rust/sglang-radix-tree/src/python_bindings.rs @@ -2320,6 +2320,20 @@ impl TreeCoreBinding { .map_err(node_access_error) } + fn inspect_get_component_host_lock_ref( + &self, + py: Python<'_>, + node_id: NodeId, + component_type: u8, + ) -> PyResult { + let component_type = parse_component_type(component_type)?; + py.allow_threads(|| { + self.core() + .inspect_get_component_host_lock_ref(node_id, component_type) + }) + .map_err(node_access_error) + } + fn inspect_get_node_hit_count(&self, py: Python<'_>, node_id: NodeId) -> PyResult { py.allow_threads(|| self.core().inspect_get_node_hit_count(node_id)) .map_err(node_access_error) @@ -3541,6 +3555,19 @@ macro_rules! tree_core_binding { }) } + #[cfg(feature = "inspection")] + fn inspect_get_component_host_lock_ref( + &self, + py: Python<'_>, + node_id: NodeId, + component_type: u8, + ) -> PyResult { + catch_native_panic(|| { + self.inner + .inspect_get_component_host_lock_ref(py, node_id, component_type) + }) + } + #[cfg(feature = "inspection")] fn inspect_get_node_hit_count(&self, py: Python<'_>, node_id: NodeId) -> PyResult { catch_native_panic(|| self.inner.inspect_get_node_hit_count(py, node_id)) diff --git a/rust/sglang-radix-tree/src/tests/components/full.rs b/rust/sglang-radix-tree/src/tests/components/full.rs index 4116b6f68a..e4de3ae28c 100644 --- a/rust/sglang-radix-tree/src/tests/components/full.rs +++ b/rust/sglang-radix-tree/src/tests/components/full.rs @@ -8,6 +8,10 @@ fn core() -> UnifiedTreeCore> { UnifiedTreeCore::new(CacheInitParams::default(), vec![FULL]) } +fn dec_params(lock: IncLockRefResult) -> DecLockRefParams { + lock.to_dec_params() +} + // Raw seeding for states set_value rejects: mid-split (key trimmed before the value // splits) and present-but-empty semantics pins. fn set_value_no_check( @@ -1351,6 +1355,11 @@ fn inc_host_lock_ref_pins_the_backuped_anchor() { .inc_host_lock_ref(tc.arena.node(node).id) .expect("live test node"); assert_eq!(result.delta, None); + assert!( + result + .component_host_lock_uuids + .contains_key(&(FULL.idx() as u8)) + ); assert_eq!(tc.arena.host_lock_ref(node, FULL), 1); // The pinned anchor leaves the H-leaf set; the device tier is untouched. assert!(!tc.evictable_host_leaves.contains(node)); @@ -1423,11 +1432,25 @@ fn host_lock_round_trips_on_a_root_anchor_are_noops() { .expect("live test node"); assert_eq!(result.delta, None); assert_eq!(tc.arena.host_lock_ref(root, FULL), 0); - tc.dec_host_lock_ref(tc.arena.node(root).id, &DecLockRefParams::default()) + tc.dec_host_lock_ref(tc.arena.node(root).id, &dec_params(result)) .expect("live test node"); assert_eq!(tc.arena.host_lock_ref(root, FULL), 0); } +#[test] +fn write_back_root_host_lock_round_trips_without_a_parent_boundary() { + let mut tc = write_back_core(); + let root = tc.arena.root(); + let baseline = tc.arena.host_lock_ref(root, FULL); + let root_handle = tc.arena.node(root).id; + let lock = tc.inc_host_lock_ref(root_handle).expect("live root node"); + let params = dec_params(lock); + assert_eq!(tc.arena.host_lock_ref(root, FULL), baseline + 1); + tc.dec_host_lock_ref(root_handle, ¶ms) + .expect("live root node"); + assert_eq!(tc.arena.host_lock_ref(root, FULL), baseline); +} + #[test] fn inc_host_lock_ref_under_write_back_pins_a_device_only_anchor() { let mut tc = write_back_core(); @@ -1446,9 +1469,11 @@ fn dec_host_lock_ref_unpins_and_restores_the_h_leaf_set() { let mut tc = core(); let node = host_lock_anchor(&mut tc); tc.component_state_mut(FULL).evictable_size = 7; - tc.inc_host_lock_ref(tc.arena.node(node).id) - .expect("live test node"); - tc.dec_host_lock_ref(tc.arena.node(node).id, &DecLockRefParams::default()) + let params = dec_params( + tc.inc_host_lock_ref(tc.arena.node(node).id) + .expect("live test node"), + ); + tc.dec_host_lock_ref(tc.arena.node(node).id, ¶ms) .expect("live test node"); assert_eq!(tc.arena.host_lock_ref(node, FULL), 0); assert!(tc.evictable_host_leaves.contains(node)); @@ -1471,10 +1496,12 @@ fn dec_host_lock_ref_keeps_the_counter_when_the_host_value_is_gone() { // A host-evicted anchor keeps its pin count under write-through. let mut tc = core(); let node = host_lock_anchor(&mut tc); - tc.inc_host_lock_ref(tc.arena.node(node).id) - .expect("live test node"); + let params = dec_params( + tc.inc_host_lock_ref(tc.arena.node(node).id) + .expect("live test node"), + ); let _ = tc.arena.take_host_value(node, FULL); - tc.dec_host_lock_ref(tc.arena.node(node).id, &DecLockRefParams::default()) + tc.dec_host_lock_ref(tc.arena.node(node).id, ¶ms) .expect("live test node"); assert_eq!(tc.arena.host_lock_ref(node, FULL), 1); } @@ -1483,9 +1510,11 @@ fn dec_host_lock_ref_keeps_the_counter_when_the_host_value_is_gone() { fn host_lock_round_trip_under_write_back_is_a_pure_counter() { let mut tc = write_back_core(); let (_n1, n2) = lock_chain(&mut tc); - tc.inc_host_lock_ref(tc.arena.node(n2).id) - .expect("live test node"); - tc.dec_host_lock_ref(tc.arena.node(n2).id, &DecLockRefParams::default()) + let params = dec_params( + tc.inc_host_lock_ref(tc.arena.node(n2).id) + .expect("live test node"), + ); + tc.dec_host_lock_ref(tc.arena.node(n2).id, ¶ms) .expect("live test node"); assert_eq!(tc.arena.host_lock_ref(n2, FULL), 0); let state = tc.component_state(FULL); @@ -1510,14 +1539,11 @@ fn acquire_host_arm_updates_the_h_leaf_set_without_the_dispatcher() { fn release_host_arm_updates_the_h_leaf_set_without_the_dispatcher() { let mut tc = core(); let node = host_lock_anchor(&mut tc); - tc.inc_host_lock_ref(tc.arena.node(node).id) - .expect("live test node"); - FullComponent.release_component_lock( - &mut tc, - node, - &DecLockRefParams::default(), - /* lock_host = */ true, + let params = dec_params( + tc.inc_host_lock_ref(tc.arena.node(node).id) + .expect("live test node"), ); + FullComponent.release_component_lock(&mut tc, node, ¶ms, /* lock_host = */ true); assert!(tc.evictable_host_leaves.contains(node)); } @@ -1525,15 +1551,19 @@ fn release_host_arm_updates_the_h_leaf_set_without_the_dispatcher() { fn nested_host_locks_release_pairwise() { let mut tc = core(); let node = host_lock_anchor(&mut tc); - tc.inc_host_lock_ref(tc.arena.node(node).id) - .expect("live test node"); - tc.inc_host_lock_ref(tc.arena.node(node).id) - .expect("live test node"); - tc.dec_host_lock_ref(tc.arena.node(node).id, &DecLockRefParams::default()) + let first_params = dec_params( + tc.inc_host_lock_ref(tc.arena.node(node).id) + .expect("live test node"), + ); + let second_params = dec_params( + tc.inc_host_lock_ref(tc.arena.node(node).id) + .expect("live test node"), + ); + tc.dec_host_lock_ref(tc.arena.node(node).id, &second_params) .expect("live test node"); assert_eq!(tc.arena.host_lock_ref(node, FULL), 1); assert!(!tc.evictable_host_leaves.contains(node)); - tc.dec_host_lock_ref(tc.arena.node(node).id, &DecLockRefParams::default()) + tc.dec_host_lock_ref(tc.arena.node(node).id, &first_params) .expect("live test node"); assert_eq!(tc.arena.host_lock_ref(node, FULL), 0); assert!(tc.evictable_host_leaves.contains(node)); @@ -2413,14 +2443,14 @@ fn redistribute_preserves_preexisting_parent_slot_value() { } #[test] -fn redistribute_does_not_copy_host_lock_ref() { +fn redistribute_copies_host_lock_ref() { let mut tc = core(); let (parent, child) = nodes(&mut tc); tc.arena .node_mut(child) .set_lock_ref_(ValueSlotIdx::host(FULL), 5); FullComponent.redistribute_on_node_split(&mut tc, parent, child); - assert_eq!(tc.arena.host_lock_ref(parent, FULL), 0); + assert_eq!(tc.arena.host_lock_ref(parent, FULL), 5); } #[test] diff --git a/rust/sglang-radix-tree/src/tests/components/mamba.rs b/rust/sglang-radix-tree/src/tests/components/mamba.rs index 7f5ebf1d74..ef12ea1c3d 100644 --- a/rust/sglang-radix-tree/src/tests/components/mamba.rs +++ b/rust/sglang-radix-tree/src/tests/components/mamba.rs @@ -159,6 +159,14 @@ fn lru_order(lru: &UnifiedLRUList) -> Vec { lru.iter().collect() } +#[test] +#[should_panic(expected = "get_or_fill_uuid is unsupported for Mamba")] +fn get_or_fill_uuid_rejects_mamba() { + let mut tc = mamba_core(/* page_size = */ 1); + let [node] = chain::<1>(&mut tc); + mamba_component().get_or_fill_uuid(&mut tc, node, /* host = */ false); +} + #[test] fn node_has_component_data_reads_each_layer() { let mut tc = mamba_core(/* page_size = */ 1); diff --git a/rust/sglang-radix-tree/src/tests/components/swa.rs b/rust/sglang-radix-tree/src/tests/components/swa.rs index 896c333b57..fdf23a3b3d 100644 --- a/rust/sglang-radix-tree/src/tests/components/swa.rs +++ b/rust/sglang-radix-tree/src/tests/components/swa.rs @@ -1827,9 +1827,12 @@ fn acquire_lock_walks_until_the_window_fills_and_stamps_the_crossing_node() { assert_eq!(tc.arena.device_lock_ref(b, SWA), 1); assert_eq!(tc.arena.device_lock_ref(a, SWA), 0); // The walk fills the 2-atom window at b; b carries the first minted uuid. - assert_eq!(result.component_lock_uuids[&(SWA.idx() as u8)], Some(2)); + assert_eq!( + result.component_lock_uuids[&(SWA.idx() as u8)], + Some(100_000_000_000_001) + ); assert!(result.component_host_lock_uuids.is_empty()); - assert_eq!(node_swa_uuid(&tc, b), Some(2)); + assert_eq!(node_swa_uuid(&tc, b), Some(100_000_000_000_001)); assert_eq!(node_swa_uuid(&tc, c), None); assert_eq!(tc.swa_evictable_size(), 1); assert_eq!(tc.swa_protected_size(), 2); @@ -1880,8 +1883,11 @@ fn acquire_lock_overshooting_the_window_stops_at_the_crossing_node() { assert_eq!(tc.arena.device_lock_ref(c, SWA), 1); assert_eq!(tc.arena.device_lock_ref(b, SWA), 1); assert_eq!(tc.arena.device_lock_ref(a, SWA), 0); - assert_eq!(result.component_lock_uuids[&(SWA.idx() as u8)], Some(2)); - assert_eq!(node_swa_uuid(&tc, b), Some(2)); + assert_eq!( + result.component_lock_uuids[&(SWA.idx() as u8)], + Some(100_000_000_000_001) + ); + assert_eq!(node_swa_uuid(&tc, b), Some(100_000_000_000_001)); assert_eq!(node_swa_uuid(&tc, a), None); assert_eq!(tc.swa_evictable_size(), 2); assert_eq!(tc.swa_protected_size(), 4); @@ -2059,10 +2065,10 @@ fn acquire_host_lock_walks_until_the_window_fills_and_stamps_the_host_uuid() { // The window fills at b; b carries the host uuid and leaves the host LRU. assert_eq!( result.component_host_lock_uuids[&(SWA.idx() as u8)], - Some(2) + Some(100_000_000_000_001) ); assert!(result.component_lock_uuids.is_empty()); - assert_eq!(node_swa_host_uuid(&tc, b), Some(2)); + assert_eq!(node_swa_host_uuid(&tc, b), Some(100_000_000_000_001)); assert_eq!(node_swa_host_uuid(&tc, c), None); assert!(!tc.host_lru_list(SWA).in_list(Some(c))); assert!(!tc.host_lru_list(SWA).in_list(Some(b))); @@ -2198,9 +2204,9 @@ fn acquire_host_lock_stamps_the_host_tier_uuid_field_only() { // The boundary uuid lands on the host-tier field; the device field stays clear. assert_eq!( result.component_host_lock_uuids[&(SWA.idx() as u8)], - Some(2) + Some(100_000_000_000_001) ); - assert_eq!(node_swa_host_uuid(&tc, b), Some(2)); + assert_eq!(node_swa_host_uuid(&tc, b), Some(100_000_000_000_001)); assert_eq!(node_swa_uuid(&tc, b), None); assert_eq!(node_swa_host_uuid(&tc, c), None); assert_eq!(node_swa_uuid(&tc, c), None); @@ -2228,10 +2234,16 @@ fn device_and_host_lock_walks_mint_independent_uuids() { IncLockRefResult::default(), /* lock_host = */ true, ); - assert_eq!(device.component_lock_uuids[&(SWA.idx() as u8)], Some(2)); - assert_eq!(host.component_host_lock_uuids[&(SWA.idx() as u8)], Some(3)); - assert_eq!(node_swa_uuid(&tc, b), Some(2)); - assert_eq!(node_swa_host_uuid(&tc, b), Some(3)); + assert_eq!( + device.component_lock_uuids[&(SWA.idx() as u8)], + Some(100_000_000_000_001) + ); + assert_eq!( + host.component_host_lock_uuids[&(SWA.idx() as u8)], + Some(100_000_000_000_002) + ); + assert_eq!(node_swa_uuid(&tc, b), Some(100_000_000_000_001)); + assert_eq!(node_swa_host_uuid(&tc, b), Some(100_000_000_000_002)); } #[test] @@ -3656,6 +3668,38 @@ fn redistribute_on_node_split_preserves_host_lock_state() { assert!(!tc.host_lru_list(SWA).in_list(Some(node))); } +#[test] +fn host_lock_receipt_releases_every_split_fragment() { + let mut tc = swa_core(/* window = */ 4, /* page_size = */ 1); + tc.insert(&insert_params_swa( + &vec![1, 2, 3, 4], + &[10, 11, 12, 13], + 0, + 0, + )); + let root = tc.arena.root(); + let node = child_of(&tc, root, &[1]); + set_swa_host(&mut tc, node); + + let node_handle = tc.arena.node(node).id; + let lock = tc.inc_host_lock_ref(node_handle).expect("live test node"); + assert!( + lock.component_host_lock_uuids + .contains_key(&(SWA.idx() as u8)) + ); + + let (parent, _) = tc.split_node_(node, /* split_len = */ 2); + assert_eq!(tc.arena.host_lock_ref(parent, SWA), 1); + assert_eq!(tc.arena.host_lock_ref(node, SWA), 1); + + tc.dec_host_lock_ref(node_handle, &lock.to_dec_params()) + .expect("live test node"); + assert_eq!(tc.arena.host_lock_ref(parent, SWA), 0); + assert_eq!(tc.arena.host_lock_ref(node, SWA), 0); + assert!(tc.host_lru_list(SWA).in_list(Some(parent))); + assert!(tc.host_lru_list(SWA).in_list(Some(node))); +} + #[test] fn finalize_window_arithmetic_at_page_boundaries() { let mut tc = swa_core(/* window = */ 4, /* page_size = */ 2); diff --git a/rust/sglang-radix-tree/src/tests/unified_tree_core.rs b/rust/sglang-radix-tree/src/tests/unified_tree_core.rs index e8468bd1fe..24f6a26405 100644 --- a/rust/sglang-radix-tree/src/tests/unified_tree_core.rs +++ b/rust/sglang-radix-tree/src/tests/unified_tree_core.rs @@ -697,10 +697,13 @@ fn dec_swa_lock_only_returns_device_frees_in_the_device_dict() { } #[test] -fn next_swa_uuid_counts_up_from_two() { +fn next_component_uuid_counts_up_independently() { let mut tc = core(); - assert_eq!(tc.next_swa_uuid_(), 2); - assert_eq!(tc.next_swa_uuid_(), 3); + assert_eq!(tc.next_component_uuid_(SWA), 100_000_000_000_001); + assert_eq!(tc.next_component_uuid_(SWA), 100_000_000_000_002); + assert_eq!(tc.next_component_uuid_(FULL), 200_000_000_000_001); + assert_eq!(tc.next_component_uuid_(MAMBA), 300_000_000_000_001); + assert_eq!(tc.next_component_uuid_(FULL), 200_000_000_000_002); } #[test] diff --git a/rust/sglang-radix-tree/src/unified_tree_core.rs b/rust/sglang-radix-tree/src/unified_tree_core.rs index 2436eafc67..609e57425e 100644 --- a/rust/sglang-radix-tree/src/unified_tree_core.rs +++ b/rust/sglang-radix-tree/src/unified_tree_core.rs @@ -29,6 +29,22 @@ use crate::unified_lru_list::{ // A 42-bit mask keeps digest multiplication by 1_000_003 within i64. const COEXIST_RECLAIM_DIGEST_MULTIPLIER: i64 = 1_000_003; const COEXIST_RECLAIM_DIGEST_MASK: i64 = (1 << 42) - 1; +const COMPONENT_UUID_RANGE_SIZE: i64 = 100_000_000_000_000; + +const fn component_uuid_initial_value(component_type: ComponentType) -> i64 { + let range_index = match component_type { + ComponentType::Swa => 0, + ComponentType::Full => 1, + ComponentType::Mamba => 2, + }; + (range_index + 1) * COMPONENT_UUID_RANGE_SIZE +} + +const COMPONENT_UUID_INITIAL_VALUES: [i64; NUM_COMPONENT_TYPES] = [ + component_uuid_initial_value(FULL), + component_uuid_initial_value(SWA), + component_uuid_initial_value(MAMBA), +]; fn next_coexist_reclaim_digest(current: i64, node_id: NodeId, component_idx: usize) -> i64 { let event = (node_id as i64 + 1) * NUM_COMPONENT_TYPES as i64 + component_idx as i64; @@ -625,8 +641,8 @@ pub struct UnifiedTreeCore { /// Hit count at which a node earns a host write-through backup. pub(crate) write_through_threshold: i64, - /// Monotonic source for SWA lock-window uuids. - pub(crate) swa_uuid_counter: i64, + /// Per-component monotonic sources for lock-segment UUIDs. + pub(crate) component_uuid_counters: [i64; NUM_COMPONENT_TYPES], /// Device the KV indices live on. pub(crate) device: Device, /// Shared empty device-index tensor (an empty match's indices). @@ -814,7 +830,7 @@ impl UnifiedTreeCore { kv_event_queue: Vec::new(), namespaced_event_hashes: HashMap::new(), write_through_threshold: params.write_through_threshold, - swa_uuid_counter: 1, + component_uuid_counters: COMPONENT_UUID_INITIAL_VALUES, device: params.device, empty_device_indices: Tensor::empty([0], (Kind::Int64, params.device)), ongoing_insert_walk_state: None, @@ -894,10 +910,11 @@ impl UnifiedTreeCore { new_node_id } - /// Mint the next SWA lock-window uuid. - pub(crate) fn next_swa_uuid_(&mut self) -> i64 { - self.swa_uuid_counter += 1; - self.swa_uuid_counter + /// Mint the next lock-segment UUID for a component. + pub(crate) fn next_component_uuid_(&mut self, component_type: ComponentType) -> i64 { + let counter = &mut self.component_uuid_counters[component_type.idx()]; + *counter += 1; + *counter } /// Bump the reference count on a node's component locks. Components in @@ -5095,6 +5112,17 @@ impl UnifiedTreeCore { Ok(self.arena.node(node_id).device_lock_ref(component_type)) } + /// A component's host lock count on a node. + pub fn inspect_get_component_host_lock_ref( + &self, + node_id: NodeId, + component_type: ComponentType, + ) -> Result { + let node_id = self.arena.resolve(node_id)?; + self.assert_component_enabled_(component_type); + Ok(self.arena.node(node_id).host_lock_ref(component_type)) + } + /// A node's accumulated match count. pub fn inspect_get_node_hit_count(&self, node_id: NodeId) -> Result { let node_id = self.arena.resolve(node_id)?; diff --git a/test/registered/unit/mem_cache/rust_unified_tree_core_inspector.py b/test/registered/unit/mem_cache/rust_unified_tree_core_inspector.py index 5b4d7daa57..f47d1afc63 100644 --- a/test/registered/unit/mem_cache/rust_unified_tree_core_inspector.py +++ b/test/registered/unit/mem_cache/rust_unified_tree_core_inspector.py @@ -76,6 +76,13 @@ class RustUnifiedTreeCoreInspector( node_id, int(component_type) ) + def get_component_host_lock_ref( + self, node_id: NodeId, component_type: ComponentType + ) -> int: + return self._binding.inspect_get_component_host_lock_ref( + node_id, int(component_type) + ) + def get_node_hit_count(self, node_id: NodeId) -> int: return self._binding.inspect_get_node_hit_count(node_id) diff --git a/test/registered/unit/mem_cache/test_component_uuid_counter.py b/test/registered/unit/mem_cache/test_component_uuid_counter.py new file mode 100644 index 0000000000..5866efb237 --- /dev/null +++ b/test/registered/unit/mem_cache/test_component_uuid_counter.py @@ -0,0 +1,43 @@ +import unittest + +from sglang.srt.mem_cache.unified_cache.component_type import ComponentType +from sglang.srt.mem_cache.unified_cache.components import base +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +class TestComponentUuidCounter(unittest.TestCase): + def setUp(self): + self._original_counters = base._COMPONENT_UUID_COUNTERS.copy() + + def tearDown(self): + base._COMPONENT_UUID_COUNTERS.clear() + base._COMPONENT_UUID_COUNTERS.update(self._original_counters) + + def test_component_ranges_are_disjoint_and_independent(self): + self.assertEqual( + base.next_component_uuid(ComponentType.SWA), 100_000_000_000_001 + ) + self.assertEqual( + base.next_component_uuid(ComponentType.SWA), 100_000_000_000_002 + ) + self.assertEqual( + base.next_component_uuid(ComponentType.FULL), 200_000_000_000_001 + ) + self.assertEqual( + base.next_component_uuid(ComponentType.MAMBA), 300_000_000_000_001 + ) + self.assertEqual( + base.next_component_uuid(ComponentType.C128), 400_000_000_000_001 + ) + self.assertEqual( + base.next_component_uuid(ComponentType.AUXILIARY_SWA), 500_000_000_000_001 + ) + self.assertEqual( + base.next_component_uuid(ComponentType.FULL), 200_000_000_000_002 + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/mem_cache/test_rust_tree_core_integration.py b/test/registered/unit/mem_cache/test_rust_tree_core_integration.py index 075e1e8b9d..d28df118ea 100644 --- a/test/registered/unit/mem_cache/test_rust_tree_core_integration.py +++ b/test/registered/unit/mem_cache/test_rust_tree_core_integration.py @@ -1443,8 +1443,13 @@ def test_host_lock_refs_round_trip(swa, missing_receipt): host_lock = core.inc_host_lock_ref(leaf) assert not host_lock.component_lock_uuids assert not device_lock.component_host_lock_uuids + assert host_lock.component_host_lock_uuids[ComponentType.FULL] > 0 if swa: - assert host_lock.component_host_lock_uuids == {ComponentType.SWA: None} + assert set(host_lock.component_host_lock_uuids) == { + ComponentType.FULL, + ComponentType.SWA, + } + assert host_lock.component_host_lock_uuids[ComponentType.SWA] is None params = host_lock.to_dec_params() if missing_receipt: params.component_host_lock_uuids.clear() @@ -5006,6 +5011,9 @@ def test_stale_inspection_handles_raise_key_error_or_report_absence(): "get_component_device_lock_ref": lambda: core.get_component_device_lock_ref( stale_root, ComponentType.FULL ), + "get_component_host_lock_ref": lambda: core.get_component_host_lock_ref( + stale_root, ComponentType.FULL + ), "get_node_hit_count": lambda: core.get_node_hit_count(stale_root), "get_write_through_pending_id": lambda: core.get_write_through_pending_id( stale_root @@ -5059,6 +5067,9 @@ def test_stale_inspection_handles_raise_key_error_or_report_absence(): "get_component_device_lock_ref": lambda: core.get_component_device_lock_ref( stale_root, ComponentType.SWA ), + "get_component_host_lock_ref": lambda: core.get_component_host_lock_ref( + stale_root, ComponentType.SWA + ), "set_component_device_value_raw": lambda: core.set_component_device_value_raw( stale_root, ComponentType.SWA, None ), diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py index 4b3c9bbcdb..eee309cf76 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py +++ b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py @@ -436,6 +436,10 @@ def _device_lock_ref(cache, node_id, component_type): return cache.tree_core.get_component_device_lock_ref(node_id, component_type) +def _host_lock_ref(cache, node_id, component_type): + return cache.tree_core.get_component_host_lock_ref(node_id, component_type) + + def _aux_storage_key_transfers(cache, node_id): transfers = [] if ComponentType.SWA in cache.tree_components: @@ -5590,6 +5594,106 @@ class UnifiedRadixCacheSuite: self.assertEqual(m.host_hit_length, 0) cache.sanity_check() + def test_hicache_split_preserves_inflight_full_host_pin(self): + """A split prefix keeps an outstanding storage backup's host ownership.""" + if self.cfg != CacheConfig(): + self.skipTest("single Full page-size-1 ownership regression") + + cache, allocator, req_to_token_pool = build_fixture(self.cfg) + self._init_hicache(cache, write_policy="write_back") + host_pool = cache.cache_controller.mem_pool_host + baseline_host = host_pool.available_size() + + seq = [1, 2, 3, 4] + self._insert(cache, allocator, req_to_token_pool, seq) + leaf = cache.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", seq))) + ).last_device_node + self._backup_node(cache, leaf) + + lock_params = cache.inc_host_lock_ref(leaf).to_dec_params() + self.assertIsNotNone( + lock_params.get_lock_uuid(ComponentType.FULL, lock_host=True) + ) + self._insert(cache, allocator, req_to_token_pool, [1, 2, 9, 10]) + split_parent = _node_parent(cache, leaf) + split_host = _host_value(cache, split_parent, ComponentType.FULL) + self.assertEqual(_host_lock_ref(cache, split_parent, ComponentType.FULL), 1) + self.assertEqual(_host_lock_ref(cache, leaf, ComponentType.FULL), 1) + + # Before the fix the prefix had host_lock_ref == 0 and these slots were + # reclaimed while the storage thread could still be reading them. + self.assertEqual(cache.evict_host(len(split_host)), 0) + + # A lock acquired after the split covers only the suffix and gets a new + # boundary there. Releasing it must not consume the older segment lock + # that was propagated to the prefix. + suffix_lock_params = cache.inc_host_lock_ref(leaf).to_dec_params() + self.assertEqual(_host_lock_ref(cache, split_parent, ComponentType.FULL), 1) + self.assertEqual(_host_lock_ref(cache, leaf, ComponentType.FULL), 2) + self.assertNotEqual( + suffix_lock_params.get_lock_uuid(ComponentType.FULL, lock_host=True), + lock_params.get_lock_uuid(ComponentType.FULL, lock_host=True), + ) + cache.dec_host_lock_ref(leaf, suffix_lock_params) + self.assertEqual(_host_lock_ref(cache, split_parent, ComponentType.FULL), 1) + self.assertEqual(_host_lock_ref(cache, leaf, ComponentType.FULL), 1) + self.assertEqual(cache.evict_host(len(split_host)), 0) + + cache.dec_host_lock_ref(leaf, lock_params) + self.assertEqual(_host_lock_ref(cache, split_parent, ComponentType.FULL), 0) + self.assertEqual(_host_lock_ref(cache, leaf, ComponentType.FULL), 0) + self.assertEqual(cache.evict_host(len(seq)), len(seq)) + self.assertEqual(host_pool.available_size(), baseline_host) + cache.sanity_check() + + def test_hicache_split_preserves_inflight_swa_host_pin(self): + """A split SWA host segment stays pinned and releases without leaks.""" + expected_cfg = CacheConfig( + page_size=1, + components=(ComponentType.FULL, ComponentType.SWA), + sliding_window_size=4, + ) + if self.cfg != expected_cfg: + self.skipTest("single page-size-1 Full+SWA host-lock regression") + + cache, allocator, req_to_token_pool = build_fixture(self.cfg) + self._init_hicache(cache, write_policy="write_back") + swa_host_pool = cache.swa_kv_pool_host + + seq = [1, 2, 3, 4, 5, 6, 7, 8] + self._insert(cache, allocator, req_to_token_pool, seq) + leaf = cache.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", seq))) + ).last_device_node + parent_before_split = _node_parent(cache, leaf) + self._backup_node(cache, leaf) + available_after_backup = swa_host_pool.available_size() + + lock_params = cache.inc_host_lock_ref(leaf).to_dec_params() + self.assertIsNotNone( + lock_params.get_lock_uuid(ComponentType.FULL, lock_host=True) + ) + self.assertIsNotNone( + lock_params.get_lock_uuid(ComponentType.SWA, lock_host=True) + ) + + # Diverge inside the locked SWA window. The leaf handle stays on the + # suffix while the lock boundary and copied host refs move to the new + # prefix, so neither half may be reclaimed by component host eviction. + self._insert(cache, allocator, req_to_token_pool, [1, 2, 3, 4, 5, 6, 9, 10]) + split_parent = _node_parent(cache, leaf) + self.assertNotEqual(split_parent, parent_before_split) + self.assertEqual(_node_parent(cache, split_parent), parent_before_split) + self.assertEqual(_host_lock_ref(cache, split_parent, ComponentType.SWA), 1) + self.assertEqual(_host_lock_ref(cache, leaf, ComponentType.SWA), 1) + + cache.dec_host_lock_ref(leaf, lock_params) + self.assertEqual(_host_lock_ref(cache, split_parent, ComponentType.SWA), 0) + self.assertEqual(_host_lock_ref(cache, leaf, ComponentType.SWA), 0) + self.assertEqual(swa_host_pool.available_size(), available_after_backup) + cache.sanity_check() + def _skip_unsupported_hicache_test(self): if self.cfg.has_swa and self.cfg.has_mamba: self.skipTest("HiCache unit fixture does not support SWA + Mamba stacks") @@ -10715,6 +10819,15 @@ class TestSegmentLockProtocol(_InsertWalkSuite): cur = _node_parent(cache, cur) return nodes + @staticmethod + def _full_path(cache, leaf_id): + """Non-root node ids protected by a Full device lock.""" + nodes, cur = [], leaf_id + while not cache.tree_core.is_root(cur): + nodes.append(cur) + cur = _node_parent(cache, cur) + return nodes + def _match_leaf(self, cache, seq): m = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq)))) return m.last_device_node @@ -10891,9 +11004,9 @@ class TestSegmentLockProtocol(_InsertWalkSuite): cache.sanity_check() def test_split_under_lock_releases_balanced(self): - """A mid-segment split mints a new node with copied refs and migrates - the boundary uuid; the original receipt (its anchor stays on the - deeper half) still releases exactly.""" + """A mid-segment split copies Full and SWA device refs and migrates the + SWA boundary uuid; the original receipt (whose anchor stays on the + deeper half) releases every copied ref without leaking either lock.""" sw = self.cfg.sliding_window_size cache, allocator, req_to_token_pool = build_fixture(self.cfg) seq = self._make_seq(1, 2 * sw) @@ -10901,18 +11014,25 @@ class TestSegmentLockProtocol(_InsertWalkSuite): leaf = self._match_leaf(cache, seq) lock = cache.inc_lock_ref(leaf) pre_segment = self._segment(cache, leaf, sw) + pre_full_path = self._full_path(cache, leaf) # Diverge inside the window to force a split of a locked node. fork = seq[: len(seq) - sw // 2] + self._make_seq(9000, sw) self._insert(cache, allocator, req_to_token_pool, fork) post_segment = self._segment(cache, leaf, sw) + post_full_path = self._full_path(cache, leaf) self.assertGreater(len(post_segment), len(pre_segment)) + self.assertGreater(len(post_full_path), len(pre_full_path)) for n in post_segment: self.assertEqual(self._swa_ref(cache, n), 1) + for n in post_full_path: + self.assertEqual(_device_lock_ref(cache, n, ComponentType.FULL), 1) cache.dec_lock_ref(leaf, lock.to_dec_params()) for n in post_segment: self.assertEqual(self._swa_ref(cache, n), 0) + for n in post_full_path: + self.assertEqual(_device_lock_ref(cache, n, ComponentType.FULL), 0) cache.sanity_check() def test_aux_release_readmits_the_leaf_whatever_the_release_order(self): diff --git a/test/registered/unit/mem_cache/unified_tree_core_inspection_interface.py b/test/registered/unit/mem_cache/unified_tree_core_inspection_interface.py index 4749faf040..257cdcc79c 100644 --- a/test/registered/unit/mem_cache/unified_tree_core_inspection_interface.py +++ b/test/registered/unit/mem_cache/unified_tree_core_inspection_interface.py @@ -78,6 +78,13 @@ class UnifiedTreeCoreInspectionInterface(UnifiedTreeCoreInterface): """The component's device lock count on the node.""" ... + @abstractmethod + def get_component_host_lock_ref( + self, node_id: NodeId, component_type: ComponentType + ) -> int: + """The component's host lock count on the node.""" + ... + @abstractmethod def get_node_hit_count(self, node_id: NodeId) -> int: """The node's accumulated match count.""" diff --git a/test/registered/unit/mem_cache/unified_tree_core_inspector.py b/test/registered/unit/mem_cache/unified_tree_core_inspector.py index c13d145ed4..827218445a 100644 --- a/test/registered/unit/mem_cache/unified_tree_core_inspector.py +++ b/test/registered/unit/mem_cache/unified_tree_core_inspector.py @@ -73,6 +73,12 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa """The component's device lock count on the node.""" return self.node_by_id(node_id).component_data[component_type].lock_ref + def get_component_host_lock_ref( + self, node_id: NodeId, component_type: ComponentType + ) -> int: + """The component's host lock count on the node.""" + return self.node_by_id(node_id).component_data[component_type].host_lock_ref + def get_node_hit_count(self, node_id: NodeId) -> int: """The node's accumulated match count.""" return self.node_by_id(node_id).hit_count