From b2786a670ea0ca0bd2caa6795be0660591c0a1e3 Mon Sep 17 00:00:00 2001 From: metamergebot Date: Thu, 8 Oct 2026 17:48:11 -0700 Subject: [PATCH] [MemCache] Count reclaimable tokens in the unified tree ledgers (#43185) Co-authored-by: merrymercy Co-authored-by: xiezhq-hermann --- .../unified_cache/components/base.py | 11 +++ .../unified_cache/components/full.py | 13 ++-- .../unified_cache/unified_tree_core.py | 30 ++++++-- .../test_unified_radix_cache_unittest.py | 69 +++++++++++++++++++ 4 files changed, 113 insertions(+), 10 deletions(-) diff --git a/python/sglang/srt/mem_cache/unified_cache/components/base.py b/python/sglang/srt/mem_cache/unified_cache/components/base.py index 151f81d8a2..9d6a85cae1 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/base.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/base.py @@ -346,6 +346,17 @@ class TreeComponent(ABC): value = node.component_data[self.component_type].value return len(value) if value is not None else 0 + def reclaimable_tokens(self, node: UnifiedTreeNode) -> int: + """Pool tokens the allocator gets back when this node's device value + is evicted. The evictable/protected ledgers count these (prefill + admission adds the evictable ledger to the allocator's free space, and + the eviction walk counts them toward its request), so they must be in + the allocator's units: the value length by default, less for a + component whose values are not backed one-to-one by pool rows. A + component whose per-node figure changes in place reports the delta + through ``UnifiedTreeCore.adjust_component_ledger``.""" + return self.value_len(node) + def has_host_value_only(self, node: UnifiedTreeNode) -> bool: """Whether this component's data is evicted from device but host-backed.""" cd = node.component_data[self.component_type] diff --git a/python/sglang/srt/mem_cache/unified_cache/components/full.py b/python/sglang/srt/mem_cache/unified_cache/components/full.py index f9dc55c82f..8f647f7533 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/full.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/full.py @@ -179,7 +179,9 @@ class FullComponent(TreeComponent): # Device layer if EvictLayer.DEVICE in target and cd.value is not None: device_frees[self.component_type].append(cd.value) - freed = len(cd.value) + # Progress toward an eviction request and the ledger move are + # both in allocator units (see reclaimable_tokens). + freed = self.reclaimable_tokens(node) self.tree_core.component_evictable_size_[self.component_type] -= freed # NOTE: cd.value = None is deferred to _cascade_evict (Full as trigger) # because SWA's free_swa still needs to read Full.value. @@ -307,7 +309,7 @@ class FullComponent(TreeComponent): f"FULL invariant broken: evicted ancestor {cur.id} above device-on segment" ) if cd.lock_ref == 0: - key_len = len(cd.value) + key_len = self.reclaimable_tokens(cur) self.tree_core.component_evictable_size_[ct] -= key_len self.tree_core.component_protected_size_[ct] += key_len delta += key_len @@ -356,7 +358,7 @@ class FullComponent(TreeComponent): f"FULL segment release hit lock_ref=0 on node {cur.id}" ) if cd.lock_ref == 1 and cd.value is not None: - key_len = len(cd.value) + key_len = self.reclaimable_tokens(cur) self.tree_core.component_evictable_size_[ct] += key_len self.tree_core.component_protected_size_[ct] -= key_len cd.lock_ref -= 1 @@ -447,10 +449,11 @@ class FullComponent(TreeComponent): offset += n_len # Full uses leaf sets, not LRU. A value materialized under # lock is protected; the last release moves it to evictable. + ledger = self.reclaimable_tokens(n) if cd.lock_ref > 0: - self.tree_core.component_protected_size_[ct] += n_len + self.tree_core.component_protected_size_[ct] += ledger else: - self.tree_core.component_evictable_size_[ct] += n_len + self.tree_core.component_evictable_size_[ct] += ledger self.tree_core._update_evictable_leaf_sets(n) self.tree_core._update_evictable_leaf_sets(node) diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py index a80972ac95..9914ce7e63 100644 --- a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py +++ b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py @@ -1496,7 +1496,9 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): new_node.rotation_base = rotation_base new_node.component_data[BASE_COMPONENT_TYPE].value = value.clone() parent.children[key.child_key(self.page_size)] = new_node - self.component_evictable_size_[BASE_COMPONENT_TYPE] += len(value) + self.component_evictable_size_[BASE_COMPONENT_TYPE] += self.components_by_type[ + BASE_COMPONENT_TYPE + ].reclaimable_tokens(new_node) if self.enable_storage or self.enable_external_cache_linker: new_node.hash_value = compute_node_hash_values(new_node, self.page_size) @@ -1516,8 +1518,8 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): ct = BASE_COMPONENT_TYPE cd = node.component_data[ct] assert cd.value is None - n = len(fresh_value) cd.value = fresh_value.clone() + n = self.components_by_type[ct].reclaimable_tokens(node) if cd.lock_ref > 0: self.component_protected_size_[ct] += n else: @@ -2862,10 +2864,28 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): self.lru_lists[component_type].insert_mru(node) # A value materialized under lock is protected; the last release # moves it to evictable. + ledger = self.components_by_type[component_type].reclaimable_tokens(node) if cd.lock_ref > 0: - self.component_protected_size_[component_type] += len(value) + self.component_protected_size_[component_type] += ledger else: - self.component_evictable_size_[component_type] += len(value) + self.component_evictable_size_[component_type] += ledger + + def adjust_component_ledger( + self, node: UnifiedTreeNode, component_type: ComponentType, delta: int + ) -> None: + """A component changed how many pool tokens evicting ``node`` would + reclaim (``TreeComponent.reclaimable_tokens``) while the value stays + on device: move the ledger the node is on by ``delta``. A tombstoned + value is on neither ledger.""" + if delta == 0: + return + cd = node.component_data[component_type] + if cd.value is None: + return + if cd.lock_ref > 0: + self.component_protected_size_[component_type] += delta + else: + self.component_evictable_size_[component_type] += delta def get_component_device_value( self, node_id: NodeId, component_type: ComponentType @@ -3086,7 +3106,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): continue cd = n.component_data[ct] if cd.value is not None: - toks = len(cd.value) + toks = self.components_by_type[ct].reclaimable_tokens(n) if cd.lock_ref > 0: protected += toks else: diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py index 685bc1ba61..906947371e 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py +++ b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py @@ -310,6 +310,75 @@ class TestUnifiedRadixComponentRegistryOverride(CustomTestCase): self.assertIsNot(COMPONENT_REGISTRY[ComponentType.FULL], _FakeFullComponent) +class _PagedFullComponent(FullComponent): + """FULL component whose device values are not backed one-to-one by pool + rows: a value lists two entries per row, and the component can hand rows + of a node back to the allocator while the value stays on device.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.rows_handed_back: dict[int, int] = {} + + def reclaimable_tokens(self, node): + return self.value_len(node) // 2 - self.rows_handed_back.get(node.id, 0) + + +class TestUnifiedRadixComponentLedgerUnits(CustomTestCase): + def test_ledgers_count_reclaimable_tokens(self): + """The evictable and protected ledgers are in the allocator's units: + prefill admission adds the evictable ledger to the allocator's free + space and the eviction walk counts them toward its request. So every + ledger site (insert, lock, release, an in-place change, eviction) + reads ``TreeComponent.reclaimable_tokens`` rather than the value + length. A site that goes back to ``len(value)`` drifts the ledger from + the recount in ``sanity_check`` for a component that overrides it.""" + cache, allocator, _ = build_fixture( + CacheConfig(), + component_registry_override={ComponentType.FULL: _PagedFullComponent}, + tree_core_backend="python", + ) + component = cache.components[ComponentType.FULL] + + def ledgers(): + cache.sanity_check() + return cache.evictable_size(), cache.protected_size() + + tokens = array("q", range(1, 9)) + value = allocator.alloc(len(tokens)) + self.assertIsNotNone(value) + cache.insert(InsertParams(key=RadixKey(tokens), value=value)) + self.assertEqual(ledgers(), (4, 0)) + + leaf = cache.match_prefix( + MatchPrefixParams(key=RadixKey(tokens)) + ).last_device_node + lock_params = cache.inc_lock_ref(leaf).to_dec_params() + self.assertEqual(ledgers(), (0, 4)) + + # The figure changes in place, without a lock transition. + component.rows_handed_back[leaf] = 1 + cache.tree_core.adjust_component_ledger( + node=cache.tree_core.node_by_id(leaf), + component_type=ComponentType.FULL, + delta=-1, + ) + self.assertEqual(ledgers(), (0, 3)) + + cache.dec_lock_ref(leaf, lock_params) + self.assertEqual(ledgers(), (3, 0)) + + # The eviction walk is satisfied by what the node reclaims, not by + # its value length. + cache.evict(EvictParams(num_tokens=3)) + self.assertEqual(ledgers(), (0, 0)) + self.assertEqual( + cache.match_prefix( + MatchPrefixParams(key=RadixKey(tokens)) + ).device_prefix_len, + 0, + ) + + class TestUnifiedTreeNodeGetPrefixHashValues(CustomTestCase): def test_get_prefix_hash_values_not_shared_across_calls(self): """Regression guard for cached mutable prefix hash lists (#26177)."""