[MemCache] Count reclaimable tokens in the unified tree ledgers (#43185)

Co-authored-by: merrymercy <merrymercy@users.noreply.github.com>
Co-authored-by: xiezhq-hermann <xiezhq-hermann@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-08 17:48:11 -07:00
committed by GitHub
co-authored by merrymercy xiezhq-hermann
parent b4fe282836
commit b2786a670e
4 changed files with 113 additions and 10 deletions
@@ -346,6 +346,17 @@ class TreeComponent(ABC):
value = node.component_data[self.component_type].value
return len(value) if value is not None else 0
def reclaimable_tokens(self, node: UnifiedTreeNode) -> int:
"""Pool tokens the allocator gets back when this node's device value
is evicted. The evictable/protected ledgers count these (prefill
admission adds the evictable ledger to the allocator's free space, and
the eviction walk counts them toward its request), so they must be in
the allocator's units: the value length by default, less for a
component whose values are not backed one-to-one by pool rows. A
component whose per-node figure changes in place reports the delta
through ``UnifiedTreeCore.adjust_component_ledger``."""
return self.value_len(node)
def has_host_value_only(self, node: UnifiedTreeNode) -> bool:
"""Whether this component's data is evicted from device but host-backed."""
cd = node.component_data[self.component_type]
@@ -179,7 +179,9 @@ class FullComponent(TreeComponent):
# Device layer
if EvictLayer.DEVICE in target and cd.value is not None:
device_frees[self.component_type].append(cd.value)
freed = len(cd.value)
# Progress toward an eviction request and the ledger move are
# both in allocator units (see reclaimable_tokens).
freed = self.reclaimable_tokens(node)
self.tree_core.component_evictable_size_[self.component_type] -= freed
# NOTE: cd.value = None is deferred to _cascade_evict (Full as trigger)
# because SWA's free_swa still needs to read Full.value.
@@ -307,7 +309,7 @@ class FullComponent(TreeComponent):
f"FULL invariant broken: evicted ancestor {cur.id} above device-on segment"
)
if cd.lock_ref == 0:
key_len = len(cd.value)
key_len = self.reclaimable_tokens(cur)
self.tree_core.component_evictable_size_[ct] -= key_len
self.tree_core.component_protected_size_[ct] += key_len
delta += key_len
@@ -356,7 +358,7 @@ class FullComponent(TreeComponent):
f"FULL segment release hit lock_ref=0 on node {cur.id}"
)
if cd.lock_ref == 1 and cd.value is not None:
key_len = len(cd.value)
key_len = self.reclaimable_tokens(cur)
self.tree_core.component_evictable_size_[ct] += key_len
self.tree_core.component_protected_size_[ct] -= key_len
cd.lock_ref -= 1
@@ -447,10 +449,11 @@ class FullComponent(TreeComponent):
offset += n_len
# Full uses leaf sets, not LRU. A value materialized under
# lock is protected; the last release moves it to evictable.
ledger = self.reclaimable_tokens(n)
if cd.lock_ref > 0:
self.tree_core.component_protected_size_[ct] += n_len
self.tree_core.component_protected_size_[ct] += ledger
else:
self.tree_core.component_evictable_size_[ct] += n_len
self.tree_core.component_evictable_size_[ct] += ledger
self.tree_core._update_evictable_leaf_sets(n)
self.tree_core._update_evictable_leaf_sets(node)
@@ -1496,7 +1496,9 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
new_node.rotation_base = rotation_base
new_node.component_data[BASE_COMPONENT_TYPE].value = value.clone()
parent.children[key.child_key(self.page_size)] = new_node
self.component_evictable_size_[BASE_COMPONENT_TYPE] += len(value)
self.component_evictable_size_[BASE_COMPONENT_TYPE] += self.components_by_type[
BASE_COMPONENT_TYPE
].reclaimable_tokens(new_node)
if self.enable_storage or self.enable_external_cache_linker:
new_node.hash_value = compute_node_hash_values(new_node, self.page_size)
@@ -1516,8 +1518,8 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
ct = BASE_COMPONENT_TYPE
cd = node.component_data[ct]
assert cd.value is None
n = len(fresh_value)
cd.value = fresh_value.clone()
n = self.components_by_type[ct].reclaimable_tokens(node)
if cd.lock_ref > 0:
self.component_protected_size_[ct] += n
else:
@@ -2862,10 +2864,28 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
self.lru_lists[component_type].insert_mru(node)
# A value materialized under lock is protected; the last release
# moves it to evictable.
ledger = self.components_by_type[component_type].reclaimable_tokens(node)
if cd.lock_ref > 0:
self.component_protected_size_[component_type] += len(value)
self.component_protected_size_[component_type] += ledger
else:
self.component_evictable_size_[component_type] += len(value)
self.component_evictable_size_[component_type] += ledger
def adjust_component_ledger(
self, node: UnifiedTreeNode, component_type: ComponentType, delta: int
) -> None:
"""A component changed how many pool tokens evicting ``node`` would
reclaim (``TreeComponent.reclaimable_tokens``) while the value stays
on device: move the ledger the node is on by ``delta``. A tombstoned
value is on neither ledger."""
if delta == 0:
return
cd = node.component_data[component_type]
if cd.value is None:
return
if cd.lock_ref > 0:
self.component_protected_size_[component_type] += delta
else:
self.component_evictable_size_[component_type] += delta
def get_component_device_value(
self, node_id: NodeId, component_type: ComponentType
@@ -3086,7 +3106,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
continue
cd = n.component_data[ct]
if cd.value is not None:
toks = len(cd.value)
toks = self.components_by_type[ct].reclaimable_tokens(n)
if cd.lock_ref > 0:
protected += toks
else:
@@ -310,6 +310,75 @@ class TestUnifiedRadixComponentRegistryOverride(CustomTestCase):
self.assertIsNot(COMPONENT_REGISTRY[ComponentType.FULL], _FakeFullComponent)
class _PagedFullComponent(FullComponent):
"""FULL component whose device values are not backed one-to-one by pool
rows: a value lists two entries per row, and the component can hand rows
of a node back to the allocator while the value stays on device."""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.rows_handed_back: dict[int, int] = {}
def reclaimable_tokens(self, node):
return self.value_len(node) // 2 - self.rows_handed_back.get(node.id, 0)
class TestUnifiedRadixComponentLedgerUnits(CustomTestCase):
def test_ledgers_count_reclaimable_tokens(self):
"""The evictable and protected ledgers are in the allocator's units:
prefill admission adds the evictable ledger to the allocator's free
space and the eviction walk counts them toward its request. So every
ledger site (insert, lock, release, an in-place change, eviction)
reads ``TreeComponent.reclaimable_tokens`` rather than the value
length. A site that goes back to ``len(value)`` drifts the ledger from
the recount in ``sanity_check`` for a component that overrides it."""
cache, allocator, _ = build_fixture(
CacheConfig(),
component_registry_override={ComponentType.FULL: _PagedFullComponent},
tree_core_backend="python",
)
component = cache.components[ComponentType.FULL]
def ledgers():
cache.sanity_check()
return cache.evictable_size(), cache.protected_size()
tokens = array("q", range(1, 9))
value = allocator.alloc(len(tokens))
self.assertIsNotNone(value)
cache.insert(InsertParams(key=RadixKey(tokens), value=value))
self.assertEqual(ledgers(), (4, 0))
leaf = cache.match_prefix(
MatchPrefixParams(key=RadixKey(tokens))
).last_device_node
lock_params = cache.inc_lock_ref(leaf).to_dec_params()
self.assertEqual(ledgers(), (0, 4))
# The figure changes in place, without a lock transition.
component.rows_handed_back[leaf] = 1
cache.tree_core.adjust_component_ledger(
node=cache.tree_core.node_by_id(leaf),
component_type=ComponentType.FULL,
delta=-1,
)
self.assertEqual(ledgers(), (0, 3))
cache.dec_lock_ref(leaf, lock_params)
self.assertEqual(ledgers(), (3, 0))
# The eviction walk is satisfied by what the node reclaims, not by
# its value length.
cache.evict(EvictParams(num_tokens=3))
self.assertEqual(ledgers(), (0, 0))
self.assertEqual(
cache.match_prefix(
MatchPrefixParams(key=RadixKey(tokens))
).device_prefix_len,
0,
)
class TestUnifiedTreeNodeGetPrefixHashValues(CustomTestCase):
def test_get_prefix_hash_values_not_shared_across_calls(self):
"""Regression guard for cached mutable prefix hash lists (#26177)."""