[MemCache] Add shared component size and path APIs to the tree core interface (#43722)

Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com>
Co-authored-by: Jialin <Jialin@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-11 20:14:07 -07:00
committed by GitHub
co-authored by Hanming Lu Jialin
parent 95ce9caaf2
commit 77a15cf4e2
10 changed files with 677 additions and 18 deletions
@@ -57,6 +57,7 @@ from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import (
InsertStepResult,
NodeId,
RadixCacheWalkResult,
TreePathNodeInfo,
UnifiedTreeCoreInterface,
)
from sglang.srt.mem_cache.utils import get_eviction_strategy
@@ -1138,6 +1139,17 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
def write_back_duplicate_reclaim_digest(self) -> int:
return self._binding.write_back_duplicate_reclaim_digest()
def adjust_component_size(
self, node_id: NodeId, component_type: ComponentType, delta: int
) -> None:
self._binding.adjust_component_size(node_id, int(component_type), delta)
def collect_path_info(
self, end_node_ids: Sequence[NodeId]
) -> tuple[tuple[TreePathNodeInfo, ...], ...]:
paths = self._binding.collect_path_info(list(end_node_ids))
return tuple(tuple(TreePathNodeInfo(*node) for node in path) for path in paths)
def set_component_device_value(
self, node_id: NodeId, component_type: ComponentType, value: torch.Tensor
) -> None:
@@ -354,7 +354,7 @@ class TreeComponent(ABC):
the allocator's units: the value length by default, less for a
component whose values are not backed one-to-one by pool rows. A
component whose per-node figure changes in place reports the delta
through ``UnifiedTreeCore.adjust_component_ledger``."""
through ``UnifiedTreeCore.adjust_component_size``."""
return self.value_len(node)
def has_host_value_only(self, node: UnifiedTreeNode) -> bool:
@@ -78,6 +78,7 @@ from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import (
InsertStepResult,
NodeId,
RadixCacheWalkResult,
TreePathNodeInfo,
UnifiedTreeCoreInterface,
)
from sglang.srt.mem_cache.utils import (
@@ -667,6 +668,34 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
def dfs_weight_order(self, node_ids: Sequence[NodeId]) -> list[int]:
return _dfs_weight_order(self.root_node, node_ids, self.node_by_id)
def collect_path_info(
self, end_node_ids: Sequence[NodeId]
) -> tuple[tuple[TreePathNodeInfo, ...], ...]:
paths = []
for node_id in end_node_ids:
node = self.node_by_id(node_id)
chain = []
while node is not self.root_node:
chain.append(node)
node = node.parent
path = []
start = 0
for node in reversed(chain):
end = start + len(node.key)
value = node.component_data[BASE_COMPONENT_TYPE].value
path.append(
TreePathNodeInfo(
node.id,
node.parent.id,
start,
end,
None if value is None else len(value),
)
)
start = end
paths.append(tuple(path))
return tuple(paths)
def _new_node(self, priority: int = 0) -> UnifiedTreeNode:
"""Create and register a tree node in the arena."""
node = UnifiedTreeNode(self.component_types, priority=priority)
@@ -2858,17 +2887,14 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
else:
self.component_evictable_size_[component_type] += ledger
def adjust_component_ledger(
self, node: UnifiedTreeNode, component_type: ComponentType, delta: int
def adjust_component_size(
self, node_id: NodeId, component_type: ComponentType, delta: int
) -> None:
"""A component changed how many pool tokens evicting ``node`` would
reclaim (``TreeComponent.reclaimable_tokens``) while the value stays
on device: move the ledger the node is on by ``delta``. A tombstoned
value is on neither ledger."""
if delta == 0:
return
node = self.node_by_id(node_id)
if component_type not in self.components_by_type:
raise ValueError(f"component is not enabled: {component_type}")
cd = node.component_data[component_type]
if cd.value is None:
if node is self.root_node or cd.value is None:
return
if cd.lock_ref > 0:
self.component_protected_size_[component_type] += delta
@@ -104,6 +104,18 @@ class BufferBackupState(msgspec.Struct, frozen=True):
parent_last_hash: Optional[str]
class TreePathNodeInfo(msgspec.Struct, frozen=True):
"""A node's key span on its root-to-node path."""
node_id: NodeId
parent_id: NodeId
# Half-open offsets from the root, in tokens (bigrams for bigram trees).
path_start: int
path_end: int
# Logical FULL device indices; None when the value is absent.
full_value_len: Optional[int]
class InsertStepResult(msgspec.Struct, frozen=True):
"""One step of a resumable insert: the Controller executes ``actions``, then
resumes while ``result`` is None; ``result`` is set on the final step."""
@@ -270,6 +282,17 @@ class UnifiedTreeCoreInterface(ABC):
"""Return input indices in depth-first, subtree-weight order."""
...
@abstractmethod
def collect_path_info(
self, end_node_ids: Sequence[NodeId]
) -> tuple[tuple[TreePathNodeInfo, ...], ...]:
"""Copy root-to-node paths, excluding roots, in input order.
Roots produce empty paths; missing handles raise KeyError. Callers must
serialize with tree mutations; native backends hold one lock per batch.
"""
...
@abstractmethod
def inc_lock_ref(
self, node_id: NodeId, skip_lock_components: Sequence[ComponentType] = ()
@@ -398,6 +421,17 @@ class UnifiedTreeCoreInterface(ABC):
"""Protected token count for one component (0 if absent)."""
raise NotImplementedError("This tree core does not expose per-component sizes")
@abstractmethod
def adjust_component_size(
self, node_id: NodeId, component_type: ComponentType, delta: int
) -> None:
"""Adjust protected/evictable tokens after the component backing changes.
Roots and nodes without device values are no-ops. Missing handles raise
KeyError; disabled components raise ValueError.
"""
...
@abstractmethod
def full_evictable_size(self) -> int: ...