[MemCache] Add shared component size and path APIs to the tree core interface (#43722)
Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com> Co-authored-by: Jialin <Jialin@users.noreply.github.com>
This commit is contained in:
co-authored by
Hanming Lu
Jialin
parent
95ce9caaf2
commit
77a15cf4e2
@@ -57,6 +57,7 @@ from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import (
|
||||
InsertStepResult,
|
||||
NodeId,
|
||||
RadixCacheWalkResult,
|
||||
TreePathNodeInfo,
|
||||
UnifiedTreeCoreInterface,
|
||||
)
|
||||
from sglang.srt.mem_cache.utils import get_eviction_strategy
|
||||
@@ -1138,6 +1139,17 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
def write_back_duplicate_reclaim_digest(self) -> int:
|
||||
return self._binding.write_back_duplicate_reclaim_digest()
|
||||
|
||||
def adjust_component_size(
|
||||
self, node_id: NodeId, component_type: ComponentType, delta: int
|
||||
) -> None:
|
||||
self._binding.adjust_component_size(node_id, int(component_type), delta)
|
||||
|
||||
def collect_path_info(
|
||||
self, end_node_ids: Sequence[NodeId]
|
||||
) -> tuple[tuple[TreePathNodeInfo, ...], ...]:
|
||||
paths = self._binding.collect_path_info(list(end_node_ids))
|
||||
return tuple(tuple(TreePathNodeInfo(*node) for node in path) for path in paths)
|
||||
|
||||
def set_component_device_value(
|
||||
self, node_id: NodeId, component_type: ComponentType, value: torch.Tensor
|
||||
) -> None:
|
||||
|
||||
@@ -354,7 +354,7 @@ class TreeComponent(ABC):
|
||||
the allocator's units: the value length by default, less for a
|
||||
component whose values are not backed one-to-one by pool rows. A
|
||||
component whose per-node figure changes in place reports the delta
|
||||
through ``UnifiedTreeCore.adjust_component_ledger``."""
|
||||
through ``UnifiedTreeCore.adjust_component_size``."""
|
||||
return self.value_len(node)
|
||||
|
||||
def has_host_value_only(self, node: UnifiedTreeNode) -> bool:
|
||||
|
||||
@@ -78,6 +78,7 @@ from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import (
|
||||
InsertStepResult,
|
||||
NodeId,
|
||||
RadixCacheWalkResult,
|
||||
TreePathNodeInfo,
|
||||
UnifiedTreeCoreInterface,
|
||||
)
|
||||
from sglang.srt.mem_cache.utils import (
|
||||
@@ -667,6 +668,34 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
def dfs_weight_order(self, node_ids: Sequence[NodeId]) -> list[int]:
|
||||
return _dfs_weight_order(self.root_node, node_ids, self.node_by_id)
|
||||
|
||||
def collect_path_info(
|
||||
self, end_node_ids: Sequence[NodeId]
|
||||
) -> tuple[tuple[TreePathNodeInfo, ...], ...]:
|
||||
paths = []
|
||||
for node_id in end_node_ids:
|
||||
node = self.node_by_id(node_id)
|
||||
chain = []
|
||||
while node is not self.root_node:
|
||||
chain.append(node)
|
||||
node = node.parent
|
||||
path = []
|
||||
start = 0
|
||||
for node in reversed(chain):
|
||||
end = start + len(node.key)
|
||||
value = node.component_data[BASE_COMPONENT_TYPE].value
|
||||
path.append(
|
||||
TreePathNodeInfo(
|
||||
node.id,
|
||||
node.parent.id,
|
||||
start,
|
||||
end,
|
||||
None if value is None else len(value),
|
||||
)
|
||||
)
|
||||
start = end
|
||||
paths.append(tuple(path))
|
||||
return tuple(paths)
|
||||
|
||||
def _new_node(self, priority: int = 0) -> UnifiedTreeNode:
|
||||
"""Create and register a tree node in the arena."""
|
||||
node = UnifiedTreeNode(self.component_types, priority=priority)
|
||||
@@ -2858,17 +2887,14 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
else:
|
||||
self.component_evictable_size_[component_type] += ledger
|
||||
|
||||
def adjust_component_ledger(
|
||||
self, node: UnifiedTreeNode, component_type: ComponentType, delta: int
|
||||
def adjust_component_size(
|
||||
self, node_id: NodeId, component_type: ComponentType, delta: int
|
||||
) -> None:
|
||||
"""A component changed how many pool tokens evicting ``node`` would
|
||||
reclaim (``TreeComponent.reclaimable_tokens``) while the value stays
|
||||
on device: move the ledger the node is on by ``delta``. A tombstoned
|
||||
value is on neither ledger."""
|
||||
if delta == 0:
|
||||
return
|
||||
node = self.node_by_id(node_id)
|
||||
if component_type not in self.components_by_type:
|
||||
raise ValueError(f"component is not enabled: {component_type}")
|
||||
cd = node.component_data[component_type]
|
||||
if cd.value is None:
|
||||
if node is self.root_node or cd.value is None:
|
||||
return
|
||||
if cd.lock_ref > 0:
|
||||
self.component_protected_size_[component_type] += delta
|
||||
|
||||
@@ -104,6 +104,18 @@ class BufferBackupState(msgspec.Struct, frozen=True):
|
||||
parent_last_hash: Optional[str]
|
||||
|
||||
|
||||
class TreePathNodeInfo(msgspec.Struct, frozen=True):
|
||||
"""A node's key span on its root-to-node path."""
|
||||
|
||||
node_id: NodeId
|
||||
parent_id: NodeId
|
||||
# Half-open offsets from the root, in tokens (bigrams for bigram trees).
|
||||
path_start: int
|
||||
path_end: int
|
||||
# Logical FULL device indices; None when the value is absent.
|
||||
full_value_len: Optional[int]
|
||||
|
||||
|
||||
class InsertStepResult(msgspec.Struct, frozen=True):
|
||||
"""One step of a resumable insert: the Controller executes ``actions``, then
|
||||
resumes while ``result`` is None; ``result`` is set on the final step."""
|
||||
@@ -270,6 +282,17 @@ class UnifiedTreeCoreInterface(ABC):
|
||||
"""Return input indices in depth-first, subtree-weight order."""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def collect_path_info(
|
||||
self, end_node_ids: Sequence[NodeId]
|
||||
) -> tuple[tuple[TreePathNodeInfo, ...], ...]:
|
||||
"""Copy root-to-node paths, excluding roots, in input order.
|
||||
|
||||
Roots produce empty paths; missing handles raise KeyError. Callers must
|
||||
serialize with tree mutations; native backends hold one lock per batch.
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def inc_lock_ref(
|
||||
self, node_id: NodeId, skip_lock_components: Sequence[ComponentType] = ()
|
||||
@@ -398,6 +421,17 @@ class UnifiedTreeCoreInterface(ABC):
|
||||
"""Protected token count for one component (0 if absent)."""
|
||||
raise NotImplementedError("This tree core does not expose per-component sizes")
|
||||
|
||||
@abstractmethod
|
||||
def adjust_component_size(
|
||||
self, node_id: NodeId, component_type: ComponentType, delta: int
|
||||
) -> None:
|
||||
"""Adjust protected/evictable tokens after the component backing changes.
|
||||
|
||||
Roots and nodes without device values are no-ops. Missing handles raise
|
||||
KeyError; disabled components raise ValueError.
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def full_evictable_size(self) -> int: ...
|
||||
|
||||
|
||||
Reference in New Issue
Block a user