[mem_cache] Remove unreachable RadixCache paths in KV canary and HiCache accessors (#40807)

This commit is contained in:
Liangsheng Yin
2026-09-23 13:22:58 -07:00
committed by GitHub
parent 2909f84e3b
commit 954458567e
15 changed files with 101 additions and 428 deletions
@@ -266,9 +266,6 @@ class DecodeHiCacheTransferMixin:
return True
def _process_hicache_local_restores(self, decode_reqs: List[DecodeRequest]) -> None:
if not hasattr(self.tree_cache, "is_load_back_event_done"):
return
# Filter once: keep only PENDING reqs that still need restore work;
# trivially-done reqs (no prefix_match / nothing to restore) flip to READY.
active: List[DecodeRequest] = []
@@ -86,30 +86,3 @@ def make_row_source(
read_bytes=clipped,
),
)
def make_packed_source(
*,
page_buffer: torch.Tensor,
page_size: int,
bytes_per_token: int,
read_bytes: int,
) -> Tuple[RealKvSource, ...]:
if read_bytes == 0 or page_buffer.numel() == 0:
return ()
flat = page_buffer.contiguous().view(torch.uint8)
if flat.ndim == 1:
flat = flat.reshape(1, -1)
clipped = _clip_read_bytes_aligned(
requested=read_bytes, num_bytes_per_token=bytes_per_token
)
if clipped == 0:
return ()
return (
RealKvSource(
tensor=flat,
page_size=page_size,
num_bytes_per_token=bytes_per_token,
read_bytes=clipped,
),
)
@@ -1,10 +1,7 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING
import torch
from sglang.srt.mem_cache.radix_cache import RadixCache
from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import (
RadixCacheWalkResult,
)
@@ -12,7 +9,6 @@ from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
if TYPE_CHECKING:
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache
from sglang.srt.mem_cache.radix_cache import TreeNode
def walk_radix_cache_for_canary(
@@ -28,83 +24,10 @@ def walk_radix_cache_for_canary(
per-forward HEAD/TAIL path). ``unlocked_only=True`` skips nodes still locked by a running
req. ``swa_resident_only=True`` skips SWA-tombstoned nodes (slots evicted from the SWA
window)."""
cache_type = type(radix_cache)
if cache_type is UnifiedRadixCache:
return radix_cache.tree_core.walk_for_kv_canary(
unlocked_only=unlocked_only, swa_resident_only=swa_resident_only
)
if cache_type is not RadixCache:
if type(radix_cache) is not UnifiedRadixCache:
raise NotImplementedError(
f"walk_radix_cache_for_canary does not support {cache_type.__name__}"
f"walk_radix_cache_for_canary does not support {type(radix_cache).__name__}"
)
# RadixCache has no SWA tier, so every node it holds is SWA-resident and
# swa_resident_only is a no-op here.
slot_buf: list[int] = []
position_buf: list[int] = []
prev_slot_buf: list[int] = []
_walk_radix_subtree(
node=radix_cache.root_node,
depth=0,
parent_last_slot=-1,
slot_buf=slot_buf,
position_buf=position_buf,
prev_slot_buf=prev_slot_buf,
is_root=True,
unlocked_only=unlocked_only,
return radix_cache.tree_core.walk_for_kv_canary(
unlocked_only=unlocked_only, swa_resident_only=swa_resident_only
)
slot_tensor = torch.tensor(slot_buf, dtype=torch.int64)
position_tensor = torch.tensor(position_buf, dtype=torch.int64)
prev_slot_tensor = torch.tensor(prev_slot_buf, dtype=torch.int64)
return RadixCacheWalkResult(
slot_indices=slot_tensor,
positions=position_tensor,
prev_slot_indices=prev_slot_tensor,
)
def _walk_radix_subtree(
*,
node: TreeNode,
depth: int,
parent_last_slot: int,
slot_buf: list[int],
position_buf: list[int],
prev_slot_buf: list[int],
is_root: bool,
unlocked_only: bool,
) -> None:
node_slots = _node_slots_for_canary(node=node)
emit_slots = not is_root and (not unlocked_only or node.lock_ref == 0)
chain_last_slot = parent_last_slot
for j, slot in enumerate(node_slots):
prev = parent_last_slot if j == 0 else node_slots[j - 1]
if emit_slots:
slot_buf.append(slot)
position_buf.append(depth + j)
prev_slot_buf.append(prev)
chain_last_slot = slot
child_depth = depth + len(node_slots)
for child in node.children.values():
_walk_radix_subtree(
node=child,
depth=child_depth,
parent_last_slot=chain_last_slot,
slot_buf=slot_buf,
position_buf=position_buf,
prev_slot_buf=prev_slot_buf,
is_root=False,
unlocked_only=unlocked_only,
)
def _node_slots_for_canary(*, node: TreeNode) -> list[int]:
value: Any = node.value
if isinstance(value, torch.Tensor):
return [int(s) for s in value.tolist()]
return []
@@ -140,15 +140,6 @@ class SwaDivergenceLog:
return None
return cls(**json.loads(match.group(1)))
@classmethod
def find_last(cls, text: str) -> Optional[tuple[SwaDivergenceLog, str]]:
last_match: Optional[re.Match] = None
for match in _SWA_DIVERGENCE_LINE_RE.finditer(text):
last_match = match
if last_match is None:
return None
return cls(**json.loads(last_match.group(1))), last_match.group(0)
@classmethod
def find_all(cls, text: str) -> list[tuple[SwaDivergenceLog, str]]:
return [
@@ -406,18 +406,6 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
implementations that shard trees per cache namespace."""
return self.root_node
def is_root(self, node: Any) -> bool:
"""Whether the node is a tree root."""
return node is self.root_node
def get_last_hash_value(self, node: Any) -> Optional[str]:
"""The node's last page hash, or None when it was never hashed."""
return node.get_last_hash_value()
def get_prefix_hash_values(self, node: Any) -> list[str]:
"""The hash chain of the node's ancestors, in root-to-parent order."""
return node.get_prefix_hash_values(node.parent)
def rotation_base_of(self, node: Any) -> Optional[int]:
"""Logical-page KV sharding: the rotation base stamped on ``node``.
@@ -48,7 +48,6 @@ from sglang.srt.mem_cache.base_prefix_cache import (
from sglang.srt.mem_cache.events import KVCacheEventRecorder
from sglang.srt.mem_cache.utils import (
get_eviction_strategy,
get_hash_str,
split_node_hash_value,
)
@@ -246,12 +245,6 @@ class RadixKey:
return ((self.extra_key, self.cache_salt), plain)
return plain if self.extra_key is None else (self.extra_key, plain)
def hash_page(self, start: int, end: int, prior_hash: Optional[str] = None) -> str:
"""SHA256 for logical units [start, end); bigram mode feeds overlapping (t_i, t_{i+1}) byte pairs."""
hash_value = get_hash_str(self[start:end], prior_hash)
assert isinstance(hash_value, str)
return hash_value
class TreeNode:
counter = 0
@@ -280,19 +273,6 @@ class TreeNode:
def evicted(self):
return self.value is None
def get_last_hash_value(self) -> Optional[str]:
"""Returns the hash value of the last page in this node."""
if self.hash_value is None or len(self.hash_value) == 0:
return None
return self.hash_value[-1]
def get_prefix_hash_values(self, node: TreeNode) -> List[str]:
chunks = []
while node is not None and node.hash_value is not None:
chunks.append(node.hash_value)
node = node.parent
return [value for chunk in reversed(chunks) for value in chunk]
def __lt__(self, other: TreeNode):
return self.last_access_time < other.last_access_time
+55 -21
View File
@@ -2,7 +2,8 @@ from __future__ import annotations
from dataclasses import dataclass
from types import SimpleNamespace
from typing import List, Optional
from typing import List, Optional, Tuple
from unittest.mock import MagicMock
import torch
@@ -13,7 +14,13 @@ from sglang.srt.kv_canary.config import CanaryConfig, CanaryMode
from sglang.srt.kv_canary.pool_patcher.adapters.mha import attach_mha
from sglang.srt.kv_canary.pool_patcher.adapters.swa import attach_swa
from sglang.srt.kv_canary.pool_patcher.api import register_pool_attacher
from sglang.srt.mem_cache.radix_cache import RadixCache, TreeNode
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
from sglang.srt.mem_cache.unified_cache.components import (
BASE_COMPONENT_TYPE,
ComponentType,
)
from sglang.srt.mem_cache.unified_cache.unified_tree_core import UnifiedTreeCore
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache, UnifiedTreeNode
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.utils import get_device
@@ -268,28 +275,55 @@ def make_buffer_group(
)
def make_radix_cache(
slot_lists: List[List[int]], device: torch.device = DEFAULT_DEVICE
):
cache = RadixCache.__new__(RadixCache)
cache.device = device
cache.page_size = 1
cache.disable = False
root = TreeNode()
root.value = torch.tensor(
slot_lists[0] if slot_lists else [], dtype=torch.int32, device=device
def make_unified_radix_cache(
tree_components: Tuple[ComponentType, ...] = (ComponentType.FULL,),
) -> UnifiedRadixCache:
cache = UnifiedRadixCache.__new__(UnifiedRadixCache)
cache.tree_components = tree_components
cache.components = {ct: None for ct in tree_components}
cache.is_swa_enabled = ComponentType.SWA in tree_components
cache.tree_core = UnifiedTreeCore(
CacheInitParams(
disable=False,
req_to_token_pool=None,
token_to_kv_pool_allocator=None,
page_size=1,
),
{ct: MagicMock() for ct in tree_components},
)
cache.root_node = root
return cache
current = root
for child_slots in slot_lists[1:]:
child = TreeNode()
child.value = torch.tensor(child_slots, dtype=torch.int32, device=device)
child.parent = current
current.children[child.id] = child
current = child
def add_unified_child(
cache: UnifiedRadixCache,
slots: List[int],
*,
parent: Optional[UnifiedTreeNode] = None,
lock_ref: int = 0,
swa_value: Optional[List[int]] = None,
device: torch.device = DEFAULT_DEVICE,
) -> UnifiedTreeNode:
parent = cache.root_node if parent is None else parent
child = UnifiedTreeNode(cache.tree_components)
child.parent = parent
base = child.component_data[BASE_COMPONENT_TYPE]
base.value = torch.tensor(slots, dtype=torch.int32, device=device)
base.lock_ref = lock_ref
if swa_value is not None:
child.component_data[ComponentType.SWA].value = torch.tensor(
swa_value, dtype=torch.int32, device=device
)
parent.children[child.id] = child
return child
def make_unified_radix_chain(
chain: List[List[int]], device: torch.device = DEFAULT_DEVICE
) -> UnifiedRadixCache:
cache = make_unified_radix_cache()
parent = None
for slots in chain:
parent = add_unified_child(cache, slots, parent=parent, device=device)
return cache