[mem_cache] Remove unreachable RadixCache paths in KV canary and HiCache accessors (#40807)
This commit is contained in:
@@ -266,9 +266,6 @@ class DecodeHiCacheTransferMixin:
|
||||
return True
|
||||
|
||||
def _process_hicache_local_restores(self, decode_reqs: List[DecodeRequest]) -> None:
|
||||
if not hasattr(self.tree_cache, "is_load_back_event_done"):
|
||||
return
|
||||
|
||||
# Filter once: keep only PENDING reqs that still need restore work;
|
||||
# trivially-done reqs (no prefix_match / nothing to restore) flip to READY.
|
||||
active: List[DecodeRequest] = []
|
||||
|
||||
@@ -86,30 +86,3 @@ def make_row_source(
|
||||
read_bytes=clipped,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def make_packed_source(
|
||||
*,
|
||||
page_buffer: torch.Tensor,
|
||||
page_size: int,
|
||||
bytes_per_token: int,
|
||||
read_bytes: int,
|
||||
) -> Tuple[RealKvSource, ...]:
|
||||
if read_bytes == 0 or page_buffer.numel() == 0:
|
||||
return ()
|
||||
flat = page_buffer.contiguous().view(torch.uint8)
|
||||
if flat.ndim == 1:
|
||||
flat = flat.reshape(1, -1)
|
||||
clipped = _clip_read_bytes_aligned(
|
||||
requested=read_bytes, num_bytes_per_token=bytes_per_token
|
||||
)
|
||||
if clipped == 0:
|
||||
return ()
|
||||
return (
|
||||
RealKvSource(
|
||||
tensor=flat,
|
||||
page_size=page_size,
|
||||
num_bytes_per_token=bytes_per_token,
|
||||
read_bytes=clipped,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -1,10 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.mem_cache.radix_cache import RadixCache
|
||||
from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import (
|
||||
RadixCacheWalkResult,
|
||||
)
|
||||
@@ -12,7 +9,6 @@ from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache
|
||||
from sglang.srt.mem_cache.radix_cache import TreeNode
|
||||
|
||||
|
||||
def walk_radix_cache_for_canary(
|
||||
@@ -28,83 +24,10 @@ def walk_radix_cache_for_canary(
|
||||
per-forward HEAD/TAIL path). ``unlocked_only=True`` skips nodes still locked by a running
|
||||
req. ``swa_resident_only=True`` skips SWA-tombstoned nodes (slots evicted from the SWA
|
||||
window)."""
|
||||
cache_type = type(radix_cache)
|
||||
if cache_type is UnifiedRadixCache:
|
||||
return radix_cache.tree_core.walk_for_kv_canary(
|
||||
unlocked_only=unlocked_only, swa_resident_only=swa_resident_only
|
||||
)
|
||||
if cache_type is not RadixCache:
|
||||
if type(radix_cache) is not UnifiedRadixCache:
|
||||
raise NotImplementedError(
|
||||
f"walk_radix_cache_for_canary does not support {cache_type.__name__}"
|
||||
f"walk_radix_cache_for_canary does not support {type(radix_cache).__name__}"
|
||||
)
|
||||
|
||||
# RadixCache has no SWA tier, so every node it holds is SWA-resident and
|
||||
# swa_resident_only is a no-op here.
|
||||
slot_buf: list[int] = []
|
||||
position_buf: list[int] = []
|
||||
prev_slot_buf: list[int] = []
|
||||
|
||||
_walk_radix_subtree(
|
||||
node=radix_cache.root_node,
|
||||
depth=0,
|
||||
parent_last_slot=-1,
|
||||
slot_buf=slot_buf,
|
||||
position_buf=position_buf,
|
||||
prev_slot_buf=prev_slot_buf,
|
||||
is_root=True,
|
||||
unlocked_only=unlocked_only,
|
||||
return radix_cache.tree_core.walk_for_kv_canary(
|
||||
unlocked_only=unlocked_only, swa_resident_only=swa_resident_only
|
||||
)
|
||||
|
||||
slot_tensor = torch.tensor(slot_buf, dtype=torch.int64)
|
||||
position_tensor = torch.tensor(position_buf, dtype=torch.int64)
|
||||
prev_slot_tensor = torch.tensor(prev_slot_buf, dtype=torch.int64)
|
||||
return RadixCacheWalkResult(
|
||||
slot_indices=slot_tensor,
|
||||
positions=position_tensor,
|
||||
prev_slot_indices=prev_slot_tensor,
|
||||
)
|
||||
|
||||
|
||||
def _walk_radix_subtree(
|
||||
*,
|
||||
node: TreeNode,
|
||||
depth: int,
|
||||
parent_last_slot: int,
|
||||
slot_buf: list[int],
|
||||
position_buf: list[int],
|
||||
prev_slot_buf: list[int],
|
||||
is_root: bool,
|
||||
unlocked_only: bool,
|
||||
) -> None:
|
||||
node_slots = _node_slots_for_canary(node=node)
|
||||
|
||||
emit_slots = not is_root and (not unlocked_only or node.lock_ref == 0)
|
||||
|
||||
chain_last_slot = parent_last_slot
|
||||
for j, slot in enumerate(node_slots):
|
||||
prev = parent_last_slot if j == 0 else node_slots[j - 1]
|
||||
if emit_slots:
|
||||
slot_buf.append(slot)
|
||||
position_buf.append(depth + j)
|
||||
prev_slot_buf.append(prev)
|
||||
chain_last_slot = slot
|
||||
|
||||
child_depth = depth + len(node_slots)
|
||||
for child in node.children.values():
|
||||
_walk_radix_subtree(
|
||||
node=child,
|
||||
depth=child_depth,
|
||||
parent_last_slot=chain_last_slot,
|
||||
slot_buf=slot_buf,
|
||||
position_buf=position_buf,
|
||||
prev_slot_buf=prev_slot_buf,
|
||||
is_root=False,
|
||||
unlocked_only=unlocked_only,
|
||||
)
|
||||
|
||||
|
||||
def _node_slots_for_canary(*, node: TreeNode) -> list[int]:
|
||||
value: Any = node.value
|
||||
if isinstance(value, torch.Tensor):
|
||||
return [int(s) for s in value.tolist()]
|
||||
return []
|
||||
|
||||
@@ -140,15 +140,6 @@ class SwaDivergenceLog:
|
||||
return None
|
||||
return cls(**json.loads(match.group(1)))
|
||||
|
||||
@classmethod
|
||||
def find_last(cls, text: str) -> Optional[tuple[SwaDivergenceLog, str]]:
|
||||
last_match: Optional[re.Match] = None
|
||||
for match in _SWA_DIVERGENCE_LINE_RE.finditer(text):
|
||||
last_match = match
|
||||
if last_match is None:
|
||||
return None
|
||||
return cls(**json.loads(last_match.group(1))), last_match.group(0)
|
||||
|
||||
@classmethod
|
||||
def find_all(cls, text: str) -> list[tuple[SwaDivergenceLog, str]]:
|
||||
return [
|
||||
|
||||
@@ -406,18 +406,6 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
|
||||
implementations that shard trees per cache namespace."""
|
||||
return self.root_node
|
||||
|
||||
def is_root(self, node: Any) -> bool:
|
||||
"""Whether the node is a tree root."""
|
||||
return node is self.root_node
|
||||
|
||||
def get_last_hash_value(self, node: Any) -> Optional[str]:
|
||||
"""The node's last page hash, or None when it was never hashed."""
|
||||
return node.get_last_hash_value()
|
||||
|
||||
def get_prefix_hash_values(self, node: Any) -> list[str]:
|
||||
"""The hash chain of the node's ancestors, in root-to-parent order."""
|
||||
return node.get_prefix_hash_values(node.parent)
|
||||
|
||||
def rotation_base_of(self, node: Any) -> Optional[int]:
|
||||
"""Logical-page KV sharding: the rotation base stamped on ``node``.
|
||||
|
||||
|
||||
@@ -48,7 +48,6 @@ from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
from sglang.srt.mem_cache.events import KVCacheEventRecorder
|
||||
from sglang.srt.mem_cache.utils import (
|
||||
get_eviction_strategy,
|
||||
get_hash_str,
|
||||
split_node_hash_value,
|
||||
)
|
||||
|
||||
@@ -246,12 +245,6 @@ class RadixKey:
|
||||
return ((self.extra_key, self.cache_salt), plain)
|
||||
return plain if self.extra_key is None else (self.extra_key, plain)
|
||||
|
||||
def hash_page(self, start: int, end: int, prior_hash: Optional[str] = None) -> str:
|
||||
"""SHA256 for logical units [start, end); bigram mode feeds overlapping (t_i, t_{i+1}) byte pairs."""
|
||||
hash_value = get_hash_str(self[start:end], prior_hash)
|
||||
assert isinstance(hash_value, str)
|
||||
return hash_value
|
||||
|
||||
|
||||
class TreeNode:
|
||||
counter = 0
|
||||
@@ -280,19 +273,6 @@ class TreeNode:
|
||||
def evicted(self):
|
||||
return self.value is None
|
||||
|
||||
def get_last_hash_value(self) -> Optional[str]:
|
||||
"""Returns the hash value of the last page in this node."""
|
||||
if self.hash_value is None or len(self.hash_value) == 0:
|
||||
return None
|
||||
return self.hash_value[-1]
|
||||
|
||||
def get_prefix_hash_values(self, node: TreeNode) -> List[str]:
|
||||
chunks = []
|
||||
while node is not None and node.hash_value is not None:
|
||||
chunks.append(node.hash_value)
|
||||
node = node.parent
|
||||
return [value for chunk in reversed(chunks) for value in chunk]
|
||||
|
||||
def __lt__(self, other: TreeNode):
|
||||
return self.last_access_time < other.last_access_time
|
||||
|
||||
|
||||
@@ -2,7 +2,8 @@ from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from types import SimpleNamespace
|
||||
from typing import List, Optional
|
||||
from typing import List, Optional, Tuple
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import torch
|
||||
|
||||
@@ -13,7 +14,13 @@ from sglang.srt.kv_canary.config import CanaryConfig, CanaryMode
|
||||
from sglang.srt.kv_canary.pool_patcher.adapters.mha import attach_mha
|
||||
from sglang.srt.kv_canary.pool_patcher.adapters.swa import attach_swa
|
||||
from sglang.srt.kv_canary.pool_patcher.api import register_pool_attacher
|
||||
from sglang.srt.mem_cache.radix_cache import RadixCache, TreeNode
|
||||
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
|
||||
from sglang.srt.mem_cache.unified_cache.components import (
|
||||
BASE_COMPONENT_TYPE,
|
||||
ComponentType,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_cache.unified_tree_core import UnifiedTreeCore
|
||||
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache, UnifiedTreeNode
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.utils import get_device
|
||||
|
||||
@@ -268,28 +275,55 @@ def make_buffer_group(
|
||||
)
|
||||
|
||||
|
||||
def make_radix_cache(
|
||||
slot_lists: List[List[int]], device: torch.device = DEFAULT_DEVICE
|
||||
):
|
||||
cache = RadixCache.__new__(RadixCache)
|
||||
cache.device = device
|
||||
cache.page_size = 1
|
||||
cache.disable = False
|
||||
|
||||
root = TreeNode()
|
||||
root.value = torch.tensor(
|
||||
slot_lists[0] if slot_lists else [], dtype=torch.int32, device=device
|
||||
def make_unified_radix_cache(
|
||||
tree_components: Tuple[ComponentType, ...] = (ComponentType.FULL,),
|
||||
) -> UnifiedRadixCache:
|
||||
cache = UnifiedRadixCache.__new__(UnifiedRadixCache)
|
||||
cache.tree_components = tree_components
|
||||
cache.components = {ct: None for ct in tree_components}
|
||||
cache.is_swa_enabled = ComponentType.SWA in tree_components
|
||||
cache.tree_core = UnifiedTreeCore(
|
||||
CacheInitParams(
|
||||
disable=False,
|
||||
req_to_token_pool=None,
|
||||
token_to_kv_pool_allocator=None,
|
||||
page_size=1,
|
||||
),
|
||||
{ct: MagicMock() for ct in tree_components},
|
||||
)
|
||||
cache.root_node = root
|
||||
return cache
|
||||
|
||||
current = root
|
||||
for child_slots in slot_lists[1:]:
|
||||
child = TreeNode()
|
||||
child.value = torch.tensor(child_slots, dtype=torch.int32, device=device)
|
||||
child.parent = current
|
||||
current.children[child.id] = child
|
||||
current = child
|
||||
|
||||
def add_unified_child(
|
||||
cache: UnifiedRadixCache,
|
||||
slots: List[int],
|
||||
*,
|
||||
parent: Optional[UnifiedTreeNode] = None,
|
||||
lock_ref: int = 0,
|
||||
swa_value: Optional[List[int]] = None,
|
||||
device: torch.device = DEFAULT_DEVICE,
|
||||
) -> UnifiedTreeNode:
|
||||
parent = cache.root_node if parent is None else parent
|
||||
child = UnifiedTreeNode(cache.tree_components)
|
||||
child.parent = parent
|
||||
base = child.component_data[BASE_COMPONENT_TYPE]
|
||||
base.value = torch.tensor(slots, dtype=torch.int32, device=device)
|
||||
base.lock_ref = lock_ref
|
||||
if swa_value is not None:
|
||||
child.component_data[ComponentType.SWA].value = torch.tensor(
|
||||
swa_value, dtype=torch.int32, device=device
|
||||
)
|
||||
parent.children[child.id] = child
|
||||
return child
|
||||
|
||||
|
||||
def make_unified_radix_chain(
|
||||
chain: List[List[int]], device: torch.device = DEFAULT_DEVICE
|
||||
) -> UnifiedRadixCache:
|
||||
cache = make_unified_radix_cache()
|
||||
parent = None
|
||||
for slots in chain:
|
||||
parent = add_unified_child(cache, slots, parent=parent, device=device)
|
||||
return cache
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user