[MemCache] Return only the matched length from match_prefix; read KV indices off the node path (#42923)

This commit is contained in:
Liangsheng Yin
2026-10-08 01:10:14 -07:00
committed by GitHub
parent 2d26823f94
commit 5a4d01a3ae
43 changed files with 482 additions and 467 deletions
@@ -69,7 +69,7 @@ class DecodeHiCachePreallocMixin:
Performs the optional L3 storage hit length query when decode-side
HiCache is enabled and the last host node is backed up.
"""
prefix_indices = result.device_indices
prefix_indices = self.tree_cache.prefix_device_indices(req)
l1_prefix_len = len(prefix_indices)
l2_host_hit_length = result.host_hit_length
@@ -219,6 +219,7 @@ class DecodeHiCacheTransferMixin:
cow_mamba=False,
max_prefix_len=pm.decode_prefix_len,
)
rematched_indices = self.tree_cache.prefix_device_indices(dr.req)
new_indices, restored_node = self.tree_cache.init_load_back(
InitLoadBackParams(
best_match_node=rematch.best_match_node,
@@ -230,12 +231,12 @@ class DecodeHiCacheTransferMixin:
# boundary; point it back at the prefix the prealloc matched and locked.
dr.req.last_node = pm.last_device_node
# Failback: total coverage < required prefix means device alloc likely failed.
if len(rematch.device_indices) + len(new_indices) < pm.decode_prefix_len:
if rematch.device_prefix_len + len(new_indices) < pm.decode_prefix_len:
logger.warning(
"HiCache load_back failed for rid=%s: device_indices=%d, "
"new_indices=%d, expected decode_prefix_len=%d (l1=%d, l2=%d, l3=%d)",
dr.req.rid,
len(rematch.device_indices),
rematch.device_prefix_len,
len(new_indices),
pm.decode_prefix_len,
pm.l1_prefix_len,
@@ -246,7 +247,7 @@ class DecodeHiCacheTransferMixin:
return False
dr.hicache_restored_kv_indices = torch.cat(
[rematch.device_indices[pm.l1_prefix_len :], new_indices]
[rematched_indices[pm.l1_prefix_len :], new_indices]
)[: pm.restore_token_count]
dr.hicache_restore_lock = self.tree_cache.lock(restored_node)
@@ -159,7 +159,7 @@ class C128SidecarComponent(TreeComponent):
# endpoint, so constrain that candidate to the nearest C128 boundary.
group_tokens = 128 * self.allocator.c128_attn_allocator.page_size
branching_seqlen = branching_seqlen // group_tokens * group_tokens
current_boundary = len(result.device_indices) + result.host_hit_length
current_boundary = result.device_prefix_len + result.host_hit_length
return result._replace(
swa_branching_seqlen=(
branching_seqlen if branching_seqlen > current_boundary else None
@@ -343,9 +343,8 @@ class SchedulePolicy:
match_result = zero_match_result(
self.waiting_queue_radix_tree, match_result, extra_key=extra_key
)
in_batch_matching_prefixes = match_result.device_indices
if (
len(in_batch_matching_prefixes)
match_result.device_prefix_len
>= IN_BATCH_PREFIX_CACHING_DEPRIORITIZE_THRESHOLD
):
temporary_deprioritized.add(r.rid)
@@ -271,7 +271,11 @@ class MatchResult(NamedTuple):
"""Result of a prefix match operation.
Attributes:
device_indices : Indices of the KV cache on the device matched by common prefix.
device_prefix_len: Length of the device-resident matched prefix. Its KV
indices lie on the path to ``last_device_node``, except
slots LMCache loaded but has not published (which
``prefix_device_indices`` appends) and a streaming
session's match, which lives in the lent row.
last_device_node: The last TreeNode on the device that was matched.
last_host_node : The last TreeNode on the host that was matched.
Note that if HiCache is not enabled,
@@ -299,7 +303,7 @@ class MatchResult(NamedTuple):
host, independent of other components.
"""
device_indices: torch.Tensor
device_prefix_len: int
last_device_node: Any
last_host_node: Any
best_match_node: Any
@@ -322,9 +326,7 @@ def zero_match_result(
return match_result
root = tree_cache.root_node_handle(extra_key=extra_key)
return match_result._replace(
# [:0] keeps dtype and device of the original tensor (e.g. CUDA int64)
# without allocating a fresh empty tensor.
device_indices=match_result.device_indices[:0],
device_prefix_len=0,
last_device_node=root,
last_host_node=root,
best_match_node=root,
@@ -528,10 +530,17 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
if lock is not None:
self.dec_lock_ref(lock.node, lock.receipt)
def path_device_indices(self, node: Any) -> torch.Tensor:
"""Device KV indices on the path from the root to ``node``; valid until
the next allocation or eviction unless the path is locked."""
raise NotImplementedError
def prefix_device_indices(self, req: Req) -> torch.Tensor:
"""KV indices of req's matched prefix, read off the path to its locked
match node; valid from match until allocation writes them into the row."""
raise NotImplementedError
path = self.path_device_indices(req.last_node)
assert len(path) >= req.prefix_len, (req.rid, len(path), req.prefix_len)
return path[: req.prefix_len]
def maybe_hand_to_session(self, req: Req) -> None:
"""A cache that keeps records across requests (a streaming session) takes
@@ -1193,7 +1193,7 @@ class BufferModePipeline:
cache_salt=span_key.cache_salt,
)
match = self._cache.match_prefix(MatchPrefixParams(key=key))
return len(match.device_indices) >= len(key)
return match.device_prefix_len >= len(key)
def set_prefix_ctx(
self,
@@ -1391,7 +1391,7 @@ class BufferModePipeline:
req = params.req
assert req is not None
request = req.cache_request_handle
empty = cache.tree_core.empty_match_result.device_indices
empty = cache.tree_core.empty_device_indices
unchanged = (empty, req.last_node)
f = self.staged_prefetches.get(request)
if f is None:
@@ -1632,10 +1632,12 @@ class BufferModePipeline:
aux_device_releases=aux_device_releases,
)
match = cache.match_prefix(MatchPrefixParams(key=key))
canonical = cache.path_device_indices(match.last_device_node)[
splice_base:span_end
]
self.release_anchor_lock(request)
canonical = match.device_indices[splice_base:span_end]
owned = len(match.device_indices) >= span_end and torch.equal(
match.device_indices[splice_base:span_end], device_indices
owned = match.device_prefix_len >= span_end and torch.equal(
canonical, device_indices
)
if not owned:
# Fail-stop: the insert freed or replaced slots the in-flight H2D
@@ -1644,7 +1646,7 @@ class BufferModePipeline:
"HiCache buffer load-back ownership violation "
f"req={f.request.rid}: "
f"insert prefix_len={insert_result.prefix_len} "
f"expected={splice_base}, matched={len(match.device_indices)} "
f"expected={splice_base}, matched={match.device_prefix_len} "
f"span_end={span_end} splice_base={splice_base}; "
f"in-flight H2D targets freed slots"
)
+4 -1
View File
@@ -66,12 +66,15 @@ class ChunkCache(BasePrefixCache):
def match_prefix(self, params: MatchPrefixParams) -> MatchResult:
return MatchResult(
device_indices=torch.empty((0,), dtype=torch.int64),
device_prefix_len=0,
last_device_node=None,
last_host_node=None,
best_match_node=None,
)
def path_device_indices(self, node: Any) -> torch.Tensor:
return torch.empty((0,), dtype=torch.int64)
def insert(self, params: InsertParams) -> InsertResult:
# ChunkCache does not support prefix caching, so insert is a no-op
return InsertResult(prefix_len=0)
+1 -1
View File
@@ -209,7 +209,7 @@ def match_kv_cache(
match_result = zero_match_result(
tree_cache, match_result, extra_key=req.extra_key
)
req.prefix_len = len(match_result.device_indices)
req.prefix_len = match_result.device_prefix_len
(
req.last_node,
req.last_host_node,
+13 -22
View File
@@ -340,12 +340,11 @@ class RadixCache(BasePrefixCache):
self.evictable_size_ = 0
self.protected_size_ = 0
self.evictable_leaves.clear()
self._empty_device_indices = torch.empty(
(0,), dtype=torch.int64, device=self.device
)
self._empty_match_result = MatchResult(
device_indices=torch.empty(
(0,),
dtype=torch.int64,
device=self.device,
),
device_prefix_len=0,
last_device_node=self.root_node,
last_host_node=self.root_node,
best_match_node=self.root_node,
@@ -374,9 +373,8 @@ class RadixCache(BasePrefixCache):
empty result with the root as the last node.
Returns:
MatchResult: ``device_indices`` is a 1-D ``torch.int64`` tensor of
the concatenated KV cache indices corresponding to the longest
cached prefix (may be length 0).
MatchResult: ``device_prefix_len`` is the length of the longest
cached prefix (may be 0).
``last_device_node`` and ``last_host_node`` (currently the same) are the tree node objects
representing the terminal node of the matched prefix. This method
may mutate internal structure by splitting an existing node if the
@@ -401,12 +399,8 @@ class RadixCache(BasePrefixCache):
return self._empty_match_result
value, last_node = self._match_prefix_helper(self.root_node, key)
if value:
value = torch.cat(value)
else:
value = self._empty_match_result.device_indices
return MatchResult(
device_indices=value,
device_prefix_len=sum(len(v) for v in value),
last_device_node=last_node,
last_host_node=last_node,
best_match_node=last_node,
@@ -509,10 +503,8 @@ class RadixCache(BasePrefixCache):
# The prefix indices could be updated, reuse it
match_result = self.match_prefix(MatchPrefixParams(key=radix_key))
new_indices, new_last_node = (
match_result.device_indices,
match_result.last_device_node,
)
new_last_node = match_result.last_device_node
new_indices = self.path_device_indices(new_last_node)
assert len(new_indices) == len(radix_key), (
f"{len(new_indices)=}, {len(radix_key)=}"
)
@@ -624,16 +616,15 @@ class RadixCache(BasePrefixCache):
##### Internal Helper Functions #####
def prefix_device_indices(self, req: Req) -> torch.Tensor:
def path_device_indices(self, node: TreeNode) -> torch.Tensor:
values = []
node = req.last_node
while node is not self.root_node:
values.append(node.value)
node = node.parent
if not values:
return self._empty_device_indices
values.reverse()
path = torch.cat(values)
assert len(path) >= req.prefix_len, (req.rid, len(path), req.prefix_len)
return path[: req.prefix_len]
return torch.cat(values)
def _match_prefix_helper(self, node: TreeNode, key: RadixKey):
access_time = time.monotonic()
@@ -267,7 +267,7 @@ def _insert_step_from_binding(step) -> InsertStepResult:
def _match_result_from_binding(result) -> MatchResult:
"""Build the Python MatchResult for the binding's match result."""
return MatchResult(
device_indices=result.device_indices,
device_prefix_len=result.device_prefix_len,
last_device_node=result.last_device_node_id,
last_host_node=result.last_host_node_id,
best_match_node=result.best_match_node_id,
@@ -440,6 +440,9 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
self._empty_match_result = _match_result_from_binding(
self._binding.empty_match_result()
)
self._empty_device_indices = torch.empty(
(0,), dtype=torch.int64, device=self.device
)
def _binding_class(self) -> type:
"""The extension binding class this core constructs."""
@@ -717,6 +720,10 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
def empty_match_result(self) -> MatchResult:
return self._empty_match_result
@property
def empty_device_indices(self) -> torch.Tensor:
return self._empty_device_indices
def is_full_device_evicted(self, node_id: NodeId) -> bool:
return self._binding.is_full_device_evicted(node_id)
@@ -174,29 +174,25 @@ class FlexKVRadixCache(RadixCache):
if len(key) == 0:
return base_res
device_value: torch.Tensor = base_res.device_indices
last_node: TreeNode = base_res.last_device_node
if self._mode is FlexKVMode.MP:
if params.req is None:
return base_res
return self._mp_match_prefix(
key, base_res, device_value, last_node, params.req
)
return self._ip_match_prefix(key, base_res, device_value, last_node)
return self._mp_match_prefix(key, base_res, last_node, params.req)
return self._ip_match_prefix(key, base_res, last_node)
def _mp_match_prefix(
self,
key: RadixKey,
base_res: MatchResult,
device_value: torch.Tensor,
last_node: TreeNode,
req: Req,
) -> MatchResult:
"""LOOKUP-only path. Sets ``host_hit_length`` on the result so
the scheduler later invokes :meth:`init_load_back`."""
token_ids = key.raw_token_ids()
device_len = int(device_value.numel())
device_len = base_res.device_prefix_len
if device_len >= len(token_ids):
return base_res
@@ -226,7 +222,7 @@ class FlexKVRadixCache(RadixCache):
value_numel=device_len,
)
return MatchResult(
device_indices=device_value,
device_prefix_len=device_len,
last_device_node=last_node,
last_host_node=last_node,
best_match_node=last_node,
@@ -237,13 +233,12 @@ class FlexKVRadixCache(RadixCache):
self,
key: RadixKey,
base_res: MatchResult,
device_value: torch.Tensor,
last_node: TreeNode,
) -> MatchResult:
"""Layerwise path: allocate slots and fire ``start_load_kv_layerwise``
immediately. Per-layer hook waits during forward."""
token_ids = key.raw_token_ids()
device_len = int(device_value.numel())
device_len = base_res.device_prefix_len
if device_len >= len(token_ids):
return base_res
@@ -271,7 +266,7 @@ class FlexKVRadixCache(RadixCache):
return base_res
new_slots, new_node = result
return MatchResult(
device_indices=torch.cat([device_value, new_slots]),
device_prefix_len=device_len + len(new_slots),
last_device_node=new_node,
last_host_node=new_node,
best_match_node=new_node,
@@ -90,8 +90,7 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
def prefix_device_indices(self, req: Req) -> torch.Tensor:
# match_prefix may append loaded slots not yet published to the tree,
# so the path alone can be shorter than the prefix.
root = self.root_node_handle(req.extra_key)
path = self.tree_core.collect_full_device_indices(req.last_node, root)
path = self.path_device_indices(req.last_node)
indices = path[: req.prefix_len]
missing = req.prefix_len - len(indices)
if missing > 0:
@@ -129,7 +128,7 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
return result
total_hit = min(flow.total_hit, requested_key_len, len(flow.key))
local_hit = len(result.device_indices)
local_hit = result.device_prefix_len
skip = 0
if flow.load is not None and not flow.prefix_published:
# Track private slots shadowed by a concurrent tree insertion.
@@ -165,7 +164,7 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
# Reuse an already submitted H2D load on the next admission attempt.
suffix = flow.load.device_indices[skip : skip + max(total_hit - local_hit, 0)]
return result._replace(
device_indices=torch.cat([result.device_indices, suffix]),
device_prefix_len=local_hit + len(suffix),
last_host_node=result.last_device_node,
best_match_node=result.last_device_node,
host_hit_length=0,
@@ -232,7 +231,7 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
return False
latest = super().match_prefix(MatchPrefixParams(key=flow.key))
local_hit = torch.tensor(
[len(latest.device_indices)], dtype=torch.int64, device="cpu"
[latest.device_prefix_len], dtype=torch.int64, device="cpu"
)
self.lmcache_connector.parallel_all_reduce(
local_hit, torch.distributed.ReduceOp.MIN
@@ -358,13 +357,13 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
req = params.req
if req is None:
return (
self.tree_core.empty_match_result.device_indices,
self.tree_core.empty_device_indices,
params.best_match_node,
)
flow = self._external_flows.get(req.rid)
if flow is None or flow.total_hit is None or flow.load is not None:
return (
self.tree_core.empty_match_result.device_indices,
self.tree_core.empty_device_indices,
params.best_match_node,
)
@@ -381,7 +380,7 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
# Retire the unloaded flow to release its lookup locks.
self._retire_loaded_flow(req.rid)
return (
self.tree_core.empty_match_result.device_indices,
self.tree_core.empty_device_indices,
params.best_match_node,
)
return device_indices, params.best_match_node
@@ -538,7 +537,7 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
latest = super().match_prefix(MatchPrefixParams(key=flow.key[:local_hit]))
total_hit = min(flow.total_hit, len(flow.key))
if total_hit <= local_hit:
return self.tree_core.empty_match_result.device_indices
return self.tree_core.empty_device_indices
# Pin the common L1 boundary before allocation can trigger eviction.
flow.anchor_node = latest.last_device_node
@@ -833,13 +832,13 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
matched = super().match_prefix(
MatchPrefixParams(key=key, req=req, cow_mamba=False)
)
if len(matched.device_indices) < total_hit:
if matched.device_prefix_len < total_hit:
raise RuntimeError(
"LMCache loaded prefix was inserted but is not reusable across "
f"all UnifiedRadixCache components: {len(matched.device_indices)}"
f"all UnifiedRadixCache components: {matched.device_prefix_len}"
f"/{total_hit} tokens for request {req.rid}"
)
canonical = matched.device_indices[:total_hit]
canonical = self.path_device_indices(matched.last_device_node)[:total_hit]
self.req_to_token_pool.write(
(req.kv.req_pool_idx, slice(prev_prefix_len, total_hit)),
canonical[prev_prefix_len:],
@@ -873,7 +872,7 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
matched = super().match_prefix(MatchPrefixParams(key=key))
# Store the currently resident SWA prefix; later callbacks may extend it.
resident_len = torch.tensor(
[min(len(matched.device_indices), len(key))],
[min(matched.device_prefix_len, len(key))],
dtype=torch.int64,
device="cpu",
)
@@ -907,7 +906,9 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
key = key[:resident_len]
# Re-match so the lock and Mamba state use the shortened boundary.
matched = super().match_prefix(MatchPrefixParams(key=key))
store_indices = matched.device_indices[store_start : len(key)]
store_indices = self.path_device_indices(matched.last_device_node)[
store_start : len(key)
]
if len(store_indices) != len(key) - store_start:
logger.debug(
"LMCache store skipped for %s: store indices length: %d, "
@@ -99,7 +99,7 @@ Find the longest cached prefix for a token sequence.
|--------|--------|
| **Purpose** | Walk the radix tree to find the longest prefix where **all** component validators pass |
| **Inputs** | `params.key: RadixKey` — token IDs + optional extra key for namespace isolation |
| **Output** | `MatchResult(device_indices, last_device_node, last_host_node, best_match_node, host_hit_length, mamba_branching_seqlen, ...)` |
| **Output** | `MatchResult(device_prefix_len, last_device_node, last_host_node, best_match_node, host_hit_length, mamba_branching_seqlen, ...)` |
| **Mutation** | Updates `last_access_time` on matched path; promotes matched nodes to MRU in all component LRU lists; may trigger `_split_node` if match ends mid-node |
| **Complexity** | **O(K + D·C)** |
@@ -165,7 +165,7 @@ class MambaComponent(TreeComponent):
) -> MatchResult:
last_node = result.best_match_node
mamba_boundary_len = len(result.device_indices) + result.host_hit_length
mamba_boundary_len = result.device_prefix_len + result.host_hit_length
# Full KV may extend beyond the latest reusable Mamba state. The branching
# point is the last Mamba-cache-chunk-aligned position within the Full-KV hit
@@ -356,7 +356,7 @@ class SWAComponent(TreeComponent):
best_value_len: int,
) -> MatchResult:
ct = self.component_type
swa_boundary_len = len(result.device_indices) + result.host_hit_length
swa_boundary_len = result.device_prefix_len + result.host_hit_length
# Full KV may extend beyond the latest reusable SWA window. The branching
# point is the last page-aligned position within the Full-KV hit that lies
@@ -193,7 +193,7 @@ class UnifiedCacheLinkerWrapper:
cache = self.cache
key, _ = key.maybe_to_bigram_view(cache.tree_core.is_eagle)
page = cache.page_size
device_hit_len = int(result.device_indices.numel())
device_hit_len = result.device_prefix_len
if device_hit_len >= len(key):
return result
@@ -289,7 +289,7 @@ class UnifiedCacheLinkerWrapper:
def load_back(self, req: Req) -> tuple[torch.Tensor, NodeId]:
cache = self.cache
empty_indices = cache.tree_core.empty_match_result.device_indices
empty_indices = cache.tree_core.empty_device_indices
hit = self.hit_markers.pop(req.rid, None)
if hit is None:
return empty_indices, req.last_node
@@ -525,12 +525,11 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
# Rolling digest of reclaim victim ids, cross-checked across TP ranks.
self.write_back_duplicate_reclaim_digest: int = 0
self._empty_device_indices = torch.empty(
(0,), dtype=torch.int64, device=self.device
)
self._empty_match_result = MatchResult(
device_indices=torch.empty(
(0,),
dtype=torch.int64,
device=self.device,
),
device_prefix_len=0,
last_device_node=self.root_node.id,
last_host_node=self.root_node.id,
best_match_node=self.root_node.id,
@@ -1006,12 +1005,8 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
best_match_node if self.enable_hicache else best_match_device_node
)
if best_match_device_value_len > 0:
device_indices = torch.cat(value[:best_match_device_value_len])
else:
device_indices = self._empty_match_result.device_indices
result = MatchResult(
device_indices=device_indices,
device_prefix_len=sum(len(v) for v in value[:best_match_device_value_len]),
last_device_node=best_match_device_node,
last_host_node=last_host_node,
best_match_node=best_match_node,
@@ -1036,9 +1031,13 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
@property
def empty_match_result(self) -> MatchResult:
"""A shared empty MatchResult (empty device indices + boundary NodeIds)."""
"""A shared empty MatchResult (zero device prefix + boundary NodeIds)."""
return self._empty_match_result
@property
def empty_device_indices(self) -> torch.Tensor:
return self._empty_device_indices
def is_full_device_evicted(self, node_id: NodeId) -> bool:
"""Whether the node's FULL device value has been evicted."""
return self.node_by_id(node_id).evicted
@@ -1057,7 +1056,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
prefix_chunks.append(value)
node = node.parent
if not prefix_chunks:
return self._empty_match_result.device_indices
return self._empty_device_indices
prefix_chunks.reverse()
return torch.cat(prefix_chunks)
@@ -438,7 +438,7 @@ class UnifiedTreeCoreInterface(ABC):
@abstractmethod
def match_prefix(self, params: MatchPrefixParams) -> MatchResult:
"""Match a key against the tree; returns device indices + boundary NodeIds."""
"""Match a key; returns the device prefix length + boundary NodeIds."""
...
@abstractmethod
@@ -463,7 +463,13 @@ class UnifiedTreeCoreInterface(ABC):
@property
@abstractmethod
def empty_match_result(self) -> MatchResult:
"""A shared empty MatchResult (empty device indices + boundary NodeIds)."""
"""A shared empty MatchResult (zero device prefix + boundary NodeIds)."""
...
@property
@abstractmethod
def empty_device_indices(self) -> torch.Tensor:
"""A shared empty device-index tensor on the core's device."""
...
@abstractmethod
@@ -1201,8 +1201,8 @@ class UnifiedRadixCache(BasePrefixCache):
# page-aligned boundary, so the normal match remains safe to repoint.
# The tree's own walk: a session slot must not answer for the insert.
match_result = self._match_tree(MatchPrefixParams(key=radix_key, req=req))
new_indices = match_result.device_indices
new_last_node = match_result.last_device_node
new_indices = self.path_device_indices(new_last_node)
new_prefix_len = result.prefix_len
assert req.kv.cache_protected_len <= len(new_indices) + self.page_size - 1, (
f"{req.kv.cache_protected_len=}, {len(new_indices)=}, {page_aligned_len=}"
@@ -3472,7 +3472,7 @@ class UnifiedRadixCache(BasePrefixCache):
)
if new_indices.numel() == 0:
return (
self.tree_core.empty_match_result.device_indices,
self.tree_core.empty_device_indices,
last_best_match_device_node_id,
)
@@ -3484,7 +3484,7 @@ class UnifiedRadixCache(BasePrefixCache):
return new_indices, best_match_node_id
return (
self.tree_core.empty_match_result.device_indices,
self.tree_core.empty_device_indices,
last_best_match_device_node_id,
)
@@ -3801,11 +3801,8 @@ class UnifiedRadixCache(BasePrefixCache):
# Internal callers (and the session sentinel / None) pass a non-int through.
return node_handle
def prefix_device_indices(self, req: Req) -> torch.Tensor:
root = self.root_node_handle(req.extra_key)
path = self.tree_core.collect_full_device_indices(req.last_node, root)
assert len(path) >= req.prefix_len, (req.rid, len(path), req.prefix_len)
return path[: req.prefix_len]
def path_device_indices(self, node: NodeId) -> torch.Tensor:
return self.tree_core.collect_full_device_indices(node, self.root_node_handle())
def root_node_handle(self, extra_key: Optional[str] = None) -> NodeId:
"""The root's NodeId -- URC match results carry NodeIds."""
@@ -4,8 +4,6 @@ import logging
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, Dict, Optional
import torch
from sglang.srt.managers.schedule_batch import ReqKvInfo
from sglang.srt.mem_cache.base_prefix_cache import (
MatchPrefixParams,
@@ -138,12 +136,8 @@ class StreamingSession:
self._free_tail(req.kv, prefix_len)
device_indices = self.cache.req_to_token_pool.req_to_token[
req.kv.req_pool_idx, :prefix_len
].to(dtype=torch.int64)
return MatchResult(
device_indices=device_indices,
device_prefix_len=prefix_len,
last_device_node=slot.virtual_node,
last_host_node=slot.virtual_node,
best_match_node=slot.virtual_node,