[MemCache] Return only the matched length from match_prefix; read KV indices off the node path (#42923)
This commit is contained in:
@@ -69,7 +69,7 @@ class DecodeHiCachePreallocMixin:
|
||||
Performs the optional L3 storage hit length query when decode-side
|
||||
HiCache is enabled and the last host node is backed up.
|
||||
"""
|
||||
prefix_indices = result.device_indices
|
||||
prefix_indices = self.tree_cache.prefix_device_indices(req)
|
||||
l1_prefix_len = len(prefix_indices)
|
||||
l2_host_hit_length = result.host_hit_length
|
||||
|
||||
@@ -219,6 +219,7 @@ class DecodeHiCacheTransferMixin:
|
||||
cow_mamba=False,
|
||||
max_prefix_len=pm.decode_prefix_len,
|
||||
)
|
||||
rematched_indices = self.tree_cache.prefix_device_indices(dr.req)
|
||||
new_indices, restored_node = self.tree_cache.init_load_back(
|
||||
InitLoadBackParams(
|
||||
best_match_node=rematch.best_match_node,
|
||||
@@ -230,12 +231,12 @@ class DecodeHiCacheTransferMixin:
|
||||
# boundary; point it back at the prefix the prealloc matched and locked.
|
||||
dr.req.last_node = pm.last_device_node
|
||||
# Failback: total coverage < required prefix means device alloc likely failed.
|
||||
if len(rematch.device_indices) + len(new_indices) < pm.decode_prefix_len:
|
||||
if rematch.device_prefix_len + len(new_indices) < pm.decode_prefix_len:
|
||||
logger.warning(
|
||||
"HiCache load_back failed for rid=%s: device_indices=%d, "
|
||||
"new_indices=%d, expected decode_prefix_len=%d (l1=%d, l2=%d, l3=%d)",
|
||||
dr.req.rid,
|
||||
len(rematch.device_indices),
|
||||
rematch.device_prefix_len,
|
||||
len(new_indices),
|
||||
pm.decode_prefix_len,
|
||||
pm.l1_prefix_len,
|
||||
@@ -246,7 +247,7 @@ class DecodeHiCacheTransferMixin:
|
||||
return False
|
||||
|
||||
dr.hicache_restored_kv_indices = torch.cat(
|
||||
[rematch.device_indices[pm.l1_prefix_len :], new_indices]
|
||||
[rematched_indices[pm.l1_prefix_len :], new_indices]
|
||||
)[: pm.restore_token_count]
|
||||
dr.hicache_restore_lock = self.tree_cache.lock(restored_node)
|
||||
|
||||
|
||||
@@ -159,7 +159,7 @@ class C128SidecarComponent(TreeComponent):
|
||||
# endpoint, so constrain that candidate to the nearest C128 boundary.
|
||||
group_tokens = 128 * self.allocator.c128_attn_allocator.page_size
|
||||
branching_seqlen = branching_seqlen // group_tokens * group_tokens
|
||||
current_boundary = len(result.device_indices) + result.host_hit_length
|
||||
current_boundary = result.device_prefix_len + result.host_hit_length
|
||||
return result._replace(
|
||||
swa_branching_seqlen=(
|
||||
branching_seqlen if branching_seqlen > current_boundary else None
|
||||
|
||||
@@ -343,9 +343,8 @@ class SchedulePolicy:
|
||||
match_result = zero_match_result(
|
||||
self.waiting_queue_radix_tree, match_result, extra_key=extra_key
|
||||
)
|
||||
in_batch_matching_prefixes = match_result.device_indices
|
||||
if (
|
||||
len(in_batch_matching_prefixes)
|
||||
match_result.device_prefix_len
|
||||
>= IN_BATCH_PREFIX_CACHING_DEPRIORITIZE_THRESHOLD
|
||||
):
|
||||
temporary_deprioritized.add(r.rid)
|
||||
|
||||
@@ -271,7 +271,11 @@ class MatchResult(NamedTuple):
|
||||
"""Result of a prefix match operation.
|
||||
|
||||
Attributes:
|
||||
device_indices : Indices of the KV cache on the device matched by common prefix.
|
||||
device_prefix_len: Length of the device-resident matched prefix. Its KV
|
||||
indices lie on the path to ``last_device_node``, except
|
||||
slots LMCache loaded but has not published (which
|
||||
``prefix_device_indices`` appends) and a streaming
|
||||
session's match, which lives in the lent row.
|
||||
last_device_node: The last TreeNode on the device that was matched.
|
||||
last_host_node : The last TreeNode on the host that was matched.
|
||||
Note that if HiCache is not enabled,
|
||||
@@ -299,7 +303,7 @@ class MatchResult(NamedTuple):
|
||||
host, independent of other components.
|
||||
"""
|
||||
|
||||
device_indices: torch.Tensor
|
||||
device_prefix_len: int
|
||||
last_device_node: Any
|
||||
last_host_node: Any
|
||||
best_match_node: Any
|
||||
@@ -322,9 +326,7 @@ def zero_match_result(
|
||||
return match_result
|
||||
root = tree_cache.root_node_handle(extra_key=extra_key)
|
||||
return match_result._replace(
|
||||
# [:0] keeps dtype and device of the original tensor (e.g. CUDA int64)
|
||||
# without allocating a fresh empty tensor.
|
||||
device_indices=match_result.device_indices[:0],
|
||||
device_prefix_len=0,
|
||||
last_device_node=root,
|
||||
last_host_node=root,
|
||||
best_match_node=root,
|
||||
@@ -528,10 +530,17 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
|
||||
if lock is not None:
|
||||
self.dec_lock_ref(lock.node, lock.receipt)
|
||||
|
||||
def path_device_indices(self, node: Any) -> torch.Tensor:
|
||||
"""Device KV indices on the path from the root to ``node``; valid until
|
||||
the next allocation or eviction unless the path is locked."""
|
||||
raise NotImplementedError
|
||||
|
||||
def prefix_device_indices(self, req: Req) -> torch.Tensor:
|
||||
"""KV indices of req's matched prefix, read off the path to its locked
|
||||
match node; valid from match until allocation writes them into the row."""
|
||||
raise NotImplementedError
|
||||
path = self.path_device_indices(req.last_node)
|
||||
assert len(path) >= req.prefix_len, (req.rid, len(path), req.prefix_len)
|
||||
return path[: req.prefix_len]
|
||||
|
||||
def maybe_hand_to_session(self, req: Req) -> None:
|
||||
"""A cache that keeps records across requests (a streaming session) takes
|
||||
|
||||
@@ -1193,7 +1193,7 @@ class BufferModePipeline:
|
||||
cache_salt=span_key.cache_salt,
|
||||
)
|
||||
match = self._cache.match_prefix(MatchPrefixParams(key=key))
|
||||
return len(match.device_indices) >= len(key)
|
||||
return match.device_prefix_len >= len(key)
|
||||
|
||||
def set_prefix_ctx(
|
||||
self,
|
||||
@@ -1391,7 +1391,7 @@ class BufferModePipeline:
|
||||
req = params.req
|
||||
assert req is not None
|
||||
request = req.cache_request_handle
|
||||
empty = cache.tree_core.empty_match_result.device_indices
|
||||
empty = cache.tree_core.empty_device_indices
|
||||
unchanged = (empty, req.last_node)
|
||||
f = self.staged_prefetches.get(request)
|
||||
if f is None:
|
||||
@@ -1632,10 +1632,12 @@ class BufferModePipeline:
|
||||
aux_device_releases=aux_device_releases,
|
||||
)
|
||||
match = cache.match_prefix(MatchPrefixParams(key=key))
|
||||
canonical = cache.path_device_indices(match.last_device_node)[
|
||||
splice_base:span_end
|
||||
]
|
||||
self.release_anchor_lock(request)
|
||||
canonical = match.device_indices[splice_base:span_end]
|
||||
owned = len(match.device_indices) >= span_end and torch.equal(
|
||||
match.device_indices[splice_base:span_end], device_indices
|
||||
owned = match.device_prefix_len >= span_end and torch.equal(
|
||||
canonical, device_indices
|
||||
)
|
||||
if not owned:
|
||||
# Fail-stop: the insert freed or replaced slots the in-flight H2D
|
||||
@@ -1644,7 +1646,7 @@ class BufferModePipeline:
|
||||
"HiCache buffer load-back ownership violation "
|
||||
f"req={f.request.rid}: "
|
||||
f"insert prefix_len={insert_result.prefix_len} "
|
||||
f"expected={splice_base}, matched={len(match.device_indices)} "
|
||||
f"expected={splice_base}, matched={match.device_prefix_len} "
|
||||
f"span_end={span_end} splice_base={splice_base}; "
|
||||
f"in-flight H2D targets freed slots"
|
||||
)
|
||||
|
||||
@@ -66,12 +66,15 @@ class ChunkCache(BasePrefixCache):
|
||||
|
||||
def match_prefix(self, params: MatchPrefixParams) -> MatchResult:
|
||||
return MatchResult(
|
||||
device_indices=torch.empty((0,), dtype=torch.int64),
|
||||
device_prefix_len=0,
|
||||
last_device_node=None,
|
||||
last_host_node=None,
|
||||
best_match_node=None,
|
||||
)
|
||||
|
||||
def path_device_indices(self, node: Any) -> torch.Tensor:
|
||||
return torch.empty((0,), dtype=torch.int64)
|
||||
|
||||
def insert(self, params: InsertParams) -> InsertResult:
|
||||
# ChunkCache does not support prefix caching, so insert is a no-op
|
||||
return InsertResult(prefix_len=0)
|
||||
|
||||
@@ -209,7 +209,7 @@ def match_kv_cache(
|
||||
match_result = zero_match_result(
|
||||
tree_cache, match_result, extra_key=req.extra_key
|
||||
)
|
||||
req.prefix_len = len(match_result.device_indices)
|
||||
req.prefix_len = match_result.device_prefix_len
|
||||
(
|
||||
req.last_node,
|
||||
req.last_host_node,
|
||||
|
||||
@@ -340,12 +340,11 @@ class RadixCache(BasePrefixCache):
|
||||
self.evictable_size_ = 0
|
||||
self.protected_size_ = 0
|
||||
self.evictable_leaves.clear()
|
||||
self._empty_device_indices = torch.empty(
|
||||
(0,), dtype=torch.int64, device=self.device
|
||||
)
|
||||
self._empty_match_result = MatchResult(
|
||||
device_indices=torch.empty(
|
||||
(0,),
|
||||
dtype=torch.int64,
|
||||
device=self.device,
|
||||
),
|
||||
device_prefix_len=0,
|
||||
last_device_node=self.root_node,
|
||||
last_host_node=self.root_node,
|
||||
best_match_node=self.root_node,
|
||||
@@ -374,9 +373,8 @@ class RadixCache(BasePrefixCache):
|
||||
empty result with the root as the last node.
|
||||
|
||||
Returns:
|
||||
MatchResult: ``device_indices`` is a 1-D ``torch.int64`` tensor of
|
||||
the concatenated KV cache indices corresponding to the longest
|
||||
cached prefix (may be length 0).
|
||||
MatchResult: ``device_prefix_len`` is the length of the longest
|
||||
cached prefix (may be 0).
|
||||
``last_device_node`` and ``last_host_node`` (currently the same) are the tree node objects
|
||||
representing the terminal node of the matched prefix. This method
|
||||
may mutate internal structure by splitting an existing node if the
|
||||
@@ -401,12 +399,8 @@ class RadixCache(BasePrefixCache):
|
||||
return self._empty_match_result
|
||||
|
||||
value, last_node = self._match_prefix_helper(self.root_node, key)
|
||||
if value:
|
||||
value = torch.cat(value)
|
||||
else:
|
||||
value = self._empty_match_result.device_indices
|
||||
return MatchResult(
|
||||
device_indices=value,
|
||||
device_prefix_len=sum(len(v) for v in value),
|
||||
last_device_node=last_node,
|
||||
last_host_node=last_node,
|
||||
best_match_node=last_node,
|
||||
@@ -509,10 +503,8 @@ class RadixCache(BasePrefixCache):
|
||||
|
||||
# The prefix indices could be updated, reuse it
|
||||
match_result = self.match_prefix(MatchPrefixParams(key=radix_key))
|
||||
new_indices, new_last_node = (
|
||||
match_result.device_indices,
|
||||
match_result.last_device_node,
|
||||
)
|
||||
new_last_node = match_result.last_device_node
|
||||
new_indices = self.path_device_indices(new_last_node)
|
||||
assert len(new_indices) == len(radix_key), (
|
||||
f"{len(new_indices)=}, {len(radix_key)=}"
|
||||
)
|
||||
@@ -624,16 +616,15 @@ class RadixCache(BasePrefixCache):
|
||||
|
||||
##### Internal Helper Functions #####
|
||||
|
||||
def prefix_device_indices(self, req: Req) -> torch.Tensor:
|
||||
def path_device_indices(self, node: TreeNode) -> torch.Tensor:
|
||||
values = []
|
||||
node = req.last_node
|
||||
while node is not self.root_node:
|
||||
values.append(node.value)
|
||||
node = node.parent
|
||||
if not values:
|
||||
return self._empty_device_indices
|
||||
values.reverse()
|
||||
path = torch.cat(values)
|
||||
assert len(path) >= req.prefix_len, (req.rid, len(path), req.prefix_len)
|
||||
return path[: req.prefix_len]
|
||||
return torch.cat(values)
|
||||
|
||||
def _match_prefix_helper(self, node: TreeNode, key: RadixKey):
|
||||
access_time = time.monotonic()
|
||||
|
||||
@@ -267,7 +267,7 @@ def _insert_step_from_binding(step) -> InsertStepResult:
|
||||
def _match_result_from_binding(result) -> MatchResult:
|
||||
"""Build the Python MatchResult for the binding's match result."""
|
||||
return MatchResult(
|
||||
device_indices=result.device_indices,
|
||||
device_prefix_len=result.device_prefix_len,
|
||||
last_device_node=result.last_device_node_id,
|
||||
last_host_node=result.last_host_node_id,
|
||||
best_match_node=result.best_match_node_id,
|
||||
@@ -440,6 +440,9 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
self._empty_match_result = _match_result_from_binding(
|
||||
self._binding.empty_match_result()
|
||||
)
|
||||
self._empty_device_indices = torch.empty(
|
||||
(0,), dtype=torch.int64, device=self.device
|
||||
)
|
||||
|
||||
def _binding_class(self) -> type:
|
||||
"""The extension binding class this core constructs."""
|
||||
@@ -717,6 +720,10 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
def empty_match_result(self) -> MatchResult:
|
||||
return self._empty_match_result
|
||||
|
||||
@property
|
||||
def empty_device_indices(self) -> torch.Tensor:
|
||||
return self._empty_device_indices
|
||||
|
||||
def is_full_device_evicted(self, node_id: NodeId) -> bool:
|
||||
return self._binding.is_full_device_evicted(node_id)
|
||||
|
||||
|
||||
@@ -174,29 +174,25 @@ class FlexKVRadixCache(RadixCache):
|
||||
if len(key) == 0:
|
||||
return base_res
|
||||
|
||||
device_value: torch.Tensor = base_res.device_indices
|
||||
last_node: TreeNode = base_res.last_device_node
|
||||
|
||||
if self._mode is FlexKVMode.MP:
|
||||
if params.req is None:
|
||||
return base_res
|
||||
return self._mp_match_prefix(
|
||||
key, base_res, device_value, last_node, params.req
|
||||
)
|
||||
return self._ip_match_prefix(key, base_res, device_value, last_node)
|
||||
return self._mp_match_prefix(key, base_res, last_node, params.req)
|
||||
return self._ip_match_prefix(key, base_res, last_node)
|
||||
|
||||
def _mp_match_prefix(
|
||||
self,
|
||||
key: RadixKey,
|
||||
base_res: MatchResult,
|
||||
device_value: torch.Tensor,
|
||||
last_node: TreeNode,
|
||||
req: Req,
|
||||
) -> MatchResult:
|
||||
"""LOOKUP-only path. Sets ``host_hit_length`` on the result so
|
||||
the scheduler later invokes :meth:`init_load_back`."""
|
||||
token_ids = key.raw_token_ids()
|
||||
device_len = int(device_value.numel())
|
||||
device_len = base_res.device_prefix_len
|
||||
if device_len >= len(token_ids):
|
||||
return base_res
|
||||
|
||||
@@ -226,7 +222,7 @@ class FlexKVRadixCache(RadixCache):
|
||||
value_numel=device_len,
|
||||
)
|
||||
return MatchResult(
|
||||
device_indices=device_value,
|
||||
device_prefix_len=device_len,
|
||||
last_device_node=last_node,
|
||||
last_host_node=last_node,
|
||||
best_match_node=last_node,
|
||||
@@ -237,13 +233,12 @@ class FlexKVRadixCache(RadixCache):
|
||||
self,
|
||||
key: RadixKey,
|
||||
base_res: MatchResult,
|
||||
device_value: torch.Tensor,
|
||||
last_node: TreeNode,
|
||||
) -> MatchResult:
|
||||
"""Layerwise path: allocate slots and fire ``start_load_kv_layerwise``
|
||||
immediately. Per-layer hook waits during forward."""
|
||||
token_ids = key.raw_token_ids()
|
||||
device_len = int(device_value.numel())
|
||||
device_len = base_res.device_prefix_len
|
||||
if device_len >= len(token_ids):
|
||||
return base_res
|
||||
|
||||
@@ -271,7 +266,7 @@ class FlexKVRadixCache(RadixCache):
|
||||
return base_res
|
||||
new_slots, new_node = result
|
||||
return MatchResult(
|
||||
device_indices=torch.cat([device_value, new_slots]),
|
||||
device_prefix_len=device_len + len(new_slots),
|
||||
last_device_node=new_node,
|
||||
last_host_node=new_node,
|
||||
best_match_node=new_node,
|
||||
|
||||
@@ -90,8 +90,7 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
|
||||
def prefix_device_indices(self, req: Req) -> torch.Tensor:
|
||||
# match_prefix may append loaded slots not yet published to the tree,
|
||||
# so the path alone can be shorter than the prefix.
|
||||
root = self.root_node_handle(req.extra_key)
|
||||
path = self.tree_core.collect_full_device_indices(req.last_node, root)
|
||||
path = self.path_device_indices(req.last_node)
|
||||
indices = path[: req.prefix_len]
|
||||
missing = req.prefix_len - len(indices)
|
||||
if missing > 0:
|
||||
@@ -129,7 +128,7 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
|
||||
return result
|
||||
|
||||
total_hit = min(flow.total_hit, requested_key_len, len(flow.key))
|
||||
local_hit = len(result.device_indices)
|
||||
local_hit = result.device_prefix_len
|
||||
skip = 0
|
||||
if flow.load is not None and not flow.prefix_published:
|
||||
# Track private slots shadowed by a concurrent tree insertion.
|
||||
@@ -165,7 +164,7 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
|
||||
# Reuse an already submitted H2D load on the next admission attempt.
|
||||
suffix = flow.load.device_indices[skip : skip + max(total_hit - local_hit, 0)]
|
||||
return result._replace(
|
||||
device_indices=torch.cat([result.device_indices, suffix]),
|
||||
device_prefix_len=local_hit + len(suffix),
|
||||
last_host_node=result.last_device_node,
|
||||
best_match_node=result.last_device_node,
|
||||
host_hit_length=0,
|
||||
@@ -232,7 +231,7 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
|
||||
return False
|
||||
latest = super().match_prefix(MatchPrefixParams(key=flow.key))
|
||||
local_hit = torch.tensor(
|
||||
[len(latest.device_indices)], dtype=torch.int64, device="cpu"
|
||||
[latest.device_prefix_len], dtype=torch.int64, device="cpu"
|
||||
)
|
||||
self.lmcache_connector.parallel_all_reduce(
|
||||
local_hit, torch.distributed.ReduceOp.MIN
|
||||
@@ -358,13 +357,13 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
|
||||
req = params.req
|
||||
if req is None:
|
||||
return (
|
||||
self.tree_core.empty_match_result.device_indices,
|
||||
self.tree_core.empty_device_indices,
|
||||
params.best_match_node,
|
||||
)
|
||||
flow = self._external_flows.get(req.rid)
|
||||
if flow is None or flow.total_hit is None or flow.load is not None:
|
||||
return (
|
||||
self.tree_core.empty_match_result.device_indices,
|
||||
self.tree_core.empty_device_indices,
|
||||
params.best_match_node,
|
||||
)
|
||||
|
||||
@@ -381,7 +380,7 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
|
||||
# Retire the unloaded flow to release its lookup locks.
|
||||
self._retire_loaded_flow(req.rid)
|
||||
return (
|
||||
self.tree_core.empty_match_result.device_indices,
|
||||
self.tree_core.empty_device_indices,
|
||||
params.best_match_node,
|
||||
)
|
||||
return device_indices, params.best_match_node
|
||||
@@ -538,7 +537,7 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
|
||||
latest = super().match_prefix(MatchPrefixParams(key=flow.key[:local_hit]))
|
||||
total_hit = min(flow.total_hit, len(flow.key))
|
||||
if total_hit <= local_hit:
|
||||
return self.tree_core.empty_match_result.device_indices
|
||||
return self.tree_core.empty_device_indices
|
||||
|
||||
# Pin the common L1 boundary before allocation can trigger eviction.
|
||||
flow.anchor_node = latest.last_device_node
|
||||
@@ -833,13 +832,13 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
|
||||
matched = super().match_prefix(
|
||||
MatchPrefixParams(key=key, req=req, cow_mamba=False)
|
||||
)
|
||||
if len(matched.device_indices) < total_hit:
|
||||
if matched.device_prefix_len < total_hit:
|
||||
raise RuntimeError(
|
||||
"LMCache loaded prefix was inserted but is not reusable across "
|
||||
f"all UnifiedRadixCache components: {len(matched.device_indices)}"
|
||||
f"all UnifiedRadixCache components: {matched.device_prefix_len}"
|
||||
f"/{total_hit} tokens for request {req.rid}"
|
||||
)
|
||||
canonical = matched.device_indices[:total_hit]
|
||||
canonical = self.path_device_indices(matched.last_device_node)[:total_hit]
|
||||
self.req_to_token_pool.write(
|
||||
(req.kv.req_pool_idx, slice(prev_prefix_len, total_hit)),
|
||||
canonical[prev_prefix_len:],
|
||||
@@ -873,7 +872,7 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
|
||||
matched = super().match_prefix(MatchPrefixParams(key=key))
|
||||
# Store the currently resident SWA prefix; later callbacks may extend it.
|
||||
resident_len = torch.tensor(
|
||||
[min(len(matched.device_indices), len(key))],
|
||||
[min(matched.device_prefix_len, len(key))],
|
||||
dtype=torch.int64,
|
||||
device="cpu",
|
||||
)
|
||||
@@ -907,7 +906,9 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
|
||||
key = key[:resident_len]
|
||||
# Re-match so the lock and Mamba state use the shortened boundary.
|
||||
matched = super().match_prefix(MatchPrefixParams(key=key))
|
||||
store_indices = matched.device_indices[store_start : len(key)]
|
||||
store_indices = self.path_device_indices(matched.last_device_node)[
|
||||
store_start : len(key)
|
||||
]
|
||||
if len(store_indices) != len(key) - store_start:
|
||||
logger.debug(
|
||||
"LMCache store skipped for %s: store indices length: %d, "
|
||||
|
||||
@@ -99,7 +99,7 @@ Find the longest cached prefix for a token sequence.
|
||||
|--------|--------|
|
||||
| **Purpose** | Walk the radix tree to find the longest prefix where **all** component validators pass |
|
||||
| **Inputs** | `params.key: RadixKey` — token IDs + optional extra key for namespace isolation |
|
||||
| **Output** | `MatchResult(device_indices, last_device_node, last_host_node, best_match_node, host_hit_length, mamba_branching_seqlen, ...)` |
|
||||
| **Output** | `MatchResult(device_prefix_len, last_device_node, last_host_node, best_match_node, host_hit_length, mamba_branching_seqlen, ...)` |
|
||||
| **Mutation** | Updates `last_access_time` on matched path; promotes matched nodes to MRU in all component LRU lists; may trigger `_split_node` if match ends mid-node |
|
||||
| **Complexity** | **O(K + D·C)** |
|
||||
|
||||
|
||||
@@ -165,7 +165,7 @@ class MambaComponent(TreeComponent):
|
||||
) -> MatchResult:
|
||||
last_node = result.best_match_node
|
||||
|
||||
mamba_boundary_len = len(result.device_indices) + result.host_hit_length
|
||||
mamba_boundary_len = result.device_prefix_len + result.host_hit_length
|
||||
|
||||
# Full KV may extend beyond the latest reusable Mamba state. The branching
|
||||
# point is the last Mamba-cache-chunk-aligned position within the Full-KV hit
|
||||
|
||||
@@ -356,7 +356,7 @@ class SWAComponent(TreeComponent):
|
||||
best_value_len: int,
|
||||
) -> MatchResult:
|
||||
ct = self.component_type
|
||||
swa_boundary_len = len(result.device_indices) + result.host_hit_length
|
||||
swa_boundary_len = result.device_prefix_len + result.host_hit_length
|
||||
|
||||
# Full KV may extend beyond the latest reusable SWA window. The branching
|
||||
# point is the last page-aligned position within the Full-KV hit that lies
|
||||
|
||||
@@ -193,7 +193,7 @@ class UnifiedCacheLinkerWrapper:
|
||||
cache = self.cache
|
||||
key, _ = key.maybe_to_bigram_view(cache.tree_core.is_eagle)
|
||||
page = cache.page_size
|
||||
device_hit_len = int(result.device_indices.numel())
|
||||
device_hit_len = result.device_prefix_len
|
||||
if device_hit_len >= len(key):
|
||||
return result
|
||||
|
||||
@@ -289,7 +289,7 @@ class UnifiedCacheLinkerWrapper:
|
||||
|
||||
def load_back(self, req: Req) -> tuple[torch.Tensor, NodeId]:
|
||||
cache = self.cache
|
||||
empty_indices = cache.tree_core.empty_match_result.device_indices
|
||||
empty_indices = cache.tree_core.empty_device_indices
|
||||
hit = self.hit_markers.pop(req.rid, None)
|
||||
if hit is None:
|
||||
return empty_indices, req.last_node
|
||||
|
||||
@@ -525,12 +525,11 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
# Rolling digest of reclaim victim ids, cross-checked across TP ranks.
|
||||
self.write_back_duplicate_reclaim_digest: int = 0
|
||||
|
||||
self._empty_device_indices = torch.empty(
|
||||
(0,), dtype=torch.int64, device=self.device
|
||||
)
|
||||
self._empty_match_result = MatchResult(
|
||||
device_indices=torch.empty(
|
||||
(0,),
|
||||
dtype=torch.int64,
|
||||
device=self.device,
|
||||
),
|
||||
device_prefix_len=0,
|
||||
last_device_node=self.root_node.id,
|
||||
last_host_node=self.root_node.id,
|
||||
best_match_node=self.root_node.id,
|
||||
@@ -1006,12 +1005,8 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
best_match_node if self.enable_hicache else best_match_device_node
|
||||
)
|
||||
|
||||
if best_match_device_value_len > 0:
|
||||
device_indices = torch.cat(value[:best_match_device_value_len])
|
||||
else:
|
||||
device_indices = self._empty_match_result.device_indices
|
||||
result = MatchResult(
|
||||
device_indices=device_indices,
|
||||
device_prefix_len=sum(len(v) for v in value[:best_match_device_value_len]),
|
||||
last_device_node=best_match_device_node,
|
||||
last_host_node=last_host_node,
|
||||
best_match_node=best_match_node,
|
||||
@@ -1036,9 +1031,13 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
|
||||
@property
|
||||
def empty_match_result(self) -> MatchResult:
|
||||
"""A shared empty MatchResult (empty device indices + boundary NodeIds)."""
|
||||
"""A shared empty MatchResult (zero device prefix + boundary NodeIds)."""
|
||||
return self._empty_match_result
|
||||
|
||||
@property
|
||||
def empty_device_indices(self) -> torch.Tensor:
|
||||
return self._empty_device_indices
|
||||
|
||||
def is_full_device_evicted(self, node_id: NodeId) -> bool:
|
||||
"""Whether the node's FULL device value has been evicted."""
|
||||
return self.node_by_id(node_id).evicted
|
||||
@@ -1057,7 +1056,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
prefix_chunks.append(value)
|
||||
node = node.parent
|
||||
if not prefix_chunks:
|
||||
return self._empty_match_result.device_indices
|
||||
return self._empty_device_indices
|
||||
prefix_chunks.reverse()
|
||||
return torch.cat(prefix_chunks)
|
||||
|
||||
|
||||
@@ -438,7 +438,7 @@ class UnifiedTreeCoreInterface(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def match_prefix(self, params: MatchPrefixParams) -> MatchResult:
|
||||
"""Match a key against the tree; returns device indices + boundary NodeIds."""
|
||||
"""Match a key; returns the device prefix length + boundary NodeIds."""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
@@ -463,7 +463,13 @@ class UnifiedTreeCoreInterface(ABC):
|
||||
@property
|
||||
@abstractmethod
|
||||
def empty_match_result(self) -> MatchResult:
|
||||
"""A shared empty MatchResult (empty device indices + boundary NodeIds)."""
|
||||
"""A shared empty MatchResult (zero device prefix + boundary NodeIds)."""
|
||||
...
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def empty_device_indices(self) -> torch.Tensor:
|
||||
"""A shared empty device-index tensor on the core's device."""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
|
||||
@@ -1201,8 +1201,8 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
# page-aligned boundary, so the normal match remains safe to repoint.
|
||||
# The tree's own walk: a session slot must not answer for the insert.
|
||||
match_result = self._match_tree(MatchPrefixParams(key=radix_key, req=req))
|
||||
new_indices = match_result.device_indices
|
||||
new_last_node = match_result.last_device_node
|
||||
new_indices = self.path_device_indices(new_last_node)
|
||||
new_prefix_len = result.prefix_len
|
||||
assert req.kv.cache_protected_len <= len(new_indices) + self.page_size - 1, (
|
||||
f"{req.kv.cache_protected_len=}, {len(new_indices)=}, {page_aligned_len=}"
|
||||
@@ -3472,7 +3472,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
)
|
||||
if new_indices.numel() == 0:
|
||||
return (
|
||||
self.tree_core.empty_match_result.device_indices,
|
||||
self.tree_core.empty_device_indices,
|
||||
last_best_match_device_node_id,
|
||||
)
|
||||
|
||||
@@ -3484,7 +3484,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
return new_indices, best_match_node_id
|
||||
|
||||
return (
|
||||
self.tree_core.empty_match_result.device_indices,
|
||||
self.tree_core.empty_device_indices,
|
||||
last_best_match_device_node_id,
|
||||
)
|
||||
|
||||
@@ -3801,11 +3801,8 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
# Internal callers (and the session sentinel / None) pass a non-int through.
|
||||
return node_handle
|
||||
|
||||
def prefix_device_indices(self, req: Req) -> torch.Tensor:
|
||||
root = self.root_node_handle(req.extra_key)
|
||||
path = self.tree_core.collect_full_device_indices(req.last_node, root)
|
||||
assert len(path) >= req.prefix_len, (req.rid, len(path), req.prefix_len)
|
||||
return path[: req.prefix_len]
|
||||
def path_device_indices(self, node: NodeId) -> torch.Tensor:
|
||||
return self.tree_core.collect_full_device_indices(node, self.root_node_handle())
|
||||
|
||||
def root_node_handle(self, extra_key: Optional[str] = None) -> NodeId:
|
||||
"""The root's NodeId -- URC match results carry NodeIds."""
|
||||
|
||||
@@ -4,8 +4,6 @@ import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Any, Dict, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.managers.schedule_batch import ReqKvInfo
|
||||
from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
MatchPrefixParams,
|
||||
@@ -138,12 +136,8 @@ class StreamingSession:
|
||||
|
||||
self._free_tail(req.kv, prefix_len)
|
||||
|
||||
device_indices = self.cache.req_to_token_pool.req_to_token[
|
||||
req.kv.req_pool_idx, :prefix_len
|
||||
].to(dtype=torch.int64)
|
||||
|
||||
return MatchResult(
|
||||
device_indices=device_indices,
|
||||
device_prefix_len=prefix_len,
|
||||
last_device_node=slot.virtual_node,
|
||||
last_host_node=slot.virtual_node,
|
||||
best_match_node=slot.virtual_node,
|
||||
|
||||
Reference in New Issue
Block a user