From 5a4d01a3ae631ff77ffc393655b0bb71e86bd38f Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Thu, 8 Oct 2026 01:10:14 -0700 Subject: [PATCH] [MemCache] Return only the matched length from `match_prefix`; read KV indices off the node path (#42923) --- .../disaggregation/decode_hicache_mixin.py | 9 +- .../npu/dsv4/c128_sidecar_component.py | 2 +- python/sglang/srt/managers/schedule_policy.py | 3 +- .../sglang/srt/mem_cache/base_prefix_cache.py | 21 +- .../srt/mem_cache/buffer_mode/pipeline.py | 14 +- python/sglang/srt/mem_cache/chunk_cache.py | 5 +- python/sglang/srt/mem_cache/common.py | 2 +- python/sglang/srt/mem_cache/radix_cache.py | 35 +-- .../srt/mem_cache/rust_tree_core/adapter.py | 9 +- .../storage/flexkv/flexkv_radix_cache.py | 17 +- .../lmcache/lmcache_unified_radix_cache.py | 29 +- .../unified_cache/components/README.md | 2 +- .../unified_cache/components/mamba.py | 2 +- .../mem_cache/unified_cache/components/swa.py | 2 +- .../unified_cache/unified_cache_linker.py | 4 +- .../unified_cache/unified_tree_core.py | 23 +- .../unified_tree_core_interface.py | 10 +- .../srt/mem_cache/unified_radix_cache.py | 13 +- .../sglang/srt/session/streaming_session.py | 8 +- .../sglang-radix-tree/src/components/mamba.rs | 2 +- rust/sglang-radix-tree/src/components/swa.rs | 2 +- rust/sglang-radix-tree/src/python_bindings.rs | 12 +- .../src/tests/components/mamba.rs | 4 +- .../src/tests/components/swa.rs | 10 +- .../sglang-radix-tree/src/tests/test_utils.rs | 11 +- .../src/tests/unified_tree_core.rs | 165 +++-------- .../src/unified_tree_core.rs | 22 +- .../test_decode_hicache_tree_core.py | 3 +- .../mem_cache/test_decode_radix_lock_ref.py | 10 +- .../test_lmcache_component_cursors.py | 7 +- .../mem_cache/test_page_interleave_shard.py | 2 +- .../mem_cache/test_pure_swa_radix_cache.py | 2 +- .../test_radix_cache_slru_accuracy.py | 4 +- .../unit/mem_cache/test_radix_cache_unit.py | 56 ++-- .../unit/mem_cache/test_radix_force_miss.py | 13 +- .../unit/mem_cache/test_rust_tree_core.py | 12 +- .../test_rust_tree_core_integration.py | 85 +++--- .../test_session_unified_radix_cache.py | 8 +- .../test_storage_prefetch_lifecycle.py | 5 +- .../mem_cache/test_streaming_session_unit.py | 6 +- .../mem_cache/test_unified_cache_linker.py | 20 +- .../test_unified_radix_cache_bench.py | 10 +- .../test_unified_radix_cache_unittest.py | 268 ++++++++++-------- 43 files changed, 482 insertions(+), 467 deletions(-) diff --git a/python/sglang/srt/disaggregation/decode_hicache_mixin.py b/python/sglang/srt/disaggregation/decode_hicache_mixin.py index 2538f30819..459cd7e9a1 100644 --- a/python/sglang/srt/disaggregation/decode_hicache_mixin.py +++ b/python/sglang/srt/disaggregation/decode_hicache_mixin.py @@ -69,7 +69,7 @@ class DecodeHiCachePreallocMixin: Performs the optional L3 storage hit length query when decode-side HiCache is enabled and the last host node is backed up. """ - prefix_indices = result.device_indices + prefix_indices = self.tree_cache.prefix_device_indices(req) l1_prefix_len = len(prefix_indices) l2_host_hit_length = result.host_hit_length @@ -219,6 +219,7 @@ class DecodeHiCacheTransferMixin: cow_mamba=False, max_prefix_len=pm.decode_prefix_len, ) + rematched_indices = self.tree_cache.prefix_device_indices(dr.req) new_indices, restored_node = self.tree_cache.init_load_back( InitLoadBackParams( best_match_node=rematch.best_match_node, @@ -230,12 +231,12 @@ class DecodeHiCacheTransferMixin: # boundary; point it back at the prefix the prealloc matched and locked. dr.req.last_node = pm.last_device_node # Failback: total coverage < required prefix means device alloc likely failed. - if len(rematch.device_indices) + len(new_indices) < pm.decode_prefix_len: + if rematch.device_prefix_len + len(new_indices) < pm.decode_prefix_len: logger.warning( "HiCache load_back failed for rid=%s: device_indices=%d, " "new_indices=%d, expected decode_prefix_len=%d (l1=%d, l2=%d, l3=%d)", dr.req.rid, - len(rematch.device_indices), + rematch.device_prefix_len, len(new_indices), pm.decode_prefix_len, pm.l1_prefix_len, @@ -246,7 +247,7 @@ class DecodeHiCacheTransferMixin: return False dr.hicache_restored_kv_indices = torch.cat( - [rematch.device_indices[pm.l1_prefix_len :], new_indices] + [rematched_indices[pm.l1_prefix_len :], new_indices] )[: pm.restore_token_count] dr.hicache_restore_lock = self.tree_cache.lock(restored_node) diff --git a/python/sglang/srt/hardware_backend/npu/dsv4/c128_sidecar_component.py b/python/sglang/srt/hardware_backend/npu/dsv4/c128_sidecar_component.py index 1749ea1e34..a7942aef2c 100644 --- a/python/sglang/srt/hardware_backend/npu/dsv4/c128_sidecar_component.py +++ b/python/sglang/srt/hardware_backend/npu/dsv4/c128_sidecar_component.py @@ -159,7 +159,7 @@ class C128SidecarComponent(TreeComponent): # endpoint, so constrain that candidate to the nearest C128 boundary. group_tokens = 128 * self.allocator.c128_attn_allocator.page_size branching_seqlen = branching_seqlen // group_tokens * group_tokens - current_boundary = len(result.device_indices) + result.host_hit_length + current_boundary = result.device_prefix_len + result.host_hit_length return result._replace( swa_branching_seqlen=( branching_seqlen if branching_seqlen > current_boundary else None diff --git a/python/sglang/srt/managers/schedule_policy.py b/python/sglang/srt/managers/schedule_policy.py index fe41d2d675..4504dc50e4 100644 --- a/python/sglang/srt/managers/schedule_policy.py +++ b/python/sglang/srt/managers/schedule_policy.py @@ -343,9 +343,8 @@ class SchedulePolicy: match_result = zero_match_result( self.waiting_queue_radix_tree, match_result, extra_key=extra_key ) - in_batch_matching_prefixes = match_result.device_indices if ( - len(in_batch_matching_prefixes) + match_result.device_prefix_len >= IN_BATCH_PREFIX_CACHING_DEPRIORITIZE_THRESHOLD ): temporary_deprioritized.add(r.rid) diff --git a/python/sglang/srt/mem_cache/base_prefix_cache.py b/python/sglang/srt/mem_cache/base_prefix_cache.py index a871059f62..8d4f51b730 100644 --- a/python/sglang/srt/mem_cache/base_prefix_cache.py +++ b/python/sglang/srt/mem_cache/base_prefix_cache.py @@ -271,7 +271,11 @@ class MatchResult(NamedTuple): """Result of a prefix match operation. Attributes: - device_indices : Indices of the KV cache on the device matched by common prefix. + device_prefix_len: Length of the device-resident matched prefix. Its KV + indices lie on the path to ``last_device_node``, except + slots LMCache loaded but has not published (which + ``prefix_device_indices`` appends) and a streaming + session's match, which lives in the lent row. last_device_node: The last TreeNode on the device that was matched. last_host_node : The last TreeNode on the host that was matched. Note that if HiCache is not enabled, @@ -299,7 +303,7 @@ class MatchResult(NamedTuple): host, independent of other components. """ - device_indices: torch.Tensor + device_prefix_len: int last_device_node: Any last_host_node: Any best_match_node: Any @@ -322,9 +326,7 @@ def zero_match_result( return match_result root = tree_cache.root_node_handle(extra_key=extra_key) return match_result._replace( - # [:0] keeps dtype and device of the original tensor (e.g. CUDA int64) - # without allocating a fresh empty tensor. - device_indices=match_result.device_indices[:0], + device_prefix_len=0, last_device_node=root, last_host_node=root, best_match_node=root, @@ -528,10 +530,17 @@ class BasePrefixCache(ABC, PrefixCacheTrait): if lock is not None: self.dec_lock_ref(lock.node, lock.receipt) + def path_device_indices(self, node: Any) -> torch.Tensor: + """Device KV indices on the path from the root to ``node``; valid until + the next allocation or eviction unless the path is locked.""" + raise NotImplementedError + def prefix_device_indices(self, req: Req) -> torch.Tensor: """KV indices of req's matched prefix, read off the path to its locked match node; valid from match until allocation writes them into the row.""" - raise NotImplementedError + path = self.path_device_indices(req.last_node) + assert len(path) >= req.prefix_len, (req.rid, len(path), req.prefix_len) + return path[: req.prefix_len] def maybe_hand_to_session(self, req: Req) -> None: """A cache that keeps records across requests (a streaming session) takes diff --git a/python/sglang/srt/mem_cache/buffer_mode/pipeline.py b/python/sglang/srt/mem_cache/buffer_mode/pipeline.py index 5c378616e4..4b38884c11 100644 --- a/python/sglang/srt/mem_cache/buffer_mode/pipeline.py +++ b/python/sglang/srt/mem_cache/buffer_mode/pipeline.py @@ -1193,7 +1193,7 @@ class BufferModePipeline: cache_salt=span_key.cache_salt, ) match = self._cache.match_prefix(MatchPrefixParams(key=key)) - return len(match.device_indices) >= len(key) + return match.device_prefix_len >= len(key) def set_prefix_ctx( self, @@ -1391,7 +1391,7 @@ class BufferModePipeline: req = params.req assert req is not None request = req.cache_request_handle - empty = cache.tree_core.empty_match_result.device_indices + empty = cache.tree_core.empty_device_indices unchanged = (empty, req.last_node) f = self.staged_prefetches.get(request) if f is None: @@ -1632,10 +1632,12 @@ class BufferModePipeline: aux_device_releases=aux_device_releases, ) match = cache.match_prefix(MatchPrefixParams(key=key)) + canonical = cache.path_device_indices(match.last_device_node)[ + splice_base:span_end + ] self.release_anchor_lock(request) - canonical = match.device_indices[splice_base:span_end] - owned = len(match.device_indices) >= span_end and torch.equal( - match.device_indices[splice_base:span_end], device_indices + owned = match.device_prefix_len >= span_end and torch.equal( + canonical, device_indices ) if not owned: # Fail-stop: the insert freed or replaced slots the in-flight H2D @@ -1644,7 +1646,7 @@ class BufferModePipeline: "HiCache buffer load-back ownership violation " f"req={f.request.rid}: " f"insert prefix_len={insert_result.prefix_len} " - f"expected={splice_base}, matched={len(match.device_indices)} " + f"expected={splice_base}, matched={match.device_prefix_len} " f"span_end={span_end} splice_base={splice_base}; " f"in-flight H2D targets freed slots" ) diff --git a/python/sglang/srt/mem_cache/chunk_cache.py b/python/sglang/srt/mem_cache/chunk_cache.py index b1932b6b64..7fafebddc2 100644 --- a/python/sglang/srt/mem_cache/chunk_cache.py +++ b/python/sglang/srt/mem_cache/chunk_cache.py @@ -66,12 +66,15 @@ class ChunkCache(BasePrefixCache): def match_prefix(self, params: MatchPrefixParams) -> MatchResult: return MatchResult( - device_indices=torch.empty((0,), dtype=torch.int64), + device_prefix_len=0, last_device_node=None, last_host_node=None, best_match_node=None, ) + def path_device_indices(self, node: Any) -> torch.Tensor: + return torch.empty((0,), dtype=torch.int64) + def insert(self, params: InsertParams) -> InsertResult: # ChunkCache does not support prefix caching, so insert is a no-op return InsertResult(prefix_len=0) diff --git a/python/sglang/srt/mem_cache/common.py b/python/sglang/srt/mem_cache/common.py index 8f8b948ff9..b3cb527cdc 100644 --- a/python/sglang/srt/mem_cache/common.py +++ b/python/sglang/srt/mem_cache/common.py @@ -209,7 +209,7 @@ def match_kv_cache( match_result = zero_match_result( tree_cache, match_result, extra_key=req.extra_key ) - req.prefix_len = len(match_result.device_indices) + req.prefix_len = match_result.device_prefix_len ( req.last_node, req.last_host_node, diff --git a/python/sglang/srt/mem_cache/radix_cache.py b/python/sglang/srt/mem_cache/radix_cache.py index e1d25aa7c0..de60ce33f2 100644 --- a/python/sglang/srt/mem_cache/radix_cache.py +++ b/python/sglang/srt/mem_cache/radix_cache.py @@ -340,12 +340,11 @@ class RadixCache(BasePrefixCache): self.evictable_size_ = 0 self.protected_size_ = 0 self.evictable_leaves.clear() + self._empty_device_indices = torch.empty( + (0,), dtype=torch.int64, device=self.device + ) self._empty_match_result = MatchResult( - device_indices=torch.empty( - (0,), - dtype=torch.int64, - device=self.device, - ), + device_prefix_len=0, last_device_node=self.root_node, last_host_node=self.root_node, best_match_node=self.root_node, @@ -374,9 +373,8 @@ class RadixCache(BasePrefixCache): empty result with the root as the last node. Returns: - MatchResult: ``device_indices`` is a 1-D ``torch.int64`` tensor of - the concatenated KV cache indices corresponding to the longest - cached prefix (may be length 0). + MatchResult: ``device_prefix_len`` is the length of the longest + cached prefix (may be 0). ``last_device_node`` and ``last_host_node`` (currently the same) are the tree node objects representing the terminal node of the matched prefix. This method may mutate internal structure by splitting an existing node if the @@ -401,12 +399,8 @@ class RadixCache(BasePrefixCache): return self._empty_match_result value, last_node = self._match_prefix_helper(self.root_node, key) - if value: - value = torch.cat(value) - else: - value = self._empty_match_result.device_indices return MatchResult( - device_indices=value, + device_prefix_len=sum(len(v) for v in value), last_device_node=last_node, last_host_node=last_node, best_match_node=last_node, @@ -509,10 +503,8 @@ class RadixCache(BasePrefixCache): # The prefix indices could be updated, reuse it match_result = self.match_prefix(MatchPrefixParams(key=radix_key)) - new_indices, new_last_node = ( - match_result.device_indices, - match_result.last_device_node, - ) + new_last_node = match_result.last_device_node + new_indices = self.path_device_indices(new_last_node) assert len(new_indices) == len(radix_key), ( f"{len(new_indices)=}, {len(radix_key)=}" ) @@ -624,16 +616,15 @@ class RadixCache(BasePrefixCache): ##### Internal Helper Functions ##### - def prefix_device_indices(self, req: Req) -> torch.Tensor: + def path_device_indices(self, node: TreeNode) -> torch.Tensor: values = [] - node = req.last_node while node is not self.root_node: values.append(node.value) node = node.parent + if not values: + return self._empty_device_indices values.reverse() - path = torch.cat(values) - assert len(path) >= req.prefix_len, (req.rid, len(path), req.prefix_len) - return path[: req.prefix_len] + return torch.cat(values) def _match_prefix_helper(self, node: TreeNode, key: RadixKey): access_time = time.monotonic() diff --git a/python/sglang/srt/mem_cache/rust_tree_core/adapter.py b/python/sglang/srt/mem_cache/rust_tree_core/adapter.py index ae458af945..9b65ae9955 100644 --- a/python/sglang/srt/mem_cache/rust_tree_core/adapter.py +++ b/python/sglang/srt/mem_cache/rust_tree_core/adapter.py @@ -267,7 +267,7 @@ def _insert_step_from_binding(step) -> InsertStepResult: def _match_result_from_binding(result) -> MatchResult: """Build the Python MatchResult for the binding's match result.""" return MatchResult( - device_indices=result.device_indices, + device_prefix_len=result.device_prefix_len, last_device_node=result.last_device_node_id, last_host_node=result.last_host_node_id, best_match_node=result.best_match_node_id, @@ -440,6 +440,9 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): self._empty_match_result = _match_result_from_binding( self._binding.empty_match_result() ) + self._empty_device_indices = torch.empty( + (0,), dtype=torch.int64, device=self.device + ) def _binding_class(self) -> type: """The extension binding class this core constructs.""" @@ -717,6 +720,10 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): def empty_match_result(self) -> MatchResult: return self._empty_match_result + @property + def empty_device_indices(self) -> torch.Tensor: + return self._empty_device_indices + def is_full_device_evicted(self, node_id: NodeId) -> bool: return self._binding.is_full_device_evicted(node_id) diff --git a/python/sglang/srt/mem_cache/storage/flexkv/flexkv_radix_cache.py b/python/sglang/srt/mem_cache/storage/flexkv/flexkv_radix_cache.py index 216cd98f7f..acc0435e3d 100644 --- a/python/sglang/srt/mem_cache/storage/flexkv/flexkv_radix_cache.py +++ b/python/sglang/srt/mem_cache/storage/flexkv/flexkv_radix_cache.py @@ -174,29 +174,25 @@ class FlexKVRadixCache(RadixCache): if len(key) == 0: return base_res - device_value: torch.Tensor = base_res.device_indices last_node: TreeNode = base_res.last_device_node if self._mode is FlexKVMode.MP: if params.req is None: return base_res - return self._mp_match_prefix( - key, base_res, device_value, last_node, params.req - ) - return self._ip_match_prefix(key, base_res, device_value, last_node) + return self._mp_match_prefix(key, base_res, last_node, params.req) + return self._ip_match_prefix(key, base_res, last_node) def _mp_match_prefix( self, key: RadixKey, base_res: MatchResult, - device_value: torch.Tensor, last_node: TreeNode, req: Req, ) -> MatchResult: """LOOKUP-only path. Sets ``host_hit_length`` on the result so the scheduler later invokes :meth:`init_load_back`.""" token_ids = key.raw_token_ids() - device_len = int(device_value.numel()) + device_len = base_res.device_prefix_len if device_len >= len(token_ids): return base_res @@ -226,7 +222,7 @@ class FlexKVRadixCache(RadixCache): value_numel=device_len, ) return MatchResult( - device_indices=device_value, + device_prefix_len=device_len, last_device_node=last_node, last_host_node=last_node, best_match_node=last_node, @@ -237,13 +233,12 @@ class FlexKVRadixCache(RadixCache): self, key: RadixKey, base_res: MatchResult, - device_value: torch.Tensor, last_node: TreeNode, ) -> MatchResult: """Layerwise path: allocate slots and fire ``start_load_kv_layerwise`` immediately. Per-layer hook waits during forward.""" token_ids = key.raw_token_ids() - device_len = int(device_value.numel()) + device_len = base_res.device_prefix_len if device_len >= len(token_ids): return base_res @@ -271,7 +266,7 @@ class FlexKVRadixCache(RadixCache): return base_res new_slots, new_node = result return MatchResult( - device_indices=torch.cat([device_value, new_slots]), + device_prefix_len=device_len + len(new_slots), last_device_node=new_node, last_host_node=new_node, best_match_node=new_node, diff --git a/python/sglang/srt/mem_cache/storage/lmcache/lmcache_unified_radix_cache.py b/python/sglang/srt/mem_cache/storage/lmcache/lmcache_unified_radix_cache.py index 37d817cac9..a9d1fa304b 100644 --- a/python/sglang/srt/mem_cache/storage/lmcache/lmcache_unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/storage/lmcache/lmcache_unified_radix_cache.py @@ -90,8 +90,7 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache): def prefix_device_indices(self, req: Req) -> torch.Tensor: # match_prefix may append loaded slots not yet published to the tree, # so the path alone can be shorter than the prefix. - root = self.root_node_handle(req.extra_key) - path = self.tree_core.collect_full_device_indices(req.last_node, root) + path = self.path_device_indices(req.last_node) indices = path[: req.prefix_len] missing = req.prefix_len - len(indices) if missing > 0: @@ -129,7 +128,7 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache): return result total_hit = min(flow.total_hit, requested_key_len, len(flow.key)) - local_hit = len(result.device_indices) + local_hit = result.device_prefix_len skip = 0 if flow.load is not None and not flow.prefix_published: # Track private slots shadowed by a concurrent tree insertion. @@ -165,7 +164,7 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache): # Reuse an already submitted H2D load on the next admission attempt. suffix = flow.load.device_indices[skip : skip + max(total_hit - local_hit, 0)] return result._replace( - device_indices=torch.cat([result.device_indices, suffix]), + device_prefix_len=local_hit + len(suffix), last_host_node=result.last_device_node, best_match_node=result.last_device_node, host_hit_length=0, @@ -232,7 +231,7 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache): return False latest = super().match_prefix(MatchPrefixParams(key=flow.key)) local_hit = torch.tensor( - [len(latest.device_indices)], dtype=torch.int64, device="cpu" + [latest.device_prefix_len], dtype=torch.int64, device="cpu" ) self.lmcache_connector.parallel_all_reduce( local_hit, torch.distributed.ReduceOp.MIN @@ -358,13 +357,13 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache): req = params.req if req is None: return ( - self.tree_core.empty_match_result.device_indices, + self.tree_core.empty_device_indices, params.best_match_node, ) flow = self._external_flows.get(req.rid) if flow is None or flow.total_hit is None or flow.load is not None: return ( - self.tree_core.empty_match_result.device_indices, + self.tree_core.empty_device_indices, params.best_match_node, ) @@ -381,7 +380,7 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache): # Retire the unloaded flow to release its lookup locks. self._retire_loaded_flow(req.rid) return ( - self.tree_core.empty_match_result.device_indices, + self.tree_core.empty_device_indices, params.best_match_node, ) return device_indices, params.best_match_node @@ -538,7 +537,7 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache): latest = super().match_prefix(MatchPrefixParams(key=flow.key[:local_hit])) total_hit = min(flow.total_hit, len(flow.key)) if total_hit <= local_hit: - return self.tree_core.empty_match_result.device_indices + return self.tree_core.empty_device_indices # Pin the common L1 boundary before allocation can trigger eviction. flow.anchor_node = latest.last_device_node @@ -833,13 +832,13 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache): matched = super().match_prefix( MatchPrefixParams(key=key, req=req, cow_mamba=False) ) - if len(matched.device_indices) < total_hit: + if matched.device_prefix_len < total_hit: raise RuntimeError( "LMCache loaded prefix was inserted but is not reusable across " - f"all UnifiedRadixCache components: {len(matched.device_indices)}" + f"all UnifiedRadixCache components: {matched.device_prefix_len}" f"/{total_hit} tokens for request {req.rid}" ) - canonical = matched.device_indices[:total_hit] + canonical = self.path_device_indices(matched.last_device_node)[:total_hit] self.req_to_token_pool.write( (req.kv.req_pool_idx, slice(prev_prefix_len, total_hit)), canonical[prev_prefix_len:], @@ -873,7 +872,7 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache): matched = super().match_prefix(MatchPrefixParams(key=key)) # Store the currently resident SWA prefix; later callbacks may extend it. resident_len = torch.tensor( - [min(len(matched.device_indices), len(key))], + [min(matched.device_prefix_len, len(key))], dtype=torch.int64, device="cpu", ) @@ -907,7 +906,9 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache): key = key[:resident_len] # Re-match so the lock and Mamba state use the shortened boundary. matched = super().match_prefix(MatchPrefixParams(key=key)) - store_indices = matched.device_indices[store_start : len(key)] + store_indices = self.path_device_indices(matched.last_device_node)[ + store_start : len(key) + ] if len(store_indices) != len(key) - store_start: logger.debug( "LMCache store skipped for %s: store indices length: %d, " diff --git a/python/sglang/srt/mem_cache/unified_cache/components/README.md b/python/sglang/srt/mem_cache/unified_cache/components/README.md index b4a765db10..bc10dd06f3 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/README.md +++ b/python/sglang/srt/mem_cache/unified_cache/components/README.md @@ -99,7 +99,7 @@ Find the longest cached prefix for a token sequence. |--------|--------| | **Purpose** | Walk the radix tree to find the longest prefix where **all** component validators pass | | **Inputs** | `params.key: RadixKey` — token IDs + optional extra key for namespace isolation | -| **Output** | `MatchResult(device_indices, last_device_node, last_host_node, best_match_node, host_hit_length, mamba_branching_seqlen, ...)` | +| **Output** | `MatchResult(device_prefix_len, last_device_node, last_host_node, best_match_node, host_hit_length, mamba_branching_seqlen, ...)` | | **Mutation** | Updates `last_access_time` on matched path; promotes matched nodes to MRU in all component LRU lists; may trigger `_split_node` if match ends mid-node | | **Complexity** | **O(K + D·C)** | diff --git a/python/sglang/srt/mem_cache/unified_cache/components/mamba.py b/python/sglang/srt/mem_cache/unified_cache/components/mamba.py index 14e466917f..4df7314d0f 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/mamba.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/mamba.py @@ -165,7 +165,7 @@ class MambaComponent(TreeComponent): ) -> MatchResult: last_node = result.best_match_node - mamba_boundary_len = len(result.device_indices) + result.host_hit_length + mamba_boundary_len = result.device_prefix_len + result.host_hit_length # Full KV may extend beyond the latest reusable Mamba state. The branching # point is the last Mamba-cache-chunk-aligned position within the Full-KV hit diff --git a/python/sglang/srt/mem_cache/unified_cache/components/swa.py b/python/sglang/srt/mem_cache/unified_cache/components/swa.py index f2f0f267a4..d432ef3976 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/swa.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/swa.py @@ -356,7 +356,7 @@ class SWAComponent(TreeComponent): best_value_len: int, ) -> MatchResult: ct = self.component_type - swa_boundary_len = len(result.device_indices) + result.host_hit_length + swa_boundary_len = result.device_prefix_len + result.host_hit_length # Full KV may extend beyond the latest reusable SWA window. The branching # point is the last page-aligned position within the Full-KV hit that lies diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_cache_linker.py b/python/sglang/srt/mem_cache/unified_cache/unified_cache_linker.py index c3e6d213f0..150ccdf9cd 100644 --- a/python/sglang/srt/mem_cache/unified_cache/unified_cache_linker.py +++ b/python/sglang/srt/mem_cache/unified_cache/unified_cache_linker.py @@ -193,7 +193,7 @@ class UnifiedCacheLinkerWrapper: cache = self.cache key, _ = key.maybe_to_bigram_view(cache.tree_core.is_eagle) page = cache.page_size - device_hit_len = int(result.device_indices.numel()) + device_hit_len = result.device_prefix_len if device_hit_len >= len(key): return result @@ -289,7 +289,7 @@ class UnifiedCacheLinkerWrapper: def load_back(self, req: Req) -> tuple[torch.Tensor, NodeId]: cache = self.cache - empty_indices = cache.tree_core.empty_match_result.device_indices + empty_indices = cache.tree_core.empty_device_indices hit = self.hit_markers.pop(req.rid, None) if hit is None: return empty_indices, req.last_node diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py index 93d640ed10..779b91783b 100644 --- a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py +++ b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py @@ -525,12 +525,11 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): # Rolling digest of reclaim victim ids, cross-checked across TP ranks. self.write_back_duplicate_reclaim_digest: int = 0 + self._empty_device_indices = torch.empty( + (0,), dtype=torch.int64, device=self.device + ) self._empty_match_result = MatchResult( - device_indices=torch.empty( - (0,), - dtype=torch.int64, - device=self.device, - ), + device_prefix_len=0, last_device_node=self.root_node.id, last_host_node=self.root_node.id, best_match_node=self.root_node.id, @@ -1006,12 +1005,8 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): best_match_node if self.enable_hicache else best_match_device_node ) - if best_match_device_value_len > 0: - device_indices = torch.cat(value[:best_match_device_value_len]) - else: - device_indices = self._empty_match_result.device_indices result = MatchResult( - device_indices=device_indices, + device_prefix_len=sum(len(v) for v in value[:best_match_device_value_len]), last_device_node=best_match_device_node, last_host_node=last_host_node, best_match_node=best_match_node, @@ -1036,9 +1031,13 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): @property def empty_match_result(self) -> MatchResult: - """A shared empty MatchResult (empty device indices + boundary NodeIds).""" + """A shared empty MatchResult (zero device prefix + boundary NodeIds).""" return self._empty_match_result + @property + def empty_device_indices(self) -> torch.Tensor: + return self._empty_device_indices + def is_full_device_evicted(self, node_id: NodeId) -> bool: """Whether the node's FULL device value has been evicted.""" return self.node_by_id(node_id).evicted @@ -1057,7 +1056,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): prefix_chunks.append(value) node = node.parent if not prefix_chunks: - return self._empty_match_result.device_indices + return self._empty_device_indices prefix_chunks.reverse() return torch.cat(prefix_chunks) diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py index b7fdc129fe..921f671288 100644 --- a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py +++ b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py @@ -438,7 +438,7 @@ class UnifiedTreeCoreInterface(ABC): @abstractmethod def match_prefix(self, params: MatchPrefixParams) -> MatchResult: - """Match a key against the tree; returns device indices + boundary NodeIds.""" + """Match a key; returns the device prefix length + boundary NodeIds.""" ... @abstractmethod @@ -463,7 +463,13 @@ class UnifiedTreeCoreInterface(ABC): @property @abstractmethod def empty_match_result(self) -> MatchResult: - """A shared empty MatchResult (empty device indices + boundary NodeIds).""" + """A shared empty MatchResult (zero device prefix + boundary NodeIds).""" + ... + + @property + @abstractmethod + def empty_device_indices(self) -> torch.Tensor: + """A shared empty device-index tensor on the core's device.""" ... @abstractmethod diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index 6f2b9774ed..d7cd7bb4ca 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -1201,8 +1201,8 @@ class UnifiedRadixCache(BasePrefixCache): # page-aligned boundary, so the normal match remains safe to repoint. # The tree's own walk: a session slot must not answer for the insert. match_result = self._match_tree(MatchPrefixParams(key=radix_key, req=req)) - new_indices = match_result.device_indices new_last_node = match_result.last_device_node + new_indices = self.path_device_indices(new_last_node) new_prefix_len = result.prefix_len assert req.kv.cache_protected_len <= len(new_indices) + self.page_size - 1, ( f"{req.kv.cache_protected_len=}, {len(new_indices)=}, {page_aligned_len=}" @@ -3472,7 +3472,7 @@ class UnifiedRadixCache(BasePrefixCache): ) if new_indices.numel() == 0: return ( - self.tree_core.empty_match_result.device_indices, + self.tree_core.empty_device_indices, last_best_match_device_node_id, ) @@ -3484,7 +3484,7 @@ class UnifiedRadixCache(BasePrefixCache): return new_indices, best_match_node_id return ( - self.tree_core.empty_match_result.device_indices, + self.tree_core.empty_device_indices, last_best_match_device_node_id, ) @@ -3801,11 +3801,8 @@ class UnifiedRadixCache(BasePrefixCache): # Internal callers (and the session sentinel / None) pass a non-int through. return node_handle - def prefix_device_indices(self, req: Req) -> torch.Tensor: - root = self.root_node_handle(req.extra_key) - path = self.tree_core.collect_full_device_indices(req.last_node, root) - assert len(path) >= req.prefix_len, (req.rid, len(path), req.prefix_len) - return path[: req.prefix_len] + def path_device_indices(self, node: NodeId) -> torch.Tensor: + return self.tree_core.collect_full_device_indices(node, self.root_node_handle()) def root_node_handle(self, extra_key: Optional[str] = None) -> NodeId: """The root's NodeId -- URC match results carry NodeIds.""" diff --git a/python/sglang/srt/session/streaming_session.py b/python/sglang/srt/session/streaming_session.py index f93cb8a6d3..817266f62e 100644 --- a/python/sglang/srt/session/streaming_session.py +++ b/python/sglang/srt/session/streaming_session.py @@ -4,8 +4,6 @@ import logging from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Dict, Optional -import torch - from sglang.srt.managers.schedule_batch import ReqKvInfo from sglang.srt.mem_cache.base_prefix_cache import ( MatchPrefixParams, @@ -138,12 +136,8 @@ class StreamingSession: self._free_tail(req.kv, prefix_len) - device_indices = self.cache.req_to_token_pool.req_to_token[ - req.kv.req_pool_idx, :prefix_len - ].to(dtype=torch.int64) - return MatchResult( - device_indices=device_indices, + device_prefix_len=prefix_len, last_device_node=slot.virtual_node, last_host_node=slot.virtual_node, best_match_node=slot.virtual_node, diff --git a/rust/sglang-radix-tree/src/components/mamba.rs b/rust/sglang-radix-tree/src/components/mamba.rs index 9dae4cfa52..9c04131e83 100644 --- a/rust/sglang-radix-tree/src/components/mamba.rs +++ b/rust/sglang-radix-tree/src/components/mamba.rs @@ -133,7 +133,7 @@ impl TreeComponent for MambaComponent { _value_chunks: &[Tensor], _best_value_len: usize, ) -> MatchResult { - let mamba_boundary_len = result.device_indices.size()[0] as usize + result.host_hit_length; + let mamba_boundary_len = result.device_prefix_len + result.host_hit_length; // Full KV may extend beyond the latest reusable Mamba state. The branching // point is the last checkpoint-grid-aligned position within the Full-KV hit diff --git a/rust/sglang-radix-tree/src/components/swa.rs b/rust/sglang-radix-tree/src/components/swa.rs index 3bde51e79d..1c68fb8088 100644 --- a/rust/sglang-radix-tree/src/components/swa.rs +++ b/rust/sglang-radix-tree/src/components/swa.rs @@ -393,7 +393,7 @@ impl TreeComponent for SwaComponent { value_chunks: &[Tensor], best_value_len: usize, ) -> MatchResult { - let swa_boundary_len = result.device_indices.size()[0] as usize + result.host_hit_length; + let swa_boundary_len = result.device_prefix_len + result.host_hit_length; // Branch at the last page-aligned Full-KV position past the SWA boundary. let page_aligned_full_hit_len = diff --git a/rust/sglang-radix-tree/src/python_bindings.rs b/rust/sglang-radix-tree/src/python_bindings.rs index f580134ff3..18836d1644 100644 --- a/rust/sglang-radix-tree/src/python_bindings.rs +++ b/rust/sglang-radix-tree/src/python_bindings.rs @@ -328,7 +328,7 @@ type TransferArgs = ( #[derive(FromPyObject)] struct InspectionMatchResultInput { #[pyo3(attribute)] - device_indices: PyTensor, + device_prefix_len: usize, #[pyo3(attribute)] last_device_node: NodeId, #[pyo3(attribute)] @@ -651,10 +651,10 @@ impl InsertParamsBinding { } } -/// Python-visible match result; tensors and actions are Python-held. +/// Python-visible match result; actions are Python-held. #[pyclass(get_all)] pub struct MatchResultBinding { - device_indices: Py, + device_prefix_len: usize, last_device_node_id: NodeId, last_host_node_id: NodeId, best_match_node_id: NodeId, @@ -671,7 +671,7 @@ impl MatchResultBinding { /// Move a core match result across the boundary. fn from_match_result(py: Python<'_>, result: MatchResult) -> PyResult { Ok(MatchResultBinding { - device_indices: tensor_to_py(py, result.device_indices)?, + device_prefix_len: result.device_prefix_len, last_device_node_id: result.last_device_node_id, last_host_node_id: result.last_host_node_id, best_match_node_id: result.best_match_node_id, @@ -2621,7 +2621,7 @@ impl TreeCoreBinding { let component_type = parse_component_type(component_type)?; let key = K::key_from(Cow::Owned(py_array_to_vec_i64(py, key)?)).into_owned(); let InspectionMatchResultInput { - device_indices, + device_prefix_len, last_device_node: last_device_node_id, last_host_node: last_host_node_id, best_match_node: best_match_node_id, @@ -2633,7 +2633,7 @@ impl TreeCoreBinding { full_kv_hit_length, } = result; let result = MatchResult { - device_indices: device_indices.0, + device_prefix_len, last_device_node_id, last_host_node_id, best_match_node_id, diff --git a/rust/sglang-radix-tree/src/tests/components/mamba.rs b/rust/sglang-radix-tree/src/tests/components/mamba.rs index ef12ea1c3d..86f5df84af 100644 --- a/rust/sglang-radix-tree/src/tests/components/mamba.rs +++ b/rust/sglang-radix-tree/src/tests/components/mamba.rs @@ -2402,7 +2402,7 @@ fn branching_from_a_host_full_hit_is_reusable_after_insert() { let result = tc.match_prefix(&match_params(&vec![1, 2, 3, 4, 5, 6, 7])); assert_eq!(result.best_match_node_id, a); assert_eq!(result.last_device_node_id, a); - assert_eq!(result.device_indices.numel(), 3); + assert_eq!(result.device_prefix_len, 3); assert_eq!(result.host_hit_length, 0); assert_eq!(result.full_kv_hit_length, 7); assert_eq!(result.mamba_branching_seqlen, Some(6)); @@ -2414,7 +2414,7 @@ fn branching_from_a_host_full_hit_is_reusable_after_insert() { )); assert!(!insert_result.mamba_exist); let second = tc.match_prefix(&match_params(&vec![1, 2, 3, 4, 5, 6, 7])); - assert_eq!(second.device_indices.numel(), 6); + assert_eq!(second.device_prefix_len, 6); assert_eq!(second.mamba_branching_seqlen, None); } diff --git a/rust/sglang-radix-tree/src/tests/components/swa.rs b/rust/sglang-radix-tree/src/tests/components/swa.rs index fdf23a3b3d..563e96026c 100644 --- a/rust/sglang-radix-tree/src/tests/components/swa.rs +++ b/rust/sglang-radix-tree/src/tests/components/swa.rs @@ -1,6 +1,6 @@ use super::*; use crate::components::{ComponentSet, FULL, MAMBA, SWA}; -use crate::test_utils::{accumulate_step, action_kinds}; +use crate::test_utils::{accumulate_step, action_kinds, matched_device_indices}; use crate::unified_tree_core::CacheInitParams; #[test] @@ -630,7 +630,7 @@ fn match_prefix_uses_request_ring_layout_instead_of_hicache_pool_presence() { assert_eq!(result.full_kv_hit_length, 8); let expected = if swa_req_ring { &values[..8] } else { &[] }; assert!( - result.device_indices.equal(&Tensor::from_slice(expected)), + matched_device_indices(&tc, &result).equal(&Tensor::from_slice(expected)), "ring={swa_req_ring}, hicache={enable_hicache}, swa_host={has_swa_host_pool}" ); } @@ -3852,7 +3852,7 @@ fn match_prefix_with_an_empty_key_on_a_swa_core_is_a_clean_miss() { key: &Vec::new(), namespace: Default::default(), }); - assert_eq!(result.device_indices.size()[0], 0); + assert_eq!(result.device_prefix_len, 0); assert_eq!(result.swa_host_hit_length, 0); let root = tc.arena.root(); assert_eq!(result.best_match_node_id, tc.arena.node(root).id); @@ -5915,7 +5915,7 @@ fn deep_request_ring_tree_survives_full_backup_evict_and_load_back_rounds() { tc.sanity_check(&[], &[]); } let matched = tc.match_prefix(&match_params(&vec![1, 2, 3, 4, 5, 6])); - assert_eq!(matched.device_indices.numel(), 6); + assert_eq!(matched.device_prefix_len, 6); tc.sanity_check(&[], &[]); } @@ -6372,7 +6372,7 @@ fn finalize_branching( .finalize_match_result_in_tree_core( tc, MatchResult { - device_indices: Tensor::from_slice(&vec![0i64; device_len]), + device_prefix_len: device_len, host_hit_length, full_kv_hit_length, ..tc.empty_match_result() diff --git a/rust/sglang-radix-tree/src/tests/test_utils.rs b/rust/sglang-radix-tree/src/tests/test_utils.rs index 258e8bb7ca..d82dbe86cf 100644 --- a/rust/sglang-radix-tree/src/tests/test_utils.rs +++ b/rust/sglang-radix-tree/src/tests/test_utils.rs @@ -5,7 +5,16 @@ use std::collections::HashMap; use tch::Tensor; use crate::components::ComponentType; -use crate::unified_tree_core::{CacheAction, EvictionStepResult}; +use crate::node::ChildKeyType; +use crate::unified_tree_core::{CacheAction, EvictionStepResult, MatchResult, UnifiedTreeCore}; + +pub(crate) fn matched_device_indices( + tc: &UnifiedTreeCore, + result: &MatchResult, +) -> Tensor { + tc.collect_full_device_indices(result.last_device_node_id, tc.root_node_handle(None)) + .expect("live match node") +} /// Fold an eviction step into a caller's running accumulators (the Controller /// consumption contract: deltas add, freed tensors append). diff --git a/rust/sglang-radix-tree/src/tests/unified_tree_core.rs b/rust/sglang-radix-tree/src/tests/unified_tree_core.rs index 0cd2e63c60..36076a57db 100644 --- a/rust/sglang-radix-tree/src/tests/unified_tree_core.rs +++ b/rust/sglang-radix-tree/src/tests/unified_tree_core.rs @@ -5,7 +5,7 @@ use tch::Tensor; use super::*; use crate::components::{ComponentSet, FULL, MAMBA, SWA}; use crate::node::{NodeAccessError, ValueSlotIdx}; -use crate::test_utils::{accumulate_step, action_kinds}; +use crate::test_utils::{accumulate_step, action_kinds, matched_device_indices}; fn core() -> UnifiedTreeCore> { UnifiedTreeCore::new(CacheInitParams::default(), vec![FULL]) @@ -1236,11 +1236,7 @@ fn match_prefix_returns_the_full_hit() { let mut tc = core(); let (_a, b) = matched_chain(&mut tc); let result = tc.match_prefix(&match_params(&vec![1, 2, 3])); - assert!( - result - .device_indices - .equal(&Tensor::from_slice(&[10i64, 11, 12])) - ); + assert!(matched_device_indices(&tc, &result).equal(&Tensor::from_slice(&[10i64, 11, 12]))); assert_eq!(result.last_device_node_id, tc.arena.node(b).id); assert_eq!(result.last_host_node_id, tc.arena.node(b).id); assert_eq!(result.best_match_node_id, tc.arena.node(b).id); @@ -1253,11 +1249,7 @@ fn match_prefix_stops_at_the_matched_depth() { let mut tc = core(); let (a, _b) = matched_chain(&mut tc); let result = tc.match_prefix(&match_params(&vec![1, 2, 9])); - assert!( - result - .device_indices - .equal(&Tensor::from_slice(&[10i64, 11])) - ); + assert!(matched_device_indices(&tc, &result).equal(&Tensor::from_slice(&[10i64, 11]))); assert_eq!(result.best_match_node_id, tc.arena.node(a).id); } @@ -1267,7 +1259,7 @@ fn match_prefix_miss_anchors_at_the_root() { matched_chain(&mut tc); let root = tc.arena.root(); let result = tc.match_prefix(&match_params(&vec![9])); - assert_eq!(result.device_indices.numel(), 0); + assert_eq!(result.device_prefix_len, 0); assert_eq!(result.best_match_node_id, tc.arena.node(root).id); assert_eq!(result.last_device_node_id, tc.arena.node(root).id); } @@ -1280,7 +1272,7 @@ fn match_prefix_splits_on_a_partial_match() { // The walk split a at 1: the new prefix node holds [10] and anchors the result. let prefix_node = result.best_match_node_id; assert_ne!(prefix_node, tc.arena.node(a).id); - assert!(result.device_indices.equal(&Tensor::from_slice(&[10i64]))); + assert!(matched_device_indices(&tc, &result).equal(&Tensor::from_slice(&[10i64]))); assert_eq!( tc.arena .node(tc.arena.resolve(prefix_node).expect("live test node")) @@ -1329,7 +1321,7 @@ fn match_prefix_stops_at_a_dead_node() { .unwrap(); let _ = a; let result = tc.match_prefix(&match_params(&vec![1, 2])); - assert_eq!(result.device_indices.numel(), 0); + assert_eq!(result.device_prefix_len, 0); assert_eq!(result.best_match_node_id, tc.arena.node(root).id); } @@ -1350,11 +1342,7 @@ fn match_prefix_page_aligns_the_query() { ); // The trailing partial page is dropped before the walk. let result = tc.match_prefix(&match_params(&vec![1, 2, 3])); - assert!( - result - .device_indices - .equal(&Tensor::from_slice(&[10i64, 11])) - ); + assert!(matched_device_indices(&tc, &result).equal(&Tensor::from_slice(&[10i64, 11]))); assert_eq!(result.best_match_node_id, tc.arena.node(a).id); } @@ -1388,7 +1376,7 @@ fn insert_first_write_creates_the_namespace() { key: &vec![1, 2], namespace: KeyNamespaceRef::new(Some("lora-1"), None), }); - assert_eq!(result.device_indices.numel(), 0); + assert_eq!(result.device_prefix_len, 0); assert_eq!(result.best_match_node_id, tc.root_node_handle(None)); } @@ -1418,11 +1406,7 @@ fn match_prefix_skips_host_only_nodes_without_hicache() { let taken = tc.arena.take_device_value(b, FULL); tc.arena.set_host_value(b, FULL, taken); let result = tc.match_prefix(&match_params(&vec![1, 2, 3])); - assert!( - result - .device_indices - .equal(&Tensor::from_slice(&[10i64, 11])) - ); + assert!(matched_device_indices(&tc, &result).equal(&Tensor::from_slice(&[10i64, 11]))); assert_eq!(result.best_match_node_id, tc.arena.node(a).id); assert_eq!(result.last_device_node_id, tc.arena.node(a).id); } @@ -1437,11 +1421,7 @@ fn match_prefix_with_hicache_advances_best_match_onto_host_nodes() { let taken = tc.arena.take_device_value(b, FULL); tc.arena.set_host_value(b, FULL, taken); let result = tc.match_prefix(&match_params(&vec![1, 2, 3])); - assert!( - result - .device_indices - .equal(&Tensor::from_slice(&[10i64, 11])) - ); + assert!(matched_device_indices(&tc, &result).equal(&Tensor::from_slice(&[10i64, 11]))); assert_eq!(result.last_device_node_id, tc.arena.node(a).id); assert_eq!(result.best_match_node_id, tc.arena.node(b).id); assert_eq!(result.last_host_node_id, tc.arena.node(b).id); @@ -1483,11 +1463,7 @@ fn match_prefix_with_hicache_sums_the_host_span_length() { let taken = tc.arena.take_device_value(c, FULL); tc.arena.set_host_value(c, FULL, taken); let result = tc.match_prefix(&match_params(&vec![1, 2, 3, 4, 5])); - assert!( - result - .device_indices - .equal(&Tensor::from_slice(&[10i64, 11])) - ); + assert!(matched_device_indices(&tc, &result).equal(&Tensor::from_slice(&[10i64, 11]))); assert_eq!(result.last_device_node_id, tc.arena.node(a).id); assert_eq!(result.best_match_node_id, tc.arena.node(c).id); assert_eq!(result.last_host_node_id, tc.arena.node(c).id); @@ -1598,11 +1574,7 @@ fn match_walk_swa_state_resets_at_a_full_rejected_node() { let result = tc.match_prefix(&match_params(&vec![1, 2, 3, 4])); assert_eq!(result.best_match_node_id, tc.arena.node(a).id); assert_eq!(result.last_device_node_id, tc.arena.node(a).id); - assert!( - result - .device_indices - .equal(&Tensor::from_slice(&[10i64, 11])) - ); + assert!(matched_device_indices(&tc, &result).equal(&Tensor::from_slice(&[10i64, 11]))); } #[test] @@ -1636,11 +1608,7 @@ fn match_prefix_swa_tombstone_holds_the_best_match_below_the_window() { set_swa_device_and_list(&mut tc, c); let result = tc.match_prefix(&match_params(&vec![1, 2, 3, 4])); assert_eq!(result.best_match_node_id, tc.arena.node(a).id); - assert!( - result - .device_indices - .equal(&Tensor::from_slice(&[10i64, 11])) - ); + assert!(matched_device_indices(&tc, &result).equal(&Tensor::from_slice(&[10i64, 11]))); } #[test] @@ -1675,9 +1643,7 @@ fn match_prefix_swa_best_match_advances_at_the_window() { let result = tc.match_prefix(&match_params(&vec![1, 2, 3, 4, 5])); assert_eq!(result.best_match_node_id, tc.arena.node(c).id); assert!( - result - .device_indices - .equal(&Tensor::from_slice(&[10i64, 11, 12, 13, 14])) + matched_device_indices(&tc, &result).equal(&Tensor::from_slice(&[10i64, 11, 12, 13, 14])) ); } @@ -1831,9 +1797,7 @@ fn repeated_deep_swa_matches_keep_the_tree_sane() { for _ in 0..3 { let result = tc.match_prefix(&match_params(&vec![1, 2, 3, 4])); assert!( - result - .device_indices - .equal(&Tensor::from_slice(&[10i64, 11, 12, 13])) + matched_device_indices(&tc, &result).equal(&Tensor::from_slice(&[10i64, 11, 12, 13])) ); tc.sanity_check(&[], &[]); } @@ -1867,11 +1831,7 @@ fn swa_host_backed_node_advances_best_match_but_keeps_the_device_anchor() { .value = Some(Tensor::from_slice(&[0i64])); tc.host_lru_list_mut(SWA).insert_mru(b); let result = tc.match_prefix(&match_params(&vec![1, 2, 3])); - assert!( - result - .device_indices - .equal(&Tensor::from_slice(&[10i64, 11])) - ); + assert!(matched_device_indices(&tc, &result).equal(&Tensor::from_slice(&[10i64, 11]))); assert_eq!(result.last_device_node_id, tc.arena.node(a).id); assert_eq!(result.best_match_node_id, tc.arena.node(b).id); assert_eq!(result.host_hit_length, 0); @@ -2182,11 +2142,7 @@ fn insert_creates_a_leaf_and_matches_back() { assert_eq!(tc.evictable_size_(FULL), 3); let matched = tc.match_prefix(&match_params(&vec![1, 2, 3])); assert_eq!(result.last_device_node_id, Some(matched.best_match_node_id)); - assert!( - matched - .device_indices - .equal(&Tensor::from_slice(&[10i64, 11, 12])) - ); + assert!(matched_device_indices(&tc, &matched).equal(&Tensor::from_slice(&[10i64, 11, 12]))); } #[test] @@ -2241,9 +2197,7 @@ fn insert_extends_an_existing_prefix() { assert!(freed[0].equal(&Tensor::from_slice(&[20i64, 21, 22]))); let matched = tc.match_prefix(&match_params(&vec![1, 2, 3, 4, 5])); assert!( - matched - .device_indices - .equal(&Tensor::from_slice(&[10i64, 11, 12, 13, 14])) + matched_device_indices(&tc, &matched).equal(&Tensor::from_slice(&[10i64, 11, 12, 13, 14])) ); } @@ -2300,17 +2254,9 @@ fn insert_splits_on_a_partial_overlap() { assert_eq!(result.prefix_len, 2); // Both suffixes live under the split prefix node. let matched = tc.match_prefix(&match_params(&vec![1, 2, 9])); - assert!( - matched - .device_indices - .equal(&Tensor::from_slice(&[10i64, 11, 29])) - ); + assert!(matched_device_indices(&tc, &matched).equal(&Tensor::from_slice(&[10i64, 11, 29]))); let matched = tc.match_prefix(&match_params(&vec![1, 2, 3])); - assert!( - matched - .device_indices - .equal(&Tensor::from_slice(&[10i64, 11, 12])) - ); + assert!(matched_device_indices(&tc, &matched).equal(&Tensor::from_slice(&[10i64, 11, 12]))); } #[test] @@ -4540,11 +4486,7 @@ fn commit_load_back_reattaches_device_slices_and_restores_the_match() { assert!(tc.full_coexisting_host_nodes.contains(child)); tc.sanity_check(&[], &[]); let result = tc.match_prefix(&match_params(&vec![1, 2, 3, 4])); - assert!( - result - .device_indices - .equal(&Tensor::from_slice(&[50i64, 51, 52, 53])) - ); + assert!(matched_device_indices(&tc, &result).equal(&Tensor::from_slice(&[50i64, 51, 52, 53]))); } #[test] @@ -4768,7 +4710,7 @@ fn match_prefix_with_hicache_splits_a_host_only_backuped_node() { tc.demote(leaf).expect("backuped live test node"); // The partial match splits the host-only node; the host prefix stays usable. let result = tc.match_prefix(&match_params(&vec![1, 2, 9])); - assert_eq!(result.device_indices.numel(), 0); + assert_eq!(result.device_prefix_len, 0); let root = tc.arena.root(); assert_eq!(result.last_device_node_id, tc.arena.node(root).id); assert_eq!(result.host_hit_length, 2); @@ -4848,14 +4790,12 @@ fn mixed_backup_evict_insert_keeps_the_leaf_sets_disjoint() { // The unbacked chains died outright. assert_eq!( tc.match_prefix(&match_params(&vec![301, 302])) - .device_indices - .numel(), + .device_prefix_len, 0 ); assert_eq!( tc.match_prefix(&match_params(&vec![401, 402])) - .device_indices - .numel(), + .device_prefix_len, 0 ); tc.insert(&insert_params(&vec![501, 502], &[60, 61])); @@ -4922,7 +4862,7 @@ fn drop_subtree_no_host_frees_the_leaf_and_its_host_descendants() { assert!(hf[&FULL][0].equal(&Tensor::from_slice(&[20i64, 21]))); assert_eq!(tc.arena.len(), 1); let result = tc.match_prefix(&match_params(&vec![1, 2])); - assert_eq!(result.device_indices.numel(), 0); + assert_eq!(result.device_prefix_len, 0); tc.sanity_check(&[], &[]); } @@ -5172,12 +5112,12 @@ fn insert_into_a_named_namespace_is_isolated() { }); // The default namespace stays empty; the named namespace hits. let miss = tc.match_prefix(&match_params(&vec![1, 2])); - assert_eq!(miss.device_indices.numel(), 0); + assert_eq!(miss.device_prefix_len, 0); let hit = tc.match_prefix(&MatchPrefixParams { key: &vec![1, 2], namespace: KeyNamespaceRef::new(Some("lora-1"), None), }); - assert!(hit.device_indices.equal(&Tensor::from_slice(&[10i64, 11]))); + assert!(matched_device_indices(&tc, &hit).equal(&Tensor::from_slice(&[10i64, 11]))); } fn page2_core() -> UnifiedTreeCore> { @@ -5255,17 +5195,9 @@ fn insert_page_size_two_splits_mid_page_divergence_at_the_page_boundary() { vec![1, 2] ); let matched = tc.match_prefix(&match_params(&vec![1, 2, 3, 4])); - assert!( - matched - .device_indices - .equal(&Tensor::from_slice(&[10i64, 11, 12, 13])) - ); + assert!(matched_device_indices(&tc, &matched).equal(&Tensor::from_slice(&[10i64, 11, 12, 13]))); let matched = tc.match_prefix(&match_params(&vec![1, 2, 3, 9])); - assert!( - matched - .device_indices - .equal(&Tensor::from_slice(&[10i64, 11, 22, 29])) - ); + assert!(matched_device_indices(&tc, &matched).equal(&Tensor::from_slice(&[10i64, 11, 22, 29]))); tc.sanity_check(&[], &[]); } @@ -5275,11 +5207,7 @@ fn match_prefix_page_size_two_splits_at_a_page_boundary() { tc.insert(&insert_params(&vec![1, 2, 3, 4], &[10, 11, 12, 13])); // The query shares 3 atoms; pages quantize the split down to 2. let result = tc.match_prefix(&match_params(&vec![1, 2, 3, 9])); - assert!( - result - .device_indices - .equal(&Tensor::from_slice(&[10i64, 11])) - ); + assert!(matched_device_indices(&tc, &result).equal(&Tensor::from_slice(&[10i64, 11]))); assert_eq!( tc.arena .node( @@ -5292,11 +5220,7 @@ fn match_prefix_page_size_two_splits_at_a_page_boundary() { ); // The split child stays reachable through its own page key. let matched = tc.match_prefix(&match_params(&vec![1, 2, 3, 4])); - assert!( - matched - .device_indices - .equal(&Tensor::from_slice(&[10i64, 11, 12, 13])) - ); + assert!(matched_device_indices(&tc, &matched).equal(&Tensor::from_slice(&[10i64, 11, 12, 13]))); tc.sanity_check(&[], &[]); } @@ -5306,7 +5230,7 @@ fn match_prefix_page_size_two_sub_page_query_is_an_empty_match() { tc.insert(&insert_params(&vec![1, 2], &[10, 11])); let root = tc.arena.root(); let result = tc.match_prefix(&match_params(&vec![1])); - assert_eq!(result.device_indices.numel(), 0); + assert_eq!(result.device_prefix_len, 0); assert_eq!(result.best_match_node_id, tc.arena.node(root).id); } @@ -6226,8 +6150,8 @@ fn empty_match_result_anchors_all_boundaries_at_the_root() { assert_eq!(result.last_host_node_id, tc.arena.node(root).id); assert_eq!(result.best_match_node_id, tc.arena.node(root).id); assert_eq!(result.host_hit_length, 0); - assert_eq!(result.device_indices.numel(), 0); - assert_eq!(result.device_indices.kind(), Kind::Int64); + assert_eq!(result.device_prefix_len, 0); + assert_eq!(matched_device_indices(&tc, &result).kind(), Kind::Int64); } #[test] @@ -7002,7 +6926,7 @@ fn reset_restores_a_fresh_tree() { assert_eq!(tc.device_lru_list(SWA).len(), 0); assert_eq!(tc.host_lru_list(SWA).len(), 0); let matched = tc.match_prefix(&match_params(&vec![1, 2, 3])); - assert_eq!(matched.device_indices.numel(), 0); + assert_eq!(matched.device_prefix_len, 0); // The tree accepts fresh inserts after the reset. tc.insert(&insert_params(&vec![4, 5], &[30, 31])); assert_eq!(tc.evictable_size(), 2); @@ -7988,7 +7912,7 @@ fn match_prefix_on_an_unknown_namespace_allocates_nothing() { key: &vec![1, 2, 3], namespace: KeyNamespaceRef::new(Some("ghost"), None), }); - assert_eq!(result.device_indices.numel(), 0); + assert_eq!(result.device_prefix_len, 0); assert_eq!(tc.arena.len(), arena_len); // The empty result anchors at the default root (the namespace has no root). let default_root = tc.arena.root(); @@ -8315,11 +8239,7 @@ fn match_ragged_query_stops_at_the_aligned_window_page_size_two() { tc.insert(&insert_params(&vec![1, 2, 3, 4], &[10, 11, 12, 13])); // The node key runs past the query's aligned window; the ragged atom matches too. let result = tc.match_prefix(&match_params(&vec![1, 2, 3])); - assert!( - result - .device_indices - .equal(&Tensor::from_slice(&[10i64, 11])) - ); + assert!(matched_device_indices(&tc, &result).equal(&Tensor::from_slice(&[10i64, 11]))); } #[test] @@ -8602,7 +8522,7 @@ fn full_kv_hit_length_counts_the_split_fragment() { let result = tc.match_prefix(&match_params(&vec![1, 2, 99])); // The mid-node partial match splits the node; the fragment still counts. assert_eq!(result.full_kv_hit_length, 2); - assert_eq!(result.device_indices.size()[0], 2); + assert_eq!(result.device_prefix_len, 2); } #[test] @@ -8731,8 +8651,7 @@ fn reset_invalidates_every_prior_handle() { tc.insert(&insert_params(&vec![1, 2], &[10, 11])); assert_eq!( tc.match_prefix(&match_params(&vec![1, 2])) - .device_indices - .size()[0], + .device_prefix_len, 2 ); } @@ -8873,8 +8792,8 @@ fn run_random_op_sequence(mut tc: UnifiedTreeCore>, page: usize, mamba: } 1 => { // Consecutive matches of the same key agree (idempotency). - let first = tc.match_prefix(&match_params(&key)).device_indices.numel(); - let second = tc.match_prefix(&match_params(&key)).device_indices.numel(); + let first = tc.match_prefix(&match_params(&key)).device_prefix_len; + let second = tc.match_prefix(&match_params(&key)).device_prefix_len; assert_eq!(first, second); } 2 => { @@ -8891,7 +8810,7 @@ fn run_random_op_sequence(mut tc: UnifiedTreeCore>, page: usize, mamba: // Insert-while-locked churn, the cache_finished_req shape. let matched = tc.match_prefix(&match_params(&key)); let anchor = matched.best_match_node_id; - let matched_len = matched.device_indices.numel() as usize; + let matched_len = matched.device_prefix_len; let lock = tc .inc_lock_ref(anchor, ComponentSet::EMPTY) .expect("live match anchor"); diff --git a/rust/sglang-radix-tree/src/unified_tree_core.rs b/rust/sglang-radix-tree/src/unified_tree_core.rs index 303202fbbb..b5ae8f5ce6 100644 --- a/rust/sglang-radix-tree/src/unified_tree_core.rs +++ b/rust/sglang-radix-tree/src/unified_tree_core.rs @@ -116,8 +116,9 @@ pub struct DecLockRefResult {} /// Result of a prefix match. pub struct MatchResult { - /// Device KV indices matched by the common prefix. - pub device_indices: Tensor, + /// Length of the device-resident matched prefix; its KV indices are read + /// off the path to `last_device_node_id` (`collect_full_device_indices`). + pub device_prefix_len: usize, /// Last matched node still resident on device. pub last_device_node_id: NodeId, /// Last matched node on host; equals `last_device_node_id` without HiCache. @@ -1160,7 +1161,7 @@ impl UnifiedTreeCore { Ok(DecLockRefResult::default()) } - /// Match a key against the tree; returns device indices + boundary NodeIds. + /// Match a key; returns the device prefix length + boundary NodeIds. pub fn match_prefix(&mut self, params: &MatchPrefixParams<'_, K>) -> MatchResult { // Bigram view conversion happens at the boundary; the key arrives typed. let aligned_key_len = params.key.atom_len() / self.page_size * self.page_size; @@ -1423,13 +1424,12 @@ impl UnifiedTreeCore { best_match_device_node_id }; - let device_indices = if best_match_device_value_len > 0 { - Tensor::cat(&value[..best_match_device_value_len], 0) - } else { - self.empty_device_indices.shallow_clone() - }; + let device_prefix_len = value[..best_match_device_value_len] + .iter() + .map(|chunk| chunk.size()[0] as usize) + .sum(); let mut result = MatchResult { - device_indices, + device_prefix_len, last_device_node_id: self.arena.node(best_match_device_node_id).id, last_host_node_id: self.arena.node(last_host_node_id).id, best_match_node_id: self.arena.node(best_match_node_id).id, @@ -1457,11 +1457,11 @@ impl UnifiedTreeCore { result } - /// An empty match: no device indices, every boundary anchored at the root. + /// An empty match: no device prefix, every boundary anchored at the root. pub fn empty_match_result(&self) -> MatchResult { let root_id = self.arena.node(self.arena.root()).id; MatchResult { - device_indices: self.empty_device_indices.shallow_clone(), + device_prefix_len: 0, last_device_node_id: root_id, last_host_node_id: root_id, best_match_node_id: root_id, diff --git a/test/registered/unit/disaggregation/test_decode_hicache_tree_core.py b/test/registered/unit/disaggregation/test_decode_hicache_tree_core.py index adc255ed65..792a52c99b 100644 --- a/test/registered/unit/disaggregation/test_decode_hicache_tree_core.py +++ b/test/registered/unit/disaggregation/test_decode_hicache_tree_core.py @@ -34,6 +34,7 @@ class TestDecodeHiCacheTreeCore(CustomTestCase): get_prefix_hash_values=Mock(return_value=["h0", "h1"]), query_storage_hit_length=Mock(return_value=2), prefetch_from_storage=Mock(side_effect=register_prefetch), + prefix_device_indices=Mock(return_value=torch.tensor([10, 11])), ) harness = SimpleNamespace( scheduler=SimpleNamespace(enable_decode_hicache=True), @@ -47,7 +48,7 @@ class TestDecodeHiCacheTreeCore(CustomTestCase): cache_salt="tenant-a", ) result = SimpleNamespace( - device_indices=torch.tensor([10, 11]), + device_prefix_len=2, host_hit_length=2, last_device_node=11, last_host_node=22, diff --git a/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py b/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py index 5e36e0d4f2..3b88728fed 100644 --- a/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py +++ b/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py @@ -242,7 +242,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): # Match prefix (simulates _match_prefix_and_lock in pop_preallocated) result = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", prefix)))) matched_node = result.last_device_node - prefix_len = len(result.device_indices) + prefix_len = result.device_prefix_len self.assertEqual(prefix_len, 3) # Step 1: lock (pop_preallocated locks the matched node) @@ -291,7 +291,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): MatchPrefixParams(key=RadixKey(array("q", full_ids))) ) matched_node = result.last_device_node - self.assertEqual(len(result.device_indices), 0) # no match + self.assertEqual(result.device_prefix_len, 0) # no match # matched_node is root root_lock_before = cache.root_node.lock_ref @@ -344,7 +344,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): # Match and lock result = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", prefix)))) matched_node = result.last_device_node - prefix_len = len(result.device_indices) + prefix_len = result.device_prefix_len lock = cache.lock(matched_node) # Prefix tokens should now be protected (locked) @@ -393,7 +393,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): result = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", prefix)))) matched_node = result.last_device_node - prefix_len = len(result.device_indices) + prefix_len = result.device_prefix_len lock = cache.lock(matched_node) # Token sequence is 5 long; a 6th KV slot is committed with no token id. @@ -678,7 +678,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): MatchPrefixParams(key=RadixKey(array("q", prefix))) ) matched_node = result.last_device_node - prefix_len = len(result.device_indices) + prefix_len = result.device_prefix_len lock = cache.lock(matched_node) diff --git a/test/registered/unit/mem_cache/test_lmcache_component_cursors.py b/test/registered/unit/mem_cache/test_lmcache_component_cursors.py index 3c82186fbe..adf9d2c873 100644 --- a/test/registered/unit/mem_cache/test_lmcache_component_cursors.py +++ b/test/registered/unit/mem_cache/test_lmcache_component_cursors.py @@ -98,13 +98,16 @@ def test_mamba_publication_snapshots_component_cursors(cache_class, swa_enabled) cache.insert = mock.Mock(return_value=InsertResult(prefix_len=8)) cache.inc_lock_ref = mock.Mock(return_value=IncLockRefResult(node_id=42)) matched = MatchResult( - device_indices=torch.arange(8), + device_prefix_len=8, last_device_node=42, last_host_node=None, best_match_node=None, ) - with mock.patch.object(UnifiedRadixCache, "match_prefix", return_value=matched): + with ( + mock.patch.object(UnifiedRadixCache, "match_prefix", return_value=matched), + mock.patch.object(cache, "path_device_indices", return_value=torch.arange(8)), + ): cache._publish_external_loaded_prefix(req, token_ids_len=12) insert_params = cache.insert.call_args.args[0] diff --git a/test/registered/unit/mem_cache/test_page_interleave_shard.py b/test/registered/unit/mem_cache/test_page_interleave_shard.py index 31bbeca139..cf87c93d48 100644 --- a/test/registered/unit/mem_cache/test_page_interleave_shard.py +++ b/test/registered/unit/mem_cache/test_page_interleave_shard.py @@ -652,7 +652,7 @@ def _insert(tree, tokens, rotation_base=None, value=None): def _match_len(tree, tokens): res = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", tokens)))) - return len(res.device_indices) + return res.device_prefix_len class _TreeCoreBackendCase(CustomTestCase): diff --git a/test/registered/unit/mem_cache/test_pure_swa_radix_cache.py b/test/registered/unit/mem_cache/test_pure_swa_radix_cache.py index 1cb8e02e13..aca959b135 100644 --- a/test/registered/unit/mem_cache/test_pure_swa_radix_cache.py +++ b/test/registered/unit/mem_cache/test_pure_swa_radix_cache.py @@ -85,7 +85,7 @@ class TestPureSWARadixCache(CustomTestCase): # [0, 4) went into the tree; [4, 6) was window-evicted; [6, 8) is freed. match = cache.match_prefix(MatchPrefixParams(key=RadixKey(token_ids))) - self.assertEqual(len(match.device_indices), 4) + self.assertEqual(match.device_prefix_len, 4) self.assertEqual(allocator.freed, [6, 7]) self.assertEqual(allocator.skipped, [4, 5]) diff --git a/test/registered/unit/mem_cache/test_radix_cache_slru_accuracy.py b/test/registered/unit/mem_cache/test_radix_cache_slru_accuracy.py index 1a28b67b64..1e81868314 100644 --- a/test/registered/unit/mem_cache/test_radix_cache_slru_accuracy.py +++ b/test/registered/unit/mem_cache/test_radix_cache_slru_accuracy.py @@ -117,7 +117,7 @@ class TestSLRUAccuracy(unittest.TestCase): ) # Check if the tensor is empty, which indicates the key was not found (evicted) - is_frequent_key_present = frequent_match_result.device_indices.numel() > 0 + is_frequent_key_present = frequent_match_result.device_prefix_len > 0 self.assertTrue( is_frequent_key_present, "Frequently accessed key should still be in cache after evictions", @@ -126,7 +126,7 @@ class TestSLRUAccuracy(unittest.TestCase): # Verify the first low-frequency key has been evicted # The device_indices tensor should be empty when the key is not found is_first_low_freq_key_present = ( - first_low_freq_match_result.device_indices.numel() > 0 + first_low_freq_match_result.device_prefix_len > 0 ) self.assertFalse( is_first_low_freq_key_present, diff --git a/test/registered/unit/mem_cache/test_radix_cache_unit.py b/test/registered/unit/mem_cache/test_radix_cache_unit.py index 27ea456f18..bb469c74f8 100644 --- a/test/registered/unit/mem_cache/test_radix_cache_unit.py +++ b/test/registered/unit/mem_cache/test_radix_cache_unit.py @@ -367,16 +367,19 @@ class TestRadixCache(CustomTestCase): result = cache.match_prefix( MatchPrefixParams(key=RadixKey(array("q", [1, 2, 3]))) ) - self.assertEqual(len(result.device_indices), 3) - torch.testing.assert_close(result.device_indices, value) + self.assertEqual(result.device_prefix_len, 3) + torch.testing.assert_close( + cache.path_device_indices(result.last_device_node), value + ) # Test partial match result = cache.match_prefix( MatchPrefixParams(key=RadixKey(array("q", [1, 2]))) ) - self.assertEqual(len(result.device_indices), 2) + self.assertEqual(result.device_prefix_len, 2) torch.testing.assert_close( - result.device_indices, torch.tensor([10, 20], dtype=torch.int64) + cache.path_device_indices(result.last_device_node), + torch.tensor([10, 20], dtype=torch.int64), ) def test_insert_with_none_value(self): @@ -515,7 +518,7 @@ class TestRadixCache(CustomTestCase): match = cache.match_prefix( MatchPrefixParams(key=RadixKey(prompt_ids + output_ids)) ) - self.assertEqual(len(match.device_indices), len(prompt_ids)) + self.assertEqual(match.device_prefix_len, len(prompt_ids)) def test_kv_cache_events(self): """Test KV cache events functionality.""" @@ -642,23 +645,26 @@ class TestRadixCache(CustomTestCase): ) # Each should match only its own data - self.assertEqual(len(result1.device_indices), 3) + self.assertEqual(result1.device_prefix_len, 3) torch.testing.assert_close( - result1.device_indices, torch.tensor([10, 20, 30], dtype=torch.int64) + cache.path_device_indices(result1.last_device_node), + torch.tensor([10, 20, 30], dtype=torch.int64), ) - self.assertEqual(len(result2.device_indices), 3) + self.assertEqual(result2.device_prefix_len, 3) torch.testing.assert_close( - result2.device_indices, torch.tensor([40, 50, 60], dtype=torch.int64) + cache.path_device_indices(result2.last_device_node), + torch.tensor([40, 50, 60], dtype=torch.int64), ) - self.assertEqual(len(result3.device_indices), 3) + self.assertEqual(result3.device_prefix_len, 3) torch.testing.assert_close( - result3.device_indices, torch.tensor([70, 80, 90], dtype=torch.int64) + cache.path_device_indices(result3.last_device_node), + torch.tensor([70, 80, 90], dtype=torch.int64), ) # Non-existent extra_key should not match - self.assertEqual(len(result4.device_indices), 0) + self.assertEqual(result4.device_prefix_len, 0) def test_cache_salt_isolation_is_independent_of_extra_key(self): cache = RadixCache.create_simulated() @@ -684,10 +690,12 @@ class TestRadixCache(CustomTestCase): MatchPrefixParams(key=RadixKey(tokens, extra_key="c", cache_salt="ab")) ) torch.testing.assert_close( - first.device_indices, torch.tensor([10, 20, 30], dtype=torch.int64) + cache.path_device_indices(first.last_device_node), + torch.tensor([10, 20, 30], dtype=torch.int64), ) torch.testing.assert_close( - second.device_indices, torch.tensor([40, 50, 60], dtype=torch.int64) + cache.path_device_indices(second.last_device_node), + torch.tensor([40, 50, 60], dtype=torch.int64), ) def test_cache_salt_is_included_in_store_and_remove_events(self): @@ -873,10 +881,10 @@ class TestRadixCache(CustomTestCase): result = cache.match_prefix( MatchPrefixParams(key=RadixKey(array("q", tokens))) ) - self.assertGreater(len(result.device_indices), 0) + self.assertGreater(result.device_prefix_len, 0) # Match length should be page-aligned - match_len = len(result.device_indices) + match_len = result.device_prefix_len self.assertEqual(match_len % page_size, 0) def test_advanced_prefix_match_with_node_splits(self): @@ -906,7 +914,9 @@ class TestRadixCache(CustomTestCase): result1 = cache.match_prefix( MatchPrefixParams(key=RadixKey(array("q", query1))) ) - torch.testing.assert_close(result1.device_indices, val1[:4]) + torch.testing.assert_close( + cache.path_device_indices(result1.last_device_node), val1[:4] + ) # No data change after structural split during matching. self.assertEqual(cache.total_size(), baseline_total) @@ -914,21 +924,27 @@ class TestRadixCache(CustomTestCase): result_full = cache.match_prefix( MatchPrefixParams(key=RadixKey(array("q", seq1))) ) - torch.testing.assert_close(result_full.device_indices, val1) + torch.testing.assert_close( + cache.path_device_indices(result_full.last_device_node), val1 + ) # Another split deeper on the path (after matching 6 tokens, then diverge). query2 = [1, 2, 3, 4, 5, 6, 777, 888] result2 = cache.match_prefix( MatchPrefixParams(key=RadixKey(array("q", query2))) ) - torch.testing.assert_close(result2.device_indices, val1[:6]) + torch.testing.assert_close( + cache.path_device_indices(result2.last_device_node), val1[:6] + ) self.assertEqual(cache.total_size(), baseline_total) # Matching the short diverging branch should return exactly its indices. result_branch = cache.match_prefix( MatchPrefixParams(key=RadixKey(array("q", seq2))) ) - torch.testing.assert_close(result_branch.device_indices, val2) + torch.testing.assert_close( + cache.path_device_indices(result_branch.last_device_node), val2 + ) def test_hash_value_storage(self): """Test that hash_value is stored correctly after insert operations.""" diff --git a/test/registered/unit/mem_cache/test_radix_force_miss.py b/test/registered/unit/mem_cache/test_radix_force_miss.py index 8f2671c9ab..a70b46c289 100644 --- a/test/registered/unit/mem_cache/test_radix_force_miss.py +++ b/test/registered/unit/mem_cache/test_radix_force_miss.py @@ -13,8 +13,6 @@ import unittest from array import array from types import SimpleNamespace -import torch - from sglang.srt.environ import envs from sglang.srt.mem_cache.base_prefix_cache import ( InsertParams, @@ -52,16 +50,13 @@ class TestZeroMatchResult(unittest.TestCase): match = tree.match_prefix( MatchPrefixParams(key=RadixKey(token_ids=array("q", [1, 2, 3, 9]))) ) - self.assertGreater(len(match.device_indices), 0) + self.assertGreater(match.device_prefix_len, 0) zeroed = zero_match_result(tree, match) - self.assertEqual(int(zeroed.device_indices.numel()), 0) + self.assertEqual(int(zeroed.device_prefix_len), 0) self.assertIs(zeroed.last_device_node, tree.root_node) self.assertIs(zeroed.last_host_node, tree.root_node) self.assertIs(zeroed.best_match_node, tree.root_node) self.assertEqual(zeroed.host_hit_length, 0) - # dtype/device preserved (slice-not-allocate). - self.assertEqual(zeroed.device_indices.dtype, match.device_indices.dtype) - self.assertEqual(zeroed.device_indices.device, match.device_indices.device) def test_chunk_cache_is_passthrough(self): class _StubChunkCache: @@ -69,7 +64,7 @@ class TestZeroMatchResult(unittest.TestCase): return False original = MatchResult( - device_indices=torch.empty((0,), dtype=torch.int64), + device_prefix_len=0, last_device_node=None, last_host_node=None, best_match_node=None, @@ -86,7 +81,7 @@ class TestMatchKvCacheForceMiss(unittest.TestCase): def match_prefix(self, params): return MatchResult( - device_indices=torch.empty((0,), dtype=torch.int64), + device_prefix_len=0, last_device_node=None, last_host_node=None, best_match_node=None, diff --git a/test/registered/unit/mem_cache/test_rust_tree_core.py b/test/registered/unit/mem_cache/test_rust_tree_core.py index b7b0968459..2fff5bf118 100644 --- a/test/registered/unit/mem_cache/test_rust_tree_core.py +++ b/test/registered/unit/mem_cache/test_rust_tree_core.py @@ -81,7 +81,10 @@ def test_insert_then_match_round_trips(): ), ) matched = core.match_prefix(MatchPrefixParams(key=_key([1, 2, 3]))) - assert matched.device_indices.tolist() == [10, 11, 12] + path = core.collect_full_device_indices( + matched.last_device_node, core.root_node_handle(None) + ) + assert path.tolist() == [10, 11, 12] def test_lock_moves_tokens_between_evictable_and_protected(): @@ -108,9 +111,12 @@ def test_namespaces_isolate_the_same_tokens(): ), ) salted = core.match_prefix(MatchPrefixParams(key=_key([1, 2], extra_key="chat"))) - assert salted.device_indices.tolist() == [20, 21] + path = core.collect_full_device_indices( + salted.last_device_node, core.root_node_handle("chat") + ) + assert path.tolist() == [20, 21] unsalted = core.match_prefix(MatchPrefixParams(key=_key([1, 2]))) - assert unsalted.device_indices.numel() == 0 + assert unsalted.device_prefix_len == 0 def test_backfill_hashes_existing_nodes_in_parent_order(): diff --git a/test/registered/unit/mem_cache/test_rust_tree_core_integration.py b/test/registered/unit/mem_cache/test_rust_tree_core_integration.py index 1d5e2a9bec..5b0773e070 100644 --- a/test/registered/unit/mem_cache/test_rust_tree_core_integration.py +++ b/test/registered/unit/mem_cache/test_rust_tree_core_integration.py @@ -72,6 +72,10 @@ def _tree_core(**params_overrides) -> RustUnifiedTreeCore: return RustUnifiedTreeCore(CacheInitParams(**params)) +def _path_indices(core: RustUnifiedTreeCore, node_id) -> torch.Tensor: + return core.collect_full_device_indices(node_id, core.root_node_handle(None)) + + def _key(token_ids: list[int]) -> RadixKey: return RadixKey(array("q", token_ids)) @@ -128,7 +132,7 @@ def _accumulate_step(result, tracker, device_frees, host_frees): def test_match_on_the_empty_tree_returns_no_indices(): core = _tree_core() result = core.match_prefix(MatchPrefixParams(key=_key([1, 2, 3]))) - assert result.device_indices.numel() == 0 + assert result.device_prefix_len == 0 @pytest.mark.parametrize("instance_backend", [None, "rust"]) @@ -153,7 +157,7 @@ def test_default_backend_constructs_real_rust_cpu_cache(monkeypatch, instance_ba assert isinstance(cache.tree_core, RustUnifiedTreeCore) cache.insert(InsertParams(key=_key([1, 2, 3]), value=torch.tensor([11, 12, 13]))) matched = cache.match_prefix(MatchPrefixParams(key=_key([1, 2, 3]))) - assert matched.device_indices.tolist() == [11, 12, 13] + assert cache.path_device_indices(matched.last_device_node).tolist() == [11, 12, 13] cache.tree_core.sanity_check([], []) @@ -164,7 +168,7 @@ def test_insert_then_match_back_returns_the_exact_indices(): assert result.cache_actions == [] matched = core.match_prefix(MatchPrefixParams(key=_key([1, 2, 3]))) assert result.last_device_node == matched.last_device_node - assert matched.device_indices.tolist() == [10, 11, 12] + assert _path_indices(core, matched.last_device_node).tolist() == [10, 11, 12] def test_root_node_handle_is_namespace_independent(): @@ -708,7 +712,7 @@ def test_stale_match_finalizer_handles_raise_key_error_without_poisoning_the_cor core.reset() live_root = core.root_node_handle() result = MatchResult( - device_indices=torch.empty(0, dtype=torch.int64), + device_prefix_len=0, last_device_node=live_root, last_host_node=live_root, best_match_node=live_root, @@ -945,7 +949,13 @@ def test_extension_insert_frees_the_duplicate_overlap(): assert isinstance(action, FreeDeviceKV) assert torch.cat(action.indices).tolist() == [20, 21, 22] matched = core.match_prefix(MatchPrefixParams(key=_key([1, 2, 3, 4, 5]))) - assert matched.device_indices.tolist() == [10, 11, 12, 13, 14] + assert _path_indices(core, matched.last_device_node).tolist() == [ + 10, + 11, + 12, + 13, + 14, + ] def test_lock_and_unlock_move_tokens_between_protected_and_evictable(): @@ -1094,7 +1104,9 @@ def test_binding_stays_usable_after_a_failed_insert(): ) assert result.prefix_len == 0 matched = binding.match_prefix(mem_cache.MatchParamsBinding(array("q", [1, 2, 3]))) - assert matched.device_indices.tolist() == [10, 11, 12] + assert binding.collect_full_device_indices( + matched.last_device_node_id, binding.root_node_handle(None) + ).tolist() == [10, 11, 12] @pytest.mark.parametrize("prior_hash", ["abcd", "z" * 64]) @@ -1152,13 +1164,13 @@ def test_extra_key_isolates_namespaces(): salted = core.match_prefix( MatchPrefixParams(key=RadixKey(array("q", [1, 2, 3]), extra_key="salt")) ) - assert salted.device_indices.tolist() == [10, 11, 12] + assert _path_indices(core, salted.last_device_node).tolist() == [10, 11, 12] unsalted = core.match_prefix(MatchPrefixParams(key=_key([1, 2, 3]))) - assert unsalted.device_indices.numel() == 0 + assert unsalted.device_prefix_len == 0 other = core.match_prefix( MatchPrefixParams(key=RadixKey(array("q", [1, 2, 3]), extra_key="other")) ) - assert other.device_indices.numel() == 0 + assert other.device_prefix_len == 0 assert core.prefetch_anchor_info(salted.best_match_node) == ("salt", None) assert core.prefetch_anchor_info(core.root_node_handle()) == (None, None) @@ -1177,20 +1189,23 @@ def test_cache_salt_is_supported_by_all_key_entry_points(): InsertParams(key=second_key, value=torch.tensor([20, 21], dtype=torch.int64)), ) - assert core.match_prefix( - MatchPrefixParams(key=first_key) - ).device_indices.tolist() == [ + assert _path_indices( + core, core.match_prefix(MatchPrefixParams(key=first_key)).last_device_node + ).tolist() == [ 10, 11, ] - assert core.match_prefix( - MatchPrefixParams(key=second_key) - ).device_indices.tolist() == [ + assert _path_indices( + core, core.match_prefix(MatchPrefixParams(key=second_key)).last_device_node + ).tolist() == [ 20, 21, ] assert ( - core.match_prefix(MatchPrefixParams(key=_key([1, 2]))).device_indices.numel() + _path_indices( + core, + core.match_prefix(MatchPrefixParams(key=_key([1, 2]))).last_device_node, + ).numel() == 0 ) @@ -1249,7 +1264,7 @@ def test_page_size_two_drops_the_ragged_tail(): ) assert result.prefix_len == 0 matched = core.match_prefix(MatchPrefixParams(key=_key([1, 2, 3, 4, 5]))) - assert matched.device_indices.tolist() == [10, 11, 12, 13] + assert _path_indices(core, matched.last_device_node).tolist() == [10, 11, 12, 13] def test_insert_value_none_materializes_the_token_ids(): @@ -1257,14 +1272,14 @@ def test_insert_value_none_materializes_the_token_ids(): result = _pump_insert(core, InsertParams(key=_key([1, 2, 3]))) assert result.prefix_len == 0 matched = core.match_prefix(MatchPrefixParams(key=_key([1, 2, 3]))) - assert matched.device_indices.tolist() == [1, 2, 3] + assert _path_indices(core, matched.last_device_node).tolist() == [1, 2, 3] @pytest.mark.parametrize("swa", [False, True]) def test_empty_match_result_is_root_anchored(swa): core = _swa_tree_core() if swa else _tree_core() empty = core.empty_match_result - assert empty.device_indices.numel() == 0 + assert empty.device_prefix_len == 0 assert empty.host_hit_length == 0 probe = core.match_prefix(MatchPrefixParams(key=_key([9]))) assert empty.best_match_node == probe.best_match_node @@ -1312,7 +1327,7 @@ def test_hicache_write_through_and_load_back_round_trip(): ) assert actions == [] result = core.match_prefix(MatchPrefixParams(key=_key([1, 2]))) - assert result.device_indices.tolist() == [50, 51] + assert _path_indices(core, result.last_device_node).tolist() == [50, 51] core.finish_load_back(leaf) core.sanity_check([], []) @@ -1852,7 +1867,9 @@ def test_empty_cache_salt_uses_the_default_namespace_at_the_binding(): ) ) result = binding.match_prefix(mem_cache.MatchParamsBinding(array("q", [1]))) - assert result.device_indices.tolist() == [10] + assert binding.collect_full_device_indices( + result.last_device_node_id, binding.root_node_handle(None) + ).tolist() == [10] @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") @@ -1870,13 +1887,13 @@ def test_cuda_core_resolves_the_current_device(): ) assert result.prefix_len == 0 matched = core.match_prefix(MatchPrefixParams(key=_key([1, 2, 3]))) - assert matched.device_indices.device == core.device - assert matched.device_indices.tolist() == [10, 11, 12] + assert _path_indices(core, matched.last_device_node).device == core.device + assert _path_indices(core, matched.last_device_node).tolist() == [10, 11, 12] # The value=None fallback also lands on the resolved device. fallback = _pump_insert(core, InsertParams(key=_key([7, 8]))) assert fallback.prefix_len == 0 matched = core.match_prefix(MatchPrefixParams(key=_key([7, 8]))) - assert matched.device_indices.tolist() == [7, 8] + assert _path_indices(core, matched.last_device_node).tolist() == [7, 8] def test_unsupported_component_sets_are_rejected(): @@ -1934,7 +1951,7 @@ def test_write_back_eviction_backs_up_then_drop_subtree_falls_back(): assert tracker[ComponentType.FULL] == 2 assert [t.tolist() for t in device_frees[ComponentType.FULL]] == [[10, 11]] result = core.match_prefix(MatchPrefixParams(key=_key([1, 2]))) - assert result.device_indices.numel() == 0 + assert result.device_prefix_len == 0 core.sanity_check([], []) @@ -2725,7 +2742,7 @@ def test_internal_swa_write_back_preserves_window_until_ack( MatchPrefixParams(key=_key(list(range(case.segment)))) ) assert matched.best_match_node == parent - assert matched.device_indices.numel() == 0 + assert matched.device_prefix_len == 0 assert matched.full_kv_hit_length == case.segment assert matched.swa_host_hit_length == case.segment assert matched.mamba_host_hit_length == int(with_mamba) @@ -3801,7 +3818,9 @@ def test_swa_match_uses_allocator_layout(backend, ring, hicache, has_swa_host_po cache.tree_core.has_swa_host_pool = has_swa_host_pool matched = cache.match_prefix(MatchPrefixParams(key=_key(list(range(8))))) assert matched.full_kv_hit_length == 8 - assert matched.device_indices.tolist() == (indices[:8].tolist() if ring else []) + assert cache.path_device_indices(matched.last_device_node).tolist() == ( + indices[:8].tolist() if ring else [] + ) def test_buffer_backup_snapshot_round_trips_and_detects_a_split(): @@ -4196,7 +4215,7 @@ def test_mamba_tree_round_trips_through_the_adapter(): assert not result.mamba_exist matched = core.match_prefix(MatchPrefixParams(key=_key([1, 2]))) - assert matched.device_indices.tolist() == [10, 11] + assert _path_indices(core, matched.last_device_node).tolist() == [10, 11] assert matched.mamba_host_hit_length == 0 assert core.mamba_evictable_size() == 1 assert core.all_mamba_values_flatten().tolist() == [7] @@ -5001,7 +5020,7 @@ def test_bigram_insert_then_a_longer_match_returns_the_inserted_prefix(): ) assert result.prefix_len == 0 matched = core.match_prefix(MatchPrefixParams(key=_bigram_key([1, 2, 3, 4, 5]))) - assert matched.device_indices.tolist() == [10, 11, 12] + assert _path_indices(core, matched.last_device_node).tolist() == [10, 11, 12] def test_bigram_match_diverges_on_the_pair_not_the_token(): @@ -5015,7 +5034,7 @@ def test_bigram_match_diverges_on_the_pair_not_the_token(): ) # (1, 2) matches; (2, 9) diverges from (2, 3) despite the shared token 2. matched = core.match_prefix(MatchPrefixParams(key=_bigram_key([1, 2, 9]))) - assert matched.device_indices.tolist() == [10] + assert _path_indices(core, matched.last_device_node).tolist() == [10] def test_bigram_empty_and_single_token_keys_match_nothing(): @@ -5028,9 +5047,9 @@ def test_bigram_empty_and_single_token_keys_match_nothing(): ), ) empty = core.match_prefix(MatchPrefixParams(key=_bigram_key([]))) - assert empty.device_indices.numel() == 0 + assert empty.device_prefix_len == 0 single = core.match_prefix(MatchPrefixParams(key=_bigram_key([1]))) - assert single.device_indices.numel() == 0 + assert single.device_prefix_len == 0 def test_bigram_insert_truncates_a_raw_length_value_to_the_bigram_count(): @@ -5044,7 +5063,7 @@ def test_bigram_insert_truncates_a_raw_length_value_to_the_bigram_count(): ) assert result.prefix_len == 0 matched = core.match_prefix(MatchPrefixParams(key=_bigram_key([1, 2, 3]))) - assert matched.device_indices.tolist() == [10, 11] + assert _path_indices(core, matched.last_device_node).tolist() == [10, 11] def test_bigram_insert_value_shorter_than_the_bigram_count_raises(): diff --git a/test/registered/unit/mem_cache/test_session_unified_radix_cache.py b/test/registered/unit/mem_cache/test_session_unified_radix_cache.py index 719ab577ca..cd84be3c55 100644 --- a/test/registered/unit/mem_cache/test_session_unified_radix_cache.py +++ b/test/registered/unit/mem_cache/test_session_unified_radix_cache.py @@ -75,11 +75,9 @@ def insert(cache, token_ids): def match_len(cache, token_ids) -> int: - return len( - cache.match_prefix( - MatchPrefixParams(key=RadixKey(array("q", token_ids))) - ).device_indices - ) + return cache.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", token_ids))) + ).device_prefix_len def register(cache, token_ids, session_id, generation=None): diff --git a/test/registered/unit/mem_cache/test_storage_prefetch_lifecycle.py b/test/registered/unit/mem_cache/test_storage_prefetch_lifecycle.py index 8c4e1ad11b..a28c756623 100644 --- a/test/registered/unit/mem_cache/test_storage_prefetch_lifecycle.py +++ b/test/registered/unit/mem_cache/test_storage_prefetch_lifecycle.py @@ -50,9 +50,8 @@ def _staged_fixture(full_match=2): root_node_handle=lambda extra_key=None: 0, inc_full_pin=Mock(), dec_full_pin=Mock(), - empty_match_result=SimpleNamespace( - last_device_node=0, device_indices=torch.arange(0) - ), + empty_match_result=SimpleNamespace(last_device_node=0), + empty_device_indices=torch.arange(0), ) cache.host_memory_mode = "buffer_only" cache.linker = None diff --git a/test/registered/unit/mem_cache/test_streaming_session_unit.py b/test/registered/unit/mem_cache/test_streaming_session_unit.py index 81d481c792..11949f6951 100644 --- a/test/registered/unit/mem_cache/test_streaming_session_unit.py +++ b/test/registered/unit/mem_cache/test_streaming_session_unit.py @@ -157,7 +157,7 @@ def test_preabort_detaches_session_and_preserves_slot(): page_size=16, match_results=[ MatchResult( - device_indices=torch.tensor([], dtype=torch.int64), + device_prefix_len=0, last_device_node=None, last_host_node=None, best_match_node=None, @@ -189,7 +189,7 @@ def test_preabort_detaches_session_and_preserves_slot(): assert slot.kv.req_pool_idx == 0 assert slot.kv.kv_committed_len == 48 assert slot.kv.kv_allocated_len == 48 - assert len(result.device_indices) == 0 + assert result.device_prefix_len == 0 def test_trim_overshoot_postcondition(): @@ -242,7 +242,7 @@ def test_session_rewind_keeps_component_cursors_page_aligned(operation, componen tree_cache.session.slots["session-a"] = SessionSlot(kv=req.detach_kv()) req = _FakeReq("session-a", req_pool_idx=0, committed=0, allocated=0) result = tree_cache.match_prefix(SimpleNamespace(req=req, key=list(range(38)))) - assert result.device_indices.tolist() == list(range(32)) + assert result.device_prefix_len == 32 # Rewound to floor_align(38) = 32; every cursor lands page-aligned. assert req.kv.kv_allocated_len == 32 diff --git a/test/registered/unit/mem_cache/test_unified_cache_linker.py b/test/registered/unit/mem_cache/test_unified_cache_linker.py index b08a84e779..4cb726cbd9 100644 --- a/test/registered/unit/mem_cache/test_unified_cache_linker.py +++ b/test/registered/unit/mem_cache/test_unified_cache_linker.py @@ -389,7 +389,7 @@ class TestUnifiedCacheLinkerPythonBackend(_TreeCoreBackendTestMixin, _InsertWalk match = consumer.match_prefix( MatchPrefixParams(key=RadixKey(array("q", tokens)), req=req) ) - self.assertEqual(match.device_indices.numel(), 0) + self.assertEqual(match.device_prefix_len, 0) self.assertEqual(match.host_hit_length, len(tokens)) self._apply_match_to_req(req, match) @@ -414,7 +414,7 @@ class TestUnifiedCacheLinkerPythonBackend(_TreeCoreBackendTestMixin, _InsertWalk final_match = consumer.match_prefix( MatchPrefixParams(key=RadixKey(array("q", tokens))) ) - self.assertEqual(final_match.device_indices.numel(), len(tokens)) + self.assertEqual(final_match.device_prefix_len, len(tokens)) self.assertEqual(consumer_linker.offload_calls, []) consumer.sanity_check() @@ -455,7 +455,7 @@ class TestUnifiedCacheLinkerPythonBackend(_TreeCoreBackendTestMixin, _InsertWalk match = consumer.match_prefix(MatchPrefixParams(key=lookup_key, req=req)) self.assertTrue(lookup_key.is_bigram) - self.assertEqual(match.device_indices.numel(), 4) + self.assertEqual(match.device_prefix_len, 4) self.assertEqual(match.host_hit_length, 4) self.assertEqual(len(consumer_linker.lookup_calls), 1) _, transfers = consumer_linker.lookup_calls[0] @@ -566,7 +566,7 @@ class TestUnifiedCacheLinkerPythonBackend(_TreeCoreBackendTestMixin, _InsertWalk match = consumer.match_prefix( MatchPrefixParams(key=RadixKey(array("q", tokens)), req=req) ) - self.assertEqual(match.device_indices.numel(), 2) + self.assertEqual(match.device_prefix_len, 2) self.assertEqual(match.host_hit_length, 2) self.assertEqual(match.swa_host_hit_length, 2) self._apply_match_to_req(req, match) @@ -575,7 +575,9 @@ class TestUnifiedCacheLinkerPythonBackend(_TreeCoreBackendTestMixin, _InsertWalk raced_match = consumer.match_prefix( MatchPrefixParams(key=RadixKey(array("q", tokens[:3]))) ) - raced_full = raced_match.device_indices[-1:].clone() + raced_full = consumer.path_device_indices(raced_match.last_device_node)[ + -1: + ].clone() raced_swa = consumer_allocator.translate_loc_from_full_to_swa(raced_full) loaded, loaded_node = consumer.init_load_back( @@ -609,7 +611,7 @@ class TestUnifiedCacheLinkerPythonBackend(_TreeCoreBackendTestMixin, _InsertWalk final_match = consumer.match_prefix( MatchPrefixParams(key=RadixKey(array("q", tokens[:4]))) ) - self.assertEqual(final_match.device_indices.numel(), 4) + self.assertEqual(final_match.device_prefix_len, 4) self.assertEqual(_device_lock_ref(consumer, loaded_node, ComponentType.FULL), 0) consumer.sanity_check() @@ -1171,7 +1173,7 @@ def test_linker_filters_request_relative_swa_from_lookup( wrapper = UnifiedCacheLinkerWrapper(cache, backend) assert wrapper._components == (full,) result = MatchResult( - device_indices=torch.empty(0, dtype=torch.int64), + device_prefix_len=0, last_device_node=0, last_host_node=0, best_match_node=0, @@ -1284,9 +1286,7 @@ def test_linker_load_preserves_swa_boundaries( components={ComponentType.FULL: full, ComponentType.SWA: swa}, token_to_kv_pool_allocator=_swa_allocator(swa_req_ring), tree_core=SimpleNamespace( - empty_match_result=SimpleNamespace( - device_indices=torch.empty(0, dtype=torch.int64) - ), + empty_device_indices=torch.empty(0, dtype=torch.int64), collect_full_device_indices=lambda node, ancestor: full_indices, mark_external_cache_stored_path=MagicMock(), ), diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_bench.py b/test/registered/unit/mem_cache/test_unified_radix_cache_bench.py index e11306e038..2c2350e704 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_cache_bench.py +++ b/test/registered/unit/mem_cache/test_unified_radix_cache_bench.py @@ -509,7 +509,7 @@ def bench_match_prefix( k = RadixKey(array("q", q)) r1 = env.tree.match_prefix(MatchPrefixParams(key=k)) r2 = env.tree.match_prefix(MatchPrefixParams(key=k)) - assert len(r1.device_indices) == len(r2.device_indices), "match not idempotent" + assert r1.device_prefix_len == r2.device_prefix_len, "match not idempotent" warmup = min(20, len(queries) // 10) return bench_api( @@ -613,7 +613,7 @@ def bench_release( for seq in env.seqs: key = RadixKey(array("q", seq)) mr = env.tree.match_prefix(MatchPrefixParams(key=key)) - matched_len = len(mr.device_indices) + matched_len = mr.device_prefix_len node = mr.last_device_node lr = env.tree.inc_lock_ref(node) @@ -626,9 +626,11 @@ def bench_release( lr.to_dec_params(), ) continue - kv_indices = torch.cat([mr.device_indices, v]) + kv_indices = torch.cat( + [env.tree.path_device_indices(mr.last_device_node), v] + ) else: - kv_indices = mr.device_indices + kv_indices = env.tree.path_device_indices(mr.last_device_node) req = env.make_req() req.origin_input_ids = array("q", seq) diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py index 7622b0192b..2e0c80ae6e 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py +++ b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py @@ -1075,7 +1075,7 @@ class TestUnifiedRadixCacheEagleHiCacheStorageKey(CustomTestCase): ) ) ) - self.assertEqual(len(match.device_indices), len(prefix_tokens) - 1) + self.assertEqual(match.device_prefix_len, len(prefix_tokens) - 1) req_id = CacheRequestHandle("bigram-anchor", 0) prefetch_key = RadixKey( @@ -1596,7 +1596,7 @@ class UnifiedRadixCacheSuite: return req def _apply_match_to_req(self, req, match): - req.prefix_len = len(match.device_indices) + req.prefix_len = match.device_prefix_len req.last_node = match.last_device_node req.last_host_node = match.last_host_node req.best_match_node = match.best_match_node @@ -1660,17 +1660,17 @@ class UnifiedRadixCacheSuite: m = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq_b)))) self.assertEqual(result.last_device_node, m.last_device_node) - self.assertEqual(len(m.device_indices), len(seq_b)) + self.assertEqual(m.device_prefix_len, len(seq_b)) m = cache.match_prefix( MatchPrefixParams(key=RadixKey(array("q", seq_a + self._make_seq(9000, 1)))) ) - self.assertEqual(len(m.device_indices), len(seq_a)) + self.assertEqual(m.device_prefix_len, len(seq_a)) m = cache.match_prefix( MatchPrefixParams(key=RadixKey(array("q", self._make_seq(5000, 2)))) ) - self.assertEqual(len(m.device_indices), 0) + self.assertEqual(m.device_prefix_len, 0) cache.sanity_check() @@ -1710,9 +1710,9 @@ class UnifiedRadixCacheSuite: default_match = cache.match_prefix( MatchPrefixParams(key=RadixKey(array("q", seq))) ) - self.assertEqual(len(first_match.device_indices), len(seq)) - self.assertEqual(len(second_match.device_indices), len(seq)) - self.assertEqual(len(default_match.device_indices), 0) + self.assertEqual(first_match.device_prefix_len, len(seq)) + self.assertEqual(second_match.device_prefix_len, len(seq)) + self.assertEqual(default_match.device_prefix_len, 0) self.assertNotEqual(first_match.last_device_node, second_match.last_device_node) cache.sanity_check() @@ -1731,12 +1731,12 @@ class UnifiedRadixCacheSuite: for seq in (branch_a, branch_b): m = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq)))) - self.assertEqual(len(m.device_indices), len(seq)) + self.assertEqual(m.device_prefix_len, len(seq)) m = cache.match_prefix( MatchPrefixParams(key=RadixKey(array("q", base + self._make_seq(999, 1)))) ) - self.assertEqual(len(m.device_indices), len(base)) + self.assertEqual(m.device_prefix_len, len(base)) cache.sanity_check() def test_evict_respects_lock_ref(self): @@ -1754,7 +1754,7 @@ class UnifiedRadixCacheSuite: self.assertGreaterEqual(result.num_tokens_evicted, len(seq_b)) m = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq_a)))) - self.assertEqual(len(m.device_indices), len(seq_a)) + self.assertEqual(m.device_prefix_len, len(seq_a)) # Unlock -> should now be evictable cache.dec_lock_ref( @@ -1864,7 +1864,7 @@ class UnifiedRadixCacheSuite: m = cache.match_prefix( MatchPrefixParams(key=RadixKey(array("q", all_ids[:aligned_len]))) ) - self.assertEqual(len(m.device_indices), aligned_len) + self.assertEqual(m.device_prefix_len, aligned_len) prompt_aligned_len = (len(input_ids) // ps) * ps if self.cfg.components == (ComponentType.FULL,): @@ -1886,7 +1886,7 @@ class UnifiedRadixCacheSuite: prompt_only = cache.match_prefix( MatchPrefixParams(key=RadixKey(array("q", all_ids[:aligned_len]))) ) - self.assertEqual(len(prompt_only.device_indices), prompt_aligned_len) + self.assertEqual(prompt_only.device_prefix_len, prompt_aligned_len) cache.sanity_check() def test_checkpoint_strips_thinking(self): @@ -1939,7 +1939,7 @@ class UnifiedRadixCacheSuite: m = cache.match_prefix( MatchPrefixParams(key=RadixKey(array("q", prompt_ids + output_ids))) ) - self.assertEqual(len(m.device_indices), prompt_aligned) + self.assertEqual(m.device_prefix_len, prompt_aligned) if self.cfg.has_mamba and self.cfg.enable_mamba_extra_buffer: node_value = cache.tree_core.get_component_device_value( m.last_device_node, ComponentType.MAMBA @@ -1979,7 +1979,7 @@ class UnifiedRadixCacheSuite: self.assertEqual(allocator.available_size(), avail_before + kv_len) m = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", tokens)))) - self.assertEqual(len(m.device_indices), 0) + self.assertEqual(m.device_prefix_len, 0) cache.sanity_check() def test_checkpoint_mid_flight(self): @@ -2046,7 +2046,7 @@ class UnifiedRadixCacheSuite: self.assertIsNotNone(_device_value(cache, live, ComponentType.SWA)) m = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", tokens)))) - self.assertEqual(len(m.device_indices), len(tokens)) + self.assertEqual(m.device_prefix_len, len(tokens)) cache.unlock(req.lock) cache.sanity_check() @@ -2079,13 +2079,13 @@ class UnifiedRadixCacheSuite: # Tree truncates unaligned tail internally, so it matches the seq prefix. unaligned = seq + list(range(9000, 9000 + ps - 1)) m = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", unaligned)))) - self.assertEqual(len(m.device_indices), len(seq)) + self.assertEqual(m.device_prefix_len, len(seq)) # Below-page-size key aligns to 0 -> no match. m = cache.match_prefix( MatchPrefixParams(key=RadixKey(array("q", seq[: ps - 1]))) ) - self.assertEqual(len(m.device_indices), 0) + self.assertEqual(m.device_prefix_len, 0) cache.sanity_check() @@ -2104,12 +2104,12 @@ class UnifiedRadixCacheSuite: # Mismatch in second page → only first page matches bad_page2 = seq[:ps] + [9999] * ps m = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", bad_page2)))) - self.assertEqual(len(m.device_indices), ps) + self.assertEqual(m.device_prefix_len, ps) # Mismatch in first page → 0 match bad_page1 = [9999] + seq[1:] m = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", bad_page1)))) - self.assertEqual(len(m.device_indices), 0) + self.assertEqual(m.device_prefix_len, 0) cache.sanity_check() def test_paged_checkpoint_unaligned_tail_freed(self): @@ -2144,7 +2144,7 @@ class UnifiedRadixCacheSuite: self.assertEqual(allocator.available_size(), avail_before + tail_extra) aligned = input_ids[: (len(input_ids) // ps) * ps] m = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", aligned)))) - self.assertEqual(len(m.device_indices), len(aligned)) + self.assertEqual(m.device_prefix_len, len(aligned)) cache.sanity_check() def test_mamba_evict_only(self): @@ -2175,7 +2175,7 @@ class UnifiedRadixCacheSuite: self.assertEqual(cache.mamba_evictable_size(), 0) m = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq_long)))) - self.assertEqual(len(m.device_indices), 0) + self.assertEqual(m.device_prefix_len, 0) cache.sanity_check() def test_mamba_evict_cascades_on_full_leaf(self): @@ -2203,7 +2203,7 @@ class UnifiedRadixCacheSuite: m = cache.match_prefix( MatchPrefixParams(key=RadixKey(array("q", seq)), cow_mamba=True, req=req2) ) - self.assertEqual(len(m.device_indices), len(seq)) + self.assertEqual(m.device_prefix_len, len(seq)) self.assertIsNotNone(req2.kv.mamba_pool_idx) src_value = _device_value(cache, m.last_device_node, ComponentType.MAMBA) @@ -2862,7 +2862,7 @@ class UnifiedRadixCacheSuite: side_node, c_node, b_node, a_node = pre[0], pre[2], pre[4], pre[6] m = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq_abc)))) - self.assertEqual(len(m.device_indices), len(seq_abc)) + self.assertEqual(m.device_prefix_len, len(seq_abc)) post = self._swa_lru_order(cache) # Matching seq_abc refreshes only the window cushion (C's capped nodes) @@ -2897,12 +2897,12 @@ class UnifiedRadixCacheSuite: self._insert(cache, allocator, req_to_token_pool, seq_side) m = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq_abc)))) - self.assertEqual(len(m.device_indices), len(seq_abc)) + self.assertEqual(m.device_prefix_len, len(seq_abc)) m_side_before = cache.match_prefix( MatchPrefixParams(key=RadixKey(array("q", seq_side))) ) - self.assertEqual(len(m_side_before.device_indices), len(seq_side)) + self.assertEqual(m_side_before.device_prefix_len, len(seq_side)) cache.evict(EvictParams(num_tokens=0, swa_num_tokens=self.cfg.page_size)) @@ -2910,7 +2910,7 @@ class UnifiedRadixCacheSuite: MatchPrefixParams(key=RadixKey(array("q", seq_side))) ) self.assertEqual( - len(m_side_after.device_indices), + m_side_after.device_prefix_len, len(seq_side), "Side branch SWA must survive eviction; oldest ancestors (A) " "should be evicted first under bounded SWA LRU refresh.", @@ -3054,7 +3054,7 @@ class UnifiedRadixCacheSuite: for _ in range(3): m = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq_abc)))) - self.assertEqual(len(m.device_indices), len(seq_abc)) + self.assertEqual(m.device_prefix_len, len(seq_abc)) cache.sanity_check() def test_tombstone_cleanup_respects_locked_parent(self): @@ -3227,8 +3227,8 @@ class UnifiedRadixCacheSuite: # seq_old should be gone (LRU), seq_new should remain m_old = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq_old)))) m_new = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq_new)))) - self.assertEqual(len(m_old.device_indices), 0) - self.assertEqual(len(m_new.device_indices), len(seq_new)) + self.assertEqual(m_old.device_prefix_len, 0) + self.assertEqual(m_new.device_prefix_len, len(seq_new)) cache.sanity_check() def test_evict_respects_priority_policy(self): @@ -3248,8 +3248,8 @@ class UnifiedRadixCacheSuite: MatchPrefixParams(key=RadixKey(array("q", seq_high))) ) m_low = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq_low)))) - self.assertEqual(len(m_high.device_indices), len(seq_high)) - self.assertEqual(len(m_low.device_indices), 0) + self.assertEqual(m_high.device_prefix_len, len(seq_high)) + self.assertEqual(m_low.device_prefix_len, 0) cache.sanity_check() def test_evict_multiple_independent_leaves(self): @@ -3294,7 +3294,7 @@ class UnifiedRadixCacheSuite: cache.evict(EvictParams(num_tokens=len(branch_a))) m_b = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", branch_b)))) - self.assertEqual(len(m_b.device_indices), len(branch_b)) + self.assertEqual(m_b.device_prefix_len, len(branch_b)) cache.dec_lock_ref( m.last_device_node, @@ -3327,7 +3327,7 @@ class UnifiedRadixCacheSuite: seq_b = self._make_seq(500, 2) self._insert(cache, allocator, req_to_token_pool, seq_b) m = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq_b)))) - self.assertEqual(len(m.device_indices), len(seq_b)) + self.assertEqual(m.device_prefix_len, len(seq_b)) cache.sanity_check() def test_swa_evict_internal_tombstone(self): @@ -3467,8 +3467,8 @@ class UnifiedRadixCacheSuite: ) ) ) - req.prefix_len = len(joint.device_indices) - joint_covers_span = len(joint.device_indices) >= len(f.key_tokens) + req.prefix_len = joint.device_prefix_len + joint_covers_span = joint.device_prefix_len >= len(f.key_tokens) adder = PrefillAdder.__new__(PrefillAdder) adder.tree_cache = cache with ( @@ -3500,11 +3500,11 @@ class UnifiedRadixCacheSuite: full_match.assert_not_called() else: full_match.assert_called_once() - collect_indices.assert_called_once() + # A load-back committing here reads the path again for its + # slot-ownership check. + collect_indices.assert_called() new_indices = ( - cache.tree_core.empty_match_result.device_indices - if loaded is None - else loaded[0] + cache.tree_core.empty_device_indices if loaded is None else loaded[0] ) if on_dispatched is not None: on_dispatched() @@ -3595,8 +3595,12 @@ class UnifiedRadixCacheSuite: self._insert(prod, prod_alloc, prod_rtp, seq) mp = prod.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq)))) prod_leaf = mp.last_device_node - self._fill_full_kv(prod_alloc, mp.device_indices, marker=7) - expected_k, expected_v = self._snapshot_full_kv(prod_alloc, mp.device_indices) + self._fill_full_kv( + prod_alloc, prod.path_device_indices(mp.last_device_node), marker=7 + ) + expected_k, expected_v = self._snapshot_full_kv( + prod_alloc, prod.path_device_indices(mp.last_device_node) + ) self._backup_node(prod, prod_leaf) self._write_path_to_l3(prod, prod_leaf) self._flush_l3_backups(prod) @@ -3624,9 +3628,11 @@ class UnifiedRadixCacheSuite: # Load the reloaded host prefix back to device and verify KV bytes. self._load_back_node(cons, host_node) - loaded_indices = cons.match_prefix( - MatchPrefixParams(key=RadixKey(array("q", seq))) - ).device_indices + loaded_indices = cons.path_device_indices( + cons.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", seq))) + ).last_device_node + ) self.assertEqual(len(loaded_indices), len(seq)) loaded_k, loaded_v = self._snapshot_full_kv(cons_alloc, loaded_indices) self.assertTrue(torch.equal(loaded_k, expected_k)) @@ -4021,8 +4027,12 @@ class UnifiedRadixCacheSuite: expected = None if marker is not None: m = prod.match_prefix(MatchPrefixParams(key=key)) - self._fill_full_kv(prod_alloc, m.device_indices, marker=marker) - expected = self._snapshot_full_kv(prod_alloc, m.device_indices) + self._fill_full_kv( + prod_alloc, prod.path_device_indices(m.last_device_node), marker=marker + ) + expected = self._snapshot_full_kv( + prod_alloc, prod.path_device_indices(m.last_device_node) + ) self._buffer_backup_and_wait(prod, leaf) return leaf, expected @@ -4449,9 +4459,11 @@ class UnifiedRadixCacheSuite: self.assertEqual(cons.token_to_kv_pool_allocator.available_size(), dev_avail0) self.assertEqual( len( - cons.match_prefix( - MatchPrefixParams(key=RadixKey(array("q", seq))) - ).device_indices + cons.path_device_indices( + cons.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", seq))) + ).last_device_node + ) ), 0, ) @@ -4495,7 +4507,9 @@ class UnifiedRadixCacheSuite: self.assertEqual(int(spliced.numel()), len(seq)) cons.ready_to_load_host_cache() m = cons.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq)))) - self.assertTrue(torch.equal(m.device_indices, spliced)) + self.assertTrue( + torch.equal(cons.path_device_indices(m.last_device_node), spliced) + ) # Ack: bounce freed, beliefs fed, tree holds no host values, and the # loaded KV bytes equal the producer's. @@ -4509,7 +4523,7 @@ class UnifiedRadixCacheSuite: ) mc = cons.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq)))) self.assertEqual(mc.host_hit_length, 0) - self.assertEqual(len(mc.device_indices), len(seq)) + self.assertEqual(mc.device_prefix_len, len(seq)) leaf = mc.last_device_node for cur in self._path_chain(cons, leaf): for component_type in cons.tree_components: @@ -4527,7 +4541,9 @@ class UnifiedRadixCacheSuite: transfer.keys ) ) - loaded_k, loaded_v = self._snapshot_full_kv(cons_alloc, mc.device_indices) + loaded_k, loaded_v = self._snapshot_full_kv( + cons_alloc, cons.path_device_indices(mc.last_device_node) + ) self.assertTrue(torch.equal(loaded_k, expected_k)) self.assertTrue(torch.equal(loaded_v, expected_v)) self.assertEqual(cons.cache_controller.prefetch_tokens_occupied, 0) @@ -4605,7 +4621,7 @@ class UnifiedRadixCacheSuite: self.assertEqual(len(loaded), len(seq)) key = RadixKey(array("q", seq), extra_key=extra_key, cache_salt=cache_salt) self.assertEqual( - len(cons.match_prefix(MatchPrefixParams(key=key)).device_indices), len(seq) + cons.match_prefix(MatchPrefixParams(key=key)).device_prefix_len, len(seq) ) for miss in ( RadixKey(array("q", seq)), @@ -4613,7 +4629,7 @@ class UnifiedRadixCacheSuite: RadixKey(array("q", seq), extra_key="adapter-b", cache_salt=cache_salt), ): self.assertEqual( - len(cons.match_prefix(MatchPrefixParams(key=miss)).device_indices), 0 + cons.match_prefix(MatchPrefixParams(key=miss)).device_prefix_len, 0 ) cons.sanity_check() @@ -4678,7 +4694,7 @@ class UnifiedRadixCacheSuite: self.assertEqual(cons2.buffer_pipeline.anchor_locked_tokens_, 0) self.assertEqual(_device_lock_ref(cons2, anchor, ComponentType.FULL), lock_ref) self.assertEqual( - len(cons2.match_prefix(MatchPrefixParams(key=key)).device_indices), len(seq) + cons2.match_prefix(MatchPrefixParams(key=key)).device_prefix_len, len(seq) ) cons2.sanity_check() @@ -4851,7 +4867,7 @@ class UnifiedRadixCacheSuite: req = SimpleNamespace( rid=req_id.rid, cache_request_handle=req_id, - prefix_len=len(cons.tree_core.empty_match_result.device_indices), + prefix_len=cons.tree_core.empty_match_result.device_prefix_len, kv=SimpleNamespace(cache_protected_len=0), ) self.assertTrue(cons.buffer_pipeline.prepare_staged_prefetch(req)) @@ -4953,9 +4969,13 @@ class UnifiedRadixCacheSuite: # Sibling publishes the identical span (live FULL + SWA). self._insert(cons, cons_alloc, cons_rtp, seq) sib = cons.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq)))) - self.assertEqual(len(sib.device_indices), len(seq)) - self._fill_full_kv(cons_alloc, sib.device_indices, marker=3) - sib_kv = self._snapshot_full_kv(cons_alloc, sib.device_indices) + self.assertEqual(sib.device_prefix_len, len(seq)) + self._fill_full_kv( + cons_alloc, cons.path_device_indices(sib.last_device_node), marker=3 + ) + sib_kv = self._snapshot_full_kv( + cons_alloc, cons.path_device_indices(sib.last_device_node) + ) dev_avail0 = cons.token_to_kv_pool_allocator.available_size() # Consume with the batch-stale empty prefix view: the live unified @@ -4970,8 +4990,15 @@ class UnifiedRadixCacheSuite: self.assertEqual(cons.cache_controller.prefetch_tokens_occupied, 0) self.assertEqual(self._host_avail_sizes(cons), avail0) m = cons.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq)))) - self.assertTrue(torch.equal(m.device_indices, sib.device_indices)) - k, v = self._snapshot_full_kv(cons_alloc, m.device_indices) + self.assertTrue( + torch.equal( + cons.path_device_indices(m.last_device_node), + cons.path_device_indices(sib.last_device_node), + ) + ) + k, v = self._snapshot_full_kv( + cons_alloc, cons.path_device_indices(m.last_device_node) + ) self.assertTrue(torch.equal(k, sib_kv[0])) self.assertTrue(torch.equal(v, sib_kv[1])) cons.storage_metrics_collector.log_storage_prefetch_unfulfilled_tokens.assert_called_once_with( @@ -5019,7 +5046,7 @@ class UnifiedRadixCacheSuite: ) ) masked = cons.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq)))) - self.assertEqual(len(masked.device_indices), 0, "unified match not masked") + self.assertEqual(masked.device_prefix_len, 0, "unified match not masked") self.assertEqual(masked.full_kv_hit_length, len(seq), "live FULL not resident") masked_full = value[: len(seq)].clone() # Model the real SWA-eviction path: tombstoned positions have their @@ -5113,7 +5140,7 @@ class UnifiedRadixCacheSuite: self.assertEqual(self._host_avail_sizes(cons), avail0) after = cons.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq)))) self.assertEqual(after.full_kv_hit_length, len(seq)) - self.assertEqual(len(after.device_indices), len(seq)) + self.assertEqual(after.device_prefix_len, len(seq)) # Only the fetched SWA tail made the resident FULL reusable. self.assertEqual((req.storage_hit_start, req.storage_hit_length), (0, len(seq))) cons.storage_metrics_collector.log_storage_prefetch_unfulfilled_tokens.assert_not_called() @@ -5146,7 +5173,7 @@ class UnifiedRadixCacheSuite: ) ) live = cons.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq)))) - self.assertEqual(len(live.device_indices), 0, "unified match not masked") + self.assertEqual(live.device_prefix_len, 0, "unified match not masked") self.assertEqual(live.full_kv_hit_length, split_at, "FULL run not partial") head_full = value[:split_at].clone() cons_alloc.free_swa(head_full) @@ -5375,9 +5402,13 @@ class UnifiedRadixCacheSuite: head = seq[: self.cfg.page_size] self._insert(cons, cons_alloc, cons_rtp, head) sib = cons.match_prefix(MatchPrefixParams(key=RadixKey(array("q", head)))) - self.assertEqual(len(sib.device_indices), len(head)) - self._fill_full_kv(cons_alloc, sib.device_indices, marker=3) - head_k, head_v = self._snapshot_full_kv(cons_alloc, sib.device_indices) + self.assertEqual(sib.device_prefix_len, len(head)) + self._fill_full_kv( + cons_alloc, cons.path_device_indices(sib.last_device_node), marker=3 + ) + head_k, head_v = self._snapshot_full_kv( + cons_alloc, cons.path_device_indices(sib.last_device_node) + ) if window_sized: # FULL shrank below a window; the complete SWA staging must survive. @@ -5396,8 +5427,8 @@ class UnifiedRadixCacheSuite: req = SimpleNamespace( rid=req_id.rid, cache_request_handle=req_id, - prefix_len=len(sib.device_indices), - kv=SimpleNamespace(cache_protected_len=len(sib.device_indices)), + prefix_len=sib.device_prefix_len, + kv=SimpleNamespace(cache_protected_len=sib.device_prefix_len), ) self.assertTrue(cons.buffer_pipeline.prepare_staged_prefetch(req)) self.assertEqual(req.host_hit_length, len(seq) - len(head)) @@ -5407,12 +5438,21 @@ class UnifiedRadixCacheSuite: self.assertEqual(int(spliced.numel()), len(seq) - len(head)) m = cons.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq)))) - self.assertEqual(len(m.device_indices), len(seq)) - self.assertTrue(torch.equal(m.device_indices[: len(head)], sib.device_indices)) - k, v = self._snapshot_full_kv(cons_alloc, m.device_indices[len(head) :]) + self.assertEqual(m.device_prefix_len, len(seq)) + self.assertTrue( + torch.equal( + cons.path_device_indices(m.last_device_node)[: len(head)], + cons.path_device_indices(sib.last_device_node), + ) + ) + k, v = self._snapshot_full_kv( + cons_alloc, cons.path_device_indices(m.last_device_node)[len(head) :] + ) self.assertTrue(torch.equal(k, expected_k[len(head) :])) self.assertTrue(torch.equal(v, expected_v[len(head) :])) - hk, hv = self._snapshot_full_kv(cons_alloc, m.device_indices[: len(head)]) + hk, hv = self._snapshot_full_kv( + cons_alloc, cons.path_device_indices(m.last_device_node)[: len(head)] + ) self.assertTrue(torch.equal(hk, head_k)) self.assertTrue(torch.equal(hv, head_v)) @@ -5461,7 +5501,7 @@ class UnifiedRadixCacheSuite: req = mock.Mock( rid=req_id.rid, cache_request_handle=req_id, - prefix_len=len(live.device_indices), + prefix_len=live.device_prefix_len, last_node=live.last_device_node, ) self.assertTrue(cons.buffer_pipeline.prepare_staged_prefetch(req)) @@ -5622,9 +5662,11 @@ class UnifiedRadixCacheSuite: self.assertEqual(int(spliced.numel()), len(seq_ab) - len(seq_a)) self.assertEqual( len( - cons2.match_prefix( - MatchPrefixParams(key=RadixKey(array("q", seq_ab))) - ).device_indices + cons2.path_device_indices( + cons2.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", seq_ab))) + ).last_device_node + ) ), len(seq_ab), ) @@ -5653,9 +5695,11 @@ class UnifiedRadixCacheSuite: self.assertEqual(operation.storage_hit_count, len(stored)) self.assertEqual( len( - cons3.match_prefix( - MatchPrefixParams(key=RadixKey(array("q", stored))) - ).device_indices + cons3.path_device_indices( + cons3.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", stored))) + ).last_device_node + ) ), 0, "sub-window hit should not be loaded", @@ -5846,7 +5890,7 @@ class UnifiedRadixCacheSuite: # The chain is gone entirely: no device hit, no host hit. m = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq)))) - self.assertEqual(len(m.device_indices), 0) + self.assertEqual(m.device_prefix_len, 0) self.assertEqual(m.host_hit_length, 0) cache.sanity_check() @@ -5896,7 +5940,7 @@ class UnifiedRadixCacheSuite: m = cache.match_prefix( MatchPrefixParams(key=RadixKey(array("q", parent_seq))) ) - self.assertEqual(len(m.device_indices), len(parent_seq)) + self.assertEqual(m.device_prefix_len, len(parent_seq)) cache.dec_host_lock_ref(child, child_lock_params) # Unpinned: the subtree drops and the child's host slots return. @@ -5904,7 +5948,7 @@ class UnifiedRadixCacheSuite: self.assertGreaterEqual(result.num_tokens_evicted, len(parent_seq)) self.assertEqual(host_pool.available_size(), baseline_host) m = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", child_seq)))) - self.assertEqual(len(m.device_indices), 0) + self.assertEqual(m.device_prefix_len, 0) self.assertEqual(m.host_hit_length, 0) cache.sanity_check() @@ -6268,7 +6312,7 @@ class UnifiedRadixCacheSuite: lr.to_dec_params(), ) m = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", leaf)))) - self.assertGreaterEqual(len(m.device_indices), len(base)) + self.assertGreaterEqual(m.device_prefix_len, len(base)) cache.sanity_check() def test_hicache_partial_match_splits_evicted_backed_up_node(self): @@ -6293,7 +6337,7 @@ class UnifiedRadixCacheSuite: m = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", query)))) - self.assertEqual(len(m.device_indices), 0) + self.assertEqual(m.device_prefix_len, 0) self.assertEqual(m.last_device_node, cache.root_node_handle()) # Locate the host prefix via last_host_node and rebuild prefix/suffix @@ -6387,7 +6431,7 @@ class UnifiedRadixCacheSuite: m = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", base)))) node = m.last_device_node - original_device_indices = m.device_indices.clone() + original_device_indices = cache.path_device_indices(m.last_device_node).clone() self._fill_full_kv(allocator, original_device_indices, marker=3) expected_k, expected_v = self._snapshot_full_kv( allocator, original_device_indices @@ -6415,9 +6459,11 @@ class UnifiedRadixCacheSuite: self.assertFalse(cache.tree_core.is_full_device_evicted(node)) self.assertIsNotNone(_device_value(cache, node, ComponentType.FULL)) # Gather the whole reloaded prefix via match (a leaf may be split). - loaded_indices = cache.match_prefix( - MatchPrefixParams(key=RadixKey(array("q", base))) - ).device_indices + loaded_indices = cache.path_device_indices( + cache.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", base))) + ).last_device_node + ) loaded_k, loaded_v = self._snapshot_full_kv(allocator, loaded_indices) self.assertTrue(torch.equal(loaded_k, expected_k)) self.assertTrue(torch.equal(loaded_v, expected_v)) @@ -6656,7 +6702,7 @@ class UnifiedRadixCacheSuite: ) ) self.assertEqual(last_node, node) - self.assertEqual(len(m.device_indices) + len(new_indices), len(seq_a)) + self.assertEqual(m.device_prefix_len + len(new_indices), len(seq_a)) self.assertIsNotNone( _device_value(cache, node, ComponentType.MAMBA), "mamba state revived on device", @@ -6719,7 +6765,7 @@ class UnifiedRadixCacheSuite: m = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq_a)))) self.assertEqual( - len(m.device_indices), + m.device_prefix_len, 0, "write_through keeps the legacy drop: frontier capped at root", ) @@ -6768,7 +6814,7 @@ class UnifiedRadixCacheSuite: "SWA KV demoted to host, not dropped", ) self.assertEqual( - len(m.device_indices) + m.host_hit_length, + m.device_prefix_len + m.host_hit_length, len(seq_a), "the full prefix stays servable", ) @@ -6788,7 +6834,7 @@ class UnifiedRadixCacheSuite: ) ) self.assertEqual(last_node, node) - self.assertEqual(len(m.device_indices) + len(new_indices), len(seq_a)) + self.assertEqual(m.device_prefix_len + len(new_indices), len(seq_a)) self.assertIsNotNone( _device_value(cache, node, ComponentType.SWA), "SWA KV revived on device", @@ -6881,7 +6927,7 @@ class UnifiedRadixCacheSuite: result = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq)))) - self.assertEqual(len(result.device_indices), len(seq)) + self.assertEqual(result.device_prefix_len, len(seq)) self.assertEqual(result.best_match_node, result.last_device_node) self.assertEqual(result.last_host_node, result.last_device_node) self.assertEqual(result.host_hit_length, 0) @@ -6962,7 +7008,7 @@ class UnifiedRadixCacheSuite: zeroed = zero_match_result(cache, result) - self.assertEqual(len(zeroed.device_indices), 0) + self.assertEqual(zeroed.device_prefix_len, 0) # zeroed results must carry the root's NodeId, not the raw node self.assertEqual(zeroed.best_match_node, cache.root_node_handle()) # The env-gated force-miss path passes the request's extra key. @@ -7021,7 +7067,7 @@ class UnifiedRadixCacheSuite: self.assertEqual(result.best_match_node, leaf) self.assertEqual(result.last_device_node, parent) self.assertEqual( - len(result.device_indices), len(tokens) - _node_key_length(cache, leaf) + result.device_prefix_len, len(tokens) - _node_key_length(cache, leaf) ) self.assertEqual(result.host_hit_length, _node_key_length(cache, leaf)) @@ -7212,7 +7258,7 @@ class UnifiedRadixCacheSuite: self._run_prefetch_to_completion(cons, CacheRequestHandle("r", 0)) cons.drain_storage_control_queues() m = cons.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq)))) - self.assertEqual(len(m.device_indices), len(seq)) + self.assertEqual(m.device_prefix_len, len(seq)) cons.sanity_check() def test_hicache_swa_host_best_match_keeps_device_anchor(self): @@ -7235,7 +7281,7 @@ class UnifiedRadixCacheSuite: self.assertEqual(result.best_match_node, leaf) self.assertEqual(result.last_device_node, parent) self.assertEqual( - len(result.device_indices), len(tokens) - _node_key_length(cache, leaf) + result.device_prefix_len, len(tokens) - _node_key_length(cache, leaf) ) self.assertEqual(result.host_hit_length, _node_key_length(cache, leaf)) self.assertEqual(result.swa_host_hit_length, _node_key_length(cache, leaf)) @@ -7255,7 +7301,7 @@ class UnifiedRadixCacheSuite: self.assertEqual(result.best_match_node, leaf) self.assertEqual(result.last_device_node, parent) self.assertEqual( - len(result.device_indices), len(tokens) - _node_key_length(cache, leaf) + result.device_prefix_len, len(tokens) - _node_key_length(cache, leaf) ) self.assertEqual(result.host_hit_length, 0) self.assertEqual(result.swa_host_hit_length, _node_key_length(cache, leaf)) @@ -7301,7 +7347,7 @@ class UnifiedRadixCacheSuite: rematch = cache.match_prefix( MatchPrefixParams(key=RadixKey(array("q", tokens))) ) - self.assertEqual(len(rematch.device_indices), result.swa_branching_seqlen) + self.assertEqual(rematch.device_prefix_len, result.swa_branching_seqlen) self.assertIsNone(rematch.swa_branching_seqlen) def test_swa_branching_seqlen_uses_host_full_hit(self): @@ -7361,7 +7407,7 @@ class UnifiedRadixCacheSuite: rematch = cache.match_prefix( MatchPrefixParams(key=RadixKey(array("q", tokens))) ) - self.assertEqual(len(rematch.device_indices), result.swa_branching_seqlen) + self.assertEqual(rematch.device_prefix_len, result.swa_branching_seqlen) self.assertIsNone(rematch.swa_branching_seqlen) def test_swa_branching_seqlen_caps_insert_after_forward(self): @@ -7526,7 +7572,7 @@ class UnifiedRadixCacheSuite: self.assertEqual(result.best_match_node, parent) self.assertEqual(result.last_device_node, parent) - self.assertEqual(len(result.device_indices), chunk_size) + self.assertEqual(result.device_prefix_len, chunk_size) self.assertEqual(result.host_hit_length, 0) self.assertEqual(result.full_kv_hit_length, len(tokens)) self.assertEqual(result.mamba_branching_seqlen, 2 * chunk_size) @@ -7570,7 +7616,7 @@ class UnifiedRadixCacheSuite: self.assertEqual(result.best_match_node, parent) self.assertEqual(result.last_device_node, parent) - self.assertEqual(len(result.device_indices), chunk_size) + self.assertEqual(result.device_prefix_len, chunk_size) self.assertEqual(result.host_hit_length, 0) self.assertEqual(result.full_kv_hit_length, len(tokens)) branching_seqlen = 2 * chunk_size @@ -7597,7 +7643,7 @@ class UnifiedRadixCacheSuite: MatchPrefixParams(key=RadixKey(array("q", tokens))) ) - self.assertEqual(len(second_match.device_indices), branching_seqlen) + self.assertEqual(second_match.device_prefix_len, branching_seqlen) self.assertIsNone(second_match.mamba_branching_seqlen) self.assertIsNotNone( _host_value(cache, second_match.last_device_node, ComponentType.MAMBA) @@ -8348,9 +8394,7 @@ class UnifiedRadixCacheSuite: ) result = MatchResult( - device_indices=torch.empty( - (0,), dtype=torch.int64, device=cache.device - ), + device_prefix_len=0, last_device_node=leaf, last_host_node=leaf, best_match_node=leaf, @@ -8547,7 +8591,7 @@ class UnifiedRadixCacheSuite: ps = self.cfg.page_size base = MatchResult( - device_indices=torch.empty((0,), dtype=torch.int64, device=cache.device), + device_prefix_len=0, last_device_node=x, last_host_node=y, best_match_node=x, @@ -9202,7 +9246,7 @@ class TestMambaCheckpointGrid(CustomTestCase): cache.tree_core.set_component_device_value_raw(leaf, ComponentType.MAMBA, None) result = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", tokens)))) self.assertEqual(result.full_kv_hit_length, full_hit_length) - self.assertEqual(len(result.device_indices), tree_page_size) + self.assertEqual(result.device_prefix_len, tree_page_size) return result.mamba_branching_seqlen def test_grid_follows_the_widened_tree_page(self): @@ -9264,7 +9308,7 @@ class TestMambaFinishedOvershootCheckpoint(CustomTestCase): match = cache.match_prefix( MatchPrefixParams(key=RadixKey(array("q", tokens))) ) - self.assertEqual(len(match.device_indices), expected_len) + self.assertEqual(match.device_prefix_len, expected_len) if expected_len: node_value = cache.tree_core.get_component_device_value( match.last_device_node, ComponentType.MAMBA @@ -11619,7 +11663,7 @@ class TestStreamingSessionLockLifecycle(CustomTestCase): match = cache.match_prefix( MatchPrefixParams(key=RadixKey(array("q", range(1, 13)))) ) - self.assertEqual(len(match.device_indices), len(tokens)) + self.assertEqual(match.device_prefix_len, len(tokens)) cache.session.release_session("s") cache.sanity_check()