diff --git a/python/sglang/srt/disaggregation/decode.py b/python/sglang/srt/disaggregation/decode.py index efe1b69a19..bc732bca26 100644 --- a/python/sglang/srt/disaggregation/decode.py +++ b/python/sglang/srt/disaggregation/decode.py @@ -83,8 +83,8 @@ from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator from sglang.srt.mem_cache.allocator.swa import is_swa_req_ring from sglang.srt.mem_cache.base_prefix_cache import ( BasePrefixCache, - DecLockRefParams, EvictParams, + TreeLock, ) from sglang.srt.mem_cache.common import ( RetractionBackup, @@ -147,7 +147,7 @@ def _bind_root_prefix(req: Req, tree_cache: BasePrefixCache) -> None: req.last_node = tree_cache.root_node_handle(req.extra_key) req.last_host_node = req.last_node req.best_match_node = req.last_node - req.lock_receipt = DecLockRefParams() + req.lock = None req.kv.cache_protected_len = 0 req.kv.cache_inserted_len = 0 req.num_matched_prefix_tokens = 0 @@ -351,9 +351,8 @@ class DecodeRequest: # HiCache Status prefix_match: Optional[DecodePrefixMatch] = None hicache_restored_kv_indices: Optional[torch.Tensor] = None - hicache_restored_node: Any = None - # Receipt for the inc_lock_ref held on hicache_restored_node. - hicache_restore_lock_receipt: Optional[DecLockRefParams] = None + # The lock held on the restored node until the commit hands it to the req. + hicache_restore_lock: Optional[TreeLock] = None hicache_load_consumer_index: int = -1 hicache_restore_status: HiCacheRestoreResult = HiCacheRestoreResult.PENDING @@ -503,13 +502,6 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): swa_budget_tokens=swa_allocatable_tokens, ) - def _release_matched_prefix_lock(self, req: Req) -> None: - if req.swa_prefix_lock_released: - self.tree_cache.dec_lock_ref(req.last_node, req.lock_receipt, skip_swa=True) - req.swa_prefix_lock_released = False - else: - self.tree_cache.dec_lock_ref(req.last_node, req.lock_receipt) - def _reclaim_swa_tail_capacity( self, swa_tail_len: int, req_id: str, *, full_len: int = 0 ) -> Optional[str]: @@ -801,12 +793,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): include_req=True, max_prefix_len=max_prefix_len, ) - # Keep aggregated scheduling semantics while preserving the SWA lock - # boundary needed for the matching dec_lock_ref; the full receipt - # travels on the req so every later release mirrors this acquire. - req.lock_receipt = self.tree_cache.inc_lock_ref( - result.last_device_node - ).to_dec_params() + req.lock = self.tree_cache.lock(result.last_device_node) return self._build_decode_prefix_match( req, result, max_prefix_len=max_prefix_len ) @@ -1436,12 +1423,9 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): if ( uses_swa_tail_prealloc and prefix_match.l1_prefix_len > 0 - and hasattr(self.tree_cache, "dec_swa_lock_only") + and hasattr(self.tree_cache, "release_swa") ): - self.tree_cache.dec_swa_lock_only( - decode_req.req.last_node, decode_req.req.lock_receipt - ) - decode_req.req.swa_prefix_lock_released = True + self.tree_cache.release_swa(decode_req.req.lock) required_alloc_tokens = self._required_alloc_tokens( fill_len=fill_len, prefix_len=prefix_len @@ -1494,7 +1478,8 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): swa_allocatable_tokens=swa_allocatable_tokens, ): if prefix_match is not None and prefix_match.l1_prefix_len > 0: - self._release_matched_prefix_lock(decode_req.req) + self.tree_cache.unlock(decode_req.req.lock) + decode_req.req.lock = None break if swa_allocatable_tokens is not None: @@ -1505,7 +1490,8 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): ) if reclaim_error is not None: if prefix_match is not None and prefix_match.l1_prefix_len > 0: - self._release_matched_prefix_lock(decode_req.req) + self.tree_cache.unlock(decode_req.req.lock) + decode_req.req.lock = None logger.error(reclaim_error) prepare_abort( decode_req.req, diff --git a/python/sglang/srt/disaggregation/decode_hicache_mixin.py b/python/sglang/srt/disaggregation/decode_hicache_mixin.py index 4127f68545..9e1bcbdd4c 100644 --- a/python/sglang/srt/disaggregation/decode_hicache_mixin.py +++ b/python/sglang/srt/disaggregation/decode_hicache_mixin.py @@ -161,7 +161,7 @@ class DecodeHiCachePreallocMixin: for dr in self.transfer_queue.queue if dr.prefix_match is not None and dr.hicache_restore_status == HiCacheRestoreResult.PENDING - and dr.hicache_restored_node is None + and dr.hicache_restore_lock is None ) @@ -192,19 +192,14 @@ class DecodeHiCacheTransferMixin: self.tree_cache.finish( decode_req.req.cache_request_handle, CacheRequestOutcome.ABORT ) - if decode_req.hicache_restored_node is not None: - self.tree_cache.dec_lock_ref( - decode_req.hicache_restored_node, - decode_req.hicache_restore_lock_receipt, - ) - decode_req.hicache_restored_node = None - decode_req.hicache_restore_lock_receipt = None + self.tree_cache.unlock(decode_req.hicache_restore_lock) + decode_req.hicache_restore_lock = None def _try_hicache_queue_load_back(self, dr: DecodeRequest) -> bool: """Queue one L2->L1 load_back op for ``dr``; True iff a DMA was queued. - On success, ``dr.hicache_restored_node`` and ``hicache_restored_kv_indices`` - are populated, and an inc_lock_ref is held until commit/abort. + On success, ``dr.hicache_restore_lock`` and ``hicache_restored_kv_indices`` + are populated; the lock is held until commit/abort. Trivial cases (all-on-device / no needed coverage) auto-flip to READY. Failback paths flip to FAILED. """ @@ -233,10 +228,7 @@ class DecodeHiCacheTransferMixin: ) ) # The rematch repointed req.last_node to feed init_load_back's device - # boundary, but the prealloc lock and the receipt on the req still - # belong to pm.last_device_node; restore the pairing so any release - # before the commit hands over the restored lock hits the right node - # (the receipt's anchor makes a mispaired release assert). + # boundary; point it back at the prefix the prealloc matched and locked. dr.req.last_node = pm.last_device_node # Failback: total coverage < required prefix means device alloc likely failed. if len(rematch.device_indices) + len(new_indices) < pm.decode_prefix_len: @@ -257,10 +249,7 @@ class DecodeHiCacheTransferMixin: dr.hicache_restored_kv_indices = torch.cat( [rematch.device_indices[pm.l1_prefix_len :], new_indices] )[: pm.restore_token_count] - dr.hicache_restored_node = restored_node - dr.hicache_restore_lock_receipt = self.tree_cache.inc_lock_ref( - restored_node - ).to_dec_params() + dr.hicache_restore_lock = self.tree_cache.lock(restored_node) if len(new_indices) == 0: # Whole prefix already on device; no DMA needed. @@ -284,7 +273,7 @@ class DecodeHiCacheTransferMixin: # Phase A: advance in-flight DMAs to READY. for dr in active: if ( - dr.hicache_restored_node is not None + dr.hicache_restore_lock is not None and self.tree_cache.is_load_back_event_done( dr.hicache_load_consumer_index ) @@ -297,8 +286,7 @@ class DecodeHiCacheTransferMixin: queued = [ dr for dr in active - if dr.hicache_restored_node is None - and self._try_hicache_queue_load_back(dr) + if dr.hicache_restore_lock is None and self._try_hicache_queue_load_back(dr) ] if not queued: return @@ -318,16 +306,9 @@ class DecodeHiCacheTransferMixin: return req = decode_req.req - restored_node = decode_req.hicache_restored_node - restored_lock_receipt = decode_req.hicache_restore_lock_receipt - assert restored_node is not None - assert restored_lock_receipt is not None - # Release preallocation before installing the restored lock receipt. - self.tree_cache.dec_lock_ref( - prefix_match.last_device_node, - req.lock_receipt, - skip_swa=req.swa_prefix_lock_released, - ) + restore_lock = decode_req.hicache_restore_lock + assert restore_lock is not None + self.tree_cache.unlock(req.lock) self.tree_cache.req_to_token_pool.write( ( @@ -339,9 +320,7 @@ class DecodeHiCacheTransferMixin: req.prefix_indices = torch.cat( [prefix_match.prefix_indices, decode_req.hicache_restored_kv_indices] ) - req.last_node = restored_node - req.lock_receipt = restored_lock_receipt - req.swa_prefix_lock_released = False + req.last_node = restore_lock.node + req.lock = restore_lock # Prevent abort cleanup from releasing the transferred lock. - decode_req.hicache_restored_node = None - decode_req.hicache_restore_lock_receipt = None + decode_req.hicache_restore_lock = None diff --git a/python/sglang/srt/disaggregation/decode_kvcache_offload_manager.py b/python/sglang/srt/disaggregation/decode_kvcache_offload_manager.py index bd83c8167b..6ee0838bef 100644 --- a/python/sglang/srt/disaggregation/decode_kvcache_offload_manager.py +++ b/python/sglang/srt/disaggregation/decode_kvcache_offload_manager.py @@ -337,7 +337,7 @@ class DecodeKVCacheOffloadManager: def _release_finished_req(self, req: Req): # Defensive guard: ReqToTokenPool.free sets req_pool_idx to None, # so a previously-released request must be skipped here to avoid - # non-idempotent side effects (double free, double unpin). + # non-idempotent side effects (double free, double unlock). if req.kv.req_pool_idx is None or req.kv.req_pool_idx == -1: return @@ -347,7 +347,8 @@ class DecodeKVCacheOffloadManager: self.tree_cache.free_kv_row( req.kv, [(req.kv.cache_protected_len, req.kv.kv_allocated_len)] ) - self.tree_cache.unpin(req) + self.tree_cache.unlock(req.lock) + req.lock = None self.req_to_token_pool.free(req) req.kv.mark_kv_released() self.offloaded_state.pop(req, None) diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index d1e411660b..eef421653b 100755 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -107,8 +107,8 @@ from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator from sglang.srt.mem_cache.base_prefix_cache import ( BasePrefixCache, CacheRequestHandle, - DecLockRefParams, MatchPrefixParams, + TreeLock, zero_match_result, ) from sglang.srt.mem_cache.common import ( @@ -1207,14 +1207,12 @@ class Req(ReqDllmMixin): self.host_hit_is_storage = False self.storage_prefetch_retry_attempts = 0 self.staged_prefetch_plan: Optional[StagedPrefetchPlan] = None - # Receipt of the tree lock held on last_node (anchor, SWA boundary, - # skipped components); every release replays it unchanged. - self.lock_receipt: DecLockRefParams = DecLockRefParams() + # The tree lock this request holds; None while it runs on a session + # slot's record or holds no lock. + self.lock: Optional[TreeLock] = None # Device/host prefix used to plan the latest L3 lookup. Admission uses # it to detect newly exposed storage demand after queue-time eviction. self.storage_prefetch_last_match_len: Optional[int] = None - # Whether the prefill-time SWA tree lock has been released early - self.swa_prefix_lock_released: bool = False # Logical-page KV sharding: rotation base of the chain this request # extends (owner of position-page P is (base + P) % shard_size). # Refreshed at every sharded alloc — read through last_node, or drawn @@ -1984,8 +1982,7 @@ class Req(ReqDllmMixin): self.kv.cache_inserted_len = 0 self.kv_rotation_base = None self.num_matched_prefix_tokens = 0 - self.lock_receipt = DecLockRefParams() - self.swa_prefix_lock_released = False + self.lock = None self.swa_branching_seqlen = None self.extend_range = None self.dllm_initialized = False diff --git a/python/sglang/srt/managers/schedule_policy.py b/python/sglang/srt/managers/schedule_policy.py index e7d10082d0..db470c8335 100644 --- a/python/sglang/srt/managers/schedule_policy.py +++ b/python/sglang/srt/managers/schedule_policy.py @@ -1038,10 +1038,6 @@ class PrefillAdder: ) self._account_prefill_cache_admission(req, prefix_len) - def _req_inc_lock_ref(self, req: Req): - # Persist the release receipt. - req.lock_receipt = self.tree_cache.inc_lock_ref(req.last_node).to_dec_params() - def _kv_shard_reserve_scratch(self, prefix_len: int, extend_len: int) -> bool: """Reserve scratch or return False to defer; no-op when sharding is off. @@ -1205,13 +1201,11 @@ class PrefillAdder: else None ) try: - # Replay the acquire's receipt (SWA boundary uuid, mamba flag) so the - # release takes back exactly what this temporary lock took. - dec_lock_params = self.tree_cache.inc_lock_ref(last_node).to_dec_params() + lock = self.tree_cache.lock(last_node) try: yield None finally: - self.tree_cache.dec_lock_ref(last_node, dec_lock_params) + self.tree_cache.unlock(lock) finally: if host_lock_params is not None: self.tree_cache.dec_host_lock_ref(last_node, host_lock_params) @@ -1612,7 +1606,7 @@ class PrefillAdder: req.set_extend_range( admission.prefix_len, admission.prefix_len + admission.extend_len ) - self._req_inc_lock_ref(req) + req.lock = self.tree_cache.lock(req.last_node) self.can_run_list.append(req) if admission.is_chunked: self.new_chunked_req = req diff --git a/python/sglang/srt/managers/scheduler_components/dynamic_chunk_sizer.py b/python/sglang/srt/managers/scheduler_components/dynamic_chunk_sizer.py index 2942787dbd..76919db379 100644 --- a/python/sglang/srt/managers/scheduler_components/dynamic_chunk_sizer.py +++ b/python/sglang/srt/managers/scheduler_components/dynamic_chunk_sizer.py @@ -169,9 +169,7 @@ class DynamicChunkSizer: # Walk the same match -> lock -> alloc lifecycle as a scheduled # request so release_kv_cache can release it symmetrically. req.init_next_round_input(self.tree_cache) - req.lock_receipt = self.tree_cache.inc_lock_ref( - req.last_node - ).to_dec_params() + req.lock = self.tree_cache.lock(req.last_node) req.set_extend_range( len(req.prefix_indices), len(req.full_untruncated_fill_ids) ) diff --git a/python/sglang/srt/mem_cache/base_prefix_cache.py b/python/sglang/srt/mem_cache/base_prefix_cache.py index 8974991abc..7bed675515 100644 --- a/python/sglang/srt/mem_cache/base_prefix_cache.py +++ b/python/sglang/srt/mem_cache/base_prefix_cache.py @@ -240,6 +240,16 @@ class DecLockRefParams: return uuids[component_type] +@dataclasses.dataclass +class TreeLock: + """``receipt`` replays the acquire on release; ``swa_released`` marks the + SWA part released early, so neither release takes it twice.""" + + node: Any + receipt: DecLockRefParams + swa_released: bool = False + + @dataclasses.dataclass class DecLockRefResult: """Result of an dec_lock_ref operation.""" @@ -510,11 +520,13 @@ class BasePrefixCache(ABC, PrefixCacheTrait): ) -> DecLockRefResult: pass - def unpin(self, req: Req) -> None: - """Drop the tree lock the request holds on ``req.last_node``; a cache - whose acquire returned a receipt releases with it here.""" - if req.last_node is not None: - self.dec_lock_ref(req.last_node) + def lock(self, node: Any) -> Optional[TreeLock]: + """Take a tree lock on ``node`` for one holder; ``unlock`` releases it.""" + return TreeLock(node, self.inc_lock_ref(node).to_dec_params()) + + def unlock(self, lock: Optional[TreeLock]) -> None: + if lock is not None: + self.dec_lock_ref(lock.node, lock.receipt) def maybe_hand_to_session(self, req: Req) -> None: """A cache that keeps records across requests (a streaming session) takes diff --git a/python/sglang/srt/mem_cache/common.py b/python/sglang/srt/mem_cache/common.py index d34dff6ab9..db6241b6d2 100644 --- a/python/sglang/srt/mem_cache/common.py +++ b/python/sglang/srt/mem_cache/common.py @@ -328,7 +328,8 @@ def release_kv_cache(req: Req, tree_cache: BasePrefixCache, *, checkpoint: bool) tree_cache.checkpoint(req, up_to=owned_kv_len) # The protected prefix is not this req's to free. tree_cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, owned_kv_len)]) - tree_cache.unpin(req) + tree_cache.unlock(req.lock) + req.lock = None _release_overallocated_kv_indices( req, owned_kv_len, req.kv.kv_allocated_len, tree_cache ) diff --git a/python/sglang/srt/mem_cache/radix_cache.py b/python/sglang/srt/mem_cache/radix_cache.py index 1df2aa6c28..6a0968eab0 100644 --- a/python/sglang/srt/mem_cache/radix_cache.py +++ b/python/sglang/srt/mem_cache/radix_cache.py @@ -527,8 +527,8 @@ class RadixCache(BasePrefixCache): # checkpoint or release_kv_cache frees the rest. req.kv.cache_protected_len = len(new_indices) - self.dec_lock_ref(req.last_node) - self.inc_lock_ref(new_last_node) + self.unlock(req.lock) + req.lock = self.lock(new_last_node) # `req.prefix_indices` will be used in `PrefillAdder::add_chunked_req` later # - page_size != 1: there is a partial page at the end, keep the full kv_indices diff --git a/python/sglang/srt/mem_cache/storage/lmcache/lmcache_unified_radix_cache.py b/python/sglang/srt/mem_cache/storage/lmcache/lmcache_unified_radix_cache.py index 4c3ce66b50..1fdf3811fd 100644 --- a/python/sglang/srt/mem_cache/storage/lmcache/lmcache_unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/storage/lmcache/lmcache_unified_radix_cache.py @@ -832,9 +832,8 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache): ) # Move the request lock to the published boundary. - if req.last_node is not None: - self._dec_req_lock(req) - lock_result = self.inc_lock_ref(matched.last_device_node) + self.unlock(req.lock) + req.lock = self.lock(matched.last_device_node) if total_hit < token_ids_len: req.prefix_indices = torch.cat( [canonical, kv_indices[total_hit:].to(dtype=torch.int64, copy=True)] @@ -843,8 +842,6 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache): req.prefix_indices = canonical req.kv.cache_protected_len = total_hit req.last_node = matched.last_device_node - req.lock_receipt = lock_result.to_dec_params() - req.swa_prefix_lock_released = False flow.request_mamba_value = None flow.allocated_request_mamba_for_load = False flow.load_req = None diff --git a/python/sglang/srt/mem_cache/unified_cache/components/README.md b/python/sglang/srt/mem_cache/unified_cache/components/README.md index 51bd8045b1..8ad58d06b5 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/README.md +++ b/python/sglang/srt/mem_cache/unified_cache/components/README.md @@ -237,10 +237,10 @@ Insert a request's KV into the tree: at every checkpoint while it runs, and once | Aspect | Detail | |--------|--------| -| **Purpose** | Publish the request's KV `[cache_protected_len, up_to)` so other requests can match it; nodes past `req.kv.cache_inserted_len` count one hit (a request counts each node once). `checkpoint_kv_cache` calls it with `req.extend_range.end` while the request runs; `release_kv_cache` calls it with the request-owned length when it finishes (`req.finished()`), then frees `[cache_protected_len, up_to)` and everything past it and unpins; with `checkpoint=False` it skips the insert and calls `on_release(req, checkpointed=False)` for component cleanup | +| **Purpose** | Publish the request's KV `[cache_protected_len, up_to)` so other requests can match it; nodes past `req.kv.cache_inserted_len` count one hit (a request counts each node once). `checkpoint_kv_cache` calls it with `req.extend_range.end` while the request runs; `release_kv_cache` calls it with the request-owned length when it finishes (`req.finished()`), then frees `[cache_protected_len, up_to)` and everything past it and unlocks `req.lock`; with `checkpoint=False` it skips the insert and calls `on_release(req, checkpointed=False)` for component cleanup | | **Inputs** | `req` — the request; `up_to` — row position the insert may read up to | | **Output** | `None` | -| **Mutation** | Component hooks → `insert` → re-match → writes the tree's indices back into the row → moves the request's lock from the old `req.last_node` to the node the insert ended on → updates `req.prefix_indices`, `req.kv.cache_protected_len`, `req.kv.cache_inserted_len`, `req.last_node` → component cleanup. A finished request's component state (Mamba) is handed to the tree instead of forked, and what it still held is freed. Frees no KV slot: `release_kv_cache` does that afterwards. | +| **Mutation** | Component hooks → `insert` → re-match → writes the tree's indices back into the row → moves `req.lock` to the node the insert ended on → updates `req.prefix_indices`, `req.kv.cache_protected_len`, `req.kv.cache_inserted_len`, `req.last_node` → component cleanup. A finished request's component state (Mamba) is handed to the tree instead of forked, and what it still held is freed. Frees no KV slot: `release_kv_cache` does that afterwards. | | **Complexity** | **O(K + D·C)** — insert O(K + D·C) + re-match O(K + D·C) + lock transfer O(D). Simplifies to **O(K)**. | **Algorithm detail:** @@ -248,7 +248,7 @@ Insert a request's KV into the tree: at every checkpoint while it runs, and once 2. Truncates the key to `effective_cache_len`; the row past it stays the request's 3. Converts token IDs (bigram if EAGLE), page-aligns the key, then calls `insert()`; a single-component tree re-inserts the prompt part so eviction can drop the output without the prompt 4. `match_prefix()` on the inserted key and writes the matched indices into the row -5. `dec_lock_ref()` on the old `req.last_node`, `inc_lock_ref()` on the new one +5. `unlock(req.lock)`, then `req.lock = lock(new_last_node)` 6. `cleanup_after_caching_req()` per component (Mamba: frees the forked `mamba_value` when the tree already had one, frees the finished request's slot) --- diff --git a/python/sglang/srt/mem_cache/unified_cache/session_ref_tracker.py b/python/sglang/srt/mem_cache/unified_cache/session_ref_tracker.py index 50c739859f..73f03a004f 100644 --- a/python/sglang/srt/mem_cache/unified_cache/session_ref_tracker.py +++ b/python/sglang/srt/mem_cache/unified_cache/session_ref_tracker.py @@ -55,7 +55,7 @@ class UnifiedSessionRefTracker: def register_session_ref(self, req: Req, leaf: NodeId) -> None: """Register the leaf a finished request's insert ended on with each - component; the lock anchor ``req.last_node`` is a different node.""" + component; the request's lock sits on a different node.""" if not self.enable_session_radix_cache: return diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index cded5258bf..88b370be3c 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -29,6 +29,7 @@ from sglang.srt.mem_cache.base_prefix_cache import ( InsertResult, MatchPrefixParams, MatchResult, + TreeLock, ) from sglang.srt.mem_cache.buffer_mode.pipeline import ( BufferModePipeline, @@ -93,7 +94,7 @@ from sglang.srt.observability.metrics_collector import ( StorageMetricsCollector, ) from sglang.srt.runtime_context import get_memory, get_model, get_observability -from sglang.srt.session.streaming_session import StreamingSession +from sglang.srt.session.streaming_session import StreamingSession, is_virtual_node from sglang.srt.utils.common import ceil_align if TYPE_CHECKING: @@ -975,9 +976,6 @@ class UnifiedRadixCache(BasePrefixCache): def inc_lock_ref( self, node_id: NodeId, skip_lock_components: Sequence[ComponentType] = () ) -> IncLockRefResult: - result = self.session.try_inc_lock_ref(node_id) - if result is not None: - return result if self.disable: return IncLockRefResult() return self.tree_core.inc_lock_ref(node_id, skip_lock_components) @@ -988,45 +986,41 @@ class UnifiedRadixCache(BasePrefixCache): params: DecLockRefParams, skip_swa: bool = False, ) -> DecLockRefResult: - result = self.session.try_dec_lock_ref(node_id, params) - if result is not None: - return result if self.disable: return DecLockRefResult() return self.tree_core.dec_lock_ref(node_id, params, skip_swa) - def _dec_req_lock(self, req: Req, *, skip_swa: bool = False) -> None: - """Release the tree lock a request holds on its last_node with the - receipt its acquire returned, so it never drops a lock it never took.""" - self.dec_lock_ref(req.last_node, req.lock_receipt, skip_swa=skip_swa) + def lock( + self, node_id: NodeId, skip_lock_components: Sequence[ComponentType] = () + ) -> Optional[TreeLock]: + # A session's virtual node stands for a lock its slot already holds. + if is_virtual_node(node_id): + return None + result = self.inc_lock_ref(node_id, skip_lock_components) + return TreeLock(node_id, result.to_dec_params()) - def unpin(self, req: Req) -> None: - # Synthetic profiling requests may own KV without locking a tree node. - if req.last_node is not None: - self._dec_req_lock(req, skip_swa=req.swa_prefix_lock_released) - - def dec_swa_lock_only( - self, - node_id: NodeId, - params: DecLockRefParams, - ) -> None: - if self.disable: + def unlock(self, lock: Optional[TreeLock]) -> None: + if lock is None: return - result = self.tree_core.dec_swa_lock_only(node_id, params) + self.dec_lock_ref(lock.node, lock.receipt, skip_swa=lock.swa_released) + + def release_swa(self, lock: Optional[TreeLock]) -> None: + """Release the SWA part of ``lock`` early, once.""" + if self.disable or lock is None or lock.swa_released: + return + result = self.tree_core.dec_swa_lock_only(lock.node, lock.receipt) self._free_values(result.device_frees, result.host_frees) + lock.swa_released = True def release_swa_prefix_lock(self, req: Req) -> None: """The request's window has moved past its prefix: leave the prefix's SWA evictable. A session turn releases its slot's lock, once per session.""" - holder = self.session.borrowed_slot(req) or req + lock = (self.session.borrowed_slot(req) or req).lock if ( - holder.swa_prefix_lock_released - or holder.last_node is None - or holder.lock_receipt.component_lock_uuids.get(ComponentType.SWA) is None + lock is not None + and lock.receipt.component_lock_uuids.get(ComponentType.SWA) is not None ): - return - self.dec_swa_lock_only(holder.last_node, holder.lock_receipt) - holder.swa_prefix_lock_released = True + self.release_swa(lock) def inc_host_lock_ref(self, node_id: NodeId) -> IncLockRefResult: if self.disable: @@ -1208,14 +1202,14 @@ class UnifiedRadixCache(BasePrefixCache): new_indices[req.kv.cache_protected_len :], ) - self._dec_req_lock(req, skip_swa=req.swa_prefix_lock_released) + self.unlock(req.lock) # Opt-in: leave the matched-prefix mamba evictable during decode (it is # already COW'd to the request's own slot, never read from this node again). # Safe only because any future COW source is the COWing request's own # admission-locked last_node (recorded only if still present, locked before # the next alloc) -- not this evictable node. A scheduler that matched a # whole batch before locking would break that. Off = original full lock. - lock_result = self.inc_lock_ref( + req.lock = self.lock( new_last_node, skip_lock_components=( (ComponentType.MAMBA,) @@ -1233,10 +1227,6 @@ class UnifiedRadixCache(BasePrefixCache): req.prefix_indices = new_indices req.kv.cache_protected_len = len(new_indices) req.last_node = new_last_node - # Carry the receipt so this node's dec releases only what we locked. - req.lock_receipt = lock_result.to_dec_params() - # The rematch acquired a new SWA prefix lock. - req.swa_prefix_lock_released = False # cleanup for comp in self._components_tuple: diff --git a/python/sglang/srt/session/streaming_session.py b/python/sglang/srt/session/streaming_session.py index df915bcaac..efd521eaac 100644 --- a/python/sglang/srt/session/streaming_session.py +++ b/python/sglang/srt/session/streaming_session.py @@ -8,11 +8,9 @@ import torch from sglang.srt.managers.schedule_batch import ReqKvInfo from sglang.srt.mem_cache.base_prefix_cache import ( - DecLockRefParams, - DecLockRefResult, - IncLockRefResult, MatchPrefixParams, MatchResult, + TreeLock, ) from sglang.srt.utils.common import ceil_align, is_npu @@ -41,27 +39,22 @@ class SessionSlot: # KV pool state kv: ReqKvInfo = field(default_factory=ReqKvInfo) - # Tree lock on the session's tree-owned prefix, and its receipt. - last_node: Any = None - lock_receipt: DecLockRefParams = field(default_factory=DecLockRefParams) - # Whether the SWA part of that lock was released early. - swa_prefix_lock_released: bool = False + # Tree lock on the session's tree-owned prefix. + lock: Optional[TreeLock] = None # Until the first turn finishes or is retracted, its checkpoints publish # the prompt into the tree and move the lock onto the deepest published node. publishes_prompt: bool = False +def is_virtual_node(node: Any) -> bool: + """A slot's stand-in node: locking it is a no-op.""" + return isinstance(node, _VirtualNode) + + def _is_streaming(req: Optional[Req]) -> bool: return req is not None and req.session is not None and req.session.streaming -def _move_tree_lock(src: Any, dst: Any) -> None: - """Hand the tree lock ``src`` holds (a request or a slot) to ``dst``.""" - dst.last_node = src.last_node - dst.lock_receipt = src.lock_receipt - dst.swa_prefix_lock_released = src.swa_prefix_lock_released - - class StreamingSession: """Streaming-session KV records, owned by ``UnifiedRadixCache``. @@ -80,20 +73,6 @@ class StreamingSession: # -- Try-handle entries (see class docstring) -- - def try_inc_lock_ref(self, node: Any) -> Optional[IncLockRefResult]: - """No-op lock if ``node`` is a session-internal sentinel; returns - None to tell the caller to run its raw tree lock path.""" - if isinstance(node, _VirtualNode): - return IncLockRefResult() - return None - - def try_dec_lock_ref( - self, node: Any, params: Optional[DecLockRefParams] = None - ) -> Optional[DecLockRefResult]: - if isinstance(node, _VirtualNode): - return DecLockRefResult() - return None - def find_active_slot(self, req: Req) -> Optional[SessionSlot]: """A pre-aborted req (to_finish set) is detached from the session and gets None; the slot stays for the next request.""" @@ -186,7 +165,7 @@ class StreamingSession: if isinstance(req.finished_reason, FINISH_ABORT): # Hand the record and the tree lock back; the caller releases them. del self.slots[req.session.session_id] - _move_tree_lock(slot, req) + req.lock, slot.lock = slot.lock, None req.session.abort_req() return False @@ -217,7 +196,7 @@ class StreamingSession: if slot is None: return False if slot.publishes_prompt: - _move_tree_lock(slot, req) + req.lock, slot.lock = slot.lock, None self.cache.checkpoint_into_tree(req, up_to=up_to, **kwargs) self._lock_to_slot(req, slot) return True @@ -255,7 +234,6 @@ class StreamingSession: if slot is None: return protected_len = slot.kv.cache_protected_len - lock_node = slot.last_node tokens_freed = ( max(0, slot.kv.kv_allocated_len - protected_len) if slot.kv.holds_kv else 0 ) @@ -263,11 +241,7 @@ class StreamingSession: "Session KV released: %s (%d tokens freed)", session_id, tokens_freed ) - if lock_node is not None: - # skip_swa is an SWA-cache extension kwarg; a slot can only have - # early-released when the cache supports SWA locks. - skip = {"skip_swa": True} if slot.swa_prefix_lock_released else {} - self.cache.dec_lock_ref(lock_node, slot.lock_receipt, **skip) + self.cache.unlock(slot.lock) if slot.kv.holds_kv: self.cache.free_kv_row(slot.kv, [(protected_len, slot.kv.kv_allocated_len)]) @@ -291,12 +265,8 @@ class StreamingSession: # -- Internal helpers (streaming body bits) -- def _lock_to_slot(self, req: Req, slot: SessionSlot) -> None: - """Move the request's tree lock to the slot; the request is left on the - slot's virtual node, where locking is a no-op.""" - _move_tree_lock(req, slot) + slot.lock, req.lock = req.lock, None req.last_node = slot.virtual_node - req.lock_receipt = DecLockRefParams() - req.swa_prefix_lock_released = False def _free_tail(self, kv: ReqKvInfo, prefix_len: int) -> None: """Free [prefix_len, allocated) before alloc_for_extend overwrites it: diff --git a/python/sglang/test/mem_cache_utils.py b/python/sglang/test/mem_cache_utils.py index 683f4a4885..3b1b388e1a 100644 --- a/python/sglang/test/mem_cache_utils.py +++ b/python/sglang/test/mem_cache_utils.py @@ -11,4 +11,5 @@ def finish_req(cache, req, up_to): req.refresh_fill_ids() cache.checkpoint(req, up_to=up_to) cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, up_to)]) - cache.unpin(req) + cache.unlock(req.lock) + req.lock = None diff --git a/test/registered/unit/disaggregation/test_specv2_kvcache_offloading.py b/test/registered/unit/disaggregation/test_specv2_kvcache_offloading.py index a01ded8d07..024e8901d4 100644 --- a/test/registered/unit/disaggregation/test_specv2_kvcache_offloading.py +++ b/test/registered/unit/disaggregation/test_specv2_kvcache_offloading.py @@ -205,10 +205,12 @@ class TestReleaseFinishedReq(unittest.TestCase): prefix_indices_len=5, ) req.kv.cache_protected_len = 5 + lock = req.lock manager._release_finished_req(req) - manager.tree_cache.unpin.assert_called_once_with(req) + manager.tree_cache.unlock.assert_called_once_with(lock) + self.assertIsNone(req.lock) self.assertTrue(torch.equal(freed[0], torch.arange(5, 20, dtype=torch.int64))) def test_release_finished_req_frees_prefill_and_pops_state(self): diff --git a/test/registered/unit/layers/test_minicpm_sparse_cache.py b/test/registered/unit/layers/test_minicpm_sparse_cache.py index 640bc47c83..f1b04b74d3 100644 --- a/test/registered/unit/layers/test_minicpm_sparse_cache.py +++ b/test/registered/unit/layers/test_minicpm_sparse_cache.py @@ -247,6 +247,7 @@ def test_streaming_session_release_frees_compressed_slots(): req_to_token_pool=pool, token_to_kv_pool_allocator=allocator, page_size=1, + unlock=lambda lock: None, ) cache.free_kv_row = functools.partial(BasePrefixCache.free_kv_row, cache) session = StreamingSession(cache) diff --git a/test/registered/unit/managers/test_prefill_adder.py b/test/registered/unit/managers/test_prefill_adder.py index 85071759fd..f987dab49b 100644 --- a/test/registered/unit/managers/test_prefill_adder.py +++ b/test/registered/unit/managers/test_prefill_adder.py @@ -17,6 +17,7 @@ from sglang.srt.mem_cache.base_prefix_cache import ( CacheRequestHandle, DecLockRefResult, IncLockRefResult, + TreeLock, ) from sglang.srt.mem_cache.page_interleave import PageShardSpec, make_page_shard_spec from sglang.srt.mem_cache.prefill_budget import ( @@ -69,6 +70,13 @@ class TestPrefillAdder(CustomTestCase): tree_cache.disable = False tree_cache.inc_lock_ref.return_value = IncLockRefResult() tree_cache.dec_lock_ref.return_value = DecLockRefResult() + # Route lock/unlock through inc/dec_lock_ref so tests can hook those. + tree_cache.lock.side_effect = lambda node: TreeLock( + node, tree_cache.inc_lock_ref(node).to_dec_params() + ) + tree_cache.unlock.side_effect = lambda lock: ( + lock is not None and tree_cache.dec_lock_ref(lock.node, lock.receipt) + ) tree_cache.buffer_pipeline = None return tree_cache diff --git a/test/registered/unit/managers/test_scheduler_chunked_req_gate.py b/test/registered/unit/managers/test_scheduler_chunked_req_gate.py index fe81e4c4a3..a9198ed10a 100644 --- a/test/registered/unit/managers/test_scheduler_chunked_req_gate.py +++ b/test/registered/unit/managers/test_scheduler_chunked_req_gate.py @@ -7,7 +7,6 @@ from unittest.mock import MagicMock import torch -from sglang.srt.mem_cache.base_prefix_cache import DecLockRefParams from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel @@ -43,7 +42,7 @@ def _make_req( req.skip_radix_cache_insert = False req.finished_reason = None req.last_node = None - req.lock_receipt = DecLockRefParams() + req.lock = None req.session = None req.return_logprob = False req.logprob_start_len = -1 diff --git a/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py b/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py index 6a07f45151..d5c0ccfa12 100644 --- a/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py +++ b/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py @@ -11,7 +11,7 @@ transfer scenarios identified in PR #19746: inc_lock_ref(get_new_prebuilt_batch) -> dec+inc(checkpoint) -> dec(release_kv_cache) 3. Incremental transfer & failure (prefix match > 0, transfer fails) - inc_lock_ref(pop_preallocated) -> dec(unpin via release_kv_cache checkpoint=False) + lock(pop_preallocated) -> unlock(release_kv_cache checkpoint=False) 4. Full transfer & failure (prefix match == 0, transfer fails) no inc_lock_ref -> dec(root_node) is no-op since root lock_ref starts at 1 @@ -47,6 +47,7 @@ from sglang.srt.mem_cache.base_prefix_cache import ( DecLockRefParams, InsertParams, MatchPrefixParams, + TreeLock, ) from sglang.srt.mem_cache.common import release_kv_cache from sglang.srt.mem_cache.radix_cache import RadixCache, RadixKey @@ -86,7 +87,9 @@ def _make_cache_with_pools(page_size=1): class MockReq: """Minimal mock Req with fields needed by checkpoint / release_kv_cache.""" - def __init__(self, fill_ids, req_pool_idx=0, cache_protected_len=0, last_node=None): + def __init__( + self, fill_ids, req_pool_idx=0, cache_protected_len=0, last_node=None, lock=None + ): self.full_untruncated_fill_ids = array("q", fill_ids) self.extend_range = Range(0, len(self.full_untruncated_fill_ids)) self.origin_input_ids = array( @@ -94,6 +97,7 @@ class MockReq: ) self.output_ids = array("q", [fill_ids[-1]] if len(fill_ids) > 1 else []) self.last_node = last_node + self.lock = lock self.extra_key = None self.cache_salt = None self.prefix_indices = torch.empty(0, dtype=torch.int64) @@ -123,8 +127,10 @@ class MockReq: return self.kv.kv_committed_len -def _make_req(fill_ids, req_pool_idx=0, cache_protected_len=0, last_node=None): - return MockReq(fill_ids, req_pool_idx, cache_protected_len, last_node) +def _make_req( + fill_ids, req_pool_idx=0, cache_protected_len=0, last_node=None, lock=None +): + return MockReq(fill_ids, req_pool_idx, cache_protected_len, last_node, lock) class TestDecodeLockRefScenarios(CustomTestCase): @@ -240,8 +246,8 @@ class TestDecodeLockRefScenarios(CustomTestCase): prefix_len = len(result.device_indices) self.assertEqual(prefix_len, 3) - # Step 1: inc_lock_ref (pop_preallocated locks the matched node) - cache.inc_lock_ref(matched_node) + # Step 1: lock (pop_preallocated locks the matched node) + lock = cache.lock(matched_node) self.assertGreater(matched_node.lock_ref, 0) # Simulate _pre_alloc: write prefix + new tokens to req_to_token @@ -254,6 +260,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): req_pool_idx=0, cache_protected_len=prefix_len, last_node=matched_node, + lock=lock, ) # Step 2: checkpoint (dec old lock, inc new lock) @@ -289,10 +296,10 @@ class TestDecodeLockRefScenarios(CustomTestCase): # matched_node is root root_lock_before = cache.root_node.lock_ref - # Step 1: inc_lock_ref on root (simulates get_new_prebuilt_batch) + # Step 1: lock root (simulates get_new_prebuilt_batch) # Note: inc/dec_lock_ref skip the root node (while node != root_node), # so this is a no-op. Root always keeps lock_ref=1. - cache.inc_lock_ref(matched_node) + lock = cache.lock(matched_node) self.assertEqual(cache.root_node.lock_ref, root_lock_before) # no-op on root # Write full KV to pool @@ -304,6 +311,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): req_pool_idx=0, cache_protected_len=0, last_node=matched_node, + lock=lock, ) # Step 2: checkpoint (dec root=no-op, inc new leaf) @@ -324,8 +332,8 @@ class TestDecodeLockRefScenarios(CustomTestCase): The final committed KV slot has no corresponding output token. Cleanup must preserve the matched prefix and release the full request-owned suffix. - Flow: inc_lock_ref(pop_preallocated) - -> dec_lock_ref(unpin via release_kv_cache checkpoint=False) + Flow: lock(pop_preallocated) + -> unlock(release_kv_cache checkpoint=False) """ cache, req_to_token = _make_cache_with_pools() @@ -339,7 +347,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): matched_node = result.last_device_node prefix_len = len(result.device_indices) - cache.inc_lock_ref(matched_node) + lock = cache.lock(matched_node) # Prefix tokens should now be protected (locked) self.assertGreater(cache.protected_size(), 0) @@ -353,6 +361,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): req_pool_idx=0, cache_protected_len=prefix_len, last_node=matched_node, + lock=lock, ) req.output_ids = array("q") @@ -386,7 +395,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): result = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", prefix)))) matched_node = result.last_device_node prefix_len = len(result.device_indices) - cache.inc_lock_ref(matched_node) + lock = cache.lock(matched_node) # Token sequence is 5 long; a 6th KV slot is committed with no token id. full_ids = [1, 2, 3, 4, 5] @@ -398,6 +407,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): req_pool_idx=0, cache_protected_len=prefix_len, last_node=matched_node, + lock=lock, ) req.kv.kv_committed_len = len(row_vals) req.kv.kv_allocated_len = len(row_vals) @@ -438,8 +448,8 @@ class TestDecodeLockRefScenarios(CustomTestCase): matched_node = result.last_device_node self.assertIs(matched_node, cache.root_node) - # inc_lock_ref(root) is a no-op - cache.inc_lock_ref(matched_node) + # lock(root) is a no-op + lock = cache.lock(matched_node) self.assertEqual(cache.root_node.lock_ref, root_lock_before) full_vals = [100, 200, 300] @@ -451,6 +461,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): req_pool_idx=0, cache_protected_len=0, last_node=matched_node, + lock=lock, ) # Transfer fails -> discard without inserting; dec_lock_ref(root) is a no-op @@ -472,10 +483,11 @@ class TestDecodeLockRefScenarios(CustomTestCase): req.last_node = object() req.finished_reason = None req.kv.cache_protected_len = 0 - req.lock_receipt = DecLockRefParams( - component_lock_uuids={ComponentType.SWA: 123} + lock = TreeLock( + req.last_node, + DecLockRefParams(component_lock_uuids={ComponentType.SWA: 123}), ) - req.swa_prefix_lock_released = False + req.lock = lock req.pd_rebootstrap_in_progress = False req.sampling_params.max_new_tokens = 16 @@ -511,7 +523,6 @@ class TestDecodeLockRefScenarios(CustomTestCase): ) queue.transfer_queue = MagicMock(queue=[], enable_staging=False) queue.tree_cache = MagicMock() - queue.tree_cache.dec_lock_ref = MagicMock() queue.req_to_token_pool = MagicMock() queue.req_to_token_pool.available_size.return_value = 1 # Non-hybrid pools have no mamba allocator; MagicMock would otherwise @@ -554,38 +565,34 @@ class TestDecodeLockRefScenarios(CustomTestCase): self.assertEqual(preallocated, []) self.assertEqual(failed, []) queue._pre_alloc.assert_not_called() - queue.tree_cache.dec_swa_lock_only.assert_called_once_with( - req.last_node, - DecLockRefParams(component_lock_uuids={ComponentType.SWA: 123}), - ) - queue.tree_cache.dec_lock_ref.assert_called_once_with( - req.last_node, - DecLockRefParams(component_lock_uuids={ComponentType.SWA: 123}), - skip_swa=True, - ) - self.assertFalse(req.swa_prefix_lock_released) + queue.tree_cache.release_swa.assert_called_once_with(lock) + queue.tree_cache.unlock.assert_called_once_with(lock) + lock_calls = [ + name + for name, _, _ in queue.tree_cache.method_calls + if name in ("release_swa", "unlock") + ] + self.assertEqual(lock_calls, ["release_swa", "unlock"]) + self.assertIsNone(req.lock) queue._swa_tail_len.assert_called_with(8) queue._allocatable_token_budgets.assert_called_once() - def test_hicache_restore_commit_hands_over_lock_with_receipt(self): - """The hicache-restore commit must release the prealloc lock with the - req's receipt, honoring a prior early SWA release (skip_swa), and hand - the restored node's lock to the req atomically: receipt fields move - with last_node, the early-release flag resets (the restored lock is - fresh), and the decode_req drops ownership so a post-commit abort - cannot release the restored lock a second time.""" + def test_hicache_restore_commit_hands_over_lock(self): + """The commit releases the prealloc lock and hands the restored lock to + the req; the decode_req drops it so a post-commit abort cannot release + it a second time.""" q = DecodeHiCacheTransferMixin.__new__(DecodeHiCacheTransferMixin) q.tree_cache = MagicMock() req = MagicMock() req.req_pool_idx = 0 - req.lock_receipt = DecLockRefParams( - component_lock_uuids={ComponentType.SWA: 123} - ) - req.swa_prefix_lock_released = True # SWA tail-prealloc released early - prealloc_node = object() restored_node = object() + prealloc_lock = TreeLock( + prealloc_node, + DecLockRefParams(component_lock_uuids={ComponentType.SWA: 123}), + ) + req.lock = prealloc_lock decode_req = MagicMock() decode_req.req = req decode_req.prefix_match = DecodePrefixMatch( @@ -594,26 +601,19 @@ class TestDecodeLockRefScenarios(CustomTestCase): l3_storage_hit_length=0, last_device_node=prealloc_node, ) - decode_req.hicache_restored_node = restored_node - decode_req.hicache_restore_lock_receipt = DecLockRefParams( - component_lock_uuids={ComponentType.SWA: 456}, - skipped_lock_components=(ComponentType.MAMBA,), + restore_lock = TreeLock( + restored_node, + DecLockRefParams(component_lock_uuids={ComponentType.SWA: 456}), ) + decode_req.hicache_restore_lock = restore_lock decode_req.hicache_restored_kv_indices = torch.arange(4, 8, dtype=torch.int64) q._commit_hicache_local_restore_to_req(decode_req) - q.tree_cache.dec_lock_ref.assert_called_once_with( - prealloc_node, - DecLockRefParams(component_lock_uuids={ComponentType.SWA: 123}), - skip_swa=True, - ) + q.tree_cache.unlock.assert_called_once_with(prealloc_lock) self.assertIs(req.last_node, restored_node) - self.assertEqual(req.lock_receipt.component_lock_uuids[ComponentType.SWA], 456) - self.assertIn(ComponentType.MAMBA, req.lock_receipt.skipped_lock_components) - self.assertFalse(req.swa_prefix_lock_released) - self.assertIsNone(decode_req.hicache_restored_node) - self.assertIsNone(decode_req.hicache_restore_lock_receipt) + self.assertIs(req.lock, restore_lock) + self.assertIsNone(decode_req.hicache_restore_lock) def test_resumed_retracted_req_keeps_other_holders_lock(self): """A retracted request resumed through the prebuilt batch must not release @@ -681,7 +681,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): matched_node = result.last_device_node prefix_len = len(result.device_indices) - cache.inc_lock_ref(matched_node) + lock = cache.lock(matched_node) suffix_token = 40 + iteration full_ids = prefix + [suffix_token] @@ -695,6 +695,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): req_pool_idx=0, cache_protected_len=prefix_len, last_node=matched_node, + lock=lock, ) cache.checkpoint(req, up_to=req.extend_range.end) diff --git a/test/registered/unit/mem_cache/test_lmcache_component_cursors.py b/test/registered/unit/mem_cache/test_lmcache_component_cursors.py index 904e3fdfc2..5313b2fc83 100644 --- a/test/registered/unit/mem_cache/test_lmcache_component_cursors.py +++ b/test/registered/unit/mem_cache/test_lmcache_component_cursors.py @@ -66,6 +66,7 @@ def _cache_and_request(cache_class, *, swa_enabled, swa_cursor=0): ), priority=0, last_node=None, + lock=None, ) if swa_enabled: req.kv.set_evicted_seqlen(ComponentType.SWA, swa_cursor) @@ -117,7 +118,7 @@ def test_mamba_publication_snapshots_component_cursors(cache_class, swa_enabled) req.kv.set_evicted_seqlen(ComponentType.AUXILIARY_SWA, 99) assert insert_params.get_evicted_seqlen(ComponentType.AUXILIARY_SWA) == 6 assert req.kv.cache_protected_len == 8 - assert req.lock_receipt.node_id == 42 + assert req.lock.receipt.node_id == 42 assert flow.prefix_published assert flow.mamba_value is None assert req.prefix_indices.tolist() == list(range(12)) diff --git a/test/registered/unit/mem_cache/test_page_interleave_shard.py b/test/registered/unit/mem_cache/test_page_interleave_shard.py index 4216ee6fee..e0a8592b02 100644 --- a/test/registered/unit/mem_cache/test_page_interleave_shard.py +++ b/test/registered/unit/mem_cache/test_page_interleave_shard.py @@ -64,7 +64,6 @@ from sglang.srt.mem_cache.allocator.page_interleave import ( ) from sglang.srt.mem_cache.allocator.paged import PagedTokenToKVPoolAllocator from sglang.srt.mem_cache.base_prefix_cache import ( - DecLockRefParams, EvictResult, InsertParams, MatchPrefixParams, @@ -789,8 +788,7 @@ class _GraftReq: self.last_node = None self.priority = 0 self.kv_rotation_base = None - self.lock_receipt = DecLockRefParams() - self.swa_prefix_lock_released = False + self.lock = None self.finished_reason = None self.session = None self.session_id = None diff --git a/test/registered/unit/mem_cache/test_pure_swa_chunk_cache.py b/test/registered/unit/mem_cache/test_pure_swa_chunk_cache.py index 85a4de7d22..e0a2305c90 100644 --- a/test/registered/unit/mem_cache/test_pure_swa_chunk_cache.py +++ b/test/registered/unit/mem_cache/test_pure_swa_chunk_cache.py @@ -38,6 +38,7 @@ class _FakeAllocator: def _make_req(): return SimpleNamespace( last_node=None, + lock=None, kv=ReqKvInfo( req_pool_idx=0, cache_protected_len=2, @@ -63,7 +64,7 @@ class TestPureSWAChunkCache(CustomTestCase): req = _make_req() cache.checkpoint(req, up_to=8) cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, 8)]) - cache.unpin(req) + cache.unlock(req.lock) self.assertEqual(cache.token_to_kv_pool_allocator.freed, [2, 6, 7]) self.assertEqual(cache.token_to_kv_pool_allocator.skipped, [3, 4, 5]) diff --git a/test/registered/unit/mem_cache/test_pure_swa_radix_cache.py b/test/registered/unit/mem_cache/test_pure_swa_radix_cache.py index 20ff609e52..1cb8e02e13 100644 --- a/test/registered/unit/mem_cache/test_pure_swa_radix_cache.py +++ b/test/registered/unit/mem_cache/test_pure_swa_radix_cache.py @@ -70,6 +70,7 @@ class TestPureSWARadixCache(CustomTestCase): extra_key=None, cache_salt=None, last_node=cache.root_node, + lock=None, priority=0, kv=ReqKvInfo( req_pool_idx=0, @@ -80,7 +81,7 @@ class TestPureSWARadixCache(CustomTestCase): cache.checkpoint(req, up_to=8) cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, 8)]) - cache.unpin(req) + cache.unlock(req.lock) # [0, 4) went into the tree; [4, 6) was window-evicted; [6, 8) is freed. match = cache.match_prefix(MatchPrefixParams(key=RadixKey(token_ids))) diff --git a/test/registered/unit/mem_cache/test_radix_cache_unit.py b/test/registered/unit/mem_cache/test_radix_cache_unit.py index ba42160b4d..27ea456f18 100644 --- a/test/registered/unit/mem_cache/test_radix_cache_unit.py +++ b/test/registered/unit/mem_cache/test_radix_cache_unit.py @@ -447,6 +447,7 @@ class TestRadixCache(CustomTestCase): cache_salt=None, priority=0, last_node=cache.root_node, + lock=None, ) req.full_untruncated_fill_ids = token_ids req.origin_input_ids = token_ids @@ -496,12 +497,13 @@ class TestRadixCache(CustomTestCase): cache_salt=None, priority=0, last_node=cache.root_node, + lock=None, ) up_to = len(prompt_ids) + len(output_ids) cache.checkpoint(req, up_to=up_to) cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, up_to)]) - cache.unpin(req) + cache.unlock(req.lock) (prompt_node,) = cache.root_node.children.values() (output_node,) = prompt_node.children.values() diff --git a/test/registered/unit/mem_cache/test_rust_tree_core_integration.py b/test/registered/unit/mem_cache/test_rust_tree_core_integration.py index 4c252cc426..1d5e2a9bec 100644 --- a/test/registered/unit/mem_cache/test_rust_tree_core_integration.py +++ b/test/registered/unit/mem_cache/test_rust_tree_core_integration.py @@ -29,6 +29,7 @@ from sglang.srt.mem_cache.base_prefix_cache import ( InsertResult, MatchPrefixParams, MatchResult, + TreeLock, ) from sglang.srt.mem_cache.cache_init_params import CacheInitParams from sglang.srt.mem_cache.evict_policy import TLRUStrategy @@ -4938,7 +4939,7 @@ def test_recover_with_locked_full_applies_through_the_python_allocator(): # The decode advanced past the window: the SWA lock releases early, then # window eviction tombstones the SWA slot under the FULL lock (the state a # locked-full overlap recovers from); its frees return to the allocator. - cache.dec_swa_lock_only(node, lock.to_dec_params()) + cache.release_swa(TreeLock(node, lock.to_dec_params())) tracker = {ComponentType.FULL: 0, ComponentType.SWA: 0} device_frees: dict = {} host_frees: dict = {} diff --git a/test/registered/unit/mem_cache/test_streaming_session_unit.py b/test/registered/unit/mem_cache/test_streaming_session_unit.py index 5baeb6cfd4..81d481c792 100644 --- a/test/registered/unit/mem_cache/test_streaming_session_unit.py +++ b/test/registered/unit/mem_cache/test_streaming_session_unit.py @@ -7,8 +7,6 @@ from sglang.srt.managers.schedule_batch import FINISH_ABORT, FINISH_LENGTH, ReqK from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator from sglang.srt.mem_cache.base_prefix_cache import ( BasePrefixCache, - DecLockRefParams, - IncLockRefResult, MatchResult, ) from sglang.srt.mem_cache.unified_cache.component_type import ComponentType @@ -63,9 +61,6 @@ class _FakeInnerCache: self.token_to_kv_pool_allocator = allocator self.page_size = page_size self.match_results = list(match_results or []) - self.dec_lock_ref_calls = [] - self.dec_lock_ref_params = [] - self.dec_lock_ref_skip_swa = [] self.session = StreamingSession(self) def checkpoint(self, req, *, up_to): @@ -82,12 +77,8 @@ class _FakeInnerCache: raise AssertionError("Unexpected match_prefix call") return self.match_results.pop(0) - def dec_lock_ref(self, node, *args, **kwargs): - if self.session.try_dec_lock_ref(node) is not None: - return - self.dec_lock_ref_calls.append(node) - self.dec_lock_ref_params.append(args[0] if args else kwargs.get("params")) - self.dec_lock_ref_skip_swa.append(kwargs.get("skip_swa", False)) + def unlock(self, lock): + pass class _FakeReq: @@ -113,8 +104,7 @@ class _FakeReq: self.cache_salt = None self.last_node = None self.swa_branching_seqlen = None - self.lock_receipt = DecLockRefParams() - self.swa_prefix_lock_released = False + self.lock = None self.to_finish = None self.finished_reason = None self.finished_len = None @@ -202,57 +192,6 @@ def test_preabort_detaches_session_and_preserves_slot(): assert len(result.device_indices) == 0 -@pytest.mark.parametrize("uuid", [None, 17]) -def test_release_session_preserves_component_lock_receipt(uuid): - """Closing a session releases only the component locks it acquired.""" - req_to_token = torch.arange(256, dtype=torch.int32).reshape(2, 128) - req_to_token_pool = _FakeReqToTokenPool(req_to_token) - allocator = _FakeAllocator() - inner = _FakeInnerCache(req_to_token_pool, allocator, page_size=1) - tree_cache = inner - - lock_node = SimpleNamespace(id=42) - acquired = IncLockRefResult( - node_id=42, - skipped_lock_components=(ComponentType.MAMBA,), - ) - acquired.set_lock_uuid(ComponentType.SWA, uuid) - acquired.set_lock_uuid(ComponentType.SWA, 19, lock_host=True) - acquired.set_lock_uuid(ComponentType.AUXILIARY_SWA, 23) - acquired.set_lock_uuid(ComponentType.AUXILIARY_SWA, None, lock_host=True) - tree_cache.session.slots["session-a"] = SessionSlot( - kv=ReqKvInfo( - req_pool_idx=0, - kv_committed_len=50, - kv_allocated_len=50, - cache_protected_len=0, - ), - last_node=lock_node, - lock_receipt=acquired.to_dec_params(), - ) - - acquired.set_lock_uuid(ComponentType.SWA, 99) - acquired.set_lock_uuid(ComponentType.SWA, 99, lock_host=True) - acquired.set_lock_uuid(ComponentType.AUXILIARY_SWA, 99) - acquired.set_lock_uuid(ComponentType.AUXILIARY_SWA, 99, lock_host=True) - tree_cache.session.release_session("session-a") - - assert inner.dec_lock_ref_calls == [lock_node] - params = inner.dec_lock_ref_params[0] - assert params is not None - assert params.skipped_lock_components == (ComponentType.MAMBA,) - assert params.get_lock_uuid(ComponentType.SWA) == uuid - assert params.get_lock_uuid(ComponentType.SWA, lock_host=True) == 19 - assert params.get_lock_uuid(ComponentType.AUXILIARY_SWA) == 23 - assert params.get_lock_uuid(ComponentType.AUXILIARY_SWA, lock_host=True) is None - for lock_host in (False, True): - with pytest.raises(KeyError): - params.get_lock_uuid(ComponentType.MAMBA, lock_host=lock_host) - with pytest.raises(KeyError): - DecLockRefParams().get_lock_uuid(ComponentType.SWA, lock_host=lock_host) - assert inner.dec_lock_ref_skip_swa == [False] - - def test_trim_overshoot_postcondition(): """Every per-request KV cursor is capped at origin + finished_len and the tail past it is freed.""" diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_bench.py b/test/registered/unit/mem_cache/test_unified_radix_cache_bench.py index 4eb6bcb50e..c52cf12c5f 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_cache_bench.py +++ b/test/registered/unit/mem_cache/test_unified_radix_cache_bench.py @@ -28,6 +28,7 @@ from sglang.srt.mem_cache.base_prefix_cache import ( EvictParams, InsertParams, MatchPrefixParams, + TreeLock, ) from sglang.srt.mem_cache.cache_init_params import CacheInitParams from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool, HybridReqToTokenPool @@ -603,7 +604,7 @@ def bench_release( ): """Request release throughput — full request lifecycle. - Simulates: match_prefix → inc_lock_ref → alloc → fill req_to_token → checkpoint + free + unpin. + Simulates: match_prefix → lock → alloc → fill req_to_token → checkpoint + free + unlock. """ env = _make_env(num_seqs, chunk_len, kv_size, components, page_size) @@ -640,7 +641,7 @@ def bench_release( req.kv.cache_protected_len = matched_len req.kv.kv_committed_len = len(seq) if hasattr(lr, "to_dec_params"): - req.lock_receipt = lr.to_dec_params() + req.lock = TreeLock(node, lr.to_dec_params()) env.rtp.req_to_token[req.kv.req_pool_idx, : len(kv_indices)] = kv_indices req_items.append(req) diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py index 8ea8c4a328..b271af60cf 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py +++ b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py @@ -45,6 +45,7 @@ from sglang.srt.mem_cache.base_prefix_cache import ( InsertParams, MatchPrefixParams, MatchResult, + TreeLock, zero_match_result, ) from sglang.srt.mem_cache.cache_init_params import CacheInitParams @@ -112,7 +113,6 @@ from sglang.srt.server_args import ( ServerArgs, set_global_server_args_for_scheduler, ) -from sglang.srt.session.streaming_session import SessionSlot from sglang.srt.utils import get_device from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.mem_cache_utils import finish_req @@ -1798,7 +1798,7 @@ class UnifiedRadixCacheSuite: req.kv.kv_committed_len = kv_len req.last_node = cache.root_node_handle() req.kv.cache_protected_len = 0 - req.lock_receipt = DecLockRefParams() + req.lock = None req.extra_key = None req.full_untruncated_fill_ids = array("q", input_ids + output_ids) req.set_extend_range( @@ -1859,7 +1859,7 @@ class UnifiedRadixCacheSuite: req.kv.kv_allocated_len = kv_len req.last_node = cache.root_node_handle() req.kv.cache_protected_len = 0 - req.lock_receipt = DecLockRefParams() + req.lock = None req.extra_key = None if self.cfg.has_mamba: req.kv.mamba_last_track_seqlen = kv_len @@ -1921,8 +1921,7 @@ class UnifiedRadixCacheSuite: req.kv.kv_allocated_len = kv_len req.last_node = cache.root_node_handle() req.kv.cache_protected_len = 0 - req.lock_receipt = DecLockRefParams() - req.swa_prefix_lock_released = True + req.lock = None req.extra_key = None req.full_untruncated_fill_ids = array("q", tokens) req.set_extend_range( @@ -1954,7 +1953,7 @@ class UnifiedRadixCacheSuite: req.kv.kv_committed_len = kv_len req.last_node = cache.root_node_handle() req.kv.cache_protected_len = 0 - req.lock_receipt = DecLockRefParams() + req.lock = None req.extra_key = None if self.cfg.has_mamba: req.kv.mamba_last_track_seqlen = kv_len @@ -1964,12 +1963,9 @@ class UnifiedRadixCacheSuite: self.assertGreater(len(req.prefix_indices), 0) self.assertEqual(req.kv.cache_protected_len, len(req.prefix_indices)) self.assertIsNotNone(req.last_node) - self.assertFalse(req.swa_prefix_lock_released) + self.assertFalse(req.lock.swa_released) - cache.dec_lock_ref( - req.last_node, - req.lock_receipt, - ) + cache.unlock(req.lock) cache.sanity_check() def test_swa_unfinished_req_preserves_existing_eviction_boundary(self): @@ -1993,7 +1989,7 @@ class UnifiedRadixCacheSuite: req.kv.kv_committed_len = len(tokens) req.last_node = cache.root_node_handle() req.kv.cache_protected_len = 0 - req.lock_receipt = DecLockRefParams() + req.lock = None req.extra_key = None req.kv.set_evicted_seqlen(ComponentType.SWA, evicted_len) @@ -2009,10 +2005,7 @@ class UnifiedRadixCacheSuite: m = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", tokens)))) self.assertEqual(len(m.device_indices), len(tokens)) - cache.dec_lock_ref( - req.last_node, - req.lock_receipt, - ) + cache.unlock(req.lock) cache.sanity_check() def test_diagnostics(self): @@ -2095,7 +2088,7 @@ class UnifiedRadixCacheSuite: req.kv.kv_committed_len = kv_len req.last_node = cache.root_node_handle() req.kv.cache_protected_len = 0 - req.lock_receipt = DecLockRefParams() + req.lock = None req.extra_key = None req.full_untruncated_fill_ids = array("q", input_ids) req.set_extend_range( @@ -2218,7 +2211,7 @@ class UnifiedRadixCacheSuite: req.kv.kv_committed_len = kv_len req.last_node = cache.root_node_handle() req.kv.cache_protected_len = 0 - req.lock_receipt = DecLockRefParams() + req.lock = None req.extra_key = None req.kv.set_evicted_seqlen(ComponentType.SWA, 0) @@ -2246,10 +2239,7 @@ class UnifiedRadixCacheSuite: ) self.assertEqual(req.kv.cache_protected_len, len(tokens)) - cache.dec_lock_ref( - req.last_node, - req.lock_receipt, - ) + cache.unlock(req.lock) cache.dec_lock_ref(last_device_node, lock_result.to_dec_params()) cache.sanity_check() @@ -2314,7 +2304,7 @@ class UnifiedRadixCacheSuite: 1, "Mamba locked before release", ) - cache.dec_swa_lock_only(node_a, lock_result.to_dec_params()) + cache.release_swa(TreeLock(node_a, lock_result.to_dec_params())) self.assertEqual(_device_lock_ref(cache, node_a, ComponentType.SWA), 0) self.assertEqual( _device_lock_ref(cache, node_a, ComponentType.MAMBA), @@ -2394,7 +2384,7 @@ class UnifiedRadixCacheSuite: ) # B: early SWA release, then final release -- both replay B's receipt. - cache.dec_swa_lock_only(node, lock_b.to_dec_params()) + cache.release_swa(TreeLock(node, lock_b.to_dec_params())) self.assertEqual( _device_lock_ref(cache, node, ComponentType.MAMBA), 1, @@ -2407,7 +2397,7 @@ class UnifiedRadixCacheSuite: "B's final release spares A", ) - cache.dec_swa_lock_only(node, lock_a.to_dec_params()) + cache.release_swa(TreeLock(node, lock_a.to_dec_params())) self.assertEqual( _device_lock_ref(cache, node, ComponentType.MAMBA), 0, @@ -2448,7 +2438,7 @@ class UnifiedRadixCacheSuite: # Early SWA release (decode advanced past the window), via the public # path the scheduler calls. The leaf's SWA is tombstoned and the # co-located lower-tier Mamba lock must drop in the same release. - cache.dec_swa_lock_only(node_a, lock_result.to_dec_params()) + cache.release_swa(TreeLock(node_a, lock_result.to_dec_params())) self.assertEqual( _device_lock_ref(cache, node_a, ComponentType.SWA), 0, "SWA early-released" ) @@ -2484,7 +2474,7 @@ class UnifiedRadixCacheSuite: self.assertIn(ComponentType.MAMBA, skipped.skipped_lock_components) self.assertEqual(_device_lock_ref(cache, node_a, ComponentType.MAMBA), 1) - cache.dec_swa_lock_only(node_a, skipped.to_dec_params()) + cache.release_swa(TreeLock(node_a, skipped.to_dec_params())) self.assertEqual( _device_lock_ref(cache, node_a, ComponentType.MAMBA), 1, @@ -2649,7 +2639,7 @@ class UnifiedRadixCacheSuite: self.assertGreaterEqual(_device_lock_ref(cache, node_a, ComponentType.MAMBA), 1) self.assertGreaterEqual(_device_lock_ref(cache, node_a, ComponentType.FULL), 1) - cache.dec_swa_lock_only(node_a, lock_result.to_dec_params()) + cache.release_swa(TreeLock(node_a, lock_result.to_dec_params())) self.assertEqual( _device_lock_ref(cache, node_a, ComponentType.SWA), 0, "SWA released" ) @@ -2907,7 +2897,7 @@ class UnifiedRadixCacheSuite: req.kv.kv_committed_len = pre_len req.last_node = cache.root_node_handle() req.kv.cache_protected_len = 0 - req.lock_receipt = DecLockRefParams() + req.lock = None req.extra_key = None swa_avail_before = allocator.swa_attn_allocator.available_size() @@ -2932,10 +2922,7 @@ class UnifiedRadixCacheSuite: f"before={swa_avail_before}, after={swa_avail_after}", ) - cache.dec_lock_ref( - req.last_node, - req.lock_receipt, - ) + cache.unlock(req.lock) cache.sanity_check() def test_swa_lru_fresh_leaf_cap_rebuilds_both_nodes(self): @@ -2996,7 +2983,7 @@ class UnifiedRadixCacheSuite: req.kv.kv_committed_len = pre_len req.last_node = cache.root_node_handle() req.kv.cache_protected_len = 0 - req.lock_receipt = DecLockRefParams() + req.lock = None req.extra_key = None with envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS.override(True): @@ -3008,10 +2995,7 @@ class UnifiedRadixCacheSuite: "Nothing should be evicted when prefill fits inside the cushion", ) - cache.dec_lock_ref( - req.last_node, - req.lock_receipt, - ) + cache.unlock(req.lock) cache.sanity_check() def test_swa_sanity_check_passes_after_deep_match(self): @@ -3085,10 +3069,7 @@ class UnifiedRadixCacheSuite: else: lock_result = cache.inc_lock_ref(node) self.assertGreater(_device_lock_ref(cache, node, aux), 0) - cache.dec_swa_lock_only( - node, - lock_result.to_dec_params(), - ) + cache.release_swa(TreeLock(node, lock_result.to_dec_params())) # FULL still locked -> not a device leaf -> no inline evict; the # value stays evictable for the explicit aux eviction below. self.assertIsNotNone(_device_value(cache, node, aux)) @@ -4610,7 +4591,7 @@ class UnifiedRadixCacheSuite: ) self.assertEqual(int(spliced.numel()), len(seq), "load-back degraded") cons.ready_to_load_host_cache() - cons.inc_lock_ref(last_node) # _req_inc_lock_ref + cons.inc_lock_ref(last_node) # the admission lock self.assertEqual(cons.swa_evictable_size(), 0) # window is protected # FULL stays roomy: only SWA can fail below. @@ -7724,7 +7705,7 @@ class UnifiedRadixCacheSuite: temporal, _ = self._snapshot_mamba_state(pool, slot) self.assertTrue(torch.all(temporal == 11)) pool.mamba_allocator.free(other_slots) - cache.dec_lock_ref(req.last_node, req.lock_receipt) + cache.unlock(req.lock) def test_load_back_success_copies_mamba_state_into_request_slot(self): if not self.cfg.has_mamba or self.cfg.has_swa or self.cfg.page_size != 1: @@ -9038,7 +9019,7 @@ class TestUnifiedRadixCacheInt8MambaCheckpoint(CustomTestCase): req.kv.kv_committed_len = len(tokens) req.kv.kv_allocated_len = len(tokens) req.kv.cache_protected_len = 0 - req.lock_receipt = DecLockRefParams() + req.lock = None req.extra_key = None req.kv.mamba_last_track_seqlen = len(tokens) return req @@ -9935,7 +9916,7 @@ class TestResumableInsertWalkSWA(_InsertWalkSuite): lock_result = cache.inc_lock_ref(node) self.assertGreaterEqual(_device_lock_ref(cache, node, ComponentType.SWA), 1) - cache.dec_swa_lock_only(node, lock_result.to_dec_params()) + cache.release_swa(TreeLock(node, lock_result.to_dec_params())) self.assertEqual(_device_lock_ref(cache, node, ComponentType.SWA), 0) self.assertGreaterEqual(_device_lock_ref(cache, node, ComponentType.FULL), 1) @@ -10069,7 +10050,7 @@ class TestReturnedValuesDrain(_InsertWalkSuite): ( "dec_swa_lock_only", lambda: make(DecSwaLockOnlyResult), - lambda: cache.dec_swa_lock_only(node, DecLockRefParams()), + lambda: cache.release_swa(TreeLock(node, DecLockRefParams())), None, ), ] @@ -10592,7 +10573,7 @@ class TestSWAWindowUnderBigramKey(CustomTestCase): req.kv.kv_committed_len = seq_len req.last_node = cache.root_node_handle() req.kv.cache_protected_len = 0 - req.lock_receipt = DecLockRefParams() + req.lock = None req.extra_key = None with envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS.override(True): @@ -10612,10 +10593,7 @@ class TestSWAWindowUnderBigramKey(CustomTestCase): "the match after the insert must reach the leaf the insert created", ) - cache.dec_lock_ref( - req.last_node, - req.lock_receipt, - ) + cache.unlock(req.lock) cache.sanity_check() @@ -11078,21 +11056,23 @@ class TestSegmentLockProtocol(_InsertWalkSuite): self._insert(cache, allocator, req_to_token_pool, seq) leaf = self._match_leaf(cache, seq) lock = cache.inc_lock_ref(leaf) - cache.dec_swa_lock_only(leaf, lock.to_dec_params()) + cache.release_swa(TreeLock(leaf, lock.to_dec_params())) self._assert_protocol_violation( lambda: cache.dec_lock_ref(leaf, lock.to_dec_params()), "lock_ref=0" ) def test_finish_after_early_release_with_skip_swa(self): - """The correct F2 flow: skip_swa honors the early release.""" + """The correct F2 flow: unlock honors the early release.""" sw = self.cfg.sliding_window_size cache, allocator, req_to_token_pool = build_fixture(self.cfg) seq = self._make_seq(1, 2 * sw) self._insert(cache, allocator, req_to_token_pool, seq) leaf = self._match_leaf(cache, seq) - lock = cache.inc_lock_ref(leaf) - cache.dec_swa_lock_only(leaf, lock.to_dec_params()) - cache.dec_lock_ref(leaf, lock.to_dec_params(), skip_swa=True) + lock = cache.lock(leaf) + cache.release_swa(lock) + self.assertTrue(lock.swa_released) + cache.release_swa(lock) # a repeat is a no-op + cache.unlock(lock) self.assertEqual(self._swa_ref(cache, leaf), 0) self.assertEqual(_device_lock_ref(cache, leaf, ComponentType.FULL), 0) cache.sanity_check() @@ -11241,10 +11221,7 @@ class TestSegmentLockFuzz(_InsertWalkSuite): or receipt.component_lock_uuids[ComponentType.SWA] is None ): continue - cache.dec_swa_lock_only( - node_id, - receipt.to_dec_params(), - ) + cache.release_swa(TreeLock(node_id, receipt.to_dec_params())) held[idx][2] = True else: cache.evict( @@ -11300,17 +11277,6 @@ class TestStreamingSessionLockLifecycle(CustomTestCase): max_context_len=64, ) - def _lock_and_early_release(self, cache, allocator): - tokens = array("q", range(1, 9)) - value = allocator.alloc(len(tokens)) - cache.insert(InsertParams(key=RadixKey(tokens), value=value)) - match = cache.match_prefix(MatchPrefixParams(key=RadixKey(tokens))) - node = match.last_device_node - lock = cache.inc_lock_ref(node) - self.assertIsNotNone(lock.component_lock_uuids[ComponentType.SWA]) - cache.dec_swa_lock_only(node, lock.to_dec_params()) - return node, lock - def _session(self, session_id): return SimpleNamespace( session_id=session_id, @@ -11343,45 +11309,9 @@ class TestStreamingSessionLockLifecycle(CustomTestCase): req.kv.kv_committed_len = len(tokens) req.kv.kv_allocated_len = len(tokens) req.last_node = cache.root_node_handle() - req.lock_receipt = cache.inc_lock_ref(req.last_node).to_dec_params() + req.lock = cache.lock(req.last_node) return req - def test_close_after_early_release_releases_swa_once(self): - cache, allocator, _ = build_fixture(self.cfg) - node, lock = self._lock_and_early_release(cache, allocator) - cache.session.slots["s"] = SessionSlot( - last_node=node, - lock_receipt=lock.to_dec_params(), - swa_prefix_lock_released=True, - ) - cache.session.release_session("s") - cache.sanity_check() - - def test_abort_hands_early_released_lock_back(self): - """The handed-back lock keeps its SWA part released; unpin skips it.""" - cache, allocator, _ = build_fixture(self.cfg) - node, lock = self._lock_and_early_release(cache, allocator) - req = SimpleNamespace( - rid="r", - kv=ReqKvInfo(), - last_node=node, - lock_receipt=lock.to_dec_params(), - swa_prefix_lock_released=True, - session=self._session("s2"), - finished_reason=None, - ) - cache.maybe_hand_to_session(req) - slot = cache.session.slots["s2"] - self.assertIs(req.last_node, slot.virtual_node) - - req.finished_reason = FINISH_ABORT() - self.assertFalse(cache.session.try_cache_finished_req(req)) - self.assertNotIn("s2", cache.session.slots) - self.assertEqual(req.last_node, node) - self.assertTrue(req.swa_prefix_lock_released) - cache.unpin(req) - cache.sanity_check() - def test_slot_publishes_first_prompt_but_not_its_output(self): cache, allocator, pool = build_fixture(self.cfg) tokens = list(range(1, 9)) @@ -11391,7 +11321,7 @@ class TestStreamingSessionLockLifecycle(CustomTestCase): cache.checkpoint(req, up_to=len(tokens)) match = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", tokens)))) - self.assertEqual(slot.last_node, match.last_device_node) + self.assertEqual(slot.lock.node, match.last_device_node) self.assertIs(req.last_node, slot.virtual_node) req.output_ids = array("q", range(9, 13)) @@ -11419,8 +11349,8 @@ class TestStreamingSessionLockLifecycle(CustomTestCase): slot = cache.session.slots["s"] cache.release_swa_prefix_lock(req) - self.assertTrue(slot.swa_prefix_lock_released) - self.assertFalse(req.swa_prefix_lock_released) + self.assertTrue(slot.lock.swa_released) + self.assertIsNone(req.lock) cache.release_swa_prefix_lock(req) req.finished_reason = FINISH_LENGTH(length=0) @@ -11452,10 +11382,10 @@ class TestStreamingSessionLockLifecycle(CustomTestCase): def test_later_turn_borrows_without_publishing(self): cache, allocator, pool = build_fixture(self.cfg) slot, req = self._later_turn(cache, allocator, pool) - slot_lock_node = slot.last_node + slot_lock = slot.lock cache.checkpoint(req, up_to=12) - self.assertIs(slot.last_node, slot_lock_node) + self.assertIs(slot.lock, slot_lock) req.finished_reason = FINISH_LENGTH(length=0) self.assertTrue(cache.session.try_cache_finished_req(req)) @@ -11463,9 +11393,13 @@ class TestStreamingSessionLockLifecycle(CustomTestCase): cache.sanity_check() def test_later_turn_abort_releases_like_any_request(self): - """The record and the slot's lock go back before the KV is checkpointed.""" + """The record and the slot's lock, SWA part already released, go back + before the KV is checkpointed.""" cache, allocator, pool = build_fixture(self.cfg) - _, req = self._later_turn(cache, allocator, pool) + slot, req = self._later_turn(cache, allocator, pool) + slot_lock = slot.lock + cache.release_swa_prefix_lock(req) + self.assertTrue(slot_lock.swa_released) req.finished_reason = FINISH_ABORT() release_kv_cache(req, cache, checkpoint=True) diff --git a/test/registered/unit/mem_cache/test_unified_radix_lock_ref.py b/test/registered/unit/mem_cache/test_unified_radix_lock_ref.py index 4a38206884..bf40e7f1d5 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_lock_ref.py +++ b/test/registered/unit/mem_cache/test_unified_radix_lock_ref.py @@ -16,7 +16,7 @@ register_cpu_ci(est_time=7, suite="base-a-test-cpu") class TestUnifiedRadixLockRefScenarios(unittest.TestCase): - def test_no_insert_without_last_node_skips_lock_release(self): + def test_no_insert_without_lock_skips_lock_release(self): cache = object.__new__(UnifiedRadixCache) cache.cache_controller = None cache.session = MagicMock() @@ -26,7 +26,7 @@ class TestUnifiedRadixLockRefScenarios(unittest.TestCase): cache.token_to_kv_pool_allocator = SimpleNamespace(page_size=1) cache.req_to_token_pool.req_to_token = torch.arange(8).reshape(1, 8) cache.free_kv_row = MagicMock() - cache._dec_req_lock = MagicMock() + cache.dec_lock_ref = MagicMock() cache._components_tuple = () cache.enable_session_radix_cache = False @@ -41,7 +41,7 @@ class TestUnifiedRadixLockRefScenarios(unittest.TestCase): output_ids=array("q"), kv=kv, last_node=None, - swa_prefix_lock_released=False, + lock=None, owned_kv_len=lambda: 3, ) @@ -58,7 +58,7 @@ class TestUnifiedRadixLockRefScenarios(unittest.TestCase): release_kv_cache(req, cache, checkpoint=False) cache.free_kv_row.assert_called_once_with(kv, [(0, 3)]) - cache._dec_req_lock.assert_not_called() + cache.dec_lock_ref.assert_not_called() cache.req_to_token_pool.free.assert_called_once_with(req)