diff --git a/python/sglang/srt/mem_cache/base_prefix_cache.py b/python/sglang/srt/mem_cache/base_prefix_cache.py index 607fa61370..41bfbe815d 100644 --- a/python/sglang/srt/mem_cache/base_prefix_cache.py +++ b/python/sglang/srt/mem_cache/base_prefix_cache.py @@ -455,12 +455,11 @@ class BasePrefixCache(ABC, PrefixCacheTrait): return None @abstractmethod - def cache_finished_req(self, req: Req, *, owned_kv_len: int, **kwargs): - """Hand a finished request's KV to the tree: insert what can be keyed - (advancing ``cache_protected_len``), ``free_kv_row`` the rest of - ``[cache_protected_len, owned_kv_len)``, ``unpin``. Slicing the row by - token count instead strands the slots up to ``owned_kv_len``; the - caller frees everything past it.""" + def insert_req(self, req: Req, *, up_to: int, **kwargs): + """Hand a finished request's KV up to row position ``up_to`` to the + tree: insert what can be keyed and advance ``cache_protected_len`` + past it. The caller then frees ``[cache_protected_len, up_to)`` and + everything after, and unpins; nothing here releases a slot.""" @abstractmethod def cache_unfinished_req(self, req: Req, **kwargs): diff --git a/python/sglang/srt/mem_cache/chunk_cache.py b/python/sglang/srt/mem_cache/chunk_cache.py index 41d59fb0cf..07c2c558fa 100644 --- a/python/sglang/srt/mem_cache/chunk_cache.py +++ b/python/sglang/srt/mem_cache/chunk_cache.py @@ -76,11 +76,8 @@ class ChunkCache(BasePrefixCache): # ChunkCache does not support prefix caching, so insert is a no-op return InsertResult(prefix_len=0) - def cache_finished_req(self, req: Req, *, owned_kv_len: int): - # For decode server: if req.output_ids is empty, we want to free all req.origin_input_ids - # The protected prefix is not this req's to free. - self.free_kv_row(req.kv, [(req.kv.cache_protected_len, owned_kv_len)]) - self.unpin(req) + def insert_req(self, req: Req, *, up_to: int): + pass def cache_unfinished_req(self, req: Req, chunked=False): kv_indices = self.req_to_token_pool.req_to_token[ diff --git a/python/sglang/srt/mem_cache/common.py b/python/sglang/srt/mem_cache/common.py index 508ba0f7de..e49dc3b8bb 100644 --- a/python/sglang/srt/mem_cache/common.py +++ b/python/sglang/srt/mem_cache/common.py @@ -313,11 +313,10 @@ def release_kv_cache(req: Req, tree_cache: BasePrefixCache, is_insert: bool = Tr owned_kv_len = req.owned_kv_len() is_insert = is_insert and not req.skip_radix_cache_insert if is_insert: - tree_cache.cache_finished_req(req, owned_kv_len=owned_kv_len) - else: - # The protected prefix is not this req's to free. - tree_cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, owned_kv_len)]) - tree_cache.unpin(req) + tree_cache.insert_req(req, up_to=owned_kv_len) + # The protected prefix is not this req's to free. + tree_cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, owned_kv_len)]) + tree_cache.unpin(req) _release_overallocated_kv_indices( req, owned_kv_len, req.kv.kv_allocated_len, tree_cache ) @@ -360,7 +359,7 @@ def _release_overallocated_kv_indices( if start_p < end_p: # start_p is aligned to the allocator's page above, so it never shares a - # page with cache_finished_req's tail free in this group. + # page with the tail free_kv_row in this group. tree_cache.free_kv_row(req.kv, [(start_p, end_p)]) diff --git a/python/sglang/srt/mem_cache/memory_pool.py b/python/sglang/srt/mem_cache/memory_pool.py index 3e0e2e5798..89008bfe21 100644 --- a/python/sglang/srt/mem_cache/memory_pool.py +++ b/python/sglang/srt/mem_cache/memory_pool.py @@ -1036,7 +1036,7 @@ class MambaPool: (``write_pos[src] == 0``). Only ``temporal`` is copied, not the ring, so an un-flushed source would drop its last ``write_pos`` updates. Callers comply: COW copies radix checkpoints; ``cache_unfinished_req`` copies an - active slot only during prefill (ring empty); ``cache_finished_req`` + active slot only during prefill (ring empty); ``insert_req`` caps the donate to the last flush boundary. The dst cursor is reset to 0 (the copied checkpoint has no pending ring entries). """ diff --git a/python/sglang/srt/mem_cache/pure_swa_radix_cache.py b/python/sglang/srt/mem_cache/pure_swa_radix_cache.py index 6ad7c227e6..663e0d2839 100644 --- a/python/sglang/srt/mem_cache/pure_swa_radix_cache.py +++ b/python/sglang/srt/mem_cache/pure_swa_radix_cache.py @@ -57,22 +57,17 @@ class PureSWARadixCache(RadixCache): num_tokens = max(params.num_tokens, params.swa_num_tokens) return super().evict(EvictParams(num_tokens=num_tokens)) - def cache_finished_req(self, req: Req, *, owned_kv_len: int): - """Insert only the prefill portion [0, evict_floor); free_kv_row skips - the span _evict_swa already freed during decode.""" - if not self.disable: - token_ids = (req.origin_input_ids + req.output_ids)[:owned_kv_len] - swa_evict_floor = req.kv.swa_evict_floor - key_limit = ( - ceil_align(swa_evict_floor, self.page_size) - if swa_evict_floor > 0 - else None - ) - radix_key, _, _ = self._adopt(req, token_ids, key_limit=key_limit) - req.kv.cache_protected_len = len(radix_key) - # The protected prefix is not this req's to free. - self.free_kv_row(req.kv, [(req.kv.cache_protected_len, owned_kv_len)]) - self.unpin(req) + def insert_req(self, req: Req, *, up_to: int): + """Insert only the prefill portion [0, swa_evict_floor).""" + if self.disable: + return + token_ids = (req.origin_input_ids + req.output_ids)[:up_to] + swa_evict_floor = req.kv.swa_evict_floor + key_limit = ( + ceil_align(swa_evict_floor, self.page_size) if swa_evict_floor > 0 else None + ) + radix_key, _, _ = self._insert_cache(req, token_ids, key_limit=key_limit) + req.kv.cache_protected_len = len(radix_key) def cache_unfinished_req(self, req: Req, chunked=False): """During chunked prefill, swa_evicted_seqlen is 0 and no SWA eviction diff --git a/python/sglang/srt/mem_cache/radix_cache.py b/python/sglang/srt/mem_cache/radix_cache.py index 586e991eeb..fa5f9df2e6 100644 --- a/python/sglang/srt/mem_cache/radix_cache.py +++ b/python/sglang/srt/mem_cache/radix_cache.py @@ -433,7 +433,7 @@ class RadixCache(BasePrefixCache): ) return InsertResult(prefix_len=prefix_len, last_device_node=last_node) - def _adopt( + def _insert_cache( self, req: Req, token_ids, @@ -492,22 +492,21 @@ class RadixCache(BasePrefixCache): ) return radix_key, kv_indices, result.prefix_len - def cache_finished_req(self, req: Req, *, owned_kv_len: int): - """Cache request when it finishes.""" - if not self.disable: - token_ids = (req.origin_input_ids + req.output_ids)[:owned_kv_len] - radix_key, _, _ = self._adopt(req, token_ids, split_prompt=True) - req.kv.cache_protected_len = len(radix_key) - # The protected prefix is not this req's to free. - self.free_kv_row(req.kv, [(req.kv.cache_protected_len, owned_kv_len)]) - self.unpin(req) + def insert_req(self, req: Req, *, up_to: int): + if self.disable: + return + token_ids = (req.origin_input_ids + req.output_ids)[:up_to] + radix_key, _, _ = self._insert_cache(req, token_ids, split_prompt=True) + req.kv.cache_protected_len = len(radix_key) def cache_unfinished_req(self, req: Req, chunked=False): """Cache request when it is unfinished.""" if self.disable: return - radix_key, kv_indices, _ = self._adopt(req, req.get_fill_ids(), chunked=chunked) + radix_key, kv_indices, _ = self._insert_cache( + req, req.get_fill_ids(), chunked=chunked + ) # The prefix indices could be updated, reuse it match_result = self.match_prefix(MatchPrefixParams(key=radix_key)) @@ -524,10 +523,9 @@ class RadixCache(BasePrefixCache): new_indices[req.kv.cache_protected_len :], ) - # The cache_protected_len is not always equal to len(req.prefix_indices) - # since for page_size > 1, the partial part is added to req.prefix_indices, but that part of kv indices is not added to the tree. - # It should be freed in the next cache_unfinished_req and final cache_finished_req to avoid memory leak. - # So we introduce this `cache_protected_len` field to make sure the partial part can be freed correctly. + # With page_size > 1 the partial page sits in req.prefix_indices but not + # in the tree; cache_protected_len marks the tree-owned part so the next + # cache_unfinished_req or release_kv_cache frees the rest. req.kv.cache_protected_len = len(new_indices) self.dec_lock_ref(req.last_node) diff --git a/python/sglang/srt/mem_cache/storage/flexkv/flexkv_radix_cache.py b/python/sglang/srt/mem_cache/storage/flexkv/flexkv_radix_cache.py index 2b770987f6..9357f8af8d 100644 --- a/python/sglang/srt/mem_cache/storage/flexkv/flexkv_radix_cache.py +++ b/python/sglang/srt/mem_cache/storage/flexkv/flexkv_radix_cache.py @@ -382,18 +382,16 @@ class FlexKVRadixCache(RadixCache): return fetched_slots, new_node # ------------------------------------------------------------------ - # cache_finished_req (STORE) + # insert_req (STORE) # ------------------------------------------------------------------ def on_release(self, req: Req, *, inserted: bool) -> None: if not inserted: self._load_markers.pop(req.cache_request_handle, None) - def cache_finished_req( # type: ignore[override] - self, req: Req, *, owned_kv_len: int - ) -> None: - """Base cache_finished_req then fire an async FlexKV store.""" - super().cache_finished_req(req, owned_kv_len=owned_kv_len) + def insert_req(self, req: Req, *, up_to: int) -> None: # type: ignore[override] + """Base insert_req then fire an async FlexKV store.""" + super().insert_req(req, up_to=up_to) # Compute the committed prefix. topk = get_spec().speculative_eagle_topk diff --git a/python/sglang/srt/mem_cache/storage/lmcache/lmcache_unified_radix_cache.py b/python/sglang/srt/mem_cache/storage/lmcache/lmcache_unified_radix_cache.py index bf526ec0e4..1e0b9c6a41 100644 --- a/python/sglang/srt/mem_cache/storage/lmcache/lmcache_unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/storage/lmcache/lmcache_unified_radix_cache.py @@ -259,11 +259,11 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache): if not inserted: self.release_aborted_request(req.cache_request_handle) - def cache_finished_req(self, req: Req, *, owned_kv_len: int, **kwargs) -> None: - self._publish_external_loaded_prefix(req, token_ids_len=owned_kv_len) - super().cache_finished_req(req, owned_kv_len=owned_kv_len, **kwargs) + def insert_req(self, req: Req, *, up_to: int, **kwargs) -> None: + self._publish_external_loaded_prefix(req, token_ids_len=up_to) + super().insert_req(req, up_to=up_to, **kwargs) self._retire_loaded_flow(req.rid) - token_ids = (req.origin_input_ids + req.output_ids)[:owned_kv_len] + token_ids = (req.origin_input_ids + req.output_ids)[:up_to] self._submit_store(req, token_ids) self._request_session_finish(req.rid) diff --git a/python/sglang/srt/mem_cache/unified_cache/components/README.md b/python/sglang/srt/mem_cache/unified_cache/components/README.md index 3144ebdd1d..24e142a1ba 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/README.md +++ b/python/sglang/srt/mem_cache/unified_cache/components/README.md @@ -228,16 +228,16 @@ receipt proves were taken. The eventual full release must pass --- -### `cache_finished_req(req: Req, *, owned_kv_len: int)` +### `insert_req(req: Req, *, up_to: int)` Cache a completed request's KV data into the tree. | Aspect | Detail | |--------|--------| | **Purpose** | After a request finishes, insert its token/KV data into the tree for future reuse | -| **Inputs** | `req` — the finished request; `owned_kv_len` — end of the request-owned KV range; slots past it are freed by `release_kv_cache`. A request that leaves without inserting goes through `release_kv_cache(is_insert=False)` instead, which frees the row and calls `on_release(req, inserted=False)` for component cleanup | +| **Inputs** | `req` — the finished request; `up_to` — end of the request-owned KV range. `release_kv_cache` frees `[cache_protected_len, up_to)` and everything past it and unpins; with `is_insert=False` it skips the insert and calls `on_release(req, inserted=False)` for component cleanup | | **Output** | `None` | -| **Mutation** | Calls component hooks → `insert` → `dec_lock_ref` → component cleanup. Frees unaligned tail KV indices. | +| **Mutation** | Calls component hooks → `insert` → component cleanup; advances `cache_protected_len` past the inserted key. Frees nothing and drops no lock: `release_kv_cache` does both afterwards. | | **Complexity** | **O(K + D·C)** — insert O(K + D·C) + lock release O(D). Simplifies to **O(K)**. | **Algorithm detail:** diff --git a/python/sglang/srt/mem_cache/unified_cache/session_ref_tracker.py b/python/sglang/srt/mem_cache/unified_cache/session_ref_tracker.py index 406069aeb2..50c739859f 100644 --- a/python/sglang/srt/mem_cache/unified_cache/session_ref_tracker.py +++ b/python/sglang/srt/mem_cache/unified_cache/session_ref_tracker.py @@ -10,6 +10,8 @@ from collections import OrderedDict from dataclasses import dataclass from typing import TYPE_CHECKING, Optional +from sglang.srt.mem_cache.unified_cache.unified_tree_core import NodeId + if TYPE_CHECKING: from sglang.srt.managers.schedule_batch import Req from sglang.srt.mem_cache.unified_cache.components.base import ( @@ -51,8 +53,9 @@ class UnifiedSessionRefTracker: session_id = req.session.session_id return session_id - def register_session_ref(self, req: Req) -> None: - """Register a non-streaming request's reusable leaves with each component.""" + def register_session_ref(self, req: Req, leaf: NodeId) -> None: + """Register the leaf a finished request's insert ended on with each + component; the lock anchor ``req.last_node`` is a different node.""" if not self.enable_session_radix_cache: return @@ -69,14 +72,13 @@ class UnifiedSessionRefTracker: logger.warning("register_session_ref called for stale request; Skip it.") return - assert req.last_node is not None - last_node = self.tree_core.node_by_id(req.last_node) - if last_node is self.tree_core.root_node: + node = self.tree_core.node_by_id(leaf) + if node is self.tree_core.root_node: return for component in self.components: - leaf = component.resolve_session_leaf(req, last_node) - component.register_session_leaf(session_id, leaf) + component_leaf = component.resolve_session_leaf(req, node) + component.register_session_leaf(session_id, component_leaf) def _remember_closed_session(self, session_id: str) -> None: self._closed_session_ids[session_id] = None diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index 8d1dda3351..f9c0fcc04f 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -975,18 +975,15 @@ class UnifiedRadixCache(BasePrefixCache): for comp in self._components_tuple: comp.cleanup_after_caching_req(req, is_finished=True) - @rank_consensus(same_params=["req.rid", "owned_kv_len"]) - def cache_finished_req(self, req: Req, *, owned_kv_len: int, **kwargs) -> None: + @rank_consensus(same_params=["req.rid", "up_to"]) + def insert_req(self, req: Req, *, up_to: int, **kwargs) -> None: if self.disable: - self.free_kv_row(req.kv, [(req.kv.cache_protected_len, owned_kv_len)]) for comp in self._components_tuple: comp.cleanup_after_caching_req(req, is_finished=True) return - token_ids = (req.origin_input_ids + req.output_ids)[:owned_kv_len] - kv_indices = self.req_to_token_pool.req_to_token[ - req.kv.req_pool_idx, :owned_kv_len - ] + token_ids = (req.origin_input_ids + req.output_ids)[:up_to] + kv_indices = self.req_to_token_pool.req_to_token[req.kv.req_pool_idx, :up_to] insert_params = InsertParams( prev_prefix_len=req.kv.cache_protected_len, @@ -1060,8 +1057,9 @@ class UnifiedRadixCache(BasePrefixCache): ) ) - # Everything past the inserted key goes back, the protected prefix - # never does. After a rotation decline nothing was inserted. + # Everything past the inserted key goes back to the caller, the + # protected prefix never does. After a rotation decline nothing was + # inserted. free_from = ( min(req.kv.cache_protected_len, len(kv_indices)) if result.rotation_tail_declined @@ -1074,12 +1072,7 @@ class UnifiedRadixCache(BasePrefixCache): f"{free_from=} {len(kv_indices)=} {req.kv.cache_protected_len=}" ) free_from = tail_free_start - self.free_kv_row(req.kv, [(free_from, len(kv_indices_full))]) - - self.unpin(req) - - if result is not None and result.last_device_node is not None: - req.last_node = result.last_device_node + req.kv.cache_protected_len = free_from # cleanup for comp in self._components_tuple: @@ -1093,7 +1086,9 @@ class UnifiedRadixCache(BasePrefixCache): if req.finished_reason is not None and not isinstance( req.finished_reason, FINISH_ABORT ): - self.session_refs.register_session_ref(req) + self.session_refs.register_session_ref( + req, leaf=result.last_device_node + ) @rank_consensus(same_params=["req.rid", "chunked"]) def advance_unpublished_req(self, req: Req, chunked: bool = False) -> None: @@ -1217,7 +1212,7 @@ class UnifiedRadixCache(BasePrefixCache): # gather contract forbids -- keep the request entirely on its own # pages: no dedup free, no rebind, no protection change. The insert # declined before its walk, so nothing was freed underneath us. The - # final cache_finished_req releases everything past the protected + # release_kv_cache releases everything past the protected # prefix. req.prefix_indices = kv_indices_orig.to(dtype=torch.int64, copy=True) for comp in self._components_tuple: diff --git a/python/sglang/srt/session/session_controller.py b/python/sglang/srt/session/session_controller.py index 7f28b8594a..979b887fa5 100644 --- a/python/sglang/srt/session/session_controller.py +++ b/python/sglang/srt/session/session_controller.py @@ -423,7 +423,7 @@ class SessionController: # An in-flight request is still decoding on this session's KV # memory. Freeing now would corrupt the scheduler. Mark the # session for deferred cleanup: the request keeps its session - # reference so cache_finished_req takes the streaming path, + # reference so release_kv_cache takes the streaming path, # and we schedule release_session for after it completes. session.close_on_finish = True logger.info( diff --git a/python/sglang/srt/session/streaming_session.py b/python/sglang/srt/session/streaming_session.py index 038744af33..a90e612355 100644 --- a/python/sglang/srt/session/streaming_session.py +++ b/python/sglang/srt/session/streaming_session.py @@ -174,7 +174,7 @@ class StreamingSession(BasePrefixCache): """Returns an active slot for this req, or None. Side effect: if req is pre-aborted (to_finish set, e.g. input too - long), detach it from the session so cache_finished_req treats it + long), detach it from the session so release_kv_cache treats it as a normal req. The slot stays intact for the next request. """ if not _is_streaming(req): @@ -359,8 +359,8 @@ class StreamingSession(BasePrefixCache): def on_release(self, req: Req, *, inserted: bool) -> None: self.inner.on_release(req, inserted=inserted) - def cache_finished_req(self, req: Req, **kwargs): - self.inner.cache_finished_req(req, **kwargs) + def insert_req(self, req: Req, **kwargs): + self.inner.insert_req(req, **kwargs) def cache_unfinished_req(self, req: Req, **kwargs): if self.try_cache_unfinished_req(req, **kwargs): diff --git a/python/sglang/test/mem_cache_utils.py b/python/sglang/test/mem_cache_utils.py new file mode 100644 index 0000000000..9ed558e13a --- /dev/null +++ b/python/sglang/test/mem_cache_utils.py @@ -0,0 +1,8 @@ +"""Shared helpers for mem_cache unit tests.""" + + +def finish_req(cache, req, up_to): + """insert_req, then what release_kv_cache does after it: free the rest, drop the lock.""" + cache.insert_req(req, up_to=up_to) + cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, up_to)]) + cache.unpin(req) diff --git a/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py b/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py index cf44dbf6d0..107d8a5e5d 100644 --- a/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py +++ b/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py @@ -109,6 +109,8 @@ class MockReq: def get_fill_ids(self): return self.full_untruncated_fill_ids[: self.extend_range.end] + skip_radix_cache_insert = False + def owned_kv_len(self): return self.kv.kv_committed_len @@ -249,8 +251,8 @@ class TestDecodeLockRefScenarios(CustomTestCase): # Step 2: cache_unfinished_req (dec old lock, inc new lock) cache.cache_unfinished_req(req) - # Step 3: cache_finished_req (dec lock) - cache.cache_finished_req(req, owned_kv_len=req.kv.kv_committed_len) + # Step 3: release_kv_cache (insert, free the rest, dec lock) + release_kv_cache(req, cache) # Verify: all non-root nodes should have lock_ref == 0 # (root always has lock_ref == 1) @@ -299,7 +301,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): cache.cache_unfinished_req(req) # Step 3: cache_finished_req (dec leaf) - cache.cache_finished_req(req, owned_kv_len=req.kv.kv_committed_len) + release_kv_cache(req, cache) # Root lock unchanged, all nodes unlocked self.assertEqual(cache.root_node.lock_ref, root_lock_before) @@ -391,7 +393,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): req.kv.kv_allocated_len = len(row_vals) cache.token_to_kv_pool_allocator.reset_mock() - cache.cache_finished_req(req, owned_kv_len=req.kv.kv_committed_len) + release_kv_cache(req, cache) # The unnamed tail slot is freed as the segment past the radix key. segments = cache.token_to_kv_pool_allocator.free_segments.call_args.args[0] @@ -686,7 +688,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): ) cache.cache_unfinished_req(req) - cache.cache_finished_req(req, owned_kv_len=req.kv.kv_committed_len) + release_kv_cache(req, cache) # After all iterations, root lock should be 1, no protected nodes self.assertEqual(cache.root_node.lock_ref, 1) diff --git a/test/registered/unit/mem_cache/test_free_kv_row_coalesce.py b/test/registered/unit/mem_cache/test_free_kv_row_coalesce.py index b08f9d5a47..a3f60887be 100644 --- a/test/registered/unit/mem_cache/test_free_kv_row_coalesce.py +++ b/test/registered/unit/mem_cache/test_free_kv_row_coalesce.py @@ -39,7 +39,7 @@ class _RowCache(BasePrefixCache): def match_prefix(self, params): raise NotImplementedError - def cache_finished_req(self, req, **kwargs): + def insert_req(self, req, **kwargs): raise NotImplementedError def cache_unfinished_req(self, req, **kwargs): diff --git a/test/registered/unit/mem_cache/test_page_interleave_shard.py b/test/registered/unit/mem_cache/test_page_interleave_shard.py index 8a9ca91888..275edb238a 100644 --- a/test/registered/unit/mem_cache/test_page_interleave_shard.py +++ b/test/registered/unit/mem_cache/test_page_interleave_shard.py @@ -89,6 +89,7 @@ from sglang.srt.runtime_context import get_parallel, publish from sglang.srt.server_args import ServerArgs from sglang.srt.utils import ceil_div from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.mem_cache_utils import finish_req from sglang.test.test_utils import CustomTestCase, publish_build_topology register_cpu_ci(est_time=30, suite="base-a-test-cpu") @@ -761,7 +762,7 @@ class TestShardedCoreGate(CustomTestCase): class _GraftReq: - """Minimal Req stand-in for cache_unfinished/finished_req.""" + """Minimal Req stand-in for cache_unfinished_req / insert_req.""" def __init__(self, fill_ids, req_pool_idx=0): self.fill_ids = list(fill_ids) @@ -904,13 +905,13 @@ class TestRotationGraftDecline(CustomTestCase): torch.equal(tree.req_to_token_pool.req_to_token[0, :12], own_locs) ) - def test_cache_finished_decline_frees_duplicates_and_suffix(self): + def test_insert_req_decline_frees_duplicates_and_suffix(self): tree, freed = self._tree_with_spy() self._seed_chain(tree, list(range(8)), base=1) req = _GraftReq(list(range(8)) + [90, 91, 92, 93]) req.kv_rotation_base = 3 own_locs = self._own_row(tree, req, 12) - tree.cache_finished_req(req, owned_kv_len=12) + finish_req(tree, req, 12) released = torch.cat(freed) # Everything past the protected prefix is released: the duplicates of # the matched region AND the declined tail (nothing leaks, nothing is @@ -918,7 +919,7 @@ class TestRotationGraftDecline(CustomTestCase): self.assertEqual(set(released.tolist()), set(own_locs.tolist())) self.assertEqual(_match_len(tree, req.fill_ids), 8) - def test_cache_finished_same_base_keeps_the_tail_cached(self): + def test_insert_req_same_base_keeps_the_tail_cached(self): """Control for the decline test: with an agreeing base the tail is grafted and only the matched duplicates are freed.""" tree, freed = self._tree_with_spy() @@ -926,7 +927,7 @@ class TestRotationGraftDecline(CustomTestCase): req = _GraftReq(list(range(8)) + [90, 91, 92, 93]) req.kv_rotation_base = 1 own_locs = self._own_row(tree, req, 12) - tree.cache_finished_req(req, owned_kv_len=12) + finish_req(tree, req, 12) self.assertEqual(_match_len(tree, req.fill_ids), 12) released = torch.cat(freed) if freed else torch.empty(0, dtype=torch.int64) # Only the 8 duplicate rows go back; the tail stays live in the tree. diff --git a/test/registered/unit/mem_cache/test_pure_swa_chunk_cache.py b/test/registered/unit/mem_cache/test_pure_swa_chunk_cache.py index b74469e52d..b7e3fe2455 100644 --- a/test/registered/unit/mem_cache/test_pure_swa_chunk_cache.py +++ b/test/registered/unit/mem_cache/test_pure_swa_chunk_cache.py @@ -60,7 +60,10 @@ class TestPureSWAChunkCache(CustomTestCase): cache = self._make_cache() # protected 2, floor 3, cursor 6: [2, 3) and [6, 8) go back, [3, 6) is dead - cache.cache_finished_req(_make_req(), owned_kv_len=8) + req = _make_req() + cache.insert_req(req, up_to=8) + cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, 8)]) + cache.unpin(req) self.assertEqual(cache.token_to_kv_pool_allocator.freed, [2, 6, 7]) self.assertEqual(cache.token_to_kv_pool_allocator.skipped, [3, 4, 5]) diff --git a/test/registered/unit/mem_cache/test_pure_swa_radix_cache.py b/test/registered/unit/mem_cache/test_pure_swa_radix_cache.py index d8595defd2..7e1c3c2c33 100644 --- a/test/registered/unit/mem_cache/test_pure_swa_radix_cache.py +++ b/test/registered/unit/mem_cache/test_pure_swa_radix_cache.py @@ -69,7 +69,9 @@ class TestPureSWARadixCache(CustomTestCase): ), ) - cache.cache_finished_req(req, owned_kv_len=8) + cache.insert_req(req, up_to=8) + cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, 8)]) + cache.unpin(req) # [0, 4) went into the tree; [4, 6) was window-evicted; [6, 8) is freed. match = cache.match_prefix(MatchPrefixParams(key=RadixKey(token_ids))) diff --git a/test/registered/unit/mem_cache/test_radix_cache_unit.py b/test/registered/unit/mem_cache/test_radix_cache_unit.py index 53002f2f5f..2ab69da3cc 100644 --- a/test/registered/unit/mem_cache/test_radix_cache_unit.py +++ b/test/registered/unit/mem_cache/test_radix_cache_unit.py @@ -492,10 +492,10 @@ class TestRadixCache(CustomTestCase): last_node=cache.root_node, ) - cache.cache_finished_req( - req, - owned_kv_len=len(prompt_ids) + len(output_ids), - ) + up_to = len(prompt_ids) + len(output_ids) + cache.insert_req(req, up_to=up_to) + cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, up_to)]) + cache.unpin(req) (prompt_node,) = cache.root_node.children.values() (output_node,) = prompt_node.children.values() diff --git a/test/registered/unit/mem_cache/test_session_unified_radix_cache.py b/test/registered/unit/mem_cache/test_session_unified_radix_cache.py index 2678b61976..41af9ece2d 100644 --- a/test/registered/unit/mem_cache/test_session_unified_radix_cache.py +++ b/test/registered/unit/mem_cache/test_session_unified_radix_cache.py @@ -83,18 +83,19 @@ def match_len(cache, token_ids) -> int: def register(cache, token_ids, session_id, generation=None): if generation is None: generation = cache.ensure_session_generation(session_id) + leaf = cache.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", token_ids))) + ).last_device_node cache.session_refs.register_session_ref( SimpleNamespace( session_id=session_id, session_generation=generation, session=None, - last_node=cache.match_prefix( - MatchPrefixParams(key=RadixKey(array("q", token_ids))) - ).last_device_node, origin_input_ids=array("q", token_ids), output_ids=array("q"), extra_key=None, - ) + ), + leaf=leaf, ) diff --git a/test/registered/unit/mem_cache/test_streaming_session_unit.py b/test/registered/unit/mem_cache/test_streaming_session_unit.py index 3813c84585..5c1d66dcd1 100644 --- a/test/registered/unit/mem_cache/test_streaming_session_unit.py +++ b/test/registered/unit/mem_cache/test_streaming_session_unit.py @@ -63,7 +63,7 @@ class _FakeInnerCache: self.dec_lock_ref_params = [] self.dec_lock_ref_skip_swa = [] - def cache_finished_req(self, *args, **kwargs): + def insert_req(self, *args, **kwargs): raise AssertionError("Streaming requests should not delegate to inner cache") def match_prefix(self, *args, **kwargs): diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_bench.py b/test/registered/unit/mem_cache/test_unified_radix_cache_bench.py index 0bf4a5a57d..a0c2efbc0d 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_cache_bench.py +++ b/test/registered/unit/mem_cache/test_unified_radix_cache_bench.py @@ -37,6 +37,7 @@ from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler from sglang.srt.utils import get_device from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci +from sglang.test.mem_cache_utils import finish_req from sglang.test.test_utils import CustomTestCase register_cuda_ci(est_time=29, stage="base-b", runner_config="1-gpu-small") @@ -592,7 +593,7 @@ def bench_lock_unlock( ) -def bench_cache_finished( +def bench_release( num_seqs=5000, chunk_len=256, kv_size=500_000, @@ -600,9 +601,9 @@ def bench_cache_finished( verify=False, page_size=1, ): - """cache_finished_req throughput — full request lifecycle. + """Request release throughput — full request lifecycle. - Simulates: match_prefix → inc_lock_ref → alloc → fill req_to_token → cache_finished_req. + Simulates: match_prefix → inc_lock_ref → alloc → fill req_to_token → insert_req + free + unpin. """ env = _make_env(num_seqs, chunk_len, kv_size, components, page_size) @@ -644,15 +645,13 @@ def bench_cache_finished( req_items.append(req) if not req_items: - return BenchResult("cache_finished", 0, 0, 0, []) + return BenchResult("release", 0, 0, 0, []) warmup = min(20, len(req_items) // 10) return bench_api( - "cache_finished", + "release", lambda: req_items, - lambda req: env.tree.cache_finished_req( - req, owned_kv_len=req.kv.kv_committed_len - ), + lambda req: finish_req(env.tree, req, req.kv.kv_committed_len), len(req_items) - warmup, env.avg_tokens, warmup, @@ -669,7 +668,7 @@ ALL_BENCHMARKS = { "match": bench_match_prefix, "evict": bench_evict, "lock": bench_lock_unlock, - "cache_finished": bench_cache_finished, + "release": bench_release, } @@ -791,8 +790,8 @@ class _BenchSuite: def test_bench_lock_unlock(self): self._run(bench_lock_unlock) - def test_bench_cache_finished(self): - self._run(bench_cache_finished) + def test_bench_release(self): + self._run(bench_release) for _cfg in _CI_BENCH_CONFIGS: @@ -837,7 +836,7 @@ def _run_bench_cli(): "--benchmarks", nargs="+", default=["all"], - help="insert match evict lock cache_finished all", + help="insert match evict lock release all", ) args, _ = parser.parse_known_args() diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py index 5c19e6debe..3a22143790 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py +++ b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py @@ -107,6 +107,7 @@ from sglang.srt.server_args import ( from sglang.srt.session.streaming_session import SessionSlot from sglang.srt.utils import get_device from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci +from sglang.test.mem_cache_utils import finish_req from sglang.test.test_utils import CustomTestCase register_cuda_ci(est_time=60, stage="base-b", runner_config="1-gpu-small") @@ -1731,7 +1732,7 @@ class UnifiedRadixCacheSuite: self.assertEqual(allocator.available_size(), avail_before - len(seq_3p)) cache.sanity_check() - def test_cache_finished_req_insert(self): + def test_insert_req(self): cache, allocator, req_to_token_pool = build_fixture(self.cfg) ps = self.cfg.page_size @@ -1755,7 +1756,7 @@ class UnifiedRadixCacheSuite: if self.cfg.has_mamba: req.kv.mamba_last_track_seqlen = kv_len - cache.cache_finished_req(req, owned_kv_len=req.owned_kv_len()) + finish_req(cache, req, req.owned_kv_len()) all_ids = input_ids + output_ids aligned_len = (len(all_ids) // ps) * ps @@ -1787,7 +1788,7 @@ class UnifiedRadixCacheSuite: self.assertEqual(len(prompt_only.device_indices), prompt_aligned_len) cache.sanity_check() - def test_cache_finished_req_strips_thinking(self): + def test_insert_req_strips_thinking(self): cache, allocator, req_to_token_pool = build_fixture(self.cfg) ps = self.cfg.page_size @@ -1813,10 +1814,10 @@ class UnifiedRadixCacheSuite: req.kv.mamba_last_track_seqlen = kv_len req.reasoning_tokens = 1 - # cache_finished_req reads get_serving().strip_thinking_cache + # owned_kv_len reads get_serving().strip_thinking_cache with get_serving().override(strip_thinking_cache=True): avail_before = allocator.available_size() - cache.cache_finished_req(req, owned_kv_len=req.owned_kv_len()) + finish_req(cache, req, req.owned_kv_len()) start_p, end_p = req.owned_kv_len(), req.kv.kv_allocated_len if ps > 1: start_p = ((start_p + ps - 1) // ps) * ps @@ -1837,7 +1838,7 @@ class UnifiedRadixCacheSuite: ) cache.sanity_check() - def test_cache_finished_req_no_insert(self): + def test_release_without_insert(self): cache, allocator, req_to_token_pool = build_fixture(self.cfg) req = self._make_req(req_to_token_pool) tokens = self._make_seq(1, 2) @@ -2005,7 +2006,7 @@ class UnifiedRadixCacheSuite: self.assertEqual(len(m.device_indices), 0) cache.sanity_check() - def test_paged_cache_finished_unaligned_tail_freed(self): + def test_paged_insert_req_unaligned_tail_freed(self): if self.cfg.page_size == 1: self.skipTest("page_size > 1 only") if self.cfg.has_swa: @@ -2034,7 +2035,7 @@ class UnifiedRadixCacheSuite: req.kv.mamba_last_track_seqlen = kv_len avail_before = allocator.available_size() - cache.cache_finished_req(req, owned_kv_len=req.owned_kv_len()) + finish_req(cache, req, req.owned_kv_len()) self.assertEqual(allocator.available_size(), avail_before + tail_extra) aligned = input_ids[: (len(input_ids) // ps) * ps] @@ -8775,7 +8776,7 @@ class TestUnifiedRadixCacheInt8MambaCheckpoint(CustomTestCase): req.kv.mamba_last_track_seqlen = len(tokens) return req - def _cache_finished(self, cache, allocator, req_to_token_pool, tokens): + def _finished_req(self, cache, allocator, req_to_token_pool, tokens): req = self._make_req(req_to_token_pool, tokens) kv_indices = allocator.alloc(len(tokens)) self.assertIsNotNone(kv_indices) @@ -8784,7 +8785,7 @@ class TestUnifiedRadixCacheInt8MambaCheckpoint(CustomTestCase): ) req.last_node = cache.root_node_handle() - cache.cache_finished_req(req, owned_kv_len=req.owned_kv_len()) + finish_req(cache, req, req.owned_kv_len()) def test_finished_req_stores_radix_mamba_state_in_int8_pool(self): cache, allocator, req_to_token_pool = build_fixture(self.cfg) @@ -8794,14 +8795,14 @@ class TestUnifiedRadixCacheInt8MambaCheckpoint(CustomTestCase): ckpt_initial = ckpt_pool.available_size() tokens = [1, 2, 3, 4] - self._cache_finished(cache, allocator, req_to_token_pool, tokens) + self._finished_req(cache, allocator, req_to_token_pool, tokens) self.assertEqual( req_to_token_pool.mamba_allocator.available_size(), active_initial ) self.assertEqual(ckpt_pool.available_size(), ckpt_initial - 1) self.assertEqual(cache.mamba_evictable_size(), 1) - self._cache_finished(cache, allocator, req_to_token_pool, tokens) + self._finished_req(cache, allocator, req_to_token_pool, tokens) self.assertEqual( req_to_token_pool.mamba_allocator.available_size(), active_initial )