[mem_cache] Replace cache_finished_req with insert_req; release_kv_cache frees and unpins (#41281)

This commit is contained in:
Liangsheng Yin
2026-09-27 02:53:16 -07:00
committed by GitHub
parent 0bc074db83
commit 37556c1b91
24 changed files with 136 additions and 136 deletions
@@ -455,12 +455,11 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
return None
@abstractmethod
def cache_finished_req(self, req: Req, *, owned_kv_len: int, **kwargs):
"""Hand a finished request's KV to the tree: insert what can be keyed
(advancing ``cache_protected_len``), ``free_kv_row`` the rest of
``[cache_protected_len, owned_kv_len)``, ``unpin``. Slicing the row by
token count instead strands the slots up to ``owned_kv_len``; the
caller frees everything past it."""
def insert_req(self, req: Req, *, up_to: int, **kwargs):
"""Hand a finished request's KV up to row position ``up_to`` to the
tree: insert what can be keyed and advance ``cache_protected_len``
past it. The caller then frees ``[cache_protected_len, up_to)`` and
everything after, and unpins; nothing here releases a slot."""
@abstractmethod
def cache_unfinished_req(self, req: Req, **kwargs):
+2 -5
View File
@@ -76,11 +76,8 @@ class ChunkCache(BasePrefixCache):
# ChunkCache does not support prefix caching, so insert is a no-op
return InsertResult(prefix_len=0)
def cache_finished_req(self, req: Req, *, owned_kv_len: int):
# For decode server: if req.output_ids is empty, we want to free all req.origin_input_ids
# The protected prefix is not this req's to free.
self.free_kv_row(req.kv, [(req.kv.cache_protected_len, owned_kv_len)])
self.unpin(req)
def insert_req(self, req: Req, *, up_to: int):
pass
def cache_unfinished_req(self, req: Req, chunked=False):
kv_indices = self.req_to_token_pool.req_to_token[
+5 -6
View File
@@ -313,11 +313,10 @@ def release_kv_cache(req: Req, tree_cache: BasePrefixCache, is_insert: bool = Tr
owned_kv_len = req.owned_kv_len()
is_insert = is_insert and not req.skip_radix_cache_insert
if is_insert:
tree_cache.cache_finished_req(req, owned_kv_len=owned_kv_len)
else:
# The protected prefix is not this req's to free.
tree_cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, owned_kv_len)])
tree_cache.unpin(req)
tree_cache.insert_req(req, up_to=owned_kv_len)
# The protected prefix is not this req's to free.
tree_cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, owned_kv_len)])
tree_cache.unpin(req)
_release_overallocated_kv_indices(
req, owned_kv_len, req.kv.kv_allocated_len, tree_cache
)
@@ -360,7 +359,7 @@ def _release_overallocated_kv_indices(
if start_p < end_p:
# start_p is aligned to the allocator's page above, so it never shares a
# page with cache_finished_req's tail free in this group.
# page with the tail free_kv_row in this group.
tree_cache.free_kv_row(req.kv, [(start_p, end_p)])
+1 -1
View File
@@ -1036,7 +1036,7 @@ class MambaPool:
(``write_pos[src] == 0``). Only ``temporal`` is copied, not the ring, so
an un-flushed source would drop its last ``write_pos`` updates. Callers
comply: COW copies radix checkpoints; ``cache_unfinished_req`` copies an
active slot only during prefill (ring empty); ``cache_finished_req``
active slot only during prefill (ring empty); ``insert_req``
caps the donate to the last flush boundary. The dst cursor is reset to 0
(the copied checkpoint has no pending ring entries).
"""
@@ -57,22 +57,17 @@ class PureSWARadixCache(RadixCache):
num_tokens = max(params.num_tokens, params.swa_num_tokens)
return super().evict(EvictParams(num_tokens=num_tokens))
def cache_finished_req(self, req: Req, *, owned_kv_len: int):
"""Insert only the prefill portion [0, evict_floor); free_kv_row skips
the span _evict_swa already freed during decode."""
if not self.disable:
token_ids = (req.origin_input_ids + req.output_ids)[:owned_kv_len]
swa_evict_floor = req.kv.swa_evict_floor
key_limit = (
ceil_align(swa_evict_floor, self.page_size)
if swa_evict_floor > 0
else None
)
radix_key, _, _ = self._adopt(req, token_ids, key_limit=key_limit)
req.kv.cache_protected_len = len(radix_key)
# The protected prefix is not this req's to free.
self.free_kv_row(req.kv, [(req.kv.cache_protected_len, owned_kv_len)])
self.unpin(req)
def insert_req(self, req: Req, *, up_to: int):
"""Insert only the prefill portion [0, swa_evict_floor)."""
if self.disable:
return
token_ids = (req.origin_input_ids + req.output_ids)[:up_to]
swa_evict_floor = req.kv.swa_evict_floor
key_limit = (
ceil_align(swa_evict_floor, self.page_size) if swa_evict_floor > 0 else None
)
radix_key, _, _ = self._insert_cache(req, token_ids, key_limit=key_limit)
req.kv.cache_protected_len = len(radix_key)
def cache_unfinished_req(self, req: Req, chunked=False):
"""During chunked prefill, swa_evicted_seqlen is 0 and no SWA eviction
+13 -15
View File
@@ -433,7 +433,7 @@ class RadixCache(BasePrefixCache):
)
return InsertResult(prefix_len=prefix_len, last_device_node=last_node)
def _adopt(
def _insert_cache(
self,
req: Req,
token_ids,
@@ -492,22 +492,21 @@ class RadixCache(BasePrefixCache):
)
return radix_key, kv_indices, result.prefix_len
def cache_finished_req(self, req: Req, *, owned_kv_len: int):
"""Cache request when it finishes."""
if not self.disable:
token_ids = (req.origin_input_ids + req.output_ids)[:owned_kv_len]
radix_key, _, _ = self._adopt(req, token_ids, split_prompt=True)
req.kv.cache_protected_len = len(radix_key)
# The protected prefix is not this req's to free.
self.free_kv_row(req.kv, [(req.kv.cache_protected_len, owned_kv_len)])
self.unpin(req)
def insert_req(self, req: Req, *, up_to: int):
if self.disable:
return
token_ids = (req.origin_input_ids + req.output_ids)[:up_to]
radix_key, _, _ = self._insert_cache(req, token_ids, split_prompt=True)
req.kv.cache_protected_len = len(radix_key)
def cache_unfinished_req(self, req: Req, chunked=False):
"""Cache request when it is unfinished."""
if self.disable:
return
radix_key, kv_indices, _ = self._adopt(req, req.get_fill_ids(), chunked=chunked)
radix_key, kv_indices, _ = self._insert_cache(
req, req.get_fill_ids(), chunked=chunked
)
# The prefix indices could be updated, reuse it
match_result = self.match_prefix(MatchPrefixParams(key=radix_key))
@@ -524,10 +523,9 @@ class RadixCache(BasePrefixCache):
new_indices[req.kv.cache_protected_len :],
)
# The cache_protected_len is not always equal to len(req.prefix_indices)
# since for page_size > 1, the partial part is added to req.prefix_indices, but that part of kv indices is not added to the tree.
# It should be freed in the next cache_unfinished_req and final cache_finished_req to avoid memory leak.
# So we introduce this `cache_protected_len` field to make sure the partial part can be freed correctly.
# With page_size > 1 the partial page sits in req.prefix_indices but not
# in the tree; cache_protected_len marks the tree-owned part so the next
# cache_unfinished_req or release_kv_cache frees the rest.
req.kv.cache_protected_len = len(new_indices)
self.dec_lock_ref(req.last_node)
@@ -382,18 +382,16 @@ class FlexKVRadixCache(RadixCache):
return fetched_slots, new_node
# ------------------------------------------------------------------
# cache_finished_req (STORE)
# insert_req (STORE)
# ------------------------------------------------------------------
def on_release(self, req: Req, *, inserted: bool) -> None:
if not inserted:
self._load_markers.pop(req.cache_request_handle, None)
def cache_finished_req( # type: ignore[override]
self, req: Req, *, owned_kv_len: int
) -> None:
"""Base cache_finished_req then fire an async FlexKV store."""
super().cache_finished_req(req, owned_kv_len=owned_kv_len)
def insert_req(self, req: Req, *, up_to: int) -> None: # type: ignore[override]
"""Base insert_req then fire an async FlexKV store."""
super().insert_req(req, up_to=up_to)
# Compute the committed prefix.
topk = get_spec().speculative_eagle_topk
@@ -259,11 +259,11 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
if not inserted:
self.release_aborted_request(req.cache_request_handle)
def cache_finished_req(self, req: Req, *, owned_kv_len: int, **kwargs) -> None:
self._publish_external_loaded_prefix(req, token_ids_len=owned_kv_len)
super().cache_finished_req(req, owned_kv_len=owned_kv_len, **kwargs)
def insert_req(self, req: Req, *, up_to: int, **kwargs) -> None:
self._publish_external_loaded_prefix(req, token_ids_len=up_to)
super().insert_req(req, up_to=up_to, **kwargs)
self._retire_loaded_flow(req.rid)
token_ids = (req.origin_input_ids + req.output_ids)[:owned_kv_len]
token_ids = (req.origin_input_ids + req.output_ids)[:up_to]
self._submit_store(req, token_ids)
self._request_session_finish(req.rid)
@@ -228,16 +228,16 @@ receipt proves were taken. The eventual full release must pass
---
### `cache_finished_req(req: Req, *, owned_kv_len: int)`
### `insert_req(req: Req, *, up_to: int)`
Cache a completed request's KV data into the tree.
| Aspect | Detail |
|--------|--------|
| **Purpose** | After a request finishes, insert its token/KV data into the tree for future reuse |
| **Inputs** | `req` — the finished request; `owned_kv_len` — end of the request-owned KV range; slots past it are freed by `release_kv_cache`. A request that leaves without inserting goes through `release_kv_cache(is_insert=False)` instead, which frees the row and calls `on_release(req, inserted=False)` for component cleanup |
| **Inputs** | `req` — the finished request; `up_to` — end of the request-owned KV range. `release_kv_cache` frees `[cache_protected_len, up_to)` and everything past it and unpins; with `is_insert=False` it skips the insert and calls `on_release(req, inserted=False)` for component cleanup |
| **Output** | `None` |
| **Mutation** | Calls component hooks → `insert` → `dec_lock_ref` → component cleanup. Frees unaligned tail KV indices. |
| **Mutation** | Calls component hooks → `insert` → component cleanup; advances `cache_protected_len` past the inserted key. Frees nothing and drops no lock: `release_kv_cache` does both afterwards. |
| **Complexity** | **O(K + D·C)** — insert O(K + D·C) + lock release O(D). Simplifies to **O(K)**. |
**Algorithm detail:**
@@ -10,6 +10,8 @@ from collections import OrderedDict
from dataclasses import dataclass
from typing import TYPE_CHECKING, Optional
from sglang.srt.mem_cache.unified_cache.unified_tree_core import NodeId
if TYPE_CHECKING:
from sglang.srt.managers.schedule_batch import Req
from sglang.srt.mem_cache.unified_cache.components.base import (
@@ -51,8 +53,9 @@ class UnifiedSessionRefTracker:
session_id = req.session.session_id
return session_id
def register_session_ref(self, req: Req) -> None:
"""Register a non-streaming request's reusable leaves with each component."""
def register_session_ref(self, req: Req, leaf: NodeId) -> None:
"""Register the leaf a finished request's insert ended on with each
component; the lock anchor ``req.last_node`` is a different node."""
if not self.enable_session_radix_cache:
return
@@ -69,14 +72,13 @@ class UnifiedSessionRefTracker:
logger.warning("register_session_ref called for stale request; Skip it.")
return
assert req.last_node is not None
last_node = self.tree_core.node_by_id(req.last_node)
if last_node is self.tree_core.root_node:
node = self.tree_core.node_by_id(leaf)
if node is self.tree_core.root_node:
return
for component in self.components:
leaf = component.resolve_session_leaf(req, last_node)
component.register_session_leaf(session_id, leaf)
component_leaf = component.resolve_session_leaf(req, node)
component.register_session_leaf(session_id, component_leaf)
def _remember_closed_session(self, session_id: str) -> None:
self._closed_session_ids[session_id] = None
@@ -975,18 +975,15 @@ class UnifiedRadixCache(BasePrefixCache):
for comp in self._components_tuple:
comp.cleanup_after_caching_req(req, is_finished=True)
@rank_consensus(same_params=["req.rid", "owned_kv_len"])
def cache_finished_req(self, req: Req, *, owned_kv_len: int, **kwargs) -> None:
@rank_consensus(same_params=["req.rid", "up_to"])
def insert_req(self, req: Req, *, up_to: int, **kwargs) -> None:
if self.disable:
self.free_kv_row(req.kv, [(req.kv.cache_protected_len, owned_kv_len)])
for comp in self._components_tuple:
comp.cleanup_after_caching_req(req, is_finished=True)
return
token_ids = (req.origin_input_ids + req.output_ids)[:owned_kv_len]
kv_indices = self.req_to_token_pool.req_to_token[
req.kv.req_pool_idx, :owned_kv_len
]
token_ids = (req.origin_input_ids + req.output_ids)[:up_to]
kv_indices = self.req_to_token_pool.req_to_token[req.kv.req_pool_idx, :up_to]
insert_params = InsertParams(
prev_prefix_len=req.kv.cache_protected_len,
@@ -1060,8 +1057,9 @@ class UnifiedRadixCache(BasePrefixCache):
)
)
# Everything past the inserted key goes back, the protected prefix
# never does. After a rotation decline nothing was inserted.
# Everything past the inserted key goes back to the caller, the
# protected prefix never does. After a rotation decline nothing was
# inserted.
free_from = (
min(req.kv.cache_protected_len, len(kv_indices))
if result.rotation_tail_declined
@@ -1074,12 +1072,7 @@ class UnifiedRadixCache(BasePrefixCache):
f"{free_from=} {len(kv_indices)=} {req.kv.cache_protected_len=}"
)
free_from = tail_free_start
self.free_kv_row(req.kv, [(free_from, len(kv_indices_full))])
self.unpin(req)
if result is not None and result.last_device_node is not None:
req.last_node = result.last_device_node
req.kv.cache_protected_len = free_from
# cleanup
for comp in self._components_tuple:
@@ -1093,7 +1086,9 @@ class UnifiedRadixCache(BasePrefixCache):
if req.finished_reason is not None and not isinstance(
req.finished_reason, FINISH_ABORT
):
self.session_refs.register_session_ref(req)
self.session_refs.register_session_ref(
req, leaf=result.last_device_node
)
@rank_consensus(same_params=["req.rid", "chunked"])
def advance_unpublished_req(self, req: Req, chunked: bool = False) -> None:
@@ -1217,7 +1212,7 @@ class UnifiedRadixCache(BasePrefixCache):
# gather contract forbids -- keep the request entirely on its own
# pages: no dedup free, no rebind, no protection change. The insert
# declined before its walk, so nothing was freed underneath us. The
# final cache_finished_req releases everything past the protected
# release_kv_cache releases everything past the protected
# prefix.
req.prefix_indices = kv_indices_orig.to(dtype=torch.int64, copy=True)
for comp in self._components_tuple:
@@ -423,7 +423,7 @@ class SessionController:
# An in-flight request is still decoding on this session's KV
# memory. Freeing now would corrupt the scheduler. Mark the
# session for deferred cleanup: the request keeps its session
# reference so cache_finished_req takes the streaming path,
# reference so release_kv_cache takes the streaming path,
# and we schedule release_session for after it completes.
session.close_on_finish = True
logger.info(
@@ -174,7 +174,7 @@ class StreamingSession(BasePrefixCache):
"""Returns an active slot for this req, or None.
Side effect: if req is pre-aborted (to_finish set, e.g. input too
long), detach it from the session so cache_finished_req treats it
long), detach it from the session so release_kv_cache treats it
as a normal req. The slot stays intact for the next request.
"""
if not _is_streaming(req):
@@ -359,8 +359,8 @@ class StreamingSession(BasePrefixCache):
def on_release(self, req: Req, *, inserted: bool) -> None:
self.inner.on_release(req, inserted=inserted)
def cache_finished_req(self, req: Req, **kwargs):
self.inner.cache_finished_req(req, **kwargs)
def insert_req(self, req: Req, **kwargs):
self.inner.insert_req(req, **kwargs)
def cache_unfinished_req(self, req: Req, **kwargs):
if self.try_cache_unfinished_req(req, **kwargs):
+8
View File
@@ -0,0 +1,8 @@
"""Shared helpers for mem_cache unit tests."""
def finish_req(cache, req, up_to):
"""insert_req, then what release_kv_cache does after it: free the rest, drop the lock."""
cache.insert_req(req, up_to=up_to)
cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, up_to)])
cache.unpin(req)