[mem_cache] Never free the protected prefix on request release (#41312)

This commit is contained in:
Liangsheng Yin
2026-09-26 22:00:02 -07:00
committed by GitHub
parent b4584ab838
commit d27efca353
3 changed files with 23 additions and 25 deletions
@@ -315,18 +315,19 @@ class DecodeKVCacheOffloadManager:
def _release_finished_req(self, req: Req):
# Defensive guard: ReqToTokenPool.free sets req_pool_idx to None,
# so a previously-released request must be skipped here to avoid
# non-idempotent side effects (e.g. tree_cache.protected_size_
# double-decrement, host pool double-free).
# non-idempotent side effects (double free, double unpin).
if req.kv.req_pool_idx is None or req.kv.req_pool_idx == -1:
return
# Released only at request finish; a mid-decode free races with
# concurrent admission over live slots.
self.tree_cache.free_kv_row(req.kv, [(0, req.kv.kv_allocated_len)])
# concurrent admission over live slots. The prefix the prealloc match
# took from the tree stays with the tree: not freed, only unlocked.
self.tree_cache.free_kv_row(
req.kv, [(req.kv.cache_protected_len, req.kv.kv_allocated_len)]
)
self.tree_cache.unpin(req)
self.req_to_token_pool.free(req)
req.kv.mark_kv_released()
self.tree_cache.protected_size_ -= len(req.prefix_indices)
self.offloaded_state.pop(req, None)
def _check_backup_progress(self, finish_count):
@@ -1060,25 +1060,21 @@ class UnifiedRadixCache(BasePrefixCache):
)
)
# Free the unaligned tail and the deferred truncation tail; after a
# rotation decline nothing was inserted, so everything past the
# protected prefix goes.
# Everything past the inserted key goes back, the protected prefix
# never does. After a rotation decline nothing was inserted.
free_from = (
# min(): the protected prefix can already run past a truncated
# cache_len, and free_kv_row takes ascending ranges only.
min(req.kv.cache_protected_len, len(kv_indices))
if result.rotation_tail_declined
else page_aligned_len
)
ranges = [(free_from, len(kv_indices))]
if tail_free_start is not None:
if free_from < len(kv_indices) and tail_free_start <= len(kv_indices):
# The two halves touch at the truncation boundary and share
# that page; free the union as one range.
ranges[0] = (free_from, len(kv_indices_full))
else:
ranges.append((tail_free_start, len(kv_indices_full)))
self.free_kv_row(req.kv, ranges)
if tail_free_start is not None and tail_free_start > len(kv_indices):
# Truncated below the protected prefix: only an untracked mamba
# request gets here, with an empty key, and owns nothing before it.
assert free_from == len(kv_indices), (
f"{free_from=} {len(kv_indices)=} {req.kv.cache_protected_len=}"
)
free_from = tail_free_start
self.free_kv_row(req.kv, [(free_from, len(kv_indices_full))])
self.unpin(req)