[mem_cache] Never free the protected prefix on request release (#41312)
This commit is contained in:
@@ -315,18 +315,19 @@ class DecodeKVCacheOffloadManager:
|
||||
def _release_finished_req(self, req: Req):
|
||||
# Defensive guard: ReqToTokenPool.free sets req_pool_idx to None,
|
||||
# so a previously-released request must be skipped here to avoid
|
||||
# non-idempotent side effects (e.g. tree_cache.protected_size_
|
||||
# double-decrement, host pool double-free).
|
||||
# non-idempotent side effects (double free, double unpin).
|
||||
if req.kv.req_pool_idx is None or req.kv.req_pool_idx == -1:
|
||||
return
|
||||
|
||||
# Released only at request finish; a mid-decode free races with
|
||||
# concurrent admission over live slots.
|
||||
self.tree_cache.free_kv_row(req.kv, [(0, req.kv.kv_allocated_len)])
|
||||
|
||||
# concurrent admission over live slots. The prefix the prealloc match
|
||||
# took from the tree stays with the tree: not freed, only unlocked.
|
||||
self.tree_cache.free_kv_row(
|
||||
req.kv, [(req.kv.cache_protected_len, req.kv.kv_allocated_len)]
|
||||
)
|
||||
self.tree_cache.unpin(req)
|
||||
self.req_to_token_pool.free(req)
|
||||
req.kv.mark_kv_released()
|
||||
self.tree_cache.protected_size_ -= len(req.prefix_indices)
|
||||
self.offloaded_state.pop(req, None)
|
||||
|
||||
def _check_backup_progress(self, finish_count):
|
||||
|
||||
@@ -1060,25 +1060,21 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
)
|
||||
)
|
||||
|
||||
# Free the unaligned tail and the deferred truncation tail; after a
|
||||
# rotation decline nothing was inserted, so everything past the
|
||||
# protected prefix goes.
|
||||
# Everything past the inserted key goes back, the protected prefix
|
||||
# never does. After a rotation decline nothing was inserted.
|
||||
free_from = (
|
||||
# min(): the protected prefix can already run past a truncated
|
||||
# cache_len, and free_kv_row takes ascending ranges only.
|
||||
min(req.kv.cache_protected_len, len(kv_indices))
|
||||
if result.rotation_tail_declined
|
||||
else page_aligned_len
|
||||
)
|
||||
ranges = [(free_from, len(kv_indices))]
|
||||
if tail_free_start is not None:
|
||||
if free_from < len(kv_indices) and tail_free_start <= len(kv_indices):
|
||||
# The two halves touch at the truncation boundary and share
|
||||
# that page; free the union as one range.
|
||||
ranges[0] = (free_from, len(kv_indices_full))
|
||||
else:
|
||||
ranges.append((tail_free_start, len(kv_indices_full)))
|
||||
self.free_kv_row(req.kv, ranges)
|
||||
if tail_free_start is not None and tail_free_start > len(kv_indices):
|
||||
# Truncated below the protected prefix: only an untracked mamba
|
||||
# request gets here, with an empty key, and owns nothing before it.
|
||||
assert free_from == len(kv_indices), (
|
||||
f"{free_from=} {len(kv_indices)=} {req.kv.cache_protected_len=}"
|
||||
)
|
||||
free_from = tail_free_start
|
||||
self.free_kv_row(req.kv, [(free_from, len(kv_indices_full))])
|
||||
|
||||
self.unpin(req)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user