[mem_cache] Free the rows below the SWA evict floor on all-SWA request release (#40798)

This commit is contained in:
Liangsheng Yin
2026-09-23 14:46:28 -07:00
committed by GitHub
parent ab0c31bd94
commit c39a1c0663
9 changed files with 123 additions and 136 deletions
@@ -441,13 +441,15 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
"""
from sglang.srt.mem_cache.common import coalesce_ranges, free_kv_row_segments
allocator = self.token_to_kv_pool_allocator
row = self.req_to_token_pool.req_to_token[kv.req_pool_idx]
# Adjacent pieces whose seam falls inside one (DCP-widened) page would
# free that page twice; the allocator rejects that, so merge them first.
free_kv_row_segments(
self.token_to_kv_pool_allocator,
allocator,
[(row[start:end], start) for start, end in coalesce_ranges(ranges)],
swa_evicted_seqlen=kv.swa_evicted_seqlen,
swa_dead_lo=kv.swa_dead_lo(allocator.page_size),
)
@abstractmethod
+2 -35
View File
@@ -137,38 +137,5 @@ class SWAChunkCache(ChunkCache):
class PureSWAChunkCache(SWAChunkCache):
"""ChunkCache for all-SWA models (no full attention layers).
For hybrid models, full_to_swa_index_mapping prevents SWA double-free.
All-SWA models lack this mapping, so on request completion we must
explicitly skip the range already freed by ``free_swa_out_of_window_slots``
(a.k.a. _evict_swa) during decode.
``req.kv.swa_evict_floor`` shields the prompt/image KV from window eviction
only while the request is active, so that range IS released here on
finish. Distinct from the ``cache_protected_len`` prefix, which is owned
elsewhere and never freed by this path.
"""
def cache_finished_req(
self, req: Req, is_insert: bool = True, *, owned_kv_len: int
):
kv_indices = self.req_to_token_pool.req_to_token[
req.kv.req_pool_idx, :owned_kv_len
]
# The cache_protected_len prefix is not this req's to free.
protected_len = req.kv.cache_protected_len
evict_floor = req.kv.swa_evict_floor
evicted_seqlen = req.kv.swa_evicted_seqlen
if evicted_seqlen > evict_floor:
parts = []
if evict_floor > protected_len:
parts.append(kv_indices[protected_len:evict_floor])
if evicted_seqlen < owned_kv_len:
parts.append(
kv_indices[max(evicted_seqlen, protected_len) : owned_kv_len]
)
if parts:
self.token_to_kv_pool_allocator.free(torch.cat(parts))
else:
self.token_to_kv_pool_allocator.free(kv_indices[protected_len:])
"""ChunkCache for all-SWA models (no full attention layers): no
full_to_swa_index_mapping, so free_kv_row must skip the window-evicted span."""
+18 -12
View File
@@ -123,28 +123,34 @@ def free_kv_row_segments(
segments: list[tuple[torch.Tensor, int]],
*,
swa_evicted_seqlen: int,
swa_dead_lo: int = 0,
) -> None:
"""Free ascending disjoint ``(kv_indices, start_pos)`` segments of one
request's kv row, split at the SWA eviction floor."""
request's kv row; ``[swa_dead_lo, swa_evicted_seqlen)`` goes back full-side only."""
dead_lo, dead_hi = swa_dead_lo, max(swa_evicted_seqlen, swa_dead_lo)
swa_dead: list[tuple[torch.Tensor, int]] = []
swa_alive: list[tuple[torch.Tensor, int]] = []
for kv_indices, start_pos in segments:
num_indices = kv_indices.numel()
if num_indices == 0:
end_pos = start_pos + kv_indices.numel()
lo = min(max(dead_lo, start_pos), end_pos)
hi = min(max(dead_hi, start_pos), end_pos)
# start_pos <= lo <= hi <= end_pos
# inside [lo, hi) is the swa dead segment
if hi <= lo:
swa_alive.append((kv_indices, start_pos))
continue
# Below the floor the SWA peers are already gone -- window eviction, or
# the deliberately unmapped prefix of a PD decode SWA-tail prealloc.
num_dead = min(max(swa_evicted_seqlen - start_pos, 0), num_indices)
if num_dead > 0:
swa_dead.append((kv_indices[:num_dead], start_pos))
if num_dead < num_indices:
swa_alive.append((kv_indices[num_dead:], start_pos + num_dead))
swa_dead.append((kv_indices[lo - start_pos : hi - start_pos], lo))
if lo > start_pos:
swa_alive.append((kv_indices[: lo - start_pos], start_pos))
if end_pos > hi:
swa_alive.append((kv_indices[hi - start_pos :], hi))
if swa_dead and swa_alive:
# The two sides are separate calls, so neither one's page-disjointness
# check sees a floor that splits a page between them.
# check sees a boundary that splits a page between them.
assert swa_evicted_seqlen % allocator.page_size == 0, (
f"SWA eviction floor {swa_evicted_seqlen} splits a page "
f"SWA eviction cursor {swa_evicted_seqlen} splits a page "
f"(page_size {allocator.page_size})"
)
if swa_dead:
@@ -65,18 +65,10 @@ class PureSWARadixCache(RadixCache):
def cache_finished_req(
self, req: Req, is_insert: bool = True, *, owned_kv_len: int
):
"""Cache request when it finishes.
Only inserts the prefill portion [0, evict_floor) into the radix tree.
The window portion [swa_evicted_seqlen, committed_len) is freed back
to the allocator. The range [evict_floor, swa_evicted_seqlen) was already
freed by _evict_swa during decode — we skip it to avoid double-free.
"""
"""Insert only the prefill portion [0, evict_floor); free_kv_row skips
the span _evict_swa already freed during decode."""
if self.disable:
kv_indices = self.req_to_token_pool.req_to_token[
req.kv.req_pool_idx, :owned_kv_len
]
self.token_to_kv_pool_allocator.free(kv_indices)
self.free_kv_row(req.kv, [(req.kv.cache_protected_len, owned_kv_len)])
return
token_ids = (req.origin_input_ids + req.output_ids)[:owned_kv_len]
@@ -94,42 +86,26 @@ class PureSWARadixCache(RadixCache):
old_prefix_len = req.kv.cache_protected_len
swa_evict_floor = req.kv.swa_evict_floor
swa_evicted_seqlen = req.kv.swa_evicted_seqlen
if self.page_size > 1 and swa_evict_floor > 0:
swa_evict_floor = -(-swa_evict_floor // self.page_size) * self.page_size
if swa_evict_floor > 0:
insert_end = min(swa_evict_floor, keys_len)
else:
insert_end = keys_len
release_from = old_prefix_len
if is_insert and insert_end > 0:
insert_values = kv_indices[:insert_end].to(dtype=torch.int64, copy=True)
result = self.insert(
InsertParams(key=radix_key[:insert_end], value=insert_values)
)
new_prefix_len = result.prefix_len
if new_prefix_len > old_prefix_len:
self.token_to_kv_pool_allocator.free(
kv_indices[old_prefix_len:new_prefix_len]
)
alive_start = max(swa_evicted_seqlen, insert_end)
if alive_start < keys_len:
self.token_to_kv_pool_allocator.free(kv_indices[alive_start:keys_len])
else:
free_end = (
min(swa_evict_floor, keys_len) if swa_evict_floor > 0 else keys_len
self.token_to_kv_pool_allocator.free_segment(
kv_indices[old_prefix_len : result.prefix_len],
start_pos=old_prefix_len,
)
if free_end > old_prefix_len:
self.token_to_kv_pool_allocator.free(
kv_indices[old_prefix_len:free_end]
)
alive_start = max(swa_evicted_seqlen, free_end)
if alive_start < keys_len:
self.token_to_kv_pool_allocator.free(kv_indices[alive_start:keys_len])
release_from = insert_end
self.token_to_kv_pool_allocator.free(kv_indices[keys_len:])
self.free_kv_row(req.kv, [(release_from, owned_kv_len)])
if req.last_node is not None:
self.dec_lock_ref(req.last_node)