[mem_cache] Rename cache_unfinished_req to checkpoint_req and count cache hits only when a request finishes (#41520)

This commit is contained in:
Liangsheng Yin
2026-10-01 21:41:45 -07:00
committed by GitHub
parent b906cd3f45
commit 50be533d09
49 changed files with 350 additions and 447 deletions
+3 -2
View File
@@ -150,6 +150,7 @@ def _bind_root_prefix(req: Req, tree_cache: BasePrefixCache) -> None:
req.best_match_node = req.last_node
req.lock_receipt = DecLockRefParams()
req.kv.cache_protected_len = 0
req.kv.cache_inserted_len = 0
req.num_matched_prefix_tokens = 0
req.host_hit_length = 0
@@ -2200,7 +2201,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
kv_loc,
)
# Truncate fill_len to kv_committed_len so cache_unfinished_req only
# Truncate fill_len to kv_committed_len so insert_req only
# inserts committed KV into the radix tree. The last output token
# hasn't had KV committed yet (output_ids is 1 ahead).
req.full_untruncated_fill_ids = req.origin_input_ids + req.output_ids
@@ -3124,7 +3125,7 @@ class SchedulerDisaggregationDecodeMixin:
else:
tree_cache = self.tree_cache
req.init_next_round_input(tree_cache)
# Truncate fill_len to kv_committed_len so cache_unfinished_req
# Truncate fill_len to kv_committed_len so insert_req
# only sees committed KV (full array includes one uncommitted
# token because init_next_round_input rebuilt it as full).
if req.kv.kv_committed_len is not None:
@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, List
import torch
from sglang.srt.managers.overlap_utils import RelayPayload
from sglang.srt.mem_cache.common import maybe_cache_unfinished_req
from sglang.srt.mem_cache.common import checkpoint_kv_cache
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo
@@ -120,7 +120,7 @@ class ScheduleBatchDisaggregationDecodeMixin:
# PREBUILT does not materialize a local SWA branching window.
if req.swa_branching_seqlen is not None:
req.swa_branching_seqlen = None
maybe_cache_unfinished_req(req, self.tree_cache)
checkpoint_kv_cache(req, self.tree_cache)
if req.grammar is not None:
# FIXME: this try-except block is for handling unexpected xgrammar issue.
try:
+10 -10
View File
@@ -73,9 +73,9 @@ from sglang.srt.managers.schedule_batch import (
)
from sglang.srt.mem_cache.base_prefix_cache import CacheRequestOutcome
from sglang.srt.mem_cache.common import (
checkpoint_kv_cache,
kv_to_page_indices,
kv_to_page_num,
maybe_cache_unfinished_req,
release_kv_cache,
)
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
@@ -581,15 +581,13 @@ class SchedulerDisaggregationPrefillMixin:
if room is not None and room in kv_mgr.transfer_infos:
prefetch(room)
def cache_unfinished_disagg_prefill(
self: Scheduler, req: Req, *, chunked: bool = False
) -> None:
def checkpoint_disagg_prefill(self: Scheduler, req: Req) -> None:
cache = self.tree_cache
if req.pending_bootstrap and _uses_write_through_cache(cache):
cache.advance_unpublished_req(req, chunked=chunked)
cache.advance_unpublished_req(req)
return
maybe_cache_unfinished_req(req, cache, chunked=chunked)
checkpoint_kv_cache(req, cache)
def release_aborted_prefill_waiting_req(self: Scheduler, req: Req) -> None:
self.clear_pending_chunk_send(req)
@@ -897,7 +895,7 @@ class SchedulerDisaggregationPrefillMixin:
advance_logprob_pt(i, req)
continue
self.cache_unfinished_disagg_prefill(req)
self.checkpoint_disagg_prefill(req)
self.disagg_prefill_inflight_queue.append(req)
if self.spec_algorithm.is_eagle() and draft_input is not None:
req.output_topk_p = draft_input.topk_p[i]
@@ -1140,6 +1138,8 @@ class SchedulerDisaggregationPrefillMixin:
else:
logger.warning(error_message)
req.time_stats.trace_ctx.abort(abort_info={"reason": error_message})
if req.finished_reason is None:
req.finished_reason = FINISH_LENGTH(length=0)
release_kv_cache(req, self.tree_cache) # unlock the tree
self._release_aborted_request(req)
if not isinstance(req.finished_reason, FINISH_ABORT):
@@ -1228,7 +1228,7 @@ class SchedulerDisaggregationPrefillMixin:
# Metadata buffer was allocated in pop_bootstrapped before
# the request entered the waiting queue, so finalize should not fail.
assert self.disagg_prefill_bootstrap_queue.finalize_bootstrap(req)
self.cache_unfinished_disagg_prefill(req)
self.checkpoint_disagg_prefill(req)
return True
else:
raise RuntimeError(
@@ -1255,7 +1255,7 @@ class SchedulerDisaggregationPrefillMixin:
chunked_req_to_exclude = set()
if (req := self.chunked_req) is not None:
chunked_req_to_exclude.add(req)
self.cache_unfinished_disagg_prefill(req, chunked=True)
self.checkpoint_disagg_prefill(req)
if not self.check_bootstrap(req):
if is_aborted(req):
@@ -1555,7 +1555,7 @@ class SchedulerDisaggregationPrefillMixin:
max_attempts = get_disagg().optimistic_prefill_attempts
uses_write_through_cache = _uses_write_through_cache(self.tree_cache)
if not uses_write_through_cache:
maybe_cache_unfinished_req(req, self.tree_cache)
checkpoint_kv_cache(req, self.tree_cache)
# The cached prefix is evictable once the KV is released. Its length
# (capped at what a retry can match) seeds the retry's storage baseline,
# so an evicted prefix is looked up in L3 once before it is recomputed.
+6 -2
View File
@@ -933,6 +933,9 @@ class ReqKvInfo:
# The request's own KV is [cache_protected_len, kv_allocated_len).
cache_protected_len: int = 0 # Tree cache owns [0, here) (matched or inserted)
# This request already inserted [0, here) into the tree; later inserts count
# a hit only on the nodes past it, so a request counts each node once.
cache_inserted_len: int = 0
kv_committed_len: int = 0 # KV content committed up to here, <= kv_allocated_len
kv_allocated_len: int = 0
@@ -1217,7 +1220,7 @@ class Req(ReqDllmMixin):
# Refreshed at every sharded alloc — read through last_node, or drawn
# least-full for a new chain — and consumed by the radix insert to
# stamp new tree nodes. Allocation itself must NOT read it back when
# a tree node is available (the cache_unfinished_req dedup rebind
# a tree node is available (the insert_req dedup rebind
# would make it stale); the only allocation-time reader is the
# ChunkCache fallback, which has no tree nodes and no rebind.
self.kv_rotation_base: Optional[int] = None
@@ -1978,6 +1981,7 @@ class Req(ReqDllmMixin):
self.indexer_topk = None
self.last_node = None
self.kv.cache_protected_len = 0
self.kv.cache_inserted_len = 0
self.kv_rotation_base = None
self.num_matched_prefix_tokens = 0
self.lock_receipt = DecLockRefParams()
@@ -3257,7 +3261,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
for req, seq_len in zip(self.reqs, seq_lens, strict=True):
req._refresh_fill_ids()
# end runs one past full_untruncated_fill_ids while output_ids
# trails; safe only while decoding_reqs suppresses cache_unfinished_req.
# trails; safe only while decoding_reqs suppresses the checkpoint insert.
req.set_extend_range(seq_len - 1, seq_len)
self.prefix_lens = [seq_len - 1 for seq_len in seq_lens]
+5 -3
View File
@@ -291,8 +291,8 @@ from sglang.srt.mem_cache import kv_cache_builder
from sglang.srt.mem_cache.allocator.page_interleave import page_interleave_shard_size
from sglang.srt.mem_cache.base_prefix_cache import CacheRequestOutcome
from sglang.srt.mem_cache.common import (
checkpoint_kv_cache,
discard_kv_cache_backup,
maybe_cache_unfinished_req,
release_kv_cache,
)
from sglang.srt.model_executor.forward_batch_info import PPProxyTensors
@@ -3533,9 +3533,9 @@ class Scheduler(
def stash_chunked_request(self, req: Req):
if self.disaggregation_mode == DisaggregationMode.PREFILL:
self.cache_unfinished_disagg_prefill(req, chunked=True)
self.checkpoint_disagg_prefill(req)
else:
maybe_cache_unfinished_req(req, self.tree_cache, chunked=True)
checkpoint_kv_cache(req, self.tree_cache)
def process_pending_chunked_abort(self) -> None:
"""Abort an in-flight chunked-prefill request once it is safe to do so.
@@ -5321,6 +5321,8 @@ class Scheduler(
discard_kv_cache_backup(req, self.tree_cache, "host_pool")
if self.enable_hisparse:
self.hisparse_coordinator.request_finished(req)
if req.finished_reason is None:
req.finished_reason = FINISH_ABORT()
release_kv_cache(req, self.tree_cache)
if self.disaggregation_mode == DisaggregationMode.PREFILL:
self.release_aborted_prefill_waiting_req(req)
@@ -30,7 +30,7 @@ from sglang.srt.managers.schedule_batch import (
mamba_lazy_spec_in_window,
)
from sglang.srt.mem_cache.common import (
maybe_cache_unfinished_req,
checkpoint_kv_cache,
release_kv_cache,
)
from sglang.srt.model_executor.forward_batch_info import (
@@ -381,7 +381,7 @@ class SchedulerBatchResultProcessor:
)
req.time_stats.set_completion_time()
elif not batch.decoding_reqs or req not in batch.decoding_reqs:
maybe_cache_unfinished_req(req, self.tree_cache)
checkpoint_kv_cache(req, self.tree_cache)
if get_memory().enable_hisparse:
self.hisparse_coordinator.admit_request_into_staging(req)
@@ -472,7 +472,7 @@ class SchedulerBatchResultProcessor:
release_kv_cache(req, self.tree_cache)
req.time_stats.set_completion_time()
else:
maybe_cache_unfinished_req(req, self.tree_cache)
checkpoint_kv_cache(req, self.tree_cache)
else:
# being chunked reqs' prefill is not finished
req.inflight_middle_chunks -= 1
+1 -1
View File
@@ -255,7 +255,7 @@ def _kv_shard_rotation_bases(
The owner class of position-page P is ``(b_i + P) % shard_size``. Rules:
- Read through ``req.last_node`` at alloc time, never a value cached on
the request: ``cache_unfinished_req`` can rebind a chunked request onto
the request: a checkpoint ``insert_req`` can rebind a chunked request onto
another chain's canonical locs between chunks, changing the base. The
read goes through ``tree_cache.rotation_base_of`` because the node
handle is tree-specific (a NodeId on the unified tree).
@@ -95,7 +95,9 @@ class InsertParams:
component_evicted_seqlens: dict[ComponentType, int] = dataclasses.field(
default_factory=dict, kw_only=True
)
chunked: bool = False
# The inserting request already inserted [0, here) (req.kv.cache_inserted_len);
# only the nodes past it count a hit, so a request counts each node once.
inserted_len: int = 0
priority: int = 0
session_id: Optional[str] = None
track_adopted_ranges: bool = False
@@ -458,14 +460,14 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
@abstractmethod
def insert_req(self, req: Req, *, up_to: int, **kwargs):
"""Hand a finished request's KV up to row position ``up_to`` to the
tree: insert what can be keyed and advance ``cache_protected_len``
past it. The caller then frees ``[cache_protected_len, up_to)`` and
everything after, and unpins; nothing here releases a slot."""
@abstractmethod
def cache_unfinished_req(self, req: Req, **kwargs):
pass
"""Insert the request's KV up to row position ``up_to`` into the tree,
repoint the row onto the tree's copy, re-anchor ``req.last_node`` on
the node the insert ended on and advance ``cache_protected_len``.
Called at every checkpoint of a running request and once more when
it finishes (``req.finished()``), when the tree also takes over the
component state the request no longer needs. Nothing here frees a
slot: ``release_kv_cache`` frees ``[cache_protected_len, up_to)`` and
everything after, and unpins."""
def free_kv_row(self, kv: Any, ranges: list[tuple[int, int]]) -> None:
"""Give back ascending, disjoint, half-open row-position ranges
+1 -6
View File
@@ -77,12 +77,7 @@ class ChunkCache(BasePrefixCache):
return InsertResult(prefix_len=0)
def insert_req(self, req: Req, *, up_to: int):
pass
def cache_unfinished_req(self, req: Req, chunked=False):
kv_indices = self.req_to_token_pool.req_to_token[
req.kv.req_pool_idx, : req.extend_range.end
]
kv_indices = self.req_to_token_pool.req_to_token[req.kv.req_pool_idx, :up_to]
# `req.prefix_indices` will be used in `PrefillAdder::add_chunked_req` later
req.prefix_indices = kv_indices.to(dtype=torch.int64, copy=True)
+14 -2
View File
@@ -169,11 +169,16 @@ def free_kv_row_segments(
allocator.free_segments(swa_alive)
def maybe_cache_unfinished_req(req: Req, tree_cache: BasePrefixCache, **kwargs):
def checkpoint_kv_cache(req: Req, tree_cache: BasePrefixCache) -> None:
"""Publish what the running request has computed so far, unless it is
barred from the tree."""
# The tree reads req.finished() to tell a checkpoint from the final
# insert; a finished request belongs in release_kv_cache.
assert not req.finished(), f"checkpointing finished request {req.rid}"
if req.skip_radix_cache_insert:
return
tree_cache.cache_unfinished_req(req, **kwargs)
tree_cache.insert_req(req, up_to=req.extend_range.end)
def evict_from_tree_cache(
@@ -313,6 +318,13 @@ def release_kv_cache(req: Req, tree_cache: BasePrefixCache, is_insert: bool = Tr
owned_kv_len = req.owned_kv_len()
is_insert = is_insert and not req.skip_radix_cache_insert
if is_insert:
# A tree that takes over component state (mamba) must see the request
# finished, or the insert forks the state and the slot leaks.
assert req.finished() or not tree_cache.supports_mamba(), (
f"releasing unfinished request {req.rid} into a mamba tree"
)
# The fill-id array lags output_ids until the next prepare_for_decode.
req._refresh_fill_ids()
tree_cache.insert_req(req, up_to=owned_kv_len)
# The protected prefix is not this req's to free.
tree_cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, owned_kv_len)])
+1 -1
View File
@@ -1034,7 +1034,7 @@ class MambaPool:
ReplaySSM invariant: the SOURCE must be a fully-flushed checkpoint
(``write_pos[src] == 0``). Only ``temporal`` is copied, not the ring, so
an un-flushed source would drop its last ``write_pos`` updates. Callers
comply: COW copies radix checkpoints; ``cache_unfinished_req`` copies an
comply: COW copies radix checkpoints; a checkpoint ``insert_req`` copies an
active slot only during prefill (ring empty); ``insert_req``
caps the donate to the last flush boundary. The dst cursor is reset to 0
(the copied checkpoint has no pending ring entries).
@@ -3,15 +3,10 @@
from __future__ import annotations
import logging
from typing import TYPE_CHECKING
from sglang.srt.mem_cache.base_prefix_cache import EvictParams, EvictResult
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
from sglang.srt.mem_cache.radix_cache import RadixCache
from sglang.srt.utils.common import ceil_align
if TYPE_CHECKING:
from sglang.srt.managers.schedule_batch import Req
logger = logging.getLogger(__name__)
@@ -57,23 +52,6 @@ class PureSWARadixCache(RadixCache):
num_tokens = max(params.num_tokens, params.swa_num_tokens)
return super().evict(EvictParams(num_tokens=num_tokens))
def insert_req(self, req: Req, *, up_to: int):
"""Insert only the prefill portion [0, swa_evict_floor)."""
if self.disable:
return
token_ids = (req.origin_input_ids + req.output_ids)[:up_to]
swa_evict_floor = req.kv.swa_evict_floor
key_limit = (
ceil_align(swa_evict_floor, self.page_size) if swa_evict_floor > 0 else None
)
radix_key, _, _ = self._insert_cache(req, token_ids, key_limit=key_limit)
req.kv.cache_protected_len = len(radix_key)
def cache_unfinished_req(self, req: Req, chunked=False):
"""During chunked prefill, swa_evicted_seqlen is 0 and no SWA eviction
has happened yet, so standard RadixCache logic is correct."""
super().cache_unfinished_req(req, chunked=chunked)
def available_and_evictable_str(self) -> str:
allocator = self.token_to_kv_pool_allocator
swa_available = allocator.swa_available_size()
+24 -29
View File
@@ -50,6 +50,7 @@ from sglang.srt.mem_cache.utils import (
get_eviction_strategy,
split_node_hash_value,
)
from sglang.srt.utils.common import ceil_align
if TYPE_CHECKING:
from sglang.srt.managers.schedule_batch import Req
@@ -418,7 +419,7 @@ class RadixCache(BasePrefixCache):
key = params.key
value = params.value
priority = params.priority
chunked = params.chunked
inserted_len = params.inserted_len
key, value = key.maybe_to_bigram_view(self.is_eagle, value)
key = key.page_aligned(self.page_size)
@@ -429,7 +430,7 @@ class RadixCache(BasePrefixCache):
value = torch.tensor(key.token_ids[: len(key)], dtype=torch.int64)
prefix_len, last_node = self._insert_helper(
self.root_node, key, value, priority, chunked
self.root_node, key, value, priority, inserted_len
)
return InsertResult(prefix_len=prefix_len, last_device_node=last_node)
@@ -438,7 +439,6 @@ class RadixCache(BasePrefixCache):
req: Req,
token_ids,
*,
chunked: bool = False,
key_limit: Optional[int] = None,
split_prompt: bool = False,
) -> Tuple[RadixKey, torch.Tensor, int]:
@@ -462,9 +462,13 @@ class RadixCache(BasePrefixCache):
priority = req.priority or 0
result = self.insert(
InsertParams(
key=radix_key, value=values, chunked=chunked, priority=priority
key=radix_key,
value=values,
inserted_len=req.kv.cache_inserted_len,
priority=priority,
)
)
req.kv.cache_inserted_len = max(req.kv.cache_inserted_len, key_len)
if split_prompt:
# Split the leaf at the prompt boundary so eviction can drop the
# output KV without the prompt; a prefix re-insert only changes topology.
@@ -479,10 +483,8 @@ class RadixCache(BasePrefixCache):
InsertParams(
key=prompt_key,
value=values[: len(prompt_key)],
inserted_len=len(prompt_key),
priority=priority + 1,
# The request created these nodes moments ago; another
# hit_count bump would promote every prompt node.
chunked=True,
)
)
@@ -495,17 +497,14 @@ class RadixCache(BasePrefixCache):
def insert_req(self, req: Req, *, up_to: int):
if self.disable:
return
token_ids = (req.origin_input_ids + req.output_ids)[:up_to]
radix_key, _, _ = self._insert_cache(req, token_ids, split_prompt=True)
req.kv.cache_protected_len = len(radix_key)
def cache_unfinished_req(self, req: Req, chunked=False):
"""Cache request when it is unfinished."""
if self.disable:
return
token_ids = req.full_untruncated_fill_ids[:up_to]
# Pure-SWA trees never cache what the window already evicted.
swa_evict_floor = req.kv.swa_evict_floor
key_limit = (
ceil_align(swa_evict_floor, self.page_size) if swa_evict_floor > 0 else None
)
radix_key, kv_indices, _ = self._insert_cache(
req, req.get_fill_ids(), chunked=chunked
req, token_ids, key_limit=key_limit, split_prompt=True
)
# The prefix indices could be updated, reuse it
@@ -525,7 +524,7 @@ class RadixCache(BasePrefixCache):
# With page_size > 1 the partial page sits in req.prefix_indices but not
# in the tree; cache_protected_len marks the tree-owned part so the next
# cache_unfinished_req or release_kv_cache frees the rest.
# insert_req or release_kv_cache frees the rest.
req.kv.cache_protected_len = len(new_indices)
self.dec_lock_ref(req.last_node)
@@ -540,7 +539,6 @@ class RadixCache(BasePrefixCache):
)
else:
req.prefix_indices = new_indices
req.last_node = new_last_node
def pretty_print(self):
@@ -687,12 +685,7 @@ class RadixCache(BasePrefixCache):
return new_node
def _inc_hit_count(self, node: TreeNode, chunked: bool = False):
# Skip the hit count update for chunked requests to avoid self-referencing
# inflation where a chunked request increments hit_count on nodes it created
# in previous chunks.
if chunked:
return
def _inc_hit_count(self, node: TreeNode):
node.hit_count += 1
def _insert_helper(
@@ -701,7 +694,7 @@ class RadixCache(BasePrefixCache):
key: RadixKey,
value,
priority: int = 0,
chunked: bool = False,
inserted_len: int = 0,
):
# Convert None priority to 0
if priority is None:
@@ -727,11 +720,12 @@ class RadixCache(BasePrefixCache):
if prefix_len < len(node.key):
new_node = self._split_node(node.key, node, prefix_len)
new_node.priority = max(new_node.priority, priority)
self._inc_hit_count(new_node, chunked)
node = new_node
else:
node.priority = max(node.priority, priority)
self._inc_hit_count(node, chunked)
# Nodes this request already inserted were counted back then.
if total_prefix_length > inserted_len:
self._inc_hit_count(node)
if len(key):
child_key = key.child_key(self.page_size)
@@ -740,7 +734,8 @@ class RadixCache(BasePrefixCache):
new_node.parent = node
new_node.key = key
new_node.value = value.clone()
self._inc_hit_count(new_node, chunked)
if total_prefix_length + len(key) > inserted_len:
self._inc_hit_count(new_node)
node.children[child_key] = new_node
self.evictable_size_ += len(key)
self._update_leaf_status(node)
@@ -726,7 +726,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
prev_prefix_len=params.prev_prefix_len,
swa_evicted_seqlen=params.get_evicted_seqlen(ComponentType.SWA),
swa_branching_seqlen=params.swa_branching_seqlen,
chunked=params.chunked,
inserted_len=params.inserted_len,
priority=0 if params.priority is None else params.priority,
track_adopted_ranges=params.track_adopted_ranges,
rotation_base=params.rotation_base,
@@ -390,8 +390,10 @@ class FlexKVRadixCache(RadixCache):
self._load_markers.pop(req.cache_request_handle, None)
def insert_req(self, req: Req, *, up_to: int) -> None: # type: ignore[override]
"""Base insert_req then fire an async FlexKV store."""
"""Base insert_req; a finished request also fires an async FlexKV store."""
super().insert_req(req, up_to=up_to)
if not req.finished():
return
# Compute the committed prefix.
topk = get_spec().speculative_eagle_topk
@@ -248,12 +248,6 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
return True
def cache_unfinished_req(self, req: Req, chunked: bool = False, **kwargs) -> None:
self._publish_external_loaded_prefix(req, token_ids_len=len(req.get_fill_ids()))
super().cache_unfinished_req(req, chunked=chunked, **kwargs)
self._retire_loaded_flow(req.rid)
self._submit_store(req, req.get_fill_ids())
def on_release(self, req: Req, *, inserted: bool) -> None:
super().on_release(req, inserted=inserted)
if not inserted:
@@ -263,9 +257,9 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
self._publish_external_loaded_prefix(req, token_ids_len=up_to)
super().insert_req(req, up_to=up_to, **kwargs)
self._retire_loaded_flow(req.rid)
token_ids = (req.origin_input_ids + req.output_ids)[:up_to]
self._submit_store(req, token_ids)
self._request_session_finish(req.rid)
self._submit_store(req, req.full_untruncated_fill_ids[:up_to])
if req.finished():
self._request_session_finish(req.rid)
def check_hicache_events(self) -> None:
"""Poll LMCache retrieve/store futures at the scheduler safe point."""
@@ -811,7 +805,7 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
mamba_value=checkpoint,
prev_prefix_len=prev_prefix_len,
component_evicted_seqlens=req.kv.component_evicted_seqlens.copy(),
chunked=True,
inserted_len=len(key),
priority=req.priority or 0,
)
)
@@ -230,47 +230,23 @@ receipt proves were taken. The eventual full release must pass
### `insert_req(req: Req, *, up_to: int)`
Cache a completed request's KV data into the tree.
Insert a request's KV into the tree: at every checkpoint while it runs, and once more when it finishes.
| Aspect | Detail |
|--------|--------|
| **Purpose** | After a request finishes, insert its token/KV data into the tree for future reuse |
| **Inputs** | `req` — the finished request; `up_to` — end of the request-owned KV range. `release_kv_cache` frees `[cache_protected_len, up_to)` and everything past it and unpins; with `is_insert=False` it skips the insert and calls `on_release(req, inserted=False)` for component cleanup |
| **Purpose** | Publish the request's KV `[cache_protected_len, up_to)` so other requests can match it; nodes past `req.kv.cache_inserted_len` count one hit (a request counts each node once). `checkpoint_kv_cache` calls it with `req.extend_range.end` while the request runs; `release_kv_cache` calls it with the request-owned length when it finishes (`req.finished()`), then frees `[cache_protected_len, up_to)` and everything past it and unpins; with `is_insert=False` it skips the insert and calls `on_release(req, inserted=False)` for component cleanup |
| **Inputs** | `req` — the request; `up_to` — row position the insert may read up to |
| **Output** | `None` |
| **Mutation** | Calls component hooks → `insert` → component cleanup; advances `cache_protected_len` past the inserted key. Frees nothing and drops no lock: `release_kv_cache` does both afterwards. |
| **Complexity** | **O(K + D·C)** — insert O(K + D·C) + lock release O(D). Simplifies to **O(K)**. |
| **Mutation** | Component hooks → `insert` → re-match → writes the tree's indices back into the row → moves the request's lock from the old `req.last_node` to the node the insert ended on → updates `req.prefix_indices`, `req.kv.cache_protected_len`, `req.kv.cache_inserted_len`, `req.last_node` → component cleanup. A finished request's component state (Mamba) is handed to the tree instead of forked, and what it still held is freed. Frees no KV slot: `release_kv_cache` does that afterwards. |
| **Complexity** | **O(K + D·C)** — insert O(K + D·C) + re-match O(K + D·C) + lock transfer O(D). Simplifies to **O(K)**. |
**Algorithm detail:**
1. `prepare_for_caching_req()` per component — sets component-specific insert params, returns effective cache length (SWA: copies its cursor with `set_evicted_seqlen`; Mamba: prepares `mamba_value` from ping-pong buffer, returns `mamba_last_track_seqlen` as truncation hint)
2. Truncates if `effective_cache_len < len(token_ids)`: frees excess pool indices
3. Converts token IDs (bigram if EAGLE), page-aligns keys, then calls `insert()`
4. Frees unaligned tail KV indices beyond page boundary
5. Calls `dec_lock_ref()` on the previous `req.last_node`
6. `cleanup_after_caching_req()` per component (Mamba: frees forked mamba_value based on `mamba_exist`, handles ping-pong buffer cleanup)
---
### `cache_unfinished_req(req: Req, chunked=False)`
Cache an in-progress request's partial KV data (chunked prefill).
| Aspect | Detail |
|--------|--------|
| **Purpose** | During chunked prefill, insert partial results so the next chunk can match the prefix |
| **Inputs** | `req` — the in-progress request |
| **Output** | `None` |
| **Mutation** | Inserts partial KV → re-matches prefix → updates `req.prefix_indices`, `req.kv.cache_protected_len`, `req.last_node`; transfers lock from old node to new node |
| **Complexity** | **O(K + D·C)** — two tree traversals: insert O(K + D·C) + re-match O(K + D·C) + lock transfer O(D). Simplifies to **O(K)**. |
**Algorithm detail:**
1. `prepare_for_caching_req()` per component
2. `insert()` — first tree traversal
3. `match_prefix()` — **second** tree traversal to get updated indices
4. Writes new prefix indices into `req_to_token_pool`
5. `dec_lock_ref()` on old `req.last_node`
6. `inc_lock_ref()` on new matched node
7. Updates `req.prefix_indices`, `req.kv.cache_protected_len`, `req.last_node`
8. `cleanup_after_caching_req()` per component
1. `prepare_for_caching_req(is_finished=req.finished())` per component — sets component-specific insert params, returns effective cache length (SWA: copies its cursor with `set_evicted_seqlen`, may return a branching boundary; Mamba: prepares `mamba_value`, donated when finished and forked otherwise, returns `mamba_last_track_seqlen` as truncation hint)
2. Truncates the key to `effective_cache_len`; the row past it stays the request's
3. Converts token IDs (bigram if EAGLE), page-aligns the key, then calls `insert()`; a single-component tree re-inserts the prompt part so eviction can drop the output without the prompt
4. `match_prefix()` on the inserted key and writes the matched indices into the row
5. `dec_lock_ref()` on the old `req.last_node`, `inc_lock_ref()` on the new one
6. `cleanup_after_caching_req()` per component (Mamba: frees the forked `mamba_value` when the tree already had one, frees the finished request's slot)
---
@@ -363,7 +363,7 @@ class UnifiedCacheLinkerWrapper:
if req.kv is not None
else {}
),
chunked=True,
inserted_len=len(hit.prefix_key),
priority=req.priority or 0,
track_adopted_ranges=True,
)
@@ -1053,11 +1053,9 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
continue
comp.refresh_lru(LRURefreshPhase.WALKDOWN, node, self.root_node)
def _inc_hit_count_and_check(
self, node: UnifiedTreeNode, chunked: bool = False
) -> bool:
def _inc_hit_count_and_check(self, node: UnifiedTreeNode) -> bool:
"""Increment hit count; check whether a write backup should be fired."""
if node.evicted or chunked:
if node.evicted:
return False
if self.is_write_back:
return False
@@ -1307,7 +1305,9 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
if swa_already_freed < dup.numel():
step_actions.append(FreeDeviceKV([dup[swa_already_freed:]]))
if self._inc_hit_count_and_check(node, state.params.chunked):
# Nodes this request already inserted were counted back then.
node_end = state.total_prefix_length + prefix_len
if node_end > state.params.inserted_len and self._inc_hit_count_and_check(node):
step_actions.append(self._build_backup_kv_action(node))
state.node = node
state.total_prefix_length += prefix_len
@@ -1368,8 +1368,10 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
def _should_backup_after_insert(self, state: _InsertWalkState) -> bool:
"""Check whether the insert target needs a Host backup."""
if state.is_new_leaf:
return self._inc_hit_count_and_check(
state.target_node, state.params.chunked
leaf_end = state.total_prefix_length + len(state.target_node.key)
return (
leaf_end > state.params.inserted_len
and self._inc_hit_count_and_check(state.target_node)
)
node = state.target_node
@@ -1004,171 +1004,21 @@ class UnifiedRadixCache(BasePrefixCache):
@rank_consensus(same_params=["req.rid", "up_to"])
def insert_req(self, req: Req, *, up_to: int, **kwargs) -> None:
if self.disable:
for comp in self._components_tuple:
comp.cleanup_after_caching_req(req, is_finished=True)
if self.session.try_insert_req(req, up_to=up_to, **kwargs):
return
token_ids = (req.origin_input_ids + req.output_ids)[:up_to]
kv_indices = self.req_to_token_pool.req_to_token[req.kv.req_pool_idx, :up_to]
insert_params = InsertParams(
prev_prefix_len=req.kv.cache_protected_len,
priority=req.priority or 0,
session_id=req.session_id,
rotation_base=req.kv_rotation_base,
)
# components prepare insert data + return effective cache_len
effective_cache_len = len(token_ids)
for comp in self._components_tuple:
cl = comp.prepare_for_caching_req(
req=req,
insert_params=insert_params,
token_ids_len=len(token_ids),
is_finished=True,
)
if cl is not None:
effective_cache_len = min(effective_cache_len, cl)
for comp in self._components_tuple:
effective_cache_len = comp.floor_cache_len(effective_cache_len)
# Truncate if needed; the tail free is deferred and batched with
# the unaligned tail below so a shared boundary page is emitted once.
kv_indices_full = kv_indices
tail_free_start = None
if effective_cache_len < len(token_ids):
tail_free_start = max(effective_cache_len, req.kv.cache_protected_len)
token_ids = token_ids[:effective_cache_len]
kv_indices = kv_indices[:effective_cache_len]
radix_key = RadixKey(
token_ids,
req.extra_key,
is_bigram=self.tree_core.is_eagle,
cache_salt=req.cache_salt,
).page_aligned(self.page_size)
page_aligned_len = len(radix_key)
values = kv_indices[:page_aligned_len].to(dtype=torch.int64, copy=True)
insert_params.key = radix_key
insert_params.value = values
result = self.insert(insert_params)
# Split the leaf at the prompt boundary so eviction can drop the output
# KV without the prompt. prev_prefix_len keeps the overlapping indices
# from being freed as duplicates; skipped after a declined rotation,
# whose rows are freed below.
prompt_key = RadixKey(
req.origin_input_ids,
req.extra_key,
is_bigram=self.tree_core.is_eagle,
cache_salt=req.cache_salt,
).page_aligned(self.page_size)
if (
not result.rotation_tail_declined
and len(self._components_tuple) == 1
and self._components_tuple[0].component_type == BASE_COMPONENT_TYPE
and 0 < len(prompt_key) < len(radix_key)
):
self.insert(
replace(
insert_params,
key=prompt_key,
value=values[: len(prompt_key)],
prev_prefix_len=len(prompt_key),
priority=insert_params.priority + 1,
# The request created these nodes moments ago; another
# hit_count bump would promote every prompt node.
chunked=True,
)
)
# Everything past the inserted key goes back to the caller, the
# protected prefix never does. After a rotation decline nothing was
# inserted.
free_from = (
min(req.kv.cache_protected_len, len(kv_indices))
if result.rotation_tail_declined
else page_aligned_len
)
if tail_free_start is not None and tail_free_start > len(kv_indices):
# Truncated below the protected prefix: only an untracked mamba
# request gets here, with an empty key, and owns nothing before it.
assert free_from == len(kv_indices), (
f"{free_from=} {len(kv_indices)=} {req.kv.cache_protected_len=}"
)
free_from = tail_free_start
req.kv.cache_protected_len = free_from
# cleanup
for comp in self._components_tuple:
comp.cleanup_after_caching_req(
req, is_finished=True, insert_result=result, insert_params=insert_params
)
if self.enable_session_radix_cache and result is not None:
from sglang.srt.managers.schedule_batch import FINISH_ABORT
if req.finished_reason is not None and not isinstance(
req.finished_reason, FINISH_ABORT
):
self.session_refs.register_session_ref(
req, leaf=result.last_device_node
)
@rank_consensus(same_params=["req.rid", "chunked"])
def advance_unpublished_req(self, req: Req, chunked: bool = False) -> None:
assert not self.supports_mamba()
token_ids = req.get_fill_ids()
kv_indices = self.req_to_token_pool.req_to_token[
req.kv.req_pool_idx, : len(token_ids)
]
insert_params = InsertParams(
prev_prefix_len=req.kv.cache_protected_len,
chunked=chunked,
priority=req.priority or 0,
rotation_base=req.kv_rotation_base,
)
effective_cache_len = len(token_ids)
for comp in self._components_tuple:
cache_len = comp.prepare_for_caching_req(
req=req,
insert_params=insert_params,
token_ids_len=len(token_ids),
is_finished=False,
)
if cache_len is not None:
effective_cache_len = min(effective_cache_len, cache_len)
radix_key = RadixKey(
token_ids[:effective_cache_len],
req.extra_key,
is_bigram=self.tree_core.is_eagle,
cache_salt=req.cache_salt,
)
if envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS.get():
for comp in self._components_tuple:
comp.free_out_of_window_slots(req, len(radix_key) - 1, insert_params)
req.prefix_indices = kv_indices.to(dtype=torch.int64, copy=True)
for comp in self._components_tuple:
comp.cleanup_after_caching_req(
req, is_finished=False, insert_params=insert_params
)
@rank_consensus(same_params=["req.rid", "chunked"])
def cache_unfinished_req(self, req: Req, chunked: bool = False, **kwargs) -> None:
if self.session.try_cache_unfinished_req(req, chunked=chunked, **kwargs):
return
token_ids = req.get_fill_ids()
# A finished request hands its component state (mamba) to the tree
# instead of forking it, and the tree frees what the request still held.
is_finished = req.finished()
token_ids = req.full_untruncated_fill_ids[:up_to]
if self.disable:
kv_indices = self.req_to_token_pool.req_to_token[
req.kv.req_pool_idx, : len(token_ids)
]
req.prefix_indices = kv_indices.to(dtype=torch.int64, copy=True)
if is_finished:
for comp in self._components_tuple:
comp.cleanup_after_caching_req(req, is_finished=True)
return
kv_indices_orig = self.req_to_token_pool.req_to_token[
@@ -1178,7 +1028,7 @@ class UnifiedRadixCache(BasePrefixCache):
# components prepare insert data + return effective cache_len
insert_params = InsertParams(
prev_prefix_len=req.kv.cache_protected_len,
chunked=chunked,
inserted_len=req.kv.cache_inserted_len,
priority=req.priority or 0,
session_id=req.session_id,
rotation_base=req.kv_rotation_base,
@@ -1189,7 +1039,7 @@ class UnifiedRadixCache(BasePrefixCache):
req=req,
insert_params=insert_params,
token_ids_len=len(token_ids),
is_finished=False,
is_finished=is_finished,
)
if cl is not None:
effective_cache_len = min(effective_cache_len, cl)
@@ -1204,7 +1054,10 @@ class UnifiedRadixCache(BasePrefixCache):
cache_salt=req.cache_salt,
)
if envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS.get():
if (
not is_finished
and envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS.get()
):
# The frontier lands a page below page_floor(pre_len + 1), which has to
# be where the insert stops, or the leaf it creates keeps less than a
# sliding window of live SWA and the match after the insert rejects it.
@@ -1217,7 +1070,7 @@ class UnifiedRadixCache(BasePrefixCache):
req.prefix_indices = kv_indices_orig.to(dtype=torch.int64, copy=True)
for comp in self._components_tuple:
comp.cleanup_after_caching_req(
req, is_finished=False, insert_params=insert_params
req, is_finished=is_finished, insert_params=insert_params
)
return
@@ -1244,9 +1097,35 @@ class UnifiedRadixCache(BasePrefixCache):
req.prefix_indices = kv_indices_orig.to(dtype=torch.int64, copy=True)
for comp in self._components_tuple:
comp.cleanup_after_caching_req(
req, is_finished=False, insert_params=insert_params
req, is_finished=is_finished, insert_params=insert_params
)
return
req.kv.cache_inserted_len = max(req.kv.cache_inserted_len, page_aligned_len)
# Split the leaf at the prompt boundary so eviction can drop the output
# KV without the prompt. prev_prefix_len keeps the overlapping indices
# from being freed as duplicates.
prompt_key = RadixKey(
req.origin_input_ids,
req.extra_key,
is_bigram=self.tree_core.is_eagle,
cache_salt=req.cache_salt,
).page_aligned(self.page_size)
if (
len(self._components_tuple) == 1
and self._components_tuple[0].component_type == BASE_COMPONENT_TYPE
and 0 < len(prompt_key) < len(radix_key)
):
self.insert(
replace(
insert_params,
key=prompt_key,
value=values[: len(prompt_key)],
prev_prefix_len=len(prompt_key),
inserted_len=len(prompt_key),
priority=insert_params.priority + 1,
)
)
# Match prefix. SWA insertion retains one extra window before the
# page-aligned boundary, so the normal match remains safe to repoint.
@@ -1299,11 +1178,58 @@ class UnifiedRadixCache(BasePrefixCache):
for comp in self._components_tuple:
comp.cleanup_after_caching_req(
req,
is_finished=False,
is_finished=is_finished,
insert_result=result,
insert_params=insert_params,
)
if self.enable_session_radix_cache and is_finished:
from sglang.srt.managers.schedule_batch import FINISH_ABORT
if not isinstance(req.finished_reason, FINISH_ABORT):
self.session_refs.register_session_ref(
req, leaf=result.last_device_node
)
@rank_consensus(same_params=["req.rid"])
def advance_unpublished_req(self, req: Req) -> None:
assert not self.supports_mamba()
token_ids = req.get_fill_ids()
kv_indices = self.req_to_token_pool.req_to_token[
req.kv.req_pool_idx, : len(token_ids)
]
insert_params = InsertParams(
prev_prefix_len=req.kv.cache_protected_len,
priority=req.priority or 0,
rotation_base=req.kv_rotation_base,
)
effective_cache_len = len(token_ids)
for comp in self._components_tuple:
cache_len = comp.prepare_for_caching_req(
req=req,
insert_params=insert_params,
token_ids_len=len(token_ids),
is_finished=False,
)
if cache_len is not None:
effective_cache_len = min(effective_cache_len, cache_len)
radix_key = RadixKey(
token_ids[:effective_cache_len],
req.extra_key,
is_bigram=self.tree_core.is_eagle,
cache_salt=req.cache_salt,
)
if envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS.get():
for comp in self._components_tuple:
comp.free_out_of_window_slots(req, len(radix_key) - 1, insert_params)
req.prefix_indices = kv_indices.to(dtype=torch.int64, copy=True)
for comp in self._components_tuple:
comp.cleanup_after_caching_req(
req, is_finished=False, insert_params=insert_params
)
# ---- Internal Helpers ----
def _apply_cache_actions(
+10 -23
View File
@@ -325,25 +325,15 @@ class StreamingSession(BasePrefixCache):
return True
def try_cache_unfinished_req(
self, req: Req, chunked: bool = False, **kwargs
) -> bool:
"""Handles a streaming-session mid-flight cache op:
- chunked prefill: snapshot current KV as prefix, skip radix
- subsequent turn: skip radix (slot already holds KV)
Returns False for first-turn non-chunked (caller must run raw radix
insert to set up the initial tree lock)."""
if not _is_streaming(req):
def try_insert_req(self, req: Req, *, up_to: int, **kwargs) -> bool:
"""A first turn checkpoints into the tree like any request (its
prompt prefix is tree-owned and the slot inherits that lock); later
turns run on the slot's KV, so only the chunk cursor is kept."""
if not _is_streaming(req) or req.session.session_id not in self.slots:
return False
if chunked:
kv_indices = self.req_to_token_pool.req_to_token[
req.kv.req_pool_idx, : req.extend_range.end
]
req.prefix_indices = kv_indices.to(dtype=torch.int64, copy=True)
return True
if req.session.session_id in self.slots:
return True
return False
kv_indices = self.req_to_token_pool.req_to_token[req.kv.req_pool_idx, :up_to]
req.prefix_indices = kv_indices.to(dtype=torch.int64, copy=True)
return True
# -- BasePrefixCache abstract methods: thin adapters over try_handle_* --
@@ -360,12 +350,9 @@ class StreamingSession(BasePrefixCache):
self.inner.on_release(req, inserted=inserted)
def insert_req(self, req: Req, **kwargs):
self.inner.insert_req(req, **kwargs)
def cache_unfinished_req(self, req: Req, **kwargs):
if self.try_cache_unfinished_req(req, **kwargs):
if self.try_insert_req(req, **kwargs):
return
self.inner.cache_unfinished_req(req, **kwargs)
self.inner.insert_req(req, **kwargs)
def unpin(self, req: Req) -> None:
self.inner.unpin(req)
+1 -1
View File
@@ -4727,7 +4727,7 @@ def get_extend_input_len_swa_limit(
sliding_window_size: int, chunked_prefill_size: int, page_size: int
) -> int:
# 1. a factor of 2x is because each prefill contains chunked_prefill_size tokens,
# and between prefills, we run the tree cache's cache_unfinished_req(),
# and between prefills, we run the tree cache's insert_req(),
# so we unlock the previously locked nodes.
# 2. max is to handle the case that chunked_prefill_size is larger than sliding_window_size.
# in that case, each prefill contains chunked_prefill_size tokens,
+8 -1
View File
@@ -1,8 +1,15 @@
"""Shared helpers for mem_cache unit tests."""
from sglang.srt.managers.schedule_batch import FINISH_LENGTH
def finish_req(cache, req, up_to):
"""insert_req, then what release_kv_cache does after it: free the rest, drop the lock."""
"""What release_kv_cache does for a request that keeps its KV: mark it
finished, insert_req, free the rest, drop the lock."""
if getattr(req, "finished_reason", None) is None:
req.finished_reason = FINISH_LENGTH(length=0)
if hasattr(req, "_refresh_fill_ids"):
req._refresh_fill_ids()
cache.insert_req(req, up_to=up_to)
cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, up_to)])
cache.unpin(req)