From 50be533d09bde329bface6f1ac0631af93b528bf Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Thu, 1 Oct 2026 21:41:45 -0700 Subject: [PATCH] [mem_cache] Rename `cache_unfinished_req` to `checkpoint_req` and count cache hits only when a request finishes (#41520) --- python/sglang/srt/disaggregation/decode.py | 5 +- .../decode_schedule_batch_mixin.py | 4 +- python/sglang/srt/disaggregation/prefill.py | 20 +- python/sglang/srt/managers/schedule_batch.py | 8 +- python/sglang/srt/managers/scheduler.py | 8 +- .../batch_result_processor.py | 6 +- python/sglang/srt/mem_cache/allocation.py | 2 +- .../sglang/srt/mem_cache/base_prefix_cache.py | 20 +- python/sglang/srt/mem_cache/chunk_cache.py | 7 +- python/sglang/srt/mem_cache/common.py | 16 +- python/sglang/srt/mem_cache/memory_pool.py | 2 +- .../srt/mem_cache/pure_swa_radix_cache.py | 22 -- python/sglang/srt/mem_cache/radix_cache.py | 53 ++-- .../srt/mem_cache/rust_tree_core/adapter.py | 2 +- .../storage/flexkv/flexkv_radix_cache.py | 4 +- .../lmcache/lmcache_unified_radix_cache.py | 14 +- .../unified_cache/components/README.md | 46 +--- .../unified_cache/unified_cache_linker.py | 2 +- .../unified_cache/unified_tree_core.py | 16 +- .../srt/mem_cache/unified_radix_cache.py | 254 +++++++----------- .../sglang/srt/session/streaming_session.py | 33 +-- python/sglang/srt/utils/common.py | 2 +- python/sglang/test/mem_cache_utils.py | 9 +- rust/sglang-radix-tree/src/python_bindings.rs | 12 +- .../src/tests/components/base.rs | 2 +- .../src/tests/components/full.rs | 4 +- .../src/tests/components/mamba.rs | 2 +- .../src/tests/components/swa.rs | 2 +- .../src/tests/unified_tree_core.rs | 37 +-- .../src/unified_tree_core.rs | 25 +- .../chunked_prefill/test_scripted_radix.py | 2 +- .../test_scripted_regression.py | 2 +- .../dsv4/test_dsv4_swa_radix_retract.py | 2 +- .../test_scripted_core_1gpu.py | 10 +- .../kv_canary/test_self_e2e_pd_perturb.py | 2 +- .../test_prefill_abort_result_cleanup.py | 12 +- .../unit/managers/test_auxiliary_output.py | 4 +- ...st_batch_result_processor_hidden_states.py | 2 +- ...test_priority_scheduling_disaggregation.py | 15 +- .../test_scheduler_chunked_req_gate.py | 3 +- .../mem_cache/test_decode_radix_lock_ref.py | 36 ++- .../test_decode_retraction_backup.py | 4 +- .../mem_cache/test_free_kv_row_coalesce.py | 3 - .../test_mamba_donated_alloc_ratio.py | 2 +- .../mem_cache/test_page_interleave_shard.py | 11 +- .../mem_cache/test_pure_swa_radix_cache.py | 15 +- .../unit/mem_cache/test_radix_cache_unit.py | 12 +- .../test_swa_locked_full_recover_unified.py | 2 +- .../test_unified_radix_cache_unittest.py | 19 +- 49 files changed, 350 insertions(+), 447 deletions(-) diff --git a/python/sglang/srt/disaggregation/decode.py b/python/sglang/srt/disaggregation/decode.py index 3b36880617..a921e11363 100644 --- a/python/sglang/srt/disaggregation/decode.py +++ b/python/sglang/srt/disaggregation/decode.py @@ -150,6 +150,7 @@ def _bind_root_prefix(req: Req, tree_cache: BasePrefixCache) -> None: req.best_match_node = req.last_node req.lock_receipt = DecLockRefParams() req.kv.cache_protected_len = 0 + req.kv.cache_inserted_len = 0 req.num_matched_prefix_tokens = 0 req.host_hit_length = 0 @@ -2200,7 +2201,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): kv_loc, ) - # Truncate fill_len to kv_committed_len so cache_unfinished_req only + # Truncate fill_len to kv_committed_len so insert_req only # inserts committed KV into the radix tree. The last output token # hasn't had KV committed yet (output_ids is 1 ahead). req.full_untruncated_fill_ids = req.origin_input_ids + req.output_ids @@ -3124,7 +3125,7 @@ class SchedulerDisaggregationDecodeMixin: else: tree_cache = self.tree_cache req.init_next_round_input(tree_cache) - # Truncate fill_len to kv_committed_len so cache_unfinished_req + # Truncate fill_len to kv_committed_len so insert_req # only sees committed KV (full array includes one uncommitted # token because init_next_round_input rebuilt it as full). if req.kv.kv_committed_len is not None: diff --git a/python/sglang/srt/disaggregation/decode_schedule_batch_mixin.py b/python/sglang/srt/disaggregation/decode_schedule_batch_mixin.py index 5b5aada9f8..11fafbab4b 100644 --- a/python/sglang/srt/disaggregation/decode_schedule_batch_mixin.py +++ b/python/sglang/srt/disaggregation/decode_schedule_batch_mixin.py @@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, List import torch from sglang.srt.managers.overlap_utils import RelayPayload -from sglang.srt.mem_cache.common import maybe_cache_unfinished_req +from sglang.srt.mem_cache.common import checkpoint_kv_cache from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo @@ -120,7 +120,7 @@ class ScheduleBatchDisaggregationDecodeMixin: # PREBUILT does not materialize a local SWA branching window. if req.swa_branching_seqlen is not None: req.swa_branching_seqlen = None - maybe_cache_unfinished_req(req, self.tree_cache) + checkpoint_kv_cache(req, self.tree_cache) if req.grammar is not None: # FIXME: this try-except block is for handling unexpected xgrammar issue. try: diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index 63af9f669b..c0fb2d82e9 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -73,9 +73,9 @@ from sglang.srt.managers.schedule_batch import ( ) from sglang.srt.mem_cache.base_prefix_cache import CacheRequestOutcome from sglang.srt.mem_cache.common import ( + checkpoint_kv_cache, kv_to_page_indices, kv_to_page_num, - maybe_cache_unfinished_req, release_kv_cache, ) from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool @@ -581,15 +581,13 @@ class SchedulerDisaggregationPrefillMixin: if room is not None and room in kv_mgr.transfer_infos: prefetch(room) - def cache_unfinished_disagg_prefill( - self: Scheduler, req: Req, *, chunked: bool = False - ) -> None: + def checkpoint_disagg_prefill(self: Scheduler, req: Req) -> None: cache = self.tree_cache if req.pending_bootstrap and _uses_write_through_cache(cache): - cache.advance_unpublished_req(req, chunked=chunked) + cache.advance_unpublished_req(req) return - maybe_cache_unfinished_req(req, cache, chunked=chunked) + checkpoint_kv_cache(req, cache) def release_aborted_prefill_waiting_req(self: Scheduler, req: Req) -> None: self.clear_pending_chunk_send(req) @@ -897,7 +895,7 @@ class SchedulerDisaggregationPrefillMixin: advance_logprob_pt(i, req) continue - self.cache_unfinished_disagg_prefill(req) + self.checkpoint_disagg_prefill(req) self.disagg_prefill_inflight_queue.append(req) if self.spec_algorithm.is_eagle() and draft_input is not None: req.output_topk_p = draft_input.topk_p[i] @@ -1140,6 +1138,8 @@ class SchedulerDisaggregationPrefillMixin: else: logger.warning(error_message) req.time_stats.trace_ctx.abort(abort_info={"reason": error_message}) + if req.finished_reason is None: + req.finished_reason = FINISH_LENGTH(length=0) release_kv_cache(req, self.tree_cache) # unlock the tree self._release_aborted_request(req) if not isinstance(req.finished_reason, FINISH_ABORT): @@ -1228,7 +1228,7 @@ class SchedulerDisaggregationPrefillMixin: # Metadata buffer was allocated in pop_bootstrapped before # the request entered the waiting queue, so finalize should not fail. assert self.disagg_prefill_bootstrap_queue.finalize_bootstrap(req) - self.cache_unfinished_disagg_prefill(req) + self.checkpoint_disagg_prefill(req) return True else: raise RuntimeError( @@ -1255,7 +1255,7 @@ class SchedulerDisaggregationPrefillMixin: chunked_req_to_exclude = set() if (req := self.chunked_req) is not None: chunked_req_to_exclude.add(req) - self.cache_unfinished_disagg_prefill(req, chunked=True) + self.checkpoint_disagg_prefill(req) if not self.check_bootstrap(req): if is_aborted(req): @@ -1555,7 +1555,7 @@ class SchedulerDisaggregationPrefillMixin: max_attempts = get_disagg().optimistic_prefill_attempts uses_write_through_cache = _uses_write_through_cache(self.tree_cache) if not uses_write_through_cache: - maybe_cache_unfinished_req(req, self.tree_cache) + checkpoint_kv_cache(req, self.tree_cache) # The cached prefix is evictable once the KV is released. Its length # (capped at what a retry can match) seeds the retry's storage baseline, # so an evicted prefix is looked up in L3 once before it is recomputed. diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index e16872afa4..acfe19a3e5 100755 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -933,6 +933,9 @@ class ReqKvInfo: # The request's own KV is [cache_protected_len, kv_allocated_len). cache_protected_len: int = 0 # Tree cache owns [0, here) (matched or inserted) + # This request already inserted [0, here) into the tree; later inserts count + # a hit only on the nodes past it, so a request counts each node once. + cache_inserted_len: int = 0 kv_committed_len: int = 0 # KV content committed up to here, <= kv_allocated_len kv_allocated_len: int = 0 @@ -1217,7 +1220,7 @@ class Req(ReqDllmMixin): # Refreshed at every sharded alloc — read through last_node, or drawn # least-full for a new chain — and consumed by the radix insert to # stamp new tree nodes. Allocation itself must NOT read it back when - # a tree node is available (the cache_unfinished_req dedup rebind + # a tree node is available (the insert_req dedup rebind # would make it stale); the only allocation-time reader is the # ChunkCache fallback, which has no tree nodes and no rebind. self.kv_rotation_base: Optional[int] = None @@ -1978,6 +1981,7 @@ class Req(ReqDllmMixin): self.indexer_topk = None self.last_node = None self.kv.cache_protected_len = 0 + self.kv.cache_inserted_len = 0 self.kv_rotation_base = None self.num_matched_prefix_tokens = 0 self.lock_receipt = DecLockRefParams() @@ -3257,7 +3261,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): for req, seq_len in zip(self.reqs, seq_lens, strict=True): req._refresh_fill_ids() # end runs one past full_untruncated_fill_ids while output_ids - # trails; safe only while decoding_reqs suppresses cache_unfinished_req. + # trails; safe only while decoding_reqs suppresses the checkpoint insert. req.set_extend_range(seq_len - 1, seq_len) self.prefix_lens = [seq_len - 1 for seq_len in seq_lens] diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 615890cddb..eaa1944f8e 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -291,8 +291,8 @@ from sglang.srt.mem_cache import kv_cache_builder from sglang.srt.mem_cache.allocator.page_interleave import page_interleave_shard_size from sglang.srt.mem_cache.base_prefix_cache import CacheRequestOutcome from sglang.srt.mem_cache.common import ( + checkpoint_kv_cache, discard_kv_cache_backup, - maybe_cache_unfinished_req, release_kv_cache, ) from sglang.srt.model_executor.forward_batch_info import PPProxyTensors @@ -3533,9 +3533,9 @@ class Scheduler( def stash_chunked_request(self, req: Req): if self.disaggregation_mode == DisaggregationMode.PREFILL: - self.cache_unfinished_disagg_prefill(req, chunked=True) + self.checkpoint_disagg_prefill(req) else: - maybe_cache_unfinished_req(req, self.tree_cache, chunked=True) + checkpoint_kv_cache(req, self.tree_cache) def process_pending_chunked_abort(self) -> None: """Abort an in-flight chunked-prefill request once it is safe to do so. @@ -5321,6 +5321,8 @@ class Scheduler( discard_kv_cache_backup(req, self.tree_cache, "host_pool") if self.enable_hisparse: self.hisparse_coordinator.request_finished(req) + if req.finished_reason is None: + req.finished_reason = FINISH_ABORT() release_kv_cache(req, self.tree_cache) if self.disaggregation_mode == DisaggregationMode.PREFILL: self.release_aborted_prefill_waiting_req(req) diff --git a/python/sglang/srt/managers/scheduler_components/batch_result_processor.py b/python/sglang/srt/managers/scheduler_components/batch_result_processor.py index a10d0aa7bf..1dff4fde47 100644 --- a/python/sglang/srt/managers/scheduler_components/batch_result_processor.py +++ b/python/sglang/srt/managers/scheduler_components/batch_result_processor.py @@ -30,7 +30,7 @@ from sglang.srt.managers.schedule_batch import ( mamba_lazy_spec_in_window, ) from sglang.srt.mem_cache.common import ( - maybe_cache_unfinished_req, + checkpoint_kv_cache, release_kv_cache, ) from sglang.srt.model_executor.forward_batch_info import ( @@ -381,7 +381,7 @@ class SchedulerBatchResultProcessor: ) req.time_stats.set_completion_time() elif not batch.decoding_reqs or req not in batch.decoding_reqs: - maybe_cache_unfinished_req(req, self.tree_cache) + checkpoint_kv_cache(req, self.tree_cache) if get_memory().enable_hisparse: self.hisparse_coordinator.admit_request_into_staging(req) @@ -472,7 +472,7 @@ class SchedulerBatchResultProcessor: release_kv_cache(req, self.tree_cache) req.time_stats.set_completion_time() else: - maybe_cache_unfinished_req(req, self.tree_cache) + checkpoint_kv_cache(req, self.tree_cache) else: # being chunked reqs' prefill is not finished req.inflight_middle_chunks -= 1 diff --git a/python/sglang/srt/mem_cache/allocation.py b/python/sglang/srt/mem_cache/allocation.py index 0032a8ccc0..a17695bf86 100644 --- a/python/sglang/srt/mem_cache/allocation.py +++ b/python/sglang/srt/mem_cache/allocation.py @@ -255,7 +255,7 @@ def _kv_shard_rotation_bases( The owner class of position-page P is ``(b_i + P) % shard_size``. Rules: - Read through ``req.last_node`` at alloc time, never a value cached on - the request: ``cache_unfinished_req`` can rebind a chunked request onto + the request: a checkpoint ``insert_req`` can rebind a chunked request onto another chain's canonical locs between chunks, changing the base. The read goes through ``tree_cache.rotation_base_of`` because the node handle is tree-specific (a NodeId on the unified tree). diff --git a/python/sglang/srt/mem_cache/base_prefix_cache.py b/python/sglang/srt/mem_cache/base_prefix_cache.py index b1c207e1e7..56fe67fb1b 100644 --- a/python/sglang/srt/mem_cache/base_prefix_cache.py +++ b/python/sglang/srt/mem_cache/base_prefix_cache.py @@ -95,7 +95,9 @@ class InsertParams: component_evicted_seqlens: dict[ComponentType, int] = dataclasses.field( default_factory=dict, kw_only=True ) - chunked: bool = False + # The inserting request already inserted [0, here) (req.kv.cache_inserted_len); + # only the nodes past it count a hit, so a request counts each node once. + inserted_len: int = 0 priority: int = 0 session_id: Optional[str] = None track_adopted_ranges: bool = False @@ -458,14 +460,14 @@ class BasePrefixCache(ABC, PrefixCacheTrait): @abstractmethod def insert_req(self, req: Req, *, up_to: int, **kwargs): - """Hand a finished request's KV up to row position ``up_to`` to the - tree: insert what can be keyed and advance ``cache_protected_len`` - past it. The caller then frees ``[cache_protected_len, up_to)`` and - everything after, and unpins; nothing here releases a slot.""" - - @abstractmethod - def cache_unfinished_req(self, req: Req, **kwargs): - pass + """Insert the request's KV up to row position ``up_to`` into the tree, + repoint the row onto the tree's copy, re-anchor ``req.last_node`` on + the node the insert ended on and advance ``cache_protected_len``. + Called at every checkpoint of a running request and once more when + it finishes (``req.finished()``), when the tree also takes over the + component state the request no longer needs. Nothing here frees a + slot: ``release_kv_cache`` frees ``[cache_protected_len, up_to)`` and + everything after, and unpins.""" def free_kv_row(self, kv: Any, ranges: list[tuple[int, int]]) -> None: """Give back ascending, disjoint, half-open row-position ranges diff --git a/python/sglang/srt/mem_cache/chunk_cache.py b/python/sglang/srt/mem_cache/chunk_cache.py index 07c2c558fa..4b60dbdb87 100644 --- a/python/sglang/srt/mem_cache/chunk_cache.py +++ b/python/sglang/srt/mem_cache/chunk_cache.py @@ -77,12 +77,7 @@ class ChunkCache(BasePrefixCache): return InsertResult(prefix_len=0) def insert_req(self, req: Req, *, up_to: int): - pass - - def cache_unfinished_req(self, req: Req, chunked=False): - kv_indices = self.req_to_token_pool.req_to_token[ - req.kv.req_pool_idx, : req.extend_range.end - ] + kv_indices = self.req_to_token_pool.req_to_token[req.kv.req_pool_idx, :up_to] # `req.prefix_indices` will be used in `PrefillAdder::add_chunked_req` later req.prefix_indices = kv_indices.to(dtype=torch.int64, copy=True) diff --git a/python/sglang/srt/mem_cache/common.py b/python/sglang/srt/mem_cache/common.py index 2439265ad2..01fde5cf6a 100644 --- a/python/sglang/srt/mem_cache/common.py +++ b/python/sglang/srt/mem_cache/common.py @@ -169,11 +169,16 @@ def free_kv_row_segments( allocator.free_segments(swa_alive) -def maybe_cache_unfinished_req(req: Req, tree_cache: BasePrefixCache, **kwargs): +def checkpoint_kv_cache(req: Req, tree_cache: BasePrefixCache) -> None: + """Publish what the running request has computed so far, unless it is + barred from the tree.""" + # The tree reads req.finished() to tell a checkpoint from the final + # insert; a finished request belongs in release_kv_cache. + assert not req.finished(), f"checkpointing finished request {req.rid}" if req.skip_radix_cache_insert: return - tree_cache.cache_unfinished_req(req, **kwargs) + tree_cache.insert_req(req, up_to=req.extend_range.end) def evict_from_tree_cache( @@ -313,6 +318,13 @@ def release_kv_cache(req: Req, tree_cache: BasePrefixCache, is_insert: bool = Tr owned_kv_len = req.owned_kv_len() is_insert = is_insert and not req.skip_radix_cache_insert if is_insert: + # A tree that takes over component state (mamba) must see the request + # finished, or the insert forks the state and the slot leaks. + assert req.finished() or not tree_cache.supports_mamba(), ( + f"releasing unfinished request {req.rid} into a mamba tree" + ) + # The fill-id array lags output_ids until the next prepare_for_decode. + req._refresh_fill_ids() tree_cache.insert_req(req, up_to=owned_kv_len) # The protected prefix is not this req's to free. tree_cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, owned_kv_len)]) diff --git a/python/sglang/srt/mem_cache/memory_pool.py b/python/sglang/srt/mem_cache/memory_pool.py index 00b00400aa..a8d521304f 100644 --- a/python/sglang/srt/mem_cache/memory_pool.py +++ b/python/sglang/srt/mem_cache/memory_pool.py @@ -1034,7 +1034,7 @@ class MambaPool: ReplaySSM invariant: the SOURCE must be a fully-flushed checkpoint (``write_pos[src] == 0``). Only ``temporal`` is copied, not the ring, so an un-flushed source would drop its last ``write_pos`` updates. Callers - comply: COW copies radix checkpoints; ``cache_unfinished_req`` copies an + comply: COW copies radix checkpoints; a checkpoint ``insert_req`` copies an active slot only during prefill (ring empty); ``insert_req`` caps the donate to the last flush boundary. The dst cursor is reset to 0 (the copied checkpoint has no pending ring entries). diff --git a/python/sglang/srt/mem_cache/pure_swa_radix_cache.py b/python/sglang/srt/mem_cache/pure_swa_radix_cache.py index 663e0d2839..b51dada1fa 100644 --- a/python/sglang/srt/mem_cache/pure_swa_radix_cache.py +++ b/python/sglang/srt/mem_cache/pure_swa_radix_cache.py @@ -3,15 +3,10 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING from sglang.srt.mem_cache.base_prefix_cache import EvictParams, EvictResult from sglang.srt.mem_cache.cache_init_params import CacheInitParams from sglang.srt.mem_cache.radix_cache import RadixCache -from sglang.srt.utils.common import ceil_align - -if TYPE_CHECKING: - from sglang.srt.managers.schedule_batch import Req logger = logging.getLogger(__name__) @@ -57,23 +52,6 @@ class PureSWARadixCache(RadixCache): num_tokens = max(params.num_tokens, params.swa_num_tokens) return super().evict(EvictParams(num_tokens=num_tokens)) - def insert_req(self, req: Req, *, up_to: int): - """Insert only the prefill portion [0, swa_evict_floor).""" - if self.disable: - return - token_ids = (req.origin_input_ids + req.output_ids)[:up_to] - swa_evict_floor = req.kv.swa_evict_floor - key_limit = ( - ceil_align(swa_evict_floor, self.page_size) if swa_evict_floor > 0 else None - ) - radix_key, _, _ = self._insert_cache(req, token_ids, key_limit=key_limit) - req.kv.cache_protected_len = len(radix_key) - - def cache_unfinished_req(self, req: Req, chunked=False): - """During chunked prefill, swa_evicted_seqlen is 0 and no SWA eviction - has happened yet, so standard RadixCache logic is correct.""" - super().cache_unfinished_req(req, chunked=chunked) - def available_and_evictable_str(self) -> str: allocator = self.token_to_kv_pool_allocator swa_available = allocator.swa_available_size() diff --git a/python/sglang/srt/mem_cache/radix_cache.py b/python/sglang/srt/mem_cache/radix_cache.py index fa5f9df2e6..8a7530c466 100644 --- a/python/sglang/srt/mem_cache/radix_cache.py +++ b/python/sglang/srt/mem_cache/radix_cache.py @@ -50,6 +50,7 @@ from sglang.srt.mem_cache.utils import ( get_eviction_strategy, split_node_hash_value, ) +from sglang.srt.utils.common import ceil_align if TYPE_CHECKING: from sglang.srt.managers.schedule_batch import Req @@ -418,7 +419,7 @@ class RadixCache(BasePrefixCache): key = params.key value = params.value priority = params.priority - chunked = params.chunked + inserted_len = params.inserted_len key, value = key.maybe_to_bigram_view(self.is_eagle, value) key = key.page_aligned(self.page_size) @@ -429,7 +430,7 @@ class RadixCache(BasePrefixCache): value = torch.tensor(key.token_ids[: len(key)], dtype=torch.int64) prefix_len, last_node = self._insert_helper( - self.root_node, key, value, priority, chunked + self.root_node, key, value, priority, inserted_len ) return InsertResult(prefix_len=prefix_len, last_device_node=last_node) @@ -438,7 +439,6 @@ class RadixCache(BasePrefixCache): req: Req, token_ids, *, - chunked: bool = False, key_limit: Optional[int] = None, split_prompt: bool = False, ) -> Tuple[RadixKey, torch.Tensor, int]: @@ -462,9 +462,13 @@ class RadixCache(BasePrefixCache): priority = req.priority or 0 result = self.insert( InsertParams( - key=radix_key, value=values, chunked=chunked, priority=priority + key=radix_key, + value=values, + inserted_len=req.kv.cache_inserted_len, + priority=priority, ) ) + req.kv.cache_inserted_len = max(req.kv.cache_inserted_len, key_len) if split_prompt: # Split the leaf at the prompt boundary so eviction can drop the # output KV without the prompt; a prefix re-insert only changes topology. @@ -479,10 +483,8 @@ class RadixCache(BasePrefixCache): InsertParams( key=prompt_key, value=values[: len(prompt_key)], + inserted_len=len(prompt_key), priority=priority + 1, - # The request created these nodes moments ago; another - # hit_count bump would promote every prompt node. - chunked=True, ) ) @@ -495,17 +497,14 @@ class RadixCache(BasePrefixCache): def insert_req(self, req: Req, *, up_to: int): if self.disable: return - token_ids = (req.origin_input_ids + req.output_ids)[:up_to] - radix_key, _, _ = self._insert_cache(req, token_ids, split_prompt=True) - req.kv.cache_protected_len = len(radix_key) - - def cache_unfinished_req(self, req: Req, chunked=False): - """Cache request when it is unfinished.""" - if self.disable: - return - + token_ids = req.full_untruncated_fill_ids[:up_to] + # Pure-SWA trees never cache what the window already evicted. + swa_evict_floor = req.kv.swa_evict_floor + key_limit = ( + ceil_align(swa_evict_floor, self.page_size) if swa_evict_floor > 0 else None + ) radix_key, kv_indices, _ = self._insert_cache( - req, req.get_fill_ids(), chunked=chunked + req, token_ids, key_limit=key_limit, split_prompt=True ) # The prefix indices could be updated, reuse it @@ -525,7 +524,7 @@ class RadixCache(BasePrefixCache): # With page_size > 1 the partial page sits in req.prefix_indices but not # in the tree; cache_protected_len marks the tree-owned part so the next - # cache_unfinished_req or release_kv_cache frees the rest. + # insert_req or release_kv_cache frees the rest. req.kv.cache_protected_len = len(new_indices) self.dec_lock_ref(req.last_node) @@ -540,7 +539,6 @@ class RadixCache(BasePrefixCache): ) else: req.prefix_indices = new_indices - req.last_node = new_last_node def pretty_print(self): @@ -687,12 +685,7 @@ class RadixCache(BasePrefixCache): return new_node - def _inc_hit_count(self, node: TreeNode, chunked: bool = False): - # Skip the hit count update for chunked requests to avoid self-referencing - # inflation where a chunked request increments hit_count on nodes it created - # in previous chunks. - if chunked: - return + def _inc_hit_count(self, node: TreeNode): node.hit_count += 1 def _insert_helper( @@ -701,7 +694,7 @@ class RadixCache(BasePrefixCache): key: RadixKey, value, priority: int = 0, - chunked: bool = False, + inserted_len: int = 0, ): # Convert None priority to 0 if priority is None: @@ -727,11 +720,12 @@ class RadixCache(BasePrefixCache): if prefix_len < len(node.key): new_node = self._split_node(node.key, node, prefix_len) new_node.priority = max(new_node.priority, priority) - self._inc_hit_count(new_node, chunked) node = new_node else: node.priority = max(node.priority, priority) - self._inc_hit_count(node, chunked) + # Nodes this request already inserted were counted back then. + if total_prefix_length > inserted_len: + self._inc_hit_count(node) if len(key): child_key = key.child_key(self.page_size) @@ -740,7 +734,8 @@ class RadixCache(BasePrefixCache): new_node.parent = node new_node.key = key new_node.value = value.clone() - self._inc_hit_count(new_node, chunked) + if total_prefix_length + len(key) > inserted_len: + self._inc_hit_count(new_node) node.children[child_key] = new_node self.evictable_size_ += len(key) self._update_leaf_status(node) diff --git a/python/sglang/srt/mem_cache/rust_tree_core/adapter.py b/python/sglang/srt/mem_cache/rust_tree_core/adapter.py index 22671f7dbd..cd45c5a84d 100644 --- a/python/sglang/srt/mem_cache/rust_tree_core/adapter.py +++ b/python/sglang/srt/mem_cache/rust_tree_core/adapter.py @@ -726,7 +726,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): prev_prefix_len=params.prev_prefix_len, swa_evicted_seqlen=params.get_evicted_seqlen(ComponentType.SWA), swa_branching_seqlen=params.swa_branching_seqlen, - chunked=params.chunked, + inserted_len=params.inserted_len, priority=0 if params.priority is None else params.priority, track_adopted_ranges=params.track_adopted_ranges, rotation_base=params.rotation_base, diff --git a/python/sglang/srt/mem_cache/storage/flexkv/flexkv_radix_cache.py b/python/sglang/srt/mem_cache/storage/flexkv/flexkv_radix_cache.py index 9357f8af8d..5c1dc179ed 100644 --- a/python/sglang/srt/mem_cache/storage/flexkv/flexkv_radix_cache.py +++ b/python/sglang/srt/mem_cache/storage/flexkv/flexkv_radix_cache.py @@ -390,8 +390,10 @@ class FlexKVRadixCache(RadixCache): self._load_markers.pop(req.cache_request_handle, None) def insert_req(self, req: Req, *, up_to: int) -> None: # type: ignore[override] - """Base insert_req then fire an async FlexKV store.""" + """Base insert_req; a finished request also fires an async FlexKV store.""" super().insert_req(req, up_to=up_to) + if not req.finished(): + return # Compute the committed prefix. topk = get_spec().speculative_eagle_topk diff --git a/python/sglang/srt/mem_cache/storage/lmcache/lmcache_unified_radix_cache.py b/python/sglang/srt/mem_cache/storage/lmcache/lmcache_unified_radix_cache.py index 1e0b9c6a41..5d028d6169 100644 --- a/python/sglang/srt/mem_cache/storage/lmcache/lmcache_unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/storage/lmcache/lmcache_unified_radix_cache.py @@ -248,12 +248,6 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache): return True - def cache_unfinished_req(self, req: Req, chunked: bool = False, **kwargs) -> None: - self._publish_external_loaded_prefix(req, token_ids_len=len(req.get_fill_ids())) - super().cache_unfinished_req(req, chunked=chunked, **kwargs) - self._retire_loaded_flow(req.rid) - self._submit_store(req, req.get_fill_ids()) - def on_release(self, req: Req, *, inserted: bool) -> None: super().on_release(req, inserted=inserted) if not inserted: @@ -263,9 +257,9 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache): self._publish_external_loaded_prefix(req, token_ids_len=up_to) super().insert_req(req, up_to=up_to, **kwargs) self._retire_loaded_flow(req.rid) - token_ids = (req.origin_input_ids + req.output_ids)[:up_to] - self._submit_store(req, token_ids) - self._request_session_finish(req.rid) + self._submit_store(req, req.full_untruncated_fill_ids[:up_to]) + if req.finished(): + self._request_session_finish(req.rid) def check_hicache_events(self) -> None: """Poll LMCache retrieve/store futures at the scheduler safe point.""" @@ -811,7 +805,7 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache): mamba_value=checkpoint, prev_prefix_len=prev_prefix_len, component_evicted_seqlens=req.kv.component_evicted_seqlens.copy(), - chunked=True, + inserted_len=len(key), priority=req.priority or 0, ) ) diff --git a/python/sglang/srt/mem_cache/unified_cache/components/README.md b/python/sglang/srt/mem_cache/unified_cache/components/README.md index 24e142a1ba..35487cd219 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/README.md +++ b/python/sglang/srt/mem_cache/unified_cache/components/README.md @@ -230,47 +230,23 @@ receipt proves were taken. The eventual full release must pass ### `insert_req(req: Req, *, up_to: int)` -Cache a completed request's KV data into the tree. +Insert a request's KV into the tree: at every checkpoint while it runs, and once more when it finishes. | Aspect | Detail | |--------|--------| -| **Purpose** | After a request finishes, insert its token/KV data into the tree for future reuse | -| **Inputs** | `req` — the finished request; `up_to` — end of the request-owned KV range. `release_kv_cache` frees `[cache_protected_len, up_to)` and everything past it and unpins; with `is_insert=False` it skips the insert and calls `on_release(req, inserted=False)` for component cleanup | +| **Purpose** | Publish the request's KV `[cache_protected_len, up_to)` so other requests can match it; nodes past `req.kv.cache_inserted_len` count one hit (a request counts each node once). `checkpoint_kv_cache` calls it with `req.extend_range.end` while the request runs; `release_kv_cache` calls it with the request-owned length when it finishes (`req.finished()`), then frees `[cache_protected_len, up_to)` and everything past it and unpins; with `is_insert=False` it skips the insert and calls `on_release(req, inserted=False)` for component cleanup | +| **Inputs** | `req` — the request; `up_to` — row position the insert may read up to | | **Output** | `None` | -| **Mutation** | Calls component hooks → `insert` → component cleanup; advances `cache_protected_len` past the inserted key. Frees nothing and drops no lock: `release_kv_cache` does both afterwards. | -| **Complexity** | **O(K + D·C)** — insert O(K + D·C) + lock release O(D). Simplifies to **O(K)**. | +| **Mutation** | Component hooks → `insert` → re-match → writes the tree's indices back into the row → moves the request's lock from the old `req.last_node` to the node the insert ended on → updates `req.prefix_indices`, `req.kv.cache_protected_len`, `req.kv.cache_inserted_len`, `req.last_node` → component cleanup. A finished request's component state (Mamba) is handed to the tree instead of forked, and what it still held is freed. Frees no KV slot: `release_kv_cache` does that afterwards. | +| **Complexity** | **O(K + D·C)** — insert O(K + D·C) + re-match O(K + D·C) + lock transfer O(D). Simplifies to **O(K)**. | **Algorithm detail:** -1. `prepare_for_caching_req()` per component — sets component-specific insert params, returns effective cache length (SWA: copies its cursor with `set_evicted_seqlen`; Mamba: prepares `mamba_value` from ping-pong buffer, returns `mamba_last_track_seqlen` as truncation hint) -2. Truncates if `effective_cache_len < len(token_ids)`: frees excess pool indices -3. Converts token IDs (bigram if EAGLE), page-aligns keys, then calls `insert()` -4. Frees unaligned tail KV indices beyond page boundary -5. Calls `dec_lock_ref()` on the previous `req.last_node` -6. `cleanup_after_caching_req()` per component (Mamba: frees forked mamba_value based on `mamba_exist`, handles ping-pong buffer cleanup) - ---- - -### `cache_unfinished_req(req: Req, chunked=False)` - -Cache an in-progress request's partial KV data (chunked prefill). - -| Aspect | Detail | -|--------|--------| -| **Purpose** | During chunked prefill, insert partial results so the next chunk can match the prefix | -| **Inputs** | `req` — the in-progress request | -| **Output** | `None` | -| **Mutation** | Inserts partial KV → re-matches prefix → updates `req.prefix_indices`, `req.kv.cache_protected_len`, `req.last_node`; transfers lock from old node to new node | -| **Complexity** | **O(K + D·C)** — two tree traversals: insert O(K + D·C) + re-match O(K + D·C) + lock transfer O(D). Simplifies to **O(K)**. | - -**Algorithm detail:** -1. `prepare_for_caching_req()` per component -2. `insert()` — first tree traversal -3. `match_prefix()` — **second** tree traversal to get updated indices -4. Writes new prefix indices into `req_to_token_pool` -5. `dec_lock_ref()` on old `req.last_node` -6. `inc_lock_ref()` on new matched node -7. Updates `req.prefix_indices`, `req.kv.cache_protected_len`, `req.last_node` -8. `cleanup_after_caching_req()` per component +1. `prepare_for_caching_req(is_finished=req.finished())` per component — sets component-specific insert params, returns effective cache length (SWA: copies its cursor with `set_evicted_seqlen`, may return a branching boundary; Mamba: prepares `mamba_value`, donated when finished and forked otherwise, returns `mamba_last_track_seqlen` as truncation hint) +2. Truncates the key to `effective_cache_len`; the row past it stays the request's +3. Converts token IDs (bigram if EAGLE), page-aligns the key, then calls `insert()`; a single-component tree re-inserts the prompt part so eviction can drop the output without the prompt +4. `match_prefix()` on the inserted key and writes the matched indices into the row +5. `dec_lock_ref()` on the old `req.last_node`, `inc_lock_ref()` on the new one +6. `cleanup_after_caching_req()` per component (Mamba: frees the forked `mamba_value` when the tree already had one, frees the finished request's slot) --- diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_cache_linker.py b/python/sglang/srt/mem_cache/unified_cache/unified_cache_linker.py index 9d322580fd..a3734c278d 100644 --- a/python/sglang/srt/mem_cache/unified_cache/unified_cache_linker.py +++ b/python/sglang/srt/mem_cache/unified_cache/unified_cache_linker.py @@ -363,7 +363,7 @@ class UnifiedCacheLinkerWrapper: if req.kv is not None else {} ), - chunked=True, + inserted_len=len(hit.prefix_key), priority=req.priority or 0, track_adopted_ranges=True, ) diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py index 70482bf879..deffec7fba 100644 --- a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py +++ b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py @@ -1053,11 +1053,9 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): continue comp.refresh_lru(LRURefreshPhase.WALKDOWN, node, self.root_node) - def _inc_hit_count_and_check( - self, node: UnifiedTreeNode, chunked: bool = False - ) -> bool: + def _inc_hit_count_and_check(self, node: UnifiedTreeNode) -> bool: """Increment hit count; check whether a write backup should be fired.""" - if node.evicted or chunked: + if node.evicted: return False if self.is_write_back: return False @@ -1307,7 +1305,9 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): if swa_already_freed < dup.numel(): step_actions.append(FreeDeviceKV([dup[swa_already_freed:]])) - if self._inc_hit_count_and_check(node, state.params.chunked): + # Nodes this request already inserted were counted back then. + node_end = state.total_prefix_length + prefix_len + if node_end > state.params.inserted_len and self._inc_hit_count_and_check(node): step_actions.append(self._build_backup_kv_action(node)) state.node = node state.total_prefix_length += prefix_len @@ -1368,8 +1368,10 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): def _should_backup_after_insert(self, state: _InsertWalkState) -> bool: """Check whether the insert target needs a Host backup.""" if state.is_new_leaf: - return self._inc_hit_count_and_check( - state.target_node, state.params.chunked + leaf_end = state.total_prefix_length + len(state.target_node.key) + return ( + leaf_end > state.params.inserted_len + and self._inc_hit_count_and_check(state.target_node) ) node = state.target_node diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index 0e502fe1ac..b999571adc 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -1004,171 +1004,21 @@ class UnifiedRadixCache(BasePrefixCache): @rank_consensus(same_params=["req.rid", "up_to"]) def insert_req(self, req: Req, *, up_to: int, **kwargs) -> None: - if self.disable: - for comp in self._components_tuple: - comp.cleanup_after_caching_req(req, is_finished=True) + if self.session.try_insert_req(req, up_to=up_to, **kwargs): return - - token_ids = (req.origin_input_ids + req.output_ids)[:up_to] - kv_indices = self.req_to_token_pool.req_to_token[req.kv.req_pool_idx, :up_to] - - insert_params = InsertParams( - prev_prefix_len=req.kv.cache_protected_len, - priority=req.priority or 0, - session_id=req.session_id, - rotation_base=req.kv_rotation_base, - ) - - # components prepare insert data + return effective cache_len - effective_cache_len = len(token_ids) - for comp in self._components_tuple: - cl = comp.prepare_for_caching_req( - req=req, - insert_params=insert_params, - token_ids_len=len(token_ids), - is_finished=True, - ) - if cl is not None: - effective_cache_len = min(effective_cache_len, cl) - for comp in self._components_tuple: - effective_cache_len = comp.floor_cache_len(effective_cache_len) - - # Truncate if needed; the tail free is deferred and batched with - # the unaligned tail below so a shared boundary page is emitted once. - kv_indices_full = kv_indices - tail_free_start = None - if effective_cache_len < len(token_ids): - tail_free_start = max(effective_cache_len, req.kv.cache_protected_len) - token_ids = token_ids[:effective_cache_len] - kv_indices = kv_indices[:effective_cache_len] - - radix_key = RadixKey( - token_ids, - req.extra_key, - is_bigram=self.tree_core.is_eagle, - cache_salt=req.cache_salt, - ).page_aligned(self.page_size) - page_aligned_len = len(radix_key) - values = kv_indices[:page_aligned_len].to(dtype=torch.int64, copy=True) - - insert_params.key = radix_key - insert_params.value = values - result = self.insert(insert_params) - - # Split the leaf at the prompt boundary so eviction can drop the output - # KV without the prompt. prev_prefix_len keeps the overlapping indices - # from being freed as duplicates; skipped after a declined rotation, - # whose rows are freed below. - prompt_key = RadixKey( - req.origin_input_ids, - req.extra_key, - is_bigram=self.tree_core.is_eagle, - cache_salt=req.cache_salt, - ).page_aligned(self.page_size) - if ( - not result.rotation_tail_declined - and len(self._components_tuple) == 1 - and self._components_tuple[0].component_type == BASE_COMPONENT_TYPE - and 0 < len(prompt_key) < len(radix_key) - ): - self.insert( - replace( - insert_params, - key=prompt_key, - value=values[: len(prompt_key)], - prev_prefix_len=len(prompt_key), - priority=insert_params.priority + 1, - # The request created these nodes moments ago; another - # hit_count bump would promote every prompt node. - chunked=True, - ) - ) - - # Everything past the inserted key goes back to the caller, the - # protected prefix never does. After a rotation decline nothing was - # inserted. - free_from = ( - min(req.kv.cache_protected_len, len(kv_indices)) - if result.rotation_tail_declined - else page_aligned_len - ) - if tail_free_start is not None and tail_free_start > len(kv_indices): - # Truncated below the protected prefix: only an untracked mamba - # request gets here, with an empty key, and owns nothing before it. - assert free_from == len(kv_indices), ( - f"{free_from=} {len(kv_indices)=} {req.kv.cache_protected_len=}" - ) - free_from = tail_free_start - req.kv.cache_protected_len = free_from - - # cleanup - for comp in self._components_tuple: - comp.cleanup_after_caching_req( - req, is_finished=True, insert_result=result, insert_params=insert_params - ) - - if self.enable_session_radix_cache and result is not None: - from sglang.srt.managers.schedule_batch import FINISH_ABORT - - if req.finished_reason is not None and not isinstance( - req.finished_reason, FINISH_ABORT - ): - self.session_refs.register_session_ref( - req, leaf=result.last_device_node - ) - - @rank_consensus(same_params=["req.rid", "chunked"]) - def advance_unpublished_req(self, req: Req, chunked: bool = False) -> None: - assert not self.supports_mamba() - token_ids = req.get_fill_ids() - kv_indices = self.req_to_token_pool.req_to_token[ - req.kv.req_pool_idx, : len(token_ids) - ] - insert_params = InsertParams( - prev_prefix_len=req.kv.cache_protected_len, - chunked=chunked, - priority=req.priority or 0, - rotation_base=req.kv_rotation_base, - ) - effective_cache_len = len(token_ids) - for comp in self._components_tuple: - cache_len = comp.prepare_for_caching_req( - req=req, - insert_params=insert_params, - token_ids_len=len(token_ids), - is_finished=False, - ) - if cache_len is not None: - effective_cache_len = min(effective_cache_len, cache_len) - - radix_key = RadixKey( - token_ids[:effective_cache_len], - req.extra_key, - is_bigram=self.tree_core.is_eagle, - cache_salt=req.cache_salt, - ) - if envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS.get(): - for comp in self._components_tuple: - comp.free_out_of_window_slots(req, len(radix_key) - 1, insert_params) - - req.prefix_indices = kv_indices.to(dtype=torch.int64, copy=True) - for comp in self._components_tuple: - comp.cleanup_after_caching_req( - req, is_finished=False, insert_params=insert_params - ) - - @rank_consensus(same_params=["req.rid", "chunked"]) - def cache_unfinished_req(self, req: Req, chunked: bool = False, **kwargs) -> None: - if self.session.try_cache_unfinished_req(req, chunked=chunked, **kwargs): - return - - token_ids = req.get_fill_ids() + # A finished request hands its component state (mamba) to the tree + # instead of forking it, and the tree frees what the request still held. + is_finished = req.finished() + token_ids = req.full_untruncated_fill_ids[:up_to] if self.disable: kv_indices = self.req_to_token_pool.req_to_token[ req.kv.req_pool_idx, : len(token_ids) ] req.prefix_indices = kv_indices.to(dtype=torch.int64, copy=True) + if is_finished: + for comp in self._components_tuple: + comp.cleanup_after_caching_req(req, is_finished=True) return kv_indices_orig = self.req_to_token_pool.req_to_token[ @@ -1178,7 +1028,7 @@ class UnifiedRadixCache(BasePrefixCache): # components prepare insert data + return effective cache_len insert_params = InsertParams( prev_prefix_len=req.kv.cache_protected_len, - chunked=chunked, + inserted_len=req.kv.cache_inserted_len, priority=req.priority or 0, session_id=req.session_id, rotation_base=req.kv_rotation_base, @@ -1189,7 +1039,7 @@ class UnifiedRadixCache(BasePrefixCache): req=req, insert_params=insert_params, token_ids_len=len(token_ids), - is_finished=False, + is_finished=is_finished, ) if cl is not None: effective_cache_len = min(effective_cache_len, cl) @@ -1204,7 +1054,10 @@ class UnifiedRadixCache(BasePrefixCache): cache_salt=req.cache_salt, ) - if envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS.get(): + if ( + not is_finished + and envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS.get() + ): # The frontier lands a page below page_floor(pre_len + 1), which has to # be where the insert stops, or the leaf it creates keeps less than a # sliding window of live SWA and the match after the insert rejects it. @@ -1217,7 +1070,7 @@ class UnifiedRadixCache(BasePrefixCache): req.prefix_indices = kv_indices_orig.to(dtype=torch.int64, copy=True) for comp in self._components_tuple: comp.cleanup_after_caching_req( - req, is_finished=False, insert_params=insert_params + req, is_finished=is_finished, insert_params=insert_params ) return @@ -1244,9 +1097,35 @@ class UnifiedRadixCache(BasePrefixCache): req.prefix_indices = kv_indices_orig.to(dtype=torch.int64, copy=True) for comp in self._components_tuple: comp.cleanup_after_caching_req( - req, is_finished=False, insert_params=insert_params + req, is_finished=is_finished, insert_params=insert_params ) return + req.kv.cache_inserted_len = max(req.kv.cache_inserted_len, page_aligned_len) + + # Split the leaf at the prompt boundary so eviction can drop the output + # KV without the prompt. prev_prefix_len keeps the overlapping indices + # from being freed as duplicates. + prompt_key = RadixKey( + req.origin_input_ids, + req.extra_key, + is_bigram=self.tree_core.is_eagle, + cache_salt=req.cache_salt, + ).page_aligned(self.page_size) + if ( + len(self._components_tuple) == 1 + and self._components_tuple[0].component_type == BASE_COMPONENT_TYPE + and 0 < len(prompt_key) < len(radix_key) + ): + self.insert( + replace( + insert_params, + key=prompt_key, + value=values[: len(prompt_key)], + prev_prefix_len=len(prompt_key), + inserted_len=len(prompt_key), + priority=insert_params.priority + 1, + ) + ) # Match prefix. SWA insertion retains one extra window before the # page-aligned boundary, so the normal match remains safe to repoint. @@ -1299,11 +1178,58 @@ class UnifiedRadixCache(BasePrefixCache): for comp in self._components_tuple: comp.cleanup_after_caching_req( req, - is_finished=False, + is_finished=is_finished, insert_result=result, insert_params=insert_params, ) + if self.enable_session_radix_cache and is_finished: + from sglang.srt.managers.schedule_batch import FINISH_ABORT + + if not isinstance(req.finished_reason, FINISH_ABORT): + self.session_refs.register_session_ref( + req, leaf=result.last_device_node + ) + + @rank_consensus(same_params=["req.rid"]) + def advance_unpublished_req(self, req: Req) -> None: + assert not self.supports_mamba() + token_ids = req.get_fill_ids() + kv_indices = self.req_to_token_pool.req_to_token[ + req.kv.req_pool_idx, : len(token_ids) + ] + insert_params = InsertParams( + prev_prefix_len=req.kv.cache_protected_len, + priority=req.priority or 0, + rotation_base=req.kv_rotation_base, + ) + effective_cache_len = len(token_ids) + for comp in self._components_tuple: + cache_len = comp.prepare_for_caching_req( + req=req, + insert_params=insert_params, + token_ids_len=len(token_ids), + is_finished=False, + ) + if cache_len is not None: + effective_cache_len = min(effective_cache_len, cache_len) + + radix_key = RadixKey( + token_ids[:effective_cache_len], + req.extra_key, + is_bigram=self.tree_core.is_eagle, + cache_salt=req.cache_salt, + ) + if envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS.get(): + for comp in self._components_tuple: + comp.free_out_of_window_slots(req, len(radix_key) - 1, insert_params) + + req.prefix_indices = kv_indices.to(dtype=torch.int64, copy=True) + for comp in self._components_tuple: + comp.cleanup_after_caching_req( + req, is_finished=False, insert_params=insert_params + ) + # ---- Internal Helpers ---- def _apply_cache_actions( diff --git a/python/sglang/srt/session/streaming_session.py b/python/sglang/srt/session/streaming_session.py index a90e612355..2064d9931d 100644 --- a/python/sglang/srt/session/streaming_session.py +++ b/python/sglang/srt/session/streaming_session.py @@ -325,25 +325,15 @@ class StreamingSession(BasePrefixCache): return True - def try_cache_unfinished_req( - self, req: Req, chunked: bool = False, **kwargs - ) -> bool: - """Handles a streaming-session mid-flight cache op: - - chunked prefill: snapshot current KV as prefix, skip radix - - subsequent turn: skip radix (slot already holds KV) - Returns False for first-turn non-chunked (caller must run raw radix - insert to set up the initial tree lock).""" - if not _is_streaming(req): + def try_insert_req(self, req: Req, *, up_to: int, **kwargs) -> bool: + """A first turn checkpoints into the tree like any request (its + prompt prefix is tree-owned and the slot inherits that lock); later + turns run on the slot's KV, so only the chunk cursor is kept.""" + if not _is_streaming(req) or req.session.session_id not in self.slots: return False - if chunked: - kv_indices = self.req_to_token_pool.req_to_token[ - req.kv.req_pool_idx, : req.extend_range.end - ] - req.prefix_indices = kv_indices.to(dtype=torch.int64, copy=True) - return True - if req.session.session_id in self.slots: - return True - return False + kv_indices = self.req_to_token_pool.req_to_token[req.kv.req_pool_idx, :up_to] + req.prefix_indices = kv_indices.to(dtype=torch.int64, copy=True) + return True # -- BasePrefixCache abstract methods: thin adapters over try_handle_* -- @@ -360,12 +350,9 @@ class StreamingSession(BasePrefixCache): self.inner.on_release(req, inserted=inserted) def insert_req(self, req: Req, **kwargs): - self.inner.insert_req(req, **kwargs) - - def cache_unfinished_req(self, req: Req, **kwargs): - if self.try_cache_unfinished_req(req, **kwargs): + if self.try_insert_req(req, **kwargs): return - self.inner.cache_unfinished_req(req, **kwargs) + self.inner.insert_req(req, **kwargs) def unpin(self, req: Req) -> None: self.inner.unpin(req) diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index cacf1fb243..6f80415863 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -4727,7 +4727,7 @@ def get_extend_input_len_swa_limit( sliding_window_size: int, chunked_prefill_size: int, page_size: int ) -> int: # 1. a factor of 2x is because each prefill contains chunked_prefill_size tokens, - # and between prefills, we run the tree cache's cache_unfinished_req(), + # and between prefills, we run the tree cache's insert_req(), # so we unlock the previously locked nodes. # 2. max is to handle the case that chunked_prefill_size is larger than sliding_window_size. # in that case, each prefill contains chunked_prefill_size tokens, diff --git a/python/sglang/test/mem_cache_utils.py b/python/sglang/test/mem_cache_utils.py index 9ed558e13a..69362e56fc 100644 --- a/python/sglang/test/mem_cache_utils.py +++ b/python/sglang/test/mem_cache_utils.py @@ -1,8 +1,15 @@ """Shared helpers for mem_cache unit tests.""" +from sglang.srt.managers.schedule_batch import FINISH_LENGTH + def finish_req(cache, req, up_to): - """insert_req, then what release_kv_cache does after it: free the rest, drop the lock.""" + """What release_kv_cache does for a request that keeps its KV: mark it + finished, insert_req, free the rest, drop the lock.""" + if getattr(req, "finished_reason", None) is None: + req.finished_reason = FINISH_LENGTH(length=0) + if hasattr(req, "_refresh_fill_ids"): + req._refresh_fill_ids() cache.insert_req(req, up_to=up_to) cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, up_to)]) cache.unpin(req) diff --git a/rust/sglang-radix-tree/src/python_bindings.rs b/rust/sglang-radix-tree/src/python_bindings.rs index 3b9308569b..c7a9bdfbd2 100644 --- a/rust/sglang-radix-tree/src/python_bindings.rs +++ b/rust/sglang-radix-tree/src/python_bindings.rs @@ -608,7 +608,7 @@ pub struct InsertParamsBinding { pub prev_prefix_len: usize, pub swa_evicted_seqlen: usize, pub swa_branching_seqlen: Option, - pub chunked: bool, + pub inserted_len: usize, pub priority: i64, pub track_adopted_ranges: bool, } @@ -616,7 +616,7 @@ pub struct InsertParamsBinding { #[pymethods] impl InsertParamsBinding { #[new] - #[pyo3(signature = (key, value, extra_key = None, cache_salt = None, session_id = None, prev_prefix_len = 0, swa_evicted_seqlen = 0, swa_branching_seqlen = None, chunked = false, priority = 0, mamba_value = None, track_adopted_ranges = false, rotation_base = None))] + #[pyo3(signature = (key, value, extra_key = None, cache_salt = None, session_id = None, prev_prefix_len = 0, swa_evicted_seqlen = 0, swa_branching_seqlen = None, inserted_len = 0, priority = 0, mamba_value = None, track_adopted_ranges = false, rotation_base = None))] fn new( py: Python<'_>, key: &Bound<'_, PyAny>, @@ -627,7 +627,7 @@ impl InsertParamsBinding { prev_prefix_len: usize, swa_evicted_seqlen: usize, swa_branching_seqlen: Option, - chunked: bool, + inserted_len: usize, priority: i64, mamba_value: Option>, track_adopted_ranges: bool, @@ -644,7 +644,7 @@ impl InsertParamsBinding { prev_prefix_len, swa_evicted_seqlen, swa_branching_seqlen, - chunked, + inserted_len, priority, track_adopted_ranges, }) @@ -1187,7 +1187,7 @@ impl TreeCoreBinding { prev_prefix_len: params.prev_prefix_len, swa_evicted_seqlen: params.swa_evicted_seqlen, swa_branching_seqlen: params.swa_branching_seqlen, - chunked: params.chunked, + inserted_len: params.inserted_len, priority: params.priority, track_adopted_ranges: params.track_adopted_ranges, }; @@ -1225,7 +1225,7 @@ impl TreeCoreBinding { prev_prefix_len: params.prev_prefix_len, swa_evicted_seqlen: params.swa_evicted_seqlen, swa_branching_seqlen: params.swa_branching_seqlen, - chunked: params.chunked, + inserted_len: params.inserted_len, priority: params.priority, track_adopted_ranges: params.track_adopted_ranges, }; diff --git a/rust/sglang-radix-tree/src/tests/components/base.rs b/rust/sglang-radix-tree/src/tests/components/base.rs index cddae0bb04..b53f313fd5 100644 --- a/rust/sglang-radix-tree/src/tests/components/base.rs +++ b/rust/sglang-radix-tree/src/tests/components/base.rs @@ -97,7 +97,7 @@ fn insert_overlap_default_consumes_nothing() { prev_prefix_len: 0, swa_evicted_seqlen: 0, swa_branching_seqlen: None, - chunked: false, + inserted_len: 0, priority: 0, session_id: None, track_adopted_ranges: false, diff --git a/rust/sglang-radix-tree/src/tests/components/full.rs b/rust/sglang-radix-tree/src/tests/components/full.rs index 1f4e51967f..4116b6f68a 100644 --- a/rust/sglang-radix-tree/src/tests/components/full.rs +++ b/rust/sglang-radix-tree/src/tests/components/full.rs @@ -74,7 +74,7 @@ fn insert(tc: &mut UnifiedTreeCore>, key: &Vec, value: &[i64]) { prev_prefix_len: 0, swa_evicted_seqlen: 0, swa_branching_seqlen: None, - chunked: false, + inserted_len: 0, priority: 0, session_id: None, track_adopted_ranges: false, @@ -602,7 +602,7 @@ fn host_drive_is_a_noop_without_host_leaves() { prev_prefix_len: 0, swa_evicted_seqlen: 0, swa_branching_seqlen: None, - chunked: false, + inserted_len: 0, priority: 0, session_id: None, track_adopted_ranges: false, diff --git a/rust/sglang-radix-tree/src/tests/components/mamba.rs b/rust/sglang-radix-tree/src/tests/components/mamba.rs index 66b87ec6b9..7f5ebf1d74 100644 --- a/rust/sglang-radix-tree/src/tests/components/mamba.rs +++ b/rust/sglang-radix-tree/src/tests/components/mamba.rs @@ -67,7 +67,7 @@ fn insert_params_mamba<'k>( prev_prefix_len: 0, swa_evicted_seqlen: 0, swa_branching_seqlen: None, - chunked: false, + inserted_len: 0, priority: 0, session_id: None, track_adopted_ranges: false, diff --git a/rust/sglang-radix-tree/src/tests/components/swa.rs b/rust/sglang-radix-tree/src/tests/components/swa.rs index fd4ca8475d..896c333b57 100644 --- a/rust/sglang-radix-tree/src/tests/components/swa.rs +++ b/rust/sglang-radix-tree/src/tests/components/swa.rs @@ -871,7 +871,7 @@ fn insert_params_swa<'k>( prev_prefix_len, swa_evicted_seqlen, swa_branching_seqlen: None, - chunked: false, + inserted_len: 0, priority: 0, session_id: None, track_adopted_ranges: false, diff --git a/rust/sglang-radix-tree/src/tests/unified_tree_core.rs b/rust/sglang-radix-tree/src/tests/unified_tree_core.rs index d683888d9d..3f43e267d9 100644 --- a/rust/sglang-radix-tree/src/tests/unified_tree_core.rs +++ b/rust/sglang-radix-tree/src/tests/unified_tree_core.rs @@ -1885,7 +1885,7 @@ fn insert_params<'k>(key: &'k Vec, value: &[i64]) -> InsertParams<'k, Vec( prev_prefix_len, swa_evicted_seqlen: 0, swa_branching_seqlen: None, - chunked: false, + inserted_len: 0, priority: 0, session_id: None, track_adopted_ranges: false, diff --git a/rust/sglang-radix-tree/src/unified_tree_core.rs b/rust/sglang-radix-tree/src/unified_tree_core.rs index e39fd3743a..f1a4ee175e 100644 --- a/rust/sglang-radix-tree/src/unified_tree_core.rs +++ b/rust/sglang-radix-tree/src/unified_tree_core.rs @@ -156,8 +156,9 @@ pub struct InsertParams<'k, K: ChildKeyType> { pub swa_branching_seqlen: Option, /// The donated mamba slot for the insert target leaf; None on non-mamba trees. pub mamba_value: Option, - /// Whether this is a chunked-prefill insert (no hit-count bump). - pub chunked: bool, + /// The inserting request already inserted [0, here); only the nodes past it + /// count a hit (and get threshold-checked), so a request counts each node once. + pub inserted_len: usize, /// Eviction priority floor applied along the walked path. pub priority: i64, /// Whether the result should report which incoming ranges the tree retained. @@ -245,7 +246,7 @@ pub struct InsertWalkState { swa_evicted_seqlen: usize, swa_branching_seqlen: Option, mamba_value: Option, - chunked: bool, + inserted_len: usize, priority: i64, track_adopted_ranges: bool, total_prefix_length: usize, @@ -1490,9 +1491,9 @@ impl UnifiedTreeCore { } /// Increment hit count; check whether a write backup should be fired. - pub fn inc_hit_count_and_check_(&mut self, node_id: NodeIdx_, chunked: bool) -> bool { + pub fn inc_hit_count_and_check_(&mut self, node_id: NodeIdx_) -> bool { let node = self.arena.node_mut(node_id); - if node.evicted() || chunked { + if node.evicted() { return false; } if self.is_write_back { @@ -1609,7 +1610,7 @@ impl UnifiedTreeCore { swa_evicted_seqlen: params.swa_evicted_seqlen, swa_branching_seqlen: params.swa_branching_seqlen, mamba_value: params.mamba_value.as_ref().map(Tensor::shallow_clone), - chunked: params.chunked, + inserted_len: params.inserted_len, priority: params.priority, track_adopted_ranges: params.track_adopted_ranges, total_prefix_length: 0, @@ -1771,7 +1772,7 @@ impl UnifiedTreeCore { swa_evicted_seqlen: state.swa_evicted_seqlen, swa_branching_seqlen: state.swa_branching_seqlen, mamba_value: state.mamba_value.as_ref().map(Tensor::shallow_clone), - chunked: state.chunked, + inserted_len: state.inserted_len, priority: state.priority, track_adopted_ranges: state.track_adopted_ranges, }; @@ -1859,7 +1860,9 @@ impl UnifiedTreeCore { } } - if self.inc_hit_count_and_check_(node_id, state.chunked) { + // Nodes this request already inserted were counted back then. + let node_end = state.total_prefix_length + prefix_len; + if node_end > state.inserted_len && self.inc_hit_count_and_check_(node_id) { let backup = self .build_backup_kv_action_(self.arena.node(node_id), /* write_back = */ false); state.pending_actions.push(CacheAction::BackupKV(backup)); @@ -1928,7 +1931,7 @@ impl UnifiedTreeCore { swa_evicted_seqlen: state.swa_evicted_seqlen, swa_branching_seqlen: state.swa_branching_seqlen, mamba_value: state.mamba_value.as_ref().map(Tensor::shallow_clone), - chunked: state.chunked, + inserted_len: state.inserted_len, priority: state.priority, track_adopted_ranges: state.track_adopted_ranges, }; @@ -1965,7 +1968,9 @@ impl UnifiedTreeCore { target_node_id: NodeIdx_, ) -> bool { if state.is_new_leaf { - return self.inc_hit_count_and_check_(target_node_id, state.chunked); + // The new leaf runs to the end of the aligned key. + return state.aligned_key_len > state.inserted_len + && self.inc_hit_count_and_check_(target_node_id); } let node = self.arena.node(target_node_id); diff --git a/test/manual/chunked_prefill/test_scripted_radix.py b/test/manual/chunked_prefill/test_scripted_radix.py index 280e651a21..430f7f7354 100644 --- a/test/manual/chunked_prefill/test_scripted_radix.py +++ b/test/manual/chunked_prefill/test_scripted_radix.py @@ -358,7 +358,7 @@ class TestRadixHitCountInvariant(ScriptedTestCase): for node_id, base_count in baseline.items(): if node_id in cur: assert cur[node_id] == base_count, ( - f"_inc_hit_count(chunked=True) inflated existing " + f"re-inserting an earlier chunk inflated existing " f"node id={node_id} hit_count: baseline={base_count}, " f"now={cur[node_id]}" ) diff --git a/test/manual/chunked_prefill/test_scripted_regression.py b/test/manual/chunked_prefill/test_scripted_regression.py index ab6c238242..259ab29c08 100644 --- a/test/manual/chunked_prefill/test_scripted_regression.py +++ b/test/manual/chunked_prefill/test_scripted_regression.py @@ -517,7 +517,7 @@ class TestRegressionGptOss(ScriptedTestCase): assert committed > 0 assert len(r.req.prefix_indices) <= committed, ( - f"cache_unfinished_req over-read past kv_committed_len: " + f"insert_req over-read past kv_committed_len: " f"prefix_indices_len={len(r.req.prefix_indices)}, " f"kv_committed_len={committed}" ) diff --git a/test/manual/dsv4/test_dsv4_swa_radix_retract.py b/test/manual/dsv4/test_dsv4_swa_radix_retract.py index bdc8fa0413..36ca376b6c 100644 --- a/test/manual/dsv4/test_dsv4_swa_radix_retract.py +++ b/test/manual/dsv4/test_dsv4_swa_radix_retract.py @@ -1,6 +1,6 @@ """DSV4 stress test for SWA radix cache + tombstone + retract interaction. -Regression test for the former SWA `cache_unfinished_req` assertion: +Regression test for the former SWA `insert_req` assertion: assert old_prefix_len <= len(new_indices) The unified cache reads `req.kv.cache_protected_len` and tolerates page_size - 1 of alignment slack, so this reproduces the historical trip conditions rather diff --git a/test/registered/chunked_prefill/test_scripted_core_1gpu.py b/test/registered/chunked_prefill/test_scripted_core_1gpu.py index db4cbf9f44..c095bbac1d 100644 --- a/test/registered/chunked_prefill/test_scripted_core_1gpu.py +++ b/test/registered/chunked_prefill/test_scripted_core_1gpu.py @@ -145,7 +145,7 @@ class TestScriptedCore(ScriptedTestCase): r = t.start_req(prompt_len=_PROMPT_LEN, max_new_tokens=2) yield from run_until_finished(r) assert r.finished - _assert_prefill_twice_decode_once(t, prompt_len=_PROMPT_LEN) + _assert_every_node_hit_once(t, prompt_len=_PROMPT_LEN) def test_nonchunked_prefill_radix_hit_count(self): self.server.execute_script(self._script_nonchunked_prefill_radix_hit_count) @@ -156,10 +156,10 @@ class TestScriptedCore(ScriptedTestCase): r = t.start_req(prompt_len=prompt_len, max_new_tokens=2) yield from run_until_finished(r) assert r.finished - _assert_prefill_twice_decode_once(t, prompt_len=prompt_len) + _assert_every_node_hit_once(t, prompt_len=prompt_len) -def _assert_prefill_twice_decode_once(t: ScriptedContext, prompt_len: int) -> None: +def _assert_every_node_hit_once(t: ScriptedContext, prompt_len: int) -> None: core = t.scheduler.tree_cache.tree_core prefill_hits: list[int] = [] decode_hits: list[int] = [] @@ -178,8 +178,8 @@ def _assert_prefill_twice_decode_once(t: ScriptedContext, prompt_len: int) -> No f"expected both prefill and decode radix nodes; " f"prefill={prefill_hits}, decode={decode_hits}, prompt_len={prompt_len}" ) - assert all(h == 2 for h in prefill_hits), ( - f"each prefill node must be hit exactly twice; " + assert all(h == 1 for h in prefill_hits), ( + f"each prefill node must be hit exactly once; " f"prefill={prefill_hits}, decode={decode_hits}" ) assert all(h == 1 for h in decode_hits), ( diff --git a/test/registered/kv_canary/test_self_e2e_pd_perturb.py b/test/registered/kv_canary/test_self_e2e_pd_perturb.py index c2ea1ff0a0..73f61ace69 100644 --- a/test/registered/kv_canary/test_self_e2e_pd_perturb.py +++ b/test/registered/kv_canary/test_self_e2e_pd_perturb.py @@ -42,7 +42,7 @@ class _PDPerturbBase(CanaryPDFixture): # distinct_prompts is required for the violation to surface reliably: # with a shared prompt, P-side radix caching rewrites each request's # req_to_token row to the first-inserted (canonical) copy's slots in - # cache_unfinished_req BEFORE send_kv_chunk snapshots the indices, so a + # insert_req BEFORE send_kv_chunk snapshots the indices, so a # flip on a deduped duplicate slot is freed untransferred and never # re-verified on either side. self.send_parallel_short_requests(n=4, distinct_prompts=True) diff --git a/test/registered/unit/disaggregation/test_prefill_abort_result_cleanup.py b/test/registered/unit/disaggregation/test_prefill_abort_result_cleanup.py index 4286a29f55..8152440032 100644 --- a/test/registered/unit/disaggregation/test_prefill_abort_result_cleanup.py +++ b/test/registered/unit/disaggregation/test_prefill_abort_result_cleanup.py @@ -97,9 +97,9 @@ def _free_req(req, _tree_cache, *, is_insert): @patch("sglang.srt.disaggregation.prefill.release_kv_cache", side_effect=_free_req) -@patch("sglang.srt.disaggregation.prefill.maybe_cache_unfinished_req") +@patch("sglang.srt.disaggregation.prefill.checkpoint_kv_cache") def test_aborted_final_result_releases_hybrid_cache( - maybe_cache_unfinished_req, release_kv_cache + checkpoint_kv_cache, release_kv_cache ): scheduler = _Scheduler() req = _Req(inflight_middle_chunks=0) @@ -107,7 +107,7 @@ def test_aborted_final_result_releases_hybrid_cache( scheduler.process_batch_result_disagg_prefill(_batch(req), _result()) release_kv_cache.assert_called_once_with(req, scheduler.tree_cache, is_insert=False) - maybe_cache_unfinished_req.assert_not_called() + checkpoint_kv_cache.assert_not_called() req.disagg_kv_sender.abort.assert_called_once_with() scheduler.req_to_metadata_buffer_idx_allocator.free.assert_called_once_with(7) scheduler.tree_cache.finish.assert_called_once_with( @@ -178,9 +178,9 @@ def test_sender_abort_failure_does_not_skip_local_cleanup(release_kv_cache): @patch("sglang.srt.disaggregation.prefill.release_kv_cache", side_effect=_free_req) -@patch("sglang.srt.disaggregation.prefill.maybe_cache_unfinished_req") +@patch("sglang.srt.disaggregation.prefill.checkpoint_kv_cache") def test_grammar_rejection_retires_prefill_before_transfer( - maybe_cache_unfinished_req, release_kv_cache + checkpoint_kv_cache, release_kv_cache ): scheduler = _Scheduler() req = _Req(inflight_middle_chunks=0) @@ -194,7 +194,7 @@ def test_grammar_rejection_retires_prefill_before_transfer( assert req.grammar.finished assert req.finished() release_kv_cache.assert_called_once_with(req, scheduler.tree_cache, is_insert=False) - maybe_cache_unfinished_req.assert_not_called() + checkpoint_kv_cache.assert_not_called() scheduler.send_kv_chunk.assert_not_called() scheduler.output_streamer.stream_output.assert_called_once_with([req], False) assert scheduler.disagg_prefill_inflight_queue == [] diff --git a/test/registered/unit/managers/test_auxiliary_output.py b/test/registered/unit/managers/test_auxiliary_output.py index 7545473fa6..ea6d1b8052 100644 --- a/test/registered/unit/managers/test_auxiliary_output.py +++ b/test/registered/unit/managers/test_auxiliary_output.py @@ -605,7 +605,7 @@ def test_disaggregated_prefill_consumes_auxiliary_output_after_commit(): spec_algorithm=SimpleNamespace(is_eagle=lambda: False), tree_cache=object(), disagg_prefill_inflight_queue=[], - cache_unfinished_disagg_prefill=Mock(), + checkpoint_disagg_prefill=Mock(), send_kv_chunk=Mock(), metrics_reporter=SimpleNamespace(report_prefill_stats=Mock()), maybe_send_health_check_signal=Mock(), @@ -618,7 +618,7 @@ def test_disaggregated_prefill_consumes_auxiliary_output_after_commit(): ) assert req.output_ids == [7] - scheduler.cache_unfinished_disagg_prefill.assert_called_once_with(req) + scheduler.checkpoint_disagg_prefill.assert_called_once_with(req) snapshot_auxiliary_output_starts.assert_called_once_with(batch, result) processor.consume_auxiliary_output.assert_called_once_with( batch, diff --git a/test/registered/unit/managers/test_batch_result_processor_hidden_states.py b/test/registered/unit/managers/test_batch_result_processor_hidden_states.py index 8122187d51..f2463febae 100644 --- a/test/registered/unit/managers/test_batch_result_processor_hidden_states.py +++ b/test/registered/unit/managers/test_batch_result_processor_hidden_states.py @@ -257,7 +257,7 @@ class TestPrefillHiddenStateOffsets(CustomTestCase): with ( patch( "sglang.srt.managers.scheduler_components." - "batch_result_processor.maybe_cache_unfinished_req" + "batch_result_processor.checkpoint_kv_cache" ), patch( "sglang.srt.managers.scheduler_components." diff --git a/test/registered/unit/managers/test_priority_scheduling_disaggregation.py b/test/registered/unit/managers/test_priority_scheduling_disaggregation.py index 97babd8751..e74a12de4e 100644 --- a/test/registered/unit/managers/test_priority_scheduling_disaggregation.py +++ b/test/registered/unit/managers/test_priority_scheduling_disaggregation.py @@ -277,7 +277,7 @@ class TestOptimisticPrefillCacheOwnership(unittest.TestCase): last_node = req.last_node with envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS.override(True): - cache.advance_unpublished_req(req, chunked=True) + cache.advance_unpublished_req(req) self.assertTrue(torch.equal(req.prefix_indices, torch.tensor([8, 9, 10, 11]))) self.assertEqual(req.kv.cache_protected_len, 2) @@ -285,7 +285,6 @@ class TestOptimisticPrefillCacheOwnership(unittest.TestCase): prepare_params = component.prepare_for_caching_req.call_args.kwargs[ "insert_params" ] - self.assertTrue(prepare_params.chunked) self.assertEqual(prepare_params.prev_prefix_len, 2) component.free_out_of_window_slots.assert_called_once_with( req, 3, prepare_params @@ -328,7 +327,7 @@ class TestOptimisticPrefillCacheOwnership(unittest.TestCase): return_value=SimpleNamespace(tp_rank=0), ), ): - cache.advance_unpublished_req(req, chunked=True) + cache.advance_unpublished_req(req) SchedulerDisaggregationPrefillMixin.handle_bootstrap_failure(scheduler, req) component.free_out_of_window_slots.assert_called_once() @@ -347,16 +346,14 @@ class TestOptimisticPrefillCacheOwnership(unittest.TestCase): scheduler.tree_cache = cache req = SimpleNamespace(pending_bootstrap=True) - SchedulerDisaggregationPrefillMixin.cache_unfinished_disagg_prefill( - scheduler, req, chunked=True - ) + SchedulerDisaggregationPrefillMixin.checkpoint_disagg_prefill(scheduler, req) - cache.advance_unpublished_req.assert_called_once_with(req, chunked=True) + cache.advance_unpublished_req.assert_called_once_with(req) def test_bootstrap_success_publishes_once(self): scheduler = SimpleNamespace( disagg_prefill_bootstrap_queue=MagicMock(), - cache_unfinished_disagg_prefill=MagicMock(), + checkpoint_disagg_prefill=MagicMock(), ) scheduler.disagg_prefill_bootstrap_queue.finalize_bootstrap.return_value = True req = SimpleNamespace(rid="req") @@ -369,7 +366,7 @@ class TestOptimisticPrefillCacheOwnership(unittest.TestCase): scheduler.disagg_prefill_bootstrap_queue.finalize_bootstrap.assert_called_once_with( req ) - scheduler.cache_unfinished_disagg_prefill.assert_called_once_with(req) + scheduler.checkpoint_disagg_prefill.assert_called_once_with(req) def test_waiting_abort_releases_without_insert(self): scheduler = SimpleNamespace( diff --git a/test/registered/unit/managers/test_scheduler_chunked_req_gate.py b/test/registered/unit/managers/test_scheduler_chunked_req_gate.py index 200eed26b1..2dd769ee03 100644 --- a/test/registered/unit/managers/test_scheduler_chunked_req_gate.py +++ b/test/registered/unit/managers/test_scheduler_chunked_req_gate.py @@ -41,6 +41,7 @@ def _make_req( req.host_hit_length = 0 req.kv = ReqKvInfo(req_pool_idx=req_pool_idx) req.skip_radix_cache_insert = False + req.finished_reason = None req.last_node = None req.lock_receipt = DecLockRefParams() req.session = None @@ -118,7 +119,7 @@ def _scheduler_for_get_next_batch(*, tree_cache, chunked_req) -> Scheduler: class TestStashGatePreservesPrefixIndices(CustomTestCase): - """Consumer side: real ChunkCache.cache_unfinished_req mutates + """Consumer side: real ChunkCache.insert_req mutates req.prefix_indices iff stash actually runs, so prefix_indices content is the bug-detection signal. The stash gate is content-based: `fill_len > len(prefix_indices)` means there is freshly computed KV to diff --git a/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py b/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py index 107d8a5e5d..df2321507b 100644 --- a/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py +++ b/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py @@ -5,10 +5,10 @@ Verifies that inc_lock_ref / dec_lock_ref are balanced across the four transfer scenarios identified in PR #19746: 1. Incremental transfer & success (prefix match > 0) - inc_lock_ref(pop_preallocated) -> dec+inc(cache_unfinished_req) -> dec(cache_finished_req) + inc_lock_ref(pop_preallocated) -> dec+inc(insert_req) -> dec(release_kv_cache) 2. Full transfer & success (prefix match == 0, full KV transferred) - inc_lock_ref(get_new_prebuilt_batch) -> dec+inc(cache_unfinished_req) -> dec(cache_finished_req) + inc_lock_ref(get_new_prebuilt_batch) -> dec+inc(insert_req) -> dec(release_kv_cache) 3. Incremental transfer & failure (prefix match > 0, transfer fails) inc_lock_ref(pop_preallocated) -> dec(unpin via release_kv_cache is_insert=False) @@ -60,7 +60,7 @@ from sglang.test.test_utils import CustomTestCase def _make_cache_with_pools(page_size=1): - """Create a RadixCache with mock pools sufficient for cache_unfinished/finished_req.""" + """Create a RadixCache with mock pools sufficient for insert_req / insert_req.""" mock_allocator = MagicMock() mock_allocator.device = torch.device("cpu") mock_allocator.page_size = page_size @@ -84,7 +84,7 @@ def _make_cache_with_pools(page_size=1): class MockReq: - """Minimal mock Req with fields needed by cache_unfinished/finished_req.""" + """Minimal mock Req with fields needed by insert_req / release_kv_cache.""" def __init__(self, fill_ids, req_pool_idx=0, cache_protected_len=0, last_node=None): self.full_untruncated_fill_ids = array("q", fill_ids) @@ -109,7 +109,15 @@ class MockReq: def get_fill_ids(self): return self.full_untruncated_fill_ids[: self.extend_range.end] + rid = "mock-req" skip_radix_cache_insert = False + finished_reason = None + + def finished(self): + return self.finished_reason is not None + + def _refresh_fill_ids(self): + pass def owned_kv_len(self): return self.kv.kv_committed_len @@ -216,7 +224,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): """Scenario 1: prefix match > 0, transfer succeeds. Flow: inc_lock_ref(pop_preallocated) - -> dec_lock_ref + inc_lock_ref(cache_unfinished_req) + -> dec_lock_ref + inc_lock_ref(insert_req) -> dec_lock_ref(cache_finished_req) """ cache, req_to_token = _make_cache_with_pools() @@ -248,10 +256,11 @@ class TestDecodeLockRefScenarios(CustomTestCase): last_node=matched_node, ) - # Step 2: cache_unfinished_req (dec old lock, inc new lock) - cache.cache_unfinished_req(req) + # Step 2: insert_req (dec old lock, inc new lock) + cache.insert_req(req, up_to=req.extend_range.end) # Step 3: release_kv_cache (insert, free the rest, dec lock) + req.finished_reason = "finished" release_kv_cache(req, cache) # Verify: all non-root nodes should have lock_ref == 0 @@ -265,7 +274,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): """Scenario 2: no prefix match, full KV transferred, succeeds. Flow: inc_lock_ref(root, via init_next_round_input/get_new_prebuilt_batch) - -> dec_lock_ref + inc_lock_ref(cache_unfinished_req) + -> dec_lock_ref + inc_lock_ref(insert_req) -> dec_lock_ref(cache_finished_req) """ cache, req_to_token = _make_cache_with_pools() @@ -297,10 +306,11 @@ class TestDecodeLockRefScenarios(CustomTestCase): last_node=matched_node, ) - # Step 2: cache_unfinished_req (dec root=no-op, inc new leaf) - cache.cache_unfinished_req(req) + # Step 2: insert_req (dec root=no-op, inc new leaf) + cache.insert_req(req, up_to=req.extend_range.end) # Step 3: cache_finished_req (dec leaf) + req.finished_reason = "finished" release_kv_cache(req, cache) # Root lock unchanged, all nodes unlocked @@ -393,6 +403,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): req.kv.kv_allocated_len = len(row_vals) cache.token_to_kv_pool_allocator.reset_mock() + req.finished_reason = "finished" release_kv_cache(req, cache) # The unnamed tail slot is freed as the segment past the radix key. @@ -651,7 +662,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): SchedulerDisaggregationDecodeMixin._get_new_prebuilt_batch( scheduler, SimpleNamespace(batch_size=lambda: 0) ) - cache.cache_unfinished_req(req) + cache.insert_req(req, up_to=req.extend_range.end) self.assertEqual(shared.lock_ref, 2) self.assertEqual(req_to_token[0, :3].tolist(), prefix_vals) @@ -687,7 +698,8 @@ class TestDecodeLockRefScenarios(CustomTestCase): last_node=matched_node, ) - cache.cache_unfinished_req(req) + cache.insert_req(req, up_to=req.extend_range.end) + req.finished_reason = "finished" release_kv_cache(req, cache) # After all iterations, root lock should be 1, no protected nodes diff --git a/test/registered/unit/mem_cache/test_decode_retraction_backup.py b/test/registered/unit/mem_cache/test_decode_retraction_backup.py index 9745851c2d..9d588be7c6 100644 --- a/test/registered/unit/mem_cache/test_decode_retraction_backup.py +++ b/test/registered/unit/mem_cache/test_decode_retraction_backup.py @@ -591,7 +591,7 @@ class TestDecodeRetractionBackup(CustomTestCase): cached_indices = queue._pre_alloc(cached) self._seed_pool(env.target_pool, cached_indices, base=500) cached_values = self._snapshot_pool(env.target_pool, cached_indices) - cache.cache_unfinished_req(cached) + cache.insert_req(cached, up_to=cached.extend_range.end) pressure = env.allocator.alloc(self.pool_size // 2 - 4) req = make_req("receiving", 8) @@ -620,7 +620,7 @@ class TestDecodeRetractionBackup(CustomTestCase): restore_kv_cache(req, cache, env.req_to_token_pool, env.allocator, "host_pool") self.assertEqual(cache.host_pool_group.available_size(), host_free_before) free_before_insert = env.allocator.available_size() - cache.cache_unfinished_req(req) + cache.insert_req(req, up_to=req.extend_range.end) row = env.req_to_token_pool.req_to_token[req.kv.req_pool_idx, :8] self.assertTrue(torch.equal(row[:4], cached_indices)) diff --git a/test/registered/unit/mem_cache/test_free_kv_row_coalesce.py b/test/registered/unit/mem_cache/test_free_kv_row_coalesce.py index a3f60887be..687f234983 100644 --- a/test/registered/unit/mem_cache/test_free_kv_row_coalesce.py +++ b/test/registered/unit/mem_cache/test_free_kv_row_coalesce.py @@ -42,9 +42,6 @@ class _RowCache(BasePrefixCache): def insert_req(self, req, **kwargs): raise NotImplementedError - def cache_unfinished_req(self, req, **kwargs): - raise NotImplementedError - def evict(self, params): raise NotImplementedError diff --git a/test/registered/unit/mem_cache/test_mamba_donated_alloc_ratio.py b/test/registered/unit/mem_cache/test_mamba_donated_alloc_ratio.py index abbf74bf2f..e65cfb2729 100644 --- a/test/registered/unit/mem_cache/test_mamba_donated_alloc_ratio.py +++ b/test/registered/unit/mem_cache/test_mamba_donated_alloc_ratio.py @@ -1,7 +1,7 @@ """CPU-only unit tests for the mamba pool ratio vs the prefill->decode peak. Pins the sizing invariant behind MAMBA_CACHE_SIZE_MAX_RUNNING_REQUESTS_RATIO: -at the first cache_unfinished_req, a request still holds its admission-locked +at the first insert_req, a request still holds its admission-locked matched-prefix mamba (protected) plus its own COW slot, and then allocates a donated slot. With N distinct-prefix requests that peak is N own + N locked + 1 donated. An effective ratio of 2 (pool = 2N) leaves no evictable victim and diff --git a/test/registered/unit/mem_cache/test_page_interleave_shard.py b/test/registered/unit/mem_cache/test_page_interleave_shard.py index e714aed944..a616f5b577 100644 --- a/test/registered/unit/mem_cache/test_page_interleave_shard.py +++ b/test/registered/unit/mem_cache/test_page_interleave_shard.py @@ -775,11 +775,12 @@ class TestShardedCoreGate(_TreeCoreBackendCase): class _GraftReq: - """Minimal Req stand-in for cache_unfinished_req / insert_req.""" + """Minimal Req stand-in for insert_req.""" def __init__(self, fill_ids, req_pool_idx=0): self.fill_ids = list(fill_ids) self.origin_input_ids = array("q", fill_ids) + self.full_untruncated_fill_ids = self.origin_input_ids self.output_ids = array("q", []) self.kv = ReqKvInfo(req_pool_idx=req_pool_idx) self.extra_key = None @@ -797,6 +798,9 @@ class _GraftReq: def get_fill_ids(self): return array("q", self.fill_ids) + def finished(self): + return self.finished_reason is not None + @parameterized_class(("tree_core_backend",), [("python",), ("rust",)]) class TestRotationGraftDecline(_TreeCoreBackendCase): @@ -904,13 +908,13 @@ class TestRotationGraftDecline(_TreeCoreBackendCase): res = _insert(tree, list(range(12)), rotation_base=3) self.assertTrue(res.rotation_tail_declined) - def test_cache_unfinished_decline_keeps_request_on_own_pages(self): + def test_insert_decline_keeps_request_on_own_pages(self): tree, freed = self._tree_with_spy() self._seed_chain(tree, list(range(8)), base=1) req = _GraftReq(list(range(8)) + [90, 91, 92, 93]) req.kv_rotation_base = 3 own_locs = self._own_row(tree, req, 12) - tree.cache_unfinished_req(req) + tree.insert_req(req, up_to=len(req.fill_ids)) # No dedup free, no rebind: the request keeps its own locs whole. self.assertEqual([t.tolist() for t in freed], []) self.assertTrue(torch.equal(req.prefix_indices, own_locs)) @@ -941,6 +945,7 @@ class TestRotationGraftDecline(_TreeCoreBackendCase): req = _GraftReq(list(range(8)) + [90, 91, 92, 93]) req.kv_rotation_base = 1 own_locs = self._own_row(tree, req, 12) + req.last_node = tree.root_node_handle() finish_req(tree, req, 12) self.assertEqual(_match_len(tree, req.fill_ids), 12) released = torch.cat(freed) if freed else torch.empty(0, dtype=torch.int64) diff --git a/test/registered/unit/mem_cache/test_pure_swa_radix_cache.py b/test/registered/unit/mem_cache/test_pure_swa_radix_cache.py index 7e1c3c2c33..56c705d479 100644 --- a/test/registered/unit/mem_cache/test_pure_swa_radix_cache.py +++ b/test/registered/unit/mem_cache/test_pure_swa_radix_cache.py @@ -18,6 +18,14 @@ from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=10, suite="base-a-test-cpu") +class _FakeReqToTokenPool: + def __init__(self, req_to_token): + self.req_to_token = req_to_token + + def write(self, indices, values): + self.req_to_token[indices] = values + + class _FakeAllocator: """Rows routed to the full side are skipped: all-SWA has no full pool.""" @@ -46,8 +54,8 @@ class TestPureSWARadixCache(CustomTestCase): cache = PureSWARadixCache( CacheInitParams( disable=False, - req_to_token_pool=SimpleNamespace( - req_to_token=torch.arange(8, dtype=torch.int64).unsqueeze(0) + req_to_token_pool=_FakeReqToTokenPool( + torch.arange(8, dtype=torch.int64).unsqueeze(0) ), token_to_kv_pool_allocator=allocator, page_size=1, @@ -58,9 +66,10 @@ class TestPureSWARadixCache(CustomTestCase): req = SimpleNamespace( origin_input_ids=token_ids, output_ids=array("q"), + full_untruncated_fill_ids=token_ids, extra_key=None, cache_salt=None, - last_node=None, + last_node=cache.root_node, priority=0, kv=ReqKvInfo( req_pool_idx=0, diff --git a/test/registered/unit/mem_cache/test_radix_cache_unit.py b/test/registered/unit/mem_cache/test_radix_cache_unit.py index 2ab69da3cc..2fec456cd3 100644 --- a/test/registered/unit/mem_cache/test_radix_cache_unit.py +++ b/test/registered/unit/mem_cache/test_radix_cache_unit.py @@ -413,7 +413,7 @@ class TestRadixCache(CustomTestCase): ) self.assertEqual(cache.total_size(), 5) - def test_cache_unfinished_req_deferred_free_owns_original_indices(self): + def test_insert_req_deferred_free_owns_original_indices(self): class ReqToTokenPool: def __init__(self, row): self.req_to_token = row.unsqueeze(0) @@ -448,11 +448,13 @@ class TestRadixCache(CustomTestCase): priority=0, last_node=cache.root_node, ) - req.get_fill_ids.return_value = token_ids + req.full_untruncated_fill_ids = token_ids + req.origin_input_ids = token_ids + req.output_ids = array("q") available_before_free = allocator.available_size() allocator.free_group_begin() - cache.cache_unfinished_req(req) + cache.insert_req(req, up_to=len(token_ids)) allocator.free_group_end() self.assertEqual( @@ -469,6 +471,9 @@ class TestRadixCache(CustomTestCase): def __init__(self, row): self.req_to_token = row.unsqueeze(0) + def write(self, indices, values): + self.req_to_token[indices] = values + allocator = TokenToKVPoolAllocator( size=16, dtype=torch.float16, @@ -485,6 +490,7 @@ class TestRadixCache(CustomTestCase): req = unittest.mock.Mock( origin_input_ids=prompt_ids, output_ids=output_ids, + full_untruncated_fill_ids=prompt_ids + output_ids, kv=ReqKvInfo(req_pool_idx=0, cache_protected_len=0), extra_key=None, cache_salt=None, diff --git a/test/registered/unit/mem_cache/test_swa_locked_full_recover_unified.py b/test/registered/unit/mem_cache/test_swa_locked_full_recover_unified.py index 2be88d3516..4909e753e5 100644 --- a/test/registered/unit/mem_cache/test_swa_locked_full_recover_unified.py +++ b/test/registered/unit/mem_cache/test_swa_locked_full_recover_unified.py @@ -24,7 +24,7 @@ The recovery must succeed rather than decline: the TreeCore insert walk counts the node in `prefix_len` regardless of component consumption, while the SWA match validator rejects a `value is None` node, so a declined recovery reports a prefix `match_prefix` cannot honor and trips -`new_prefix_len <= len(new_indices)` in `cache_unfinished_req`. +`new_prefix_len <= len(new_indices)` in `insert_req`. """ import unittest diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py index b48ba04376..1b79f9c575 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py +++ b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py @@ -1883,7 +1883,7 @@ class UnifiedRadixCacheSuite: self.assertEqual(len(m.device_indices), 0) cache.sanity_check() - def test_cache_unfinished_req(self): + def test_insert_req_mid_flight(self): cache, allocator, req_to_token_pool = build_fixture(self.cfg) req = self._make_req(req_to_token_pool) @@ -1905,7 +1905,7 @@ class UnifiedRadixCacheSuite: if self.cfg.has_mamba: req.kv.mamba_last_track_seqlen = kv_len - cache.cache_unfinished_req(req) + cache.insert_req(req, up_to=req.extend_range.end) self.assertGreater(len(req.prefix_indices), 0) self.assertEqual(req.kv.cache_protected_len, len(req.prefix_indices)) @@ -1943,7 +1943,7 @@ class UnifiedRadixCacheSuite: req.extra_key = None req.kv.set_evicted_seqlen(ComponentType.SWA, evicted_len) - cache.cache_unfinished_req(req) + cache.insert_req(req, up_to=req.extend_range.end) (first,) = _node_children(cache, cache.root_node_handle()) self.assertEqual(_node_key_length(cache, first), evicted_len) @@ -2170,7 +2170,7 @@ class UnifiedRadixCacheSuite: full_available_before_insert = allocator.full_attn_allocator.available_size() - cache.cache_unfinished_req(req) + cache.insert_req(req, up_to=req.extend_range.end) self.assertEqual( allocator.full_attn_allocator.available_size(), @@ -2859,7 +2859,7 @@ class UnifiedRadixCacheSuite: swa_avail_before = allocator.swa_attn_allocator.available_size() with envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS.override(True): - cache.cache_unfinished_req(req) + cache.insert_req(req, up_to=req.extend_range.end) cushion = max(self.cfg.sliding_window_size, self.cfg.page_size) expected_evicted = (pre_len - 1) - cushion @@ -2946,7 +2946,7 @@ class UnifiedRadixCacheSuite: req.extra_key = None with envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS.override(True): - cache.cache_unfinished_req(req) + cache.insert_req(req, up_to=req.extend_range.end) self.assertEqual( req.kv.get_evicted_seqlen(ComponentType.SWA), @@ -9342,7 +9342,8 @@ class TestResumableInsertWalk(_InsertWalkSuite): # Suspend an insert at its crossing barrier by pumping it directly. params = InsertParams( - key=RadixKey(array("q", [1, 2, 3, 4])), value=self._alloc(allocator, 4) + key=RadixKey(array("q", [1, 2, 3, 4])), + value=self._alloc(allocator, 4), ) step = cache.tree_core.begin_insert(params) self.assertIsNone(step.result) @@ -10288,7 +10289,7 @@ class TestUnifiedRadixPrefetchCorruption(CustomTestCase): class TestSWAWindowUnderBigramKey(CustomTestCase): - """`cache_unfinished_req` has to leave the leaf it inserts holding a full + """`insert_req` has to leave the leaf it inserts holding a full sliding window of live SWA. Otherwise the match that follows the insert refuses that leaf, `cache_protected_len` never advances, and the next insert frees KV the tree already owns as if it were the request's duplicate. @@ -10344,7 +10345,7 @@ class TestSWAWindowUnderBigramKey(CustomTestCase): req.extra_key = None with envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS.override(True): - cache.cache_unfinished_req(req) + cache.insert_req(req, up_to=req.extend_range.end) boundary = (seq_len - 1) // page_size * page_size self.assertGreaterEqual(