From ad7a9b53ccad9d53599cb872233e2cedee1fe154 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Mon, 5 Oct 2026 17:32:07 -0700 Subject: [PATCH] [mem_cache] Make `release_kv_cache`'s final checkpoint an explicit `checkpoint` flag (#42270) --- python/sglang/srt/disaggregation/decode.py | 6 +- python/sglang/srt/disaggregation/prefill.py | 14 +- python/sglang/srt/dllm/mixin/scheduler.py | 2 +- python/sglang/srt/managers/schedule_batch.py | 4 +- python/sglang/srt/managers/scheduler.py | 8 +- .../batch_result_processor.py | 18 +- .../dynamic_chunk_sizer.py | 2 +- .../sglang/srt/mem_cache/base_prefix_cache.py | 6 +- python/sglang/srt/mem_cache/common.py | 10 +- .../storage/flexkv/flexkv_radix_cache.py | 4 +- .../lmcache/lmcache_unified_radix_cache.py | 6 +- .../unified_cache/components/README.md | 2 +- .../srt/mem_cache/unified_radix_cache.py | 6 +- .../test_decode_queue_cleanup.py | 6 +- .../test_deferred_decode_kv_release.py | 8 +- .../test_prefill_abort_result_cleanup.py | 28 ++- .../test_prefill_result_polling.py | 6 +- .../dllm/test_gemma4_uniform_lifecycle.py | 4 +- .../mlx/test_attention_patching.py | 4 +- ...st_batch_result_processor_hidden_states.py | 2 +- ...test_priority_scheduling_disaggregation.py | 4 +- .../mem_cache/test_decode_radix_lock_ref.py | 16 +- .../test_decode_retraction_backup.py | 6 +- .../mem_cache/test_streaming_session_unit.py | 166 +----------------- .../test_unified_radix_cache_unittest.py | 76 ++++---- .../mem_cache/test_unified_radix_lock_ref.py | 2 +- 26 files changed, 138 insertions(+), 278 deletions(-) diff --git a/python/sglang/srt/disaggregation/decode.py b/python/sglang/srt/disaggregation/decode.py index c9aa043fb4..efe1b69a19 100644 --- a/python/sglang/srt/disaggregation/decode.py +++ b/python/sglang/srt/disaggregation/decode.py @@ -1719,7 +1719,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): ] for decode_req in failed_reqs: if decode_req.req.kv.holds_mamba and not decode_req.req.kv.holds_kv: - release_kv_cache(decode_req.req, self.tree_cache, is_insert=False) + release_kv_cache(decode_req.req, self.tree_cache, checkpoint=False) self.queue = [ entry for i, entry in enumerate(self.queue) if i not in indices_to_remove @@ -2622,7 +2622,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): discard_kv_cache_backup(decode_req.req, self.tree_cache, "host_pool") if decode_req.host_staged: return - release_kv_cache(decode_req.req, self.tree_cache, is_insert=False) + release_kv_cache(decode_req.req, self.tree_cache, checkpoint=False) def pop_transferred(self, rids_to_check: Optional[List[str]] = None) -> List[Req]: if not self.queue: @@ -3126,7 +3126,7 @@ class SchedulerDisaggregationDecodeMixin: status_code=HTTPStatus.INTERNAL_SERVER_ERROR, ) self.output_streamer.stream_output([req], req.return_logprob) - release_kv_cache(req, self.tree_cache, is_insert=False) + release_kv_cache(req, self.tree_cache, checkpoint=False) if self.metrics_reporter.enable_metrics: self.metrics_collector.increment_transfer_failed_reqs() diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index 9d89cb4a39..df2974c485 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -604,7 +604,7 @@ class SchedulerDisaggregationPrefillMixin: sender.abort() maybe_release_metadata_buffer(req, self.req_to_metadata_buffer_idx_allocator) if req.kv.holds_kv or req.kv.holds_mamba: - release_kv_cache(req, self.tree_cache, is_insert=False) + release_kv_cache(req, self.tree_cache, checkpoint=False) req.pending_bootstrap = False @scheduler_stage_method(SCHEDULER_STAGE_PROCESS_QUEUE) @@ -1185,7 +1185,8 @@ class SchedulerDisaggregationPrefillMixin: elif poll == KVPoll.Success: # transfer done if not isinstance(req.finished_reason, FINISH_ABORT): req.finished_reason = FINISH_LENGTH(length=0) - release_kv_cache(req, self.tree_cache) # unlock the tree + # unlock the tree + release_kv_cache(req, self.tree_cache, checkpoint=True) self.tree_cache.finish( req.cache_request_handle, CacheRequestOutcome.SUCCESS ) @@ -1261,7 +1262,8 @@ class SchedulerDisaggregationPrefillMixin: req.time_stats.trace_ctx.abort(abort_info={"reason": error_message}) if req.finished_reason is None: req.finished_reason = FINISH_LENGTH(length=0) - release_kv_cache(req, self.tree_cache) # unlock the tree + # unlock the tree + release_kv_cache(req, self.tree_cache, checkpoint=True) self._release_aborted_request(req) if not isinstance(req.finished_reason, FINISH_ABORT): prepare_abort( @@ -1305,7 +1307,7 @@ class SchedulerDisaggregationPrefillMixin: req.pending_bootstrap = False self.tree_cache.finish(req.cache_request_handle, CacheRequestOutcome.ABORT) if req.kv.holds_kv or req.kv.holds_mamba: - release_kv_cache(req, self.tree_cache, is_insert=False) + release_kv_cache(req, self.tree_cache, checkpoint=False) return True def handle_bootstrap_failure(self: Scheduler, req: Req) -> None: @@ -1327,7 +1329,7 @@ class SchedulerDisaggregationPrefillMixin: logger.warning(error_message) req.time_stats.trace_ctx.abort(abort_info={"reason": error_message}) if req.kv.holds_kv or req.kv.holds_mamba: - release_kv_cache(req, self.tree_cache, is_insert=False) + release_kv_cache(req, self.tree_cache, checkpoint=False) maybe_release_metadata_buffer(req, self.req_to_metadata_buffer_idx_allocator) req.pending_bootstrap = False prepare_abort(req, error_message, status_code=HTTPStatus.INTERNAL_SERVER_ERROR) @@ -1696,7 +1698,7 @@ class SchedulerDisaggregationPrefillMixin: self._release_aborted_request(req) # The checkpoint above already handed the prefill KV to the tree; the # request is not finished, so the release only frees the rest. - release_kv_cache(req, self.tree_cache, is_insert=False) + release_kv_cache(req, self.tree_cache, checkpoint=False) req.reset_for_retract() req.output_ids = array("q") req.start_send_idx = 0 diff --git a/python/sglang/srt/dllm/mixin/scheduler.py b/python/sglang/srt/dllm/mixin/scheduler.py index 7de817bb32..b4d7f6dc67 100644 --- a/python/sglang/srt/dllm/mixin/scheduler.py +++ b/python/sglang/srt/dllm/mixin/scheduler.py @@ -179,7 +179,7 @@ class SchedulerDllmMixin: release_kv_cache( req, self.tree_cache, - is_insert=not self.dllm_config.requires_separate_context_encoding, + checkpoint=not self.dllm_config.requires_separate_context_encoding, ) req.time_stats.set_completion_time() diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index e0ab7b7b9d..d1e411660b 100755 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -1137,7 +1137,7 @@ class Req(ReqDllmMixin): # Lazy extra buffer: skip radix cache insert when prealloc failed at # boundary — the forward overwrites the only slot, corrupting the state. - self.mamba_lazy_is_insert: bool = True + self.mamba_lazy_checkpoint: bool = True # Check finish self.tokenizer = None @@ -2290,7 +2290,7 @@ def release_req( get_disagg().disaggregation_decode_retraction_backup, ) # TODO (csy): for preempted requests, we may want to insert into the tree - release_kv_cache(req, tree_cache, is_insert=False) + release_kv_cache(req, tree_cache, checkpoint=False) # NOTE(lsyin): we should use the newly evictable memory instantly. num_tokens = remaing_req_count * envs.SGLANG_RETRACT_DECODE_STEPS.get() evict_from_tree_cache(tree_cache, num_tokens) diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index be1f64ec9a..0aafc6baa5 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -3578,7 +3578,7 @@ class Scheduler( ) req.pending_bootstrap = False self._release_aborted_request(req) - release_kv_cache(req, self.tree_cache, is_insert=False) + release_kv_cache(req, self.tree_cache, checkpoint=False) self.chunked_req = None self._pending_chunked_abort_req = None @@ -5335,7 +5335,7 @@ class Scheduler( self.hisparse_coordinator.request_finished(req) if req.finished_reason is None: req.finished_reason = FINISH_ABORT() - release_kv_cache(req, self.tree_cache) + release_kv_cache(req, self.tree_cache, checkpoint=True) if self.disaggregation_mode == DisaggregationMode.PREFILL: self.release_aborted_prefill_waiting_req(req) @@ -5344,7 +5344,7 @@ class Scheduler( DisaggregationMode.PREFILL, DisaggregationMode.DECODE, ): - release_kv_cache(req, self.tree_cache, is_insert=False) + release_kv_cache(req, self.tree_cache, checkpoint=False) logger.debug(f"Abort queued request. {req.rid=}") if self.dllm_config is not None: @@ -5356,7 +5356,7 @@ class Scheduler( _make_abort_req(req), req ) if req.kv.holds_kv or req.kv.holds_mamba: - release_kv_cache(req, self.tree_cache, is_insert=False) + release_kv_cache(req, self.tree_cache, checkpoint=False) logger.debug(f"Abort dLLM queued request. {req.rid=}") # Delete the requests in the grammar queue diff --git a/python/sglang/srt/managers/scheduler_components/batch_result_processor.py b/python/sglang/srt/managers/scheduler_components/batch_result_processor.py index 1dff4fde47..c0fbc5e80c 100644 --- a/python/sglang/srt/managers/scheduler_components/batch_result_processor.py +++ b/python/sglang/srt/managers/scheduler_components/batch_result_processor.py @@ -126,7 +126,7 @@ class SchedulerBatchResultProcessor: req.time_stats.set_quick_finish_time() if get_memory().enable_hisparse: self.hisparse_coordinator.request_finished(req) - release_kv_cache(req, self.tree_cache) + release_kv_cache(req, self.tree_cache, checkpoint=True) # Note: Logprobs should be handled on the prefill engine. self.output_streamer.stream_output(batch.reqs, batch.return_logprob) @@ -377,7 +377,7 @@ class SchedulerBatchResultProcessor: release_kv_cache( req, self.tree_cache, - is_insert=sampling_mask_finish_reason is None, + checkpoint=sampling_mask_finish_reason is None, ) req.time_stats.set_completion_time() elif not batch.decoding_reqs or req not in batch.decoding_reqs: @@ -469,7 +469,7 @@ class SchedulerBatchResultProcessor: req.update_finish_state() if req.finished(): - release_kv_cache(req, self.tree_cache) + release_kv_cache(req, self.tree_cache, checkpoint=True) req.time_stats.set_completion_time() else: checkpoint_kv_cache(req, self.tree_cache) @@ -1337,7 +1337,7 @@ class SchedulerBatchResultProcessor: ) if callable(prepare_release): prepare_release(req) - release_kv_cache(req, self.tree_cache, is_insert=False) + release_kv_cache(req, self.tree_cache, checkpoint=False) req.time_stats.set_completion_time() def _handle_finish_state_updated_req( @@ -1377,7 +1377,7 @@ class SchedulerBatchResultProcessor: and known_mamba_boundary and req.kv.mamba_next_track_idx == req.kv.mamba_last_track_idx ): - req.mamba_lazy_is_insert = False + req.mamba_lazy_checkpoint = False # Called here (after update_finish_state) so req.finished() is valid # for mamba_lazy_post_decode_at_boundary inside. @@ -1424,12 +1424,12 @@ class SchedulerBatchResultProcessor: ) if callable(prepare_release): prepare_release(req) - is_insert = ( - req.mamba_lazy_is_insert + checkpoint = ( + req.mamba_lazy_checkpoint if get_exec().mamba.enable_mamba_extra_buffer_lazy else True ) - release_kv_cache(req, self.tree_cache, is_insert=is_insert) + release_kv_cache(req, self.tree_cache, checkpoint=checkpoint) req.time_stats.set_completion_time() @@ -1552,7 +1552,7 @@ class SchedulerBatchResultProcessor: or keep_written_by_this_step or keep_may_be_written_in_flight ): - req.mamba_lazy_is_insert = False + req.mamba_lazy_checkpoint = False return if not crossed or planned_pos is None: diff --git a/python/sglang/srt/managers/scheduler_components/dynamic_chunk_sizer.py b/python/sglang/srt/managers/scheduler_components/dynamic_chunk_sizer.py index ea373c12a5..2942787dbd 100644 --- a/python/sglang/srt/managers/scheduler_components/dynamic_chunk_sizer.py +++ b/python/sglang/srt/managers/scheduler_components/dynamic_chunk_sizer.py @@ -259,7 +259,7 @@ class DynamicChunkSizer: # Release KV and Mamba cache if req.kv.holds_kv: - release_kv_cache(req, self.tree_cache, is_insert=False) + release_kv_cache(req, self.tree_cache, checkpoint=False) logger.info( f"[PP Dynamic Chunk] [PP0] Profiled {len(seq_lens)} samples: " diff --git a/python/sglang/srt/mem_cache/base_prefix_cache.py b/python/sglang/srt/mem_cache/base_prefix_cache.py index 07eb652908..8974991abc 100644 --- a/python/sglang/srt/mem_cache/base_prefix_cache.py +++ b/python/sglang/srt/mem_cache/base_prefix_cache.py @@ -525,9 +525,9 @@ class BasePrefixCache(ABC, PrefixCacheTrait): Return True after taking the row; the caller then releases nothing.""" return False - def on_release(self, req: Req, *, inserted: bool) -> None: - """The row is freed and the lock dropped; ``inserted`` says whether the - KV went into the tree first. Drop per-request state kept outside the tree.""" + def on_release(self, req: Req, *, checkpointed: bool) -> None: + """The row is freed and the lock dropped; ``checkpointed`` says whether + the KV went into the tree first. Drop per-request state kept outside the tree.""" def evictable_size(self): return 0 diff --git a/python/sglang/srt/mem_cache/common.py b/python/sglang/srt/mem_cache/common.py index 6f82227b19..d34dff6ab9 100644 --- a/python/sglang/srt/mem_cache/common.py +++ b/python/sglang/srt/mem_cache/common.py @@ -294,8 +294,8 @@ def discard_kv_cache_backup( req.kv.retraction_backup = None -def release_kv_cache(req: Req, tree_cache: BasePrefixCache, is_insert: bool = True): - """Give the request's kv row back; with ``is_insert`` the tree first keeps +def release_kv_cache(req: Req, tree_cache: BasePrefixCache, *, checkpoint: bool): + """Give the request's kv row back; with ``checkpoint`` the tree first keeps what it can key.""" assert (not req.kv.holds_kv) == req.kv.is_kv_released # A mamba-capable cache may alloc mamba state before alloc KV cache @@ -316,8 +316,8 @@ def release_kv_cache(req: Req, tree_cache: BasePrefixCache, is_insert: bool = Tr return owned_kv_len = req.owned_kv_len() - is_insert = is_insert and not req.skip_radix_cache_insert - if is_insert: + checkpoint = checkpoint and not req.skip_radix_cache_insert + if checkpoint: # A tree that takes over component state (mamba) must see the request # finished, or the insert forks the state and the slot leaks. assert req.finished() or not tree_cache.supports_mamba(), ( @@ -332,7 +332,7 @@ def release_kv_cache(req: Req, tree_cache: BasePrefixCache, is_insert: bool = Tr _release_overallocated_kv_indices( req, owned_kv_len, req.kv.kv_allocated_len, tree_cache ) - tree_cache.on_release(req, inserted=is_insert) + tree_cache.on_release(req, checkpointed=checkpoint) # The DSV4-NPU ReqToTokenPool subclass's free() additionally releases the # c4/c128 state pages; other ReqToTokenPool subclasses are a no-op here. diff --git a/python/sglang/srt/mem_cache/storage/flexkv/flexkv_radix_cache.py b/python/sglang/srt/mem_cache/storage/flexkv/flexkv_radix_cache.py index fbae1235bf..216cd98f7f 100644 --- a/python/sglang/srt/mem_cache/storage/flexkv/flexkv_radix_cache.py +++ b/python/sglang/srt/mem_cache/storage/flexkv/flexkv_radix_cache.py @@ -385,8 +385,8 @@ class FlexKVRadixCache(RadixCache): # checkpoint (STORE) # ------------------------------------------------------------------ - def on_release(self, req: Req, *, inserted: bool) -> None: - if not inserted: + def on_release(self, req: Req, *, checkpointed: bool) -> None: + if not checkpointed: self._load_markers.pop(req.cache_request_handle, None) def checkpoint(self, req: Req, *, up_to: int) -> None: # type: ignore[override] diff --git a/python/sglang/srt/mem_cache/storage/lmcache/lmcache_unified_radix_cache.py b/python/sglang/srt/mem_cache/storage/lmcache/lmcache_unified_radix_cache.py index 57976d5772..4c3ce66b50 100644 --- a/python/sglang/srt/mem_cache/storage/lmcache/lmcache_unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/storage/lmcache/lmcache_unified_radix_cache.py @@ -248,9 +248,9 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache): return True - def on_release(self, req: Req, *, inserted: bool) -> None: - super().on_release(req, inserted=inserted) - if not inserted: + def on_release(self, req: Req, *, checkpointed: bool) -> None: + super().on_release(req, checkpointed=checkpointed) + if not checkpointed: self.release_aborted_request(req.cache_request_handle) def checkpoint(self, req: Req, *, up_to: int, **kwargs) -> None: diff --git a/python/sglang/srt/mem_cache/unified_cache/components/README.md b/python/sglang/srt/mem_cache/unified_cache/components/README.md index 1005f03616..51bd8045b1 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/README.md +++ b/python/sglang/srt/mem_cache/unified_cache/components/README.md @@ -237,7 +237,7 @@ Insert a request's KV into the tree: at every checkpoint while it runs, and once | Aspect | Detail | |--------|--------| -| **Purpose** | Publish the request's KV `[cache_protected_len, up_to)` so other requests can match it; nodes past `req.kv.cache_inserted_len` count one hit (a request counts each node once). `checkpoint_kv_cache` calls it with `req.extend_range.end` while the request runs; `release_kv_cache` calls it with the request-owned length when it finishes (`req.finished()`), then frees `[cache_protected_len, up_to)` and everything past it and unpins; with `is_insert=False` it skips the insert and calls `on_release(req, inserted=False)` for component cleanup | +| **Purpose** | Publish the request's KV `[cache_protected_len, up_to)` so other requests can match it; nodes past `req.kv.cache_inserted_len` count one hit (a request counts each node once). `checkpoint_kv_cache` calls it with `req.extend_range.end` while the request runs; `release_kv_cache` calls it with the request-owned length when it finishes (`req.finished()`), then frees `[cache_protected_len, up_to)` and everything past it and unpins; with `checkpoint=False` it skips the insert and calls `on_release(req, checkpointed=False)` for component cleanup | | **Inputs** | `req` — the request; `up_to` — row position the insert may read up to | | **Output** | `None` | | **Mutation** | Component hooks → `insert` → re-match → writes the tree's indices back into the row → moves the request's lock from the old `req.last_node` to the node the insert ended on → updates `req.prefix_indices`, `req.kv.cache_protected_len`, `req.kv.cache_inserted_len`, `req.last_node` → component cleanup. A finished request's component state (Mamba) is handed to the tree instead of forked, and what it still held is freed. Frees no KV slot: `release_kv_cache` does that afterwards. | diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index 76742745ab..cded5258bf 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -1053,9 +1053,9 @@ class UnifiedRadixCache(BasePrefixCache): self.cache_controller.release_pp_prefetch(req.rid) return self.session.try_cache_finished_req(req) - @rank_consensus(same_params=["req.rid", "inserted"]) - def on_release(self, req: Req, *, inserted: bool) -> None: - if inserted: + @rank_consensus(same_params=["req.rid", "checkpointed"]) + def on_release(self, req: Req, *, checkpointed: bool) -> None: + if checkpointed: return for comp in self._components_tuple: comp.cleanup_after_caching_req(req, is_finished=True) diff --git a/test/registered/unit/disaggregation/test_decode_queue_cleanup.py b/test/registered/unit/disaggregation/test_decode_queue_cleanup.py index a997377e70..c587cba01e 100644 --- a/test/registered/unit/disaggregation/test_decode_queue_cleanup.py +++ b/test/registered/unit/disaggregation/test_decode_queue_cleanup.py @@ -403,7 +403,7 @@ class TestDecodeQueueCleanup(CustomTestCase): ) mock_prepare_abort.assert_called_once() mock_release_kv_cache.assert_called_once_with( - req, queue.tree_cache, is_insert=False + req, queue.tree_cache, checkpoint=False ) receiver = FakeReceiver() @@ -422,7 +422,7 @@ class TestDecodeQueueCleanup(CustomTestCase): self.assertIsNone(decode_req.kv_receiver) queue.req_to_metadata_buffer_idx_allocator.free.assert_called_once_with(3) mock_release_kv_cache.assert_called_once_with( - req, queue.tree_cache, is_insert=False + req, queue.tree_cache, checkpoint=False ) receiver = MagicMock() @@ -505,7 +505,7 @@ class TestDecodeQueueCleanup(CustomTestCase): self.assertFalse(receiver.abort_notified) queue._defer_release.assert_not_called() mock_release_kv_cache.assert_called_once_with( - req, queue.tree_cache, is_insert=False + req, queue.tree_cache, checkpoint=False ) def test_retracted_decode_requests_keep_scheduler_non_idle(self): diff --git a/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py b/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py index 48bb6ac02e..bec48bc1d8 100644 --- a/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py +++ b/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py @@ -746,7 +746,7 @@ class TestResolveDeferredReleases(CustomTestCase): # Both ranks acked -> released exactly once. mgr.note_abort_ack(room, 1, generation) q.resolve_deferred_releases() - rel.assert_called_once_with(dreq.req, q.tree_cache, is_insert=False) + rel.assert_called_once_with(dreq.req, q.tree_cache, checkpoint=False) # Held state fully cleaned up. self.assertEqual(q._deferred_releases, []) @@ -773,7 +773,7 @@ class TestResolveDeferredReleases(CustomTestCase): patch.object(decode_mod, "release_kv_cache") as rel, ): q.resolve_deferred_releases() - rel.assert_called_once_with(dreq.req, q.tree_cache, is_insert=False) + rel.assert_called_once_with(dreq.req, q.tree_cache, checkpoint=False) self.assertEqual(q._deferred_releases, []) self.assertEqual(q.req_to_metadata_buffer_idx_allocator.freed, [idx]) @@ -796,7 +796,7 @@ class TestResolveDeferredReleases(CustomTestCase): ): q.resolve_deferred_releases() - rel.assert_called_once_with(dreq.req, q.tree_cache, is_insert=False) + rel.assert_called_once_with(dreq.req, q.tree_cache, checkpoint=False) self.assertEqual(q.num_pending_deferred_releases(), 0) q.scheduler.metrics_collector.observe_decode_deferred_kv_release.assert_not_called() @@ -813,7 +813,7 @@ class TestResolveDeferredReleases(CustomTestCase): calls = [] - def fake_release(req, tree_cache, is_insert): + def fake_release(req, tree_cache, checkpoint): calls.append(req) if req is bad.req: raise RuntimeError("boom") diff --git a/test/registered/unit/disaggregation/test_prefill_abort_result_cleanup.py b/test/registered/unit/disaggregation/test_prefill_abort_result_cleanup.py index 5ccfef15fa..2a12b110a5 100644 --- a/test/registered/unit/disaggregation/test_prefill_abort_result_cleanup.py +++ b/test/registered/unit/disaggregation/test_prefill_abort_result_cleanup.py @@ -90,8 +90,8 @@ def _result(): return GenerationBatchResult(next_token_ids=torch.tensor([11])) -def _free_req(req, _tree_cache, *, is_insert): - assert is_insert is False +def _free_req(req, _tree_cache, *, checkpoint): + assert checkpoint is False req.kv.req_pool_idx = None req.kv.mark_kv_released() req.kv.mamba_pool_idx = None @@ -107,7 +107,9 @@ def test_aborted_final_result_releases_hybrid_cache( scheduler.process_batch_result_disagg_prefill(_batch(req), _result()) - release_kv_cache.assert_called_once_with(req, scheduler.tree_cache, is_insert=False) + release_kv_cache.assert_called_once_with( + req, scheduler.tree_cache, checkpoint=False + ) checkpoint_kv_cache.assert_not_called() req.disagg_kv_sender.abort.assert_called_once_with() scheduler.req_to_metadata_buffer_idx_allocator.free.assert_called_once_with(7) @@ -131,7 +133,9 @@ def test_aborted_middle_result_releases_after_last_chunk(release_kv_cache): scheduler.process_batch_result_disagg_prefill(_batch(req), _result()) assert req.inflight_middle_chunks == 0 - release_kv_cache.assert_called_once_with(req, scheduler.tree_cache, is_insert=False) + release_kv_cache.assert_called_once_with( + req, scheduler.tree_cache, checkpoint=False + ) scheduler.output_streamer.stream_output.assert_called_once_with([req], False) @@ -148,7 +152,9 @@ def test_aborted_middle_result_waits_for_inflight_chunk(release_kv_cache): scheduler.process_batch_result_disagg_prefill(_batch(req), _result()) - release_kv_cache.assert_called_once_with(req, scheduler.tree_cache, is_insert=False) + release_kv_cache.assert_called_once_with( + req, scheduler.tree_cache, checkpoint=False + ) scheduler.output_streamer.stream_output.assert_called_once_with([req], False) @@ -172,7 +178,9 @@ def test_sender_abort_failure_does_not_skip_local_cleanup(release_kv_cache): scheduler.process_batch_result_disagg_prefill(_batch(req), _result()) - release_kv_cache.assert_called_once_with(req, scheduler.tree_cache, is_insert=False) + release_kv_cache.assert_called_once_with( + req, scheduler.tree_cache, checkpoint=False + ) scheduler.req_to_metadata_buffer_idx_allocator.free.assert_called_once_with(7) scheduler.output_streamer.stream_output.assert_called_once_with([req], False) assert req.finished() @@ -194,7 +202,9 @@ def test_grammar_rejection_retires_prefill_before_transfer( req.grammar.accept_token.assert_called_once_with(11) assert req.grammar.finished assert req.finished() - release_kv_cache.assert_called_once_with(req, scheduler.tree_cache, is_insert=False) + release_kv_cache.assert_called_once_with( + req, scheduler.tree_cache, checkpoint=False + ) checkpoint_kv_cache.assert_not_called() scheduler.send_kv_chunk.assert_not_called() scheduler.output_streamer.stream_output.assert_called_once_with([req], False) @@ -259,7 +269,9 @@ def test_sampling_mask_abort_preserves_error_and_releases_once( assert req.metadata_buffer_index == -1 assert not req.pending_bootstrap assert req.rid not in scheduler.disagg_prefill_pending_chunk_rids - release_kv_cache.assert_called_once_with(req, scheduler.tree_cache, is_insert=False) + release_kv_cache.assert_called_once_with( + req, scheduler.tree_cache, checkpoint=False + ) req.disagg_kv_sender.abort.assert_called_once_with() scheduler.req_to_metadata_buffer_idx_allocator.free.assert_called_once_with(7) scheduler.tree_cache.finish.assert_called_once_with( diff --git a/test/registered/unit/disaggregation/test_prefill_result_polling.py b/test/registered/unit/disaggregation/test_prefill_result_polling.py index d3e55ec9f9..7e55ca2c2c 100644 --- a/test/registered/unit/disaggregation/test_prefill_result_polling.py +++ b/test/registered/unit/disaggregation/test_prefill_result_polling.py @@ -809,7 +809,7 @@ class TestPrefillResultPolling(CustomTestCase): ): _run_loop(scheduler) - release_kv.assert_called_once_with(prior, scheduler.tree_cache) + release_kv.assert_called_once_with(prior, scheduler.tree_cache, checkpoint=True) self.assertEqual(prior.metadata_buffer_index, -1) self.assertEqual(second.reqs[0].metadata_buffer_index, 0) self.assertIn((6, "launch", 2), scheduler.actions) @@ -1366,7 +1366,7 @@ class TestPrefillResultPolling(CustomTestCase): scheduler.metrics_reporter.enable_metrics = False release_steps = [] - def release(req, tree_cache, is_insert): + def release(req, tree_cache, checkpoint): self.assertEqual(req.inflight_middle_chunks, 0) release_steps.append(scheduler.iteration) req.kv.req_pool_idx = None @@ -1382,7 +1382,7 @@ class TestPrefillResultPolling(CustomTestCase): _run_loop(scheduler, optimistic_prefill_attempts=2) release_kv.assert_called_once_with( - req, scheduler.tree_cache, is_insert=False + req, scheduler.tree_cache, checkpoint=False ) self.assertEqual( release_steps, [3 if depth == 1 or yield_before_pause else 5] diff --git a/test/registered/unit/dllm/test_gemma4_uniform_lifecycle.py b/test/registered/unit/dllm/test_gemma4_uniform_lifecycle.py index 7d35a91609..f126b63b0c 100644 --- a/test/registered/unit/dllm/test_gemma4_uniform_lifecycle.py +++ b/test/registered/unit/dllm/test_gemma4_uniform_lifecycle.py @@ -322,7 +322,7 @@ class TestGemma4ContextLifecycle(unittest.TestCase): scheduler, _Batch([req]), _result([canvas]) ) - release.assert_called_once_with(req, scheduler.tree_cache, is_insert=False) + release.assert_called_once_with(req, scheduler.tree_cache, checkpoint=False) def test_context_boundary_stops_sync_and_fdfo(self): block_size = 4 @@ -346,7 +346,7 @@ class TestGemma4ContextLifecycle(unittest.TestCase): self.assertTrue(req.finished()) release.assert_called_once_with( - req, scheduler.tree_cache, is_insert=False + req, scheduler.tree_cache, checkpoint=False ) diff --git a/test/registered/unit/hardware_backend/mlx/test_attention_patching.py b/test/registered/unit/hardware_backend/mlx/test_attention_patching.py index 38964a655d..acecaed853 100644 --- a/test/registered/unit/hardware_backend/mlx/test_attention_patching.py +++ b/test/registered/unit/hardware_backend/mlx/test_attention_patching.py @@ -1237,7 +1237,7 @@ class TestMlxOverlapScheduler(unittest.TestCase): multimodal_inputs=None, session=None, return_routed_experts=False, - mamba_lazy_is_insert=True, + mamba_lazy_checkpoint=True, time_stats=SimpleNamespace( set_completion_time=lambda: events.append(("completion", "r0")) ), @@ -1253,7 +1253,7 @@ class TestMlxOverlapScheduler(unittest.TestCase): original_release = batch_result_processor_module.release_kv_cache original_get_indexer = batch_result_processor_module.get_global_indexer_capturer - def fake_release_kv_cache(release_req, tree_cache, is_insert=False): + def fake_release_kv_cache(release_req, tree_cache, checkpoint=False): events.append(("release", release_req.rid)) self.assertIs(tree_cache, processor.tree_cache) diff --git a/test/registered/unit/managers/test_batch_result_processor_hidden_states.py b/test/registered/unit/managers/test_batch_result_processor_hidden_states.py index f2463febae..6e62181374 100644 --- a/test/registered/unit/managers/test_batch_result_processor_hidden_states.py +++ b/test/registered/unit/managers/test_batch_result_processor_hidden_states.py @@ -379,7 +379,7 @@ class TestSamplingMaskStatusErrors(CustomTestCase): self.assertEqual(req.to_finish.status_code, 400) req.update_finish_state.assert_called_once_with(0) processor.model_worker.prepare_for_kv_cache_release.assert_called_once_with(req) - release.assert_called_once_with(req, processor.tree_cache, is_insert=False) + release.assert_called_once_with(req, processor.tree_cache, checkpoint=False) processor.output_streamer.stream_output.assert_called_once_with([req], False) def test_overflow_and_invalid_have_distinct_http_errors(self): diff --git a/test/registered/unit/managers/test_priority_scheduling_disaggregation.py b/test/registered/unit/managers/test_priority_scheduling_disaggregation.py index e74a12de4e..cb1960cd19 100644 --- a/test/registered/unit/managers/test_priority_scheduling_disaggregation.py +++ b/test/registered/unit/managers/test_priority_scheduling_disaggregation.py @@ -332,7 +332,7 @@ class TestOptimisticPrefillCacheOwnership(unittest.TestCase): component.free_out_of_window_slots.assert_called_once() cache.insert.assert_not_called() - release_kv_cache.assert_called_once_with(req, cache, is_insert=False) + release_kv_cache.assert_called_once_with(req, cache, checkpoint=False) cache.finish.assert_called_once_with( req.cache_request_handle, CacheRequestOutcome.ABORT ) @@ -390,7 +390,7 @@ class TestOptimisticPrefillCacheOwnership(unittest.TestCase): ) release_kv_cache.assert_called_once_with( - req, scheduler.tree_cache, is_insert=False + req, scheduler.tree_cache, checkpoint=False ) sender.abort.assert_called_once() self.assertFalse(req.pending_bootstrap) diff --git a/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py b/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py index 832a59630e..6a07f45151 100644 --- a/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py +++ b/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py @@ -11,7 +11,7 @@ transfer scenarios identified in PR #19746: inc_lock_ref(get_new_prebuilt_batch) -> dec+inc(checkpoint) -> dec(release_kv_cache) 3. Incremental transfer & failure (prefix match > 0, transfer fails) - inc_lock_ref(pop_preallocated) -> dec(unpin via release_kv_cache is_insert=False) + inc_lock_ref(pop_preallocated) -> dec(unpin via release_kv_cache checkpoint=False) 4. Full transfer & failure (prefix match == 0, transfer fails) no inc_lock_ref -> dec(root_node) is no-op since root lock_ref starts at 1 @@ -261,7 +261,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): # Step 3: release_kv_cache (insert, free the rest, dec lock) req.finished_reason = "finished" - release_kv_cache(req, cache) + release_kv_cache(req, cache, checkpoint=True) # Verify: all non-root nodes should have lock_ref == 0 # (root always has lock_ref == 1) @@ -311,7 +311,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): # Step 3: cache_finished_req (dec leaf) req.finished_reason = "finished" - release_kv_cache(req, cache) + release_kv_cache(req, cache, checkpoint=True) # Root lock unchanged, all nodes unlocked self.assertEqual(cache.root_node.lock_ref, root_lock_before) @@ -325,7 +325,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): must preserve the matched prefix and release the full request-owned suffix. Flow: inc_lock_ref(pop_preallocated) - -> dec_lock_ref(unpin via release_kv_cache is_insert=False) + -> dec_lock_ref(unpin via release_kv_cache checkpoint=False) """ cache, req_to_token = _make_cache_with_pools() @@ -358,7 +358,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): # Transfer fails -> discard without inserting cache.token_to_kv_pool_allocator.reset_mock() - release_kv_cache(req, cache, is_insert=False) + release_kv_cache(req, cache, checkpoint=False) ((indices, start_pos),) = ( cache.token_to_kv_pool_allocator.free_segments.call_args.args[0] @@ -404,7 +404,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): cache.token_to_kv_pool_allocator.reset_mock() req.finished_reason = "finished" - release_kv_cache(req, cache) + release_kv_cache(req, cache, checkpoint=True) # The unnamed tail slot is freed as the segment past the radix key. segments = cache.token_to_kv_pool_allocator.free_segments.call_args.args[0] @@ -454,7 +454,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): ) # Transfer fails -> discard without inserting; dec_lock_ref(root) is a no-op - release_kv_cache(req, cache, is_insert=False) + release_kv_cache(req, cache, checkpoint=False) # Root lock unchanged, nothing protected or evictable self.assertEqual(cache.root_node.lock_ref, root_lock_before) @@ -699,7 +699,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): cache.checkpoint(req, up_to=req.extend_range.end) req.finished_reason = "finished" - release_kv_cache(req, cache) + release_kv_cache(req, cache, checkpoint=True) # After all iterations, root lock should be 1, no protected nodes self.assertEqual(cache.root_node.lock_ref, 1) diff --git a/test/registered/unit/mem_cache/test_decode_retraction_backup.py b/test/registered/unit/mem_cache/test_decode_retraction_backup.py index c9cc5b6536..1c9e07c0d0 100644 --- a/test/registered/unit/mem_cache/test_decode_retraction_backup.py +++ b/test/registered/unit/mem_cache/test_decode_retraction_backup.py @@ -403,7 +403,7 @@ class TestDecodeRetractionBackup(CustomTestCase): if host_staged: env.cache.discard_kv_cache_backup(req.kv.retraction_backup) else: - release_kv_cache(req, env.cache, is_insert=False) + release_kv_cache(req, env.cache, checkpoint=False) env.allocator.free(pressure) self.assertEqual(env.allocator.available_size(), self.pool_size) @@ -626,9 +626,9 @@ class TestDecodeRetractionBackup(CustomTestCase): self._assert_pool_equal(env.target_pool, cached_indices, cached_values) self.assertEqual(env.allocator.available_size(), free_before_insert + 4) self.assertEqual(cache.protected_size(), 8) - release_kv_cache(req, cache, is_insert=False) + release_kv_cache(req, cache, checkpoint=False) self.assertEqual(cache.protected_size(), 4) - release_kv_cache(cached, cache, is_insert=False) + release_kv_cache(cached, cache, checkpoint=False) self.assertEqual(cache.protected_size(), 0) diff --git a/test/registered/unit/mem_cache/test_streaming_session_unit.py b/test/registered/unit/mem_cache/test_streaming_session_unit.py index eb52f9ae54..5baeb6cfd4 100644 --- a/test/registered/unit/mem_cache/test_streaming_session_unit.py +++ b/test/registered/unit/mem_cache/test_streaming_session_unit.py @@ -11,10 +11,7 @@ from sglang.srt.mem_cache.base_prefix_cache import ( IncLockRefResult, MatchResult, ) -from sglang.srt.mem_cache.common import release_kv_cache from sglang.srt.mem_cache.unified_cache.component_type import ComponentType -from sglang.srt.runtime_context import publish, reset_context -from sglang.srt.server_args import ServerArgs from sglang.srt.session.streaming_session import SessionSlot, StreamingSession from sglang.test.ci.ci_register import register_cpu_ci @@ -56,15 +53,6 @@ class _FakeReqToTokenPool: req.kv.req_pool_idx = None -@pytest.fixture -def published_config(): - # release_kv_cache reads the spec / serving bags. - reset_context() - publish(ServerArgs(model_path="dummy"), role="scheduler") - yield - reset_context() - - class _FakeInnerCache: """Stands in for UnifiedRadixCache: owns the session and tries it first.""" @@ -86,18 +74,6 @@ class _FakeInnerCache: def maybe_hand_to_session(self, req): self.session.take(req) - def claim_kv_row(self, req): - return self.session.try_cache_finished_req(req) - - def unpin(self, req): - if req.last_node is not None: - self.dec_lock_ref( - req.last_node, req.lock_receipt, skip_swa=req.swa_prefix_lock_released - ) - - def on_release(self, req, *, inserted): - pass - def match_prefix(self, params): result = self.session.try_match_prefix(params) if result is not None: @@ -113,12 +89,6 @@ class _FakeInnerCache: self.dec_lock_ref_params.append(args[0] if args else kwargs.get("params")) self.dec_lock_ref_skip_swa.append(kwargs.get("skip_swa", False)) - def supports_mamba(self): - return False - - def sanity_check(self): - return None - class _FakeReq: def __init__( @@ -149,21 +119,10 @@ class _FakeReq: self.finished_reason = None self.finished_len = None - skip_radix_cache_insert = False - def detach_kv(self): kv, self.kv = self.kv, ReqKvInfo() return kv - def finished(self): - return self.finished_reason is not None - - def owned_kv_len(self): - return self.kv.kv_committed_len - - def refresh_fill_ids(self): - pass - def _single_row_cache(): req_to_token = torch.arange(128, dtype=torch.int32).reshape(1, 128) @@ -181,8 +140,6 @@ def _finished_turn(tree_cache, req): def test_finished_turn_leaves_its_record_in_the_slot(): - # Component cursors ride in the record the slot keeps; the request is - # detached, and its per-turn SWA branch observation is dropped. req = _FakeReq("session-a", req_pool_idx=0, committed=4, allocated=4) record = req.kv record.mamba_next_track_idx = 1 @@ -199,8 +156,8 @@ def test_finished_turn_leaves_its_record_in_the_slot(): def test_preabort_detaches_session_and_preserves_slot(): - """Pre-aborted req (to_finish set before match_prefix) is detached from - the session: session=None, abort_req() called. Slot stays intact.""" + """A request aborted before its match is detached from the session; the + slot stays intact.""" req_to_token = torch.arange(256, dtype=torch.int32).reshape(2, 128) req_to_token_pool = _FakeReqToTokenPool(req_to_token) allocator = _FakeAllocator(page_size=16) @@ -237,9 +194,7 @@ def test_preabort_detaches_session_and_preserves_slot(): ) ) - # Req detached from session. assert req.session is None - # Slot untouched. slot = tree_cache.session.slots["session-a"] assert slot.kv.req_pool_idx == 0 assert slot.kv.kv_committed_len == 48 @@ -247,71 +202,6 @@ def test_preabort_detaches_session_and_preserves_slot(): assert len(result.device_indices) == 0 -def test_first_mid_abort_releases_like_any_request(published_config): - """First-turn abort: the session took the record and the admission lock at - allocation; the abort hands both back, so the request is released like - any aborted request.""" - page_size = 1 - req_to_token = torch.arange(128, dtype=torch.int32).reshape(1, 128) - req_to_token_pool = _FakeReqToTokenPool(req_to_token) - allocator = _FakeAllocator() - inner = _FakeInnerCache(req_to_token_pool, allocator, page_size) - tree_cache = inner - - req = _FakeReq("session-a", req_pool_idx=0, committed=20, allocated=20) - lock_node = SimpleNamespace(id=7) - req.last_node = lock_node - tree_cache.maybe_hand_to_session(req) - assert tree_cache.session.slots["session-a"].last_node is lock_node - req.finished_reason = FINISH_ABORT("input too long") - - release_kv_cache(req, tree_cache) - - assert "session-a" not in tree_cache.session.slots - assert inner.dec_lock_ref_calls == [lock_node] - # Transient pool slot freed. - assert req.kv.req_pool_idx is None - assert req_to_token_pool.free_slots == [0] - assert len(allocator.freed) == 1 - assert allocator.freed[0].tolist() == list(range(20)) - - -def test_nth_mid_abort_drops_session_slot(published_config): - """Later-turn abort: the request ran on the slot's record, so releasing it - frees the whole row and drops the slot with its tree lock (skipping an - early-released SWA lock); the session re-prefills next turn.""" - page_size = 1 - req_to_token = torch.arange(256, dtype=torch.int32).reshape(2, 128) - req_to_token_pool = _FakeReqToTokenPool(req_to_token) - allocator = _FakeAllocator() - inner = _FakeInnerCache(req_to_token_pool, allocator, page_size) - tree_cache = inner - - # match_prefix lent the slot's record, so the req runs on it. - req = _FakeReq("session-a", req_pool_idx=0, committed=65, allocated=65) - req.finished_reason = FINISH_ABORT("client disconnected") - lock_node = SimpleNamespace(id=42) - tree_cache.session.slots["session-a"] = SessionSlot( - kv=req.kv, - last_node=lock_node, - lock_receipt=DecLockRefParams( - node_id=42, component_lock_uuids={ComponentType.SWA: 7} - ), - swa_prefix_lock_released=True, - ) - - release_kv_cache(req, tree_cache) - - assert "session-a" not in tree_cache.session.slots - assert inner.dec_lock_ref_calls == [lock_node] - assert inner.dec_lock_ref_skip_swa == [True] - assert len(allocator.freed) == 1 - assert allocator.freed[0].tolist() == list(range(65)) - # Pool slot returned. - assert req_to_token_pool.free_slots == [0] - assert req.kv.req_pool_idx is None - - @pytest.mark.parametrize("uuid", [None, 17]) def test_release_session_preserves_component_lock_receipt(uuid): """Closing a session releases only the component locks it acquired.""" @@ -363,59 +253,16 @@ def test_release_session_preserves_component_lock_receipt(uuid): assert inner.dec_lock_ref_skip_swa == [False] -def test_release_session_skips_swa_after_early_release(): - """A slot whose SWA lock was released early must close with skip_swa, or - the SWA segment is released twice.""" - req_to_token = torch.arange(256, dtype=torch.int32).reshape(2, 128) - req_to_token_pool = _FakeReqToTokenPool(req_to_token) - allocator = _FakeAllocator() - inner = _FakeInnerCache(req_to_token_pool, allocator, page_size=1) - tree_cache = inner - - lock_node = SimpleNamespace(id=42) - tree_cache.session.slots["session-a"] = SessionSlot( - kv=ReqKvInfo( - req_pool_idx=0, - kv_committed_len=50, - kv_allocated_len=50, - cache_protected_len=0, - ), - last_node=lock_node, - lock_receipt=DecLockRefParams( - node_id=42, component_lock_uuids={ComponentType.SWA: 7} - ), - swa_prefix_lock_released=True, - ) - - tree_cache.session.release_session("session-a") - - assert inner.dec_lock_ref_calls == [lock_node] - assert inner.dec_lock_ref_params[0].component_lock_uuids[ComponentType.SWA] == 7 - assert inner.dec_lock_ref_skip_swa == [True] - - -# Shrink tests removed: streaming sessions are append-only after the -# rollback fix in session_controller (rollback_aborted_req). The shrink -# code path in cache_finished_req no longer exists. - - def test_trim_overshoot_postcondition(): - """`_trim_overshoot` postcondition: every per-req KV field is capped at - target = origin+finished_len, output_ids is truncated, and the tail - KV slots are freed. Covers both non-SWA fields (kv_committed_len, - kv_allocated_len, output_ids) and SWA bookkeeping (swa_evicted_seqlen) - in one shot — same invariant `_free_tail` enforces on the match_prefix - path. - """ + """Every per-request KV cursor is capped at origin + finished_len and the + tail past it is freed.""" page_size = 1 req_to_token = torch.arange(128, dtype=torch.int32).reshape(1, 128) req_to_token_pool = _FakeReqToTokenPool(req_to_token) allocator = _FakeAllocator() tree_cache = _FakeInnerCache(req_to_token_pool, allocator, page_size) - # Overshoot scenario: origin=26, finished_len=12 -> target=38. - # committed=40 (overshoot 2), allocated=44, swa_evicted=42 (> target), - # output_ids extended to 14 by the overshoot round. + # target = 26 + 12 = 38; the overshoot committed 40 and allocated 44. req = _FakeReq("session-a", req_pool_idx=0, committed=40, allocated=44) req.origin_input_ids = list(range(26)) req.output_ids = list(range(14)) @@ -428,8 +275,7 @@ def test_trim_overshoot_postcondition(): assert req.kv.kv_allocated_len == target assert req.kv.get_evicted_seqlen(ComponentType.SWA) == target assert len(req.output_ids) == 12 - # Tail [38, 44) freed by _free_kv_aligned, split at the pre-trim eviction - # floor 42: [38, 42) gave its SWA peers back already, so it goes back full-only. + # [38, 42) already gave its SWA back, so the free splits at 42. assert [t.tolist() for t in allocator.freed] == [[38, 39, 40, 41], [42, 43]] diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py index 1d59f546b9..8ea8c4a328 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py +++ b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py @@ -1930,7 +1930,7 @@ class UnifiedRadixCacheSuite: ) avail_before = allocator.available_size() - release_kv_cache(req, cache, is_insert=False) + release_kv_cache(req, cache, checkpoint=False) self.assertEqual(allocator.available_size(), avail_before + kv_len) m = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", tokens)))) @@ -11289,9 +11289,8 @@ class TestSegmentLockFuzzWithMamba(TestSegmentLockFuzz): class TestStreamingSessionLockLifecycle(CustomTestCase): - """A streaming session owns its record and the tree lock on its prefix - from the first turn's row allocation; every move of that lock keeps the - tree's lock counts balanced.""" + """A streaming session owns its record and prefix lock from the first + turn's row allocation; every move of that lock keeps the tree balanced.""" cfg = CacheConfig( page_size=1, @@ -11334,8 +11333,7 @@ class TestStreamingSessionLockLifecycle(CustomTestCase): return req def _admitted_turn(self, cache, allocator, pool, tokens, session): - """A first turn after admission and row allocation, before the session - takes it: its own row, and the root lock admission took.""" + """A first turn with its own row and the root lock admission took.""" req = self._turn(tokens, session) pool.alloc([req]) pool.write( @@ -11360,8 +11358,7 @@ class TestStreamingSessionLockLifecycle(CustomTestCase): cache.sanity_check() def test_abort_hands_early_released_lock_back(self): - """The aborted turn gets the slot's lock with its SWA part already - released, so its own unpin must not release the SWA segment again.""" + """The handed-back lock keeps its SWA part released; unpin skips it.""" cache, allocator, _ = build_fixture(self.cfg) node, lock = self._lock_and_early_release(cache, allocator) req = SimpleNamespace( @@ -11385,7 +11382,7 @@ class TestStreamingSessionLockLifecycle(CustomTestCase): cache.unpin(req) cache.sanity_check() - def test_slot_publishes_first_prompt_and_keeps_its_lock(self): + def test_slot_publishes_first_prompt_but_not_its_output(self): cache, allocator, pool = build_fixture(self.cfg) tokens = list(range(1, 9)) req = self._admitted_turn(cache, allocator, pool, tokens, self._session("s")) @@ -11393,35 +11390,26 @@ class TestStreamingSessionLockLifecycle(CustomTestCase): slot = cache.session.slots["s"] cache.checkpoint(req, up_to=len(tokens)) - - # The prompt is in the tree for anyone, and the slot's lock moved onto it. match = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", tokens)))) - self.assertEqual(len(match.device_indices), len(tokens)) self.assertEqual(slot.last_node, match.last_device_node) - self.assertEqual(slot.kv.cache_protected_len, len(tokens)) self.assertIs(req.last_node, slot.virtual_node) - req.finished_reason = FINISH_LENGTH(length=0) - self.assertTrue(cache.session.try_cache_finished_req(req)) - self.assertFalse(slot.publishes_prompt) - self.assertFalse(req.kv.holds_kv) + req.output_ids = array("q", range(9, 13)) + req.refresh_fill_ids() + pool.write((req.kv.req_pool_idx, slice(8, 12)), allocator.alloc(4)) + req.kv.kv_committed_len = 12 + req.kv.kv_allocated_len = 12 + req.finished_reason = FINISH_LENGTH(length=4) + release_kv_cache(req, cache, checkpoint=True) + + self.assertIs(cache.session.slots["s"], slot) + match = cache.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", range(1, 13)))) + ) + self.assertEqual(len(match.device_indices), len(tokens)) cache.session.release_session("s") cache.sanity_check() - def test_abort_after_publish_releases_like_any_request(self): - cache, allocator, pool = build_fixture(self.cfg) - tokens = list(range(1, 9)) - req = self._admitted_turn(cache, allocator, pool, tokens, self._session("s")) - cache.maybe_hand_to_session(req) - cache.checkpoint(req, up_to=len(tokens)) - slot = cache.session.slots["s"] - - req.finished_reason = FINISH_ABORT() - self.assertFalse(cache.session.try_cache_finished_req(req)) - self.assertEqual(req.last_node, slot.last_node) - finish_req(cache, req, len(tokens)) - cache.sanity_check() - def test_session_turn_releases_slot_swa_prefix_lock_once(self): cache, allocator, pool = build_fixture(self.cfg) tokens = list(range(1, 9)) @@ -11433,7 +11421,6 @@ class TestStreamingSessionLockLifecycle(CustomTestCase): cache.release_swa_prefix_lock(req) self.assertTrue(slot.swa_prefix_lock_released) self.assertFalse(req.swa_prefix_lock_released) - # A second call finds the SWA part already released. cache.release_swa_prefix_lock(req) req.finished_reason = FINISH_LENGTH(length=0) @@ -11441,18 +11428,14 @@ class TestStreamingSessionLockLifecycle(CustomTestCase): cache.session.release_session("s") cache.sanity_check() - def test_later_turn_borrows_without_publishing(self): - """A later turn runs on the slot's record: its allocation hands nothing - over, and its checkpoints neither insert nor move the slot's lock.""" - cache, allocator, pool = build_fixture(self.cfg) + def _later_turn(self, cache, allocator, pool): + """A finished first turn, then a second turn on the slot's record.""" session = self._session("s") first = self._admitted_turn(cache, allocator, pool, list(range(1, 9)), session) cache.maybe_hand_to_session(first) cache.checkpoint(first, up_to=8) first.finished_reason = FINISH_LENGTH(length=0) self.assertTrue(cache.session.try_cache_finished_req(first)) - slot = cache.session.slots["s"] - slot_lock_node = slot.last_node tokens = list(range(1, 13)) req = self._turn(tokens, session) @@ -11464,6 +11447,13 @@ class TestStreamingSessionLockLifecycle(CustomTestCase): req.kv.kv_committed_len = 12 req.kv.kv_allocated_len = 12 cache.maybe_hand_to_session(req) + return cache.session.slots["s"], req + + def test_later_turn_borrows_without_publishing(self): + cache, allocator, pool = build_fixture(self.cfg) + slot, req = self._later_turn(cache, allocator, pool) + slot_lock_node = slot.last_node + cache.checkpoint(req, up_to=12) self.assertIs(slot.last_node, slot_lock_node) @@ -11472,6 +11462,16 @@ class TestStreamingSessionLockLifecycle(CustomTestCase): cache.session.release_session("s") cache.sanity_check() + def test_later_turn_abort_releases_like_any_request(self): + """The record and the slot's lock go back before the KV is checkpointed.""" + cache, allocator, pool = build_fixture(self.cfg) + _, req = self._later_turn(cache, allocator, pool) + + req.finished_reason = FINISH_ABORT() + release_kv_cache(req, cache, checkpoint=True) + self.assertNotIn("s", cache.session.slots) + cache.sanity_check() + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/mem_cache/test_unified_radix_lock_ref.py b/test/registered/unit/mem_cache/test_unified_radix_lock_ref.py index 42c87a9bb9..4a38206884 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_lock_ref.py +++ b/test/registered/unit/mem_cache/test_unified_radix_lock_ref.py @@ -55,7 +55,7 @@ class TestUnifiedRadixLockRefScenarios(unittest.TestCase): return_value=SimpleNamespace(strip_thinking_cache=False), ), ): - release_kv_cache(req, cache, is_insert=False) + release_kv_cache(req, cache, checkpoint=False) cache.free_kv_row.assert_called_once_with(kv, [(0, 3)]) cache._dec_req_lock.assert_not_called()