[mem_cache] Make release_kv_cache's final checkpoint an explicit checkpoint flag (#42270)

This commit is contained in:
Liangsheng Yin
2026-10-05 17:32:07 -07:00
committed by GitHub
parent 62af5f56a5
commit ad7a9b53cc
26 changed files with 138 additions and 278 deletions
+3 -3
View File
@@ -1719,7 +1719,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
]
for decode_req in failed_reqs:
if decode_req.req.kv.holds_mamba and not decode_req.req.kv.holds_kv:
release_kv_cache(decode_req.req, self.tree_cache, is_insert=False)
release_kv_cache(decode_req.req, self.tree_cache, checkpoint=False)
self.queue = [
entry for i, entry in enumerate(self.queue) if i not in indices_to_remove
@@ -2622,7 +2622,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
discard_kv_cache_backup(decode_req.req, self.tree_cache, "host_pool")
if decode_req.host_staged:
return
release_kv_cache(decode_req.req, self.tree_cache, is_insert=False)
release_kv_cache(decode_req.req, self.tree_cache, checkpoint=False)
def pop_transferred(self, rids_to_check: Optional[List[str]] = None) -> List[Req]:
if not self.queue:
@@ -3126,7 +3126,7 @@ class SchedulerDisaggregationDecodeMixin:
status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
)
self.output_streamer.stream_output([req], req.return_logprob)
release_kv_cache(req, self.tree_cache, is_insert=False)
release_kv_cache(req, self.tree_cache, checkpoint=False)
if self.metrics_reporter.enable_metrics:
self.metrics_collector.increment_transfer_failed_reqs()
+8 -6
View File
@@ -604,7 +604,7 @@ class SchedulerDisaggregationPrefillMixin:
sender.abort()
maybe_release_metadata_buffer(req, self.req_to_metadata_buffer_idx_allocator)
if req.kv.holds_kv or req.kv.holds_mamba:
release_kv_cache(req, self.tree_cache, is_insert=False)
release_kv_cache(req, self.tree_cache, checkpoint=False)
req.pending_bootstrap = False
@scheduler_stage_method(SCHEDULER_STAGE_PROCESS_QUEUE)
@@ -1185,7 +1185,8 @@ class SchedulerDisaggregationPrefillMixin:
elif poll == KVPoll.Success: # transfer done
if not isinstance(req.finished_reason, FINISH_ABORT):
req.finished_reason = FINISH_LENGTH(length=0)
release_kv_cache(req, self.tree_cache) # unlock the tree
# unlock the tree
release_kv_cache(req, self.tree_cache, checkpoint=True)
self.tree_cache.finish(
req.cache_request_handle, CacheRequestOutcome.SUCCESS
)
@@ -1261,7 +1262,8 @@ class SchedulerDisaggregationPrefillMixin:
req.time_stats.trace_ctx.abort(abort_info={"reason": error_message})
if req.finished_reason is None:
req.finished_reason = FINISH_LENGTH(length=0)
release_kv_cache(req, self.tree_cache) # unlock the tree
# unlock the tree
release_kv_cache(req, self.tree_cache, checkpoint=True)
self._release_aborted_request(req)
if not isinstance(req.finished_reason, FINISH_ABORT):
prepare_abort(
@@ -1305,7 +1307,7 @@ class SchedulerDisaggregationPrefillMixin:
req.pending_bootstrap = False
self.tree_cache.finish(req.cache_request_handle, CacheRequestOutcome.ABORT)
if req.kv.holds_kv or req.kv.holds_mamba:
release_kv_cache(req, self.tree_cache, is_insert=False)
release_kv_cache(req, self.tree_cache, checkpoint=False)
return True
def handle_bootstrap_failure(self: Scheduler, req: Req) -> None:
@@ -1327,7 +1329,7 @@ class SchedulerDisaggregationPrefillMixin:
logger.warning(error_message)
req.time_stats.trace_ctx.abort(abort_info={"reason": error_message})
if req.kv.holds_kv or req.kv.holds_mamba:
release_kv_cache(req, self.tree_cache, is_insert=False)
release_kv_cache(req, self.tree_cache, checkpoint=False)
maybe_release_metadata_buffer(req, self.req_to_metadata_buffer_idx_allocator)
req.pending_bootstrap = False
prepare_abort(req, error_message, status_code=HTTPStatus.INTERNAL_SERVER_ERROR)
@@ -1696,7 +1698,7 @@ class SchedulerDisaggregationPrefillMixin:
self._release_aborted_request(req)
# The checkpoint above already handed the prefill KV to the tree; the
# request is not finished, so the release only frees the rest.
release_kv_cache(req, self.tree_cache, is_insert=False)
release_kv_cache(req, self.tree_cache, checkpoint=False)
req.reset_for_retract()
req.output_ids = array("q")
req.start_send_idx = 0
+1 -1
View File
@@ -179,7 +179,7 @@ class SchedulerDllmMixin:
release_kv_cache(
req,
self.tree_cache,
is_insert=not self.dllm_config.requires_separate_context_encoding,
checkpoint=not self.dllm_config.requires_separate_context_encoding,
)
req.time_stats.set_completion_time()
+2 -2
View File
@@ -1137,7 +1137,7 @@ class Req(ReqDllmMixin):
# Lazy extra buffer: skip radix cache insert when prealloc failed at
# boundary — the forward overwrites the only slot, corrupting the state.
self.mamba_lazy_is_insert: bool = True
self.mamba_lazy_checkpoint: bool = True
# Check finish
self.tokenizer = None
@@ -2290,7 +2290,7 @@ def release_req(
get_disagg().disaggregation_decode_retraction_backup,
)
# TODO (csy): for preempted requests, we may want to insert into the tree
release_kv_cache(req, tree_cache, is_insert=False)
release_kv_cache(req, tree_cache, checkpoint=False)
# NOTE(lsyin): we should use the newly evictable memory instantly.
num_tokens = remaing_req_count * envs.SGLANG_RETRACT_DECODE_STEPS.get()
evict_from_tree_cache(tree_cache, num_tokens)
+4 -4
View File
@@ -3578,7 +3578,7 @@ class Scheduler(
)
req.pending_bootstrap = False
self._release_aborted_request(req)
release_kv_cache(req, self.tree_cache, is_insert=False)
release_kv_cache(req, self.tree_cache, checkpoint=False)
self.chunked_req = None
self._pending_chunked_abort_req = None
@@ -5335,7 +5335,7 @@ class Scheduler(
self.hisparse_coordinator.request_finished(req)
if req.finished_reason is None:
req.finished_reason = FINISH_ABORT()
release_kv_cache(req, self.tree_cache)
release_kv_cache(req, self.tree_cache, checkpoint=True)
if self.disaggregation_mode == DisaggregationMode.PREFILL:
self.release_aborted_prefill_waiting_req(req)
@@ -5344,7 +5344,7 @@ class Scheduler(
DisaggregationMode.PREFILL,
DisaggregationMode.DECODE,
):
release_kv_cache(req, self.tree_cache, is_insert=False)
release_kv_cache(req, self.tree_cache, checkpoint=False)
logger.debug(f"Abort queued request. {req.rid=}")
if self.dllm_config is not None:
@@ -5356,7 +5356,7 @@ class Scheduler(
_make_abort_req(req), req
)
if req.kv.holds_kv or req.kv.holds_mamba:
release_kv_cache(req, self.tree_cache, is_insert=False)
release_kv_cache(req, self.tree_cache, checkpoint=False)
logger.debug(f"Abort dLLM queued request. {req.rid=}")
# Delete the requests in the grammar queue
@@ -126,7 +126,7 @@ class SchedulerBatchResultProcessor:
req.time_stats.set_quick_finish_time()
if get_memory().enable_hisparse:
self.hisparse_coordinator.request_finished(req)
release_kv_cache(req, self.tree_cache)
release_kv_cache(req, self.tree_cache, checkpoint=True)
# Note: Logprobs should be handled on the prefill engine.
self.output_streamer.stream_output(batch.reqs, batch.return_logprob)
@@ -377,7 +377,7 @@ class SchedulerBatchResultProcessor:
release_kv_cache(
req,
self.tree_cache,
is_insert=sampling_mask_finish_reason is None,
checkpoint=sampling_mask_finish_reason is None,
)
req.time_stats.set_completion_time()
elif not batch.decoding_reqs or req not in batch.decoding_reqs:
@@ -469,7 +469,7 @@ class SchedulerBatchResultProcessor:
req.update_finish_state()
if req.finished():
release_kv_cache(req, self.tree_cache)
release_kv_cache(req, self.tree_cache, checkpoint=True)
req.time_stats.set_completion_time()
else:
checkpoint_kv_cache(req, self.tree_cache)
@@ -1337,7 +1337,7 @@ class SchedulerBatchResultProcessor:
)
if callable(prepare_release):
prepare_release(req)
release_kv_cache(req, self.tree_cache, is_insert=False)
release_kv_cache(req, self.tree_cache, checkpoint=False)
req.time_stats.set_completion_time()
def _handle_finish_state_updated_req(
@@ -1377,7 +1377,7 @@ class SchedulerBatchResultProcessor:
and known_mamba_boundary
and req.kv.mamba_next_track_idx == req.kv.mamba_last_track_idx
):
req.mamba_lazy_is_insert = False
req.mamba_lazy_checkpoint = False
# Called here (after update_finish_state) so req.finished() is valid
# for mamba_lazy_post_decode_at_boundary inside.
@@ -1424,12 +1424,12 @@ class SchedulerBatchResultProcessor:
)
if callable(prepare_release):
prepare_release(req)
is_insert = (
req.mamba_lazy_is_insert
checkpoint = (
req.mamba_lazy_checkpoint
if get_exec().mamba.enable_mamba_extra_buffer_lazy
else True
)
release_kv_cache(req, self.tree_cache, is_insert=is_insert)
release_kv_cache(req, self.tree_cache, checkpoint=checkpoint)
req.time_stats.set_completion_time()
@@ -1552,7 +1552,7 @@ class SchedulerBatchResultProcessor:
or keep_written_by_this_step
or keep_may_be_written_in_flight
):
req.mamba_lazy_is_insert = False
req.mamba_lazy_checkpoint = False
return
if not crossed or planned_pos is None:
@@ -259,7 +259,7 @@ class DynamicChunkSizer:
# Release KV and Mamba cache
if req.kv.holds_kv:
release_kv_cache(req, self.tree_cache, is_insert=False)
release_kv_cache(req, self.tree_cache, checkpoint=False)
logger.info(
f"[PP Dynamic Chunk] [PP0] Profiled {len(seq_lens)} samples: "
@@ -525,9 +525,9 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
Return True after taking the row; the caller then releases nothing."""
return False
def on_release(self, req: Req, *, inserted: bool) -> None:
"""The row is freed and the lock dropped; ``inserted`` says whether the
KV went into the tree first. Drop per-request state kept outside the tree."""
def on_release(self, req: Req, *, checkpointed: bool) -> None:
"""The row is freed and the lock dropped; ``checkpointed`` says whether
the KV went into the tree first. Drop per-request state kept outside the tree."""
def evictable_size(self):
return 0
+5 -5
View File
@@ -294,8 +294,8 @@ def discard_kv_cache_backup(
req.kv.retraction_backup = None
def release_kv_cache(req: Req, tree_cache: BasePrefixCache, is_insert: bool = True):
"""Give the request's kv row back; with ``is_insert`` the tree first keeps
def release_kv_cache(req: Req, tree_cache: BasePrefixCache, *, checkpoint: bool):
"""Give the request's kv row back; with ``checkpoint`` the tree first keeps
what it can key."""
assert (not req.kv.holds_kv) == req.kv.is_kv_released
# A mamba-capable cache may alloc mamba state before alloc KV cache
@@ -316,8 +316,8 @@ def release_kv_cache(req: Req, tree_cache: BasePrefixCache, is_insert: bool = Tr
return
owned_kv_len = req.owned_kv_len()
is_insert = is_insert and not req.skip_radix_cache_insert
if is_insert:
checkpoint = checkpoint and not req.skip_radix_cache_insert
if checkpoint:
# A tree that takes over component state (mamba) must see the request
# finished, or the insert forks the state and the slot leaks.
assert req.finished() or not tree_cache.supports_mamba(), (
@@ -332,7 +332,7 @@ def release_kv_cache(req: Req, tree_cache: BasePrefixCache, is_insert: bool = Tr
_release_overallocated_kv_indices(
req, owned_kv_len, req.kv.kv_allocated_len, tree_cache
)
tree_cache.on_release(req, inserted=is_insert)
tree_cache.on_release(req, checkpointed=checkpoint)
# The DSV4-NPU ReqToTokenPool subclass's free() additionally releases the
# c4/c128 state pages; other ReqToTokenPool subclasses are a no-op here.
@@ -385,8 +385,8 @@ class FlexKVRadixCache(RadixCache):
# checkpoint (STORE)
# ------------------------------------------------------------------
def on_release(self, req: Req, *, inserted: bool) -> None:
if not inserted:
def on_release(self, req: Req, *, checkpointed: bool) -> None:
if not checkpointed:
self._load_markers.pop(req.cache_request_handle, None)
def checkpoint(self, req: Req, *, up_to: int) -> None: # type: ignore[override]
@@ -248,9 +248,9 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
return True
def on_release(self, req: Req, *, inserted: bool) -> None:
super().on_release(req, inserted=inserted)
if not inserted:
def on_release(self, req: Req, *, checkpointed: bool) -> None:
super().on_release(req, checkpointed=checkpointed)
if not checkpointed:
self.release_aborted_request(req.cache_request_handle)
def checkpoint(self, req: Req, *, up_to: int, **kwargs) -> None:
@@ -237,7 +237,7 @@ Insert a request's KV into the tree: at every checkpoint while it runs, and once
| Aspect | Detail |
|--------|--------|
| **Purpose** | Publish the request's KV `[cache_protected_len, up_to)` so other requests can match it; nodes past `req.kv.cache_inserted_len` count one hit (a request counts each node once). `checkpoint_kv_cache` calls it with `req.extend_range.end` while the request runs; `release_kv_cache` calls it with the request-owned length when it finishes (`req.finished()`), then frees `[cache_protected_len, up_to)` and everything past it and unpins; with `is_insert=False` it skips the insert and calls `on_release(req, inserted=False)` for component cleanup |
| **Purpose** | Publish the request's KV `[cache_protected_len, up_to)` so other requests can match it; nodes past `req.kv.cache_inserted_len` count one hit (a request counts each node once). `checkpoint_kv_cache` calls it with `req.extend_range.end` while the request runs; `release_kv_cache` calls it with the request-owned length when it finishes (`req.finished()`), then frees `[cache_protected_len, up_to)` and everything past it and unpins; with `checkpoint=False` it skips the insert and calls `on_release(req, checkpointed=False)` for component cleanup |
| **Inputs** | `req` — the request; `up_to` — row position the insert may read up to |
| **Output** | `None` |
| **Mutation** | Component hooks → `insert` → re-match → writes the tree's indices back into the row → moves the request's lock from the old `req.last_node` to the node the insert ended on → updates `req.prefix_indices`, `req.kv.cache_protected_len`, `req.kv.cache_inserted_len`, `req.last_node` → component cleanup. A finished request's component state (Mamba) is handed to the tree instead of forked, and what it still held is freed. Frees no KV slot: `release_kv_cache` does that afterwards. |
@@ -1053,9 +1053,9 @@ class UnifiedRadixCache(BasePrefixCache):
self.cache_controller.release_pp_prefetch(req.rid)
return self.session.try_cache_finished_req(req)
@rank_consensus(same_params=["req.rid", "inserted"])
def on_release(self, req: Req, *, inserted: bool) -> None:
if inserted:
@rank_consensus(same_params=["req.rid", "checkpointed"])
def on_release(self, req: Req, *, checkpointed: bool) -> None:
if checkpointed:
return
for comp in self._components_tuple:
comp.cleanup_after_caching_req(req, is_finished=True)