[mem_cache] Make release_kv_cache's final checkpoint an explicit checkpoint flag (#42270)
This commit is contained in:
@@ -1719,7 +1719,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
]
|
||||
for decode_req in failed_reqs:
|
||||
if decode_req.req.kv.holds_mamba and not decode_req.req.kv.holds_kv:
|
||||
release_kv_cache(decode_req.req, self.tree_cache, is_insert=False)
|
||||
release_kv_cache(decode_req.req, self.tree_cache, checkpoint=False)
|
||||
|
||||
self.queue = [
|
||||
entry for i, entry in enumerate(self.queue) if i not in indices_to_remove
|
||||
@@ -2622,7 +2622,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
discard_kv_cache_backup(decode_req.req, self.tree_cache, "host_pool")
|
||||
if decode_req.host_staged:
|
||||
return
|
||||
release_kv_cache(decode_req.req, self.tree_cache, is_insert=False)
|
||||
release_kv_cache(decode_req.req, self.tree_cache, checkpoint=False)
|
||||
|
||||
def pop_transferred(self, rids_to_check: Optional[List[str]] = None) -> List[Req]:
|
||||
if not self.queue:
|
||||
@@ -3126,7 +3126,7 @@ class SchedulerDisaggregationDecodeMixin:
|
||||
status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
self.output_streamer.stream_output([req], req.return_logprob)
|
||||
release_kv_cache(req, self.tree_cache, is_insert=False)
|
||||
release_kv_cache(req, self.tree_cache, checkpoint=False)
|
||||
if self.metrics_reporter.enable_metrics:
|
||||
self.metrics_collector.increment_transfer_failed_reqs()
|
||||
|
||||
|
||||
@@ -604,7 +604,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
sender.abort()
|
||||
maybe_release_metadata_buffer(req, self.req_to_metadata_buffer_idx_allocator)
|
||||
if req.kv.holds_kv or req.kv.holds_mamba:
|
||||
release_kv_cache(req, self.tree_cache, is_insert=False)
|
||||
release_kv_cache(req, self.tree_cache, checkpoint=False)
|
||||
req.pending_bootstrap = False
|
||||
|
||||
@scheduler_stage_method(SCHEDULER_STAGE_PROCESS_QUEUE)
|
||||
@@ -1185,7 +1185,8 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
elif poll == KVPoll.Success: # transfer done
|
||||
if not isinstance(req.finished_reason, FINISH_ABORT):
|
||||
req.finished_reason = FINISH_LENGTH(length=0)
|
||||
release_kv_cache(req, self.tree_cache) # unlock the tree
|
||||
# unlock the tree
|
||||
release_kv_cache(req, self.tree_cache, checkpoint=True)
|
||||
self.tree_cache.finish(
|
||||
req.cache_request_handle, CacheRequestOutcome.SUCCESS
|
||||
)
|
||||
@@ -1261,7 +1262,8 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
req.time_stats.trace_ctx.abort(abort_info={"reason": error_message})
|
||||
if req.finished_reason is None:
|
||||
req.finished_reason = FINISH_LENGTH(length=0)
|
||||
release_kv_cache(req, self.tree_cache) # unlock the tree
|
||||
# unlock the tree
|
||||
release_kv_cache(req, self.tree_cache, checkpoint=True)
|
||||
self._release_aborted_request(req)
|
||||
if not isinstance(req.finished_reason, FINISH_ABORT):
|
||||
prepare_abort(
|
||||
@@ -1305,7 +1307,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
req.pending_bootstrap = False
|
||||
self.tree_cache.finish(req.cache_request_handle, CacheRequestOutcome.ABORT)
|
||||
if req.kv.holds_kv or req.kv.holds_mamba:
|
||||
release_kv_cache(req, self.tree_cache, is_insert=False)
|
||||
release_kv_cache(req, self.tree_cache, checkpoint=False)
|
||||
return True
|
||||
|
||||
def handle_bootstrap_failure(self: Scheduler, req: Req) -> None:
|
||||
@@ -1327,7 +1329,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
logger.warning(error_message)
|
||||
req.time_stats.trace_ctx.abort(abort_info={"reason": error_message})
|
||||
if req.kv.holds_kv or req.kv.holds_mamba:
|
||||
release_kv_cache(req, self.tree_cache, is_insert=False)
|
||||
release_kv_cache(req, self.tree_cache, checkpoint=False)
|
||||
maybe_release_metadata_buffer(req, self.req_to_metadata_buffer_idx_allocator)
|
||||
req.pending_bootstrap = False
|
||||
prepare_abort(req, error_message, status_code=HTTPStatus.INTERNAL_SERVER_ERROR)
|
||||
@@ -1696,7 +1698,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
self._release_aborted_request(req)
|
||||
# The checkpoint above already handed the prefill KV to the tree; the
|
||||
# request is not finished, so the release only frees the rest.
|
||||
release_kv_cache(req, self.tree_cache, is_insert=False)
|
||||
release_kv_cache(req, self.tree_cache, checkpoint=False)
|
||||
req.reset_for_retract()
|
||||
req.output_ids = array("q")
|
||||
req.start_send_idx = 0
|
||||
|
||||
@@ -179,7 +179,7 @@ class SchedulerDllmMixin:
|
||||
release_kv_cache(
|
||||
req,
|
||||
self.tree_cache,
|
||||
is_insert=not self.dllm_config.requires_separate_context_encoding,
|
||||
checkpoint=not self.dllm_config.requires_separate_context_encoding,
|
||||
)
|
||||
req.time_stats.set_completion_time()
|
||||
|
||||
|
||||
@@ -1137,7 +1137,7 @@ class Req(ReqDllmMixin):
|
||||
|
||||
# Lazy extra buffer: skip radix cache insert when prealloc failed at
|
||||
# boundary — the forward overwrites the only slot, corrupting the state.
|
||||
self.mamba_lazy_is_insert: bool = True
|
||||
self.mamba_lazy_checkpoint: bool = True
|
||||
|
||||
# Check finish
|
||||
self.tokenizer = None
|
||||
@@ -2290,7 +2290,7 @@ def release_req(
|
||||
get_disagg().disaggregation_decode_retraction_backup,
|
||||
)
|
||||
# TODO (csy): for preempted requests, we may want to insert into the tree
|
||||
release_kv_cache(req, tree_cache, is_insert=False)
|
||||
release_kv_cache(req, tree_cache, checkpoint=False)
|
||||
# NOTE(lsyin): we should use the newly evictable memory instantly.
|
||||
num_tokens = remaing_req_count * envs.SGLANG_RETRACT_DECODE_STEPS.get()
|
||||
evict_from_tree_cache(tree_cache, num_tokens)
|
||||
|
||||
@@ -3578,7 +3578,7 @@ class Scheduler(
|
||||
)
|
||||
req.pending_bootstrap = False
|
||||
self._release_aborted_request(req)
|
||||
release_kv_cache(req, self.tree_cache, is_insert=False)
|
||||
release_kv_cache(req, self.tree_cache, checkpoint=False)
|
||||
|
||||
self.chunked_req = None
|
||||
self._pending_chunked_abort_req = None
|
||||
@@ -5335,7 +5335,7 @@ class Scheduler(
|
||||
self.hisparse_coordinator.request_finished(req)
|
||||
if req.finished_reason is None:
|
||||
req.finished_reason = FINISH_ABORT()
|
||||
release_kv_cache(req, self.tree_cache)
|
||||
release_kv_cache(req, self.tree_cache, checkpoint=True)
|
||||
if self.disaggregation_mode == DisaggregationMode.PREFILL:
|
||||
self.release_aborted_prefill_waiting_req(req)
|
||||
|
||||
@@ -5344,7 +5344,7 @@ class Scheduler(
|
||||
DisaggregationMode.PREFILL,
|
||||
DisaggregationMode.DECODE,
|
||||
):
|
||||
release_kv_cache(req, self.tree_cache, is_insert=False)
|
||||
release_kv_cache(req, self.tree_cache, checkpoint=False)
|
||||
logger.debug(f"Abort queued request. {req.rid=}")
|
||||
|
||||
if self.dllm_config is not None:
|
||||
@@ -5356,7 +5356,7 @@ class Scheduler(
|
||||
_make_abort_req(req), req
|
||||
)
|
||||
if req.kv.holds_kv or req.kv.holds_mamba:
|
||||
release_kv_cache(req, self.tree_cache, is_insert=False)
|
||||
release_kv_cache(req, self.tree_cache, checkpoint=False)
|
||||
logger.debug(f"Abort dLLM queued request. {req.rid=}")
|
||||
|
||||
# Delete the requests in the grammar queue
|
||||
|
||||
@@ -126,7 +126,7 @@ class SchedulerBatchResultProcessor:
|
||||
req.time_stats.set_quick_finish_time()
|
||||
if get_memory().enable_hisparse:
|
||||
self.hisparse_coordinator.request_finished(req)
|
||||
release_kv_cache(req, self.tree_cache)
|
||||
release_kv_cache(req, self.tree_cache, checkpoint=True)
|
||||
|
||||
# Note: Logprobs should be handled on the prefill engine.
|
||||
self.output_streamer.stream_output(batch.reqs, batch.return_logprob)
|
||||
@@ -377,7 +377,7 @@ class SchedulerBatchResultProcessor:
|
||||
release_kv_cache(
|
||||
req,
|
||||
self.tree_cache,
|
||||
is_insert=sampling_mask_finish_reason is None,
|
||||
checkpoint=sampling_mask_finish_reason is None,
|
||||
)
|
||||
req.time_stats.set_completion_time()
|
||||
elif not batch.decoding_reqs or req not in batch.decoding_reqs:
|
||||
@@ -469,7 +469,7 @@ class SchedulerBatchResultProcessor:
|
||||
req.update_finish_state()
|
||||
|
||||
if req.finished():
|
||||
release_kv_cache(req, self.tree_cache)
|
||||
release_kv_cache(req, self.tree_cache, checkpoint=True)
|
||||
req.time_stats.set_completion_time()
|
||||
else:
|
||||
checkpoint_kv_cache(req, self.tree_cache)
|
||||
@@ -1337,7 +1337,7 @@ class SchedulerBatchResultProcessor:
|
||||
)
|
||||
if callable(prepare_release):
|
||||
prepare_release(req)
|
||||
release_kv_cache(req, self.tree_cache, is_insert=False)
|
||||
release_kv_cache(req, self.tree_cache, checkpoint=False)
|
||||
req.time_stats.set_completion_time()
|
||||
|
||||
def _handle_finish_state_updated_req(
|
||||
@@ -1377,7 +1377,7 @@ class SchedulerBatchResultProcessor:
|
||||
and known_mamba_boundary
|
||||
and req.kv.mamba_next_track_idx == req.kv.mamba_last_track_idx
|
||||
):
|
||||
req.mamba_lazy_is_insert = False
|
||||
req.mamba_lazy_checkpoint = False
|
||||
|
||||
# Called here (after update_finish_state) so req.finished() is valid
|
||||
# for mamba_lazy_post_decode_at_boundary inside.
|
||||
@@ -1424,12 +1424,12 @@ class SchedulerBatchResultProcessor:
|
||||
)
|
||||
if callable(prepare_release):
|
||||
prepare_release(req)
|
||||
is_insert = (
|
||||
req.mamba_lazy_is_insert
|
||||
checkpoint = (
|
||||
req.mamba_lazy_checkpoint
|
||||
if get_exec().mamba.enable_mamba_extra_buffer_lazy
|
||||
else True
|
||||
)
|
||||
release_kv_cache(req, self.tree_cache, is_insert=is_insert)
|
||||
release_kv_cache(req, self.tree_cache, checkpoint=checkpoint)
|
||||
|
||||
req.time_stats.set_completion_time()
|
||||
|
||||
@@ -1552,7 +1552,7 @@ class SchedulerBatchResultProcessor:
|
||||
or keep_written_by_this_step
|
||||
or keep_may_be_written_in_flight
|
||||
):
|
||||
req.mamba_lazy_is_insert = False
|
||||
req.mamba_lazy_checkpoint = False
|
||||
return
|
||||
|
||||
if not crossed or planned_pos is None:
|
||||
|
||||
@@ -259,7 +259,7 @@ class DynamicChunkSizer:
|
||||
|
||||
# Release KV and Mamba cache
|
||||
if req.kv.holds_kv:
|
||||
release_kv_cache(req, self.tree_cache, is_insert=False)
|
||||
release_kv_cache(req, self.tree_cache, checkpoint=False)
|
||||
|
||||
logger.info(
|
||||
f"[PP Dynamic Chunk] [PP0] Profiled {len(seq_lens)} samples: "
|
||||
|
||||
@@ -525,9 +525,9 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
|
||||
Return True after taking the row; the caller then releases nothing."""
|
||||
return False
|
||||
|
||||
def on_release(self, req: Req, *, inserted: bool) -> None:
|
||||
"""The row is freed and the lock dropped; ``inserted`` says whether the
|
||||
KV went into the tree first. Drop per-request state kept outside the tree."""
|
||||
def on_release(self, req: Req, *, checkpointed: bool) -> None:
|
||||
"""The row is freed and the lock dropped; ``checkpointed`` says whether
|
||||
the KV went into the tree first. Drop per-request state kept outside the tree."""
|
||||
|
||||
def evictable_size(self):
|
||||
return 0
|
||||
|
||||
@@ -294,8 +294,8 @@ def discard_kv_cache_backup(
|
||||
req.kv.retraction_backup = None
|
||||
|
||||
|
||||
def release_kv_cache(req: Req, tree_cache: BasePrefixCache, is_insert: bool = True):
|
||||
"""Give the request's kv row back; with ``is_insert`` the tree first keeps
|
||||
def release_kv_cache(req: Req, tree_cache: BasePrefixCache, *, checkpoint: bool):
|
||||
"""Give the request's kv row back; with ``checkpoint`` the tree first keeps
|
||||
what it can key."""
|
||||
assert (not req.kv.holds_kv) == req.kv.is_kv_released
|
||||
# A mamba-capable cache may alloc mamba state before alloc KV cache
|
||||
@@ -316,8 +316,8 @@ def release_kv_cache(req: Req, tree_cache: BasePrefixCache, is_insert: bool = Tr
|
||||
return
|
||||
|
||||
owned_kv_len = req.owned_kv_len()
|
||||
is_insert = is_insert and not req.skip_radix_cache_insert
|
||||
if is_insert:
|
||||
checkpoint = checkpoint and not req.skip_radix_cache_insert
|
||||
if checkpoint:
|
||||
# A tree that takes over component state (mamba) must see the request
|
||||
# finished, or the insert forks the state and the slot leaks.
|
||||
assert req.finished() or not tree_cache.supports_mamba(), (
|
||||
@@ -332,7 +332,7 @@ def release_kv_cache(req: Req, tree_cache: BasePrefixCache, is_insert: bool = Tr
|
||||
_release_overallocated_kv_indices(
|
||||
req, owned_kv_len, req.kv.kv_allocated_len, tree_cache
|
||||
)
|
||||
tree_cache.on_release(req, inserted=is_insert)
|
||||
tree_cache.on_release(req, checkpointed=checkpoint)
|
||||
|
||||
# The DSV4-NPU ReqToTokenPool subclass's free() additionally releases the
|
||||
# c4/c128 state pages; other ReqToTokenPool subclasses are a no-op here.
|
||||
|
||||
@@ -385,8 +385,8 @@ class FlexKVRadixCache(RadixCache):
|
||||
# checkpoint (STORE)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def on_release(self, req: Req, *, inserted: bool) -> None:
|
||||
if not inserted:
|
||||
def on_release(self, req: Req, *, checkpointed: bool) -> None:
|
||||
if not checkpointed:
|
||||
self._load_markers.pop(req.cache_request_handle, None)
|
||||
|
||||
def checkpoint(self, req: Req, *, up_to: int) -> None: # type: ignore[override]
|
||||
|
||||
@@ -248,9 +248,9 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
|
||||
|
||||
return True
|
||||
|
||||
def on_release(self, req: Req, *, inserted: bool) -> None:
|
||||
super().on_release(req, inserted=inserted)
|
||||
if not inserted:
|
||||
def on_release(self, req: Req, *, checkpointed: bool) -> None:
|
||||
super().on_release(req, checkpointed=checkpointed)
|
||||
if not checkpointed:
|
||||
self.release_aborted_request(req.cache_request_handle)
|
||||
|
||||
def checkpoint(self, req: Req, *, up_to: int, **kwargs) -> None:
|
||||
|
||||
@@ -237,7 +237,7 @@ Insert a request's KV into the tree: at every checkpoint while it runs, and once
|
||||
|
||||
| Aspect | Detail |
|
||||
|--------|--------|
|
||||
| **Purpose** | Publish the request's KV `[cache_protected_len, up_to)` so other requests can match it; nodes past `req.kv.cache_inserted_len` count one hit (a request counts each node once). `checkpoint_kv_cache` calls it with `req.extend_range.end` while the request runs; `release_kv_cache` calls it with the request-owned length when it finishes (`req.finished()`), then frees `[cache_protected_len, up_to)` and everything past it and unpins; with `is_insert=False` it skips the insert and calls `on_release(req, inserted=False)` for component cleanup |
|
||||
| **Purpose** | Publish the request's KV `[cache_protected_len, up_to)` so other requests can match it; nodes past `req.kv.cache_inserted_len` count one hit (a request counts each node once). `checkpoint_kv_cache` calls it with `req.extend_range.end` while the request runs; `release_kv_cache` calls it with the request-owned length when it finishes (`req.finished()`), then frees `[cache_protected_len, up_to)` and everything past it and unpins; with `checkpoint=False` it skips the insert and calls `on_release(req, checkpointed=False)` for component cleanup |
|
||||
| **Inputs** | `req` — the request; `up_to` — row position the insert may read up to |
|
||||
| **Output** | `None` |
|
||||
| **Mutation** | Component hooks → `insert` → re-match → writes the tree's indices back into the row → moves the request's lock from the old `req.last_node` to the node the insert ended on → updates `req.prefix_indices`, `req.kv.cache_protected_len`, `req.kv.cache_inserted_len`, `req.last_node` → component cleanup. A finished request's component state (Mamba) is handed to the tree instead of forked, and what it still held is freed. Frees no KV slot: `release_kv_cache` does that afterwards. |
|
||||
|
||||
@@ -1053,9 +1053,9 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
self.cache_controller.release_pp_prefetch(req.rid)
|
||||
return self.session.try_cache_finished_req(req)
|
||||
|
||||
@rank_consensus(same_params=["req.rid", "inserted"])
|
||||
def on_release(self, req: Req, *, inserted: bool) -> None:
|
||||
if inserted:
|
||||
@rank_consensus(same_params=["req.rid", "checkpointed"])
|
||||
def on_release(self, req: Req, *, checkpointed: bool) -> None:
|
||||
if checkpointed:
|
||||
return
|
||||
for comp in self._components_tuple:
|
||||
comp.cleanup_after_caching_req(req, is_finished=True)
|
||||
|
||||
Reference in New Issue
Block a user