[mem_cache] Rename the tree's request-level insert_req to checkpoint (#42202)

This commit is contained in:
Liangsheng Yin
2026-10-02 14:42:35 -07:00
committed by GitHub
parent 2ec9e3b11d
commit 8a328e867b
31 changed files with 71 additions and 71 deletions
+2 -2
View File
@@ -2201,7 +2201,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
kv_loc,
)
# Truncate fill_len to kv_committed_len so insert_req only
# Truncate fill_len to kv_committed_len so checkpoint only
# inserts committed KV into the radix tree. The last output token
# hasn't had KV committed yet (output_ids is 1 ahead).
req.full_untruncated_fill_ids = req.origin_input_ids + req.output_ids
@@ -3125,7 +3125,7 @@ class SchedulerDisaggregationDecodeMixin:
else:
tree_cache = self.tree_cache
req.init_next_round_input(tree_cache)
# Truncate fill_len to kv_committed_len so insert_req
# Truncate fill_len to kv_committed_len so checkpoint
# only sees committed KV (full array includes one uncommitted
# token because init_next_round_input rebuilt it as full).
if req.kv.kv_committed_len is not None:
+1 -1
View File
@@ -1220,7 +1220,7 @@ class Req(ReqDllmMixin):
# Refreshed at every sharded alloc — read through last_node, or drawn
# least-full for a new chain — and consumed by the radix insert to
# stamp new tree nodes. Allocation itself must NOT read it back when
# a tree node is available (the insert_req dedup rebind
# a tree node is available (the checkpoint dedup rebind
# would make it stale); the only allocation-time reader is the
# ChunkCache fallback, which has no tree nodes and no rebind.
self.kv_rotation_base: Optional[int] = None
+1 -1
View File
@@ -255,7 +255,7 @@ def _kv_shard_rotation_bases(
The owner class of position-page P is ``(b_i + P) % shard_size``. Rules:
- Read through ``req.last_node`` at alloc time, never a value cached on
the request: a checkpoint ``insert_req`` can rebind a chunked request onto
the request: a ``checkpoint`` can rebind a chunked request onto
another chain's canonical locs between chunks, changing the base. The
read goes through ``tree_cache.rotation_base_of`` because the node
handle is tree-specific (a NodeId on the unified tree).
@@ -459,7 +459,7 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
return None
@abstractmethod
def insert_req(self, req: Req, *, up_to: int, **kwargs):
def checkpoint(self, req: Req, *, up_to: int, **kwargs):
"""Insert the request's KV up to row position ``up_to`` into the tree,
repoint the row onto the tree's copy, re-anchor ``req.last_node`` on
the node the insert ended on and advance ``cache_protected_len``.
+1 -1
View File
@@ -76,7 +76,7 @@ class ChunkCache(BasePrefixCache):
# ChunkCache does not support prefix caching, so insert is a no-op
return InsertResult(prefix_len=0)
def insert_req(self, req: Req, *, up_to: int):
def checkpoint(self, req: Req, *, up_to: int):
kv_indices = self.req_to_token_pool.req_to_token[req.kv.req_pool_idx, :up_to]
# `req.prefix_indices` will be used in `PrefillAdder::add_chunked_req` later
req.prefix_indices = kv_indices.to(dtype=torch.int64, copy=True)
+2 -2
View File
@@ -178,7 +178,7 @@ def checkpoint_kv_cache(req: Req, tree_cache: BasePrefixCache) -> None:
if req.skip_radix_cache_insert:
return
tree_cache.insert_req(req, up_to=req.extend_range.end)
tree_cache.checkpoint(req, up_to=req.extend_range.end)
def evict_from_tree_cache(
@@ -325,7 +325,7 @@ def release_kv_cache(req: Req, tree_cache: BasePrefixCache, is_insert: bool = Tr
)
# The fill-id array lags output_ids until the next prepare_for_decode.
req.refresh_fill_ids()
tree_cache.insert_req(req, up_to=owned_kv_len)
tree_cache.checkpoint(req, up_to=owned_kv_len)
# The protected prefix is not this req's to free.
tree_cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, owned_kv_len)])
tree_cache.unpin(req)
+2 -2
View File
@@ -1051,8 +1051,8 @@ class MambaPool:
ReplaySSM invariant: the SOURCE must be a fully-flushed checkpoint
(``write_pos[src] == 0``). Only ``temporal`` is copied, not the ring, so
an un-flushed source would drop its last ``write_pos`` updates. Callers
comply: COW copies radix checkpoints; a checkpoint ``insert_req`` copies an
active slot only during prefill (ring empty); ``insert_req``
comply: COW copies radix checkpoints; a ``checkpoint`` copies an
active slot only during prefill (ring empty); ``checkpoint``
caps the donate to the last flush boundary. The dst cursor is reset to 0
(the copied checkpoint has no pending ring entries).
"""
+2 -2
View File
@@ -494,7 +494,7 @@ class RadixCache(BasePrefixCache):
)
return radix_key, kv_indices, result.prefix_len
def insert_req(self, req: Req, *, up_to: int):
def checkpoint(self, req: Req, *, up_to: int):
if self.disable:
return
token_ids = req.full_untruncated_fill_ids[:up_to]
@@ -524,7 +524,7 @@ class RadixCache(BasePrefixCache):
# With page_size > 1 the partial page sits in req.prefix_indices but not
# in the tree; cache_protected_len marks the tree-owned part so the next
# insert_req or release_kv_cache frees the rest.
# checkpoint or release_kv_cache frees the rest.
req.kv.cache_protected_len = len(new_indices)
self.dec_lock_ref(req.last_node)
@@ -382,16 +382,16 @@ class FlexKVRadixCache(RadixCache):
return fetched_slots, new_node
# ------------------------------------------------------------------
# insert_req (STORE)
# checkpoint (STORE)
# ------------------------------------------------------------------
def on_release(self, req: Req, *, inserted: bool) -> None:
if not inserted:
self._load_markers.pop(req.cache_request_handle, None)
def insert_req(self, req: Req, *, up_to: int) -> None: # type: ignore[override]
"""Base insert_req; a finished request also fires an async FlexKV store."""
super().insert_req(req, up_to=up_to)
def checkpoint(self, req: Req, *, up_to: int) -> None: # type: ignore[override]
"""Base checkpoint; a finished request also fires an async FlexKV store."""
super().checkpoint(req, up_to=up_to)
if not req.finished():
return
@@ -253,9 +253,9 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
if not inserted:
self.release_aborted_request(req.cache_request_handle)
def insert_req(self, req: Req, *, up_to: int, **kwargs) -> None:
def checkpoint(self, req: Req, *, up_to: int, **kwargs) -> None:
self._publish_external_loaded_prefix(req, token_ids_len=up_to)
super().insert_req(req, up_to=up_to, **kwargs)
super().checkpoint(req, up_to=up_to, **kwargs)
self._retire_loaded_flow(req.rid)
self._submit_store(req, req.full_untruncated_fill_ids[:up_to])
if req.finished():
@@ -228,7 +228,7 @@ receipt proves were taken. The eventual full release must pass
---
### `insert_req(req: Req, *, up_to: int)`
### `checkpoint(req: Req, *, up_to: int)`
Insert a request's KV into the tree: at every checkpoint while it runs, and once more when it finishes.
@@ -1003,8 +1003,8 @@ class UnifiedRadixCache(BasePrefixCache):
comp.cleanup_after_caching_req(req, is_finished=True)
@rank_consensus(same_params=["req.rid", "up_to"])
def insert_req(self, req: Req, *, up_to: int, **kwargs) -> None:
if self.session.try_insert_req(req, up_to=up_to, **kwargs):
def checkpoint(self, req: Req, *, up_to: int, **kwargs) -> None:
if self.session.try_checkpoint(req, up_to=up_to, **kwargs):
return
# A finished request hands its component state (mamba) to the tree
# instead of forking it, and the tree frees what the request still held.
@@ -325,7 +325,7 @@ class StreamingSession(BasePrefixCache):
return True
def try_insert_req(self, req: Req, *, up_to: int, **kwargs) -> bool:
def try_checkpoint(self, req: Req, *, up_to: int, **kwargs) -> bool:
"""A first turn checkpoints into the tree like any request (its
prompt prefix is tree-owned and the slot inherits that lock); later
turns run on the slot's KV, so only the chunk cursor is kept."""
@@ -349,10 +349,10 @@ class StreamingSession(BasePrefixCache):
def on_release(self, req: Req, *, inserted: bool) -> None:
self.inner.on_release(req, inserted=inserted)
def insert_req(self, req: Req, **kwargs):
if self.try_insert_req(req, **kwargs):
def checkpoint(self, req: Req, **kwargs):
if self.try_checkpoint(req, **kwargs):
return
self.inner.insert_req(req, **kwargs)
self.inner.checkpoint(req, **kwargs)
def unpin(self, req: Req) -> None:
self.inner.unpin(req)
+1 -1
View File
@@ -4727,7 +4727,7 @@ def get_extend_input_len_swa_limit(
sliding_window_size: int, chunked_prefill_size: int, page_size: int
) -> int:
# 1. a factor of 2x is because each prefill contains chunked_prefill_size tokens,
# and between prefills, we run the tree cache's insert_req(),
# and between prefills, we run the tree cache's checkpoint(),
# so we unlock the previously locked nodes.
# 2. max is to handle the case that chunked_prefill_size is larger than sliding_window_size.
# in that case, each prefill contains chunked_prefill_size tokens,
+2 -2
View File
@@ -5,10 +5,10 @@ from sglang.srt.managers.schedule_batch import FINISH_LENGTH
def finish_req(cache, req, up_to):
"""What release_kv_cache does for a request that keeps its KV: mark it
finished, insert_req, free the rest, drop the lock."""
finished, checkpoint, free the rest, drop the lock."""
if getattr(req, "finished_reason", None) is None:
req.finished_reason = FINISH_LENGTH(length=0)
req.refresh_fill_ids()
cache.insert_req(req, up_to=up_to)
cache.checkpoint(req, up_to=up_to)
cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, up_to)])
cache.unpin(req)