[mem_cache] Rename the tree's request-level insert_req to checkpoint (#42202)
This commit is contained in:
@@ -2201,7 +2201,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
kv_loc,
|
||||
)
|
||||
|
||||
# Truncate fill_len to kv_committed_len so insert_req only
|
||||
# Truncate fill_len to kv_committed_len so checkpoint only
|
||||
# inserts committed KV into the radix tree. The last output token
|
||||
# hasn't had KV committed yet (output_ids is 1 ahead).
|
||||
req.full_untruncated_fill_ids = req.origin_input_ids + req.output_ids
|
||||
@@ -3125,7 +3125,7 @@ class SchedulerDisaggregationDecodeMixin:
|
||||
else:
|
||||
tree_cache = self.tree_cache
|
||||
req.init_next_round_input(tree_cache)
|
||||
# Truncate fill_len to kv_committed_len so insert_req
|
||||
# Truncate fill_len to kv_committed_len so checkpoint
|
||||
# only sees committed KV (full array includes one uncommitted
|
||||
# token because init_next_round_input rebuilt it as full).
|
||||
if req.kv.kv_committed_len is not None:
|
||||
|
||||
@@ -1220,7 +1220,7 @@ class Req(ReqDllmMixin):
|
||||
# Refreshed at every sharded alloc — read through last_node, or drawn
|
||||
# least-full for a new chain — and consumed by the radix insert to
|
||||
# stamp new tree nodes. Allocation itself must NOT read it back when
|
||||
# a tree node is available (the insert_req dedup rebind
|
||||
# a tree node is available (the checkpoint dedup rebind
|
||||
# would make it stale); the only allocation-time reader is the
|
||||
# ChunkCache fallback, which has no tree nodes and no rebind.
|
||||
self.kv_rotation_base: Optional[int] = None
|
||||
|
||||
@@ -255,7 +255,7 @@ def _kv_shard_rotation_bases(
|
||||
The owner class of position-page P is ``(b_i + P) % shard_size``. Rules:
|
||||
|
||||
- Read through ``req.last_node`` at alloc time, never a value cached on
|
||||
the request: a checkpoint ``insert_req`` can rebind a chunked request onto
|
||||
the request: a ``checkpoint`` can rebind a chunked request onto
|
||||
another chain's canonical locs between chunks, changing the base. The
|
||||
read goes through ``tree_cache.rotation_base_of`` because the node
|
||||
handle is tree-specific (a NodeId on the unified tree).
|
||||
|
||||
@@ -459,7 +459,7 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
|
||||
return None
|
||||
|
||||
@abstractmethod
|
||||
def insert_req(self, req: Req, *, up_to: int, **kwargs):
|
||||
def checkpoint(self, req: Req, *, up_to: int, **kwargs):
|
||||
"""Insert the request's KV up to row position ``up_to`` into the tree,
|
||||
repoint the row onto the tree's copy, re-anchor ``req.last_node`` on
|
||||
the node the insert ended on and advance ``cache_protected_len``.
|
||||
|
||||
@@ -76,7 +76,7 @@ class ChunkCache(BasePrefixCache):
|
||||
# ChunkCache does not support prefix caching, so insert is a no-op
|
||||
return InsertResult(prefix_len=0)
|
||||
|
||||
def insert_req(self, req: Req, *, up_to: int):
|
||||
def checkpoint(self, req: Req, *, up_to: int):
|
||||
kv_indices = self.req_to_token_pool.req_to_token[req.kv.req_pool_idx, :up_to]
|
||||
# `req.prefix_indices` will be used in `PrefillAdder::add_chunked_req` later
|
||||
req.prefix_indices = kv_indices.to(dtype=torch.int64, copy=True)
|
||||
|
||||
@@ -178,7 +178,7 @@ def checkpoint_kv_cache(req: Req, tree_cache: BasePrefixCache) -> None:
|
||||
if req.skip_radix_cache_insert:
|
||||
return
|
||||
|
||||
tree_cache.insert_req(req, up_to=req.extend_range.end)
|
||||
tree_cache.checkpoint(req, up_to=req.extend_range.end)
|
||||
|
||||
|
||||
def evict_from_tree_cache(
|
||||
@@ -325,7 +325,7 @@ def release_kv_cache(req: Req, tree_cache: BasePrefixCache, is_insert: bool = Tr
|
||||
)
|
||||
# The fill-id array lags output_ids until the next prepare_for_decode.
|
||||
req.refresh_fill_ids()
|
||||
tree_cache.insert_req(req, up_to=owned_kv_len)
|
||||
tree_cache.checkpoint(req, up_to=owned_kv_len)
|
||||
# The protected prefix is not this req's to free.
|
||||
tree_cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, owned_kv_len)])
|
||||
tree_cache.unpin(req)
|
||||
|
||||
@@ -1051,8 +1051,8 @@ class MambaPool:
|
||||
ReplaySSM invariant: the SOURCE must be a fully-flushed checkpoint
|
||||
(``write_pos[src] == 0``). Only ``temporal`` is copied, not the ring, so
|
||||
an un-flushed source would drop its last ``write_pos`` updates. Callers
|
||||
comply: COW copies radix checkpoints; a checkpoint ``insert_req`` copies an
|
||||
active slot only during prefill (ring empty); ``insert_req``
|
||||
comply: COW copies radix checkpoints; a ``checkpoint`` copies an
|
||||
active slot only during prefill (ring empty); ``checkpoint``
|
||||
caps the donate to the last flush boundary. The dst cursor is reset to 0
|
||||
(the copied checkpoint has no pending ring entries).
|
||||
"""
|
||||
|
||||
@@ -494,7 +494,7 @@ class RadixCache(BasePrefixCache):
|
||||
)
|
||||
return radix_key, kv_indices, result.prefix_len
|
||||
|
||||
def insert_req(self, req: Req, *, up_to: int):
|
||||
def checkpoint(self, req: Req, *, up_to: int):
|
||||
if self.disable:
|
||||
return
|
||||
token_ids = req.full_untruncated_fill_ids[:up_to]
|
||||
@@ -524,7 +524,7 @@ class RadixCache(BasePrefixCache):
|
||||
|
||||
# With page_size > 1 the partial page sits in req.prefix_indices but not
|
||||
# in the tree; cache_protected_len marks the tree-owned part so the next
|
||||
# insert_req or release_kv_cache frees the rest.
|
||||
# checkpoint or release_kv_cache frees the rest.
|
||||
req.kv.cache_protected_len = len(new_indices)
|
||||
|
||||
self.dec_lock_ref(req.last_node)
|
||||
|
||||
@@ -382,16 +382,16 @@ class FlexKVRadixCache(RadixCache):
|
||||
return fetched_slots, new_node
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# insert_req (STORE)
|
||||
# checkpoint (STORE)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def on_release(self, req: Req, *, inserted: bool) -> None:
|
||||
if not inserted:
|
||||
self._load_markers.pop(req.cache_request_handle, None)
|
||||
|
||||
def insert_req(self, req: Req, *, up_to: int) -> None: # type: ignore[override]
|
||||
"""Base insert_req; a finished request also fires an async FlexKV store."""
|
||||
super().insert_req(req, up_to=up_to)
|
||||
def checkpoint(self, req: Req, *, up_to: int) -> None: # type: ignore[override]
|
||||
"""Base checkpoint; a finished request also fires an async FlexKV store."""
|
||||
super().checkpoint(req, up_to=up_to)
|
||||
if not req.finished():
|
||||
return
|
||||
|
||||
|
||||
@@ -253,9 +253,9 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
|
||||
if not inserted:
|
||||
self.release_aborted_request(req.cache_request_handle)
|
||||
|
||||
def insert_req(self, req: Req, *, up_to: int, **kwargs) -> None:
|
||||
def checkpoint(self, req: Req, *, up_to: int, **kwargs) -> None:
|
||||
self._publish_external_loaded_prefix(req, token_ids_len=up_to)
|
||||
super().insert_req(req, up_to=up_to, **kwargs)
|
||||
super().checkpoint(req, up_to=up_to, **kwargs)
|
||||
self._retire_loaded_flow(req.rid)
|
||||
self._submit_store(req, req.full_untruncated_fill_ids[:up_to])
|
||||
if req.finished():
|
||||
|
||||
@@ -228,7 +228,7 @@ receipt proves were taken. The eventual full release must pass
|
||||
|
||||
---
|
||||
|
||||
### `insert_req(req: Req, *, up_to: int)`
|
||||
### `checkpoint(req: Req, *, up_to: int)`
|
||||
|
||||
Insert a request's KV into the tree: at every checkpoint while it runs, and once more when it finishes.
|
||||
|
||||
|
||||
@@ -1003,8 +1003,8 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
comp.cleanup_after_caching_req(req, is_finished=True)
|
||||
|
||||
@rank_consensus(same_params=["req.rid", "up_to"])
|
||||
def insert_req(self, req: Req, *, up_to: int, **kwargs) -> None:
|
||||
if self.session.try_insert_req(req, up_to=up_to, **kwargs):
|
||||
def checkpoint(self, req: Req, *, up_to: int, **kwargs) -> None:
|
||||
if self.session.try_checkpoint(req, up_to=up_to, **kwargs):
|
||||
return
|
||||
# A finished request hands its component state (mamba) to the tree
|
||||
# instead of forking it, and the tree frees what the request still held.
|
||||
|
||||
@@ -325,7 +325,7 @@ class StreamingSession(BasePrefixCache):
|
||||
|
||||
return True
|
||||
|
||||
def try_insert_req(self, req: Req, *, up_to: int, **kwargs) -> bool:
|
||||
def try_checkpoint(self, req: Req, *, up_to: int, **kwargs) -> bool:
|
||||
"""A first turn checkpoints into the tree like any request (its
|
||||
prompt prefix is tree-owned and the slot inherits that lock); later
|
||||
turns run on the slot's KV, so only the chunk cursor is kept."""
|
||||
@@ -349,10 +349,10 @@ class StreamingSession(BasePrefixCache):
|
||||
def on_release(self, req: Req, *, inserted: bool) -> None:
|
||||
self.inner.on_release(req, inserted=inserted)
|
||||
|
||||
def insert_req(self, req: Req, **kwargs):
|
||||
if self.try_insert_req(req, **kwargs):
|
||||
def checkpoint(self, req: Req, **kwargs):
|
||||
if self.try_checkpoint(req, **kwargs):
|
||||
return
|
||||
self.inner.insert_req(req, **kwargs)
|
||||
self.inner.checkpoint(req, **kwargs)
|
||||
|
||||
def unpin(self, req: Req) -> None:
|
||||
self.inner.unpin(req)
|
||||
|
||||
@@ -4727,7 +4727,7 @@ def get_extend_input_len_swa_limit(
|
||||
sliding_window_size: int, chunked_prefill_size: int, page_size: int
|
||||
) -> int:
|
||||
# 1. a factor of 2x is because each prefill contains chunked_prefill_size tokens,
|
||||
# and between prefills, we run the tree cache's insert_req(),
|
||||
# and between prefills, we run the tree cache's checkpoint(),
|
||||
# so we unlock the previously locked nodes.
|
||||
# 2. max is to handle the case that chunked_prefill_size is larger than sliding_window_size.
|
||||
# in that case, each prefill contains chunked_prefill_size tokens,
|
||||
|
||||
@@ -5,10 +5,10 @@ from sglang.srt.managers.schedule_batch import FINISH_LENGTH
|
||||
|
||||
def finish_req(cache, req, up_to):
|
||||
"""What release_kv_cache does for a request that keeps its KV: mark it
|
||||
finished, insert_req, free the rest, drop the lock."""
|
||||
finished, checkpoint, free the rest, drop the lock."""
|
||||
if getattr(req, "finished_reason", None) is None:
|
||||
req.finished_reason = FINISH_LENGTH(length=0)
|
||||
req.refresh_fill_ids()
|
||||
cache.insert_req(req, up_to=up_to)
|
||||
cache.checkpoint(req, up_to=up_to)
|
||||
cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, up_to)])
|
||||
cache.unpin(req)
|
||||
|
||||
Reference in New Issue
Block a user