[mem_cache] Rename cache_unfinished_req to checkpoint_req and count cache hits only when a request finishes (#41520)
This commit is contained in:
@@ -150,6 +150,7 @@ def _bind_root_prefix(req: Req, tree_cache: BasePrefixCache) -> None:
|
||||
req.best_match_node = req.last_node
|
||||
req.lock_receipt = DecLockRefParams()
|
||||
req.kv.cache_protected_len = 0
|
||||
req.kv.cache_inserted_len = 0
|
||||
req.num_matched_prefix_tokens = 0
|
||||
req.host_hit_length = 0
|
||||
|
||||
@@ -2200,7 +2201,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
kv_loc,
|
||||
)
|
||||
|
||||
# Truncate fill_len to kv_committed_len so cache_unfinished_req only
|
||||
# Truncate fill_len to kv_committed_len so insert_req only
|
||||
# inserts committed KV into the radix tree. The last output token
|
||||
# hasn't had KV committed yet (output_ids is 1 ahead).
|
||||
req.full_untruncated_fill_ids = req.origin_input_ids + req.output_ids
|
||||
@@ -3124,7 +3125,7 @@ class SchedulerDisaggregationDecodeMixin:
|
||||
else:
|
||||
tree_cache = self.tree_cache
|
||||
req.init_next_round_input(tree_cache)
|
||||
# Truncate fill_len to kv_committed_len so cache_unfinished_req
|
||||
# Truncate fill_len to kv_committed_len so insert_req
|
||||
# only sees committed KV (full array includes one uncommitted
|
||||
# token because init_next_round_input rebuilt it as full).
|
||||
if req.kv.kv_committed_len is not None:
|
||||
|
||||
@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, List
|
||||
import torch
|
||||
|
||||
from sglang.srt.managers.overlap_utils import RelayPayload
|
||||
from sglang.srt.mem_cache.common import maybe_cache_unfinished_req
|
||||
from sglang.srt.mem_cache.common import checkpoint_kv_cache
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo
|
||||
|
||||
@@ -120,7 +120,7 @@ class ScheduleBatchDisaggregationDecodeMixin:
|
||||
# PREBUILT does not materialize a local SWA branching window.
|
||||
if req.swa_branching_seqlen is not None:
|
||||
req.swa_branching_seqlen = None
|
||||
maybe_cache_unfinished_req(req, self.tree_cache)
|
||||
checkpoint_kv_cache(req, self.tree_cache)
|
||||
if req.grammar is not None:
|
||||
# FIXME: this try-except block is for handling unexpected xgrammar issue.
|
||||
try:
|
||||
|
||||
@@ -73,9 +73,9 @@ from sglang.srt.managers.schedule_batch import (
|
||||
)
|
||||
from sglang.srt.mem_cache.base_prefix_cache import CacheRequestOutcome
|
||||
from sglang.srt.mem_cache.common import (
|
||||
checkpoint_kv_cache,
|
||||
kv_to_page_indices,
|
||||
kv_to_page_num,
|
||||
maybe_cache_unfinished_req,
|
||||
release_kv_cache,
|
||||
)
|
||||
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
|
||||
@@ -581,15 +581,13 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
if room is not None and room in kv_mgr.transfer_infos:
|
||||
prefetch(room)
|
||||
|
||||
def cache_unfinished_disagg_prefill(
|
||||
self: Scheduler, req: Req, *, chunked: bool = False
|
||||
) -> None:
|
||||
def checkpoint_disagg_prefill(self: Scheduler, req: Req) -> None:
|
||||
cache = self.tree_cache
|
||||
if req.pending_bootstrap and _uses_write_through_cache(cache):
|
||||
cache.advance_unpublished_req(req, chunked=chunked)
|
||||
cache.advance_unpublished_req(req)
|
||||
return
|
||||
|
||||
maybe_cache_unfinished_req(req, cache, chunked=chunked)
|
||||
checkpoint_kv_cache(req, cache)
|
||||
|
||||
def release_aborted_prefill_waiting_req(self: Scheduler, req: Req) -> None:
|
||||
self.clear_pending_chunk_send(req)
|
||||
@@ -897,7 +895,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
advance_logprob_pt(i, req)
|
||||
continue
|
||||
|
||||
self.cache_unfinished_disagg_prefill(req)
|
||||
self.checkpoint_disagg_prefill(req)
|
||||
self.disagg_prefill_inflight_queue.append(req)
|
||||
if self.spec_algorithm.is_eagle() and draft_input is not None:
|
||||
req.output_topk_p = draft_input.topk_p[i]
|
||||
@@ -1140,6 +1138,8 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
else:
|
||||
logger.warning(error_message)
|
||||
req.time_stats.trace_ctx.abort(abort_info={"reason": error_message})
|
||||
if req.finished_reason is None:
|
||||
req.finished_reason = FINISH_LENGTH(length=0)
|
||||
release_kv_cache(req, self.tree_cache) # unlock the tree
|
||||
self._release_aborted_request(req)
|
||||
if not isinstance(req.finished_reason, FINISH_ABORT):
|
||||
@@ -1228,7 +1228,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
# Metadata buffer was allocated in pop_bootstrapped before
|
||||
# the request entered the waiting queue, so finalize should not fail.
|
||||
assert self.disagg_prefill_bootstrap_queue.finalize_bootstrap(req)
|
||||
self.cache_unfinished_disagg_prefill(req)
|
||||
self.checkpoint_disagg_prefill(req)
|
||||
return True
|
||||
else:
|
||||
raise RuntimeError(
|
||||
@@ -1255,7 +1255,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
chunked_req_to_exclude = set()
|
||||
if (req := self.chunked_req) is not None:
|
||||
chunked_req_to_exclude.add(req)
|
||||
self.cache_unfinished_disagg_prefill(req, chunked=True)
|
||||
self.checkpoint_disagg_prefill(req)
|
||||
|
||||
if not self.check_bootstrap(req):
|
||||
if is_aborted(req):
|
||||
@@ -1555,7 +1555,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
max_attempts = get_disagg().optimistic_prefill_attempts
|
||||
uses_write_through_cache = _uses_write_through_cache(self.tree_cache)
|
||||
if not uses_write_through_cache:
|
||||
maybe_cache_unfinished_req(req, self.tree_cache)
|
||||
checkpoint_kv_cache(req, self.tree_cache)
|
||||
# The cached prefix is evictable once the KV is released. Its length
|
||||
# (capped at what a retry can match) seeds the retry's storage baseline,
|
||||
# so an evicted prefix is looked up in L3 once before it is recomputed.
|
||||
|
||||
@@ -933,6 +933,9 @@ class ReqKvInfo:
|
||||
|
||||
# The request's own KV is [cache_protected_len, kv_allocated_len).
|
||||
cache_protected_len: int = 0 # Tree cache owns [0, here) (matched or inserted)
|
||||
# This request already inserted [0, here) into the tree; later inserts count
|
||||
# a hit only on the nodes past it, so a request counts each node once.
|
||||
cache_inserted_len: int = 0
|
||||
kv_committed_len: int = 0 # KV content committed up to here, <= kv_allocated_len
|
||||
kv_allocated_len: int = 0
|
||||
|
||||
@@ -1217,7 +1220,7 @@ class Req(ReqDllmMixin):
|
||||
# Refreshed at every sharded alloc — read through last_node, or drawn
|
||||
# least-full for a new chain — and consumed by the radix insert to
|
||||
# stamp new tree nodes. Allocation itself must NOT read it back when
|
||||
# a tree node is available (the cache_unfinished_req dedup rebind
|
||||
# a tree node is available (the insert_req dedup rebind
|
||||
# would make it stale); the only allocation-time reader is the
|
||||
# ChunkCache fallback, which has no tree nodes and no rebind.
|
||||
self.kv_rotation_base: Optional[int] = None
|
||||
@@ -1978,6 +1981,7 @@ class Req(ReqDllmMixin):
|
||||
self.indexer_topk = None
|
||||
self.last_node = None
|
||||
self.kv.cache_protected_len = 0
|
||||
self.kv.cache_inserted_len = 0
|
||||
self.kv_rotation_base = None
|
||||
self.num_matched_prefix_tokens = 0
|
||||
self.lock_receipt = DecLockRefParams()
|
||||
@@ -3257,7 +3261,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
for req, seq_len in zip(self.reqs, seq_lens, strict=True):
|
||||
req._refresh_fill_ids()
|
||||
# end runs one past full_untruncated_fill_ids while output_ids
|
||||
# trails; safe only while decoding_reqs suppresses cache_unfinished_req.
|
||||
# trails; safe only while decoding_reqs suppresses the checkpoint insert.
|
||||
req.set_extend_range(seq_len - 1, seq_len)
|
||||
|
||||
self.prefix_lens = [seq_len - 1 for seq_len in seq_lens]
|
||||
|
||||
@@ -291,8 +291,8 @@ from sglang.srt.mem_cache import kv_cache_builder
|
||||
from sglang.srt.mem_cache.allocator.page_interleave import page_interleave_shard_size
|
||||
from sglang.srt.mem_cache.base_prefix_cache import CacheRequestOutcome
|
||||
from sglang.srt.mem_cache.common import (
|
||||
checkpoint_kv_cache,
|
||||
discard_kv_cache_backup,
|
||||
maybe_cache_unfinished_req,
|
||||
release_kv_cache,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import PPProxyTensors
|
||||
@@ -3533,9 +3533,9 @@ class Scheduler(
|
||||
|
||||
def stash_chunked_request(self, req: Req):
|
||||
if self.disaggregation_mode == DisaggregationMode.PREFILL:
|
||||
self.cache_unfinished_disagg_prefill(req, chunked=True)
|
||||
self.checkpoint_disagg_prefill(req)
|
||||
else:
|
||||
maybe_cache_unfinished_req(req, self.tree_cache, chunked=True)
|
||||
checkpoint_kv_cache(req, self.tree_cache)
|
||||
|
||||
def process_pending_chunked_abort(self) -> None:
|
||||
"""Abort an in-flight chunked-prefill request once it is safe to do so.
|
||||
@@ -5321,6 +5321,8 @@ class Scheduler(
|
||||
discard_kv_cache_backup(req, self.tree_cache, "host_pool")
|
||||
if self.enable_hisparse:
|
||||
self.hisparse_coordinator.request_finished(req)
|
||||
if req.finished_reason is None:
|
||||
req.finished_reason = FINISH_ABORT()
|
||||
release_kv_cache(req, self.tree_cache)
|
||||
if self.disaggregation_mode == DisaggregationMode.PREFILL:
|
||||
self.release_aborted_prefill_waiting_req(req)
|
||||
|
||||
@@ -30,7 +30,7 @@ from sglang.srt.managers.schedule_batch import (
|
||||
mamba_lazy_spec_in_window,
|
||||
)
|
||||
from sglang.srt.mem_cache.common import (
|
||||
maybe_cache_unfinished_req,
|
||||
checkpoint_kv_cache,
|
||||
release_kv_cache,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import (
|
||||
@@ -381,7 +381,7 @@ class SchedulerBatchResultProcessor:
|
||||
)
|
||||
req.time_stats.set_completion_time()
|
||||
elif not batch.decoding_reqs or req not in batch.decoding_reqs:
|
||||
maybe_cache_unfinished_req(req, self.tree_cache)
|
||||
checkpoint_kv_cache(req, self.tree_cache)
|
||||
if get_memory().enable_hisparse:
|
||||
self.hisparse_coordinator.admit_request_into_staging(req)
|
||||
|
||||
@@ -472,7 +472,7 @@ class SchedulerBatchResultProcessor:
|
||||
release_kv_cache(req, self.tree_cache)
|
||||
req.time_stats.set_completion_time()
|
||||
else:
|
||||
maybe_cache_unfinished_req(req, self.tree_cache)
|
||||
checkpoint_kv_cache(req, self.tree_cache)
|
||||
else:
|
||||
# being chunked reqs' prefill is not finished
|
||||
req.inflight_middle_chunks -= 1
|
||||
|
||||
@@ -255,7 +255,7 @@ def _kv_shard_rotation_bases(
|
||||
The owner class of position-page P is ``(b_i + P) % shard_size``. Rules:
|
||||
|
||||
- Read through ``req.last_node`` at alloc time, never a value cached on
|
||||
the request: ``cache_unfinished_req`` can rebind a chunked request onto
|
||||
the request: a checkpoint ``insert_req`` can rebind a chunked request onto
|
||||
another chain's canonical locs between chunks, changing the base. The
|
||||
read goes through ``tree_cache.rotation_base_of`` because the node
|
||||
handle is tree-specific (a NodeId on the unified tree).
|
||||
|
||||
@@ -95,7 +95,9 @@ class InsertParams:
|
||||
component_evicted_seqlens: dict[ComponentType, int] = dataclasses.field(
|
||||
default_factory=dict, kw_only=True
|
||||
)
|
||||
chunked: bool = False
|
||||
# The inserting request already inserted [0, here) (req.kv.cache_inserted_len);
|
||||
# only the nodes past it count a hit, so a request counts each node once.
|
||||
inserted_len: int = 0
|
||||
priority: int = 0
|
||||
session_id: Optional[str] = None
|
||||
track_adopted_ranges: bool = False
|
||||
@@ -458,14 +460,14 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
|
||||
|
||||
@abstractmethod
|
||||
def insert_req(self, req: Req, *, up_to: int, **kwargs):
|
||||
"""Hand a finished request's KV up to row position ``up_to`` to the
|
||||
tree: insert what can be keyed and advance ``cache_protected_len``
|
||||
past it. The caller then frees ``[cache_protected_len, up_to)`` and
|
||||
everything after, and unpins; nothing here releases a slot."""
|
||||
|
||||
@abstractmethod
|
||||
def cache_unfinished_req(self, req: Req, **kwargs):
|
||||
pass
|
||||
"""Insert the request's KV up to row position ``up_to`` into the tree,
|
||||
repoint the row onto the tree's copy, re-anchor ``req.last_node`` on
|
||||
the node the insert ended on and advance ``cache_protected_len``.
|
||||
Called at every checkpoint of a running request and once more when
|
||||
it finishes (``req.finished()``), when the tree also takes over the
|
||||
component state the request no longer needs. Nothing here frees a
|
||||
slot: ``release_kv_cache`` frees ``[cache_protected_len, up_to)`` and
|
||||
everything after, and unpins."""
|
||||
|
||||
def free_kv_row(self, kv: Any, ranges: list[tuple[int, int]]) -> None:
|
||||
"""Give back ascending, disjoint, half-open row-position ranges
|
||||
|
||||
@@ -77,12 +77,7 @@ class ChunkCache(BasePrefixCache):
|
||||
return InsertResult(prefix_len=0)
|
||||
|
||||
def insert_req(self, req: Req, *, up_to: int):
|
||||
pass
|
||||
|
||||
def cache_unfinished_req(self, req: Req, chunked=False):
|
||||
kv_indices = self.req_to_token_pool.req_to_token[
|
||||
req.kv.req_pool_idx, : req.extend_range.end
|
||||
]
|
||||
kv_indices = self.req_to_token_pool.req_to_token[req.kv.req_pool_idx, :up_to]
|
||||
# `req.prefix_indices` will be used in `PrefillAdder::add_chunked_req` later
|
||||
req.prefix_indices = kv_indices.to(dtype=torch.int64, copy=True)
|
||||
|
||||
|
||||
@@ -169,11 +169,16 @@ def free_kv_row_segments(
|
||||
allocator.free_segments(swa_alive)
|
||||
|
||||
|
||||
def maybe_cache_unfinished_req(req: Req, tree_cache: BasePrefixCache, **kwargs):
|
||||
def checkpoint_kv_cache(req: Req, tree_cache: BasePrefixCache) -> None:
|
||||
"""Publish what the running request has computed so far, unless it is
|
||||
barred from the tree."""
|
||||
# The tree reads req.finished() to tell a checkpoint from the final
|
||||
# insert; a finished request belongs in release_kv_cache.
|
||||
assert not req.finished(), f"checkpointing finished request {req.rid}"
|
||||
if req.skip_radix_cache_insert:
|
||||
return
|
||||
|
||||
tree_cache.cache_unfinished_req(req, **kwargs)
|
||||
tree_cache.insert_req(req, up_to=req.extend_range.end)
|
||||
|
||||
|
||||
def evict_from_tree_cache(
|
||||
@@ -313,6 +318,13 @@ def release_kv_cache(req: Req, tree_cache: BasePrefixCache, is_insert: bool = Tr
|
||||
owned_kv_len = req.owned_kv_len()
|
||||
is_insert = is_insert and not req.skip_radix_cache_insert
|
||||
if is_insert:
|
||||
# A tree that takes over component state (mamba) must see the request
|
||||
# finished, or the insert forks the state and the slot leaks.
|
||||
assert req.finished() or not tree_cache.supports_mamba(), (
|
||||
f"releasing unfinished request {req.rid} into a mamba tree"
|
||||
)
|
||||
# The fill-id array lags output_ids until the next prepare_for_decode.
|
||||
req._refresh_fill_ids()
|
||||
tree_cache.insert_req(req, up_to=owned_kv_len)
|
||||
# The protected prefix is not this req's to free.
|
||||
tree_cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, owned_kv_len)])
|
||||
|
||||
@@ -1034,7 +1034,7 @@ class MambaPool:
|
||||
ReplaySSM invariant: the SOURCE must be a fully-flushed checkpoint
|
||||
(``write_pos[src] == 0``). Only ``temporal`` is copied, not the ring, so
|
||||
an un-flushed source would drop its last ``write_pos`` updates. Callers
|
||||
comply: COW copies radix checkpoints; ``cache_unfinished_req`` copies an
|
||||
comply: COW copies radix checkpoints; a checkpoint ``insert_req`` copies an
|
||||
active slot only during prefill (ring empty); ``insert_req``
|
||||
caps the donate to the last flush boundary. The dst cursor is reset to 0
|
||||
(the copied checkpoint has no pending ring entries).
|
||||
|
||||
@@ -3,15 +3,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sglang.srt.mem_cache.base_prefix_cache import EvictParams, EvictResult
|
||||
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
|
||||
from sglang.srt.mem_cache.radix_cache import RadixCache
|
||||
from sglang.srt.utils.common import ceil_align
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.schedule_batch import Req
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -57,23 +52,6 @@ class PureSWARadixCache(RadixCache):
|
||||
num_tokens = max(params.num_tokens, params.swa_num_tokens)
|
||||
return super().evict(EvictParams(num_tokens=num_tokens))
|
||||
|
||||
def insert_req(self, req: Req, *, up_to: int):
|
||||
"""Insert only the prefill portion [0, swa_evict_floor)."""
|
||||
if self.disable:
|
||||
return
|
||||
token_ids = (req.origin_input_ids + req.output_ids)[:up_to]
|
||||
swa_evict_floor = req.kv.swa_evict_floor
|
||||
key_limit = (
|
||||
ceil_align(swa_evict_floor, self.page_size) if swa_evict_floor > 0 else None
|
||||
)
|
||||
radix_key, _, _ = self._insert_cache(req, token_ids, key_limit=key_limit)
|
||||
req.kv.cache_protected_len = len(radix_key)
|
||||
|
||||
def cache_unfinished_req(self, req: Req, chunked=False):
|
||||
"""During chunked prefill, swa_evicted_seqlen is 0 and no SWA eviction
|
||||
has happened yet, so standard RadixCache logic is correct."""
|
||||
super().cache_unfinished_req(req, chunked=chunked)
|
||||
|
||||
def available_and_evictable_str(self) -> str:
|
||||
allocator = self.token_to_kv_pool_allocator
|
||||
swa_available = allocator.swa_available_size()
|
||||
|
||||
@@ -50,6 +50,7 @@ from sglang.srt.mem_cache.utils import (
|
||||
get_eviction_strategy,
|
||||
split_node_hash_value,
|
||||
)
|
||||
from sglang.srt.utils.common import ceil_align
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.schedule_batch import Req
|
||||
@@ -418,7 +419,7 @@ class RadixCache(BasePrefixCache):
|
||||
key = params.key
|
||||
value = params.value
|
||||
priority = params.priority
|
||||
chunked = params.chunked
|
||||
inserted_len = params.inserted_len
|
||||
|
||||
key, value = key.maybe_to_bigram_view(self.is_eagle, value)
|
||||
key = key.page_aligned(self.page_size)
|
||||
@@ -429,7 +430,7 @@ class RadixCache(BasePrefixCache):
|
||||
value = torch.tensor(key.token_ids[: len(key)], dtype=torch.int64)
|
||||
|
||||
prefix_len, last_node = self._insert_helper(
|
||||
self.root_node, key, value, priority, chunked
|
||||
self.root_node, key, value, priority, inserted_len
|
||||
)
|
||||
return InsertResult(prefix_len=prefix_len, last_device_node=last_node)
|
||||
|
||||
@@ -438,7 +439,6 @@ class RadixCache(BasePrefixCache):
|
||||
req: Req,
|
||||
token_ids,
|
||||
*,
|
||||
chunked: bool = False,
|
||||
key_limit: Optional[int] = None,
|
||||
split_prompt: bool = False,
|
||||
) -> Tuple[RadixKey, torch.Tensor, int]:
|
||||
@@ -462,9 +462,13 @@ class RadixCache(BasePrefixCache):
|
||||
priority = req.priority or 0
|
||||
result = self.insert(
|
||||
InsertParams(
|
||||
key=radix_key, value=values, chunked=chunked, priority=priority
|
||||
key=radix_key,
|
||||
value=values,
|
||||
inserted_len=req.kv.cache_inserted_len,
|
||||
priority=priority,
|
||||
)
|
||||
)
|
||||
req.kv.cache_inserted_len = max(req.kv.cache_inserted_len, key_len)
|
||||
if split_prompt:
|
||||
# Split the leaf at the prompt boundary so eviction can drop the
|
||||
# output KV without the prompt; a prefix re-insert only changes topology.
|
||||
@@ -479,10 +483,8 @@ class RadixCache(BasePrefixCache):
|
||||
InsertParams(
|
||||
key=prompt_key,
|
||||
value=values[: len(prompt_key)],
|
||||
inserted_len=len(prompt_key),
|
||||
priority=priority + 1,
|
||||
# The request created these nodes moments ago; another
|
||||
# hit_count bump would promote every prompt node.
|
||||
chunked=True,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -495,17 +497,14 @@ class RadixCache(BasePrefixCache):
|
||||
def insert_req(self, req: Req, *, up_to: int):
|
||||
if self.disable:
|
||||
return
|
||||
token_ids = (req.origin_input_ids + req.output_ids)[:up_to]
|
||||
radix_key, _, _ = self._insert_cache(req, token_ids, split_prompt=True)
|
||||
req.kv.cache_protected_len = len(radix_key)
|
||||
|
||||
def cache_unfinished_req(self, req: Req, chunked=False):
|
||||
"""Cache request when it is unfinished."""
|
||||
if self.disable:
|
||||
return
|
||||
|
||||
token_ids = req.full_untruncated_fill_ids[:up_to]
|
||||
# Pure-SWA trees never cache what the window already evicted.
|
||||
swa_evict_floor = req.kv.swa_evict_floor
|
||||
key_limit = (
|
||||
ceil_align(swa_evict_floor, self.page_size) if swa_evict_floor > 0 else None
|
||||
)
|
||||
radix_key, kv_indices, _ = self._insert_cache(
|
||||
req, req.get_fill_ids(), chunked=chunked
|
||||
req, token_ids, key_limit=key_limit, split_prompt=True
|
||||
)
|
||||
|
||||
# The prefix indices could be updated, reuse it
|
||||
@@ -525,7 +524,7 @@ class RadixCache(BasePrefixCache):
|
||||
|
||||
# With page_size > 1 the partial page sits in req.prefix_indices but not
|
||||
# in the tree; cache_protected_len marks the tree-owned part so the next
|
||||
# cache_unfinished_req or release_kv_cache frees the rest.
|
||||
# insert_req or release_kv_cache frees the rest.
|
||||
req.kv.cache_protected_len = len(new_indices)
|
||||
|
||||
self.dec_lock_ref(req.last_node)
|
||||
@@ -540,7 +539,6 @@ class RadixCache(BasePrefixCache):
|
||||
)
|
||||
else:
|
||||
req.prefix_indices = new_indices
|
||||
|
||||
req.last_node = new_last_node
|
||||
|
||||
def pretty_print(self):
|
||||
@@ -687,12 +685,7 @@ class RadixCache(BasePrefixCache):
|
||||
|
||||
return new_node
|
||||
|
||||
def _inc_hit_count(self, node: TreeNode, chunked: bool = False):
|
||||
# Skip the hit count update for chunked requests to avoid self-referencing
|
||||
# inflation where a chunked request increments hit_count on nodes it created
|
||||
# in previous chunks.
|
||||
if chunked:
|
||||
return
|
||||
def _inc_hit_count(self, node: TreeNode):
|
||||
node.hit_count += 1
|
||||
|
||||
def _insert_helper(
|
||||
@@ -701,7 +694,7 @@ class RadixCache(BasePrefixCache):
|
||||
key: RadixKey,
|
||||
value,
|
||||
priority: int = 0,
|
||||
chunked: bool = False,
|
||||
inserted_len: int = 0,
|
||||
):
|
||||
# Convert None priority to 0
|
||||
if priority is None:
|
||||
@@ -727,11 +720,12 @@ class RadixCache(BasePrefixCache):
|
||||
if prefix_len < len(node.key):
|
||||
new_node = self._split_node(node.key, node, prefix_len)
|
||||
new_node.priority = max(new_node.priority, priority)
|
||||
self._inc_hit_count(new_node, chunked)
|
||||
node = new_node
|
||||
else:
|
||||
node.priority = max(node.priority, priority)
|
||||
self._inc_hit_count(node, chunked)
|
||||
# Nodes this request already inserted were counted back then.
|
||||
if total_prefix_length > inserted_len:
|
||||
self._inc_hit_count(node)
|
||||
if len(key):
|
||||
child_key = key.child_key(self.page_size)
|
||||
|
||||
@@ -740,7 +734,8 @@ class RadixCache(BasePrefixCache):
|
||||
new_node.parent = node
|
||||
new_node.key = key
|
||||
new_node.value = value.clone()
|
||||
self._inc_hit_count(new_node, chunked)
|
||||
if total_prefix_length + len(key) > inserted_len:
|
||||
self._inc_hit_count(new_node)
|
||||
node.children[child_key] = new_node
|
||||
self.evictable_size_ += len(key)
|
||||
self._update_leaf_status(node)
|
||||
|
||||
@@ -726,7 +726,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
prev_prefix_len=params.prev_prefix_len,
|
||||
swa_evicted_seqlen=params.get_evicted_seqlen(ComponentType.SWA),
|
||||
swa_branching_seqlen=params.swa_branching_seqlen,
|
||||
chunked=params.chunked,
|
||||
inserted_len=params.inserted_len,
|
||||
priority=0 if params.priority is None else params.priority,
|
||||
track_adopted_ranges=params.track_adopted_ranges,
|
||||
rotation_base=params.rotation_base,
|
||||
|
||||
@@ -390,8 +390,10 @@ class FlexKVRadixCache(RadixCache):
|
||||
self._load_markers.pop(req.cache_request_handle, None)
|
||||
|
||||
def insert_req(self, req: Req, *, up_to: int) -> None: # type: ignore[override]
|
||||
"""Base insert_req then fire an async FlexKV store."""
|
||||
"""Base insert_req; a finished request also fires an async FlexKV store."""
|
||||
super().insert_req(req, up_to=up_to)
|
||||
if not req.finished():
|
||||
return
|
||||
|
||||
# Compute the committed prefix.
|
||||
topk = get_spec().speculative_eagle_topk
|
||||
|
||||
@@ -248,12 +248,6 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
|
||||
|
||||
return True
|
||||
|
||||
def cache_unfinished_req(self, req: Req, chunked: bool = False, **kwargs) -> None:
|
||||
self._publish_external_loaded_prefix(req, token_ids_len=len(req.get_fill_ids()))
|
||||
super().cache_unfinished_req(req, chunked=chunked, **kwargs)
|
||||
self._retire_loaded_flow(req.rid)
|
||||
self._submit_store(req, req.get_fill_ids())
|
||||
|
||||
def on_release(self, req: Req, *, inserted: bool) -> None:
|
||||
super().on_release(req, inserted=inserted)
|
||||
if not inserted:
|
||||
@@ -263,9 +257,9 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
|
||||
self._publish_external_loaded_prefix(req, token_ids_len=up_to)
|
||||
super().insert_req(req, up_to=up_to, **kwargs)
|
||||
self._retire_loaded_flow(req.rid)
|
||||
token_ids = (req.origin_input_ids + req.output_ids)[:up_to]
|
||||
self._submit_store(req, token_ids)
|
||||
self._request_session_finish(req.rid)
|
||||
self._submit_store(req, req.full_untruncated_fill_ids[:up_to])
|
||||
if req.finished():
|
||||
self._request_session_finish(req.rid)
|
||||
|
||||
def check_hicache_events(self) -> None:
|
||||
"""Poll LMCache retrieve/store futures at the scheduler safe point."""
|
||||
@@ -811,7 +805,7 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
|
||||
mamba_value=checkpoint,
|
||||
prev_prefix_len=prev_prefix_len,
|
||||
component_evicted_seqlens=req.kv.component_evicted_seqlens.copy(),
|
||||
chunked=True,
|
||||
inserted_len=len(key),
|
||||
priority=req.priority or 0,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -230,47 +230,23 @@ receipt proves were taken. The eventual full release must pass
|
||||
|
||||
### `insert_req(req: Req, *, up_to: int)`
|
||||
|
||||
Cache a completed request's KV data into the tree.
|
||||
Insert a request's KV into the tree: at every checkpoint while it runs, and once more when it finishes.
|
||||
|
||||
| Aspect | Detail |
|
||||
|--------|--------|
|
||||
| **Purpose** | After a request finishes, insert its token/KV data into the tree for future reuse |
|
||||
| **Inputs** | `req` — the finished request; `up_to` — end of the request-owned KV range. `release_kv_cache` frees `[cache_protected_len, up_to)` and everything past it and unpins; with `is_insert=False` it skips the insert and calls `on_release(req, inserted=False)` for component cleanup |
|
||||
| **Purpose** | Publish the request's KV `[cache_protected_len, up_to)` so other requests can match it; nodes past `req.kv.cache_inserted_len` count one hit (a request counts each node once). `checkpoint_kv_cache` calls it with `req.extend_range.end` while the request runs; `release_kv_cache` calls it with the request-owned length when it finishes (`req.finished()`), then frees `[cache_protected_len, up_to)` and everything past it and unpins; with `is_insert=False` it skips the insert and calls `on_release(req, inserted=False)` for component cleanup |
|
||||
| **Inputs** | `req` — the request; `up_to` — row position the insert may read up to |
|
||||
| **Output** | `None` |
|
||||
| **Mutation** | Calls component hooks → `insert` → component cleanup; advances `cache_protected_len` past the inserted key. Frees nothing and drops no lock: `release_kv_cache` does both afterwards. |
|
||||
| **Complexity** | **O(K + D·C)** — insert O(K + D·C) + lock release O(D). Simplifies to **O(K)**. |
|
||||
| **Mutation** | Component hooks → `insert` → re-match → writes the tree's indices back into the row → moves the request's lock from the old `req.last_node` to the node the insert ended on → updates `req.prefix_indices`, `req.kv.cache_protected_len`, `req.kv.cache_inserted_len`, `req.last_node` → component cleanup. A finished request's component state (Mamba) is handed to the tree instead of forked, and what it still held is freed. Frees no KV slot: `release_kv_cache` does that afterwards. |
|
||||
| **Complexity** | **O(K + D·C)** — insert O(K + D·C) + re-match O(K + D·C) + lock transfer O(D). Simplifies to **O(K)**. |
|
||||
|
||||
**Algorithm detail:**
|
||||
1. `prepare_for_caching_req()` per component — sets component-specific insert params, returns effective cache length (SWA: copies its cursor with `set_evicted_seqlen`; Mamba: prepares `mamba_value` from ping-pong buffer, returns `mamba_last_track_seqlen` as truncation hint)
|
||||
2. Truncates if `effective_cache_len < len(token_ids)`: frees excess pool indices
|
||||
3. Converts token IDs (bigram if EAGLE), page-aligns keys, then calls `insert()`
|
||||
4. Frees unaligned tail KV indices beyond page boundary
|
||||
5. Calls `dec_lock_ref()` on the previous `req.last_node`
|
||||
6. `cleanup_after_caching_req()` per component (Mamba: frees forked mamba_value based on `mamba_exist`, handles ping-pong buffer cleanup)
|
||||
|
||||
---
|
||||
|
||||
### `cache_unfinished_req(req: Req, chunked=False)`
|
||||
|
||||
Cache an in-progress request's partial KV data (chunked prefill).
|
||||
|
||||
| Aspect | Detail |
|
||||
|--------|--------|
|
||||
| **Purpose** | During chunked prefill, insert partial results so the next chunk can match the prefix |
|
||||
| **Inputs** | `req` — the in-progress request |
|
||||
| **Output** | `None` |
|
||||
| **Mutation** | Inserts partial KV → re-matches prefix → updates `req.prefix_indices`, `req.kv.cache_protected_len`, `req.last_node`; transfers lock from old node to new node |
|
||||
| **Complexity** | **O(K + D·C)** — two tree traversals: insert O(K + D·C) + re-match O(K + D·C) + lock transfer O(D). Simplifies to **O(K)**. |
|
||||
|
||||
**Algorithm detail:**
|
||||
1. `prepare_for_caching_req()` per component
|
||||
2. `insert()` — first tree traversal
|
||||
3. `match_prefix()` — **second** tree traversal to get updated indices
|
||||
4. Writes new prefix indices into `req_to_token_pool`
|
||||
5. `dec_lock_ref()` on old `req.last_node`
|
||||
6. `inc_lock_ref()` on new matched node
|
||||
7. Updates `req.prefix_indices`, `req.kv.cache_protected_len`, `req.last_node`
|
||||
8. `cleanup_after_caching_req()` per component
|
||||
1. `prepare_for_caching_req(is_finished=req.finished())` per component — sets component-specific insert params, returns effective cache length (SWA: copies its cursor with `set_evicted_seqlen`, may return a branching boundary; Mamba: prepares `mamba_value`, donated when finished and forked otherwise, returns `mamba_last_track_seqlen` as truncation hint)
|
||||
2. Truncates the key to `effective_cache_len`; the row past it stays the request's
|
||||
3. Converts token IDs (bigram if EAGLE), page-aligns the key, then calls `insert()`; a single-component tree re-inserts the prompt part so eviction can drop the output without the prompt
|
||||
4. `match_prefix()` on the inserted key and writes the matched indices into the row
|
||||
5. `dec_lock_ref()` on the old `req.last_node`, `inc_lock_ref()` on the new one
|
||||
6. `cleanup_after_caching_req()` per component (Mamba: frees the forked `mamba_value` when the tree already had one, frees the finished request's slot)
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -363,7 +363,7 @@ class UnifiedCacheLinkerWrapper:
|
||||
if req.kv is not None
|
||||
else {}
|
||||
),
|
||||
chunked=True,
|
||||
inserted_len=len(hit.prefix_key),
|
||||
priority=req.priority or 0,
|
||||
track_adopted_ranges=True,
|
||||
)
|
||||
|
||||
@@ -1053,11 +1053,9 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
continue
|
||||
comp.refresh_lru(LRURefreshPhase.WALKDOWN, node, self.root_node)
|
||||
|
||||
def _inc_hit_count_and_check(
|
||||
self, node: UnifiedTreeNode, chunked: bool = False
|
||||
) -> bool:
|
||||
def _inc_hit_count_and_check(self, node: UnifiedTreeNode) -> bool:
|
||||
"""Increment hit count; check whether a write backup should be fired."""
|
||||
if node.evicted or chunked:
|
||||
if node.evicted:
|
||||
return False
|
||||
if self.is_write_back:
|
||||
return False
|
||||
@@ -1307,7 +1305,9 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
if swa_already_freed < dup.numel():
|
||||
step_actions.append(FreeDeviceKV([dup[swa_already_freed:]]))
|
||||
|
||||
if self._inc_hit_count_and_check(node, state.params.chunked):
|
||||
# Nodes this request already inserted were counted back then.
|
||||
node_end = state.total_prefix_length + prefix_len
|
||||
if node_end > state.params.inserted_len and self._inc_hit_count_and_check(node):
|
||||
step_actions.append(self._build_backup_kv_action(node))
|
||||
state.node = node
|
||||
state.total_prefix_length += prefix_len
|
||||
@@ -1368,8 +1368,10 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
def _should_backup_after_insert(self, state: _InsertWalkState) -> bool:
|
||||
"""Check whether the insert target needs a Host backup."""
|
||||
if state.is_new_leaf:
|
||||
return self._inc_hit_count_and_check(
|
||||
state.target_node, state.params.chunked
|
||||
leaf_end = state.total_prefix_length + len(state.target_node.key)
|
||||
return (
|
||||
leaf_end > state.params.inserted_len
|
||||
and self._inc_hit_count_and_check(state.target_node)
|
||||
)
|
||||
|
||||
node = state.target_node
|
||||
|
||||
@@ -1004,171 +1004,21 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
|
||||
@rank_consensus(same_params=["req.rid", "up_to"])
|
||||
def insert_req(self, req: Req, *, up_to: int, **kwargs) -> None:
|
||||
if self.disable:
|
||||
for comp in self._components_tuple:
|
||||
comp.cleanup_after_caching_req(req, is_finished=True)
|
||||
if self.session.try_insert_req(req, up_to=up_to, **kwargs):
|
||||
return
|
||||
|
||||
token_ids = (req.origin_input_ids + req.output_ids)[:up_to]
|
||||
kv_indices = self.req_to_token_pool.req_to_token[req.kv.req_pool_idx, :up_to]
|
||||
|
||||
insert_params = InsertParams(
|
||||
prev_prefix_len=req.kv.cache_protected_len,
|
||||
priority=req.priority or 0,
|
||||
session_id=req.session_id,
|
||||
rotation_base=req.kv_rotation_base,
|
||||
)
|
||||
|
||||
# components prepare insert data + return effective cache_len
|
||||
effective_cache_len = len(token_ids)
|
||||
for comp in self._components_tuple:
|
||||
cl = comp.prepare_for_caching_req(
|
||||
req=req,
|
||||
insert_params=insert_params,
|
||||
token_ids_len=len(token_ids),
|
||||
is_finished=True,
|
||||
)
|
||||
if cl is not None:
|
||||
effective_cache_len = min(effective_cache_len, cl)
|
||||
for comp in self._components_tuple:
|
||||
effective_cache_len = comp.floor_cache_len(effective_cache_len)
|
||||
|
||||
# Truncate if needed; the tail free is deferred and batched with
|
||||
# the unaligned tail below so a shared boundary page is emitted once.
|
||||
kv_indices_full = kv_indices
|
||||
tail_free_start = None
|
||||
if effective_cache_len < len(token_ids):
|
||||
tail_free_start = max(effective_cache_len, req.kv.cache_protected_len)
|
||||
token_ids = token_ids[:effective_cache_len]
|
||||
kv_indices = kv_indices[:effective_cache_len]
|
||||
|
||||
radix_key = RadixKey(
|
||||
token_ids,
|
||||
req.extra_key,
|
||||
is_bigram=self.tree_core.is_eagle,
|
||||
cache_salt=req.cache_salt,
|
||||
).page_aligned(self.page_size)
|
||||
page_aligned_len = len(radix_key)
|
||||
values = kv_indices[:page_aligned_len].to(dtype=torch.int64, copy=True)
|
||||
|
||||
insert_params.key = radix_key
|
||||
insert_params.value = values
|
||||
result = self.insert(insert_params)
|
||||
|
||||
# Split the leaf at the prompt boundary so eviction can drop the output
|
||||
# KV without the prompt. prev_prefix_len keeps the overlapping indices
|
||||
# from being freed as duplicates; skipped after a declined rotation,
|
||||
# whose rows are freed below.
|
||||
prompt_key = RadixKey(
|
||||
req.origin_input_ids,
|
||||
req.extra_key,
|
||||
is_bigram=self.tree_core.is_eagle,
|
||||
cache_salt=req.cache_salt,
|
||||
).page_aligned(self.page_size)
|
||||
if (
|
||||
not result.rotation_tail_declined
|
||||
and len(self._components_tuple) == 1
|
||||
and self._components_tuple[0].component_type == BASE_COMPONENT_TYPE
|
||||
and 0 < len(prompt_key) < len(radix_key)
|
||||
):
|
||||
self.insert(
|
||||
replace(
|
||||
insert_params,
|
||||
key=prompt_key,
|
||||
value=values[: len(prompt_key)],
|
||||
prev_prefix_len=len(prompt_key),
|
||||
priority=insert_params.priority + 1,
|
||||
# The request created these nodes moments ago; another
|
||||
# hit_count bump would promote every prompt node.
|
||||
chunked=True,
|
||||
)
|
||||
)
|
||||
|
||||
# Everything past the inserted key goes back to the caller, the
|
||||
# protected prefix never does. After a rotation decline nothing was
|
||||
# inserted.
|
||||
free_from = (
|
||||
min(req.kv.cache_protected_len, len(kv_indices))
|
||||
if result.rotation_tail_declined
|
||||
else page_aligned_len
|
||||
)
|
||||
if tail_free_start is not None and tail_free_start > len(kv_indices):
|
||||
# Truncated below the protected prefix: only an untracked mamba
|
||||
# request gets here, with an empty key, and owns nothing before it.
|
||||
assert free_from == len(kv_indices), (
|
||||
f"{free_from=} {len(kv_indices)=} {req.kv.cache_protected_len=}"
|
||||
)
|
||||
free_from = tail_free_start
|
||||
req.kv.cache_protected_len = free_from
|
||||
|
||||
# cleanup
|
||||
for comp in self._components_tuple:
|
||||
comp.cleanup_after_caching_req(
|
||||
req, is_finished=True, insert_result=result, insert_params=insert_params
|
||||
)
|
||||
|
||||
if self.enable_session_radix_cache and result is not None:
|
||||
from sglang.srt.managers.schedule_batch import FINISH_ABORT
|
||||
|
||||
if req.finished_reason is not None and not isinstance(
|
||||
req.finished_reason, FINISH_ABORT
|
||||
):
|
||||
self.session_refs.register_session_ref(
|
||||
req, leaf=result.last_device_node
|
||||
)
|
||||
|
||||
@rank_consensus(same_params=["req.rid", "chunked"])
|
||||
def advance_unpublished_req(self, req: Req, chunked: bool = False) -> None:
|
||||
assert not self.supports_mamba()
|
||||
token_ids = req.get_fill_ids()
|
||||
kv_indices = self.req_to_token_pool.req_to_token[
|
||||
req.kv.req_pool_idx, : len(token_ids)
|
||||
]
|
||||
insert_params = InsertParams(
|
||||
prev_prefix_len=req.kv.cache_protected_len,
|
||||
chunked=chunked,
|
||||
priority=req.priority or 0,
|
||||
rotation_base=req.kv_rotation_base,
|
||||
)
|
||||
effective_cache_len = len(token_ids)
|
||||
for comp in self._components_tuple:
|
||||
cache_len = comp.prepare_for_caching_req(
|
||||
req=req,
|
||||
insert_params=insert_params,
|
||||
token_ids_len=len(token_ids),
|
||||
is_finished=False,
|
||||
)
|
||||
if cache_len is not None:
|
||||
effective_cache_len = min(effective_cache_len, cache_len)
|
||||
|
||||
radix_key = RadixKey(
|
||||
token_ids[:effective_cache_len],
|
||||
req.extra_key,
|
||||
is_bigram=self.tree_core.is_eagle,
|
||||
cache_salt=req.cache_salt,
|
||||
)
|
||||
if envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS.get():
|
||||
for comp in self._components_tuple:
|
||||
comp.free_out_of_window_slots(req, len(radix_key) - 1, insert_params)
|
||||
|
||||
req.prefix_indices = kv_indices.to(dtype=torch.int64, copy=True)
|
||||
for comp in self._components_tuple:
|
||||
comp.cleanup_after_caching_req(
|
||||
req, is_finished=False, insert_params=insert_params
|
||||
)
|
||||
|
||||
@rank_consensus(same_params=["req.rid", "chunked"])
|
||||
def cache_unfinished_req(self, req: Req, chunked: bool = False, **kwargs) -> None:
|
||||
if self.session.try_cache_unfinished_req(req, chunked=chunked, **kwargs):
|
||||
return
|
||||
|
||||
token_ids = req.get_fill_ids()
|
||||
# A finished request hands its component state (mamba) to the tree
|
||||
# instead of forking it, and the tree frees what the request still held.
|
||||
is_finished = req.finished()
|
||||
token_ids = req.full_untruncated_fill_ids[:up_to]
|
||||
|
||||
if self.disable:
|
||||
kv_indices = self.req_to_token_pool.req_to_token[
|
||||
req.kv.req_pool_idx, : len(token_ids)
|
||||
]
|
||||
req.prefix_indices = kv_indices.to(dtype=torch.int64, copy=True)
|
||||
if is_finished:
|
||||
for comp in self._components_tuple:
|
||||
comp.cleanup_after_caching_req(req, is_finished=True)
|
||||
return
|
||||
|
||||
kv_indices_orig = self.req_to_token_pool.req_to_token[
|
||||
@@ -1178,7 +1028,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
# components prepare insert data + return effective cache_len
|
||||
insert_params = InsertParams(
|
||||
prev_prefix_len=req.kv.cache_protected_len,
|
||||
chunked=chunked,
|
||||
inserted_len=req.kv.cache_inserted_len,
|
||||
priority=req.priority or 0,
|
||||
session_id=req.session_id,
|
||||
rotation_base=req.kv_rotation_base,
|
||||
@@ -1189,7 +1039,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
req=req,
|
||||
insert_params=insert_params,
|
||||
token_ids_len=len(token_ids),
|
||||
is_finished=False,
|
||||
is_finished=is_finished,
|
||||
)
|
||||
if cl is not None:
|
||||
effective_cache_len = min(effective_cache_len, cl)
|
||||
@@ -1204,7 +1054,10 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
cache_salt=req.cache_salt,
|
||||
)
|
||||
|
||||
if envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS.get():
|
||||
if (
|
||||
not is_finished
|
||||
and envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS.get()
|
||||
):
|
||||
# The frontier lands a page below page_floor(pre_len + 1), which has to
|
||||
# be where the insert stops, or the leaf it creates keeps less than a
|
||||
# sliding window of live SWA and the match after the insert rejects it.
|
||||
@@ -1217,7 +1070,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
req.prefix_indices = kv_indices_orig.to(dtype=torch.int64, copy=True)
|
||||
for comp in self._components_tuple:
|
||||
comp.cleanup_after_caching_req(
|
||||
req, is_finished=False, insert_params=insert_params
|
||||
req, is_finished=is_finished, insert_params=insert_params
|
||||
)
|
||||
return
|
||||
|
||||
@@ -1244,9 +1097,35 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
req.prefix_indices = kv_indices_orig.to(dtype=torch.int64, copy=True)
|
||||
for comp in self._components_tuple:
|
||||
comp.cleanup_after_caching_req(
|
||||
req, is_finished=False, insert_params=insert_params
|
||||
req, is_finished=is_finished, insert_params=insert_params
|
||||
)
|
||||
return
|
||||
req.kv.cache_inserted_len = max(req.kv.cache_inserted_len, page_aligned_len)
|
||||
|
||||
# Split the leaf at the prompt boundary so eviction can drop the output
|
||||
# KV without the prompt. prev_prefix_len keeps the overlapping indices
|
||||
# from being freed as duplicates.
|
||||
prompt_key = RadixKey(
|
||||
req.origin_input_ids,
|
||||
req.extra_key,
|
||||
is_bigram=self.tree_core.is_eagle,
|
||||
cache_salt=req.cache_salt,
|
||||
).page_aligned(self.page_size)
|
||||
if (
|
||||
len(self._components_tuple) == 1
|
||||
and self._components_tuple[0].component_type == BASE_COMPONENT_TYPE
|
||||
and 0 < len(prompt_key) < len(radix_key)
|
||||
):
|
||||
self.insert(
|
||||
replace(
|
||||
insert_params,
|
||||
key=prompt_key,
|
||||
value=values[: len(prompt_key)],
|
||||
prev_prefix_len=len(prompt_key),
|
||||
inserted_len=len(prompt_key),
|
||||
priority=insert_params.priority + 1,
|
||||
)
|
||||
)
|
||||
|
||||
# Match prefix. SWA insertion retains one extra window before the
|
||||
# page-aligned boundary, so the normal match remains safe to repoint.
|
||||
@@ -1299,11 +1178,58 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
for comp in self._components_tuple:
|
||||
comp.cleanup_after_caching_req(
|
||||
req,
|
||||
is_finished=False,
|
||||
is_finished=is_finished,
|
||||
insert_result=result,
|
||||
insert_params=insert_params,
|
||||
)
|
||||
|
||||
if self.enable_session_radix_cache and is_finished:
|
||||
from sglang.srt.managers.schedule_batch import FINISH_ABORT
|
||||
|
||||
if not isinstance(req.finished_reason, FINISH_ABORT):
|
||||
self.session_refs.register_session_ref(
|
||||
req, leaf=result.last_device_node
|
||||
)
|
||||
|
||||
@rank_consensus(same_params=["req.rid"])
|
||||
def advance_unpublished_req(self, req: Req) -> None:
|
||||
assert not self.supports_mamba()
|
||||
token_ids = req.get_fill_ids()
|
||||
kv_indices = self.req_to_token_pool.req_to_token[
|
||||
req.kv.req_pool_idx, : len(token_ids)
|
||||
]
|
||||
insert_params = InsertParams(
|
||||
prev_prefix_len=req.kv.cache_protected_len,
|
||||
priority=req.priority or 0,
|
||||
rotation_base=req.kv_rotation_base,
|
||||
)
|
||||
effective_cache_len = len(token_ids)
|
||||
for comp in self._components_tuple:
|
||||
cache_len = comp.prepare_for_caching_req(
|
||||
req=req,
|
||||
insert_params=insert_params,
|
||||
token_ids_len=len(token_ids),
|
||||
is_finished=False,
|
||||
)
|
||||
if cache_len is not None:
|
||||
effective_cache_len = min(effective_cache_len, cache_len)
|
||||
|
||||
radix_key = RadixKey(
|
||||
token_ids[:effective_cache_len],
|
||||
req.extra_key,
|
||||
is_bigram=self.tree_core.is_eagle,
|
||||
cache_salt=req.cache_salt,
|
||||
)
|
||||
if envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS.get():
|
||||
for comp in self._components_tuple:
|
||||
comp.free_out_of_window_slots(req, len(radix_key) - 1, insert_params)
|
||||
|
||||
req.prefix_indices = kv_indices.to(dtype=torch.int64, copy=True)
|
||||
for comp in self._components_tuple:
|
||||
comp.cleanup_after_caching_req(
|
||||
req, is_finished=False, insert_params=insert_params
|
||||
)
|
||||
|
||||
# ---- Internal Helpers ----
|
||||
|
||||
def _apply_cache_actions(
|
||||
|
||||
@@ -325,25 +325,15 @@ class StreamingSession(BasePrefixCache):
|
||||
|
||||
return True
|
||||
|
||||
def try_cache_unfinished_req(
|
||||
self, req: Req, chunked: bool = False, **kwargs
|
||||
) -> bool:
|
||||
"""Handles a streaming-session mid-flight cache op:
|
||||
- chunked prefill: snapshot current KV as prefix, skip radix
|
||||
- subsequent turn: skip radix (slot already holds KV)
|
||||
Returns False for first-turn non-chunked (caller must run raw radix
|
||||
insert to set up the initial tree lock)."""
|
||||
if not _is_streaming(req):
|
||||
def try_insert_req(self, req: Req, *, up_to: int, **kwargs) -> bool:
|
||||
"""A first turn checkpoints into the tree like any request (its
|
||||
prompt prefix is tree-owned and the slot inherits that lock); later
|
||||
turns run on the slot's KV, so only the chunk cursor is kept."""
|
||||
if not _is_streaming(req) or req.session.session_id not in self.slots:
|
||||
return False
|
||||
if chunked:
|
||||
kv_indices = self.req_to_token_pool.req_to_token[
|
||||
req.kv.req_pool_idx, : req.extend_range.end
|
||||
]
|
||||
req.prefix_indices = kv_indices.to(dtype=torch.int64, copy=True)
|
||||
return True
|
||||
if req.session.session_id in self.slots:
|
||||
return True
|
||||
return False
|
||||
kv_indices = self.req_to_token_pool.req_to_token[req.kv.req_pool_idx, :up_to]
|
||||
req.prefix_indices = kv_indices.to(dtype=torch.int64, copy=True)
|
||||
return True
|
||||
|
||||
# -- BasePrefixCache abstract methods: thin adapters over try_handle_* --
|
||||
|
||||
@@ -360,12 +350,9 @@ class StreamingSession(BasePrefixCache):
|
||||
self.inner.on_release(req, inserted=inserted)
|
||||
|
||||
def insert_req(self, req: Req, **kwargs):
|
||||
self.inner.insert_req(req, **kwargs)
|
||||
|
||||
def cache_unfinished_req(self, req: Req, **kwargs):
|
||||
if self.try_cache_unfinished_req(req, **kwargs):
|
||||
if self.try_insert_req(req, **kwargs):
|
||||
return
|
||||
self.inner.cache_unfinished_req(req, **kwargs)
|
||||
self.inner.insert_req(req, **kwargs)
|
||||
|
||||
def unpin(self, req: Req) -> None:
|
||||
self.inner.unpin(req)
|
||||
|
||||
@@ -4727,7 +4727,7 @@ def get_extend_input_len_swa_limit(
|
||||
sliding_window_size: int, chunked_prefill_size: int, page_size: int
|
||||
) -> int:
|
||||
# 1. a factor of 2x is because each prefill contains chunked_prefill_size tokens,
|
||||
# and between prefills, we run the tree cache's cache_unfinished_req(),
|
||||
# and between prefills, we run the tree cache's insert_req(),
|
||||
# so we unlock the previously locked nodes.
|
||||
# 2. max is to handle the case that chunked_prefill_size is larger than sliding_window_size.
|
||||
# in that case, each prefill contains chunked_prefill_size tokens,
|
||||
|
||||
@@ -1,8 +1,15 @@
|
||||
"""Shared helpers for mem_cache unit tests."""
|
||||
|
||||
from sglang.srt.managers.schedule_batch import FINISH_LENGTH
|
||||
|
||||
|
||||
def finish_req(cache, req, up_to):
|
||||
"""insert_req, then what release_kv_cache does after it: free the rest, drop the lock."""
|
||||
"""What release_kv_cache does for a request that keeps its KV: mark it
|
||||
finished, insert_req, free the rest, drop the lock."""
|
||||
if getattr(req, "finished_reason", None) is None:
|
||||
req.finished_reason = FINISH_LENGTH(length=0)
|
||||
if hasattr(req, "_refresh_fill_ids"):
|
||||
req._refresh_fill_ids()
|
||||
cache.insert_req(req, up_to=up_to)
|
||||
cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, up_to)])
|
||||
cache.unpin(req)
|
||||
|
||||
Reference in New Issue
Block a user