Parent span scheduler.loop.prefill_batch_init records wall + the scheduler thread's CPU at both existing close points (empty admission and normal return); the CPU clock is only read when loop detail is enabled and sampled, through the safe batch_telemetry.thread_cpu_ns() wrapper so a missing clock degrades to 'not measured' instead of failing the scheduling path. Fixed children, in call order: schedule_batch_init, ready_to_load_host_cache, prepare_for_extend, alloc_for_extend (+ prefix_device_indices attribute-only, sampling_batch_info), evict_to_free_tokens, evict_for_alloc, prefill_stats (+ pending_tokens), mixed_chunk_tail. * prefill_stats wraps the whole assignment so _get_num_pending_tokens() is included; mixed_chunk_tail only exists when that branch is taken. * prefill-only shape counters (n_reqs/n_new/n_cached) are snapshotted before mixed-chunk folding; a mixed batch additionally publishes decode_reqs and merged totals under explicit *_merged_* names. * eviction is timed at the call site (outside the allocator wrapper) so rank-consensus waits are included, and covers base, SWA (x2), UnifiedSWA (x3) and the extend Mamba reclaim through evict_for_alloc_observed(). Demand, shortfall and plan are separate counters; Mamba plan and realized counts are both recorded (including zero). All of it is gated by detail on a sampled loop.
586 lines
23 KiB
Python
586 lines
23 KiB
Python
from __future__ import annotations
|
|
|
|
import logging
|
|
from array import array
|
|
from typing import TYPE_CHECKING, Any, Callable, NamedTuple, Optional, cast
|
|
|
|
import numpy as np
|
|
import torch
|
|
|
|
from sglang.kernels.ops.memory.common import (
|
|
_get_last_loc_safe_kernel as _get_last_loc_safe_kernel,
|
|
)
|
|
from sglang.kernels.ops.memory.common import get_last_loc_kernel as get_last_loc_kernel
|
|
from sglang.srt.environ import envs
|
|
from sglang.srt.mem_cache.allocator.page_interleave import page_interleave_shard_size
|
|
from sglang.srt.mem_cache.base_prefix_cache import (
|
|
BasePrefixCache,
|
|
EvictParams,
|
|
MatchPrefixParams,
|
|
MatchResult,
|
|
zero_match_result,
|
|
)
|
|
from sglang.srt.mem_cache.hicache_storage import PoolTransfer
|
|
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
|
|
from sglang.srt.mem_cache.radix_cache import RadixKey
|
|
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
|
|
from sglang.srt.observability import batch_telemetry
|
|
from sglang.srt.runtime_context import get_serving, get_spec
|
|
from sglang.srt.utils.common import ceil_align
|
|
|
|
if TYPE_CHECKING:
|
|
from sglang.srt.managers.schedule_batch import Req
|
|
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
|
|
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class RetractionBackup(NamedTuple):
|
|
cpu_tensors: Any = None
|
|
host_indices: Optional[torch.Tensor] = None
|
|
pool_transfers: Optional[list[PoolTransfer]] = None
|
|
# Set when the KV pool leaves the recurrent state to the caller.
|
|
mamba_cpu: Any = None
|
|
|
|
|
|
def kv_to_page_indices(kv_indices: torch.Tensor, page_size: int) -> np.ndarray:
|
|
return (kv_indices[::page_size] // page_size).cpu().numpy()
|
|
|
|
|
|
def kv_to_page_indices_pinned(
|
|
kv_indices: torch.Tensor, page_size: int
|
|
) -> torch.Tensor:
|
|
"""Same values/dtype as :func:`kv_to_page_indices`, in a pinned CPU tensor.
|
|
|
|
The device-to-host copy is non-blocking on the current stream. The caller
|
|
must record a CUDA event on that stream after all such copies and hand it
|
|
to the transfer worker, which materializes the tensor only after the event
|
|
(see ``materialize_state_indices``).
|
|
"""
|
|
src = kv_indices[::page_size] // page_size
|
|
out = torch.empty(src.shape, dtype=src.dtype, device="cpu", pin_memory=True)
|
|
out.copy_(src, non_blocking=True)
|
|
return out
|
|
|
|
|
|
def suffix_page_indices(
|
|
req_to_token: torch.Tensor,
|
|
req_pool_idx: int,
|
|
prefix_len: int,
|
|
seq_len: int,
|
|
page_size: int,
|
|
) -> np.ndarray:
|
|
"""Pages of a request's tokens [prefix_len, seq_len).
|
|
|
|
PD prefill and decode both call this with the same page-aligned prefix
|
|
(decode_prefix_len), so their lists pair up page by page even though their
|
|
rows point at different physical pages.
|
|
"""
|
|
return kv_to_page_indices(req_to_token[req_pool_idx, prefix_len:seq_len], page_size)
|
|
|
|
|
|
def kv_to_page_num(num_kv_indices: int, page_size: int):
|
|
return (num_kv_indices + page_size - 1) // page_size
|
|
|
|
|
|
def page_align_floor(length: int, page_size: int) -> int:
|
|
return (length // page_size) * page_size
|
|
|
|
|
|
def free_swa_out_of_window_slots(
|
|
req: Req,
|
|
pre_len: int,
|
|
*,
|
|
sliding_window_size: int,
|
|
page_size: int,
|
|
req_to_token_pool: ReqToTokenPool,
|
|
token_to_kv_pool_allocator: BaseTokenToKVPoolAllocator,
|
|
supports_prefix_sharing: bool = True,
|
|
retain_floor: int | None = None,
|
|
component_type: ComponentType = ComponentType.SWA,
|
|
free_segment: Callable[..., None] | None = None,
|
|
eviction_interval: int = 1,
|
|
reprefill_tail_tokens: int = 0,
|
|
) -> None:
|
|
if not req.kv.holds_kv:
|
|
return
|
|
|
|
# For SWA-capable tree caches, we need to evict the tokens that are not in the tree cache and also not in the sliding window
|
|
assert req.kv.cache_protected_len % page_size == 0, (
|
|
"cache_protected_len must be page aligned"
|
|
)
|
|
# Protected rows limit what can be freed, not where the interval starts.
|
|
evicted_seqlen = req.kv.get_evicted_seqlen(component_type)
|
|
if pre_len - sliding_window_size < evicted_seqlen + eviction_interval:
|
|
return
|
|
dead_lo = (
|
|
req.kv.swa_dead_lo(page_size)
|
|
if component_type == ComponentType.SWA
|
|
else req.kv.cache_protected_len
|
|
)
|
|
evicted_seqlen = max(evicted_seqlen, dead_lo)
|
|
req.kv.set_evicted_seqlen(component_type, evicted_seqlen)
|
|
|
|
if not supports_prefix_sharing:
|
|
# Nothing is inserted into a tree, so no tombstone-leaf concern; evict
|
|
# up to the window boundary (the trailing floor keeps it page-aligned).
|
|
evict_threshold = pre_len - sliding_window_size
|
|
else:
|
|
# Prefix-sharing cache: keep max(window, page) rounded up to whole pages, so
|
|
# a key that ends at page_floor(pre_len) still has a full window of live SWA,
|
|
# and the last leaf, below page_floor(seq_len), is never all-tombstone.
|
|
evict_threshold = pre_len - ceil_align(
|
|
max(sliding_window_size, page_size), page_size
|
|
)
|
|
if reprefill_tail_tokens:
|
|
# Matching excludes the replay tail before page alignment. Keep the
|
|
# SWA window behind that earlier boundary too, or a first replay
|
|
# falls back to an older checkpoint despite a Full-KV cache hit.
|
|
replay_boundary = (pre_len - reprefill_tail_tokens) // page_size * page_size
|
|
evict_threshold = min(
|
|
evict_threshold, replay_boundary - sliding_window_size
|
|
)
|
|
if retain_floor is not None and supports_prefix_sharing:
|
|
# The caller owns where the floor is (see BasePrefixCache.swa_retain_floor);
|
|
# this only promises not to free past it. Without prefix sharing a retained
|
|
# checkpoint could never be matched, so holding it is pure cost.
|
|
evict_threshold = min(evict_threshold, retain_floor)
|
|
|
|
new_evicted_seqlen = max(evicted_seqlen, evict_threshold)
|
|
|
|
if page_size > 1:
|
|
new_evicted_seqlen = (new_evicted_seqlen // page_size) * page_size
|
|
|
|
if new_evicted_seqlen > evicted_seqlen:
|
|
free_slots = req_to_token_pool.req_to_token[
|
|
req.kv.req_pool_idx, evicted_seqlen:new_evicted_seqlen
|
|
]
|
|
if free_segment is None:
|
|
assert component_type == ComponentType.SWA
|
|
free_segment = token_to_kv_pool_allocator.free_swa_segment
|
|
free_segment(free_slots, start_pos=evicted_seqlen)
|
|
req.kv.set_evicted_seqlen(component_type, new_evicted_seqlen)
|
|
|
|
|
|
def coalesce_ranges(ranges: list[tuple[int, int]]) -> list[tuple[int, int]]:
|
|
"""Merge adjacent half-open ranges so a split that falls mid-page frees that page once."""
|
|
merged: list[tuple[int, int]] = []
|
|
for start, end in ranges:
|
|
if merged and start == merged[-1][1]:
|
|
merged[-1] = (merged[-1][0], end)
|
|
else:
|
|
merged.append((start, end))
|
|
return merged
|
|
|
|
|
|
def free_kv_row_segments(
|
|
allocator: BaseTokenToKVPoolAllocator,
|
|
segments: list[tuple[torch.Tensor, int]],
|
|
*,
|
|
swa_evicted_seqlen: int,
|
|
swa_dead_lo: int = 0,
|
|
) -> None:
|
|
"""Free ascending disjoint ``(kv_indices, start_pos)`` segments of one
|
|
request's kv row; ``[swa_dead_lo, swa_evicted_seqlen)`` goes back full-side only."""
|
|
dead_lo, dead_hi = swa_dead_lo, max(swa_evicted_seqlen, swa_dead_lo)
|
|
swa_dead: list[tuple[torch.Tensor, int]] = []
|
|
swa_alive: list[tuple[torch.Tensor, int]] = []
|
|
for kv_indices, start_pos in segments:
|
|
end_pos = start_pos + kv_indices.numel()
|
|
lo = min(max(dead_lo, start_pos), end_pos)
|
|
hi = min(max(dead_hi, start_pos), end_pos)
|
|
# start_pos <= lo <= hi <= end_pos
|
|
# inside [lo, hi) is the swa dead segment
|
|
if hi <= lo:
|
|
swa_alive.append((kv_indices, start_pos))
|
|
continue
|
|
swa_dead.append((kv_indices[lo - start_pos : hi - start_pos], lo))
|
|
|
|
if lo > start_pos:
|
|
swa_alive.append((kv_indices[: lo - start_pos], start_pos))
|
|
if end_pos > hi:
|
|
swa_alive.append((kv_indices[hi - start_pos :], hi))
|
|
|
|
if swa_dead and swa_alive:
|
|
# The two sides are separate calls, so neither one's page-disjointness
|
|
# check sees a boundary that splits a page between them.
|
|
assert swa_evicted_seqlen % allocator.page_size == 0, (
|
|
f"SWA eviction cursor {swa_evicted_seqlen} splits a page "
|
|
f"(page_size {allocator.page_size})"
|
|
)
|
|
if swa_dead:
|
|
allocator.free_full_segments(swa_dead)
|
|
if swa_alive:
|
|
allocator.free_segments(swa_alive)
|
|
|
|
|
|
def _req_radix_key(
|
|
tree_cache: BasePrefixCache,
|
|
req: Req,
|
|
token_ids: array[int],
|
|
max_prefix_len: Optional[int] = None,
|
|
) -> RadixKey:
|
|
# A reused prefix may carry stale or unwritten SWA (the unified_kv ring, decoder
|
|
# bounded replay); cap the match so the window is re-prefilled.
|
|
reprefill_tail = tree_cache.swa_reprefill_tail_tokens(req)
|
|
key_limit = max(0, len(token_ids) - reprefill_tail) if reprefill_tail else None
|
|
if max_prefix_len is not None:
|
|
key_limit = (
|
|
max_prefix_len if key_limit is None else min(key_limit, max_prefix_len)
|
|
)
|
|
return RadixKey(
|
|
token_ids=token_ids,
|
|
extra_key=req.extra_key,
|
|
limit=key_limit,
|
|
cache_salt=req.cache_salt,
|
|
)
|
|
|
|
|
|
def touch_waiting_prefix(req: Req, tree_cache: BasePrefixCache) -> None:
|
|
tree_cache.touch_prefix(
|
|
_req_radix_key(tree_cache, req, req.origin_input_ids + req.output_ids)
|
|
)
|
|
|
|
|
|
def match_kv_cache(
|
|
req: Req,
|
|
tree_cache: BasePrefixCache,
|
|
token_ids: Optional[array] = None,
|
|
*,
|
|
cow_mamba: bool = False,
|
|
max_prefix_len: Optional[int] = None,
|
|
) -> MatchResult:
|
|
"""Match token_ids against tree_cache and adopt the hit as req's prefix."""
|
|
if token_ids is None:
|
|
token_ids = req.origin_input_ids + req.output_ids
|
|
|
|
match_result = tree_cache.match_prefix(
|
|
MatchPrefixParams(
|
|
key=_req_radix_key(tree_cache, req, token_ids, max_prefix_len),
|
|
cow_mamba=cow_mamba,
|
|
req=req,
|
|
)
|
|
)
|
|
if envs.SGLANG_RADIX_FORCE_MISS.get():
|
|
match_result = zero_match_result(
|
|
tree_cache, match_result, extra_key=req.extra_key
|
|
)
|
|
req.prefix_len = match_result.device_prefix_len
|
|
(
|
|
req.last_node,
|
|
req.last_host_node,
|
|
req.best_match_node,
|
|
req.host_hit_length,
|
|
req.swa_host_hit_length,
|
|
req.mamba_host_hit_length,
|
|
) = (
|
|
match_result.last_device_node,
|
|
match_result.last_host_node,
|
|
match_result.best_match_node,
|
|
match_result.host_hit_length,
|
|
match_result.swa_host_hit_length,
|
|
match_result.mamba_host_hit_length,
|
|
)
|
|
max_len = req._compute_max_prefix_len(len(token_ids))
|
|
req.num_matched_prefix_tokens = min(req.prefix_len + req.host_hit_length, max_len)
|
|
req.swa_branching_seqlen = match_result.swa_branching_seqlen
|
|
# Observability: record the Full-KV length the components could have used.
|
|
# This function also runs on waiting-queue probes, PD decode and HiCache
|
|
# rematches, so it only stores the gap; the metric is emitted exactly once
|
|
# per prefill admission in PrefillAdder._commit_prefill_admission, where the
|
|
# accepted prefix length is final (after any host load-back).
|
|
req.full_kv_hit_length = match_result.full_kv_hit_length
|
|
req.swa_gate_reject_tokens = max(
|
|
0,
|
|
match_result.full_kv_hit_length
|
|
- (match_result.device_prefix_len + match_result.host_hit_length),
|
|
)
|
|
# A probe match keeps what it did not report; a new round resets both.
|
|
if match_result.mamba_branching_seqlen is not None:
|
|
req.mamba_branching_seqlen = match_result.mamba_branching_seqlen
|
|
if match_result.cache_protected_len is not None:
|
|
req.kv.cache_protected_len = match_result.cache_protected_len
|
|
return match_result
|
|
|
|
|
|
def checkpoint_kv_cache(req: Req, tree_cache: BasePrefixCache) -> None:
|
|
"""Publish what the running request has computed so far, unless it is
|
|
barred from the tree."""
|
|
# The tree reads req.finished() to tell a checkpoint from the final
|
|
# insert; a finished request belongs in release_kv_cache.
|
|
assert not req.finished(), f"checkpointing finished request {req.rid}"
|
|
if not req.skip_radix_cache_insert:
|
|
tree_cache.checkpoint(req, up_to=req.extend_end)
|
|
# The next extend resumes after this one, published or not.
|
|
req.prefix_len = req.extend_end
|
|
|
|
|
|
def _rank_consensus_enabled() -> bool:
|
|
"""Whether the rank-consensus checker is active (cheap, best effort)."""
|
|
try:
|
|
from sglang.srt.utils import rank_consensus_checker
|
|
|
|
return bool(rank_consensus_checker.enabled())
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
def evict_for_alloc_observed(tree_cache, params, **kwargs):
|
|
"""Time ``tree_cache.evict_for_alloc`` and record what it realized.
|
|
|
|
Kept at the call site (not inside the callee) so decorator/consensus waits
|
|
are part of the measurement. ``kwargs`` forwards the unified allocators'
|
|
``allocation_reclaim_satisfied`` predicate.
|
|
"""
|
|
with batch_telemetry.detail_phase("evict_for_alloc"):
|
|
result = tree_cache.evict_for_alloc(params, **kwargs)
|
|
if batch_telemetry.loop_detail_enabled() and result is not None:
|
|
batch_telemetry.detail_count("evict_for_alloc_calls")
|
|
batch_telemetry.detail_attr_sum(
|
|
"evict_plan_requested_tokens", int(getattr(params, "num_tokens", 0) or 0)
|
|
)
|
|
batch_telemetry.detail_attr_sum(
|
|
"evict_plan_requested_swa_tokens",
|
|
int(getattr(params, "swa_num_tokens", 0) or 0),
|
|
)
|
|
batch_telemetry.detail_attr_sum(
|
|
"evict_plan_requested_mamba",
|
|
int(getattr(params, "mamba_num", 0) or 0),
|
|
)
|
|
batch_telemetry.detail_attr_sum(
|
|
"evict_evicted_tokens",
|
|
int(getattr(result, "num_tokens_evicted", 0) or 0),
|
|
)
|
|
batch_telemetry.detail_attr_sum(
|
|
"evict_evicted_swa_tokens",
|
|
int(getattr(result, "swa_num_tokens_evicted", 0) or 0),
|
|
)
|
|
# Record the realized Mamba count unconditionally (including zero) so
|
|
# "no Mamba demand" and "demand but nothing reclaimed" stay distinct.
|
|
batch_telemetry.detail_attr_sum(
|
|
"evict_evicted_mamba",
|
|
int(getattr(result, "mamba_num_evicted", 0) or 0),
|
|
)
|
|
return result
|
|
|
|
|
|
def evict_from_tree_cache(
|
|
tree_cache: BasePrefixCache | None, num_tokens: int
|
|
) -> bool | None:
|
|
if tree_cache is not None and tree_cache.supports_prefix_sharing():
|
|
allocator = tree_cache.token_to_kv_pool_allocator
|
|
with batch_telemetry.detail_phase("evict_to_free_tokens"):
|
|
if batch_telemetry.loop_detail_enabled():
|
|
available_before = allocator.available_size()
|
|
batch_telemetry.detail_count("evict_calls")
|
|
# Demand is what the batch wants allocatable; shortfall is the
|
|
# part that eviction must actually recover. Keep them apart:
|
|
# summing demand as "requested eviction" overstates eviction.
|
|
batch_telemetry.detail_attr_sum(
|
|
"evict_demand_tokens", int(num_tokens)
|
|
)
|
|
batch_telemetry.detail_attr_sum(
|
|
"evict_shortfall_tokens",
|
|
max(0, int(num_tokens) - int(available_before)),
|
|
)
|
|
batch_telemetry.detail_attr_setdefault(
|
|
"evict_free_tokens_before", int(available_before)
|
|
)
|
|
batch_telemetry.detail_attr(
|
|
"evict_rank_consensus_checker", _rank_consensus_enabled()
|
|
)
|
|
result = allocator.evict_to_free_tokens(tree_cache, num_tokens)
|
|
if batch_telemetry.loop_detail_enabled():
|
|
batch_telemetry.detail_attr(
|
|
"evict_free_tokens_after", int(allocator.available_size())
|
|
)
|
|
return result
|
|
|
|
|
|
def _evict_until_allocatable(
|
|
tree_cache: BasePrefixCache, allocator, num_tokens: int
|
|
) -> None:
|
|
"""Keep evicting the shortfall until `num_tokens` are allocatable.
|
|
|
|
Under classed page sharding available_size() reports the MIN-CLASS
|
|
capacity floor, so a single evict() sized in tokens can raise that floor by
|
|
less than the number of tokens it freed: the evicted pages spread across
|
|
all owner classes. Looping is deterministic, so it stays mirrored across
|
|
the ranks of a shard group. Stock allocators need no extra pass.
|
|
"""
|
|
if page_interleave_shard_size(allocator) <= 1:
|
|
return
|
|
while True:
|
|
available_size = allocator.available_size()
|
|
if available_size >= num_tokens:
|
|
return
|
|
shortfall = num_tokens - available_size
|
|
result = tree_cache.evict(
|
|
EvictParams(num_tokens=max(shortfall, allocator.page_size))
|
|
)
|
|
if result.num_tokens_evicted == 0:
|
|
return
|
|
|
|
|
|
def dsv4_needs_rebootstrap(
|
|
token_to_kv_pool_allocator: BaseTokenToKVPoolAllocator,
|
|
) -> bool:
|
|
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
|
|
|
|
pool = token_to_kv_pool_allocator.get_kvcache()
|
|
# No DeepSeek-V4 pool keeps a host copy of a request's KV; the prefill recomputes it.
|
|
return isinstance(pool, DeepSeekV4TokenToKVPool)
|
|
|
|
|
|
def backup_kv_cache(
|
|
req: Req,
|
|
tree_cache: BasePrefixCache,
|
|
req_to_token_pool: ReqToTokenPool,
|
|
token_to_kv_pool_allocator: BaseTokenToKVPoolAllocator,
|
|
backend: str,
|
|
) -> bool:
|
|
"""Returns False when the request can be neither backed up nor recomputed;
|
|
the caller aborts it."""
|
|
if backend == "none":
|
|
return False
|
|
if dsv4_needs_rebootstrap(token_to_kv_pool_allocator):
|
|
if req.multimodal_inputs is not None:
|
|
# The recompute payload carries token ids only; images would be lost.
|
|
return False
|
|
# Drain the in-flight forward before its slots can receive recomputed KV.
|
|
device = token_to_kv_pool_allocator.get_kvcache().device
|
|
torch.get_device_module(device).synchronize(device)
|
|
return True
|
|
if backend == "cpu_tensor":
|
|
req.offload_kv_cache(req_to_token_pool, token_to_kv_pool_allocator)
|
|
return True
|
|
if backend != "host_pool":
|
|
raise ValueError(f"Unknown retraction backup backend: {backend}")
|
|
if req.seqlen <= 1:
|
|
return True
|
|
|
|
unified_cache = cast("UnifiedRadixCache", tree_cache)
|
|
req.kv.retraction_backup = unified_cache.backup_kv_cache(req)
|
|
return req.kv.retraction_backup is not None
|
|
|
|
|
|
def restore_kv_cache(
|
|
req: Req,
|
|
tree_cache: BasePrefixCache,
|
|
req_to_token_pool: ReqToTokenPool,
|
|
token_to_kv_pool_allocator: BaseTokenToKVPoolAllocator,
|
|
backend: str,
|
|
) -> None:
|
|
if backend == "cpu_tensor":
|
|
req.load_kv_cache(req_to_token_pool, token_to_kv_pool_allocator)
|
|
return
|
|
if backend != "host_pool":
|
|
raise ValueError(f"Unknown retraction backup backend: {backend}")
|
|
if req.seqlen <= 1:
|
|
return
|
|
|
|
unified_cache = cast("UnifiedRadixCache", tree_cache)
|
|
assert req.kv.retraction_backup is not None
|
|
unified_cache.restore_kv_cache(req, req.kv.retraction_backup)
|
|
req.kv.retraction_backup = None
|
|
|
|
|
|
def discard_kv_cache_backup(
|
|
req: Req, tree_cache: BasePrefixCache, backend: str
|
|
) -> None:
|
|
if backend == "cpu_tensor":
|
|
req.kv.retraction_backup = None
|
|
return
|
|
if backend != "host_pool":
|
|
raise ValueError(f"Unknown retraction backup backend: {backend}")
|
|
if req.kv.retraction_backup is None:
|
|
return
|
|
|
|
unified_cache = cast("UnifiedRadixCache", tree_cache)
|
|
unified_cache.discard_kv_cache_backup(req.kv.retraction_backup)
|
|
req.kv.retraction_backup = None
|
|
|
|
|
|
def release_kv_cache(req: Req, tree_cache: BasePrefixCache, *, checkpoint: bool):
|
|
"""Give the request's kv row back; with ``checkpoint`` the tree first keeps
|
|
what it can key."""
|
|
assert (not req.kv.holds_kv) == req.kv.is_kv_released
|
|
# A mamba-capable cache may alloc mamba state before alloc KV cache
|
|
if not req.kv.holds_kv:
|
|
assert tree_cache.supports_mamba(), (
|
|
"Only a mamba-capable tree cache allows freeing before alloc"
|
|
)
|
|
# TODO (csy, hanming): clean up this early allocation logic
|
|
if req.kv.holds_mamba:
|
|
tree_cache.req_to_token_pool.mamba_allocator.free(
|
|
req.kv.mamba_pool_idx.unsqueeze(-1)
|
|
)
|
|
req.kv.mamba_pool_idx = None
|
|
return
|
|
if tree_cache.claim_kv_row(req):
|
|
# A streaming session detached the kv record to keep the row.
|
|
assert not req.kv.holds_kv
|
|
return
|
|
|
|
owned_kv_len = req.owned_kv_len()
|
|
checkpoint = checkpoint and not req.skip_radix_cache_insert
|
|
if checkpoint:
|
|
# A tree that takes over component state (mamba) must see the request
|
|
# finished, or the insert forks the state and the slot leaks.
|
|
assert req.finished() or not tree_cache.supports_mamba(), (
|
|
f"releasing unfinished request {req.rid} into a mamba tree"
|
|
)
|
|
# The fill-id array lags output_ids until the next prepare_for_decode.
|
|
req.refresh_fill_ids()
|
|
tree_cache.checkpoint(req, up_to=owned_kv_len)
|
|
# The protected prefix is not this req's to free.
|
|
tree_cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, owned_kv_len)])
|
|
tree_cache.unlock(req.lock)
|
|
req.lock = None
|
|
_release_overallocated_kv_indices(
|
|
req, owned_kv_len, req.kv.kv_allocated_len, tree_cache
|
|
)
|
|
tree_cache.on_release(req, checkpointed=checkpoint)
|
|
|
|
# The DSV4-NPU ReqToTokenPool subclass's free() additionally releases the
|
|
# c4/c128 state pages; other ReqToTokenPool subclasses are a no-op here.
|
|
tree_cache.req_to_token_pool.free(req)
|
|
req.kv.mark_kv_released()
|
|
|
|
|
|
def _release_overallocated_kv_indices(
|
|
req: Req, start_p: int, end_p: int, tree_cache: BasePrefixCache
|
|
) -> None:
|
|
allocator = tree_cache.token_to_kv_pool_allocator
|
|
page_size = allocator.page_size
|
|
spec_algo = get_spec().speculative_algorithm
|
|
|
|
# strip_thinking_cache intentionally reports output tokens as overallocated
|
|
# so they fall into the free path below (#22373).
|
|
if spec_algo is None and not get_serving().strip_thinking_cache:
|
|
# A stop landing before the last committed token does the same, via
|
|
# effective_kv_committed_len().
|
|
assert start_p == end_p or (
|
|
req.finished_len is not None
|
|
and len(req.origin_input_ids) + req.finished_len < req.kv.kv_committed_len
|
|
), (
|
|
f"Unexpected overallocated KV cache, {req.kv.kv_committed_len=}, {req.kv.kv_allocated_len=}"
|
|
)
|
|
|
|
# Align to the ALLOCATOR's page, which under DCP is wider than the kernel
|
|
# page: paged free() releases the whole page containing any freed index, so
|
|
# a boundary aligned only to the kernel page could free a widened page whose
|
|
# head rows are still live.
|
|
if page_size > 1:
|
|
start_p = ceil_align(start_p, page_size)
|
|
|
|
if start_p < end_p:
|
|
# start_p is aligned to the allocator's page above, so it never shares a
|
|
# page with the tail free_kv_row in this group.
|
|
tree_cache.free_kv_row(req.kv, [(start_p, end_p)])
|
|
|
|
|
|
def available_and_evictable_str(tree_cache: BasePrefixCache) -> str:
|
|
return tree_cache.available_and_evictable_str()
|