[mem_cache] Hold a request's tree lock as one TreeLock (#42686)

This commit is contained in:
Liangsheng Yin
2026-10-06 00:59:38 -07:00
committed by GitHub
parent 05340e9ecb
commit c7f3ae1dd5
30 changed files with 246 additions and 431 deletions
+11 -25
View File
@@ -83,8 +83,8 @@ from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
from sglang.srt.mem_cache.allocator.swa import is_swa_req_ring
from sglang.srt.mem_cache.base_prefix_cache import (
BasePrefixCache,
DecLockRefParams,
EvictParams,
TreeLock,
)
from sglang.srt.mem_cache.common import (
RetractionBackup,
@@ -147,7 +147,7 @@ def _bind_root_prefix(req: Req, tree_cache: BasePrefixCache) -> None:
req.last_node = tree_cache.root_node_handle(req.extra_key)
req.last_host_node = req.last_node
req.best_match_node = req.last_node
req.lock_receipt = DecLockRefParams()
req.lock = None
req.kv.cache_protected_len = 0
req.kv.cache_inserted_len = 0
req.num_matched_prefix_tokens = 0
@@ -351,9 +351,8 @@ class DecodeRequest:
# HiCache Status
prefix_match: Optional[DecodePrefixMatch] = None
hicache_restored_kv_indices: Optional[torch.Tensor] = None
hicache_restored_node: Any = None
# Receipt for the inc_lock_ref held on hicache_restored_node.
hicache_restore_lock_receipt: Optional[DecLockRefParams] = None
# The lock held on the restored node until the commit hands it to the req.
hicache_restore_lock: Optional[TreeLock] = None
hicache_load_consumer_index: int = -1
hicache_restore_status: HiCacheRestoreResult = HiCacheRestoreResult.PENDING
@@ -503,13 +502,6 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
swa_budget_tokens=swa_allocatable_tokens,
)
def _release_matched_prefix_lock(self, req: Req) -> None:
if req.swa_prefix_lock_released:
self.tree_cache.dec_lock_ref(req.last_node, req.lock_receipt, skip_swa=True)
req.swa_prefix_lock_released = False
else:
self.tree_cache.dec_lock_ref(req.last_node, req.lock_receipt)
def _reclaim_swa_tail_capacity(
self, swa_tail_len: int, req_id: str, *, full_len: int = 0
) -> Optional[str]:
@@ -801,12 +793,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
include_req=True,
max_prefix_len=max_prefix_len,
)
# Keep aggregated scheduling semantics while preserving the SWA lock
# boundary needed for the matching dec_lock_ref; the full receipt
# travels on the req so every later release mirrors this acquire.
req.lock_receipt = self.tree_cache.inc_lock_ref(
result.last_device_node
).to_dec_params()
req.lock = self.tree_cache.lock(result.last_device_node)
return self._build_decode_prefix_match(
req, result, max_prefix_len=max_prefix_len
)
@@ -1436,12 +1423,9 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
if (
uses_swa_tail_prealloc
and prefix_match.l1_prefix_len > 0
and hasattr(self.tree_cache, "dec_swa_lock_only")
and hasattr(self.tree_cache, "release_swa")
):
self.tree_cache.dec_swa_lock_only(
decode_req.req.last_node, decode_req.req.lock_receipt
)
decode_req.req.swa_prefix_lock_released = True
self.tree_cache.release_swa(decode_req.req.lock)
required_alloc_tokens = self._required_alloc_tokens(
fill_len=fill_len, prefix_len=prefix_len
@@ -1494,7 +1478,8 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
swa_allocatable_tokens=swa_allocatable_tokens,
):
if prefix_match is not None and prefix_match.l1_prefix_len > 0:
self._release_matched_prefix_lock(decode_req.req)
self.tree_cache.unlock(decode_req.req.lock)
decode_req.req.lock = None
break
if swa_allocatable_tokens is not None:
@@ -1505,7 +1490,8 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
)
if reclaim_error is not None:
if prefix_match is not None and prefix_match.l1_prefix_len > 0:
self._release_matched_prefix_lock(decode_req.req)
self.tree_cache.unlock(decode_req.req.lock)
decode_req.req.lock = None
logger.error(reclaim_error)
prepare_abort(
decode_req.req,
@@ -161,7 +161,7 @@ class DecodeHiCachePreallocMixin:
for dr in self.transfer_queue.queue
if dr.prefix_match is not None
and dr.hicache_restore_status == HiCacheRestoreResult.PENDING
and dr.hicache_restored_node is None
and dr.hicache_restore_lock is None
)
@@ -192,19 +192,14 @@ class DecodeHiCacheTransferMixin:
self.tree_cache.finish(
decode_req.req.cache_request_handle, CacheRequestOutcome.ABORT
)
if decode_req.hicache_restored_node is not None:
self.tree_cache.dec_lock_ref(
decode_req.hicache_restored_node,
decode_req.hicache_restore_lock_receipt,
)
decode_req.hicache_restored_node = None
decode_req.hicache_restore_lock_receipt = None
self.tree_cache.unlock(decode_req.hicache_restore_lock)
decode_req.hicache_restore_lock = None
def _try_hicache_queue_load_back(self, dr: DecodeRequest) -> bool:
"""Queue one L2->L1 load_back op for ``dr``; True iff a DMA was queued.
On success, ``dr.hicache_restored_node`` and ``hicache_restored_kv_indices``
are populated, and an inc_lock_ref is held until commit/abort.
On success, ``dr.hicache_restore_lock`` and ``hicache_restored_kv_indices``
are populated; the lock is held until commit/abort.
Trivial cases (all-on-device / no needed coverage) auto-flip to READY.
Failback paths flip to FAILED.
"""
@@ -233,10 +228,7 @@ class DecodeHiCacheTransferMixin:
)
)
# The rematch repointed req.last_node to feed init_load_back's device
# boundary, but the prealloc lock and the receipt on the req still
# belong to pm.last_device_node; restore the pairing so any release
# before the commit hands over the restored lock hits the right node
# (the receipt's anchor makes a mispaired release assert).
# boundary; point it back at the prefix the prealloc matched and locked.
dr.req.last_node = pm.last_device_node
# Failback: total coverage < required prefix means device alloc likely failed.
if len(rematch.device_indices) + len(new_indices) < pm.decode_prefix_len:
@@ -257,10 +249,7 @@ class DecodeHiCacheTransferMixin:
dr.hicache_restored_kv_indices = torch.cat(
[rematch.device_indices[pm.l1_prefix_len :], new_indices]
)[: pm.restore_token_count]
dr.hicache_restored_node = restored_node
dr.hicache_restore_lock_receipt = self.tree_cache.inc_lock_ref(
restored_node
).to_dec_params()
dr.hicache_restore_lock = self.tree_cache.lock(restored_node)
if len(new_indices) == 0:
# Whole prefix already on device; no DMA needed.
@@ -284,7 +273,7 @@ class DecodeHiCacheTransferMixin:
# Phase A: advance in-flight DMAs to READY.
for dr in active:
if (
dr.hicache_restored_node is not None
dr.hicache_restore_lock is not None
and self.tree_cache.is_load_back_event_done(
dr.hicache_load_consumer_index
)
@@ -297,8 +286,7 @@ class DecodeHiCacheTransferMixin:
queued = [
dr
for dr in active
if dr.hicache_restored_node is None
and self._try_hicache_queue_load_back(dr)
if dr.hicache_restore_lock is None and self._try_hicache_queue_load_back(dr)
]
if not queued:
return
@@ -318,16 +306,9 @@ class DecodeHiCacheTransferMixin:
return
req = decode_req.req
restored_node = decode_req.hicache_restored_node
restored_lock_receipt = decode_req.hicache_restore_lock_receipt
assert restored_node is not None
assert restored_lock_receipt is not None
# Release preallocation before installing the restored lock receipt.
self.tree_cache.dec_lock_ref(
prefix_match.last_device_node,
req.lock_receipt,
skip_swa=req.swa_prefix_lock_released,
)
restore_lock = decode_req.hicache_restore_lock
assert restore_lock is not None
self.tree_cache.unlock(req.lock)
self.tree_cache.req_to_token_pool.write(
(
@@ -339,9 +320,7 @@ class DecodeHiCacheTransferMixin:
req.prefix_indices = torch.cat(
[prefix_match.prefix_indices, decode_req.hicache_restored_kv_indices]
)
req.last_node = restored_node
req.lock_receipt = restored_lock_receipt
req.swa_prefix_lock_released = False
req.last_node = restore_lock.node
req.lock = restore_lock
# Prevent abort cleanup from releasing the transferred lock.
decode_req.hicache_restored_node = None
decode_req.hicache_restore_lock_receipt = None
decode_req.hicache_restore_lock = None
@@ -337,7 +337,7 @@ class DecodeKVCacheOffloadManager:
def _release_finished_req(self, req: Req):
# Defensive guard: ReqToTokenPool.free sets req_pool_idx to None,
# so a previously-released request must be skipped here to avoid
# non-idempotent side effects (double free, double unpin).
# non-idempotent side effects (double free, double unlock).
if req.kv.req_pool_idx is None or req.kv.req_pool_idx == -1:
return
@@ -347,7 +347,8 @@ class DecodeKVCacheOffloadManager:
self.tree_cache.free_kv_row(
req.kv, [(req.kv.cache_protected_len, req.kv.kv_allocated_len)]
)
self.tree_cache.unpin(req)
self.tree_cache.unlock(req.lock)
req.lock = None
self.req_to_token_pool.free(req)
req.kv.mark_kv_released()
self.offloaded_state.pop(req, None)
+5 -8
View File
@@ -107,8 +107,8 @@ from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
from sglang.srt.mem_cache.base_prefix_cache import (
BasePrefixCache,
CacheRequestHandle,
DecLockRefParams,
MatchPrefixParams,
TreeLock,
zero_match_result,
)
from sglang.srt.mem_cache.common import (
@@ -1207,14 +1207,12 @@ class Req(ReqDllmMixin):
self.host_hit_is_storage = False
self.storage_prefetch_retry_attempts = 0
self.staged_prefetch_plan: Optional[StagedPrefetchPlan] = None
# Receipt of the tree lock held on last_node (anchor, SWA boundary,
# skipped components); every release replays it unchanged.
self.lock_receipt: DecLockRefParams = DecLockRefParams()
# The tree lock this request holds; None while it runs on a session
# slot's record or holds no lock.
self.lock: Optional[TreeLock] = None
# Device/host prefix used to plan the latest L3 lookup. Admission uses
# it to detect newly exposed storage demand after queue-time eviction.
self.storage_prefetch_last_match_len: Optional[int] = None
# Whether the prefill-time SWA tree lock has been released early
self.swa_prefix_lock_released: bool = False
# Logical-page KV sharding: rotation base of the chain this request
# extends (owner of position-page P is (base + P) % shard_size).
# Refreshed at every sharded alloc — read through last_node, or drawn
@@ -1984,8 +1982,7 @@ class Req(ReqDllmMixin):
self.kv.cache_inserted_len = 0
self.kv_rotation_base = None
self.num_matched_prefix_tokens = 0
self.lock_receipt = DecLockRefParams()
self.swa_prefix_lock_released = False
self.lock = None
self.swa_branching_seqlen = None
self.extend_range = None
self.dllm_initialized = False
@@ -1038,10 +1038,6 @@ class PrefillAdder:
)
self._account_prefill_cache_admission(req, prefix_len)
def _req_inc_lock_ref(self, req: Req):
# Persist the release receipt.
req.lock_receipt = self.tree_cache.inc_lock_ref(req.last_node).to_dec_params()
def _kv_shard_reserve_scratch(self, prefix_len: int, extend_len: int) -> bool:
"""Reserve scratch or return False to defer; no-op when sharding is off.
@@ -1205,13 +1201,11 @@ class PrefillAdder:
else None
)
try:
# Replay the acquire's receipt (SWA boundary uuid, mamba flag) so the
# release takes back exactly what this temporary lock took.
dec_lock_params = self.tree_cache.inc_lock_ref(last_node).to_dec_params()
lock = self.tree_cache.lock(last_node)
try:
yield None
finally:
self.tree_cache.dec_lock_ref(last_node, dec_lock_params)
self.tree_cache.unlock(lock)
finally:
if host_lock_params is not None:
self.tree_cache.dec_host_lock_ref(last_node, host_lock_params)
@@ -1612,7 +1606,7 @@ class PrefillAdder:
req.set_extend_range(
admission.prefix_len, admission.prefix_len + admission.extend_len
)
self._req_inc_lock_ref(req)
req.lock = self.tree_cache.lock(req.last_node)
self.can_run_list.append(req)
if admission.is_chunked:
self.new_chunked_req = req
@@ -169,9 +169,7 @@ class DynamicChunkSizer:
# Walk the same match -> lock -> alloc lifecycle as a scheduled
# request so release_kv_cache can release it symmetrically.
req.init_next_round_input(self.tree_cache)
req.lock_receipt = self.tree_cache.inc_lock_ref(
req.last_node
).to_dec_params()
req.lock = self.tree_cache.lock(req.last_node)
req.set_extend_range(
len(req.prefix_indices), len(req.full_untruncated_fill_ids)
)
@@ -240,6 +240,16 @@ class DecLockRefParams:
return uuids[component_type]
@dataclasses.dataclass
class TreeLock:
"""``receipt`` replays the acquire on release; ``swa_released`` marks the
SWA part released early, so neither release takes it twice."""
node: Any
receipt: DecLockRefParams
swa_released: bool = False
@dataclasses.dataclass
class DecLockRefResult:
"""Result of an dec_lock_ref operation."""
@@ -510,11 +520,13 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
) -> DecLockRefResult:
pass
def unpin(self, req: Req) -> None:
"""Drop the tree lock the request holds on ``req.last_node``; a cache
whose acquire returned a receipt releases with it here."""
if req.last_node is not None:
self.dec_lock_ref(req.last_node)
def lock(self, node: Any) -> Optional[TreeLock]:
"""Take a tree lock on ``node`` for one holder; ``unlock`` releases it."""
return TreeLock(node, self.inc_lock_ref(node).to_dec_params())
def unlock(self, lock: Optional[TreeLock]) -> None:
if lock is not None:
self.dec_lock_ref(lock.node, lock.receipt)
def maybe_hand_to_session(self, req: Req) -> None:
"""A cache that keeps records across requests (a streaming session) takes
+2 -1
View File
@@ -328,7 +328,8 @@ def release_kv_cache(req: Req, tree_cache: BasePrefixCache, *, checkpoint: bool)
tree_cache.checkpoint(req, up_to=owned_kv_len)
# The protected prefix is not this req's to free.
tree_cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, owned_kv_len)])
tree_cache.unpin(req)
tree_cache.unlock(req.lock)
req.lock = None
_release_overallocated_kv_indices(
req, owned_kv_len, req.kv.kv_allocated_len, tree_cache
)
+2 -2
View File
@@ -527,8 +527,8 @@ class RadixCache(BasePrefixCache):
# checkpoint or release_kv_cache frees the rest.
req.kv.cache_protected_len = len(new_indices)
self.dec_lock_ref(req.last_node)
self.inc_lock_ref(new_last_node)
self.unlock(req.lock)
req.lock = self.lock(new_last_node)
# `req.prefix_indices` will be used in `PrefillAdder::add_chunked_req` later
# - page_size != 1: there is a partial page at the end, keep the full kv_indices
@@ -832,9 +832,8 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
)
# Move the request lock to the published boundary.
if req.last_node is not None:
self._dec_req_lock(req)
lock_result = self.inc_lock_ref(matched.last_device_node)
self.unlock(req.lock)
req.lock = self.lock(matched.last_device_node)
if total_hit < token_ids_len:
req.prefix_indices = torch.cat(
[canonical, kv_indices[total_hit:].to(dtype=torch.int64, copy=True)]
@@ -843,8 +842,6 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
req.prefix_indices = canonical
req.kv.cache_protected_len = total_hit
req.last_node = matched.last_device_node
req.lock_receipt = lock_result.to_dec_params()
req.swa_prefix_lock_released = False
flow.request_mamba_value = None
flow.allocated_request_mamba_for_load = False
flow.load_req = None
@@ -237,10 +237,10 @@ Insert a request's KV into the tree: at every checkpoint while it runs, and once
| Aspect | Detail |
|--------|--------|
| **Purpose** | Publish the request's KV `[cache_protected_len, up_to)` so other requests can match it; nodes past `req.kv.cache_inserted_len` count one hit (a request counts each node once). `checkpoint_kv_cache` calls it with `req.extend_range.end` while the request runs; `release_kv_cache` calls it with the request-owned length when it finishes (`req.finished()`), then frees `[cache_protected_len, up_to)` and everything past it and unpins; with `checkpoint=False` it skips the insert and calls `on_release(req, checkpointed=False)` for component cleanup |
| **Purpose** | Publish the request's KV `[cache_protected_len, up_to)` so other requests can match it; nodes past `req.kv.cache_inserted_len` count one hit (a request counts each node once). `checkpoint_kv_cache` calls it with `req.extend_range.end` while the request runs; `release_kv_cache` calls it with the request-owned length when it finishes (`req.finished()`), then frees `[cache_protected_len, up_to)` and everything past it and unlocks `req.lock`; with `checkpoint=False` it skips the insert and calls `on_release(req, checkpointed=False)` for component cleanup |
| **Inputs** | `req` — the request; `up_to` — row position the insert may read up to |
| **Output** | `None` |
| **Mutation** | Component hooks → `insert` → re-match → writes the tree's indices back into the row → moves the request's lock from the old `req.last_node` to the node the insert ended on → updates `req.prefix_indices`, `req.kv.cache_protected_len`, `req.kv.cache_inserted_len`, `req.last_node` → component cleanup. A finished request's component state (Mamba) is handed to the tree instead of forked, and what it still held is freed. Frees no KV slot: `release_kv_cache` does that afterwards. |
| **Mutation** | Component hooks → `insert` → re-match → writes the tree's indices back into the row → moves `req.lock` to the node the insert ended on → updates `req.prefix_indices`, `req.kv.cache_protected_len`, `req.kv.cache_inserted_len`, `req.last_node` → component cleanup. A finished request's component state (Mamba) is handed to the tree instead of forked, and what it still held is freed. Frees no KV slot: `release_kv_cache` does that afterwards. |
| **Complexity** | **O(K + D·C)** — insert O(K + D·C) + re-match O(K + D·C) + lock transfer O(D). Simplifies to **O(K)**. |
**Algorithm detail:**
@@ -248,7 +248,7 @@ Insert a request's KV into the tree: at every checkpoint while it runs, and once
2. Truncates the key to `effective_cache_len`; the row past it stays the request's
3. Converts token IDs (bigram if EAGLE), page-aligns the key, then calls `insert()`; a single-component tree re-inserts the prompt part so eviction can drop the output without the prompt
4. `match_prefix()` on the inserted key and writes the matched indices into the row
5. `dec_lock_ref()` on the old `req.last_node`, `inc_lock_ref()` on the new one
5. `unlock(req.lock)`, then `req.lock = lock(new_last_node)`
6. `cleanup_after_caching_req()` per component (Mamba: frees the forked `mamba_value` when the tree already had one, frees the finished request's slot)
---
@@ -55,7 +55,7 @@ class UnifiedSessionRefTracker:
def register_session_ref(self, req: Req, leaf: NodeId) -> None:
"""Register the leaf a finished request's insert ended on with each
component; the lock anchor ``req.last_node`` is a different node."""
component; the request's lock sits on a different node."""
if not self.enable_session_radix_cache:
return
@@ -29,6 +29,7 @@ from sglang.srt.mem_cache.base_prefix_cache import (
InsertResult,
MatchPrefixParams,
MatchResult,
TreeLock,
)
from sglang.srt.mem_cache.buffer_mode.pipeline import (
BufferModePipeline,
@@ -93,7 +94,7 @@ from sglang.srt.observability.metrics_collector import (
StorageMetricsCollector,
)
from sglang.srt.runtime_context import get_memory, get_model, get_observability
from sglang.srt.session.streaming_session import StreamingSession
from sglang.srt.session.streaming_session import StreamingSession, is_virtual_node
from sglang.srt.utils.common import ceil_align
if TYPE_CHECKING:
@@ -975,9 +976,6 @@ class UnifiedRadixCache(BasePrefixCache):
def inc_lock_ref(
self, node_id: NodeId, skip_lock_components: Sequence[ComponentType] = ()
) -> IncLockRefResult:
result = self.session.try_inc_lock_ref(node_id)
if result is not None:
return result
if self.disable:
return IncLockRefResult()
return self.tree_core.inc_lock_ref(node_id, skip_lock_components)
@@ -988,45 +986,41 @@ class UnifiedRadixCache(BasePrefixCache):
params: DecLockRefParams,
skip_swa: bool = False,
) -> DecLockRefResult:
result = self.session.try_dec_lock_ref(node_id, params)
if result is not None:
return result
if self.disable:
return DecLockRefResult()
return self.tree_core.dec_lock_ref(node_id, params, skip_swa)
def _dec_req_lock(self, req: Req, *, skip_swa: bool = False) -> None:
"""Release the tree lock a request holds on its last_node with the
receipt its acquire returned, so it never drops a lock it never took."""
self.dec_lock_ref(req.last_node, req.lock_receipt, skip_swa=skip_swa)
def lock(
self, node_id: NodeId, skip_lock_components: Sequence[ComponentType] = ()
) -> Optional[TreeLock]:
# A session's virtual node stands for a lock its slot already holds.
if is_virtual_node(node_id):
return None
result = self.inc_lock_ref(node_id, skip_lock_components)
return TreeLock(node_id, result.to_dec_params())
def unpin(self, req: Req) -> None:
# Synthetic profiling requests may own KV without locking a tree node.
if req.last_node is not None:
self._dec_req_lock(req, skip_swa=req.swa_prefix_lock_released)
def dec_swa_lock_only(
self,
node_id: NodeId,
params: DecLockRefParams,
) -> None:
if self.disable:
def unlock(self, lock: Optional[TreeLock]) -> None:
if lock is None:
return
result = self.tree_core.dec_swa_lock_only(node_id, params)
self.dec_lock_ref(lock.node, lock.receipt, skip_swa=lock.swa_released)
def release_swa(self, lock: Optional[TreeLock]) -> None:
"""Release the SWA part of ``lock`` early, once."""
if self.disable or lock is None or lock.swa_released:
return
result = self.tree_core.dec_swa_lock_only(lock.node, lock.receipt)
self._free_values(result.device_frees, result.host_frees)
lock.swa_released = True
def release_swa_prefix_lock(self, req: Req) -> None:
"""The request's window has moved past its prefix: leave the prefix's
SWA evictable. A session turn releases its slot's lock, once per session."""
holder = self.session.borrowed_slot(req) or req
lock = (self.session.borrowed_slot(req) or req).lock
if (
holder.swa_prefix_lock_released
or holder.last_node is None
or holder.lock_receipt.component_lock_uuids.get(ComponentType.SWA) is None
lock is not None
and lock.receipt.component_lock_uuids.get(ComponentType.SWA) is not None
):
return
self.dec_swa_lock_only(holder.last_node, holder.lock_receipt)
holder.swa_prefix_lock_released = True
self.release_swa(lock)
def inc_host_lock_ref(self, node_id: NodeId) -> IncLockRefResult:
if self.disable:
@@ -1208,14 +1202,14 @@ class UnifiedRadixCache(BasePrefixCache):
new_indices[req.kv.cache_protected_len :],
)
self._dec_req_lock(req, skip_swa=req.swa_prefix_lock_released)
self.unlock(req.lock)
# Opt-in: leave the matched-prefix mamba evictable during decode (it is
# already COW'd to the request's own slot, never read from this node again).
# Safe only because any future COW source is the COWing request's own
# admission-locked last_node (recorded only if still present, locked before
# the next alloc) -- not this evictable node. A scheduler that matched a
# whole batch before locking would break that. Off = original full lock.
lock_result = self.inc_lock_ref(
req.lock = self.lock(
new_last_node,
skip_lock_components=(
(ComponentType.MAMBA,)
@@ -1233,10 +1227,6 @@ class UnifiedRadixCache(BasePrefixCache):
req.prefix_indices = new_indices
req.kv.cache_protected_len = len(new_indices)
req.last_node = new_last_node
# Carry the receipt so this node's dec releases only what we locked.
req.lock_receipt = lock_result.to_dec_params()
# The rematch acquired a new SWA prefix lock.
req.swa_prefix_lock_released = False
# cleanup
for comp in self._components_tuple:
+12 -42
View File
@@ -8,11 +8,9 @@ import torch
from sglang.srt.managers.schedule_batch import ReqKvInfo
from sglang.srt.mem_cache.base_prefix_cache import (
DecLockRefParams,
DecLockRefResult,
IncLockRefResult,
MatchPrefixParams,
MatchResult,
TreeLock,
)
from sglang.srt.utils.common import ceil_align, is_npu
@@ -41,27 +39,22 @@ class SessionSlot:
# KV pool state
kv: ReqKvInfo = field(default_factory=ReqKvInfo)
# Tree lock on the session's tree-owned prefix, and its receipt.
last_node: Any = None
lock_receipt: DecLockRefParams = field(default_factory=DecLockRefParams)
# Whether the SWA part of that lock was released early.
swa_prefix_lock_released: bool = False
# Tree lock on the session's tree-owned prefix.
lock: Optional[TreeLock] = None
# Until the first turn finishes or is retracted, its checkpoints publish
# the prompt into the tree and move the lock onto the deepest published node.
publishes_prompt: bool = False
def is_virtual_node(node: Any) -> bool:
"""A slot's stand-in node: locking it is a no-op."""
return isinstance(node, _VirtualNode)
def _is_streaming(req: Optional[Req]) -> bool:
return req is not None and req.session is not None and req.session.streaming
def _move_tree_lock(src: Any, dst: Any) -> None:
"""Hand the tree lock ``src`` holds (a request or a slot) to ``dst``."""
dst.last_node = src.last_node
dst.lock_receipt = src.lock_receipt
dst.swa_prefix_lock_released = src.swa_prefix_lock_released
class StreamingSession:
"""Streaming-session KV records, owned by ``UnifiedRadixCache``.
@@ -80,20 +73,6 @@ class StreamingSession:
# -- Try-handle entries (see class docstring) --
def try_inc_lock_ref(self, node: Any) -> Optional[IncLockRefResult]:
"""No-op lock if ``node`` is a session-internal sentinel; returns
None to tell the caller to run its raw tree lock path."""
if isinstance(node, _VirtualNode):
return IncLockRefResult()
return None
def try_dec_lock_ref(
self, node: Any, params: Optional[DecLockRefParams] = None
) -> Optional[DecLockRefResult]:
if isinstance(node, _VirtualNode):
return DecLockRefResult()
return None
def find_active_slot(self, req: Req) -> Optional[SessionSlot]:
"""A pre-aborted req (to_finish set) is detached from the session and
gets None; the slot stays for the next request."""
@@ -186,7 +165,7 @@ class StreamingSession:
if isinstance(req.finished_reason, FINISH_ABORT):
# Hand the record and the tree lock back; the caller releases them.
del self.slots[req.session.session_id]
_move_tree_lock(slot, req)
req.lock, slot.lock = slot.lock, None
req.session.abort_req()
return False
@@ -217,7 +196,7 @@ class StreamingSession:
if slot is None:
return False
if slot.publishes_prompt:
_move_tree_lock(slot, req)
req.lock, slot.lock = slot.lock, None
self.cache.checkpoint_into_tree(req, up_to=up_to, **kwargs)
self._lock_to_slot(req, slot)
return True
@@ -255,7 +234,6 @@ class StreamingSession:
if slot is None:
return
protected_len = slot.kv.cache_protected_len
lock_node = slot.last_node
tokens_freed = (
max(0, slot.kv.kv_allocated_len - protected_len) if slot.kv.holds_kv else 0
)
@@ -263,11 +241,7 @@ class StreamingSession:
"Session KV released: %s (%d tokens freed)", session_id, tokens_freed
)
if lock_node is not None:
# skip_swa is an SWA-cache extension kwarg; a slot can only have
# early-released when the cache supports SWA locks.
skip = {"skip_swa": True} if slot.swa_prefix_lock_released else {}
self.cache.dec_lock_ref(lock_node, slot.lock_receipt, **skip)
self.cache.unlock(slot.lock)
if slot.kv.holds_kv:
self.cache.free_kv_row(slot.kv, [(protected_len, slot.kv.kv_allocated_len)])
@@ -291,12 +265,8 @@ class StreamingSession:
# -- Internal helpers (streaming body bits) --
def _lock_to_slot(self, req: Req, slot: SessionSlot) -> None:
"""Move the request's tree lock to the slot; the request is left on the
slot's virtual node, where locking is a no-op."""
_move_tree_lock(req, slot)
slot.lock, req.lock = req.lock, None
req.last_node = slot.virtual_node
req.lock_receipt = DecLockRefParams()
req.swa_prefix_lock_released = False
def _free_tail(self, kv: ReqKvInfo, prefix_len: int) -> None:
"""Free [prefix_len, allocated) before alloc_for_extend overwrites it:
+2 -1
View File
@@ -11,4 +11,5 @@ def finish_req(cache, req, up_to):
req.refresh_fill_ids()
cache.checkpoint(req, up_to=up_to)
cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, up_to)])
cache.unpin(req)
cache.unlock(req.lock)
req.lock = None