[mem_cache] Hold a request's tree lock as one TreeLock (#42686)
This commit is contained in:
@@ -83,8 +83,8 @@ from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
|
||||
from sglang.srt.mem_cache.allocator.swa import is_swa_req_ring
|
||||
from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
BasePrefixCache,
|
||||
DecLockRefParams,
|
||||
EvictParams,
|
||||
TreeLock,
|
||||
)
|
||||
from sglang.srt.mem_cache.common import (
|
||||
RetractionBackup,
|
||||
@@ -147,7 +147,7 @@ def _bind_root_prefix(req: Req, tree_cache: BasePrefixCache) -> None:
|
||||
req.last_node = tree_cache.root_node_handle(req.extra_key)
|
||||
req.last_host_node = req.last_node
|
||||
req.best_match_node = req.last_node
|
||||
req.lock_receipt = DecLockRefParams()
|
||||
req.lock = None
|
||||
req.kv.cache_protected_len = 0
|
||||
req.kv.cache_inserted_len = 0
|
||||
req.num_matched_prefix_tokens = 0
|
||||
@@ -351,9 +351,8 @@ class DecodeRequest:
|
||||
# HiCache Status
|
||||
prefix_match: Optional[DecodePrefixMatch] = None
|
||||
hicache_restored_kv_indices: Optional[torch.Tensor] = None
|
||||
hicache_restored_node: Any = None
|
||||
# Receipt for the inc_lock_ref held on hicache_restored_node.
|
||||
hicache_restore_lock_receipt: Optional[DecLockRefParams] = None
|
||||
# The lock held on the restored node until the commit hands it to the req.
|
||||
hicache_restore_lock: Optional[TreeLock] = None
|
||||
hicache_load_consumer_index: int = -1
|
||||
hicache_restore_status: HiCacheRestoreResult = HiCacheRestoreResult.PENDING
|
||||
|
||||
@@ -503,13 +502,6 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
swa_budget_tokens=swa_allocatable_tokens,
|
||||
)
|
||||
|
||||
def _release_matched_prefix_lock(self, req: Req) -> None:
|
||||
if req.swa_prefix_lock_released:
|
||||
self.tree_cache.dec_lock_ref(req.last_node, req.lock_receipt, skip_swa=True)
|
||||
req.swa_prefix_lock_released = False
|
||||
else:
|
||||
self.tree_cache.dec_lock_ref(req.last_node, req.lock_receipt)
|
||||
|
||||
def _reclaim_swa_tail_capacity(
|
||||
self, swa_tail_len: int, req_id: str, *, full_len: int = 0
|
||||
) -> Optional[str]:
|
||||
@@ -801,12 +793,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
include_req=True,
|
||||
max_prefix_len=max_prefix_len,
|
||||
)
|
||||
# Keep aggregated scheduling semantics while preserving the SWA lock
|
||||
# boundary needed for the matching dec_lock_ref; the full receipt
|
||||
# travels on the req so every later release mirrors this acquire.
|
||||
req.lock_receipt = self.tree_cache.inc_lock_ref(
|
||||
result.last_device_node
|
||||
).to_dec_params()
|
||||
req.lock = self.tree_cache.lock(result.last_device_node)
|
||||
return self._build_decode_prefix_match(
|
||||
req, result, max_prefix_len=max_prefix_len
|
||||
)
|
||||
@@ -1436,12 +1423,9 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
if (
|
||||
uses_swa_tail_prealloc
|
||||
and prefix_match.l1_prefix_len > 0
|
||||
and hasattr(self.tree_cache, "dec_swa_lock_only")
|
||||
and hasattr(self.tree_cache, "release_swa")
|
||||
):
|
||||
self.tree_cache.dec_swa_lock_only(
|
||||
decode_req.req.last_node, decode_req.req.lock_receipt
|
||||
)
|
||||
decode_req.req.swa_prefix_lock_released = True
|
||||
self.tree_cache.release_swa(decode_req.req.lock)
|
||||
|
||||
required_alloc_tokens = self._required_alloc_tokens(
|
||||
fill_len=fill_len, prefix_len=prefix_len
|
||||
@@ -1494,7 +1478,8 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
swa_allocatable_tokens=swa_allocatable_tokens,
|
||||
):
|
||||
if prefix_match is not None and prefix_match.l1_prefix_len > 0:
|
||||
self._release_matched_prefix_lock(decode_req.req)
|
||||
self.tree_cache.unlock(decode_req.req.lock)
|
||||
decode_req.req.lock = None
|
||||
break
|
||||
|
||||
if swa_allocatable_tokens is not None:
|
||||
@@ -1505,7 +1490,8 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
)
|
||||
if reclaim_error is not None:
|
||||
if prefix_match is not None and prefix_match.l1_prefix_len > 0:
|
||||
self._release_matched_prefix_lock(decode_req.req)
|
||||
self.tree_cache.unlock(decode_req.req.lock)
|
||||
decode_req.req.lock = None
|
||||
logger.error(reclaim_error)
|
||||
prepare_abort(
|
||||
decode_req.req,
|
||||
|
||||
@@ -161,7 +161,7 @@ class DecodeHiCachePreallocMixin:
|
||||
for dr in self.transfer_queue.queue
|
||||
if dr.prefix_match is not None
|
||||
and dr.hicache_restore_status == HiCacheRestoreResult.PENDING
|
||||
and dr.hicache_restored_node is None
|
||||
and dr.hicache_restore_lock is None
|
||||
)
|
||||
|
||||
|
||||
@@ -192,19 +192,14 @@ class DecodeHiCacheTransferMixin:
|
||||
self.tree_cache.finish(
|
||||
decode_req.req.cache_request_handle, CacheRequestOutcome.ABORT
|
||||
)
|
||||
if decode_req.hicache_restored_node is not None:
|
||||
self.tree_cache.dec_lock_ref(
|
||||
decode_req.hicache_restored_node,
|
||||
decode_req.hicache_restore_lock_receipt,
|
||||
)
|
||||
decode_req.hicache_restored_node = None
|
||||
decode_req.hicache_restore_lock_receipt = None
|
||||
self.tree_cache.unlock(decode_req.hicache_restore_lock)
|
||||
decode_req.hicache_restore_lock = None
|
||||
|
||||
def _try_hicache_queue_load_back(self, dr: DecodeRequest) -> bool:
|
||||
"""Queue one L2->L1 load_back op for ``dr``; True iff a DMA was queued.
|
||||
|
||||
On success, ``dr.hicache_restored_node`` and ``hicache_restored_kv_indices``
|
||||
are populated, and an inc_lock_ref is held until commit/abort.
|
||||
On success, ``dr.hicache_restore_lock`` and ``hicache_restored_kv_indices``
|
||||
are populated; the lock is held until commit/abort.
|
||||
Trivial cases (all-on-device / no needed coverage) auto-flip to READY.
|
||||
Failback paths flip to FAILED.
|
||||
"""
|
||||
@@ -233,10 +228,7 @@ class DecodeHiCacheTransferMixin:
|
||||
)
|
||||
)
|
||||
# The rematch repointed req.last_node to feed init_load_back's device
|
||||
# boundary, but the prealloc lock and the receipt on the req still
|
||||
# belong to pm.last_device_node; restore the pairing so any release
|
||||
# before the commit hands over the restored lock hits the right node
|
||||
# (the receipt's anchor makes a mispaired release assert).
|
||||
# boundary; point it back at the prefix the prealloc matched and locked.
|
||||
dr.req.last_node = pm.last_device_node
|
||||
# Failback: total coverage < required prefix means device alloc likely failed.
|
||||
if len(rematch.device_indices) + len(new_indices) < pm.decode_prefix_len:
|
||||
@@ -257,10 +249,7 @@ class DecodeHiCacheTransferMixin:
|
||||
dr.hicache_restored_kv_indices = torch.cat(
|
||||
[rematch.device_indices[pm.l1_prefix_len :], new_indices]
|
||||
)[: pm.restore_token_count]
|
||||
dr.hicache_restored_node = restored_node
|
||||
dr.hicache_restore_lock_receipt = self.tree_cache.inc_lock_ref(
|
||||
restored_node
|
||||
).to_dec_params()
|
||||
dr.hicache_restore_lock = self.tree_cache.lock(restored_node)
|
||||
|
||||
if len(new_indices) == 0:
|
||||
# Whole prefix already on device; no DMA needed.
|
||||
@@ -284,7 +273,7 @@ class DecodeHiCacheTransferMixin:
|
||||
# Phase A: advance in-flight DMAs to READY.
|
||||
for dr in active:
|
||||
if (
|
||||
dr.hicache_restored_node is not None
|
||||
dr.hicache_restore_lock is not None
|
||||
and self.tree_cache.is_load_back_event_done(
|
||||
dr.hicache_load_consumer_index
|
||||
)
|
||||
@@ -297,8 +286,7 @@ class DecodeHiCacheTransferMixin:
|
||||
queued = [
|
||||
dr
|
||||
for dr in active
|
||||
if dr.hicache_restored_node is None
|
||||
and self._try_hicache_queue_load_back(dr)
|
||||
if dr.hicache_restore_lock is None and self._try_hicache_queue_load_back(dr)
|
||||
]
|
||||
if not queued:
|
||||
return
|
||||
@@ -318,16 +306,9 @@ class DecodeHiCacheTransferMixin:
|
||||
return
|
||||
|
||||
req = decode_req.req
|
||||
restored_node = decode_req.hicache_restored_node
|
||||
restored_lock_receipt = decode_req.hicache_restore_lock_receipt
|
||||
assert restored_node is not None
|
||||
assert restored_lock_receipt is not None
|
||||
# Release preallocation before installing the restored lock receipt.
|
||||
self.tree_cache.dec_lock_ref(
|
||||
prefix_match.last_device_node,
|
||||
req.lock_receipt,
|
||||
skip_swa=req.swa_prefix_lock_released,
|
||||
)
|
||||
restore_lock = decode_req.hicache_restore_lock
|
||||
assert restore_lock is not None
|
||||
self.tree_cache.unlock(req.lock)
|
||||
|
||||
self.tree_cache.req_to_token_pool.write(
|
||||
(
|
||||
@@ -339,9 +320,7 @@ class DecodeHiCacheTransferMixin:
|
||||
req.prefix_indices = torch.cat(
|
||||
[prefix_match.prefix_indices, decode_req.hicache_restored_kv_indices]
|
||||
)
|
||||
req.last_node = restored_node
|
||||
req.lock_receipt = restored_lock_receipt
|
||||
req.swa_prefix_lock_released = False
|
||||
req.last_node = restore_lock.node
|
||||
req.lock = restore_lock
|
||||
# Prevent abort cleanup from releasing the transferred lock.
|
||||
decode_req.hicache_restored_node = None
|
||||
decode_req.hicache_restore_lock_receipt = None
|
||||
decode_req.hicache_restore_lock = None
|
||||
|
||||
@@ -337,7 +337,7 @@ class DecodeKVCacheOffloadManager:
|
||||
def _release_finished_req(self, req: Req):
|
||||
# Defensive guard: ReqToTokenPool.free sets req_pool_idx to None,
|
||||
# so a previously-released request must be skipped here to avoid
|
||||
# non-idempotent side effects (double free, double unpin).
|
||||
# non-idempotent side effects (double free, double unlock).
|
||||
if req.kv.req_pool_idx is None or req.kv.req_pool_idx == -1:
|
||||
return
|
||||
|
||||
@@ -347,7 +347,8 @@ class DecodeKVCacheOffloadManager:
|
||||
self.tree_cache.free_kv_row(
|
||||
req.kv, [(req.kv.cache_protected_len, req.kv.kv_allocated_len)]
|
||||
)
|
||||
self.tree_cache.unpin(req)
|
||||
self.tree_cache.unlock(req.lock)
|
||||
req.lock = None
|
||||
self.req_to_token_pool.free(req)
|
||||
req.kv.mark_kv_released()
|
||||
self.offloaded_state.pop(req, None)
|
||||
|
||||
@@ -107,8 +107,8 @@ from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
|
||||
from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
BasePrefixCache,
|
||||
CacheRequestHandle,
|
||||
DecLockRefParams,
|
||||
MatchPrefixParams,
|
||||
TreeLock,
|
||||
zero_match_result,
|
||||
)
|
||||
from sglang.srt.mem_cache.common import (
|
||||
@@ -1207,14 +1207,12 @@ class Req(ReqDllmMixin):
|
||||
self.host_hit_is_storage = False
|
||||
self.storage_prefetch_retry_attempts = 0
|
||||
self.staged_prefetch_plan: Optional[StagedPrefetchPlan] = None
|
||||
# Receipt of the tree lock held on last_node (anchor, SWA boundary,
|
||||
# skipped components); every release replays it unchanged.
|
||||
self.lock_receipt: DecLockRefParams = DecLockRefParams()
|
||||
# The tree lock this request holds; None while it runs on a session
|
||||
# slot's record or holds no lock.
|
||||
self.lock: Optional[TreeLock] = None
|
||||
# Device/host prefix used to plan the latest L3 lookup. Admission uses
|
||||
# it to detect newly exposed storage demand after queue-time eviction.
|
||||
self.storage_prefetch_last_match_len: Optional[int] = None
|
||||
# Whether the prefill-time SWA tree lock has been released early
|
||||
self.swa_prefix_lock_released: bool = False
|
||||
# Logical-page KV sharding: rotation base of the chain this request
|
||||
# extends (owner of position-page P is (base + P) % shard_size).
|
||||
# Refreshed at every sharded alloc — read through last_node, or drawn
|
||||
@@ -1984,8 +1982,7 @@ class Req(ReqDllmMixin):
|
||||
self.kv.cache_inserted_len = 0
|
||||
self.kv_rotation_base = None
|
||||
self.num_matched_prefix_tokens = 0
|
||||
self.lock_receipt = DecLockRefParams()
|
||||
self.swa_prefix_lock_released = False
|
||||
self.lock = None
|
||||
self.swa_branching_seqlen = None
|
||||
self.extend_range = None
|
||||
self.dllm_initialized = False
|
||||
|
||||
@@ -1038,10 +1038,6 @@ class PrefillAdder:
|
||||
)
|
||||
self._account_prefill_cache_admission(req, prefix_len)
|
||||
|
||||
def _req_inc_lock_ref(self, req: Req):
|
||||
# Persist the release receipt.
|
||||
req.lock_receipt = self.tree_cache.inc_lock_ref(req.last_node).to_dec_params()
|
||||
|
||||
def _kv_shard_reserve_scratch(self, prefix_len: int, extend_len: int) -> bool:
|
||||
"""Reserve scratch or return False to defer; no-op when sharding is off.
|
||||
|
||||
@@ -1205,13 +1201,11 @@ class PrefillAdder:
|
||||
else None
|
||||
)
|
||||
try:
|
||||
# Replay the acquire's receipt (SWA boundary uuid, mamba flag) so the
|
||||
# release takes back exactly what this temporary lock took.
|
||||
dec_lock_params = self.tree_cache.inc_lock_ref(last_node).to_dec_params()
|
||||
lock = self.tree_cache.lock(last_node)
|
||||
try:
|
||||
yield None
|
||||
finally:
|
||||
self.tree_cache.dec_lock_ref(last_node, dec_lock_params)
|
||||
self.tree_cache.unlock(lock)
|
||||
finally:
|
||||
if host_lock_params is not None:
|
||||
self.tree_cache.dec_host_lock_ref(last_node, host_lock_params)
|
||||
@@ -1612,7 +1606,7 @@ class PrefillAdder:
|
||||
req.set_extend_range(
|
||||
admission.prefix_len, admission.prefix_len + admission.extend_len
|
||||
)
|
||||
self._req_inc_lock_ref(req)
|
||||
req.lock = self.tree_cache.lock(req.last_node)
|
||||
self.can_run_list.append(req)
|
||||
if admission.is_chunked:
|
||||
self.new_chunked_req = req
|
||||
|
||||
@@ -169,9 +169,7 @@ class DynamicChunkSizer:
|
||||
# Walk the same match -> lock -> alloc lifecycle as a scheduled
|
||||
# request so release_kv_cache can release it symmetrically.
|
||||
req.init_next_round_input(self.tree_cache)
|
||||
req.lock_receipt = self.tree_cache.inc_lock_ref(
|
||||
req.last_node
|
||||
).to_dec_params()
|
||||
req.lock = self.tree_cache.lock(req.last_node)
|
||||
req.set_extend_range(
|
||||
len(req.prefix_indices), len(req.full_untruncated_fill_ids)
|
||||
)
|
||||
|
||||
@@ -240,6 +240,16 @@ class DecLockRefParams:
|
||||
return uuids[component_type]
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class TreeLock:
|
||||
"""``receipt`` replays the acquire on release; ``swa_released`` marks the
|
||||
SWA part released early, so neither release takes it twice."""
|
||||
|
||||
node: Any
|
||||
receipt: DecLockRefParams
|
||||
swa_released: bool = False
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class DecLockRefResult:
|
||||
"""Result of an dec_lock_ref operation."""
|
||||
@@ -510,11 +520,13 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
|
||||
) -> DecLockRefResult:
|
||||
pass
|
||||
|
||||
def unpin(self, req: Req) -> None:
|
||||
"""Drop the tree lock the request holds on ``req.last_node``; a cache
|
||||
whose acquire returned a receipt releases with it here."""
|
||||
if req.last_node is not None:
|
||||
self.dec_lock_ref(req.last_node)
|
||||
def lock(self, node: Any) -> Optional[TreeLock]:
|
||||
"""Take a tree lock on ``node`` for one holder; ``unlock`` releases it."""
|
||||
return TreeLock(node, self.inc_lock_ref(node).to_dec_params())
|
||||
|
||||
def unlock(self, lock: Optional[TreeLock]) -> None:
|
||||
if lock is not None:
|
||||
self.dec_lock_ref(lock.node, lock.receipt)
|
||||
|
||||
def maybe_hand_to_session(self, req: Req) -> None:
|
||||
"""A cache that keeps records across requests (a streaming session) takes
|
||||
|
||||
@@ -328,7 +328,8 @@ def release_kv_cache(req: Req, tree_cache: BasePrefixCache, *, checkpoint: bool)
|
||||
tree_cache.checkpoint(req, up_to=owned_kv_len)
|
||||
# The protected prefix is not this req's to free.
|
||||
tree_cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, owned_kv_len)])
|
||||
tree_cache.unpin(req)
|
||||
tree_cache.unlock(req.lock)
|
||||
req.lock = None
|
||||
_release_overallocated_kv_indices(
|
||||
req, owned_kv_len, req.kv.kv_allocated_len, tree_cache
|
||||
)
|
||||
|
||||
@@ -527,8 +527,8 @@ class RadixCache(BasePrefixCache):
|
||||
# checkpoint or release_kv_cache frees the rest.
|
||||
req.kv.cache_protected_len = len(new_indices)
|
||||
|
||||
self.dec_lock_ref(req.last_node)
|
||||
self.inc_lock_ref(new_last_node)
|
||||
self.unlock(req.lock)
|
||||
req.lock = self.lock(new_last_node)
|
||||
|
||||
# `req.prefix_indices` will be used in `PrefillAdder::add_chunked_req` later
|
||||
# - page_size != 1: there is a partial page at the end, keep the full kv_indices
|
||||
|
||||
@@ -832,9 +832,8 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
|
||||
)
|
||||
|
||||
# Move the request lock to the published boundary.
|
||||
if req.last_node is not None:
|
||||
self._dec_req_lock(req)
|
||||
lock_result = self.inc_lock_ref(matched.last_device_node)
|
||||
self.unlock(req.lock)
|
||||
req.lock = self.lock(matched.last_device_node)
|
||||
if total_hit < token_ids_len:
|
||||
req.prefix_indices = torch.cat(
|
||||
[canonical, kv_indices[total_hit:].to(dtype=torch.int64, copy=True)]
|
||||
@@ -843,8 +842,6 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
|
||||
req.prefix_indices = canonical
|
||||
req.kv.cache_protected_len = total_hit
|
||||
req.last_node = matched.last_device_node
|
||||
req.lock_receipt = lock_result.to_dec_params()
|
||||
req.swa_prefix_lock_released = False
|
||||
flow.request_mamba_value = None
|
||||
flow.allocated_request_mamba_for_load = False
|
||||
flow.load_req = None
|
||||
|
||||
@@ -237,10 +237,10 @@ Insert a request's KV into the tree: at every checkpoint while it runs, and once
|
||||
|
||||
| Aspect | Detail |
|
||||
|--------|--------|
|
||||
| **Purpose** | Publish the request's KV `[cache_protected_len, up_to)` so other requests can match it; nodes past `req.kv.cache_inserted_len` count one hit (a request counts each node once). `checkpoint_kv_cache` calls it with `req.extend_range.end` while the request runs; `release_kv_cache` calls it with the request-owned length when it finishes (`req.finished()`), then frees `[cache_protected_len, up_to)` and everything past it and unpins; with `checkpoint=False` it skips the insert and calls `on_release(req, checkpointed=False)` for component cleanup |
|
||||
| **Purpose** | Publish the request's KV `[cache_protected_len, up_to)` so other requests can match it; nodes past `req.kv.cache_inserted_len` count one hit (a request counts each node once). `checkpoint_kv_cache` calls it with `req.extend_range.end` while the request runs; `release_kv_cache` calls it with the request-owned length when it finishes (`req.finished()`), then frees `[cache_protected_len, up_to)` and everything past it and unlocks `req.lock`; with `checkpoint=False` it skips the insert and calls `on_release(req, checkpointed=False)` for component cleanup |
|
||||
| **Inputs** | `req` — the request; `up_to` — row position the insert may read up to |
|
||||
| **Output** | `None` |
|
||||
| **Mutation** | Component hooks → `insert` → re-match → writes the tree's indices back into the row → moves the request's lock from the old `req.last_node` to the node the insert ended on → updates `req.prefix_indices`, `req.kv.cache_protected_len`, `req.kv.cache_inserted_len`, `req.last_node` → component cleanup. A finished request's component state (Mamba) is handed to the tree instead of forked, and what it still held is freed. Frees no KV slot: `release_kv_cache` does that afterwards. |
|
||||
| **Mutation** | Component hooks → `insert` → re-match → writes the tree's indices back into the row → moves `req.lock` to the node the insert ended on → updates `req.prefix_indices`, `req.kv.cache_protected_len`, `req.kv.cache_inserted_len`, `req.last_node` → component cleanup. A finished request's component state (Mamba) is handed to the tree instead of forked, and what it still held is freed. Frees no KV slot: `release_kv_cache` does that afterwards. |
|
||||
| **Complexity** | **O(K + D·C)** — insert O(K + D·C) + re-match O(K + D·C) + lock transfer O(D). Simplifies to **O(K)**. |
|
||||
|
||||
**Algorithm detail:**
|
||||
@@ -248,7 +248,7 @@ Insert a request's KV into the tree: at every checkpoint while it runs, and once
|
||||
2. Truncates the key to `effective_cache_len`; the row past it stays the request's
|
||||
3. Converts token IDs (bigram if EAGLE), page-aligns the key, then calls `insert()`; a single-component tree re-inserts the prompt part so eviction can drop the output without the prompt
|
||||
4. `match_prefix()` on the inserted key and writes the matched indices into the row
|
||||
5. `dec_lock_ref()` on the old `req.last_node`, `inc_lock_ref()` on the new one
|
||||
5. `unlock(req.lock)`, then `req.lock = lock(new_last_node)`
|
||||
6. `cleanup_after_caching_req()` per component (Mamba: frees the forked `mamba_value` when the tree already had one, frees the finished request's slot)
|
||||
|
||||
---
|
||||
|
||||
@@ -55,7 +55,7 @@ class UnifiedSessionRefTracker:
|
||||
|
||||
def register_session_ref(self, req: Req, leaf: NodeId) -> None:
|
||||
"""Register the leaf a finished request's insert ended on with each
|
||||
component; the lock anchor ``req.last_node`` is a different node."""
|
||||
component; the request's lock sits on a different node."""
|
||||
if not self.enable_session_radix_cache:
|
||||
return
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
InsertResult,
|
||||
MatchPrefixParams,
|
||||
MatchResult,
|
||||
TreeLock,
|
||||
)
|
||||
from sglang.srt.mem_cache.buffer_mode.pipeline import (
|
||||
BufferModePipeline,
|
||||
@@ -93,7 +94,7 @@ from sglang.srt.observability.metrics_collector import (
|
||||
StorageMetricsCollector,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_memory, get_model, get_observability
|
||||
from sglang.srt.session.streaming_session import StreamingSession
|
||||
from sglang.srt.session.streaming_session import StreamingSession, is_virtual_node
|
||||
from sglang.srt.utils.common import ceil_align
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -975,9 +976,6 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
def inc_lock_ref(
|
||||
self, node_id: NodeId, skip_lock_components: Sequence[ComponentType] = ()
|
||||
) -> IncLockRefResult:
|
||||
result = self.session.try_inc_lock_ref(node_id)
|
||||
if result is not None:
|
||||
return result
|
||||
if self.disable:
|
||||
return IncLockRefResult()
|
||||
return self.tree_core.inc_lock_ref(node_id, skip_lock_components)
|
||||
@@ -988,45 +986,41 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
params: DecLockRefParams,
|
||||
skip_swa: bool = False,
|
||||
) -> DecLockRefResult:
|
||||
result = self.session.try_dec_lock_ref(node_id, params)
|
||||
if result is not None:
|
||||
return result
|
||||
if self.disable:
|
||||
return DecLockRefResult()
|
||||
return self.tree_core.dec_lock_ref(node_id, params, skip_swa)
|
||||
|
||||
def _dec_req_lock(self, req: Req, *, skip_swa: bool = False) -> None:
|
||||
"""Release the tree lock a request holds on its last_node with the
|
||||
receipt its acquire returned, so it never drops a lock it never took."""
|
||||
self.dec_lock_ref(req.last_node, req.lock_receipt, skip_swa=skip_swa)
|
||||
def lock(
|
||||
self, node_id: NodeId, skip_lock_components: Sequence[ComponentType] = ()
|
||||
) -> Optional[TreeLock]:
|
||||
# A session's virtual node stands for a lock its slot already holds.
|
||||
if is_virtual_node(node_id):
|
||||
return None
|
||||
result = self.inc_lock_ref(node_id, skip_lock_components)
|
||||
return TreeLock(node_id, result.to_dec_params())
|
||||
|
||||
def unpin(self, req: Req) -> None:
|
||||
# Synthetic profiling requests may own KV without locking a tree node.
|
||||
if req.last_node is not None:
|
||||
self._dec_req_lock(req, skip_swa=req.swa_prefix_lock_released)
|
||||
|
||||
def dec_swa_lock_only(
|
||||
self,
|
||||
node_id: NodeId,
|
||||
params: DecLockRefParams,
|
||||
) -> None:
|
||||
if self.disable:
|
||||
def unlock(self, lock: Optional[TreeLock]) -> None:
|
||||
if lock is None:
|
||||
return
|
||||
result = self.tree_core.dec_swa_lock_only(node_id, params)
|
||||
self.dec_lock_ref(lock.node, lock.receipt, skip_swa=lock.swa_released)
|
||||
|
||||
def release_swa(self, lock: Optional[TreeLock]) -> None:
|
||||
"""Release the SWA part of ``lock`` early, once."""
|
||||
if self.disable or lock is None or lock.swa_released:
|
||||
return
|
||||
result = self.tree_core.dec_swa_lock_only(lock.node, lock.receipt)
|
||||
self._free_values(result.device_frees, result.host_frees)
|
||||
lock.swa_released = True
|
||||
|
||||
def release_swa_prefix_lock(self, req: Req) -> None:
|
||||
"""The request's window has moved past its prefix: leave the prefix's
|
||||
SWA evictable. A session turn releases its slot's lock, once per session."""
|
||||
holder = self.session.borrowed_slot(req) or req
|
||||
lock = (self.session.borrowed_slot(req) or req).lock
|
||||
if (
|
||||
holder.swa_prefix_lock_released
|
||||
or holder.last_node is None
|
||||
or holder.lock_receipt.component_lock_uuids.get(ComponentType.SWA) is None
|
||||
lock is not None
|
||||
and lock.receipt.component_lock_uuids.get(ComponentType.SWA) is not None
|
||||
):
|
||||
return
|
||||
self.dec_swa_lock_only(holder.last_node, holder.lock_receipt)
|
||||
holder.swa_prefix_lock_released = True
|
||||
self.release_swa(lock)
|
||||
|
||||
def inc_host_lock_ref(self, node_id: NodeId) -> IncLockRefResult:
|
||||
if self.disable:
|
||||
@@ -1208,14 +1202,14 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
new_indices[req.kv.cache_protected_len :],
|
||||
)
|
||||
|
||||
self._dec_req_lock(req, skip_swa=req.swa_prefix_lock_released)
|
||||
self.unlock(req.lock)
|
||||
# Opt-in: leave the matched-prefix mamba evictable during decode (it is
|
||||
# already COW'd to the request's own slot, never read from this node again).
|
||||
# Safe only because any future COW source is the COWing request's own
|
||||
# admission-locked last_node (recorded only if still present, locked before
|
||||
# the next alloc) -- not this evictable node. A scheduler that matched a
|
||||
# whole batch before locking would break that. Off = original full lock.
|
||||
lock_result = self.inc_lock_ref(
|
||||
req.lock = self.lock(
|
||||
new_last_node,
|
||||
skip_lock_components=(
|
||||
(ComponentType.MAMBA,)
|
||||
@@ -1233,10 +1227,6 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
req.prefix_indices = new_indices
|
||||
req.kv.cache_protected_len = len(new_indices)
|
||||
req.last_node = new_last_node
|
||||
# Carry the receipt so this node's dec releases only what we locked.
|
||||
req.lock_receipt = lock_result.to_dec_params()
|
||||
# The rematch acquired a new SWA prefix lock.
|
||||
req.swa_prefix_lock_released = False
|
||||
|
||||
# cleanup
|
||||
for comp in self._components_tuple:
|
||||
|
||||
@@ -8,11 +8,9 @@ import torch
|
||||
|
||||
from sglang.srt.managers.schedule_batch import ReqKvInfo
|
||||
from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
DecLockRefParams,
|
||||
DecLockRefResult,
|
||||
IncLockRefResult,
|
||||
MatchPrefixParams,
|
||||
MatchResult,
|
||||
TreeLock,
|
||||
)
|
||||
from sglang.srt.utils.common import ceil_align, is_npu
|
||||
|
||||
@@ -41,27 +39,22 @@ class SessionSlot:
|
||||
# KV pool state
|
||||
kv: ReqKvInfo = field(default_factory=ReqKvInfo)
|
||||
|
||||
# Tree lock on the session's tree-owned prefix, and its receipt.
|
||||
last_node: Any = None
|
||||
lock_receipt: DecLockRefParams = field(default_factory=DecLockRefParams)
|
||||
# Whether the SWA part of that lock was released early.
|
||||
swa_prefix_lock_released: bool = False
|
||||
# Tree lock on the session's tree-owned prefix.
|
||||
lock: Optional[TreeLock] = None
|
||||
# Until the first turn finishes or is retracted, its checkpoints publish
|
||||
# the prompt into the tree and move the lock onto the deepest published node.
|
||||
publishes_prompt: bool = False
|
||||
|
||||
|
||||
def is_virtual_node(node: Any) -> bool:
|
||||
"""A slot's stand-in node: locking it is a no-op."""
|
||||
return isinstance(node, _VirtualNode)
|
||||
|
||||
|
||||
def _is_streaming(req: Optional[Req]) -> bool:
|
||||
return req is not None and req.session is not None and req.session.streaming
|
||||
|
||||
|
||||
def _move_tree_lock(src: Any, dst: Any) -> None:
|
||||
"""Hand the tree lock ``src`` holds (a request or a slot) to ``dst``."""
|
||||
dst.last_node = src.last_node
|
||||
dst.lock_receipt = src.lock_receipt
|
||||
dst.swa_prefix_lock_released = src.swa_prefix_lock_released
|
||||
|
||||
|
||||
class StreamingSession:
|
||||
"""Streaming-session KV records, owned by ``UnifiedRadixCache``.
|
||||
|
||||
@@ -80,20 +73,6 @@ class StreamingSession:
|
||||
|
||||
# -- Try-handle entries (see class docstring) --
|
||||
|
||||
def try_inc_lock_ref(self, node: Any) -> Optional[IncLockRefResult]:
|
||||
"""No-op lock if ``node`` is a session-internal sentinel; returns
|
||||
None to tell the caller to run its raw tree lock path."""
|
||||
if isinstance(node, _VirtualNode):
|
||||
return IncLockRefResult()
|
||||
return None
|
||||
|
||||
def try_dec_lock_ref(
|
||||
self, node: Any, params: Optional[DecLockRefParams] = None
|
||||
) -> Optional[DecLockRefResult]:
|
||||
if isinstance(node, _VirtualNode):
|
||||
return DecLockRefResult()
|
||||
return None
|
||||
|
||||
def find_active_slot(self, req: Req) -> Optional[SessionSlot]:
|
||||
"""A pre-aborted req (to_finish set) is detached from the session and
|
||||
gets None; the slot stays for the next request."""
|
||||
@@ -186,7 +165,7 @@ class StreamingSession:
|
||||
if isinstance(req.finished_reason, FINISH_ABORT):
|
||||
# Hand the record and the tree lock back; the caller releases them.
|
||||
del self.slots[req.session.session_id]
|
||||
_move_tree_lock(slot, req)
|
||||
req.lock, slot.lock = slot.lock, None
|
||||
req.session.abort_req()
|
||||
return False
|
||||
|
||||
@@ -217,7 +196,7 @@ class StreamingSession:
|
||||
if slot is None:
|
||||
return False
|
||||
if slot.publishes_prompt:
|
||||
_move_tree_lock(slot, req)
|
||||
req.lock, slot.lock = slot.lock, None
|
||||
self.cache.checkpoint_into_tree(req, up_to=up_to, **kwargs)
|
||||
self._lock_to_slot(req, slot)
|
||||
return True
|
||||
@@ -255,7 +234,6 @@ class StreamingSession:
|
||||
if slot is None:
|
||||
return
|
||||
protected_len = slot.kv.cache_protected_len
|
||||
lock_node = slot.last_node
|
||||
tokens_freed = (
|
||||
max(0, slot.kv.kv_allocated_len - protected_len) if slot.kv.holds_kv else 0
|
||||
)
|
||||
@@ -263,11 +241,7 @@ class StreamingSession:
|
||||
"Session KV released: %s (%d tokens freed)", session_id, tokens_freed
|
||||
)
|
||||
|
||||
if lock_node is not None:
|
||||
# skip_swa is an SWA-cache extension kwarg; a slot can only have
|
||||
# early-released when the cache supports SWA locks.
|
||||
skip = {"skip_swa": True} if slot.swa_prefix_lock_released else {}
|
||||
self.cache.dec_lock_ref(lock_node, slot.lock_receipt, **skip)
|
||||
self.cache.unlock(slot.lock)
|
||||
|
||||
if slot.kv.holds_kv:
|
||||
self.cache.free_kv_row(slot.kv, [(protected_len, slot.kv.kv_allocated_len)])
|
||||
@@ -291,12 +265,8 @@ class StreamingSession:
|
||||
# -- Internal helpers (streaming body bits) --
|
||||
|
||||
def _lock_to_slot(self, req: Req, slot: SessionSlot) -> None:
|
||||
"""Move the request's tree lock to the slot; the request is left on the
|
||||
slot's virtual node, where locking is a no-op."""
|
||||
_move_tree_lock(req, slot)
|
||||
slot.lock, req.lock = req.lock, None
|
||||
req.last_node = slot.virtual_node
|
||||
req.lock_receipt = DecLockRefParams()
|
||||
req.swa_prefix_lock_released = False
|
||||
|
||||
def _free_tail(self, kv: ReqKvInfo, prefix_len: int) -> None:
|
||||
"""Free [prefix_len, allocated) before alloc_for_extend overwrites it:
|
||||
|
||||
@@ -11,4 +11,5 @@ def finish_req(cache, req, up_to):
|
||||
req.refresh_fill_ids()
|
||||
cache.checkpoint(req, up_to=up_to)
|
||||
cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, up_to)])
|
||||
cache.unpin(req)
|
||||
cache.unlock(req.lock)
|
||||
req.lock = None
|
||||
|
||||
Reference in New Issue
Block a user