[Session] Let a streaming session own its KV record and tree lock from the first turn (#42537)

This commit is contained in:
Liangsheng Yin
2026-10-05 17:28:09 -07:00
committed by GitHub
parent f3fe753475
commit c01d59eb90
11 changed files with 319 additions and 144 deletions
@@ -2221,6 +2221,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
prefix_indices if prefix_len > 0 else torch.empty((0,), dtype=torch.int64)
)
req.set_extend_range(total_prefix_len, req.kv.kv_committed_len)
self.tree_cache.maybe_hand_to_session(req)
# Return the transfer destination indices:
if self.scheduler.enable_hisparse:
+2 -9
View File
@@ -3920,7 +3920,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
has_auxiliary_swa = self.tree_cache.supports_auxiliary_swa()
release_leaf_lock = envs.SGLANG_OPT_RELEASE_PREFILL_SWA.get() and hasattr(
self.tree_cache, "dec_swa_lock_only"
self.tree_cache, "release_swa_prefix_lock"
)
eviction_interval = max(1, envs.SGLANG_SWA_EVICTION_INTERVAL.get())
@@ -3952,16 +3952,9 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
# release that part of the tree lock so SWA LRU can reclaim it.
if (
release_leaf_lock
and not req.swa_prefix_lock_released
and req.lock_receipt.component_lock_uuids.get(ComponentType.SWA)
is not None
and req.last_node is not None
and req.decode_batch_idx >= sliding_window_size
):
self.tree_cache.dec_swa_lock_only(
req.last_node, req.lock_receipt
)
req.swa_prefix_lock_released = True
self.tree_cache.release_swa_prefix_lock(req)
elif (
self.forward_mode.is_extend()
and not self.tree_cache.supports_prefix_sharing()
@@ -447,6 +447,7 @@ def alloc_for_extend(
for req, seq_len in zip(batch.reqs, batch.seq_lens_cpu.tolist()):
req.kv.kv_allocated_len = seq_len
req.kv.kv_committed_len = seq_len
batch.tree_cache.maybe_hand_to_session(req)
return out_cache_loc, req_pool_indices_device, req_pool_indices_cpu
@@ -516,6 +516,10 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
if req.last_node is not None:
self.dec_lock_ref(req.last_node)
def maybe_hand_to_session(self, req: Req) -> None:
"""A cache that keeps records across requests (a streaming session) takes
the just-allocated row and the request's tree lock; the request borrows it."""
def claim_kv_row(self, req: Req) -> bool:
"""A streaming session keeps the request's kv row for the next turn.
Return True after taking the row; the caller then releases nothing."""
@@ -563,6 +563,9 @@ class UnifiedRadixCache(BasePrefixCache):
result = self.session.try_match_prefix(params)
if result is not None:
return result
return self._match_tree(params)
def _match_tree(self, params: MatchPrefixParams) -> MatchResult:
if self.disable:
return self.tree_core.empty_match_result
result = self.tree_core.match_prefix(params)
@@ -1012,6 +1015,19 @@ class UnifiedRadixCache(BasePrefixCache):
result = self.tree_core.dec_swa_lock_only(node_id, params)
self._free_values(result.device_frees, result.host_frees)
def release_swa_prefix_lock(self, req: Req) -> None:
"""The request's window has moved past its prefix: leave the prefix's
SWA evictable. A session turn releases its slot's lock, once per session."""
holder = self.session.borrowed_slot(req) or req
if (
holder.swa_prefix_lock_released
or holder.last_node is None
or holder.lock_receipt.component_lock_uuids.get(ComponentType.SWA) is None
):
return
self.dec_swa_lock_only(holder.last_node, holder.lock_receipt)
holder.swa_prefix_lock_released = True
def inc_host_lock_ref(self, node_id: NodeId) -> IncLockRefResult:
if self.disable:
return IncLockRefResult()
@@ -1024,6 +1040,9 @@ class UnifiedRadixCache(BasePrefixCache):
return DecLockRefResult()
return self.tree_core.dec_host_lock_ref(node_id, params)
def maybe_hand_to_session(self, req: Req) -> None:
self.session.take(req)
def claim_kv_row(self, req: Req) -> bool:
# Retraction also enters here: retain its ticket until actual finish.
if (
@@ -1045,6 +1064,11 @@ class UnifiedRadixCache(BasePrefixCache):
def checkpoint(self, req: Req, *, up_to: int, **kwargs) -> None:
if self.session.try_checkpoint(req, up_to=up_to, **kwargs):
return
self.checkpoint_into_tree(req, up_to=up_to, **kwargs)
def checkpoint_into_tree(self, req: Req, *, up_to: int, **kwargs) -> None:
"""Insert ``[cache_protected_len, up_to)`` of the request's record and
move its tree lock onto the node the insert ended on."""
# A finished request hands its component state (mamba) to the tree
# instead of forking it, and the tree frees what the request still held.
is_finished = req.finished()
@@ -1168,7 +1192,8 @@ class UnifiedRadixCache(BasePrefixCache):
# Match prefix. SWA insertion retains one extra window before the
# page-aligned boundary, so the normal match remains safe to repoint.
match_result = self.match_prefix(MatchPrefixParams(key=radix_key, req=req))
# The tree's own walk: a session slot must not answer for the insert.
match_result = self._match_tree(MatchPrefixParams(key=radix_key, req=req))
new_indices = match_result.device_indices
new_last_node = match_result.last_device_node
new_prefix_len = result.prefix_len
@@ -3640,7 +3665,7 @@ class UnifiedRadixCache(BasePrefixCache):
"""
# Skip when streaming sessions hold tree locks: the check asserts
# all nodes are unlocked during idle, which streaming sessions break
# by design (they hold a first-turn lock across turns).
# by design (each slot holds a lock on its prefix across turns).
if self.session.any_holding_kv():
return
+76 -57
View File
@@ -32,54 +32,43 @@ class _VirtualNode:
@dataclass
class SessionSlot:
"""Holds KV state between streaming session turns."""
"""A streaming session's KV record and the tree lock on its prefix. The
session owns both from its first turn's row allocation until it closes or
a turn aborts; every turn runs on the record as a borrower."""
virtual_node: _VirtualNode = field(default_factory=_VirtualNode)
# KV pool state
kv: ReqKvInfo = field(default_factory=ReqKvInfo)
# First req's radix tree node (for dec_lock_ref on session close)
# Tree lock on the session's tree-owned prefix, and its receipt.
last_node: Any = None
# Receipt of the first request's tree lock on last_node.
lock_receipt: DecLockRefParams = field(default_factory=DecLockRefParams)
# Whether the first request already released its SWA lock.
# Whether the SWA part of that lock was released early.
swa_prefix_lock_released: bool = False
def save_from_req(self, req: Req, is_first: bool):
"""Save KV state from a finishing request into this slot."""
kv = req.detach_kv()
if is_first:
self.last_node = req.last_node
self.lock_receipt = req.lock_receipt
self.swa_prefix_lock_released = req.swa_prefix_lock_released
# The slot takes over the request's KV record.
self.kv = kv
else:
# Later turns run on the slot's record (see restore_to_req).
assert kv is self.kv
req.swa_branching_seqlen = None
def restore_to_req(self, req: Req):
"""Restore KV state from this slot into an incoming request."""
req.kv = self.kv
req.lock_receipt = self.lock_receipt
req.swa_prefix_lock_released = self.swa_prefix_lock_released
# The slot keeps sharing the record: a rejected chunked request calls
# match_prefix -> restore_to_req again next cycle.
# Until the first turn finishes or is retracted, its checkpoints publish
# the prompt into the tree and move the lock onto the deepest published node.
publishes_prompt: bool = False
def _is_streaming(req: Optional[Req]) -> bool:
return req is not None and req.session is not None and req.session.streaming
class StreamingSession:
"""Streaming-session KV save/restore, owned by ``UnifiedRadixCache``.
def _move_tree_lock(src: Any, dst: Any) -> None:
"""Hand the tree lock ``src`` holds (a request or a slot) to ``dst``."""
dst.last_node = src.last_node
dst.lock_receipt = src.lock_receipt
dst.swa_prefix_lock_released = src.swa_prefix_lock_released
The cache calls the ``try_*`` entries first; each runs the session body
when it applies and tells the cache whether to run its own path.
class StreamingSession:
"""Streaming-session KV records, owned by ``UnifiedRadixCache``.
A session takes its first turn's record and tree lock at row allocation
(``take``); every turn then borrows them. The cache calls the ``try_*``
entries first; each runs the session body when it applies and tells the
cache whether to run its own path.
"""
def __init__(self, cache: UnifiedRadixCache):
@@ -143,7 +132,9 @@ class StreamingSession:
self.release_session(req.session.session_id)
return None
slot.restore_to_req(req)
# Lend the record; the slot keeps the tree lock. A rejected chunked
# request matches again next cycle and borrows the same record.
req.kv = slot.kv
# token_ids = get_fill_ids()[:input_len-1] (1-token logit reserve
# already applied). min handles retract retry where committed_len
@@ -181,41 +172,33 @@ class StreamingSession:
)
def try_cache_finished_req(self, req: Req) -> bool:
"""Hands a turn's row to the session slot when it finishes or is
retracted. Returns False for non-streaming requests and aborts, which
the caller releases."""
"""Keeps a finished or retracted turn's record in the session slot.
An aborted turn gets the record and the tree lock back and is released
like any request (returns False); the session re-prefills from its
last finished request next turn."""
if not _is_streaming(req):
return False
from sglang.srt.managers.schedule_batch import FINISH_ABORT
session_id = req.session.session_id
slot = self.slots.get(session_id)
slot = self.borrowed_slot(req)
assert slot is not None, f"streaming {req.rid=} does not run on its slot"
if isinstance(req.finished_reason, FINISH_ABORT):
if slot is not None:
# The turn ran on the slot's record, which the caller releases
# (row and mamba state): drop the slot with its tree lock. The
# session keeps its last finished request and re-prefills next turn.
assert slot.kv is req.kv
del self.slots[session_id]
if slot.last_node is not None:
skip = {"skip_swa": True} if slot.swa_prefix_lock_released else {}
self.cache.dec_lock_ref(slot.last_node, slot.lock_receipt, **skip)
# Hand the record and the tree lock back; the caller releases them.
del self.slots[req.session.session_id]
_move_tree_lock(slot, req)
req.session.abort_req()
return False
is_first = slot is None
if is_first:
slot = SessionSlot()
self.slots[session_id] = slot
finished_len = (
req.finished_len if req.finished_len is not None else len(req.output_ids)
)
target = len(req.origin_input_ids) + finished_len
self._trim_overshoot(req, finished_len)
slot.save_from_req(req, is_first=is_first)
req.detach_kv()
req.swa_branching_seqlen = None
slot.publishes_prompt = False
# Use the finished length, not the req clock (it lags an in-flight verify
# by ~1 under overlap); clamp so committed <= allocated.
slot.kv.kv_committed_len = min(target, slot.kv.kv_allocated_len)
@@ -226,17 +209,45 @@ class StreamingSession:
return True
def try_checkpoint(self, req: Req, *, up_to: int, **kwargs) -> bool:
"""A first turn checkpoints into the tree like any request (its
prompt prefix is tree-owned and the slot inherits that lock); later
turns run on the slot's KV, so only the chunk cursor is kept."""
if not _is_streaming(req) or req.session.session_id not in self.slots:
"""A turn on the slot's record publishes nothing of its own, so only
the chunk cursor is kept. The exception is the first prompt: the slot
publishes it for other requests to share, and its lock follows the
insert."""
slot = self.borrowed_slot(req)
if slot is None:
return False
if slot.publishes_prompt:
_move_tree_lock(slot, req)
self.cache.checkpoint_into_tree(req, up_to=up_to, **kwargs)
self._lock_to_slot(req, slot)
return True
kv_indices = self.cache.req_to_token_pool.req_to_token[
req.kv.req_pool_idx, :up_to
]
req.prefix_indices = kv_indices.to(dtype=torch.int64, copy=True)
return True
# -- Record ownership --
def take(self, req: Req) -> None:
"""A streaming turn's first row allocation: the session takes the
request's record and the tree lock it took at admission, and the
request borrows them from here on."""
if not _is_streaming(req) or self.borrowed_slot(req) is not None:
return
session_id = req.session.session_id
assert session_id not in self.slots, f"{session_id=} already has a slot"
slot = SessionSlot(kv=req.kv, publishes_prompt=True)
self._lock_to_slot(req, slot)
self.slots[session_id] = slot
def borrowed_slot(self, req: Req) -> Optional[SessionSlot]:
"""The slot whose record the request runs on, if any."""
if not _is_streaming(req):
return None
slot = self.slots.get(req.session.session_id)
return slot if slot is not None and slot.kv is req.kv else None
# -- Session lifecycle --
def release_session(self, session_id: str) -> None:
@@ -279,6 +290,14 @@ class StreamingSession:
# -- Internal helpers (streaming body bits) --
def _lock_to_slot(self, req: Req, slot: SessionSlot) -> None:
"""Move the request's tree lock to the slot; the request is left on the
slot's virtual node, where locking is a no-op."""
_move_tree_lock(req, slot)
req.last_node = slot.virtual_node
req.lock_receipt = DecLockRefParams()
req.swa_prefix_lock_released = False
def _free_tail(self, kv: ReqKvInfo, prefix_len: int) -> None:
"""Free [prefix_len, allocated) before alloc_for_extend overwrites it:
spec decoding or a retract retry leaves stale indices there."""