[Session] Let a streaming session own its KV record and tree lock from the first turn (#42537)
This commit is contained in:
@@ -2221,6 +2221,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
prefix_indices if prefix_len > 0 else torch.empty((0,), dtype=torch.int64)
|
||||
)
|
||||
req.set_extend_range(total_prefix_len, req.kv.kv_committed_len)
|
||||
self.tree_cache.maybe_hand_to_session(req)
|
||||
|
||||
# Return the transfer destination indices:
|
||||
if self.scheduler.enable_hisparse:
|
||||
|
||||
@@ -3920,7 +3920,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
has_auxiliary_swa = self.tree_cache.supports_auxiliary_swa()
|
||||
|
||||
release_leaf_lock = envs.SGLANG_OPT_RELEASE_PREFILL_SWA.get() and hasattr(
|
||||
self.tree_cache, "dec_swa_lock_only"
|
||||
self.tree_cache, "release_swa_prefix_lock"
|
||||
)
|
||||
|
||||
eviction_interval = max(1, envs.SGLANG_SWA_EVICTION_INTERVAL.get())
|
||||
@@ -3952,16 +3952,9 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
# release that part of the tree lock so SWA LRU can reclaim it.
|
||||
if (
|
||||
release_leaf_lock
|
||||
and not req.swa_prefix_lock_released
|
||||
and req.lock_receipt.component_lock_uuids.get(ComponentType.SWA)
|
||||
is not None
|
||||
and req.last_node is not None
|
||||
and req.decode_batch_idx >= sliding_window_size
|
||||
):
|
||||
self.tree_cache.dec_swa_lock_only(
|
||||
req.last_node, req.lock_receipt
|
||||
)
|
||||
req.swa_prefix_lock_released = True
|
||||
self.tree_cache.release_swa_prefix_lock(req)
|
||||
elif (
|
||||
self.forward_mode.is_extend()
|
||||
and not self.tree_cache.supports_prefix_sharing()
|
||||
|
||||
@@ -447,6 +447,7 @@ def alloc_for_extend(
|
||||
for req, seq_len in zip(batch.reqs, batch.seq_lens_cpu.tolist()):
|
||||
req.kv.kv_allocated_len = seq_len
|
||||
req.kv.kv_committed_len = seq_len
|
||||
batch.tree_cache.maybe_hand_to_session(req)
|
||||
|
||||
return out_cache_loc, req_pool_indices_device, req_pool_indices_cpu
|
||||
|
||||
|
||||
@@ -516,6 +516,10 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
|
||||
if req.last_node is not None:
|
||||
self.dec_lock_ref(req.last_node)
|
||||
|
||||
def maybe_hand_to_session(self, req: Req) -> None:
|
||||
"""A cache that keeps records across requests (a streaming session) takes
|
||||
the just-allocated row and the request's tree lock; the request borrows it."""
|
||||
|
||||
def claim_kv_row(self, req: Req) -> bool:
|
||||
"""A streaming session keeps the request's kv row for the next turn.
|
||||
Return True after taking the row; the caller then releases nothing."""
|
||||
|
||||
@@ -563,6 +563,9 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
result = self.session.try_match_prefix(params)
|
||||
if result is not None:
|
||||
return result
|
||||
return self._match_tree(params)
|
||||
|
||||
def _match_tree(self, params: MatchPrefixParams) -> MatchResult:
|
||||
if self.disable:
|
||||
return self.tree_core.empty_match_result
|
||||
result = self.tree_core.match_prefix(params)
|
||||
@@ -1012,6 +1015,19 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
result = self.tree_core.dec_swa_lock_only(node_id, params)
|
||||
self._free_values(result.device_frees, result.host_frees)
|
||||
|
||||
def release_swa_prefix_lock(self, req: Req) -> None:
|
||||
"""The request's window has moved past its prefix: leave the prefix's
|
||||
SWA evictable. A session turn releases its slot's lock, once per session."""
|
||||
holder = self.session.borrowed_slot(req) or req
|
||||
if (
|
||||
holder.swa_prefix_lock_released
|
||||
or holder.last_node is None
|
||||
or holder.lock_receipt.component_lock_uuids.get(ComponentType.SWA) is None
|
||||
):
|
||||
return
|
||||
self.dec_swa_lock_only(holder.last_node, holder.lock_receipt)
|
||||
holder.swa_prefix_lock_released = True
|
||||
|
||||
def inc_host_lock_ref(self, node_id: NodeId) -> IncLockRefResult:
|
||||
if self.disable:
|
||||
return IncLockRefResult()
|
||||
@@ -1024,6 +1040,9 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
return DecLockRefResult()
|
||||
return self.tree_core.dec_host_lock_ref(node_id, params)
|
||||
|
||||
def maybe_hand_to_session(self, req: Req) -> None:
|
||||
self.session.take(req)
|
||||
|
||||
def claim_kv_row(self, req: Req) -> bool:
|
||||
# Retraction also enters here: retain its ticket until actual finish.
|
||||
if (
|
||||
@@ -1045,6 +1064,11 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
def checkpoint(self, req: Req, *, up_to: int, **kwargs) -> None:
|
||||
if self.session.try_checkpoint(req, up_to=up_to, **kwargs):
|
||||
return
|
||||
self.checkpoint_into_tree(req, up_to=up_to, **kwargs)
|
||||
|
||||
def checkpoint_into_tree(self, req: Req, *, up_to: int, **kwargs) -> None:
|
||||
"""Insert ``[cache_protected_len, up_to)`` of the request's record and
|
||||
move its tree lock onto the node the insert ended on."""
|
||||
# A finished request hands its component state (mamba) to the tree
|
||||
# instead of forking it, and the tree frees what the request still held.
|
||||
is_finished = req.finished()
|
||||
@@ -1168,7 +1192,8 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
|
||||
# Match prefix. SWA insertion retains one extra window before the
|
||||
# page-aligned boundary, so the normal match remains safe to repoint.
|
||||
match_result = self.match_prefix(MatchPrefixParams(key=radix_key, req=req))
|
||||
# The tree's own walk: a session slot must not answer for the insert.
|
||||
match_result = self._match_tree(MatchPrefixParams(key=radix_key, req=req))
|
||||
new_indices = match_result.device_indices
|
||||
new_last_node = match_result.last_device_node
|
||||
new_prefix_len = result.prefix_len
|
||||
@@ -3640,7 +3665,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
"""
|
||||
# Skip when streaming sessions hold tree locks: the check asserts
|
||||
# all nodes are unlocked during idle, which streaming sessions break
|
||||
# by design (they hold a first-turn lock across turns).
|
||||
# by design (each slot holds a lock on its prefix across turns).
|
||||
if self.session.any_holding_kv():
|
||||
return
|
||||
|
||||
|
||||
@@ -32,54 +32,43 @@ class _VirtualNode:
|
||||
|
||||
@dataclass
|
||||
class SessionSlot:
|
||||
"""Holds KV state between streaming session turns."""
|
||||
"""A streaming session's KV record and the tree lock on its prefix. The
|
||||
session owns both from its first turn's row allocation until it closes or
|
||||
a turn aborts; every turn runs on the record as a borrower."""
|
||||
|
||||
virtual_node: _VirtualNode = field(default_factory=_VirtualNode)
|
||||
|
||||
# KV pool state
|
||||
kv: ReqKvInfo = field(default_factory=ReqKvInfo)
|
||||
|
||||
# First req's radix tree node (for dec_lock_ref on session close)
|
||||
# Tree lock on the session's tree-owned prefix, and its receipt.
|
||||
last_node: Any = None
|
||||
# Receipt of the first request's tree lock on last_node.
|
||||
lock_receipt: DecLockRefParams = field(default_factory=DecLockRefParams)
|
||||
# Whether the first request already released its SWA lock.
|
||||
# Whether the SWA part of that lock was released early.
|
||||
swa_prefix_lock_released: bool = False
|
||||
|
||||
def save_from_req(self, req: Req, is_first: bool):
|
||||
"""Save KV state from a finishing request into this slot."""
|
||||
kv = req.detach_kv()
|
||||
if is_first:
|
||||
self.last_node = req.last_node
|
||||
self.lock_receipt = req.lock_receipt
|
||||
self.swa_prefix_lock_released = req.swa_prefix_lock_released
|
||||
# The slot takes over the request's KV record.
|
||||
self.kv = kv
|
||||
else:
|
||||
# Later turns run on the slot's record (see restore_to_req).
|
||||
assert kv is self.kv
|
||||
|
||||
req.swa_branching_seqlen = None
|
||||
|
||||
def restore_to_req(self, req: Req):
|
||||
"""Restore KV state from this slot into an incoming request."""
|
||||
req.kv = self.kv
|
||||
req.lock_receipt = self.lock_receipt
|
||||
req.swa_prefix_lock_released = self.swa_prefix_lock_released
|
||||
|
||||
# The slot keeps sharing the record: a rejected chunked request calls
|
||||
# match_prefix -> restore_to_req again next cycle.
|
||||
# Until the first turn finishes or is retracted, its checkpoints publish
|
||||
# the prompt into the tree and move the lock onto the deepest published node.
|
||||
publishes_prompt: bool = False
|
||||
|
||||
|
||||
def _is_streaming(req: Optional[Req]) -> bool:
|
||||
return req is not None and req.session is not None and req.session.streaming
|
||||
|
||||
|
||||
class StreamingSession:
|
||||
"""Streaming-session KV save/restore, owned by ``UnifiedRadixCache``.
|
||||
def _move_tree_lock(src: Any, dst: Any) -> None:
|
||||
"""Hand the tree lock ``src`` holds (a request or a slot) to ``dst``."""
|
||||
dst.last_node = src.last_node
|
||||
dst.lock_receipt = src.lock_receipt
|
||||
dst.swa_prefix_lock_released = src.swa_prefix_lock_released
|
||||
|
||||
The cache calls the ``try_*`` entries first; each runs the session body
|
||||
when it applies and tells the cache whether to run its own path.
|
||||
|
||||
class StreamingSession:
|
||||
"""Streaming-session KV records, owned by ``UnifiedRadixCache``.
|
||||
|
||||
A session takes its first turn's record and tree lock at row allocation
|
||||
(``take``); every turn then borrows them. The cache calls the ``try_*``
|
||||
entries first; each runs the session body when it applies and tells the
|
||||
cache whether to run its own path.
|
||||
"""
|
||||
|
||||
def __init__(self, cache: UnifiedRadixCache):
|
||||
@@ -143,7 +132,9 @@ class StreamingSession:
|
||||
self.release_session(req.session.session_id)
|
||||
return None
|
||||
|
||||
slot.restore_to_req(req)
|
||||
# Lend the record; the slot keeps the tree lock. A rejected chunked
|
||||
# request matches again next cycle and borrows the same record.
|
||||
req.kv = slot.kv
|
||||
|
||||
# token_ids = get_fill_ids()[:input_len-1] (1-token logit reserve
|
||||
# already applied). min handles retract retry where committed_len
|
||||
@@ -181,41 +172,33 @@ class StreamingSession:
|
||||
)
|
||||
|
||||
def try_cache_finished_req(self, req: Req) -> bool:
|
||||
"""Hands a turn's row to the session slot when it finishes or is
|
||||
retracted. Returns False for non-streaming requests and aborts, which
|
||||
the caller releases."""
|
||||
"""Keeps a finished or retracted turn's record in the session slot.
|
||||
An aborted turn gets the record and the tree lock back and is released
|
||||
like any request (returns False); the session re-prefills from its
|
||||
last finished request next turn."""
|
||||
if not _is_streaming(req):
|
||||
return False
|
||||
|
||||
from sglang.srt.managers.schedule_batch import FINISH_ABORT
|
||||
|
||||
session_id = req.session.session_id
|
||||
slot = self.slots.get(session_id)
|
||||
slot = self.borrowed_slot(req)
|
||||
assert slot is not None, f"streaming {req.rid=} does not run on its slot"
|
||||
if isinstance(req.finished_reason, FINISH_ABORT):
|
||||
if slot is not None:
|
||||
# The turn ran on the slot's record, which the caller releases
|
||||
# (row and mamba state): drop the slot with its tree lock. The
|
||||
# session keeps its last finished request and re-prefills next turn.
|
||||
assert slot.kv is req.kv
|
||||
del self.slots[session_id]
|
||||
if slot.last_node is not None:
|
||||
skip = {"skip_swa": True} if slot.swa_prefix_lock_released else {}
|
||||
self.cache.dec_lock_ref(slot.last_node, slot.lock_receipt, **skip)
|
||||
# Hand the record and the tree lock back; the caller releases them.
|
||||
del self.slots[req.session.session_id]
|
||||
_move_tree_lock(slot, req)
|
||||
req.session.abort_req()
|
||||
return False
|
||||
|
||||
is_first = slot is None
|
||||
if is_first:
|
||||
slot = SessionSlot()
|
||||
self.slots[session_id] = slot
|
||||
|
||||
finished_len = (
|
||||
req.finished_len if req.finished_len is not None else len(req.output_ids)
|
||||
)
|
||||
target = len(req.origin_input_ids) + finished_len
|
||||
self._trim_overshoot(req, finished_len)
|
||||
|
||||
slot.save_from_req(req, is_first=is_first)
|
||||
req.detach_kv()
|
||||
req.swa_branching_seqlen = None
|
||||
slot.publishes_prompt = False
|
||||
# Use the finished length, not the req clock (it lags an in-flight verify
|
||||
# by ~1 under overlap); clamp so committed <= allocated.
|
||||
slot.kv.kv_committed_len = min(target, slot.kv.kv_allocated_len)
|
||||
@@ -226,17 +209,45 @@ class StreamingSession:
|
||||
return True
|
||||
|
||||
def try_checkpoint(self, req: Req, *, up_to: int, **kwargs) -> bool:
|
||||
"""A first turn checkpoints into the tree like any request (its
|
||||
prompt prefix is tree-owned and the slot inherits that lock); later
|
||||
turns run on the slot's KV, so only the chunk cursor is kept."""
|
||||
if not _is_streaming(req) or req.session.session_id not in self.slots:
|
||||
"""A turn on the slot's record publishes nothing of its own, so only
|
||||
the chunk cursor is kept. The exception is the first prompt: the slot
|
||||
publishes it for other requests to share, and its lock follows the
|
||||
insert."""
|
||||
slot = self.borrowed_slot(req)
|
||||
if slot is None:
|
||||
return False
|
||||
if slot.publishes_prompt:
|
||||
_move_tree_lock(slot, req)
|
||||
self.cache.checkpoint_into_tree(req, up_to=up_to, **kwargs)
|
||||
self._lock_to_slot(req, slot)
|
||||
return True
|
||||
kv_indices = self.cache.req_to_token_pool.req_to_token[
|
||||
req.kv.req_pool_idx, :up_to
|
||||
]
|
||||
req.prefix_indices = kv_indices.to(dtype=torch.int64, copy=True)
|
||||
return True
|
||||
|
||||
# -- Record ownership --
|
||||
|
||||
def take(self, req: Req) -> None:
|
||||
"""A streaming turn's first row allocation: the session takes the
|
||||
request's record and the tree lock it took at admission, and the
|
||||
request borrows them from here on."""
|
||||
if not _is_streaming(req) or self.borrowed_slot(req) is not None:
|
||||
return
|
||||
session_id = req.session.session_id
|
||||
assert session_id not in self.slots, f"{session_id=} already has a slot"
|
||||
slot = SessionSlot(kv=req.kv, publishes_prompt=True)
|
||||
self._lock_to_slot(req, slot)
|
||||
self.slots[session_id] = slot
|
||||
|
||||
def borrowed_slot(self, req: Req) -> Optional[SessionSlot]:
|
||||
"""The slot whose record the request runs on, if any."""
|
||||
if not _is_streaming(req):
|
||||
return None
|
||||
slot = self.slots.get(req.session.session_id)
|
||||
return slot if slot is not None and slot.kv is req.kv else None
|
||||
|
||||
# -- Session lifecycle --
|
||||
|
||||
def release_session(self, session_id: str) -> None:
|
||||
@@ -279,6 +290,14 @@ class StreamingSession:
|
||||
|
||||
# -- Internal helpers (streaming body bits) --
|
||||
|
||||
def _lock_to_slot(self, req: Req, slot: SessionSlot) -> None:
|
||||
"""Move the request's tree lock to the slot; the request is left on the
|
||||
slot's virtual node, where locking is a no-op."""
|
||||
_move_tree_lock(req, slot)
|
||||
req.last_node = slot.virtual_node
|
||||
req.lock_receipt = DecLockRefParams()
|
||||
req.swa_prefix_lock_released = False
|
||||
|
||||
def _free_tail(self, kv: ReqKvInfo, prefix_len: int) -> None:
|
||||
"""Free [prefix_len, allocated) before alloc_for_extend overwrites it:
|
||||
spec decoding or a retract retry leaves stale indices there."""
|
||||
|
||||
Reference in New Issue
Block a user