diff --git a/python/sglang/srt/disaggregation/decode.py b/python/sglang/srt/disaggregation/decode.py index f3956f561a..c9aa043fb4 100644 --- a/python/sglang/srt/disaggregation/decode.py +++ b/python/sglang/srt/disaggregation/decode.py @@ -2221,6 +2221,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): prefix_indices if prefix_len > 0 else torch.empty((0,), dtype=torch.int64) ) req.set_extend_range(total_prefix_len, req.kv.kv_committed_len) + self.tree_cache.maybe_hand_to_session(req) # Return the transfer destination indices: if self.scheduler.enable_hisparse: diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index 9179c55c35..e0ab7b7b9d 100755 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -3920,7 +3920,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): has_auxiliary_swa = self.tree_cache.supports_auxiliary_swa() release_leaf_lock = envs.SGLANG_OPT_RELEASE_PREFILL_SWA.get() and hasattr( - self.tree_cache, "dec_swa_lock_only" + self.tree_cache, "release_swa_prefix_lock" ) eviction_interval = max(1, envs.SGLANG_SWA_EVICTION_INTERVAL.get()) @@ -3952,16 +3952,9 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): # release that part of the tree lock so SWA LRU can reclaim it. if ( release_leaf_lock - and not req.swa_prefix_lock_released - and req.lock_receipt.component_lock_uuids.get(ComponentType.SWA) - is not None - and req.last_node is not None and req.decode_batch_idx >= sliding_window_size ): - self.tree_cache.dec_swa_lock_only( - req.last_node, req.lock_receipt - ) - req.swa_prefix_lock_released = True + self.tree_cache.release_swa_prefix_lock(req) elif ( self.forward_mode.is_extend() and not self.tree_cache.supports_prefix_sharing() diff --git a/python/sglang/srt/mem_cache/allocation.py b/python/sglang/srt/mem_cache/allocation.py index 80aad30f1e..a6db9227cd 100644 --- a/python/sglang/srt/mem_cache/allocation.py +++ b/python/sglang/srt/mem_cache/allocation.py @@ -447,6 +447,7 @@ def alloc_for_extend( for req, seq_len in zip(batch.reqs, batch.seq_lens_cpu.tolist()): req.kv.kv_allocated_len = seq_len req.kv.kv_committed_len = seq_len + batch.tree_cache.maybe_hand_to_session(req) return out_cache_loc, req_pool_indices_device, req_pool_indices_cpu diff --git a/python/sglang/srt/mem_cache/base_prefix_cache.py b/python/sglang/srt/mem_cache/base_prefix_cache.py index 81f4ceb037..07eb652908 100644 --- a/python/sglang/srt/mem_cache/base_prefix_cache.py +++ b/python/sglang/srt/mem_cache/base_prefix_cache.py @@ -516,6 +516,10 @@ class BasePrefixCache(ABC, PrefixCacheTrait): if req.last_node is not None: self.dec_lock_ref(req.last_node) + def maybe_hand_to_session(self, req: Req) -> None: + """A cache that keeps records across requests (a streaming session) takes + the just-allocated row and the request's tree lock; the request borrows it.""" + def claim_kv_row(self, req: Req) -> bool: """A streaming session keeps the request's kv row for the next turn. Return True after taking the row; the caller then releases nothing.""" diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index e333f8bbcc..76742745ab 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -563,6 +563,9 @@ class UnifiedRadixCache(BasePrefixCache): result = self.session.try_match_prefix(params) if result is not None: return result + return self._match_tree(params) + + def _match_tree(self, params: MatchPrefixParams) -> MatchResult: if self.disable: return self.tree_core.empty_match_result result = self.tree_core.match_prefix(params) @@ -1012,6 +1015,19 @@ class UnifiedRadixCache(BasePrefixCache): result = self.tree_core.dec_swa_lock_only(node_id, params) self._free_values(result.device_frees, result.host_frees) + def release_swa_prefix_lock(self, req: Req) -> None: + """The request's window has moved past its prefix: leave the prefix's + SWA evictable. A session turn releases its slot's lock, once per session.""" + holder = self.session.borrowed_slot(req) or req + if ( + holder.swa_prefix_lock_released + or holder.last_node is None + or holder.lock_receipt.component_lock_uuids.get(ComponentType.SWA) is None + ): + return + self.dec_swa_lock_only(holder.last_node, holder.lock_receipt) + holder.swa_prefix_lock_released = True + def inc_host_lock_ref(self, node_id: NodeId) -> IncLockRefResult: if self.disable: return IncLockRefResult() @@ -1024,6 +1040,9 @@ class UnifiedRadixCache(BasePrefixCache): return DecLockRefResult() return self.tree_core.dec_host_lock_ref(node_id, params) + def maybe_hand_to_session(self, req: Req) -> None: + self.session.take(req) + def claim_kv_row(self, req: Req) -> bool: # Retraction also enters here: retain its ticket until actual finish. if ( @@ -1045,6 +1064,11 @@ class UnifiedRadixCache(BasePrefixCache): def checkpoint(self, req: Req, *, up_to: int, **kwargs) -> None: if self.session.try_checkpoint(req, up_to=up_to, **kwargs): return + self.checkpoint_into_tree(req, up_to=up_to, **kwargs) + + def checkpoint_into_tree(self, req: Req, *, up_to: int, **kwargs) -> None: + """Insert ``[cache_protected_len, up_to)`` of the request's record and + move its tree lock onto the node the insert ended on.""" # A finished request hands its component state (mamba) to the tree # instead of forking it, and the tree frees what the request still held. is_finished = req.finished() @@ -1168,7 +1192,8 @@ class UnifiedRadixCache(BasePrefixCache): # Match prefix. SWA insertion retains one extra window before the # page-aligned boundary, so the normal match remains safe to repoint. - match_result = self.match_prefix(MatchPrefixParams(key=radix_key, req=req)) + # The tree's own walk: a session slot must not answer for the insert. + match_result = self._match_tree(MatchPrefixParams(key=radix_key, req=req)) new_indices = match_result.device_indices new_last_node = match_result.last_device_node new_prefix_len = result.prefix_len @@ -3640,7 +3665,7 @@ class UnifiedRadixCache(BasePrefixCache): """ # Skip when streaming sessions hold tree locks: the check asserts # all nodes are unlocked during idle, which streaming sessions break - # by design (they hold a first-turn lock across turns). + # by design (each slot holds a lock on its prefix across turns). if self.session.any_holding_kv(): return diff --git a/python/sglang/srt/session/streaming_session.py b/python/sglang/srt/session/streaming_session.py index 2a55ebea91..df915bcaac 100644 --- a/python/sglang/srt/session/streaming_session.py +++ b/python/sglang/srt/session/streaming_session.py @@ -32,54 +32,43 @@ class _VirtualNode: @dataclass class SessionSlot: - """Holds KV state between streaming session turns.""" + """A streaming session's KV record and the tree lock on its prefix. The + session owns both from its first turn's row allocation until it closes or + a turn aborts; every turn runs on the record as a borrower.""" virtual_node: _VirtualNode = field(default_factory=_VirtualNode) # KV pool state kv: ReqKvInfo = field(default_factory=ReqKvInfo) - # First req's radix tree node (for dec_lock_ref on session close) + # Tree lock on the session's tree-owned prefix, and its receipt. last_node: Any = None - # Receipt of the first request's tree lock on last_node. lock_receipt: DecLockRefParams = field(default_factory=DecLockRefParams) - # Whether the first request already released its SWA lock. + # Whether the SWA part of that lock was released early. swa_prefix_lock_released: bool = False - - def save_from_req(self, req: Req, is_first: bool): - """Save KV state from a finishing request into this slot.""" - kv = req.detach_kv() - if is_first: - self.last_node = req.last_node - self.lock_receipt = req.lock_receipt - self.swa_prefix_lock_released = req.swa_prefix_lock_released - # The slot takes over the request's KV record. - self.kv = kv - else: - # Later turns run on the slot's record (see restore_to_req). - assert kv is self.kv - - req.swa_branching_seqlen = None - - def restore_to_req(self, req: Req): - """Restore KV state from this slot into an incoming request.""" - req.kv = self.kv - req.lock_receipt = self.lock_receipt - req.swa_prefix_lock_released = self.swa_prefix_lock_released - - # The slot keeps sharing the record: a rejected chunked request calls - # match_prefix -> restore_to_req again next cycle. + # Until the first turn finishes or is retracted, its checkpoints publish + # the prompt into the tree and move the lock onto the deepest published node. + publishes_prompt: bool = False def _is_streaming(req: Optional[Req]) -> bool: return req is not None and req.session is not None and req.session.streaming -class StreamingSession: - """Streaming-session KV save/restore, owned by ``UnifiedRadixCache``. +def _move_tree_lock(src: Any, dst: Any) -> None: + """Hand the tree lock ``src`` holds (a request or a slot) to ``dst``.""" + dst.last_node = src.last_node + dst.lock_receipt = src.lock_receipt + dst.swa_prefix_lock_released = src.swa_prefix_lock_released - The cache calls the ``try_*`` entries first; each runs the session body - when it applies and tells the cache whether to run its own path. + +class StreamingSession: + """Streaming-session KV records, owned by ``UnifiedRadixCache``. + + A session takes its first turn's record and tree lock at row allocation + (``take``); every turn then borrows them. The cache calls the ``try_*`` + entries first; each runs the session body when it applies and tells the + cache whether to run its own path. """ def __init__(self, cache: UnifiedRadixCache): @@ -143,7 +132,9 @@ class StreamingSession: self.release_session(req.session.session_id) return None - slot.restore_to_req(req) + # Lend the record; the slot keeps the tree lock. A rejected chunked + # request matches again next cycle and borrows the same record. + req.kv = slot.kv # token_ids = get_fill_ids()[:input_len-1] (1-token logit reserve # already applied). min handles retract retry where committed_len @@ -181,41 +172,33 @@ class StreamingSession: ) def try_cache_finished_req(self, req: Req) -> bool: - """Hands a turn's row to the session slot when it finishes or is - retracted. Returns False for non-streaming requests and aborts, which - the caller releases.""" + """Keeps a finished or retracted turn's record in the session slot. + An aborted turn gets the record and the tree lock back and is released + like any request (returns False); the session re-prefills from its + last finished request next turn.""" if not _is_streaming(req): return False from sglang.srt.managers.schedule_batch import FINISH_ABORT - session_id = req.session.session_id - slot = self.slots.get(session_id) + slot = self.borrowed_slot(req) + assert slot is not None, f"streaming {req.rid=} does not run on its slot" if isinstance(req.finished_reason, FINISH_ABORT): - if slot is not None: - # The turn ran on the slot's record, which the caller releases - # (row and mamba state): drop the slot with its tree lock. The - # session keeps its last finished request and re-prefills next turn. - assert slot.kv is req.kv - del self.slots[session_id] - if slot.last_node is not None: - skip = {"skip_swa": True} if slot.swa_prefix_lock_released else {} - self.cache.dec_lock_ref(slot.last_node, slot.lock_receipt, **skip) + # Hand the record and the tree lock back; the caller releases them. + del self.slots[req.session.session_id] + _move_tree_lock(slot, req) req.session.abort_req() return False - is_first = slot is None - if is_first: - slot = SessionSlot() - self.slots[session_id] = slot - finished_len = ( req.finished_len if req.finished_len is not None else len(req.output_ids) ) target = len(req.origin_input_ids) + finished_len self._trim_overshoot(req, finished_len) - slot.save_from_req(req, is_first=is_first) + req.detach_kv() + req.swa_branching_seqlen = None + slot.publishes_prompt = False # Use the finished length, not the req clock (it lags an in-flight verify # by ~1 under overlap); clamp so committed <= allocated. slot.kv.kv_committed_len = min(target, slot.kv.kv_allocated_len) @@ -226,17 +209,45 @@ class StreamingSession: return True def try_checkpoint(self, req: Req, *, up_to: int, **kwargs) -> bool: - """A first turn checkpoints into the tree like any request (its - prompt prefix is tree-owned and the slot inherits that lock); later - turns run on the slot's KV, so only the chunk cursor is kept.""" - if not _is_streaming(req) or req.session.session_id not in self.slots: + """A turn on the slot's record publishes nothing of its own, so only + the chunk cursor is kept. The exception is the first prompt: the slot + publishes it for other requests to share, and its lock follows the + insert.""" + slot = self.borrowed_slot(req) + if slot is None: return False + if slot.publishes_prompt: + _move_tree_lock(slot, req) + self.cache.checkpoint_into_tree(req, up_to=up_to, **kwargs) + self._lock_to_slot(req, slot) + return True kv_indices = self.cache.req_to_token_pool.req_to_token[ req.kv.req_pool_idx, :up_to ] req.prefix_indices = kv_indices.to(dtype=torch.int64, copy=True) return True + # -- Record ownership -- + + def take(self, req: Req) -> None: + """A streaming turn's first row allocation: the session takes the + request's record and the tree lock it took at admission, and the + request borrows them from here on.""" + if not _is_streaming(req) or self.borrowed_slot(req) is not None: + return + session_id = req.session.session_id + assert session_id not in self.slots, f"{session_id=} already has a slot" + slot = SessionSlot(kv=req.kv, publishes_prompt=True) + self._lock_to_slot(req, slot) + self.slots[session_id] = slot + + def borrowed_slot(self, req: Req) -> Optional[SessionSlot]: + """The slot whose record the request runs on, if any.""" + if not _is_streaming(req): + return None + slot = self.slots.get(req.session.session_id) + return slot if slot is not None and slot.kv is req.kv else None + # -- Session lifecycle -- def release_session(self, session_id: str) -> None: @@ -279,6 +290,14 @@ class StreamingSession: # -- Internal helpers (streaming body bits) -- + def _lock_to_slot(self, req: Req, slot: SessionSlot) -> None: + """Move the request's tree lock to the slot; the request is left on the + slot's virtual node, where locking is a no-op.""" + _move_tree_lock(req, slot) + req.last_node = slot.virtual_node + req.lock_receipt = DecLockRefParams() + req.swa_prefix_lock_released = False + def _free_tail(self, kv: ReqKvInfo, prefix_len: int) -> None: """Free [prefix_len, allocated) before alloc_for_extend overwrites it: spec decoding or a retract retry leaves stale indices there.""" diff --git a/test/registered/unit/managers/test_hisparse_unit.py b/test/registered/unit/managers/test_hisparse_unit.py index 459c100054..c296d999d0 100644 --- a/test/registered/unit/managers/test_hisparse_unit.py +++ b/test/registered/unit/managers/test_hisparse_unit.py @@ -773,6 +773,7 @@ class TestHiSparseUnit(unittest.TestCase): queue.tree_cache = SimpleNamespace( evictable_size=lambda: 0, protected_size=lambda: 0, + maybe_hand_to_session=lambda req: None, ) queue.scheduler = SimpleNamespace( enable_hisparse=True, diff --git a/test/registered/unit/mem_cache/test_dllm_fdfo_kv_reuse.py b/test/registered/unit/mem_cache/test_dllm_fdfo_kv_reuse.py index 8d602dc24d..1f8ac19d38 100644 --- a/test/registered/unit/mem_cache/test_dllm_fdfo_kv_reuse.py +++ b/test/registered/unit/mem_cache/test_dllm_fdfo_kv_reuse.py @@ -57,6 +57,9 @@ class _FakeTreeCache: def supports_prefix_sharing(self): return False + def maybe_hand_to_session(self, req): + pass + def _make_req(rid, prefix, block_size, *, req_pool_idx=None, reuse=False): return SimpleNamespace( diff --git a/test/registered/unit/mem_cache/test_hisparse_allocator.py b/test/registered/unit/mem_cache/test_hisparse_allocator.py index e603d992b3..6da4199370 100644 --- a/test/registered/unit/mem_cache/test_hisparse_allocator.py +++ b/test/registered/unit/mem_cache/test_hisparse_allocator.py @@ -281,6 +281,7 @@ class TestDeepSeekV4HiSparseAllocator(CustomTestCase): queue.tree_cache = SimpleNamespace( evictable_size=MagicMock(return_value=0), protected_size=MagicMock(return_value=0), + maybe_hand_to_session=lambda req: None, ) queue.scheduler = SimpleNamespace( enable_hisparse=True, diff --git a/test/registered/unit/mem_cache/test_streaming_session_unit.py b/test/registered/unit/mem_cache/test_streaming_session_unit.py index a27738f788..eb52f9ae54 100644 --- a/test/registered/unit/mem_cache/test_streaming_session_unit.py +++ b/test/registered/unit/mem_cache/test_streaming_session_unit.py @@ -3,7 +3,7 @@ from types import SimpleNamespace import pytest import torch -from sglang.srt.managers.schedule_batch import FINISH_ABORT, ReqKvInfo +from sglang.srt.managers.schedule_batch import FINISH_ABORT, FINISH_LENGTH, ReqKvInfo from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator from sglang.srt.mem_cache.base_prefix_cache import ( BasePrefixCache, @@ -83,11 +83,17 @@ class _FakeInnerCache: def checkpoint(self, req, *, up_to): pass + def maybe_hand_to_session(self, req): + self.session.take(req) + def claim_kv_row(self, req): return self.session.try_cache_finished_req(req) def unpin(self, req): - pass + if req.last_node is not None: + self.dec_lock_ref( + req.last_node, req.lock_receipt, skip_swa=req.swa_prefix_lock_released + ) def on_release(self, req, *, inserted): pass @@ -101,6 +107,8 @@ class _FakeInnerCache: return self.match_results.pop(0) def dec_lock_ref(self, node, *args, **kwargs): + if self.session.try_dec_lock_ref(node) is not None: + return self.dec_lock_ref_calls.append(node) self.dec_lock_ref_params.append(args[0] if args else kwargs.get("params")) self.dec_lock_ref_skip_swa.append(kwargs.get("skip_swa", False)) @@ -157,33 +165,37 @@ class _FakeReq: pass -def test_session_slot_round_trip_preserves_component_state(): - # The mamba state rides in the shared ReqKvInfo record. mamba_branching_seqlen - # is a per-turn match observation on the Req and is not preserved by the slot. +def _single_row_cache(): + req_to_token = torch.arange(128, dtype=torch.int32).reshape(1, 128) + return _FakeInnerCache( + _FakeReqToTokenPool(req_to_token), _FakeAllocator(), page_size=1 + ) + + +def _finished_turn(tree_cache, req): + """The session took the record at allocation; the turn then finished.""" + tree_cache.maybe_hand_to_session(req) + req.finished_reason = FINISH_LENGTH(length=0) + assert tree_cache.session.try_cache_finished_req(req) + return tree_cache.session.slots[req.session.session_id] + + +def test_finished_turn_leaves_its_record_in_the_slot(): + # Component cursors ride in the record the slot keeps; the request is + # detached, and its per-turn SWA branch observation is dropped. req = _FakeReq("session-a", req_pool_idx=0, committed=4, allocated=4) - req.kv.mamba_next_track_idx = 1 - req.kv.mamba_last_track_idx = 0 - req.kv.mamba_last_track_seqlen = 3 - req.kv.set_evicted_seqlen(ComponentType.SWA, 2) - req.kv.set_evicted_seqlen(ComponentType.AUXILIARY_SWA, 3) + record = req.kv + record.mamba_next_track_idx = 1 + record.set_evicted_seqlen(ComponentType.SWA, 2) + req.swa_branching_seqlen = 8 - slot = SessionSlot() - slot.save_from_req(req, is_first=True) + slot = _finished_turn(_single_row_cache(), req) - next_req = _FakeReq("session-a", req_pool_idx=1, committed=0, allocated=0) - slot.restore_to_req(next_req) - - assert next_req.kv.mamba_next_track_idx == 1 - assert next_req.kv.mamba_last_track_idx == 0 - assert next_req.kv.mamba_last_track_seqlen == 3 - assert next_req.kv.component_evicted_seqlens == { - ComponentType.SWA: 2, - ComponentType.AUXILIARY_SWA: 3, - } - assert req.kv.component_evicted_seqlens == {} - next_req.kv.mark_kv_released() - assert next_req.kv.is_kv_released - assert next_req.kv.component_evicted_seqlens == {} + assert slot.kv is record + assert slot.kv.mamba_next_track_idx == 1 + assert slot.kv.component_evicted_seqlens == {ComponentType.SWA: 2} + assert req.kv is not record + assert req.swa_branching_seqlen is None def test_preabort_detaches_session_and_preserves_slot(): @@ -236,8 +248,9 @@ def test_preabort_detaches_session_and_preserves_slot(): def test_first_mid_abort_releases_like_any_request(published_config): - """First-turn abort: no slot exists, so the request is released like any - aborted request and no slot is created.""" + """First-turn abort: the session took the record and the admission lock at + allocation; the abort hands both back, so the request is released like + any aborted request.""" page_size = 1 req_to_token = torch.arange(128, dtype=torch.int32).reshape(1, 128) req_to_token_pool = _FakeReqToTokenPool(req_to_token) @@ -245,14 +258,17 @@ def test_first_mid_abort_releases_like_any_request(published_config): inner = _FakeInnerCache(req_to_token_pool, allocator, page_size) tree_cache = inner - # No slot exists yet (first request). req = _FakeReq("session-a", req_pool_idx=0, committed=20, allocated=20) + lock_node = SimpleNamespace(id=7) + req.last_node = lock_node + tree_cache.maybe_hand_to_session(req) + assert tree_cache.session.slots["session-a"].last_node is lock_node req.finished_reason = FINISH_ABORT("input too long") release_kv_cache(req, tree_cache) - # Slot must NOT be created. assert "session-a" not in tree_cache.session.slots + assert inner.dec_lock_ref_calls == [lock_node] # Transient pool slot freed. assert req.kv.req_pool_idx is None assert req_to_token_pool.free_slots == [0] @@ -271,7 +287,7 @@ def test_nth_mid_abort_drops_session_slot(published_config): inner = _FakeInnerCache(req_to_token_pool, allocator, page_size) tree_cache = inner - # restore_to_req ran, so the req runs on the slot's record. + # match_prefix lent the slot's record, so the req runs on it. req = _FakeReq("session-a", req_pool_idx=0, committed=65, allocated=65) req.finished_reason = FINISH_ABORT("client disconnected") lock_node = SimpleNamespace(id=42) @@ -348,9 +364,8 @@ def test_release_session_preserves_component_lock_receipt(uuid): def test_release_session_skips_swa_after_early_release(): - """A slot saved from a req that early-released its SWA lock - (swa_prefix_lock_released) must release with skip_swa, or the session - close double-releases the SWA segment.""" + """A slot whose SWA lock was released early must close with skip_swa, or + the SWA segment is released twice.""" req_to_token = torch.arange(256, dtype=torch.int32).reshape(2, 128) req_to_token_pool = _FakeReqToTokenPool(req_to_token) allocator = _FakeAllocator() @@ -379,20 +394,6 @@ def test_release_session_skips_swa_after_early_release(): assert inner.dec_lock_ref_skip_swa == [True] -def test_session_slot_does_not_restore_swa_branching_seqlen(): - req = _FakeReq("session-a", req_pool_idx=0, committed=4, allocated=4) - req.swa_branching_seqlen = 8 - - slot = SessionSlot() - slot.save_from_req(req, is_first=True) - - next_req = _FakeReq("session-a", req_pool_idx=1, committed=0, allocated=0) - slot.restore_to_req(next_req) - - assert req.swa_branching_seqlen is None - assert next_req.swa_branching_seqlen is None - - # Shrink tests removed: streaming sessions are append-only after the # rollback fix in session_controller (rollback_aborted_req). The shrink # code path in cache_finished_req no longer exists. @@ -453,9 +454,7 @@ def test_session_rewind_keeps_component_cursors_page_aligned(operation, componen tree_cache.session._trim_overshoot(req, finished_len=12) assert len(req.output_ids) == 12 else: - slot = SessionSlot() - slot.save_from_req(req, is_first=True) - tree_cache.session.slots["session-a"] = slot + tree_cache.session.slots["session-a"] = SessionSlot(kv=req.detach_kv()) req = _FakeReq("session-a", req_pool_idx=0, committed=0, allocated=0) result = tree_cache.match_prefix(SimpleNamespace(req=req, key=list(range(38)))) assert result.device_indices.tolist() == list(range(32)) diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py index 7649519070..1d59f546b9 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py +++ b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py @@ -27,7 +27,12 @@ from sglang.srt.disaggregation.kv_events import ( StorageMedium, ) from sglang.srt.environ import envs -from sglang.srt.managers.schedule_batch import FINISH_ABORT, Req, ReqKvInfo +from sglang.srt.managers.schedule_batch import ( + FINISH_ABORT, + FINISH_LENGTH, + Req, + ReqKvInfo, +) from sglang.srt.managers.schedule_policy import PrefillAdder from sglang.srt.mem_cache.allocator import TokenToKVPoolAllocator from sglang.srt.mem_cache.allocator.swa import SWATokenToKVPoolAllocator @@ -11284,9 +11289,9 @@ class TestSegmentLockFuzzWithMamba(TestSegmentLockFuzz): class TestStreamingSessionLockLifecycle(CustomTestCase): - """A streaming session must persist swa_prefix_lock_released: closing or - aborting a session whose first turn early-released its SWA lock must not - release the SWA segment a second time.""" + """A streaming session owns its record and the tree lock on its prefix + from the first turn's row allocation; every move of that lock keeps the + tree's lock counts balanced.""" cfg = CacheConfig( page_size=1, @@ -11307,43 +11312,166 @@ class TestStreamingSessionLockLifecycle(CustomTestCase): cache.dec_swa_lock_only(node, lock.to_dec_params()) return node, lock - def _streaming_req(self, node, lock, *, session): - # No KV row is held: the slot only carries the tree lock receipt. - kv = ReqKvInfo() + def _session(self, session_id): return SimpleNamespace( - kv=kv, - detach_kv=lambda: kv, - last_node=node, - lock_receipt=lock.to_dec_params(), - swa_prefix_lock_released=True, - session=session, - finished_reason=None, + session_id=session_id, + streaming=True, + finish_req=lambda req: None, + abort_req=lambda: None, ) + def _turn(self, tokens, session): + req = Req( + rid="turn", + origin_input_text="", + origin_input_ids=array("q", tokens), + sampling_params=SamplingParams(temperature=0, max_new_tokens=1), + ) + req.session = session + req.extra_key = None + req.output_ids = array("q") + req.refresh_fill_ids() + return req + + def _admitted_turn(self, cache, allocator, pool, tokens, session): + """A first turn after admission and row allocation, before the session + takes it: its own row, and the root lock admission took.""" + req = self._turn(tokens, session) + pool.alloc([req]) + pool.write( + (req.kv.req_pool_idx, slice(0, len(tokens))), + allocator.alloc(len(tokens)), + ) + req.kv.kv_committed_len = len(tokens) + req.kv.kv_allocated_len = len(tokens) + req.last_node = cache.root_node_handle() + req.lock_receipt = cache.inc_lock_ref(req.last_node).to_dec_params() + return req + def test_close_after_early_release_releases_swa_once(self): cache, allocator, _ = build_fixture(self.cfg) node, lock = self._lock_and_early_release(cache, allocator) - req = self._streaming_req(node, lock, session=None) - slot = SessionSlot() - cache.session.slots["s"] = slot - slot.save_from_req(req, is_first=True) + cache.session.slots["s"] = SessionSlot( + last_node=node, + lock_receipt=lock.to_dec_params(), + swa_prefix_lock_released=True, + ) cache.session.release_session("s") cache.sanity_check() - def test_first_req_mid_abort_after_early_release(self): - cache, allocator, pool = build_fixture(self.cfg) + def test_abort_hands_early_released_lock_back(self): + """The aborted turn gets the slot's lock with its SWA part already + released, so its own unpin must not release the SWA segment again.""" + cache, allocator, _ = build_fixture(self.cfg) node, lock = self._lock_and_early_release(cache, allocator) - session = SimpleNamespace( - session_id="s2", streaming=True, abort_req=lambda: None + req = SimpleNamespace( + rid="r", + kv=ReqKvInfo(), + last_node=node, + lock_receipt=lock.to_dec_params(), + swa_prefix_lock_released=True, + session=self._session("s2"), + finished_reason=None, ) - req = self._streaming_req(node, lock, session=session) + cache.maybe_hand_to_session(req) + slot = cache.session.slots["s2"] + self.assertIs(req.last_node, slot.virtual_node) + req.finished_reason = FINISH_ABORT() - # The session leaves an abort to the normal release, whose unpin - # must not release the SWA lock a second time. self.assertFalse(cache.session.try_cache_finished_req(req)) + self.assertNotIn("s2", cache.session.slots) + self.assertEqual(req.last_node, node) + self.assertTrue(req.swa_prefix_lock_released) cache.unpin(req) cache.sanity_check() + def test_slot_publishes_first_prompt_and_keeps_its_lock(self): + cache, allocator, pool = build_fixture(self.cfg) + tokens = list(range(1, 9)) + req = self._admitted_turn(cache, allocator, pool, tokens, self._session("s")) + cache.maybe_hand_to_session(req) + slot = cache.session.slots["s"] + + cache.checkpoint(req, up_to=len(tokens)) + + # The prompt is in the tree for anyone, and the slot's lock moved onto it. + match = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", tokens)))) + self.assertEqual(len(match.device_indices), len(tokens)) + self.assertEqual(slot.last_node, match.last_device_node) + self.assertEqual(slot.kv.cache_protected_len, len(tokens)) + self.assertIs(req.last_node, slot.virtual_node) + + req.finished_reason = FINISH_LENGTH(length=0) + self.assertTrue(cache.session.try_cache_finished_req(req)) + self.assertFalse(slot.publishes_prompt) + self.assertFalse(req.kv.holds_kv) + cache.session.release_session("s") + cache.sanity_check() + + def test_abort_after_publish_releases_like_any_request(self): + cache, allocator, pool = build_fixture(self.cfg) + tokens = list(range(1, 9)) + req = self._admitted_turn(cache, allocator, pool, tokens, self._session("s")) + cache.maybe_hand_to_session(req) + cache.checkpoint(req, up_to=len(tokens)) + slot = cache.session.slots["s"] + + req.finished_reason = FINISH_ABORT() + self.assertFalse(cache.session.try_cache_finished_req(req)) + self.assertEqual(req.last_node, slot.last_node) + finish_req(cache, req, len(tokens)) + cache.sanity_check() + + def test_session_turn_releases_slot_swa_prefix_lock_once(self): + cache, allocator, pool = build_fixture(self.cfg) + tokens = list(range(1, 9)) + req = self._admitted_turn(cache, allocator, pool, tokens, self._session("s")) + cache.maybe_hand_to_session(req) + cache.checkpoint(req, up_to=len(tokens)) + slot = cache.session.slots["s"] + + cache.release_swa_prefix_lock(req) + self.assertTrue(slot.swa_prefix_lock_released) + self.assertFalse(req.swa_prefix_lock_released) + # A second call finds the SWA part already released. + cache.release_swa_prefix_lock(req) + + req.finished_reason = FINISH_LENGTH(length=0) + self.assertTrue(cache.session.try_cache_finished_req(req)) + cache.session.release_session("s") + cache.sanity_check() + + def test_later_turn_borrows_without_publishing(self): + """A later turn runs on the slot's record: its allocation hands nothing + over, and its checkpoints neither insert nor move the slot's lock.""" + cache, allocator, pool = build_fixture(self.cfg) + session = self._session("s") + first = self._admitted_turn(cache, allocator, pool, list(range(1, 9)), session) + cache.maybe_hand_to_session(first) + cache.checkpoint(first, up_to=8) + first.finished_reason = FINISH_LENGTH(length=0) + self.assertTrue(cache.session.try_cache_finished_req(first)) + slot = cache.session.slots["s"] + slot_lock_node = slot.last_node + + tokens = list(range(1, 13)) + req = self._turn(tokens, session) + match = cache.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", tokens)), req=req) + ) + req.last_node = match.last_device_node + pool.write((req.kv.req_pool_idx, slice(8, 12)), allocator.alloc(4)) + req.kv.kv_committed_len = 12 + req.kv.kv_allocated_len = 12 + cache.maybe_hand_to_session(req) + cache.checkpoint(req, up_to=12) + self.assertIs(slot.last_node, slot_lock_node) + + req.finished_reason = FINISH_LENGTH(length=0) + self.assertTrue(cache.session.try_cache_finished_req(req)) + cache.session.release_session("s") + cache.sanity_check() + if __name__ == "__main__": unittest.main()