diff --git a/python/sglang/srt/managers/schedule_policy.py b/python/sglang/srt/managers/schedule_policy.py index 28b1797bad..53fca050c0 100644 --- a/python/sglang/srt/managers/schedule_policy.py +++ b/python/sglang/srt/managers/schedule_policy.py @@ -1188,19 +1188,13 @@ class PrefillAdder: @contextmanager def _lock_node(self, last_node: TreeNode): - dec_lock_params = None + # Replay the acquire's receipt (SWA boundary uuid, mamba flag) so the + # release takes back exactly what this temporary lock took. + dec_lock_params = self.tree_cache.inc_lock_ref(last_node).to_dec_params() try: - result = self.tree_cache.inc_lock_ref(last_node) - if self.tree_cache.is_tree_cache(): - # Replay the acquire's receipt (SWA boundary uuid, mamba flag) - # so release takes back exactly what this temporary lock took. - dec_lock_params = result.to_dec_params() yield None finally: - if dec_lock_params is not None: - self.tree_cache.dec_lock_ref(last_node, dec_lock_params) - else: - self.tree_cache.dec_lock_ref(last_node) + self.tree_cache.dec_lock_ref(last_node, dec_lock_params) def add_one_req_ignore_eos(self, req: Req): cand_extend_input_len = len(req.full_untruncated_fill_ids) - len( diff --git a/python/sglang/srt/mem_cache/base_prefix_cache.py b/python/sglang/srt/mem_cache/base_prefix_cache.py index 7bf22f64e2..206f557d3c 100644 --- a/python/sglang/srt/mem_cache/base_prefix_cache.py +++ b/python/sglang/srt/mem_cache/base_prefix_cache.py @@ -645,9 +645,6 @@ class BasePrefixCache(ABC, PrefixCacheTrait): def supports_mamba(self) -> bool: return False - def supports_streaming_session(self) -> bool: - return False - def release_session(self, session_id: str) -> None: pass diff --git a/python/sglang/srt/mem_cache/registry.py b/python/sglang/srt/mem_cache/registry.py index d04399bec6..c4d033f031 100644 --- a/python/sglang/srt/mem_cache/registry.py +++ b/python/sglang/srt/mem_cache/registry.py @@ -81,9 +81,12 @@ def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache: """Built-in Radix Cache selection chain.""" params = ctx.params - if ( - ctx.disable_radix_cache - and get_disagg().disaggregation_decode_retraction_backup == "host_pool" + is_pure_swa = ctx.is_hybrid_swa and ctx.full_tokens_per_layer == 0 + if ctx.disable_radix_cache and ( + get_disagg().disaggregation_decode_retraction_backup == "host_pool" + # Streaming sessions live in UnifiedRadixCache; its disabled mode + # stands in for the chunk caches. Pure-SWA has no unified layout. + or (get_serving().enable_streaming_session and not is_pure_swa) ): return create_unified_radix_cache(ctx) @@ -278,25 +281,25 @@ def create_tree_cache(ctx: TreeCacheBuildContext) -> BasePrefixCache: "option that selected another tree cache for this model." ) - hicache_attached = cache.cache_controller is not None - streaming_wrapped = False - if ( - get_serving().enable_streaming_session - and not cache.supports_streaming_session() + from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache + + if get_serving().enable_streaming_session and not isinstance( + cache, UnifiedRadixCache ): - from sglang.srt.session.streaming_session import StreamingSession - - cache = StreamingSession(cache) - streaming_wrapped = True + raise NotImplementedError( + f"--enable-streaming-session is not verified with {type(cache).__name__}; " + "streaming sessions run on UnifiedRadixCache. Please open an issue or " + "a PR at https://github.com/sgl-project/sglang if you need this." + ) + hicache_attached = cache.cache_controller is not None logger.info( "Tree cache initialized: source=%s impl=%s hybrid_swa=%s hybrid_ssm=%s " - "hicache_attached=%s streaming_wrapped=%s", + "hicache_attached=%s", source, type(cache).__name__, ctx.is_hybrid_swa, ctx.is_hybrid_ssm, hicache_attached, - streaming_wrapped, ) return cache diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index 0b9c7898be..9d4e6a1807 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -242,12 +242,8 @@ class UnifiedRadixCache(BasePrefixCache): self.sidecar_pool_specs: list[SidecarPoolSpec] = [] - # Streaming session: embedded StreamingSession with self as inner. - # Always on -- zero overhead when no streaming session is open (the - # try_* entries short-circuit on non-streaming reqs / real TreeNodes). - # Dispatch methods below pre-check conditions so the session's - # internal fall-through to self.inner.xxx never fires -- no recursion. - self.session = StreamingSession(inner=self) + # Always on; the try_* entries short-circuit on non-streaming reqs. + self.session = StreamingSession(self) self.tp_group = params.tp_cache_group self.attn_cp_group = params.attn_cp_cache_group @@ -3531,10 +3527,7 @@ class UnifiedRadixCache(BasePrefixCache): def release_radix_session(self, session_id: str) -> int: return self.session_refs.release_radix_session(session_id) - # ---- Streaming session API (delegates to composed StreamingSession) ---- - - def supports_streaming_session(self) -> bool: - return True + # ---- Streaming session API (delegates to self.session) ---- def release_session(self, session_id: str) -> None: self.session.release_session(session_id) diff --git a/python/sglang/srt/session/streaming_session.py b/python/sglang/srt/session/streaming_session.py index 6049849902..5caf890b4b 100644 --- a/python/sglang/srt/session/streaming_session.py +++ b/python/sglang/srt/session/streaming_session.py @@ -8,15 +8,9 @@ import torch from sglang.srt.managers.schedule_batch import ReqKvInfo from sglang.srt.mem_cache.base_prefix_cache import ( - BasePrefixCache, - CacheRequestHandle, - CacheRequestOutcome, DecLockRefParams, DecLockRefResult, - EvictParams, - EvictResult, IncLockRefResult, - InitLoadBackParams, MatchPrefixParams, MatchResult, ) @@ -25,19 +19,14 @@ from sglang.srt.utils.common import ceil_align, is_npu if TYPE_CHECKING: from sglang.srt.managers.schedule_batch import Req - from sglang.srt.mem_cache.buffer_mode.pipeline import BufferModePipeline - from sglang.srt.mem_cache.storage_prefetch import StoragePrefetchRetries + from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache logger = logging.getLogger(__name__) class _VirtualNode: - """Sentinel node for streaming session requests. - - Passed to inc_lock_ref / dec_lock_ref so the cache can distinguish - streaming-session locks (no-op) from real radix-tree locks (forwarded). - """ + """Lock target for session-owned KV; locking it is a no-op.""" pass @@ -79,82 +68,29 @@ class SessionSlot: req.lock_receipt = self.lock_receipt req.swa_prefix_lock_released = self.swa_prefix_lock_released - # NOTE: the slot keeps sharing the record it just handed out. During - # chunked prefill, a request may be rejected by - # the scheduler (e.g. budget exhausted) and retried in the next cycle. - # Each retry calls match_prefix -> restore_to_req again, so the slot - # must remain intact for idempotent restoration. + # The slot keeps sharing the record: a rejected chunked request calls + # match_prefix -> restore_to_req again next cycle. def _is_streaming(req: Optional[Req]) -> bool: return req is not None and req.session is not None and req.session.streaming -class StreamingSession(BasePrefixCache): - """Adds streaming-session KV save/restore on top of any BasePrefixCache. +class StreamingSession: + """Streaming-session KV save/restore, owned by ``UnifiedRadixCache``. - Works both as an external wrapper (``StreamingSession(RadixCache(...))``) - and in embedded composition (``StreamingSession(inner=self)``). For the - embedded case, the composing cache must pre-check dispatch conditions - (``_is_streaming`` / ``find_active_slot`` / ``has_slot``) so the internal - fall-through to ``self.inner.xxx`` never fires -- otherwise it recurses. + The cache calls the ``try_*`` entries first; each runs the session body + when it applies and tells the cache whether to run its own path. """ - def __init__(self, inner: BasePrefixCache): - self.inner = inner + def __init__(self, cache: UnifiedRadixCache): + self.cache = cache self.slots: Dict[str, SessionSlot] = {} - # -- Forward PrefixCacheTrait properties to inner cache -- - - @property - def req_to_token_pool(self): - return self.inner.req_to_token_pool - - @req_to_token_pool.setter - def req_to_token_pool(self, value): - self.inner.req_to_token_pool = value - - @property - def token_to_kv_pool_allocator(self): - return self.inner.token_to_kv_pool_allocator - - @token_to_kv_pool_allocator.setter - def token_to_kv_pool_allocator(self, value): - self.inner.token_to_kv_pool_allocator = value - - @property - def page_size(self): - return self.inner.page_size - - @page_size.setter - def page_size(self, value): - self.inner.page_size = value - - @property - def disable(self): - return self.inner.disable - - @disable.setter - def disable(self, value): - self.inner.disable = value - - @property - def metrics_collector(self): - return self.inner.metrics_collector - - @metrics_collector.setter - def metrics_collector(self, value): - self.inner.metrics_collector = value - - # -- Condition helpers (used by embedded-mode callers for pre-dispatch) -- - - def has_slot(self, session_id: str) -> bool: - return session_id in self.slots - def any_holding_kv(self) -> bool: return any(s.kv.holds_kv for s in self.slots.values()) - # -- Try-handle entries for composition (see class docstring) -- + # -- Try-handle entries (see class docstring) -- def try_inc_lock_ref(self, node: Any) -> Optional[IncLockRefResult]: """No-op lock if ``node`` is a session-internal sentinel; returns @@ -171,12 +107,8 @@ class StreamingSession(BasePrefixCache): return None def find_active_slot(self, req: Req) -> Optional[SessionSlot]: - """Returns an active slot for this req, or None. - - Side effect: if req is pre-aborted (to_finish set, e.g. input too - long), detach it from the session so release_kv_cache treats it - as a normal req. The slot stays intact for the next request. - """ + """A pre-aborted req (to_finish set) is detached from the session and + gets None; the slot stays for the next request.""" if not _is_streaming(req): return None slot = self.slots.get(req.session.session_id) @@ -188,17 +120,6 @@ class StreamingSession(BasePrefixCache): return None return slot - # -- BasePrefixCache abstract methods -- - - def reset(self): - self.slots.clear() - self.inner.reset() - - # -- Streaming entries: contract with embedded composers (e.g. - # UnifiedRadixCache) is a uniform "try_handle_*" pattern. Each method - # executes the streaming body if applicable and signals whether the - # caller still needs to run its raw path. - def try_match_prefix(self, params: MatchPrefixParams) -> Optional[MatchResult]: """Returns a MatchResult iff the request hits an active session slot; otherwise None (caller falls back to its raw match).""" @@ -208,14 +129,12 @@ class StreamingSession(BasePrefixCache): req = params.req - # [NPU] When aligned context < page_size, release the slot's KV and - # fall back to radix cache (full prefill). Once context >= page_size, - # streaming session kicks in with page-aligned KV reuse. - if is_npu() and self.page_size > 1: + # [NPU] Below one aligned page, drop the slot's KV and fully prefill. + if is_npu() and self.cache.page_size > 1: expected_prefix_len = min(slot.kv.kv_committed_len, len(params.key)) aligned_prefix_len = ( - expected_prefix_len // self.page_size - ) * self.page_size + expected_prefix_len // self.cache.page_size + ) * self.cache.page_size if ( aligned_prefix_len < slot.kv.cache_protected_len or aligned_prefix_len == 0 @@ -242,18 +161,15 @@ class StreamingSession(BasePrefixCache): # NPU requires page-aligned KV reuse; a rewind below the SWA eviction # cursor must also land on a page boundary -- free_kv_row_segments # splits dead/alive at the cursor, and a mid-page cut frees a page twice. - if self.page_size > 1 and (is_npu() or req.kv.max_evicted_seqlen > prefix_len): - prefix_len = (prefix_len // self.page_size) * self.page_size + if self.cache.page_size > 1 and ( + is_npu() or req.kv.max_evicted_seqlen > prefix_len + ): + prefix_len = (prefix_len // self.cache.page_size) * self.cache.page_size req.kv.kv_committed_len = min(req.kv.kv_committed_len, prefix_len) - # Free orphaned tail: alloc_for_extend will overwrite - # req_to_token[prefix_len:] with new indices. The range - # [prefix_len, kv_allocated_len) has stale indices from the - # previous turn's decode (e.g. alloc-commit gap on retract, - # or speculative draft tokens). self._free_tail(req.kv, prefix_len) - device_indices = self.req_to_token_pool.req_to_token[ + device_indices = self.cache.req_to_token_pool.req_to_token[ req.kv.req_pool_idx, :prefix_len ].to(dtype=torch.int64) @@ -277,19 +193,13 @@ class StreamingSession(BasePrefixCache): slot = self.slots.get(session_id) is_first = slot is None - # Mid-processing abort only. Pre-aborted reqs have session=None - # (set in find_active_slot) and never reach here. - # Nuke all KV via release_session, delete slot. Token IDs stay - # in req_nodes (finish_req was never called -> last successful - # req). Next request re-prefills from scratch. + # Mid-processing abort: free all session KV and drop the slot; req_nodes + # still points at the last finished request, so the next turn re-prefills. if isinstance(req.finished_reason, FINISH_ABORT): kv = req.detach_kv() if slot is None: - # First-request mid-processing abort: create ephemeral - # slot from req state so release_session handles cleanup; - # the detached record carries the mamba refs for - # _free_slot_mamba, and last_node lets release_session - # dec_lock_ref the tree lock. + # First turn: a throwaway slot lets release_session free the + # record (mamba refs included) and drop the tree lock. slot = SessionSlot( kv=kv, last_node=req.last_node, @@ -314,10 +224,8 @@ class StreamingSession(BasePrefixCache): self._trim_overshoot(req, finished_len) slot.save_from_req(req, is_first=is_first) - # Inherit the authoritative finished length on the slot, not the lagging - # req clock (under overlap + honest committed the clock lags the in-flight - # verify by ~1, which would short-change inheritance). Clamp to allocated - # to keep committed <= allocated for prepare_for_decode. + # Use the finished length, not the req clock (it lags an in-flight verify + # by ~1 under overlap); clamp so committed <= allocated. slot.kv.kv_committed_len = min(target, slot.kv.kv_allocated_len) # Update req_nodes to this successfully finished request. @@ -331,58 +239,12 @@ class StreamingSession(BasePrefixCache): turns run on the slot's KV, so only the chunk cursor is kept.""" if not _is_streaming(req) or req.session.session_id not in self.slots: return False - kv_indices = self.req_to_token_pool.req_to_token[req.kv.req_pool_idx, :up_to] + kv_indices = self.cache.req_to_token_pool.req_to_token[ + req.kv.req_pool_idx, :up_to + ] req.prefix_indices = kv_indices.to(dtype=torch.int64, copy=True) return True - # -- BasePrefixCache abstract methods: thin adapters over try_handle_* -- - - def match_prefix(self, params: MatchPrefixParams) -> MatchResult: - result = self.try_match_prefix(params) - if result is not None: - return result - return self.inner.match_prefix(params) - - def claim_kv_row(self, req: Req) -> bool: - return self.try_cache_finished_req(req) - - def on_release(self, req: Req, *, inserted: bool) -> None: - self.inner.on_release(req, inserted=inserted) - - def checkpoint(self, req: Req, **kwargs): - if self.try_checkpoint(req, **kwargs): - return - self.inner.checkpoint(req, **kwargs) - - def unpin(self, req: Req) -> None: - self.inner.unpin(req) - - def finish(self, handle: CacheRequestHandle, outcome: CacheRequestOutcome) -> None: - self.inner.finish(handle, outcome) - - def release_aborted_request(self, handle: CacheRequestHandle) -> None: - self.inner.release_aborted_request(handle) - - def evict(self, params: EvictParams) -> EvictResult: - return self.inner.evict(params) - - def evict_for_alloc(self, params: EvictParams) -> EvictResult: - return self.inner.evict_for_alloc(params) - - def inc_lock_ref(self, node: Any) -> IncLockRefResult: - result = self.try_inc_lock_ref(node) - if result is not None: - return result - return self.inner.inc_lock_ref(node) - - def dec_lock_ref( - self, node: Any, params: Optional[DecLockRefParams] = None - ) -> DecLockRefResult: - result = self.try_dec_lock_ref(node, params) - if result is not None: - return result - return self.inner.dec_lock_ref(node, params) - # -- Session lifecycle -- def release_session(self, session_id: str) -> None: @@ -400,26 +262,19 @@ class StreamingSession(BasePrefixCache): if lock_node is not None: # skip_swa is an SWA-cache extension kwarg; a slot can only have - # early-released when the inner cache supports SWA locks. + # early-released when the cache supports SWA locks. skip = {"skip_swa": True} if slot.swa_prefix_lock_released else {} - self.inner.dec_lock_ref(lock_node, slot.lock_receipt, **skip) + self.cache.dec_lock_ref(lock_node, slot.lock_receipt, **skip) if slot.kv.holds_kv: - self.free_kv_row(slot.kv, [(protected_len, slot.kv.kv_allocated_len)]) - self.req_to_token_pool.free(slot) + self.cache.free_kv_row(slot.kv, [(protected_len, slot.kv.kv_allocated_len)]) + self.cache.req_to_token_pool.free(slot) self._free_slot_mamba(slot) - def release_radix_session(self, session_id: str) -> None: - self.inner.release_radix_session(session_id) - def session_held_tokens(self, active_pool_idxs: Optional[set] = None) -> int: - """Total KV tokens held by session slots, not tracked by the tree. - - Excludes slots whose KV is currently owned by an owning request -- - those tokens are counted via uncached_size in the busy mem check. - A slot's pool_idx being in active_pool_idxs indicates a req owns it. - """ + """KV tokens held by idle session slots; a slot whose pool idx is in + ``active_pool_idxs`` is counted via uncached_size instead.""" total = 0 for slot in self.slots.values(): in_batch = ( @@ -427,12 +282,11 @@ class StreamingSession(BasePrefixCache): and slot.kv.req_pool_idx in active_pool_idxs ) if slot.kv.holds_kv and not in_batch: - allocated = ceil_align(slot.kv.kv_allocated_len, self.page_size) + allocated = ceil_align(slot.kv.kv_allocated_len, self.cache.page_size) total += allocated - slot.kv.cache_protected_len return total def session_held_full_tokens(self, active_pool_idxs: Optional[set] = None) -> int: - """An alias to align the naming style of SWA""" return self.session_held_tokens(active_pool_idxs) def session_held_swa_tokens(self, active_pool_idxs: Optional[set] = None) -> int: @@ -444,7 +298,7 @@ class StreamingSession(BasePrefixCache): and slot.kv.req_pool_idx in active_pool_idxs ) if slot.kv.holds_kv and not in_batch: - allocated = ceil_align(slot.kv.kv_allocated_len, self.page_size) + allocated = ceil_align(slot.kv.kv_allocated_len, self.cache.page_size) total += allocated - max( slot.kv.cache_protected_len, slot.kv.get_evicted_seqlen(ComponentType.SWA), @@ -463,12 +317,8 @@ class StreamingSession(BasePrefixCache): return sum(_owned(s) for s in self.slots.values()) def session_held_mamba_slots(self, active_pool_idxs: Optional[set] = None) -> int: - """Total mamba_pool entries held by session slots (mamba_pool_idx + - mamba_ping_pong_track_buffer). Excludes slots whose owning req is - currently in the batch -- those slots are counted via the normal - alloc/free paths (same convention as the sibling ``session_held_*`` - accessors). - """ + """mamba_pool entries held by idle session slots (same exclusion as + ``session_held_tokens``).""" total = 0 for slot in self.slots.values(): in_batch = ( @@ -485,7 +335,7 @@ class StreamingSession(BasePrefixCache): def _free_slot_mamba(self, slot: SessionSlot) -> None: """Return a session slot's mamba pool state to the allocator.""" - mamba_allocator = getattr(self.req_to_token_pool, "mamba_allocator", None) + mamba_allocator = getattr(self.cache.req_to_token_pool, "mamba_allocator", None) if mamba_allocator is None: return if slot.kv.holds_mamba: @@ -498,27 +348,21 @@ class StreamingSession(BasePrefixCache): # -- Internal helpers (streaming body bits) -- def _free_tail(self, kv: ReqKvInfo, prefix_len: int) -> None: - """match_prefix path: free orphaned KV in [prefix_len, kv_allocated_len) - before alloc_for_extend overwrites it. The gap appears when spec - decoding pushes allocated above committed, or when retract retry's - logit-reserve pulls prefix_len below committed. - """ + """Free [prefix_len, allocated) before alloc_for_extend overwrites it: + spec decoding or a retract retry leaves stale indices there.""" self._free_kv_aligned(kv, prefix_len, kv.kv_allocated_len) kv.kv_allocated_len = prefix_len kv.kv_committed_len = min(kv.kv_committed_len, prefix_len) kv.clamp_evicted_seqlens(prefix_len) def _trim_overshoot(self, req: Req, finished_len: int) -> None: - """Trim slot KV to finished_len boundary. Spec v2 may overshoot - max_new_tokens (verify round commits M+1 at a time); next turn's - input is output_ids[:finished_len], so positions past that must - be released to avoid token/KV mismatch. - """ + """Spec v2 can commit past max_new_tokens; the next turn's input is + output_ids[:finished_len], so release the KV past it.""" target = len(req.origin_input_ids) + finished_len - if self.page_size > 1 and req.kv.max_evicted_seqlen > target: + if self.cache.page_size > 1 and req.kv.max_evicted_seqlen > target: # Same hazard as the match-path rewind: the cursor must stay # page-aligned; the partial page is re-prefilled next turn. - target = (target // self.page_size) * self.page_size + target = (target // self.cache.page_size) * self.cache.page_size self._free_kv_aligned(req.kv, target, req.kv.kv_allocated_len) req.kv.kv_allocated_len = min(req.kv.kv_allocated_len, target) req.kv.kv_committed_len = min(req.kv.kv_committed_len, target) @@ -526,110 +370,11 @@ class StreamingSession(BasePrefixCache): req.output_ids = req.output_ids[:finished_len] def _free_kv_aligned(self, kv: ReqKvInfo, target: int, end: int) -> None: - """Free the record's kv row over [ceil_align(target), end). Page-aligned - because PagedTokenToKVPoolAllocator.free returns whole pages - (free_index // page_size), so partial-page free would corrupt pages - still holding committed tokens. The range [target, ceil_align(target)) - stays attached until release_session frees the whole page. - """ + """Free [ceil_align(target), end): paged free returns whole pages, so + the partial page stays until release_session.""" if end <= target: return start = target - if self.page_size > 1: - start = ceil_align(start, self.page_size) - self.free_kv_row(kv, [(start, end)]) - - # -- Pass-through methods -- - - def evictable_size(self): - return self.inner.evictable_size() - - def full_evictable_size(self): - return self.inner.full_evictable_size() - - def swa_evictable_size(self): - return self.inner.swa_evictable_size() - - def protected_size(self): - return self.inner.protected_size() - - def full_protected_size(self): - return self.inner.full_protected_size() - - def swa_protected_size(self): - return self.inner.swa_protected_size() - - def total_size(self): - return self.inner.total_size() - - def pretty_print(self): - return self.inner.pretty_print() - - def init_load_back(self, params: InitLoadBackParams): - return self.inner.init_load_back(params) - - @property - def buffer_pipeline(self) -> Optional[BufferModePipeline]: - return self.inner.buffer_pipeline - - @property - def storage_prefetch_retries(self) -> Optional[StoragePrefetchRetries]: - return self.inner.storage_prefetch_retries - - def pop_prefetch_loaded_span( - self, handle: CacheRequestHandle - ) -> tuple[int, Optional[int]]: - return self.inner.pop_prefetch_loaded_span(handle) - - def finish_storage_prefetch_admission( - self, handle: CacheRequestHandle, fulfilled_tokens: int, reason: Optional[str] - ) -> None: - self.inner.finish_storage_prefetch_admission(handle, fulfilled_tokens, reason) - - def discard_storage_prefetch_accounting(self, handle: CacheRequestHandle) -> None: - self.inner.discard_storage_prefetch_accounting(handle) - - def ready_to_load_host_cache(self): - return self.inner.ready_to_load_host_cache() - - def check_hicache_events(self): - return self.inner.check_hicache_events() - - def flush_pending_backups(self) -> None: - self.inner.flush_pending_backups() - - def take_events(self): - return self.inner.take_events() - - def supports_swa(self): - return self.inner.supports_swa() - - def supports_mamba(self): - return self.inner.supports_mamba() - - def supports_streaming_session(self) -> bool: - return True - - def is_chunk_cache(self): - return self.inner.is_chunk_cache() - - def is_tree_cache(self): - return self.inner.is_tree_cache() - - def available_and_evictable_str(self): - return self.inner.available_and_evictable_str() - - def init_metrics_collector(self): - return self.inner.init_metrics_collector() - - def sanity_check(self): - # Skip inner sanity check when sessions hold tree locks, because - # the check asserts all nodes are unlocked during idle. - if self.any_holding_kv(): - return - self.inner.sanity_check() - - # Forward attribute access for cache-specific methods (e.g. - # sliding_window_size, all_values_flatten, etc.) - def __getattr__(self, name): - return getattr(self.inner, name) + if self.cache.page_size > 1: + start = ceil_align(start, self.cache.page_size) + self.cache.free_kv_row(kv, [(start, end)]) diff --git a/test/registered/sessions/test_streaming_session.py b/test/registered/sessions/test_streaming_session.py index 06f15f1366..47da830ca6 100644 --- a/test/registered/sessions/test_streaming_session.py +++ b/test/registered/sessions/test_streaming_session.py @@ -33,6 +33,12 @@ class TestStreamingSession(StreamingSessionServerBase, StreamingSessionKitMixin) extra_args = ["--chunked-prefill-size", "512"] +class TestStreamingSessionDisableRadixCache(TestStreamingSession): + """--disable-radix-cache keeps streaming sessions on UnifiedRadixCache.""" + + extra_args = ["--chunked-prefill-size", "512", "--disable-radix-cache"] + + class TestStreamingSessionEagleV2RetractLargePage(TestStreamingSession): """EAGLE3 spec v2 + retract + page=256.""" diff --git a/test/registered/unit/layers/test_minicpm_sparse_cache.py b/test/registered/unit/layers/test_minicpm_sparse_cache.py index 5369270a1c..cf8d93b0e1 100644 --- a/test/registered/unit/layers/test_minicpm_sparse_cache.py +++ b/test/registered/unit/layers/test_minicpm_sparse_cache.py @@ -1,3 +1,4 @@ +import functools import sys from types import SimpleNamespace @@ -15,6 +16,7 @@ from sglang.srt.managers.scheduler_components.pool_stats_observer import ( SchedulerPoolStatsObserver, ) from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator +from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache from sglang.srt.mem_cache.memory_pool import ReqToTokenPool from sglang.srt.runtime_context import publish, reset_context from sglang.srt.server_args import ServerArgs @@ -240,13 +242,13 @@ def test_streaming_session_release_frees_compressed_slots(): compressed_cache = pool._aux_cache assert len(compressed_cache.free_slots) < len(compressed_cache.reserved_slots) - session = StreamingSession( - SimpleNamespace( - req_to_token_pool=pool, - token_to_kv_pool_allocator=allocator, - page_size=1, - ) + cache = SimpleNamespace( + req_to_token_pool=pool, + token_to_kv_pool_allocator=allocator, + page_size=1, ) + cache.free_kv_row = functools.partial(BasePrefixCache.free_kv_row, cache) + session = StreamingSession(cache) session.slots["session-a"] = SessionSlot( kv=ReqKvInfo(req_pool_idx=req_pool_idx, kv_allocated_len=16), ) diff --git a/test/registered/unit/mem_cache/test_registry.py b/test/registered/unit/mem_cache/test_registry.py index d5441e9476..3fec71a4c9 100644 --- a/test/registered/unit/mem_cache/test_registry.py +++ b/test/registered/unit/mem_cache/test_registry.py @@ -18,6 +18,7 @@ from sglang.srt.mem_cache.registry import ( register_radix_cache_backend, registered_radix_cache_backends, ) +from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache from sglang.test.test_utils import CustomTestCase, enter_override @@ -115,8 +116,7 @@ class TestRegisterRadixCacheBackend(_RegistryIsolationMixin, CustomTestCase): class TestCreateTreeCacheRouting(_RegistryIsolationMixin, CustomTestCase): def test_dispatches_to_registered_factory(self): - cache = MagicMock() - cache.supports_streaming_session.return_value = True + cache = MagicMock(spec=UnifiedRadixCache) factory = MagicMock(return_value=cache) register_radix_cache_backend("custom", factory) @@ -131,8 +131,7 @@ class TestCreateTreeCacheRouting(_RegistryIsolationMixin, CustomTestCase): @patch("sglang.srt.mem_cache.registry.default_radix_cache_factory") def test_unset_backend_falls_back_to_default(self, default_factory): - cache = MagicMock() - cache.supports_streaming_session.return_value = True + cache = MagicMock(spec=UnifiedRadixCache) default_factory.return_value = cache result = create_tree_cache(_make_ctx(self, backend=None)) @@ -140,42 +139,15 @@ class TestCreateTreeCacheRouting(_RegistryIsolationMixin, CustomTestCase): default_factory.assert_called_once() self.assertIs(result, cache) - def test_streaming_wrap_when_cache_does_not_support_it(self): + def test_streaming_rejected_on_non_unified_cache(self): inner = MagicMock() - inner.supports_streaming_session.return_value = False register_radix_cache_backend("nonstreaming", MagicMock(return_value=inner)) - with patch( - "sglang.srt.session.streaming_session.StreamingSession" - ) as session_cls: - session_cls.return_value = MagicMock(name="wrapped") - result = create_tree_cache( + with self.assertRaisesRegex(NotImplementedError, "not verified"): + create_tree_cache( _make_ctx(self, backend="nonstreaming", enable_streaming=True) ) - session_cls.assert_called_once_with(inner) - self.assertIs(result, session_cls.return_value) - - def test_no_streaming_wrap_when_cache_supports_it(self): - inner = MagicMock() - inner.supports_streaming_session.return_value = True - register_radix_cache_backend("streaming", MagicMock(return_value=inner)) - - result = create_tree_cache( - _make_ctx(self, backend="streaming", enable_streaming=True) - ) - - self.assertIs(result, inner) - - -class TestDefaultRadixCacheFactory(CustomTestCase): - """Branch coverage for the built-in radix cache selection chain. - - Each cache class is imported lazily inside the factory, so we patch - the class at its definition site to verify routing without depending - on each cache's real constructor or runtime state. - """ - def test_chunk_cache_when_chunked_prefill_and_disable_radix(self): ctx = _make_ctx( self, effective_chunked_prefill_size=512, disable_radix_cache=True @@ -215,6 +187,27 @@ class TestDefaultRadixCacheFactory(CustomTestCase): PureSWAChunkCache.assert_called_once_with(ctx.params) self.assertIs(result, PureSWAChunkCache.return_value) + def test_streaming_with_disable_radix_keeps_pure_swa_off_unified(self): + ctx = _make_ctx( + self, + effective_chunked_prefill_size=512, + disable_radix_cache=True, + enable_streaming=True, + is_hybrid_swa=True, + full_tokens_per_layer=0, + ) + with ( + patch( + "sglang.srt.mem_cache.registry.create_unified_radix_cache" + ) as create_unified, + patch( + "sglang.srt.mem_cache.chunk_cache.PureSWAChunkCache" + ) as PureSWAChunkCache, + ): + result = default_radix_cache_factory(ctx) + create_unified.assert_not_called() + self.assertIs(result, PureSWAChunkCache.return_value) + def test_unified_radix_cache_is_the_default(self): ctx = _make_ctx( self, diff --git a/test/registered/unit/mem_cache/test_streaming_session_unit.py b/test/registered/unit/mem_cache/test_streaming_session_unit.py index 06b7b31731..3d81f02823 100644 --- a/test/registered/unit/mem_cache/test_streaming_session_unit.py +++ b/test/registered/unit/mem_cache/test_streaming_session_unit.py @@ -6,6 +6,7 @@ import torch from sglang.srt.managers.schedule_batch import FINISH_ABORT, ReqKvInfo from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator from sglang.srt.mem_cache.base_prefix_cache import ( + BasePrefixCache, DecLockRefParams, IncLockRefResult, MatchResult, @@ -54,6 +55,10 @@ class _FakeReqToTokenPool: class _FakeInnerCache: + """Stands in for UnifiedRadixCache: owns the session and tries it first.""" + + free_kv_row = BasePrefixCache.free_kv_row + def __init__(self, req_to_token_pool, allocator, page_size, match_results=None): self.req_to_token_pool = req_to_token_pool self.token_to_kv_pool_allocator = allocator @@ -62,11 +67,18 @@ class _FakeInnerCache: self.dec_lock_ref_calls = [] self.dec_lock_ref_params = [] self.dec_lock_ref_skip_swa = [] + self.session = StreamingSession(self) def checkpoint(self, *args, **kwargs): raise AssertionError("Streaming requests should not delegate to inner cache") - def match_prefix(self, *args, **kwargs): + def claim_kv_row(self, req): + return self.session.try_cache_finished_req(req) + + def match_prefix(self, params): + result = self.session.try_match_prefix(params) + if result is not None: + return result if not self.match_results: raise AssertionError("Unexpected match_prefix call") return self.match_results.pop(0) @@ -165,8 +177,8 @@ def test_preabort_detaches_session_and_preserves_slot(): ) ], ) - tree_cache = StreamingSession(inner) - tree_cache.slots["session-a"] = SessionSlot( + tree_cache = inner + tree_cache.session.slots["session-a"] = SessionSlot( kv=ReqKvInfo( req_pool_idx=0, kv_committed_len=48, @@ -188,7 +200,7 @@ def test_preabort_detaches_session_and_preserves_slot(): # Req detached from session. assert req.session is None # Slot untouched. - slot = tree_cache.slots["session-a"] + slot = tree_cache.session.slots["session-a"] assert slot.kv.req_pool_idx == 0 assert slot.kv.kv_committed_len == 48 assert slot.kv.kv_allocated_len == 48 @@ -203,7 +215,7 @@ def test_first_mid_abort_nukes_ephemeral_slot(): req_to_token_pool = _FakeReqToTokenPool(req_to_token) allocator = _FakeAllocator() inner = _FakeInnerCache(req_to_token_pool, allocator, page_size) - tree_cache = StreamingSession(inner) + tree_cache = inner # No slot exists yet (first request). req = _FakeReq("session-a", req_pool_idx=0, committed=0, allocated=20) @@ -212,7 +224,7 @@ def test_first_mid_abort_nukes_ephemeral_slot(): release_kv_cache(req, tree_cache) # Slot must NOT be created. - assert "session-a" not in tree_cache.slots + assert "session-a" not in tree_cache.session.slots # Transient pool slot freed. assert req.kv.req_pool_idx is None assert req_to_token_pool.free_slots == [0] @@ -229,18 +241,18 @@ def test_nth_mid_abort_nukes_session_slot(): req_to_token_pool = _FakeReqToTokenPool(req_to_token) allocator = _FakeAllocator() inner = _FakeInnerCache(req_to_token_pool, allocator, page_size) - tree_cache = StreamingSession(inner) + tree_cache = inner # Mid-processing abort: restore_to_req ran, so the req runs on the slot's # record, which this turn has grown to committed=60 / allocated=65. req = _FakeReq("session-a", req_pool_idx=0, committed=60, allocated=65) req.finished_reason = FINISH_ABORT("client disconnected") - tree_cache.slots["session-a"] = SessionSlot(kv=req.kv, last_node=None) + tree_cache.session.slots["session-a"] = SessionSlot(kv=req.kv, last_node=None) release_kv_cache(req, tree_cache) # Slot wiped — deleted from slots dict. - assert "session-a" not in tree_cache.slots + assert "session-a" not in tree_cache.session.slots # All KV freed: [0, 65) from release_session. assert len(allocator.freed) == 1 assert allocator.freed[0].tolist() == list(range(65)) @@ -256,7 +268,7 @@ def test_release_session_preserves_component_lock_receipt(uuid): req_to_token_pool = _FakeReqToTokenPool(req_to_token) allocator = _FakeAllocator() inner = _FakeInnerCache(req_to_token_pool, allocator, page_size=1) - tree_cache = StreamingSession(inner) + tree_cache = inner lock_node = SimpleNamespace(id=42) acquired = IncLockRefResult( @@ -267,7 +279,7 @@ def test_release_session_preserves_component_lock_receipt(uuid): acquired.set_lock_uuid(ComponentType.SWA, 19, lock_host=True) acquired.set_lock_uuid(ComponentType.AUXILIARY_SWA, 23) acquired.set_lock_uuid(ComponentType.AUXILIARY_SWA, None, lock_host=True) - tree_cache.slots["session-a"] = SessionSlot( + tree_cache.session.slots["session-a"] = SessionSlot( kv=ReqKvInfo( req_pool_idx=0, kv_committed_len=50, @@ -282,7 +294,7 @@ def test_release_session_preserves_component_lock_receipt(uuid): acquired.set_lock_uuid(ComponentType.SWA, 99, lock_host=True) acquired.set_lock_uuid(ComponentType.AUXILIARY_SWA, 99) acquired.set_lock_uuid(ComponentType.AUXILIARY_SWA, 99, lock_host=True) - tree_cache.release_session("session-a") + tree_cache.session.release_session("session-a") assert inner.dec_lock_ref_calls == [lock_node] params = inner.dec_lock_ref_params[0] @@ -308,10 +320,10 @@ def test_release_session_skips_swa_after_early_release(): req_to_token_pool = _FakeReqToTokenPool(req_to_token) allocator = _FakeAllocator() inner = _FakeInnerCache(req_to_token_pool, allocator, page_size=1) - tree_cache = StreamingSession(inner) + tree_cache = inner lock_node = SimpleNamespace(id=42) - tree_cache.slots["session-a"] = SessionSlot( + tree_cache.session.slots["session-a"] = SessionSlot( kv=ReqKvInfo( req_pool_idx=0, kv_committed_len=50, @@ -325,7 +337,7 @@ def test_release_session_skips_swa_after_early_release(): swa_prefix_lock_released=True, ) - tree_cache.release_session("session-a") + tree_cache.session.release_session("session-a") assert inner.dec_lock_ref_calls == [lock_node] assert inner.dec_lock_ref_params[0].component_lock_uuids[ComponentType.SWA] == 7 @@ -363,9 +375,7 @@ def test_trim_overshoot_postcondition(): req_to_token = torch.arange(128, dtype=torch.int32).reshape(1, 128) req_to_token_pool = _FakeReqToTokenPool(req_to_token) allocator = _FakeAllocator() - tree_cache = StreamingSession( - _FakeInnerCache(req_to_token_pool, allocator, page_size) - ) + tree_cache = _FakeInnerCache(req_to_token_pool, allocator, page_size) # Overshoot scenario: origin=26, finished_len=12 -> target=38. # committed=40 (overshoot 2), allocated=44, swa_evicted=42 (> target), @@ -375,7 +385,7 @@ def test_trim_overshoot_postcondition(): req.output_ids = list(range(14)) req.kv.set_evicted_seqlen(ComponentType.SWA, 42) - tree_cache._trim_overshoot(req, finished_len=12) + tree_cache.session._trim_overshoot(req, finished_len=12) target = 38 assert req.kv.kv_committed_len == target @@ -395,9 +405,7 @@ def test_session_rewind_keeps_component_cursors_page_aligned(operation, componen req_to_token = torch.arange(128, dtype=torch.int32).reshape(1, 128) req_to_token_pool = _FakeReqToTokenPool(req_to_token) allocator = _FakeAllocator(page_size=page_size) - tree_cache = StreamingSession( - _FakeInnerCache(req_to_token_pool, allocator, page_size) - ) + tree_cache = _FakeInnerCache(req_to_token_pool, allocator, page_size) # origin=26, finished=12 -> raw target 38 (mid-page); cursor 48 > target. req = _FakeReq("session-a", req_pool_idx=0, committed=52, allocated=64) @@ -407,12 +415,12 @@ def test_session_rewind_keeps_component_cursors_page_aligned(operation, componen req.kv.set_evicted_seqlen(component, 48) if operation == "trim": - tree_cache._trim_overshoot(req, finished_len=12) + tree_cache.session._trim_overshoot(req, finished_len=12) assert len(req.output_ids) == 12 else: slot = SessionSlot() slot.save_from_req(req, is_first=True) - tree_cache.slots["session-a"] = slot + tree_cache.session.slots["session-a"] = slot req = _FakeReq("session-a", req_pool_idx=0, committed=0, allocated=0) result = tree_cache.match_prefix(SimpleNamespace(req=req, key=list(range(38)))) assert result.device_indices.tolist() == list(range(32))