From ee30dc4d40993045273f0a90260db7efa302c214 Mon Sep 17 00:00:00 2001 From: SuperSong Date: Sat, 17 Oct 2026 20:46:48 +0900 Subject: [PATCH] [Fix] unified-memory: Bound joint reclaim and preserve cached prefixes (#39294) Bound joint FULL/SWA/Mamba reclaim so feasible allocations recover capacity without evicting unnecessary cached prefixes, and impossible demands preserve the cache. Use exact unsharded paged-extend demand and update allocation regression fixtures. --- python/sglang/srt/mem_cache/allocation.py | 11 +- python/sglang/srt/mem_cache/allocator/base.py | 11 + .../mem_cache/allocator/unified_hybrid_swa.py | 320 ++++- .../sglang/srt/mem_cache/base_prefix_cache.py | 24 +- .../srt/mem_cache/unified_radix_cache.py | 47 +- .../sglang/test/unified_allocator_fixtures.py | 113 +- .../unit/mem_cache/test_dllm_fdfo_kv_reuse.py | 10 + .../mem_cache/test_prefill_memory_budget.py | 36 +- .../test_unified_joint_allocation_eviction.py | 545 ++++++++ .../test_unified_tri_joint_reclaim.py | 1158 +++++++++++++++++ 10 files changed, 2208 insertions(+), 67 deletions(-) create mode 100644 test/registered/unit/mem_cache/test_unified_joint_allocation_eviction.py create mode 100644 test/registered/unit/mem_cache/test_unified_tri_joint_reclaim.py diff --git a/python/sglang/srt/mem_cache/allocation.py b/python/sglang/srt/mem_cache/allocation.py index 4a5393779c..42beda47f5 100644 --- a/python/sglang/srt/mem_cache/allocation.py +++ b/python/sglang/srt/mem_cache/allocation.py @@ -188,8 +188,15 @@ def alloc_paged_token_slots_extend( # availability floor must cover every request's ceil(K_i/N) rounding, # matching the N*ps per-request admission reserve). allocator = tree_cache.token_to_kv_pool_allocator - num_tokens = extend_num_tokens + len(seq_lens_cpu) * ( - allocator.page_size * page_interleave_shard_size(allocator) + shard_size = page_interleave_shard_size(allocator) + conservative_num_tokens = extend_num_tokens + len(seq_lens_cpu) * ( + allocator.page_size * shard_size + ) + num_tokens = allocator.get_extend_allocation_demand( + prefix_lens_cpu, + seq_lens_cpu, + conservative_num_tokens=conservative_num_tokens, + shard_size=shard_size, ) evict_from_tree_cache(tree_cache, num_tokens) diff --git a/python/sglang/srt/mem_cache/allocator/base.py b/python/sglang/srt/mem_cache/allocator/base.py index a5d1aca7dc..6c27e6411b 100644 --- a/python/sglang/srt/mem_cache/allocator/base.py +++ b/python/sglang/srt/mem_cache/allocator/base.py @@ -141,6 +141,17 @@ class BaseTokenToKVPoolAllocator(abc.ABC): swa_budget_tokens is None or swa_tokens <= swa_budget_tokens ) + def get_extend_allocation_demand( + self, + prefix_lens_cpu: torch.Tensor, + seq_lens_cpu: torch.Tensor, + *, + conservative_num_tokens: int, + shard_size: int, + ) -> int: + """Token demand for eviction before paged extend.""" + return conservative_num_tokens + def evict_to_free_tokens(self, tree_cache, num_tokens: int) -> bool | None: """Evict unlocked prefix-cache entries until this allocator can serve ``num_tokens`` or nothing evictable remains. diff --git a/python/sglang/srt/mem_cache/allocator/unified_hybrid_swa.py b/python/sglang/srt/mem_cache/allocator/unified_hybrid_swa.py index 662d463f6b..e2d6f44b5c 100644 --- a/python/sglang/srt/mem_cache/allocator/unified_hybrid_swa.py +++ b/python/sglang/srt/mem_cache/allocator/unified_hybrid_swa.py @@ -41,6 +41,7 @@ from sglang.srt.mem_cache.allocator.unified_sub_pool import ( from sglang.srt.mem_cache.unified_memory_pool import UnifiedKVPool from sglang.srt.runtime_context import get_disagg, get_memory from sglang.srt.utils.common import get_num_new_pages +from sglang.srt.utils.rank_consensus_checker import rank_consensus logger = logging.getLogger(__name__) @@ -1183,6 +1184,23 @@ class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase): hi = mid - 1 return lo + def get_extend_allocation_demand( + self, + prefix_lens_cpu: torch.Tensor, + seq_lens_cpu: torch.Tensor, + *, + conservative_num_tokens: int, + shard_size: int, + ) -> int: + if shard_size != 1: + return conservative_num_tokens + return self.page_size * get_num_new_pages( + seq_lens=seq_lens_cpu, + page_size=self.page_size, + prefix_lens=prefix_lens_cpu, + ) + + @rank_consensus(same_params=["num_tokens", "swa_num_tokens"], same_results=True) def evict_to_free_tokens( self, tree_cache, num_tokens: int, *, swa_num_tokens: Optional[int] = None ) -> bool | None: @@ -1197,6 +1215,10 @@ class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase): and memory.hicache_storage_backend is not None ) or get_disagg().disaggregation_mode == "decode" required_swa = num_tokens if swa_num_tokens is None else swa_num_tokens + _flush_deferred_free_group( + self, + (self.free_group, self.free_page_reps_group, self.full_free_group), + ) reclaim_plan = self.reclaim_plan( num_tokens, required_swa, @@ -1208,11 +1230,11 @@ class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase): return full_reclaim, swa_reclaim = reclaim_plan if full_reclaim or swa_reclaim: - # The shared-byte plan returns cumulative eviction quotas. - # Per-component capacity targets can count the same shared bytes - # independently and stop before the joint allocation fits. - tree_cache.evict( - EvictParams(num_tokens=full_reclaim, swa_num_tokens=swa_reclaim) + tree_cache.evict_for_alloc( + EvictParams(num_tokens=full_reclaim, swa_num_tokens=swa_reclaim), + allocation_reclaim_satisfied=lambda: ( + self.reclaim_plan(num_tokens, required_swa) == (0, 0) + ), ) # A zero-reclaim plan can still depend on compaction before allocation. return self.ensure_capacity(num_tokens, required_swa) @@ -1436,19 +1458,177 @@ class UnifiedMambaSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase): return False if self.can_reserve(full_tokens, swa_tokens): return True - for allocator in self._flush_targets(): - allocator.flush_for_allocation() - if self.can_reserve(full_tokens, swa_tokens): + if full_tokens != swa_tokens: + for allocator in self._flush_targets(): + allocator.flush_for_allocation() + if self.can_reserve(full_tokens, swa_tokens): + return True + _float_open_short_side( + self.swa_attn_allocator, + { + self.full_attn_allocator: -(-full_tokens // self.page_size), + self.swa_attn_allocator: -(-swa_tokens // self.page_size), + self.mamba_allocator: 0, + }, + ) + return self.can_reserve(full_tokens, swa_tokens) + need_tokens = -(-int(full_tokens) // self.page_size) * self.page_size + if not self._token_id_capacity(need_tokens): + return False + if need_tokens <= self.available_size(): return True - _float_open_short_side( + if self._token_end_reclaim_satisfied(need_tokens): + # The certificate preserves FLOAT holes in place. Absorbing FLOAT + # boundaries first would describe a different allocation layout. + self.full_attn_allocator.flush_for_allocation() + self.mamba_allocator.flush_for_allocation() + return need_tokens <= self.available_size() + if self._token_float_high_target(need_tokens) is not None: + self.full_attn_allocator.flush_for_allocation() + self.mamba_allocator.flush_for_allocation() + if not self._token_id_capacity(need_tokens): + return False + if need_tokens <= self.available_size(): + return True + # Recompute from actual frontiers: no credit for another END flush. + target = self._token_float_high_target(need_tokens, flush_ends=False) + if target is None: + return False + self.swa_attn_allocator.make_room(side="high", min_bytes=target) + return self._token_id_capacity(need_tokens) and ( + need_tokens <= self.available_size() + ) + if self._token_allocation_byte_shortfall(need_tokens) > 0: + return False + return _relieve_for_alloc(self, need_tokens) + + def _token_id_capacity(self, num_tokens: int) -> bool: + sa = self.swa_attn_allocator + return ( + num_tokens // self.page_size + <= self.full_attn_allocator.free_virtual_ids.numel() + and num_tokens // self.page_size + <= sa.num_pages - sa.min_page_index - sa._live_pages() + ) + + def _token_reclaim_satisfied(self, num_tokens: int) -> bool: + """Pure sufficient recovery query; False leaves FLOAT geometry unknown.""" + return self._token_end_reclaim_satisfied(num_tokens) or ( + self._token_float_high_target(num_tokens) is not None + ) + + def _token_end_frontiers(self, num_tokens: int, *, flush_ends: bool = True): + """FULL before/after allocation and Mamba after permitted END flushes. + + Pending reuse is never credited. Unflushed FULL holes remain allocatable. + """ + fa, ma = self.full_attn_allocator, self.mamba_allocator + + def end_credit(member): + if not flush_ends or not member.lazy_compaction: + return 0 + if member.moves_blocked(): + return 0 + return member._free_phys_pages.numel() + + full_credit, state_credit = end_credit(fa), end_credit(ma) + full_holes = fa._free_phys_pages.numel() if fa.lazy_compaction else 0 + pages = num_tokens // self.page_size + full_extension = max(0, pages - (full_holes - full_credit)) + full_before = fa._byte_low_frontier() + full_credit * fa.entry_bytes_per_page + full_after = full_before - full_extension * fa.entry_bytes_per_page + state_after = ma._byte_high_frontier() - state_credit * ma.entry_bytes_per_page + return full_before, full_after, state_after + + def _token_end_reclaim_satisfied(self, num_tokens: int) -> bool: + """Sufficient allocation certificate requiring only allowed END flushes.""" + if not self._token_id_capacity(num_tokens): + return False + if num_tokens <= self.available_size(): + return True + fa, sa = self.full_attn_allocator, self.swa_attn_allocator + _, full_after, state_after = self._token_end_frontiers(num_tokens) + pages = num_tokens // self.page_size + if full_after < fa.min_page_index * fa.entry_bytes_per_page: + return False + if sa._is_frontier_transparent(): + return pages <= sa.pages_in_band(low_byte=state_after, high_byte=full_after) + if full_after < sa._byte_high_frontier(): + return False + swa_extension = max(0, pages - sa._hole_pages()) + return swa_extension <= max( + sa.pages_in_band(low_byte=state_after, high_byte=sa._byte_low_frontier()), + sa.pages_in_band(low_byte=sa._byte_high_frontier(), high_byte=full_after), + ) + + def _token_float_high_target( + self, num_tokens: int, *, flush_ends: bool = True + ) -> Optional[int]: + """Absolute HIGH band target after END preparation; None means unknown. + + Reserve fresh LOW destinations for every moved FLOAT page and fresh HIGH + pages for SWA after FULL allocation. No FLOAT hole positions or pending + reuse are credited. The callback performs no movement or device reads. + """ + if not self._token_id_capacity(num_tokens): + return None + fa, sa = self.full_attn_allocator, self.swa_attn_allocator + if sa._is_frontier_transparent(): + return None + if sa.moves_blocked(): + return None + full_before, full_after, state_after = self._token_end_frontiers( + num_tokens, flush_ends=flush_ends + ) + if full_after < fa.min_page_index * fa.entry_bytes_per_page: + return None + e_s = sa.entry_bytes_per_page + low = max(-(-state_after // e_s), sa.min_page_index) + high = min(full_before // e_s, sa.num_pages) + high_after = min(full_after // e_s, sa.num_pages) + limit = high_after - num_tokens // self.page_size + retreat = sa.high_wm_page - limit + if retreat <= 0 or sa.low_wm_page - low < retreat: + return None + # The far gap supplies all retreat destinations even with adverse holes. + # Retreating HIGH to limit leaves the requested SWA pages after FULL. + return (high - limit) * e_s + + def _token_allocation_byte_shortfall( + self, + num_tokens: int, + *, + full_reclaim: int = 0, + swa_reclaim: int = 0, + state_reclaim: int = 0, + ) -> int: + """Optimistic byte lower bound; zero does not certify FLOAT geometry.""" + fa, sa, ma = ( + self.full_attn_allocator, self.swa_attn_allocator, - { - self.full_attn_allocator: -(-full_tokens // self.page_size), - self.swa_attn_allocator: -(-swa_tokens // self.page_size), - self.mamba_allocator: 0, - }, + self.mamba_allocator, + ) + live_bytes = sum( + max(0, member.allocated_count() - reclaim) * member.entry_bytes + for member, reclaim in ( + (fa, full_reclaim), + (sa, swa_reclaim), + (ma, state_reclaim), + ) + ) + sink_bytes = min( + member.min_page_index * member.entry_bytes_per_page + for member in (fa, sa, ma) + ) + required_bytes = ( + num_tokens + // self.page_size + * (fa.entry_bytes_per_page + sa.entry_bytes_per_page) + ) + return max( + 0, + required_bytes + live_bytes + sink_bytes - self.unified_buffer.total_bytes, ) - return self.can_reserve(full_tokens, swa_tokens) def _fits_page_demand(self, full_pages: int, swa_pages: int) -> bool: """Price FULL first, then SWA in one contiguous band on the float grid.""" @@ -1615,17 +1795,105 @@ class UnifiedMambaSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase): (swa_capacity, self.conserve_swa_available_size()), ) - def evict_to_free_tokens(self, tree_cache, num_tokens: int) -> None: - """Joint-aware eviction: one tri-lifetime node frees bytes on several sides - at once, so re-check the JOINT gate instead of the per-side shortfall.""" - # Arbitrary retry bound; a round that frees nothing ends the loop anyway. - for _ in range(4): - before = self.available_size() - if before >= num_tokens: - return - SWATokenToKVPoolAllocator.evict_to_free_tokens(self, tree_cache, num_tokens) - if self.available_size() <= before: - return # no progress + def get_extend_allocation_demand( + self, + prefix_lens_cpu: torch.Tensor, + seq_lens_cpu: torch.Tensor, + *, + conservative_num_tokens: int, + shard_size: int, + ) -> int: + if shard_size != 1: + return conservative_num_tokens + return self.page_size * get_num_new_pages( + seq_lens=seq_lens_cpu, + page_size=self.page_size, + prefix_lens=prefix_lens_cpu, + ) + + @rank_consensus(same_params=["num_tokens"], same_results=True) + def evict_to_free_tokens(self, tree_cache, num_tokens: int) -> bool | None: + if tree_cache is None or not tree_cache.supports_prefix_sharing(): + return + if num_tokens <= 0: + return True + if self.free_group is not None: + self.flush_deferred_full_frees() + num_tokens = -(-num_tokens // self.page_size) * self.page_size + # Immediate capacity needs no reclaim plan or layout preparation. + if self._token_id_capacity(num_tokens) and num_tokens <= self.available_size(): + return True + + from sglang.srt.mem_cache.base_prefix_cache import EvictParams + + full_reclaim = tree_cache.full_evictable_size() + swa_reclaim = tree_cache.swa_evictable_size() + state_reclaim = tree_cache.mamba_evictable_size() + fa, sa, ma = ( + self.full_attn_allocator, + self.swa_attn_allocator, + self.mamba_allocator, + ) + if ( + num_tokens // self.page_size + > fa.free_virtual_ids.numel() + full_reclaim // self.page_size + or num_tokens // self.page_size + > sa.num_pages + - sa.min_page_index + - sa._live_pages() + + swa_reclaim // self.page_size + or self._token_allocation_byte_shortfall( + num_tokens, + full_reclaim=full_reclaim, + swa_reclaim=swa_reclaim, + state_reclaim=state_reclaim, + ) + > 0 + ): + return False + if self.ensure_capacity(num_tokens, num_tokens): + return True + + # Quotas are cumulative logical counts within each phase. Token reclaim + # precedes state reclaim so Mamba-first tree ordering cannot skip it. + state_freed = 0 + if full_reclaim or swa_reclaim: + result = tree_cache.evict_for_alloc( + EvictParams(num_tokens=num_tokens, swa_num_tokens=num_tokens), + allocation_reclaim_satisfied=lambda: self._token_reclaim_satisfied( + num_tokens + ), + ) + state_freed = result.mamba_num_evicted + if ( + result.num_tokens_evicted + or result.swa_num_tokens_evicted + or state_freed + ) and self.ensure_capacity(num_tokens, num_tokens): + return True + + pair_bytes = ( + num_tokens + // self.page_size + * (fa.entry_bytes_per_page + sa.entry_bytes_per_page) + ) + state_quota = max(0, -(-pair_bytes // ma.entry_bytes) - state_freed) + if state_quota and tree_cache.mamba_evictable_size(): + result = tree_cache.evict_for_alloc( + EvictParams(mamba_num=state_quota), + allocation_reclaim_satisfied=lambda: self._token_reclaim_satisfied( + num_tokens + ), + ) + if ( + result.num_tokens_evicted + or result.swa_num_tokens_evicted + or result.mamba_num_evicted + ): + return self.ensure_capacity(num_tokens, num_tokens) + return ( + self._token_id_capacity(num_tokens) and num_tokens <= self.available_size() + ) def verify_byte_accounting(self) -> List[str]: return ( diff --git a/python/sglang/srt/mem_cache/base_prefix_cache.py b/python/sglang/srt/mem_cache/base_prefix_cache.py index 526cd7c4ba..0cbb696c84 100644 --- a/python/sglang/srt/mem_cache/base_prefix_cache.py +++ b/python/sglang/srt/mem_cache/base_prefix_cache.py @@ -505,14 +505,26 @@ class BasePrefixCache(ABC, PrefixCacheTrait): def evict(self, params: EvictParams) -> EvictResult: pass - def evict_for_alloc(self, params: EvictParams) -> EvictResult: - """Evict cache entries to cover allocator shortfalls. + def evict_for_alloc( + self, + params: EvictParams, + *, + allocation_reclaim_satisfied: Optional[Callable[[], bool]] = None, + ) -> EvictResult: + """Evict for allocation using shortfalls or joint-reclaim quotas. - The default implementation preserves the component-count semantics of - :meth:`evict`. Multi-component caches backed by shared memory can - override this entry point to stop once collateral frees make the - requested allocation feasible. + Without ``allocation_reclaim_satisfied``, callers pass component + shortfalls in ``params``. The default implementation forwards these + counts to :meth:`evict`; multi-component caches may stop earlier when + collateral frees satisfy the requested allocation. + + With ``allocation_reclaim_satisfied``, ``params`` instead contains + cumulative reclaim quotas. Supporting caches may stop at the pure + predicate before reaching those quotas. The allocator still owns + preparation and the actual capacity check after the walk. """ + if allocation_reclaim_satisfied is not None: + raise NotImplementedError("This cache does not support joint reclaim") return self.evict(params) @abstractmethod diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index d43d5445c8..e5b0e1f6cf 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -8,7 +8,15 @@ from array import array from dataclasses import replace from http import HTTPStatus from queue import Queue -from typing import TYPE_CHECKING, Iterator, NamedTuple, Optional, Sequence, TypeVar +from typing import ( + TYPE_CHECKING, + Callable, + Iterator, + NamedTuple, + Optional, + Sequence, + TypeVar, +) import msgspec import torch @@ -656,17 +664,33 @@ class UnifiedRadixCache(BasePrefixCache): def evict(self, params: EvictParams) -> EvictResult: return self._evict(params) - @rank_consensus(same_params=True, same_results=True) - def evict_for_alloc(self, params: EvictParams) -> EvictResult: - """Evict until the requested component allocations become feasible. + @rank_consensus( + same_params=["params", "allocation_reclaim_satisfied is not None"], + same_results=True, + ) + def evict_for_alloc( + self, + params: EvictParams, + *, + allocation_reclaim_satisfied: Optional[Callable[[], bool]] = None, + ) -> EvictResult: + """Evict using component shortfalls or cumulative joint-reclaim quotas. - ``params`` contains allocator shortfalls, not absolute eviction quotas. - A component eviction can cascade to its peers; with a shared memory pool, - those collateral frees can satisfy the original allocation before the - triggering component's requested count is reached. + Without ``allocation_reclaim_satisfied``, ``params`` contains component + shortfalls. These become availability targets, allowing collateral frees + to satisfy an allocation before its requested eviction count is reached. + + With ``allocation_reclaim_satisfied``, ``params`` instead contains + cumulative reclaim quotas and component-local targets are bypassed. + The pure predicate may stop the walk before those quotas are reached; + the allocator must prepare and recheck actual capacity after the walk. """ if self.disable: return EvictResult() + if allocation_reclaim_satisfied is not None: + return self._evict( + params, allocation_reclaim_satisfied=allocation_reclaim_satisfied + ) request_by_type = self._evict_request_by_type(params) available_size_targets = { @@ -758,6 +782,8 @@ class UnifiedRadixCache(BasePrefixCache): available_size_targets: Optional[ dict[ComponentType, tuple[ComponentType, int]] ] = None, + *, + allocation_reclaim_satisfied: Optional[Callable[[], bool]] = None, ) -> EvictResult: if self.disable: return EvictResult() @@ -769,6 +795,7 @@ class UnifiedRadixCache(BasePrefixCache): request_by_type, tracker, available_size_targets=available_size_targets, + allocation_reclaim_satisfied=allocation_reclaim_satisfied, ) if ( @@ -890,6 +917,8 @@ class UnifiedRadixCache(BasePrefixCache): available_size_targets: Optional[ dict[ComponentType, tuple[ComponentType, int]] ] = None, + *, + allocation_reclaim_satisfied: Optional[Callable[[], bool]] = None, ) -> None: # Buffer mode: eviction always wins over queued backup intents — a # destroyed victim's intent is stale-swept and the content rewrites @@ -899,6 +928,8 @@ class UnifiedRadixCache(BasePrefixCache): def target_reached(component_type: ComponentType) -> bool: nonlocal last_mamba_donor_check, mamba_donor_prepared + if allocation_reclaim_satisfied is not None: + return allocation_reclaim_satisfied() if available_size_targets is None: return False target = available_size_targets.get(component_type) diff --git a/python/sglang/test/unified_allocator_fixtures.py b/python/sglang/test/unified_allocator_fixtures.py index fa331e7236..0e74d0a2ca 100644 --- a/python/sglang/test/unified_allocator_fixtures.py +++ b/python/sglang/test/unified_allocator_fixtures.py @@ -1,9 +1,15 @@ """Small unified-pool builders and fakes shared by allocator regression tests.""" +from array import array from types import SimpleNamespace import torch +from sglang.srt.mem_cache.base_prefix_cache import InsertParams +from sglang.srt.mem_cache.cache_init_params import CacheInitParams +from sglang.srt.mem_cache.memory_pool import ReqToTokenPool +from sglang.srt.mem_cache.radix_cache import RadixKey +from sglang.srt.mem_cache.unified_cache.components import ComponentType from sglang.srt.mem_cache.unified_memory_pool import ( MambaSubPoolSpec, MHASubPoolSpec, @@ -11,17 +17,22 @@ from sglang.srt.mem_cache.unified_memory_pool import ( init_unified_mamba_swa_pools, init_unified_swa_pools, ) +from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache from sglang.srt.runtime_context import publish, reset_context from sglang.srt.server_args import ServerArgs def setup_allocator_context(): reset_context() - publish(ServerArgs(model_path="dummy", device="cpu"), role="tokenizer") + args = ServerArgs(model_path="dummy", device="cpu") + # Mamba cache setup must not resolve the dummy model through Hugging Face. + args._mamba_cache_chunk_size = 64 + publish(args, role="tokenizer") -def build_swa_pool(*, page_size=1): - """Lazy full + SWA composite over real CPU pools, 100 pages per side.""" +def build_swa_pool(*, page_size=1, lazy=True, ratio=(1, 1), total_bytes=None): + """Full + SWA composite over real CPU pools, 100 pages per side.""" + full_layers, swa_layers = ratio bundle = init_unified_swa_pools( device="cpu", kv_cache_dtype=torch.float16, @@ -33,23 +44,25 @@ def build_swa_pool(*, page_size=1): swa_v_head_dim=8, page_size=page_size, start_layer=0, - end_layer=2, - swa_attention_layer_ids=[1], - full_attention_layer_ids=[0], + end_layer=full_layers + swa_layers, + swa_attention_layer_ids=list(range(full_layers, full_layers + swa_layers)), + full_attention_layer_ids=list(range(full_layers)), full_max_total_num_tokens=100 * page_size, swa_max_total_num_tokens=100 * page_size, enable_memory_saver=False, need_sort=False, - lazy_compaction=True, + lazy_compaction=lazy, + total_bytes=total_bytes, ) return bundle.token_to_kv_pool_allocator -def build_tri_pool(*, page_size=1): - """Lazy full + SWA + mamba composite over real CPU pools; returns the pool +def build_tri_pool(*, page_size=1, lazy=True, ratio=(1, 1), temporal=(1, 4, 8)): + """Full + SWA + Mamba composite over real CPU pools; returns the pool bundle and its allocator.""" + full_layers, swa_layers = ratio config = SimpleNamespace( - shape=SimpleNamespace(conv=[(3, 8)], temporal=(1, 4, 8)), + shape=SimpleNamespace(conv=[(3, 8)], temporal=temporal), dtype=SimpleNamespace(conv=torch.bfloat16, temporal=torch.float32), layers=[0], ) @@ -64,14 +77,14 @@ def build_tri_pool(*, page_size=1): swa_v_head_dim=8, page_size=page_size, start_layer=0, - end_layer=2, - swa_attention_layer_ids=[1], - full_attention_layer_ids=[0], + end_layer=full_layers + swa_layers, + swa_attention_layer_ids=list(range(full_layers, full_layers + swa_layers)), + full_attention_layer_ids=list(range(full_layers)), full_max_total_num_tokens=100 * page_size, swa_max_total_num_tokens=100 * page_size, enable_memory_saver=False, need_sort=False, - lazy_compaction=True, + lazy_compaction=lazy, mamba_layer_ids=[0], mamba2_cache_params=config, max_mamba_cache_size=8, @@ -150,3 +163,75 @@ def tri_sub_pool_specs( grow_direction="up", ) return full, swa, mamba + + +def build_swa_cache( + *, + occupancy=96, + lazy=False, + page_size=1, + ratio=(1, 1), + total_bytes=None, + sessions=False, + window=None, +): + allocator = build_swa_pool( + page_size=page_size, lazy=lazy, ratio=ratio, total_bytes=total_bytes + ) + req_pool = ReqToTokenPool( + size=4, + max_context_len=256 * page_size, + device="cpu", + enable_memory_saver=False, + ) + cache = UnifiedRadixCache( + CacheInitParams( + disable=False, + enable_session_radix_cache=sessions, + req_to_token_pool=req_pool, + token_to_kv_pool_allocator=allocator, + page_size=page_size, + sliding_window_size=128 * page_size if window is None else window, + tree_components=(ComponentType.FULL, ComponentType.SWA), + ) + ) + slots = allocator.alloc(occupancy * page_size) + assert slots is not None, "Fixture allocation must fit" + for i in range(occupancy): + cache.insert( + InsertParams( + key=RadixKey(array("q", range(i * 1000, i * 1000 + page_size))), + value=slots[i * page_size : (i + 1) * page_size], + ) + ) + return allocator, cache + + +def build_tri_cache( + *, + lazy=True, + temporal=(0, 0, 0), + page_size=1, + ratio=(1, 1), + sessions=False, + state_cache=True, +): + bundle, a = build_tri_pool( + page_size=page_size, lazy=lazy, ratio=ratio, temporal=temporal + ) + c = UnifiedRadixCache( + CacheInitParams( + disable=False, + req_to_token_pool=bundle.req_to_token_pool, + token_to_kv_pool_allocator=a, + page_size=page_size, + sliding_window_size=128 * page_size, + tree_components=( + (ComponentType.FULL, ComponentType.SWA, ComponentType.MAMBA) + if state_cache + else (ComponentType.FULL, ComponentType.SWA) + ), + enable_session_radix_cache=sessions, + ) + ) + return bundle, a, c diff --git a/test/registered/unit/mem_cache/test_dllm_fdfo_kv_reuse.py b/test/registered/unit/mem_cache/test_dllm_fdfo_kv_reuse.py index cc92a6b4b1..bfd901ce3d 100644 --- a/test/registered/unit/mem_cache/test_dllm_fdfo_kv_reuse.py +++ b/test/registered/unit/mem_cache/test_dllm_fdfo_kv_reuse.py @@ -27,6 +27,16 @@ class _FakeAllocator: def available_size(self): return 1 << 30 + def get_extend_allocation_demand( + self, + prefix_lens_cpu, + seq_lens_cpu, + *, + conservative_num_tokens, + shard_size, + ): + return conservative_num_tokens + def alloc(self, need_size): self.alloc_calls.append(need_size) return torch.arange(self.base, self.base + need_size, dtype=torch.int64) diff --git a/test/registered/unit/mem_cache/test_prefill_memory_budget.py b/test/registered/unit/mem_cache/test_prefill_memory_budget.py index eadb5f7894..8963f338f8 100644 --- a/test/registered/unit/mem_cache/test_prefill_memory_budget.py +++ b/test/registered/unit/mem_cache/test_prefill_memory_budget.py @@ -346,17 +346,31 @@ class TestFixedPrefillMemoryBudget(unittest.TestCase): ) def test_tri_pool_eviction_does_not_reenter_common(self): - allocator = self._allocator(UnifiedMambaSWATokenToKVPoolAllocator) - allocator.available_size = lambda: 0 - allocator.full_available_size = lambda: 0 - allocator.swa_available_size = lambda: 0 - cache = _cache() - cache.token_to_kv_pool_allocator = allocator - cache.evict_for_alloc = MagicMock() - evict_from_tree_cache(cache, 8) - cache.evict_for_alloc.assert_called_once() - params = cache.evict_for_alloc.call_args.args[0] - self.assertEqual((params.num_tokens, params.swa_num_tokens), (8, 8)) + from array import array + + from sglang.srt.mem_cache.base_prefix_cache import InsertParams + from sglang.srt.mem_cache.radix_cache import RadixKey + from sglang.srt.runtime_context import reset_context + from sglang.test.unified_allocator_fixtures import ( + build_tri_cache, + setup_allocator_context, + ) + + setup_allocator_context() + self.addCleanup(reset_context) + bundle, allocator, cache = build_tri_cache(state_cache=False) + states = bundle.req_to_token_pool.mamba_allocator.alloc(8) + slots = allocator.alloc(96) + self.assertIsNotNone(states) + self.assertIsNotNone(slots) + for i in range(96): + cache.insert( + InsertParams(key=RadixKey(array("q", [i])), value=slots[i : i + 1]) + ) + self.assertLess(allocator.available_size(), 8) + self.assertTrue(evict_from_tree_cache(cache, 8)) + self.assertIsNotNone(allocator.alloc(8)) + self.assertFalse(allocator.verify_byte_accounting()) if __name__ == "__main__": diff --git a/test/registered/unit/mem_cache/test_unified_joint_allocation_eviction.py b/test/registered/unit/mem_cache/test_unified_joint_allocation_eviction.py new file mode 100644 index 0000000000..b56c2beec0 --- /dev/null +++ b/test/registered/unit/mem_cache/test_unified_joint_allocation_eviction.py @@ -0,0 +1,545 @@ +"""Real shared-pool allocation after allocation-driven cache eviction.""" + +import unittest +from array import array +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.srt.mem_cache.base_prefix_cache import ( + EvictParams, + InsertParams, + MatchPrefixParams, +) +from sglang.srt.mem_cache.common import evict_from_tree_cache +from sglang.srt.mem_cache.radix_cache import RadixKey +from sglang.srt.runtime_context import reset_context +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase +from sglang.test.unified_allocator_fixtures import ( + build_swa_cache, + setup_allocator_context, +) + +register_cpu_ci(est_time=30, suite="base-a-test-cpu") + + +class TestUnifiedJointAllocationEviction(CustomTestCase): + def setUp(self): + self.addCleanup(reset_context) + setup_allocator_context() + + def tearDown(self): + reset_context() + + build_cache = staticmethod(build_swa_cache) + + def mark_prefixes(self, allocator, cache, occupancy, page_size=1): + records = [] + kv = allocator.get_kvcache() + for i in range(occupancy): + key = RadixKey(array("q", range(i * 1000, i * 1000 + page_size))) + slots = cache.path_device_indices( + cache.match_prefix(MatchPrefixParams(key=key)).last_device_node + ).clone() + self.assertEqual(len(slots), page_size) + for layer, member in enumerate( + (allocator.full_attn_allocator, allocator.swa_attn_allocator) + ): + ids = member.translate_kv_loc(slots) + marker = slots.to(torch.float16).view(-1, 1, 1) + 1000 * layer + kv.get_key_buffer(layer)[ids] = marker + kv.get_value_buffer(layer)[ids] = marker + 100 + records.append((key, slots)) + return records + + def check_prefixes(self, allocator, cache, records, expected): + kv = allocator.get_kvcache() + kept = [] + for i, (key, slots) in enumerate(records): + actual = cache.path_device_indices( + cache.match_prefix(MatchPrefixParams(key=key)).last_device_node + ) + if i not in expected: + self.assertEqual(len(actual), 0, i) + continue + kept.append(i) + torch.testing.assert_close(actual, slots) + for layer, member in enumerate( + (allocator.full_attn_allocator, allocator.swa_attn_allocator) + ): + ids = member.translate_kv_loc(slots) + marker = slots.to(torch.float16).view(-1, 1, 1) + 1000 * layer + self.assertTrue(torch.all(kv.get_key_buffer(layer)[ids] == marker)) + self.assertTrue( + torch.all(kv.get_value_buffer(layer)[ids] == marker + 100) + ) + self.assertEqual(kept, list(expected)) + self.assertFalse(allocator.verify_byte_accounting()) + + def test_joint_reclaim_stops_after_actual_cascade(self): + for lazy in (False, True): + for page_size in (1, 4, 16): + for demand, first in ((3, 0), (4, 1), (6, 3), (90, 87)): + with self.subTest(lazy=lazy, page_size=page_size, demand=demand): + allocator, cache = self.build_cache( + lazy=lazy, page_size=page_size + ) + records = self.mark_prefixes(allocator, cache, 96, page_size) + self.assertTrue( + evict_from_tree_cache(cache, demand * page_size) + ) + self.assertIsNotNone(allocator.alloc(demand * page_size)) + self.check_prefixes(allocator, cache, records, range(first, 96)) + + def test_impossible_byte_demand_preserves_cache(self): + for lazy in (False, True): + with self.subTest(lazy=lazy): + allocator, cache = self.build_cache(lazy=lazy) + records = self.mark_prefixes(allocator, cache, 96) + self.assertIsNone(allocator.reclaim_plan(101, 101, empty_pool=True)) + for _ in range(3): + evict_from_tree_cache(cache, 101) + self.assertIsNone(allocator.alloc(101)) + self.check_prefixes(allocator, cache, records, range(96)) + + def test_pending_groups_reclaim_without_prefix_eviction(self): + for lazy in (False, True): + for mode, page_size in (("ordinary", 1), ("representatives", 4)): + with self.subTest(lazy=lazy, mode=mode): + allocator, cache = self.build_cache( + occupancy=94, lazy=lazy, page_size=page_size + ) + records = self.mark_prefixes(allocator, cache, 94, page_size) + temporary = allocator.alloc(2 * page_size) + allocator.free_group_begin() + if mode == "ordinary": + allocator.free(temporary) + else: + allocator.free_segment(temporary, start_pos=0) + self.assertLess(allocator.available_size(), 4 * page_size) + self.assertTrue(evict_from_tree_cache(cache, 4 * page_size)) + out = allocator.alloc(4 * page_size) + self.assertIsNotNone(out) + self.assertEqual(allocator.free_group, []) + self.assertEqual(allocator.free_page_reps_group, []) + self.check_prefixes(allocator, cache, records, range(94)) + allocator.free_group_end() + allocator.free(out) + cache.evict(EvictParams(num_tokens=94 * page_size)) + self.assertEqual(allocator.full_attn_allocator.allocated_count(), 0) + self.assertEqual(allocator.swa_attn_allocator.allocated_count(), 0) + self.assertFalse(allocator.verify_byte_accounting()) + + def test_locked_prefixes_preserve_cache(self): + allocator, cache = self.build_cache() + records = self.mark_prefixes(allocator, cache, 96) + locks = [] + for key, _ in records: + node = cache.match_prefix(MatchPrefixParams(key=key)).last_device_node + locks.append((node, cache.inc_lock_ref(node))) + try: + evict_from_tree_cache(cache, 4) + self.assertIsNone(allocator.alloc(4)) + self.check_prefixes(allocator, cache, records, range(96)) + finally: + for node, lock in locks: + cache.dec_lock_ref(node, lock.to_dec_params()) + self.assertTrue(evict_from_tree_cache(cache, 4)) + self.assertIsNotNone(allocator.alloc(4)) + self.check_prefixes(allocator, cache, records, range(1, 96)) + + def test_explicit_and_legacy_allocation_eviction_keep_count_contract(self): + for legacy in (False, True): + with self.subTest(legacy=legacy): + allocator, cache = self.build_cache() + records = self.mark_prefixes(allocator, cache, 96) + method = cache.evict_for_alloc if legacy else cache.evict + result = method(EvictParams(num_tokens=2)) + self.assertEqual(result.num_tokens_evicted, 2) + self.check_prefixes(allocator, cache, records, range(2, 96)) + + def test_schedule_batch_decode_uses_allocator_contract(self): + from sglang.srt.managers.schedule_batch import ScheduleBatch + from sglang.srt.speculative.spec_info import SpeculativeAlgorithm + + allocator, cache = self.build_cache() + records = self.mark_prefixes(allocator, cache, 96) + batch = object.__new__(ScheduleBatch) + batch.token_to_kv_pool_allocator = allocator + batch.tree_cache = cache + batch.spec_algorithm = SpeculativeAlgorithm.NONE + batch.reqs = [ + SimpleNamespace(kv=SimpleNamespace(kv_committed_len=8), beam_group=None) + for _ in range(6) + ] + self.assertTrue(batch.check_decode_mem()) + self.assertIsNotNone(allocator.alloc(6)) + self.check_prefixes(allocator, cache, records, range(3, 96)) + + def test_exact_extend_demand_preserves_other_allocator_dispatch(self): + from sglang.srt.mem_cache.allocator.base import BaseTokenToKVPoolAllocator + from sglang.srt.mem_cache.allocator.unified_hybrid_swa import ( + UnifiedMambaSWATokenToKVPoolAllocator, + UnifiedSWAAllocatorBase, + ) + + allocator, _ = self.build_cache(occupancy=0, page_size=4) + prefix = torch.tensor([1, 4, 7]) + seq = torch.tensor([3, 5, 9]) + kwargs = dict(conservative_num_tokens=19, shard_size=1) + self.assertEqual( + allocator.get_extend_allocation_demand(prefix, seq, **kwargs), 8 + ) + self.assertEqual( + allocator.get_extend_allocation_demand( + prefix, seq, conservative_num_tokens=31, shard_size=2 + ), + 31, + ) + self.assertIs( + UnifiedSWAAllocatorBase.get_extend_allocation_demand, + BaseTokenToKVPoolAllocator.get_extend_allocation_demand, + ) + self.assertEqual( + UnifiedMambaSWATokenToKVPoolAllocator.get_extend_allocation_demand( + allocator, prefix, seq, **kwargs + ), + 8, + ) + self.assertEqual( + BaseTokenToKVPoolAllocator.get_extend_allocation_demand( + allocator, prefix, seq, **kwargs + ), + 19, + ) + + def test_page128_partial_extend_and_conservative_fallback(self): + from sglang.srt.mem_cache.allocator.base import BaseTokenToKVPoolAllocator + from sglang.srt.mem_cache.allocator.unified_hybrid_swa import ( + UnifiedMambaSWATokenToKVPoolAllocator, + ) + + allocator, _ = self.build_cache(occupancy=0, page_size=128) + for prefix, seq, expected in ( + ([0], [1], 128), + ([127], [128], 0), + ([128], [129], 128), + ([129], [130], 0), + ([127], [129], 128), + ([0, 127, 128, 129], [1, 128, 129, 257], 384), + ): + with self.subTest(prefix=prefix, seq=seq): + p, q = torch.tensor(prefix), torch.tensor(seq) + for query in ( + type(allocator).get_extend_allocation_demand, + UnifiedMambaSWATokenToKVPoolAllocator.get_extend_allocation_demand, + ): + self.assertEqual( + query( + allocator, p, q, conservative_num_tokens=1024, shard_size=1 + ), + expected, + ) + self.assertEqual( + query( + allocator, p, q, conservative_num_tokens=1024, shard_size=2 + ), + 1024, + ) + self.assertEqual( + BaseTokenToKVPoolAllocator.get_extend_allocation_demand( + allocator, p, q, conservative_num_tokens=1024, shard_size=1 + ), + 1024, + ) + + def test_pending_full_only_reclaim_is_needed_before_walk(self): + for lazy in (False, True): + with self.subTest(lazy=lazy): + allocator, cache = self.build_cache(occupancy=94, lazy=lazy) + records = self.mark_prefixes(allocator, cache, 94) + extra = allocator.full_attn_allocator.alloc(4) + self.assertIsNotNone(extra) + allocator.free_group_begin() + allocator.free_full(extra[:2]) + self.assertLess(allocator.available_size(), 4) + self.assertTrue(evict_from_tree_cache(cache, 4)) + out = allocator.alloc(4) + self.assertIsNotNone(out) + self.check_prefixes(allocator, cache, records, range(94)) + allocator.free_group_end() + allocator.free(out) + allocator.free_full(extra[2:]) + cache.evict(EvictParams(num_tokens=94)) + self.assertEqual(allocator.full_attn_allocator.allocated_count(), 0) + self.assertFalse(allocator.verify_byte_accounting()) + + def test_shared_virtual_id_limit_is_independent_of_swa_bytes(self): + allocator, cache = self.build_cache(occupancy=0, ratio=(4, 1)) + fa, sa = allocator.full_attn_allocator, allocator.swa_attn_allocator + demand = fa.num_virtual_ids - fa.min_page_index + 1 + # Asymmetric SWA restore demand fits bytes/physical SWA slots, but + # cannot address more virtual pages than its FULL ID owner provides. + self.assertLess( + demand * sa.entry_bytes_per_page, allocator._empty_shared_gap_bytes + ) + self.assertLess(demand, sa.num_pages - sa.min_page_index) + self.assertIsNone(allocator.reclaim_plan(0, demand, empty_pool=True)) + self.assertFalse(allocator.ensure_capacity(0, demand)) + self.assertEqual(cache.full_evictable_size(), 0) + self.assertFalse(allocator.verify_byte_accounting()) + + def test_exact_extend_caller_reclaims_only_new_pages(self): + from sglang.srt.mem_cache.allocation import alloc_paged_token_slots_extend + + class CPUReferenceKernel: + # Only the external Triton index-generation kernel is replaced; + # the actual caller, capacity checks and page bindings run on CPU. + def __getitem__(self, grid): + def launch(prefix, seq, last, free, out, batch_size, page_size): + next_page = offset = 0 + for i in range(len(prefix)): + for pos in range(int(prefix[i]), int(seq[i])): + if pos % page_size == 0: + page = int(free[next_page]) + next_page += 1 + elif pos == int(prefix[i]): + page = int(last[i]) // page_size + out[offset] = page * page_size + pos % page_size + offset += 1 + + return launch + + cases = [ + (2, [0], [396], 0), + (0, [0], [396], 0), + (94, [1, 4, 7], [3, 5, 9], 1), + (96, [1], [3], 0), + (96, [4], [8], 0), + ] + for occupancy, prefixes, lengths, evicted in cases: + with self.subTest(occupancy=occupancy, prefixes=prefixes, lengths=lengths): + allocator, cache = self.build_cache(occupancy=occupancy, page_size=4) + records = self.mark_prefixes(allocator, cache, occupancy, 4) + prefix_slots = [ + allocator.alloc((n + 3) // 4 * 4) + if n + else torch.empty(0, dtype=torch.int64) + for n in prefixes + ] + for slots in prefix_slots: + self.assertIsNotNone(slots) + last = torch.tensor( + [ + int(slots[n - 1]) if n else -1 + for slots, n in zip(prefix_slots, prefixes) + ], + dtype=torch.int64, + ) + prefix = torch.tensor(prefixes, dtype=torch.int64) + seq = torch.tensor(lengths, dtype=torch.int64) + extra_tokens = sum(b - a for a, b in zip(prefixes, lengths)) + before_pages = allocator.full_attn_allocator.allocated_count() // 4 + with patch( + "sglang.srt.mem_cache.allocator.unified_sub_pool.alloc_extend_kernel", + CPUReferenceKernel(), + ): + out = alloc_paged_token_slots_extend( + cache, prefix, prefix, seq, seq, last, extra_tokens + ) + self.assertEqual(len(out), extra_tokens) + self.assertEqual(len(set(out.tolist())), extra_tokens) + expected_pages = sum( + (b + 3) // 4 - (a + 3) // 4 for a, b in zip(prefixes, lengths) + ) + actual_evicted = occupancy if lengths == [396] else evicted + self.assertEqual( + allocator.full_attn_allocator.allocated_count() // 4, + before_pages - actual_evicted + expected_pages, + ) + offset = 0 + for n, end, slots in zip(prefixes, lengths, prefix_slots): + reused = min(end, (n + 3) // 4 * 4) - n + if reused: + torch.testing.assert_close( + out[offset : offset + reused], slots[n : n + reused] + ) + offset += end - n + self.check_prefixes( + allocator, cache, records, range(actual_evicted, occupancy) + ) + + def test_reclaim_predicate_does_not_drain_pending_groups_or_move_pages(self): + allocator, cache = self.build_cache(occupancy=94, lazy=True) + temporary = allocator.alloc(2) + allocator.free_swa(temporary[:1]) + allocator.free_group_begin() + allocator.free(temporary) + + def snapshot(): + groups = [ + [t.tolist() for t in group] + for group in ( + allocator.free_group, + allocator.free_page_reps_group, + allocator.full_free_group, + ) + ] + members = [] + for member in allocator._flush_targets(): + members.append( + ( + member.virtual_to_physical.tolist(), + member.physical_to_virtual.tolist(), + member._free_phys_pages.tolist(), + member._byte_low_frontier(), + member._byte_high_frontier(), + member.allocated_count(), + ) + ) + return groups, members + + before = snapshot() + for gate in (False, True): + for member in allocator._flush_targets(): + member.disagg_move_gate = lambda: gate + for _ in range(20): + allocator.reclaim_plan(4, 4) + self.assertEqual(snapshot(), before) + for member in allocator._flush_targets(): + member.disagg_move_gate = None + self.assertTrue(evict_from_tree_cache(cache, 4)) + self.assertIsNotNone(allocator.alloc(4)) + self.assertEqual(cache.full_evictable_size(), 94) + allocator.free_group_end() + self.assertFalse(allocator.verify_byte_accounting()) + + def _check_internal_swa_reclaim(self, *, sessions): + allocator, cache = self.build_cache(occupancy=0, sessions=sessions, window=1) + # Each incoming branch owns its duplicate prefix until insert + # deduplicates it: two of these 98 slots will be returned. + slots = allocator.alloc(98) + branch_keys = [] + for offset, first in ((0, 10), (6, 20)): + first_key = RadixKey(array("q", [first, first + 1, first + 2])) + second_key = RadixKey(array("q", [first, first + 3, first + 4])) + branch_keys.extend((first_key, second_key)) + cache.insert( + InsertParams( + key=first_key, + value=slots[offset : offset + 3], + ) + ) + cache.insert( + InsertParams( + key=second_key, + value=slots[offset + 3 : offset + 6], + ) + ) + for i in range(12, 98): + cache.insert( + InsertParams( + key=RadixKey(array("q", [1000 + i])), value=slots[i : i + 1] + ) + ) + # A short SWA window refreshes the two-token child, leaving + # its internal ancestor as the oldest SWA reclaim candidate. + for key in branch_keys: + cache.match_prefix(MatchPrefixParams(key=key)) + self.assertEqual(allocator.full_attn_allocator.allocated_count(), 96) + self.assertEqual(allocator.available_size(), 3) + kv = allocator.get_kvcache() + for layer, member in enumerate(allocator._flush_targets()): + ids = member.translate_kv_loc(slots[12:13]) + kv.get_key_buffer(layer)[ids] = 17 + layer + kv.get_value_buffer(layer)[ids] = 117 + layer + original = cache._evict_device_next_node + observed = [] + + def step(component, tracker): + result = original(component, tracker) + observed.append((result, allocator.reclaim_plan(4, 4) == (0, 0))) + return result + + with patch.object(cache, "_evict_device_next_node", side_effect=step): + cache.evict_for_alloc( + EvictParams(swa_num_tokens=2), + allocation_reclaim_satisfied=lambda: ( + allocator.reclaim_plan(4, 4) == (0, 0) + ), + ) + self.assertTrue( + any(node is None and progress for (node, progress), _ in observed), + observed, + ) + self.assertTrue(observed[-1][1]) + self.assertEqual(len(observed), 1) + self.assertEqual(allocator.full_attn_allocator.allocated_count(), 96) + # The arena's half-pair slack plus one SWA page funds the + # demand. Do not consume the second internal victim/quota. + self.assertEqual(allocator.swa_attn_allocator.allocated_count(), 95) + self.assertIsNotNone(allocator.alloc(4)) + # The next independent leaf survives the internal tombstones. + actual = cache.path_device_indices( + cache.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", [1012]))) + ).last_device_node + ) + torch.testing.assert_close(actual, slots[12:13]) + for layer, member in enumerate(allocator._flush_targets()): + ids = member.translate_kv_loc(actual) + self.assertTrue(torch.all(kv.get_key_buffer(layer)[ids] == 17 + layer)) + self.assertTrue(torch.all(kv.get_value_buffer(layer)[ids] == 117 + layer)) + self.assertFalse(allocator.verify_byte_accounting()) + + def test_internal_swa_reclaim_drains_before_joint_stop(self): + self._check_internal_swa_reclaim(sessions=False) + + def test_session_cursor_stops_after_internal_reclaim(self): + self._check_internal_swa_reclaim(sessions=True) + + def test_rank_consensus_ignores_callback_identity(self): + import os + import subprocess + import sys + + script = r""" +import queue, runpy, sys, threading +from unittest.mock import patch +from sglang.srt.utils import rank_consensus_checker as checker +module = runpy.run_path(sys.argv[1], run_name="rank_fixture") +fixture = module["TestUnifiedJointAllocationEviction"]() +traces = [] +for _ in range(2): + fixture.setUp() + try: + allocator, cache = fixture.build_cache() + records = fixture.mark_prefixes(allocator, cache, 96) + q = queue.Queue() + with patch.object(checker, "_q", q), patch.object(checker, "_scheduler_thread", threading.current_thread()): + assert module["evict_from_tree_cache"](cache, 6) + assert allocator.alloc(6) is not None + fixture.check_prefixes(allocator, cache, records, range(3, 96)) + traces.append(list(q.queue)) + finally: + fixture.tearDown() +assert traces[0] == traces[1], traces +assert any("num_tokens=6" in event for event in traces[0]), traces +assert any("allocation_reclaim_satisfied is not None=True" in event for event in traces[0]), traces +assert not any(" 0) + ) + ids = member.translate_kv_loc(slots) + values = slots.to(torch.float16).view(-1, 1, 1) + layer * 100 + torch.testing.assert_close( + kv.get_key_buffer(layer)[ids], + values.expand_as(kv.get_key_buffer(layer)[ids]), + ) + torch.testing.assert_close( + kv.get_value_buffer(layer)[ids], + (values + 500).expand_as(kv.get_value_buffer(layer)[ids]), + ) + mc = b.req_to_token_pool.mamba_pool.mamba_cache + self.assertTrue(torch.all(a.mamba_allocator.virtual_to_physical[states] > 0)) + for j, view in enumerate([*mc.conv, mc.temporal]): + if view.numel(): + mids = a.mamba_allocator.translate_kv_loc(states) + for v, mid in zip(states.tolist(), mids): + i = original_states.tolist().index(v) + self.assertTrue(torch.all(view[:, mid] == i + 10 * j + 1)) + + def insert_parts(self, c, slots, states, *, page_size=1): + records = [] + for i, (lo, hi) in enumerate( + ((0, page_size), (page_size, 2 * page_size), (2 * page_size, len(slots))) + ): + key = RadixKey(array("q", range(i * 10000, i * 10000 + hi - lo))) + c.insert( + InsertParams(key=key, value=slots[lo:hi], mamba_value=states[i : i + 1]) + ) + records.append((key, slots[lo:hi].clone())) + return records + + def test_joint_reclaim_keeps_live_prefix_and_state(self): + for lazy in (False, True): + for occupancy, count, temporal, outside_count in ( + (96, 8, (0, 0, 0), 0), + (80, 24, (0, 0, 0), 0), + (96, 5, (1, 4, 8), 1), + ): + with self.subTest(lazy=lazy, occupancy=occupancy, states=count): + b, a, c = self.build(lazy=lazy, temporal=temporal) + slots = a.alloc(occupancy) + states = b.req_to_token_pool.mamba_allocator.alloc(count) + outside = a.alloc(outside_count) + self.assertIsNotNone(slots) + self.assertIsNotNone(states) + self.assertIsNotNone(outside) + self.stamp(b, a, torch.cat((slots, outside)), states) + records = self.insert_parts(c, slots, states) + demand = 4 if count == 8 else a.available_size() + 1 + if count == 24: + demand = min(a.full_available_size(), a.swa_available_size()) + with patch.object( + a, "ensure_capacity", wraps=a.ensure_capacity + ) as prep: + evict_from_tree_cache(c, demand) + got = a.alloc(demand) + self.assertIsNotNone(got) + self.assertLessEqual(prep.call_count, 4) + expected = ( + [] if count == 24 else ([1, 2] if count == 8 else [0, 1, 2]) + ) + for i, (key, prior) in enumerate(records): + match = c.path_device_indices( + c.match_prefix(MatchPrefixParams(key=key)).last_device_node + ) + if i in expected: + torch.testing.assert_close(match, prior) + self.check_payload(b, a, match, states[i : i + 1], states) + else: + self.assertEqual(len(match), 0) + self.check_payload(b, a, outside, states[3:], states) + if count == 8: + self.assertEqual(c.full_evictable_size(), 95) + self.assertFalse(a.verify_byte_accounting()) + + def test_impossible_and_locked_demands_preserve_cache(self): + for lazy in (False, True): + for locked in (False, True): + with self.subTest(lazy=lazy, locked=locked): + b, a, c = self.build(lazy=lazy) + slots = a.alloc(96) + states = b.req_to_token_pool.mamba_allocator.alloc(8) + self.stamp(b, a, slots, states) + records = self.insert_parts(c, slots, states) + receipts = [] + if locked: + for key, _ in records: + node = c.match_prefix( + MatchPrefixParams(key=key) + ).last_device_node + receipts.append((node, c.inc_lock_ref(node))) + try: + with patch.object( + c, "evict_for_alloc", wraps=c.evict_for_alloc + ) as walks: + for _ in range(3): + evict_from_tree_cache(c, 4 if locked else 200) + self.assertEqual(walks.call_count, 0) + for key, prior in records: + torch.testing.assert_close( + c.path_device_indices( + c.match_prefix( + MatchPrefixParams(key=key) + ).last_device_node + ), + prior, + ) + self.check_payload(b, a, slots, states, states) + finally: + for node, receipt in receipts: + c.dec_lock_ref(node, receipt.to_dec_params()) + self.assertFalse(a.verify_byte_accounting()) + + def test_state_phase_preserves_locked_token_bindings(self): + b, a, c = self.build(temporal=(1, 4, 8)) + states = b.req_to_token_pool.mamba_allocator.alloc(8) + slots = a.alloc(96) + self.assertIsNotNone(states) + self.assertIsNotNone(slots) + self.stamp(b, a, slots, states) + records = self.insert_parts(c, slots, states) + receipts = [] + for key, _ in records: + node = c.match_prefix(MatchPrefixParams(key=key)).last_device_node + receipts.append( + ( + node, + c.inc_lock_ref(node, skip_lock_components=(ComponentType.MAMBA,)), + ) + ) + demand = min(a.conserve_full_available_size(), a.conserve_swa_available_size()) + try: + self.assertEqual(c.full_evictable_size(), 0) + self.assertGreater(c.mamba_evictable_size(), 0) + phases = [] + original = c.evict_for_alloc + + def walk(params, **kw): + phases.append(params) + return original(params, **kw) + + with patch.object(c, "evict_for_alloc", side_effect=walk): + evict_from_tree_cache(c, demand) + got = a.alloc(demand) + self.assertIsNotNone(got) + self.assertEqual(len(phases), 1) + self.assertEqual(phases[0].num_tokens, 0) + self.assertGreater(phases[0].mamba_num, 0) + for (node, _), (_, prior) in zip(receipts, records): + torch.testing.assert_close( + c.tree_core.get_component_device_value(node, ComponentType.FULL), + prior, + ) + self.check_payload(b, a, slots, states[3:], states) + finally: + for node, receipt in receipts: + c.dec_lock_ref(node, receipt.to_dec_params()) + self.assertFalse(a.verify_byte_accounting()) + + def test_exact_extend_demand_avoids_impossible_probe(self): + from sglang.srt.mem_cache.allocation import alloc_paged_token_slots_extend + + class CPUReferenceKernel: + def __getitem__(self, grid): + def launch(prefix, seq, last, free, out, batch_size, page_size): + next_page = offset = 0 + for i in range(len(prefix)): + for pos in range(int(prefix[i]), int(seq[i])): + if pos % page_size == 0: + page = int(free[next_page]) + next_page += 1 + elif pos == int(prefix[i]): + page = int(last[i]) // page_size + out[offset] = page * page_size + pos % page_size + offset += 1 + + return launch + + for occupancy in (0, 2): + with self.subTest(cached_pages=occupancy): + b, a, c = self.build(page_size=4, state_cache=False) + states = b.req_to_token_pool.mamba_allocator.alloc(8) + self.assertIsNotNone(states) + slots = a.alloc(occupancy * 4) + self.assertIsNotNone(slots) + self.stamp(b, a, slots, states) + for i in range(occupancy): + c.insert( + InsertParams( + key=RadixKey(array("q", range(i * 100, i * 100 + 4))), + value=slots[i * 4 : (i + 1) * 4], + ) + ) + self.assertGreater( + a._token_allocation_byte_shortfall( + 400, + full_reclaim=c.full_evictable_size(), + swa_reclaim=c.swa_evictable_size(), + state_reclaim=c.mamba_evictable_size(), + ), + 0, + ) + prefix = torch.tensor([0]) + seq = torch.tensor([396]) + last = torch.tensor([-1]) + with patch( + "sglang.srt.mem_cache.allocator.unified_sub_pool.alloc_extend_kernel", + CPUReferenceKernel(), + ): + got = alloc_paged_token_slots_extend( + c, prefix, prefix, seq, seq, last, 396 + ) + self.assertEqual(len(got), 396) + self.assertEqual(len(set(got.tolist())), 396) + self.check_payload(b, a, slots[:0], states, states) + self.assertFalse(a.verify_byte_accounting()) + + def test_grouped_frees_preserve_cache(self): + for grouped in ("ordinary", "representatives", "full_only"): + with self.subTest(grouped=grouped): + b, a, c = self.build() + slots = a.alloc(96) + states = b.req_to_token_pool.mamba_allocator.alloc(8) + self.stamp(b, a, slots, states) + records = self.insert_parts(c, slots, states) + pending = a.alloc(2) + self.assertIsNotNone(pending) + a.free_group_begin() + try: + if grouped == "ordinary": + a.free(pending) + elif grouped == "representatives": + a.free_segment(pending, start_pos=0) + else: + a.free_swa(pending) + a.free_full(pending) + with patch.object( + c, "evict_for_alloc", wraps=c.evict_for_alloc + ) as walks: + evict_from_tree_cache(c, 2) + got = a.alloc(2) + self.assertIsNotNone(got) + self.assertIsNotNone(a.free_group) + self.assertEqual(walks.call_count, 0) + for key, prior in records: + torch.testing.assert_close( + c.path_device_indices( + c.match_prefix( + MatchPrefixParams(key=key) + ).last_device_node + ), + prior, + ) + self.check_payload(b, a, slots, states, states) + a.free(got) + finally: + a.free_group_end() + self.assertFalse(a.verify_byte_accounting()) + + def test_session_cursor_joint_reclaim(self): + b, a, c = self.build(sessions=True) + slots = a.alloc(96) + states = b.req_to_token_pool.mamba_allocator.alloc(8) + self.stamp(b, a, slots, states) + records = self.insert_parts(c, slots, states) + evict_from_tree_cache(c, 4) + self.assertIsNotNone(a.alloc(4)) + self.assertEqual(c.full_evictable_size(), 95) + for key, prior in records[1:]: + torch.testing.assert_close( + c.path_device_indices( + c.match_prefix(MatchPrefixParams(key=key)).last_device_node + ), + prior, + ) + self.check_payload(b, a, slots[1:], states[1:], states) + self.assertFalse(a.verify_byte_accounting()) + + def test_ready_allocation_preserves_payload_with_closed_move_gates(self): + for page_size, lazy, grouped in itertools.product( + (1, 4), (False, True), (False, True) + ): + with self.subTest(page_size=page_size, lazy=lazy, grouped=grouped): + b, a, c = self.build( + page_size=page_size, lazy=lazy, state_cache=page_size == 1 + ) + slots = a.alloc(96 * page_size) + states = a.mamba_allocator.alloc(8) + self.assertIsNotNone(slots) + self.assertIsNotNone(states) + self.stamp(b, a, slots, states) + if page_size == 1: + records = self.insert_parts(c, slots, states) + else: + # Paged cache owns KV; the request retains all Mamba states. + key = RadixKey(array("q", range(len(slots)))) + c.insert(InsertParams(key=key, value=slots)) + records = [(key, slots.clone())] + for member in a._flush_targets(): + member.disagg_move_gate = lambda: False + if grouped: + a.free_group_begin() + self.assertGreaterEqual(a.available_size(), page_size) + before = snapshot(a) + self.assertTrue(a.evict_to_free_tokens(c, page_size)) + self.assertEqual(snapshot(a), before) + got = a.alloc(page_size) + self.assertIsNotNone(got) + self.assertEqual(len(got), page_size) + for key, prior in records: + torch.testing.assert_close( + c.path_device_indices( + c.match_prefix(MatchPrefixParams(key=key)).last_device_node + ), + prior, + ) + self.check_payload(b, a, slots, states, states) + self.assertFalse(a.verify_byte_accounting()) + if grouped: + self.assertIsNotNone(a.free_group) + a.free_group_end() + + def test_ready_and_no_progress_skip_repeated_preparation(self): + b, a, c = self.build() + slots = a.alloc(96) + states = b.req_to_token_pool.mamba_allocator.alloc(8) + self.stamp(b, a, slots, states) + self.insert_parts(c, slots, states) + with ( + patch.object(a, "ensure_capacity", wraps=a.ensure_capacity) as prep, + patch.object(c, "evict_for_alloc", wraps=c.evict_for_alloc) as walks, + ): + evict_from_tree_cache(c, 0) + self.assertEqual(prep.call_count, 0) + evict_from_tree_cache(c, 1) + self.assertEqual(walks.call_count, 0) + # Close every movement gate. The finite request quotas may still reclaim + # cache bytes; actual allocation and each phase remain separately visible. + for member in a._flush_targets(): + member.disagg_move_gate = lambda: False + with ( + patch.object(a, "ensure_capacity", wraps=a.ensure_capacity) as prep, + patch.object(c, "evict_for_alloc", wraps=c.evict_for_alloc) as walks, + ): + evict_from_tree_cache(c, 90) + got = a.alloc(90) + self.assertLessEqual(prep.call_count, 4) + self.assertLessEqual(walks.call_count, 2) + self.assertIsNotNone(got) + self.assertFalse(a.verify_byte_accounting()) + + def test_actual_decode_contract(self): + from sglang.srt.managers.schedule_batch import ScheduleBatch + from sglang.srt.speculative.spec_info import SpeculativeAlgorithm + + b, a, c = self.build() + slots = a.alloc(96) + states = b.req_to_token_pool.mamba_allocator.alloc(8) + self.stamp(b, a, slots, states) + records = self.insert_parts(c, slots, states) + batch = object.__new__(ScheduleBatch) + batch.token_to_kv_pool_allocator = a + batch.tree_cache = c + batch.spec_algorithm = SpeculativeAlgorithm.NONE + batch.reqs = [ + SimpleNamespace(kv=SimpleNamespace(kv_committed_len=8), beam_group=None) + for _ in range(4) + ] + self.assertTrue(batch.check_decode_mem()) + self.assertIsNotNone(a.alloc(4)) + self.assertEqual(c.full_evictable_size(), 95) + self.check_payload(b, a, slots[1:], states[1:], states) + + def test_physical_capacity_above_nominal_partition(self): + for entry in ("direct", "helper", "decode"): + with self.subTest(entry=entry): + b, a, c = self.build(state_cache=False) + demand = a.available_size() + self.assertGreater(demand, a.conserve_full_available_size()) + self.assertTrue(a._token_reclaim_satisfied(demand)) + if entry == "direct": + self.assertTrue(a.ensure_capacity(demand, demand)) + elif entry == "helper": + self.assertTrue(evict_from_tree_cache(c, demand)) + else: + self.assertTrue( + a.check_decode_capacity( + num_tokens=demand, + tree_cache=c, + requests=[], + spec_algorithm=None, + ) + ) + self.assertIsNotNone(a.alloc(demand)) + self.assertFalse(a.verify_byte_accounting()) + + def test_float_recovery_preserves_prefix_when_full_band_is_short(self): + for lazy in (False, True): + for recovery in ("prepare", "eviction", "decode"): + with self.subTest(lazy=lazy, recovery=recovery): + temporal = (0, 0, 0) + state_config = SimpleNamespace( + shape=SimpleNamespace(conv=[(3, 8)], temporal=temporal), + dtype=SimpleNamespace( + conv=torch.bfloat16, temporal=torch.float32 + ), + layers=[0, 1], + ) + bundle = init_unified_mamba_swa_pools( + device="cpu", + kv_cache_dtype=torch.float16, + head_num=1, + head_dim=8, + v_head_dim=8, + swa_head_num=1, + swa_head_dim=8, + swa_v_head_dim=8, + page_size=1, + start_layer=0, + end_layer=6, + swa_attention_layer_ids=[4, 5], + full_attention_layer_ids=[0, 1, 2, 3], + full_max_total_num_tokens=32, + swa_max_total_num_tokens=24, + enable_memory_saver=False, + need_sort=False, + lazy_compaction=lazy, + mamba_layer_ids=[0, 1], + mamba2_cache_params=state_config, + max_mamba_cache_size=8, + model_context_len=256, + extra_max_context_len=1, + max_num_reqs=4, + enable_mamba_extra_buffer=False, + enable_mamba_extra_buffer_lazy=False, + disable_overlap_schedule=True, + sliding_window_size=8, + ) + allocator = bundle.token_to_kv_pool_allocator + cache = UnifiedRadixCache( + CacheInitParams( + disable=False, + req_to_token_pool=bundle.req_to_token_pool, + token_to_kv_pool_allocator=allocator, + page_size=1, + sliding_window_size=128, + tree_components=(ComponentType.FULL, ComponentType.SWA), + ) + ) + slots = allocator.alloc(4) + self.assertIsNotNone(slots) + kv_pool = bundle.token_to_kv_pool + full = allocator.full_attn_allocator + swa = allocator.swa_attn_allocator + markers = slots.to(torch.float16).view(-1, 1, 1) + for layer in range(6): + member = full if layer < 4 else swa + kernel_ids = member.translate_kv_loc(slots) + kv_pool.get_key_buffer(layer)[kernel_ids] = markers + 10 * layer + kv_pool.get_value_buffer(layer)[kernel_ids] = ( + markers + 100 + 10 * layer + ) + extra = full.alloc(full.available_size() - 2) + self.assertIsNotNone(extra) + key = RadixKey(array("q", [1, 2, 3, 4])) + cache.insert(InsertParams(key=key, value=slots)) + self.assertEqual(allocator.full_available_size(), 2) + self.assertGreaterEqual(allocator.conserve_full_available_size(), 6) + self.assertGreaterEqual(allocator.swa_available_size(), 6) + self.assertLess(allocator.available_size(), 6) + self.assertEqual(allocator._token_allocation_byte_shortfall(6), 0) + if recovery == "prepare": + self.assertTrue(allocator.ensure_capacity(6, 6)) + elif recovery == "decode": + self.assertTrue( + allocator.check_decode_capacity( + num_tokens=6, tree_cache=cache + ) + ) + else: + evict_from_tree_cache(cache, 6) + self.assertTrue(allocator.available_size() >= 6) + self.assertEqual(cache.full_evictable_size(), 4) + matched = cache.match_prefix(MatchPrefixParams(key=key)) + torch.testing.assert_close( + cache.path_device_indices(matched.last_device_node), slots + ) + self.assertIsNotNone(allocator.alloc(6)) + for layer in range(6): + member = full if layer < 4 else swa + kernel_ids = member.translate_kv_loc(slots) + self.assertTrue( + torch.all( + kv_pool.get_key_buffer(layer)[kernel_ids] + == markers + 10 * layer + ) + ) + self.assertTrue( + torch.all( + kv_pool.get_value_buffer(layer)[kernel_ids] + == markers + 100 + 10 * layer + ) + ) + self.assertFalse(allocator.verify_byte_accounting()) + + def test_deep_reclaim_preserves_first_sufficient_prefix_set(self): + for lazy, demand, first in ( + (False, 6, 3), + (True, 6, 3), + (False, 90, 87), + (True, 90, 87), + ): + with self.subTest(lazy=lazy, demand=demand): + b, a, c = self.build(lazy=lazy, state_cache=False) + states = b.req_to_token_pool.mamba_allocator.alloc(8) + slots = a.alloc(96) + self.stamp(b, a, slots, states) + keys = [RadixKey(array("q", [i])) for i in range(96)] + for i, key in enumerate(keys): + c.insert(InsertParams(key=key, value=slots[i : i + 1])) + with patch.object( + a, "ensure_capacity", wraps=a.ensure_capacity + ) as prep: + evict_from_tree_cache(c, demand) + self.assertIsNotNone(a.alloc(demand)) + self.assertLessEqual(prep.call_count, 3) + for i, key in enumerate(keys): + actual = c.path_device_indices( + c.match_prefix(MatchPrefixParams(key=key)).last_device_node + ) + if i < first: + self.assertEqual(len(actual), 0) + else: + torch.testing.assert_close(actual, slots[i : i + 1]) + self.check_payload(b, a, slots[first:], states, states) + self.assertFalse(a.verify_byte_accounting()) + + def test_rank_consensus_uses_stable_tri_demand(self): + import os + import subprocess + import sys + + script = r""" +import queue,runpy,sys,threading +from pathlib import Path +from unittest.mock import patch +from sglang.srt.utils import rank_consensus_checker as checker +sys.path.insert(0,str(Path(sys.argv[1]).parent)) +module=runpy.run_path(sys.argv[1],run_name='rank_fixture') +fixture=module['TestTriJointReclaim']() +traces=[] +for _ in range(2): + fixture.setUp() + try: + b,a,c=fixture.build() + slots=a.alloc(96);states=b.req_to_token_pool.mamba_allocator.alloc(8) + fixture.stamp(b,a,slots,states);fixture.insert_parts(c,slots,states) + q=queue.Queue() + with patch.object(checker,'_q',q),patch.object(checker,'_scheduler_thread',threading.current_thread()): + assert module['evict_from_tree_cache'](c,4) + assert a.alloc(4) is not None + fixture.check_payload(b,a,slots[1:],states[1:],states) + traces.append(list(q.queue)) + finally:fixture.tearDown() +assert traces[0]==traces[1],traces +assert any('num_tokens=4'in event for event in traces[0]),traces +assert any('allocation_reclaim_satisfied is not None=True'in event for event in traces[0]),traces +assert not any('= member.min_page_index + ) + ) + torch.testing.assert_close(buf[member.translate_kv_loc(ids)], values) + self.assertFalse(a.verify_byte_accounting()) + + def test_float_fresh_destination_reservation_boundary(self): + for ps, extra_pages in ((1, 18), (4, 9), (16, 6)): + for one_short in (False, True): + with self.subTest(ps=ps, one_short=one_short): + a, kv, mk = build_geometry(ps, (1, 1), True, "none", False) + ma, fa, sa = ( + a.mamba_allocator, + a.full_attn_allocator, + a.swa_attn_allocator, + ) + live = torch.where(ma.virtual_to_physical >= ma.min_page_index)[0] + live = live[live >= ma.min_page_index] + ma.free(live[:-1]) + fa.flush_for_allocation() + ma.flush_for_allocation() + extra = fa.alloc(extra_pages * ps) + self.assertIsNotNone(extra) + demand = 2 * ps + target = a._token_float_high_target(demand) + self.assertIsNotNone(target) + lo, hi = sa._region_bounds_pages() + retreat = target // sa.entry_bytes_per_page - (hi - sa.high_wm_page) + self.assertEqual(sa.low_wm_page - lo, retreat) + self.assertLess(retreat, sa._live_pages()) + self.assertFalse(a._token_end_reclaim_satisfied(demand)) + if one_short: + # Bind an existing FULL-only ID into the roomier LOW gap. + low_before = sa.low_wm_page + sa.alloc_with_virtual(extra[:1] // ps) + self.assertEqual(sa.low_wm_page, low_before - 1) + before = snapshot(a) + self.assertIsNone(a._token_float_high_target(demand)) + self.assertEqual(snapshot(a), before) + self.assertFalse(a.verify_byte_accounting()) + continue + saved = payload(a, kv, mk) + with ( + patch.object( + sa, + "_relocate_to_positions", + wraps=sa._relocate_to_positions, + ) as whole, + patch( + "sglang.srt.mem_cache.allocator.unified_hybrid_swa._relieve_for_alloc", + side_effect=AssertionError( + "positive certificate used fallback" + ), + ), + ): + self.assertTrue(a.ensure_capacity(demand, demand)) + self.assertIsNotNone(a.alloc(demand)) + self.assertEqual(whole.call_count, 0) + self.check_saved(a, saved) + + def test_float_certificates_match_executor_across_page_grids(self): + recognized = set() + for ps, ratio, lazy, pattern, gated, npage in itertools.product( + (1, 4, 16), + ((1, 1), (1, 4), (4, 1)), + (False, True), + ("none", "paired", "swa_edges", "swa_internal", "empty", "mixed"), + (False, True), + (5, 7, 10), + ): + a, kv, mk = build_geometry(ps, ratio, lazy, pattern, gated) + prior = snapshot(a) + target = a._token_float_high_target(npage * ps) + self.assertEqual(snapshot(a), prior) + if gated or pattern == "empty": + self.assertIsNone(target) + if target is None or a._token_end_reclaim_satisfied(npage * ps): + continue + with self.subTest( + ps=ps, ratio=ratio, lazy=lazy, pattern=pattern, npage=npage + ): + recognized.add(ps) + saved = payload(a, kv, mk) + with patch( + "sglang.srt.mem_cache.allocator.unified_hybrid_swa._relieve_for_alloc", + side_effect=AssertionError("FLOAT certificate used fallback"), + ): + self.assertTrue(a.ensure_capacity(npage * ps, npage * ps)) + self.assertIsNotNone(a.alloc(npage * ps)) + self.check_saved(a, saved) + self.assertEqual(recognized, {1, 4, 16}) + + def test_nontrivial_end_certificates_preserve_float(self): + recognized = set() + for ps, ratio, cut, npage in itertools.product( + (1, 4, 16), ((1, 1), (1, 4), (4, 1)), (2, 3, 4), range(1, 12) + ): + a, kv, mk = build_geometry(ps, ratio, True, "none", False) + ma = a.mamba_allocator + extra = ma.alloc(ma.available_size()) + self.assertIsNotNone(extra) + vp = torch.where(ma.virtual_to_physical >= ma.min_page_index)[0] + vp = vp[vp >= ma.min_page_index] + mk.buf[ma.virtual_to_physical[vp]] = vp + 1000 + ma.free(vp[1:-1:cut]) + if npage * ps <= a.available_size() or not a._token_end_reclaim_satisfied( + npage * ps + ): + continue + with self.subTest(page_size=ps, ratio=ratio, cut=cut, demand=npage): + recognized.add(ps) + saved = payload(a, kv, mk) + prior = snapshot(a) + for _ in range(20): + self.assertTrue(a._token_end_reclaim_satisfied(npage * ps)) + self.assertEqual(snapshot(a), prior) + sa = a.swa_attn_allocator + with ( + patch.object(sa, "make_room", wraps=sa.make_room) as moves, + patch.object( + sa, "flush_for_allocation", wraps=sa.flush_for_allocation + ) as flush, + ): + self.assertTrue(a.ensure_capacity(npage * ps, npage * ps)) + self.assertIsNotNone(a.alloc(npage * ps)) + self.assertEqual(moves.call_count, 0) + self.assertEqual(flush.call_count, 0) + self.check_saved(a, saved) + self.assertEqual(recognized, {1, 4, 16}) + + def test_gate_change_invalidates_end_certificate(self): + for pd_open, host_open in ((True, False), (False, True), (False, False)): + with self.subTest(pd_open=pd_open, host_open=host_open): + a, kv, mk = build_geometry(1, (1, 4), True, "swa_edges", False) + self.assertLess(a.available_size(), 5) + self.assertTrue(a._token_reclaim_satisfied(5)) + saved = payload(a, kv, mk) + for member in a._flush_targets(): + member.disagg_move_gate = lambda: pd_open + member.host_transfer_move_gate = lambda: host_open + before = snapshot(a) + self.assertFalse(a._token_reclaim_satisfied(5)) + with patch.object( + kv.swa_kv_pool, + "move_physical_pages", + wraps=kv.swa_kv_pool.move_physical_pages, + ) as copies: + self.assertFalse(a.ensure_capacity(5, 5)) + self.assertEqual(copies.call_count, 0) + # Zero-copy FLOAT boundary absorption is legal even with movement gated. + self.check_saved(a, saved) + for member in a._flush_targets(): + member.disagg_move_gate = None + member.host_transfer_move_gate = None + self.assertTrue(a.ensure_capacity(5, 5)) + self.assertIsNotNone(a.alloc(5)) + self.check_saved(a, saved) + + def test_pending_reuse_is_not_certificate_credit(self): + class Event: + fired = False + + def query(self): + return self.fired + + class Stream: + waits = 0 + + def wait_event(self, event): + self.waits += 1 + + for ps, urgent in itertools.product((1, 4, 16), (False, True)): + with self.subTest(page_size=ps, urgent=urgent): + a, kv, mk = build_geometry(ps, (1, 4), True, "none", False) + fa = a.full_attn_allocator + event = Event() + stream = Stream() + fa.set_latest_forward_done_event(event) + fa.set_inflight_forward(event, None) + vp = torch.where(fa.virtual_to_physical >= fa.min_page_index)[0] + vp = vp[vp >= fa.min_page_index][10:11] + tokens = (vp[:, None] * ps + torch.arange(ps)[None, :]).flatten() + a.free(tokens) + self.assertGreater(fa._flush(urgent=False), 0) + self.assertGreater(len(fa._pending_reuse_pages_cpu), 0) + saved = payload(a, kv, mk) + for member in a._flush_targets(): + member.disagg_move_gate = lambda: False + before = snapshot(a) + predictions = [a._token_reclaim_satisfied(n * ps) for n in range(1, 10)] + for _ in range(20): + self.assertEqual( + predictions, + [a._token_reclaim_satisfied(n * ps) for n in range(1, 10)], + ) + self.assertEqual(snapshot(a), before) + for member in a._flush_targets(): + member.disagg_move_gate = None + if not urgent: + event.fired = True + fa.set_latest_forward_done_event(None) + with patch("torch.cuda.current_stream", return_value=stream): + fa.flush_for_allocation() + self.assertFalse(fa._pending_reuse) + n = max( + n for n in range(1, 10) if a._token_reclaim_satisfied(n * ps) + ) + self.assertTrue(a.ensure_capacity(n * ps, n * ps)) + self.assertIsNotNone(a.alloc(n * ps)) + self.assertEqual(stream.waits > 0, urgent) + self.check_saved(a, saved) + + +if __name__ == "__main__": + unittest.main()