[Fix] unified-memory: Bound joint reclaim and preserve cached prefixes (#39294)

Bound joint FULL/SWA/Mamba reclaim so feasible allocations recover capacity without evicting unnecessary cached prefixes, and impossible demands preserve the cache. Use exact unsharded paged-extend demand and update allocation regression fixtures.
This commit is contained in:
SuperSong
2026-10-17 04:46:48 -07:00
committed by GitHub
parent ae28306a74
commit ee30dc4d40
10 changed files with 2208 additions and 67 deletions
+9 -2
View File
@@ -188,8 +188,15 @@ def alloc_paged_token_slots_extend(
# availability floor must cover every request's ceil(K_i/N) rounding,
# matching the N*ps per-request admission reserve).
allocator = tree_cache.token_to_kv_pool_allocator
num_tokens = extend_num_tokens + len(seq_lens_cpu) * (
allocator.page_size * page_interleave_shard_size(allocator)
shard_size = page_interleave_shard_size(allocator)
conservative_num_tokens = extend_num_tokens + len(seq_lens_cpu) * (
allocator.page_size * shard_size
)
num_tokens = allocator.get_extend_allocation_demand(
prefix_lens_cpu,
seq_lens_cpu,
conservative_num_tokens=conservative_num_tokens,
shard_size=shard_size,
)
evict_from_tree_cache(tree_cache, num_tokens)
@@ -141,6 +141,17 @@ class BaseTokenToKVPoolAllocator(abc.ABC):
swa_budget_tokens is None or swa_tokens <= swa_budget_tokens
)
def get_extend_allocation_demand(
self,
prefix_lens_cpu: torch.Tensor,
seq_lens_cpu: torch.Tensor,
*,
conservative_num_tokens: int,
shard_size: int,
) -> int:
"""Token demand for eviction before paged extend."""
return conservative_num_tokens
def evict_to_free_tokens(self, tree_cache, num_tokens: int) -> bool | None:
"""Evict unlocked prefix-cache entries until this allocator can serve
``num_tokens`` or nothing evictable remains.
@@ -41,6 +41,7 @@ from sglang.srt.mem_cache.allocator.unified_sub_pool import (
from sglang.srt.mem_cache.unified_memory_pool import UnifiedKVPool
from sglang.srt.runtime_context import get_disagg, get_memory
from sglang.srt.utils.common import get_num_new_pages
from sglang.srt.utils.rank_consensus_checker import rank_consensus
logger = logging.getLogger(__name__)
@@ -1183,6 +1184,23 @@ class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
hi = mid - 1
return lo
def get_extend_allocation_demand(
self,
prefix_lens_cpu: torch.Tensor,
seq_lens_cpu: torch.Tensor,
*,
conservative_num_tokens: int,
shard_size: int,
) -> int:
if shard_size != 1:
return conservative_num_tokens
return self.page_size * get_num_new_pages(
seq_lens=seq_lens_cpu,
page_size=self.page_size,
prefix_lens=prefix_lens_cpu,
)
@rank_consensus(same_params=["num_tokens", "swa_num_tokens"], same_results=True)
def evict_to_free_tokens(
self, tree_cache, num_tokens: int, *, swa_num_tokens: Optional[int] = None
) -> bool | None:
@@ -1197,6 +1215,10 @@ class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
and memory.hicache_storage_backend is not None
) or get_disagg().disaggregation_mode == "decode"
required_swa = num_tokens if swa_num_tokens is None else swa_num_tokens
_flush_deferred_free_group(
self,
(self.free_group, self.free_page_reps_group, self.full_free_group),
)
reclaim_plan = self.reclaim_plan(
num_tokens,
required_swa,
@@ -1208,11 +1230,11 @@ class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
return
full_reclaim, swa_reclaim = reclaim_plan
if full_reclaim or swa_reclaim:
# The shared-byte plan returns cumulative eviction quotas.
# Per-component capacity targets can count the same shared bytes
# independently and stop before the joint allocation fits.
tree_cache.evict(
EvictParams(num_tokens=full_reclaim, swa_num_tokens=swa_reclaim)
tree_cache.evict_for_alloc(
EvictParams(num_tokens=full_reclaim, swa_num_tokens=swa_reclaim),
allocation_reclaim_satisfied=lambda: (
self.reclaim_plan(num_tokens, required_swa) == (0, 0)
),
)
# A zero-reclaim plan can still depend on compaction before allocation.
return self.ensure_capacity(num_tokens, required_swa)
@@ -1436,19 +1458,177 @@ class UnifiedMambaSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
return False
if self.can_reserve(full_tokens, swa_tokens):
return True
for allocator in self._flush_targets():
allocator.flush_for_allocation()
if self.can_reserve(full_tokens, swa_tokens):
if full_tokens != swa_tokens:
for allocator in self._flush_targets():
allocator.flush_for_allocation()
if self.can_reserve(full_tokens, swa_tokens):
return True
_float_open_short_side(
self.swa_attn_allocator,
{
self.full_attn_allocator: -(-full_tokens // self.page_size),
self.swa_attn_allocator: -(-swa_tokens // self.page_size),
self.mamba_allocator: 0,
},
)
return self.can_reserve(full_tokens, swa_tokens)
need_tokens = -(-int(full_tokens) // self.page_size) * self.page_size
if not self._token_id_capacity(need_tokens):
return False
if need_tokens <= self.available_size():
return True
_float_open_short_side(
if self._token_end_reclaim_satisfied(need_tokens):
# The certificate preserves FLOAT holes in place. Absorbing FLOAT
# boundaries first would describe a different allocation layout.
self.full_attn_allocator.flush_for_allocation()
self.mamba_allocator.flush_for_allocation()
return need_tokens <= self.available_size()
if self._token_float_high_target(need_tokens) is not None:
self.full_attn_allocator.flush_for_allocation()
self.mamba_allocator.flush_for_allocation()
if not self._token_id_capacity(need_tokens):
return False
if need_tokens <= self.available_size():
return True
# Recompute from actual frontiers: no credit for another END flush.
target = self._token_float_high_target(need_tokens, flush_ends=False)
if target is None:
return False
self.swa_attn_allocator.make_room(side="high", min_bytes=target)
return self._token_id_capacity(need_tokens) and (
need_tokens <= self.available_size()
)
if self._token_allocation_byte_shortfall(need_tokens) > 0:
return False
return _relieve_for_alloc(self, need_tokens)
def _token_id_capacity(self, num_tokens: int) -> bool:
sa = self.swa_attn_allocator
return (
num_tokens // self.page_size
<= self.full_attn_allocator.free_virtual_ids.numel()
and num_tokens // self.page_size
<= sa.num_pages - sa.min_page_index - sa._live_pages()
)
def _token_reclaim_satisfied(self, num_tokens: int) -> bool:
"""Pure sufficient recovery query; False leaves FLOAT geometry unknown."""
return self._token_end_reclaim_satisfied(num_tokens) or (
self._token_float_high_target(num_tokens) is not None
)
def _token_end_frontiers(self, num_tokens: int, *, flush_ends: bool = True):
"""FULL before/after allocation and Mamba after permitted END flushes.
Pending reuse is never credited. Unflushed FULL holes remain allocatable.
"""
fa, ma = self.full_attn_allocator, self.mamba_allocator
def end_credit(member):
if not flush_ends or not member.lazy_compaction:
return 0
if member.moves_blocked():
return 0
return member._free_phys_pages.numel()
full_credit, state_credit = end_credit(fa), end_credit(ma)
full_holes = fa._free_phys_pages.numel() if fa.lazy_compaction else 0
pages = num_tokens // self.page_size
full_extension = max(0, pages - (full_holes - full_credit))
full_before = fa._byte_low_frontier() + full_credit * fa.entry_bytes_per_page
full_after = full_before - full_extension * fa.entry_bytes_per_page
state_after = ma._byte_high_frontier() - state_credit * ma.entry_bytes_per_page
return full_before, full_after, state_after
def _token_end_reclaim_satisfied(self, num_tokens: int) -> bool:
"""Sufficient allocation certificate requiring only allowed END flushes."""
if not self._token_id_capacity(num_tokens):
return False
if num_tokens <= self.available_size():
return True
fa, sa = self.full_attn_allocator, self.swa_attn_allocator
_, full_after, state_after = self._token_end_frontiers(num_tokens)
pages = num_tokens // self.page_size
if full_after < fa.min_page_index * fa.entry_bytes_per_page:
return False
if sa._is_frontier_transparent():
return pages <= sa.pages_in_band(low_byte=state_after, high_byte=full_after)
if full_after < sa._byte_high_frontier():
return False
swa_extension = max(0, pages - sa._hole_pages())
return swa_extension <= max(
sa.pages_in_band(low_byte=state_after, high_byte=sa._byte_low_frontier()),
sa.pages_in_band(low_byte=sa._byte_high_frontier(), high_byte=full_after),
)
def _token_float_high_target(
self, num_tokens: int, *, flush_ends: bool = True
) -> Optional[int]:
"""Absolute HIGH band target after END preparation; None means unknown.
Reserve fresh LOW destinations for every moved FLOAT page and fresh HIGH
pages for SWA after FULL allocation. No FLOAT hole positions or pending
reuse are credited. The callback performs no movement or device reads.
"""
if not self._token_id_capacity(num_tokens):
return None
fa, sa = self.full_attn_allocator, self.swa_attn_allocator
if sa._is_frontier_transparent():
return None
if sa.moves_blocked():
return None
full_before, full_after, state_after = self._token_end_frontiers(
num_tokens, flush_ends=flush_ends
)
if full_after < fa.min_page_index * fa.entry_bytes_per_page:
return None
e_s = sa.entry_bytes_per_page
low = max(-(-state_after // e_s), sa.min_page_index)
high = min(full_before // e_s, sa.num_pages)
high_after = min(full_after // e_s, sa.num_pages)
limit = high_after - num_tokens // self.page_size
retreat = sa.high_wm_page - limit
if retreat <= 0 or sa.low_wm_page - low < retreat:
return None
# The far gap supplies all retreat destinations even with adverse holes.
# Retreating HIGH to limit leaves the requested SWA pages after FULL.
return (high - limit) * e_s
def _token_allocation_byte_shortfall(
self,
num_tokens: int,
*,
full_reclaim: int = 0,
swa_reclaim: int = 0,
state_reclaim: int = 0,
) -> int:
"""Optimistic byte lower bound; zero does not certify FLOAT geometry."""
fa, sa, ma = (
self.full_attn_allocator,
self.swa_attn_allocator,
{
self.full_attn_allocator: -(-full_tokens // self.page_size),
self.swa_attn_allocator: -(-swa_tokens // self.page_size),
self.mamba_allocator: 0,
},
self.mamba_allocator,
)
live_bytes = sum(
max(0, member.allocated_count() - reclaim) * member.entry_bytes
for member, reclaim in (
(fa, full_reclaim),
(sa, swa_reclaim),
(ma, state_reclaim),
)
)
sink_bytes = min(
member.min_page_index * member.entry_bytes_per_page
for member in (fa, sa, ma)
)
required_bytes = (
num_tokens
// self.page_size
* (fa.entry_bytes_per_page + sa.entry_bytes_per_page)
)
return max(
0,
required_bytes + live_bytes + sink_bytes - self.unified_buffer.total_bytes,
)
return self.can_reserve(full_tokens, swa_tokens)
def _fits_page_demand(self, full_pages: int, swa_pages: int) -> bool:
"""Price FULL first, then SWA in one contiguous band on the float grid."""
@@ -1615,17 +1795,105 @@ class UnifiedMambaSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
(swa_capacity, self.conserve_swa_available_size()),
)
def evict_to_free_tokens(self, tree_cache, num_tokens: int) -> None:
"""Joint-aware eviction: one tri-lifetime node frees bytes on several sides
at once, so re-check the JOINT gate instead of the per-side shortfall."""
# Arbitrary retry bound; a round that frees nothing ends the loop anyway.
for _ in range(4):
before = self.available_size()
if before >= num_tokens:
return
SWATokenToKVPoolAllocator.evict_to_free_tokens(self, tree_cache, num_tokens)
if self.available_size() <= before:
return # no progress
def get_extend_allocation_demand(
self,
prefix_lens_cpu: torch.Tensor,
seq_lens_cpu: torch.Tensor,
*,
conservative_num_tokens: int,
shard_size: int,
) -> int:
if shard_size != 1:
return conservative_num_tokens
return self.page_size * get_num_new_pages(
seq_lens=seq_lens_cpu,
page_size=self.page_size,
prefix_lens=prefix_lens_cpu,
)
@rank_consensus(same_params=["num_tokens"], same_results=True)
def evict_to_free_tokens(self, tree_cache, num_tokens: int) -> bool | None:
if tree_cache is None or not tree_cache.supports_prefix_sharing():
return
if num_tokens <= 0:
return True
if self.free_group is not None:
self.flush_deferred_full_frees()
num_tokens = -(-num_tokens // self.page_size) * self.page_size
# Immediate capacity needs no reclaim plan or layout preparation.
if self._token_id_capacity(num_tokens) and num_tokens <= self.available_size():
return True
from sglang.srt.mem_cache.base_prefix_cache import EvictParams
full_reclaim = tree_cache.full_evictable_size()
swa_reclaim = tree_cache.swa_evictable_size()
state_reclaim = tree_cache.mamba_evictable_size()
fa, sa, ma = (
self.full_attn_allocator,
self.swa_attn_allocator,
self.mamba_allocator,
)
if (
num_tokens // self.page_size
> fa.free_virtual_ids.numel() + full_reclaim // self.page_size
or num_tokens // self.page_size
> sa.num_pages
- sa.min_page_index
- sa._live_pages()
+ swa_reclaim // self.page_size
or self._token_allocation_byte_shortfall(
num_tokens,
full_reclaim=full_reclaim,
swa_reclaim=swa_reclaim,
state_reclaim=state_reclaim,
)
> 0
):
return False
if self.ensure_capacity(num_tokens, num_tokens):
return True
# Quotas are cumulative logical counts within each phase. Token reclaim
# precedes state reclaim so Mamba-first tree ordering cannot skip it.
state_freed = 0
if full_reclaim or swa_reclaim:
result = tree_cache.evict_for_alloc(
EvictParams(num_tokens=num_tokens, swa_num_tokens=num_tokens),
allocation_reclaim_satisfied=lambda: self._token_reclaim_satisfied(
num_tokens
),
)
state_freed = result.mamba_num_evicted
if (
result.num_tokens_evicted
or result.swa_num_tokens_evicted
or state_freed
) and self.ensure_capacity(num_tokens, num_tokens):
return True
pair_bytes = (
num_tokens
// self.page_size
* (fa.entry_bytes_per_page + sa.entry_bytes_per_page)
)
state_quota = max(0, -(-pair_bytes // ma.entry_bytes) - state_freed)
if state_quota and tree_cache.mamba_evictable_size():
result = tree_cache.evict_for_alloc(
EvictParams(mamba_num=state_quota),
allocation_reclaim_satisfied=lambda: self._token_reclaim_satisfied(
num_tokens
),
)
if (
result.num_tokens_evicted
or result.swa_num_tokens_evicted
or result.mamba_num_evicted
):
return self.ensure_capacity(num_tokens, num_tokens)
return (
self._token_id_capacity(num_tokens) and num_tokens <= self.available_size()
)
def verify_byte_accounting(self) -> List[str]:
return (
@@ -505,14 +505,26 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
def evict(self, params: EvictParams) -> EvictResult:
pass
def evict_for_alloc(self, params: EvictParams) -> EvictResult:
"""Evict cache entries to cover allocator shortfalls.
def evict_for_alloc(
self,
params: EvictParams,
*,
allocation_reclaim_satisfied: Optional[Callable[[], bool]] = None,
) -> EvictResult:
"""Evict for allocation using shortfalls or joint-reclaim quotas.
The default implementation preserves the component-count semantics of
:meth:`evict`. Multi-component caches backed by shared memory can
override this entry point to stop once collateral frees make the
requested allocation feasible.
Without ``allocation_reclaim_satisfied``, callers pass component
shortfalls in ``params``. The default implementation forwards these
counts to :meth:`evict`; multi-component caches may stop earlier when
collateral frees satisfy the requested allocation.
With ``allocation_reclaim_satisfied``, ``params`` instead contains
cumulative reclaim quotas. Supporting caches may stop at the pure
predicate before reaching those quotas. The allocator still owns
preparation and the actual capacity check after the walk.
"""
if allocation_reclaim_satisfied is not None:
raise NotImplementedError("This cache does not support joint reclaim")
return self.evict(params)
@abstractmethod
@@ -8,7 +8,15 @@ from array import array
from dataclasses import replace
from http import HTTPStatus
from queue import Queue
from typing import TYPE_CHECKING, Iterator, NamedTuple, Optional, Sequence, TypeVar
from typing import (
TYPE_CHECKING,
Callable,
Iterator,
NamedTuple,
Optional,
Sequence,
TypeVar,
)
import msgspec
import torch
@@ -656,17 +664,33 @@ class UnifiedRadixCache(BasePrefixCache):
def evict(self, params: EvictParams) -> EvictResult:
return self._evict(params)
@rank_consensus(same_params=True, same_results=True)
def evict_for_alloc(self, params: EvictParams) -> EvictResult:
"""Evict until the requested component allocations become feasible.
@rank_consensus(
same_params=["params", "allocation_reclaim_satisfied is not None"],
same_results=True,
)
def evict_for_alloc(
self,
params: EvictParams,
*,
allocation_reclaim_satisfied: Optional[Callable[[], bool]] = None,
) -> EvictResult:
"""Evict using component shortfalls or cumulative joint-reclaim quotas.
``params`` contains allocator shortfalls, not absolute eviction quotas.
A component eviction can cascade to its peers; with a shared memory pool,
those collateral frees can satisfy the original allocation before the
triggering component's requested count is reached.
Without ``allocation_reclaim_satisfied``, ``params`` contains component
shortfalls. These become availability targets, allowing collateral frees
to satisfy an allocation before its requested eviction count is reached.
With ``allocation_reclaim_satisfied``, ``params`` instead contains
cumulative reclaim quotas and component-local targets are bypassed.
The pure predicate may stop the walk before those quotas are reached;
the allocator must prepare and recheck actual capacity after the walk.
"""
if self.disable:
return EvictResult()
if allocation_reclaim_satisfied is not None:
return self._evict(
params, allocation_reclaim_satisfied=allocation_reclaim_satisfied
)
request_by_type = self._evict_request_by_type(params)
available_size_targets = {
@@ -758,6 +782,8 @@ class UnifiedRadixCache(BasePrefixCache):
available_size_targets: Optional[
dict[ComponentType, tuple[ComponentType, int]]
] = None,
*,
allocation_reclaim_satisfied: Optional[Callable[[], bool]] = None,
) -> EvictResult:
if self.disable:
return EvictResult()
@@ -769,6 +795,7 @@ class UnifiedRadixCache(BasePrefixCache):
request_by_type,
tracker,
available_size_targets=available_size_targets,
allocation_reclaim_satisfied=allocation_reclaim_satisfied,
)
if (
@@ -890,6 +917,8 @@ class UnifiedRadixCache(BasePrefixCache):
available_size_targets: Optional[
dict[ComponentType, tuple[ComponentType, int]]
] = None,
*,
allocation_reclaim_satisfied: Optional[Callable[[], bool]] = None,
) -> None:
# Buffer mode: eviction always wins over queued backup intents — a
# destroyed victim's intent is stale-swept and the content rewrites
@@ -899,6 +928,8 @@ class UnifiedRadixCache(BasePrefixCache):
def target_reached(component_type: ComponentType) -> bool:
nonlocal last_mamba_donor_check, mamba_donor_prepared
if allocation_reclaim_satisfied is not None:
return allocation_reclaim_satisfied()
if available_size_targets is None:
return False
target = available_size_targets.get(component_type)
@@ -1,9 +1,15 @@
"""Small unified-pool builders and fakes shared by allocator regression tests."""
from array import array
from types import SimpleNamespace
import torch
from sglang.srt.mem_cache.base_prefix_cache import InsertParams
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
from sglang.srt.mem_cache.radix_cache import RadixKey
from sglang.srt.mem_cache.unified_cache.components import ComponentType
from sglang.srt.mem_cache.unified_memory_pool import (
MambaSubPoolSpec,
MHASubPoolSpec,
@@ -11,17 +17,22 @@ from sglang.srt.mem_cache.unified_memory_pool import (
init_unified_mamba_swa_pools,
init_unified_swa_pools,
)
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
from sglang.srt.runtime_context import publish, reset_context
from sglang.srt.server_args import ServerArgs
def setup_allocator_context():
reset_context()
publish(ServerArgs(model_path="dummy", device="cpu"), role="tokenizer")
args = ServerArgs(model_path="dummy", device="cpu")
# Mamba cache setup must not resolve the dummy model through Hugging Face.
args._mamba_cache_chunk_size = 64
publish(args, role="tokenizer")
def build_swa_pool(*, page_size=1):
"""Lazy full + SWA composite over real CPU pools, 100 pages per side."""
def build_swa_pool(*, page_size=1, lazy=True, ratio=(1, 1), total_bytes=None):
"""Full + SWA composite over real CPU pools, 100 pages per side."""
full_layers, swa_layers = ratio
bundle = init_unified_swa_pools(
device="cpu",
kv_cache_dtype=torch.float16,
@@ -33,23 +44,25 @@ def build_swa_pool(*, page_size=1):
swa_v_head_dim=8,
page_size=page_size,
start_layer=0,
end_layer=2,
swa_attention_layer_ids=[1],
full_attention_layer_ids=[0],
end_layer=full_layers + swa_layers,
swa_attention_layer_ids=list(range(full_layers, full_layers + swa_layers)),
full_attention_layer_ids=list(range(full_layers)),
full_max_total_num_tokens=100 * page_size,
swa_max_total_num_tokens=100 * page_size,
enable_memory_saver=False,
need_sort=False,
lazy_compaction=True,
lazy_compaction=lazy,
total_bytes=total_bytes,
)
return bundle.token_to_kv_pool_allocator
def build_tri_pool(*, page_size=1):
"""Lazy full + SWA + mamba composite over real CPU pools; returns the pool
def build_tri_pool(*, page_size=1, lazy=True, ratio=(1, 1), temporal=(1, 4, 8)):
"""Full + SWA + Mamba composite over real CPU pools; returns the pool
bundle and its allocator."""
full_layers, swa_layers = ratio
config = SimpleNamespace(
shape=SimpleNamespace(conv=[(3, 8)], temporal=(1, 4, 8)),
shape=SimpleNamespace(conv=[(3, 8)], temporal=temporal),
dtype=SimpleNamespace(conv=torch.bfloat16, temporal=torch.float32),
layers=[0],
)
@@ -64,14 +77,14 @@ def build_tri_pool(*, page_size=1):
swa_v_head_dim=8,
page_size=page_size,
start_layer=0,
end_layer=2,
swa_attention_layer_ids=[1],
full_attention_layer_ids=[0],
end_layer=full_layers + swa_layers,
swa_attention_layer_ids=list(range(full_layers, full_layers + swa_layers)),
full_attention_layer_ids=list(range(full_layers)),
full_max_total_num_tokens=100 * page_size,
swa_max_total_num_tokens=100 * page_size,
enable_memory_saver=False,
need_sort=False,
lazy_compaction=True,
lazy_compaction=lazy,
mamba_layer_ids=[0],
mamba2_cache_params=config,
max_mamba_cache_size=8,
@@ -150,3 +163,75 @@ def tri_sub_pool_specs(
grow_direction="up",
)
return full, swa, mamba
def build_swa_cache(
*,
occupancy=96,
lazy=False,
page_size=1,
ratio=(1, 1),
total_bytes=None,
sessions=False,
window=None,
):
allocator = build_swa_pool(
page_size=page_size, lazy=lazy, ratio=ratio, total_bytes=total_bytes
)
req_pool = ReqToTokenPool(
size=4,
max_context_len=256 * page_size,
device="cpu",
enable_memory_saver=False,
)
cache = UnifiedRadixCache(
CacheInitParams(
disable=False,
enable_session_radix_cache=sessions,
req_to_token_pool=req_pool,
token_to_kv_pool_allocator=allocator,
page_size=page_size,
sliding_window_size=128 * page_size if window is None else window,
tree_components=(ComponentType.FULL, ComponentType.SWA),
)
)
slots = allocator.alloc(occupancy * page_size)
assert slots is not None, "Fixture allocation must fit"
for i in range(occupancy):
cache.insert(
InsertParams(
key=RadixKey(array("q", range(i * 1000, i * 1000 + page_size))),
value=slots[i * page_size : (i + 1) * page_size],
)
)
return allocator, cache
def build_tri_cache(
*,
lazy=True,
temporal=(0, 0, 0),
page_size=1,
ratio=(1, 1),
sessions=False,
state_cache=True,
):
bundle, a = build_tri_pool(
page_size=page_size, lazy=lazy, ratio=ratio, temporal=temporal
)
c = UnifiedRadixCache(
CacheInitParams(
disable=False,
req_to_token_pool=bundle.req_to_token_pool,
token_to_kv_pool_allocator=a,
page_size=page_size,
sliding_window_size=128 * page_size,
tree_components=(
(ComponentType.FULL, ComponentType.SWA, ComponentType.MAMBA)
if state_cache
else (ComponentType.FULL, ComponentType.SWA)
),
enable_session_radix_cache=sessions,
)
)
return bundle, a, c