[CI] radix-cache: Build unified radix cache test states through production paths (#45334)

This commit is contained in:
Liangsheng Yin
2026-10-22 13:04:23 -07:00
committed by GitHub
parent d67d1ae02c
commit 4f96807ea8
15 changed files with 521 additions and 1267 deletions
@@ -13,7 +13,6 @@ from sglang.test.test_utils import CustomTestCase
@dataclass(frozen=True)
class CacheConfig:
# Tree
page_size: int = 1
components: tuple[ComponentType, ...] = (ComponentType.FULL,)
@@ -21,12 +20,10 @@ class CacheConfig:
num_layers: int = 24
full_attention_layer_ids: tuple[int, ...] = (3, 7, 11, 15, 19, 23)
# SWA
sliding_window_size: Optional[int] = None
draft_swa_window: int = 0
decoder_swa_bounded_replay: bool = False
# Mamba
enable_mamba_extra_buffer: bool = False
mamba_cache_size: int = 20
mamba_intermediate_size: int = 256
@@ -37,7 +34,6 @@ class CacheConfig:
mamba_conv_kernel: int = 4
enable_int8_mamba_checkpoint: bool = False
# Model / pool
kv_size: int = 256
max_num_reqs: int = 10
max_context_len: int = 512
@@ -193,11 +189,8 @@ CONFIGS: list[CacheConfig] = [
def requires(predicate, reason):
"""Keep a test only under the configs where ``predicate`` holds.
Under any other config the test is removed from the bound class, not
skipped. ``reason`` documents the requirement and is not read at runtime.
Stacked requirements are checked top to bottom and stop at the first that
fails, so a lower one may assume the ones above it hold.
Elsewhere the test is removed, not skipped. Stacked requirements are checked
top to bottom, so a lower one may assume the ones above it hold.
"""
def decorate(test):
@@ -86,9 +86,7 @@ class FakeFullComponent(TreeComponent):
class PagedFullComponent(FullComponent):
"""FULL component whose device values are not backed one-to-one by pool
rows: a value lists two entries per row, and the component can hand rows
of a node back to the allocator while the value stays on device."""
"""FULL component whose device values are not backed one-to-one by pool rows."""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@@ -99,7 +97,6 @@ class PagedFullComponent(FullComponent):
def component_with_cache(component_type, cache):
"""A registry component instance bound to a (mock) cache and its tree_core."""
component = object.__new__(COMPONENT_REGISTRY[component_type])
component.cache = cache
component.tree_core = cache.tree_core
@@ -52,9 +52,7 @@ from sglang.test.kits.unified_radix_cache.unified_tree_core_inspector import (
UnifiedTreeCoreInspector,
)
# The Rust re-run entry point overrides this without changing the process-wide
# production backend selection. Every cache built by build_fixture follows it;
# tests that construct a Python core directly stay Python-only.
# The Rust re-run overrides this; every cache built by build_fixture follows it.
TREE_CORE_TEST_BACKEND: Optional[str] = "python"
@@ -63,9 +61,8 @@ def selected_tree_core_test_backend() -> str:
def drop_hicache_atexit_pin(cache):
"""init_hicache registers ``cache.shutdown`` with atexit, which pins the
fixture's device and host pools until process exit; hundreds of HiCache
fixtures in one test process would otherwise accumulate on the GPU."""
"""init_hicache registers cache.shutdown with atexit, which pins the fixture's GPU
memory until exit."""
atexit.unregister(cache.shutdown)
@@ -128,19 +125,17 @@ def build_fixture(
):
"""Create (tree, allocator, req_to_token_pool) from a CacheConfig.
``tree_page_size`` stands in for DCP, which widens the tree page past the
``page_size`` the rest of the config still sees. It only reaches values
derived from the tree page: the allocator keeps ``cfg.page_size``, whereas
DCP sets the two equal, so do not read insert or match behaviour off it.
``tree_page_size`` stands in for DCP but only reaches values derived from the
tree page; the allocator keeps ``cfg.page_size``, so do not read insert or
match behaviour off it.
"""
server_args = ServerArgs(
model_path="dummy",
page_size=cfg.page_size,
enable_int8_mamba_checkpoint=cfg.enable_int8_mamba_checkpoint,
)
# The mamba component reads mamba_cache_chunk_size, whose property otherwise
# loads the HF config for self.model_path — impossible for the dummy model.
# Mirror the property's default for a dummy HF config: FLA_CHUNK_SIZE.
# The dummy HF config cannot supply mamba_cache_chunk_size, so preset
# FLA_CHUNK_SIZE.
server_args._mamba_cache_chunk_size = (
max(FLA_CHUNK_SIZE, cfg.page_size)
if mamba_cache_chunk_size is None
@@ -33,7 +33,6 @@ class ResumableInsertWalkSWAFixtures(UnifiedRadixCacheFixtures):
seq = list(range(1, self.cfg.sliding_window_size + 1))
key = RadixKey(array("q", seq))
evicted = self._alloc(allocator, len(seq))
# Window eviction already released the peers below the floor.
allocator.free_swa(evicted)
cache.insert(
InsertParams(
@@ -229,7 +228,6 @@ class SegmentLockFuzzFixtures(UnifiedRadixCacheFixtures):
raise
cache.sanity_check()
# Drain remaining locks; the tree must come back exactly balanced.
for node_id, receipt, released in held:
cache.dec_lock_ref(node_id, receipt.to_dec_params(), skip_swa=released)
cache.sanity_check()
@@ -81,11 +81,10 @@ class UnifiedRadixCacheKVEventsFixtures:
return leaf
def _init_hicache(self, cache, *, write_policy: str = "write_through"):
# Production config: kernel IO backend + page_first layout with
# PINNED host pools (kernel GPU DMA requires them). Pools track their
# cudaHostRegister'd pointers and unregister on destroy()/GC, so the
# many fixtures sharing this process cannot collide on recycled
# address ranges (rc=712).
# Production config: kernel IO backend + page_first layout.
# Pinned host pools (kernel GPU DMA requires them) track their registered
# pointers and unregister on destroy/GC, so fixtures sharing a process
# cannot collide on recycled address ranges.
server_args = ServerArgs(
model_path="dummy",
page_size=self.cfg.page_size,
@@ -11,11 +11,14 @@ from sglang.srt.managers.schedule_batch import (
Req,
)
from sglang.srt.mem_cache.base_prefix_cache import (
InitLoadBackParams,
InsertParams,
MatchPrefixParams,
)
from sglang.srt.mem_cache.radix_cache import RadixKey
from sglang.srt.mem_cache.unified_cache.components.base import (
ComponentType,
EvictLayer,
)
from sglang.srt.sampling.sampling_params import SamplingParams
from sglang.srt.server_args import (
@@ -73,9 +76,8 @@ class UnifiedRadixCacheFixtures:
if not (self.cfg.has_swa and self.cfg.page_size > 1):
return allocator.alloc(need_size)
# SWATokenToKVPoolAllocator.alloc() asserts page_size == 1, and
# alloc_extend() requires batch tensors unsuitable for unit tests.
# Replicate alloc_extend's core logic here.
# The SWA allocator's alloc asserts page_size == 1 and alloc_extend needs
# batch tensors, so replicate alloc_extend's core logic.
ps = self.cfg.page_size
aligned = ((need_size + ps - 1) // ps) * ps
if aligned > allocator.full_attn_allocator.available_size():
@@ -195,9 +197,8 @@ class UnifiedRadixCacheFixtures:
if storage_backend == "file":
from sglang.srt.runtime_context import get_parallel
# The file-backend storage config records TP/PP rank/size. These unit
# fixtures run without initializing distributed parallel state, so
# force the local single-rank topology the fixture represents.
# Distributed state is not initialized in unit fixtures; force the
# local single-rank topology.
parallel_override = get_parallel().override(
tp_rank=0, tp_size=1, pp_rank=0, pp_size=1
)
@@ -216,7 +217,6 @@ class UnifiedRadixCacheFixtures:
extra.update(storage_extra)
storage_extra_config = json.dumps(extra) if extra else None
# See _init_hicache: production kernel IO backend, pinned pools.
server_args = ServerArgs(
model_path="dummy",
page_size=self.cfg.page_size,
@@ -228,7 +228,6 @@ class UnifiedRadixCacheFixtures:
hicache_host_memory_mode=host_memory_mode,
context_length=context_length,
)
# See build_fixture for why _mamba_cache_chunk_size is preset.
server_args._mamba_cache_chunk_size = max(FLA_CHUNK_SIZE, self.cfg.page_size)
set_global_server_args_for_scheduler(server_args)
cache.init_hicache(server_args, cache.cache_init_params)
@@ -237,15 +236,12 @@ class UnifiedRadixCacheFixtures:
cache.write_through_threshold = 1 << 30
cache.load_back_threshold = 0
if storage_backend is not None:
# Unit fixtures size host/device pools equally, which makes the
# production prefetch capacity limit (host - device) zero. Keep the
# L3 tests focused on storage round trips by allowing one fixture
# worth of prefetch tokens.
# Fixtures size host and device pools equally, which zeroes the
# prefetch capacity limit; allow one fixture of prefetch tokens.
cache.cache_controller.prefetch_capacity_limit = max(
cache.cache_controller.prefetch_capacity_limit,
cache.cache_controller.mem_pool_host.size,
)
# Background prefetch/backup threads are daemon; stop them per-test.
self.addCleanup(cache.cache_controller._stop_storage_threads)
def _build_hicache_fixture(self, tree_core_backend: Optional[str] = None):
@@ -307,10 +303,9 @@ class UnifiedRadixCacheFixtures:
)
def _build_chain_pages(self, cache, allocator, req_to_token_pool, num_pages):
"""Insert an incremental chain of single-page extensions.
"""Insert a chain of single-page extensions; return it root-to-leaf.
Returns the chain root-to-leaf. Length may differ from num_pages
when the radix tree merges or splits nodes.
The length may differ from num_pages when the tree merges or splits nodes.
"""
seq: list[int] = []
for i in range(num_pages):
@@ -340,6 +335,31 @@ class UnifiedRadixCacheFixtures:
ack.finish_event.synchronize()
cache.loading_check()
def _evict_device_component(self, cache, node, component_type):
"""Evict one node's device value of one component, as the eviction
walk does for the victim it picks."""
result = cache.tree_core.evict_component(
node, component_type, EvictLayer.DEVICE
)
cache._free_values(result.device_frees, result.host_frees)
def _load_back_through_admission(self, cache, req_to_token_pool, tokens):
"""Match ``tokens`` and start the scheduler's load-back of the host
hit. The load-back holds its path lock until it is released."""
req = self._make_req(req_to_token_pool)
match = cache.match_prefix(
MatchPrefixParams(key=RadixKey(array("q", tokens)), req=req)
)
self._apply_match_to_req(req, match)
cache.init_load_back(
InitLoadBackParams(
best_match_node=req.best_match_node,
host_hit_length=req.host_hit_length,
req=req,
)
)
return req
def _match_tokens_for_chain(self, cache, chain):
tokens: list[int] = []
for node in chain:
@@ -363,9 +383,7 @@ class UnifiedRadixCacheFixtures:
)
def _swa_finalize_setup(self):
"""Build a SWA chain long enough to fill at least the window
plus one extra page, and host-back every node so we can flip
SWA tombstones at will."""
"""Build a SWA chain longer than the window, host-backed at every node."""
ps = self.cfg.page_size
sw = self.cfg.sliding_window_size
window_pages = (sw + ps - 1) // ps
@@ -134,10 +134,9 @@ class UnifiedRadixCacheStorageFixtures(UnifiedRadixCacheFixtures):
def _run_prefetch_to_completion(self, cache, req_id, timeout: float = 10.0):
deadline = time.time() + timeout
while time.time() < deadline:
# Host memory is reserved (and IO started) by the scheduler-thread
# drain once the L3 hit count is known, and the buffer-mode fill
# commits at its H2D ack in loading_check — pump the full event
# round like the real scheduler loop does.
# Pump the full event round like the scheduler loop: host reservation
# happens at the scheduler-thread drain, and the buffer-mode fill
# commits at its H2D ack.
cache.check_hicache_events()
if cache.check_prefetch_progress(req_id):
# Buffer mode parks a completed fetch as a staged prefetch;
@@ -175,7 +174,7 @@ class UnifiedRadixCacheStorageFixtures(UnifiedRadixCacheFixtures):
f = cache.buffer_pipeline.staged_prefetches[req_id]
if prefix_len is None:
prefix_len = f.matched_len
req = mock.Mock()
req = self._make_req(cache.req_to_token_pool)
req.rid = req_id.rid
req.cache_request_handle = req_id
req.extra_key = extra_key
@@ -391,9 +390,8 @@ class UnifiedRadixCacheStorageFixtures(UnifiedRadixCacheFixtures):
cons.enable_storage_metrics = True
cons.storage_metrics_collector = mock.Mock()
# Masked state: nodes born with live FULL under SWA tombstones (a
# sibling insert whose SWA ring had slid past the span). live_tail is
# how far short of the span end the ring stopped.
# Nodes born with live FULL under SWA tombstones; live_tail is how far short
# of the span end the ring stopped.
live_from = len(seq) - live_tail
value = self._alloc(cons_alloc, len(seq))
cons.insert(
@@ -407,8 +405,7 @@ class UnifiedRadixCacheStorageFixtures(UnifiedRadixCacheFixtures):
self.assertEqual(masked.device_prefix_len, 0, "unified match not masked")
self.assertEqual(masked.full_kv_hit_length, len(seq), "live FULL not resident")
masked_full = value[: len(seq)].clone()
# Model the real SWA-eviction path: tombstoned positions have their
# allocator mapping cleared, while the live trailing portion remains.
# Tombstoned positions have their allocator mapping cleared; the tail stays.
cons_alloc.free_swa(masked_full[:live_from])
self._fill_full_kv(cons_alloc, masked_full, marker=7)
masked_kv = self._snapshot_full_kv(cons_alloc, masked_full)
@@ -421,7 +418,6 @@ class UnifiedRadixCacheStorageFixtures(UnifiedRadixCacheFixtures):
cons.prefetch_from_storage(
req_id, cons.root_node_handle(), array("q", seq), None, None
)
# A parked SWA-only retry has no FULL suffix, but must re-pin its prefix.
with mock.patch.object(cons.swa_kv_pool_host, "alloc", return_value=None):
self._pump_hicache_until(
cons,
@@ -500,7 +496,6 @@ class UnifiedRadixCacheStorageFixtures(UnifiedRadixCacheFixtures):
after = match_prefix(cons, seq)
self.assertEqual(after.full_kv_hit_length, len(seq))
self.assertEqual(after.device_prefix_len, len(seq))
# Only the fetched SWA tail made the resident FULL reusable.
self.assertEqual((req.storage_hit_start, req.storage_hit_length), (0, len(seq)))
cons.storage_metrics_collector.log_storage_prefetch_unfulfilled_tokens.assert_not_called()
cons.sanity_check()
@@ -552,7 +547,6 @@ class UnifiedRadixCacheStorageFixtures(UnifiedRadixCacheFixtures):
)
self.assertGreater(aux_max, len(seq) - len(head))
# The surfaced host hit is the splice-able tail, not the full span.
req = SimpleNamespace(
rid=req_id.rid,
cache_request_handle=req_id,
@@ -585,7 +579,6 @@ class UnifiedRadixCacheStorageFixtures(UnifiedRadixCacheFixtures):
self.assertTrue(torch.equal(hk, head_k))
self.assertTrue(torch.equal(hv, head_v))
# The whole bounce (trimmed head included) frees at the ack.
self.assertEqual(self._host_avail_sizes(cons), avail0)
self.assertEqual(cons.cache_controller.prefetch_tokens_occupied, 0)
cons.sanity_check()
@@ -596,7 +589,6 @@ class UnifiedRadixCacheStorageFixtures(UnifiedRadixCacheFixtures):
cc = cache.cache_controller
# Fake _reduce_prefetch_ack to drop SWA pool_hits when drop_swa is True.
def fake_reduce(ack):
if drop_swa and ack.pool_hits is not None:
if PoolName.SWA.value in ack.pool_hits:
@@ -608,7 +600,6 @@ class UnifiedRadixCacheStorageFixtures(UnifiedRadixCacheFixtures):
p_reduce.start()
self.addCleanup(p_reduce.stop)
# Make all_reduce no-op. The real all_reduce fails in unit tests with tp_world_size=2.
p_dist = mock.patch.object(dist, "all_reduce", return_value=None)
p_dist.start()
self.addCleanup(p_dist.stop)
@@ -774,25 +765,26 @@ class UnifiedRadixCacheStorageAttachBackfillFixtures:
return hashes
class AnchorLockOutcomePolicyFixtures:
def _make_pipeline(self, cache, cap_tokens=10_000):
from sglang.srt.mem_cache.buffer_mode.pipeline import BufferModePipeline
class AnchorLockOutcomePolicyFixtures(UnifiedRadixCacheStorageFixtures):
def _insert_tokens(self, cache, tokens):
value = cache.token_to_kv_pool_allocator.alloc(len(tokens))
self.assertIsNotNone(value)
cache.insert(InsertParams(key=RadixKey(array("q", tokens)), value=value))
pipeline = BufferModePipeline.__new__(BufferModePipeline)
pipeline.anchor_locks = {}
pipeline.anchor_locked_tokens_ = 0
pipeline.anchor_lock_cap_tokens = cap_tokens
pipeline._anchor_lock_cap_skips = 0
pipeline._prefetch_prefix_ctx = {self._REQ: (list(self._PREFIX), None, None)}
pipeline._cache = cache
return pipeline
def _make_cache(self, live_match_len, pinned_len=None):
cache = mock.MagicMock()
cache.tree_core.is_eagle = False
cache.tree_core.match_full_device_prefix.return_value = (
live_match_len,
99,
live_match_len if pinned_len is None else pinned_len,
def _make_buffer_cache(
self, resident=(), cap_tokens=None, prefix=(), storage_dir=None
):
"""A buffer-mode cache holding ``resident`` on device, with the
request's matched prefix recorded as prefetch enqueue records it."""
cache, _, _ = self._build_buffer_hicache_fixture(
storage_dir or self._make_storage_dir()
)
return cache
if resident:
self._insert_tokens(cache, list(resident))
pipeline = cache.buffer_pipeline
if cap_tokens is not None:
pipeline.anchor_lock_cap_tokens = cap_tokens
pipeline.set_prefix_ctx(
self._REQ, self._PREFIX if prefix == () else (prefix or [])
)
return cache, pipeline
@@ -27,8 +27,6 @@ if TYPE_CHECKING:
class UnifiedTreeCoreInspectionInterface(UnifiedTreeCoreInterface):
"""Test-only inspection and control contract for shared backend tests."""
# ==== Read-only inspection ====
@abstractmethod
def contains_node(self, node_id: NodeId) -> bool: ...
@@ -125,8 +123,6 @@ class UnifiedTreeCoreInspectionInterface(UnifiedTreeCoreInterface):
"""Protected token count for one component (0 if the component is absent)."""
...
# ==== White-box state controls ====
@abstractmethod
def set_node_hash_values(
self, node_id: NodeId, hash_values: Optional[list[str]]
@@ -194,8 +190,6 @@ class UnifiedTreeCoreInspectionInterface(UnifiedTreeCoreInterface):
"""Refresh duplicate-host tracking for the node."""
...
# ==== Targeted white-box operations ====
@abstractmethod
def advance_insert_walk_once(self) -> None:
"""Advance one suspended insert walk step without flushing its actions."""
@@ -30,7 +30,6 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa
return node_id in self._node_arena
def get_parent_node_id(self, node_id: NodeId) -> Optional[NodeId]:
"""The parent node id, or None for the root."""
parent = self.node_by_id(node_id).parent
return None if parent is None else parent.id
@@ -45,13 +44,11 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa
return len(key)
def get_node_token_ids(self, node_id: NodeId) -> list[int]:
"""The raw token ids spanned by the node's radix key."""
key = self.node_by_id(node_id).key
assert key is not None
return list(key.raw_token_ids())
def is_node_key_bigram(self, node_id: NodeId) -> bool:
"""Whether the node's radix key uses bigram atoms."""
key = self.node_by_id(node_id).key
assert key is not None
return key.is_bigram
@@ -59,19 +56,16 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa
def get_component_host_value(
self, node_id: NodeId, component_type: ComponentType
) -> Optional[torch.Tensor]:
"""The component's host value on the node, or None if absent."""
return self.node_by_id(node_id).component_data[component_type].host_value
def get_component_device_lock_ref(
self, node_id: NodeId, component_type: ComponentType
) -> int:
"""The component's device lock count on the node."""
return self.node_by_id(node_id).component_data[component_type].lock_ref
def get_component_host_lock_ref(
self, node_id: NodeId, component_type: ComponentType
) -> int:
"""The component's host lock count on the node."""
return self.node_by_id(node_id).component_data[component_type].host_lock_ref
def get_node_hit_count(self, node_id: NodeId) -> int:
@@ -83,20 +77,17 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa
return self.node_by_id(node_id).write_through_pending_id
def is_external_cache_stored(self, node_id: NodeId) -> bool:
"""Whether the node is known to be stored in the external cache."""
return self.node_by_id(node_id).external_cache_stored
def is_node_in_device_lru(
self, node_id: NodeId, component_type: ComponentType
) -> bool:
"""Whether the node belongs to the component's device LRU."""
lru = self.lru_lists.get(component_type)
return lru is not None and lru.in_list(self.node_by_id(node_id))
def is_node_in_host_lru(
self, node_id: NodeId, component_type: ComponentType
) -> bool:
"""Whether the node belongs to the component's host LRU."""
lru = self.host_lru_lists.get(component_type)
return lru is not None and lru.in_list(self.node_by_id(node_id))
@@ -114,7 +105,6 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa
def get_component_device_lru_node_ids(
self, component_type: ComponentType
) -> list[NodeId]:
"""The component's device LRU members from most to least recent."""
lru = self.lru_lists.get(component_type)
return [] if lru is None else self._lru_node_ids(lru)
@@ -129,7 +119,6 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa
return node is not None and node in self.evictable_host_leaves
def is_device_leaf(self, node_id: NodeId) -> bool:
"""Whether the node has no device-resident descendants."""
return self._is_device_leaf(self.node_by_id(node_id))
def get_all_node_ids(self) -> list[NodeId]:
@@ -137,13 +126,11 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa
return [node.id for node in self._collect_all_nodes()]
def component_protected_size(self, component_type: ComponentType) -> int:
"""Protected token count for one component (0 if the component is absent)."""
return self.component_protected_size_.get(component_type, 0)
def set_node_hash_values(
self, node_id: NodeId, hash_values: Optional[list[str]]
) -> None:
"""Replace the node's page-hash field."""
self.node_by_id(node_id).hash_value = hash_values
def set_component_device_value_raw(
@@ -152,7 +139,6 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa
component_type: ComponentType,
value: Optional[torch.Tensor],
) -> None:
"""Replace the device-value field without updating tree bookkeeping."""
self.node_by_id(node_id).component_data[component_type].value = value
def set_component_host_value_raw(
@@ -161,48 +147,40 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa
component_type: ComponentType,
value: Optional[torch.Tensor],
) -> None:
"""Replace the host-value field without updating tree bookkeeping."""
self.node_by_id(node_id).component_data[component_type].host_value = value
def set_component_device_lock_ref(
self, node_id: NodeId, component_type: ComponentType, lock_ref: int
) -> None:
"""Replace the component's device lock count."""
assert lock_ref >= 0
self.node_by_id(node_id).component_data[component_type].lock_ref = lock_ref
def remove_node_from_device_lru(
self, node_id: NodeId, component_type: ComponentType
) -> None:
"""Remove the node from the component's device LRU."""
self.lru_lists[component_type].remove_node(self.node_by_id(node_id))
def insert_node_into_host_lru(
self, node_id: NodeId, component_type: ComponentType
) -> None:
"""Insert the node as the component's most-recent host entry."""
self.host_lru_lists[component_type].insert_mru(self.node_by_id(node_id))
def set_component_evictable_size(
self, component_type: ComponentType, value: int
) -> None:
"""Replace the component's evictable device-token count."""
assert value >= 0
self.component_evictable_size_[component_type] = value
def set_component_protected_size(
self, component_type: ComponentType, value: int
) -> None:
"""Replace the component's protected device-token count."""
assert value >= 0
self.component_protected_size_[component_type] = value
def update_duplicate_tracking(self, node_id: NodeId) -> None:
"""Refresh duplicate-host tracking for the node."""
self._update_duplicate_tracking(self.node_by_id(node_id))
def advance_insert_walk_once(self) -> None:
"""Advance one suspended insert walk step without flushing its actions."""
state = self._ongoing_insert_walk_state
if state is None:
raise RuntimeError("no in-flight insert")
@@ -216,7 +194,6 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa
component_type: ComponentType,
target: EvictLayer,
) -> BaseEvictionResult:
"""Evict one component layer from a node and detach its LRU entry."""
result = BaseEvictionResult()
self._evict_component_and_detach_lru(
self.node_by_id(node_id),
@@ -235,7 +212,6 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa
component_type: ComponentType,
target: EvictLayer,
) -> None:
"""Validate the locks for a component-triggered cascade eviction."""
node = self.node_by_id(node_id)
trigger = self.components_by_type[component_type]
is_leaf = self._is_cascade_evict_leaf(node, target)
@@ -243,7 +219,6 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa
self._should_cascade_evict_component(node, trigger, comp, target, is_leaf)
def cleanup_tombstone_ancestors(self, node_id: NodeId) -> BaseEvictionResult:
"""Delete childless tombstone ancestors until a live or locked node is reached."""
result = BaseEvictionResult()
self._iteratively_delete_tombstone_ancestors(
self.node_by_id(node_id),
@@ -262,7 +237,6 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa
value_chunks: list[torch.Tensor],
best_value_len: int,
) -> MatchResult:
"""Run one component's match finalizer with NodeId boundaries."""
node_result = result._replace(
last_device_node=self.node_by_id(result.last_device_node),
last_host_node=self.node_by_id(result.last_host_node),
@@ -285,7 +259,6 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa
def build_backup_node_ids(
self, node_id: NodeId, write_back: bool = False
) -> list[NodeId]:
"""Build the ordered node list for a device-to-host backup."""
return self._build_backup_kv_action(
self.node_by_id(node_id), write_back
).node_ids
@@ -1,5 +1,3 @@
"""Insert, match, lock, eviction, checkpoint, SWA and Mamba tests for UnifiedRadixCache."""
import unittest
from array import array
from dataclasses import replace
@@ -121,13 +119,8 @@ class TestUnifiedRadixComponentRegistryOverride(CustomTestCase):
class TestUnifiedRadixComponentLedgerUnits(CustomTestCase):
def test_ledgers_count_reclaimable_tokens(self):
"""The evictable and protected ledgers are in the allocator's units:
prefill admission adds the evictable ledger to the allocator's free
space and the eviction walk counts them toward its request. So every
ledger site (insert, lock, release, an in-place change, eviction)
reads ``TreeComponent.reclaimable_value_len`` rather than the value
length. A site that goes back to ``len(value)`` drifts the ledger from
the recount in ``sanity_check`` for a component that overrides it."""
"""Ledgers count allocator units: every site reads reclaimable_value_len,
not len(value), or the ledger drifts from sanity_check's recount."""
cache, allocator, _ = build_fixture(
CacheConfig(),
component_registry_override={ComponentType.FULL: PagedFullComponent},
@@ -183,7 +176,7 @@ class TestDisabledUnifiedRadixCache(DisabledUnifiedRadixCacheFixtures, CustomTes
class TestUnifiedTreeNodeGetPrefixHashValues(CustomTestCase):
def test_get_prefix_hash_values_not_shared_across_calls(self):
"""Regression guard for cached mutable prefix hash lists (#26177)."""
"""Cached prefix hash lists must not be mutated by callers."""
def make_node():
return UnifiedTreeNode(tree_components=(ComponentType.FULL,))
@@ -458,7 +451,6 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
self._insert(cache, allocator, req_to_token_pool, seq_1p)
self.assertEqual(allocator.available_size(), initial_avail - len(seq_1p))
# Frees the 1-page overlap.
key_2p = RadixKey(array("q", seq_2p))
value_2p = self._alloc(allocator, len(seq_2p))
params = InsertParams(
@@ -481,7 +473,6 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
initial_avail - len(seq_1p) - (len(seq_2p) - len(seq_1p)),
)
# prev_prefix_len covers the whole overlap: nothing freed.
avail_before = allocator.available_size()
key_3p = RadixKey(array("q", seq_3p))
value_3p = self._alloc(allocator, len(seq_3p))
@@ -637,10 +628,9 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
@requires_swa_only
@requires_ps1_window4
def test_swa_unfinished_req_repoints_whole_insert_below_window(self):
# A request whose live SWA tail inside the inserted range is shorter
# than the window must still end up protected for the whole insert:
# the tree owns those slots now, and a shorter protected length lets
# the request free tree-owned SWA pages and re-insert them at finish.
# Even a live SWA tail shorter than the window must be protected for the whole
# insert, else the request frees tree-owned SWA pages and re-inserts them at
# finish.
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
full_avail = allocator.full_attn_allocator.available_size()
swa_avail = allocator.swa_attn_allocator.available_size()
@@ -730,12 +720,10 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
seq = self._make_seq(1, 2)
self._insert(cache, allocator, req_to_token_pool, seq)
# Tree truncates unaligned tail internally, so it matches the seq prefix.
unaligned = seq + list(range(9000, 9000 + ps - 1))
m = match_prefix(cache, unaligned)
self.assertEqual(m.device_prefix_len, len(seq))
# Below-page-size key aligns to 0 -> no match.
m = match_prefix(cache, seq[: ps - 1])
self.assertEqual(m.device_prefix_len, 0)
@@ -752,12 +740,10 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
self._insert(cache, allocator, req_to_token_pool, first_page)
self._insert(cache, allocator, req_to_token_pool, seq)
# Mismatch in second page → only first page matches
bad_page2 = seq[:ps] + [9999] * ps
m = match_prefix(cache, bad_page2)
self.assertEqual(m.device_prefix_len, ps)
# Mismatch in first page → 0 match
bad_page1 = [9999] + seq[1:]
m = match_prefix(cache, bad_page1)
self.assertEqual(m.device_prefix_len, 0)
@@ -896,7 +882,6 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
# past the SWA window (swa_evicted_seqlen >= total), the Full leaf must
# be materialized (and the Full KV kept) so the prefix stays cacheable.
# Runs across all SWA configs, including page_size > sliding_window_size
# (the dsv4-style edge case).
cache, allocator, _ = build_fixture(self.cfg)
tokens = self._make_seq(1, 2)
@@ -938,11 +923,8 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
node_a = m.last_device_node
self.assertGreater(len(node_children(cache, node_a)), 0, "A must be internal")
# A request locks A, then decodes past the window → early-release the SWA
# portion. On this internal node, dec_swa_lock_only also drops the
# strictly-lower-tier Mamba lock (the co-located Mamba is useless once SWA
# is gone), leaving only the Full path-lock held. This is what guarantees
# the later SWA-eviction cascade never meets a legitimately-locked Mamba.
# Early SWA release on an internal node also drops the lower-tier Mamba lock,
# so later SWA eviction never meets a locked Mamba.
lock_result = cache.inc_lock_ref(node_a)
self.assertGreaterEqual(
node_device_lock_ref(cache, node_a, ComponentType.MAMBA),
@@ -963,9 +945,7 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
cache.tree_core.is_node_in_device_lru(node_a, ComponentType.SWA)
)
# Evict the child branch (Full/device eviction only) → A becomes a
# Full-locked leaf with its now-unlocked SWA still in the SWA LRU. We do
# NOT tombstone aux at the leaf-transition; the held Full pins the node.
# Evicting the child makes A a Full-locked leaf with its SWA still in the LRU.
cache.evict(EvictParams(num_tokens=len(seq_ab)))
self.assertEqual(len(node_children(cache, node_a)), 0, "A should now be a leaf")
@@ -979,10 +959,7 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
"A's unlocked SWA stays in the LRU (not tombstoned at transition)",
)
# SWA eviction now selects A (it is in the SWA LRU). A's Full lock is a
# higher-or-equal internal tier, so the cascade skips it and spares the
# Full KV. The unlocked lower-tier Mamba is cascaded as part of the atomic
# leaf teardown. This used to assert `cd.lock_ref == 0` on the locked Full.
# SWA eviction skips A's Full lock and cascades the unlocked lower-tier Mamba.
cache.evict(EvictParams(num_tokens=0, swa_num_tokens=len(seq_a)))
self.assertGreaterEqual(
node_device_lock_ref(cache, node_a, ComponentType.FULL),
@@ -1006,8 +983,7 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
def test_mamba_opt_out_holder_cannot_release_another_holders_mamba_lock(self):
"""Holder A takes the mamba lock; holder B opts out (skip_lock_components=(ComponentType.MAMBA,))
on the same node. B's early SWA release and final release must leave
A's mamba lock intact -- a lost/defaulted receipt on B's side used to
decrement A's lock without tripping any assert."""
A's mamba lock intact."""
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
seq = self._make_seq(
@@ -1064,7 +1040,6 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
"A must hold a Mamba checkpoint",
)
# Natural lock acquisition — records inc_lock_ref in the lock trace.
lock_result = cache.inc_lock_ref(node_a)
self.assertGreaterEqual(
node_device_lock_ref(cache, node_a, ComponentType.SWA), 1, "SWA locked"
@@ -1096,46 +1071,6 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
"Full stays locked",
)
@requires_swa_and_mamba
def test_cascade_evict_asserts_on_locked_internal_mamba(self):
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
n_short = (self.cfg.sliding_window_size // self.cfg.page_size) + 4
seq_a = self._make_seq(1, n_short)
seq_ab = seq_a + self._make_seq(7000, 2)
self._insert(cache, allocator, req_to_token_pool, seq_a)
self._insert(cache, allocator, req_to_token_pool, seq_ab)
node_a = match_prefix(cache, seq_a).last_device_node
self.assertGreater(len(node_children(cache, node_a)), 0, "A must be internal")
self.assertIsNotNone(
node_device_value(cache, node_a, ComponentType.MAMBA),
"A must hold a Mamba checkpoint",
)
# Lock ONLY Mamba — a stranded lower-priority lock that no supported path
# produces. The cascade must surface it rather than silently skip.
cache.tree_core.set_component_device_lock_ref(node_a, ComponentType.MAMBA, 1)
self.assertGreaterEqual(
node_device_lock_ref(cache, node_a, ComponentType.MAMBA), 1, "Mamba locked"
)
self.assertEqual(
node_device_lock_ref(cache, node_a, ComponentType.FULL), 0, "Full unlocked"
)
result = cache.tree_core.evict_component(
node_a, ComponentType.SWA, EvictLayer.DEVICE
)
cache._free_values(result.device_frees, result.host_frees)
# No higher-or-equal tier pins the node, so even with early-release on
# the stranded Mamba lock must trip the hard-invariant assert.
with self.assertRaises(AssertionError):
cache.tree_core.validate_cascade_evict(
node_a, ComponentType.SWA, EvictLayer.DEVICE
)
cache.tree_core.set_component_device_lock_ref(node_a, ComponentType.MAMBA, 0)
@requires_swa_and_mamba
def test_cascade_evict_asserts_on_locked_leaf_mamba(self):
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
@@ -1173,9 +1108,6 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
@requires_swa
def test_swa_leaf_capped_to_window_on_insert(self):
"""A long SWA leaf is split so locking it protects one window of SWA
while full attention still protects the whole sequence."""
ps = self.cfg.page_size
window = self.cfg.sliding_window_size
tail_size = ((window + ps - 1) // ps) * ps
@@ -1194,13 +1126,11 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
self.assertIsNotNone(swa_val)
if case == "long_splits":
# Capped to one page-aligned window; prefix is a real ancestor.
self.assertEqual(len(swa_val), tail_size)
self.assertNotEqual(
node_parent(cache, leaf), cache.root_node_handle()
)
else:
# Already within one window — no split.
self.assertEqual(len(swa_val), len(seq))
self.assertEqual(node_parent(cache, leaf), cache.root_node_handle())
@@ -1447,19 +1377,6 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
cache.unlock(req.lock)
cache.sanity_check()
def test_tombstone_cleanup_respects_locked_parent(self):
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
tokens = self._make_seq(1, 1)
self._insert(cache, allocator, req_to_token_pool, tokens)
parent = match_prefix(cache, tokens).last_device_node
cache.tree_core.set_component_device_lock_ref(parent, ComponentType.FULL, 1)
result = cache.tree_core.cleanup_tombstone_ancestors(parent)
cache._free_values(result.device_frees, result.host_frees)
self.assertIn(parent, node_children(cache, cache.root_node_handle()))
self.assertTrue(all(evicted == 0 for evicted in result.tracker.values()))
def test_aux_evict_full_locked_leaf_tombstones_aux_only(self):
aux_types = [
ct
@@ -1473,7 +1390,6 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
aux = aux_types[0]
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
# One page stays within a single SWA window (no leaf-cap split).
seq = self._make_seq(1, 1)
self._insert(cache, allocator, req_to_token_pool, seq)
@@ -1483,9 +1399,8 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
self.assertIsNotNone(node_device_value(cache, node, ComponentType.FULL))
self.assertIsNotNone(node_device_value(cache, node, aux))
# Reach the "FULL locked, aux unlocked" state the way production does:
# mamba via the decode-hold opt-out (skip_lock_components=(ComponentType.MAMBA,)), SWA via the
# early window release (its own first-class op).
# Reach "FULL locked, aux unlocked" as production does: Mamba via the
# decode-hold opt-out, SWA via early window release.
aux_len = len(node_device_value(cache, node, aux))
if aux == ComponentType.MAMBA:
lock_result = cache.inc_lock_ref(
@@ -1495,8 +1410,6 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
lock_result = cache.inc_lock_ref(node)
self.assertGreater(node_device_lock_ref(cache, node, aux), 0)
cache.release_swa(TreeLock(node, lock_result.to_dec_params()))
# FULL still locked -> not a device leaf -> no inline evict; the
# value stays evictable for the explicit aux eviction below.
self.assertIsNotNone(node_device_value(cache, node, aux))
self.assertGreater(node_device_lock_ref(cache, node, ComponentType.FULL), 0)
self.assertEqual(node_device_lock_ref(cache, node, aux), 0)
@@ -1525,25 +1438,6 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
)
cache.sanity_check()
def test_evict_leaf_frees_all_components(self):
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
seq = self._make_seq(1, 3)
self._insert(cache, allocator, req_to_token_pool, seq)
full_before = cache.full_evictable_size()
mamba_before = cache.mamba_evictable_size() if self.cfg.has_mamba else 0
swa_before = cache.swa_evictable_size() if self.cfg.has_swa else 0
self.assertGreater(full_before, 0)
result = cache.evict(EvictParams(num_tokens=full_before * 2))
self.assertGreaterEqual(result.num_tokens_evicted, full_before)
self.assertEqual(cache.full_evictable_size(), 0)
if self.cfg.has_mamba:
self.assertEqual(cache.mamba_evictable_size(), 0)
if self.cfg.has_swa:
self.assertEqual(cache.swa_evictable_size(), 0)
cache.sanity_check()
def test_evict_cascade_parent_becomes_d_leaf(self):
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
base = self._make_seq(1, 2)
@@ -1580,6 +1474,10 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
result = cache.evict(EvictParams(num_tokens=initial_evictable * 2))
self.assertGreaterEqual(result.num_tokens_evicted, initial_evictable)
self.assertEqual(cache.full_evictable_size(), 0)
if self.cfg.has_mamba:
self.assertEqual(cache.mamba_evictable_size(), 0)
if self.cfg.has_swa:
self.assertEqual(cache.swa_evictable_size(), 0)
self.assertEqual(len(node_children(cache, cache.root_node_handle())), 0)
cache.sanity_check()
@@ -1623,27 +1521,6 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
self.assertEqual(m_low.device_prefix_len, 0)
cache.sanity_check()
def test_evict_multiple_independent_leaves(self):
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
seqs = [self._make_seq(i * 100, 2) for i in range(4)]
for s in seqs:
self._insert(cache, allocator, req_to_token_pool, s)
total = sum(len(s) for s in seqs)
self.assertEqual(cache.full_evictable_size(), total)
half = total // 2
result = cache.evict(EvictParams(num_tokens=half))
self.assertGreaterEqual(result.num_tokens_evicted, half)
self.assertLessEqual(cache.full_evictable_size(), total - half)
cache.sanity_check()
remaining = cache.full_evictable_size()
result = cache.evict(EvictParams(num_tokens=remaining * 2))
self.assertGreaterEqual(result.num_tokens_evicted, remaining)
self.assertEqual(cache.full_evictable_size(), 0)
cache.sanity_check()
def test_evict_result_accounting_matches_actual(self):
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
seqs = [self._make_seq(i * 100, 2) for i in range(5)]
@@ -1651,9 +1528,15 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
self._insert(cache, allocator, req_to_token_pool, s)
before = cache.full_evictable_size()
result = cache.evict(EvictParams(num_tokens=before))
half = before // 2
result = cache.evict(EvictParams(num_tokens=half))
after = cache.full_evictable_size()
self.assertGreaterEqual(result.num_tokens_evicted, half)
self.assertEqual(result.num_tokens_evicted, before - after)
remaining = cache.full_evictable_size()
cache.evict(EvictParams(num_tokens=remaining * 2))
self.assertEqual(cache.full_evictable_size(), 0)
cache.sanity_check()
@requires_swa
@@ -1706,54 +1589,37 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
@requires_full_swa_ps1_window4
def test_swa_branch_insert_releases_forward_overshoot(self):
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
swa = cache.components[ComponentType.SWA]
branching_seqlen = 8
forward_len = 20
req = self._make_req(req_to_token_pool)
tokens = self._make_seq(1, forward_len)
req.origin_input_ids = tokens
req.output_ids = []
req.full_untruncated_fill_ids = array("q", tokens)
req.extend_end = forward_len
req.kv.cache_protected_len = branching_seqlen
kv_indices = self._alloc(allocator, forward_len)
req_to_token_pool.write(
(req.kv.req_pool_idx, slice(0, forward_len)),
kv_indices,
req, kv_indices = self._make_inflight_req(
cache, allocator, req_to_token_pool, tokens
)
params = InsertParams(
key=RadixKey(array("q", tokens[:branching_seqlen])),
swa_branching_seqlen=branching_seqlen,
)
result = mock.Mock(swa_branch_inserted=False)
swa.commit_insert_component_data(
mock.Mock(),
False,
params,
result,
[],
)
self.assertTrue(result.swa_branch_inserted)
req.swa_branching_seqlen = branching_seqlen
with envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS.override(True):
swa.cleanup_after_caching_req(
req,
is_finished=False,
insert_result=result,
)
cache.checkpoint(req, up_to=forward_len)
# Only the branch point is cached; the forward past it is released.
self.assertEqual(
match_prefix(cache, tokens).device_prefix_len, branching_seqlen
)
expected_evicted = forward_len - 1 - self.cfg.sliding_window_size
self.assertEqual(req.kv.get_evicted_seqlen(ComponentType.SWA), expected_evicted)
# The cached branch keeps SWA for its last window; the forward overshoot
# past it is released, and the request keeps its own live window.
branch_freed = branching_seqlen - 1 - self.cfg.sliding_window_size
mapping = allocator.full_to_swa_index_mapping
self.assertEqual(
torch.count_nonzero(mapping[kv_indices[:branching_seqlen]]).item(),
branching_seqlen,
torch.count_nonzero(mapping[kv_indices[:branch_freed]]).item(), 0
)
self.assertEqual(
torch.count_nonzero(
mapping[kv_indices[branch_freed:branching_seqlen]]
).item(),
branching_seqlen - branch_freed,
)
self.assertEqual(
torch.count_nonzero(
@@ -1765,6 +1631,8 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
torch.count_nonzero(mapping[kv_indices[expected_evicted:]]).item(),
forward_len - expected_evicted,
)
cache.unlock(req.lock)
cache.sanity_check()
define_config_test_classes(UnifiedRadixCacheCoreSuite, globals())
@@ -1786,22 +1654,18 @@ class UnifiedLRUListBoundedRefreshTest(
lru.reset_node_and_window_ancestors_mru(
d, root, window_size=5, should_include=lambda n: n in included
)
# D, C refreshed; B contributes 2 to acc (now 6 >= 5) but is skipped;
# A is not visited because the walk stops at B.
self.assertEqual(self._lru_order(lru), [d, c, a])
def test_bounded_refresh_visits_only_until_window_filled(self):
root, [a, b, c, d] = self._build_chain([3, 3, 3, 3])
lru = UnifiedLRUList(ComponentType.SWA, self.components)
# MRU->LRU: A, B, C, D (deepest is at the LRU tail, oldest)
for n in (d, c, b, a):
lru.insert_mru(n)
self.assertEqual(self._lru_order(lru), [a, b, c, d])
# window=5: walking up from D visits D(acc=3<5) and C(acc=6>=5, stop).
# Order expected: [D, C, A, B]. Why: D and C move to head in that
# order (D first, then C right after D). A and B keep their relative
# positions (they were the surviving prefix [A, B] before).
# D and C move to the head in that order; A and B keep their relative
# positions.
lru.reset_node_and_window_ancestors_mru(
d, root, window_size=5, should_include=lambda _n: True
)
@@ -1835,13 +1699,8 @@ class UnifiedLRUListBoundedRefreshTest(
class TestUnifiedMambaLRUMatchRefresh(
UnifiedMambaLRUMatchRefreshFixtures, CustomTestCase
):
"""A prefix-cache hit must refresh only best_match_node's mamba state in the
mamba LRU, not its ancestors. The base TreeComponent refresh is whole-chain
(reset_node_and_parents_mru), which clusters a session's states so that under
mamba-pool pressure eviction drops whole cold sessions instead of the
intermediate states reuse never needs. Guards MambaComponent.refresh_lru's
single-node MATCH_END override.
"""
"""A hit refreshes only best_match_node's mamba state, so pressure evicts whole
cold sessions rather than intermediate states."""
cfg = CacheConfig(page_size=1, components=(ComponentType.FULL, ComponentType.MAMBA))
@@ -1917,8 +1776,6 @@ class TestMambaCheckpointGrid(MambaCheckpointGridFixtures, CustomTestCase):
class TestMambaFinishedOvershootCheckpoint(
MambaFinishedOvershootCheckpointFixtures, CustomTestCase
):
"""A donated state must have consumed exactly the prefix its key names."""
_rid = 0
cfg = CacheConfig(
page_size=4,
@@ -2001,7 +1858,6 @@ class TestUnifiedRadixCacheInt8MambaCheckpoint(
def _component_with_cache(component_type, cache):
"""A registry component instance bound to a (mock) cache and its tree_core."""
component = object.__new__(COMPONENT_REGISTRY[component_type])
component.cache = cache
component.tree_core = cache.tree_core
@@ -2009,8 +1865,6 @@ def _component_with_cache(component_type, cache):
class TestUnifiedRadixCacheActionRouting(CustomTestCase):
"""CacheAction routing: each type forwards to the right Controller API."""
def test_retraction_uses_engine_transfer_index_domains(self):
cache = object.__new__(UnifiedRadixCache)
cache.is_swa_enabled = True
@@ -2087,19 +1941,6 @@ class TestUnifiedRadixCacheActionRouting(CustomTestCase):
5, ComponentType.SWA, swa_value
)
def test_apply_cache_actions_applies_each_in_order(self):
cache = mock.MagicMock()
first = UpdatePendingBackupOnNodeSplit(
ack_id=1, old_node_id=1, new_node_id=2, new_child_node_id=1
)
second = UpdatePendingBackupOnNodeSplit(
ack_id=2, old_node_id=3, new_node_id=4, new_child_node_id=3
)
UnifiedRadixCache._apply_cache_actions(cache, [first, second])
cache._apply_cache_action.assert_has_calls(
[mock.call(first), mock.call(second)]
)
def test_eviction_free_failure_still_drains_actions_in_order(self):
cache = object.__new__(UnifiedRadixCache)
first = UpdatePendingBackupOnNodeSplit(
@@ -2147,7 +1988,6 @@ class TestUnifiedRadixCacheActionRouting(CustomTestCase):
def apply(cache, action):
UnifiedRadixCache._apply_cache_action(cache, action)
# split X(10) -> [A(11), X(10)], then fragment A(11) -> [B(12), A(11)]
first = UpdatePendingBackupOnNodeSplit(
ack_id=7, old_node_id=10, new_node_id=11, new_child_node_id=10
)
@@ -2155,7 +1995,6 @@ class TestUnifiedRadixCacheActionRouting(CustomTestCase):
ack_id=7, old_node_id=11, new_node_id=12, new_child_node_id=11
)
# in list order, the publish list threads through both replaces
cache = make_cache()
apply(cache, first)
apply(cache, second)
@@ -2169,8 +2008,8 @@ class TestUnifiedRadixCacheActionRouting(CustomTestCase):
self.assertEqual(receiver.call_args_list, [mock.call(first), mock.call(second)])
self.assertEqual(cache.linker.replace_pending_offload_node.call_count, 2)
# reversed order silently drops the second fragment - documents why
# emission order must be preserved end to end
# Reversed order silently drops the second fragment, so emission order
# must be preserved end to end.
cache = make_cache()
apply(cache, second)
apply(cache, first)
@@ -2195,8 +2034,7 @@ class TestUnifiedRadixCacheActionRouting(CustomTestCase):
class TestSWADraftHistory(SWADraftHistoryFixtures, CustomTestCase):
"""A widened draft needs SWA history before the target window. A running
checkpoint must lock that history, and an incomplete cached window must
not be reused even when the target alone could read it. The Rust rerun module
runs these contracts against both Python and Rust TreeCore.
not be reused even when the target alone could read it.
"""
cfg = CacheConfig(
@@ -2271,15 +2109,10 @@ class TestSWADraftHistory(SWADraftHistoryFixtures, CustomTestCase):
class TestSWAWindowUnderBigramKey(SWAWindowUnderBigramKeyFixtures, CustomTestCase):
"""`checkpoint` has to leave the leaf it inserts holding a full
sliding window of live SWA. Otherwise the match that follows the insert
refuses that leaf, `cache_protected_len` never advances, and the next insert
frees KV the tree already owns as if it were the request's duplicate.
An EAGLE bigram key holds one entry less than the tokens it spans, so the
leaf stops at page_floor(len(key)) rather than page_floor(seq_len), a page
lower. The eviction frontier has to be measured against the key.
"""
"""checkpoint must leave its leaf with a full live SWA window, else
cache_protected_len stalls and the next insert frees tree-owned KV. EAGLE
bigram keys hold one entry fewer than seq_len, so measure the frontier
against the key."""
cfg = CacheConfig(
page_size=4,
@@ -1,12 +1,8 @@
"""HiCache device <-> host backup, demotion and eviction tests for UnifiedRadixCache."""
import unittest
from array import array
from dataclasses import replace
from unittest import mock
import torch
from sglang.srt.mem_cache.base_prefix_cache import (
EvictParams,
InitLoadBackParams,
@@ -18,10 +14,8 @@ from sglang.srt.mem_cache.base_prefix_cache import (
from sglang.srt.mem_cache.radix_cache import RadixKey
from sglang.srt.mem_cache.unified_cache.cache_action import (
BackupKV,
FreeDeviceKV,
)
from sglang.srt.mem_cache.unified_cache.components.base import (
CacheTransferPhase,
ComponentType,
)
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
@@ -36,7 +30,6 @@ from sglang.test.kits.unified_radix_cache.config import (
requires_mamba,
requires_swa,
requires_swa_and_mamba,
requires_swa_only,
)
from sglang.test.kits.unified_radix_cache.fixtures import (
build_fixture,
@@ -158,8 +151,7 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
@requires_hicache
def test_hicache_write_back_evict_drops_unbacked_leaf_when_host_full(self):
"""Write-back eviction will keep freeing device KV when the host pool
is exhausted and host eviction cannot free space to prevent OOM."""
"""Write-back eviction keeps freeing device KV when the host pool is full."""
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
self._init_hicache(cache, write_policy="write_back")
@@ -179,7 +171,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
result = cache.evict(EvictParams(num_tokens=len(seq)))
self.assertGreaterEqual(result.num_tokens_evicted, len(seq))
# The chain is gone entirely: no device hit, no host hit.
m = match_prefix(cache, seq)
self.assertEqual(m.device_prefix_len, 0)
self.assertEqual(m.host_hit_length, 0)
@@ -187,9 +178,8 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
@requires_hicache
def test_hicache_write_back_drop_respects_pins_then_frees_subtree(self):
"""The host-pressure drop fallback must decline while any node in the
unbacked subtree is host-pinned, then reclaim the whole subtree --
including a demoted child's host backup -- once unpinned."""
"""The host-pressure drop fallback declines while any node in the unbacked
subtree is host-pinned, then reclaims the whole subtree once unpinned."""
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
self._init_hicache(cache, write_policy="write_back")
host_pool = cache.cache_controller.mem_pool_host
@@ -217,13 +207,11 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
# From here every backup fails (controller.write returns None), so
# each evict() attempts the drop fallback on the parent.
with mock.patch.object(cache.cache_controller, "write", return_value=None):
# Pinned subtree root: drop declines, chain stays intact.
parent_lock_params = cache.inc_host_lock_ref(parent).to_dec_params()
result = cache.evict(EvictParams(num_tokens=len(parent_seq)))
self.assertEqual(result.num_tokens_evicted, 0)
cache.dec_host_lock_ref(parent, parent_lock_params)
# Pinned host-only descendant: drop declines as well.
child_lock_params = cache.inc_host_lock_ref(child).to_dec_params()
result = cache.evict(EvictParams(num_tokens=len(parent_seq)))
self.assertEqual(result.num_tokens_evicted, 0)
@@ -231,7 +219,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
self.assertEqual(m.device_prefix_len, len(parent_seq))
cache.dec_host_lock_ref(child, child_lock_params)
# Unpinned: the subtree drops and the child's host slots return.
result = cache.evict(EvictParams(num_tokens=len(parent_seq)))
self.assertGreaterEqual(result.num_tokens_evicted, len(parent_seq))
self.assertEqual(host_pool.available_size(), baseline_host)
@@ -244,8 +231,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
lambda cfg: cfg == CacheConfig(), "single Full page-size-1 ownership regression"
)
def test_hicache_split_preserves_inflight_full_host_pin(self):
"""A split prefix keeps an outstanding storage backup's host ownership."""
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
self._init_hicache(cache, write_policy="write_back")
host_pool = cache.cache_controller.mem_pool_host
@@ -266,8 +251,8 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
self.assertEqual(node_host_lock_ref(cache, split_parent, ComponentType.FULL), 1)
self.assertEqual(node_host_lock_ref(cache, leaf, ComponentType.FULL), 1)
# Before the fix the prefix had host_lock_ref == 0 and these slots were
# reclaimed while the storage thread could still be reading them.
# The prefix must hold a host lock, else these slots are reclaimed while
# the storage thread still reads them.
self.assertEqual(cache.evict_host(len(split_host)), 0)
# A lock acquired after the split covers only the suffix and gets a new
@@ -293,7 +278,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
cache.sanity_check()
def test_hicache_split_preserves_inflight_swa_host_pin(self):
"""A split SWA host segment stays pinned and releases without leaks."""
expected_cfg = CacheConfig(
page_size=1,
components=(ComponentType.FULL, ComponentType.SWA),
@@ -321,9 +305,8 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
lock_params.get_lock_uuid(ComponentType.SWA, lock_host=True)
)
# Diverge inside the locked SWA window. The leaf handle stays on the
# suffix while the lock boundary and copied host refs move to the new
# prefix, so neither half may be reclaimed by component host eviction.
# Diverging inside the locked SWA window moves the lock boundary and copied
# host refs to the new prefix; neither half may be reclaimed.
self._insert(cache, allocator, req_to_token_pool, [1, 2, 3, 4, 5, 6, 9, 10])
split_parent = node_parent(cache, leaf)
self.assertNotEqual(split_parent, parent_before_split)
@@ -339,9 +322,8 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
@requires_hicache
def test_hicache_evict_device_leaf_aborts_demote_when_backup_fails(self):
"""The tree op defers a write-back victim's demote: it returns a BackupKV
without freeing device, and _demote requires a completed backup, so a
failed backup leaves the node recoverable on device."""
"""The tree op defers a write-back victim's demote, so a failed backup
leaves the node recoverable on device."""
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
self._init_hicache(cache, write_policy="write_back")
ct = ComponentType.FULL
@@ -354,7 +336,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
self.assertFalse(cache.tree_core.is_backuped(node))
self.assertFalse(cache.tree_core.is_full_device_evicted(node))
# Tree op defers: returns a BackupKV and leaves the node on device (no demote).
leaf_result = cache.tree_core.evict_device_leaf(node, is_write_back=True)
self.assertIsNotNone(leaf_result.backup_kv)
cache._free_values(leaf_result.device_frees, leaf_result.host_frees)
@@ -369,33 +350,8 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
cache.sanity_check()
@requires_hicache
def test_hicache_evict_to_host(self):
cache, allocator, req_to_token_pool = self._build_hicache_fixture()
seq = self._make_seq(1, 2)
self._insert(cache, allocator, req_to_token_pool, seq)
m = match_prefix(cache, seq)
node = m.last_device_node
self._backup_node(cache, node)
self.assertTrue(cache.tree_core.is_backuped(node))
result = cache.evict(EvictParams(num_tokens=len(seq)))
self.assertGreaterEqual(result.num_tokens_evicted, len(seq))
self.assertTrue(cache.tree_core.is_full_device_evicted(node))
self.assertTrue(cache.tree_core.is_backuped(node))
self.assertIsNone(node_device_value(cache, node, ComponentType.FULL))
self.assertIsNotNone(node_host_value(cache, node, ComponentType.FULL))
self.assertFalse(cache.tree_core.is_device_evictable_leaf(node))
self.assertTrue(cache.tree_core.is_host_evictable_leaf(node))
cache.sanity_check()
@requires_hicache
def test_hicache_partial_match_splits_evicted_backed_up_node(self):
"""Partial matches on host-only nodes must keep the host prefix usable."""
cache, allocator, req_to_token_pool = self._build_hicache_fixture()
ps = self.cfg.page_size
seq = self._make_seq(1, 4)
@@ -428,7 +384,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
self.assertNotEqual(split_parent, cache.root_node_handle())
self.assertTrue(cache.tree_core.is_full_device_evicted(split_parent))
self.assertTrue(cache.tree_core.is_backuped(split_parent))
# root -> split_parent keys reconstruct expected_prefix
prefix_tokens: list[int] = []
chain = []
cur = split_parent
@@ -438,7 +393,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
for n in reversed(chain):
prefix_tokens.extend(node_token_ids(cache, n))
self.assertEqual(prefix_tokens, expected_prefix)
# the diverged suffix stays as evicted+backuped descendant(s)
suffix_tokens: list[int] = []
cur = split_parent
while children := node_children(cache, cur):
@@ -462,6 +416,10 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
self._backup_node(cache, node)
cache.evict(EvictParams(num_tokens=len(seq)))
self.assertTrue(cache.tree_core.is_backuped(node))
self.assertIsNone(node_device_value(cache, node, ComponentType.FULL))
self.assertIsNotNone(node_host_value(cache, node, ComponentType.FULL))
self.assertFalse(cache.tree_core.is_device_evictable_leaf(node))
self.assertTrue(cache.tree_core.is_full_device_evicted(node))
self.assertTrue(cache.tree_core.is_host_evictable_leaf(node))
@@ -483,7 +441,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
self.assertFalse(cache.tree_core.is_backuped(node))
self.assertFalse(cache.tree_core.is_full_device_evicted(node))
# Backup failure falls back to dropping the unlocked subtree.
with mock.patch.object(cache, "_execute_and_commit_kv_backup", return_value=0):
result = cache.evict(EvictParams(num_tokens=len(seq)))
@@ -497,7 +454,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
lambda cfg: not cfg.has_mamba, "SWA-only path keeps the split setup simple"
)
def test_hicache_write_through_offloads_swa_split_leaf(self):
ps = self.cfg.page_size
cache, allocator, _ = build_fixture(self.cfg)
self._init_hicache(cache)
@@ -525,7 +481,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
cache.sanity_check()
def test_hicache_evict_to_host_updates_aux_lru(self):
"""Aux components (MAMBA / SWA) move from device LRU to host LRU on D->H eviction."""
aux_types = [
ct
for ct in (ComponentType.MAMBA, ComponentType.SWA)
@@ -556,9 +511,8 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
@requires_mamba
def test_hicache_write_back_internal_mamba_evict_demotes_state(self):
"""write_back: an internal node's tombstoned mamba state is demoted
to host, keeping the node a valid match boundary so the KV beneath
it stays servable."""
"""write_back: an internal node's tombstoned mamba state is demoted to host,
keeping the node a valid match boundary."""
cache, req_to_token_pool, seq_a = self._build_internal_mamba_fixture(
"write_back"
)
@@ -638,8 +592,8 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
# The backed parent stops the Full backup chain, while the SWA window
# reaches the unbacked grandparent and marks it under the anchor's ack.
insert(3 * page)
# Queue the asynchronous window backup explicitly: write-back insertion
# no longer publishes it, but Mamba eviction must still respect its ack.
# Queue the asynchronous window backup explicitly; Mamba eviction must
# respect its ack.
cache._execute_and_commit_kv_backup(BackupKV(node_ids=[anchor]))
self.assertEqual(
cache.ongoing_write_through[anchor].publish_node_ids, nodes[:3]
@@ -666,7 +620,7 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
@requires_mamba
def test_hicache_write_through_internal_mamba_evict_keeps_drop(self):
"""Non-write_back policies keep the legacy tombstone-and-drop."""
"""Non-write_back policies tombstone and drop."""
cache, _, seq_a = self._build_internal_mamba_fixture("write_through")
cache.evict(EvictParams(num_tokens=0, mamba_num=10))
@@ -675,15 +629,14 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
self.assertEqual(
m.device_prefix_len,
0,
"write_through keeps the legacy drop: frontier capped at root",
"write_through keeps the drop: frontier capped at root",
)
cache.sanity_check()
@requires_swa
def test_hicache_write_back_internal_swa_evict_demotes_kv(self):
"""write_back: an internal node's tombstoned SWA KV is demoted to
host, keeping the node a valid match boundary so the KV beneath it
stays servable instead of losing a window of matchable prefix."""
"""write_back: an internal node's tombstoned SWA KV is demoted to host,
keeping the node a valid match boundary."""
cache, req_to_token_pool, seq_a, seq_b = self._build_internal_swa_fixture(
"write_back"
)
@@ -753,44 +706,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
self.assertEqual(result.last_host_node, result.last_device_node)
self.assertEqual(result.host_hit_length, 0)
@requires_swa_only
@requires(
lambda cfg: cfg.sliding_window_size > cfg.page_size,
"the window must reach past the leaf's own page",
)
def test_swa_backup_collector_is_shared_by_both_call_sites(self):
"""needs_incremental_backup and the BACKUP_HOST transfer read one
collector: cache mode walks the window past a host-backed target to
its device-only ancestor, buffer mode stages the target alone."""
# This test exercises the Python component's collector directly.
cache, allocator, req_to_token_pool = self._build_hicache_fixture(
tree_core_backend="python"
)
chain = self._build_chain_pages(cache, allocator, req_to_token_pool, 2)
if len(chain) < 2:
self.skipTest("chain too short")
parent, leaf = chain[-2], chain[-1]
swa = cache.components[ComponentType.SWA]
leaf_node = cache.tree_core.node_by_id(leaf)
cache.tree_core.set_component_host_value_raw(
leaf,
ComponentType.SWA,
node_device_value(cache, leaf, ComponentType.SWA).clone(),
)
self.assertTrue(swa.needs_incremental_backup(leaf_node))
xfer = cache.tree_core.build_hicache_transfers(
ComponentType.SWA, leaf, CacheTransferPhase.BACKUP_HOST
)[0]
self.assertEqual(xfer.nodes_to_load, [parent])
cache.tree_core.set_host_memory_buffer_only()
self.assertTrue(swa.needs_incremental_backup(leaf_node))
xfer = cache.tree_core.build_hicache_transfers(
ComponentType.SWA, leaf, CacheTransferPhase.BACKUP_HOST
)[0]
self.assertEqual(xfer.nodes_to_load, [leaf])
def test_evict_host_drains_freed_host_values_to_the_pools(self):
cache, allocator, req_to_token_pool = self._build_hicache_fixture()
chain = self._build_chain_pages(cache, allocator, req_to_token_pool, 3)
@@ -807,7 +722,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
ComponentType.MAMBA: "mamba_pool_host",
}
for ct in cache.tree_components:
# set by the hicache assembler for every enabled component
pool = getattr(cache, host_pool_attr[ct])
if pool is None:
continue
@@ -815,7 +729,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
evicted = cache.evict_host(pool.size, ct)
if evicted == 0:
continue
# the drain must return every freed host value to the pool
self.assertGreater(pool.available_size(), available_before)
@requires_full_mamba_ps1
@@ -841,38 +754,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
)
self.assertEqual(result.host_hit_length, node_key_length(cache, leaf))
@requires_full_mamba_ps1
def test_mamba_has_host_value_only_predicate(self):
"""Needs a slot only when mamba is host-only (device evicted, host backed up)."""
cache, allocator, req_to_token_pool = self._build_hicache_fixture()
chain = self._build_chain_pages(cache, allocator, req_to_token_pool, 3)
if not chain:
self.skipTest("chain too short")
node = chain[-1]
dev = torch.tensor([0], dtype=torch.int64)
host = torch.tensor([0], dtype=torch.int64)
# host-only: device evicted, host backup present -> restore needs a slot
cache.tree_core.set_component_device_value_raw(node, ComponentType.MAMBA, None)
cache.tree_core.set_component_host_value_raw(node, ComponentType.MAMBA, host)
self.assertTrue(
cache.tree_core.component_has_host_value_only(node, ComponentType.MAMBA)
)
# device + host: device value present -> no restore, no slot (the D+H bug case)
cache.tree_core.set_component_device_value_raw(node, ComponentType.MAMBA, dev)
self.assertFalse(
cache.tree_core.component_has_host_value_only(node, ComponentType.MAMBA)
)
# device-only / neither: nothing to restore
cache.tree_core.set_component_host_value_raw(node, ComponentType.MAMBA, None)
self.assertFalse(
cache.tree_core.component_has_host_value_only(node, ComponentType.MAMBA)
)
cache.tree_core.set_component_device_value_raw(node, ComponentType.MAMBA, None)
self.assertFalse(
cache.tree_core.component_has_host_value_only(node, ComponentType.MAMBA)
)
@requires_full_swa_ps1
def test_hicache_swa_host_best_match_keeps_device_anchor(self):
cache, allocator, req_to_token_pool = self._build_hicache_fixture()
@@ -919,25 +800,16 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
@requires_full_swa_ps1_window4
def test_hicache_swa_backup_window_stops_at_pending_ancestor(self):
cache, allocator, req_to_token_pool = self._build_hicache_fixture()
chain = self._build_chain_pages(cache, allocator, req_to_token_pool, 3)
if len(chain) < 3:
self.skipTest("chain collapsed below the pending ancestor test")
c = chain[-1]
c_swa = node_device_value(cache, c, ComponentType.SWA).clone()
# First transfer: publish Full for C only. C's SWA is a device tombstone
# (decode-evicted, never backed up), so the write-through ack stays
# pending on a node the SWA backup window has nothing to send for.
cache.tree_core.set_component_device_value_raw(c, ComponentType.SWA, None)
if cache.tree_core.is_node_in_device_lru(c, ComponentType.SWA):
cache.tree_core.remove_node_from_device_lru(c, ComponentType.SWA)
cache.tree_core.set_component_evictable_size(
ComponentType.SWA,
cache.tree_core.component_evictable_size(ComponentType.SWA) - len(c_swa),
)
tokens = self._match_tokens_for_chain(cache, chain)
# C's SWA is decode-evicted before it is ever backed up, so the
# write-through of C publishes Full only and its ack stays pending on a
# node the SWA backup window has nothing to send for.
self._evict_device_component(cache, c, ComponentType.SWA)
self.assertGreater(
cache._execute_and_commit_kv_backup(BackupKV(node_ids=[c])),
0,
@@ -945,25 +817,19 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
self.assertEqual(cache.tree_core.get_write_through_pending_id(c), c)
self.assertIsNotNone(node_host_value(cache, c, ComponentType.FULL))
self.assertIsNone(node_host_value(cache, c, ComponentType.SWA))
# SWA is reconstructed on device before the first ack, the way a
# load-back commit stores it: under the pending segment lock the value
# counts as protected until the ack releases it. The next incremental
# SWA backup must treat C as the boundary and back up only the newly
# inserted descendant.
cache.tree_core.set_component_device_value(c, ComponentType.SWA, c_swa)
tokens = self._match_tokens_for_chain(cache, chain)
# A request re-inserting C's tokens recovers its SWA on device before
# the first ack. The next incremental SWA backup must treat C as the
# boundary and back up only the newly inserted descendant.
self._insert(cache, allocator, req_to_token_pool, tokens)
self.assertIsNotNone(node_device_value(cache, c, ComponentType.SWA))
next_tokens = tokens + self._make_seq(9000, 1)
cache.write_through_threshold = 1
self._insert(cache, allocator, req_to_token_pool, next_tokens)
d = match_prefix(cache, next_tokens).last_device_node
self.assertEqual(cache.tree_core.get_write_through_pending_id(c), c)
self.assertEqual(cache.tree_core.get_write_through_pending_id(d), d)
self.assertIsNone(node_host_value(cache, c, ComponentType.SWA))
self.assertIsNotNone(node_host_value(cache, d, ComponentType.SWA))
cache.writing_check(write_back=True)
self.assertIsNone(cache.tree_core.get_write_through_pending_id(c))
self.assertIsNone(cache.tree_core.get_write_through_pending_id(d))
@@ -974,59 +840,39 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
lambda cfg: not cfg.has_mamba,
"SWA-only path keeps the chain construction simple",
)
def test_hicache_swa_finalize_match_result(self):
"""finalize_match_result accumulates host_value lengths of SWA tombstones
within the trailing sliding window into ``swa_host_hit_length``. Out-of-window
tombstones and chains fully on device must leave ``swa_host_hit_length`` at 0.
``host_hit_length`` is Full-KV only and is never written by SWA.
"""
cache, _, _, chain, window_pages = self._swa_finalize_setup()
leaf = chain[-1]
def test_hicache_swa_host_hit_counts_tombstones_inside_the_window(self):
"""A host-backed SWA tombstone inside the trailing sliding window stops
the device match before it: the Full-backed tail from the tombstone on
is a host hit, and the tombstone's own SWA counts in
``swa_host_hit_length``."""
ps = self.cfg.page_size
cases = [
("all_on_device", None, 0),
("tombstone_in_window", chain[-window_pages], ps),
("tombstone_outside_window", chain[-(window_pages + 1)], 0),
]
for name, victim, expected in cases:
window_pages = (self.cfg.sliding_window_size + ps - 1) // ps
for name, victim in (
("all_on_device", None),
("tombstone_in_window", -window_pages),
("tombstone_outside_window", -(window_pages + 1)),
):
with self.subTest(name):
# Reset SWA state for each subcase.
for n in chain:
device_value = node_device_value(cache, n, ComponentType.SWA)
host_value = node_host_value(cache, n, ComponentType.SWA)
if device_value is None and host_value is not None:
cache.tree_core.set_component_device_value_raw(
n, ComponentType.SWA, host_value.clone()
)
cache, allocator, req_to_token_pool = self._build_hicache_fixture()
chain = self._build_chain_pages(
cache, allocator, req_to_token_pool, window_pages + 2
)
if len(chain) <= window_pages:
self.skipTest("chain collapsed below the window length")
tokens = self._match_tokens_for_chain(cache, chain)
self._backup_node(cache, chain[-1])
host_tail, swa_host = 0, 0
if victim is not None:
cache.tree_core.set_component_device_value_raw(
victim, ComponentType.SWA, None
self._evict_device_component(
cache, chain[victim], ComponentType.SWA
)
result = MatchResult(
device_prefix_len=0,
last_device_node=leaf,
last_host_node=leaf,
best_match_node=leaf,
host_hit_length=0,
cache_protected_len=7,
cache_actions=(FreeDeviceKV(indices=[]),),
)
result = cache.tree_core.finalize_component_match_result(
ComponentType.SWA,
result=result,
params=MatchPrefixParams(
key=RadixKey(array("q", self._make_seq(1, 1)))
),
value_chunks=[],
best_value_len=0,
)
self.assertEqual(result.host_hit_length, 0)
self.assertEqual(result.swa_host_hit_length, expected)
self.assertEqual(result.cache_protected_len, 7)
self.assertEqual(result.cache_actions, (FreeDeviceKV(indices=[]),))
if name == "tombstone_in_window":
host_tail = sum(node_key_length(cache, n) for n in chain[victim:])
swa_host = node_key_length(cache, chain[victim])
result = match_prefix(cache, tokens)
self.assertEqual(result.device_prefix_len, len(tokens) - host_tail)
self.assertEqual(result.host_hit_length, host_tail)
self.assertEqual(result.swa_host_hit_length, swa_host)
def test_hicache_swa_finalize_anchored_on_best_match_node(self):
cache, _, _, y, x, _ = self._swa_anchor_setup()
@@ -1057,62 +903,41 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
)
def test_hicache_swa_temp_lock_does_not_release_restored_tombstone(self):
"""A temporary scheduler lock that skipped a SWA tombstone must not
release later load-back/request locks after the tombstone is restored.
"""
cache, allocator, _, chain, _ = self._swa_finalize_setup()
release later load-back/request locks after the tombstone is restored."""
cache, allocator, req_to_token_pool = self._build_hicache_fixture()
chain = self._build_chain_pages(cache, allocator, req_to_token_pool, 3)
if len(chain) < 3:
self.skipTest("chain too short")
leaf = chain[-1]
tombstone = leaf
old_swa = node_device_value(cache, tombstone, ComponentType.SWA)
self.assertIsNotNone(old_swa)
tokens = self._match_tokens_for_chain(cache, chain)
self._backup_node(cache, leaf)
self.assertIsNotNone(node_host_value(cache, leaf, ComponentType.SWA))
self._evict_device_component(cache, leaf, ComponentType.SWA)
self.assertIsNone(node_device_value(cache, leaf, ComponentType.SWA))
cache.tree_core.set_component_device_value_raw(
tombstone, ComponentType.SWA, None
)
cache.tree_core.remove_node_from_device_lru(tombstone, ComponentType.SWA)
cache.tree_core.insert_node_into_host_lru(tombstone, ComponentType.SWA)
cache.tree_core.set_component_evictable_size(
ComponentType.SWA,
cache.tree_core.component_evictable_size(ComponentType.SWA) - len(old_swa),
)
def swa_ref():
return node_device_lock_ref(cache, leaf, ComponentType.SWA)
temp_lock = cache.inc_lock_ref(leaf)
self.assertEqual(node_device_lock_ref(cache, tombstone, ComponentType.SWA), 1)
xfer = cache.tree_core.build_hicache_transfers(
ComponentType.SWA, leaf, CacheTransferPhase.LOAD_BACK
)[0]
new_swa = allocator.swa_attn_allocator.alloc(int(xfer.host_indices.numel()))
self.assertIsNotNone(new_swa)
xfer.device_indices = new_swa
load_actions = []
cache.tree_core.commit_hicache_transfers(
leaf,
CacheTransferPhase.LOAD_BACK,
{ComponentType.SWA: [xfer]},
cache_actions=load_actions,
)
cache._apply_cache_actions(load_actions)
load_back_lock = cache.inc_lock_ref(leaf)
self.assertEqual(swa_ref(), 1)
self._load_back_through_admission(cache, req_to_token_pool, tokens)
self.assertIsNotNone(node_device_value(cache, leaf, ComponentType.SWA))
request_lock = cache.inc_lock_ref(leaf)
self.assertEqual(node_device_lock_ref(cache, tombstone, ComponentType.SWA), 3)
self.assertEqual(swa_ref(), 3)
cache.dec_lock_ref(leaf, temp_lock.to_dec_params())
self.assertEqual(node_device_lock_ref(cache, tombstone, ComponentType.SWA), 2)
cache.dec_lock_ref(leaf, load_back_lock.to_dec_params())
self.assertEqual(swa_ref(), 2)
self._finish_pending_loads(cache)
self._release_ongoing_load_back_locks(cache)
cache.dec_lock_ref(leaf, request_lock.to_dec_params())
self.assertEqual(node_device_lock_ref(cache, tombstone, ComponentType.SWA), 0)
self.assertEqual(swa_ref(), 0)
cache.sanity_check()
@requires_hicache
def test_hicache_full_temp_lock_covers_evicted_anchor_and_mirrors_on_release(
self,
):
"""Segment locks count the evicted anchor too (no skip receipts), so
a value restored mid-hold stays correctly attributed: each release
takes back exactly its own ref regardless of interleaved holders.
"""
"""Segment locks count the evicted anchor too (no skip receipts), so a value
restored mid-hold stays correctly attributed."""
ps = self.cfg.page_size
if 3 * ps > self.cfg.kv_size // 2:
self.skipTest("kv_size too small")
@@ -1120,53 +945,26 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
chain = self._build_chain_pages(cache, allocator, req_to_token_pool, 3)
if len(chain) < 3:
self.skipTest("chain too short")
a, y, anchor = chain
self._simulate_backup_tree(cache)
anchor = chain[-1]
self._backup_node(cache, anchor)
cache.evict(EvictParams(num_tokens=node_key_length(cache, anchor)))
self.assertTrue(cache.tree_core.is_full_device_evicted(anchor))
anchor_value = node_device_value(cache, anchor, ComponentType.FULL)
# Simulate the anchor's FULL device eviction: drop the value and take
# its tokens out of the evictable ledger, as a real evict would.
cache.tree_core.set_component_device_value_raw(anchor, ComponentType.FULL, None)
cache.tree_core.set_component_evictable_size(
ComponentType.FULL,
cache.tree_core.component_evictable_size(ComponentType.FULL)
- len(anchor_value),
)
self.assertEqual(node_device_lock_ref(cache, anchor, ComponentType.FULL), 0)
self.assertEqual(node_device_lock_ref(cache, y, ComponentType.FULL), 0)
self.assertEqual(node_device_lock_ref(cache, a, ComponentType.FULL), 0)
def full_refs():
return [node_device_lock_ref(cache, n, ComponentType.FULL) for n in chain]
self.assertEqual(full_refs(), [0, 0, 0])
temp_lock = cache.inc_lock_ref(anchor)
self.assertEqual(node_device_lock_ref(cache, anchor, ComponentType.FULL), 1)
self.assertEqual(node_device_lock_ref(cache, y, ComponentType.FULL), 1)
self.assertEqual(node_device_lock_ref(cache, a, ComponentType.FULL), 1)
# Restore the value mid-hold: a value materialized under lock is
# protected until the last release, exactly as a load-back credits it.
cache.tree_core.set_component_device_value_raw(
anchor, ComponentType.FULL, anchor_value
)
cache.tree_core.set_component_protected_size(
ComponentType.FULL,
cache.tree_core.component_protected_size(ComponentType.FULL)
+ len(anchor_value),
)
self.assertEqual(full_refs(), [1, 1, 1])
# Restore the anchor mid-hold.
self._load_back_node(cache, anchor)
second_lock = cache.inc_lock_ref(anchor)
self.assertEqual(node_device_lock_ref(cache, anchor, ComponentType.FULL), 2)
self.assertEqual(node_device_lock_ref(cache, y, ComponentType.FULL), 2)
self.assertEqual(node_device_lock_ref(cache, a, ComponentType.FULL), 2)
self.assertEqual(full_refs(), [2, 2, 2])
cache.dec_lock_ref(anchor, temp_lock.to_dec_params())
self.assertEqual(node_device_lock_ref(cache, anchor, ComponentType.FULL), 1)
self.assertEqual(node_device_lock_ref(cache, y, ComponentType.FULL), 1)
self.assertEqual(node_device_lock_ref(cache, a, ComponentType.FULL), 1)
self.assertEqual(full_refs(), [1, 1, 1])
cache.dec_lock_ref(anchor, second_lock.to_dec_params())
self.assertEqual(node_device_lock_ref(cache, anchor, ComponentType.FULL), 0)
self.assertEqual(node_device_lock_ref(cache, y, ComponentType.FULL), 0)
self.assertEqual(node_device_lock_ref(cache, a, ComponentType.FULL), 0)
self.assertEqual(full_refs(), [0, 0, 0])
cache.sanity_check()
@requires_mamba
@requires(
@@ -1175,54 +973,32 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
)
def test_hicache_mamba_temp_lock_does_not_release_restored_tombstone(self):
"""A temporary scheduler lock that skipped a Mamba tombstone must not
release later load-back/request locks after the tombstone is restored.
"""
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
release later load-back/request locks after the tombstone is restored."""
cache, allocator, req_to_token_pool = self._build_hicache_fixture()
seq = self._make_seq(1, 2)
self._insert(cache, allocator, req_to_token_pool, seq)
m = match_prefix(cache, seq)
node = m.last_device_node
old_mamba = node_device_value(cache, node, ComponentType.MAMBA)
self.assertIsNotNone(old_mamba)
self._simulate_backup(cache, node)
node = match_prefix(cache, seq).last_device_node
self._backup_node(cache, node)
self.assertIsNotNone(node_host_value(cache, node, ComponentType.MAMBA))
self._evict_device_component(cache, node, ComponentType.MAMBA)
self.assertIsNone(node_device_value(cache, node, ComponentType.MAMBA))
cache.tree_core.set_component_device_value_raw(node, ComponentType.MAMBA, None)
cache.tree_core.remove_node_from_device_lru(node, ComponentType.MAMBA)
cache.tree_core.insert_node_into_host_lru(node, ComponentType.MAMBA)
cache.tree_core.set_component_evictable_size(
ComponentType.MAMBA,
cache.tree_core.component_evictable_size(ComponentType.MAMBA)
- len(old_mamba),
)
def mamba_ref():
return node_device_lock_ref(cache, node, ComponentType.MAMBA)
temp_lock = cache.inc_lock_ref(node)
self.assertEqual(node_device_lock_ref(cache, node, ComponentType.MAMBA), 1)
xfer = cache.tree_core.build_hicache_transfers(
ComponentType.MAMBA, node, CacheTransferPhase.LOAD_BACK
)[0]
new_mamba = req_to_token_pool.mamba_allocator.alloc(1)
self.assertIsNotNone(new_mamba)
xfer.device_indices = new_mamba
cache.tree_core.commit_hicache_transfers(
node,
CacheTransferPhase.LOAD_BACK,
{ComponentType.MAMBA: [xfer]},
cache_actions=[],
)
load_back_lock = cache.inc_lock_ref(node)
self.assertEqual(mamba_ref(), 1)
self._load_back_through_admission(cache, req_to_token_pool, seq)
self.assertIsNotNone(node_device_value(cache, node, ComponentType.MAMBA))
request_lock = cache.inc_lock_ref(node)
self.assertEqual(node_device_lock_ref(cache, node, ComponentType.MAMBA), 3)
self.assertEqual(mamba_ref(), 3)
cache.dec_lock_ref(node, temp_lock.to_dec_params())
self.assertEqual(node_device_lock_ref(cache, node, ComponentType.MAMBA), 2)
cache.dec_lock_ref(node, load_back_lock.to_dec_params())
self.assertEqual(mamba_ref(), 2)
self._finish_pending_loads(cache)
self._release_ongoing_load_back_locks(cache)
cache.dec_lock_ref(node, request_lock.to_dec_params())
self.assertEqual(node_device_lock_ref(cache, node, ComponentType.MAMBA), 0)
# The commit ran under held locks: the restored value must have been
self.assertEqual(mamba_ref(), 0)
# The restore ran under held locks: the restored value must have been
# credited to protected, or the ledger drifts on the final release.
cache.sanity_check()
@@ -1274,7 +1050,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
"SWA boundary splits deepen the chain non-deterministically",
)
def test_build_backup_kv_action_orders_ancestors_first(self):
"""write-through backs up unbacked ancestors parent-first; write-back only the leaf."""
cache, allocator, req_to_token_pool = self._build_hicache_fixture()
top_seq = self._make_seq(1, 1)
@@ -1293,11 +1068,9 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
self.assertFalse(cache.tree_core.is_backuped(mid))
self.assertFalse(cache.tree_core.is_backuped(top))
# write-through: every unbacked ancestor, top-most first, child last
write_through = cache.tree_core.build_backup_node_ids(leaf, write_back=False)
self.assertEqual(write_through, [top, mid, leaf])
# write-back: only the eviction victim, even with unbacked ancestors
write_back = cache.tree_core.build_backup_node_ids(leaf, write_back=True)
self.assertEqual(write_back, [leaf])
@@ -1306,7 +1079,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
@requires_swa
@requires_hicache
def test_write_back_defers_swa_window_backup_until_eviction(self):
"""Reinserting backed Full nodes must not publish freshly recovered SWA."""
ps = self.cfg.page_size
if self.cfg.sliding_window_size <= 2 * ps:
self.skipTest("window must span past the leaf and its parent to reach c")
@@ -1338,7 +1110,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
self.assertIsNotNone(node_device_value(cache, node, ComponentType.SWA))
self.assertIsNone(node_host_value(cache, node, ComponentType.SWA))
# Two requests in one batch finish at t1 and t2; neither ack drains.
with mock.patch.object(
cache,
"_execute_and_commit_kv_backup",
@@ -1352,7 +1123,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
self.assertIsNone(cache.tree_core.get_write_through_pending_id(node))
self.assertIsNone(node_host_value(cache, node, ComponentType.SWA))
self.assertEqual(cache.cache_controller.write_queue, [])
# Write-back defers the unbacked Full prefix to eviction.
self.assertFalse(cache.tree_core.is_backuped(c))
self.assertFalse(cache.tree_core.is_backuped(d))
@@ -1,5 +1,3 @@
"""Resumable insert walk, returned-value drain and segment lock tests for UnifiedRadixCache."""
import sys
import unittest
from array import array
@@ -134,7 +132,6 @@ class TestResumableInsertWalk(UnifiedRadixCacheFixtures, CustomTestCase):
self._insert(cache, allocator, req_to_token_pool, [1, 2, 3, 4])
self.assertFalse(cache.tree_core.has_ongoing_insert())
# The tree stays usable and the crossing re-fires on the next walk.
self._insert(cache, allocator, req_to_token_pool, [1, 2, 3, 4])
cache.writing_check(write_back=True)
(ancestor,) = node_children(cache, cache.root_node_handle())
@@ -231,7 +228,6 @@ class TestResumableInsertWalk(UnifiedRadixCacheFixtures, CustomTestCase):
self.assertTrue(cache.tree_core.is_backuped(middle))
self.assertFalse(cache.tree_core.is_backuped(top))
# The device insert unevicts the middle and adds the deep leaf.
self._insert(cache, allocator, req_to_token_pool, list(range(1, 13)))
(deep,) = node_children(cache, middle)
@@ -386,11 +382,9 @@ class TestResumableInsertWalkSWA(ResumableInsertWalkSWAFixtures, CustomTestCase)
(window_node,) = node_children(cache, prefix_node)
self.assertIsNone(node_device_value(cache, prefix_node, ComponentType.SWA))
# Re-inserting fully in-window recovers the prefix span's SWA data.
cache.insert(InsertParams(key=key, value=self._alloc(allocator, len(seq))))
self.assertIsNotNone(node_device_value(cache, prefix_node, ComponentType.SWA))
# SWA eviction takes the recovered span and keeps the window leaf.
cache.evict(EvictParams(num_tokens=0, swa_num_tokens=sw))
self.assertIsNone(node_device_value(cache, prefix_node, ComponentType.SWA))
self.assertIsNotNone(node_device_value(cache, window_node, ComponentType.SWA))
@@ -457,14 +451,13 @@ class TestResumableInsertWalkWriteBack(UnifiedRadixCacheFixtures, CustomTestCase
def test_drop_fallback_frees_host_for_later_backups_same_round(self):
"""Host slots reclaimed by the drop fallback itself (an interior
host-only descendant) must be reusable by later write-back backups in
the same eviction round (pre-split freed them inline)."""
the same eviction round."""
if self._skip_unsupported_hicache_test():
return
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
self._init_hicache(cache, write_policy="write_back")
host_pool = cache.cache_controller.mem_pool_host
# Chain 1: unbacked parent -> host-only child -> host-only grandchild.
p1 = self._make_seq(1, 2)
self._insert(cache, allocator, req_to_token_pool, p1)
c1 = p1 + self._make_seq(1000, 2)
@@ -518,8 +511,7 @@ class TestResumableInsertWalkWriteBack(UnifiedRadixCacheFixtures, CustomTestCase
@unittest.skipUnless(torch.cuda.is_available(), "cache fixtures need CUDA")
class TestReturnedValuesDrain(UnifiedRadixCacheFixtures, CustomTestCase):
"""Drain contract of the returned-values eviction API: every tree-core step
result is drained exactly once, in per-component insertion order."""
"""Each tree-core step result is drained exactly once, in insertion order."""
cfg = CacheConfig()
@@ -629,45 +621,6 @@ class TestReturnedValuesDrain(UnifiedRadixCacheFixtures, CustomTestCase):
self.assertIs(device_frees[ComponentType.FULL][0], sentinel)
self.assertIs(host_frees[ComponentType.FULL][0], sentinel)
def test_free_values_frees_in_component_insertion_order(self):
"""Device frees apply before host frees, each in per-component
insertion order — the allocator free-list order the pre-split inline
frees produced."""
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
order = [ComponentType.FULL, ComponentType.SWA, ComponentType.MAMBA]
device_frees = defaultdict(list)
host_frees = defaultdict(list)
for ct in order:
device_frees[ct].append(torch.tensor([1]))
host_frees[ct].append(torch.tensor([2]))
freed = []
fake_components = {
ct: mock.MagicMock(
free_host_values=mock.MagicMock(
side_effect=lambda values, ct=ct: freed.append(("host", ct))
)
)
for ct in order
}
with (
mock.patch.object(
cache,
"_apply_cache_action",
side_effect=lambda action: freed.append(
("device", action.component_type)
),
),
mock.patch.dict(cache.components, fake_components),
):
cache._free_values(device_frees, host_frees)
self.assertEqual(
freed, [("device", ct) for ct in order] + [("host", ct) for ct in order]
)
self.assertFalse(device_frees)
self.assertFalse(host_frees)
def test_free_values_mid_drain_failure_cannot_replay_freed_entries(self):
"""A device free that raises must leave only un-attempted entries in
the dict (no replay of successes) while host frees still drain."""
@@ -723,25 +676,17 @@ class TestReturnedValuesDrain(UnifiedRadixCacheFixtures, CustomTestCase):
@unittest.skipUnless(torch.cuda.is_available(), "cache fixtures need CUDA")
class TestSegmentLockProtocol(SegmentLockProtocolFixtures, CustomTestCase):
"""Segment-lock protocol regressions, replaying the production lock-theft
failure classes (F1/F2) and the split hazards.
The protocol: a lock covers the contiguous node segment
[start, boundary-uuid], counting every node (tombstones included), so a
release needs only the receipt (anchor node, boundary uuid, skipped
components) and any ref==0 met inside the segment is a hard protocol
violation. The replays read the tree through the inspection interface, so
they run unchanged against the Python and Rust cores.
"""
"""A lock covers the node segment [start, boundary-uuid] counting tombstones;
a release needs only its receipt, and any ref==0 inside the segment is a
violation."""
cfg = CacheConfig(
components=(ComponentType.FULL, ComponentType.SWA), sliding_window_size=8
)
def test_rebuilt_tombstone_relock_release_no_theft(self):
"""F1 attribution replay: A locks a window containing a tombstone; the
tombstone is rebuilt and locked by B mid-hold; A's release must leave
B's refs intact (the old skip-set protocol decremented B's lock)."""
"""A locks a window containing a tombstone; B rebuilds and locks it
mid-hold; A's release must leave B's refs intact."""
sw = self.cfg.sliding_window_size
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
seq = self._make_seq(1, 2 * sw)
@@ -794,7 +739,6 @@ class TestSegmentLockProtocol(SegmentLockProtocolFixtures, CustomTestCase):
for n in segment:
self.assertEqual(self._swa_ref(cache, n), 2)
# THE regression: A's release takes back exactly A's refs.
cache.dec_lock_ref(leaf, lock_a.to_dec_params())
for n in segment:
self.assertEqual(self._swa_ref(cache, n), 1)
@@ -821,7 +765,6 @@ class TestSegmentLockProtocol(SegmentLockProtocolFixtures, CustomTestCase):
cache.sanity_check()
def test_root_acquire_receipt_releases_on_root(self):
"""A cold request locks the root, so its receipt is anchored on root."""
cache, _, _ = build_fixture(self.cfg)
root = cache.root_node_handle()
lock = cache.inc_lock_ref(root).to_dec_params()
@@ -830,7 +773,6 @@ class TestSegmentLockProtocol(SegmentLockProtocolFixtures, CustomTestCase):
cache.sanity_check()
def test_release_with_incorrect_root_boundary_fails_loud(self):
"""An incorrect root boundary must fail at the acquired segment's edge."""
sw = self.cfg.sliding_window_size
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
seq = self._make_seq(1, 3 * sw)
@@ -869,8 +811,8 @@ class TestSegmentLockProtocol(SegmentLockProtocolFixtures, CustomTestCase):
)
def test_unanchored_device_release_fails_loud(self):
"""#41261 replay: a receipt that lost its anchor must not walk the
node's segment and drop locks other holders own."""
"""A receipt that lost its anchor must not walk the node's segment and
drop locks other holders own."""
cache, allocator, leaf = self._leaf_with_lock_path()
lock = cache.inc_lock_ref(leaf).to_dec_params()
self._assert_unanchored_release_rejected(
@@ -920,8 +862,6 @@ class TestSegmentLockProtocol(SegmentLockProtocolFixtures, CustomTestCase):
)
def test_unanchored_root_release_fails_loud(self):
"""Every acquire anchors its receipt, root included; an unanchored
receipt lost its anchor upstream even where releasing is a no-op."""
cache, _, _ = build_fixture(self.cfg)
root = cache.root_node_handle()
self._assert_protocol_violation(
@@ -941,8 +881,8 @@ class TestSegmentLockProtocol(SegmentLockProtocolFixtures, CustomTestCase):
)
def test_finish_after_early_release_without_skip_swa_fails_loud(self):
"""F2 replay: retraction-after-early-release used to run a second SWA
walk that stole ancestors' locks; now it dies at the first node."""
"""A finish after early SWA release without skip_swa asserts at the first
node instead of stealing ancestors' locks."""
sw = self.cfg.sliding_window_size
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
seq = self._make_seq(1, 2 * sw)
@@ -955,7 +895,7 @@ class TestSegmentLockProtocol(SegmentLockProtocolFixtures, CustomTestCase):
)
def test_finish_after_early_release_with_skip_swa(self):
"""The correct F2 flow: unlock honors the early release."""
"""Finish with skip_swa honors the early release."""
sw = self.cfg.sliding_window_size
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
seq = self._make_seq(1, 2 * sw)
@@ -983,7 +923,6 @@ class TestSegmentLockProtocol(SegmentLockProtocolFixtures, CustomTestCase):
pre_segment = self._segment(cache, leaf, sw)
pre_full_path = self._full_path(cache, leaf)
# Diverge inside the window to force a split of a locked node.
fork = seq[: len(seq) - sw // 2] + self._make_seq(9000, sw)
self._insert(cache, allocator, req_to_token_pool, fork)
post_segment = self._segment(cache, leaf, sw)
@@ -1018,7 +957,6 @@ class TestSegmentLockProtocol(SegmentLockProtocolFixtures, CustomTestCase):
params = cache.inc_lock_ref(leaf).to_dec_params()
self.assertNotIn(node, cache.tree_core.evictable_device_leaves)
# Full first: its walk still sees the SWA lock, so the leaf stays out.
cache.components[ComponentType.FULL].release_component_lock(node, params)
self.assertNotIn(node, cache.tree_core.evictable_device_leaves)
# The SWA release drops the last lock and must readmit the leaf itself.
@@ -1029,8 +967,7 @@ class TestSegmentLockProtocol(SegmentLockProtocolFixtures, CustomTestCase):
@unittest.skipUnless(torch.cuda.is_available(), "cache fixtures need CUDA")
class TestSegmentLockFuzz(SegmentLockFuzzFixtures, CustomTestCase):
"""Random lock/insert/evict interleavings with the tree's own ledger
recomputation (sanity_check) as the per-step oracle, on both cores."""
"""Random lock/insert/evict interleavings, with sanity_check as the oracle."""
cfg = CacheConfig(
components=(ComponentType.FULL, ComponentType.SWA),
@@ -1051,9 +988,7 @@ class TestSegmentLockFuzz(SegmentLockFuzzFixtures, CustomTestCase):
@unittest.skipUnless(torch.cuda.is_available(), "cache fixtures need CUDA")
class TestSegmentLockFuzzWithMamba(TestSegmentLockFuzz):
"""The FULL+SWA+MAMBA (Inkling) shape: the Mamba opt-out receipt, the
lower-priority cascade on early SWA release, and Mamba evictions all join
the interleavings."""
"""The FULL+SWA+MAMBA shape adds the Mamba opt-out and Mamba evictions."""
cfg = CacheConfig(
components=(ComponentType.FULL, ComponentType.SWA, ComponentType.MAMBA),
@@ -1067,8 +1002,8 @@ class TestSegmentLockFuzzWithMamba(TestSegmentLockFuzz):
class TestStreamingSessionLockLifecycle(
StreamingSessionLockLifecycleFixtures, CustomTestCase
):
"""A streaming session owns its record and prefix lock from the first
turn's row allocation; every move of that lock keeps the tree balanced."""
"""A streaming session's prefix lock keeps the tree balanced through every
move."""
cfg = CacheConfig(
page_size=1,
@@ -1,5 +1,3 @@
"""KV cache event tests for UnifiedRadixCache."""
import unittest
from array import array
from unittest import mock
@@ -174,7 +172,6 @@ class TestUnifiedRadixCacheKVEvents(UnifiedRadixCacheKVEventsFixtures, CustomTes
)
def test_lora_external_ids_separate_adapter_loads(self):
"""External IDs distinguish loads; routers match tokens and public names."""
lora_names, _reqs = make_lora_names(
**{
"uuid-a": "adapter-a",
@@ -207,7 +204,6 @@ class TestUnifiedRadixCacheKVEvents(UnifiedRadixCacheKVEventsFixtures, CustomTes
for h in self._event_hashes(base)
],
)
# Another load of the same public name has distinct external IDs.
self.assertNotEqual(
self._event_hashes(store_then_evict("user|uuid-a2")),
self._event_hashes(lora_a),
@@ -385,7 +381,6 @@ class TestUnifiedRadixCacheKVEvents(UnifiedRadixCacheKVEventsFixtures, CustomTes
self.assertCountEqual(self._event_hashes(removed_cpu), stored_hashes)
def test_hicache_storage_prefetch_publishes_host_only_suffix(self):
"""A storage-prefetch refill has no write-through ack to publish it."""
from sglang.srt.mem_cache.utils import get_hash_str, hash_str_to_int64
cache, _, _ = build_fixture(self.cfg, enable_kv_cache_events=True)
@@ -403,7 +398,6 @@ class TestUnifiedRadixCacheKVEvents(UnifiedRadixCacheKVEventsFixtures, CustomTes
self.assertFalse(result.host_insert_dropped)
self.assertIsNotNone(result.inserted_host_node)
# The recorder coalesces the parent-linked pages into one event.
stored_cpu = self._stored_events(cache, StorageMedium.CPU)
self.assertEqual(len(stored_cpu), 1)
self.assertEqual(list(stored_cpu[0].token_ids), tokens)
@@ -441,7 +435,6 @@ class TestUnifiedRadixCacheKVEvents(UnifiedRadixCacheKVEventsFixtures, CustomTes
[[1, 2], [3, 4]],
)
# Both split fragments must be published as one parent-linked batch.
stored_cpu = self._stored_events(cache, StorageMedium.CPU)
self.assertEqual(len(stored_cpu), 1)
self.assertEqual(list(stored_cpu[0].token_ids), [1, 2, 3, 4])
@@ -1,5 +1,3 @@
"""HiCache load-back, admission and branching tests for UnifiedRadixCache."""
import unittest
from array import array
from dataclasses import replace
@@ -9,7 +7,6 @@ import torch
from sglang.srt.managers.schedule_batch import (
Req,
ReqKvInfo,
)
from sglang.srt.managers.schedule_policy import PrefillAdder
from sglang.srt.mem_cache.base_prefix_cache import (
@@ -141,7 +138,7 @@ class TestUnifiedTreeCoreLoadBackOwnershipBackends(CustomTestCase):
sliding_window_size=4,
)
def test_auxiliary_load_does_not_reuse_full_pending_pin(self):
def test_auxiliary_load_restores_swa_and_rebuilds_mapping(self):
cache, _, _ = build_fixture(self.cfg)
core = cache.tree_core
core.is_write_back = True
@@ -240,7 +237,6 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures):
self._load_back_node(cache, node)
self.assertFalse(cache.tree_core.is_full_device_evicted(node))
self.assertIsNotNone(node_device_value(cache, node, ComponentType.FULL))
# Gather the whole reloaded prefix via match (a leaf may be split).
loaded_indices = cache.path_device_indices(
match_prefix(cache, base).last_device_node
)
@@ -288,7 +284,6 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures):
self.assertEqual(result.swa_branching_seqlen, len(tokens))
self.assertEqual(result.swa_branching_seqlen % self.cfg.page_size, 0)
# Simulate forward producing fresh SWA KV at the branching point.
self._insert(
cache,
allocator,
@@ -352,15 +347,12 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures):
@requires_full_swa_ps1_window4
def test_swa_branching_seqlen_caps_insert_after_forward(self):
cache, _, _ = build_fixture(self.cfg)
cache, _, req_to_token_pool = build_fixture(self.cfg)
swa = cache.components[ComponentType.SWA]
req = mock.Mock(
swa_branching_seqlen=8,
kv=ReqKvInfo(
cache_protected_len=4,
component_evicted_seqlens={ComponentType.SWA: 3},
),
)
req = self._make_req(req_to_token_pool)
req.swa_branching_seqlen = 8
req.kv.cache_protected_len = 4
req.kv.set_evicted_seqlen(ComponentType.SWA, 3)
for is_finished in (False, True):
params = InsertParams(component_evicted_seqlens={ComponentType.MAMBA: 6})
@@ -378,12 +370,7 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures):
cache.tree_core.is_eagle = True
req.kv.cache_protected_len = 4
params = InsertParams()
self.assertEqual(swa.prepare_for_caching_req(req, params, 12, False), 9)
params.key = RadixKey(array("q", range(9)), is_bigram=True)
result = mock.Mock(swa_branch_inserted=False)
swa.commit_insert_component_data(mock.Mock(), False, params, result, [])
self.assertTrue(result.swa_branch_inserted)
self.assertEqual(swa.prepare_for_caching_req(req, InsertParams(), 12, False), 9)
@requires_full_mamba_ps1
def test_mamba_branching_seqlen_disabled_under_hicache(self):
@@ -615,7 +602,6 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures):
self.assertEqual(new_node, match.last_device_node)
self.assertIsNone(node_device_value(cache, leaf, ComponentType.FULL))
self.assertIsNone(node_device_value(cache, leaf, ComponentType.MAMBA))
# A failed load-back must roll back the pre-allocated mamba slot.
self.assertIsNone(req.kv.mamba_pool_idx)
self.assertEqual(
req_to_token_pool.mamba_allocator.available_size(), avail_before
@@ -639,10 +625,8 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures):
)
self._apply_match_to_req(req, match)
# Simulate a request without its own mamba slot so load-back allocates one.
req.kv.mamba_pool_idx = None
avail_before = req_to_token_pool.mamba_allocator.available_size()
# H->D load fails after the mamba slot is pre-allocated -> must free it.
with mock.patch.object(cache.cache_controller, "load", return_value=None):
loaded_len, _ = cache.init_load_back(
InitLoadBackParams(
@@ -677,7 +661,6 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures):
)
self._apply_match_to_req(req, match)
# Simulate a request without its own mamba slot so load-back allocates one.
req.kv.mamba_pool_idx = None
avail_before = req_to_token_pool.mamba_allocator.available_size()
# No device room and eviction frees nothing -> load-back bails after the
@@ -762,36 +745,6 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures):
)
self._release_ongoing_load_back_locks(cache)
@requires_full_mamba_ps1
def test_load_back_success_publishes_fresh_mamba_slot(self):
cache, allocator, req_to_token_pool = self._build_hicache_fixture()
chain = self._build_chain_pages(cache, allocator, req_to_token_pool, 3)
if len(chain) < 3:
self.skipTest("chain too short")
leaf = chain[-1]
self._backup_node(cache, leaf)
cache.evict(EvictParams(num_tokens=node_key_length(cache, leaf)))
self.assertTrue(cache.tree_core.is_full_device_evicted(leaf))
req = self._make_req(req_to_token_pool)
req_to_token_pool.mamba_allocator.free(req.kv.mamba_pool_idx.unsqueeze(0))
req.kv.mamba_pool_idx = None
mamba_avail = req_to_token_pool.mamba_allocator.available_size()
loaded = cache.load_back(leaf, req=req)
self.assertTrue(loaded)
# the successful load must keep the freshly allocated slot published
self.assertIsNotNone(req.kv.mamba_pool_idx)
self.assertIsNotNone(node_device_value(cache, leaf, ComponentType.MAMBA))
# one slot restores the node's mamba value, one is the request's CoW slot
self.assertEqual(
req_to_token_pool.mamba_allocator.available_size(), mamba_avail - 2
)
self._finish_pending_loads(cache)
self._release_ongoing_load_back_locks(cache)
@requires(
lambda cfg: (
cfg.has_mamba
@@ -835,9 +788,8 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures):
)
adder.add_one_req(req, False, None)
# The load pins 12 FULL slots, leaving 8 free. Rechecking the original
# 18-token demand would reject and scheduler cleanup would free the CoW
# destination even though the queued H2D still targets it.
# Rechecking the original 18-token demand would reject, and scheduler
# cleanup would free the CoW destination while the queued H2D targets it.
self.assertEqual(adder.can_run_list, [req])
self.assertEqual(req.prefix_len, 12)
self.assertEqual(req.kv.cache_protected_len, 12)
@@ -858,7 +810,6 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures):
self.skipTest("chain too short")
leaf = chain[-1]
# Stamp the node's mamba state so the host backup carries it.
node_mamba_indices = node_device_value(cache, leaf, ComponentType.MAMBA).clone()
self._fill_mamba_state(req_to_token_pool, node_mamba_indices, marker=11)
expected_temporal, expected_conv = self._snapshot_mamba_state(
@@ -872,10 +823,16 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures):
req = self._make_req(req_to_token_pool)
req_to_token_pool.mamba_allocator.free(req.kv.mamba_pool_idx.unsqueeze(0))
req.kv.mamba_pool_idx = None
mamba_avail = req_to_token_pool.mamba_allocator.available_size()
loaded = cache.load_back(leaf, req=req)
self.assertTrue(loaded)
self.assertIsNotNone(req.kv.mamba_pool_idx)
self.assertIsNotNone(node_device_value(cache, leaf, ComponentType.MAMBA))
# one slot restores the node's mamba value, one is the request's CoW slot
self.assertEqual(
req_to_token_pool.mamba_allocator.available_size(), mamba_avail - 2
)
self._finish_pending_loads(cache)
# The CoW slot must actually hold the backed-up mamba state, not merely exist.
@@ -902,28 +859,23 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures):
req_to_token_pool.mamba_allocator.free(req.kv.mamba_pool_idx.unsqueeze(0))
req.kv.mamba_pool_idx = None
# device value still present -> nothing to prepare even though host-backed
self.assertIsNone(comp.prepare_load_back(leaf, req=req).allocated_mamba_slot)
self.assertIsNone(req.kv.mamba_pool_idx)
cache.evict(EvictParams(num_tokens=node_key_length(cache, leaf)))
self.assertTrue(cache.tree_core.is_full_device_evicted(leaf))
# a request that already owns a slot -> nothing to prepare
req_owned = self._make_req(req_to_token_pool)
self.assertIsNone(
comp.prepare_load_back(leaf, req=req_owned).allocated_mamba_slot
)
# no request -> nothing to prepare
self.assertIsNone(comp.prepare_load_back(leaf, req=None).allocated_mamba_slot)
# fresh request + host-only mamba -> allocates and publishes onto req
prep = comp.prepare_load_back(leaf, req=req)
self.assertIsNotNone(prep.allocated_mamba_slot)
self.assertEqual(int(req.kv.mamba_pool_idx), int(prep.allocated_mamba_slot[0]))
# node without host-backed mamba -> nothing to prepare
req2 = self._make_req(req_to_token_pool)
req_to_token_pool.mamba_allocator.free(req2.kv.mamba_pool_idx.unsqueeze(0))
req2.kv.mamba_pool_idx = None
@@ -987,9 +939,7 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures):
self._simulate_backup_tree(cache)
# Tombstone every chain node on the device side without going through
# the tree-wide eviction loop. This isolates build_hicache_transfers
# from LRU and cascade ordering.
# Tombstone every chain node directly, bypassing the eviction loop.
for n in chain:
cache.tree_core.set_component_device_value_raw(n, ComponentType.FULL, None)
cache.tree_core.set_component_device_value_raw(n, ComponentType.SWA, None)
@@ -1003,7 +953,6 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures):
xfer = transfers[0]
self.assertEqual(xfer.name, PoolName.SWA)
self.assertEqual(len(xfer.nodes_to_load), expected_pages)
# host_indices must cover exactly the expected suffix tokens (>= sw).
self.assertEqual(int(xfer.host_indices.numel()), expected_pages * ps)
self.assertGreaterEqual(int(xfer.host_indices.numel()), sw)
self.assertEqual(xfer.nodes_to_load, chain[-expected_pages:])
@@ -1018,17 +967,11 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures):
"window must span past the leaf to reach the pin",
)
def test_hicache_swa_load_back_rejects_foreign_pinned_window(self):
"""A second load-back must not claim nodes pinned by an in-flight one.
commit_load_back republishes Full device values before the DMA acks, so
a later anchor can claim just the still-host-only SWA window of a
pinned node and hit the commit pin assert. The spec build must degrade
to an empty spec instead (the caller recomputes).
"""
"""A second load-back must not claim nodes pinned by an in-flight one;
the spec build degrades to empty (caller recomputes)."""
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
# commit_load_back pins its source nodes only under write-back; run the
# scenario in that mode so the foreign-pin path is reachable.
# commit_load_back pins its source nodes only under write-back.
cache.is_write_back = True
chain = self._build_chain_pages(cache, allocator, req_to_token_pool, 3)
if len(chain) < 3:
@@ -1058,7 +1001,6 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures):
self.assertEqual(kv_xfer.nodes_to_load, [])
self.assertEqual(comp_xfers, {})
# After the ack unpins, the same spec builds fully.
cache.tree_core.finish_load_back(a)
kv_xfer, comp_xfers = cache.tree_core.build_load_back_spec(b)
self.assertEqual(kv_xfer.nodes_to_load, [b])
@@ -1070,78 +1012,42 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures):
"SWA-only path keeps the chain construction simple",
)
def test_hicache_swa_commit_load_back_rebuilds_mapping(self):
"""LOAD_BACK commit must:
(1) restore SWA cd.value via _restore_device_value (host LRU -> device LRU),
(2) rewrite full_to_swa_index_mapping[full_idx] = new_swa_idx for every
loaded chunk so subsequent SWA reads via translate_loc_from_full_to_swa
return the freshly allocated SWA slot."""
cache, allocator, _, chain, window_pages = self._swa_finalize_setup()
# Tombstone every SWA node in the trailing window.
"""A SWA load-back restores each tombstoned window node's SWA to device
and maps restored Full slots to their SWA slots."""
ps = self.cfg.page_size
window_pages = (self.cfg.sliding_window_size + ps - 1) // ps
cache, allocator, req_to_token_pool = self._build_hicache_fixture()
chain = self._build_chain_pages(
cache, allocator, req_to_token_pool, window_pages + 2
)
if len(chain) <= window_pages:
self.skipTest("chain collapsed below the window length")
tokens = self._match_tokens_for_chain(cache, chain)
self._backup_node(cache, chain[-1])
loaded_nodes = chain[-window_pages:]
for n in loaded_nodes:
cache.tree_core.set_component_device_value_raw(n, ComponentType.SWA, None)
# SWA LRU bookkeeping must reflect tombstone state for the
# _restore_device_value path to exercise the host->device move.
cache.tree_core.remove_node_from_device_lru(n, ComponentType.SWA)
cache.tree_core.insert_node_into_host_lru(n, ComponentType.SWA)
# Build the LOAD_BACK transfer the same way load_back() would.
transfers = cache.tree_core.build_hicache_transfers(
ComponentType.SWA, chain[-1], CacheTransferPhase.LOAD_BACK
)
self.assertIsNotNone(transfers)
xfer = transfers[0]
self.assertEqual(xfer.nodes_to_load, loaded_nodes)
# Allocate SWA device slots from the inner allocator (mirrors how
# _resolve_device_transfers routes via device_alloc_fn ->
# swa_attn_allocator.alloc on the load-back path).
n_swa = int(xfer.host_indices.numel())
new_swa = allocator.swa_attn_allocator.alloc(n_swa)
self.assertIsNotNone(new_swa)
xfer.device_indices = new_swa
# Snapshot pre-commit state for invariants checks.
self._evict_device_component(cache, n, ComponentType.SWA)
pre_evictable = cache.tree_core.component_evictable_size(ComponentType.SWA)
load_actions = []
cache.tree_core.commit_hicache_transfers(
chain[-1],
CacheTransferPhase.LOAD_BACK,
{ComponentType.SWA: transfers},
cache_actions=load_actions,
)
cache._apply_cache_actions(load_actions)
self._load_back_through_admission(cache, req_to_token_pool, tokens)
self._finish_pending_loads(cache)
self._release_ongoing_load_back_locks(cache)
# (1) cd.value restored, host LRU -> device LRU swap done.
offset = 0
restored = 0
for n in loaded_nodes:
value = node_device_value(cache, n, ComponentType.SWA)
self.assertIsNotNone(value)
chunk_len = int(value.numel())
self.assertEqual(
value.tolist(),
new_swa[offset : offset + chunk_len].tolist(),
)
offset += chunk_len
swa_idx = node_device_value(cache, n, ComponentType.SWA)
self.assertIsNotNone(swa_idx)
self.assertTrue(cache.tree_core.is_node_in_device_lru(n, ComponentType.SWA))
self.assertFalse(cache.tree_core.is_node_in_host_lru(n, ComponentType.SWA))
self.assertEqual(offset, n_swa)
# (2) full_to_swa_index_mapping rebuilt for every loaded chunk.
for n in loaded_nodes:
full_idx = node_device_value(cache, n, ComponentType.FULL)
swa_idx = node_device_value(cache, n, ComponentType.SWA)
translated = allocator.translate_loc_from_full_to_swa(full_idx)
self.assertEqual(translated.tolist(), swa_idx.tolist())
# Evictable size moved up by the restored token count.
restored += int(swa_idx.numel())
self.assertEqual(
cache.tree_core.component_evictable_size(ComponentType.SWA) - pre_evictable,
n_swa,
restored,
)
cache.sanity_check()
def test_hicache_swa_load_back_anchored_on_best_match_node(self):
cache, _, _, y, x, _ = self._swa_anchor_setup()
File diff suppressed because it is too large Load Diff