[CI] radix-cache: Build unified radix cache test states through production paths (#45334)
This commit is contained in:
@@ -13,7 +13,6 @@ from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CacheConfig:
|
||||
# Tree
|
||||
page_size: int = 1
|
||||
components: tuple[ComponentType, ...] = (ComponentType.FULL,)
|
||||
|
||||
@@ -21,12 +20,10 @@ class CacheConfig:
|
||||
num_layers: int = 24
|
||||
full_attention_layer_ids: tuple[int, ...] = (3, 7, 11, 15, 19, 23)
|
||||
|
||||
# SWA
|
||||
sliding_window_size: Optional[int] = None
|
||||
draft_swa_window: int = 0
|
||||
decoder_swa_bounded_replay: bool = False
|
||||
|
||||
# Mamba
|
||||
enable_mamba_extra_buffer: bool = False
|
||||
mamba_cache_size: int = 20
|
||||
mamba_intermediate_size: int = 256
|
||||
@@ -37,7 +34,6 @@ class CacheConfig:
|
||||
mamba_conv_kernel: int = 4
|
||||
enable_int8_mamba_checkpoint: bool = False
|
||||
|
||||
# Model / pool
|
||||
kv_size: int = 256
|
||||
max_num_reqs: int = 10
|
||||
max_context_len: int = 512
|
||||
@@ -193,11 +189,8 @@ CONFIGS: list[CacheConfig] = [
|
||||
def requires(predicate, reason):
|
||||
"""Keep a test only under the configs where ``predicate`` holds.
|
||||
|
||||
Under any other config the test is removed from the bound class, not
|
||||
skipped. ``reason`` documents the requirement and is not read at runtime.
|
||||
|
||||
Stacked requirements are checked top to bottom and stop at the first that
|
||||
fails, so a lower one may assume the ones above it hold.
|
||||
Elsewhere the test is removed, not skipped. Stacked requirements are checked
|
||||
top to bottom, so a lower one may assume the ones above it hold.
|
||||
"""
|
||||
|
||||
def decorate(test):
|
||||
|
||||
@@ -86,9 +86,7 @@ class FakeFullComponent(TreeComponent):
|
||||
|
||||
|
||||
class PagedFullComponent(FullComponent):
|
||||
"""FULL component whose device values are not backed one-to-one by pool
|
||||
rows: a value lists two entries per row, and the component can hand rows
|
||||
of a node back to the allocator while the value stays on device."""
|
||||
"""FULL component whose device values are not backed one-to-one by pool rows."""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
@@ -99,7 +97,6 @@ class PagedFullComponent(FullComponent):
|
||||
|
||||
|
||||
def component_with_cache(component_type, cache):
|
||||
"""A registry component instance bound to a (mock) cache and its tree_core."""
|
||||
component = object.__new__(COMPONENT_REGISTRY[component_type])
|
||||
component.cache = cache
|
||||
component.tree_core = cache.tree_core
|
||||
|
||||
@@ -52,9 +52,7 @@ from sglang.test.kits.unified_radix_cache.unified_tree_core_inspector import (
|
||||
UnifiedTreeCoreInspector,
|
||||
)
|
||||
|
||||
# The Rust re-run entry point overrides this without changing the process-wide
|
||||
# production backend selection. Every cache built by build_fixture follows it;
|
||||
# tests that construct a Python core directly stay Python-only.
|
||||
# The Rust re-run overrides this; every cache built by build_fixture follows it.
|
||||
TREE_CORE_TEST_BACKEND: Optional[str] = "python"
|
||||
|
||||
|
||||
@@ -63,9 +61,8 @@ def selected_tree_core_test_backend() -> str:
|
||||
|
||||
|
||||
def drop_hicache_atexit_pin(cache):
|
||||
"""init_hicache registers ``cache.shutdown`` with atexit, which pins the
|
||||
fixture's device and host pools until process exit; hundreds of HiCache
|
||||
fixtures in one test process would otherwise accumulate on the GPU."""
|
||||
"""init_hicache registers cache.shutdown with atexit, which pins the fixture's GPU
|
||||
memory until exit."""
|
||||
atexit.unregister(cache.shutdown)
|
||||
|
||||
|
||||
@@ -128,19 +125,17 @@ def build_fixture(
|
||||
):
|
||||
"""Create (tree, allocator, req_to_token_pool) from a CacheConfig.
|
||||
|
||||
``tree_page_size`` stands in for DCP, which widens the tree page past the
|
||||
``page_size`` the rest of the config still sees. It only reaches values
|
||||
derived from the tree page: the allocator keeps ``cfg.page_size``, whereas
|
||||
DCP sets the two equal, so do not read insert or match behaviour off it.
|
||||
``tree_page_size`` stands in for DCP but only reaches values derived from the
|
||||
tree page; the allocator keeps ``cfg.page_size``, so do not read insert or
|
||||
match behaviour off it.
|
||||
"""
|
||||
server_args = ServerArgs(
|
||||
model_path="dummy",
|
||||
page_size=cfg.page_size,
|
||||
enable_int8_mamba_checkpoint=cfg.enable_int8_mamba_checkpoint,
|
||||
)
|
||||
# The mamba component reads mamba_cache_chunk_size, whose property otherwise
|
||||
# loads the HF config for self.model_path — impossible for the dummy model.
|
||||
# Mirror the property's default for a dummy HF config: FLA_CHUNK_SIZE.
|
||||
# The dummy HF config cannot supply mamba_cache_chunk_size, so preset
|
||||
# FLA_CHUNK_SIZE.
|
||||
server_args._mamba_cache_chunk_size = (
|
||||
max(FLA_CHUNK_SIZE, cfg.page_size)
|
||||
if mamba_cache_chunk_size is None
|
||||
|
||||
@@ -33,7 +33,6 @@ class ResumableInsertWalkSWAFixtures(UnifiedRadixCacheFixtures):
|
||||
seq = list(range(1, self.cfg.sliding_window_size + 1))
|
||||
key = RadixKey(array("q", seq))
|
||||
evicted = self._alloc(allocator, len(seq))
|
||||
# Window eviction already released the peers below the floor.
|
||||
allocator.free_swa(evicted)
|
||||
cache.insert(
|
||||
InsertParams(
|
||||
@@ -229,7 +228,6 @@ class SegmentLockFuzzFixtures(UnifiedRadixCacheFixtures):
|
||||
raise
|
||||
cache.sanity_check()
|
||||
|
||||
# Drain remaining locks; the tree must come back exactly balanced.
|
||||
for node_id, receipt, released in held:
|
||||
cache.dec_lock_ref(node_id, receipt.to_dec_params(), skip_swa=released)
|
||||
cache.sanity_check()
|
||||
|
||||
@@ -81,11 +81,10 @@ class UnifiedRadixCacheKVEventsFixtures:
|
||||
return leaf
|
||||
|
||||
def _init_hicache(self, cache, *, write_policy: str = "write_through"):
|
||||
# Production config: kernel IO backend + page_first layout with
|
||||
# PINNED host pools (kernel GPU DMA requires them). Pools track their
|
||||
# cudaHostRegister'd pointers and unregister on destroy()/GC, so the
|
||||
# many fixtures sharing this process cannot collide on recycled
|
||||
# address ranges (rc=712).
|
||||
# Production config: kernel IO backend + page_first layout.
|
||||
# Pinned host pools (kernel GPU DMA requires them) track their registered
|
||||
# pointers and unregister on destroy/GC, so fixtures sharing a process
|
||||
# cannot collide on recycled address ranges.
|
||||
server_args = ServerArgs(
|
||||
model_path="dummy",
|
||||
page_size=self.cfg.page_size,
|
||||
|
||||
@@ -11,11 +11,14 @@ from sglang.srt.managers.schedule_batch import (
|
||||
Req,
|
||||
)
|
||||
from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
InitLoadBackParams,
|
||||
InsertParams,
|
||||
MatchPrefixParams,
|
||||
)
|
||||
from sglang.srt.mem_cache.radix_cache import RadixKey
|
||||
from sglang.srt.mem_cache.unified_cache.components.base import (
|
||||
ComponentType,
|
||||
EvictLayer,
|
||||
)
|
||||
from sglang.srt.sampling.sampling_params import SamplingParams
|
||||
from sglang.srt.server_args import (
|
||||
@@ -73,9 +76,8 @@ class UnifiedRadixCacheFixtures:
|
||||
if not (self.cfg.has_swa and self.cfg.page_size > 1):
|
||||
return allocator.alloc(need_size)
|
||||
|
||||
# SWATokenToKVPoolAllocator.alloc() asserts page_size == 1, and
|
||||
# alloc_extend() requires batch tensors unsuitable for unit tests.
|
||||
# Replicate alloc_extend's core logic here.
|
||||
# The SWA allocator's alloc asserts page_size == 1 and alloc_extend needs
|
||||
# batch tensors, so replicate alloc_extend's core logic.
|
||||
ps = self.cfg.page_size
|
||||
aligned = ((need_size + ps - 1) // ps) * ps
|
||||
if aligned > allocator.full_attn_allocator.available_size():
|
||||
@@ -195,9 +197,8 @@ class UnifiedRadixCacheFixtures:
|
||||
if storage_backend == "file":
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
# The file-backend storage config records TP/PP rank/size. These unit
|
||||
# fixtures run without initializing distributed parallel state, so
|
||||
# force the local single-rank topology the fixture represents.
|
||||
# Distributed state is not initialized in unit fixtures; force the
|
||||
# local single-rank topology.
|
||||
parallel_override = get_parallel().override(
|
||||
tp_rank=0, tp_size=1, pp_rank=0, pp_size=1
|
||||
)
|
||||
@@ -216,7 +217,6 @@ class UnifiedRadixCacheFixtures:
|
||||
extra.update(storage_extra)
|
||||
storage_extra_config = json.dumps(extra) if extra else None
|
||||
|
||||
# See _init_hicache: production kernel IO backend, pinned pools.
|
||||
server_args = ServerArgs(
|
||||
model_path="dummy",
|
||||
page_size=self.cfg.page_size,
|
||||
@@ -228,7 +228,6 @@ class UnifiedRadixCacheFixtures:
|
||||
hicache_host_memory_mode=host_memory_mode,
|
||||
context_length=context_length,
|
||||
)
|
||||
# See build_fixture for why _mamba_cache_chunk_size is preset.
|
||||
server_args._mamba_cache_chunk_size = max(FLA_CHUNK_SIZE, self.cfg.page_size)
|
||||
set_global_server_args_for_scheduler(server_args)
|
||||
cache.init_hicache(server_args, cache.cache_init_params)
|
||||
@@ -237,15 +236,12 @@ class UnifiedRadixCacheFixtures:
|
||||
cache.write_through_threshold = 1 << 30
|
||||
cache.load_back_threshold = 0
|
||||
if storage_backend is not None:
|
||||
# Unit fixtures size host/device pools equally, which makes the
|
||||
# production prefetch capacity limit (host - device) zero. Keep the
|
||||
# L3 tests focused on storage round trips by allowing one fixture
|
||||
# worth of prefetch tokens.
|
||||
# Fixtures size host and device pools equally, which zeroes the
|
||||
# prefetch capacity limit; allow one fixture of prefetch tokens.
|
||||
cache.cache_controller.prefetch_capacity_limit = max(
|
||||
cache.cache_controller.prefetch_capacity_limit,
|
||||
cache.cache_controller.mem_pool_host.size,
|
||||
)
|
||||
# Background prefetch/backup threads are daemon; stop them per-test.
|
||||
self.addCleanup(cache.cache_controller._stop_storage_threads)
|
||||
|
||||
def _build_hicache_fixture(self, tree_core_backend: Optional[str] = None):
|
||||
@@ -307,10 +303,9 @@ class UnifiedRadixCacheFixtures:
|
||||
)
|
||||
|
||||
def _build_chain_pages(self, cache, allocator, req_to_token_pool, num_pages):
|
||||
"""Insert an incremental chain of single-page extensions.
|
||||
"""Insert a chain of single-page extensions; return it root-to-leaf.
|
||||
|
||||
Returns the chain root-to-leaf. Length may differ from num_pages
|
||||
when the radix tree merges or splits nodes.
|
||||
The length may differ from num_pages when the tree merges or splits nodes.
|
||||
"""
|
||||
seq: list[int] = []
|
||||
for i in range(num_pages):
|
||||
@@ -340,6 +335,31 @@ class UnifiedRadixCacheFixtures:
|
||||
ack.finish_event.synchronize()
|
||||
cache.loading_check()
|
||||
|
||||
def _evict_device_component(self, cache, node, component_type):
|
||||
"""Evict one node's device value of one component, as the eviction
|
||||
walk does for the victim it picks."""
|
||||
result = cache.tree_core.evict_component(
|
||||
node, component_type, EvictLayer.DEVICE
|
||||
)
|
||||
cache._free_values(result.device_frees, result.host_frees)
|
||||
|
||||
def _load_back_through_admission(self, cache, req_to_token_pool, tokens):
|
||||
"""Match ``tokens`` and start the scheduler's load-back of the host
|
||||
hit. The load-back holds its path lock until it is released."""
|
||||
req = self._make_req(req_to_token_pool)
|
||||
match = cache.match_prefix(
|
||||
MatchPrefixParams(key=RadixKey(array("q", tokens)), req=req)
|
||||
)
|
||||
self._apply_match_to_req(req, match)
|
||||
cache.init_load_back(
|
||||
InitLoadBackParams(
|
||||
best_match_node=req.best_match_node,
|
||||
host_hit_length=req.host_hit_length,
|
||||
req=req,
|
||||
)
|
||||
)
|
||||
return req
|
||||
|
||||
def _match_tokens_for_chain(self, cache, chain):
|
||||
tokens: list[int] = []
|
||||
for node in chain:
|
||||
@@ -363,9 +383,7 @@ class UnifiedRadixCacheFixtures:
|
||||
)
|
||||
|
||||
def _swa_finalize_setup(self):
|
||||
"""Build a SWA chain long enough to fill at least the window
|
||||
plus one extra page, and host-back every node so we can flip
|
||||
SWA tombstones at will."""
|
||||
"""Build a SWA chain longer than the window, host-backed at every node."""
|
||||
ps = self.cfg.page_size
|
||||
sw = self.cfg.sliding_window_size
|
||||
window_pages = (sw + ps - 1) // ps
|
||||
|
||||
@@ -134,10 +134,9 @@ class UnifiedRadixCacheStorageFixtures(UnifiedRadixCacheFixtures):
|
||||
def _run_prefetch_to_completion(self, cache, req_id, timeout: float = 10.0):
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
# Host memory is reserved (and IO started) by the scheduler-thread
|
||||
# drain once the L3 hit count is known, and the buffer-mode fill
|
||||
# commits at its H2D ack in loading_check — pump the full event
|
||||
# round like the real scheduler loop does.
|
||||
# Pump the full event round like the scheduler loop: host reservation
|
||||
# happens at the scheduler-thread drain, and the buffer-mode fill
|
||||
# commits at its H2D ack.
|
||||
cache.check_hicache_events()
|
||||
if cache.check_prefetch_progress(req_id):
|
||||
# Buffer mode parks a completed fetch as a staged prefetch;
|
||||
@@ -175,7 +174,7 @@ class UnifiedRadixCacheStorageFixtures(UnifiedRadixCacheFixtures):
|
||||
f = cache.buffer_pipeline.staged_prefetches[req_id]
|
||||
if prefix_len is None:
|
||||
prefix_len = f.matched_len
|
||||
req = mock.Mock()
|
||||
req = self._make_req(cache.req_to_token_pool)
|
||||
req.rid = req_id.rid
|
||||
req.cache_request_handle = req_id
|
||||
req.extra_key = extra_key
|
||||
@@ -391,9 +390,8 @@ class UnifiedRadixCacheStorageFixtures(UnifiedRadixCacheFixtures):
|
||||
cons.enable_storage_metrics = True
|
||||
cons.storage_metrics_collector = mock.Mock()
|
||||
|
||||
# Masked state: nodes born with live FULL under SWA tombstones (a
|
||||
# sibling insert whose SWA ring had slid past the span). live_tail is
|
||||
# how far short of the span end the ring stopped.
|
||||
# Nodes born with live FULL under SWA tombstones; live_tail is how far short
|
||||
# of the span end the ring stopped.
|
||||
live_from = len(seq) - live_tail
|
||||
value = self._alloc(cons_alloc, len(seq))
|
||||
cons.insert(
|
||||
@@ -407,8 +405,7 @@ class UnifiedRadixCacheStorageFixtures(UnifiedRadixCacheFixtures):
|
||||
self.assertEqual(masked.device_prefix_len, 0, "unified match not masked")
|
||||
self.assertEqual(masked.full_kv_hit_length, len(seq), "live FULL not resident")
|
||||
masked_full = value[: len(seq)].clone()
|
||||
# Model the real SWA-eviction path: tombstoned positions have their
|
||||
# allocator mapping cleared, while the live trailing portion remains.
|
||||
# Tombstoned positions have their allocator mapping cleared; the tail stays.
|
||||
cons_alloc.free_swa(masked_full[:live_from])
|
||||
self._fill_full_kv(cons_alloc, masked_full, marker=7)
|
||||
masked_kv = self._snapshot_full_kv(cons_alloc, masked_full)
|
||||
@@ -421,7 +418,6 @@ class UnifiedRadixCacheStorageFixtures(UnifiedRadixCacheFixtures):
|
||||
cons.prefetch_from_storage(
|
||||
req_id, cons.root_node_handle(), array("q", seq), None, None
|
||||
)
|
||||
# A parked SWA-only retry has no FULL suffix, but must re-pin its prefix.
|
||||
with mock.patch.object(cons.swa_kv_pool_host, "alloc", return_value=None):
|
||||
self._pump_hicache_until(
|
||||
cons,
|
||||
@@ -500,7 +496,6 @@ class UnifiedRadixCacheStorageFixtures(UnifiedRadixCacheFixtures):
|
||||
after = match_prefix(cons, seq)
|
||||
self.assertEqual(after.full_kv_hit_length, len(seq))
|
||||
self.assertEqual(after.device_prefix_len, len(seq))
|
||||
# Only the fetched SWA tail made the resident FULL reusable.
|
||||
self.assertEqual((req.storage_hit_start, req.storage_hit_length), (0, len(seq)))
|
||||
cons.storage_metrics_collector.log_storage_prefetch_unfulfilled_tokens.assert_not_called()
|
||||
cons.sanity_check()
|
||||
@@ -552,7 +547,6 @@ class UnifiedRadixCacheStorageFixtures(UnifiedRadixCacheFixtures):
|
||||
)
|
||||
self.assertGreater(aux_max, len(seq) - len(head))
|
||||
|
||||
# The surfaced host hit is the splice-able tail, not the full span.
|
||||
req = SimpleNamespace(
|
||||
rid=req_id.rid,
|
||||
cache_request_handle=req_id,
|
||||
@@ -585,7 +579,6 @@ class UnifiedRadixCacheStorageFixtures(UnifiedRadixCacheFixtures):
|
||||
self.assertTrue(torch.equal(hk, head_k))
|
||||
self.assertTrue(torch.equal(hv, head_v))
|
||||
|
||||
# The whole bounce (trimmed head included) frees at the ack.
|
||||
self.assertEqual(self._host_avail_sizes(cons), avail0)
|
||||
self.assertEqual(cons.cache_controller.prefetch_tokens_occupied, 0)
|
||||
cons.sanity_check()
|
||||
@@ -596,7 +589,6 @@ class UnifiedRadixCacheStorageFixtures(UnifiedRadixCacheFixtures):
|
||||
|
||||
cc = cache.cache_controller
|
||||
|
||||
# Fake _reduce_prefetch_ack to drop SWA pool_hits when drop_swa is True.
|
||||
def fake_reduce(ack):
|
||||
if drop_swa and ack.pool_hits is not None:
|
||||
if PoolName.SWA.value in ack.pool_hits:
|
||||
@@ -608,7 +600,6 @@ class UnifiedRadixCacheStorageFixtures(UnifiedRadixCacheFixtures):
|
||||
p_reduce.start()
|
||||
self.addCleanup(p_reduce.stop)
|
||||
|
||||
# Make all_reduce no-op. The real all_reduce fails in unit tests with tp_world_size=2.
|
||||
p_dist = mock.patch.object(dist, "all_reduce", return_value=None)
|
||||
p_dist.start()
|
||||
self.addCleanup(p_dist.stop)
|
||||
@@ -774,25 +765,26 @@ class UnifiedRadixCacheStorageAttachBackfillFixtures:
|
||||
return hashes
|
||||
|
||||
|
||||
class AnchorLockOutcomePolicyFixtures:
|
||||
def _make_pipeline(self, cache, cap_tokens=10_000):
|
||||
from sglang.srt.mem_cache.buffer_mode.pipeline import BufferModePipeline
|
||||
class AnchorLockOutcomePolicyFixtures(UnifiedRadixCacheStorageFixtures):
|
||||
def _insert_tokens(self, cache, tokens):
|
||||
value = cache.token_to_kv_pool_allocator.alloc(len(tokens))
|
||||
self.assertIsNotNone(value)
|
||||
cache.insert(InsertParams(key=RadixKey(array("q", tokens)), value=value))
|
||||
|
||||
pipeline = BufferModePipeline.__new__(BufferModePipeline)
|
||||
pipeline.anchor_locks = {}
|
||||
pipeline.anchor_locked_tokens_ = 0
|
||||
pipeline.anchor_lock_cap_tokens = cap_tokens
|
||||
pipeline._anchor_lock_cap_skips = 0
|
||||
pipeline._prefetch_prefix_ctx = {self._REQ: (list(self._PREFIX), None, None)}
|
||||
pipeline._cache = cache
|
||||
return pipeline
|
||||
|
||||
def _make_cache(self, live_match_len, pinned_len=None):
|
||||
cache = mock.MagicMock()
|
||||
cache.tree_core.is_eagle = False
|
||||
cache.tree_core.match_full_device_prefix.return_value = (
|
||||
live_match_len,
|
||||
99,
|
||||
live_match_len if pinned_len is None else pinned_len,
|
||||
def _make_buffer_cache(
|
||||
self, resident=(), cap_tokens=None, prefix=(), storage_dir=None
|
||||
):
|
||||
"""A buffer-mode cache holding ``resident`` on device, with the
|
||||
request's matched prefix recorded as prefetch enqueue records it."""
|
||||
cache, _, _ = self._build_buffer_hicache_fixture(
|
||||
storage_dir or self._make_storage_dir()
|
||||
)
|
||||
return cache
|
||||
if resident:
|
||||
self._insert_tokens(cache, list(resident))
|
||||
pipeline = cache.buffer_pipeline
|
||||
if cap_tokens is not None:
|
||||
pipeline.anchor_lock_cap_tokens = cap_tokens
|
||||
pipeline.set_prefix_ctx(
|
||||
self._REQ, self._PREFIX if prefix == () else (prefix or [])
|
||||
)
|
||||
return cache, pipeline
|
||||
|
||||
@@ -27,8 +27,6 @@ if TYPE_CHECKING:
|
||||
class UnifiedTreeCoreInspectionInterface(UnifiedTreeCoreInterface):
|
||||
"""Test-only inspection and control contract for shared backend tests."""
|
||||
|
||||
# ==== Read-only inspection ====
|
||||
|
||||
@abstractmethod
|
||||
def contains_node(self, node_id: NodeId) -> bool: ...
|
||||
|
||||
@@ -125,8 +123,6 @@ class UnifiedTreeCoreInspectionInterface(UnifiedTreeCoreInterface):
|
||||
"""Protected token count for one component (0 if the component is absent)."""
|
||||
...
|
||||
|
||||
# ==== White-box state controls ====
|
||||
|
||||
@abstractmethod
|
||||
def set_node_hash_values(
|
||||
self, node_id: NodeId, hash_values: Optional[list[str]]
|
||||
@@ -194,8 +190,6 @@ class UnifiedTreeCoreInspectionInterface(UnifiedTreeCoreInterface):
|
||||
"""Refresh duplicate-host tracking for the node."""
|
||||
...
|
||||
|
||||
# ==== Targeted white-box operations ====
|
||||
|
||||
@abstractmethod
|
||||
def advance_insert_walk_once(self) -> None:
|
||||
"""Advance one suspended insert walk step without flushing its actions."""
|
||||
|
||||
@@ -30,7 +30,6 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa
|
||||
return node_id in self._node_arena
|
||||
|
||||
def get_parent_node_id(self, node_id: NodeId) -> Optional[NodeId]:
|
||||
"""The parent node id, or None for the root."""
|
||||
parent = self.node_by_id(node_id).parent
|
||||
return None if parent is None else parent.id
|
||||
|
||||
@@ -45,13 +44,11 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa
|
||||
return len(key)
|
||||
|
||||
def get_node_token_ids(self, node_id: NodeId) -> list[int]:
|
||||
"""The raw token ids spanned by the node's radix key."""
|
||||
key = self.node_by_id(node_id).key
|
||||
assert key is not None
|
||||
return list(key.raw_token_ids())
|
||||
|
||||
def is_node_key_bigram(self, node_id: NodeId) -> bool:
|
||||
"""Whether the node's radix key uses bigram atoms."""
|
||||
key = self.node_by_id(node_id).key
|
||||
assert key is not None
|
||||
return key.is_bigram
|
||||
@@ -59,19 +56,16 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa
|
||||
def get_component_host_value(
|
||||
self, node_id: NodeId, component_type: ComponentType
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""The component's host value on the node, or None if absent."""
|
||||
return self.node_by_id(node_id).component_data[component_type].host_value
|
||||
|
||||
def get_component_device_lock_ref(
|
||||
self, node_id: NodeId, component_type: ComponentType
|
||||
) -> int:
|
||||
"""The component's device lock count on the node."""
|
||||
return self.node_by_id(node_id).component_data[component_type].lock_ref
|
||||
|
||||
def get_component_host_lock_ref(
|
||||
self, node_id: NodeId, component_type: ComponentType
|
||||
) -> int:
|
||||
"""The component's host lock count on the node."""
|
||||
return self.node_by_id(node_id).component_data[component_type].host_lock_ref
|
||||
|
||||
def get_node_hit_count(self, node_id: NodeId) -> int:
|
||||
@@ -83,20 +77,17 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa
|
||||
return self.node_by_id(node_id).write_through_pending_id
|
||||
|
||||
def is_external_cache_stored(self, node_id: NodeId) -> bool:
|
||||
"""Whether the node is known to be stored in the external cache."""
|
||||
return self.node_by_id(node_id).external_cache_stored
|
||||
|
||||
def is_node_in_device_lru(
|
||||
self, node_id: NodeId, component_type: ComponentType
|
||||
) -> bool:
|
||||
"""Whether the node belongs to the component's device LRU."""
|
||||
lru = self.lru_lists.get(component_type)
|
||||
return lru is not None and lru.in_list(self.node_by_id(node_id))
|
||||
|
||||
def is_node_in_host_lru(
|
||||
self, node_id: NodeId, component_type: ComponentType
|
||||
) -> bool:
|
||||
"""Whether the node belongs to the component's host LRU."""
|
||||
lru = self.host_lru_lists.get(component_type)
|
||||
return lru is not None and lru.in_list(self.node_by_id(node_id))
|
||||
|
||||
@@ -114,7 +105,6 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa
|
||||
def get_component_device_lru_node_ids(
|
||||
self, component_type: ComponentType
|
||||
) -> list[NodeId]:
|
||||
"""The component's device LRU members from most to least recent."""
|
||||
lru = self.lru_lists.get(component_type)
|
||||
return [] if lru is None else self._lru_node_ids(lru)
|
||||
|
||||
@@ -129,7 +119,6 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa
|
||||
return node is not None and node in self.evictable_host_leaves
|
||||
|
||||
def is_device_leaf(self, node_id: NodeId) -> bool:
|
||||
"""Whether the node has no device-resident descendants."""
|
||||
return self._is_device_leaf(self.node_by_id(node_id))
|
||||
|
||||
def get_all_node_ids(self) -> list[NodeId]:
|
||||
@@ -137,13 +126,11 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa
|
||||
return [node.id for node in self._collect_all_nodes()]
|
||||
|
||||
def component_protected_size(self, component_type: ComponentType) -> int:
|
||||
"""Protected token count for one component (0 if the component is absent)."""
|
||||
return self.component_protected_size_.get(component_type, 0)
|
||||
|
||||
def set_node_hash_values(
|
||||
self, node_id: NodeId, hash_values: Optional[list[str]]
|
||||
) -> None:
|
||||
"""Replace the node's page-hash field."""
|
||||
self.node_by_id(node_id).hash_value = hash_values
|
||||
|
||||
def set_component_device_value_raw(
|
||||
@@ -152,7 +139,6 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa
|
||||
component_type: ComponentType,
|
||||
value: Optional[torch.Tensor],
|
||||
) -> None:
|
||||
"""Replace the device-value field without updating tree bookkeeping."""
|
||||
self.node_by_id(node_id).component_data[component_type].value = value
|
||||
|
||||
def set_component_host_value_raw(
|
||||
@@ -161,48 +147,40 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa
|
||||
component_type: ComponentType,
|
||||
value: Optional[torch.Tensor],
|
||||
) -> None:
|
||||
"""Replace the host-value field without updating tree bookkeeping."""
|
||||
self.node_by_id(node_id).component_data[component_type].host_value = value
|
||||
|
||||
def set_component_device_lock_ref(
|
||||
self, node_id: NodeId, component_type: ComponentType, lock_ref: int
|
||||
) -> None:
|
||||
"""Replace the component's device lock count."""
|
||||
assert lock_ref >= 0
|
||||
self.node_by_id(node_id).component_data[component_type].lock_ref = lock_ref
|
||||
|
||||
def remove_node_from_device_lru(
|
||||
self, node_id: NodeId, component_type: ComponentType
|
||||
) -> None:
|
||||
"""Remove the node from the component's device LRU."""
|
||||
self.lru_lists[component_type].remove_node(self.node_by_id(node_id))
|
||||
|
||||
def insert_node_into_host_lru(
|
||||
self, node_id: NodeId, component_type: ComponentType
|
||||
) -> None:
|
||||
"""Insert the node as the component's most-recent host entry."""
|
||||
self.host_lru_lists[component_type].insert_mru(self.node_by_id(node_id))
|
||||
|
||||
def set_component_evictable_size(
|
||||
self, component_type: ComponentType, value: int
|
||||
) -> None:
|
||||
"""Replace the component's evictable device-token count."""
|
||||
assert value >= 0
|
||||
self.component_evictable_size_[component_type] = value
|
||||
|
||||
def set_component_protected_size(
|
||||
self, component_type: ComponentType, value: int
|
||||
) -> None:
|
||||
"""Replace the component's protected device-token count."""
|
||||
assert value >= 0
|
||||
self.component_protected_size_[component_type] = value
|
||||
|
||||
def update_duplicate_tracking(self, node_id: NodeId) -> None:
|
||||
"""Refresh duplicate-host tracking for the node."""
|
||||
self._update_duplicate_tracking(self.node_by_id(node_id))
|
||||
|
||||
def advance_insert_walk_once(self) -> None:
|
||||
"""Advance one suspended insert walk step without flushing its actions."""
|
||||
state = self._ongoing_insert_walk_state
|
||||
if state is None:
|
||||
raise RuntimeError("no in-flight insert")
|
||||
@@ -216,7 +194,6 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa
|
||||
component_type: ComponentType,
|
||||
target: EvictLayer,
|
||||
) -> BaseEvictionResult:
|
||||
"""Evict one component layer from a node and detach its LRU entry."""
|
||||
result = BaseEvictionResult()
|
||||
self._evict_component_and_detach_lru(
|
||||
self.node_by_id(node_id),
|
||||
@@ -235,7 +212,6 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa
|
||||
component_type: ComponentType,
|
||||
target: EvictLayer,
|
||||
) -> None:
|
||||
"""Validate the locks for a component-triggered cascade eviction."""
|
||||
node = self.node_by_id(node_id)
|
||||
trigger = self.components_by_type[component_type]
|
||||
is_leaf = self._is_cascade_evict_leaf(node, target)
|
||||
@@ -243,7 +219,6 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa
|
||||
self._should_cascade_evict_component(node, trigger, comp, target, is_leaf)
|
||||
|
||||
def cleanup_tombstone_ancestors(self, node_id: NodeId) -> BaseEvictionResult:
|
||||
"""Delete childless tombstone ancestors until a live or locked node is reached."""
|
||||
result = BaseEvictionResult()
|
||||
self._iteratively_delete_tombstone_ancestors(
|
||||
self.node_by_id(node_id),
|
||||
@@ -262,7 +237,6 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa
|
||||
value_chunks: list[torch.Tensor],
|
||||
best_value_len: int,
|
||||
) -> MatchResult:
|
||||
"""Run one component's match finalizer with NodeId boundaries."""
|
||||
node_result = result._replace(
|
||||
last_device_node=self.node_by_id(result.last_device_node),
|
||||
last_host_node=self.node_by_id(result.last_host_node),
|
||||
@@ -285,7 +259,6 @@ class UnifiedTreeCoreInspector(UnifiedTreeCore, UnifiedTreeCoreInspectionInterfa
|
||||
def build_backup_node_ids(
|
||||
self, node_id: NodeId, write_back: bool = False
|
||||
) -> list[NodeId]:
|
||||
"""Build the ordered node list for a device-to-host backup."""
|
||||
return self._build_backup_kv_action(
|
||||
self.node_by_id(node_id), write_back
|
||||
).node_ids
|
||||
|
||||
+55
-222
@@ -1,5 +1,3 @@
|
||||
"""Insert, match, lock, eviction, checkpoint, SWA and Mamba tests for UnifiedRadixCache."""
|
||||
|
||||
import unittest
|
||||
from array import array
|
||||
from dataclasses import replace
|
||||
@@ -121,13 +119,8 @@ class TestUnifiedRadixComponentRegistryOverride(CustomTestCase):
|
||||
|
||||
class TestUnifiedRadixComponentLedgerUnits(CustomTestCase):
|
||||
def test_ledgers_count_reclaimable_tokens(self):
|
||||
"""The evictable and protected ledgers are in the allocator's units:
|
||||
prefill admission adds the evictable ledger to the allocator's free
|
||||
space and the eviction walk counts them toward its request. So every
|
||||
ledger site (insert, lock, release, an in-place change, eviction)
|
||||
reads ``TreeComponent.reclaimable_value_len`` rather than the value
|
||||
length. A site that goes back to ``len(value)`` drifts the ledger from
|
||||
the recount in ``sanity_check`` for a component that overrides it."""
|
||||
"""Ledgers count allocator units: every site reads reclaimable_value_len,
|
||||
not len(value), or the ledger drifts from sanity_check's recount."""
|
||||
cache, allocator, _ = build_fixture(
|
||||
CacheConfig(),
|
||||
component_registry_override={ComponentType.FULL: PagedFullComponent},
|
||||
@@ -183,7 +176,7 @@ class TestDisabledUnifiedRadixCache(DisabledUnifiedRadixCacheFixtures, CustomTes
|
||||
|
||||
class TestUnifiedTreeNodeGetPrefixHashValues(CustomTestCase):
|
||||
def test_get_prefix_hash_values_not_shared_across_calls(self):
|
||||
"""Regression guard for cached mutable prefix hash lists (#26177)."""
|
||||
"""Cached prefix hash lists must not be mutated by callers."""
|
||||
|
||||
def make_node():
|
||||
return UnifiedTreeNode(tree_components=(ComponentType.FULL,))
|
||||
@@ -458,7 +451,6 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
|
||||
self._insert(cache, allocator, req_to_token_pool, seq_1p)
|
||||
self.assertEqual(allocator.available_size(), initial_avail - len(seq_1p))
|
||||
|
||||
# Frees the 1-page overlap.
|
||||
key_2p = RadixKey(array("q", seq_2p))
|
||||
value_2p = self._alloc(allocator, len(seq_2p))
|
||||
params = InsertParams(
|
||||
@@ -481,7 +473,6 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
|
||||
initial_avail - len(seq_1p) - (len(seq_2p) - len(seq_1p)),
|
||||
)
|
||||
|
||||
# prev_prefix_len covers the whole overlap: nothing freed.
|
||||
avail_before = allocator.available_size()
|
||||
key_3p = RadixKey(array("q", seq_3p))
|
||||
value_3p = self._alloc(allocator, len(seq_3p))
|
||||
@@ -637,10 +628,9 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
|
||||
@requires_swa_only
|
||||
@requires_ps1_window4
|
||||
def test_swa_unfinished_req_repoints_whole_insert_below_window(self):
|
||||
# A request whose live SWA tail inside the inserted range is shorter
|
||||
# than the window must still end up protected for the whole insert:
|
||||
# the tree owns those slots now, and a shorter protected length lets
|
||||
# the request free tree-owned SWA pages and re-insert them at finish.
|
||||
# Even a live SWA tail shorter than the window must be protected for the whole
|
||||
# insert, else the request frees tree-owned SWA pages and re-inserts them at
|
||||
# finish.
|
||||
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
|
||||
full_avail = allocator.full_attn_allocator.available_size()
|
||||
swa_avail = allocator.swa_attn_allocator.available_size()
|
||||
@@ -730,12 +720,10 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
|
||||
seq = self._make_seq(1, 2)
|
||||
self._insert(cache, allocator, req_to_token_pool, seq)
|
||||
|
||||
# Tree truncates unaligned tail internally, so it matches the seq prefix.
|
||||
unaligned = seq + list(range(9000, 9000 + ps - 1))
|
||||
m = match_prefix(cache, unaligned)
|
||||
self.assertEqual(m.device_prefix_len, len(seq))
|
||||
|
||||
# Below-page-size key aligns to 0 -> no match.
|
||||
m = match_prefix(cache, seq[: ps - 1])
|
||||
self.assertEqual(m.device_prefix_len, 0)
|
||||
|
||||
@@ -752,12 +740,10 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
|
||||
self._insert(cache, allocator, req_to_token_pool, first_page)
|
||||
self._insert(cache, allocator, req_to_token_pool, seq)
|
||||
|
||||
# Mismatch in second page → only first page matches
|
||||
bad_page2 = seq[:ps] + [9999] * ps
|
||||
m = match_prefix(cache, bad_page2)
|
||||
self.assertEqual(m.device_prefix_len, ps)
|
||||
|
||||
# Mismatch in first page → 0 match
|
||||
bad_page1 = [9999] + seq[1:]
|
||||
m = match_prefix(cache, bad_page1)
|
||||
self.assertEqual(m.device_prefix_len, 0)
|
||||
@@ -896,7 +882,6 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
|
||||
# past the SWA window (swa_evicted_seqlen >= total), the Full leaf must
|
||||
# be materialized (and the Full KV kept) so the prefix stays cacheable.
|
||||
# Runs across all SWA configs, including page_size > sliding_window_size
|
||||
# (the dsv4-style edge case).
|
||||
cache, allocator, _ = build_fixture(self.cfg)
|
||||
|
||||
tokens = self._make_seq(1, 2)
|
||||
@@ -938,11 +923,8 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
|
||||
node_a = m.last_device_node
|
||||
self.assertGreater(len(node_children(cache, node_a)), 0, "A must be internal")
|
||||
|
||||
# A request locks A, then decodes past the window → early-release the SWA
|
||||
# portion. On this internal node, dec_swa_lock_only also drops the
|
||||
# strictly-lower-tier Mamba lock (the co-located Mamba is useless once SWA
|
||||
# is gone), leaving only the Full path-lock held. This is what guarantees
|
||||
# the later SWA-eviction cascade never meets a legitimately-locked Mamba.
|
||||
# Early SWA release on an internal node also drops the lower-tier Mamba lock,
|
||||
# so later SWA eviction never meets a locked Mamba.
|
||||
lock_result = cache.inc_lock_ref(node_a)
|
||||
self.assertGreaterEqual(
|
||||
node_device_lock_ref(cache, node_a, ComponentType.MAMBA),
|
||||
@@ -963,9 +945,7 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
|
||||
cache.tree_core.is_node_in_device_lru(node_a, ComponentType.SWA)
|
||||
)
|
||||
|
||||
# Evict the child branch (Full/device eviction only) → A becomes a
|
||||
# Full-locked leaf with its now-unlocked SWA still in the SWA LRU. We do
|
||||
# NOT tombstone aux at the leaf-transition; the held Full pins the node.
|
||||
# Evicting the child makes A a Full-locked leaf with its SWA still in the LRU.
|
||||
cache.evict(EvictParams(num_tokens=len(seq_ab)))
|
||||
|
||||
self.assertEqual(len(node_children(cache, node_a)), 0, "A should now be a leaf")
|
||||
@@ -979,10 +959,7 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
|
||||
"A's unlocked SWA stays in the LRU (not tombstoned at transition)",
|
||||
)
|
||||
|
||||
# SWA eviction now selects A (it is in the SWA LRU). A's Full lock is a
|
||||
# higher-or-equal internal tier, so the cascade skips it and spares the
|
||||
# Full KV. The unlocked lower-tier Mamba is cascaded as part of the atomic
|
||||
# leaf teardown. This used to assert `cd.lock_ref == 0` on the locked Full.
|
||||
# SWA eviction skips A's Full lock and cascades the unlocked lower-tier Mamba.
|
||||
cache.evict(EvictParams(num_tokens=0, swa_num_tokens=len(seq_a)))
|
||||
self.assertGreaterEqual(
|
||||
node_device_lock_ref(cache, node_a, ComponentType.FULL),
|
||||
@@ -1006,8 +983,7 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
|
||||
def test_mamba_opt_out_holder_cannot_release_another_holders_mamba_lock(self):
|
||||
"""Holder A takes the mamba lock; holder B opts out (skip_lock_components=(ComponentType.MAMBA,))
|
||||
on the same node. B's early SWA release and final release must leave
|
||||
A's mamba lock intact -- a lost/defaulted receipt on B's side used to
|
||||
decrement A's lock without tripping any assert."""
|
||||
A's mamba lock intact."""
|
||||
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
|
||||
|
||||
seq = self._make_seq(
|
||||
@@ -1064,7 +1040,6 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
|
||||
"A must hold a Mamba checkpoint",
|
||||
)
|
||||
|
||||
# Natural lock acquisition — records inc_lock_ref in the lock trace.
|
||||
lock_result = cache.inc_lock_ref(node_a)
|
||||
self.assertGreaterEqual(
|
||||
node_device_lock_ref(cache, node_a, ComponentType.SWA), 1, "SWA locked"
|
||||
@@ -1096,46 +1071,6 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
|
||||
"Full stays locked",
|
||||
)
|
||||
|
||||
@requires_swa_and_mamba
|
||||
def test_cascade_evict_asserts_on_locked_internal_mamba(self):
|
||||
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
|
||||
|
||||
n_short = (self.cfg.sliding_window_size // self.cfg.page_size) + 4
|
||||
seq_a = self._make_seq(1, n_short)
|
||||
seq_ab = seq_a + self._make_seq(7000, 2)
|
||||
self._insert(cache, allocator, req_to_token_pool, seq_a)
|
||||
self._insert(cache, allocator, req_to_token_pool, seq_ab)
|
||||
|
||||
node_a = match_prefix(cache, seq_a).last_device_node
|
||||
self.assertGreater(len(node_children(cache, node_a)), 0, "A must be internal")
|
||||
self.assertIsNotNone(
|
||||
node_device_value(cache, node_a, ComponentType.MAMBA),
|
||||
"A must hold a Mamba checkpoint",
|
||||
)
|
||||
|
||||
# Lock ONLY Mamba — a stranded lower-priority lock that no supported path
|
||||
# produces. The cascade must surface it rather than silently skip.
|
||||
cache.tree_core.set_component_device_lock_ref(node_a, ComponentType.MAMBA, 1)
|
||||
self.assertGreaterEqual(
|
||||
node_device_lock_ref(cache, node_a, ComponentType.MAMBA), 1, "Mamba locked"
|
||||
)
|
||||
self.assertEqual(
|
||||
node_device_lock_ref(cache, node_a, ComponentType.FULL), 0, "Full unlocked"
|
||||
)
|
||||
|
||||
result = cache.tree_core.evict_component(
|
||||
node_a, ComponentType.SWA, EvictLayer.DEVICE
|
||||
)
|
||||
cache._free_values(result.device_frees, result.host_frees)
|
||||
# No higher-or-equal tier pins the node, so even with early-release on
|
||||
# the stranded Mamba lock must trip the hard-invariant assert.
|
||||
with self.assertRaises(AssertionError):
|
||||
cache.tree_core.validate_cascade_evict(
|
||||
node_a, ComponentType.SWA, EvictLayer.DEVICE
|
||||
)
|
||||
|
||||
cache.tree_core.set_component_device_lock_ref(node_a, ComponentType.MAMBA, 0)
|
||||
|
||||
@requires_swa_and_mamba
|
||||
def test_cascade_evict_asserts_on_locked_leaf_mamba(self):
|
||||
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
|
||||
@@ -1173,9 +1108,6 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
|
||||
|
||||
@requires_swa
|
||||
def test_swa_leaf_capped_to_window_on_insert(self):
|
||||
"""A long SWA leaf is split so locking it protects one window of SWA
|
||||
while full attention still protects the whole sequence."""
|
||||
|
||||
ps = self.cfg.page_size
|
||||
window = self.cfg.sliding_window_size
|
||||
tail_size = ((window + ps - 1) // ps) * ps
|
||||
@@ -1194,13 +1126,11 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
|
||||
self.assertIsNotNone(swa_val)
|
||||
|
||||
if case == "long_splits":
|
||||
# Capped to one page-aligned window; prefix is a real ancestor.
|
||||
self.assertEqual(len(swa_val), tail_size)
|
||||
self.assertNotEqual(
|
||||
node_parent(cache, leaf), cache.root_node_handle()
|
||||
)
|
||||
else:
|
||||
# Already within one window — no split.
|
||||
self.assertEqual(len(swa_val), len(seq))
|
||||
self.assertEqual(node_parent(cache, leaf), cache.root_node_handle())
|
||||
|
||||
@@ -1447,19 +1377,6 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
|
||||
cache.unlock(req.lock)
|
||||
cache.sanity_check()
|
||||
|
||||
def test_tombstone_cleanup_respects_locked_parent(self):
|
||||
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
|
||||
tokens = self._make_seq(1, 1)
|
||||
self._insert(cache, allocator, req_to_token_pool, tokens)
|
||||
parent = match_prefix(cache, tokens).last_device_node
|
||||
cache.tree_core.set_component_device_lock_ref(parent, ComponentType.FULL, 1)
|
||||
|
||||
result = cache.tree_core.cleanup_tombstone_ancestors(parent)
|
||||
cache._free_values(result.device_frees, result.host_frees)
|
||||
|
||||
self.assertIn(parent, node_children(cache, cache.root_node_handle()))
|
||||
self.assertTrue(all(evicted == 0 for evicted in result.tracker.values()))
|
||||
|
||||
def test_aux_evict_full_locked_leaf_tombstones_aux_only(self):
|
||||
aux_types = [
|
||||
ct
|
||||
@@ -1473,7 +1390,6 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
|
||||
aux = aux_types[0]
|
||||
|
||||
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
|
||||
# One page stays within a single SWA window (no leaf-cap split).
|
||||
seq = self._make_seq(1, 1)
|
||||
self._insert(cache, allocator, req_to_token_pool, seq)
|
||||
|
||||
@@ -1483,9 +1399,8 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
|
||||
self.assertIsNotNone(node_device_value(cache, node, ComponentType.FULL))
|
||||
self.assertIsNotNone(node_device_value(cache, node, aux))
|
||||
|
||||
# Reach the "FULL locked, aux unlocked" state the way production does:
|
||||
# mamba via the decode-hold opt-out (skip_lock_components=(ComponentType.MAMBA,)), SWA via the
|
||||
# early window release (its own first-class op).
|
||||
# Reach "FULL locked, aux unlocked" as production does: Mamba via the
|
||||
# decode-hold opt-out, SWA via early window release.
|
||||
aux_len = len(node_device_value(cache, node, aux))
|
||||
if aux == ComponentType.MAMBA:
|
||||
lock_result = cache.inc_lock_ref(
|
||||
@@ -1495,8 +1410,6 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
|
||||
lock_result = cache.inc_lock_ref(node)
|
||||
self.assertGreater(node_device_lock_ref(cache, node, aux), 0)
|
||||
cache.release_swa(TreeLock(node, lock_result.to_dec_params()))
|
||||
# FULL still locked -> not a device leaf -> no inline evict; the
|
||||
# value stays evictable for the explicit aux eviction below.
|
||||
self.assertIsNotNone(node_device_value(cache, node, aux))
|
||||
self.assertGreater(node_device_lock_ref(cache, node, ComponentType.FULL), 0)
|
||||
self.assertEqual(node_device_lock_ref(cache, node, aux), 0)
|
||||
@@ -1525,25 +1438,6 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
|
||||
)
|
||||
cache.sanity_check()
|
||||
|
||||
def test_evict_leaf_frees_all_components(self):
|
||||
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
|
||||
seq = self._make_seq(1, 3)
|
||||
self._insert(cache, allocator, req_to_token_pool, seq)
|
||||
|
||||
full_before = cache.full_evictable_size()
|
||||
mamba_before = cache.mamba_evictable_size() if self.cfg.has_mamba else 0
|
||||
swa_before = cache.swa_evictable_size() if self.cfg.has_swa else 0
|
||||
self.assertGreater(full_before, 0)
|
||||
|
||||
result = cache.evict(EvictParams(num_tokens=full_before * 2))
|
||||
self.assertGreaterEqual(result.num_tokens_evicted, full_before)
|
||||
self.assertEqual(cache.full_evictable_size(), 0)
|
||||
if self.cfg.has_mamba:
|
||||
self.assertEqual(cache.mamba_evictable_size(), 0)
|
||||
if self.cfg.has_swa:
|
||||
self.assertEqual(cache.swa_evictable_size(), 0)
|
||||
cache.sanity_check()
|
||||
|
||||
def test_evict_cascade_parent_becomes_d_leaf(self):
|
||||
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
|
||||
base = self._make_seq(1, 2)
|
||||
@@ -1580,6 +1474,10 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
|
||||
result = cache.evict(EvictParams(num_tokens=initial_evictable * 2))
|
||||
self.assertGreaterEqual(result.num_tokens_evicted, initial_evictable)
|
||||
self.assertEqual(cache.full_evictable_size(), 0)
|
||||
if self.cfg.has_mamba:
|
||||
self.assertEqual(cache.mamba_evictable_size(), 0)
|
||||
if self.cfg.has_swa:
|
||||
self.assertEqual(cache.swa_evictable_size(), 0)
|
||||
self.assertEqual(len(node_children(cache, cache.root_node_handle())), 0)
|
||||
cache.sanity_check()
|
||||
|
||||
@@ -1623,27 +1521,6 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
|
||||
self.assertEqual(m_low.device_prefix_len, 0)
|
||||
cache.sanity_check()
|
||||
|
||||
def test_evict_multiple_independent_leaves(self):
|
||||
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
|
||||
seqs = [self._make_seq(i * 100, 2) for i in range(4)]
|
||||
for s in seqs:
|
||||
self._insert(cache, allocator, req_to_token_pool, s)
|
||||
|
||||
total = sum(len(s) for s in seqs)
|
||||
self.assertEqual(cache.full_evictable_size(), total)
|
||||
|
||||
half = total // 2
|
||||
result = cache.evict(EvictParams(num_tokens=half))
|
||||
self.assertGreaterEqual(result.num_tokens_evicted, half)
|
||||
self.assertLessEqual(cache.full_evictable_size(), total - half)
|
||||
cache.sanity_check()
|
||||
|
||||
remaining = cache.full_evictable_size()
|
||||
result = cache.evict(EvictParams(num_tokens=remaining * 2))
|
||||
self.assertGreaterEqual(result.num_tokens_evicted, remaining)
|
||||
self.assertEqual(cache.full_evictable_size(), 0)
|
||||
cache.sanity_check()
|
||||
|
||||
def test_evict_result_accounting_matches_actual(self):
|
||||
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
|
||||
seqs = [self._make_seq(i * 100, 2) for i in range(5)]
|
||||
@@ -1651,9 +1528,15 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
|
||||
self._insert(cache, allocator, req_to_token_pool, s)
|
||||
|
||||
before = cache.full_evictable_size()
|
||||
result = cache.evict(EvictParams(num_tokens=before))
|
||||
half = before // 2
|
||||
result = cache.evict(EvictParams(num_tokens=half))
|
||||
after = cache.full_evictable_size()
|
||||
self.assertGreaterEqual(result.num_tokens_evicted, half)
|
||||
self.assertEqual(result.num_tokens_evicted, before - after)
|
||||
|
||||
remaining = cache.full_evictable_size()
|
||||
cache.evict(EvictParams(num_tokens=remaining * 2))
|
||||
self.assertEqual(cache.full_evictable_size(), 0)
|
||||
cache.sanity_check()
|
||||
|
||||
@requires_swa
|
||||
@@ -1706,54 +1589,37 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
|
||||
|
||||
@requires_full_swa_ps1_window4
|
||||
def test_swa_branch_insert_releases_forward_overshoot(self):
|
||||
|
||||
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
|
||||
swa = cache.components[ComponentType.SWA]
|
||||
|
||||
branching_seqlen = 8
|
||||
forward_len = 20
|
||||
req = self._make_req(req_to_token_pool)
|
||||
tokens = self._make_seq(1, forward_len)
|
||||
req.origin_input_ids = tokens
|
||||
req.output_ids = []
|
||||
req.full_untruncated_fill_ids = array("q", tokens)
|
||||
req.extend_end = forward_len
|
||||
req.kv.cache_protected_len = branching_seqlen
|
||||
|
||||
kv_indices = self._alloc(allocator, forward_len)
|
||||
req_to_token_pool.write(
|
||||
(req.kv.req_pool_idx, slice(0, forward_len)),
|
||||
kv_indices,
|
||||
req, kv_indices = self._make_inflight_req(
|
||||
cache, allocator, req_to_token_pool, tokens
|
||||
)
|
||||
|
||||
params = InsertParams(
|
||||
key=RadixKey(array("q", tokens[:branching_seqlen])),
|
||||
swa_branching_seqlen=branching_seqlen,
|
||||
)
|
||||
result = mock.Mock(swa_branch_inserted=False)
|
||||
swa.commit_insert_component_data(
|
||||
mock.Mock(),
|
||||
False,
|
||||
params,
|
||||
result,
|
||||
[],
|
||||
)
|
||||
self.assertTrue(result.swa_branch_inserted)
|
||||
req.swa_branching_seqlen = branching_seqlen
|
||||
|
||||
with envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS.override(True):
|
||||
swa.cleanup_after_caching_req(
|
||||
req,
|
||||
is_finished=False,
|
||||
insert_result=result,
|
||||
)
|
||||
cache.checkpoint(req, up_to=forward_len)
|
||||
|
||||
# Only the branch point is cached; the forward past it is released.
|
||||
self.assertEqual(
|
||||
match_prefix(cache, tokens).device_prefix_len, branching_seqlen
|
||||
)
|
||||
expected_evicted = forward_len - 1 - self.cfg.sliding_window_size
|
||||
self.assertEqual(req.kv.get_evicted_seqlen(ComponentType.SWA), expected_evicted)
|
||||
|
||||
# The cached branch keeps SWA for its last window; the forward overshoot
|
||||
# past it is released, and the request keeps its own live window.
|
||||
branch_freed = branching_seqlen - 1 - self.cfg.sliding_window_size
|
||||
mapping = allocator.full_to_swa_index_mapping
|
||||
self.assertEqual(
|
||||
torch.count_nonzero(mapping[kv_indices[:branching_seqlen]]).item(),
|
||||
branching_seqlen,
|
||||
torch.count_nonzero(mapping[kv_indices[:branch_freed]]).item(), 0
|
||||
)
|
||||
self.assertEqual(
|
||||
torch.count_nonzero(
|
||||
mapping[kv_indices[branch_freed:branching_seqlen]]
|
||||
).item(),
|
||||
branching_seqlen - branch_freed,
|
||||
)
|
||||
self.assertEqual(
|
||||
torch.count_nonzero(
|
||||
@@ -1765,6 +1631,8 @@ class UnifiedRadixCacheCoreSuite(UnifiedRadixCacheCoreFixtures):
|
||||
torch.count_nonzero(mapping[kv_indices[expected_evicted:]]).item(),
|
||||
forward_len - expected_evicted,
|
||||
)
|
||||
cache.unlock(req.lock)
|
||||
cache.sanity_check()
|
||||
|
||||
|
||||
define_config_test_classes(UnifiedRadixCacheCoreSuite, globals())
|
||||
@@ -1786,22 +1654,18 @@ class UnifiedLRUListBoundedRefreshTest(
|
||||
lru.reset_node_and_window_ancestors_mru(
|
||||
d, root, window_size=5, should_include=lambda n: n in included
|
||||
)
|
||||
# D, C refreshed; B contributes 2 to acc (now 6 >= 5) but is skipped;
|
||||
# A is not visited because the walk stops at B.
|
||||
self.assertEqual(self._lru_order(lru), [d, c, a])
|
||||
|
||||
def test_bounded_refresh_visits_only_until_window_filled(self):
|
||||
root, [a, b, c, d] = self._build_chain([3, 3, 3, 3])
|
||||
lru = UnifiedLRUList(ComponentType.SWA, self.components)
|
||||
# MRU->LRU: A, B, C, D (deepest is at the LRU tail, oldest)
|
||||
for n in (d, c, b, a):
|
||||
lru.insert_mru(n)
|
||||
self.assertEqual(self._lru_order(lru), [a, b, c, d])
|
||||
|
||||
# window=5: walking up from D visits D(acc=3<5) and C(acc=6>=5, stop).
|
||||
# Order expected: [D, C, A, B]. Why: D and C move to head in that
|
||||
# order (D first, then C right after D). A and B keep their relative
|
||||
# positions (they were the surviving prefix [A, B] before).
|
||||
# D and C move to the head in that order; A and B keep their relative
|
||||
# positions.
|
||||
lru.reset_node_and_window_ancestors_mru(
|
||||
d, root, window_size=5, should_include=lambda _n: True
|
||||
)
|
||||
@@ -1835,13 +1699,8 @@ class UnifiedLRUListBoundedRefreshTest(
|
||||
class TestUnifiedMambaLRUMatchRefresh(
|
||||
UnifiedMambaLRUMatchRefreshFixtures, CustomTestCase
|
||||
):
|
||||
"""A prefix-cache hit must refresh only best_match_node's mamba state in the
|
||||
mamba LRU, not its ancestors. The base TreeComponent refresh is whole-chain
|
||||
(reset_node_and_parents_mru), which clusters a session's states so that under
|
||||
mamba-pool pressure eviction drops whole cold sessions instead of the
|
||||
intermediate states reuse never needs. Guards MambaComponent.refresh_lru's
|
||||
single-node MATCH_END override.
|
||||
"""
|
||||
"""A hit refreshes only best_match_node's mamba state, so pressure evicts whole
|
||||
cold sessions rather than intermediate states."""
|
||||
|
||||
cfg = CacheConfig(page_size=1, components=(ComponentType.FULL, ComponentType.MAMBA))
|
||||
|
||||
@@ -1917,8 +1776,6 @@ class TestMambaCheckpointGrid(MambaCheckpointGridFixtures, CustomTestCase):
|
||||
class TestMambaFinishedOvershootCheckpoint(
|
||||
MambaFinishedOvershootCheckpointFixtures, CustomTestCase
|
||||
):
|
||||
"""A donated state must have consumed exactly the prefix its key names."""
|
||||
|
||||
_rid = 0
|
||||
cfg = CacheConfig(
|
||||
page_size=4,
|
||||
@@ -2001,7 +1858,6 @@ class TestUnifiedRadixCacheInt8MambaCheckpoint(
|
||||
|
||||
|
||||
def _component_with_cache(component_type, cache):
|
||||
"""A registry component instance bound to a (mock) cache and its tree_core."""
|
||||
component = object.__new__(COMPONENT_REGISTRY[component_type])
|
||||
component.cache = cache
|
||||
component.tree_core = cache.tree_core
|
||||
@@ -2009,8 +1865,6 @@ def _component_with_cache(component_type, cache):
|
||||
|
||||
|
||||
class TestUnifiedRadixCacheActionRouting(CustomTestCase):
|
||||
"""CacheAction routing: each type forwards to the right Controller API."""
|
||||
|
||||
def test_retraction_uses_engine_transfer_index_domains(self):
|
||||
cache = object.__new__(UnifiedRadixCache)
|
||||
cache.is_swa_enabled = True
|
||||
@@ -2087,19 +1941,6 @@ class TestUnifiedRadixCacheActionRouting(CustomTestCase):
|
||||
5, ComponentType.SWA, swa_value
|
||||
)
|
||||
|
||||
def test_apply_cache_actions_applies_each_in_order(self):
|
||||
cache = mock.MagicMock()
|
||||
first = UpdatePendingBackupOnNodeSplit(
|
||||
ack_id=1, old_node_id=1, new_node_id=2, new_child_node_id=1
|
||||
)
|
||||
second = UpdatePendingBackupOnNodeSplit(
|
||||
ack_id=2, old_node_id=3, new_node_id=4, new_child_node_id=3
|
||||
)
|
||||
UnifiedRadixCache._apply_cache_actions(cache, [first, second])
|
||||
cache._apply_cache_action.assert_has_calls(
|
||||
[mock.call(first), mock.call(second)]
|
||||
)
|
||||
|
||||
def test_eviction_free_failure_still_drains_actions_in_order(self):
|
||||
cache = object.__new__(UnifiedRadixCache)
|
||||
first = UpdatePendingBackupOnNodeSplit(
|
||||
@@ -2147,7 +1988,6 @@ class TestUnifiedRadixCacheActionRouting(CustomTestCase):
|
||||
def apply(cache, action):
|
||||
UnifiedRadixCache._apply_cache_action(cache, action)
|
||||
|
||||
# split X(10) -> [A(11), X(10)], then fragment A(11) -> [B(12), A(11)]
|
||||
first = UpdatePendingBackupOnNodeSplit(
|
||||
ack_id=7, old_node_id=10, new_node_id=11, new_child_node_id=10
|
||||
)
|
||||
@@ -2155,7 +1995,6 @@ class TestUnifiedRadixCacheActionRouting(CustomTestCase):
|
||||
ack_id=7, old_node_id=11, new_node_id=12, new_child_node_id=11
|
||||
)
|
||||
|
||||
# in list order, the publish list threads through both replaces
|
||||
cache = make_cache()
|
||||
apply(cache, first)
|
||||
apply(cache, second)
|
||||
@@ -2169,8 +2008,8 @@ class TestUnifiedRadixCacheActionRouting(CustomTestCase):
|
||||
self.assertEqual(receiver.call_args_list, [mock.call(first), mock.call(second)])
|
||||
self.assertEqual(cache.linker.replace_pending_offload_node.call_count, 2)
|
||||
|
||||
# reversed order silently drops the second fragment - documents why
|
||||
# emission order must be preserved end to end
|
||||
# Reversed order silently drops the second fragment, so emission order
|
||||
# must be preserved end to end.
|
||||
cache = make_cache()
|
||||
apply(cache, second)
|
||||
apply(cache, first)
|
||||
@@ -2195,8 +2034,7 @@ class TestUnifiedRadixCacheActionRouting(CustomTestCase):
|
||||
class TestSWADraftHistory(SWADraftHistoryFixtures, CustomTestCase):
|
||||
"""A widened draft needs SWA history before the target window. A running
|
||||
checkpoint must lock that history, and an incomplete cached window must
|
||||
not be reused even when the target alone could read it. The Rust rerun module
|
||||
runs these contracts against both Python and Rust TreeCore.
|
||||
not be reused even when the target alone could read it.
|
||||
"""
|
||||
|
||||
cfg = CacheConfig(
|
||||
@@ -2271,15 +2109,10 @@ class TestSWADraftHistory(SWADraftHistoryFixtures, CustomTestCase):
|
||||
|
||||
|
||||
class TestSWAWindowUnderBigramKey(SWAWindowUnderBigramKeyFixtures, CustomTestCase):
|
||||
"""`checkpoint` has to leave the leaf it inserts holding a full
|
||||
sliding window of live SWA. Otherwise the match that follows the insert
|
||||
refuses that leaf, `cache_protected_len` never advances, and the next insert
|
||||
frees KV the tree already owns as if it were the request's duplicate.
|
||||
|
||||
An EAGLE bigram key holds one entry less than the tokens it spans, so the
|
||||
leaf stops at page_floor(len(key)) rather than page_floor(seq_len), a page
|
||||
lower. The eviction frontier has to be measured against the key.
|
||||
"""
|
||||
"""checkpoint must leave its leaf with a full live SWA window, else
|
||||
cache_protected_len stalls and the next insert frees tree-owned KV. EAGLE
|
||||
bigram keys hold one entry fewer than seq_len, so measure the frontier
|
||||
against the key."""
|
||||
|
||||
cfg = CacheConfig(
|
||||
page_size=4,
|
||||
|
||||
+115
-345
@@ -1,12 +1,8 @@
|
||||
"""HiCache device <-> host backup, demotion and eviction tests for UnifiedRadixCache."""
|
||||
|
||||
import unittest
|
||||
from array import array
|
||||
from dataclasses import replace
|
||||
from unittest import mock
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
EvictParams,
|
||||
InitLoadBackParams,
|
||||
@@ -18,10 +14,8 @@ from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
from sglang.srt.mem_cache.radix_cache import RadixKey
|
||||
from sglang.srt.mem_cache.unified_cache.cache_action import (
|
||||
BackupKV,
|
||||
FreeDeviceKV,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_cache.components.base import (
|
||||
CacheTransferPhase,
|
||||
ComponentType,
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
@@ -36,7 +30,6 @@ from sglang.test.kits.unified_radix_cache.config import (
|
||||
requires_mamba,
|
||||
requires_swa,
|
||||
requires_swa_and_mamba,
|
||||
requires_swa_only,
|
||||
)
|
||||
from sglang.test.kits.unified_radix_cache.fixtures import (
|
||||
build_fixture,
|
||||
@@ -158,8 +151,7 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
|
||||
@requires_hicache
|
||||
def test_hicache_write_back_evict_drops_unbacked_leaf_when_host_full(self):
|
||||
"""Write-back eviction will keep freeing device KV when the host pool
|
||||
is exhausted and host eviction cannot free space to prevent OOM."""
|
||||
"""Write-back eviction keeps freeing device KV when the host pool is full."""
|
||||
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
|
||||
self._init_hicache(cache, write_policy="write_back")
|
||||
|
||||
@@ -179,7 +171,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
result = cache.evict(EvictParams(num_tokens=len(seq)))
|
||||
self.assertGreaterEqual(result.num_tokens_evicted, len(seq))
|
||||
|
||||
# The chain is gone entirely: no device hit, no host hit.
|
||||
m = match_prefix(cache, seq)
|
||||
self.assertEqual(m.device_prefix_len, 0)
|
||||
self.assertEqual(m.host_hit_length, 0)
|
||||
@@ -187,9 +178,8 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
|
||||
@requires_hicache
|
||||
def test_hicache_write_back_drop_respects_pins_then_frees_subtree(self):
|
||||
"""The host-pressure drop fallback must decline while any node in the
|
||||
unbacked subtree is host-pinned, then reclaim the whole subtree --
|
||||
including a demoted child's host backup -- once unpinned."""
|
||||
"""The host-pressure drop fallback declines while any node in the unbacked
|
||||
subtree is host-pinned, then reclaims the whole subtree once unpinned."""
|
||||
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
|
||||
self._init_hicache(cache, write_policy="write_back")
|
||||
host_pool = cache.cache_controller.mem_pool_host
|
||||
@@ -217,13 +207,11 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
# From here every backup fails (controller.write returns None), so
|
||||
# each evict() attempts the drop fallback on the parent.
|
||||
with mock.patch.object(cache.cache_controller, "write", return_value=None):
|
||||
# Pinned subtree root: drop declines, chain stays intact.
|
||||
parent_lock_params = cache.inc_host_lock_ref(parent).to_dec_params()
|
||||
result = cache.evict(EvictParams(num_tokens=len(parent_seq)))
|
||||
self.assertEqual(result.num_tokens_evicted, 0)
|
||||
cache.dec_host_lock_ref(parent, parent_lock_params)
|
||||
|
||||
# Pinned host-only descendant: drop declines as well.
|
||||
child_lock_params = cache.inc_host_lock_ref(child).to_dec_params()
|
||||
result = cache.evict(EvictParams(num_tokens=len(parent_seq)))
|
||||
self.assertEqual(result.num_tokens_evicted, 0)
|
||||
@@ -231,7 +219,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
self.assertEqual(m.device_prefix_len, len(parent_seq))
|
||||
cache.dec_host_lock_ref(child, child_lock_params)
|
||||
|
||||
# Unpinned: the subtree drops and the child's host slots return.
|
||||
result = cache.evict(EvictParams(num_tokens=len(parent_seq)))
|
||||
self.assertGreaterEqual(result.num_tokens_evicted, len(parent_seq))
|
||||
self.assertEqual(host_pool.available_size(), baseline_host)
|
||||
@@ -244,8 +231,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
lambda cfg: cfg == CacheConfig(), "single Full page-size-1 ownership regression"
|
||||
)
|
||||
def test_hicache_split_preserves_inflight_full_host_pin(self):
|
||||
"""A split prefix keeps an outstanding storage backup's host ownership."""
|
||||
|
||||
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
|
||||
self._init_hicache(cache, write_policy="write_back")
|
||||
host_pool = cache.cache_controller.mem_pool_host
|
||||
@@ -266,8 +251,8 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
self.assertEqual(node_host_lock_ref(cache, split_parent, ComponentType.FULL), 1)
|
||||
self.assertEqual(node_host_lock_ref(cache, leaf, ComponentType.FULL), 1)
|
||||
|
||||
# Before the fix the prefix had host_lock_ref == 0 and these slots were
|
||||
# reclaimed while the storage thread could still be reading them.
|
||||
# The prefix must hold a host lock, else these slots are reclaimed while
|
||||
# the storage thread still reads them.
|
||||
self.assertEqual(cache.evict_host(len(split_host)), 0)
|
||||
|
||||
# A lock acquired after the split covers only the suffix and gets a new
|
||||
@@ -293,7 +278,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
cache.sanity_check()
|
||||
|
||||
def test_hicache_split_preserves_inflight_swa_host_pin(self):
|
||||
"""A split SWA host segment stays pinned and releases without leaks."""
|
||||
expected_cfg = CacheConfig(
|
||||
page_size=1,
|
||||
components=(ComponentType.FULL, ComponentType.SWA),
|
||||
@@ -321,9 +305,8 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
lock_params.get_lock_uuid(ComponentType.SWA, lock_host=True)
|
||||
)
|
||||
|
||||
# Diverge inside the locked SWA window. The leaf handle stays on the
|
||||
# suffix while the lock boundary and copied host refs move to the new
|
||||
# prefix, so neither half may be reclaimed by component host eviction.
|
||||
# Diverging inside the locked SWA window moves the lock boundary and copied
|
||||
# host refs to the new prefix; neither half may be reclaimed.
|
||||
self._insert(cache, allocator, req_to_token_pool, [1, 2, 3, 4, 5, 6, 9, 10])
|
||||
split_parent = node_parent(cache, leaf)
|
||||
self.assertNotEqual(split_parent, parent_before_split)
|
||||
@@ -339,9 +322,8 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
|
||||
@requires_hicache
|
||||
def test_hicache_evict_device_leaf_aborts_demote_when_backup_fails(self):
|
||||
"""The tree op defers a write-back victim's demote: it returns a BackupKV
|
||||
without freeing device, and _demote requires a completed backup, so a
|
||||
failed backup leaves the node recoverable on device."""
|
||||
"""The tree op defers a write-back victim's demote, so a failed backup
|
||||
leaves the node recoverable on device."""
|
||||
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
|
||||
self._init_hicache(cache, write_policy="write_back")
|
||||
ct = ComponentType.FULL
|
||||
@@ -354,7 +336,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
self.assertFalse(cache.tree_core.is_backuped(node))
|
||||
self.assertFalse(cache.tree_core.is_full_device_evicted(node))
|
||||
|
||||
# Tree op defers: returns a BackupKV and leaves the node on device (no demote).
|
||||
leaf_result = cache.tree_core.evict_device_leaf(node, is_write_back=True)
|
||||
self.assertIsNotNone(leaf_result.backup_kv)
|
||||
cache._free_values(leaf_result.device_frees, leaf_result.host_frees)
|
||||
@@ -369,33 +350,8 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
|
||||
cache.sanity_check()
|
||||
|
||||
@requires_hicache
|
||||
def test_hicache_evict_to_host(self):
|
||||
cache, allocator, req_to_token_pool = self._build_hicache_fixture()
|
||||
seq = self._make_seq(1, 2)
|
||||
self._insert(cache, allocator, req_to_token_pool, seq)
|
||||
|
||||
m = match_prefix(cache, seq)
|
||||
node = m.last_device_node
|
||||
|
||||
self._backup_node(cache, node)
|
||||
self.assertTrue(cache.tree_core.is_backuped(node))
|
||||
|
||||
result = cache.evict(EvictParams(num_tokens=len(seq)))
|
||||
self.assertGreaterEqual(result.num_tokens_evicted, len(seq))
|
||||
|
||||
self.assertTrue(cache.tree_core.is_full_device_evicted(node))
|
||||
self.assertTrue(cache.tree_core.is_backuped(node))
|
||||
self.assertIsNone(node_device_value(cache, node, ComponentType.FULL))
|
||||
self.assertIsNotNone(node_host_value(cache, node, ComponentType.FULL))
|
||||
|
||||
self.assertFalse(cache.tree_core.is_device_evictable_leaf(node))
|
||||
self.assertTrue(cache.tree_core.is_host_evictable_leaf(node))
|
||||
cache.sanity_check()
|
||||
|
||||
@requires_hicache
|
||||
def test_hicache_partial_match_splits_evicted_backed_up_node(self):
|
||||
"""Partial matches on host-only nodes must keep the host prefix usable."""
|
||||
cache, allocator, req_to_token_pool = self._build_hicache_fixture()
|
||||
ps = self.cfg.page_size
|
||||
seq = self._make_seq(1, 4)
|
||||
@@ -428,7 +384,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
self.assertNotEqual(split_parent, cache.root_node_handle())
|
||||
self.assertTrue(cache.tree_core.is_full_device_evicted(split_parent))
|
||||
self.assertTrue(cache.tree_core.is_backuped(split_parent))
|
||||
# root -> split_parent keys reconstruct expected_prefix
|
||||
prefix_tokens: list[int] = []
|
||||
chain = []
|
||||
cur = split_parent
|
||||
@@ -438,7 +393,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
for n in reversed(chain):
|
||||
prefix_tokens.extend(node_token_ids(cache, n))
|
||||
self.assertEqual(prefix_tokens, expected_prefix)
|
||||
# the diverged suffix stays as evicted+backuped descendant(s)
|
||||
suffix_tokens: list[int] = []
|
||||
cur = split_parent
|
||||
while children := node_children(cache, cur):
|
||||
@@ -462,6 +416,10 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
self._backup_node(cache, node)
|
||||
cache.evict(EvictParams(num_tokens=len(seq)))
|
||||
|
||||
self.assertTrue(cache.tree_core.is_backuped(node))
|
||||
self.assertIsNone(node_device_value(cache, node, ComponentType.FULL))
|
||||
self.assertIsNotNone(node_host_value(cache, node, ComponentType.FULL))
|
||||
self.assertFalse(cache.tree_core.is_device_evictable_leaf(node))
|
||||
self.assertTrue(cache.tree_core.is_full_device_evicted(node))
|
||||
self.assertTrue(cache.tree_core.is_host_evictable_leaf(node))
|
||||
|
||||
@@ -483,7 +441,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
self.assertFalse(cache.tree_core.is_backuped(node))
|
||||
self.assertFalse(cache.tree_core.is_full_device_evicted(node))
|
||||
|
||||
# Backup failure falls back to dropping the unlocked subtree.
|
||||
with mock.patch.object(cache, "_execute_and_commit_kv_backup", return_value=0):
|
||||
result = cache.evict(EvictParams(num_tokens=len(seq)))
|
||||
|
||||
@@ -497,7 +454,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
lambda cfg: not cfg.has_mamba, "SWA-only path keeps the split setup simple"
|
||||
)
|
||||
def test_hicache_write_through_offloads_swa_split_leaf(self):
|
||||
|
||||
ps = self.cfg.page_size
|
||||
cache, allocator, _ = build_fixture(self.cfg)
|
||||
self._init_hicache(cache)
|
||||
@@ -525,7 +481,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
cache.sanity_check()
|
||||
|
||||
def test_hicache_evict_to_host_updates_aux_lru(self):
|
||||
"""Aux components (MAMBA / SWA) move from device LRU to host LRU on D->H eviction."""
|
||||
aux_types = [
|
||||
ct
|
||||
for ct in (ComponentType.MAMBA, ComponentType.SWA)
|
||||
@@ -556,9 +511,8 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
|
||||
@requires_mamba
|
||||
def test_hicache_write_back_internal_mamba_evict_demotes_state(self):
|
||||
"""write_back: an internal node's tombstoned mamba state is demoted
|
||||
to host, keeping the node a valid match boundary so the KV beneath
|
||||
it stays servable."""
|
||||
"""write_back: an internal node's tombstoned mamba state is demoted to host,
|
||||
keeping the node a valid match boundary."""
|
||||
cache, req_to_token_pool, seq_a = self._build_internal_mamba_fixture(
|
||||
"write_back"
|
||||
)
|
||||
@@ -638,8 +592,8 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
# The backed parent stops the Full backup chain, while the SWA window
|
||||
# reaches the unbacked grandparent and marks it under the anchor's ack.
|
||||
insert(3 * page)
|
||||
# Queue the asynchronous window backup explicitly: write-back insertion
|
||||
# no longer publishes it, but Mamba eviction must still respect its ack.
|
||||
# Queue the asynchronous window backup explicitly; Mamba eviction must
|
||||
# respect its ack.
|
||||
cache._execute_and_commit_kv_backup(BackupKV(node_ids=[anchor]))
|
||||
self.assertEqual(
|
||||
cache.ongoing_write_through[anchor].publish_node_ids, nodes[:3]
|
||||
@@ -666,7 +620,7 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
|
||||
@requires_mamba
|
||||
def test_hicache_write_through_internal_mamba_evict_keeps_drop(self):
|
||||
"""Non-write_back policies keep the legacy tombstone-and-drop."""
|
||||
"""Non-write_back policies tombstone and drop."""
|
||||
cache, _, seq_a = self._build_internal_mamba_fixture("write_through")
|
||||
|
||||
cache.evict(EvictParams(num_tokens=0, mamba_num=10))
|
||||
@@ -675,15 +629,14 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
self.assertEqual(
|
||||
m.device_prefix_len,
|
||||
0,
|
||||
"write_through keeps the legacy drop: frontier capped at root",
|
||||
"write_through keeps the drop: frontier capped at root",
|
||||
)
|
||||
cache.sanity_check()
|
||||
|
||||
@requires_swa
|
||||
def test_hicache_write_back_internal_swa_evict_demotes_kv(self):
|
||||
"""write_back: an internal node's tombstoned SWA KV is demoted to
|
||||
host, keeping the node a valid match boundary so the KV beneath it
|
||||
stays servable instead of losing a window of matchable prefix."""
|
||||
"""write_back: an internal node's tombstoned SWA KV is demoted to host,
|
||||
keeping the node a valid match boundary."""
|
||||
cache, req_to_token_pool, seq_a, seq_b = self._build_internal_swa_fixture(
|
||||
"write_back"
|
||||
)
|
||||
@@ -753,44 +706,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
self.assertEqual(result.last_host_node, result.last_device_node)
|
||||
self.assertEqual(result.host_hit_length, 0)
|
||||
|
||||
@requires_swa_only
|
||||
@requires(
|
||||
lambda cfg: cfg.sliding_window_size > cfg.page_size,
|
||||
"the window must reach past the leaf's own page",
|
||||
)
|
||||
def test_swa_backup_collector_is_shared_by_both_call_sites(self):
|
||||
"""needs_incremental_backup and the BACKUP_HOST transfer read one
|
||||
collector: cache mode walks the window past a host-backed target to
|
||||
its device-only ancestor, buffer mode stages the target alone."""
|
||||
# This test exercises the Python component's collector directly.
|
||||
cache, allocator, req_to_token_pool = self._build_hicache_fixture(
|
||||
tree_core_backend="python"
|
||||
)
|
||||
chain = self._build_chain_pages(cache, allocator, req_to_token_pool, 2)
|
||||
if len(chain) < 2:
|
||||
self.skipTest("chain too short")
|
||||
parent, leaf = chain[-2], chain[-1]
|
||||
swa = cache.components[ComponentType.SWA]
|
||||
leaf_node = cache.tree_core.node_by_id(leaf)
|
||||
cache.tree_core.set_component_host_value_raw(
|
||||
leaf,
|
||||
ComponentType.SWA,
|
||||
node_device_value(cache, leaf, ComponentType.SWA).clone(),
|
||||
)
|
||||
|
||||
self.assertTrue(swa.needs_incremental_backup(leaf_node))
|
||||
xfer = cache.tree_core.build_hicache_transfers(
|
||||
ComponentType.SWA, leaf, CacheTransferPhase.BACKUP_HOST
|
||||
)[0]
|
||||
self.assertEqual(xfer.nodes_to_load, [parent])
|
||||
|
||||
cache.tree_core.set_host_memory_buffer_only()
|
||||
self.assertTrue(swa.needs_incremental_backup(leaf_node))
|
||||
xfer = cache.tree_core.build_hicache_transfers(
|
||||
ComponentType.SWA, leaf, CacheTransferPhase.BACKUP_HOST
|
||||
)[0]
|
||||
self.assertEqual(xfer.nodes_to_load, [leaf])
|
||||
|
||||
def test_evict_host_drains_freed_host_values_to_the_pools(self):
|
||||
cache, allocator, req_to_token_pool = self._build_hicache_fixture()
|
||||
chain = self._build_chain_pages(cache, allocator, req_to_token_pool, 3)
|
||||
@@ -807,7 +722,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
ComponentType.MAMBA: "mamba_pool_host",
|
||||
}
|
||||
for ct in cache.tree_components:
|
||||
# set by the hicache assembler for every enabled component
|
||||
pool = getattr(cache, host_pool_attr[ct])
|
||||
if pool is None:
|
||||
continue
|
||||
@@ -815,7 +729,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
evicted = cache.evict_host(pool.size, ct)
|
||||
if evicted == 0:
|
||||
continue
|
||||
# the drain must return every freed host value to the pool
|
||||
self.assertGreater(pool.available_size(), available_before)
|
||||
|
||||
@requires_full_mamba_ps1
|
||||
@@ -841,38 +754,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
)
|
||||
self.assertEqual(result.host_hit_length, node_key_length(cache, leaf))
|
||||
|
||||
@requires_full_mamba_ps1
|
||||
def test_mamba_has_host_value_only_predicate(self):
|
||||
"""Needs a slot only when mamba is host-only (device evicted, host backed up)."""
|
||||
cache, allocator, req_to_token_pool = self._build_hicache_fixture()
|
||||
chain = self._build_chain_pages(cache, allocator, req_to_token_pool, 3)
|
||||
if not chain:
|
||||
self.skipTest("chain too short")
|
||||
node = chain[-1]
|
||||
dev = torch.tensor([0], dtype=torch.int64)
|
||||
host = torch.tensor([0], dtype=torch.int64)
|
||||
|
||||
# host-only: device evicted, host backup present -> restore needs a slot
|
||||
cache.tree_core.set_component_device_value_raw(node, ComponentType.MAMBA, None)
|
||||
cache.tree_core.set_component_host_value_raw(node, ComponentType.MAMBA, host)
|
||||
self.assertTrue(
|
||||
cache.tree_core.component_has_host_value_only(node, ComponentType.MAMBA)
|
||||
)
|
||||
# device + host: device value present -> no restore, no slot (the D+H bug case)
|
||||
cache.tree_core.set_component_device_value_raw(node, ComponentType.MAMBA, dev)
|
||||
self.assertFalse(
|
||||
cache.tree_core.component_has_host_value_only(node, ComponentType.MAMBA)
|
||||
)
|
||||
# device-only / neither: nothing to restore
|
||||
cache.tree_core.set_component_host_value_raw(node, ComponentType.MAMBA, None)
|
||||
self.assertFalse(
|
||||
cache.tree_core.component_has_host_value_only(node, ComponentType.MAMBA)
|
||||
)
|
||||
cache.tree_core.set_component_device_value_raw(node, ComponentType.MAMBA, None)
|
||||
self.assertFalse(
|
||||
cache.tree_core.component_has_host_value_only(node, ComponentType.MAMBA)
|
||||
)
|
||||
|
||||
@requires_full_swa_ps1
|
||||
def test_hicache_swa_host_best_match_keeps_device_anchor(self):
|
||||
cache, allocator, req_to_token_pool = self._build_hicache_fixture()
|
||||
@@ -919,25 +800,16 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
|
||||
@requires_full_swa_ps1_window4
|
||||
def test_hicache_swa_backup_window_stops_at_pending_ancestor(self):
|
||||
|
||||
cache, allocator, req_to_token_pool = self._build_hicache_fixture()
|
||||
chain = self._build_chain_pages(cache, allocator, req_to_token_pool, 3)
|
||||
if len(chain) < 3:
|
||||
self.skipTest("chain collapsed below the pending ancestor test")
|
||||
|
||||
c = chain[-1]
|
||||
c_swa = node_device_value(cache, c, ComponentType.SWA).clone()
|
||||
|
||||
# First transfer: publish Full for C only. C's SWA is a device tombstone
|
||||
# (decode-evicted, never backed up), so the write-through ack stays
|
||||
# pending on a node the SWA backup window has nothing to send for.
|
||||
cache.tree_core.set_component_device_value_raw(c, ComponentType.SWA, None)
|
||||
if cache.tree_core.is_node_in_device_lru(c, ComponentType.SWA):
|
||||
cache.tree_core.remove_node_from_device_lru(c, ComponentType.SWA)
|
||||
cache.tree_core.set_component_evictable_size(
|
||||
ComponentType.SWA,
|
||||
cache.tree_core.component_evictable_size(ComponentType.SWA) - len(c_swa),
|
||||
)
|
||||
tokens = self._match_tokens_for_chain(cache, chain)
|
||||
# C's SWA is decode-evicted before it is ever backed up, so the
|
||||
# write-through of C publishes Full only and its ack stays pending on a
|
||||
# node the SWA backup window has nothing to send for.
|
||||
self._evict_device_component(cache, c, ComponentType.SWA)
|
||||
self.assertGreater(
|
||||
cache._execute_and_commit_kv_backup(BackupKV(node_ids=[c])),
|
||||
0,
|
||||
@@ -945,25 +817,19 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
self.assertEqual(cache.tree_core.get_write_through_pending_id(c), c)
|
||||
self.assertIsNotNone(node_host_value(cache, c, ComponentType.FULL))
|
||||
self.assertIsNone(node_host_value(cache, c, ComponentType.SWA))
|
||||
|
||||
# SWA is reconstructed on device before the first ack, the way a
|
||||
# load-back commit stores it: under the pending segment lock the value
|
||||
# counts as protected until the ack releases it. The next incremental
|
||||
# SWA backup must treat C as the boundary and back up only the newly
|
||||
# inserted descendant.
|
||||
cache.tree_core.set_component_device_value(c, ComponentType.SWA, c_swa)
|
||||
tokens = self._match_tokens_for_chain(cache, chain)
|
||||
# A request re-inserting C's tokens recovers its SWA on device before
|
||||
# the first ack. The next incremental SWA backup must treat C as the
|
||||
# boundary and back up only the newly inserted descendant.
|
||||
self._insert(cache, allocator, req_to_token_pool, tokens)
|
||||
self.assertIsNotNone(node_device_value(cache, c, ComponentType.SWA))
|
||||
next_tokens = tokens + self._make_seq(9000, 1)
|
||||
cache.write_through_threshold = 1
|
||||
|
||||
self._insert(cache, allocator, req_to_token_pool, next_tokens)
|
||||
|
||||
d = match_prefix(cache, next_tokens).last_device_node
|
||||
self.assertEqual(cache.tree_core.get_write_through_pending_id(c), c)
|
||||
self.assertEqual(cache.tree_core.get_write_through_pending_id(d), d)
|
||||
self.assertIsNone(node_host_value(cache, c, ComponentType.SWA))
|
||||
self.assertIsNotNone(node_host_value(cache, d, ComponentType.SWA))
|
||||
|
||||
cache.writing_check(write_back=True)
|
||||
self.assertIsNone(cache.tree_core.get_write_through_pending_id(c))
|
||||
self.assertIsNone(cache.tree_core.get_write_through_pending_id(d))
|
||||
@@ -974,59 +840,39 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
lambda cfg: not cfg.has_mamba,
|
||||
"SWA-only path keeps the chain construction simple",
|
||||
)
|
||||
def test_hicache_swa_finalize_match_result(self):
|
||||
"""finalize_match_result accumulates host_value lengths of SWA tombstones
|
||||
within the trailing sliding window into ``swa_host_hit_length``. Out-of-window
|
||||
tombstones and chains fully on device must leave ``swa_host_hit_length`` at 0.
|
||||
``host_hit_length`` is Full-KV only and is never written by SWA.
|
||||
"""
|
||||
|
||||
cache, _, _, chain, window_pages = self._swa_finalize_setup()
|
||||
leaf = chain[-1]
|
||||
def test_hicache_swa_host_hit_counts_tombstones_inside_the_window(self):
|
||||
"""A host-backed SWA tombstone inside the trailing sliding window stops
|
||||
the device match before it: the Full-backed tail from the tombstone on
|
||||
is a host hit, and the tombstone's own SWA counts in
|
||||
``swa_host_hit_length``."""
|
||||
ps = self.cfg.page_size
|
||||
|
||||
cases = [
|
||||
("all_on_device", None, 0),
|
||||
("tombstone_in_window", chain[-window_pages], ps),
|
||||
("tombstone_outside_window", chain[-(window_pages + 1)], 0),
|
||||
]
|
||||
for name, victim, expected in cases:
|
||||
window_pages = (self.cfg.sliding_window_size + ps - 1) // ps
|
||||
for name, victim in (
|
||||
("all_on_device", None),
|
||||
("tombstone_in_window", -window_pages),
|
||||
("tombstone_outside_window", -(window_pages + 1)),
|
||||
):
|
||||
with self.subTest(name):
|
||||
# Reset SWA state for each subcase.
|
||||
for n in chain:
|
||||
device_value = node_device_value(cache, n, ComponentType.SWA)
|
||||
host_value = node_host_value(cache, n, ComponentType.SWA)
|
||||
if device_value is None and host_value is not None:
|
||||
cache.tree_core.set_component_device_value_raw(
|
||||
n, ComponentType.SWA, host_value.clone()
|
||||
)
|
||||
cache, allocator, req_to_token_pool = self._build_hicache_fixture()
|
||||
chain = self._build_chain_pages(
|
||||
cache, allocator, req_to_token_pool, window_pages + 2
|
||||
)
|
||||
if len(chain) <= window_pages:
|
||||
self.skipTest("chain collapsed below the window length")
|
||||
tokens = self._match_tokens_for_chain(cache, chain)
|
||||
self._backup_node(cache, chain[-1])
|
||||
host_tail, swa_host = 0, 0
|
||||
if victim is not None:
|
||||
cache.tree_core.set_component_device_value_raw(
|
||||
victim, ComponentType.SWA, None
|
||||
self._evict_device_component(
|
||||
cache, chain[victim], ComponentType.SWA
|
||||
)
|
||||
|
||||
result = MatchResult(
|
||||
device_prefix_len=0,
|
||||
last_device_node=leaf,
|
||||
last_host_node=leaf,
|
||||
best_match_node=leaf,
|
||||
host_hit_length=0,
|
||||
cache_protected_len=7,
|
||||
cache_actions=(FreeDeviceKV(indices=[]),),
|
||||
)
|
||||
result = cache.tree_core.finalize_component_match_result(
|
||||
ComponentType.SWA,
|
||||
result=result,
|
||||
params=MatchPrefixParams(
|
||||
key=RadixKey(array("q", self._make_seq(1, 1)))
|
||||
),
|
||||
value_chunks=[],
|
||||
best_value_len=0,
|
||||
)
|
||||
self.assertEqual(result.host_hit_length, 0)
|
||||
self.assertEqual(result.swa_host_hit_length, expected)
|
||||
self.assertEqual(result.cache_protected_len, 7)
|
||||
self.assertEqual(result.cache_actions, (FreeDeviceKV(indices=[]),))
|
||||
if name == "tombstone_in_window":
|
||||
host_tail = sum(node_key_length(cache, n) for n in chain[victim:])
|
||||
swa_host = node_key_length(cache, chain[victim])
|
||||
result = match_prefix(cache, tokens)
|
||||
self.assertEqual(result.device_prefix_len, len(tokens) - host_tail)
|
||||
self.assertEqual(result.host_hit_length, host_tail)
|
||||
self.assertEqual(result.swa_host_hit_length, swa_host)
|
||||
|
||||
def test_hicache_swa_finalize_anchored_on_best_match_node(self):
|
||||
cache, _, _, y, x, _ = self._swa_anchor_setup()
|
||||
@@ -1057,62 +903,41 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
)
|
||||
def test_hicache_swa_temp_lock_does_not_release_restored_tombstone(self):
|
||||
"""A temporary scheduler lock that skipped a SWA tombstone must not
|
||||
release later load-back/request locks after the tombstone is restored.
|
||||
"""
|
||||
|
||||
cache, allocator, _, chain, _ = self._swa_finalize_setup()
|
||||
release later load-back/request locks after the tombstone is restored."""
|
||||
cache, allocator, req_to_token_pool = self._build_hicache_fixture()
|
||||
chain = self._build_chain_pages(cache, allocator, req_to_token_pool, 3)
|
||||
if len(chain) < 3:
|
||||
self.skipTest("chain too short")
|
||||
leaf = chain[-1]
|
||||
tombstone = leaf
|
||||
old_swa = node_device_value(cache, tombstone, ComponentType.SWA)
|
||||
self.assertIsNotNone(old_swa)
|
||||
tokens = self._match_tokens_for_chain(cache, chain)
|
||||
self._backup_node(cache, leaf)
|
||||
self.assertIsNotNone(node_host_value(cache, leaf, ComponentType.SWA))
|
||||
self._evict_device_component(cache, leaf, ComponentType.SWA)
|
||||
self.assertIsNone(node_device_value(cache, leaf, ComponentType.SWA))
|
||||
|
||||
cache.tree_core.set_component_device_value_raw(
|
||||
tombstone, ComponentType.SWA, None
|
||||
)
|
||||
cache.tree_core.remove_node_from_device_lru(tombstone, ComponentType.SWA)
|
||||
cache.tree_core.insert_node_into_host_lru(tombstone, ComponentType.SWA)
|
||||
cache.tree_core.set_component_evictable_size(
|
||||
ComponentType.SWA,
|
||||
cache.tree_core.component_evictable_size(ComponentType.SWA) - len(old_swa),
|
||||
)
|
||||
def swa_ref():
|
||||
return node_device_lock_ref(cache, leaf, ComponentType.SWA)
|
||||
|
||||
temp_lock = cache.inc_lock_ref(leaf)
|
||||
self.assertEqual(node_device_lock_ref(cache, tombstone, ComponentType.SWA), 1)
|
||||
|
||||
xfer = cache.tree_core.build_hicache_transfers(
|
||||
ComponentType.SWA, leaf, CacheTransferPhase.LOAD_BACK
|
||||
)[0]
|
||||
new_swa = allocator.swa_attn_allocator.alloc(int(xfer.host_indices.numel()))
|
||||
self.assertIsNotNone(new_swa)
|
||||
xfer.device_indices = new_swa
|
||||
load_actions = []
|
||||
cache.tree_core.commit_hicache_transfers(
|
||||
leaf,
|
||||
CacheTransferPhase.LOAD_BACK,
|
||||
{ComponentType.SWA: [xfer]},
|
||||
cache_actions=load_actions,
|
||||
)
|
||||
cache._apply_cache_actions(load_actions)
|
||||
|
||||
load_back_lock = cache.inc_lock_ref(leaf)
|
||||
self.assertEqual(swa_ref(), 1)
|
||||
self._load_back_through_admission(cache, req_to_token_pool, tokens)
|
||||
self.assertIsNotNone(node_device_value(cache, leaf, ComponentType.SWA))
|
||||
request_lock = cache.inc_lock_ref(leaf)
|
||||
self.assertEqual(node_device_lock_ref(cache, tombstone, ComponentType.SWA), 3)
|
||||
|
||||
self.assertEqual(swa_ref(), 3)
|
||||
cache.dec_lock_ref(leaf, temp_lock.to_dec_params())
|
||||
self.assertEqual(node_device_lock_ref(cache, tombstone, ComponentType.SWA), 2)
|
||||
|
||||
cache.dec_lock_ref(leaf, load_back_lock.to_dec_params())
|
||||
self.assertEqual(swa_ref(), 2)
|
||||
self._finish_pending_loads(cache)
|
||||
self._release_ongoing_load_back_locks(cache)
|
||||
cache.dec_lock_ref(leaf, request_lock.to_dec_params())
|
||||
self.assertEqual(node_device_lock_ref(cache, tombstone, ComponentType.SWA), 0)
|
||||
self.assertEqual(swa_ref(), 0)
|
||||
cache.sanity_check()
|
||||
|
||||
@requires_hicache
|
||||
def test_hicache_full_temp_lock_covers_evicted_anchor_and_mirrors_on_release(
|
||||
self,
|
||||
):
|
||||
"""Segment locks count the evicted anchor too (no skip receipts), so
|
||||
a value restored mid-hold stays correctly attributed: each release
|
||||
takes back exactly its own ref regardless of interleaved holders.
|
||||
"""
|
||||
"""Segment locks count the evicted anchor too (no skip receipts), so a value
|
||||
restored mid-hold stays correctly attributed."""
|
||||
ps = self.cfg.page_size
|
||||
if 3 * ps > self.cfg.kv_size // 2:
|
||||
self.skipTest("kv_size too small")
|
||||
@@ -1120,53 +945,26 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
chain = self._build_chain_pages(cache, allocator, req_to_token_pool, 3)
|
||||
if len(chain) < 3:
|
||||
self.skipTest("chain too short")
|
||||
a, y, anchor = chain
|
||||
self._simulate_backup_tree(cache)
|
||||
anchor = chain[-1]
|
||||
self._backup_node(cache, anchor)
|
||||
cache.evict(EvictParams(num_tokens=node_key_length(cache, anchor)))
|
||||
self.assertTrue(cache.tree_core.is_full_device_evicted(anchor))
|
||||
|
||||
anchor_value = node_device_value(cache, anchor, ComponentType.FULL)
|
||||
# Simulate the anchor's FULL device eviction: drop the value and take
|
||||
# its tokens out of the evictable ledger, as a real evict would.
|
||||
cache.tree_core.set_component_device_value_raw(anchor, ComponentType.FULL, None)
|
||||
cache.tree_core.set_component_evictable_size(
|
||||
ComponentType.FULL,
|
||||
cache.tree_core.component_evictable_size(ComponentType.FULL)
|
||||
- len(anchor_value),
|
||||
)
|
||||
|
||||
self.assertEqual(node_device_lock_ref(cache, anchor, ComponentType.FULL), 0)
|
||||
self.assertEqual(node_device_lock_ref(cache, y, ComponentType.FULL), 0)
|
||||
self.assertEqual(node_device_lock_ref(cache, a, ComponentType.FULL), 0)
|
||||
def full_refs():
|
||||
return [node_device_lock_ref(cache, n, ComponentType.FULL) for n in chain]
|
||||
|
||||
self.assertEqual(full_refs(), [0, 0, 0])
|
||||
temp_lock = cache.inc_lock_ref(anchor)
|
||||
self.assertEqual(node_device_lock_ref(cache, anchor, ComponentType.FULL), 1)
|
||||
self.assertEqual(node_device_lock_ref(cache, y, ComponentType.FULL), 1)
|
||||
self.assertEqual(node_device_lock_ref(cache, a, ComponentType.FULL), 1)
|
||||
|
||||
# Restore the value mid-hold: a value materialized under lock is
|
||||
# protected until the last release, exactly as a load-back credits it.
|
||||
cache.tree_core.set_component_device_value_raw(
|
||||
anchor, ComponentType.FULL, anchor_value
|
||||
)
|
||||
cache.tree_core.set_component_protected_size(
|
||||
ComponentType.FULL,
|
||||
cache.tree_core.component_protected_size(ComponentType.FULL)
|
||||
+ len(anchor_value),
|
||||
)
|
||||
|
||||
self.assertEqual(full_refs(), [1, 1, 1])
|
||||
# Restore the anchor mid-hold.
|
||||
self._load_back_node(cache, anchor)
|
||||
second_lock = cache.inc_lock_ref(anchor)
|
||||
self.assertEqual(node_device_lock_ref(cache, anchor, ComponentType.FULL), 2)
|
||||
self.assertEqual(node_device_lock_ref(cache, y, ComponentType.FULL), 2)
|
||||
self.assertEqual(node_device_lock_ref(cache, a, ComponentType.FULL), 2)
|
||||
|
||||
self.assertEqual(full_refs(), [2, 2, 2])
|
||||
cache.dec_lock_ref(anchor, temp_lock.to_dec_params())
|
||||
self.assertEqual(node_device_lock_ref(cache, anchor, ComponentType.FULL), 1)
|
||||
self.assertEqual(node_device_lock_ref(cache, y, ComponentType.FULL), 1)
|
||||
self.assertEqual(node_device_lock_ref(cache, a, ComponentType.FULL), 1)
|
||||
|
||||
self.assertEqual(full_refs(), [1, 1, 1])
|
||||
cache.dec_lock_ref(anchor, second_lock.to_dec_params())
|
||||
self.assertEqual(node_device_lock_ref(cache, anchor, ComponentType.FULL), 0)
|
||||
self.assertEqual(node_device_lock_ref(cache, y, ComponentType.FULL), 0)
|
||||
self.assertEqual(node_device_lock_ref(cache, a, ComponentType.FULL), 0)
|
||||
self.assertEqual(full_refs(), [0, 0, 0])
|
||||
cache.sanity_check()
|
||||
|
||||
@requires_mamba
|
||||
@requires(
|
||||
@@ -1175,54 +973,32 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
)
|
||||
def test_hicache_mamba_temp_lock_does_not_release_restored_tombstone(self):
|
||||
"""A temporary scheduler lock that skipped a Mamba tombstone must not
|
||||
release later load-back/request locks after the tombstone is restored.
|
||||
"""
|
||||
|
||||
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
|
||||
release later load-back/request locks after the tombstone is restored."""
|
||||
cache, allocator, req_to_token_pool = self._build_hicache_fixture()
|
||||
seq = self._make_seq(1, 2)
|
||||
self._insert(cache, allocator, req_to_token_pool, seq)
|
||||
m = match_prefix(cache, seq)
|
||||
node = m.last_device_node
|
||||
old_mamba = node_device_value(cache, node, ComponentType.MAMBA)
|
||||
self.assertIsNotNone(old_mamba)
|
||||
self._simulate_backup(cache, node)
|
||||
node = match_prefix(cache, seq).last_device_node
|
||||
self._backup_node(cache, node)
|
||||
self.assertIsNotNone(node_host_value(cache, node, ComponentType.MAMBA))
|
||||
self._evict_device_component(cache, node, ComponentType.MAMBA)
|
||||
self.assertIsNone(node_device_value(cache, node, ComponentType.MAMBA))
|
||||
|
||||
cache.tree_core.set_component_device_value_raw(node, ComponentType.MAMBA, None)
|
||||
cache.tree_core.remove_node_from_device_lru(node, ComponentType.MAMBA)
|
||||
cache.tree_core.insert_node_into_host_lru(node, ComponentType.MAMBA)
|
||||
cache.tree_core.set_component_evictable_size(
|
||||
ComponentType.MAMBA,
|
||||
cache.tree_core.component_evictable_size(ComponentType.MAMBA)
|
||||
- len(old_mamba),
|
||||
)
|
||||
def mamba_ref():
|
||||
return node_device_lock_ref(cache, node, ComponentType.MAMBA)
|
||||
|
||||
temp_lock = cache.inc_lock_ref(node)
|
||||
self.assertEqual(node_device_lock_ref(cache, node, ComponentType.MAMBA), 1)
|
||||
|
||||
xfer = cache.tree_core.build_hicache_transfers(
|
||||
ComponentType.MAMBA, node, CacheTransferPhase.LOAD_BACK
|
||||
)[0]
|
||||
new_mamba = req_to_token_pool.mamba_allocator.alloc(1)
|
||||
self.assertIsNotNone(new_mamba)
|
||||
xfer.device_indices = new_mamba
|
||||
cache.tree_core.commit_hicache_transfers(
|
||||
node,
|
||||
CacheTransferPhase.LOAD_BACK,
|
||||
{ComponentType.MAMBA: [xfer]},
|
||||
cache_actions=[],
|
||||
)
|
||||
|
||||
load_back_lock = cache.inc_lock_ref(node)
|
||||
self.assertEqual(mamba_ref(), 1)
|
||||
self._load_back_through_admission(cache, req_to_token_pool, seq)
|
||||
self.assertIsNotNone(node_device_value(cache, node, ComponentType.MAMBA))
|
||||
request_lock = cache.inc_lock_ref(node)
|
||||
self.assertEqual(node_device_lock_ref(cache, node, ComponentType.MAMBA), 3)
|
||||
|
||||
self.assertEqual(mamba_ref(), 3)
|
||||
cache.dec_lock_ref(node, temp_lock.to_dec_params())
|
||||
self.assertEqual(node_device_lock_ref(cache, node, ComponentType.MAMBA), 2)
|
||||
|
||||
cache.dec_lock_ref(node, load_back_lock.to_dec_params())
|
||||
self.assertEqual(mamba_ref(), 2)
|
||||
self._finish_pending_loads(cache)
|
||||
self._release_ongoing_load_back_locks(cache)
|
||||
cache.dec_lock_ref(node, request_lock.to_dec_params())
|
||||
self.assertEqual(node_device_lock_ref(cache, node, ComponentType.MAMBA), 0)
|
||||
# The commit ran under held locks: the restored value must have been
|
||||
self.assertEqual(mamba_ref(), 0)
|
||||
# The restore ran under held locks: the restored value must have been
|
||||
# credited to protected, or the ledger drifts on the final release.
|
||||
cache.sanity_check()
|
||||
|
||||
@@ -1274,7 +1050,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
"SWA boundary splits deepen the chain non-deterministically",
|
||||
)
|
||||
def test_build_backup_kv_action_orders_ancestors_first(self):
|
||||
"""write-through backs up unbacked ancestors parent-first; write-back only the leaf."""
|
||||
cache, allocator, req_to_token_pool = self._build_hicache_fixture()
|
||||
|
||||
top_seq = self._make_seq(1, 1)
|
||||
@@ -1293,11 +1068,9 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
self.assertFalse(cache.tree_core.is_backuped(mid))
|
||||
self.assertFalse(cache.tree_core.is_backuped(top))
|
||||
|
||||
# write-through: every unbacked ancestor, top-most first, child last
|
||||
write_through = cache.tree_core.build_backup_node_ids(leaf, write_back=False)
|
||||
self.assertEqual(write_through, [top, mid, leaf])
|
||||
|
||||
# write-back: only the eviction victim, even with unbacked ancestors
|
||||
write_back = cache.tree_core.build_backup_node_ids(leaf, write_back=True)
|
||||
self.assertEqual(write_back, [leaf])
|
||||
|
||||
@@ -1306,7 +1079,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
@requires_swa
|
||||
@requires_hicache
|
||||
def test_write_back_defers_swa_window_backup_until_eviction(self):
|
||||
"""Reinserting backed Full nodes must not publish freshly recovered SWA."""
|
||||
ps = self.cfg.page_size
|
||||
if self.cfg.sliding_window_size <= 2 * ps:
|
||||
self.skipTest("window must span past the leaf and its parent to reach c")
|
||||
@@ -1338,7 +1110,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
self.assertIsNotNone(node_device_value(cache, node, ComponentType.SWA))
|
||||
self.assertIsNone(node_host_value(cache, node, ComponentType.SWA))
|
||||
|
||||
# Two requests in one batch finish at t1 and t2; neither ack drains.
|
||||
with mock.patch.object(
|
||||
cache,
|
||||
"_execute_and_commit_kv_backup",
|
||||
@@ -1352,7 +1123,6 @@ class UnifiedRadixCacheHiCacheSuite(UnifiedRadixCacheHiCacheFixtures):
|
||||
self.assertIsNone(cache.tree_core.get_write_through_pending_id(node))
|
||||
self.assertIsNone(node_host_value(cache, node, ComponentType.SWA))
|
||||
self.assertEqual(cache.cache_controller.write_queue, [])
|
||||
# Write-back defers the unbacked Full prefix to eviction.
|
||||
self.assertFalse(cache.tree_core.is_backuped(c))
|
||||
self.assertFalse(cache.tree_core.is_backuped(d))
|
||||
|
||||
|
||||
+16
-81
@@ -1,5 +1,3 @@
|
||||
"""Resumable insert walk, returned-value drain and segment lock tests for UnifiedRadixCache."""
|
||||
|
||||
import sys
|
||||
import unittest
|
||||
from array import array
|
||||
@@ -134,7 +132,6 @@ class TestResumableInsertWalk(UnifiedRadixCacheFixtures, CustomTestCase):
|
||||
self._insert(cache, allocator, req_to_token_pool, [1, 2, 3, 4])
|
||||
self.assertFalse(cache.tree_core.has_ongoing_insert())
|
||||
|
||||
# The tree stays usable and the crossing re-fires on the next walk.
|
||||
self._insert(cache, allocator, req_to_token_pool, [1, 2, 3, 4])
|
||||
cache.writing_check(write_back=True)
|
||||
(ancestor,) = node_children(cache, cache.root_node_handle())
|
||||
@@ -231,7 +228,6 @@ class TestResumableInsertWalk(UnifiedRadixCacheFixtures, CustomTestCase):
|
||||
self.assertTrue(cache.tree_core.is_backuped(middle))
|
||||
self.assertFalse(cache.tree_core.is_backuped(top))
|
||||
|
||||
# The device insert unevicts the middle and adds the deep leaf.
|
||||
self._insert(cache, allocator, req_to_token_pool, list(range(1, 13)))
|
||||
(deep,) = node_children(cache, middle)
|
||||
|
||||
@@ -386,11 +382,9 @@ class TestResumableInsertWalkSWA(ResumableInsertWalkSWAFixtures, CustomTestCase)
|
||||
(window_node,) = node_children(cache, prefix_node)
|
||||
self.assertIsNone(node_device_value(cache, prefix_node, ComponentType.SWA))
|
||||
|
||||
# Re-inserting fully in-window recovers the prefix span's SWA data.
|
||||
cache.insert(InsertParams(key=key, value=self._alloc(allocator, len(seq))))
|
||||
self.assertIsNotNone(node_device_value(cache, prefix_node, ComponentType.SWA))
|
||||
|
||||
# SWA eviction takes the recovered span and keeps the window leaf.
|
||||
cache.evict(EvictParams(num_tokens=0, swa_num_tokens=sw))
|
||||
self.assertIsNone(node_device_value(cache, prefix_node, ComponentType.SWA))
|
||||
self.assertIsNotNone(node_device_value(cache, window_node, ComponentType.SWA))
|
||||
@@ -457,14 +451,13 @@ class TestResumableInsertWalkWriteBack(UnifiedRadixCacheFixtures, CustomTestCase
|
||||
def test_drop_fallback_frees_host_for_later_backups_same_round(self):
|
||||
"""Host slots reclaimed by the drop fallback itself (an interior
|
||||
host-only descendant) must be reusable by later write-back backups in
|
||||
the same eviction round (pre-split freed them inline)."""
|
||||
the same eviction round."""
|
||||
if self._skip_unsupported_hicache_test():
|
||||
return
|
||||
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
|
||||
self._init_hicache(cache, write_policy="write_back")
|
||||
host_pool = cache.cache_controller.mem_pool_host
|
||||
|
||||
# Chain 1: unbacked parent -> host-only child -> host-only grandchild.
|
||||
p1 = self._make_seq(1, 2)
|
||||
self._insert(cache, allocator, req_to_token_pool, p1)
|
||||
c1 = p1 + self._make_seq(1000, 2)
|
||||
@@ -518,8 +511,7 @@ class TestResumableInsertWalkWriteBack(UnifiedRadixCacheFixtures, CustomTestCase
|
||||
|
||||
@unittest.skipUnless(torch.cuda.is_available(), "cache fixtures need CUDA")
|
||||
class TestReturnedValuesDrain(UnifiedRadixCacheFixtures, CustomTestCase):
|
||||
"""Drain contract of the returned-values eviction API: every tree-core step
|
||||
result is drained exactly once, in per-component insertion order."""
|
||||
"""Each tree-core step result is drained exactly once, in insertion order."""
|
||||
|
||||
cfg = CacheConfig()
|
||||
|
||||
@@ -629,45 +621,6 @@ class TestReturnedValuesDrain(UnifiedRadixCacheFixtures, CustomTestCase):
|
||||
self.assertIs(device_frees[ComponentType.FULL][0], sentinel)
|
||||
self.assertIs(host_frees[ComponentType.FULL][0], sentinel)
|
||||
|
||||
def test_free_values_frees_in_component_insertion_order(self):
|
||||
"""Device frees apply before host frees, each in per-component
|
||||
insertion order — the allocator free-list order the pre-split inline
|
||||
frees produced."""
|
||||
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
|
||||
order = [ComponentType.FULL, ComponentType.SWA, ComponentType.MAMBA]
|
||||
device_frees = defaultdict(list)
|
||||
host_frees = defaultdict(list)
|
||||
for ct in order:
|
||||
device_frees[ct].append(torch.tensor([1]))
|
||||
host_frees[ct].append(torch.tensor([2]))
|
||||
|
||||
freed = []
|
||||
fake_components = {
|
||||
ct: mock.MagicMock(
|
||||
free_host_values=mock.MagicMock(
|
||||
side_effect=lambda values, ct=ct: freed.append(("host", ct))
|
||||
)
|
||||
)
|
||||
for ct in order
|
||||
}
|
||||
with (
|
||||
mock.patch.object(
|
||||
cache,
|
||||
"_apply_cache_action",
|
||||
side_effect=lambda action: freed.append(
|
||||
("device", action.component_type)
|
||||
),
|
||||
),
|
||||
mock.patch.dict(cache.components, fake_components),
|
||||
):
|
||||
cache._free_values(device_frees, host_frees)
|
||||
|
||||
self.assertEqual(
|
||||
freed, [("device", ct) for ct in order] + [("host", ct) for ct in order]
|
||||
)
|
||||
self.assertFalse(device_frees)
|
||||
self.assertFalse(host_frees)
|
||||
|
||||
def test_free_values_mid_drain_failure_cannot_replay_freed_entries(self):
|
||||
"""A device free that raises must leave only un-attempted entries in
|
||||
the dict (no replay of successes) while host frees still drain."""
|
||||
@@ -723,25 +676,17 @@ class TestReturnedValuesDrain(UnifiedRadixCacheFixtures, CustomTestCase):
|
||||
|
||||
@unittest.skipUnless(torch.cuda.is_available(), "cache fixtures need CUDA")
|
||||
class TestSegmentLockProtocol(SegmentLockProtocolFixtures, CustomTestCase):
|
||||
"""Segment-lock protocol regressions, replaying the production lock-theft
|
||||
failure classes (F1/F2) and the split hazards.
|
||||
|
||||
The protocol: a lock covers the contiguous node segment
|
||||
[start, boundary-uuid], counting every node (tombstones included), so a
|
||||
release needs only the receipt (anchor node, boundary uuid, skipped
|
||||
components) and any ref==0 met inside the segment is a hard protocol
|
||||
violation. The replays read the tree through the inspection interface, so
|
||||
they run unchanged against the Python and Rust cores.
|
||||
"""
|
||||
"""A lock covers the node segment [start, boundary-uuid] counting tombstones;
|
||||
a release needs only its receipt, and any ref==0 inside the segment is a
|
||||
violation."""
|
||||
|
||||
cfg = CacheConfig(
|
||||
components=(ComponentType.FULL, ComponentType.SWA), sliding_window_size=8
|
||||
)
|
||||
|
||||
def test_rebuilt_tombstone_relock_release_no_theft(self):
|
||||
"""F1 attribution replay: A locks a window containing a tombstone; the
|
||||
tombstone is rebuilt and locked by B mid-hold; A's release must leave
|
||||
B's refs intact (the old skip-set protocol decremented B's lock)."""
|
||||
"""A locks a window containing a tombstone; B rebuilds and locks it
|
||||
mid-hold; A's release must leave B's refs intact."""
|
||||
sw = self.cfg.sliding_window_size
|
||||
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
|
||||
seq = self._make_seq(1, 2 * sw)
|
||||
@@ -794,7 +739,6 @@ class TestSegmentLockProtocol(SegmentLockProtocolFixtures, CustomTestCase):
|
||||
for n in segment:
|
||||
self.assertEqual(self._swa_ref(cache, n), 2)
|
||||
|
||||
# THE regression: A's release takes back exactly A's refs.
|
||||
cache.dec_lock_ref(leaf, lock_a.to_dec_params())
|
||||
for n in segment:
|
||||
self.assertEqual(self._swa_ref(cache, n), 1)
|
||||
@@ -821,7 +765,6 @@ class TestSegmentLockProtocol(SegmentLockProtocolFixtures, CustomTestCase):
|
||||
cache.sanity_check()
|
||||
|
||||
def test_root_acquire_receipt_releases_on_root(self):
|
||||
"""A cold request locks the root, so its receipt is anchored on root."""
|
||||
cache, _, _ = build_fixture(self.cfg)
|
||||
root = cache.root_node_handle()
|
||||
lock = cache.inc_lock_ref(root).to_dec_params()
|
||||
@@ -830,7 +773,6 @@ class TestSegmentLockProtocol(SegmentLockProtocolFixtures, CustomTestCase):
|
||||
cache.sanity_check()
|
||||
|
||||
def test_release_with_incorrect_root_boundary_fails_loud(self):
|
||||
"""An incorrect root boundary must fail at the acquired segment's edge."""
|
||||
sw = self.cfg.sliding_window_size
|
||||
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
|
||||
seq = self._make_seq(1, 3 * sw)
|
||||
@@ -869,8 +811,8 @@ class TestSegmentLockProtocol(SegmentLockProtocolFixtures, CustomTestCase):
|
||||
)
|
||||
|
||||
def test_unanchored_device_release_fails_loud(self):
|
||||
"""#41261 replay: a receipt that lost its anchor must not walk the
|
||||
node's segment and drop locks other holders own."""
|
||||
"""A receipt that lost its anchor must not walk the node's segment and
|
||||
drop locks other holders own."""
|
||||
cache, allocator, leaf = self._leaf_with_lock_path()
|
||||
lock = cache.inc_lock_ref(leaf).to_dec_params()
|
||||
self._assert_unanchored_release_rejected(
|
||||
@@ -920,8 +862,6 @@ class TestSegmentLockProtocol(SegmentLockProtocolFixtures, CustomTestCase):
|
||||
)
|
||||
|
||||
def test_unanchored_root_release_fails_loud(self):
|
||||
"""Every acquire anchors its receipt, root included; an unanchored
|
||||
receipt lost its anchor upstream even where releasing is a no-op."""
|
||||
cache, _, _ = build_fixture(self.cfg)
|
||||
root = cache.root_node_handle()
|
||||
self._assert_protocol_violation(
|
||||
@@ -941,8 +881,8 @@ class TestSegmentLockProtocol(SegmentLockProtocolFixtures, CustomTestCase):
|
||||
)
|
||||
|
||||
def test_finish_after_early_release_without_skip_swa_fails_loud(self):
|
||||
"""F2 replay: retraction-after-early-release used to run a second SWA
|
||||
walk that stole ancestors' locks; now it dies at the first node."""
|
||||
"""A finish after early SWA release without skip_swa asserts at the first
|
||||
node instead of stealing ancestors' locks."""
|
||||
sw = self.cfg.sliding_window_size
|
||||
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
|
||||
seq = self._make_seq(1, 2 * sw)
|
||||
@@ -955,7 +895,7 @@ class TestSegmentLockProtocol(SegmentLockProtocolFixtures, CustomTestCase):
|
||||
)
|
||||
|
||||
def test_finish_after_early_release_with_skip_swa(self):
|
||||
"""The correct F2 flow: unlock honors the early release."""
|
||||
"""Finish with skip_swa honors the early release."""
|
||||
sw = self.cfg.sliding_window_size
|
||||
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
|
||||
seq = self._make_seq(1, 2 * sw)
|
||||
@@ -983,7 +923,6 @@ class TestSegmentLockProtocol(SegmentLockProtocolFixtures, CustomTestCase):
|
||||
pre_segment = self._segment(cache, leaf, sw)
|
||||
pre_full_path = self._full_path(cache, leaf)
|
||||
|
||||
# Diverge inside the window to force a split of a locked node.
|
||||
fork = seq[: len(seq) - sw // 2] + self._make_seq(9000, sw)
|
||||
self._insert(cache, allocator, req_to_token_pool, fork)
|
||||
post_segment = self._segment(cache, leaf, sw)
|
||||
@@ -1018,7 +957,6 @@ class TestSegmentLockProtocol(SegmentLockProtocolFixtures, CustomTestCase):
|
||||
|
||||
params = cache.inc_lock_ref(leaf).to_dec_params()
|
||||
self.assertNotIn(node, cache.tree_core.evictable_device_leaves)
|
||||
# Full first: its walk still sees the SWA lock, so the leaf stays out.
|
||||
cache.components[ComponentType.FULL].release_component_lock(node, params)
|
||||
self.assertNotIn(node, cache.tree_core.evictable_device_leaves)
|
||||
# The SWA release drops the last lock and must readmit the leaf itself.
|
||||
@@ -1029,8 +967,7 @@ class TestSegmentLockProtocol(SegmentLockProtocolFixtures, CustomTestCase):
|
||||
|
||||
@unittest.skipUnless(torch.cuda.is_available(), "cache fixtures need CUDA")
|
||||
class TestSegmentLockFuzz(SegmentLockFuzzFixtures, CustomTestCase):
|
||||
"""Random lock/insert/evict interleavings with the tree's own ledger
|
||||
recomputation (sanity_check) as the per-step oracle, on both cores."""
|
||||
"""Random lock/insert/evict interleavings, with sanity_check as the oracle."""
|
||||
|
||||
cfg = CacheConfig(
|
||||
components=(ComponentType.FULL, ComponentType.SWA),
|
||||
@@ -1051,9 +988,7 @@ class TestSegmentLockFuzz(SegmentLockFuzzFixtures, CustomTestCase):
|
||||
|
||||
@unittest.skipUnless(torch.cuda.is_available(), "cache fixtures need CUDA")
|
||||
class TestSegmentLockFuzzWithMamba(TestSegmentLockFuzz):
|
||||
"""The FULL+SWA+MAMBA (Inkling) shape: the Mamba opt-out receipt, the
|
||||
lower-priority cascade on early SWA release, and Mamba evictions all join
|
||||
the interleavings."""
|
||||
"""The FULL+SWA+MAMBA shape adds the Mamba opt-out and Mamba evictions."""
|
||||
|
||||
cfg = CacheConfig(
|
||||
components=(ComponentType.FULL, ComponentType.SWA, ComponentType.MAMBA),
|
||||
@@ -1067,8 +1002,8 @@ class TestSegmentLockFuzzWithMamba(TestSegmentLockFuzz):
|
||||
class TestStreamingSessionLockLifecycle(
|
||||
StreamingSessionLockLifecycleFixtures, CustomTestCase
|
||||
):
|
||||
"""A streaming session owns its record and prefix lock from the first
|
||||
turn's row allocation; every move of that lock keeps the tree balanced."""
|
||||
"""A streaming session's prefix lock keeps the tree balanced through every
|
||||
move."""
|
||||
|
||||
cfg = CacheConfig(
|
||||
page_size=1,
|
||||
|
||||
-7
@@ -1,5 +1,3 @@
|
||||
"""KV cache event tests for UnifiedRadixCache."""
|
||||
|
||||
import unittest
|
||||
from array import array
|
||||
from unittest import mock
|
||||
@@ -174,7 +172,6 @@ class TestUnifiedRadixCacheKVEvents(UnifiedRadixCacheKVEventsFixtures, CustomTes
|
||||
)
|
||||
|
||||
def test_lora_external_ids_separate_adapter_loads(self):
|
||||
"""External IDs distinguish loads; routers match tokens and public names."""
|
||||
lora_names, _reqs = make_lora_names(
|
||||
**{
|
||||
"uuid-a": "adapter-a",
|
||||
@@ -207,7 +204,6 @@ class TestUnifiedRadixCacheKVEvents(UnifiedRadixCacheKVEventsFixtures, CustomTes
|
||||
for h in self._event_hashes(base)
|
||||
],
|
||||
)
|
||||
# Another load of the same public name has distinct external IDs.
|
||||
self.assertNotEqual(
|
||||
self._event_hashes(store_then_evict("user|uuid-a2")),
|
||||
self._event_hashes(lora_a),
|
||||
@@ -385,7 +381,6 @@ class TestUnifiedRadixCacheKVEvents(UnifiedRadixCacheKVEventsFixtures, CustomTes
|
||||
self.assertCountEqual(self._event_hashes(removed_cpu), stored_hashes)
|
||||
|
||||
def test_hicache_storage_prefetch_publishes_host_only_suffix(self):
|
||||
"""A storage-prefetch refill has no write-through ack to publish it."""
|
||||
from sglang.srt.mem_cache.utils import get_hash_str, hash_str_to_int64
|
||||
|
||||
cache, _, _ = build_fixture(self.cfg, enable_kv_cache_events=True)
|
||||
@@ -403,7 +398,6 @@ class TestUnifiedRadixCacheKVEvents(UnifiedRadixCacheKVEventsFixtures, CustomTes
|
||||
self.assertFalse(result.host_insert_dropped)
|
||||
self.assertIsNotNone(result.inserted_host_node)
|
||||
|
||||
# The recorder coalesces the parent-linked pages into one event.
|
||||
stored_cpu = self._stored_events(cache, StorageMedium.CPU)
|
||||
self.assertEqual(len(stored_cpu), 1)
|
||||
self.assertEqual(list(stored_cpu[0].token_ids), tokens)
|
||||
@@ -441,7 +435,6 @@ class TestUnifiedRadixCacheKVEvents(UnifiedRadixCacheKVEventsFixtures, CustomTes
|
||||
[[1, 2], [3, 4]],
|
||||
)
|
||||
|
||||
# Both split fragments must be published as one parent-linked batch.
|
||||
stored_cpu = self._stored_events(cache, StorageMedium.CPU)
|
||||
self.assertEqual(len(stored_cpu), 1)
|
||||
self.assertEqual(list(stored_cpu[0].token_ids), [1, 2, 3, 4])
|
||||
|
||||
+41
-135
@@ -1,5 +1,3 @@
|
||||
"""HiCache load-back, admission and branching tests for UnifiedRadixCache."""
|
||||
|
||||
import unittest
|
||||
from array import array
|
||||
from dataclasses import replace
|
||||
@@ -9,7 +7,6 @@ import torch
|
||||
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Req,
|
||||
ReqKvInfo,
|
||||
)
|
||||
from sglang.srt.managers.schedule_policy import PrefillAdder
|
||||
from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
@@ -141,7 +138,7 @@ class TestUnifiedTreeCoreLoadBackOwnershipBackends(CustomTestCase):
|
||||
sliding_window_size=4,
|
||||
)
|
||||
|
||||
def test_auxiliary_load_does_not_reuse_full_pending_pin(self):
|
||||
def test_auxiliary_load_restores_swa_and_rebuilds_mapping(self):
|
||||
cache, _, _ = build_fixture(self.cfg)
|
||||
core = cache.tree_core
|
||||
core.is_write_back = True
|
||||
@@ -240,7 +237,6 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures):
|
||||
self._load_back_node(cache, node)
|
||||
self.assertFalse(cache.tree_core.is_full_device_evicted(node))
|
||||
self.assertIsNotNone(node_device_value(cache, node, ComponentType.FULL))
|
||||
# Gather the whole reloaded prefix via match (a leaf may be split).
|
||||
loaded_indices = cache.path_device_indices(
|
||||
match_prefix(cache, base).last_device_node
|
||||
)
|
||||
@@ -288,7 +284,6 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures):
|
||||
self.assertEqual(result.swa_branching_seqlen, len(tokens))
|
||||
self.assertEqual(result.swa_branching_seqlen % self.cfg.page_size, 0)
|
||||
|
||||
# Simulate forward producing fresh SWA KV at the branching point.
|
||||
self._insert(
|
||||
cache,
|
||||
allocator,
|
||||
@@ -352,15 +347,12 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures):
|
||||
|
||||
@requires_full_swa_ps1_window4
|
||||
def test_swa_branching_seqlen_caps_insert_after_forward(self):
|
||||
cache, _, _ = build_fixture(self.cfg)
|
||||
cache, _, req_to_token_pool = build_fixture(self.cfg)
|
||||
swa = cache.components[ComponentType.SWA]
|
||||
req = mock.Mock(
|
||||
swa_branching_seqlen=8,
|
||||
kv=ReqKvInfo(
|
||||
cache_protected_len=4,
|
||||
component_evicted_seqlens={ComponentType.SWA: 3},
|
||||
),
|
||||
)
|
||||
req = self._make_req(req_to_token_pool)
|
||||
req.swa_branching_seqlen = 8
|
||||
req.kv.cache_protected_len = 4
|
||||
req.kv.set_evicted_seqlen(ComponentType.SWA, 3)
|
||||
|
||||
for is_finished in (False, True):
|
||||
params = InsertParams(component_evicted_seqlens={ComponentType.MAMBA: 6})
|
||||
@@ -378,12 +370,7 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures):
|
||||
|
||||
cache.tree_core.is_eagle = True
|
||||
req.kv.cache_protected_len = 4
|
||||
params = InsertParams()
|
||||
self.assertEqual(swa.prepare_for_caching_req(req, params, 12, False), 9)
|
||||
params.key = RadixKey(array("q", range(9)), is_bigram=True)
|
||||
result = mock.Mock(swa_branch_inserted=False)
|
||||
swa.commit_insert_component_data(mock.Mock(), False, params, result, [])
|
||||
self.assertTrue(result.swa_branch_inserted)
|
||||
self.assertEqual(swa.prepare_for_caching_req(req, InsertParams(), 12, False), 9)
|
||||
|
||||
@requires_full_mamba_ps1
|
||||
def test_mamba_branching_seqlen_disabled_under_hicache(self):
|
||||
@@ -615,7 +602,6 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures):
|
||||
self.assertEqual(new_node, match.last_device_node)
|
||||
self.assertIsNone(node_device_value(cache, leaf, ComponentType.FULL))
|
||||
self.assertIsNone(node_device_value(cache, leaf, ComponentType.MAMBA))
|
||||
# A failed load-back must roll back the pre-allocated mamba slot.
|
||||
self.assertIsNone(req.kv.mamba_pool_idx)
|
||||
self.assertEqual(
|
||||
req_to_token_pool.mamba_allocator.available_size(), avail_before
|
||||
@@ -639,10 +625,8 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures):
|
||||
)
|
||||
self._apply_match_to_req(req, match)
|
||||
|
||||
# Simulate a request without its own mamba slot so load-back allocates one.
|
||||
req.kv.mamba_pool_idx = None
|
||||
avail_before = req_to_token_pool.mamba_allocator.available_size()
|
||||
# H->D load fails after the mamba slot is pre-allocated -> must free it.
|
||||
with mock.patch.object(cache.cache_controller, "load", return_value=None):
|
||||
loaded_len, _ = cache.init_load_back(
|
||||
InitLoadBackParams(
|
||||
@@ -677,7 +661,6 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures):
|
||||
)
|
||||
self._apply_match_to_req(req, match)
|
||||
|
||||
# Simulate a request without its own mamba slot so load-back allocates one.
|
||||
req.kv.mamba_pool_idx = None
|
||||
avail_before = req_to_token_pool.mamba_allocator.available_size()
|
||||
# No device room and eviction frees nothing -> load-back bails after the
|
||||
@@ -762,36 +745,6 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures):
|
||||
)
|
||||
self._release_ongoing_load_back_locks(cache)
|
||||
|
||||
@requires_full_mamba_ps1
|
||||
def test_load_back_success_publishes_fresh_mamba_slot(self):
|
||||
cache, allocator, req_to_token_pool = self._build_hicache_fixture()
|
||||
chain = self._build_chain_pages(cache, allocator, req_to_token_pool, 3)
|
||||
if len(chain) < 3:
|
||||
self.skipTest("chain too short")
|
||||
leaf = chain[-1]
|
||||
|
||||
self._backup_node(cache, leaf)
|
||||
cache.evict(EvictParams(num_tokens=node_key_length(cache, leaf)))
|
||||
self.assertTrue(cache.tree_core.is_full_device_evicted(leaf))
|
||||
|
||||
req = self._make_req(req_to_token_pool)
|
||||
req_to_token_pool.mamba_allocator.free(req.kv.mamba_pool_idx.unsqueeze(0))
|
||||
req.kv.mamba_pool_idx = None
|
||||
mamba_avail = req_to_token_pool.mamba_allocator.available_size()
|
||||
|
||||
loaded = cache.load_back(leaf, req=req)
|
||||
|
||||
self.assertTrue(loaded)
|
||||
# the successful load must keep the freshly allocated slot published
|
||||
self.assertIsNotNone(req.kv.mamba_pool_idx)
|
||||
self.assertIsNotNone(node_device_value(cache, leaf, ComponentType.MAMBA))
|
||||
# one slot restores the node's mamba value, one is the request's CoW slot
|
||||
self.assertEqual(
|
||||
req_to_token_pool.mamba_allocator.available_size(), mamba_avail - 2
|
||||
)
|
||||
self._finish_pending_loads(cache)
|
||||
self._release_ongoing_load_back_locks(cache)
|
||||
|
||||
@requires(
|
||||
lambda cfg: (
|
||||
cfg.has_mamba
|
||||
@@ -835,9 +788,8 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures):
|
||||
)
|
||||
adder.add_one_req(req, False, None)
|
||||
|
||||
# The load pins 12 FULL slots, leaving 8 free. Rechecking the original
|
||||
# 18-token demand would reject and scheduler cleanup would free the CoW
|
||||
# destination even though the queued H2D still targets it.
|
||||
# Rechecking the original 18-token demand would reject, and scheduler
|
||||
# cleanup would free the CoW destination while the queued H2D targets it.
|
||||
self.assertEqual(adder.can_run_list, [req])
|
||||
self.assertEqual(req.prefix_len, 12)
|
||||
self.assertEqual(req.kv.cache_protected_len, 12)
|
||||
@@ -858,7 +810,6 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures):
|
||||
self.skipTest("chain too short")
|
||||
leaf = chain[-1]
|
||||
|
||||
# Stamp the node's mamba state so the host backup carries it.
|
||||
node_mamba_indices = node_device_value(cache, leaf, ComponentType.MAMBA).clone()
|
||||
self._fill_mamba_state(req_to_token_pool, node_mamba_indices, marker=11)
|
||||
expected_temporal, expected_conv = self._snapshot_mamba_state(
|
||||
@@ -872,10 +823,16 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures):
|
||||
req = self._make_req(req_to_token_pool)
|
||||
req_to_token_pool.mamba_allocator.free(req.kv.mamba_pool_idx.unsqueeze(0))
|
||||
req.kv.mamba_pool_idx = None
|
||||
mamba_avail = req_to_token_pool.mamba_allocator.available_size()
|
||||
|
||||
loaded = cache.load_back(leaf, req=req)
|
||||
self.assertTrue(loaded)
|
||||
self.assertIsNotNone(req.kv.mamba_pool_idx)
|
||||
self.assertIsNotNone(node_device_value(cache, leaf, ComponentType.MAMBA))
|
||||
# one slot restores the node's mamba value, one is the request's CoW slot
|
||||
self.assertEqual(
|
||||
req_to_token_pool.mamba_allocator.available_size(), mamba_avail - 2
|
||||
)
|
||||
self._finish_pending_loads(cache)
|
||||
|
||||
# The CoW slot must actually hold the backed-up mamba state, not merely exist.
|
||||
@@ -902,28 +859,23 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures):
|
||||
req_to_token_pool.mamba_allocator.free(req.kv.mamba_pool_idx.unsqueeze(0))
|
||||
req.kv.mamba_pool_idx = None
|
||||
|
||||
# device value still present -> nothing to prepare even though host-backed
|
||||
self.assertIsNone(comp.prepare_load_back(leaf, req=req).allocated_mamba_slot)
|
||||
self.assertIsNone(req.kv.mamba_pool_idx)
|
||||
|
||||
cache.evict(EvictParams(num_tokens=node_key_length(cache, leaf)))
|
||||
self.assertTrue(cache.tree_core.is_full_device_evicted(leaf))
|
||||
|
||||
# a request that already owns a slot -> nothing to prepare
|
||||
req_owned = self._make_req(req_to_token_pool)
|
||||
self.assertIsNone(
|
||||
comp.prepare_load_back(leaf, req=req_owned).allocated_mamba_slot
|
||||
)
|
||||
|
||||
# no request -> nothing to prepare
|
||||
self.assertIsNone(comp.prepare_load_back(leaf, req=None).allocated_mamba_slot)
|
||||
|
||||
# fresh request + host-only mamba -> allocates and publishes onto req
|
||||
prep = comp.prepare_load_back(leaf, req=req)
|
||||
self.assertIsNotNone(prep.allocated_mamba_slot)
|
||||
self.assertEqual(int(req.kv.mamba_pool_idx), int(prep.allocated_mamba_slot[0]))
|
||||
|
||||
# node without host-backed mamba -> nothing to prepare
|
||||
req2 = self._make_req(req_to_token_pool)
|
||||
req_to_token_pool.mamba_allocator.free(req2.kv.mamba_pool_idx.unsqueeze(0))
|
||||
req2.kv.mamba_pool_idx = None
|
||||
@@ -987,9 +939,7 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures):
|
||||
|
||||
self._simulate_backup_tree(cache)
|
||||
|
||||
# Tombstone every chain node on the device side without going through
|
||||
# the tree-wide eviction loop. This isolates build_hicache_transfers
|
||||
# from LRU and cascade ordering.
|
||||
# Tombstone every chain node directly, bypassing the eviction loop.
|
||||
for n in chain:
|
||||
cache.tree_core.set_component_device_value_raw(n, ComponentType.FULL, None)
|
||||
cache.tree_core.set_component_device_value_raw(n, ComponentType.SWA, None)
|
||||
@@ -1003,7 +953,6 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures):
|
||||
xfer = transfers[0]
|
||||
self.assertEqual(xfer.name, PoolName.SWA)
|
||||
self.assertEqual(len(xfer.nodes_to_load), expected_pages)
|
||||
# host_indices must cover exactly the expected suffix tokens (>= sw).
|
||||
self.assertEqual(int(xfer.host_indices.numel()), expected_pages * ps)
|
||||
self.assertGreaterEqual(int(xfer.host_indices.numel()), sw)
|
||||
self.assertEqual(xfer.nodes_to_load, chain[-expected_pages:])
|
||||
@@ -1018,17 +967,11 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures):
|
||||
"window must span past the leaf to reach the pin",
|
||||
)
|
||||
def test_hicache_swa_load_back_rejects_foreign_pinned_window(self):
|
||||
"""A second load-back must not claim nodes pinned by an in-flight one.
|
||||
|
||||
commit_load_back republishes Full device values before the DMA acks, so
|
||||
a later anchor can claim just the still-host-only SWA window of a
|
||||
pinned node and hit the commit pin assert. The spec build must degrade
|
||||
to an empty spec instead (the caller recomputes).
|
||||
"""
|
||||
"""A second load-back must not claim nodes pinned by an in-flight one;
|
||||
the spec build degrades to empty (caller recomputes)."""
|
||||
|
||||
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
|
||||
# commit_load_back pins its source nodes only under write-back; run the
|
||||
# scenario in that mode so the foreign-pin path is reachable.
|
||||
# commit_load_back pins its source nodes only under write-back.
|
||||
cache.is_write_back = True
|
||||
chain = self._build_chain_pages(cache, allocator, req_to_token_pool, 3)
|
||||
if len(chain) < 3:
|
||||
@@ -1058,7 +1001,6 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures):
|
||||
self.assertEqual(kv_xfer.nodes_to_load, [])
|
||||
self.assertEqual(comp_xfers, {})
|
||||
|
||||
# After the ack unpins, the same spec builds fully.
|
||||
cache.tree_core.finish_load_back(a)
|
||||
kv_xfer, comp_xfers = cache.tree_core.build_load_back_spec(b)
|
||||
self.assertEqual(kv_xfer.nodes_to_load, [b])
|
||||
@@ -1070,78 +1012,42 @@ class UnifiedRadixCacheLoadBackSuite(UnifiedRadixCacheLoadBackFixtures):
|
||||
"SWA-only path keeps the chain construction simple",
|
||||
)
|
||||
def test_hicache_swa_commit_load_back_rebuilds_mapping(self):
|
||||
"""LOAD_BACK commit must:
|
||||
(1) restore SWA cd.value via _restore_device_value (host LRU -> device LRU),
|
||||
(2) rewrite full_to_swa_index_mapping[full_idx] = new_swa_idx for every
|
||||
loaded chunk so subsequent SWA reads via translate_loc_from_full_to_swa
|
||||
return the freshly allocated SWA slot."""
|
||||
|
||||
cache, allocator, _, chain, window_pages = self._swa_finalize_setup()
|
||||
|
||||
# Tombstone every SWA node in the trailing window.
|
||||
"""A SWA load-back restores each tombstoned window node's SWA to device
|
||||
and maps restored Full slots to their SWA slots."""
|
||||
ps = self.cfg.page_size
|
||||
window_pages = (self.cfg.sliding_window_size + ps - 1) // ps
|
||||
cache, allocator, req_to_token_pool = self._build_hicache_fixture()
|
||||
chain = self._build_chain_pages(
|
||||
cache, allocator, req_to_token_pool, window_pages + 2
|
||||
)
|
||||
if len(chain) <= window_pages:
|
||||
self.skipTest("chain collapsed below the window length")
|
||||
tokens = self._match_tokens_for_chain(cache, chain)
|
||||
self._backup_node(cache, chain[-1])
|
||||
loaded_nodes = chain[-window_pages:]
|
||||
for n in loaded_nodes:
|
||||
cache.tree_core.set_component_device_value_raw(n, ComponentType.SWA, None)
|
||||
# SWA LRU bookkeeping must reflect tombstone state for the
|
||||
# _restore_device_value path to exercise the host->device move.
|
||||
cache.tree_core.remove_node_from_device_lru(n, ComponentType.SWA)
|
||||
cache.tree_core.insert_node_into_host_lru(n, ComponentType.SWA)
|
||||
|
||||
# Build the LOAD_BACK transfer the same way load_back() would.
|
||||
transfers = cache.tree_core.build_hicache_transfers(
|
||||
ComponentType.SWA, chain[-1], CacheTransferPhase.LOAD_BACK
|
||||
)
|
||||
self.assertIsNotNone(transfers)
|
||||
xfer = transfers[0]
|
||||
self.assertEqual(xfer.nodes_to_load, loaded_nodes)
|
||||
|
||||
# Allocate SWA device slots from the inner allocator (mirrors how
|
||||
# _resolve_device_transfers routes via device_alloc_fn ->
|
||||
# swa_attn_allocator.alloc on the load-back path).
|
||||
n_swa = int(xfer.host_indices.numel())
|
||||
new_swa = allocator.swa_attn_allocator.alloc(n_swa)
|
||||
self.assertIsNotNone(new_swa)
|
||||
xfer.device_indices = new_swa
|
||||
|
||||
# Snapshot pre-commit state for invariants checks.
|
||||
self._evict_device_component(cache, n, ComponentType.SWA)
|
||||
pre_evictable = cache.tree_core.component_evictable_size(ComponentType.SWA)
|
||||
|
||||
load_actions = []
|
||||
cache.tree_core.commit_hicache_transfers(
|
||||
chain[-1],
|
||||
CacheTransferPhase.LOAD_BACK,
|
||||
{ComponentType.SWA: transfers},
|
||||
cache_actions=load_actions,
|
||||
)
|
||||
cache._apply_cache_actions(load_actions)
|
||||
self._load_back_through_admission(cache, req_to_token_pool, tokens)
|
||||
self._finish_pending_loads(cache)
|
||||
self._release_ongoing_load_back_locks(cache)
|
||||
|
||||
# (1) cd.value restored, host LRU -> device LRU swap done.
|
||||
offset = 0
|
||||
restored = 0
|
||||
for n in loaded_nodes:
|
||||
value = node_device_value(cache, n, ComponentType.SWA)
|
||||
self.assertIsNotNone(value)
|
||||
chunk_len = int(value.numel())
|
||||
self.assertEqual(
|
||||
value.tolist(),
|
||||
new_swa[offset : offset + chunk_len].tolist(),
|
||||
)
|
||||
offset += chunk_len
|
||||
swa_idx = node_device_value(cache, n, ComponentType.SWA)
|
||||
self.assertIsNotNone(swa_idx)
|
||||
self.assertTrue(cache.tree_core.is_node_in_device_lru(n, ComponentType.SWA))
|
||||
self.assertFalse(cache.tree_core.is_node_in_host_lru(n, ComponentType.SWA))
|
||||
self.assertEqual(offset, n_swa)
|
||||
|
||||
# (2) full_to_swa_index_mapping rebuilt for every loaded chunk.
|
||||
for n in loaded_nodes:
|
||||
full_idx = node_device_value(cache, n, ComponentType.FULL)
|
||||
swa_idx = node_device_value(cache, n, ComponentType.SWA)
|
||||
translated = allocator.translate_loc_from_full_to_swa(full_idx)
|
||||
self.assertEqual(translated.tolist(), swa_idx.tolist())
|
||||
|
||||
# Evictable size moved up by the restored token count.
|
||||
restored += int(swa_idx.numel())
|
||||
self.assertEqual(
|
||||
cache.tree_core.component_evictable_size(ComponentType.SWA) - pre_evictable,
|
||||
n_swa,
|
||||
restored,
|
||||
)
|
||||
cache.sanity_check()
|
||||
|
||||
def test_hicache_swa_load_back_anchored_on_best_match_node(self):
|
||||
cache, _, _, y, x, _ = self._swa_anchor_setup()
|
||||
|
||||
+214
-356
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user