1657 lines
62 KiB
Python
1657 lines
62 KiB
Python
import queue
|
|
import sys
|
|
import threading
|
|
import unittest
|
|
from array import array
|
|
from collections import defaultdict
|
|
from dataclasses import replace
|
|
from types import SimpleNamespace
|
|
from unittest.mock import MagicMock, call, patch
|
|
|
|
import pytest
|
|
import torch
|
|
|
|
from sglang.srt.managers.schedule_batch import ReqKvInfo
|
|
from sglang.srt.mem_cache.allocator.swa import SWATokenToKVPoolAllocator
|
|
from sglang.srt.mem_cache.base_prefix_cache import (
|
|
InitLoadBackParams,
|
|
MatchPrefixParams,
|
|
MatchResult,
|
|
)
|
|
from sglang.srt.mem_cache.common import release_kv_cache
|
|
from sglang.srt.mem_cache.hicache_storage import (
|
|
PoolHitPolicy,
|
|
PoolName,
|
|
PoolTransfer,
|
|
)
|
|
from sglang.srt.mem_cache.radix_cache import RadixKey
|
|
from sglang.srt.mem_cache.storage.mooncake_store.mooncake_direct_linker import (
|
|
MooncakeDirectLinker,
|
|
)
|
|
from sglang.srt.mem_cache.unified_cache.cache_action import (
|
|
UpdatePendingBackupOnNodeSplit,
|
|
)
|
|
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
|
|
from sglang.srt.mem_cache.unified_cache.components.base import (
|
|
ExternalLinkerLoadPhase,
|
|
LinkerTransferPhase,
|
|
)
|
|
from sglang.srt.mem_cache.unified_cache.components.swa import SWAComponent
|
|
from sglang.srt.mem_cache.unified_cache.unified_cache_linker import (
|
|
ExternalCacheHitMarker,
|
|
LayerWiseLoadCounter,
|
|
UnifiedCacheLinker,
|
|
UnifiedCacheLinkerWrapper,
|
|
)
|
|
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
|
|
from sglang.srt.runtime_context import get_disagg
|
|
from sglang.test.ci.ci_register import register_cpu_ci, register_cuda_ci
|
|
from sglang.test.kits.unified_radix_cache import fixtures as shared_cache_suite
|
|
from sglang.test.kits.unified_radix_cache.config import (
|
|
CacheConfig,
|
|
)
|
|
from sglang.test.kits.unified_radix_cache.fixtures import (
|
|
build_fixture,
|
|
node_children,
|
|
node_device_lock_ref,
|
|
node_device_value,
|
|
)
|
|
from sglang.test.kits.unified_radix_cache.mixins import UnifiedRadixCacheFixtures
|
|
from sglang.test.test_utils import CustomTestCase
|
|
|
|
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
|
|
register_cuda_ci(est_time=100, stage="base-b", runner_config="1-gpu-small")
|
|
|
|
|
|
class _FakeLinker(UnifiedCacheLinker):
|
|
def __init__(self):
|
|
self.layer_done_counter = object()
|
|
self.restorable = []
|
|
self.queued_loads = {}
|
|
self.queued_offloads = []
|
|
self.load_landed = True
|
|
self.completed_offloads = []
|
|
self.reset_count = 0
|
|
self.closed = False
|
|
self.pools = {}
|
|
|
|
def lookup(self, rid, transfers):
|
|
return list(self.restorable)
|
|
|
|
def load(self, rid, transfers):
|
|
self.queued_loads[rid] = list(transfers)
|
|
return True
|
|
|
|
def start_layer_wise_loading(self):
|
|
return 3
|
|
|
|
def cancel_queued_load(self, rid):
|
|
if rid not in self.queued_loads:
|
|
return False
|
|
del self.queued_loads[rid]
|
|
return True
|
|
|
|
def finish_layer_wise_loading(self, counter_index):
|
|
return self.load_landed
|
|
|
|
def offload(self, transfers):
|
|
self.queued_offloads.append(list(transfers))
|
|
return True
|
|
|
|
def num_completed_offloads(self):
|
|
return len(self.completed_offloads)
|
|
|
|
def pop_completed_offload(self):
|
|
return self.completed_offloads.pop(0)
|
|
|
|
def reset(self):
|
|
self.reset_count += 1
|
|
|
|
def close(self):
|
|
self.closed = True
|
|
|
|
|
|
class _MappingRecorder:
|
|
def __init__(self):
|
|
self.mapping = []
|
|
|
|
def set_full_to_swa_mapping(self, full, swa):
|
|
self.mapping.append((full.clone(), swa.clone()))
|
|
|
|
|
|
class _FakeExternalTreeCore:
|
|
def __init__(self, nodes=None, offload_transfers=None):
|
|
self.enable_external_cache_linker = False
|
|
self.nodes = nodes or {}
|
|
self.offload_transfers = offload_transfers or [
|
|
PoolTransfer(name=PoolName.KV, keys=["page"])
|
|
]
|
|
|
|
def build_external_linker_offload_transfers(self, node_id):
|
|
node = self.nodes[node_id]
|
|
if node.external_cache_stored or node.write_through_pending_id is not None:
|
|
return None
|
|
return list(self.offload_transfers)
|
|
|
|
def mark_external_linker_offload_pending(self, node_id):
|
|
node = self.nodes[node_id]
|
|
assert (
|
|
not node.external_cache_stored and node.write_through_pending_id is None
|
|
), "invalid external offload state"
|
|
node.write_through_pending_id = node_id
|
|
|
|
def finish_external_linker_offload(self, node_ids, ack_id, success):
|
|
nodes = [self.nodes[node_id] for node_id in node_ids]
|
|
assert all(node.write_through_pending_id == ack_id for node in nodes), (
|
|
"invalid external offload state"
|
|
)
|
|
for node in nodes:
|
|
node.write_through_pending_id = None
|
|
node.external_cache_stored |= success
|
|
|
|
|
|
def _cache_for_wrapper(**kwargs):
|
|
defaults = {
|
|
"_components_tuple": (),
|
|
"components": {},
|
|
"tree_core": SimpleNamespace(enable_external_cache_linker=False),
|
|
"tree_components": (ComponentType.FULL,),
|
|
"write_through_threshold": 256,
|
|
"pp_size": 1,
|
|
"pp_group": None,
|
|
}
|
|
defaults.update(kwargs)
|
|
return SimpleNamespace(**defaults)
|
|
|
|
|
|
def _swa_allocator(swa_req_ring):
|
|
if swa_req_ring is None:
|
|
return SimpleNamespace()
|
|
allocator = SWATokenToKVPoolAllocator.__new__(SWATokenToKVPoolAllocator)
|
|
allocator._swa_req_ring = swa_req_ring
|
|
return allocator
|
|
|
|
|
|
def test_cache_linker_attachment_is_backend_independent():
|
|
cache = UnifiedRadixCache.__new__(UnifiedRadixCache)
|
|
cache.tree_core = SimpleNamespace(
|
|
enable_external_cache_linker=False,
|
|
write_through_threshold=256,
|
|
)
|
|
cache.tree_components = (ComponentType.FULL,)
|
|
cache.linker = None
|
|
cache._components_tuple = ()
|
|
cache.components = {}
|
|
linker = _FakeLinker()
|
|
|
|
cache.init_cache_linker(linker)
|
|
|
|
assert cache.linker.cache_linker is linker
|
|
assert cache.tree_core.enable_external_cache_linker
|
|
assert cache.write_through_threshold == 1
|
|
assert cache.linker.layer_done_counter is linker.layer_done_counter
|
|
|
|
|
|
@pytest.mark.parametrize("component_type", [ComponentType.C128])
|
|
def test_cache_linker_rejects_unsupported_tree_components(component_type):
|
|
cache = _cache_for_wrapper(tree_components=(ComponentType.FULL, component_type))
|
|
|
|
with pytest.raises(ValueError, match=component_type.name):
|
|
UnifiedCacheLinkerWrapper(cache, _FakeLinker())
|
|
|
|
assert not cache.tree_core.enable_external_cache_linker
|
|
|
|
|
|
class _InMemoryUnifiedCacheLinker(UnifiedCacheLinker):
|
|
"""Controllable transport for shared Python/Rust TreeCore tests."""
|
|
|
|
def __init__(self, stored_keys=None):
|
|
self.layer_done_counter = object()
|
|
self.pools = dict.fromkeys(PoolName)
|
|
self.stored_keys = defaultdict(set) if stored_keys is None else stored_keys
|
|
self.lookup_calls = []
|
|
self.offload_calls = []
|
|
self.pending_offloads = []
|
|
self.queued_loads = {}
|
|
self.started_loads = []
|
|
self.load_landed = True
|
|
self.completed_offloads = []
|
|
|
|
@staticmethod
|
|
def _clone_transfer(transfer):
|
|
return replace(
|
|
transfer,
|
|
host_indices=(
|
|
None if transfer.host_indices is None else transfer.host_indices.clone()
|
|
),
|
|
device_indices=(
|
|
None
|
|
if transfer.device_indices is None
|
|
else transfer.device_indices.clone()
|
|
),
|
|
keys=None if transfer.keys is None else list(transfer.keys),
|
|
nodes_to_load=(
|
|
None if transfer.nodes_to_load is None else list(transfer.nodes_to_load)
|
|
),
|
|
)
|
|
|
|
@classmethod
|
|
def _clone_transfers(cls, transfers):
|
|
return [cls._clone_transfer(transfer) for transfer in transfers]
|
|
|
|
def lookup(self, rid, transfers):
|
|
transfers = self._clone_transfers(transfers)
|
|
self.lookup_calls.append((rid, transfers))
|
|
by_pool = {transfer.name: transfer for transfer in transfers}
|
|
kv = by_pool.get(PoolName.KV)
|
|
if kv is None or not kv.keys:
|
|
return []
|
|
|
|
restorable = []
|
|
for prefix_pages in range(1, len(kv.keys) + 1):
|
|
if not set(kv.keys[:prefix_pages]) <= self.stored_keys[PoolName.KV]:
|
|
continue
|
|
for transfer in transfers:
|
|
if transfer.name == PoolName.KV:
|
|
continue
|
|
if transfer.hit_policy == PoolHitPolicy.TRAILING_PAGES:
|
|
window_pages = max(1, len(transfer.keys or ()))
|
|
required = kv.keys[
|
|
max(0, prefix_pages - window_pages) : prefix_pages
|
|
]
|
|
else:
|
|
required = kv.keys[:prefix_pages]
|
|
if not set(required) <= self.stored_keys[transfer.name]:
|
|
break
|
|
else:
|
|
restorable.append(prefix_pages)
|
|
return restorable
|
|
|
|
def load(self, rid, transfers):
|
|
self.queued_loads[rid] = self._clone_transfers(transfers)
|
|
return True
|
|
|
|
def start_layer_wise_loading(self):
|
|
if not self.queued_loads:
|
|
return -1
|
|
rids = list(self.queued_loads)
|
|
self.started_loads.append(rids)
|
|
return len(self.started_loads) - 1
|
|
|
|
def cancel_queued_load(self, rid):
|
|
return self.queued_loads.pop(rid, None) is not None
|
|
|
|
def finish_layer_wise_loading(self, counter_index):
|
|
for rid in self.started_loads[counter_index]:
|
|
self.queued_loads.pop(rid, None)
|
|
return self.load_landed
|
|
|
|
def offload(self, transfers):
|
|
transfers = self._clone_transfers(transfers)
|
|
self.offload_calls.append(transfers)
|
|
self.pending_offloads.append(transfers)
|
|
return True
|
|
|
|
def num_completed_offloads(self):
|
|
return len(self.completed_offloads)
|
|
|
|
def pop_completed_offload(self):
|
|
return self.completed_offloads.pop(0)
|
|
|
|
def complete_next_offload(self, success):
|
|
transfers = self.pending_offloads.pop(0)
|
|
if success:
|
|
for transfer in transfers:
|
|
self.stored_keys[transfer.name].update(transfer.keys or ())
|
|
self.completed_offloads.append(success)
|
|
|
|
def reset(self):
|
|
self.queued_loads.clear()
|
|
self.pending_offloads.clear()
|
|
self.completed_offloads.clear()
|
|
|
|
def close(self):
|
|
self.reset()
|
|
|
|
|
|
class _TreeCoreBackendTestMixin:
|
|
tree_core_backend = "python"
|
|
|
|
def setUp(self):
|
|
previous = shared_cache_suite.TREE_CORE_TEST_BACKEND
|
|
self.addCleanup(
|
|
setattr,
|
|
shared_cache_suite,
|
|
"TREE_CORE_TEST_BACKEND",
|
|
previous,
|
|
)
|
|
shared_cache_suite.TREE_CORE_TEST_BACKEND = self.tree_core_backend
|
|
super().setUp()
|
|
|
|
|
|
@unittest.skipUnless(torch.cuda.is_available(), "cache fixtures need CUDA")
|
|
class TestUnifiedCacheLinkerPythonBackend(
|
|
_TreeCoreBackendTestMixin, UnifiedRadixCacheFixtures, CustomTestCase
|
|
):
|
|
def _bind_loaded_request(self, cache, req_to_token_pool, req, tokens, loaded):
|
|
"""Leave ``req`` as admission does: loaded tail written, owned by ``req``."""
|
|
req.kv.cache_protected_len = req.prefix_len
|
|
req.prefix_len += len(loaded)
|
|
prefix = cache.prefix_device_indices(req)
|
|
self.assertTrue(torch.equal(prefix[-len(loaded) :], loaded))
|
|
req_to_token_pool.write((req.kv.req_pool_idx, slice(0, len(prefix))), prefix)
|
|
req.origin_input_ids, req.output_ids = array("q", tokens), array("q")
|
|
req.full_untruncated_fill_ids = array("q", tokens)
|
|
req.extend_end = len(tokens)
|
|
req.kv.kv_committed_len = len(prefix)
|
|
req.kv.kv_allocated_len = len(prefix)
|
|
req.extra_key = None
|
|
req.lock = cache.lock(req.last_node)
|
|
|
|
def test_full_offload_load_round_trip_and_dedup(self):
|
|
cfg = CacheConfig(page_size=2, kv_size=64, max_context_len=64)
|
|
self.cfg = cfg
|
|
stored_keys = defaultdict(set)
|
|
|
|
producer, producer_allocator, producer_req_pool = build_fixture(cfg)
|
|
producer_linker = _InMemoryUnifiedCacheLinker(stored_keys)
|
|
producer.init_cache_linker(producer_linker)
|
|
tokens = list(range(1, 9))
|
|
|
|
inserted = self._insert(producer, producer_allocator, producer_req_pool, tokens)
|
|
self.assertEqual(len(producer_linker.offload_calls), 1)
|
|
(kv_offload,) = producer_linker.offload_calls[0]
|
|
self.assertEqual(kv_offload.name, PoolName.KV)
|
|
self.assertEqual(
|
|
kv_offload.keys,
|
|
producer.tree_core.get_hash_values(inserted.last_device_node),
|
|
)
|
|
self.assertTrue(
|
|
torch.equal(
|
|
kv_offload.device_indices,
|
|
node_device_value(
|
|
producer, inserted.last_device_node, ComponentType.FULL
|
|
),
|
|
)
|
|
)
|
|
self.assertEqual(
|
|
node_device_lock_ref(
|
|
producer, inserted.last_device_node, ComponentType.FULL
|
|
),
|
|
1,
|
|
)
|
|
self.assertFalse(
|
|
producer.tree_core.is_external_cache_stored(inserted.last_device_node)
|
|
)
|
|
self.assertEqual(
|
|
producer.tree_core.get_write_through_pending_id(inserted.last_device_node),
|
|
inserted.last_device_node,
|
|
)
|
|
|
|
self._insert(producer, producer_allocator, producer_req_pool, tokens)
|
|
self.assertEqual(len(producer_linker.offload_calls), 1)
|
|
|
|
producer_linker.complete_next_offload(True)
|
|
producer.check_hicache_events()
|
|
self.assertEqual(
|
|
node_device_lock_ref(
|
|
producer, inserted.last_device_node, ComponentType.FULL
|
|
),
|
|
0,
|
|
)
|
|
self.assertTrue(
|
|
producer.tree_core.is_external_cache_stored(inserted.last_device_node)
|
|
)
|
|
self._insert(producer, producer_allocator, producer_req_pool, tokens)
|
|
self.assertEqual(len(producer_linker.offload_calls), 1)
|
|
|
|
consumer, _, consumer_req_pool = build_fixture(cfg)
|
|
consumer_linker = _InMemoryUnifiedCacheLinker(stored_keys)
|
|
consumer.init_cache_linker(consumer_linker)
|
|
req = self._make_req(consumer_req_pool)
|
|
match = consumer.match_prefix(
|
|
MatchPrefixParams(key=RadixKey(array("q", tokens)), req=req)
|
|
)
|
|
self.assertEqual(match.device_prefix_len, 0)
|
|
self.assertEqual(match.host_hit_length, len(tokens))
|
|
self._apply_match_to_req(req, match)
|
|
|
|
loaded_len, loaded_node = consumer.init_load_back(
|
|
InitLoadBackParams(
|
|
best_match_node=match.best_match_node,
|
|
host_hit_length=match.host_hit_length,
|
|
req=req,
|
|
)
|
|
)
|
|
self.assertEqual(loaded_len, len(tokens))
|
|
loaded = consumer.linker.private_load_indices[req.rid]
|
|
# The tail is request-private: nothing is published until it lands.
|
|
self.assertEqual(loaded_node, req.last_node)
|
|
(kv_load,) = consumer_linker.queued_loads[req.rid]
|
|
self.assertEqual(kv_load.name, PoolName.KV)
|
|
self.assertEqual(kv_load.keys, kv_offload.keys)
|
|
self.assertTrue(torch.equal(kv_load.device_indices, loaded))
|
|
|
|
self.assertGreaterEqual(consumer.ready_to_load_host_cache(), 0)
|
|
self._bind_loaded_request(consumer, consumer_req_pool, req, tokens, loaded)
|
|
self.assertEqual(consumer.finish_external_linker_loads([req]), [])
|
|
consumer.checkpoint(req, up_to=req.extend_end)
|
|
consumer.unlock(req.lock)
|
|
req.lock = None
|
|
final_match = consumer.match_prefix(
|
|
MatchPrefixParams(key=RadixKey(array("q", tokens)))
|
|
)
|
|
self.assertTrue(
|
|
torch.equal(
|
|
consumer.path_device_indices(final_match.last_device_node), loaded
|
|
)
|
|
)
|
|
consumer_linker.complete_next_offload(True)
|
|
consumer.check_hicache_events()
|
|
consumer.sanity_check()
|
|
|
|
def test_eagle_lookup_uses_bigram_tail_hashes(self):
|
|
cfg = CacheConfig(
|
|
page_size=2,
|
|
is_eagle=True,
|
|
kv_size=64,
|
|
max_context_len=64,
|
|
)
|
|
self.cfg = cfg
|
|
stored_keys = defaultdict(set)
|
|
tokens = list(range(1, 10))
|
|
|
|
producer, producer_allocator, producer_req_pool = build_fixture(cfg)
|
|
producer_linker = _InMemoryUnifiedCacheLinker(stored_keys)
|
|
producer.init_cache_linker(producer_linker)
|
|
self._insert(producer, producer_allocator, producer_req_pool, tokens)
|
|
self.assertEqual(len(producer_linker.offload_calls), 1)
|
|
(kv_offload,) = producer_linker.offload_calls[0]
|
|
self.assertEqual(len(kv_offload.keys), 4)
|
|
producer_linker.complete_next_offload(True)
|
|
producer.check_hicache_events()
|
|
|
|
consumer, consumer_allocator, consumer_req_pool = build_fixture(cfg)
|
|
consumer_linker = _InMemoryUnifiedCacheLinker(stored_keys)
|
|
consumer.init_cache_linker(consumer_linker)
|
|
consumer.write_through_threshold = sys.maxsize
|
|
self._insert(
|
|
consumer,
|
|
consumer_allocator,
|
|
consumer_req_pool,
|
|
tokens[:5],
|
|
)
|
|
req = self._make_req(consumer_req_pool)
|
|
lookup_key = RadixKey(array("q", tokens))
|
|
|
|
match = consumer.match_prefix(MatchPrefixParams(key=lookup_key, req=req))
|
|
|
|
self.assertTrue(lookup_key.is_bigram)
|
|
self.assertEqual(match.device_prefix_len, 4)
|
|
self.assertEqual(match.host_hit_length, 4)
|
|
self.assertEqual(len(consumer_linker.lookup_calls), 1)
|
|
_, transfers = consumer_linker.lookup_calls[0]
|
|
(kv_lookup,) = transfers
|
|
self.assertEqual(kv_lookup.keys, kv_offload.keys[2:])
|
|
|
|
def test_decode_match_skips_linker_lookup(self):
|
|
cfg = CacheConfig(page_size=2, kv_size=64, max_context_len=64)
|
|
self.cfg = cfg
|
|
stored_keys = defaultdict(set)
|
|
producer, producer_allocator, producer_req_pool = build_fixture(cfg)
|
|
producer_linker = _InMemoryUnifiedCacheLinker(stored_keys)
|
|
producer.init_cache_linker(producer_linker)
|
|
tokens = list(range(1, 9))
|
|
self._insert(producer, producer_allocator, producer_req_pool, tokens)
|
|
producer_linker.complete_next_offload(True)
|
|
producer.check_hicache_events()
|
|
|
|
consumer, _, consumer_req_pool = build_fixture(cfg)
|
|
consumer_linker = _InMemoryUnifiedCacheLinker(stored_keys)
|
|
consumer.init_cache_linker(consumer_linker)
|
|
key = RadixKey(array("q", tokens))
|
|
with get_disagg().override(disaggregation_mode="decode"):
|
|
req = self._make_req(consumer_req_pool)
|
|
match = consumer.match_prefix(MatchPrefixParams(key=key, req=req))
|
|
self.assertEqual(match.host_hit_length, 0)
|
|
self.assertEqual(consumer_linker.lookup_calls, [])
|
|
self.assertFalse(consumer.linker.has_hit(req.rid))
|
|
|
|
req = self._make_req(consumer_req_pool)
|
|
match = consumer.match_prefix(MatchPrefixParams(key=key, req=req))
|
|
self.assertEqual(match.host_hit_length, len(tokens))
|
|
consumer.linker.release_request(req.rid)
|
|
|
|
def test_failed_split_offload_retries_and_reset_releases_locks(self):
|
|
cfg = CacheConfig(page_size=1, kv_size=64, max_context_len=64)
|
|
self.cfg = cfg
|
|
cache, allocator, req_to_token_pool = build_fixture(cfg)
|
|
linker = _InMemoryUnifiedCacheLinker()
|
|
cache.init_cache_linker(linker)
|
|
tokens = [1, 2, 3, 4]
|
|
|
|
inserted = self._insert(cache, allocator, req_to_token_pool, tokens)
|
|
original_node = inserted.last_device_node
|
|
self.assertEqual(len(linker.offload_calls), 1)
|
|
original_keys = linker.offload_calls[0][0].keys
|
|
|
|
self._insert(cache, allocator, req_to_token_pool, tokens[:2])
|
|
(parent,) = node_children(cache, cache.root_node_handle())
|
|
(child,) = node_children(cache, parent)
|
|
self.assertEqual(child, original_node)
|
|
self.assertEqual(
|
|
cache.linker.pending_offloads[0].publish_node_ids, [parent, child]
|
|
)
|
|
self.assertEqual(cache.tree_core.get_write_through_pending_id(parent), child)
|
|
self.assertEqual(cache.tree_core.get_write_through_pending_id(child), child)
|
|
self.assertFalse(cache.tree_core.is_external_cache_stored(parent))
|
|
self.assertFalse(cache.tree_core.is_external_cache_stored(child))
|
|
|
|
linker.complete_next_offload(False)
|
|
cache.check_hicache_events()
|
|
for node_id in (parent, child):
|
|
self.assertIsNone(cache.tree_core.get_write_through_pending_id(node_id))
|
|
self.assertFalse(cache.tree_core.is_external_cache_stored(node_id))
|
|
self.assertEqual(
|
|
node_device_lock_ref(cache, node_id, ComponentType.FULL), 0
|
|
)
|
|
|
|
self._insert(cache, allocator, req_to_token_pool, tokens)
|
|
retry_calls = linker.offload_calls[1:]
|
|
self.assertEqual(len(retry_calls), 2)
|
|
self.assertEqual(
|
|
[key for transfers in retry_calls for key in transfers[0].keys],
|
|
original_keys,
|
|
)
|
|
for _ in retry_calls:
|
|
linker.complete_next_offload(True)
|
|
cache.check_hicache_events()
|
|
for node_id in (parent, child):
|
|
self.assertIsNone(cache.tree_core.get_write_through_pending_id(node_id))
|
|
self.assertTrue(cache.tree_core.is_external_cache_stored(node_id))
|
|
self.assertEqual(
|
|
node_device_lock_ref(cache, node_id, ComponentType.FULL), 0
|
|
)
|
|
|
|
self._insert(cache, allocator, req_to_token_pool, tokens)
|
|
self.assertEqual(len(linker.offload_calls), 3)
|
|
|
|
extended = self._insert(cache, allocator, req_to_token_pool, tokens + [5, 6])
|
|
pending_node = extended.last_device_node
|
|
self.assertEqual(len(cache.linker.pending_offloads), 1)
|
|
self.assertEqual(
|
|
cache.tree_core.get_write_through_pending_id(pending_node), pending_node
|
|
)
|
|
self.assertEqual(
|
|
node_device_lock_ref(cache, pending_node, ComponentType.FULL), 1
|
|
)
|
|
|
|
cache.linker.reset()
|
|
self.assertEqual(cache.linker.pending_offloads, [])
|
|
self.assertIsNone(cache.tree_core.get_write_through_pending_id(pending_node))
|
|
self.assertEqual(
|
|
node_device_lock_ref(cache, pending_node, ComponentType.FULL), 0
|
|
)
|
|
cache.sanity_check()
|
|
cache.reset()
|
|
cache.sanity_check()
|
|
|
|
def test_swa_hit_loads_private_tail_and_dedups_on_insert(self):
|
|
self._run_swa_hit(load_landed=True)
|
|
|
|
def test_swa_failed_load_frees_private_slots_unpublished(self):
|
|
self._run_swa_hit(load_landed=False)
|
|
|
|
def _run_swa_hit(self, load_landed):
|
|
cfg = CacheConfig(
|
|
page_size=1,
|
|
components=(ComponentType.FULL, ComponentType.SWA),
|
|
sliding_window_size=2,
|
|
kv_size=64,
|
|
max_context_len=64,
|
|
)
|
|
self.cfg = cfg
|
|
stored_keys = defaultdict(set)
|
|
tokens = list(range(1, 7))
|
|
|
|
producer, producer_allocator, producer_req_pool = build_fixture(cfg)
|
|
producer_linker = _InMemoryUnifiedCacheLinker(stored_keys)
|
|
producer.init_cache_linker(producer_linker)
|
|
self._insert(producer, producer_allocator, producer_req_pool, tokens)
|
|
self.assertGreaterEqual(len(producer_linker.offload_calls), 1)
|
|
all_keys = []
|
|
for transfers in producer_linker.offload_calls:
|
|
offloads = {transfer.name: transfer for transfer in transfers}
|
|
self.assertEqual(set(offloads), {PoolName.KV, PoolName.SWA})
|
|
self.assertEqual(offloads[PoolName.KV].keys, offloads[PoolName.SWA].keys)
|
|
all_keys.extend(offloads[PoolName.KV].keys)
|
|
producer_linker.complete_next_offload(True)
|
|
producer.check_hicache_events()
|
|
|
|
stored_keys[PoolName.KV].difference_update(all_keys[-2:])
|
|
|
|
consumer, consumer_allocator, consumer_req_pool = build_fixture(cfg)
|
|
consumer_linker = _InMemoryUnifiedCacheLinker(stored_keys)
|
|
consumer.init_cache_linker(consumer_linker)
|
|
consumer.write_through_threshold = sys.maxsize
|
|
self._insert(consumer, consumer_allocator, consumer_req_pool, tokens[:2])
|
|
|
|
req = self._make_req(consumer_req_pool)
|
|
match = consumer.match_prefix(
|
|
MatchPrefixParams(key=RadixKey(array("q", tokens)), req=req)
|
|
)
|
|
self.assertEqual(match.device_prefix_len, 2)
|
|
self.assertEqual(match.host_hit_length, 2)
|
|
self.assertEqual(match.swa_host_hit_length, 2)
|
|
self._apply_match_to_req(req, match)
|
|
|
|
self._insert(consumer, consumer_allocator, consumer_req_pool, tokens[:3])
|
|
raced_match = consumer.match_prefix(
|
|
MatchPrefixParams(key=RadixKey(array("q", tokens[:3])))
|
|
)
|
|
raced_full = consumer.path_device_indices(raced_match.last_device_node)[
|
|
-1:
|
|
].clone()
|
|
|
|
loaded_len, loaded_node = consumer.init_load_back(
|
|
InitLoadBackParams(
|
|
best_match_node=match.best_match_node,
|
|
host_hit_length=match.host_hit_length,
|
|
req=req,
|
|
)
|
|
)
|
|
self.assertEqual(loaded_len, 2)
|
|
loaded = consumer.linker.private_load_indices[req.rid]
|
|
load_by_pool = {
|
|
transfer.name: transfer
|
|
for transfer in consumer_linker.queued_loads[req.rid]
|
|
}
|
|
self.assertEqual(set(load_by_pool), {PoolName.KV, PoolName.SWA})
|
|
for transfer in load_by_pool.values():
|
|
self.assertEqual(transfer.keys, all_keys[2:4])
|
|
translated = consumer_allocator.translate_loc_from_full_to_swa(loaded)
|
|
self.assertTrue(
|
|
torch.equal(translated, load_by_pool[PoolName.SWA].device_indices)
|
|
)
|
|
|
|
full_free = consumer_allocator.full_attn_allocator.available_size
|
|
swa_free = consumer_allocator.swa_attn_allocator.available_size
|
|
before = (full_free(), swa_free())
|
|
self.assertGreaterEqual(consumer.ready_to_load_host_cache(), 0)
|
|
self._bind_loaded_request(consumer, consumer_req_pool, req, tokens[:4], loaded)
|
|
if load_landed:
|
|
self.assertEqual(consumer.finish_external_linker_loads([req]), [])
|
|
consumer.checkpoint(req, up_to=req.extend_end)
|
|
consumer.unlock(req.lock)
|
|
req.lock = None
|
|
final = consumer.match_prefix(
|
|
MatchPrefixParams(key=RadixKey(array("q", tokens[:4])))
|
|
)
|
|
# The raced page stays; the request's duplicate of it is freed.
|
|
self.assertTrue(
|
|
torch.equal(
|
|
consumer.path_device_indices(final.last_device_node)[2:3],
|
|
raced_full,
|
|
)
|
|
)
|
|
self.assertTrue(
|
|
torch.equal(
|
|
consumer.path_device_indices(final.last_device_node)[3:], loaded[1:]
|
|
)
|
|
)
|
|
else:
|
|
consumer_linker.load_landed = False
|
|
self.assertEqual(consumer.finish_external_linker_loads([req]), [req])
|
|
self.assertEqual(req.discard_output_reason.status_code, 503)
|
|
self.assertTrue(req.skip_radix_cache_insert)
|
|
release_kv_cache(req, consumer, checkpoint=False)
|
|
self.assertFalse(req.kv.holds_kv)
|
|
self.assertTrue(req.kv.is_kv_released)
|
|
self.assertIsNone(req.lock)
|
|
for component in (ComponentType.FULL, ComponentType.SWA):
|
|
self.assertEqual(
|
|
node_device_lock_ref(consumer, loaded_node, component), 0
|
|
)
|
|
self.assertEqual((full_free(), swa_free()), (before[0] + 2, before[1] + 2))
|
|
unpublished = consumer.match_prefix(
|
|
MatchPrefixParams(key=RadixKey(array("q", tokens[:4])))
|
|
)
|
|
self.assertEqual(unpublished.device_prefix_len, 3)
|
|
consumer.sanity_check()
|
|
|
|
def _mamba_linker_consumer(self, tokens):
|
|
self.cfg = CacheConfig(
|
|
page_size=1,
|
|
components=(ComponentType.FULL, ComponentType.SWA, ComponentType.MAMBA),
|
|
sliding_window_size=2,
|
|
kv_size=64,
|
|
max_context_len=64,
|
|
)
|
|
stored_keys = defaultdict(set)
|
|
|
|
producer, producer_allocator, producer_req_pool = build_fixture(self.cfg)
|
|
producer_linker = _InMemoryUnifiedCacheLinker(stored_keys)
|
|
producer.init_cache_linker(producer_linker)
|
|
inserted = self._insert(producer, producer_allocator, producer_req_pool, tokens)
|
|
# Ancestors of the chain carry no state; only the checkpoint node does.
|
|
*ancestors, offload = producer_linker.offload_calls
|
|
for transfers in ancestors:
|
|
self.assertEqual(
|
|
{transfer.name for transfer in transfers}, {PoolName.KV, PoolName.SWA}
|
|
)
|
|
offloads = {transfer.name: transfer for transfer in offload}
|
|
self.assertEqual(set(offloads), {PoolName.KV, PoolName.SWA, PoolName.MAMBA})
|
|
mamba_offload = offloads[PoolName.MAMBA]
|
|
self.assertEqual(mamba_offload.keys, offloads[PoolName.KV].keys[-1:])
|
|
self.assertEqual(mamba_offload.hit_policy, PoolHitPolicy.TRAILING_PAGES)
|
|
self.assertTrue(
|
|
torch.equal(
|
|
mamba_offload.device_indices,
|
|
node_device_value(
|
|
producer, inserted.last_device_node, ComponentType.MAMBA
|
|
),
|
|
)
|
|
)
|
|
for _ in producer_linker.offload_calls:
|
|
producer_linker.complete_next_offload(True)
|
|
producer.check_hicache_events()
|
|
|
|
consumer, _, consumer_req_pool = build_fixture(self.cfg)
|
|
consumer_linker = _InMemoryUnifiedCacheLinker(stored_keys)
|
|
consumer.init_cache_linker(consumer_linker)
|
|
|
|
return consumer, consumer_req_pool, consumer_linker
|
|
|
|
def test_mamba_state_round_trips_only_at_its_checkpoint(self):
|
|
tokens = list(range(1, 7))
|
|
consumer, consumer_req_pool, consumer_linker = self._mamba_linker_consumer(
|
|
tokens
|
|
)
|
|
|
|
# Every KV and SWA page is stored, but only the full prefix has a state.
|
|
short = consumer.match_prefix(
|
|
MatchPrefixParams(
|
|
key=RadixKey(array("q", tokens[:5])),
|
|
req=self._make_req(consumer_req_pool),
|
|
)
|
|
)
|
|
self.assertEqual(short.host_hit_length, 0)
|
|
|
|
req = self._make_req(consumer_req_pool)
|
|
match = consumer.match_prefix(
|
|
MatchPrefixParams(key=RadixKey(array("q", tokens)), req=req)
|
|
)
|
|
self.assertEqual(match.host_hit_length, len(tokens))
|
|
self.assertEqual(match.mamba_host_hit_length, 1)
|
|
self._apply_match_to_req(req, match)
|
|
loaded, _ = consumer.init_load_back(
|
|
InitLoadBackParams(
|
|
best_match_node=match.best_match_node,
|
|
host_hit_length=match.host_hit_length,
|
|
req=req,
|
|
)
|
|
)
|
|
self.assertEqual(loaded, len(tokens))
|
|
loads = {
|
|
transfer.name: transfer
|
|
for transfer in consumer_linker.queued_loads[req.rid]
|
|
}
|
|
self.assertEqual(set(loads), {PoolName.KV, PoolName.SWA, PoolName.MAMBA})
|
|
self.assertEqual(loads[PoolName.MAMBA].keys, loads[PoolName.KV].keys[-1:])
|
|
loaded_state = loads[PoolName.MAMBA].device_indices
|
|
self.assertEqual(int(req.kv.mamba_pool_idx), int(loaded_state))
|
|
self.assertFalse(req.kv.mamba_needs_clear)
|
|
|
|
loaded_indices = consumer.linker.private_load_indices[req.rid]
|
|
self.assertGreaterEqual(consumer.ready_to_load_host_cache(), 0)
|
|
self._bind_loaded_request(
|
|
consumer, consumer_req_pool, req, tokens, loaded_indices
|
|
)
|
|
self.assertEqual(consumer.finish_external_linker_loads([req]), [])
|
|
consumer.checkpoint(req, up_to=req.extend_end)
|
|
consumer.unlock(req.lock)
|
|
req.lock = None
|
|
final_match = consumer.match_prefix(
|
|
MatchPrefixParams(key=RadixKey(array("q", tokens)))
|
|
)
|
|
self.assertEqual(final_match.device_prefix_len, len(tokens))
|
|
for _ in consumer_linker.offload_calls:
|
|
consumer_linker.complete_next_offload(True)
|
|
consumer.check_hicache_events()
|
|
consumer.sanity_check()
|
|
|
|
def test_mamba_load_supersedes_a_shorter_device_cow(self):
|
|
tokens = list(range(1, 7))
|
|
consumer, consumer_req_pool, consumer_linker = self._mamba_linker_consumer(
|
|
tokens
|
|
)
|
|
self._insert(
|
|
consumer,
|
|
consumer.token_to_kv_pool_allocator,
|
|
consumer_req_pool,
|
|
tokens[:3],
|
|
)
|
|
req = self._make_req(consumer_req_pool)
|
|
match = consumer.match_prefix(
|
|
MatchPrefixParams(key=RadixKey(array("q", tokens)), req=req, cow_mamba=True)
|
|
)
|
|
self.assertEqual(match.device_prefix_len, 3)
|
|
self.assertEqual(match.host_hit_length, 3)
|
|
self.assertIsNotNone(req.kv.mamba_cow_src_index)
|
|
self._apply_match_to_req(req, match)
|
|
allocator = consumer_req_pool.mamba_allocator
|
|
before = allocator.available_size()
|
|
|
|
consumer.init_load_back(
|
|
InitLoadBackParams(
|
|
best_match_node=match.best_match_node,
|
|
host_hit_length=match.host_hit_length,
|
|
req=req,
|
|
)
|
|
)
|
|
# The CoW from the shorter checkpoint would overwrite the loaded state.
|
|
loaded_state = next(
|
|
transfer.device_indices
|
|
for transfer in consumer_linker.queued_loads[req.rid]
|
|
if transfer.name == PoolName.MAMBA
|
|
)
|
|
self.assertEqual(int(req.kv.mamba_pool_idx), int(loaded_state))
|
|
self.assertIsNone(req.kv.mamba_cow_src_index)
|
|
self.assertEqual(allocator.available_size(), before)
|
|
|
|
self.assertGreaterEqual(consumer.ready_to_load_host_cache(), 0)
|
|
self.assertEqual(consumer.finish_external_linker_loads([req]), [])
|
|
consumer.sanity_check()
|
|
|
|
|
|
@unittest.skipUnless(torch.cuda.is_available(), "cache fixtures need CUDA")
|
|
class TestUnifiedCacheLinkerTreeCorePythonBackend(
|
|
_TreeCoreBackendTestMixin,
|
|
UnifiedRadixCacheFixtures,
|
|
CustomTestCase,
|
|
):
|
|
"""TreeCore linker contracts shared by the Python and Rust inspectors."""
|
|
|
|
def test_builds_opaque_external_offload_transfers(self):
|
|
cfg = CacheConfig(page_size=2, kv_size=64, max_context_len=64)
|
|
self.cfg = cfg
|
|
cache, allocator, req_to_token_pool = build_fixture(cfg)
|
|
core = cache.tree_core
|
|
core.enable_external_cache_linker = True
|
|
|
|
inserted = self._insert(cache, allocator, req_to_token_pool, list(range(1, 9)))
|
|
node_id = inserted.last_device_node
|
|
transfers = core.build_external_linker_offload_transfers(node_id)
|
|
|
|
self.assertIsNotNone(transfers)
|
|
(transfer,) = transfers
|
|
self.assertEqual(transfer.name, PoolName.KV)
|
|
self.assertIsNone(transfer.host_indices)
|
|
self.assertIsNotNone(transfer.device_indices)
|
|
self.assertEqual(transfer.keys, core.get_hash_values(node_id))
|
|
self.assertTrue(
|
|
torch.equal(
|
|
transfer.device_indices,
|
|
node_device_value(cache, node_id, ComponentType.FULL),
|
|
)
|
|
)
|
|
|
|
core.mark_external_linker_offload_pending(node_id)
|
|
self.assertFalse(core.is_external_cache_stored(node_id))
|
|
self.assertIsNone(core.build_external_linker_offload_transfers(node_id))
|
|
with self.assertRaisesRegex(AssertionError, "invalid external offload state"):
|
|
core.mark_external_linker_offload_pending(node_id)
|
|
self.assertEqual(core.get_write_through_pending_id(node_id), node_id)
|
|
self.assertFalse(core.is_external_cache_stored(node_id))
|
|
|
|
core.finish_external_linker_offload([node_id], node_id, success=True)
|
|
self.assertIsNone(core.get_write_through_pending_id(node_id))
|
|
self.assertTrue(core.is_external_cache_stored(node_id))
|
|
|
|
def test_external_state_updates_are_atomic_and_path_scoped(self):
|
|
cfg = CacheConfig(page_size=1, kv_size=64, max_context_len=64)
|
|
self.cfg = cfg
|
|
cache, allocator, req_to_token_pool = build_fixture(cfg)
|
|
core = cache.tree_core
|
|
core.enable_external_cache_linker = True
|
|
|
|
anchor = self._insert(cache, allocator, req_to_token_pool, [1]).last_device_node
|
|
middle = self._insert(
|
|
cache, allocator, req_to_token_pool, [1, 2]
|
|
).last_device_node
|
|
tail = self._insert(
|
|
cache, allocator, req_to_token_pool, [1, 2, 3, 4]
|
|
).last_device_node
|
|
unrelated = self._insert(
|
|
cache, allocator, req_to_token_pool, [9]
|
|
).last_device_node
|
|
self.assertEqual(core.get_parent_node_id(middle), anchor)
|
|
self.assertEqual(core.get_parent_node_id(tail), middle)
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "not an ancestor"):
|
|
core.mark_external_cache_stored_path(tail, unrelated)
|
|
self.assertFalse(core.is_external_cache_stored(middle))
|
|
self.assertFalse(core.is_external_cache_stored(tail))
|
|
|
|
core.mark_external_cache_stored_path(tail, anchor)
|
|
self.assertTrue(core.is_external_cache_stored(tail))
|
|
self.assertTrue(core.is_external_cache_stored(middle))
|
|
self.assertFalse(core.is_external_cache_stored(anchor))
|
|
self.assertFalse(core.is_external_cache_stored(unrelated))
|
|
with self.assertRaisesRegex(AssertionError, "invalid external offload state"):
|
|
core.mark_external_linker_offload_pending(tail)
|
|
|
|
split_tail = self._insert(
|
|
cache, allocator, req_to_token_pool, [9, 10, 11]
|
|
).last_device_node
|
|
self.assertEqual(core.get_parent_node_id(split_tail), unrelated)
|
|
core.mark_external_linker_offload_pending(split_tail)
|
|
self._insert(cache, allocator, req_to_token_pool, [9, 10])
|
|
split_parent = core.get_parent_node_id(split_tail)
|
|
self.assertIsNotNone(split_parent)
|
|
self.assertNotEqual(split_parent, unrelated)
|
|
self.assertEqual(core.get_parent_node_id(split_parent), unrelated)
|
|
for node_id in (split_parent, split_tail):
|
|
self.assertEqual(core.get_write_through_pending_id(node_id), split_tail)
|
|
self.assertFalse(core.is_external_cache_stored(node_id))
|
|
|
|
independent = self._insert(
|
|
cache, allocator, req_to_token_pool, [20]
|
|
).last_device_node
|
|
core.mark_external_linker_offload_pending(independent)
|
|
with self.assertRaisesRegex(AssertionError, "invalid external offload state"):
|
|
core.finish_external_linker_offload(
|
|
[independent, split_parent], independent, success=False
|
|
)
|
|
self.assertEqual(core.get_write_through_pending_id(independent), independent)
|
|
for node_id in (split_parent, split_tail):
|
|
self.assertEqual(core.get_write_through_pending_id(node_id), split_tail)
|
|
self.assertFalse(core.is_external_cache_stored(node_id))
|
|
|
|
core.finish_external_linker_offload([independent], independent, success=False)
|
|
core.finish_external_linker_offload(
|
|
[split_parent, split_tail], split_tail, success=False
|
|
)
|
|
for node_id in (split_parent, split_tail):
|
|
self.assertIsNone(core.get_write_through_pending_id(node_id))
|
|
self.assertFalse(core.is_external_cache_stored(node_id))
|
|
self.assertTrue(core.is_external_cache_stored(middle))
|
|
self.assertTrue(core.is_external_cache_stored(tail))
|
|
|
|
def test_failed_offload_preserves_independently_confirmed_state(self):
|
|
cfg = CacheConfig(page_size=1, kv_size=64, max_context_len=64)
|
|
self.cfg = cfg
|
|
cache, allocator, req_to_token_pool = build_fixture(cfg)
|
|
core = cache.tree_core
|
|
core.enable_external_cache_linker = True
|
|
|
|
anchor = self._insert(cache, allocator, req_to_token_pool, [1]).last_device_node
|
|
node_id = self._insert(
|
|
cache, allocator, req_to_token_pool, [1, 2]
|
|
).last_device_node
|
|
core.mark_external_linker_offload_pending(node_id)
|
|
|
|
core.mark_external_cache_stored_path(node_id, anchor)
|
|
self.assertEqual(core.get_write_through_pending_id(node_id), node_id)
|
|
self.assertTrue(core.is_external_cache_stored(node_id))
|
|
|
|
core.finish_external_linker_offload([node_id], node_id, success=False)
|
|
self.assertIsNone(core.get_write_through_pending_id(node_id))
|
|
self.assertTrue(core.is_external_cache_stored(node_id))
|
|
|
|
|
|
class TestUnifiedCacheLinkerRustBackend(TestUnifiedCacheLinkerPythonBackend):
|
|
tree_core_backend = "rust"
|
|
|
|
|
|
class TestUnifiedCacheLinkerTreeCoreRustBackend(
|
|
TestUnifiedCacheLinkerTreeCorePythonBackend
|
|
):
|
|
tree_core_backend = "rust"
|
|
|
|
|
|
def test_restorable_prefix_intersects_sparse_rank_results():
|
|
remote_mask = torch.tensor([0, 0, 1, 0, 0], dtype=torch.int)
|
|
|
|
def intersect_remote_mask(mask, op):
|
|
assert op == torch.distributed.ReduceOp.MIN
|
|
mask.copy_(torch.minimum(mask, remote_mask))
|
|
|
|
cache = _cache_for_wrapper(_all_reduce_attn_groups=intersect_remote_mask)
|
|
wrapper = UnifiedCacheLinkerWrapper(cache, _FakeLinker())
|
|
|
|
hit_pages = wrapper._sync_restorable_prefix([2, 4], num_pages=4, device_hit_pages=0)
|
|
|
|
assert hit_pages == 2
|
|
|
|
|
|
def test_async_offload_pins_node_until_completion():
|
|
linker = _FakeLinker()
|
|
lock_params = object()
|
|
locks = []
|
|
unlocks = []
|
|
|
|
def inc_lock_ref(node):
|
|
locks.append(node)
|
|
return SimpleNamespace(to_dec_params=lambda: lock_params)
|
|
|
|
node_id = 7
|
|
node = SimpleNamespace(
|
|
id=node_id,
|
|
external_cache_stored=False,
|
|
write_through_pending_id=None,
|
|
)
|
|
cache = _cache_for_wrapper(
|
|
tree_core=_FakeExternalTreeCore({node_id: node}),
|
|
inc_lock_ref=inc_lock_ref,
|
|
dec_lock_ref=lambda node, params: unlocks.append((node, params)),
|
|
)
|
|
wrapper = UnifiedCacheLinkerWrapper(cache, linker)
|
|
|
|
wrapper.offload_nodes([node_id])
|
|
|
|
assert locks == [node_id]
|
|
assert not node.external_cache_stored
|
|
assert node.write_through_pending_id == node_id
|
|
assert not unlocks
|
|
|
|
linker.completed_offloads.append(False)
|
|
completed = wrapper.take_completed_offloads(finish_count=1)
|
|
wrapper.commit_completed_offloads(completed)
|
|
|
|
assert not node.external_cache_stored
|
|
assert node.write_through_pending_id is None
|
|
assert unlocks == [(node_id, lock_params)]
|
|
|
|
|
|
def test_offload_skips_node_already_stored_by_tree_core():
|
|
linker = _FakeLinker()
|
|
node = SimpleNamespace(
|
|
id=7,
|
|
external_cache_stored=True,
|
|
write_through_pending_id=None,
|
|
)
|
|
cache = _cache_for_wrapper(
|
|
tree_core=_FakeExternalTreeCore({node.id: node}),
|
|
inc_lock_ref=lambda node_id: pytest.fail("stored node must not be locked"),
|
|
)
|
|
wrapper = UnifiedCacheLinkerWrapper(cache, linker)
|
|
|
|
wrapper.offload_nodes([node.id])
|
|
|
|
assert linker.queued_offloads == []
|
|
|
|
|
|
def test_layer_counter_reports_failure_after_the_forward():
|
|
ready = []
|
|
counter = LayerWiseLoadCounter(2, lambda index, layer: ready.append(layer))
|
|
index = counter.update_producer()
|
|
counter.set_consumer(index)
|
|
counter.complete(index, 0)
|
|
counter.fail(index, RuntimeError("storage read failed"))
|
|
|
|
counter.wait_until(0)
|
|
counter.wait_until(1) # does not raise into the forward
|
|
|
|
assert ready == [0, 1] # an MLA dedup source still broadcasts layer 1
|
|
assert counter.finish(index) is False
|
|
|
|
|
|
def test_layer_counter_finish_waits_for_layers_still_loading():
|
|
counter = LayerWiseLoadCounter(1)
|
|
index = counter.update_producer()
|
|
loader = threading.Timer(0.05, counter.complete, (index, 0))
|
|
loader.start()
|
|
|
|
assert counter.finish(index) is True # the forward never waited on layer 0
|
|
loader.join()
|
|
|
|
|
|
def test_finish_loads_adopts_a_peer_rank_failure():
|
|
reduce = MagicMock(side_effect=lambda value, op: value.fill_(0))
|
|
wrapper = UnifiedCacheLinkerWrapper(
|
|
_cache_for_wrapper(_all_reduce_attn_groups=reduce), _FakeLinker()
|
|
)
|
|
wrapper.inflight_load_rids = ["a", "b"]
|
|
wrapper.start_layer_wise_loading()
|
|
|
|
assert wrapper.finish_loads() == ["a", "b"] # this rank landed, a peer did not
|
|
assert wrapper.finish_loads() == [] # nothing loaded: no collective
|
|
reduce.assert_called_once()
|
|
|
|
|
|
@pytest.mark.parametrize("device_len", [0, 2])
|
|
@pytest.mark.parametrize("tail_len", [0, 4])
|
|
def test_prefix_device_indices_includes_private_linker_tail(device_len, tail_len):
|
|
path = torch.arange(device_len, dtype=torch.int64)
|
|
tail = torch.arange(10, 10 + tail_len, dtype=torch.int64)
|
|
cache = UnifiedRadixCache.__new__(UnifiedRadixCache)
|
|
cache.tree_core = SimpleNamespace(
|
|
root_node_handle=lambda extra_key: 0,
|
|
collect_full_device_indices=lambda node, root: path,
|
|
)
|
|
cache.linker = SimpleNamespace(
|
|
private_load_indices={"rid": tail} if tail_len else {}
|
|
)
|
|
req = SimpleNamespace(
|
|
rid="rid", extra_key=None, last_node=1, prefix_len=device_len + tail_len
|
|
)
|
|
|
|
expected = torch.cat([path, tail])
|
|
assert torch.equal(cache.prefix_device_indices(req), expected)
|
|
assert torch.equal(cache.prefix_device_indices(req), expected)
|
|
assert len(path) == device_len
|
|
|
|
|
|
@pytest.mark.parametrize("landed", [True, False])
|
|
def test_finish_loads_releases_private_index_references(landed, caplog):
|
|
backend = _FakeLinker()
|
|
backend.load_landed = landed
|
|
wrapper = UnifiedCacheLinkerWrapper(
|
|
_cache_for_wrapper(_all_reduce_attn_groups=lambda value, op: None), backend
|
|
)
|
|
wrapper.inflight_load_rids = ["rid"]
|
|
wrapper.private_load_indices["rid"] = torch.arange(4)
|
|
wrapper.start_layer_wise_loading()
|
|
|
|
assert wrapper.finish_loads() == ([] if landed else ["rid"])
|
|
assert wrapper.private_load_indices == {}
|
|
assert ("External linker load failed" in caplog.text) is (not landed)
|
|
|
|
|
|
@pytest.mark.parametrize("operation", ["reset", "close"])
|
|
def test_linker_lifecycle_releases_private_index_references(operation):
|
|
wrapper = UnifiedCacheLinkerWrapper(_cache_for_wrapper(), _FakeLinker())
|
|
wrapper.private_load_indices["rid"] = torch.arange(4)
|
|
|
|
getattr(wrapper, operation)()
|
|
|
|
assert wrapper.private_load_indices == {}
|
|
|
|
|
|
def test_release_request_cancels_queued_load():
|
|
linker = _FakeLinker()
|
|
wrapper = UnifiedCacheLinkerWrapper(_cache_for_wrapper(), linker)
|
|
wrapper.hit_markers["rid"] = object()
|
|
wrapper.inflight_load_rids = ["rid"]
|
|
linker.queued_loads["rid"] = [object()]
|
|
wrapper.private_load_indices["rid"] = torch.arange(4)
|
|
|
|
wrapper.release_request("rid")
|
|
|
|
assert wrapper.hit_markers == {}
|
|
assert wrapper.inflight_load_rids == []
|
|
assert "rid" not in linker.queued_loads
|
|
assert wrapper.private_load_indices == {}
|
|
|
|
|
|
def test_failed_offload_rolls_back_split_fragments():
|
|
linker = _FakeLinker()
|
|
lock_params = object()
|
|
unlocks = []
|
|
child = SimpleNamespace(
|
|
id=7,
|
|
external_cache_stored=False,
|
|
write_through_pending_id=None,
|
|
)
|
|
parent = SimpleNamespace(
|
|
id=8,
|
|
external_cache_stored=False,
|
|
write_through_pending_id=None,
|
|
)
|
|
nodes = {child.id: child, parent.id: parent}
|
|
|
|
cache = _cache_for_wrapper(
|
|
tree_core=_FakeExternalTreeCore(nodes),
|
|
inc_lock_ref=lambda node_id: SimpleNamespace(to_dec_params=lambda: lock_params),
|
|
dec_lock_ref=lambda node_id, params: unlocks.append((node_id, params)),
|
|
)
|
|
wrapper = UnifiedCacheLinkerWrapper(cache, linker)
|
|
wrapper.offload_nodes([child.id])
|
|
|
|
parent.external_cache_stored = child.external_cache_stored
|
|
parent.write_through_pending_id = child.write_through_pending_id
|
|
wrapper.replace_pending_offload_node(child.id, child.id, [parent.id, child.id])
|
|
linker.completed_offloads.append(False)
|
|
wrapper.commit_completed_offloads(wrapper.take_completed_offloads(finish_count=1))
|
|
|
|
assert not parent.external_cache_stored
|
|
assert not child.external_cache_stored
|
|
assert parent.write_through_pending_id is None
|
|
assert child.write_through_pending_id is None
|
|
assert unlocks == [(child.id, lock_params)]
|
|
|
|
|
|
def test_split_action_retargets_pending_external_offload():
|
|
calls = []
|
|
cache = UnifiedRadixCache.__new__(UnifiedRadixCache)
|
|
cache.linker = SimpleNamespace(
|
|
replace_pending_offload_node=lambda *args: calls.append(("linker", *args))
|
|
)
|
|
cache._replace_pending_write_through_node = lambda *args: calls.append(
|
|
("hicache", *args)
|
|
)
|
|
cache.components = {
|
|
ComponentType.FULL: SimpleNamespace(
|
|
apply_component_action=lambda action: calls.append(("full", action))
|
|
)
|
|
}
|
|
action = UpdatePendingBackupOnNodeSplit(
|
|
ack_id=7,
|
|
old_node_id=7,
|
|
new_node_id=8,
|
|
new_child_node_id=7,
|
|
)
|
|
|
|
cache._apply_cache_action(action)
|
|
|
|
assert calls == [
|
|
("hicache", 7, 7, [8, 7]),
|
|
("linker", 7, 7, [8, 7]),
|
|
("full", action),
|
|
]
|
|
|
|
|
|
def test_reset_quiesces_backend_before_releasing_pending_locks():
|
|
events = []
|
|
|
|
class _QuiescentFakeLinker(_FakeLinker):
|
|
def reset(self):
|
|
events.append("backend")
|
|
super().reset()
|
|
|
|
linker = _QuiescentFakeLinker()
|
|
node = SimpleNamespace(
|
|
id=7,
|
|
external_cache_stored=False,
|
|
write_through_pending_id=None,
|
|
)
|
|
cache = _cache_for_wrapper(
|
|
tree_core=_FakeExternalTreeCore({node.id: node}),
|
|
inc_lock_ref=lambda node_id: SimpleNamespace(to_dec_params=object),
|
|
dec_lock_ref=lambda node_id, params: events.append(("unlock", node_id)),
|
|
)
|
|
wrapper = UnifiedCacheLinkerWrapper(cache, linker)
|
|
wrapper.inflight_load_rids = ["rid"]
|
|
wrapper.offload_nodes([node.id])
|
|
|
|
wrapper.reset()
|
|
|
|
assert events == ["backend", ("unlock", node.id)]
|
|
assert wrapper.inflight_load_rids == []
|
|
assert wrapper.pending_offloads == []
|
|
assert not node.external_cache_stored
|
|
assert node.write_through_pending_id is None
|
|
|
|
|
|
def test_close_quiesces_backend_before_releasing_pending_loads():
|
|
events = []
|
|
|
|
class _ClosingFakeLinker(_FakeLinker):
|
|
def close(self):
|
|
events.append("backend")
|
|
super().close()
|
|
|
|
linker = _ClosingFakeLinker()
|
|
wrapper = UnifiedCacheLinkerWrapper(_cache_for_wrapper(), linker)
|
|
wrapper.inflight_load_rids = ["rid"]
|
|
|
|
wrapper.close()
|
|
|
|
assert events == ["backend"]
|
|
assert linker.closed
|
|
assert wrapper.inflight_load_rids == []
|
|
|
|
|
|
def test_check_hicache_events_commits_common_rank_results():
|
|
committed = []
|
|
cache = UnifiedRadixCache.__new__(UnifiedRadixCache)
|
|
cache.linker = SimpleNamespace(
|
|
num_completed_offloads=lambda: 3,
|
|
take_completed_offloads=lambda count: [True] * count,
|
|
commit_completed_offloads=committed.append,
|
|
)
|
|
|
|
reduce_calls = 0
|
|
|
|
def reduce_to_common_state(value, op):
|
|
nonlocal reduce_calls
|
|
assert op == torch.distributed.ReduceOp.MIN
|
|
reduce_calls += 1
|
|
if reduce_calls == 1:
|
|
value.copy_(torch.tensor([1]))
|
|
else:
|
|
value.fill_(0)
|
|
|
|
cache._all_reduce_attn_groups = reduce_to_common_state
|
|
|
|
cache.check_hicache_events()
|
|
|
|
assert committed == [[False]]
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"swa_req_ring",
|
|
[None, False, True],
|
|
ids=["other-allocator", "paged", "request-ring"],
|
|
)
|
|
@pytest.mark.parametrize("enable_hicache", [False, True])
|
|
def test_swa_reuse_policy_tracks_layout_without_a_tier_condition(
|
|
monkeypatch, swa_req_ring, enable_hicache
|
|
):
|
|
from sglang.srt.layers.attention.dsv4 import unified_kv_gate
|
|
|
|
unified_kv = swa_req_ring is True
|
|
monkeypatch.setattr(unified_kv_gate, "is_unified_kv_triton", lambda: unified_kv)
|
|
component = SWAComponent.__new__(SWAComponent)
|
|
component.sliding_window_size = 128
|
|
component.gates_prefix_match = not unified_kv
|
|
cache = UnifiedRadixCache.__new__(UnifiedRadixCache)
|
|
cache.token_to_kv_pool_allocator = _swa_allocator(swa_req_ring)
|
|
cache.components = {ComponentType.SWA: component}
|
|
cache.cache_controller = object() if enable_hicache else None
|
|
cache.decoder_swa_bounded_replay = False
|
|
cache.tree_core = SimpleNamespace(
|
|
enable_hicache=enable_hicache,
|
|
has_swa_host_pool=enable_hicache and not unified_kv,
|
|
)
|
|
component.cache = cache
|
|
component.tree_core = cache.tree_core
|
|
# #32759: request-relative SWA needs tail re-prefill even without HiCache.
|
|
assert cache.swa_reprefill_tail_tokens() == (128 if unified_kv else 0)
|
|
node = SimpleNamespace(
|
|
component_data={
|
|
ComponentType.SWA: SimpleNamespace(value=None, host_value=None)
|
|
},
|
|
backuped=False,
|
|
evicted=False,
|
|
)
|
|
assert component.create_match_validator(match_device_only=True)(node) is unified_kv
|
|
|
|
|
|
def test_cache_without_swa_needs_no_reprefill():
|
|
cache = UnifiedRadixCache.__new__(UnifiedRadixCache)
|
|
cache.components = {}
|
|
assert cache.swa_reprefill_tail_tokens() == 0
|
|
|
|
|
|
@pytest.mark.parametrize("swa_stored", [True, False])
|
|
def test_skip_swa_when_linker_store_lacks_swa(swa_stored):
|
|
"""A store without SWA pages must tombstone restored tails, not mark them hit."""
|
|
linker = _FakeLinker()
|
|
if swa_stored:
|
|
linker.pools[PoolName.SWA] = object()
|
|
swa = object()
|
|
cache = _cache_for_wrapper(
|
|
components={ComponentType.SWA: swa},
|
|
_components_tuple=(swa,),
|
|
tree_components=(ComponentType.FULL, ComponentType.SWA),
|
|
)
|
|
wrapper = UnifiedCacheLinkerWrapper(cache, linker)
|
|
assert wrapper._skip_swa is (not swa_stored)
|
|
assert (swa in wrapper._components) is swa_stored
|
|
|
|
|
|
@pytest.fixture
|
|
def full_linker_component():
|
|
def build_transfer(phase, node, keys):
|
|
keys = ["offload"] if phase == LinkerTransferPhase.OFFLOAD else list(keys)
|
|
return PoolTransfer(
|
|
name=PoolName.KV,
|
|
keys=keys,
|
|
device_indices=(
|
|
None
|
|
if phase == LinkerTransferPhase.LOOKUP
|
|
else torch.arange(len(keys) * 2)
|
|
),
|
|
)
|
|
|
|
return SimpleNamespace(
|
|
component_type=ComponentType.FULL,
|
|
build_external_linker_transfer=MagicMock(side_effect=build_transfer),
|
|
update_external_linker_load=lambda phase, req, full_transfer, transfer, prefix_len, **kwargs: (
|
|
transfer
|
|
),
|
|
)
|
|
|
|
|
|
def test_linker_filters_unstored_swa_from_lookup(full_linker_component):
|
|
full = full_linker_component
|
|
swa = SWAComponent.__new__(SWAComponent)
|
|
swa.build_external_linker_transfer = MagicMock(
|
|
side_effect=AssertionError("excluded SWA reached linker")
|
|
)
|
|
node = SimpleNamespace(id=1, external_cache_stored=False)
|
|
cache = _cache_for_wrapper(
|
|
_components_tuple=(full, swa),
|
|
components={ComponentType.FULL: full, ComponentType.SWA: swa},
|
|
tree_core=SimpleNamespace(enable_external_cache_linker=False, is_eagle=False),
|
|
page_size=2,
|
|
_all_reduce_attn_groups=lambda value, op: None,
|
|
get_last_hash_value=lambda node: None,
|
|
resolve_node_handle=lambda node_id: node,
|
|
inc_lock_ref=lambda node_id: SimpleNamespace(to_dec_params=lambda: object()),
|
|
dec_lock_ref=MagicMock(),
|
|
)
|
|
backend = _FakeLinker()
|
|
backend.restorable = [2]
|
|
wrapper = UnifiedCacheLinkerWrapper(cache, backend)
|
|
assert wrapper._components == (full,)
|
|
result = MatchResult(
|
|
device_prefix_len=0,
|
|
last_device_node=0,
|
|
last_host_node=0,
|
|
best_match_node=0,
|
|
)
|
|
matched = wrapper.match(
|
|
RadixKey(array("q", [1, 2, 3, 4])), SimpleNamespace(rid="match"), result
|
|
)
|
|
assert matched.host_hit_length == 4
|
|
assert [c.args[0] for c in full.build_external_linker_transfer.call_args_list] == [
|
|
LinkerTransferPhase.LOOKUP,
|
|
]
|
|
swa.build_external_linker_transfer.assert_not_called()
|
|
|
|
|
|
@pytest.mark.parametrize("swa_stored", [True, False], ids=["stored", "unstored"])
|
|
@pytest.mark.parametrize(
|
|
"completion", [True, False, None], ids=["success", "failure", "reset"]
|
|
)
|
|
def test_offload_filters_tree_core_swa_transfers_and_preserves_lifecycle(
|
|
swa_stored, completion
|
|
):
|
|
node = SimpleNamespace(
|
|
id=7, external_cache_stored=False, write_through_pending_id=None
|
|
)
|
|
transfers = [
|
|
PoolTransfer(name=PoolName.KV, keys=["page"]),
|
|
PoolTransfer(name=PoolName.SWA, keys=["page"]),
|
|
]
|
|
core = _FakeExternalTreeCore({node.id: node}, transfers)
|
|
swa = SWAComponent.__new__(SWAComponent)
|
|
lock_params = object()
|
|
cache = _cache_for_wrapper(
|
|
components={ComponentType.SWA: swa},
|
|
_components_tuple=(swa,),
|
|
tree_core=core,
|
|
inc_lock_ref=MagicMock(
|
|
return_value=SimpleNamespace(to_dec_params=lambda: lock_params)
|
|
),
|
|
dec_lock_ref=MagicMock(),
|
|
)
|
|
backend = _FakeLinker()
|
|
if swa_stored:
|
|
backend.pools[PoolName.SWA] = object()
|
|
wrapper = UnifiedCacheLinkerWrapper(cache, backend)
|
|
|
|
wrapper.offload_nodes([node.id, node.id])
|
|
expected = transfers if swa_stored else transfers[:1]
|
|
assert backend.queued_offloads == [expected]
|
|
assert core.offload_transfers == transfers
|
|
assert node.write_through_pending_id == node.id
|
|
assert not node.external_cache_stored
|
|
cache.inc_lock_ref.assert_called_once_with(node.id)
|
|
cache.dec_lock_ref.assert_not_called()
|
|
|
|
if completion is None:
|
|
wrapper.reset()
|
|
assert backend.reset_count == 1
|
|
else:
|
|
backend.completed_offloads.append(completion)
|
|
wrapper.commit_completed_offloads(wrapper.take_completed_offloads(1))
|
|
assert not wrapper.pending_offloads
|
|
assert node.write_through_pending_id is None
|
|
assert node.external_cache_stored is (completion is True)
|
|
cache.dec_lock_ref.assert_called_once_with(node.id, lock_params)
|
|
if completion is True:
|
|
wrapper.offload_nodes([node.id])
|
|
assert backend.queued_offloads == [expected]
|
|
else:
|
|
wrapper.offload_nodes([node.id])
|
|
assert backend.queued_offloads == [expected, expected]
|
|
wrapper.reset()
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"swa_stored,previous_boundary,expected_boundary",
|
|
[
|
|
pytest.param(False, None, 4, id="unstored-tombstones"),
|
|
pytest.param(False, 8, 8, id="preserve-existing-boundary"),
|
|
pytest.param(True, None, 2, id="stored-prepare-boundary"),
|
|
],
|
|
)
|
|
@pytest.mark.parametrize("load_refused", [False, True], ids=["queued", "refused"])
|
|
def test_linker_load_preserves_swa_boundaries(
|
|
full_linker_component,
|
|
swa_stored,
|
|
previous_boundary,
|
|
expected_boundary,
|
|
load_refused,
|
|
):
|
|
full = full_linker_component
|
|
swa = SWAComponent.__new__(SWAComponent)
|
|
participates = swa_stored
|
|
|
|
def prepare(phase, req, full_transfer, transfer, prefix_len, **kwargs):
|
|
if phase == ExternalLinkerLoadPhase.PREPARE:
|
|
req.kv = ReqKvInfo(
|
|
kv_allocated_len=prefix_len,
|
|
component_evicted_seqlens={ComponentType.SWA: prefix_len - 2},
|
|
)
|
|
return transfer
|
|
|
|
swa.build_external_linker_transfer = MagicMock(
|
|
return_value=PoolTransfer(
|
|
name=PoolName.SWA, keys=["a", "b"], device_indices=torch.arange(20, 24)
|
|
)
|
|
)
|
|
swa.update_external_linker_load = MagicMock(side_effect=prepare)
|
|
cache = _cache_for_wrapper(
|
|
_components_tuple=(full, swa),
|
|
page_size=2,
|
|
components={ComponentType.FULL: full, ComponentType.SWA: swa},
|
|
insert=MagicMock(side_effect=AssertionError("the tail must stay private")),
|
|
)
|
|
backend = _FakeLinker()
|
|
backend.load = MagicMock(return_value=not load_refused)
|
|
if swa_stored:
|
|
backend.pools[PoolName.SWA] = object()
|
|
wrapper = UnifiedCacheLinkerWrapper(cache, backend)
|
|
wrapper.hit_markers["rid"] = ExternalCacheHitMarker(
|
|
tail_hashes=["a", "b"],
|
|
device_hit_len=0,
|
|
)
|
|
kv = (
|
|
None
|
|
if previous_boundary is None
|
|
else ReqKvInfo(
|
|
kv_allocated_len=previous_boundary,
|
|
component_evicted_seqlens={ComponentType.SWA: previous_boundary},
|
|
)
|
|
)
|
|
req = SimpleNamespace(
|
|
rid="rid",
|
|
kv=kv,
|
|
prefix_len=0,
|
|
last_node=0,
|
|
priority=0,
|
|
swa_branching_seqlen=2, # inside the loaded tail
|
|
)
|
|
restored_len, last_node = wrapper.load_back(req)
|
|
|
|
assert last_node == 0
|
|
if load_refused:
|
|
# Refused before PREPARE: allocations undone, request untouched, recompute.
|
|
assert restored_len == 0 and req.kv is kv
|
|
assert wrapper.inflight_load_rids == []
|
|
assert wrapper.private_load_indices == {}
|
|
phases = [c.args[0] for c in swa.update_external_linker_load.call_args_list]
|
|
assert phases == ([ExternalLinkerLoadPhase.ABORT] if participates else [])
|
|
return
|
|
assert restored_len == 4
|
|
restored = wrapper.private_load_indices["rid"]
|
|
assert restored.tolist() == list(range(4))
|
|
assert wrapper.inflight_load_rids == ["rid"]
|
|
assert torch.equal(wrapper.private_load_indices["rid"], restored)
|
|
assert req.swa_branching_seqlen is None
|
|
assert req.kv.get_evicted_seqlen(ComponentType.SWA) == expected_boundary
|
|
assert req.kv.kv_allocated_len == (previous_boundary or 4)
|
|
assert [c.args[0] for c in full.build_external_linker_transfer.call_args_list] == [
|
|
LinkerTransferPhase.LOAD
|
|
]
|
|
if not participates:
|
|
swa.build_external_linker_transfer.assert_not_called()
|
|
swa.update_external_linker_load.assert_not_called()
|
|
|
|
|
|
class TestOffloadReadyEvents(CustomTestCase):
|
|
def _record(self, forward_stream):
|
|
device = MagicMock(name="device_module")
|
|
device.Event.side_effect = lambda: MagicMock(name="event")
|
|
linker = _FakeLinker()
|
|
linker.forward_stream = forward_stream
|
|
with patch(
|
|
"sglang.srt.mem_cache.unified_cache.unified_cache_linker.device_module",
|
|
device,
|
|
):
|
|
return linker.record_offload_ready()
|
|
|
|
def test_offload_waits_on_the_forward_writing_the_pages(self):
|
|
forward_stream = MagicMock(name="forward_stream")
|
|
indices_ready, kv_written = self._record(forward_stream)
|
|
indices_ready.record.assert_called_once_with()
|
|
kv_written.record.assert_called_once_with(forward_stream)
|
|
# The offload thread waits; the forward stream's own order is untouched.
|
|
self.assertEqual(forward_stream.mock_calls, [])
|
|
|
|
def test_offload_without_a_forward_stream_waits_on_the_current_one(self):
|
|
(indices_ready,) = self._record(None)
|
|
indices_ready.record.assert_called_once_with()
|
|
|
|
def test_mooncake_offload_thread_waits_on_every_event(self):
|
|
linker = MooncakeDirectLinker.__new__(MooncakeDirectLinker)
|
|
order = MagicMock()
|
|
linker.storage = order.storage
|
|
order.storage.batch_set_v2.return_value = {}
|
|
linker.stats = {"offload": 1}
|
|
linker.offload_results = queue.Queue()
|
|
linker.offload_queue = queue.Queue()
|
|
linker.offload_queue.put(
|
|
(["transfer"], 1, [order.indices_ready, order.kv_written])
|
|
)
|
|
linker.offload_queue.put(None)
|
|
linker.offload_thread_func()
|
|
self.assertEqual(
|
|
order.mock_calls,
|
|
[
|
|
call.indices_ready.synchronize(),
|
|
call.kv_written.synchronize(),
|
|
call.storage.batch_set_v2(["transfer"]),
|
|
],
|
|
)
|
|
self.assertTrue(linker.offload_results.get_nowait())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(pytest.main([__file__, "-v"]))
|