From 3eb4bd598f233d6a72eca1b99e3643bfbf136cd8 Mon Sep 17 00:00:00 2001 From: huangtingwei <141888744+huangtingwei9988@users.noreply.github.com> Date: Tue, 20 Oct 2026 11:05:00 +0800 Subject: [PATCH] [Perf] hicache: Unify PP prefetch hit queries and buffer allocation (#43344) --- .../hybrid_cache/hybrid_cache_controller.py | 102 ++-- .../storage/mooncake_store/mooncake_store.py | 24 +- .../unit/mem_cache/test_pp_prefetch_ticket.py | 565 ++++++++++++++++-- 3 files changed, 576 insertions(+), 115 deletions(-) diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py index da8eedb1a9..a0e95be806 100644 --- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py +++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py @@ -87,7 +87,6 @@ class PPPrefetchTicket: prefix_keys: Optional[List[str]] matched_prefix_tokens: List[int] pool_specs: tuple[PPPrefetchPoolSpec, ...] - storage_hit_count: int = 0 class PPPrefetchDecision(Enum): @@ -316,7 +315,6 @@ class HybridCacheController(BaseHiCacheController): ) enable_pp_ticket = ( self.host_memory_mode == "buffer_only" - and storage_backend == "mooncake" and self.pp_group is not None and torch.distributed.get_world_size(group=self.pp_group) > 1 ) @@ -1101,32 +1099,8 @@ class HybridCacheController(BaseHiCacheController): pool_transfers=pool_transfers, ) - storage_hit_count = len(ticket.prefetch_key.token_ids) - try: - config = self.storage_config - for pp_rank in range(config.tp_rank, config.pp_size, config.tp_size): - _, rank_hit_count = self._storage_hit_query(operation, pp_rank=pp_rank) - storage_hit_count = min(storage_hit_count, rank_hit_count) - except Exception: - logger.exception("PP HiCache hit query failed for req=%s.", rid) - storage_hit_count = 0 - - pp_group_ranks = set(torch.distributed.get_process_group_ranks(self.pp_group)) - local_groups = [ - group - for group in self.prefetch_hits_sync_groups - if set(torch.distributed.get_process_group_ranks(group)) != pp_group_ranks - ] - hit_tensor = torch.tensor(storage_hit_count, dtype=torch.int) - self._all_reduce(hit_tensor, torch.distributed.ReduceOp.MIN, local_groups) - storage_hit_count = int(hit_tensor.item()) - storage_hit_count -= storage_hit_count % self.page_size - if storage_hit_count < self.prefetch_threshold: - with self.pp_prefetch_state_lock: - self.pp_prefetch_decisions[rid] = PPPrefetchDecision.SKIPPED - return PrefetchSubmission(decision=False) - - ticket.storage_hit_count = storage_hit_count + # Every rank queries and allocates in ticket order on the worker. + # Admission waits for the normal completion ACK, including misses. operation.is_pp_broadcast = True state = PPPrefetchState(ticket=ticket, operation=operation) with self.pp_prefetch_state_lock: @@ -1158,7 +1132,7 @@ class HybridCacheController(BaseHiCacheController): allocated.append((name, indices)) return indices - host_indices = alloc(PoolName.KV, ticket.storage_hit_count) + host_indices = alloc(PoolName.KV, operation.storage_hit_count) if host_indices is None: return False @@ -1177,7 +1151,7 @@ class HybridCacheController(BaseHiCacheController): spec.num_slots, min( len(spec.keys or []), - ticket.storage_hit_count // self.page_size, + operation.storage_hit_count // self.page_size, ), ) if indices is None: @@ -1281,7 +1255,6 @@ class HybridCacheController(BaseHiCacheController): objects = [self.pp_prefetch_command_queue.get(timeout=60)] except Empty: pass # Complete idle broadcasts before the group times out. - operation = None try: objects = broadcast_pyobj(objects, rank, group, src=source) if not objects: @@ -1320,29 +1293,55 @@ class HybridCacheController(BaseHiCacheController): self.pp_prefetch_decisions.pop(rid) operation = state.operation - operation.hash_value = get_storage_hash_str( - ticket.prefetch_key, - ticket.last_hash, - page_size=self.page_size, - )[: ticket.storage_hit_count // self.page_size] - operation.all_hash_values = list(operation.hash_value) - operation.storage_hit_count = ticket.storage_hit_count + try: + hash_value, storage_hit_count = self._storage_hit_query(operation) + storage_hit_count = min( + storage_hit_count, + self.mem_pool_host.available_size(pool=PoolName.KV), + ) + storage_hit_count -= storage_hit_count % self.page_size + operation.storage_hit_count = storage_hit_count + if ( + storage_hit_count == 0 + or storage_hit_count < self.prefetch_threshold + or not self._allocate_pp_prefetch_buffers(ticket, operation) + ): + storage_hit_count = 0 + except Exception: + logger.exception( + "PP HiCache query or allocation failed for req=%s.", rid + ) + operation.query_pool_hit_pages = {} + hash_value, storage_hit_count = [], 0 + + # A failed query/allocation votes zero in the same MIN as hits. + storage_hit_count = self._sync_prefetch_hit_query( + operation, storage_hit_count + ) + if operation.host_indices is not None: + tail = operation.host_indices[storage_hit_count:] + if tail.numel(): + self.mem_pool_host.free(tail, pool=PoolName.KV) + self.prefetch_tokens_occupied -= len(tail) + operation.host_indices = operation.host_indices[ + :storage_hit_count + ] + operation.hash_value = hash_value[: storage_hit_count // self.page_size] + operation.storage_hit_count = storage_hit_count operation.storage_start = len(ticket.matched_prefix_tokens) operation.pool_storage_result = PoolTransferResult.empty() - if not self._allocate_pp_prefetch_buffers(ticket, operation): - operation.host_indices = torch.empty(0, dtype=torch.int64) + if storage_hit_count == 0: + if operation.host_indices is None: + operation.host_indices = torch.empty(0, dtype=torch.int64) operation.mark_terminate() + if operation.pool_transfers: + self._resolve_sidecar_kv_derived_pool_transfers(operation) self.prefetch_buffer.put(operation) except Exception: logger.exception("PP HiCache ticket processing failed.") - if operation is None or not operation.hash_value: - # Without a received ticket/ACK schedule, continuing is unsafe. - return - if operation.host_indices is None: - operation.host_indices = torch.empty(0, dtype=torch.int64) - operation.mark_terminate() - # Preserve every KV/sidecar ACK even when local allocation fails. - self.prefetch_buffer.put(operation) + # Query/allocation failures already follow the agreed ACK path. + # An unexpected failure cannot safely continue the collectives. + return finally: if is_source and objects: self.pp_prefetch_command_queue.task_done() @@ -1365,9 +1364,7 @@ class HybridCacheController(BaseHiCacheController): self.backup_queue.put(operation) return operation.id - def _storage_hit_query( - self, operation, pp_rank: Optional[int] = None - ) -> tuple[list[str], int]: + def _storage_hit_query(self, operation) -> tuple[list[str], int]: hash_value = get_storage_hash_str( operation.token_ids, operation.last_hash, page_size=self.page_size ) @@ -1384,7 +1381,6 @@ class HybridCacheController(BaseHiCacheController): extra_info = HiCacheStorageExtraInfo( prefix_keys=operation.prefix_keys.copy() if operation.prefix_keys else None, - extra_info={"pp_rank": pp_rank} if pp_rank is not None else None, ) if operation.pool_transfers: hit_result = self.storage_backend.batch_exists_v2( @@ -1703,7 +1699,7 @@ class HybridCacheController(BaseHiCacheController): continue trailing_n = len(transfer.keys) if transfer.keys else 1 transfer.keys = all_hashes[max(0, kv_hit_pages - trailing_n) : kv_hit_pages] - if transfer.host_indices is None: + if transfer.host_indices is None or transfer.indices_from_pool is not None: continue entry = self.mem_pool_host.entry_map.get(transfer.name) pool_page_size = ( diff --git a/python/sglang/srt/mem_cache/storage/mooncake_store/mooncake_store.py b/python/sglang/srt/mem_cache/storage/mooncake_store/mooncake_store.py index b7b0115b9c..5aa280fea8 100644 --- a/python/sglang/srt/mem_cache/storage/mooncake_store/mooncake_store.py +++ b/python/sglang/srt/mem_cache/storage/mooncake_store/mooncake_store.py @@ -899,11 +899,7 @@ class MooncakeStore(HiCacheStorage, MooncakeBaseStore): all_component_keys.extend(component_keys) prepared.append((transfer, key_multiplier, start, len(all_component_keys))) - all_exists = ( - self._batch_exist(all_component_keys, extra_info) - if all_component_keys - else [] - ) + all_exists = self._batch_exist(all_component_keys) if all_component_keys else [] for transfer, key_multiplier, start, end in prepared: if not restorable: break @@ -1441,7 +1437,7 @@ class MooncakeStore(HiCacheStorage, MooncakeBaseStore): query_keys.append(f"{key}_{self.mha_suffix}_v") key_multiplier = 2 - exist_result = self._batch_exist(query_keys, extra_info) + exist_result = self._batch_exist(query_keys) for i in range(len(query_keys)): if exist_result[i] != 1: return i // key_multiplier @@ -1493,21 +1489,7 @@ class MooncakeStore(HiCacheStorage, MooncakeBaseStore): ) return self.store.batch_get_into(key_strs, buffer_ptrs, buffer_sizes) - def _batch_exist( - self, key_strs: List[str], extra_info: Optional[HiCacheStorageExtraInfo] = None - ) -> List[int]: - pp_rank = ( - (extra_info.extra_info or {}).get("pp_rank") - if extra_info is not None - else None - ) - if pp_rank is not None: - # PP is the last rank field before the pool suffix. Replace from - # the right so an identical TP rank or backend tag stays unchanged. - key_strs = [ - f"_{pp_rank}_".join(key.rsplit(f"_{self.pp_rank}_", 1)) - for key in key_strs - ] + def _batch_exist(self, key_strs: List[str]) -> List[int]: return self.store.batch_is_exist(key_strs) def get_stats(self): diff --git a/test/registered/unit/mem_cache/test_pp_prefetch_ticket.py b/test/registered/unit/mem_cache/test_pp_prefetch_ticket.py index 4314040a70..b3c5e92900 100644 --- a/test/registered/unit/mem_cache/test_pp_prefetch_ticket.py +++ b/test/registered/unit/mem_cache/test_pp_prefetch_ticket.py @@ -1,22 +1,27 @@ """CPU regressions for PP ticket ordering, admission, and buffer ownership.""" import pickle +import tempfile import threading import unittest from array import array from contextlib import nullcontext +from dataclasses import replace +from pathlib import Path from queue import Empty, Queue from unittest.mock import Mock, call, patch import torch -from sglang.srt.managers.cache_controller import PrefetchAck +from sglang.srt.environ import envs +from sglang.srt.managers.cache_controller import HiCacheController, PrefetchAck from sglang.srt.mem_cache.base_prefix_cache import ( CacheRequestHandle, CacheRequestOutcome, ) from sglang.srt.mem_cache.buffer_mode.pipeline import BufferModePipeline from sglang.srt.mem_cache.hicache_storage import ( + HiCacheFile, HiCacheStorageConfig, PoolName, PoolTransfer, @@ -25,6 +30,7 @@ from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import ( HybridCacheController, PPPrefetchDecision, ) +from sglang.srt.mem_cache.memory_pool_host import LogicalHostPool from sglang.srt.mem_cache.radix_cache import RadixKey from sglang.srt.mem_cache.storage_prefetch import StoragePrefetchRetries from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache @@ -64,9 +70,12 @@ class TestPPPrefetchTicket(unittest.TestCase): c.storage_stop_event = threading.Event() c.prefetch_tokens_occupied = 0 c.mem_pool_host = Mock(page_size=4) + c.mem_pool_host.available_size.return_value = 1 << 20 c.mem_pool_host.layout_lease.side_effect = nullcontext c.mem_pool_host.alloc.side_effect = lambda size, **_: torch.arange(size) - c._storage_hit_query = Mock(return_value=(["h0", "h1"], 8)) + c.storage_backend_type = "mooncake" + c.storage_backend = Mock() + c._storage_hit_query = Mock(side_effect=self.query_hits) c._all_reduce = Mock() ranks = {"pp": [0, 2, 4, 6], "tp": [0, 1], "command": [0, 2, 4, 6]} patcher = patch.object( @@ -96,6 +105,14 @@ class TestPPPrefetchTicket(unittest.TestCase): cache.buffer_pipeline = Mock(spec=BufferModePipeline) cache._handle_prefetch_result = Mock() + def query_hits(self, operation, hit=8): + hashes = get_storage_hash_str( + operation.token_ids, operation.last_hash, page_size=self.c.page_size + ) + operation.all_hash_values = hashes + operation.query_pool_hit_pages = {PoolName.KV: hit // self.c.page_size} + return hashes[: hit // self.c.page_size], hit + def submit(self, rid="hit", pools=None, attempt_id=0, assume_stored=False): key = RadixKey( array("q", range(9)), "adapter", is_bigram=True, cache_salt="tenant" @@ -130,55 +147,433 @@ class TestPPPrefetchTicket(unittest.TestCase): ): self.c.prefetch_sync_thread_func() + def run_io(self, operations): + with ( + patch("sglang.srt.managers.cache_controller.STORAGE_BATCH_SIZE", 1), + patch.object( + self.c.storage_stop_event, + "is_set", + side_effect=[False] * operations + [True], + ), + ): + self.c.prefetch_io_aux_func() + acks = [] + while not self.c.prefetch_sync_queue.empty(): + acks.append(self.c.prefetch_sync_queue.get_nowait()) + return acks + def test_hit_and_miss_are_reused_across_enqueue_retract_and_retry(self): - c = self.c + c, cache = self.c, self.cache + cache.pp_rank = 0 + c.page_get_func = Mock(return_value=1) for hit in (0, 8): with self.subTest(hit=hit): rid = str(hit) handle = CacheRequestHandle(rid, 0) - c._storage_hit_query.return_value = ([], hit) - self.assertEqual(self.submit(rid).decision, bool(hit)) + c._storage_hit_query.side_effect = lambda operation: self.query_hits( + operation, hit + ) + self.assertTrue(self.submit(rid).decision) queries = c._storage_hit_query.call_count for _ in range(2): - self.assertEqual( - self.cache.prefetch_from_storage(handle, 0, []), bool(hit) - ) + self.assertTrue(cache.prefetch_from_storage(handle, 0, [])) self.assertEqual(c._storage_hit_query.call_count, queries) - if hit: - state = c.pp_prefetch_states[rid] - state.ready_event.set() - c.take_ready_pp_prefetch(rid) - self.assertFalse( - self.cache.prefetch_from_storage( - CacheRequestHandle(rid, 1), 0, [] - ) - ) - self.assertTrue(self.cache.check_prefetch_progress(handle)) + self.run_commands() + self.assertEqual(c._storage_hit_query.call_count, queries + 1) + self.sync_acks(*self.run_io(1)) + self.assertTrue(cache.check_prefetch_progress(handle)) + self.assertFalse( + cache.prefetch_from_storage(CacheRequestHandle(rid, 1), 0, []) + ) self.assertFalse(c.release_pp_prefetch(rid)) self.assertIsNone(c.get_prefetch_submission(rid)) - self.assertEqual(c.pp_prefetch_command_queue.qsize(), 1) - self.cache._all_reduce.assert_not_called() + self.assertTrue(c.pp_prefetch_command_queue.empty()) - def test_submission_does_not_wait_for_worker_and_uses_only_tp_consensus(self): + def test_all_backends_query_and_allocate_before_one_worker_consensus(self): c = self.c - with patch.object( - c.pp_prefetch_command_queue, - "join", - side_effect=AssertionError("scheduler blocked"), + for backend in ("file", "mooncake", "dynamic"): + with self.subTest(backend=backend): + c.storage_backend_type = backend + c._storage_hit_query.reset_mock() + c._all_reduce.reset_mock() + c.mem_pool_host.alloc.reset_mock() + c._storage_hit_query.side_effect = AssertionError( + "scheduler queried storage" + ) + c._all_reduce.side_effect = AssertionError( + "scheduler entered a collective" + ) + with patch.object( + c.pp_prefetch_command_queue, + "join", + side_effect=AssertionError("scheduler blocked"), + ): + first = self.submit(backend, assume_stored=True) + second = self.submit(f"next_{backend}") + self.assertTrue(first.decision) + self.assertTrue(second.decision) + self.assertEqual(c.pp_prefetch_command_queue.qsize(), 2) + self.assertFalse(c.is_pp_prefetch_ready(backend)) + self.assertFalse(first.operation.assume_stored) + c.mem_pool_host.alloc.assert_not_called() + c._storage_hit_query.assert_not_called() + c._all_reduce.assert_not_called() + + votes = [] + + def reduce_hits(tensor, op, groups): + self.assertEqual(op, torch.distributed.ReduceOp.MIN) + self.assertEqual(groups, c.prefetch_hits_sync_groups) + self.assertEqual(tensor.numel(), 1 + len(PoolName)) + self.assertEqual(tensor[0].item(), 8) + self.assertEqual(c.mem_pool_host.alloc.call_count, len(votes) + 1) + votes.append(tensor[0].item()) + + c._storage_hit_query.side_effect = self.query_hits + c._all_reduce.side_effect = reduce_hits + self.run_commands() + self.assertEqual( + c._storage_hit_query.call_args_list, + [call(first.operation), call(second.operation)], + ) + self.assertEqual(votes, [8, 8]) + self.assertEqual(c._all_reduce.call_count, 2) + self.assertIs(c.prefetch_buffer.get_nowait(), first.operation) + self.assertIs(c.prefetch_buffer.get_nowait(), second.operation) + + def test_file_attachment_enables_pp_tickets(self): + c = self.c + c.pp_prefetch_command_group = None + c.storage_host_pool = Mock() + c.host_memory_mode = "buffer_only" + c.storage_backend_type = "file" + with ( + patch.object(HiCacheController, "attach_storage_backend"), + patch.object(torch.distributed, "get_world_size", return_value=4), + patch( + "sglang.srt.distributed.parallel_state.create_custom_parallel_group", + return_value="command", + ), ): - self.assertTrue(self.submit(assume_stored=True).decision) - self.assertTrue(self.submit("next").decision) - self.assertEqual(c.pp_prefetch_command_queue.qsize(), 2) - self.assertFalse(c.is_pp_prefetch_ready("hit")) - c.mem_pool_host.alloc.assert_not_called() - self.assertFalse(c.pp_prefetch_states["hit"].operation.assume_stored) - self.assertEqual( - c._all_reduce.call_args.args[1:], (torch.distributed.ReduceOp.MIN, ["tp"]) + c.attach_storage_backend("file", prefetch_threshold=4) + self.assertEqual(c.pp_prefetch_command_group, "command") + + def test_file_stage_hits_trim_buffers_after_local_allocation(self): + """PP0's local files cannot reveal a shorter prefix on another stage.""" + c = self.c + key = RadixKey( + array("q", range(9)), "adapter", is_bigram=True, cache_salt="tenant" ) - self.assertEqual( - [call.kwargs["pp_rank"] for call in c._storage_hit_query.call_args_list], - [0, 2, 0, 2], + hashes = get_storage_hash_str(key, "ab" * 32, page_size=4) + local_votes = [] + with ( + tempfile.TemporaryDirectory() as directory, + patch.object( + envs.SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR, "get", return_value=None + ), + ): + backends = [] + for rank, pages in ((0, 2), (1, 1)): + config = replace( + c.storage_config, + pp_rank=rank, + pp_size=2, + extra_config={ + "file_storage_path": str(Path(directory) / str(rank)), + "enable_metadata_cache": False, + "max_size": 0, + "min_free_space": 0, + }, + ) + backend = HiCacheFile(config) + for hash_value in hashes[:pages]: + self.assertTrue(backend.set(hash_value, torch.ones(1))) + backends.append(backend) + + self.assertEqual(backends[0].batch_exists(hashes), 2) + self.assertEqual(backends[1].batch_exists(hashes), 1) + c.storage_backend = backends[0] + c.storage_backend_type = "file" + c._storage_hit_query = Mock( + wraps=HybridCacheController._storage_hit_query.__get__(c) + ) + result = self.submit(attempt_id=3) + ticket = pickle.loads(pickle.dumps(c.pp_prefetch_states["hit"].ticket)) + + for rank, peer_hit in ((0, 4), (1, 8)): + with self.subTest(rank=rank): + c.storage_config.pp_rank = rank + c.storage_backend = backends[rank] + if rank: + c.pp_prefetch_states.clear() + c._storage_hit_query.reset_mock() + c.mem_pool_host.alloc.reset_mock() + c.mem_pool_host.free.reset_mock() + c.prefetch_tokens_occupied = 0 + local_hit = 8 if rank == 0 else 4 + + def reduce_hits(tensor, op, groups): + self.assertEqual(op, torch.distributed.ReduceOp.MIN) + self.assertEqual(groups, ["pp", "tp"]) + self.assertEqual(tensor.numel(), 1 + len(PoolName)) + local_votes.append(tensor[0].item()) + c.mem_pool_host.alloc.assert_called_once_with( + local_hit, pool=PoolName.KV + ) + tensor[0] = min(tensor[0].item(), peer_hit) + kv_index = 1 + list(PoolName).index(PoolName.KV) + tensor[kv_index] = min(tensor[kv_index].item(), peer_hit // 4) + + c._all_reduce.side_effect = reduce_hits + self.run_commands(ticket) + operation = c.prefetch_buffer.get_nowait() + self.assertEqual(operation.handle, CacheRequestHandle("hit", 3)) + self.assertEqual(operation.storage_hit_count, 4) + self.assertEqual(operation.hash_value, hashes[:1]) + self.assertEqual(operation.host_indices.numel(), 4) + self.assertFalse(operation.is_terminated()) + self.assertEqual(c.prefetch_tokens_occupied, 4) + if rank == 0: + self.assertIs(operation, result.operation) + freed = c.mem_pool_host.free.call_args + self.assertEqual(freed.kwargs, {"pool": PoolName.KV}) + self.assertTrue(torch.equal(freed.args[0], torch.arange(4, 8))) + self.assertEqual(c.mem_pool_host.free.call_count, 1) + else: + c.mem_pool_host.free.assert_not_called() + c._storage_hit_query.assert_called_once_with(operation) + self.assertEqual(operation.query_pool_hit_pages, {PoolName.KV: 1}) + + self.assertEqual(local_votes, [8, 4]) + + def test_file_stage_hit_fits_capacity_before_consensus(self): + """A large local hit must not discard a prefix that fits on every stage.""" + c = self.c + c.page_size = c.prefetch_threshold = 256 + key = RadixKey(array("q", range(8193)), is_bigram=True) + hashes = get_storage_hash_str(key, page_size=256) + c.storage_backend_type = "file" + c._storage_hit_query = Mock( + wraps=HybridCacheController._storage_hit_query.__get__(c) ) + with ( + tempfile.TemporaryDirectory() as directory, + patch.object( + envs.SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR, "get", return_value=None + ), + ): + backends = [] + for rank, pages in ((0, 32), (1, 8)): + config = replace( + c.storage_config, + pp_rank=rank, + pp_size=2, + extra_config={ + "file_storage_path": str(Path(directory) / str(rank)), + "enable_metadata_cache": False, + "max_size": 0, + "min_free_space": 0, + }, + ) + backend = HiCacheFile(config) + for hash_value in hashes[:pages]: + self.assertTrue(backend.set(hash_value, torch.ones(1))) + backends.append(backend) + + for capacity in (4096, 4097): + c.storage_config.pp_rank = 0 + c.pp_prefetch_states.clear() + c.pp_prefetch_command_queue = Queue() + result = c.submit_prefetch( + CacheRequestHandle("capacity", 0), key, None, None, [], None + ) + ticket = pickle.loads( + pickle.dumps(c.pp_prefetch_states["capacity"].ticket) + ) + for rank, local_vote in ((0, 4096), (1, 2048)): + with self.subTest(capacity=capacity, rank=rank): + c.storage_config.pp_rank = rank + c.storage_backend = backends[rank] + if rank: + c.pp_prefetch_states.clear() + host_pool = LogicalHostPool(4096, page_size=256) + c.mem_pool_host.available_size.return_value = capacity + c.mem_pool_host.available_size.reset_mock() + c.mem_pool_host.alloc.reset_mock() + c.mem_pool_host.free.reset_mock() + c.mem_pool_host.alloc.side_effect = lambda size, **_: ( + host_pool.alloc(size) + ) + c.mem_pool_host.free.side_effect = lambda indices, **_: ( + host_pool.free(indices) + ) + c.prefetch_tokens_occupied = 0 + c._storage_hit_query.reset_mock() + c._all_reduce.reset_mock() + local_votes = [] + + def reduce_hits(tensor, op, groups): + self.assertEqual(op, torch.distributed.ReduceOp.MIN) + self.assertEqual(groups, c.prefetch_hits_sync_groups) + self.assertEqual(tensor.numel(), 1 + len(PoolName)) + local_votes.append(tensor[0].item()) + kv_index = 1 + list(PoolName).index(PoolName.KV) + self.assertEqual( + tensor[kv_index].item(), 32 if rank == 0 else 8 + ) + peer_vote = 2048 if rank == 0 else 4096 + tensor[0] = min(tensor[0].item(), peer_vote) + tensor[kv_index] = min(tensor[kv_index].item(), 8) + + c._all_reduce.side_effect = reduce_hits + self.run_commands(ticket) + operation = c.prefetch_buffer.get_nowait() + if rank == 0: + self.assertIs(operation, result.operation) + self.assertEqual(operation.storage_hit_count, 2048) + self.assertEqual(local_votes, [local_vote]) + c.mem_pool_host.alloc.assert_called_once_with( + local_vote, pool=PoolName.KV + ) + self.assertEqual(operation.hash_value, hashes[:8]) + self.assertEqual(operation.host_indices.numel(), 2048) + self.assertFalse(operation.is_terminated()) + self.assertEqual( + operation.query_pool_hit_pages, {PoolName.KV: 8} + ) + self.assertEqual(c.prefetch_tokens_occupied, 2048) + self.assertEqual(host_pool.available_size(), 2048) + c._storage_hit_query.assert_called_once_with(operation) + c.mem_pool_host.available_size.assert_called_once_with( + pool=PoolName.KV + ) + c._all_reduce.assert_called_once() + if rank == 0: + self.assertEqual(c.mem_pool_host.free.call_count, 1) + freed = c.mem_pool_host.free.call_args + self.assertEqual(freed.kwargs, {"pool": PoolName.KV}) + self.assertTrue( + torch.equal(freed.args[0], torch.arange(2048, 4096)) + ) + else: + c.mem_pool_host.free.assert_not_called() + + def test_local_miss_or_query_error_completes_before_next_ticket(self): + """A stage miss/error must complete admission without breaking ACK order.""" + c, cache = self.c, self.cache + c.storage_backend_type = "file" + cache.pp_rank = 0 + for name, local_hit, peer_hit, threshold in ( + ("local_miss", 0, 8, 4), + ("peer_miss", 8, 0, 4), + ("below_threshold", 7, 8, 6), + ("query_error", RuntimeError("local storage unavailable"), 8, 4), + ): + with self.subTest(name=name): + failed_rid, next_rid = name, f"next_{name}" + c.prefetch_threshold = threshold + c._storage_hit_query.reset_mock() + c._all_reduce.reset_mock() + c.mem_pool_host.alloc.reset_mock() + c.mem_pool_host.free.reset_mock() + cache._handle_prefetch_result.reset_mock() + occupied_before = c.prefetch_tokens_occupied + hits = iter((local_hit, 8)) + + def query_hits(operation): + hit = next(hits) + if isinstance(hit, Exception): + operation.query_pool_hit_pages = {PoolName.KV: 2} + raise hit + return self.query_hits(operation, hit) + + c._storage_hit_query.side_effect = query_hits + self.assertTrue(self.submit(failed_rid, attempt_id=3).decision) + self.assertTrue(self.submit(next_rid).decision) + peers = iter((peer_hit, 8)) + local_votes = [] + + def reduce_hits(tensor, op, groups): + self.assertEqual(op, torch.distributed.ReduceOp.MIN) + self.assertEqual(groups, ["pp", "tp"]) + self.assertEqual(tensor.numel(), 1 + len(PoolName)) + local_votes.append(tensor[0].item()) + if tensor[0].item(): + self.assertEqual(c.mem_pool_host.alloc.call_args.args, (8,)) + tensor[0] = min(tensor[0].item(), next(peers)) + + c._all_reduce.side_effect = reduce_hits + logs = ( + self.assertLogs(level="ERROR") + if isinstance(local_hit, Exception) + else nullcontext() + ) + with logs: + self.run_commands() + failed = c.pp_prefetch_states[failed_rid].operation + following = c.pp_prefetch_states[next_rid].operation + self.assertEqual(failed.storage_hit_count, 0) + self.assertEqual(failed.hash_value, []) + self.assertEqual(failed.host_indices.numel(), 0) + self.assertTrue(failed.is_terminated()) + self.assertEqual(following.host_indices.numel(), 8) + self.assertFalse(following.is_terminated()) + self.assertEqual( + local_votes, + [8 if name == "peer_miss" else 0, 8], + ) + self.assertEqual(c._all_reduce.call_count, 2) + self.assertEqual( + c._storage_hit_query.call_args_list, + [call(failed), call(following)], + ) + if name == "peer_miss": + self.assertEqual( + c.mem_pool_host.alloc.call_args_list, + [call(8, pool=PoolName.KV)] * 2, + ) + self.assertEqual(c.mem_pool_host.free.call_count, 1) + self.assertTrue( + torch.equal( + c.mem_pool_host.free.call_args.args[0], torch.arange(8) + ) + ) + self.assertEqual( + c.mem_pool_host.free.call_args.kwargs, {"pool": PoolName.KV} + ) + else: + c.mem_pool_host.alloc.assert_called_once_with(8, pool=PoolName.KV) + c.mem_pool_host.free.assert_not_called() + if name == "query_error": + self.assertEqual(failed.query_pool_hit_pages, {}) + self.assertEqual(c.prefetch_tokens_occupied, occupied_before + 8) + + c._all_reduce.side_effect = None + c.page_get_func = Mock(return_value=1) + acks = self.run_io(2) + self.assertEqual( + [ack.rid for ack in acks], [failed_rid] + [next_rid] * 3 + ) + self.assertTrue(acks[0].completed_req) + self.assertEqual( + [ack.completed_tokens for ack in acks[1:]], [4, 8, None] + ) + self.assertEqual(c.page_get_func.call_count, 2) + self.sync_acks(*acks) + handle = CacheRequestHandle(failed_rid, 3) + self.assertTrue(c.is_pp_prefetch_ready(failed_rid)) + self.assertTrue(cache.check_prefetch_progress(handle)) + self.assertEqual(cache.prefetch_loaded_tokens_by_reqid[handle], 0) + self.assertNotIn(handle, cache.ongoing_prefetch) + cache._handle_prefetch_result.assert_not_called() + self.assertTrue(cache.check_prefetch_progress(handle)) + self.assertFalse(c.release_pp_prefetch(failed_rid)) + self.assertTrue( + cache.check_prefetch_progress(CacheRequestHandle(next_rid, 0)) + ) + cache._handle_prefetch_result.assert_called_once_with(following) + self.assertEqual(c._storage_hit_query.call_count, 2) + self.assertEqual(c.pp_prefetch_command_queue.unfinished_tasks, 0) def test_tp_only_preserves_normal_prefetch(self): c = self.c @@ -209,7 +604,7 @@ class TestPPPrefetchTicket(unittest.TestCase): ) self.assertTrue(operation.token_ids.is_bigram) self.assertEqual(len(operation.token_ids), 8) - c._storage_hit_query.assert_not_called() + c._storage_hit_query.assert_called_once_with(operation) self.sync_acks(PrefetchAck("hit", operation, completed_tokens=8)) self.assertFalse(c.is_pp_prefetch_ready("hit")) self.sync_acks(PrefetchAck("hit", operation, completed_req=True)) @@ -244,7 +639,22 @@ class TestPPPrefetchTicket(unittest.TestCase): with self.assertLogs(level="ERROR"): self.run_commands(ticket, following) failed = c.pp_prefetch_states["hit"].operation + next_operation = c.pp_prefetch_states["next"].operation self.assertTrue(failed.is_terminated()) + self.assertEqual(failed.storage_hit_count, 0) + self.assertEqual(failed.hash_value, []) + self.assertEqual(failed.host_indices.numel(), 0) + self.assertEqual( + c._storage_hit_query.call_args_list, [call(failed), call(next_operation)] + ) + self.assertEqual(c._all_reduce.call_count, 2) + self.assertEqual( + [ + invocation.args[0][0].item() + for invocation in c._all_reduce.call_args_list + ], + [0, 8], + ) c.mem_pool_host.free.assert_called_once_with(kv, pool=PoolName.KV) c.page_get_func = Mock(return_value=1) c.storage_backend = Mock() @@ -256,12 +666,12 @@ class TestPPPrefetchTicket(unittest.TestCase): ), ): c.prefetch_io_aux_func() - acks = [c.prefetch_sync_queue.get_nowait() for _ in range(8)] - self.assertEqual([a.rid for a in acks], ["hit"] * 4 + ["next"] * 4) + acks = [c.prefetch_sync_queue.get_nowait() for _ in range(6)] + self.assertEqual([a.rid for a in acks], ["hit"] * 2 + ["next"] * 4) self.assertEqual( - [a.completed_tokens for a in acks], [0, 0, None, None, 4, 8, None, None] + [a.completed_tokens for a in acks], [None, None, 4, 8, None, None] ) - self.assertEqual((acks[2].pool_hits, acks[6].pool_hits), ({}, {"swa": 1})) + self.assertEqual((acks[0].pool_hits, acks[4].pool_hits), ({}, {"swa": 1})) self.assertEqual( c.page_get_func.call_count, 2 ) # No reads for the failed ticket. @@ -341,6 +751,16 @@ class TestPPPrefetchTicket(unittest.TestCase): c.mem_pool_host.alloc.side_effect = lambda *args, **kwargs: None self.run_commands() self.assertTrue(operation.is_terminated()) + self.assertEqual(operation.hash_value, []) + self.assertEqual(operation.storage_hit_count, 0) + c._storage_hit_query.assert_called_once_with(operation) + c.mem_pool_host.alloc.assert_called_once_with(8, pool=PoolName.KV) + c._all_reduce.assert_called_once() + packed, op, groups = c._all_reduce.call_args.args + self.assertEqual(packed[0].item(), 0) + self.assertEqual(packed.numel(), 1 + len(PoolName)) + self.assertEqual(op, torch.distributed.ReduceOp.MIN) + self.assertEqual(groups, c.prefetch_hits_sync_groups) c.mem_pool_host.free.assert_not_called() self.sync_acks( PrefetchAck("hit", operation, completed_tokens=0, completed_req=True) @@ -349,6 +769,69 @@ class TestPPPrefetchTicket(unittest.TestCase): self.assertIsNone(borrowed.host_indices) self.assertEqual(c.mem_pool_host.free.call_args.kwargs["pool"], PoolName.SWA) + def test_peer_vote_trims_kv_alias_and_defers_sidecar_free(self): + """A peer's zero allocation vote must release KV without freeing a sidecar.""" + c = self.c + for peer_vote in (4, 0): + with self.subTest(peer_vote=peer_vote): + rid = f"peer_{peer_vote}" + c._storage_hit_query.reset_mock() + c._all_reduce.reset_mock() + c.mem_pool_host.alloc.reset_mock() + c.mem_pool_host.free.reset_mock() + borrowed_indices = torch.arange(4) + operation = self.submit( + rid, + pools=[ + PoolTransfer(PoolName.SWA, host_indices=borrowed_indices), + PoolTransfer(PoolName.INDEXER, indices_from_pool=PoolName.KV), + ], + ).operation + + def reduce_hits(tensor, op, groups): + self.assertEqual(tensor[0].item(), 8) + self.assertEqual(tensor.numel(), 1 + len(PoolName)) + self.assertEqual(op, torch.distributed.ReduceOp.MIN) + self.assertEqual(groups, c.prefetch_hits_sync_groups) + c.mem_pool_host.alloc.assert_called_once_with(8, pool=PoolName.KV) + tensor[0] = peer_vote + + c._all_reduce.side_effect = reduce_hits + self.run_commands() + c._storage_hit_query.assert_called_once_with(operation) + c._all_reduce.assert_called_once() + swa, indexer = operation.pool_transfers + self.assertEqual(operation.host_indices.numel(), peer_vote) + self.assertEqual(operation.storage_hit_count, peer_vote) + self.assertEqual(operation.is_terminated(), peer_vote == 0) + self.assertIs(indexer.host_indices, operation.host_indices) + self.assertIs(indexer.keys, operation.hash_value) + self.assertIs(swa.host_indices, borrowed_indices) + self.assertEqual(c.prefetch_tokens_occupied, peer_vote) + freed = c.mem_pool_host.free.call_args + self.assertEqual(c.mem_pool_host.free.call_count, 1) + self.assertEqual(freed.kwargs, {"pool": PoolName.KV}) + self.assertTrue(torch.equal(freed.args[0], torch.arange(peer_vote, 8))) + + self.assertTrue(c.release_pp_prefetch(rid)) + self.assertIs(swa.host_indices, borrowed_indices) + self.assertEqual(c.mem_pool_host.free.call_count, 1) + c._all_reduce.side_effect = None + self.sync_acks( + PrefetchAck(rid, operation, completed_tokens=peer_vote), + PrefetchAck(rid, operation, completed_req=True), + ) + self.assertIsNone(swa.host_indices) + self.assertNotIn(rid, c.pp_prefetch_states) + self.assertEqual(c.prefetch_tokens_occupied, 0) + sidecar_frees = [ + invocation + for invocation in c.mem_pool_host.free.call_args_list + if invocation.kwargs["pool"] == PoolName.SWA + ] + self.assertEqual(len(sidecar_frees), 1) + self.assertIs(sidecar_frees[0].args[0], borrowed_indices) + def test_command_failure_finishes_queue_task_and_is_reported_on_poll(self): c = self.c with (