[Perf] hicache: Unify PP prefetch hit queries and buffer allocation (#43344)

This commit is contained in:
huangtingwei
2026-10-20 11:05:00 +08:00
committed by GitHub
parent ec1fdc22a9
commit 3eb4bd598f
3 changed files with 576 additions and 115 deletions
@@ -87,7 +87,6 @@ class PPPrefetchTicket:
prefix_keys: Optional[List[str]]
matched_prefix_tokens: List[int]
pool_specs: tuple[PPPrefetchPoolSpec, ...]
storage_hit_count: int = 0
class PPPrefetchDecision(Enum):
@@ -316,7 +315,6 @@ class HybridCacheController(BaseHiCacheController):
)
enable_pp_ticket = (
self.host_memory_mode == "buffer_only"
and storage_backend == "mooncake"
and self.pp_group is not None
and torch.distributed.get_world_size(group=self.pp_group) > 1
)
@@ -1101,32 +1099,8 @@ class HybridCacheController(BaseHiCacheController):
pool_transfers=pool_transfers,
)
storage_hit_count = len(ticket.prefetch_key.token_ids)
try:
config = self.storage_config
for pp_rank in range(config.tp_rank, config.pp_size, config.tp_size):
_, rank_hit_count = self._storage_hit_query(operation, pp_rank=pp_rank)
storage_hit_count = min(storage_hit_count, rank_hit_count)
except Exception:
logger.exception("PP HiCache hit query failed for req=%s.", rid)
storage_hit_count = 0
pp_group_ranks = set(torch.distributed.get_process_group_ranks(self.pp_group))
local_groups = [
group
for group in self.prefetch_hits_sync_groups
if set(torch.distributed.get_process_group_ranks(group)) != pp_group_ranks
]
hit_tensor = torch.tensor(storage_hit_count, dtype=torch.int)
self._all_reduce(hit_tensor, torch.distributed.ReduceOp.MIN, local_groups)
storage_hit_count = int(hit_tensor.item())
storage_hit_count -= storage_hit_count % self.page_size
if storage_hit_count < self.prefetch_threshold:
with self.pp_prefetch_state_lock:
self.pp_prefetch_decisions[rid] = PPPrefetchDecision.SKIPPED
return PrefetchSubmission(decision=False)
ticket.storage_hit_count = storage_hit_count
# Every rank queries and allocates in ticket order on the worker.
# Admission waits for the normal completion ACK, including misses.
operation.is_pp_broadcast = True
state = PPPrefetchState(ticket=ticket, operation=operation)
with self.pp_prefetch_state_lock:
@@ -1158,7 +1132,7 @@ class HybridCacheController(BaseHiCacheController):
allocated.append((name, indices))
return indices
host_indices = alloc(PoolName.KV, ticket.storage_hit_count)
host_indices = alloc(PoolName.KV, operation.storage_hit_count)
if host_indices is None:
return False
@@ -1177,7 +1151,7 @@ class HybridCacheController(BaseHiCacheController):
spec.num_slots,
min(
len(spec.keys or []),
ticket.storage_hit_count // self.page_size,
operation.storage_hit_count // self.page_size,
),
)
if indices is None:
@@ -1281,7 +1255,6 @@ class HybridCacheController(BaseHiCacheController):
objects = [self.pp_prefetch_command_queue.get(timeout=60)]
except Empty:
pass # Complete idle broadcasts before the group times out.
operation = None
try:
objects = broadcast_pyobj(objects, rank, group, src=source)
if not objects:
@@ -1320,29 +1293,55 @@ class HybridCacheController(BaseHiCacheController):
self.pp_prefetch_decisions.pop(rid)
operation = state.operation
operation.hash_value = get_storage_hash_str(
ticket.prefetch_key,
ticket.last_hash,
page_size=self.page_size,
)[: ticket.storage_hit_count // self.page_size]
operation.all_hash_values = list(operation.hash_value)
operation.storage_hit_count = ticket.storage_hit_count
try:
hash_value, storage_hit_count = self._storage_hit_query(operation)
storage_hit_count = min(
storage_hit_count,
self.mem_pool_host.available_size(pool=PoolName.KV),
)
storage_hit_count -= storage_hit_count % self.page_size
operation.storage_hit_count = storage_hit_count
if (
storage_hit_count == 0
or storage_hit_count < self.prefetch_threshold
or not self._allocate_pp_prefetch_buffers(ticket, operation)
):
storage_hit_count = 0
except Exception:
logger.exception(
"PP HiCache query or allocation failed for req=%s.", rid
)
operation.query_pool_hit_pages = {}
hash_value, storage_hit_count = [], 0
# A failed query/allocation votes zero in the same MIN as hits.
storage_hit_count = self._sync_prefetch_hit_query(
operation, storage_hit_count
)
if operation.host_indices is not None:
tail = operation.host_indices[storage_hit_count:]
if tail.numel():
self.mem_pool_host.free(tail, pool=PoolName.KV)
self.prefetch_tokens_occupied -= len(tail)
operation.host_indices = operation.host_indices[
:storage_hit_count
]
operation.hash_value = hash_value[: storage_hit_count // self.page_size]
operation.storage_hit_count = storage_hit_count
operation.storage_start = len(ticket.matched_prefix_tokens)
operation.pool_storage_result = PoolTransferResult.empty()
if not self._allocate_pp_prefetch_buffers(ticket, operation):
operation.host_indices = torch.empty(0, dtype=torch.int64)
if storage_hit_count == 0:
if operation.host_indices is None:
operation.host_indices = torch.empty(0, dtype=torch.int64)
operation.mark_terminate()
if operation.pool_transfers:
self._resolve_sidecar_kv_derived_pool_transfers(operation)
self.prefetch_buffer.put(operation)
except Exception:
logger.exception("PP HiCache ticket processing failed.")
if operation is None or not operation.hash_value:
# Without a received ticket/ACK schedule, continuing is unsafe.
return
if operation.host_indices is None:
operation.host_indices = torch.empty(0, dtype=torch.int64)
operation.mark_terminate()
# Preserve every KV/sidecar ACK even when local allocation fails.
self.prefetch_buffer.put(operation)
# Query/allocation failures already follow the agreed ACK path.
# An unexpected failure cannot safely continue the collectives.
return
finally:
if is_source and objects:
self.pp_prefetch_command_queue.task_done()
@@ -1365,9 +1364,7 @@ class HybridCacheController(BaseHiCacheController):
self.backup_queue.put(operation)
return operation.id
def _storage_hit_query(
self, operation, pp_rank: Optional[int] = None
) -> tuple[list[str], int]:
def _storage_hit_query(self, operation) -> tuple[list[str], int]:
hash_value = get_storage_hash_str(
operation.token_ids, operation.last_hash, page_size=self.page_size
)
@@ -1384,7 +1381,6 @@ class HybridCacheController(BaseHiCacheController):
extra_info = HiCacheStorageExtraInfo(
prefix_keys=operation.prefix_keys.copy() if operation.prefix_keys else None,
extra_info={"pp_rank": pp_rank} if pp_rank is not None else None,
)
if operation.pool_transfers:
hit_result = self.storage_backend.batch_exists_v2(
@@ -1703,7 +1699,7 @@ class HybridCacheController(BaseHiCacheController):
continue
trailing_n = len(transfer.keys) if transfer.keys else 1
transfer.keys = all_hashes[max(0, kv_hit_pages - trailing_n) : kv_hit_pages]
if transfer.host_indices is None:
if transfer.host_indices is None or transfer.indices_from_pool is not None:
continue
entry = self.mem_pool_host.entry_map.get(transfer.name)
pool_page_size = (
@@ -899,11 +899,7 @@ class MooncakeStore(HiCacheStorage, MooncakeBaseStore):
all_component_keys.extend(component_keys)
prepared.append((transfer, key_multiplier, start, len(all_component_keys)))
all_exists = (
self._batch_exist(all_component_keys, extra_info)
if all_component_keys
else []
)
all_exists = self._batch_exist(all_component_keys) if all_component_keys else []
for transfer, key_multiplier, start, end in prepared:
if not restorable:
break
@@ -1441,7 +1437,7 @@ class MooncakeStore(HiCacheStorage, MooncakeBaseStore):
query_keys.append(f"{key}_{self.mha_suffix}_v")
key_multiplier = 2
exist_result = self._batch_exist(query_keys, extra_info)
exist_result = self._batch_exist(query_keys)
for i in range(len(query_keys)):
if exist_result[i] != 1:
return i // key_multiplier
@@ -1493,21 +1489,7 @@ class MooncakeStore(HiCacheStorage, MooncakeBaseStore):
)
return self.store.batch_get_into(key_strs, buffer_ptrs, buffer_sizes)
def _batch_exist(
self, key_strs: List[str], extra_info: Optional[HiCacheStorageExtraInfo] = None
) -> List[int]:
pp_rank = (
(extra_info.extra_info or {}).get("pp_rank")
if extra_info is not None
else None
)
if pp_rank is not None:
# PP is the last rank field before the pool suffix. Replace from
# the right so an identical TP rank or backend tag stays unchanged.
key_strs = [
f"_{pp_rank}_".join(key.rsplit(f"_{self.pp_rank}_", 1))
for key in key_strs
]
def _batch_exist(self, key_strs: List[str]) -> List[int]:
return self.store.batch_is_exist(key_strs)
def get_stats(self):
@@ -1,22 +1,27 @@
"""CPU regressions for PP ticket ordering, admission, and buffer ownership."""
import pickle
import tempfile
import threading
import unittest
from array import array
from contextlib import nullcontext
from dataclasses import replace
from pathlib import Path
from queue import Empty, Queue
from unittest.mock import Mock, call, patch
import torch
from sglang.srt.managers.cache_controller import PrefetchAck
from sglang.srt.environ import envs
from sglang.srt.managers.cache_controller import HiCacheController, PrefetchAck
from sglang.srt.mem_cache.base_prefix_cache import (
CacheRequestHandle,
CacheRequestOutcome,
)
from sglang.srt.mem_cache.buffer_mode.pipeline import BufferModePipeline
from sglang.srt.mem_cache.hicache_storage import (
HiCacheFile,
HiCacheStorageConfig,
PoolName,
PoolTransfer,
@@ -25,6 +30,7 @@ from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import (
HybridCacheController,
PPPrefetchDecision,
)
from sglang.srt.mem_cache.memory_pool_host import LogicalHostPool
from sglang.srt.mem_cache.radix_cache import RadixKey
from sglang.srt.mem_cache.storage_prefetch import StoragePrefetchRetries
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
@@ -64,9 +70,12 @@ class TestPPPrefetchTicket(unittest.TestCase):
c.storage_stop_event = threading.Event()
c.prefetch_tokens_occupied = 0
c.mem_pool_host = Mock(page_size=4)
c.mem_pool_host.available_size.return_value = 1 << 20
c.mem_pool_host.layout_lease.side_effect = nullcontext
c.mem_pool_host.alloc.side_effect = lambda size, **_: torch.arange(size)
c._storage_hit_query = Mock(return_value=(["h0", "h1"], 8))
c.storage_backend_type = "mooncake"
c.storage_backend = Mock()
c._storage_hit_query = Mock(side_effect=self.query_hits)
c._all_reduce = Mock()
ranks = {"pp": [0, 2, 4, 6], "tp": [0, 1], "command": [0, 2, 4, 6]}
patcher = patch.object(
@@ -96,6 +105,14 @@ class TestPPPrefetchTicket(unittest.TestCase):
cache.buffer_pipeline = Mock(spec=BufferModePipeline)
cache._handle_prefetch_result = Mock()
def query_hits(self, operation, hit=8):
hashes = get_storage_hash_str(
operation.token_ids, operation.last_hash, page_size=self.c.page_size
)
operation.all_hash_values = hashes
operation.query_pool_hit_pages = {PoolName.KV: hit // self.c.page_size}
return hashes[: hit // self.c.page_size], hit
def submit(self, rid="hit", pools=None, attempt_id=0, assume_stored=False):
key = RadixKey(
array("q", range(9)), "adapter", is_bigram=True, cache_salt="tenant"
@@ -130,55 +147,433 @@ class TestPPPrefetchTicket(unittest.TestCase):
):
self.c.prefetch_sync_thread_func()
def run_io(self, operations):
with (
patch("sglang.srt.managers.cache_controller.STORAGE_BATCH_SIZE", 1),
patch.object(
self.c.storage_stop_event,
"is_set",
side_effect=[False] * operations + [True],
),
):
self.c.prefetch_io_aux_func()
acks = []
while not self.c.prefetch_sync_queue.empty():
acks.append(self.c.prefetch_sync_queue.get_nowait())
return acks
def test_hit_and_miss_are_reused_across_enqueue_retract_and_retry(self):
c = self.c
c, cache = self.c, self.cache
cache.pp_rank = 0
c.page_get_func = Mock(return_value=1)
for hit in (0, 8):
with self.subTest(hit=hit):
rid = str(hit)
handle = CacheRequestHandle(rid, 0)
c._storage_hit_query.return_value = ([], hit)
self.assertEqual(self.submit(rid).decision, bool(hit))
c._storage_hit_query.side_effect = lambda operation: self.query_hits(
operation, hit
)
self.assertTrue(self.submit(rid).decision)
queries = c._storage_hit_query.call_count
for _ in range(2):
self.assertEqual(
self.cache.prefetch_from_storage(handle, 0, []), bool(hit)
)
self.assertTrue(cache.prefetch_from_storage(handle, 0, []))
self.assertEqual(c._storage_hit_query.call_count, queries)
if hit:
state = c.pp_prefetch_states[rid]
state.ready_event.set()
c.take_ready_pp_prefetch(rid)
self.assertFalse(
self.cache.prefetch_from_storage(
CacheRequestHandle(rid, 1), 0, []
)
)
self.assertTrue(self.cache.check_prefetch_progress(handle))
self.run_commands()
self.assertEqual(c._storage_hit_query.call_count, queries + 1)
self.sync_acks(*self.run_io(1))
self.assertTrue(cache.check_prefetch_progress(handle))
self.assertFalse(
cache.prefetch_from_storage(CacheRequestHandle(rid, 1), 0, [])
)
self.assertFalse(c.release_pp_prefetch(rid))
self.assertIsNone(c.get_prefetch_submission(rid))
self.assertEqual(c.pp_prefetch_command_queue.qsize(), 1)
self.cache._all_reduce.assert_not_called()
self.assertTrue(c.pp_prefetch_command_queue.empty())
def test_submission_does_not_wait_for_worker_and_uses_only_tp_consensus(self):
def test_all_backends_query_and_allocate_before_one_worker_consensus(self):
c = self.c
with patch.object(
c.pp_prefetch_command_queue,
"join",
side_effect=AssertionError("scheduler blocked"),
for backend in ("file", "mooncake", "dynamic"):
with self.subTest(backend=backend):
c.storage_backend_type = backend
c._storage_hit_query.reset_mock()
c._all_reduce.reset_mock()
c.mem_pool_host.alloc.reset_mock()
c._storage_hit_query.side_effect = AssertionError(
"scheduler queried storage"
)
c._all_reduce.side_effect = AssertionError(
"scheduler entered a collective"
)
with patch.object(
c.pp_prefetch_command_queue,
"join",
side_effect=AssertionError("scheduler blocked"),
):
first = self.submit(backend, assume_stored=True)
second = self.submit(f"next_{backend}")
self.assertTrue(first.decision)
self.assertTrue(second.decision)
self.assertEqual(c.pp_prefetch_command_queue.qsize(), 2)
self.assertFalse(c.is_pp_prefetch_ready(backend))
self.assertFalse(first.operation.assume_stored)
c.mem_pool_host.alloc.assert_not_called()
c._storage_hit_query.assert_not_called()
c._all_reduce.assert_not_called()
votes = []
def reduce_hits(tensor, op, groups):
self.assertEqual(op, torch.distributed.ReduceOp.MIN)
self.assertEqual(groups, c.prefetch_hits_sync_groups)
self.assertEqual(tensor.numel(), 1 + len(PoolName))
self.assertEqual(tensor[0].item(), 8)
self.assertEqual(c.mem_pool_host.alloc.call_count, len(votes) + 1)
votes.append(tensor[0].item())
c._storage_hit_query.side_effect = self.query_hits
c._all_reduce.side_effect = reduce_hits
self.run_commands()
self.assertEqual(
c._storage_hit_query.call_args_list,
[call(first.operation), call(second.operation)],
)
self.assertEqual(votes, [8, 8])
self.assertEqual(c._all_reduce.call_count, 2)
self.assertIs(c.prefetch_buffer.get_nowait(), first.operation)
self.assertIs(c.prefetch_buffer.get_nowait(), second.operation)
def test_file_attachment_enables_pp_tickets(self):
c = self.c
c.pp_prefetch_command_group = None
c.storage_host_pool = Mock()
c.host_memory_mode = "buffer_only"
c.storage_backend_type = "file"
with (
patch.object(HiCacheController, "attach_storage_backend"),
patch.object(torch.distributed, "get_world_size", return_value=4),
patch(
"sglang.srt.distributed.parallel_state.create_custom_parallel_group",
return_value="command",
),
):
self.assertTrue(self.submit(assume_stored=True).decision)
self.assertTrue(self.submit("next").decision)
self.assertEqual(c.pp_prefetch_command_queue.qsize(), 2)
self.assertFalse(c.is_pp_prefetch_ready("hit"))
c.mem_pool_host.alloc.assert_not_called()
self.assertFalse(c.pp_prefetch_states["hit"].operation.assume_stored)
self.assertEqual(
c._all_reduce.call_args.args[1:], (torch.distributed.ReduceOp.MIN, ["tp"])
c.attach_storage_backend("file", prefetch_threshold=4)
self.assertEqual(c.pp_prefetch_command_group, "command")
def test_file_stage_hits_trim_buffers_after_local_allocation(self):
"""PP0's local files cannot reveal a shorter prefix on another stage."""
c = self.c
key = RadixKey(
array("q", range(9)), "adapter", is_bigram=True, cache_salt="tenant"
)
self.assertEqual(
[call.kwargs["pp_rank"] for call in c._storage_hit_query.call_args_list],
[0, 2, 0, 2],
hashes = get_storage_hash_str(key, "ab" * 32, page_size=4)
local_votes = []
with (
tempfile.TemporaryDirectory() as directory,
patch.object(
envs.SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR, "get", return_value=None
),
):
backends = []
for rank, pages in ((0, 2), (1, 1)):
config = replace(
c.storage_config,
pp_rank=rank,
pp_size=2,
extra_config={
"file_storage_path": str(Path(directory) / str(rank)),
"enable_metadata_cache": False,
"max_size": 0,
"min_free_space": 0,
},
)
backend = HiCacheFile(config)
for hash_value in hashes[:pages]:
self.assertTrue(backend.set(hash_value, torch.ones(1)))
backends.append(backend)
self.assertEqual(backends[0].batch_exists(hashes), 2)
self.assertEqual(backends[1].batch_exists(hashes), 1)
c.storage_backend = backends[0]
c.storage_backend_type = "file"
c._storage_hit_query = Mock(
wraps=HybridCacheController._storage_hit_query.__get__(c)
)
result = self.submit(attempt_id=3)
ticket = pickle.loads(pickle.dumps(c.pp_prefetch_states["hit"].ticket))
for rank, peer_hit in ((0, 4), (1, 8)):
with self.subTest(rank=rank):
c.storage_config.pp_rank = rank
c.storage_backend = backends[rank]
if rank:
c.pp_prefetch_states.clear()
c._storage_hit_query.reset_mock()
c.mem_pool_host.alloc.reset_mock()
c.mem_pool_host.free.reset_mock()
c.prefetch_tokens_occupied = 0
local_hit = 8 if rank == 0 else 4
def reduce_hits(tensor, op, groups):
self.assertEqual(op, torch.distributed.ReduceOp.MIN)
self.assertEqual(groups, ["pp", "tp"])
self.assertEqual(tensor.numel(), 1 + len(PoolName))
local_votes.append(tensor[0].item())
c.mem_pool_host.alloc.assert_called_once_with(
local_hit, pool=PoolName.KV
)
tensor[0] = min(tensor[0].item(), peer_hit)
kv_index = 1 + list(PoolName).index(PoolName.KV)
tensor[kv_index] = min(tensor[kv_index].item(), peer_hit // 4)
c._all_reduce.side_effect = reduce_hits
self.run_commands(ticket)
operation = c.prefetch_buffer.get_nowait()
self.assertEqual(operation.handle, CacheRequestHandle("hit", 3))
self.assertEqual(operation.storage_hit_count, 4)
self.assertEqual(operation.hash_value, hashes[:1])
self.assertEqual(operation.host_indices.numel(), 4)
self.assertFalse(operation.is_terminated())
self.assertEqual(c.prefetch_tokens_occupied, 4)
if rank == 0:
self.assertIs(operation, result.operation)
freed = c.mem_pool_host.free.call_args
self.assertEqual(freed.kwargs, {"pool": PoolName.KV})
self.assertTrue(torch.equal(freed.args[0], torch.arange(4, 8)))
self.assertEqual(c.mem_pool_host.free.call_count, 1)
else:
c.mem_pool_host.free.assert_not_called()
c._storage_hit_query.assert_called_once_with(operation)
self.assertEqual(operation.query_pool_hit_pages, {PoolName.KV: 1})
self.assertEqual(local_votes, [8, 4])
def test_file_stage_hit_fits_capacity_before_consensus(self):
"""A large local hit must not discard a prefix that fits on every stage."""
c = self.c
c.page_size = c.prefetch_threshold = 256
key = RadixKey(array("q", range(8193)), is_bigram=True)
hashes = get_storage_hash_str(key, page_size=256)
c.storage_backend_type = "file"
c._storage_hit_query = Mock(
wraps=HybridCacheController._storage_hit_query.__get__(c)
)
with (
tempfile.TemporaryDirectory() as directory,
patch.object(
envs.SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR, "get", return_value=None
),
):
backends = []
for rank, pages in ((0, 32), (1, 8)):
config = replace(
c.storage_config,
pp_rank=rank,
pp_size=2,
extra_config={
"file_storage_path": str(Path(directory) / str(rank)),
"enable_metadata_cache": False,
"max_size": 0,
"min_free_space": 0,
},
)
backend = HiCacheFile(config)
for hash_value in hashes[:pages]:
self.assertTrue(backend.set(hash_value, torch.ones(1)))
backends.append(backend)
for capacity in (4096, 4097):
c.storage_config.pp_rank = 0
c.pp_prefetch_states.clear()
c.pp_prefetch_command_queue = Queue()
result = c.submit_prefetch(
CacheRequestHandle("capacity", 0), key, None, None, [], None
)
ticket = pickle.loads(
pickle.dumps(c.pp_prefetch_states["capacity"].ticket)
)
for rank, local_vote in ((0, 4096), (1, 2048)):
with self.subTest(capacity=capacity, rank=rank):
c.storage_config.pp_rank = rank
c.storage_backend = backends[rank]
if rank:
c.pp_prefetch_states.clear()
host_pool = LogicalHostPool(4096, page_size=256)
c.mem_pool_host.available_size.return_value = capacity
c.mem_pool_host.available_size.reset_mock()
c.mem_pool_host.alloc.reset_mock()
c.mem_pool_host.free.reset_mock()
c.mem_pool_host.alloc.side_effect = lambda size, **_: (
host_pool.alloc(size)
)
c.mem_pool_host.free.side_effect = lambda indices, **_: (
host_pool.free(indices)
)
c.prefetch_tokens_occupied = 0
c._storage_hit_query.reset_mock()
c._all_reduce.reset_mock()
local_votes = []
def reduce_hits(tensor, op, groups):
self.assertEqual(op, torch.distributed.ReduceOp.MIN)
self.assertEqual(groups, c.prefetch_hits_sync_groups)
self.assertEqual(tensor.numel(), 1 + len(PoolName))
local_votes.append(tensor[0].item())
kv_index = 1 + list(PoolName).index(PoolName.KV)
self.assertEqual(
tensor[kv_index].item(), 32 if rank == 0 else 8
)
peer_vote = 2048 if rank == 0 else 4096
tensor[0] = min(tensor[0].item(), peer_vote)
tensor[kv_index] = min(tensor[kv_index].item(), 8)
c._all_reduce.side_effect = reduce_hits
self.run_commands(ticket)
operation = c.prefetch_buffer.get_nowait()
if rank == 0:
self.assertIs(operation, result.operation)
self.assertEqual(operation.storage_hit_count, 2048)
self.assertEqual(local_votes, [local_vote])
c.mem_pool_host.alloc.assert_called_once_with(
local_vote, pool=PoolName.KV
)
self.assertEqual(operation.hash_value, hashes[:8])
self.assertEqual(operation.host_indices.numel(), 2048)
self.assertFalse(operation.is_terminated())
self.assertEqual(
operation.query_pool_hit_pages, {PoolName.KV: 8}
)
self.assertEqual(c.prefetch_tokens_occupied, 2048)
self.assertEqual(host_pool.available_size(), 2048)
c._storage_hit_query.assert_called_once_with(operation)
c.mem_pool_host.available_size.assert_called_once_with(
pool=PoolName.KV
)
c._all_reduce.assert_called_once()
if rank == 0:
self.assertEqual(c.mem_pool_host.free.call_count, 1)
freed = c.mem_pool_host.free.call_args
self.assertEqual(freed.kwargs, {"pool": PoolName.KV})
self.assertTrue(
torch.equal(freed.args[0], torch.arange(2048, 4096))
)
else:
c.mem_pool_host.free.assert_not_called()
def test_local_miss_or_query_error_completes_before_next_ticket(self):
"""A stage miss/error must complete admission without breaking ACK order."""
c, cache = self.c, self.cache
c.storage_backend_type = "file"
cache.pp_rank = 0
for name, local_hit, peer_hit, threshold in (
("local_miss", 0, 8, 4),
("peer_miss", 8, 0, 4),
("below_threshold", 7, 8, 6),
("query_error", RuntimeError("local storage unavailable"), 8, 4),
):
with self.subTest(name=name):
failed_rid, next_rid = name, f"next_{name}"
c.prefetch_threshold = threshold
c._storage_hit_query.reset_mock()
c._all_reduce.reset_mock()
c.mem_pool_host.alloc.reset_mock()
c.mem_pool_host.free.reset_mock()
cache._handle_prefetch_result.reset_mock()
occupied_before = c.prefetch_tokens_occupied
hits = iter((local_hit, 8))
def query_hits(operation):
hit = next(hits)
if isinstance(hit, Exception):
operation.query_pool_hit_pages = {PoolName.KV: 2}
raise hit
return self.query_hits(operation, hit)
c._storage_hit_query.side_effect = query_hits
self.assertTrue(self.submit(failed_rid, attempt_id=3).decision)
self.assertTrue(self.submit(next_rid).decision)
peers = iter((peer_hit, 8))
local_votes = []
def reduce_hits(tensor, op, groups):
self.assertEqual(op, torch.distributed.ReduceOp.MIN)
self.assertEqual(groups, ["pp", "tp"])
self.assertEqual(tensor.numel(), 1 + len(PoolName))
local_votes.append(tensor[0].item())
if tensor[0].item():
self.assertEqual(c.mem_pool_host.alloc.call_args.args, (8,))
tensor[0] = min(tensor[0].item(), next(peers))
c._all_reduce.side_effect = reduce_hits
logs = (
self.assertLogs(level="ERROR")
if isinstance(local_hit, Exception)
else nullcontext()
)
with logs:
self.run_commands()
failed = c.pp_prefetch_states[failed_rid].operation
following = c.pp_prefetch_states[next_rid].operation
self.assertEqual(failed.storage_hit_count, 0)
self.assertEqual(failed.hash_value, [])
self.assertEqual(failed.host_indices.numel(), 0)
self.assertTrue(failed.is_terminated())
self.assertEqual(following.host_indices.numel(), 8)
self.assertFalse(following.is_terminated())
self.assertEqual(
local_votes,
[8 if name == "peer_miss" else 0, 8],
)
self.assertEqual(c._all_reduce.call_count, 2)
self.assertEqual(
c._storage_hit_query.call_args_list,
[call(failed), call(following)],
)
if name == "peer_miss":
self.assertEqual(
c.mem_pool_host.alloc.call_args_list,
[call(8, pool=PoolName.KV)] * 2,
)
self.assertEqual(c.mem_pool_host.free.call_count, 1)
self.assertTrue(
torch.equal(
c.mem_pool_host.free.call_args.args[0], torch.arange(8)
)
)
self.assertEqual(
c.mem_pool_host.free.call_args.kwargs, {"pool": PoolName.KV}
)
else:
c.mem_pool_host.alloc.assert_called_once_with(8, pool=PoolName.KV)
c.mem_pool_host.free.assert_not_called()
if name == "query_error":
self.assertEqual(failed.query_pool_hit_pages, {})
self.assertEqual(c.prefetch_tokens_occupied, occupied_before + 8)
c._all_reduce.side_effect = None
c.page_get_func = Mock(return_value=1)
acks = self.run_io(2)
self.assertEqual(
[ack.rid for ack in acks], [failed_rid] + [next_rid] * 3
)
self.assertTrue(acks[0].completed_req)
self.assertEqual(
[ack.completed_tokens for ack in acks[1:]], [4, 8, None]
)
self.assertEqual(c.page_get_func.call_count, 2)
self.sync_acks(*acks)
handle = CacheRequestHandle(failed_rid, 3)
self.assertTrue(c.is_pp_prefetch_ready(failed_rid))
self.assertTrue(cache.check_prefetch_progress(handle))
self.assertEqual(cache.prefetch_loaded_tokens_by_reqid[handle], 0)
self.assertNotIn(handle, cache.ongoing_prefetch)
cache._handle_prefetch_result.assert_not_called()
self.assertTrue(cache.check_prefetch_progress(handle))
self.assertFalse(c.release_pp_prefetch(failed_rid))
self.assertTrue(
cache.check_prefetch_progress(CacheRequestHandle(next_rid, 0))
)
cache._handle_prefetch_result.assert_called_once_with(following)
self.assertEqual(c._storage_hit_query.call_count, 2)
self.assertEqual(c.pp_prefetch_command_queue.unfinished_tasks, 0)
def test_tp_only_preserves_normal_prefetch(self):
c = self.c
@@ -209,7 +604,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
)
self.assertTrue(operation.token_ids.is_bigram)
self.assertEqual(len(operation.token_ids), 8)
c._storage_hit_query.assert_not_called()
c._storage_hit_query.assert_called_once_with(operation)
self.sync_acks(PrefetchAck("hit", operation, completed_tokens=8))
self.assertFalse(c.is_pp_prefetch_ready("hit"))
self.sync_acks(PrefetchAck("hit", operation, completed_req=True))
@@ -244,7 +639,22 @@ class TestPPPrefetchTicket(unittest.TestCase):
with self.assertLogs(level="ERROR"):
self.run_commands(ticket, following)
failed = c.pp_prefetch_states["hit"].operation
next_operation = c.pp_prefetch_states["next"].operation
self.assertTrue(failed.is_terminated())
self.assertEqual(failed.storage_hit_count, 0)
self.assertEqual(failed.hash_value, [])
self.assertEqual(failed.host_indices.numel(), 0)
self.assertEqual(
c._storage_hit_query.call_args_list, [call(failed), call(next_operation)]
)
self.assertEqual(c._all_reduce.call_count, 2)
self.assertEqual(
[
invocation.args[0][0].item()
for invocation in c._all_reduce.call_args_list
],
[0, 8],
)
c.mem_pool_host.free.assert_called_once_with(kv, pool=PoolName.KV)
c.page_get_func = Mock(return_value=1)
c.storage_backend = Mock()
@@ -256,12 +666,12 @@ class TestPPPrefetchTicket(unittest.TestCase):
),
):
c.prefetch_io_aux_func()
acks = [c.prefetch_sync_queue.get_nowait() for _ in range(8)]
self.assertEqual([a.rid for a in acks], ["hit"] * 4 + ["next"] * 4)
acks = [c.prefetch_sync_queue.get_nowait() for _ in range(6)]
self.assertEqual([a.rid for a in acks], ["hit"] * 2 + ["next"] * 4)
self.assertEqual(
[a.completed_tokens for a in acks], [0, 0, None, None, 4, 8, None, None]
[a.completed_tokens for a in acks], [None, None, 4, 8, None, None]
)
self.assertEqual((acks[2].pool_hits, acks[6].pool_hits), ({}, {"swa": 1}))
self.assertEqual((acks[0].pool_hits, acks[4].pool_hits), ({}, {"swa": 1}))
self.assertEqual(
c.page_get_func.call_count, 2
) # No reads for the failed ticket.
@@ -341,6 +751,16 @@ class TestPPPrefetchTicket(unittest.TestCase):
c.mem_pool_host.alloc.side_effect = lambda *args, **kwargs: None
self.run_commands()
self.assertTrue(operation.is_terminated())
self.assertEqual(operation.hash_value, [])
self.assertEqual(operation.storage_hit_count, 0)
c._storage_hit_query.assert_called_once_with(operation)
c.mem_pool_host.alloc.assert_called_once_with(8, pool=PoolName.KV)
c._all_reduce.assert_called_once()
packed, op, groups = c._all_reduce.call_args.args
self.assertEqual(packed[0].item(), 0)
self.assertEqual(packed.numel(), 1 + len(PoolName))
self.assertEqual(op, torch.distributed.ReduceOp.MIN)
self.assertEqual(groups, c.prefetch_hits_sync_groups)
c.mem_pool_host.free.assert_not_called()
self.sync_acks(
PrefetchAck("hit", operation, completed_tokens=0, completed_req=True)
@@ -349,6 +769,69 @@ class TestPPPrefetchTicket(unittest.TestCase):
self.assertIsNone(borrowed.host_indices)
self.assertEqual(c.mem_pool_host.free.call_args.kwargs["pool"], PoolName.SWA)
def test_peer_vote_trims_kv_alias_and_defers_sidecar_free(self):
"""A peer's zero allocation vote must release KV without freeing a sidecar."""
c = self.c
for peer_vote in (4, 0):
with self.subTest(peer_vote=peer_vote):
rid = f"peer_{peer_vote}"
c._storage_hit_query.reset_mock()
c._all_reduce.reset_mock()
c.mem_pool_host.alloc.reset_mock()
c.mem_pool_host.free.reset_mock()
borrowed_indices = torch.arange(4)
operation = self.submit(
rid,
pools=[
PoolTransfer(PoolName.SWA, host_indices=borrowed_indices),
PoolTransfer(PoolName.INDEXER, indices_from_pool=PoolName.KV),
],
).operation
def reduce_hits(tensor, op, groups):
self.assertEqual(tensor[0].item(), 8)
self.assertEqual(tensor.numel(), 1 + len(PoolName))
self.assertEqual(op, torch.distributed.ReduceOp.MIN)
self.assertEqual(groups, c.prefetch_hits_sync_groups)
c.mem_pool_host.alloc.assert_called_once_with(8, pool=PoolName.KV)
tensor[0] = peer_vote
c._all_reduce.side_effect = reduce_hits
self.run_commands()
c._storage_hit_query.assert_called_once_with(operation)
c._all_reduce.assert_called_once()
swa, indexer = operation.pool_transfers
self.assertEqual(operation.host_indices.numel(), peer_vote)
self.assertEqual(operation.storage_hit_count, peer_vote)
self.assertEqual(operation.is_terminated(), peer_vote == 0)
self.assertIs(indexer.host_indices, operation.host_indices)
self.assertIs(indexer.keys, operation.hash_value)
self.assertIs(swa.host_indices, borrowed_indices)
self.assertEqual(c.prefetch_tokens_occupied, peer_vote)
freed = c.mem_pool_host.free.call_args
self.assertEqual(c.mem_pool_host.free.call_count, 1)
self.assertEqual(freed.kwargs, {"pool": PoolName.KV})
self.assertTrue(torch.equal(freed.args[0], torch.arange(peer_vote, 8)))
self.assertTrue(c.release_pp_prefetch(rid))
self.assertIs(swa.host_indices, borrowed_indices)
self.assertEqual(c.mem_pool_host.free.call_count, 1)
c._all_reduce.side_effect = None
self.sync_acks(
PrefetchAck(rid, operation, completed_tokens=peer_vote),
PrefetchAck(rid, operation, completed_req=True),
)
self.assertIsNone(swa.host_indices)
self.assertNotIn(rid, c.pp_prefetch_states)
self.assertEqual(c.prefetch_tokens_occupied, 0)
sidecar_frees = [
invocation
for invocation in c.mem_pool_host.free.call_args_list
if invocation.kwargs["pool"] == PoolName.SWA
]
self.assertEqual(len(sidecar_frees), 1)
self.assertIs(sidecar_frees[0].args[0], borrowed_indices)
def test_command_failure_finishes_queue_task_and_is_reported_on_poll(self):
c = self.c
with (