[Perf] hicache: Unify PP prefetch hit queries and buffer allocation (#43344)
This commit is contained in:
@@ -87,7 +87,6 @@ class PPPrefetchTicket:
|
||||
prefix_keys: Optional[List[str]]
|
||||
matched_prefix_tokens: List[int]
|
||||
pool_specs: tuple[PPPrefetchPoolSpec, ...]
|
||||
storage_hit_count: int = 0
|
||||
|
||||
|
||||
class PPPrefetchDecision(Enum):
|
||||
@@ -316,7 +315,6 @@ class HybridCacheController(BaseHiCacheController):
|
||||
)
|
||||
enable_pp_ticket = (
|
||||
self.host_memory_mode == "buffer_only"
|
||||
and storage_backend == "mooncake"
|
||||
and self.pp_group is not None
|
||||
and torch.distributed.get_world_size(group=self.pp_group) > 1
|
||||
)
|
||||
@@ -1101,32 +1099,8 @@ class HybridCacheController(BaseHiCacheController):
|
||||
pool_transfers=pool_transfers,
|
||||
)
|
||||
|
||||
storage_hit_count = len(ticket.prefetch_key.token_ids)
|
||||
try:
|
||||
config = self.storage_config
|
||||
for pp_rank in range(config.tp_rank, config.pp_size, config.tp_size):
|
||||
_, rank_hit_count = self._storage_hit_query(operation, pp_rank=pp_rank)
|
||||
storage_hit_count = min(storage_hit_count, rank_hit_count)
|
||||
except Exception:
|
||||
logger.exception("PP HiCache hit query failed for req=%s.", rid)
|
||||
storage_hit_count = 0
|
||||
|
||||
pp_group_ranks = set(torch.distributed.get_process_group_ranks(self.pp_group))
|
||||
local_groups = [
|
||||
group
|
||||
for group in self.prefetch_hits_sync_groups
|
||||
if set(torch.distributed.get_process_group_ranks(group)) != pp_group_ranks
|
||||
]
|
||||
hit_tensor = torch.tensor(storage_hit_count, dtype=torch.int)
|
||||
self._all_reduce(hit_tensor, torch.distributed.ReduceOp.MIN, local_groups)
|
||||
storage_hit_count = int(hit_tensor.item())
|
||||
storage_hit_count -= storage_hit_count % self.page_size
|
||||
if storage_hit_count < self.prefetch_threshold:
|
||||
with self.pp_prefetch_state_lock:
|
||||
self.pp_prefetch_decisions[rid] = PPPrefetchDecision.SKIPPED
|
||||
return PrefetchSubmission(decision=False)
|
||||
|
||||
ticket.storage_hit_count = storage_hit_count
|
||||
# Every rank queries and allocates in ticket order on the worker.
|
||||
# Admission waits for the normal completion ACK, including misses.
|
||||
operation.is_pp_broadcast = True
|
||||
state = PPPrefetchState(ticket=ticket, operation=operation)
|
||||
with self.pp_prefetch_state_lock:
|
||||
@@ -1158,7 +1132,7 @@ class HybridCacheController(BaseHiCacheController):
|
||||
allocated.append((name, indices))
|
||||
return indices
|
||||
|
||||
host_indices = alloc(PoolName.KV, ticket.storage_hit_count)
|
||||
host_indices = alloc(PoolName.KV, operation.storage_hit_count)
|
||||
if host_indices is None:
|
||||
return False
|
||||
|
||||
@@ -1177,7 +1151,7 @@ class HybridCacheController(BaseHiCacheController):
|
||||
spec.num_slots,
|
||||
min(
|
||||
len(spec.keys or []),
|
||||
ticket.storage_hit_count // self.page_size,
|
||||
operation.storage_hit_count // self.page_size,
|
||||
),
|
||||
)
|
||||
if indices is None:
|
||||
@@ -1281,7 +1255,6 @@ class HybridCacheController(BaseHiCacheController):
|
||||
objects = [self.pp_prefetch_command_queue.get(timeout=60)]
|
||||
except Empty:
|
||||
pass # Complete idle broadcasts before the group times out.
|
||||
operation = None
|
||||
try:
|
||||
objects = broadcast_pyobj(objects, rank, group, src=source)
|
||||
if not objects:
|
||||
@@ -1320,29 +1293,55 @@ class HybridCacheController(BaseHiCacheController):
|
||||
self.pp_prefetch_decisions.pop(rid)
|
||||
operation = state.operation
|
||||
|
||||
operation.hash_value = get_storage_hash_str(
|
||||
ticket.prefetch_key,
|
||||
ticket.last_hash,
|
||||
page_size=self.page_size,
|
||||
)[: ticket.storage_hit_count // self.page_size]
|
||||
operation.all_hash_values = list(operation.hash_value)
|
||||
operation.storage_hit_count = ticket.storage_hit_count
|
||||
try:
|
||||
hash_value, storage_hit_count = self._storage_hit_query(operation)
|
||||
storage_hit_count = min(
|
||||
storage_hit_count,
|
||||
self.mem_pool_host.available_size(pool=PoolName.KV),
|
||||
)
|
||||
storage_hit_count -= storage_hit_count % self.page_size
|
||||
operation.storage_hit_count = storage_hit_count
|
||||
if (
|
||||
storage_hit_count == 0
|
||||
or storage_hit_count < self.prefetch_threshold
|
||||
or not self._allocate_pp_prefetch_buffers(ticket, operation)
|
||||
):
|
||||
storage_hit_count = 0
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"PP HiCache query or allocation failed for req=%s.", rid
|
||||
)
|
||||
operation.query_pool_hit_pages = {}
|
||||
hash_value, storage_hit_count = [], 0
|
||||
|
||||
# A failed query/allocation votes zero in the same MIN as hits.
|
||||
storage_hit_count = self._sync_prefetch_hit_query(
|
||||
operation, storage_hit_count
|
||||
)
|
||||
if operation.host_indices is not None:
|
||||
tail = operation.host_indices[storage_hit_count:]
|
||||
if tail.numel():
|
||||
self.mem_pool_host.free(tail, pool=PoolName.KV)
|
||||
self.prefetch_tokens_occupied -= len(tail)
|
||||
operation.host_indices = operation.host_indices[
|
||||
:storage_hit_count
|
||||
]
|
||||
operation.hash_value = hash_value[: storage_hit_count // self.page_size]
|
||||
operation.storage_hit_count = storage_hit_count
|
||||
operation.storage_start = len(ticket.matched_prefix_tokens)
|
||||
operation.pool_storage_result = PoolTransferResult.empty()
|
||||
if not self._allocate_pp_prefetch_buffers(ticket, operation):
|
||||
operation.host_indices = torch.empty(0, dtype=torch.int64)
|
||||
if storage_hit_count == 0:
|
||||
if operation.host_indices is None:
|
||||
operation.host_indices = torch.empty(0, dtype=torch.int64)
|
||||
operation.mark_terminate()
|
||||
if operation.pool_transfers:
|
||||
self._resolve_sidecar_kv_derived_pool_transfers(operation)
|
||||
self.prefetch_buffer.put(operation)
|
||||
except Exception:
|
||||
logger.exception("PP HiCache ticket processing failed.")
|
||||
if operation is None or not operation.hash_value:
|
||||
# Without a received ticket/ACK schedule, continuing is unsafe.
|
||||
return
|
||||
if operation.host_indices is None:
|
||||
operation.host_indices = torch.empty(0, dtype=torch.int64)
|
||||
operation.mark_terminate()
|
||||
# Preserve every KV/sidecar ACK even when local allocation fails.
|
||||
self.prefetch_buffer.put(operation)
|
||||
# Query/allocation failures already follow the agreed ACK path.
|
||||
# An unexpected failure cannot safely continue the collectives.
|
||||
return
|
||||
finally:
|
||||
if is_source and objects:
|
||||
self.pp_prefetch_command_queue.task_done()
|
||||
@@ -1365,9 +1364,7 @@ class HybridCacheController(BaseHiCacheController):
|
||||
self.backup_queue.put(operation)
|
||||
return operation.id
|
||||
|
||||
def _storage_hit_query(
|
||||
self, operation, pp_rank: Optional[int] = None
|
||||
) -> tuple[list[str], int]:
|
||||
def _storage_hit_query(self, operation) -> tuple[list[str], int]:
|
||||
hash_value = get_storage_hash_str(
|
||||
operation.token_ids, operation.last_hash, page_size=self.page_size
|
||||
)
|
||||
@@ -1384,7 +1381,6 @@ class HybridCacheController(BaseHiCacheController):
|
||||
|
||||
extra_info = HiCacheStorageExtraInfo(
|
||||
prefix_keys=operation.prefix_keys.copy() if operation.prefix_keys else None,
|
||||
extra_info={"pp_rank": pp_rank} if pp_rank is not None else None,
|
||||
)
|
||||
if operation.pool_transfers:
|
||||
hit_result = self.storage_backend.batch_exists_v2(
|
||||
@@ -1703,7 +1699,7 @@ class HybridCacheController(BaseHiCacheController):
|
||||
continue
|
||||
trailing_n = len(transfer.keys) if transfer.keys else 1
|
||||
transfer.keys = all_hashes[max(0, kv_hit_pages - trailing_n) : kv_hit_pages]
|
||||
if transfer.host_indices is None:
|
||||
if transfer.host_indices is None or transfer.indices_from_pool is not None:
|
||||
continue
|
||||
entry = self.mem_pool_host.entry_map.get(transfer.name)
|
||||
pool_page_size = (
|
||||
|
||||
@@ -899,11 +899,7 @@ class MooncakeStore(HiCacheStorage, MooncakeBaseStore):
|
||||
all_component_keys.extend(component_keys)
|
||||
prepared.append((transfer, key_multiplier, start, len(all_component_keys)))
|
||||
|
||||
all_exists = (
|
||||
self._batch_exist(all_component_keys, extra_info)
|
||||
if all_component_keys
|
||||
else []
|
||||
)
|
||||
all_exists = self._batch_exist(all_component_keys) if all_component_keys else []
|
||||
for transfer, key_multiplier, start, end in prepared:
|
||||
if not restorable:
|
||||
break
|
||||
@@ -1441,7 +1437,7 @@ class MooncakeStore(HiCacheStorage, MooncakeBaseStore):
|
||||
query_keys.append(f"{key}_{self.mha_suffix}_v")
|
||||
key_multiplier = 2
|
||||
|
||||
exist_result = self._batch_exist(query_keys, extra_info)
|
||||
exist_result = self._batch_exist(query_keys)
|
||||
for i in range(len(query_keys)):
|
||||
if exist_result[i] != 1:
|
||||
return i // key_multiplier
|
||||
@@ -1493,21 +1489,7 @@ class MooncakeStore(HiCacheStorage, MooncakeBaseStore):
|
||||
)
|
||||
return self.store.batch_get_into(key_strs, buffer_ptrs, buffer_sizes)
|
||||
|
||||
def _batch_exist(
|
||||
self, key_strs: List[str], extra_info: Optional[HiCacheStorageExtraInfo] = None
|
||||
) -> List[int]:
|
||||
pp_rank = (
|
||||
(extra_info.extra_info or {}).get("pp_rank")
|
||||
if extra_info is not None
|
||||
else None
|
||||
)
|
||||
if pp_rank is not None:
|
||||
# PP is the last rank field before the pool suffix. Replace from
|
||||
# the right so an identical TP rank or backend tag stays unchanged.
|
||||
key_strs = [
|
||||
f"_{pp_rank}_".join(key.rsplit(f"_{self.pp_rank}_", 1))
|
||||
for key in key_strs
|
||||
]
|
||||
def _batch_exist(self, key_strs: List[str]) -> List[int]:
|
||||
return self.store.batch_is_exist(key_strs)
|
||||
|
||||
def get_stats(self):
|
||||
|
||||
@@ -1,22 +1,27 @@
|
||||
"""CPU regressions for PP ticket ordering, admission, and buffer ownership."""
|
||||
|
||||
import pickle
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from array import array
|
||||
from contextlib import nullcontext
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from queue import Empty, Queue
|
||||
from unittest.mock import Mock, call, patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.managers.cache_controller import PrefetchAck
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.managers.cache_controller import HiCacheController, PrefetchAck
|
||||
from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
CacheRequestHandle,
|
||||
CacheRequestOutcome,
|
||||
)
|
||||
from sglang.srt.mem_cache.buffer_mode.pipeline import BufferModePipeline
|
||||
from sglang.srt.mem_cache.hicache_storage import (
|
||||
HiCacheFile,
|
||||
HiCacheStorageConfig,
|
||||
PoolName,
|
||||
PoolTransfer,
|
||||
@@ -25,6 +30,7 @@ from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import (
|
||||
HybridCacheController,
|
||||
PPPrefetchDecision,
|
||||
)
|
||||
from sglang.srt.mem_cache.memory_pool_host import LogicalHostPool
|
||||
from sglang.srt.mem_cache.radix_cache import RadixKey
|
||||
from sglang.srt.mem_cache.storage_prefetch import StoragePrefetchRetries
|
||||
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
|
||||
@@ -64,9 +70,12 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
||||
c.storage_stop_event = threading.Event()
|
||||
c.prefetch_tokens_occupied = 0
|
||||
c.mem_pool_host = Mock(page_size=4)
|
||||
c.mem_pool_host.available_size.return_value = 1 << 20
|
||||
c.mem_pool_host.layout_lease.side_effect = nullcontext
|
||||
c.mem_pool_host.alloc.side_effect = lambda size, **_: torch.arange(size)
|
||||
c._storage_hit_query = Mock(return_value=(["h0", "h1"], 8))
|
||||
c.storage_backend_type = "mooncake"
|
||||
c.storage_backend = Mock()
|
||||
c._storage_hit_query = Mock(side_effect=self.query_hits)
|
||||
c._all_reduce = Mock()
|
||||
ranks = {"pp": [0, 2, 4, 6], "tp": [0, 1], "command": [0, 2, 4, 6]}
|
||||
patcher = patch.object(
|
||||
@@ -96,6 +105,14 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
||||
cache.buffer_pipeline = Mock(spec=BufferModePipeline)
|
||||
cache._handle_prefetch_result = Mock()
|
||||
|
||||
def query_hits(self, operation, hit=8):
|
||||
hashes = get_storage_hash_str(
|
||||
operation.token_ids, operation.last_hash, page_size=self.c.page_size
|
||||
)
|
||||
operation.all_hash_values = hashes
|
||||
operation.query_pool_hit_pages = {PoolName.KV: hit // self.c.page_size}
|
||||
return hashes[: hit // self.c.page_size], hit
|
||||
|
||||
def submit(self, rid="hit", pools=None, attempt_id=0, assume_stored=False):
|
||||
key = RadixKey(
|
||||
array("q", range(9)), "adapter", is_bigram=True, cache_salt="tenant"
|
||||
@@ -130,55 +147,433 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
||||
):
|
||||
self.c.prefetch_sync_thread_func()
|
||||
|
||||
def run_io(self, operations):
|
||||
with (
|
||||
patch("sglang.srt.managers.cache_controller.STORAGE_BATCH_SIZE", 1),
|
||||
patch.object(
|
||||
self.c.storage_stop_event,
|
||||
"is_set",
|
||||
side_effect=[False] * operations + [True],
|
||||
),
|
||||
):
|
||||
self.c.prefetch_io_aux_func()
|
||||
acks = []
|
||||
while not self.c.prefetch_sync_queue.empty():
|
||||
acks.append(self.c.prefetch_sync_queue.get_nowait())
|
||||
return acks
|
||||
|
||||
def test_hit_and_miss_are_reused_across_enqueue_retract_and_retry(self):
|
||||
c = self.c
|
||||
c, cache = self.c, self.cache
|
||||
cache.pp_rank = 0
|
||||
c.page_get_func = Mock(return_value=1)
|
||||
for hit in (0, 8):
|
||||
with self.subTest(hit=hit):
|
||||
rid = str(hit)
|
||||
handle = CacheRequestHandle(rid, 0)
|
||||
c._storage_hit_query.return_value = ([], hit)
|
||||
self.assertEqual(self.submit(rid).decision, bool(hit))
|
||||
c._storage_hit_query.side_effect = lambda operation: self.query_hits(
|
||||
operation, hit
|
||||
)
|
||||
self.assertTrue(self.submit(rid).decision)
|
||||
queries = c._storage_hit_query.call_count
|
||||
for _ in range(2):
|
||||
self.assertEqual(
|
||||
self.cache.prefetch_from_storage(handle, 0, []), bool(hit)
|
||||
)
|
||||
self.assertTrue(cache.prefetch_from_storage(handle, 0, []))
|
||||
self.assertEqual(c._storage_hit_query.call_count, queries)
|
||||
if hit:
|
||||
state = c.pp_prefetch_states[rid]
|
||||
state.ready_event.set()
|
||||
c.take_ready_pp_prefetch(rid)
|
||||
self.assertFalse(
|
||||
self.cache.prefetch_from_storage(
|
||||
CacheRequestHandle(rid, 1), 0, []
|
||||
)
|
||||
)
|
||||
self.assertTrue(self.cache.check_prefetch_progress(handle))
|
||||
self.run_commands()
|
||||
self.assertEqual(c._storage_hit_query.call_count, queries + 1)
|
||||
self.sync_acks(*self.run_io(1))
|
||||
self.assertTrue(cache.check_prefetch_progress(handle))
|
||||
self.assertFalse(
|
||||
cache.prefetch_from_storage(CacheRequestHandle(rid, 1), 0, [])
|
||||
)
|
||||
self.assertFalse(c.release_pp_prefetch(rid))
|
||||
self.assertIsNone(c.get_prefetch_submission(rid))
|
||||
self.assertEqual(c.pp_prefetch_command_queue.qsize(), 1)
|
||||
self.cache._all_reduce.assert_not_called()
|
||||
self.assertTrue(c.pp_prefetch_command_queue.empty())
|
||||
|
||||
def test_submission_does_not_wait_for_worker_and_uses_only_tp_consensus(self):
|
||||
def test_all_backends_query_and_allocate_before_one_worker_consensus(self):
|
||||
c = self.c
|
||||
with patch.object(
|
||||
c.pp_prefetch_command_queue,
|
||||
"join",
|
||||
side_effect=AssertionError("scheduler blocked"),
|
||||
for backend in ("file", "mooncake", "dynamic"):
|
||||
with self.subTest(backend=backend):
|
||||
c.storage_backend_type = backend
|
||||
c._storage_hit_query.reset_mock()
|
||||
c._all_reduce.reset_mock()
|
||||
c.mem_pool_host.alloc.reset_mock()
|
||||
c._storage_hit_query.side_effect = AssertionError(
|
||||
"scheduler queried storage"
|
||||
)
|
||||
c._all_reduce.side_effect = AssertionError(
|
||||
"scheduler entered a collective"
|
||||
)
|
||||
with patch.object(
|
||||
c.pp_prefetch_command_queue,
|
||||
"join",
|
||||
side_effect=AssertionError("scheduler blocked"),
|
||||
):
|
||||
first = self.submit(backend, assume_stored=True)
|
||||
second = self.submit(f"next_{backend}")
|
||||
self.assertTrue(first.decision)
|
||||
self.assertTrue(second.decision)
|
||||
self.assertEqual(c.pp_prefetch_command_queue.qsize(), 2)
|
||||
self.assertFalse(c.is_pp_prefetch_ready(backend))
|
||||
self.assertFalse(first.operation.assume_stored)
|
||||
c.mem_pool_host.alloc.assert_not_called()
|
||||
c._storage_hit_query.assert_not_called()
|
||||
c._all_reduce.assert_not_called()
|
||||
|
||||
votes = []
|
||||
|
||||
def reduce_hits(tensor, op, groups):
|
||||
self.assertEqual(op, torch.distributed.ReduceOp.MIN)
|
||||
self.assertEqual(groups, c.prefetch_hits_sync_groups)
|
||||
self.assertEqual(tensor.numel(), 1 + len(PoolName))
|
||||
self.assertEqual(tensor[0].item(), 8)
|
||||
self.assertEqual(c.mem_pool_host.alloc.call_count, len(votes) + 1)
|
||||
votes.append(tensor[0].item())
|
||||
|
||||
c._storage_hit_query.side_effect = self.query_hits
|
||||
c._all_reduce.side_effect = reduce_hits
|
||||
self.run_commands()
|
||||
self.assertEqual(
|
||||
c._storage_hit_query.call_args_list,
|
||||
[call(first.operation), call(second.operation)],
|
||||
)
|
||||
self.assertEqual(votes, [8, 8])
|
||||
self.assertEqual(c._all_reduce.call_count, 2)
|
||||
self.assertIs(c.prefetch_buffer.get_nowait(), first.operation)
|
||||
self.assertIs(c.prefetch_buffer.get_nowait(), second.operation)
|
||||
|
||||
def test_file_attachment_enables_pp_tickets(self):
|
||||
c = self.c
|
||||
c.pp_prefetch_command_group = None
|
||||
c.storage_host_pool = Mock()
|
||||
c.host_memory_mode = "buffer_only"
|
||||
c.storage_backend_type = "file"
|
||||
with (
|
||||
patch.object(HiCacheController, "attach_storage_backend"),
|
||||
patch.object(torch.distributed, "get_world_size", return_value=4),
|
||||
patch(
|
||||
"sglang.srt.distributed.parallel_state.create_custom_parallel_group",
|
||||
return_value="command",
|
||||
),
|
||||
):
|
||||
self.assertTrue(self.submit(assume_stored=True).decision)
|
||||
self.assertTrue(self.submit("next").decision)
|
||||
self.assertEqual(c.pp_prefetch_command_queue.qsize(), 2)
|
||||
self.assertFalse(c.is_pp_prefetch_ready("hit"))
|
||||
c.mem_pool_host.alloc.assert_not_called()
|
||||
self.assertFalse(c.pp_prefetch_states["hit"].operation.assume_stored)
|
||||
self.assertEqual(
|
||||
c._all_reduce.call_args.args[1:], (torch.distributed.ReduceOp.MIN, ["tp"])
|
||||
c.attach_storage_backend("file", prefetch_threshold=4)
|
||||
self.assertEqual(c.pp_prefetch_command_group, "command")
|
||||
|
||||
def test_file_stage_hits_trim_buffers_after_local_allocation(self):
|
||||
"""PP0's local files cannot reveal a shorter prefix on another stage."""
|
||||
c = self.c
|
||||
key = RadixKey(
|
||||
array("q", range(9)), "adapter", is_bigram=True, cache_salt="tenant"
|
||||
)
|
||||
self.assertEqual(
|
||||
[call.kwargs["pp_rank"] for call in c._storage_hit_query.call_args_list],
|
||||
[0, 2, 0, 2],
|
||||
hashes = get_storage_hash_str(key, "ab" * 32, page_size=4)
|
||||
local_votes = []
|
||||
with (
|
||||
tempfile.TemporaryDirectory() as directory,
|
||||
patch.object(
|
||||
envs.SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR, "get", return_value=None
|
||||
),
|
||||
):
|
||||
backends = []
|
||||
for rank, pages in ((0, 2), (1, 1)):
|
||||
config = replace(
|
||||
c.storage_config,
|
||||
pp_rank=rank,
|
||||
pp_size=2,
|
||||
extra_config={
|
||||
"file_storage_path": str(Path(directory) / str(rank)),
|
||||
"enable_metadata_cache": False,
|
||||
"max_size": 0,
|
||||
"min_free_space": 0,
|
||||
},
|
||||
)
|
||||
backend = HiCacheFile(config)
|
||||
for hash_value in hashes[:pages]:
|
||||
self.assertTrue(backend.set(hash_value, torch.ones(1)))
|
||||
backends.append(backend)
|
||||
|
||||
self.assertEqual(backends[0].batch_exists(hashes), 2)
|
||||
self.assertEqual(backends[1].batch_exists(hashes), 1)
|
||||
c.storage_backend = backends[0]
|
||||
c.storage_backend_type = "file"
|
||||
c._storage_hit_query = Mock(
|
||||
wraps=HybridCacheController._storage_hit_query.__get__(c)
|
||||
)
|
||||
result = self.submit(attempt_id=3)
|
||||
ticket = pickle.loads(pickle.dumps(c.pp_prefetch_states["hit"].ticket))
|
||||
|
||||
for rank, peer_hit in ((0, 4), (1, 8)):
|
||||
with self.subTest(rank=rank):
|
||||
c.storage_config.pp_rank = rank
|
||||
c.storage_backend = backends[rank]
|
||||
if rank:
|
||||
c.pp_prefetch_states.clear()
|
||||
c._storage_hit_query.reset_mock()
|
||||
c.mem_pool_host.alloc.reset_mock()
|
||||
c.mem_pool_host.free.reset_mock()
|
||||
c.prefetch_tokens_occupied = 0
|
||||
local_hit = 8 if rank == 0 else 4
|
||||
|
||||
def reduce_hits(tensor, op, groups):
|
||||
self.assertEqual(op, torch.distributed.ReduceOp.MIN)
|
||||
self.assertEqual(groups, ["pp", "tp"])
|
||||
self.assertEqual(tensor.numel(), 1 + len(PoolName))
|
||||
local_votes.append(tensor[0].item())
|
||||
c.mem_pool_host.alloc.assert_called_once_with(
|
||||
local_hit, pool=PoolName.KV
|
||||
)
|
||||
tensor[0] = min(tensor[0].item(), peer_hit)
|
||||
kv_index = 1 + list(PoolName).index(PoolName.KV)
|
||||
tensor[kv_index] = min(tensor[kv_index].item(), peer_hit // 4)
|
||||
|
||||
c._all_reduce.side_effect = reduce_hits
|
||||
self.run_commands(ticket)
|
||||
operation = c.prefetch_buffer.get_nowait()
|
||||
self.assertEqual(operation.handle, CacheRequestHandle("hit", 3))
|
||||
self.assertEqual(operation.storage_hit_count, 4)
|
||||
self.assertEqual(operation.hash_value, hashes[:1])
|
||||
self.assertEqual(operation.host_indices.numel(), 4)
|
||||
self.assertFalse(operation.is_terminated())
|
||||
self.assertEqual(c.prefetch_tokens_occupied, 4)
|
||||
if rank == 0:
|
||||
self.assertIs(operation, result.operation)
|
||||
freed = c.mem_pool_host.free.call_args
|
||||
self.assertEqual(freed.kwargs, {"pool": PoolName.KV})
|
||||
self.assertTrue(torch.equal(freed.args[0], torch.arange(4, 8)))
|
||||
self.assertEqual(c.mem_pool_host.free.call_count, 1)
|
||||
else:
|
||||
c.mem_pool_host.free.assert_not_called()
|
||||
c._storage_hit_query.assert_called_once_with(operation)
|
||||
self.assertEqual(operation.query_pool_hit_pages, {PoolName.KV: 1})
|
||||
|
||||
self.assertEqual(local_votes, [8, 4])
|
||||
|
||||
def test_file_stage_hit_fits_capacity_before_consensus(self):
|
||||
"""A large local hit must not discard a prefix that fits on every stage."""
|
||||
c = self.c
|
||||
c.page_size = c.prefetch_threshold = 256
|
||||
key = RadixKey(array("q", range(8193)), is_bigram=True)
|
||||
hashes = get_storage_hash_str(key, page_size=256)
|
||||
c.storage_backend_type = "file"
|
||||
c._storage_hit_query = Mock(
|
||||
wraps=HybridCacheController._storage_hit_query.__get__(c)
|
||||
)
|
||||
with (
|
||||
tempfile.TemporaryDirectory() as directory,
|
||||
patch.object(
|
||||
envs.SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR, "get", return_value=None
|
||||
),
|
||||
):
|
||||
backends = []
|
||||
for rank, pages in ((0, 32), (1, 8)):
|
||||
config = replace(
|
||||
c.storage_config,
|
||||
pp_rank=rank,
|
||||
pp_size=2,
|
||||
extra_config={
|
||||
"file_storage_path": str(Path(directory) / str(rank)),
|
||||
"enable_metadata_cache": False,
|
||||
"max_size": 0,
|
||||
"min_free_space": 0,
|
||||
},
|
||||
)
|
||||
backend = HiCacheFile(config)
|
||||
for hash_value in hashes[:pages]:
|
||||
self.assertTrue(backend.set(hash_value, torch.ones(1)))
|
||||
backends.append(backend)
|
||||
|
||||
for capacity in (4096, 4097):
|
||||
c.storage_config.pp_rank = 0
|
||||
c.pp_prefetch_states.clear()
|
||||
c.pp_prefetch_command_queue = Queue()
|
||||
result = c.submit_prefetch(
|
||||
CacheRequestHandle("capacity", 0), key, None, None, [], None
|
||||
)
|
||||
ticket = pickle.loads(
|
||||
pickle.dumps(c.pp_prefetch_states["capacity"].ticket)
|
||||
)
|
||||
for rank, local_vote in ((0, 4096), (1, 2048)):
|
||||
with self.subTest(capacity=capacity, rank=rank):
|
||||
c.storage_config.pp_rank = rank
|
||||
c.storage_backend = backends[rank]
|
||||
if rank:
|
||||
c.pp_prefetch_states.clear()
|
||||
host_pool = LogicalHostPool(4096, page_size=256)
|
||||
c.mem_pool_host.available_size.return_value = capacity
|
||||
c.mem_pool_host.available_size.reset_mock()
|
||||
c.mem_pool_host.alloc.reset_mock()
|
||||
c.mem_pool_host.free.reset_mock()
|
||||
c.mem_pool_host.alloc.side_effect = lambda size, **_: (
|
||||
host_pool.alloc(size)
|
||||
)
|
||||
c.mem_pool_host.free.side_effect = lambda indices, **_: (
|
||||
host_pool.free(indices)
|
||||
)
|
||||
c.prefetch_tokens_occupied = 0
|
||||
c._storage_hit_query.reset_mock()
|
||||
c._all_reduce.reset_mock()
|
||||
local_votes = []
|
||||
|
||||
def reduce_hits(tensor, op, groups):
|
||||
self.assertEqual(op, torch.distributed.ReduceOp.MIN)
|
||||
self.assertEqual(groups, c.prefetch_hits_sync_groups)
|
||||
self.assertEqual(tensor.numel(), 1 + len(PoolName))
|
||||
local_votes.append(tensor[0].item())
|
||||
kv_index = 1 + list(PoolName).index(PoolName.KV)
|
||||
self.assertEqual(
|
||||
tensor[kv_index].item(), 32 if rank == 0 else 8
|
||||
)
|
||||
peer_vote = 2048 if rank == 0 else 4096
|
||||
tensor[0] = min(tensor[0].item(), peer_vote)
|
||||
tensor[kv_index] = min(tensor[kv_index].item(), 8)
|
||||
|
||||
c._all_reduce.side_effect = reduce_hits
|
||||
self.run_commands(ticket)
|
||||
operation = c.prefetch_buffer.get_nowait()
|
||||
if rank == 0:
|
||||
self.assertIs(operation, result.operation)
|
||||
self.assertEqual(operation.storage_hit_count, 2048)
|
||||
self.assertEqual(local_votes, [local_vote])
|
||||
c.mem_pool_host.alloc.assert_called_once_with(
|
||||
local_vote, pool=PoolName.KV
|
||||
)
|
||||
self.assertEqual(operation.hash_value, hashes[:8])
|
||||
self.assertEqual(operation.host_indices.numel(), 2048)
|
||||
self.assertFalse(operation.is_terminated())
|
||||
self.assertEqual(
|
||||
operation.query_pool_hit_pages, {PoolName.KV: 8}
|
||||
)
|
||||
self.assertEqual(c.prefetch_tokens_occupied, 2048)
|
||||
self.assertEqual(host_pool.available_size(), 2048)
|
||||
c._storage_hit_query.assert_called_once_with(operation)
|
||||
c.mem_pool_host.available_size.assert_called_once_with(
|
||||
pool=PoolName.KV
|
||||
)
|
||||
c._all_reduce.assert_called_once()
|
||||
if rank == 0:
|
||||
self.assertEqual(c.mem_pool_host.free.call_count, 1)
|
||||
freed = c.mem_pool_host.free.call_args
|
||||
self.assertEqual(freed.kwargs, {"pool": PoolName.KV})
|
||||
self.assertTrue(
|
||||
torch.equal(freed.args[0], torch.arange(2048, 4096))
|
||||
)
|
||||
else:
|
||||
c.mem_pool_host.free.assert_not_called()
|
||||
|
||||
def test_local_miss_or_query_error_completes_before_next_ticket(self):
|
||||
"""A stage miss/error must complete admission without breaking ACK order."""
|
||||
c, cache = self.c, self.cache
|
||||
c.storage_backend_type = "file"
|
||||
cache.pp_rank = 0
|
||||
for name, local_hit, peer_hit, threshold in (
|
||||
("local_miss", 0, 8, 4),
|
||||
("peer_miss", 8, 0, 4),
|
||||
("below_threshold", 7, 8, 6),
|
||||
("query_error", RuntimeError("local storage unavailable"), 8, 4),
|
||||
):
|
||||
with self.subTest(name=name):
|
||||
failed_rid, next_rid = name, f"next_{name}"
|
||||
c.prefetch_threshold = threshold
|
||||
c._storage_hit_query.reset_mock()
|
||||
c._all_reduce.reset_mock()
|
||||
c.mem_pool_host.alloc.reset_mock()
|
||||
c.mem_pool_host.free.reset_mock()
|
||||
cache._handle_prefetch_result.reset_mock()
|
||||
occupied_before = c.prefetch_tokens_occupied
|
||||
hits = iter((local_hit, 8))
|
||||
|
||||
def query_hits(operation):
|
||||
hit = next(hits)
|
||||
if isinstance(hit, Exception):
|
||||
operation.query_pool_hit_pages = {PoolName.KV: 2}
|
||||
raise hit
|
||||
return self.query_hits(operation, hit)
|
||||
|
||||
c._storage_hit_query.side_effect = query_hits
|
||||
self.assertTrue(self.submit(failed_rid, attempt_id=3).decision)
|
||||
self.assertTrue(self.submit(next_rid).decision)
|
||||
peers = iter((peer_hit, 8))
|
||||
local_votes = []
|
||||
|
||||
def reduce_hits(tensor, op, groups):
|
||||
self.assertEqual(op, torch.distributed.ReduceOp.MIN)
|
||||
self.assertEqual(groups, ["pp", "tp"])
|
||||
self.assertEqual(tensor.numel(), 1 + len(PoolName))
|
||||
local_votes.append(tensor[0].item())
|
||||
if tensor[0].item():
|
||||
self.assertEqual(c.mem_pool_host.alloc.call_args.args, (8,))
|
||||
tensor[0] = min(tensor[0].item(), next(peers))
|
||||
|
||||
c._all_reduce.side_effect = reduce_hits
|
||||
logs = (
|
||||
self.assertLogs(level="ERROR")
|
||||
if isinstance(local_hit, Exception)
|
||||
else nullcontext()
|
||||
)
|
||||
with logs:
|
||||
self.run_commands()
|
||||
failed = c.pp_prefetch_states[failed_rid].operation
|
||||
following = c.pp_prefetch_states[next_rid].operation
|
||||
self.assertEqual(failed.storage_hit_count, 0)
|
||||
self.assertEqual(failed.hash_value, [])
|
||||
self.assertEqual(failed.host_indices.numel(), 0)
|
||||
self.assertTrue(failed.is_terminated())
|
||||
self.assertEqual(following.host_indices.numel(), 8)
|
||||
self.assertFalse(following.is_terminated())
|
||||
self.assertEqual(
|
||||
local_votes,
|
||||
[8 if name == "peer_miss" else 0, 8],
|
||||
)
|
||||
self.assertEqual(c._all_reduce.call_count, 2)
|
||||
self.assertEqual(
|
||||
c._storage_hit_query.call_args_list,
|
||||
[call(failed), call(following)],
|
||||
)
|
||||
if name == "peer_miss":
|
||||
self.assertEqual(
|
||||
c.mem_pool_host.alloc.call_args_list,
|
||||
[call(8, pool=PoolName.KV)] * 2,
|
||||
)
|
||||
self.assertEqual(c.mem_pool_host.free.call_count, 1)
|
||||
self.assertTrue(
|
||||
torch.equal(
|
||||
c.mem_pool_host.free.call_args.args[0], torch.arange(8)
|
||||
)
|
||||
)
|
||||
self.assertEqual(
|
||||
c.mem_pool_host.free.call_args.kwargs, {"pool": PoolName.KV}
|
||||
)
|
||||
else:
|
||||
c.mem_pool_host.alloc.assert_called_once_with(8, pool=PoolName.KV)
|
||||
c.mem_pool_host.free.assert_not_called()
|
||||
if name == "query_error":
|
||||
self.assertEqual(failed.query_pool_hit_pages, {})
|
||||
self.assertEqual(c.prefetch_tokens_occupied, occupied_before + 8)
|
||||
|
||||
c._all_reduce.side_effect = None
|
||||
c.page_get_func = Mock(return_value=1)
|
||||
acks = self.run_io(2)
|
||||
self.assertEqual(
|
||||
[ack.rid for ack in acks], [failed_rid] + [next_rid] * 3
|
||||
)
|
||||
self.assertTrue(acks[0].completed_req)
|
||||
self.assertEqual(
|
||||
[ack.completed_tokens for ack in acks[1:]], [4, 8, None]
|
||||
)
|
||||
self.assertEqual(c.page_get_func.call_count, 2)
|
||||
self.sync_acks(*acks)
|
||||
handle = CacheRequestHandle(failed_rid, 3)
|
||||
self.assertTrue(c.is_pp_prefetch_ready(failed_rid))
|
||||
self.assertTrue(cache.check_prefetch_progress(handle))
|
||||
self.assertEqual(cache.prefetch_loaded_tokens_by_reqid[handle], 0)
|
||||
self.assertNotIn(handle, cache.ongoing_prefetch)
|
||||
cache._handle_prefetch_result.assert_not_called()
|
||||
self.assertTrue(cache.check_prefetch_progress(handle))
|
||||
self.assertFalse(c.release_pp_prefetch(failed_rid))
|
||||
self.assertTrue(
|
||||
cache.check_prefetch_progress(CacheRequestHandle(next_rid, 0))
|
||||
)
|
||||
cache._handle_prefetch_result.assert_called_once_with(following)
|
||||
self.assertEqual(c._storage_hit_query.call_count, 2)
|
||||
self.assertEqual(c.pp_prefetch_command_queue.unfinished_tasks, 0)
|
||||
|
||||
def test_tp_only_preserves_normal_prefetch(self):
|
||||
c = self.c
|
||||
@@ -209,7 +604,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
||||
)
|
||||
self.assertTrue(operation.token_ids.is_bigram)
|
||||
self.assertEqual(len(operation.token_ids), 8)
|
||||
c._storage_hit_query.assert_not_called()
|
||||
c._storage_hit_query.assert_called_once_with(operation)
|
||||
self.sync_acks(PrefetchAck("hit", operation, completed_tokens=8))
|
||||
self.assertFalse(c.is_pp_prefetch_ready("hit"))
|
||||
self.sync_acks(PrefetchAck("hit", operation, completed_req=True))
|
||||
@@ -244,7 +639,22 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
||||
with self.assertLogs(level="ERROR"):
|
||||
self.run_commands(ticket, following)
|
||||
failed = c.pp_prefetch_states["hit"].operation
|
||||
next_operation = c.pp_prefetch_states["next"].operation
|
||||
self.assertTrue(failed.is_terminated())
|
||||
self.assertEqual(failed.storage_hit_count, 0)
|
||||
self.assertEqual(failed.hash_value, [])
|
||||
self.assertEqual(failed.host_indices.numel(), 0)
|
||||
self.assertEqual(
|
||||
c._storage_hit_query.call_args_list, [call(failed), call(next_operation)]
|
||||
)
|
||||
self.assertEqual(c._all_reduce.call_count, 2)
|
||||
self.assertEqual(
|
||||
[
|
||||
invocation.args[0][0].item()
|
||||
for invocation in c._all_reduce.call_args_list
|
||||
],
|
||||
[0, 8],
|
||||
)
|
||||
c.mem_pool_host.free.assert_called_once_with(kv, pool=PoolName.KV)
|
||||
c.page_get_func = Mock(return_value=1)
|
||||
c.storage_backend = Mock()
|
||||
@@ -256,12 +666,12 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
||||
),
|
||||
):
|
||||
c.prefetch_io_aux_func()
|
||||
acks = [c.prefetch_sync_queue.get_nowait() for _ in range(8)]
|
||||
self.assertEqual([a.rid for a in acks], ["hit"] * 4 + ["next"] * 4)
|
||||
acks = [c.prefetch_sync_queue.get_nowait() for _ in range(6)]
|
||||
self.assertEqual([a.rid for a in acks], ["hit"] * 2 + ["next"] * 4)
|
||||
self.assertEqual(
|
||||
[a.completed_tokens for a in acks], [0, 0, None, None, 4, 8, None, None]
|
||||
[a.completed_tokens for a in acks], [None, None, 4, 8, None, None]
|
||||
)
|
||||
self.assertEqual((acks[2].pool_hits, acks[6].pool_hits), ({}, {"swa": 1}))
|
||||
self.assertEqual((acks[0].pool_hits, acks[4].pool_hits), ({}, {"swa": 1}))
|
||||
self.assertEqual(
|
||||
c.page_get_func.call_count, 2
|
||||
) # No reads for the failed ticket.
|
||||
@@ -341,6 +751,16 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
||||
c.mem_pool_host.alloc.side_effect = lambda *args, **kwargs: None
|
||||
self.run_commands()
|
||||
self.assertTrue(operation.is_terminated())
|
||||
self.assertEqual(operation.hash_value, [])
|
||||
self.assertEqual(operation.storage_hit_count, 0)
|
||||
c._storage_hit_query.assert_called_once_with(operation)
|
||||
c.mem_pool_host.alloc.assert_called_once_with(8, pool=PoolName.KV)
|
||||
c._all_reduce.assert_called_once()
|
||||
packed, op, groups = c._all_reduce.call_args.args
|
||||
self.assertEqual(packed[0].item(), 0)
|
||||
self.assertEqual(packed.numel(), 1 + len(PoolName))
|
||||
self.assertEqual(op, torch.distributed.ReduceOp.MIN)
|
||||
self.assertEqual(groups, c.prefetch_hits_sync_groups)
|
||||
c.mem_pool_host.free.assert_not_called()
|
||||
self.sync_acks(
|
||||
PrefetchAck("hit", operation, completed_tokens=0, completed_req=True)
|
||||
@@ -349,6 +769,69 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
||||
self.assertIsNone(borrowed.host_indices)
|
||||
self.assertEqual(c.mem_pool_host.free.call_args.kwargs["pool"], PoolName.SWA)
|
||||
|
||||
def test_peer_vote_trims_kv_alias_and_defers_sidecar_free(self):
|
||||
"""A peer's zero allocation vote must release KV without freeing a sidecar."""
|
||||
c = self.c
|
||||
for peer_vote in (4, 0):
|
||||
with self.subTest(peer_vote=peer_vote):
|
||||
rid = f"peer_{peer_vote}"
|
||||
c._storage_hit_query.reset_mock()
|
||||
c._all_reduce.reset_mock()
|
||||
c.mem_pool_host.alloc.reset_mock()
|
||||
c.mem_pool_host.free.reset_mock()
|
||||
borrowed_indices = torch.arange(4)
|
||||
operation = self.submit(
|
||||
rid,
|
||||
pools=[
|
||||
PoolTransfer(PoolName.SWA, host_indices=borrowed_indices),
|
||||
PoolTransfer(PoolName.INDEXER, indices_from_pool=PoolName.KV),
|
||||
],
|
||||
).operation
|
||||
|
||||
def reduce_hits(tensor, op, groups):
|
||||
self.assertEqual(tensor[0].item(), 8)
|
||||
self.assertEqual(tensor.numel(), 1 + len(PoolName))
|
||||
self.assertEqual(op, torch.distributed.ReduceOp.MIN)
|
||||
self.assertEqual(groups, c.prefetch_hits_sync_groups)
|
||||
c.mem_pool_host.alloc.assert_called_once_with(8, pool=PoolName.KV)
|
||||
tensor[0] = peer_vote
|
||||
|
||||
c._all_reduce.side_effect = reduce_hits
|
||||
self.run_commands()
|
||||
c._storage_hit_query.assert_called_once_with(operation)
|
||||
c._all_reduce.assert_called_once()
|
||||
swa, indexer = operation.pool_transfers
|
||||
self.assertEqual(operation.host_indices.numel(), peer_vote)
|
||||
self.assertEqual(operation.storage_hit_count, peer_vote)
|
||||
self.assertEqual(operation.is_terminated(), peer_vote == 0)
|
||||
self.assertIs(indexer.host_indices, operation.host_indices)
|
||||
self.assertIs(indexer.keys, operation.hash_value)
|
||||
self.assertIs(swa.host_indices, borrowed_indices)
|
||||
self.assertEqual(c.prefetch_tokens_occupied, peer_vote)
|
||||
freed = c.mem_pool_host.free.call_args
|
||||
self.assertEqual(c.mem_pool_host.free.call_count, 1)
|
||||
self.assertEqual(freed.kwargs, {"pool": PoolName.KV})
|
||||
self.assertTrue(torch.equal(freed.args[0], torch.arange(peer_vote, 8)))
|
||||
|
||||
self.assertTrue(c.release_pp_prefetch(rid))
|
||||
self.assertIs(swa.host_indices, borrowed_indices)
|
||||
self.assertEqual(c.mem_pool_host.free.call_count, 1)
|
||||
c._all_reduce.side_effect = None
|
||||
self.sync_acks(
|
||||
PrefetchAck(rid, operation, completed_tokens=peer_vote),
|
||||
PrefetchAck(rid, operation, completed_req=True),
|
||||
)
|
||||
self.assertIsNone(swa.host_indices)
|
||||
self.assertNotIn(rid, c.pp_prefetch_states)
|
||||
self.assertEqual(c.prefetch_tokens_occupied, 0)
|
||||
sidecar_frees = [
|
||||
invocation
|
||||
for invocation in c.mem_pool_host.free.call_args_list
|
||||
if invocation.kwargs["pool"] == PoolName.SWA
|
||||
]
|
||||
self.assertEqual(len(sidecar_frees), 1)
|
||||
self.assertIs(sidecar_frees[0].args[0], borrowed_indices)
|
||||
|
||||
def test_command_failure_finishes_queue_task_and_is_reported_on_poll(self):
|
||||
c = self.c
|
||||
with (
|
||||
|
||||
Reference in New Issue
Block a user