917 lines
42 KiB
Python
917 lines
42 KiB
Python
"""CPU regressions for PP ticket ordering, admission, and buffer ownership."""
|
|
|
|
import pickle
|
|
import tempfile
|
|
import threading
|
|
import unittest
|
|
from array import array
|
|
from contextlib import nullcontext
|
|
from dataclasses import replace
|
|
from pathlib import Path
|
|
from queue import Empty, Queue
|
|
from unittest.mock import Mock, call, patch
|
|
|
|
import torch
|
|
|
|
from sglang.srt.environ import envs
|
|
from sglang.srt.managers.cache_controller import HiCacheController, PrefetchAck
|
|
from sglang.srt.mem_cache.base_prefix_cache import (
|
|
CacheRequestHandle,
|
|
CacheRequestOutcome,
|
|
)
|
|
from sglang.srt.mem_cache.buffer_mode.pipeline import BufferModePipeline
|
|
from sglang.srt.mem_cache.hicache_storage import (
|
|
HiCacheFile,
|
|
HiCacheStorageConfig,
|
|
PoolName,
|
|
PoolTransfer,
|
|
)
|
|
from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import (
|
|
HybridCacheController,
|
|
PPPrefetchDecision,
|
|
)
|
|
from sglang.srt.mem_cache.memory_pool_host import LogicalHostPool
|
|
from sglang.srt.mem_cache.radix_cache import RadixKey
|
|
from sglang.srt.mem_cache.storage_prefetch import StoragePrefetchRetries
|
|
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
|
|
from sglang.srt.mem_cache.utils import get_storage_hash_str
|
|
from sglang.test.ci.ci_register import register_cpu_ci
|
|
|
|
register_cpu_ci(est_time=3, suite="base-a-test-cpu")
|
|
|
|
|
|
class TestPPPrefetchTicket(unittest.TestCase):
|
|
def setUp(self):
|
|
self.c = c = HybridCacheController.__new__(HybridCacheController)
|
|
c.page_size = c.prefetch_threshold = 4
|
|
c.storage_config = HiCacheStorageConfig(
|
|
tp_rank=0,
|
|
tp_size=2,
|
|
pp_rank=0,
|
|
pp_size=4,
|
|
attn_cp_rank=0,
|
|
attn_cp_size=1,
|
|
is_mla_model=False,
|
|
enable_storage_metrics=False,
|
|
is_page_first_layout=False,
|
|
model_name=None,
|
|
)
|
|
c.pp_group, c.pp_prefetch_command_group = "pp", "command"
|
|
c.pp_prefetch_command_thread = None
|
|
c.prefetch_hits_sync_groups = c.prefetch_completion_sync_groups = ["pp", "tp"]
|
|
c.pp_prefetch_states, c.pp_prefetch_decisions = {}, {}
|
|
c.pp_prefetch_state_lock = threading.Lock()
|
|
c.pp_prefetch_command_queue = Queue()
|
|
c.prefetch_queue = Queue()
|
|
c.prefetch_sync_queue = Queue()
|
|
c.ack_prefetch_queue = Queue()
|
|
c.host_mem_release_queue = Queue()
|
|
c.prefetch_buffer = Queue()
|
|
c.storage_stop_event = threading.Event()
|
|
c.prefetch_tokens_occupied = 0
|
|
c.mem_pool_host = Mock(page_size=4)
|
|
c.mem_pool_host.available_size.return_value = 1 << 20
|
|
c.mem_pool_host.layout_lease.side_effect = nullcontext
|
|
c.mem_pool_host.alloc.side_effect = lambda size, **_: torch.arange(size)
|
|
c.storage_backend_type = "mooncake"
|
|
c.storage_backend = Mock()
|
|
c._storage_hit_query = Mock(side_effect=self.query_hits)
|
|
c._all_reduce = Mock()
|
|
ranks = {"pp": [0, 2, 4, 6], "tp": [0, 1], "command": [0, 2, 4, 6]}
|
|
patcher = patch.object(
|
|
torch.distributed, "get_process_group_ranks", side_effect=ranks.__getitem__
|
|
)
|
|
patcher.start()
|
|
self.addCleanup(patcher.stop)
|
|
patcher = patch.object(
|
|
torch.distributed,
|
|
"get_rank",
|
|
side_effect=lambda: c.storage_config.pp_rank * 2 + c.storage_config.tp_rank,
|
|
)
|
|
patcher.start()
|
|
self.addCleanup(patcher.stop)
|
|
self.cache = cache = UnifiedRadixCache.__new__(UnifiedRadixCache)
|
|
cache.pp_rank = 1
|
|
cache.tree_core = Mock(enable_storage=True)
|
|
cache.cache_controller = c
|
|
cache._all_reduce = Mock()
|
|
cache.ongoing_prefetch = {}
|
|
cache.prefetch_loaded_tokens_by_reqid = {}
|
|
cache.prefetch_loaded_storage_start_by_reqid = {}
|
|
cache.storage_prefetch_retries = StoragePrefetchRetries()
|
|
cache._storage_prefetch_wait = {}
|
|
cache.linker = None
|
|
cache.root_node_handle = Mock(return_value=0)
|
|
cache.buffer_pipeline = Mock(spec=BufferModePipeline)
|
|
cache._handle_prefetch_result = Mock()
|
|
|
|
def query_hits(self, operation, hit=8):
|
|
hashes = get_storage_hash_str(
|
|
operation.token_ids, operation.last_hash, page_size=self.c.page_size
|
|
)
|
|
operation.all_hash_values = hashes
|
|
operation.query_pool_hit_pages = {PoolName.KV: hit // self.c.page_size}
|
|
return hashes[: hit // self.c.page_size], hit
|
|
|
|
def submit(self, rid="hit", pools=None, attempt_id=0, assume_stored=False):
|
|
key = RadixKey(
|
|
array("q", range(9)), "adapter", is_bigram=True, cache_salt="tenant"
|
|
)
|
|
return self.c.submit_prefetch(
|
|
CacheRequestHandle(rid, attempt_id),
|
|
key,
|
|
"ab" * 32,
|
|
["prefix"],
|
|
[10, 11, 12, 13],
|
|
pools,
|
|
assume_stored=assume_stored,
|
|
)
|
|
|
|
def run_commands(self, *tickets):
|
|
commands = iter((*tickets, None))
|
|
self.c.pp_prefetch_command_queue.put(None)
|
|
|
|
def broadcast(objects, *args, **kwargs):
|
|
return [next(commands)] if self.c.storage_config.pp_rank else objects
|
|
|
|
with patch(f"{HybridCacheController.__module__}.broadcast_pyobj", broadcast):
|
|
self.c.pp_prefetch_command_thread_func()
|
|
|
|
def sync_acks(self, *acks):
|
|
for ack in acks:
|
|
self.c.prefetch_sync_queue.put(ack)
|
|
with patch.object(
|
|
self.c.storage_stop_event,
|
|
"is_set",
|
|
side_effect=[False] * len(acks) + [True],
|
|
):
|
|
self.c.prefetch_sync_thread_func()
|
|
|
|
def run_io(self, operations):
|
|
with (
|
|
patch("sglang.srt.managers.cache_controller.STORAGE_BATCH_SIZE", 1),
|
|
patch.object(
|
|
self.c.storage_stop_event,
|
|
"is_set",
|
|
side_effect=[False] * operations + [True],
|
|
),
|
|
):
|
|
self.c.prefetch_io_aux_func()
|
|
acks = []
|
|
while not self.c.prefetch_sync_queue.empty():
|
|
acks.append(self.c.prefetch_sync_queue.get_nowait())
|
|
return acks
|
|
|
|
def test_hit_and_miss_are_reused_across_enqueue_retract_and_retry(self):
|
|
c, cache = self.c, self.cache
|
|
cache.pp_rank = 0
|
|
c.page_get_func = Mock(return_value=1)
|
|
for hit in (0, 8):
|
|
with self.subTest(hit=hit):
|
|
rid = str(hit)
|
|
handle = CacheRequestHandle(rid, 0)
|
|
c._storage_hit_query.side_effect = lambda operation: self.query_hits(
|
|
operation, hit
|
|
)
|
|
self.assertTrue(self.submit(rid).decision)
|
|
queries = c._storage_hit_query.call_count
|
|
for _ in range(2):
|
|
self.assertTrue(cache.prefetch_from_storage(handle, 0, []))
|
|
self.assertEqual(c._storage_hit_query.call_count, queries)
|
|
self.run_commands()
|
|
self.assertEqual(c._storage_hit_query.call_count, queries + 1)
|
|
self.sync_acks(*self.run_io(1))
|
|
self.assertTrue(cache.check_prefetch_progress(handle))
|
|
self.assertFalse(
|
|
cache.prefetch_from_storage(CacheRequestHandle(rid, 1), 0, [])
|
|
)
|
|
self.assertFalse(c.release_pp_prefetch(rid))
|
|
self.assertIsNone(c.get_prefetch_submission(rid))
|
|
self.assertTrue(c.pp_prefetch_command_queue.empty())
|
|
|
|
def test_all_backends_query_and_allocate_before_one_worker_consensus(self):
|
|
c = self.c
|
|
for backend in ("file", "mooncake", "dynamic"):
|
|
with self.subTest(backend=backend):
|
|
c.storage_backend_type = backend
|
|
c._storage_hit_query.reset_mock()
|
|
c._all_reduce.reset_mock()
|
|
c.mem_pool_host.alloc.reset_mock()
|
|
c._storage_hit_query.side_effect = AssertionError(
|
|
"scheduler queried storage"
|
|
)
|
|
c._all_reduce.side_effect = AssertionError(
|
|
"scheduler entered a collective"
|
|
)
|
|
with patch.object(
|
|
c.pp_prefetch_command_queue,
|
|
"join",
|
|
side_effect=AssertionError("scheduler blocked"),
|
|
):
|
|
first = self.submit(backend, assume_stored=True)
|
|
second = self.submit(f"next_{backend}")
|
|
self.assertTrue(first.decision)
|
|
self.assertTrue(second.decision)
|
|
self.assertEqual(c.pp_prefetch_command_queue.qsize(), 2)
|
|
self.assertFalse(c.is_pp_prefetch_ready(backend))
|
|
self.assertFalse(first.operation.assume_stored)
|
|
c.mem_pool_host.alloc.assert_not_called()
|
|
c._storage_hit_query.assert_not_called()
|
|
c._all_reduce.assert_not_called()
|
|
|
|
votes = []
|
|
|
|
def reduce_hits(tensor, op, groups):
|
|
self.assertEqual(op, torch.distributed.ReduceOp.MIN)
|
|
self.assertEqual(groups, c.prefetch_hits_sync_groups)
|
|
self.assertEqual(tensor.numel(), 1 + len(PoolName))
|
|
self.assertEqual(tensor[0].item(), 8)
|
|
self.assertEqual(c.mem_pool_host.alloc.call_count, len(votes) + 1)
|
|
votes.append(tensor[0].item())
|
|
|
|
c._storage_hit_query.side_effect = self.query_hits
|
|
c._all_reduce.side_effect = reduce_hits
|
|
self.run_commands()
|
|
self.assertEqual(
|
|
c._storage_hit_query.call_args_list,
|
|
[call(first.operation), call(second.operation)],
|
|
)
|
|
self.assertEqual(votes, [8, 8])
|
|
self.assertEqual(c._all_reduce.call_count, 2)
|
|
self.assertIs(c.prefetch_buffer.get_nowait(), first.operation)
|
|
self.assertIs(c.prefetch_buffer.get_nowait(), second.operation)
|
|
|
|
def test_file_attachment_enables_pp_tickets(self):
|
|
c = self.c
|
|
c.pp_prefetch_command_group = None
|
|
c.storage_host_pool = Mock()
|
|
c.host_memory_mode = "buffer_only"
|
|
c.storage_backend_type = "file"
|
|
with (
|
|
patch.object(HiCacheController, "attach_storage_backend"),
|
|
patch.object(torch.distributed, "get_world_size", return_value=4),
|
|
patch(
|
|
"sglang.srt.distributed.parallel_state.create_custom_parallel_group",
|
|
return_value="command",
|
|
),
|
|
):
|
|
c.attach_storage_backend("file", prefetch_threshold=4)
|
|
self.assertEqual(c.pp_prefetch_command_group, "command")
|
|
|
|
def test_file_stage_hits_trim_buffers_after_local_allocation(self):
|
|
"""PP0's local files cannot reveal a shorter prefix on another stage."""
|
|
c = self.c
|
|
key = RadixKey(
|
|
array("q", range(9)), "adapter", is_bigram=True, cache_salt="tenant"
|
|
)
|
|
hashes = get_storage_hash_str(key, "ab" * 32, page_size=4)
|
|
local_votes = []
|
|
with (
|
|
tempfile.TemporaryDirectory() as directory,
|
|
patch.object(
|
|
envs.SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR, "get", return_value=None
|
|
),
|
|
):
|
|
backends = []
|
|
for rank, pages in ((0, 2), (1, 1)):
|
|
config = replace(
|
|
c.storage_config,
|
|
pp_rank=rank,
|
|
pp_size=2,
|
|
extra_config={
|
|
"file_storage_path": str(Path(directory) / str(rank)),
|
|
"enable_metadata_cache": False,
|
|
"max_size": 0,
|
|
"min_free_space": 0,
|
|
},
|
|
)
|
|
backend = HiCacheFile(config)
|
|
for hash_value in hashes[:pages]:
|
|
self.assertTrue(backend.set(hash_value, torch.ones(1)))
|
|
backends.append(backend)
|
|
|
|
self.assertEqual(backends[0].batch_exists(hashes), 2)
|
|
self.assertEqual(backends[1].batch_exists(hashes), 1)
|
|
c.storage_backend = backends[0]
|
|
c.storage_backend_type = "file"
|
|
c._storage_hit_query = Mock(
|
|
wraps=HybridCacheController._storage_hit_query.__get__(c)
|
|
)
|
|
result = self.submit(attempt_id=3)
|
|
ticket = pickle.loads(pickle.dumps(c.pp_prefetch_states["hit"].ticket))
|
|
|
|
for rank, peer_hit in ((0, 4), (1, 8)):
|
|
with self.subTest(rank=rank):
|
|
c.storage_config.pp_rank = rank
|
|
c.storage_backend = backends[rank]
|
|
if rank:
|
|
c.pp_prefetch_states.clear()
|
|
c._storage_hit_query.reset_mock()
|
|
c.mem_pool_host.alloc.reset_mock()
|
|
c.mem_pool_host.free.reset_mock()
|
|
c.prefetch_tokens_occupied = 0
|
|
local_hit = 8 if rank == 0 else 4
|
|
|
|
def reduce_hits(tensor, op, groups):
|
|
self.assertEqual(op, torch.distributed.ReduceOp.MIN)
|
|
self.assertEqual(groups, ["pp", "tp"])
|
|
self.assertEqual(tensor.numel(), 1 + len(PoolName))
|
|
local_votes.append(tensor[0].item())
|
|
c.mem_pool_host.alloc.assert_called_once_with(
|
|
local_hit, pool=PoolName.KV
|
|
)
|
|
tensor[0] = min(tensor[0].item(), peer_hit)
|
|
kv_index = 1 + list(PoolName).index(PoolName.KV)
|
|
tensor[kv_index] = min(tensor[kv_index].item(), peer_hit // 4)
|
|
|
|
c._all_reduce.side_effect = reduce_hits
|
|
self.run_commands(ticket)
|
|
operation = c.prefetch_buffer.get_nowait()
|
|
self.assertEqual(operation.handle, CacheRequestHandle("hit", 3))
|
|
self.assertEqual(operation.storage_hit_count, 4)
|
|
self.assertEqual(operation.hash_value, hashes[:1])
|
|
self.assertEqual(operation.host_indices.numel(), 4)
|
|
self.assertFalse(operation.is_terminated())
|
|
self.assertEqual(c.prefetch_tokens_occupied, 4)
|
|
if rank == 0:
|
|
self.assertIs(operation, result.operation)
|
|
freed = c.mem_pool_host.free.call_args
|
|
self.assertEqual(freed.kwargs, {"pool": PoolName.KV})
|
|
self.assertTrue(torch.equal(freed.args[0], torch.arange(4, 8)))
|
|
self.assertEqual(c.mem_pool_host.free.call_count, 1)
|
|
else:
|
|
c.mem_pool_host.free.assert_not_called()
|
|
c._storage_hit_query.assert_called_once_with(operation)
|
|
self.assertEqual(operation.query_pool_hit_pages, {PoolName.KV: 1})
|
|
|
|
self.assertEqual(local_votes, [8, 4])
|
|
|
|
def test_file_stage_hit_fits_capacity_before_consensus(self):
|
|
"""A large local hit must not discard a prefix that fits on every stage."""
|
|
c = self.c
|
|
c.page_size = c.prefetch_threshold = 256
|
|
key = RadixKey(array("q", range(8193)), is_bigram=True)
|
|
hashes = get_storage_hash_str(key, page_size=256)
|
|
c.storage_backend_type = "file"
|
|
c._storage_hit_query = Mock(
|
|
wraps=HybridCacheController._storage_hit_query.__get__(c)
|
|
)
|
|
with (
|
|
tempfile.TemporaryDirectory() as directory,
|
|
patch.object(
|
|
envs.SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR, "get", return_value=None
|
|
),
|
|
):
|
|
backends = []
|
|
for rank, pages in ((0, 32), (1, 8)):
|
|
config = replace(
|
|
c.storage_config,
|
|
pp_rank=rank,
|
|
pp_size=2,
|
|
extra_config={
|
|
"file_storage_path": str(Path(directory) / str(rank)),
|
|
"enable_metadata_cache": False,
|
|
"max_size": 0,
|
|
"min_free_space": 0,
|
|
},
|
|
)
|
|
backend = HiCacheFile(config)
|
|
for hash_value in hashes[:pages]:
|
|
self.assertTrue(backend.set(hash_value, torch.ones(1)))
|
|
backends.append(backend)
|
|
|
|
for capacity in (4096, 4097):
|
|
c.storage_config.pp_rank = 0
|
|
c.pp_prefetch_states.clear()
|
|
c.pp_prefetch_command_queue = Queue()
|
|
result = c.submit_prefetch(
|
|
CacheRequestHandle("capacity", 0), key, None, None, [], None
|
|
)
|
|
ticket = pickle.loads(
|
|
pickle.dumps(c.pp_prefetch_states["capacity"].ticket)
|
|
)
|
|
for rank, local_vote in ((0, 4096), (1, 2048)):
|
|
with self.subTest(capacity=capacity, rank=rank):
|
|
c.storage_config.pp_rank = rank
|
|
c.storage_backend = backends[rank]
|
|
if rank:
|
|
c.pp_prefetch_states.clear()
|
|
host_pool = LogicalHostPool(4096, page_size=256)
|
|
c.mem_pool_host.available_size.return_value = capacity
|
|
c.mem_pool_host.available_size.reset_mock()
|
|
c.mem_pool_host.alloc.reset_mock()
|
|
c.mem_pool_host.free.reset_mock()
|
|
c.mem_pool_host.alloc.side_effect = lambda size, **_: (
|
|
host_pool.alloc(size)
|
|
)
|
|
c.mem_pool_host.free.side_effect = lambda indices, **_: (
|
|
host_pool.free(indices)
|
|
)
|
|
c.prefetch_tokens_occupied = 0
|
|
c._storage_hit_query.reset_mock()
|
|
c._all_reduce.reset_mock()
|
|
local_votes = []
|
|
|
|
def reduce_hits(tensor, op, groups):
|
|
self.assertEqual(op, torch.distributed.ReduceOp.MIN)
|
|
self.assertEqual(groups, c.prefetch_hits_sync_groups)
|
|
self.assertEqual(tensor.numel(), 1 + len(PoolName))
|
|
local_votes.append(tensor[0].item())
|
|
kv_index = 1 + list(PoolName).index(PoolName.KV)
|
|
self.assertEqual(
|
|
tensor[kv_index].item(), 32 if rank == 0 else 8
|
|
)
|
|
peer_vote = 2048 if rank == 0 else 4096
|
|
tensor[0] = min(tensor[0].item(), peer_vote)
|
|
tensor[kv_index] = min(tensor[kv_index].item(), 8)
|
|
|
|
c._all_reduce.side_effect = reduce_hits
|
|
self.run_commands(ticket)
|
|
operation = c.prefetch_buffer.get_nowait()
|
|
if rank == 0:
|
|
self.assertIs(operation, result.operation)
|
|
self.assertEqual(operation.storage_hit_count, 2048)
|
|
self.assertEqual(local_votes, [local_vote])
|
|
c.mem_pool_host.alloc.assert_called_once_with(
|
|
local_vote, pool=PoolName.KV
|
|
)
|
|
self.assertEqual(operation.hash_value, hashes[:8])
|
|
self.assertEqual(operation.host_indices.numel(), 2048)
|
|
self.assertFalse(operation.is_terminated())
|
|
self.assertEqual(
|
|
operation.query_pool_hit_pages, {PoolName.KV: 8}
|
|
)
|
|
self.assertEqual(c.prefetch_tokens_occupied, 2048)
|
|
self.assertEqual(host_pool.available_size(), 2048)
|
|
c._storage_hit_query.assert_called_once_with(operation)
|
|
c.mem_pool_host.available_size.assert_called_once_with(
|
|
pool=PoolName.KV
|
|
)
|
|
c._all_reduce.assert_called_once()
|
|
if rank == 0:
|
|
self.assertEqual(c.mem_pool_host.free.call_count, 1)
|
|
freed = c.mem_pool_host.free.call_args
|
|
self.assertEqual(freed.kwargs, {"pool": PoolName.KV})
|
|
self.assertTrue(
|
|
torch.equal(freed.args[0], torch.arange(2048, 4096))
|
|
)
|
|
else:
|
|
c.mem_pool_host.free.assert_not_called()
|
|
|
|
def test_local_miss_or_query_error_completes_before_next_ticket(self):
|
|
"""A stage miss/error must complete admission without breaking ACK order."""
|
|
c, cache = self.c, self.cache
|
|
c.storage_backend_type = "file"
|
|
cache.pp_rank = 0
|
|
for name, local_hit, peer_hit, threshold in (
|
|
("local_miss", 0, 8, 4),
|
|
("peer_miss", 8, 0, 4),
|
|
("below_threshold", 7, 8, 6),
|
|
("query_error", RuntimeError("local storage unavailable"), 8, 4),
|
|
):
|
|
with self.subTest(name=name):
|
|
failed_rid, next_rid = name, f"next_{name}"
|
|
c.prefetch_threshold = threshold
|
|
c._storage_hit_query.reset_mock()
|
|
c._all_reduce.reset_mock()
|
|
c.mem_pool_host.alloc.reset_mock()
|
|
c.mem_pool_host.free.reset_mock()
|
|
cache._handle_prefetch_result.reset_mock()
|
|
occupied_before = c.prefetch_tokens_occupied
|
|
hits = iter((local_hit, 8))
|
|
|
|
def query_hits(operation):
|
|
hit = next(hits)
|
|
if isinstance(hit, Exception):
|
|
operation.query_pool_hit_pages = {PoolName.KV: 2}
|
|
raise hit
|
|
return self.query_hits(operation, hit)
|
|
|
|
c._storage_hit_query.side_effect = query_hits
|
|
self.assertTrue(self.submit(failed_rid, attempt_id=3).decision)
|
|
self.assertTrue(self.submit(next_rid).decision)
|
|
peers = iter((peer_hit, 8))
|
|
local_votes = []
|
|
|
|
def reduce_hits(tensor, op, groups):
|
|
self.assertEqual(op, torch.distributed.ReduceOp.MIN)
|
|
self.assertEqual(groups, ["pp", "tp"])
|
|
self.assertEqual(tensor.numel(), 1 + len(PoolName))
|
|
local_votes.append(tensor[0].item())
|
|
if tensor[0].item():
|
|
self.assertEqual(c.mem_pool_host.alloc.call_args.args, (8,))
|
|
tensor[0] = min(tensor[0].item(), next(peers))
|
|
|
|
c._all_reduce.side_effect = reduce_hits
|
|
logs = (
|
|
self.assertLogs(level="ERROR")
|
|
if isinstance(local_hit, Exception)
|
|
else nullcontext()
|
|
)
|
|
with logs:
|
|
self.run_commands()
|
|
failed = c.pp_prefetch_states[failed_rid].operation
|
|
following = c.pp_prefetch_states[next_rid].operation
|
|
self.assertEqual(failed.storage_hit_count, 0)
|
|
self.assertEqual(failed.hash_value, [])
|
|
self.assertEqual(failed.host_indices.numel(), 0)
|
|
self.assertTrue(failed.is_terminated())
|
|
self.assertEqual(following.host_indices.numel(), 8)
|
|
self.assertFalse(following.is_terminated())
|
|
self.assertEqual(
|
|
local_votes,
|
|
[8 if name == "peer_miss" else 0, 8],
|
|
)
|
|
self.assertEqual(c._all_reduce.call_count, 2)
|
|
self.assertEqual(
|
|
c._storage_hit_query.call_args_list,
|
|
[call(failed), call(following)],
|
|
)
|
|
if name == "peer_miss":
|
|
self.assertEqual(
|
|
c.mem_pool_host.alloc.call_args_list,
|
|
[call(8, pool=PoolName.KV)] * 2,
|
|
)
|
|
self.assertEqual(c.mem_pool_host.free.call_count, 1)
|
|
self.assertTrue(
|
|
torch.equal(
|
|
c.mem_pool_host.free.call_args.args[0], torch.arange(8)
|
|
)
|
|
)
|
|
self.assertEqual(
|
|
c.mem_pool_host.free.call_args.kwargs, {"pool": PoolName.KV}
|
|
)
|
|
else:
|
|
c.mem_pool_host.alloc.assert_called_once_with(8, pool=PoolName.KV)
|
|
c.mem_pool_host.free.assert_not_called()
|
|
if name == "query_error":
|
|
self.assertEqual(failed.query_pool_hit_pages, {})
|
|
self.assertEqual(c.prefetch_tokens_occupied, occupied_before + 8)
|
|
|
|
c._all_reduce.side_effect = None
|
|
c.page_get_func = Mock(return_value=1)
|
|
acks = self.run_io(2)
|
|
self.assertEqual(
|
|
[ack.rid for ack in acks], [failed_rid] + [next_rid] * 3
|
|
)
|
|
self.assertTrue(acks[0].completed_req)
|
|
self.assertEqual(
|
|
[ack.completed_tokens for ack in acks[1:]], [4, 8, None]
|
|
)
|
|
self.assertEqual(c.page_get_func.call_count, 2)
|
|
self.sync_acks(*acks)
|
|
handle = CacheRequestHandle(failed_rid, 3)
|
|
self.assertTrue(c.is_pp_prefetch_ready(failed_rid))
|
|
self.assertTrue(cache.check_prefetch_progress(handle))
|
|
self.assertEqual(cache.prefetch_loaded_tokens_by_reqid[handle], 0)
|
|
self.assertNotIn(handle, cache.ongoing_prefetch)
|
|
cache._handle_prefetch_result.assert_not_called()
|
|
self.assertTrue(cache.check_prefetch_progress(handle))
|
|
self.assertFalse(c.release_pp_prefetch(failed_rid))
|
|
self.assertTrue(
|
|
cache.check_prefetch_progress(CacheRequestHandle(next_rid, 0))
|
|
)
|
|
cache._handle_prefetch_result.assert_called_once_with(following)
|
|
self.assertEqual(c._storage_hit_query.call_count, 2)
|
|
self.assertEqual(c.pp_prefetch_command_queue.unfinished_tasks, 0)
|
|
|
|
def test_tp_only_preserves_normal_prefetch(self):
|
|
c = self.c
|
|
c.pp_prefetch_command_group = None
|
|
result = self.submit(attempt_id=3, assume_stored=True)
|
|
self.assertIsNone(result.decision)
|
|
self.assertEqual(result.operation.handle, CacheRequestHandle("hit", 3))
|
|
self.assertTrue(result.operation.assume_stored)
|
|
self.assertIs(c.prefetch_queue.get_nowait(), result.operation)
|
|
c._storage_hit_query.assert_not_called()
|
|
|
|
def test_downstream_request_and_ticket_order_preserves_pp0_admission(self):
|
|
c, cache = self.c, self.cache
|
|
self.submit(attempt_id=3)
|
|
handle = CacheRequestHandle("hit", 3)
|
|
ticket = pickle.loads(pickle.dumps(c.pp_prefetch_states["hit"].ticket))
|
|
ticket.last_hash = None
|
|
c.storage_config.pp_rank = 1
|
|
c.pp_prefetch_states.clear()
|
|
cache.bind_prefetch_ticket("hit")
|
|
self.assertFalse(cache.check_prefetch_progress(handle))
|
|
c._storage_hit_query.reset_mock()
|
|
self.run_commands(ticket)
|
|
operation = c.prefetch_buffer.get_nowait()
|
|
self.assertEqual(operation.handle, handle)
|
|
self.assertEqual(
|
|
operation.hash_value, get_storage_hash_str(ticket.prefetch_key, page_size=4)
|
|
)
|
|
self.assertTrue(operation.token_ids.is_bigram)
|
|
self.assertEqual(len(operation.token_ids), 8)
|
|
c._storage_hit_query.assert_called_once_with(operation)
|
|
self.sync_acks(PrefetchAck("hit", operation, completed_tokens=8))
|
|
self.assertFalse(c.is_pp_prefetch_ready("hit"))
|
|
self.sync_acks(PrefetchAck("hit", operation, completed_req=True))
|
|
self.assertFalse(cache.check_prefetch_progress(handle)) # PP0 has not admitted.
|
|
cache._all_reduce.side_effect = lambda tensor, _: tensor.fill_(1)
|
|
self.assertTrue(cache.check_prefetch_progress(handle))
|
|
cache._handle_prefetch_result.assert_called_once_with(operation)
|
|
cache.buffer_pipeline.try_lock_anchor.assert_called_once_with(handle, 8)
|
|
key = cache.ongoing_prefetch[handle].prefetch_key
|
|
self.assertEqual((key.extra_key, key.cache_salt), ("adapter", "tenant"))
|
|
self.assertTrue(key.is_bigram)
|
|
self.assertFalse(c.is_pp_prefetch_ready("hit"))
|
|
|
|
def test_allocation_error_preserves_ack_sequence_and_next_ticket(self):
|
|
c = self.c
|
|
self.submit(
|
|
pools=[
|
|
PoolTransfer(PoolName.SWA, host_indices=torch.arange(4), keys=["h1"])
|
|
]
|
|
)
|
|
ticket = c.pp_prefetch_states.pop("hit").ticket
|
|
following = pickle.loads(pickle.dumps(ticket))
|
|
following.handle = CacheRequestHandle("next", 0)
|
|
c.storage_config.pp_rank = 1
|
|
kv = torch.arange(8)
|
|
c.mem_pool_host.alloc.side_effect = [
|
|
kv,
|
|
KeyError(PoolName.SWA),
|
|
torch.arange(8),
|
|
torch.arange(4),
|
|
]
|
|
with self.assertLogs(level="ERROR"):
|
|
self.run_commands(ticket, following)
|
|
failed = c.pp_prefetch_states["hit"].operation
|
|
next_operation = c.pp_prefetch_states["next"].operation
|
|
self.assertTrue(failed.is_terminated())
|
|
self.assertEqual(failed.storage_hit_count, 0)
|
|
self.assertEqual(failed.hash_value, [])
|
|
self.assertEqual(failed.host_indices.numel(), 0)
|
|
self.assertEqual(
|
|
c._storage_hit_query.call_args_list, [call(failed), call(next_operation)]
|
|
)
|
|
self.assertEqual(c._all_reduce.call_count, 2)
|
|
self.assertEqual(
|
|
[
|
|
invocation.args[0][0].item()
|
|
for invocation in c._all_reduce.call_args_list
|
|
],
|
|
[0, 8],
|
|
)
|
|
c.mem_pool_host.free.assert_called_once_with(kv, pool=PoolName.KV)
|
|
c.page_get_func = Mock(return_value=1)
|
|
c.storage_backend = Mock()
|
|
c.storage_backend.batch_get_v2.return_value = {"swa": [True]}
|
|
with (
|
|
patch("sglang.srt.managers.cache_controller.STORAGE_BATCH_SIZE", 1),
|
|
patch.object(
|
|
c.storage_stop_event, "is_set", side_effect=[False, False, True]
|
|
),
|
|
):
|
|
c.prefetch_io_aux_func()
|
|
acks = [c.prefetch_sync_queue.get_nowait() for _ in range(6)]
|
|
self.assertEqual([a.rid for a in acks], ["hit"] * 2 + ["next"] * 4)
|
|
self.assertEqual(
|
|
[a.completed_tokens for a in acks], [None, None, 4, 8, None, None]
|
|
)
|
|
self.assertEqual((acks[0].pool_hits, acks[4].pool_hits), ({}, {"swa": 1}))
|
|
self.assertEqual(
|
|
c.page_get_func.call_count, 2
|
|
) # No reads for the failed ticket.
|
|
self.sync_acks(*acks)
|
|
self.assertTrue(c.is_pp_prefetch_ready("hit"))
|
|
self.assertEqual(c.pp_prefetch_states["next"].operation.completed_tokens, 8)
|
|
self.assertEqual(c.prefetch_tokens_occupied, 8)
|
|
|
|
def test_cancel_before_ticket_defers_free_until_final_ack(self):
|
|
c, cache = self.c, self.cache
|
|
self.submit(pools=[PoolTransfer(PoolName.SWA, host_indices=torch.arange(4))])
|
|
ticket = c.pp_prefetch_states.pop("hit").ticket
|
|
c.storage_config.pp_rank = 1
|
|
cache.bind_prefetch_ticket("hit")
|
|
cache.finish(ticket.handle, CacheRequestOutcome.ABORT)
|
|
cache.finish(ticket.handle, CacheRequestOutcome.ABORT)
|
|
self.assertIs(c.pp_prefetch_decisions["hit"], PPPrefetchDecision.CANCELLED)
|
|
self.run_commands(ticket)
|
|
operation = c.prefetch_buffer.get_nowait()
|
|
self.sync_acks(PrefetchAck("hit", operation, completed_tokens=4))
|
|
c.mem_pool_host.free.assert_not_called()
|
|
self.sync_acks(PrefetchAck("hit", operation, completed_req=True))
|
|
self.assertEqual(
|
|
[call.kwargs["pool"] for call in c.mem_pool_host.free.call_args_list],
|
|
[PoolName.KV, PoolName.SWA],
|
|
)
|
|
self.assertEqual(c.prefetch_tokens_occupied, 0)
|
|
self.assertEqual(c.pp_prefetch_states, {})
|
|
self.assertEqual(c.pp_prefetch_decisions, {})
|
|
|
|
def test_lazy_sidecars_use_hit_pages_and_pool_page_size(self):
|
|
c = self.c
|
|
c.mem_pool_host.get_pool.side_effect = lambda name: Mock(
|
|
page_size=1 if name == PoolName.MAMBA else 4
|
|
)
|
|
self.submit(
|
|
pools=[
|
|
PoolTransfer(PoolName.SWA, keys=["pending"] * 3),
|
|
PoolTransfer(PoolName.MAMBA, keys=["pending"]),
|
|
PoolTransfer(PoolName.DRAFT_SWA, indices_from_pool=PoolName.SWA),
|
|
]
|
|
)
|
|
ticket = pickle.loads(pickle.dumps(c.pp_prefetch_states["hit"].ticket))
|
|
for rank in (0, 1):
|
|
with self.subTest(rank=rank):
|
|
c.storage_config.pp_rank = rank
|
|
if rank:
|
|
c.pp_prefetch_states.clear()
|
|
c.mem_pool_host.alloc.reset_mock()
|
|
self.run_commands(ticket)
|
|
operation = c.prefetch_buffer.get_nowait()
|
|
self.assertEqual(
|
|
c.mem_pool_host.alloc.call_args_list,
|
|
[
|
|
call(8, pool=PoolName.KV),
|
|
call(8, pool=PoolName.SWA),
|
|
call(1, pool=PoolName.MAMBA),
|
|
],
|
|
)
|
|
swa, mamba, draft_swa = operation.pool_transfers
|
|
self.assertIs(draft_swa.host_indices, swa.host_indices)
|
|
self.assertEqual(mamba.host_indices.numel(), 1)
|
|
|
|
# Missing pool metadata also rolls back KV, before a failed-ticket ACK.
|
|
c.pp_prefetch_states.clear()
|
|
c.mem_pool_host.get_pool.side_effect = KeyError(PoolName.SWA)
|
|
with self.assertLogs(level="ERROR"):
|
|
self.run_commands(ticket)
|
|
self.assertTrue(c.prefetch_buffer.get_nowait().is_terminated())
|
|
self.assertEqual(c.mem_pool_host.free.call_count, 1)
|
|
self.assertEqual(c.mem_pool_host.free.call_args.kwargs["pool"], PoolName.KV)
|
|
|
|
def test_failed_source_allocation_does_not_free_borrowed_sidecar_early(self):
|
|
c = self.c
|
|
borrowed = PoolTransfer(PoolName.SWA, host_indices=torch.arange(4))
|
|
operation = self.submit(pools=[borrowed]).operation
|
|
c.mem_pool_host.alloc.side_effect = lambda *args, **kwargs: None
|
|
self.run_commands()
|
|
self.assertTrue(operation.is_terminated())
|
|
self.assertEqual(operation.hash_value, [])
|
|
self.assertEqual(operation.storage_hit_count, 0)
|
|
c._storage_hit_query.assert_called_once_with(operation)
|
|
c.mem_pool_host.alloc.assert_called_once_with(8, pool=PoolName.KV)
|
|
c._all_reduce.assert_called_once()
|
|
packed, op, groups = c._all_reduce.call_args.args
|
|
self.assertEqual(packed[0].item(), 0)
|
|
self.assertEqual(packed.numel(), 1 + len(PoolName))
|
|
self.assertEqual(op, torch.distributed.ReduceOp.MIN)
|
|
self.assertEqual(groups, c.prefetch_hits_sync_groups)
|
|
c.mem_pool_host.free.assert_not_called()
|
|
self.sync_acks(
|
|
PrefetchAck("hit", operation, completed_tokens=0, completed_req=True)
|
|
)
|
|
c.take_ready_pp_prefetch("hit")
|
|
self.assertIsNone(borrowed.host_indices)
|
|
self.assertEqual(c.mem_pool_host.free.call_args.kwargs["pool"], PoolName.SWA)
|
|
|
|
def test_peer_vote_trims_kv_alias_and_defers_sidecar_free(self):
|
|
"""A peer's zero allocation vote must release KV without freeing a sidecar."""
|
|
c = self.c
|
|
for peer_vote in (4, 0):
|
|
with self.subTest(peer_vote=peer_vote):
|
|
rid = f"peer_{peer_vote}"
|
|
c._storage_hit_query.reset_mock()
|
|
c._all_reduce.reset_mock()
|
|
c.mem_pool_host.alloc.reset_mock()
|
|
c.mem_pool_host.free.reset_mock()
|
|
borrowed_indices = torch.arange(4)
|
|
operation = self.submit(
|
|
rid,
|
|
pools=[
|
|
PoolTransfer(PoolName.SWA, host_indices=borrowed_indices),
|
|
PoolTransfer(PoolName.INDEXER, indices_from_pool=PoolName.KV),
|
|
],
|
|
).operation
|
|
|
|
def reduce_hits(tensor, op, groups):
|
|
self.assertEqual(tensor[0].item(), 8)
|
|
self.assertEqual(tensor.numel(), 1 + len(PoolName))
|
|
self.assertEqual(op, torch.distributed.ReduceOp.MIN)
|
|
self.assertEqual(groups, c.prefetch_hits_sync_groups)
|
|
c.mem_pool_host.alloc.assert_called_once_with(8, pool=PoolName.KV)
|
|
tensor[0] = peer_vote
|
|
|
|
c._all_reduce.side_effect = reduce_hits
|
|
self.run_commands()
|
|
c._storage_hit_query.assert_called_once_with(operation)
|
|
c._all_reduce.assert_called_once()
|
|
swa, indexer = operation.pool_transfers
|
|
self.assertEqual(operation.host_indices.numel(), peer_vote)
|
|
self.assertEqual(operation.storage_hit_count, peer_vote)
|
|
self.assertEqual(operation.is_terminated(), peer_vote == 0)
|
|
self.assertIs(indexer.host_indices, operation.host_indices)
|
|
self.assertIs(indexer.keys, operation.hash_value)
|
|
self.assertIs(swa.host_indices, borrowed_indices)
|
|
self.assertEqual(c.prefetch_tokens_occupied, peer_vote)
|
|
freed = c.mem_pool_host.free.call_args
|
|
self.assertEqual(c.mem_pool_host.free.call_count, 1)
|
|
self.assertEqual(freed.kwargs, {"pool": PoolName.KV})
|
|
self.assertTrue(torch.equal(freed.args[0], torch.arange(peer_vote, 8)))
|
|
|
|
self.assertTrue(c.release_pp_prefetch(rid))
|
|
self.assertIs(swa.host_indices, borrowed_indices)
|
|
self.assertEqual(c.mem_pool_host.free.call_count, 1)
|
|
c._all_reduce.side_effect = None
|
|
self.sync_acks(
|
|
PrefetchAck(rid, operation, completed_tokens=peer_vote),
|
|
PrefetchAck(rid, operation, completed_req=True),
|
|
)
|
|
self.assertIsNone(swa.host_indices)
|
|
self.assertNotIn(rid, c.pp_prefetch_states)
|
|
self.assertEqual(c.prefetch_tokens_occupied, 0)
|
|
sidecar_frees = [
|
|
invocation
|
|
for invocation in c.mem_pool_host.free.call_args_list
|
|
if invocation.kwargs["pool"] == PoolName.SWA
|
|
]
|
|
self.assertEqual(len(sidecar_frees), 1)
|
|
self.assertIs(sidecar_frees[0].args[0], borrowed_indices)
|
|
|
|
def test_command_failure_finishes_queue_task_and_is_reported_on_poll(self):
|
|
c = self.c
|
|
with (
|
|
patch(
|
|
f"{HybridCacheController.__module__}.broadcast_pyobj",
|
|
side_effect=RuntimeError("broken"),
|
|
),
|
|
self.assertLogs(level="ERROR"),
|
|
):
|
|
worker = threading.Thread(
|
|
target=c.pp_prefetch_command_thread_func, daemon=True
|
|
)
|
|
c.pp_prefetch_command_thread = worker
|
|
worker.start()
|
|
self.submit()
|
|
worker.join(timeout=1)
|
|
self.assertFalse(worker.is_alive())
|
|
self.assertEqual(c.pp_prefetch_command_queue.unfinished_tasks, 0)
|
|
for rid in ("hit", "next"):
|
|
if rid == "next":
|
|
self.submit(rid)
|
|
with self.assertRaisesRegex(RuntimeError, "ticket thread exited"):
|
|
c.is_pp_prefetch_ready(rid)
|
|
|
|
def test_idle_source_broadcasts_empty_then_processes_ticket_and_stop(self):
|
|
c = self.c
|
|
c.storage_config.tp_rank = 1 # The source is a global rank, not pp_rank=0.
|
|
operation = self.submit().operation
|
|
c.pp_prefetch_command_queue.put(None)
|
|
get = c.pp_prefetch_command_queue.get
|
|
idle_polls = [True, True]
|
|
|
|
def get_after_idle(*, timeout):
|
|
self.assertEqual(timeout, 60)
|
|
if idle_polls:
|
|
idle_polls.pop()
|
|
raise Empty
|
|
return get(block=False)
|
|
|
|
with (
|
|
patch.object(c.pp_prefetch_command_queue, "get", get_after_idle),
|
|
patch.object(
|
|
torch.distributed, "get_process_group_ranks", return_value=[1, 3, 5, 7]
|
|
),
|
|
patch(
|
|
f"{HybridCacheController.__module__}.broadcast_pyobj",
|
|
side_effect=lambda objects, *args, **kwargs: objects,
|
|
) as broadcast,
|
|
):
|
|
c.pp_prefetch_command_thread_func()
|
|
calls = broadcast.call_args_list
|
|
self.assertEqual([len(c.args[0]) for c in calls], [0, 0, 1, 1])
|
|
for invocation in calls:
|
|
self.assertEqual(invocation.args[1:], (1, "command"))
|
|
self.assertEqual(invocation.kwargs, {"src": 1})
|
|
self.assertEqual(c.pp_prefetch_command_queue.unfinished_tasks, 0)
|
|
self.assertTrue(c.pp_prefetch_command_queue.empty())
|
|
self.assertIs(c.prefetch_buffer.get_nowait(), operation)
|
|
self.assertTrue(c.prefetch_buffer.empty())
|
|
c.mem_pool_host.alloc.assert_called_once()
|
|
|
|
def test_idle_downstream_skips_empty_broadcasts_before_ticket_and_stop(self):
|
|
c = self.c
|
|
self.submit()
|
|
ticket = c.pp_prefetch_states.pop("hit").ticket
|
|
c.storage_config.pp_rank = 1
|
|
with (
|
|
patch(
|
|
f"{HybridCacheController.__module__}.broadcast_pyobj",
|
|
side_effect=[[], [], [ticket], [None]],
|
|
),
|
|
patch.object(c.pp_prefetch_command_queue, "task_done") as task_done,
|
|
):
|
|
c.pp_prefetch_command_thread_func()
|
|
self.assertEqual(c.prefetch_buffer.get_nowait().request_id, "hit")
|
|
self.assertTrue(c.prefetch_buffer.empty())
|
|
c.mem_pool_host.alloc.assert_called_once()
|
|
task_done.assert_not_called()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|