Net async change (from exp-async-backup-perop-20261107) applied onto b300-pick-46884-20261107. Runtime files applied cleanly; the new tests and the timeline probe are added as-is. Interaction to validate on hardware: #46884 publishes prefill read-done before the replay, so the scheduler-stream fence that the index copy waits on is shorter; the async worker's wait should shrink and the copy may overlap more of the in-flight forward. Re-baseline the 4-arm A/B on this branch.
551 lines
20 KiB
Python
551 lines
20 KiB
Python
"""Unit tests for the async HiCache backup submission state machine.
|
|
|
|
Design: handoff/prefill-backup-overlap-next-20261107.md §8.1.
|
|
|
|
These tests exercise the scheduler/worker split without a GPU: the index copy
|
|
itself is covered by the CUDA test, and here the device stream/event and the
|
|
submit path are faked. What is checked:
|
|
|
|
* disabled / inapplicable shapes keep the sync path;
|
|
* per-op pinned parts are materialized per (pool, indices_from_pool) key;
|
|
* the worker waits every ready event before touching host buffers;
|
|
* exactly-once ACK, fail-closed errors, bounded queues, idle/stop lifecycle;
|
|
* has_inflight_device_transfers reflects queued/running results.
|
|
"""
|
|
|
|
import threading
|
|
import time
|
|
import types
|
|
import unittest
|
|
from unittest import mock
|
|
|
|
from sglang.srt.managers import cache_controller as cc_mod
|
|
from sglang.srt.mem_cache.hicache_storage import PoolName, PoolTransfer
|
|
|
|
|
|
class _FakeEvent:
|
|
def __init__(self, worker_log=None, name="ev"):
|
|
self.worker_log = worker_log
|
|
self.name = name
|
|
self._set = False
|
|
self.waited = 0
|
|
|
|
def record(self, stream=None):
|
|
self._set = True
|
|
|
|
def synchronize(self):
|
|
self.waited += 1
|
|
if self.worker_log is not None:
|
|
self.worker_log.append(("wait", self.name))
|
|
self._set = True
|
|
|
|
def query(self):
|
|
return self._set
|
|
|
|
|
|
class _FakeStream:
|
|
def __init__(self, name="stream", log=None):
|
|
self.name = name
|
|
self.log = log
|
|
self.waited = 0
|
|
|
|
def wait_stream(self, other):
|
|
self.waited += 1
|
|
if self.log is not None:
|
|
self.log.append(("wait_stream", self.name, getattr(other, "name", "?")))
|
|
|
|
def wait_event(self, event):
|
|
pass
|
|
|
|
|
|
class _FakeStreamCtx:
|
|
def __init__(self, stream):
|
|
self.stream = stream
|
|
|
|
def __enter__(self):
|
|
return self.stream
|
|
|
|
def __exit__(self, *exc):
|
|
return False
|
|
|
|
|
|
class _FakeCudaModule:
|
|
def __init__(self, log=None):
|
|
self.log = log
|
|
self.current = _FakeStream("schedule", log)
|
|
|
|
def Stream(self, *a, **k):
|
|
return _FakeStream("indices", self.log)
|
|
|
|
def StreamContext(self, stream):
|
|
return _FakeStreamCtx(stream)
|
|
|
|
def Event(self, *a, **k):
|
|
return _FakeEvent(self.log)
|
|
|
|
def current_stream(self):
|
|
return self.current
|
|
|
|
|
|
class _FakeTensor:
|
|
def __init__(self, numel=8, element_size=8, is_cuda=True, tag="dev"):
|
|
self._numel = numel
|
|
self._element_size = element_size
|
|
self.is_cuda = is_cuda
|
|
self.tag = tag
|
|
self.recorded = []
|
|
self.cpu_calls = 0
|
|
self.device = types.SimpleNamespace(type="cuda" if is_cuda else "cpu")
|
|
self.dtype = "torch.int64"
|
|
|
|
def numel(self):
|
|
return self._numel
|
|
|
|
def element_size(self):
|
|
return self._element_size
|
|
|
|
def record_stream(self, stream):
|
|
self.recorded.append(stream)
|
|
|
|
def copy_(self, src, non_blocking=False):
|
|
self.copied_from = src
|
|
|
|
@property
|
|
def shape(self):
|
|
return (self._numel,)
|
|
|
|
def cpu(self):
|
|
self.cpu_calls += 1
|
|
return _FakeTensor(self._numel, self._element_size, False, tag=self.tag + "-cpu")
|
|
|
|
def __len__(self):
|
|
return self._numel
|
|
|
|
|
|
class _FakeHostPool:
|
|
def __init__(self, shared=False):
|
|
self.shared = shared
|
|
|
|
|
|
class _FakeEntry:
|
|
def __init__(self, host_pool):
|
|
self.host_pool = host_pool
|
|
|
|
|
|
class _FakeMemPoolHost:
|
|
layout = "page_first_direct"
|
|
|
|
def __init__(self, anchor_shared=False, sidecars=None):
|
|
self.anchor_entry = _FakeEntry(_FakeHostPool(anchor_shared))
|
|
self.entry_map = {
|
|
name: _FakeEntry(pool) for name, pool in (sidecars or {}).items()
|
|
}
|
|
|
|
|
|
def _make_controller(*, enabled=True, anchor_shared=False, sidecars=None, policy=""):
|
|
c = cc_mod.HiCacheController.__new__(cc_mod.HiCacheController)
|
|
c.io_backend = "direct"
|
|
c.write_policy = policy
|
|
c.mem_pool_host = _FakeMemPoolHost(anchor_shared, sidecars)
|
|
c.write_queue = []
|
|
c.load_queue = []
|
|
c.ack_write_queue = []
|
|
c.ack_load_queue = []
|
|
c._indices_stream = None
|
|
c._write_submit_queue = None
|
|
c._write_result_queue = cc_mod.Queue()
|
|
c._write_submit_stop = threading.Event()
|
|
c._write_submit_thread = None
|
|
c._write_submit_state_lock = threading.Lock()
|
|
c._write_submit_cond = threading.Condition(c._write_submit_state_lock)
|
|
c._write_submit_active = 0
|
|
c._async_pinned_bytes = 0
|
|
c._retire_parts = []
|
|
return c
|
|
|
|
|
|
def _op(controller, *, anchor=None, transfers=(), node_id=7):
|
|
host = _FakeTensor(8, 8, is_cuda=False, tag="host")
|
|
dev = anchor or _FakeTensor(8, 8, is_cuda=True, tag="anchor")
|
|
op = cc_mod.CacheOperation(host, dev, node_id, pool_transfers=list(transfers))
|
|
return op
|
|
|
|
|
|
class AsyncBackupUnitTest(unittest.TestCase):
|
|
def setUp(self):
|
|
patcher = mock.patch.object(cc_mod, "_ASYNC_BACKUP_SUBMIT", True)
|
|
patcher.start()
|
|
self.addCleanup(patcher.stop)
|
|
# The real predicate isinstance-checks HostKVCache; the fakes expose a
|
|
# plain `shared` flag instead.
|
|
shared_patcher = mock.patch.object(
|
|
cc_mod, "uses_shared_host_layout", lambda pool: bool(getattr(pool, "shared", False))
|
|
)
|
|
shared_patcher.start()
|
|
self.addCleanup(shared_patcher.stop)
|
|
cat_patcher = mock.patch.object(
|
|
cc_mod.torch,
|
|
"cat",
|
|
side_effect=lambda parts: _FakeTensor(
|
|
sum(len(p) for p in parts), 8, False, "cat"
|
|
),
|
|
)
|
|
cat_patcher.start()
|
|
self.addCleanup(cat_patcher.stop)
|
|
# Pinned destination buffers are faked; only the copy_ call is checked.
|
|
self.pin_patch = mock.patch.object(
|
|
cc_mod.torch, "empty", side_effect=lambda *a, **k: _FakeTensor(8, 8, False, "pin")
|
|
)
|
|
self.pin_patch.start()
|
|
self.addCleanup(self.pin_patch.stop)
|
|
|
|
def test_disabled_keeps_sync_path(self):
|
|
with mock.patch.object(cc_mod, "_ASYNC_BACKUP_SUBMIT", False):
|
|
c = _make_controller()
|
|
self.assertFalse(c.async_backup_enabled())
|
|
self.assertFalse(c._async_submit_applicable())
|
|
|
|
def test_wrong_backend_or_layout_is_ineligible(self):
|
|
c = _make_controller()
|
|
c.io_backend = "kernel"
|
|
self.assertFalse(c.async_backup_enabled())
|
|
c = _make_controller()
|
|
c.mem_pool_host.layout = "page_first"
|
|
self.assertFalse(c.async_backup_enabled())
|
|
|
|
def test_index_targets_skip_shared_layout_and_cpu_tensors(self):
|
|
shared = PoolName.SWA
|
|
c = _make_controller(sidecars={shared: _FakeHostPool(shared=True)})
|
|
transfer = PoolTransfer(
|
|
name=shared, device_indices=_FakeTensor(4, 8, True, "swa")
|
|
)
|
|
cpu_anchor = _FakeTensor(8, 8, False, "anchor")
|
|
op = _op(c, anchor=cpu_anchor, transfers=[transfer])
|
|
self.assertEqual(c._async_index_targets(op), [])
|
|
|
|
c = _make_controller(sidecars={PoolName.SWA: _FakeHostPool(shared=False)})
|
|
transfer = PoolTransfer(
|
|
name=PoolName.SWA,
|
|
device_indices=_FakeTensor(4, 8, True, "swa"),
|
|
indices_from_pool=PoolName.KV,
|
|
)
|
|
anchor = _FakeTensor(8, 8, True, "anchor")
|
|
op = _op(c, anchor=anchor, transfers=[transfer])
|
|
targets = c._async_index_targets(op)
|
|
self.assertEqual([k for k, _ in targets], [(None, None), (PoolName.SWA, PoolName.KV)])
|
|
|
|
def test_start_copy_orders_after_scheduler_and_records_event(self):
|
|
log = []
|
|
c = _make_controller()
|
|
op = _op(c)
|
|
with mock.patch.object(cc_mod, "device_module", _FakeCudaModule(log)):
|
|
c._start_async_index_copy(op)
|
|
self.assertIsNotNone(op.async_parts)
|
|
self.assertEqual(op.async_parts.ready_event._set, True)
|
|
self.assertEqual(op.device_indices.recorded[0].name, "indices")
|
|
# The copy stream is ordered after the scheduler stream once.
|
|
self.assertIn(("wait_stream", "indices", "schedule"), log)
|
|
self.assertEqual(c._async_pinned_bytes, 64)
|
|
|
|
def test_pinned_cap_falls_back_to_sync(self):
|
|
c = _make_controller()
|
|
op = _op(c)
|
|
with mock.patch.object(cc_mod, "_ASYNC_BACKUP_PINNED_BYTES_MAX", 1), \
|
|
mock.patch.object(cc_mod, "device_module", _FakeCudaModule()):
|
|
c._start_async_index_copy(op)
|
|
self.assertIsNone(op.async_parts)
|
|
self.assertEqual(c._async_pinned_bytes, 0)
|
|
|
|
def test_materialize_uses_pool_keys(self):
|
|
c = _make_controller()
|
|
anchor_copy = cc_mod.AsyncIndexCopy((None, None), None, _FakeTensor(8, 8, False, "a"), 64)
|
|
deriv = PoolTransfer(
|
|
name=PoolName.SWA,
|
|
device_indices=_FakeTensor(4, 8, True, "swa"),
|
|
indices_from_pool=PoolName.KV,
|
|
)
|
|
side_copy = cc_mod.AsyncIndexCopy(
|
|
(PoolName.SWA, PoolName.KV), None, _FakeTensor(4, 8, False, "s"), 32
|
|
)
|
|
op = _op(c, transfers=[deriv])
|
|
op.async_parts = cc_mod.AsyncIndexParts(
|
|
copies=[anchor_copy, side_copy], ready_event=_FakeEvent(), nbytes=96
|
|
)
|
|
c._materialize_async_indices([op])
|
|
self.assertIs(op.device_indices, anchor_copy.dst)
|
|
self.assertIs(deriv.device_indices, side_copy.dst)
|
|
|
|
def test_worker_waits_events_then_submits_and_acks_once(self):
|
|
log = []
|
|
c = _make_controller()
|
|
events = []
|
|
|
|
def _make_op(tag):
|
|
op = _op(c, anchor=_FakeTensor(8, 8, True, tag))
|
|
ev = _FakeEvent(log, tag)
|
|
events.append(ev)
|
|
op.async_parts = cc_mod.AsyncIndexParts(
|
|
copies=[
|
|
cc_mod.AsyncIndexCopy(
|
|
(None, None), op.device_indices, _FakeTensor(8, 8, False, tag), 64
|
|
)
|
|
],
|
|
ready_event=ev,
|
|
nbytes=64,
|
|
)
|
|
return op
|
|
|
|
ops = [_make_op("a"), _make_op("b")]
|
|
c._async_pinned_bytes = 128
|
|
submitted = []
|
|
|
|
def _submit(op):
|
|
# Every ready event must have been waited before the merge is used.
|
|
submitted.append(list(op.node_ids))
|
|
return cc_mod.HiCacheAck(None, None, list(op.node_ids), num_tokens=8)
|
|
|
|
c._submit_write_operation = _submit
|
|
c._write_submit_queue = cc_mod.Queue(maxsize=4)
|
|
c._ensure_write_submit_thread = lambda: None
|
|
worker = threading.Thread(target=c._write_submit_worker, daemon=True)
|
|
c._write_submit_thread = worker
|
|
worker.start()
|
|
try:
|
|
item = cc_mod.WriteSubmission(ops=ops, tracer=None, enqueue_mono_ns=0)
|
|
c._write_submit_active = 1
|
|
c._write_submit_queue.put(item)
|
|
deadline = time.time() + 5
|
|
while c._write_submit_active and time.time() < deadline:
|
|
time.sleep(0.005)
|
|
self.assertEqual(c._write_submit_active, 0)
|
|
finally:
|
|
c._write_submit_queue.put(None)
|
|
worker.join(timeout=5)
|
|
self.assertEqual([e.waited for e in events], [1, 1])
|
|
self.assertEqual(submitted, [[7, 7]])
|
|
self.assertEqual(c._async_pinned_bytes, 0)
|
|
c.drain_write_submission_results()
|
|
self.assertEqual(len(c.ack_write_queue), 1)
|
|
# Second drain is a no-op: exactly-once ACK.
|
|
c.drain_write_submission_results()
|
|
self.assertEqual(len(c.ack_write_queue), 1)
|
|
|
|
def test_worker_error_is_fail_closed(self):
|
|
c = _make_controller()
|
|
op = _op(c)
|
|
op.async_parts = cc_mod.AsyncIndexParts([], _FakeEvent(), 0)
|
|
c._submit_write_operation = mock.Mock(side_effect=RuntimeError("boom"))
|
|
c._write_submit_queue = cc_mod.Queue(maxsize=4)
|
|
worker = threading.Thread(target=c._write_submit_worker, daemon=True)
|
|
c._write_submit_thread = worker
|
|
worker.start()
|
|
c._write_submit_active = 1
|
|
c._write_submit_queue.put(cc_mod.WriteSubmission([op], None, 0))
|
|
deadline = time.time() + 5
|
|
while c._write_submit_active and time.time() < deadline:
|
|
time.sleep(0.005)
|
|
c._write_submit_queue.put(None)
|
|
worker.join(timeout=5)
|
|
with self.assertRaises(RuntimeError):
|
|
c.drain_write_submission_results()
|
|
self.assertEqual(c.ack_write_queue, [])
|
|
|
|
def test_enqueue_failure_keeps_active_consistent(self):
|
|
c = _make_controller()
|
|
c._write_submit_queue = cc_mod.Queue(maxsize=1)
|
|
c._write_submit_queue.put("occupied")
|
|
c._ensure_write_submit_thread = lambda: None
|
|
with mock.patch.object(cc_mod, "_ASYNC_BACKUP_ENQUEUE_TIMEOUT", 0.05):
|
|
ok = c._enqueue_async_write([_op(c)], None)
|
|
self.assertFalse(ok)
|
|
self.assertEqual(c._write_submit_active, 0)
|
|
|
|
def test_queue_full_falls_back_to_sync_submit(self):
|
|
c = _make_controller()
|
|
# A permanently-full submit queue forces the sync path.
|
|
c._write_submit_queue = cc_mod.Queue(maxsize=1)
|
|
c._write_submit_queue.put("occupied")
|
|
c._ensure_write_submit_thread = lambda: None
|
|
op = _op(c)
|
|
retired_event = _FakeEvent()
|
|
retired_event.record()
|
|
op.async_parts = cc_mod.AsyncIndexParts(
|
|
[cc_mod.AsyncIndexCopy((None, None), op.device_indices, _FakeTensor(8, 8, False), 64)],
|
|
retired_event,
|
|
64,
|
|
)
|
|
c._async_pinned_bytes = 64
|
|
c.write_queue.append(op)
|
|
submitted = []
|
|
|
|
def _submit(merged):
|
|
# The fallback must hand over the original device indices, i.e. the
|
|
# pinned parts are retired rather than materialized.
|
|
submitted.append(merged.device_indices)
|
|
return cc_mod.HiCacheAck(None, None, list(merged.node_ids), num_tokens=8)
|
|
|
|
c._submit_write_operation = _submit
|
|
with mock.patch.object(cc_mod, "_ASYNC_BACKUP_ENQUEUE_TIMEOUT", 0.05):
|
|
c.start_writing()
|
|
self.assertEqual(len(submitted), 1)
|
|
self.assertIs(submitted[0], op.device_indices)
|
|
self.assertEqual(len(c.ack_write_queue), 1)
|
|
self.assertEqual(c.write_queue, [])
|
|
self.assertIsNone(op.async_parts)
|
|
self.assertEqual(c._write_submit_active, 0)
|
|
# The retired copy is released once its event completes.
|
|
c._reap_retired_parts()
|
|
self.assertEqual(c._async_pinned_bytes, 0)
|
|
|
|
def test_mixed_queue_sync_path_retires_unconsumed_parts(self):
|
|
c = _make_controller()
|
|
ev = _FakeEvent()
|
|
ev.record()
|
|
with_parts = _op(c, node_id=1)
|
|
with_parts.async_parts = cc_mod.AsyncIndexParts(
|
|
[
|
|
cc_mod.AsyncIndexCopy(
|
|
(None, None), with_parts.device_indices, _FakeTensor(8, 8, False), 64
|
|
)
|
|
],
|
|
ev,
|
|
64,
|
|
)
|
|
without_parts = _op(c, node_id=2)
|
|
c.write_queue.extend([with_parts, without_parts])
|
|
c._async_pinned_bytes = 64
|
|
submitted = []
|
|
c._submit_write_operation = lambda merged: (
|
|
submitted.append(list(merged.node_ids))
|
|
or cc_mod.HiCacheAck(None, None, list(merged.node_ids), num_tokens=8)
|
|
)
|
|
c.start_writing()
|
|
# Mixed residency () forces the sync path, but the pinned copy started
|
|
# for the first op must be retired and its budget released.
|
|
self.assertEqual(submitted, [[1, 2]])
|
|
self.assertIsNone(with_parts.async_parts)
|
|
self.assertEqual(c._async_pinned_bytes, 0)
|
|
self.assertEqual(len(c.ack_write_queue), 1)
|
|
|
|
def test_op_cap_falls_back_to_sync(self):
|
|
c = _make_controller()
|
|
with mock.patch.object(cc_mod, "_ASYNC_BACKUP_MAX_OPS", 1):
|
|
for i in range(2):
|
|
op = _op(c, node_id=i)
|
|
op.async_parts = cc_mod.AsyncIndexParts([], _FakeEvent(), 0)
|
|
c.write_queue.append(op)
|
|
self.assertFalse(c._async_submit_applicable())
|
|
|
|
def test_inflight_reflects_async_state(self):
|
|
c = _make_controller()
|
|
self.assertFalse(c.has_inflight_device_transfers())
|
|
c._write_submit_active = 1
|
|
self.assertTrue(c.has_inflight_device_transfers())
|
|
c._write_submit_active = 0
|
|
c._async_pinned_bytes = 8
|
|
self.assertTrue(c.has_inflight_device_transfers())
|
|
c._async_pinned_bytes = 0
|
|
c._write_result_queue.put(cc_mod.WriteResult(None, [], None))
|
|
self.assertTrue(c.has_inflight_device_transfers())
|
|
c._write_result_queue.get()
|
|
self.assertFalse(c.has_inflight_device_transfers())
|
|
|
|
def test_idle_wait_times_out(self):
|
|
c = _make_controller()
|
|
c._write_submit_active = 1
|
|
with self.assertRaises(RuntimeError):
|
|
c.wait_write_submissions_idle(timeout=0.05)
|
|
|
|
def test_retired_parts_release_only_after_event(self):
|
|
c = _make_controller()
|
|
ev = _FakeEvent()
|
|
ev._set = False
|
|
op = _op(c)
|
|
op.async_parts = cc_mod.AsyncIndexParts(
|
|
[cc_mod.AsyncIndexCopy((None, None), None, _FakeTensor(8, 8, False), 64)],
|
|
ev,
|
|
64,
|
|
)
|
|
c._async_pinned_bytes = 64
|
|
c._retire_async_parts([op])
|
|
self.assertIsNone(op.async_parts)
|
|
self.assertEqual(c._async_pinned_bytes, 64)
|
|
ev._set = True
|
|
c._reap_retired_parts()
|
|
self.assertEqual(c._async_pinned_bytes, 0)
|
|
self.assertEqual(c._retire_parts, [])
|
|
|
|
def test_merge_of_cpu_parts_matches_cpu_reference(self):
|
|
c = _make_controller()
|
|
ops = []
|
|
for i in range(3):
|
|
op = _op(c, anchor=_FakeTensor(4, 8, True, f"a{i}"), node_id=i)
|
|
op.async_parts = cc_mod.AsyncIndexParts(
|
|
[
|
|
cc_mod.AsyncIndexCopy(
|
|
(None, None),
|
|
op.device_indices,
|
|
_FakeTensor(4, 8, False, f"p{i}"),
|
|
32,
|
|
)
|
|
],
|
|
_FakeEvent(),
|
|
32,
|
|
)
|
|
ops.append(op)
|
|
c._materialize_async_indices(ops)
|
|
merged = cc_mod.CacheOperation.merge_ops(ops)
|
|
self.assertEqual(merged.node_ids, [0, 1, 2])
|
|
# Single-op merge must not copy: it returns the operation itself.
|
|
single = cc_mod.CacheOperation.merge_ops([ops[0]])
|
|
self.assertIs(single, ops[0])
|
|
|
|
def test_stop_waits_for_inflight_submission_and_drains_ack(self):
|
|
c = _make_controller()
|
|
release = threading.Event()
|
|
started = threading.Event()
|
|
|
|
def _submit(op):
|
|
started.set()
|
|
release.wait(10)
|
|
return cc_mod.HiCacheAck(None, None, list(op.node_ids), num_tokens=8)
|
|
|
|
c._submit_write_operation = _submit
|
|
op = _op(c)
|
|
op.async_parts = cc_mod.AsyncIndexParts([], _FakeEvent(), 0)
|
|
c._ensure_write_submit_thread()
|
|
c._write_submit_active = 1
|
|
c._write_submit_queue.put(cc_mod.WriteSubmission([op], None, 0))
|
|
self.assertTrue(started.wait(5))
|
|
|
|
stopped = threading.Event()
|
|
|
|
def _stop():
|
|
c._stop_write_submission_thread()
|
|
stopped.set()
|
|
|
|
stopper = threading.Thread(target=_stop, daemon=True)
|
|
stopper.start()
|
|
# The stop must not return while a submission is still in flight.
|
|
self.assertFalse(stopped.wait(0.05))
|
|
release.set()
|
|
self.assertTrue(stopped.wait(10))
|
|
self.assertFalse(c._write_submit_thread is not None and c._write_submit_thread.is_alive())
|
|
# The ACK produced by the in-flight submission was not dropped.
|
|
self.assertEqual(len(c.ack_write_queue), 1)
|
|
|
|
def test_stop_joins_worker_and_allows_restart(self):
|
|
c = _make_controller()
|
|
c._ensure_write_submit_thread()
|
|
self.assertTrue(c._write_submit_thread.is_alive())
|
|
c._stop_write_submission_thread()
|
|
self.assertIsNone(c._write_submit_thread)
|
|
self.assertIsNone(c._write_submit_queue)
|
|
c._ensure_write_submit_thread()
|
|
self.assertTrue(c._write_submit_thread.is_alive())
|
|
c._stop_write_submission_thread()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|