Files
sglang/test/registered/unit/disaggregation/test_async_backup_submit.py
T
minke.yu d6f1e1efbd rebase(hicache): per-op async backup on top of #46884 + #48782
Net async change (from exp-async-backup-perop-20261107) applied onto
b300-pick-46884-20261107. Runtime files applied cleanly; the new tests and
the timeline probe are added as-is.

Interaction to validate on hardware: #46884 publishes prefill read-done before
the replay, so the scheduler-stream fence that the index copy waits on is
shorter; the async worker's wait should shrink and the copy may overlap more of
the in-flight forward. Re-baseline the 4-arm A/B on this branch.
2026-11-07 22:04:33 +08:00

551 lines
20 KiB
Python

"""Unit tests for the async HiCache backup submission state machine.
Design: handoff/prefill-backup-overlap-next-20261107.md §8.1.
These tests exercise the scheduler/worker split without a GPU: the index copy
itself is covered by the CUDA test, and here the device stream/event and the
submit path are faked. What is checked:
* disabled / inapplicable shapes keep the sync path;
* per-op pinned parts are materialized per (pool, indices_from_pool) key;
* the worker waits every ready event before touching host buffers;
* exactly-once ACK, fail-closed errors, bounded queues, idle/stop lifecycle;
* has_inflight_device_transfers reflects queued/running results.
"""
import threading
import time
import types
import unittest
from unittest import mock
from sglang.srt.managers import cache_controller as cc_mod
from sglang.srt.mem_cache.hicache_storage import PoolName, PoolTransfer
class _FakeEvent:
def __init__(self, worker_log=None, name="ev"):
self.worker_log = worker_log
self.name = name
self._set = False
self.waited = 0
def record(self, stream=None):
self._set = True
def synchronize(self):
self.waited += 1
if self.worker_log is not None:
self.worker_log.append(("wait", self.name))
self._set = True
def query(self):
return self._set
class _FakeStream:
def __init__(self, name="stream", log=None):
self.name = name
self.log = log
self.waited = 0
def wait_stream(self, other):
self.waited += 1
if self.log is not None:
self.log.append(("wait_stream", self.name, getattr(other, "name", "?")))
def wait_event(self, event):
pass
class _FakeStreamCtx:
def __init__(self, stream):
self.stream = stream
def __enter__(self):
return self.stream
def __exit__(self, *exc):
return False
class _FakeCudaModule:
def __init__(self, log=None):
self.log = log
self.current = _FakeStream("schedule", log)
def Stream(self, *a, **k):
return _FakeStream("indices", self.log)
def StreamContext(self, stream):
return _FakeStreamCtx(stream)
def Event(self, *a, **k):
return _FakeEvent(self.log)
def current_stream(self):
return self.current
class _FakeTensor:
def __init__(self, numel=8, element_size=8, is_cuda=True, tag="dev"):
self._numel = numel
self._element_size = element_size
self.is_cuda = is_cuda
self.tag = tag
self.recorded = []
self.cpu_calls = 0
self.device = types.SimpleNamespace(type="cuda" if is_cuda else "cpu")
self.dtype = "torch.int64"
def numel(self):
return self._numel
def element_size(self):
return self._element_size
def record_stream(self, stream):
self.recorded.append(stream)
def copy_(self, src, non_blocking=False):
self.copied_from = src
@property
def shape(self):
return (self._numel,)
def cpu(self):
self.cpu_calls += 1
return _FakeTensor(self._numel, self._element_size, False, tag=self.tag + "-cpu")
def __len__(self):
return self._numel
class _FakeHostPool:
def __init__(self, shared=False):
self.shared = shared
class _FakeEntry:
def __init__(self, host_pool):
self.host_pool = host_pool
class _FakeMemPoolHost:
layout = "page_first_direct"
def __init__(self, anchor_shared=False, sidecars=None):
self.anchor_entry = _FakeEntry(_FakeHostPool(anchor_shared))
self.entry_map = {
name: _FakeEntry(pool) for name, pool in (sidecars or {}).items()
}
def _make_controller(*, enabled=True, anchor_shared=False, sidecars=None, policy=""):
c = cc_mod.HiCacheController.__new__(cc_mod.HiCacheController)
c.io_backend = "direct"
c.write_policy = policy
c.mem_pool_host = _FakeMemPoolHost(anchor_shared, sidecars)
c.write_queue = []
c.load_queue = []
c.ack_write_queue = []
c.ack_load_queue = []
c._indices_stream = None
c._write_submit_queue = None
c._write_result_queue = cc_mod.Queue()
c._write_submit_stop = threading.Event()
c._write_submit_thread = None
c._write_submit_state_lock = threading.Lock()
c._write_submit_cond = threading.Condition(c._write_submit_state_lock)
c._write_submit_active = 0
c._async_pinned_bytes = 0
c._retire_parts = []
return c
def _op(controller, *, anchor=None, transfers=(), node_id=7):
host = _FakeTensor(8, 8, is_cuda=False, tag="host")
dev = anchor or _FakeTensor(8, 8, is_cuda=True, tag="anchor")
op = cc_mod.CacheOperation(host, dev, node_id, pool_transfers=list(transfers))
return op
class AsyncBackupUnitTest(unittest.TestCase):
def setUp(self):
patcher = mock.patch.object(cc_mod, "_ASYNC_BACKUP_SUBMIT", True)
patcher.start()
self.addCleanup(patcher.stop)
# The real predicate isinstance-checks HostKVCache; the fakes expose a
# plain `shared` flag instead.
shared_patcher = mock.patch.object(
cc_mod, "uses_shared_host_layout", lambda pool: bool(getattr(pool, "shared", False))
)
shared_patcher.start()
self.addCleanup(shared_patcher.stop)
cat_patcher = mock.patch.object(
cc_mod.torch,
"cat",
side_effect=lambda parts: _FakeTensor(
sum(len(p) for p in parts), 8, False, "cat"
),
)
cat_patcher.start()
self.addCleanup(cat_patcher.stop)
# Pinned destination buffers are faked; only the copy_ call is checked.
self.pin_patch = mock.patch.object(
cc_mod.torch, "empty", side_effect=lambda *a, **k: _FakeTensor(8, 8, False, "pin")
)
self.pin_patch.start()
self.addCleanup(self.pin_patch.stop)
def test_disabled_keeps_sync_path(self):
with mock.patch.object(cc_mod, "_ASYNC_BACKUP_SUBMIT", False):
c = _make_controller()
self.assertFalse(c.async_backup_enabled())
self.assertFalse(c._async_submit_applicable())
def test_wrong_backend_or_layout_is_ineligible(self):
c = _make_controller()
c.io_backend = "kernel"
self.assertFalse(c.async_backup_enabled())
c = _make_controller()
c.mem_pool_host.layout = "page_first"
self.assertFalse(c.async_backup_enabled())
def test_index_targets_skip_shared_layout_and_cpu_tensors(self):
shared = PoolName.SWA
c = _make_controller(sidecars={shared: _FakeHostPool(shared=True)})
transfer = PoolTransfer(
name=shared, device_indices=_FakeTensor(4, 8, True, "swa")
)
cpu_anchor = _FakeTensor(8, 8, False, "anchor")
op = _op(c, anchor=cpu_anchor, transfers=[transfer])
self.assertEqual(c._async_index_targets(op), [])
c = _make_controller(sidecars={PoolName.SWA: _FakeHostPool(shared=False)})
transfer = PoolTransfer(
name=PoolName.SWA,
device_indices=_FakeTensor(4, 8, True, "swa"),
indices_from_pool=PoolName.KV,
)
anchor = _FakeTensor(8, 8, True, "anchor")
op = _op(c, anchor=anchor, transfers=[transfer])
targets = c._async_index_targets(op)
self.assertEqual([k for k, _ in targets], [(None, None), (PoolName.SWA, PoolName.KV)])
def test_start_copy_orders_after_scheduler_and_records_event(self):
log = []
c = _make_controller()
op = _op(c)
with mock.patch.object(cc_mod, "device_module", _FakeCudaModule(log)):
c._start_async_index_copy(op)
self.assertIsNotNone(op.async_parts)
self.assertEqual(op.async_parts.ready_event._set, True)
self.assertEqual(op.device_indices.recorded[0].name, "indices")
# The copy stream is ordered after the scheduler stream once.
self.assertIn(("wait_stream", "indices", "schedule"), log)
self.assertEqual(c._async_pinned_bytes, 64)
def test_pinned_cap_falls_back_to_sync(self):
c = _make_controller()
op = _op(c)
with mock.patch.object(cc_mod, "_ASYNC_BACKUP_PINNED_BYTES_MAX", 1), \
mock.patch.object(cc_mod, "device_module", _FakeCudaModule()):
c._start_async_index_copy(op)
self.assertIsNone(op.async_parts)
self.assertEqual(c._async_pinned_bytes, 0)
def test_materialize_uses_pool_keys(self):
c = _make_controller()
anchor_copy = cc_mod.AsyncIndexCopy((None, None), None, _FakeTensor(8, 8, False, "a"), 64)
deriv = PoolTransfer(
name=PoolName.SWA,
device_indices=_FakeTensor(4, 8, True, "swa"),
indices_from_pool=PoolName.KV,
)
side_copy = cc_mod.AsyncIndexCopy(
(PoolName.SWA, PoolName.KV), None, _FakeTensor(4, 8, False, "s"), 32
)
op = _op(c, transfers=[deriv])
op.async_parts = cc_mod.AsyncIndexParts(
copies=[anchor_copy, side_copy], ready_event=_FakeEvent(), nbytes=96
)
c._materialize_async_indices([op])
self.assertIs(op.device_indices, anchor_copy.dst)
self.assertIs(deriv.device_indices, side_copy.dst)
def test_worker_waits_events_then_submits_and_acks_once(self):
log = []
c = _make_controller()
events = []
def _make_op(tag):
op = _op(c, anchor=_FakeTensor(8, 8, True, tag))
ev = _FakeEvent(log, tag)
events.append(ev)
op.async_parts = cc_mod.AsyncIndexParts(
copies=[
cc_mod.AsyncIndexCopy(
(None, None), op.device_indices, _FakeTensor(8, 8, False, tag), 64
)
],
ready_event=ev,
nbytes=64,
)
return op
ops = [_make_op("a"), _make_op("b")]
c._async_pinned_bytes = 128
submitted = []
def _submit(op):
# Every ready event must have been waited before the merge is used.
submitted.append(list(op.node_ids))
return cc_mod.HiCacheAck(None, None, list(op.node_ids), num_tokens=8)
c._submit_write_operation = _submit
c._write_submit_queue = cc_mod.Queue(maxsize=4)
c._ensure_write_submit_thread = lambda: None
worker = threading.Thread(target=c._write_submit_worker, daemon=True)
c._write_submit_thread = worker
worker.start()
try:
item = cc_mod.WriteSubmission(ops=ops, tracer=None, enqueue_mono_ns=0)
c._write_submit_active = 1
c._write_submit_queue.put(item)
deadline = time.time() + 5
while c._write_submit_active and time.time() < deadline:
time.sleep(0.005)
self.assertEqual(c._write_submit_active, 0)
finally:
c._write_submit_queue.put(None)
worker.join(timeout=5)
self.assertEqual([e.waited for e in events], [1, 1])
self.assertEqual(submitted, [[7, 7]])
self.assertEqual(c._async_pinned_bytes, 0)
c.drain_write_submission_results()
self.assertEqual(len(c.ack_write_queue), 1)
# Second drain is a no-op: exactly-once ACK.
c.drain_write_submission_results()
self.assertEqual(len(c.ack_write_queue), 1)
def test_worker_error_is_fail_closed(self):
c = _make_controller()
op = _op(c)
op.async_parts = cc_mod.AsyncIndexParts([], _FakeEvent(), 0)
c._submit_write_operation = mock.Mock(side_effect=RuntimeError("boom"))
c._write_submit_queue = cc_mod.Queue(maxsize=4)
worker = threading.Thread(target=c._write_submit_worker, daemon=True)
c._write_submit_thread = worker
worker.start()
c._write_submit_active = 1
c._write_submit_queue.put(cc_mod.WriteSubmission([op], None, 0))
deadline = time.time() + 5
while c._write_submit_active and time.time() < deadline:
time.sleep(0.005)
c._write_submit_queue.put(None)
worker.join(timeout=5)
with self.assertRaises(RuntimeError):
c.drain_write_submission_results()
self.assertEqual(c.ack_write_queue, [])
def test_enqueue_failure_keeps_active_consistent(self):
c = _make_controller()
c._write_submit_queue = cc_mod.Queue(maxsize=1)
c._write_submit_queue.put("occupied")
c._ensure_write_submit_thread = lambda: None
with mock.patch.object(cc_mod, "_ASYNC_BACKUP_ENQUEUE_TIMEOUT", 0.05):
ok = c._enqueue_async_write([_op(c)], None)
self.assertFalse(ok)
self.assertEqual(c._write_submit_active, 0)
def test_queue_full_falls_back_to_sync_submit(self):
c = _make_controller()
# A permanently-full submit queue forces the sync path.
c._write_submit_queue = cc_mod.Queue(maxsize=1)
c._write_submit_queue.put("occupied")
c._ensure_write_submit_thread = lambda: None
op = _op(c)
retired_event = _FakeEvent()
retired_event.record()
op.async_parts = cc_mod.AsyncIndexParts(
[cc_mod.AsyncIndexCopy((None, None), op.device_indices, _FakeTensor(8, 8, False), 64)],
retired_event,
64,
)
c._async_pinned_bytes = 64
c.write_queue.append(op)
submitted = []
def _submit(merged):
# The fallback must hand over the original device indices, i.e. the
# pinned parts are retired rather than materialized.
submitted.append(merged.device_indices)
return cc_mod.HiCacheAck(None, None, list(merged.node_ids), num_tokens=8)
c._submit_write_operation = _submit
with mock.patch.object(cc_mod, "_ASYNC_BACKUP_ENQUEUE_TIMEOUT", 0.05):
c.start_writing()
self.assertEqual(len(submitted), 1)
self.assertIs(submitted[0], op.device_indices)
self.assertEqual(len(c.ack_write_queue), 1)
self.assertEqual(c.write_queue, [])
self.assertIsNone(op.async_parts)
self.assertEqual(c._write_submit_active, 0)
# The retired copy is released once its event completes.
c._reap_retired_parts()
self.assertEqual(c._async_pinned_bytes, 0)
def test_mixed_queue_sync_path_retires_unconsumed_parts(self):
c = _make_controller()
ev = _FakeEvent()
ev.record()
with_parts = _op(c, node_id=1)
with_parts.async_parts = cc_mod.AsyncIndexParts(
[
cc_mod.AsyncIndexCopy(
(None, None), with_parts.device_indices, _FakeTensor(8, 8, False), 64
)
],
ev,
64,
)
without_parts = _op(c, node_id=2)
c.write_queue.extend([with_parts, without_parts])
c._async_pinned_bytes = 64
submitted = []
c._submit_write_operation = lambda merged: (
submitted.append(list(merged.node_ids))
or cc_mod.HiCacheAck(None, None, list(merged.node_ids), num_tokens=8)
)
c.start_writing()
# Mixed residency () forces the sync path, but the pinned copy started
# for the first op must be retired and its budget released.
self.assertEqual(submitted, [[1, 2]])
self.assertIsNone(with_parts.async_parts)
self.assertEqual(c._async_pinned_bytes, 0)
self.assertEqual(len(c.ack_write_queue), 1)
def test_op_cap_falls_back_to_sync(self):
c = _make_controller()
with mock.patch.object(cc_mod, "_ASYNC_BACKUP_MAX_OPS", 1):
for i in range(2):
op = _op(c, node_id=i)
op.async_parts = cc_mod.AsyncIndexParts([], _FakeEvent(), 0)
c.write_queue.append(op)
self.assertFalse(c._async_submit_applicable())
def test_inflight_reflects_async_state(self):
c = _make_controller()
self.assertFalse(c.has_inflight_device_transfers())
c._write_submit_active = 1
self.assertTrue(c.has_inflight_device_transfers())
c._write_submit_active = 0
c._async_pinned_bytes = 8
self.assertTrue(c.has_inflight_device_transfers())
c._async_pinned_bytes = 0
c._write_result_queue.put(cc_mod.WriteResult(None, [], None))
self.assertTrue(c.has_inflight_device_transfers())
c._write_result_queue.get()
self.assertFalse(c.has_inflight_device_transfers())
def test_idle_wait_times_out(self):
c = _make_controller()
c._write_submit_active = 1
with self.assertRaises(RuntimeError):
c.wait_write_submissions_idle(timeout=0.05)
def test_retired_parts_release_only_after_event(self):
c = _make_controller()
ev = _FakeEvent()
ev._set = False
op = _op(c)
op.async_parts = cc_mod.AsyncIndexParts(
[cc_mod.AsyncIndexCopy((None, None), None, _FakeTensor(8, 8, False), 64)],
ev,
64,
)
c._async_pinned_bytes = 64
c._retire_async_parts([op])
self.assertIsNone(op.async_parts)
self.assertEqual(c._async_pinned_bytes, 64)
ev._set = True
c._reap_retired_parts()
self.assertEqual(c._async_pinned_bytes, 0)
self.assertEqual(c._retire_parts, [])
def test_merge_of_cpu_parts_matches_cpu_reference(self):
c = _make_controller()
ops = []
for i in range(3):
op = _op(c, anchor=_FakeTensor(4, 8, True, f"a{i}"), node_id=i)
op.async_parts = cc_mod.AsyncIndexParts(
[
cc_mod.AsyncIndexCopy(
(None, None),
op.device_indices,
_FakeTensor(4, 8, False, f"p{i}"),
32,
)
],
_FakeEvent(),
32,
)
ops.append(op)
c._materialize_async_indices(ops)
merged = cc_mod.CacheOperation.merge_ops(ops)
self.assertEqual(merged.node_ids, [0, 1, 2])
# Single-op merge must not copy: it returns the operation itself.
single = cc_mod.CacheOperation.merge_ops([ops[0]])
self.assertIs(single, ops[0])
def test_stop_waits_for_inflight_submission_and_drains_ack(self):
c = _make_controller()
release = threading.Event()
started = threading.Event()
def _submit(op):
started.set()
release.wait(10)
return cc_mod.HiCacheAck(None, None, list(op.node_ids), num_tokens=8)
c._submit_write_operation = _submit
op = _op(c)
op.async_parts = cc_mod.AsyncIndexParts([], _FakeEvent(), 0)
c._ensure_write_submit_thread()
c._write_submit_active = 1
c._write_submit_queue.put(cc_mod.WriteSubmission([op], None, 0))
self.assertTrue(started.wait(5))
stopped = threading.Event()
def _stop():
c._stop_write_submission_thread()
stopped.set()
stopper = threading.Thread(target=_stop, daemon=True)
stopper.start()
# The stop must not return while a submission is still in flight.
self.assertFalse(stopped.wait(0.05))
release.set()
self.assertTrue(stopped.wait(10))
self.assertFalse(c._write_submit_thread is not None and c._write_submit_thread.is_alive())
# The ACK produced by the in-flight submission was not dropped.
self.assertEqual(len(c.ack_write_queue), 1)
def test_stop_joins_worker_and_allows_restart(self):
c = _make_controller()
c._ensure_write_submit_thread()
self.assertTrue(c._write_submit_thread.is_alive())
c._stop_write_submission_thread()
self.assertIsNone(c._write_submit_thread)
self.assertIsNone(c._write_submit_queue)
c._ensure_write_submit_thread()
self.assertTrue(c._write_submit_thread.is_alive())
c._stop_write_submission_thread()
if __name__ == "__main__":
unittest.main()