Net async change (from exp-async-backup-perop-20261107) applied onto b300-pick-46884-20261107. Runtime files applied cleanly; the new tests and the timeline probe are added as-is. Interaction to validate on hardware: #46884 publishes prefill read-done before the replay, so the scheduler-stream fence that the index copy waits on is shorter; the async worker's wait should shrink and the copy may overlap more of the in-flight forward. Re-baseline the 4-arm A/B on this branch.
254 lines
9.2 KiB
Python
254 lines
9.2 KiB
Python
"""CUDA ordering/timeline tests for the per-op async backup index copy.
|
|
|
|
Plan: handoff/prefill-backup-overlap-next-20261107.md §3/§8.2.
|
|
|
|
Run inside the GPU container with the overlay source:
|
|
|
|
PYTHONPATH=/tmp/ovr python -m pytest \
|
|
test/registered/cuda/test_async_backup_index_copy_cuda.py -q
|
|
|
|
What is proven on real hardware:
|
|
|
|
1. ``_start_async_index_copy`` returns while a long kernel is still queued on
|
|
the scheduler stream, and the synchronous ``.cpu()`` on that stream blocks;
|
|
2. the copy is only released after that fence, but the *next* forward (a
|
|
marker kernel on another stream, ordered behind the same fence) runs
|
|
concurrently with it: the copy end timestamp falls strictly inside the
|
|
marker's [start, end] interval;
|
|
3. the host can enqueue the next batch before ``ready_event`` completes and
|
|
the event is not reported ready early;
|
|
4. with three ops whose copies complete out of order, the worker still merges
|
|
them in op order, identical to ``CacheOperation.merge_ops`` over the CPU
|
|
reference tensors;
|
|
5. the copy stream is distinct from the scheduler stream.
|
|
"""
|
|
|
|
import threading
|
|
import time
|
|
import unittest
|
|
|
|
import torch
|
|
|
|
from sglang.srt.managers import cache_controller as cc_mod
|
|
|
|
|
|
class _FakeHostPool:
|
|
"""Not a HostKVCache: uses_shared_host_layout() is false, i.e. the sync path
|
|
would call device_indices.cpu()."""
|
|
|
|
shared_allocation_domain = None
|
|
|
|
|
|
class _FakeEntry:
|
|
def __init__(self):
|
|
self.host_pool = _FakeHostPool()
|
|
|
|
|
|
class _FakeMemPoolHost:
|
|
layout = "page_first_direct"
|
|
|
|
def __init__(self):
|
|
self.anchor_entry = _FakeEntry()
|
|
self.entry_map = {}
|
|
|
|
|
|
def _make_controller():
|
|
c = cc_mod.HiCacheController.__new__(cc_mod.HiCacheController)
|
|
c.io_backend = "direct"
|
|
c.mem_pool_host = _FakeMemPoolHost()
|
|
c.write_queue = []
|
|
c.load_queue = []
|
|
c.ack_write_queue = []
|
|
c.ack_load_queue = []
|
|
c._indices_stream = None
|
|
c._write_submit_queue = None
|
|
c._write_result_queue = cc_mod.Queue()
|
|
c._write_submit_stop = threading.Event()
|
|
c._write_submit_thread = None
|
|
c._write_submit_state_lock = threading.Lock()
|
|
c._write_submit_cond = threading.Condition(c._write_submit_state_lock)
|
|
c._write_submit_active = 0
|
|
c._async_pinned_bytes = 0
|
|
c._retire_parts = []
|
|
return c
|
|
|
|
|
|
def _op(device_indices, node_id):
|
|
host = torch.empty(device_indices.numel(), dtype=torch.int64)
|
|
return cc_mod.CacheOperation(host, device_indices, node_id)
|
|
|
|
|
|
class AsyncBackupIndexCopyCudaTest(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
cc_mod._ASYNC_BACKUP_SUBMIT = True
|
|
cls.saved_pin = cc_mod._ASYNC_BACKUP_PINNED_BYTES_MAX
|
|
cc_mod._ASYNC_BACKUP_PINNED_BYTES_MAX = 1 << 30
|
|
|
|
@classmethod
|
|
def tearDownClass(cls):
|
|
cc_mod._ASYNC_BACKUP_PINNED_BYTES_MAX = cls.saved_pin
|
|
|
|
def test_start_does_not_block_and_copy_is_async(self):
|
|
torch.cuda.synchronize()
|
|
sched = torch.cuda.current_stream()
|
|
fwd = torch.cuda.Stream()
|
|
indices = torch.arange(4096, dtype=torch.int64, device="cuda")
|
|
|
|
# Simulated in-flight forward plus the scheduler's WAR fence.
|
|
with torch.cuda.stream(fwd):
|
|
torch.cuda._sleep(2_000_000_000)
|
|
sched.wait_stream(fwd)
|
|
|
|
c = _make_controller()
|
|
op = _op(indices, 11)
|
|
t0 = time.perf_counter()
|
|
c._start_async_index_copy(op)
|
|
start_elapsed = time.perf_counter() - t0
|
|
self.assertIsNotNone(op.async_parts)
|
|
# Starting the copy must not block on the simulated forward.
|
|
self.assertLess(start_elapsed, 0.05, start_elapsed)
|
|
self.assertFalse(op.async_parts.ready_event.query())
|
|
|
|
# The synchronous baseline on the same stream does block.
|
|
t1 = time.perf_counter()
|
|
reference = indices.cpu()
|
|
sync_elapsed = time.perf_counter() - t1
|
|
self.assertGreater(sync_elapsed, 0.1, sync_elapsed)
|
|
|
|
op.async_parts.ready_event.synchronize()
|
|
self.assertTrue(op.async_parts.ready_event.query())
|
|
c._materialize_async_indices([op])
|
|
self.assertTrue(bool(torch.equal(op.device_indices, reference)))
|
|
self.assertFalse(op.device_indices.is_cuda)
|
|
c._release_async_parts([op])
|
|
self.assertEqual(c._async_pinned_bytes, 0)
|
|
|
|
def test_next_forward_overlaps_the_index_copy(self):
|
|
torch.cuda.synchronize()
|
|
sched = torch.cuda.current_stream()
|
|
fwd = torch.cuda.Stream()
|
|
# Large enough that the copy takes long enough to observe (32 MB).
|
|
indices = torch.arange(4 * 1024 * 1024, dtype=torch.int64, device="cuda")
|
|
|
|
# 1. A long fence on the scheduler stream (the in-flight forward).
|
|
with torch.cuda.stream(sched):
|
|
fence_done = torch.cuda.Event(enable_timing=True)
|
|
fence_done.record()
|
|
torch.cuda._sleep(1_000_000_000)
|
|
|
|
# 2. Per-op pinned copy started now; it inherits the fence.
|
|
c = _make_controller()
|
|
op = _op(indices, 3)
|
|
c._start_async_index_copy(op)
|
|
self.assertIsNotNone(op.async_parts)
|
|
|
|
# 3. The next batch launch waits for the scheduler stream (as the real
|
|
# forward_stream does) and then runs a long marker kernel.
|
|
copy_end = torch.cuda.Event(enable_timing=True)
|
|
with torch.cuda.stream(c._indices_stream):
|
|
copy_end.record()
|
|
launch_host_ns = time.monotonic_ns()
|
|
with torch.cuda.stream(fwd):
|
|
fwd.wait_stream(sched)
|
|
fwd_start = torch.cuda.Event(enable_timing=True)
|
|
fwd_start.record()
|
|
torch.cuda._sleep(1_500_000_000)
|
|
fwd_end = torch.cuda.Event(enable_timing=True)
|
|
fwd_end.record()
|
|
|
|
# 4. The event must not be ready while the fence is in flight, and the
|
|
# host has already enqueued the next batch.
|
|
ready_seen_ns = [None]
|
|
|
|
def _wait_ready():
|
|
op.async_parts.ready_event.synchronize()
|
|
ready_seen_ns[0] = time.monotonic_ns()
|
|
|
|
waiter = threading.Thread(target=_wait_ready)
|
|
waiter.start()
|
|
self.assertIsNone(ready_seen_ns[0])
|
|
waiter.join(timeout=30)
|
|
self.assertIsNotNone(ready_seen_ns[0])
|
|
self.assertLess(launch_host_ns, ready_seen_ns[0])
|
|
|
|
torch.cuda.synchronize()
|
|
# CUDA timestamps: copy end strictly inside the next forward.
|
|
self.assertGreater(fwd_start.elapsed_time(copy_end), 0.0)
|
|
self.assertGreater(copy_end.elapsed_time(fwd_end), 0.0)
|
|
self.assertGreater(fwd_start.elapsed_time(fwd_end), 0.5)
|
|
|
|
c._materialize_async_indices([op])
|
|
self.assertTrue(
|
|
bool(torch.equal(op.device_indices, indices.cpu()))
|
|
)
|
|
c._release_async_parts([op])
|
|
|
|
def test_copy_stream_is_distinct_from_scheduler(self):
|
|
torch.cuda.synchronize()
|
|
c = _make_controller()
|
|
op = _op(torch.arange(8, dtype=torch.int64, device="cuda"), 0)
|
|
c._start_async_index_copy(op)
|
|
self.assertIsNotNone(c._indices_stream)
|
|
self.assertNotEqual(
|
|
c._indices_stream.cuda_stream,
|
|
torch.cuda.current_stream().cuda_stream,
|
|
)
|
|
op.async_parts.ready_event.synchronize()
|
|
c._release_async_parts([op])
|
|
|
|
def test_out_of_order_completion_keeps_op_order(self):
|
|
torch.cuda.synchronize()
|
|
c = _make_controller()
|
|
delays = {0: 600_000_000, 1: 150_000_000, 2: 300_000_000}
|
|
ops = []
|
|
references = []
|
|
for i in range(3):
|
|
src = torch.arange(8, dtype=torch.int64, device="cuda") + i * 100
|
|
op = _op(src, node_id=i)
|
|
buf = torch.empty(src.shape, dtype=src.dtype, device="cpu", pin_memory=True)
|
|
stream = torch.cuda.Stream()
|
|
with torch.cuda.stream(stream):
|
|
torch.cuda._sleep(delays[i])
|
|
buf.copy_(src, non_blocking=True)
|
|
ev = torch.cuda.Event()
|
|
ev.record(stream)
|
|
op.async_parts = cc_mod.AsyncIndexParts(
|
|
[cc_mod.AsyncIndexCopy((None, None), src, buf, buf.numel() * 8)],
|
|
ev,
|
|
buf.numel() * 8,
|
|
)
|
|
ops.append(op)
|
|
references.append(src.cpu())
|
|
|
|
captured = []
|
|
c._submit_write_operation = lambda merged: (
|
|
captured.append(merged.device_indices.clone())
|
|
or cc_mod.HiCacheAck(None, None, list(merged.node_ids), num_tokens=8)
|
|
)
|
|
c._write_submit_queue = cc_mod.Queue(maxsize=4)
|
|
worker = threading.Thread(target=c._write_submit_worker, daemon=True)
|
|
c._write_submit_thread = worker
|
|
worker.start()
|
|
try:
|
|
c._write_submit_active = 1
|
|
c._write_submit_queue.put(cc_mod.WriteSubmission(ops, None, 0))
|
|
deadline = time.time() + 30
|
|
while c._write_submit_active and time.time() < deadline:
|
|
time.sleep(0.01)
|
|
self.assertEqual(c._write_submit_active, 0)
|
|
finally:
|
|
c._write_submit_queue.put(None)
|
|
worker.join(timeout=10)
|
|
|
|
self.assertEqual(len(captured), 1)
|
|
expected = torch.cat(references)
|
|
self.assertTrue(bool(torch.equal(captured[0], expected)))
|
|
self.assertEqual(ops[0].node_ids, [0])
|
|
self.assertEqual(ops[1].node_ids, [1])
|
|
self.assertEqual(ops[2].node_ids, [2])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|