Files
sglang/test/registered/cuda/test_async_backup_index_copy_cuda.py
T
minke.yu d6f1e1efbd rebase(hicache): per-op async backup on top of #46884 + #48782
Net async change (from exp-async-backup-perop-20261107) applied onto
b300-pick-46884-20261107. Runtime files applied cleanly; the new tests and
the timeline probe are added as-is.

Interaction to validate on hardware: #46884 publishes prefill read-done before
the replay, so the scheduler-stream fence that the index copy waits on is
shorter; the async worker's wait should shrink and the copy may overlap more of
the in-flight forward. Re-baseline the 4-arm A/B on this branch.
2026-11-07 22:04:33 +08:00

254 lines
9.2 KiB
Python

"""CUDA ordering/timeline tests for the per-op async backup index copy.
Plan: handoff/prefill-backup-overlap-next-20261107.md §3/§8.2.
Run inside the GPU container with the overlay source:
PYTHONPATH=/tmp/ovr python -m pytest \
test/registered/cuda/test_async_backup_index_copy_cuda.py -q
What is proven on real hardware:
1. ``_start_async_index_copy`` returns while a long kernel is still queued on
the scheduler stream, and the synchronous ``.cpu()`` on that stream blocks;
2. the copy is only released after that fence, but the *next* forward (a
marker kernel on another stream, ordered behind the same fence) runs
concurrently with it: the copy end timestamp falls strictly inside the
marker's [start, end] interval;
3. the host can enqueue the next batch before ``ready_event`` completes and
the event is not reported ready early;
4. with three ops whose copies complete out of order, the worker still merges
them in op order, identical to ``CacheOperation.merge_ops`` over the CPU
reference tensors;
5. the copy stream is distinct from the scheduler stream.
"""
import threading
import time
import unittest
import torch
from sglang.srt.managers import cache_controller as cc_mod
class _FakeHostPool:
"""Not a HostKVCache: uses_shared_host_layout() is false, i.e. the sync path
would call device_indices.cpu()."""
shared_allocation_domain = None
class _FakeEntry:
def __init__(self):
self.host_pool = _FakeHostPool()
class _FakeMemPoolHost:
layout = "page_first_direct"
def __init__(self):
self.anchor_entry = _FakeEntry()
self.entry_map = {}
def _make_controller():
c = cc_mod.HiCacheController.__new__(cc_mod.HiCacheController)
c.io_backend = "direct"
c.mem_pool_host = _FakeMemPoolHost()
c.write_queue = []
c.load_queue = []
c.ack_write_queue = []
c.ack_load_queue = []
c._indices_stream = None
c._write_submit_queue = None
c._write_result_queue = cc_mod.Queue()
c._write_submit_stop = threading.Event()
c._write_submit_thread = None
c._write_submit_state_lock = threading.Lock()
c._write_submit_cond = threading.Condition(c._write_submit_state_lock)
c._write_submit_active = 0
c._async_pinned_bytes = 0
c._retire_parts = []
return c
def _op(device_indices, node_id):
host = torch.empty(device_indices.numel(), dtype=torch.int64)
return cc_mod.CacheOperation(host, device_indices, node_id)
class AsyncBackupIndexCopyCudaTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
cc_mod._ASYNC_BACKUP_SUBMIT = True
cls.saved_pin = cc_mod._ASYNC_BACKUP_PINNED_BYTES_MAX
cc_mod._ASYNC_BACKUP_PINNED_BYTES_MAX = 1 << 30
@classmethod
def tearDownClass(cls):
cc_mod._ASYNC_BACKUP_PINNED_BYTES_MAX = cls.saved_pin
def test_start_does_not_block_and_copy_is_async(self):
torch.cuda.synchronize()
sched = torch.cuda.current_stream()
fwd = torch.cuda.Stream()
indices = torch.arange(4096, dtype=torch.int64, device="cuda")
# Simulated in-flight forward plus the scheduler's WAR fence.
with torch.cuda.stream(fwd):
torch.cuda._sleep(2_000_000_000)
sched.wait_stream(fwd)
c = _make_controller()
op = _op(indices, 11)
t0 = time.perf_counter()
c._start_async_index_copy(op)
start_elapsed = time.perf_counter() - t0
self.assertIsNotNone(op.async_parts)
# Starting the copy must not block on the simulated forward.
self.assertLess(start_elapsed, 0.05, start_elapsed)
self.assertFalse(op.async_parts.ready_event.query())
# The synchronous baseline on the same stream does block.
t1 = time.perf_counter()
reference = indices.cpu()
sync_elapsed = time.perf_counter() - t1
self.assertGreater(sync_elapsed, 0.1, sync_elapsed)
op.async_parts.ready_event.synchronize()
self.assertTrue(op.async_parts.ready_event.query())
c._materialize_async_indices([op])
self.assertTrue(bool(torch.equal(op.device_indices, reference)))
self.assertFalse(op.device_indices.is_cuda)
c._release_async_parts([op])
self.assertEqual(c._async_pinned_bytes, 0)
def test_next_forward_overlaps_the_index_copy(self):
torch.cuda.synchronize()
sched = torch.cuda.current_stream()
fwd = torch.cuda.Stream()
# Large enough that the copy takes long enough to observe (32 MB).
indices = torch.arange(4 * 1024 * 1024, dtype=torch.int64, device="cuda")
# 1. A long fence on the scheduler stream (the in-flight forward).
with torch.cuda.stream(sched):
fence_done = torch.cuda.Event(enable_timing=True)
fence_done.record()
torch.cuda._sleep(1_000_000_000)
# 2. Per-op pinned copy started now; it inherits the fence.
c = _make_controller()
op = _op(indices, 3)
c._start_async_index_copy(op)
self.assertIsNotNone(op.async_parts)
# 3. The next batch launch waits for the scheduler stream (as the real
# forward_stream does) and then runs a long marker kernel.
copy_end = torch.cuda.Event(enable_timing=True)
with torch.cuda.stream(c._indices_stream):
copy_end.record()
launch_host_ns = time.monotonic_ns()
with torch.cuda.stream(fwd):
fwd.wait_stream(sched)
fwd_start = torch.cuda.Event(enable_timing=True)
fwd_start.record()
torch.cuda._sleep(1_500_000_000)
fwd_end = torch.cuda.Event(enable_timing=True)
fwd_end.record()
# 4. The event must not be ready while the fence is in flight, and the
# host has already enqueued the next batch.
ready_seen_ns = [None]
def _wait_ready():
op.async_parts.ready_event.synchronize()
ready_seen_ns[0] = time.monotonic_ns()
waiter = threading.Thread(target=_wait_ready)
waiter.start()
self.assertIsNone(ready_seen_ns[0])
waiter.join(timeout=30)
self.assertIsNotNone(ready_seen_ns[0])
self.assertLess(launch_host_ns, ready_seen_ns[0])
torch.cuda.synchronize()
# CUDA timestamps: copy end strictly inside the next forward.
self.assertGreater(fwd_start.elapsed_time(copy_end), 0.0)
self.assertGreater(copy_end.elapsed_time(fwd_end), 0.0)
self.assertGreater(fwd_start.elapsed_time(fwd_end), 0.5)
c._materialize_async_indices([op])
self.assertTrue(
bool(torch.equal(op.device_indices, indices.cpu()))
)
c._release_async_parts([op])
def test_copy_stream_is_distinct_from_scheduler(self):
torch.cuda.synchronize()
c = _make_controller()
op = _op(torch.arange(8, dtype=torch.int64, device="cuda"), 0)
c._start_async_index_copy(op)
self.assertIsNotNone(c._indices_stream)
self.assertNotEqual(
c._indices_stream.cuda_stream,
torch.cuda.current_stream().cuda_stream,
)
op.async_parts.ready_event.synchronize()
c._release_async_parts([op])
def test_out_of_order_completion_keeps_op_order(self):
torch.cuda.synchronize()
c = _make_controller()
delays = {0: 600_000_000, 1: 150_000_000, 2: 300_000_000}
ops = []
references = []
for i in range(3):
src = torch.arange(8, dtype=torch.int64, device="cuda") + i * 100
op = _op(src, node_id=i)
buf = torch.empty(src.shape, dtype=src.dtype, device="cpu", pin_memory=True)
stream = torch.cuda.Stream()
with torch.cuda.stream(stream):
torch.cuda._sleep(delays[i])
buf.copy_(src, non_blocking=True)
ev = torch.cuda.Event()
ev.record(stream)
op.async_parts = cc_mod.AsyncIndexParts(
[cc_mod.AsyncIndexCopy((None, None), src, buf, buf.numel() * 8)],
ev,
buf.numel() * 8,
)
ops.append(op)
references.append(src.cpu())
captured = []
c._submit_write_operation = lambda merged: (
captured.append(merged.device_indices.clone())
or cc_mod.HiCacheAck(None, None, list(merged.node_ids), num_tokens=8)
)
c._write_submit_queue = cc_mod.Queue(maxsize=4)
worker = threading.Thread(target=c._write_submit_worker, daemon=True)
c._write_submit_thread = worker
worker.start()
try:
c._write_submit_active = 1
c._write_submit_queue.put(cc_mod.WriteSubmission(ops, None, 0))
deadline = time.time() + 30
while c._write_submit_active and time.time() < deadline:
time.sleep(0.01)
self.assertEqual(c._write_submit_active, 0)
finally:
c._write_submit_queue.put(None)
worker.join(timeout=10)
self.assertEqual(len(captured), 1)
expected = torch.cat(references)
self.assertTrue(bool(torch.equal(captured[0], expected)))
self.assertEqual(ops[0].node_ids, [0])
self.assertEqual(ops[1].node_ids, [1])
self.assertEqual(ops[2].node_ids, [2])
if __name__ == "__main__":
unittest.main()