"""Print the CUDA timeline for one async index copy vs the next forward.""" import importlib.util as _u import threading import time import torch from sglang.srt.managers import cache_controller as cc_mod _spec = _u.spec_from_file_location( "t", "/data/ymk/wt-async/test/registered/cuda/test_async_backup_index_copy_cuda.py" ) _m = _u.module_from_spec(_spec) _spec.loader.exec_module(_m) _make_controller, _op = _m._make_controller, _m._op cc_mod._ASYNC_BACKUP_SUBMIT = True cc_mod._ASYNC_BACKUP_PINNED_BYTES_MAX = 1 << 30 torch.cuda.synchronize() sched = torch.cuda.current_stream() fwd = torch.cuda.Stream() indices = torch.arange(4 * 1024 * 1024, dtype=torch.int64, device="cuda") origin = torch.cuda.Event(enable_timing=True) with torch.cuda.stream(sched): origin.record() torch.cuda._sleep(1_000_000_000) fence_done = torch.cuda.Event(enable_timing=True) fence_done.record() c = _make_controller() op = _op(indices, 0) t0 = time.perf_counter() c._start_async_index_copy(op) host_ms = (time.perf_counter() - t0) * 1e3 copy_end = torch.cuda.Event(enable_timing=True) with torch.cuda.stream(c._indices_stream): copy_end.record() t_launch = time.monotonic_ns() with torch.cuda.stream(fwd): fwd.wait_stream(sched) fwd_start = torch.cuda.Event(enable_timing=True) fwd_start.record() torch.cuda._sleep(1_500_000_000) fwd_end = torch.cuda.Event(enable_timing=True) fwd_end.record() ready_ns = [None] def _w(): op.async_parts.ready_event.synchronize() ready_ns[0] = time.monotonic_ns() th = threading.Thread(target=_w) th.start() th.join() torch.cuda.synchronize() print(f"host start_async_index_copy: {host_ms:.2f} ms") print(f"host next-batch enqueue before ready: {t_launch < ready_ns[0]}") print(f"fence_done -> copy_end : {fence_done.elapsed_time(copy_end):8.2f} ms") print(f"origin -> fwd_start : {origin.elapsed_time(fwd_start):8.2f} ms") print(f"origin -> copy_end : {origin.elapsed_time(copy_end):8.2f} ms") print(f"origin -> fwd_end : {origin.elapsed_time(fwd_end):8.2f} ms") print(f"fwd duration : {fwd_start.elapsed_time(fwd_end):8.2f} ms") print( f"fwd_start -> copy_end : {fwd_start.elapsed_time(copy_end):8.2f} ms (>0 means overlap)" ) print( f"copy_end -> fwd_end : {copy_end.elapsed_time(fwd_end):8.2f} ms (>0 means copy finished inside the forward)" )