Net async change (from exp-async-backup-perop-20261107) applied onto b300-pick-46884-20261107. Runtime files applied cleanly; the new tests and the timeline probe are added as-is. Interaction to validate on hardware: #46884 publishes prefill read-done before the replay, so the scheduler-stream fence that the index copy waits on is shorter; the async worker's wait should shrink and the copy may overlap more of the in-flight forward. Re-baseline the 4-arm A/B on this branch.
77 lines
2.4 KiB
Python
77 lines
2.4 KiB
Python
"""Print the CUDA timeline for one async index copy vs the next forward."""
|
|
import importlib.util as _u
|
|
import threading
|
|
import time
|
|
|
|
import torch
|
|
|
|
from sglang.srt.managers import cache_controller as cc_mod
|
|
|
|
_spec = _u.spec_from_file_location(
|
|
"t", "/data/ymk/wt-async/test/registered/cuda/test_async_backup_index_copy_cuda.py"
|
|
)
|
|
_m = _u.module_from_spec(_spec)
|
|
_spec.loader.exec_module(_m)
|
|
_make_controller, _op = _m._make_controller, _m._op
|
|
|
|
cc_mod._ASYNC_BACKUP_SUBMIT = True
|
|
cc_mod._ASYNC_BACKUP_PINNED_BYTES_MAX = 1 << 30
|
|
|
|
torch.cuda.synchronize()
|
|
sched = torch.cuda.current_stream()
|
|
fwd = torch.cuda.Stream()
|
|
indices = torch.arange(4 * 1024 * 1024, dtype=torch.int64, device="cuda")
|
|
|
|
origin = torch.cuda.Event(enable_timing=True)
|
|
with torch.cuda.stream(sched):
|
|
origin.record()
|
|
torch.cuda._sleep(1_000_000_000)
|
|
fence_done = torch.cuda.Event(enable_timing=True)
|
|
fence_done.record()
|
|
|
|
c = _make_controller()
|
|
op = _op(indices, 0)
|
|
t0 = time.perf_counter()
|
|
c._start_async_index_copy(op)
|
|
host_ms = (time.perf_counter() - t0) * 1e3
|
|
|
|
copy_end = torch.cuda.Event(enable_timing=True)
|
|
with torch.cuda.stream(c._indices_stream):
|
|
copy_end.record()
|
|
|
|
t_launch = time.monotonic_ns()
|
|
with torch.cuda.stream(fwd):
|
|
fwd.wait_stream(sched)
|
|
fwd_start = torch.cuda.Event(enable_timing=True)
|
|
fwd_start.record()
|
|
torch.cuda._sleep(1_500_000_000)
|
|
fwd_end = torch.cuda.Event(enable_timing=True)
|
|
fwd_end.record()
|
|
|
|
ready_ns = [None]
|
|
|
|
|
|
def _w():
|
|
op.async_parts.ready_event.synchronize()
|
|
ready_ns[0] = time.monotonic_ns()
|
|
|
|
|
|
th = threading.Thread(target=_w)
|
|
th.start()
|
|
th.join()
|
|
torch.cuda.synchronize()
|
|
|
|
print(f"host start_async_index_copy: {host_ms:.2f} ms")
|
|
print(f"host next-batch enqueue before ready: {t_launch < ready_ns[0]}")
|
|
print(f"fence_done -> copy_end : {fence_done.elapsed_time(copy_end):8.2f} ms")
|
|
print(f"origin -> fwd_start : {origin.elapsed_time(fwd_start):8.2f} ms")
|
|
print(f"origin -> copy_end : {origin.elapsed_time(copy_end):8.2f} ms")
|
|
print(f"origin -> fwd_end : {origin.elapsed_time(fwd_end):8.2f} ms")
|
|
print(f"fwd duration : {fwd_start.elapsed_time(fwd_end):8.2f} ms")
|
|
print(
|
|
f"fwd_start -> copy_end : {fwd_start.elapsed_time(copy_end):8.2f} ms (>0 means overlap)"
|
|
)
|
|
print(
|
|
f"copy_end -> fwd_end : {copy_end.elapsed_time(fwd_end):8.2f} ms (>0 means copy finished inside the forward)"
|
|
)
|