Files
sglang/timeline_probe.py
minke.yu d6f1e1efbd rebase(hicache): per-op async backup on top of #46884 + #48782
Net async change (from exp-async-backup-perop-20261107) applied onto
b300-pick-46884-20261107. Runtime files applied cleanly; the new tests and
the timeline probe are added as-is.

Interaction to validate on hardware: #46884 publishes prefill read-done before
the replay, so the scheduler-stream fence that the index copy waits on is
shorter; the async worker's wait should shrink and the copy may overlap more of
the in-flight forward. Re-baseline the 4-arm A/B on this branch.
2026-11-07 22:04:33 +08:00

77 lines
2.4 KiB
Python

"""Print the CUDA timeline for one async index copy vs the next forward."""
import importlib.util as _u
import threading
import time
import torch
from sglang.srt.managers import cache_controller as cc_mod
_spec = _u.spec_from_file_location(
"t", "/data/ymk/wt-async/test/registered/cuda/test_async_backup_index_copy_cuda.py"
)
_m = _u.module_from_spec(_spec)
_spec.loader.exec_module(_m)
_make_controller, _op = _m._make_controller, _m._op
cc_mod._ASYNC_BACKUP_SUBMIT = True
cc_mod._ASYNC_BACKUP_PINNED_BYTES_MAX = 1 << 30
torch.cuda.synchronize()
sched = torch.cuda.current_stream()
fwd = torch.cuda.Stream()
indices = torch.arange(4 * 1024 * 1024, dtype=torch.int64, device="cuda")
origin = torch.cuda.Event(enable_timing=True)
with torch.cuda.stream(sched):
origin.record()
torch.cuda._sleep(1_000_000_000)
fence_done = torch.cuda.Event(enable_timing=True)
fence_done.record()
c = _make_controller()
op = _op(indices, 0)
t0 = time.perf_counter()
c._start_async_index_copy(op)
host_ms = (time.perf_counter() - t0) * 1e3
copy_end = torch.cuda.Event(enable_timing=True)
with torch.cuda.stream(c._indices_stream):
copy_end.record()
t_launch = time.monotonic_ns()
with torch.cuda.stream(fwd):
fwd.wait_stream(sched)
fwd_start = torch.cuda.Event(enable_timing=True)
fwd_start.record()
torch.cuda._sleep(1_500_000_000)
fwd_end = torch.cuda.Event(enable_timing=True)
fwd_end.record()
ready_ns = [None]
def _w():
op.async_parts.ready_event.synchronize()
ready_ns[0] = time.monotonic_ns()
th = threading.Thread(target=_w)
th.start()
th.join()
torch.cuda.synchronize()
print(f"host start_async_index_copy: {host_ms:.2f} ms")
print(f"host next-batch enqueue before ready: {t_launch < ready_ns[0]}")
print(f"fence_done -> copy_end : {fence_done.elapsed_time(copy_end):8.2f} ms")
print(f"origin -> fwd_start : {origin.elapsed_time(fwd_start):8.2f} ms")
print(f"origin -> copy_end : {origin.elapsed_time(copy_end):8.2f} ms")
print(f"origin -> fwd_end : {origin.elapsed_time(fwd_end):8.2f} ms")
print(f"fwd duration : {fwd_start.elapsed_time(fwd_end):8.2f} ms")
print(
f"fwd_start -> copy_end : {fwd_start.elapsed_time(copy_end):8.2f} ms (>0 means overlap)"
)
print(
f"copy_end -> fwd_end : {copy_end.elapsed_time(fwd_end):8.2f} ms (>0 means copy finished inside the forward)"
)