rebase(hicache): per-op async backup on top of #46884 + #48782

Net async change (from exp-async-backup-perop-20261107) applied onto
b300-pick-46884-20261107. Runtime files applied cleanly; the new tests and
the timeline probe are added as-is.

Interaction to validate on hardware: #46884 publishes prefill read-done before
the replay, so the scheduler-stream fence that the index copy waits on is
shorter; the async worker's wait should shrink and the copy may overlap more of
the in-flight forward. Re-baseline the 4-arm A/B on this branch.
This commit is contained in:
2026-11-07 22:04:33 +08:00
parent 520ca548ea
commit d6f1e1efbd
8 changed files with 1434 additions and 99 deletions
+488 -83
View File
@@ -18,7 +18,7 @@ import logging
import threading import threading
import time import time
from dataclasses import dataclass from dataclasses import dataclass
from queue import Empty, Queue from queue import Empty, Full, Queue
from typing import TYPE_CHECKING, Any, Callable, List, NamedTuple, Optional from typing import TYPE_CHECKING, Any, Callable, List, NamedTuple, Optional
import torch import torch
@@ -42,6 +42,7 @@ from sglang.srt.layers.dp_attention import (
) )
from sglang.srt.mem_cache.l2_transfer import L2Transfer, L2TransferEngine from sglang.srt.mem_cache.l2_transfer import L2Transfer, L2TransferEngine
from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool
from sglang.srt.mem_cache.pool_host.base import uses_shared_host_layout
from sglang.srt.mem_cache.utils import get_storage_hash_str from sglang.srt.mem_cache.utils import get_storage_hash_str
from sglang.srt.observability import batch_telemetry from sglang.srt.observability import batch_telemetry
from sglang.srt.runtime_context import get_parallel from sglang.srt.runtime_context import get_parallel
@@ -69,6 +70,25 @@ _HC_PREFETCH_WORKERS = int(_hc_os.getenv("SGLANG_HICACHE_PREFETCH_IO_THREADS", "
_HC_BACKUP_WORKERS = int(_hc_os.getenv("SGLANG_HICACHE_BACKUP_IO_THREADS", "1") or 1) _HC_BACKUP_WORKERS = int(_hc_os.getenv("SGLANG_HICACHE_BACKUP_IO_THREADS", "1") or 1)
_HC_TIMING = _hc_os.getenv("SGLANG_HICACHE_TIMING", "0") not in ("", "0", "false", "False") _HC_TIMING = _hc_os.getenv("SGLANG_HICACHE_TIMING", "0") not in ("", "0", "false", "False")
# [ASYNC_BACKUP] Per-op pinned index copy + worker-side merge/submit.
# Only the direct + page_first_direct shape (the one that does a blocking
# device_indices.cpu() on the scheduler thread) is eligible.
_ASYNC_BACKUP_SUBMIT = _hc_os.getenv(
"SGLANG_ASYNC_BACKUP_SUBMIT", "0"
).strip().lower() not in ("", "0", "false", "no", "off")
_ASYNC_BACKUP_ENQUEUE_TIMEOUT = float(
_hc_os.getenv("SGLANG_ASYNC_BACKUP_ENQUEUE_TIMEOUT", "60") or 60
)
_ASYNC_BACKUP_IDLE_TIMEOUT = float(
_hc_os.getenv("SGLANG_ASYNC_BACKUP_IDLE_TIMEOUT", "120") or 120
)
_ASYNC_BACKUP_PINNED_BYTES_MAX = int(
_hc_os.getenv("SGLANG_ASYNC_BACKUP_PINNED_MB", "2048") or 2048
) * 1024 * 1024
_ASYNC_BACKUP_MAX_OPS = int(
_hc_os.getenv("SGLANG_ASYNC_BACKUP_MAX_OPS", "256") or 256
)
class LayerLoadingEvent: class LayerLoadingEvent:
def __init__(self, num_layers: int): def __init__(self, num_layers: int):
self._num_layers = num_layers self._num_layers = num_layers
@@ -132,6 +152,9 @@ class CacheOperation:
self.node_ids = [node_id] self.node_ids = [node_id]
self.data = None self.data = None
self.pool_transfers = pool_transfers self.pool_transfers = pool_transfers
# Set when SGLANG_ASYNC_BACKUP_SUBMIT copied this op's device indices
# into pinned host buffers (see _start_async_index_copy).
self.async_parts = None
self.id = CacheOperation.counter self.id = CacheOperation.counter
CacheOperation.counter += 1 CacheOperation.counter += 1
@@ -192,6 +215,39 @@ class CacheOperation:
return self.priority < other.priority return self.priority < other.priority
@dataclass
class AsyncIndexCopy:
"""One device->pinned-host index copy started where the backup op was made."""
key: tuple
src: torch.Tensor
dst: torch.Tensor
nbytes: int
@dataclass
class AsyncIndexParts:
"""Per-op pinned index parts plus the event that guarantees they are final."""
copies: List[AsyncIndexCopy]
ready_event: object
nbytes: int
@dataclass
class WriteSubmission:
ops: List[CacheOperation]
tracer: object
enqueue_mono_ns: int
@dataclass
class WriteResult:
ack: Optional[HiCacheAck]
ops: List[CacheOperation]
error: Optional[BaseException]
class HiCacheAck(NamedTuple): class HiCacheAck(NamedTuple):
start_event: device_module.Event start_event: device_module.Event
finish_event: device_module.Event finish_event: device_module.Event
@@ -383,6 +439,21 @@ class HiCacheController:
self.write_fence_stream = None self.write_fence_stream = None
self.ack_write_queue: List[HiCacheAck] = [] self.ack_write_queue: List[HiCacheAck] = []
# [ASYNC_BACKUP] Pinned index copies are started where the backup op is
# created; a single worker waits the per-op events, merges the CPU
# index parts and submits the KV copy. Queues are bounded by the state
# lock/condition below; write_queue/ack_write_queue stay scheduler-only.
self._indices_stream = None
self._write_submit_queue: Optional[Queue] = None
self._write_result_queue: Queue = Queue()
self._write_submit_stop = threading.Event()
self._write_submit_thread: Optional[threading.Thread] = None
self._write_submit_state_lock = threading.Lock()
self._write_submit_cond = threading.Condition(self._write_submit_state_lock)
self._write_submit_active = 0
self._async_pinned_bytes = 0
self._retire_parts: List["AsyncIndexParts"] = []
self.l2_transfer_engine = L2TransferEngine(io_backend) self.l2_transfer_engine = L2TransferEngine(io_backend)
# If a storage backend is provided at startup, treat it as an implicit attach, # If a storage backend is provided at startup, treat it as an implicit attach,
@@ -708,6 +779,9 @@ class HiCacheController:
Requirement: no in-flight requests. This will stop storage threads and release Requirement: no in-flight requests. This will stop storage threads and release
the backend instance (best-effort close). the backend instance (best-effort close).
""" """
# Async backup submissions own pinned index buffers and ACKs; stop them
# before any queue/pool teardown so no result is lost.
self._stop_write_submission_thread()
# Idempotent cleanup: even if `enable_storage` is already False, # Idempotent cleanup: even if `enable_storage` is already False,
# we may still have leftover resources (threads/backend/process group) from a # we may still have leftover resources (threads/backend/process group) from a
# previous partial detach. We attempt cleanup whenever possible. # previous partial detach. We attempt cleanup whenever possible.
@@ -822,6 +896,7 @@ class HiCacheController:
def reset(self): def reset(self):
# Reuse detach's queue wakeups and bounded joins, and fail if any # Reuse detach's queue wakeups and bounded joins, and fail if any
# storage thread remains alive before clearing shared state. # storage thread remains alive before clearing shared state.
self._stop_write_submission_thread()
self._stop_storage_threads() self._stop_storage_threads()
self.write_queue.clear() self.write_queue.clear()
@@ -861,13 +936,311 @@ class HiCacheController:
def has_inflight_device_transfers(self) -> bool: def has_inflight_device_transfers(self) -> bool:
"""Whether queued or unacknowledged L2 transfers still use device rows.""" """Whether queued or unacknowledged L2 transfers still use device rows."""
with self._write_submit_state_lock:
async_pending = (
self._write_submit_active > 0
or self._async_pinned_bytes > 0
or not self._write_result_queue.empty()
)
return bool( return bool(
self.write_queue self.write_queue
or self.load_queue or self.load_queue
or self.ack_write_queue or self.ack_write_queue
or self.ack_load_queue or self.ack_load_queue
or async_pending
) )
# ---- async backup submission -----------------------------------------
def async_backup_enabled(self) -> bool:
return (
_ASYNC_BACKUP_SUBMIT
and self.io_backend == "direct"
and getattr(self.mem_pool_host, "layout", None) == "page_first_direct"
)
def _async_submit_applicable(self) -> bool:
return (
self.async_backup_enabled()
and len(self.write_queue) <= _ASYNC_BACKUP_MAX_OPS
and all(op.async_parts is not None for op in self.write_queue)
)
def _ensure_indices_stream(self):
if self._indices_stream is None:
self._indices_stream = device_module.Stream()
return self._indices_stream
def _needs_device_index_copy(self, host_pool, indices) -> bool:
if indices is None or not getattr(indices, "is_cuda", False):
return False
# move_indices() returns shared-layout ids untouched; converting them
# here would hand the transfer a different interface than the sync path.
return not uses_shared_host_layout(host_pool)
def _async_index_targets(self, op: CacheOperation):
targets = []
if self._needs_device_index_copy(
self.mem_pool_host.anchor_entry.host_pool, op.device_indices
):
targets.append(((None, None), op.device_indices))
for transfer in op.pool_transfers or []:
if transfer.device_indices is None:
continue
entry = self.mem_pool_host.entry_map.get(transfer.name)
if entry is None:
continue
if not self._needs_device_index_copy(
entry.host_pool, transfer.device_indices
):
continue
key = (transfer.name, transfer.indices_from_pool)
targets.append((key, transfer.device_indices))
return targets
def _start_async_index_copy(self, op: CacheOperation) -> None:
"""Start the pinned index D2H copy now, off the scheduler stream.
Runs in the scheduler thread at backup-op creation time. The copy runs
on a dedicated stream ordered after this instant, so it does not inherit
later fences (the WAR barrier of a later forward) and the scheduler
never blocks in ``.cpu()``.
"""
if not self.async_backup_enabled():
return
targets = self._async_index_targets(op)
if not targets:
return
nbytes = sum(int(t.numel()) * int(t.element_size()) for _, t in targets)
with self._write_submit_state_lock:
if self._async_pinned_bytes + nbytes > _ASYNC_BACKUP_PINNED_BYTES_MAX:
return
self._async_pinned_bytes += nbytes
try:
stream = self._ensure_indices_stream()
source_stream = device_module.current_stream()
copies: List[AsyncIndexCopy] = []
with device_module.StreamContext(stream):
stream.wait_stream(source_stream)
for key, src in targets:
# Keep the device rows alive until the copy has run, and pin
# the destination so copy_ is a real asynchronous D2H.
src.record_stream(stream)
dst = torch.empty(
src.shape,
dtype=src.dtype,
device="cpu",
pin_memory=True,
)
dst.copy_(src, non_blocking=True)
copies.append(
AsyncIndexCopy(
key=key,
src=src,
dst=dst,
nbytes=int(src.numel()) * int(src.element_size()),
)
)
ready_event = device_module.Event()
ready_event.record(stream)
except BaseException:
with self._write_submit_state_lock:
self._async_pinned_bytes -= nbytes
logger.exception("Async backup index copy failed; keeping sync path")
return
op.async_parts = AsyncIndexParts(
copies=copies, ready_event=ready_event, nbytes=nbytes
)
def _ensure_write_submit_thread(self) -> None:
if self._write_submit_queue is None:
self._write_submit_queue = Queue(maxsize=64)
thread = self._write_submit_thread
if thread is not None and thread.is_alive():
return
self._write_submit_stop.clear()
thread = threading.Thread(
target=self._write_submit_worker,
name="hicache-async-backup",
daemon=True,
)
self._write_submit_thread = thread
thread.start()
def _enqueue_async_write(self, ops, tracer) -> bool:
"""Enqueue one submission; False means "fall back to the sync path".
A full queue or a timeout is not fatal: the caller submits the same ops
synchronously. Before returning False we wait for the worker to become
idle so the fallback cannot race another submission on the shared
device_to_host_stream.
"""
self._ensure_write_submit_thread()
item = WriteSubmission(
ops=ops, tracer=tracer, enqueue_mono_ns=time.monotonic_ns()
)
with self._write_submit_cond:
self._write_submit_active += 1
try:
with batch_telemetry.backup_phase("enqueue"):
self._write_submit_queue.put(
item, timeout=_ASYNC_BACKUP_ENQUEUE_TIMEOUT
)
except Full:
with self._write_submit_cond:
self._write_submit_active -= 1
self._write_submit_cond.notify_all()
logger.warning(
"Async backup submit queue full after %.1fs; falling back to "
"the synchronous path",
_ASYNC_BACKUP_ENQUEUE_TIMEOUT,
)
self.wait_write_submissions_idle()
return False
return True
def _materialize_async_indices(self, ops) -> None:
"""Replace device index tensors with the pinned CPU copies (worker)."""
for op in ops:
parts = op.async_parts
if parts is None:
continue
mapping = {copy.key: copy.dst for copy in parts.copies}
anchor = mapping.get((None, None))
if anchor is not None:
op.device_indices = anchor
for transfer in op.pool_transfers or []:
dst = mapping.get((transfer.name, transfer.indices_from_pool))
if dst is not None:
transfer.device_indices = dst
def _write_submit_worker(self) -> None:
while True:
item = self._write_submit_queue.get()
if item is None:
break
tracer = item.tracer
previous = batch_telemetry.backup_flush_attach(tracer)
try:
with batch_telemetry.backup_phase("worker"):
for op in item.ops:
parts = op.async_parts
if parts is not None:
parts.ready_event.synchronize()
self._materialize_async_indices(item.ops)
merged = CacheOperation.merge_ops(item.ops)
ack = self._submit_write_operation(merged)
result = WriteResult(ack=ack, ops=item.ops, error=None)
except BaseException as exc: # noqa: BLE001 - fail closed
logger.exception("Async backup submission failed")
result = WriteResult(ack=None, ops=item.ops, error=exc)
finally:
batch_telemetry.backup_flush_detach(previous)
self._release_async_parts(item.ops)
try:
batch_telemetry.backup_flush_end(tracer)
except Exception:
pass
# Publish the result before marking the submission done, so an idle
# waiter cannot observe active==0 with the result still missing.
self._write_result_queue.put(result)
with self._write_submit_cond:
self._write_submit_active -= 1
self._write_submit_cond.notify_all()
def _release_async_parts(self, ops) -> None:
released = 0
for op in ops:
parts = op.async_parts
if parts is None:
continue
op.async_parts = None
parts.copies.clear()
released += parts.nbytes
if released:
with self._write_submit_state_lock:
self._async_pinned_bytes -= released
def _retire_async_parts(self, ops) -> None:
"""Release copies that will not be consumed (sync fallback)."""
for op in ops:
parts = op.async_parts
if parts is not None:
self._retire_parts.append(parts)
op.async_parts = None
self._reap_retired_parts()
def _reap_retired_parts(self) -> None:
if not self._retire_parts:
return
pending = []
for parts in self._retire_parts:
try:
done = parts.ready_event.query()
except Exception:
done = True
if done:
parts.copies.clear()
with self._write_submit_state_lock:
self._async_pinned_bytes -= parts.nbytes
else:
pending.append(parts)
self._retire_parts = pending
def drain_write_submission_results(self) -> None:
"""Move finished async ACKs into ack_write_queue (scheduler thread)."""
while True:
try:
result = self._write_result_queue.get_nowait()
except Empty:
break
if result.error is not None:
raise RuntimeError("async backup submission failed") from result.error
if result.ack is not None:
self.ack_write_queue.append(result.ack)
self._reap_retired_parts()
def wait_write_submissions_idle(
self, timeout: float = _ASYNC_BACKUP_IDLE_TIMEOUT
) -> None:
deadline = time.monotonic() + timeout
with self._write_submit_cond:
while self._write_submit_active > 0:
remaining = deadline - time.monotonic()
if remaining <= 0:
raise RuntimeError(
"Timed out waiting for async backup submissions"
)
self._write_submit_cond.wait(remaining)
def _stop_write_submission_thread(self) -> None:
if self._write_submit_thread is None:
self.drain_write_submission_results()
self._reap_retired_parts()
if self._retire_parts:
raise RuntimeError("async backup index copies still in flight")
return
try:
self.wait_write_submissions_idle()
finally:
self._write_submit_stop.set()
queue = self._write_submit_queue
if queue is not None:
try:
queue.put(None, timeout=5.0)
except Exception:
logger.exception("Failed to wake the async backup worker")
thread = self._write_submit_thread
if thread is not None:
thread.join(timeout=30.0)
if thread.is_alive():
raise RuntimeError("async backup submit worker did not stop")
self.drain_write_submission_results()
if self._retire_parts:
raise RuntimeError("async backup index copies still in flight")
self._write_submit_thread = None
self._write_submit_queue = None
def write( def write(
self, self,
device_indices: torch.Tensor, device_indices: torch.Tensor,
@@ -883,9 +1256,9 @@ class HiCacheController:
host_indices = self.mem_pool_host.alloc(len(device_indices)) host_indices = self.mem_pool_host.alloc(len(device_indices))
if host_indices is None: if host_indices is None:
return None return None
self.write_queue.append( op = CacheOperation(host_indices, device_indices, node_id, priority)
CacheOperation(host_indices, device_indices, node_id, priority) self._start_async_index_copy(op)
) self.write_queue.append(op)
self.start_writing() self.start_writing()
return host_indices return host_indices
@@ -898,98 +1271,130 @@ class HiCacheController:
batch_telemetry.detail_count("flush_empty") batch_telemetry.detail_count("flush_empty")
return return
ops = list(self.write_queue)
tracer = batch_telemetry.backup_flush_begin() tracer = batch_telemetry.backup_flush_begin()
async_owned = False
try: try:
batch_telemetry.detail_count("flush_calls") batch_telemetry.detail_count("flush_calls")
batch_telemetry.detail_count("flush_nonempty") batch_telemetry.detail_count("flush_nonempty")
batch_telemetry.flush_note( self._note_write_queue(ops, len(self.ack_write_queue))
"sglang.flush_write_queue_ops", len(self.write_queue)
)
batch_telemetry.flush_note("sglang.flush_io_backend", self.io_backend)
batch_telemetry.flush_note(
"sglang.flush_layout",
str(getattr(self.mem_pool_host, "layout", None) or ""),
)
batch_telemetry.flush_note(
"sglang.flush_ack_queue_before", len(self.ack_write_queue)
)
if self._async_submit_applicable():
if self._enqueue_async_write(ops, tracer):
self.write_queue.clear()
batch_telemetry.flush_note(
"sglang.flush_submission_mode", "async"
)
# The worker owns the tracer from here on (it emits once).
async_owned = True
return
# Queue full / enqueue timeout: retire the pinned copies and
# submit synchronously on this thread. `_enqueue_async_write`
# waited for the worker to go idle first, so the two paths
# never submit to device_to_host_stream concurrently.
batch_telemetry.flush_note("sglang.flush_fallback_sync", 1)
self._retire_async_parts(ops)
# Any parts that were started for this queue are not consumed by the
# sync path. Retire them unconditionally (they are released once
# their events complete) so the pinned-byte budget never leaks.
batch_telemetry.flush_note(
"sglang.flush_async_skip_ops",
sum(1 for op in ops if op.async_parts is None),
)
self._retire_async_parts(ops)
batch_telemetry.flush_note("sglang.flush_submission_mode", "sync")
with batch_telemetry.backup_phase("merge_ops"): with batch_telemetry.backup_phase("merge_ops"):
op = CacheOperation.merge_ops(self.write_queue) op = CacheOperation.merge_ops(self.write_queue)
tokens = len(op.device_indices)
indices = op.device_indices
batch_telemetry.flush_note("sglang.flush_tokens", int(tokens))
batch_telemetry.flush_note(
"sglang.flush_nodes_merged", len(op.node_ids)
)
batch_telemetry.flush_note(
"sglang.flush_sidecar_pools", len(op.pool_transfers or [])
)
batch_telemetry.flush_note(
"sglang.flush_indices_device", str(indices.device.type)
)
batch_telemetry.flush_note(
"sglang.flush_indices_dtype", str(indices.dtype)
)
batch_telemetry.flush_note(
"sglang.flush_indices_numel", int(indices.numel())
)
with batch_telemetry.backup_phase("move_write"):
host_indices, device_indices, pool_transfers = (
self._move_write_operation(op)
)
self.write_queue.clear() self.write_queue.clear()
batch_telemetry.flush_note( ack = self._submit_write_operation(op)
"sglang.flush_host_indices_on_cpu",
bool(getattr(host_indices, "is_cuda", False) is False),
)
batch_telemetry.flush_note(
"sglang.flush_device_indices_on_cpu",
bool(getattr(device_indices, "is_cuda", False) is False),
)
if self.write_fence_stream is not None:
# The copy may read KV a forward is still writing: under overlap
# scheduling a finished request is cached while the next forward
# writes its last token, and an evicted SWA page can belong to an
# unfinished checkpoint. Later forwards still overlap with the copy.
with batch_telemetry.backup_phase("write_fence"):
self.l2_transfer_engine.device_to_host_stream.wait_stream(
self.write_fence_stream
)
with batch_telemetry.backup_phase("build_l2"):
transfers = self._l2_transfers(
host_indices, device_indices, pool_transfers
)
batch_telemetry.flush_note("sglang.flush_l2_transfers", len(transfers))
with batch_telemetry.backup_phase("submit_d2h"):
completion = self.l2_transfer_engine.submit_device_to_host(transfers)
with batch_telemetry.backup_phase("set_done_event"):
self.mem_pool_device_allocator.set_hicache_transfer_done_event(
(id(self), "write"), completion.finish_event
)
with batch_telemetry.backup_phase("ack_enqueue"): with batch_telemetry.backup_phase("ack_enqueue"):
self.ack_write_queue.append( self.ack_write_queue.append(ack)
HiCacheAck(
start_event=completion.start_event,
finish_event=completion.finish_event,
node_ids=op.node_ids,
num_tokens=len(op.device_indices),
timing_enabled=completion.timing_enabled,
num_tokens_by_pool=self._num_tokens_by_pool(op),
num_bytes=self._transfer_num_bytes(op),
)
)
batch_telemetry.flush_note( batch_telemetry.flush_note(
"sglang.flush_ack_queue_after", len(self.ack_write_queue) "sglang.flush_ack_queue_after", len(self.ack_write_queue)
) )
finally: finally:
batch_telemetry.backup_flush_end(tracer) if not async_owned:
batch_telemetry.backup_flush_end(tracer)
def _note_write_queue(self, ops: List[CacheOperation], ack_before: int) -> None:
"""Queue-level flush dimensions (same for the sync and async paths)."""
batch_telemetry.flush_note("sglang.flush_write_queue_ops", len(ops))
batch_telemetry.flush_note("sglang.flush_io_backend", self.io_backend)
batch_telemetry.flush_note(
"sglang.flush_layout",
str(getattr(self.mem_pool_host, "layout", None) or ""),
)
batch_telemetry.flush_note("sglang.flush_ack_queue_before", ack_before)
def _submit_write_operation(self, op: CacheOperation) -> HiCacheAck:
"""Move indices, submit the D2H copy and build the ACK.
Runs on the scheduler thread (sync path) or on the async submit worker
(async path, with CPU-resident indices). It must not touch
``write_queue`` or ``ack_write_queue``; the caller owns those.
"""
tokens = len(op.device_indices)
indices = op.device_indices
batch_telemetry.flush_note("sglang.flush_tokens", int(tokens))
batch_telemetry.flush_note("sglang.flush_nodes_merged", len(op.node_ids))
batch_telemetry.flush_note(
"sglang.flush_sidecar_pools", len(op.pool_transfers or [])
)
batch_telemetry.flush_note(
"sglang.flush_indices_device", str(indices.device.type)
)
batch_telemetry.flush_note("sglang.flush_indices_dtype", str(indices.dtype))
batch_telemetry.flush_note(
"sglang.flush_indices_numel", int(indices.numel())
)
with batch_telemetry.backup_phase("move_write"):
host_indices, device_indices, pool_transfers = (
self._move_write_operation(op)
)
batch_telemetry.flush_note(
"sglang.flush_host_indices_on_cpu",
bool(getattr(host_indices, "is_cuda", False) is False),
)
batch_telemetry.flush_note(
"sglang.flush_device_indices_on_cpu",
bool(getattr(device_indices, "is_cuda", False) is False),
)
if self.write_fence_stream is not None:
# The copy may read KV a forward is still writing: under overlap
# scheduling a finished request is cached while the next forward
# writes its last token, and an evicted SWA page can belong to an
# unfinished checkpoint. Later forwards still overlap with the copy.
with batch_telemetry.backup_phase("write_fence"):
self.l2_transfer_engine.device_to_host_stream.wait_stream(
self.write_fence_stream
)
with batch_telemetry.backup_phase("build_l2"):
transfers = self._l2_transfers(
host_indices, device_indices, pool_transfers
)
batch_telemetry.flush_note("sglang.flush_l2_transfers", len(transfers))
with batch_telemetry.backup_phase("submit_d2h"):
completion = self.l2_transfer_engine.submit_device_to_host(transfers)
with batch_telemetry.backup_phase("set_done_event"):
self.mem_pool_device_allocator.set_hicache_transfer_done_event(
(id(self), "write"), completion.finish_event
)
return HiCacheAck(
start_event=completion.start_event,
finish_event=completion.finish_event,
node_ids=op.node_ids,
num_tokens=len(op.device_indices),
timing_enabled=completion.timing_enabled,
num_tokens_by_pool=self._num_tokens_by_pool(op),
num_bytes=self._transfer_num_bytes(op),
)
def _transfer_num_bytes(self, op: CacheOperation) -> int: def _transfer_num_bytes(self, op: CacheOperation) -> int:
return len(op.device_indices) * self.mem_pool_host.size_per_token return len(op.device_indices) * self.mem_pool_host.size_per_token
@@ -25,6 +25,7 @@ from __future__ import annotations
import inspect import inspect
import logging import logging
import os import os
import threading
from typing import ( from typing import (
Callable, Callable,
Dict, Dict,
@@ -440,6 +441,9 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
# site; `_flush` materializes the write-set lazily, avoiding a sync here. # site; `_flush` materializes the write-set lazily, avoiding a sync here.
self._inflight_forward: Optional[Tuple[torch.cuda.Event, torch.Tensor]] = None self._inflight_forward: Optional[Tuple[torch.cuda.Event, torch.Tensor]] = None
self._hicache_transfer_done_events: Dict[Hashable, torch.cuda.Event] = {} self._hicache_transfer_done_events: Dict[Hashable, torch.cuda.Event] = {}
# The async backup submit worker records transfer events while the
# scheduler may concurrently iterate/clear them for compaction gating.
self._hicache_transfer_done_events_lock = threading.Lock()
# HiCache reserves SWA physical pages during load-back, then binds them # HiCache reserves SWA physical pages during load-back, then binds them
# into the tree before the H2D copy is submitted. Compaction must not # into the tree before the H2D copy is submitted. Compaction must not
# move those pages in that interval because the queued PoolTransfer still # move those pages in that interval because the queued PoolTransfer still
@@ -557,7 +561,8 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
self.live_page_count = 0 self.live_page_count = 0
self._inflight_forward = None self._inflight_forward = None
self._latest_forward_done_event = None self._latest_forward_done_event = None
self._hicache_transfer_done_events.clear() with self._hicache_transfer_done_events_lock:
self._hicache_transfer_done_events.clear()
self._pending_hicache_load_pages = 0 self._pending_hicache_load_pages = 0
def clear_inverse_history(self) -> None: def clear_inverse_history(self) -> None:
@@ -2083,7 +2088,8 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
_pending_hicache_load_pages: _CapacityField[int] = _CapacityField() _pending_hicache_load_pages: _CapacityField[int] = _CapacityField()
def set_hicache_transfer_done_event(self, transfer_key: Hashable, event) -> None: def set_hicache_transfer_done_event(self, transfer_key: Hashable, event) -> None:
self._hicache_transfer_done_events[transfer_key] = event with self._hicache_transfer_done_events_lock:
self._hicache_transfer_done_events[transfer_key] = event
if transfer_key == "load" or ( if transfer_key == "load" or (
isinstance(transfer_key, tuple) and transfer_key[-1] == "load" isinstance(transfer_key, tuple) and transfer_key[-1] == "load"
): ):
@@ -2095,11 +2101,12 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
"""Order the current stream after outstanding HiCache copies. Only moves """Order the current stream after outstanding HiCache copies. Only moves
and frees need this: a copy's source and target are not free pages until and frees need this: a copy's source and target are not free pages until
its finish event is observed, so allocation never reuses them.""" its finish event is observed, so allocation never reuses them."""
if not self._hicache_transfer_done_events: with self._hicache_transfer_done_events_lock:
return if not self._hicache_transfer_done_events:
return
events = tuple(self._hicache_transfer_done_events.values())
self._hicache_transfer_done_events.clear()
current_stream = torch.cuda.current_stream() current_stream = torch.cuda.current_stream()
events = tuple(self._hicache_transfer_done_events.values())
self._hicache_transfer_done_events.clear()
for event in events: for event in events:
current_stream.wait_event(event) current_stream.wait_event(event)
@@ -556,15 +556,17 @@ class HybridCacheController(BaseHiCacheController):
return None return None
host_indices, pool_transfers = allocation host_indices, pool_transfers = allocation
self.write_queue.append( op = CacheOperation(
CacheOperation( host_indices,
host_indices, device_indices,
device_indices, node_id,
node_id, priority,
priority, pool_transfers=pool_transfers or None,
pool_transfers=pool_transfers or None,
)
) )
# Start the pinned index D2H before the (possibly deferred) flush, so
# the copy can run while the scheduler keeps going.
self._start_async_index_copy(op)
self.write_queue.append(op)
if flush: if flush:
self.start_writing() self.start_writing()
return host_indices return host_indices
@@ -3505,6 +3505,13 @@ class UnifiedRadixCache(BasePrefixCache):
if write_back: if write_back:
# Blocking: submit what is still queued, then wait for every ack. # Blocking: submit what is still queued, then wait for every ack.
cc.start_writing() cc.start_writing()
# Async submissions cover ACKs asynchronously; wait for them before
# asserting that every write-through node has been acknowledged.
if hasattr(cc, "wait_write_submissions_idle"):
cc.wait_write_submissions_idle()
_drain = getattr(cc, "drain_write_submission_results", None)
if _drain is not None:
_drain()
while self.ongoing_write_through: while self.ongoing_write_through:
for ack in cc.ack_write_queue: for ack in cc.ack_write_queue:
ack.finish_event.synchronize() ack.finish_event.synchronize()
@@ -3517,6 +3524,9 @@ class UnifiedRadixCache(BasePrefixCache):
return return
if finish_count is None: if finish_count is None:
_drain = getattr(cc, "drain_write_submission_results", None)
if _drain is not None:
_drain()
# Every rank must enter the all_reduce below; ongoing_write_through can # Every rank must enter the all_reduce below; ongoing_write_through can
# diverge across ranks (e.g. write_backup returning 0 on a subset). # diverge across ranks (e.g. write_backup returning 0 on a subset).
finish_count = 0 finish_count = 0
@@ -3694,9 +3704,16 @@ class UnifiedRadixCache(BasePrefixCache):
# Reap the previous round's PP-sync sends before issuing new ones. # Reap the previous round's PP-sync sends before issuing new ones.
self._drain_async_work() self._drain_async_work()
# Collect ACKs finished by the async submit worker before counting them.
# getattr keeps lightweight controller fakes (unit tests) working.
_drain = getattr(self.cache_controller, "drain_write_submission_results", None)
if _drain is not None:
_drain()
# Backups queued outside process_batch_result: the chunked-prefill stash # Backups queued outside process_batch_result: the chunked-prefill stash
# in get_next_batch_to_run, abort_request, and the PD prefill release. # in get_next_batch_to_run, abort_request, and the PD prefill release.
self.flush_pending_backups() self.flush_pending_backups()
if _drain is not None:
_drain()
( (
write_finish_count, write_finish_count,
@@ -1409,6 +1409,8 @@ class BackupFlushTracer:
__slots__ = ( __slots__ = (
"origin", "origin",
"start_ns", "start_ns",
"loop_seq",
"loop_ctx",
"phases", "phases",
"attrs", "attrs",
"counts", "counts",
@@ -1418,6 +1420,10 @@ class BackupFlushTracer:
def __init__(self, origin: str): def __init__(self, origin: str):
self.origin = origin self.origin = origin
# Captured on the scheduler thread: the flushed span may be emitted by
# the async submit worker, which has no scheduler loop TLS.
self.loop_seq = current_loop_seq()
self.loop_ctx = current_loop_context_for_spans()
self.start_ns = time.monotonic_ns() self.start_ns = time.monotonic_ns()
self.phases: dict[str, int] = {} self.phases: dict[str, int] = {}
self.attrs: dict[str, Any] = {} self.attrs: dict[str, Any] = {}
@@ -1463,7 +1469,7 @@ class BackupFlushTracer:
attrs = { attrs = {
"sglang.flush_origin": self.origin, "sglang.flush_origin": self.origin,
"sglang.flush_total_ms": total_ns / 1e6, "sglang.flush_total_ms": total_ns / 1e6,
"sglang.loop_seq": current_loop_seq(), "sglang.loop_seq": int(self.loop_seq or 0),
} }
for name, ns in self.phases.items(): for name, ns in self.phases.items():
attrs["sglang.flush_" + name.replace(".", "_") + "_ms"] = ns / 1e6 attrs["sglang.flush_" + name.replace(".", "_") + "_ms"] = ns / 1e6
@@ -1476,7 +1482,7 @@ class BackupFlushTracer:
try: try:
from opentelemetry import trace as otel_trace from opentelemetry import trace as otel_trace
ctx = current_loop_context_for_spans() ctx = self.loop_ctx
start_wall_ns = time.time_ns() - total_ns start_wall_ns = time.time_ns() - total_ns
span = otel_trace.get_tracer("sglang.cache.flush").start_span( span = otel_trace.get_tracer("sglang.cache.flush").start_span(
"scheduler.cache_flush_backup", "scheduler.cache_flush_backup",
@@ -1543,6 +1549,25 @@ def current_flush_tracer() -> Optional[BackupFlushTracer]:
return getattr(_FLUSH_TLS, "tracer", None) return getattr(_FLUSH_TLS, "tracer", None)
def backup_flush_attach(tracer: Optional[BackupFlushTracer]):
"""Bind ``tracer`` to this thread so sub-phases land on the same span.
Used by the async submit worker, which executes phases on a different
thread than the one that created the tracer. Returns the previous binding.
"""
if not _BACKUP_DETAIL_ENABLED:
return None
previous = getattr(_FLUSH_TLS, "tracer", None)
_FLUSH_TLS.tracer = tracer
return previous
def backup_flush_detach(previous) -> None:
if not _BACKUP_DETAIL_ENABLED:
return
_FLUSH_TLS.tracer = previous
def flush_note(key: str, value: Any) -> None: def flush_note(key: str, value: Any) -> None:
"""Attach a dimension to the active flush (no-op when disabled).""" """Attach a dimension to the active flush (no-op when disabled)."""
if not _BACKUP_DETAIL_ENABLED: if not _BACKUP_DETAIL_ENABLED:
@@ -0,0 +1,253 @@
"""CUDA ordering/timeline tests for the per-op async backup index copy.
Plan: handoff/prefill-backup-overlap-next-20261107.md §3/§8.2.
Run inside the GPU container with the overlay source:
PYTHONPATH=/tmp/ovr python -m pytest \
test/registered/cuda/test_async_backup_index_copy_cuda.py -q
What is proven on real hardware:
1. ``_start_async_index_copy`` returns while a long kernel is still queued on
the scheduler stream, and the synchronous ``.cpu()`` on that stream blocks;
2. the copy is only released after that fence, but the *next* forward (a
marker kernel on another stream, ordered behind the same fence) runs
concurrently with it: the copy end timestamp falls strictly inside the
marker's [start, end] interval;
3. the host can enqueue the next batch before ``ready_event`` completes and
the event is not reported ready early;
4. with three ops whose copies complete out of order, the worker still merges
them in op order, identical to ``CacheOperation.merge_ops`` over the CPU
reference tensors;
5. the copy stream is distinct from the scheduler stream.
"""
import threading
import time
import unittest
import torch
from sglang.srt.managers import cache_controller as cc_mod
class _FakeHostPool:
"""Not a HostKVCache: uses_shared_host_layout() is false, i.e. the sync path
would call device_indices.cpu()."""
shared_allocation_domain = None
class _FakeEntry:
def __init__(self):
self.host_pool = _FakeHostPool()
class _FakeMemPoolHost:
layout = "page_first_direct"
def __init__(self):
self.anchor_entry = _FakeEntry()
self.entry_map = {}
def _make_controller():
c = cc_mod.HiCacheController.__new__(cc_mod.HiCacheController)
c.io_backend = "direct"
c.mem_pool_host = _FakeMemPoolHost()
c.write_queue = []
c.load_queue = []
c.ack_write_queue = []
c.ack_load_queue = []
c._indices_stream = None
c._write_submit_queue = None
c._write_result_queue = cc_mod.Queue()
c._write_submit_stop = threading.Event()
c._write_submit_thread = None
c._write_submit_state_lock = threading.Lock()
c._write_submit_cond = threading.Condition(c._write_submit_state_lock)
c._write_submit_active = 0
c._async_pinned_bytes = 0
c._retire_parts = []
return c
def _op(device_indices, node_id):
host = torch.empty(device_indices.numel(), dtype=torch.int64)
return cc_mod.CacheOperation(host, device_indices, node_id)
class AsyncBackupIndexCopyCudaTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
cc_mod._ASYNC_BACKUP_SUBMIT = True
cls.saved_pin = cc_mod._ASYNC_BACKUP_PINNED_BYTES_MAX
cc_mod._ASYNC_BACKUP_PINNED_BYTES_MAX = 1 << 30
@classmethod
def tearDownClass(cls):
cc_mod._ASYNC_BACKUP_PINNED_BYTES_MAX = cls.saved_pin
def test_start_does_not_block_and_copy_is_async(self):
torch.cuda.synchronize()
sched = torch.cuda.current_stream()
fwd = torch.cuda.Stream()
indices = torch.arange(4096, dtype=torch.int64, device="cuda")
# Simulated in-flight forward plus the scheduler's WAR fence.
with torch.cuda.stream(fwd):
torch.cuda._sleep(2_000_000_000)
sched.wait_stream(fwd)
c = _make_controller()
op = _op(indices, 11)
t0 = time.perf_counter()
c._start_async_index_copy(op)
start_elapsed = time.perf_counter() - t0
self.assertIsNotNone(op.async_parts)
# Starting the copy must not block on the simulated forward.
self.assertLess(start_elapsed, 0.05, start_elapsed)
self.assertFalse(op.async_parts.ready_event.query())
# The synchronous baseline on the same stream does block.
t1 = time.perf_counter()
reference = indices.cpu()
sync_elapsed = time.perf_counter() - t1
self.assertGreater(sync_elapsed, 0.1, sync_elapsed)
op.async_parts.ready_event.synchronize()
self.assertTrue(op.async_parts.ready_event.query())
c._materialize_async_indices([op])
self.assertTrue(bool(torch.equal(op.device_indices, reference)))
self.assertFalse(op.device_indices.is_cuda)
c._release_async_parts([op])
self.assertEqual(c._async_pinned_bytes, 0)
def test_next_forward_overlaps_the_index_copy(self):
torch.cuda.synchronize()
sched = torch.cuda.current_stream()
fwd = torch.cuda.Stream()
# Large enough that the copy takes long enough to observe (32 MB).
indices = torch.arange(4 * 1024 * 1024, dtype=torch.int64, device="cuda")
# 1. A long fence on the scheduler stream (the in-flight forward).
with torch.cuda.stream(sched):
fence_done = torch.cuda.Event(enable_timing=True)
fence_done.record()
torch.cuda._sleep(1_000_000_000)
# 2. Per-op pinned copy started now; it inherits the fence.
c = _make_controller()
op = _op(indices, 3)
c._start_async_index_copy(op)
self.assertIsNotNone(op.async_parts)
# 3. The next batch launch waits for the scheduler stream (as the real
# forward_stream does) and then runs a long marker kernel.
copy_end = torch.cuda.Event(enable_timing=True)
with torch.cuda.stream(c._indices_stream):
copy_end.record()
launch_host_ns = time.monotonic_ns()
with torch.cuda.stream(fwd):
fwd.wait_stream(sched)
fwd_start = torch.cuda.Event(enable_timing=True)
fwd_start.record()
torch.cuda._sleep(1_500_000_000)
fwd_end = torch.cuda.Event(enable_timing=True)
fwd_end.record()
# 4. The event must not be ready while the fence is in flight, and the
# host has already enqueued the next batch.
ready_seen_ns = [None]
def _wait_ready():
op.async_parts.ready_event.synchronize()
ready_seen_ns[0] = time.monotonic_ns()
waiter = threading.Thread(target=_wait_ready)
waiter.start()
self.assertIsNone(ready_seen_ns[0])
waiter.join(timeout=30)
self.assertIsNotNone(ready_seen_ns[0])
self.assertLess(launch_host_ns, ready_seen_ns[0])
torch.cuda.synchronize()
# CUDA timestamps: copy end strictly inside the next forward.
self.assertGreater(fwd_start.elapsed_time(copy_end), 0.0)
self.assertGreater(copy_end.elapsed_time(fwd_end), 0.0)
self.assertGreater(fwd_start.elapsed_time(fwd_end), 0.5)
c._materialize_async_indices([op])
self.assertTrue(
bool(torch.equal(op.device_indices, indices.cpu()))
)
c._release_async_parts([op])
def test_copy_stream_is_distinct_from_scheduler(self):
torch.cuda.synchronize()
c = _make_controller()
op = _op(torch.arange(8, dtype=torch.int64, device="cuda"), 0)
c._start_async_index_copy(op)
self.assertIsNotNone(c._indices_stream)
self.assertNotEqual(
c._indices_stream.cuda_stream,
torch.cuda.current_stream().cuda_stream,
)
op.async_parts.ready_event.synchronize()
c._release_async_parts([op])
def test_out_of_order_completion_keeps_op_order(self):
torch.cuda.synchronize()
c = _make_controller()
delays = {0: 600_000_000, 1: 150_000_000, 2: 300_000_000}
ops = []
references = []
for i in range(3):
src = torch.arange(8, dtype=torch.int64, device="cuda") + i * 100
op = _op(src, node_id=i)
buf = torch.empty(src.shape, dtype=src.dtype, device="cpu", pin_memory=True)
stream = torch.cuda.Stream()
with torch.cuda.stream(stream):
torch.cuda._sleep(delays[i])
buf.copy_(src, non_blocking=True)
ev = torch.cuda.Event()
ev.record(stream)
op.async_parts = cc_mod.AsyncIndexParts(
[cc_mod.AsyncIndexCopy((None, None), src, buf, buf.numel() * 8)],
ev,
buf.numel() * 8,
)
ops.append(op)
references.append(src.cpu())
captured = []
c._submit_write_operation = lambda merged: (
captured.append(merged.device_indices.clone())
or cc_mod.HiCacheAck(None, None, list(merged.node_ids), num_tokens=8)
)
c._write_submit_queue = cc_mod.Queue(maxsize=4)
worker = threading.Thread(target=c._write_submit_worker, daemon=True)
c._write_submit_thread = worker
worker.start()
try:
c._write_submit_active = 1
c._write_submit_queue.put(cc_mod.WriteSubmission(ops, None, 0))
deadline = time.time() + 30
while c._write_submit_active and time.time() < deadline:
time.sleep(0.01)
self.assertEqual(c._write_submit_active, 0)
finally:
c._write_submit_queue.put(None)
worker.join(timeout=10)
self.assertEqual(len(captured), 1)
expected = torch.cat(references)
self.assertTrue(bool(torch.equal(captured[0], expected)))
self.assertEqual(ops[0].node_ids, [0])
self.assertEqual(ops[1].node_ids, [1])
self.assertEqual(ops[2].node_ids, [2])
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,550 @@
"""Unit tests for the async HiCache backup submission state machine.
Design: handoff/prefill-backup-overlap-next-20261107.md §8.1.
These tests exercise the scheduler/worker split without a GPU: the index copy
itself is covered by the CUDA test, and here the device stream/event and the
submit path are faked. What is checked:
* disabled / inapplicable shapes keep the sync path;
* per-op pinned parts are materialized per (pool, indices_from_pool) key;
* the worker waits every ready event before touching host buffers;
* exactly-once ACK, fail-closed errors, bounded queues, idle/stop lifecycle;
* has_inflight_device_transfers reflects queued/running results.
"""
import threading
import time
import types
import unittest
from unittest import mock
from sglang.srt.managers import cache_controller as cc_mod
from sglang.srt.mem_cache.hicache_storage import PoolName, PoolTransfer
class _FakeEvent:
def __init__(self, worker_log=None, name="ev"):
self.worker_log = worker_log
self.name = name
self._set = False
self.waited = 0
def record(self, stream=None):
self._set = True
def synchronize(self):
self.waited += 1
if self.worker_log is not None:
self.worker_log.append(("wait", self.name))
self._set = True
def query(self):
return self._set
class _FakeStream:
def __init__(self, name="stream", log=None):
self.name = name
self.log = log
self.waited = 0
def wait_stream(self, other):
self.waited += 1
if self.log is not None:
self.log.append(("wait_stream", self.name, getattr(other, "name", "?")))
def wait_event(self, event):
pass
class _FakeStreamCtx:
def __init__(self, stream):
self.stream = stream
def __enter__(self):
return self.stream
def __exit__(self, *exc):
return False
class _FakeCudaModule:
def __init__(self, log=None):
self.log = log
self.current = _FakeStream("schedule", log)
def Stream(self, *a, **k):
return _FakeStream("indices", self.log)
def StreamContext(self, stream):
return _FakeStreamCtx(stream)
def Event(self, *a, **k):
return _FakeEvent(self.log)
def current_stream(self):
return self.current
class _FakeTensor:
def __init__(self, numel=8, element_size=8, is_cuda=True, tag="dev"):
self._numel = numel
self._element_size = element_size
self.is_cuda = is_cuda
self.tag = tag
self.recorded = []
self.cpu_calls = 0
self.device = types.SimpleNamespace(type="cuda" if is_cuda else "cpu")
self.dtype = "torch.int64"
def numel(self):
return self._numel
def element_size(self):
return self._element_size
def record_stream(self, stream):
self.recorded.append(stream)
def copy_(self, src, non_blocking=False):
self.copied_from = src
@property
def shape(self):
return (self._numel,)
def cpu(self):
self.cpu_calls += 1
return _FakeTensor(self._numel, self._element_size, False, tag=self.tag + "-cpu")
def __len__(self):
return self._numel
class _FakeHostPool:
def __init__(self, shared=False):
self.shared = shared
class _FakeEntry:
def __init__(self, host_pool):
self.host_pool = host_pool
class _FakeMemPoolHost:
layout = "page_first_direct"
def __init__(self, anchor_shared=False, sidecars=None):
self.anchor_entry = _FakeEntry(_FakeHostPool(anchor_shared))
self.entry_map = {
name: _FakeEntry(pool) for name, pool in (sidecars or {}).items()
}
def _make_controller(*, enabled=True, anchor_shared=False, sidecars=None, policy=""):
c = cc_mod.HiCacheController.__new__(cc_mod.HiCacheController)
c.io_backend = "direct"
c.write_policy = policy
c.mem_pool_host = _FakeMemPoolHost(anchor_shared, sidecars)
c.write_queue = []
c.load_queue = []
c.ack_write_queue = []
c.ack_load_queue = []
c._indices_stream = None
c._write_submit_queue = None
c._write_result_queue = cc_mod.Queue()
c._write_submit_stop = threading.Event()
c._write_submit_thread = None
c._write_submit_state_lock = threading.Lock()
c._write_submit_cond = threading.Condition(c._write_submit_state_lock)
c._write_submit_active = 0
c._async_pinned_bytes = 0
c._retire_parts = []
return c
def _op(controller, *, anchor=None, transfers=(), node_id=7):
host = _FakeTensor(8, 8, is_cuda=False, tag="host")
dev = anchor or _FakeTensor(8, 8, is_cuda=True, tag="anchor")
op = cc_mod.CacheOperation(host, dev, node_id, pool_transfers=list(transfers))
return op
class AsyncBackupUnitTest(unittest.TestCase):
def setUp(self):
patcher = mock.patch.object(cc_mod, "_ASYNC_BACKUP_SUBMIT", True)
patcher.start()
self.addCleanup(patcher.stop)
# The real predicate isinstance-checks HostKVCache; the fakes expose a
# plain `shared` flag instead.
shared_patcher = mock.patch.object(
cc_mod, "uses_shared_host_layout", lambda pool: bool(getattr(pool, "shared", False))
)
shared_patcher.start()
self.addCleanup(shared_patcher.stop)
cat_patcher = mock.patch.object(
cc_mod.torch,
"cat",
side_effect=lambda parts: _FakeTensor(
sum(len(p) for p in parts), 8, False, "cat"
),
)
cat_patcher.start()
self.addCleanup(cat_patcher.stop)
# Pinned destination buffers are faked; only the copy_ call is checked.
self.pin_patch = mock.patch.object(
cc_mod.torch, "empty", side_effect=lambda *a, **k: _FakeTensor(8, 8, False, "pin")
)
self.pin_patch.start()
self.addCleanup(self.pin_patch.stop)
def test_disabled_keeps_sync_path(self):
with mock.patch.object(cc_mod, "_ASYNC_BACKUP_SUBMIT", False):
c = _make_controller()
self.assertFalse(c.async_backup_enabled())
self.assertFalse(c._async_submit_applicable())
def test_wrong_backend_or_layout_is_ineligible(self):
c = _make_controller()
c.io_backend = "kernel"
self.assertFalse(c.async_backup_enabled())
c = _make_controller()
c.mem_pool_host.layout = "page_first"
self.assertFalse(c.async_backup_enabled())
def test_index_targets_skip_shared_layout_and_cpu_tensors(self):
shared = PoolName.SWA
c = _make_controller(sidecars={shared: _FakeHostPool(shared=True)})
transfer = PoolTransfer(
name=shared, device_indices=_FakeTensor(4, 8, True, "swa")
)
cpu_anchor = _FakeTensor(8, 8, False, "anchor")
op = _op(c, anchor=cpu_anchor, transfers=[transfer])
self.assertEqual(c._async_index_targets(op), [])
c = _make_controller(sidecars={PoolName.SWA: _FakeHostPool(shared=False)})
transfer = PoolTransfer(
name=PoolName.SWA,
device_indices=_FakeTensor(4, 8, True, "swa"),
indices_from_pool=PoolName.KV,
)
anchor = _FakeTensor(8, 8, True, "anchor")
op = _op(c, anchor=anchor, transfers=[transfer])
targets = c._async_index_targets(op)
self.assertEqual([k for k, _ in targets], [(None, None), (PoolName.SWA, PoolName.KV)])
def test_start_copy_orders_after_scheduler_and_records_event(self):
log = []
c = _make_controller()
op = _op(c)
with mock.patch.object(cc_mod, "device_module", _FakeCudaModule(log)):
c._start_async_index_copy(op)
self.assertIsNotNone(op.async_parts)
self.assertEqual(op.async_parts.ready_event._set, True)
self.assertEqual(op.device_indices.recorded[0].name, "indices")
# The copy stream is ordered after the scheduler stream once.
self.assertIn(("wait_stream", "indices", "schedule"), log)
self.assertEqual(c._async_pinned_bytes, 64)
def test_pinned_cap_falls_back_to_sync(self):
c = _make_controller()
op = _op(c)
with mock.patch.object(cc_mod, "_ASYNC_BACKUP_PINNED_BYTES_MAX", 1), \
mock.patch.object(cc_mod, "device_module", _FakeCudaModule()):
c._start_async_index_copy(op)
self.assertIsNone(op.async_parts)
self.assertEqual(c._async_pinned_bytes, 0)
def test_materialize_uses_pool_keys(self):
c = _make_controller()
anchor_copy = cc_mod.AsyncIndexCopy((None, None), None, _FakeTensor(8, 8, False, "a"), 64)
deriv = PoolTransfer(
name=PoolName.SWA,
device_indices=_FakeTensor(4, 8, True, "swa"),
indices_from_pool=PoolName.KV,
)
side_copy = cc_mod.AsyncIndexCopy(
(PoolName.SWA, PoolName.KV), None, _FakeTensor(4, 8, False, "s"), 32
)
op = _op(c, transfers=[deriv])
op.async_parts = cc_mod.AsyncIndexParts(
copies=[anchor_copy, side_copy], ready_event=_FakeEvent(), nbytes=96
)
c._materialize_async_indices([op])
self.assertIs(op.device_indices, anchor_copy.dst)
self.assertIs(deriv.device_indices, side_copy.dst)
def test_worker_waits_events_then_submits_and_acks_once(self):
log = []
c = _make_controller()
events = []
def _make_op(tag):
op = _op(c, anchor=_FakeTensor(8, 8, True, tag))
ev = _FakeEvent(log, tag)
events.append(ev)
op.async_parts = cc_mod.AsyncIndexParts(
copies=[
cc_mod.AsyncIndexCopy(
(None, None), op.device_indices, _FakeTensor(8, 8, False, tag), 64
)
],
ready_event=ev,
nbytes=64,
)
return op
ops = [_make_op("a"), _make_op("b")]
c._async_pinned_bytes = 128
submitted = []
def _submit(op):
# Every ready event must have been waited before the merge is used.
submitted.append(list(op.node_ids))
return cc_mod.HiCacheAck(None, None, list(op.node_ids), num_tokens=8)
c._submit_write_operation = _submit
c._write_submit_queue = cc_mod.Queue(maxsize=4)
c._ensure_write_submit_thread = lambda: None
worker = threading.Thread(target=c._write_submit_worker, daemon=True)
c._write_submit_thread = worker
worker.start()
try:
item = cc_mod.WriteSubmission(ops=ops, tracer=None, enqueue_mono_ns=0)
c._write_submit_active = 1
c._write_submit_queue.put(item)
deadline = time.time() + 5
while c._write_submit_active and time.time() < deadline:
time.sleep(0.005)
self.assertEqual(c._write_submit_active, 0)
finally:
c._write_submit_queue.put(None)
worker.join(timeout=5)
self.assertEqual([e.waited for e in events], [1, 1])
self.assertEqual(submitted, [[7, 7]])
self.assertEqual(c._async_pinned_bytes, 0)
c.drain_write_submission_results()
self.assertEqual(len(c.ack_write_queue), 1)
# Second drain is a no-op: exactly-once ACK.
c.drain_write_submission_results()
self.assertEqual(len(c.ack_write_queue), 1)
def test_worker_error_is_fail_closed(self):
c = _make_controller()
op = _op(c)
op.async_parts = cc_mod.AsyncIndexParts([], _FakeEvent(), 0)
c._submit_write_operation = mock.Mock(side_effect=RuntimeError("boom"))
c._write_submit_queue = cc_mod.Queue(maxsize=4)
worker = threading.Thread(target=c._write_submit_worker, daemon=True)
c._write_submit_thread = worker
worker.start()
c._write_submit_active = 1
c._write_submit_queue.put(cc_mod.WriteSubmission([op], None, 0))
deadline = time.time() + 5
while c._write_submit_active and time.time() < deadline:
time.sleep(0.005)
c._write_submit_queue.put(None)
worker.join(timeout=5)
with self.assertRaises(RuntimeError):
c.drain_write_submission_results()
self.assertEqual(c.ack_write_queue, [])
def test_enqueue_failure_keeps_active_consistent(self):
c = _make_controller()
c._write_submit_queue = cc_mod.Queue(maxsize=1)
c._write_submit_queue.put("occupied")
c._ensure_write_submit_thread = lambda: None
with mock.patch.object(cc_mod, "_ASYNC_BACKUP_ENQUEUE_TIMEOUT", 0.05):
ok = c._enqueue_async_write([_op(c)], None)
self.assertFalse(ok)
self.assertEqual(c._write_submit_active, 0)
def test_queue_full_falls_back_to_sync_submit(self):
c = _make_controller()
# A permanently-full submit queue forces the sync path.
c._write_submit_queue = cc_mod.Queue(maxsize=1)
c._write_submit_queue.put("occupied")
c._ensure_write_submit_thread = lambda: None
op = _op(c)
retired_event = _FakeEvent()
retired_event.record()
op.async_parts = cc_mod.AsyncIndexParts(
[cc_mod.AsyncIndexCopy((None, None), op.device_indices, _FakeTensor(8, 8, False), 64)],
retired_event,
64,
)
c._async_pinned_bytes = 64
c.write_queue.append(op)
submitted = []
def _submit(merged):
# The fallback must hand over the original device indices, i.e. the
# pinned parts are retired rather than materialized.
submitted.append(merged.device_indices)
return cc_mod.HiCacheAck(None, None, list(merged.node_ids), num_tokens=8)
c._submit_write_operation = _submit
with mock.patch.object(cc_mod, "_ASYNC_BACKUP_ENQUEUE_TIMEOUT", 0.05):
c.start_writing()
self.assertEqual(len(submitted), 1)
self.assertIs(submitted[0], op.device_indices)
self.assertEqual(len(c.ack_write_queue), 1)
self.assertEqual(c.write_queue, [])
self.assertIsNone(op.async_parts)
self.assertEqual(c._write_submit_active, 0)
# The retired copy is released once its event completes.
c._reap_retired_parts()
self.assertEqual(c._async_pinned_bytes, 0)
def test_mixed_queue_sync_path_retires_unconsumed_parts(self):
c = _make_controller()
ev = _FakeEvent()
ev.record()
with_parts = _op(c, node_id=1)
with_parts.async_parts = cc_mod.AsyncIndexParts(
[
cc_mod.AsyncIndexCopy(
(None, None), with_parts.device_indices, _FakeTensor(8, 8, False), 64
)
],
ev,
64,
)
without_parts = _op(c, node_id=2)
c.write_queue.extend([with_parts, without_parts])
c._async_pinned_bytes = 64
submitted = []
c._submit_write_operation = lambda merged: (
submitted.append(list(merged.node_ids))
or cc_mod.HiCacheAck(None, None, list(merged.node_ids), num_tokens=8)
)
c.start_writing()
# Mixed residency () forces the sync path, but the pinned copy started
# for the first op must be retired and its budget released.
self.assertEqual(submitted, [[1, 2]])
self.assertIsNone(with_parts.async_parts)
self.assertEqual(c._async_pinned_bytes, 0)
self.assertEqual(len(c.ack_write_queue), 1)
def test_op_cap_falls_back_to_sync(self):
c = _make_controller()
with mock.patch.object(cc_mod, "_ASYNC_BACKUP_MAX_OPS", 1):
for i in range(2):
op = _op(c, node_id=i)
op.async_parts = cc_mod.AsyncIndexParts([], _FakeEvent(), 0)
c.write_queue.append(op)
self.assertFalse(c._async_submit_applicable())
def test_inflight_reflects_async_state(self):
c = _make_controller()
self.assertFalse(c.has_inflight_device_transfers())
c._write_submit_active = 1
self.assertTrue(c.has_inflight_device_transfers())
c._write_submit_active = 0
c._async_pinned_bytes = 8
self.assertTrue(c.has_inflight_device_transfers())
c._async_pinned_bytes = 0
c._write_result_queue.put(cc_mod.WriteResult(None, [], None))
self.assertTrue(c.has_inflight_device_transfers())
c._write_result_queue.get()
self.assertFalse(c.has_inflight_device_transfers())
def test_idle_wait_times_out(self):
c = _make_controller()
c._write_submit_active = 1
with self.assertRaises(RuntimeError):
c.wait_write_submissions_idle(timeout=0.05)
def test_retired_parts_release_only_after_event(self):
c = _make_controller()
ev = _FakeEvent()
ev._set = False
op = _op(c)
op.async_parts = cc_mod.AsyncIndexParts(
[cc_mod.AsyncIndexCopy((None, None), None, _FakeTensor(8, 8, False), 64)],
ev,
64,
)
c._async_pinned_bytes = 64
c._retire_async_parts([op])
self.assertIsNone(op.async_parts)
self.assertEqual(c._async_pinned_bytes, 64)
ev._set = True
c._reap_retired_parts()
self.assertEqual(c._async_pinned_bytes, 0)
self.assertEqual(c._retire_parts, [])
def test_merge_of_cpu_parts_matches_cpu_reference(self):
c = _make_controller()
ops = []
for i in range(3):
op = _op(c, anchor=_FakeTensor(4, 8, True, f"a{i}"), node_id=i)
op.async_parts = cc_mod.AsyncIndexParts(
[
cc_mod.AsyncIndexCopy(
(None, None),
op.device_indices,
_FakeTensor(4, 8, False, f"p{i}"),
32,
)
],
_FakeEvent(),
32,
)
ops.append(op)
c._materialize_async_indices(ops)
merged = cc_mod.CacheOperation.merge_ops(ops)
self.assertEqual(merged.node_ids, [0, 1, 2])
# Single-op merge must not copy: it returns the operation itself.
single = cc_mod.CacheOperation.merge_ops([ops[0]])
self.assertIs(single, ops[0])
def test_stop_waits_for_inflight_submission_and_drains_ack(self):
c = _make_controller()
release = threading.Event()
started = threading.Event()
def _submit(op):
started.set()
release.wait(10)
return cc_mod.HiCacheAck(None, None, list(op.node_ids), num_tokens=8)
c._submit_write_operation = _submit
op = _op(c)
op.async_parts = cc_mod.AsyncIndexParts([], _FakeEvent(), 0)
c._ensure_write_submit_thread()
c._write_submit_active = 1
c._write_submit_queue.put(cc_mod.WriteSubmission([op], None, 0))
self.assertTrue(started.wait(5))
stopped = threading.Event()
def _stop():
c._stop_write_submission_thread()
stopped.set()
stopper = threading.Thread(target=_stop, daemon=True)
stopper.start()
# The stop must not return while a submission is still in flight.
self.assertFalse(stopped.wait(0.05))
release.set()
self.assertTrue(stopped.wait(10))
self.assertFalse(c._write_submit_thread is not None and c._write_submit_thread.is_alive())
# The ACK produced by the in-flight submission was not dropped.
self.assertEqual(len(c.ack_write_queue), 1)
def test_stop_joins_worker_and_allows_restart(self):
c = _make_controller()
c._ensure_write_submit_thread()
self.assertTrue(c._write_submit_thread.is_alive())
c._stop_write_submission_thread()
self.assertIsNone(c._write_submit_thread)
self.assertIsNone(c._write_submit_queue)
c._ensure_write_submit_thread()
self.assertTrue(c._write_submit_thread.is_alive())
c._stop_write_submission_thread()
if __name__ == "__main__":
unittest.main()
+76
View File
@@ -0,0 +1,76 @@
"""Print the CUDA timeline for one async index copy vs the next forward."""
import importlib.util as _u
import threading
import time
import torch
from sglang.srt.managers import cache_controller as cc_mod
_spec = _u.spec_from_file_location(
"t", "/data/ymk/wt-async/test/registered/cuda/test_async_backup_index_copy_cuda.py"
)
_m = _u.module_from_spec(_spec)
_spec.loader.exec_module(_m)
_make_controller, _op = _m._make_controller, _m._op
cc_mod._ASYNC_BACKUP_SUBMIT = True
cc_mod._ASYNC_BACKUP_PINNED_BYTES_MAX = 1 << 30
torch.cuda.synchronize()
sched = torch.cuda.current_stream()
fwd = torch.cuda.Stream()
indices = torch.arange(4 * 1024 * 1024, dtype=torch.int64, device="cuda")
origin = torch.cuda.Event(enable_timing=True)
with torch.cuda.stream(sched):
origin.record()
torch.cuda._sleep(1_000_000_000)
fence_done = torch.cuda.Event(enable_timing=True)
fence_done.record()
c = _make_controller()
op = _op(indices, 0)
t0 = time.perf_counter()
c._start_async_index_copy(op)
host_ms = (time.perf_counter() - t0) * 1e3
copy_end = torch.cuda.Event(enable_timing=True)
with torch.cuda.stream(c._indices_stream):
copy_end.record()
t_launch = time.monotonic_ns()
with torch.cuda.stream(fwd):
fwd.wait_stream(sched)
fwd_start = torch.cuda.Event(enable_timing=True)
fwd_start.record()
torch.cuda._sleep(1_500_000_000)
fwd_end = torch.cuda.Event(enable_timing=True)
fwd_end.record()
ready_ns = [None]
def _w():
op.async_parts.ready_event.synchronize()
ready_ns[0] = time.monotonic_ns()
th = threading.Thread(target=_w)
th.start()
th.join()
torch.cuda.synchronize()
print(f"host start_async_index_copy: {host_ms:.2f} ms")
print(f"host next-batch enqueue before ready: {t_launch < ready_ns[0]}")
print(f"fence_done -> copy_end : {fence_done.elapsed_time(copy_end):8.2f} ms")
print(f"origin -> fwd_start : {origin.elapsed_time(fwd_start):8.2f} ms")
print(f"origin -> copy_end : {origin.elapsed_time(copy_end):8.2f} ms")
print(f"origin -> fwd_end : {origin.elapsed_time(fwd_end):8.2f} ms")
print(f"fwd duration : {fwd_start.elapsed_time(fwd_end):8.2f} ms")
print(
f"fwd_start -> copy_end : {fwd_start.elapsed_time(copy_end):8.2f} ms (>0 means overlap)"
)
print(
f"copy_end -> fwd_end : {copy_end.elapsed_time(fwd_end):8.2f} ms (>0 means copy finished inside the forward)"
)