Net async change (from exp-async-backup-perop-20261107) applied onto b300-pick-46884-20261107. Runtime files applied cleanly; the new tests and the timeline probe are added as-is. Interaction to validate on hardware: #46884 publishes prefill read-done before the replay, so the scheduler-stream fence that the index copy waits on is shorter; the async worker's wait should shrink and the copy may overlap more of the in-flight forward. Re-baseline the 4-arm A/B on this branch.
This commit is contained in:
@@ -18,7 +18,7 @@ import logging
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from queue import Empty, Queue
|
||||
from queue import Empty, Full, Queue
|
||||
from typing import TYPE_CHECKING, Any, Callable, List, NamedTuple, Optional
|
||||
|
||||
import torch
|
||||
@@ -42,6 +42,7 @@ from sglang.srt.layers.dp_attention import (
|
||||
)
|
||||
from sglang.srt.mem_cache.l2_transfer import L2Transfer, L2TransferEngine
|
||||
from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool
|
||||
from sglang.srt.mem_cache.pool_host.base import uses_shared_host_layout
|
||||
from sglang.srt.mem_cache.utils import get_storage_hash_str
|
||||
from sglang.srt.observability import batch_telemetry
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
@@ -69,6 +70,25 @@ _HC_PREFETCH_WORKERS = int(_hc_os.getenv("SGLANG_HICACHE_PREFETCH_IO_THREADS", "
|
||||
_HC_BACKUP_WORKERS = int(_hc_os.getenv("SGLANG_HICACHE_BACKUP_IO_THREADS", "1") or 1)
|
||||
_HC_TIMING = _hc_os.getenv("SGLANG_HICACHE_TIMING", "0") not in ("", "0", "false", "False")
|
||||
|
||||
# [ASYNC_BACKUP] Per-op pinned index copy + worker-side merge/submit.
|
||||
# Only the direct + page_first_direct shape (the one that does a blocking
|
||||
# device_indices.cpu() on the scheduler thread) is eligible.
|
||||
_ASYNC_BACKUP_SUBMIT = _hc_os.getenv(
|
||||
"SGLANG_ASYNC_BACKUP_SUBMIT", "0"
|
||||
).strip().lower() not in ("", "0", "false", "no", "off")
|
||||
_ASYNC_BACKUP_ENQUEUE_TIMEOUT = float(
|
||||
_hc_os.getenv("SGLANG_ASYNC_BACKUP_ENQUEUE_TIMEOUT", "60") or 60
|
||||
)
|
||||
_ASYNC_BACKUP_IDLE_TIMEOUT = float(
|
||||
_hc_os.getenv("SGLANG_ASYNC_BACKUP_IDLE_TIMEOUT", "120") or 120
|
||||
)
|
||||
_ASYNC_BACKUP_PINNED_BYTES_MAX = int(
|
||||
_hc_os.getenv("SGLANG_ASYNC_BACKUP_PINNED_MB", "2048") or 2048
|
||||
) * 1024 * 1024
|
||||
_ASYNC_BACKUP_MAX_OPS = int(
|
||||
_hc_os.getenv("SGLANG_ASYNC_BACKUP_MAX_OPS", "256") or 256
|
||||
)
|
||||
|
||||
class LayerLoadingEvent:
|
||||
def __init__(self, num_layers: int):
|
||||
self._num_layers = num_layers
|
||||
@@ -132,6 +152,9 @@ class CacheOperation:
|
||||
self.node_ids = [node_id]
|
||||
self.data = None
|
||||
self.pool_transfers = pool_transfers
|
||||
# Set when SGLANG_ASYNC_BACKUP_SUBMIT copied this op's device indices
|
||||
# into pinned host buffers (see _start_async_index_copy).
|
||||
self.async_parts = None
|
||||
|
||||
self.id = CacheOperation.counter
|
||||
CacheOperation.counter += 1
|
||||
@@ -192,6 +215,39 @@ class CacheOperation:
|
||||
return self.priority < other.priority
|
||||
|
||||
|
||||
@dataclass
|
||||
class AsyncIndexCopy:
|
||||
"""One device->pinned-host index copy started where the backup op was made."""
|
||||
|
||||
key: tuple
|
||||
src: torch.Tensor
|
||||
dst: torch.Tensor
|
||||
nbytes: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class AsyncIndexParts:
|
||||
"""Per-op pinned index parts plus the event that guarantees they are final."""
|
||||
|
||||
copies: List[AsyncIndexCopy]
|
||||
ready_event: object
|
||||
nbytes: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class WriteSubmission:
|
||||
ops: List[CacheOperation]
|
||||
tracer: object
|
||||
enqueue_mono_ns: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class WriteResult:
|
||||
ack: Optional[HiCacheAck]
|
||||
ops: List[CacheOperation]
|
||||
error: Optional[BaseException]
|
||||
|
||||
|
||||
class HiCacheAck(NamedTuple):
|
||||
start_event: device_module.Event
|
||||
finish_event: device_module.Event
|
||||
@@ -383,6 +439,21 @@ class HiCacheController:
|
||||
self.write_fence_stream = None
|
||||
self.ack_write_queue: List[HiCacheAck] = []
|
||||
|
||||
# [ASYNC_BACKUP] Pinned index copies are started where the backup op is
|
||||
# created; a single worker waits the per-op events, merges the CPU
|
||||
# index parts and submits the KV copy. Queues are bounded by the state
|
||||
# lock/condition below; write_queue/ack_write_queue stay scheduler-only.
|
||||
self._indices_stream = None
|
||||
self._write_submit_queue: Optional[Queue] = None
|
||||
self._write_result_queue: Queue = Queue()
|
||||
self._write_submit_stop = threading.Event()
|
||||
self._write_submit_thread: Optional[threading.Thread] = None
|
||||
self._write_submit_state_lock = threading.Lock()
|
||||
self._write_submit_cond = threading.Condition(self._write_submit_state_lock)
|
||||
self._write_submit_active = 0
|
||||
self._async_pinned_bytes = 0
|
||||
self._retire_parts: List["AsyncIndexParts"] = []
|
||||
|
||||
self.l2_transfer_engine = L2TransferEngine(io_backend)
|
||||
|
||||
# If a storage backend is provided at startup, treat it as an implicit attach,
|
||||
@@ -708,6 +779,9 @@ class HiCacheController:
|
||||
Requirement: no in-flight requests. This will stop storage threads and release
|
||||
the backend instance (best-effort close).
|
||||
"""
|
||||
# Async backup submissions own pinned index buffers and ACKs; stop them
|
||||
# before any queue/pool teardown so no result is lost.
|
||||
self._stop_write_submission_thread()
|
||||
# Idempotent cleanup: even if `enable_storage` is already False,
|
||||
# we may still have leftover resources (threads/backend/process group) from a
|
||||
# previous partial detach. We attempt cleanup whenever possible.
|
||||
@@ -822,6 +896,7 @@ class HiCacheController:
|
||||
def reset(self):
|
||||
# Reuse detach's queue wakeups and bounded joins, and fail if any
|
||||
# storage thread remains alive before clearing shared state.
|
||||
self._stop_write_submission_thread()
|
||||
self._stop_storage_threads()
|
||||
|
||||
self.write_queue.clear()
|
||||
@@ -861,13 +936,311 @@ class HiCacheController:
|
||||
|
||||
def has_inflight_device_transfers(self) -> bool:
|
||||
"""Whether queued or unacknowledged L2 transfers still use device rows."""
|
||||
with self._write_submit_state_lock:
|
||||
async_pending = (
|
||||
self._write_submit_active > 0
|
||||
or self._async_pinned_bytes > 0
|
||||
or not self._write_result_queue.empty()
|
||||
)
|
||||
return bool(
|
||||
self.write_queue
|
||||
or self.load_queue
|
||||
or self.ack_write_queue
|
||||
or self.ack_load_queue
|
||||
or async_pending
|
||||
)
|
||||
|
||||
# ---- async backup submission -----------------------------------------
|
||||
def async_backup_enabled(self) -> bool:
|
||||
return (
|
||||
_ASYNC_BACKUP_SUBMIT
|
||||
and self.io_backend == "direct"
|
||||
and getattr(self.mem_pool_host, "layout", None) == "page_first_direct"
|
||||
)
|
||||
|
||||
def _async_submit_applicable(self) -> bool:
|
||||
return (
|
||||
self.async_backup_enabled()
|
||||
and len(self.write_queue) <= _ASYNC_BACKUP_MAX_OPS
|
||||
and all(op.async_parts is not None for op in self.write_queue)
|
||||
)
|
||||
|
||||
def _ensure_indices_stream(self):
|
||||
if self._indices_stream is None:
|
||||
self._indices_stream = device_module.Stream()
|
||||
return self._indices_stream
|
||||
|
||||
def _needs_device_index_copy(self, host_pool, indices) -> bool:
|
||||
if indices is None or not getattr(indices, "is_cuda", False):
|
||||
return False
|
||||
# move_indices() returns shared-layout ids untouched; converting them
|
||||
# here would hand the transfer a different interface than the sync path.
|
||||
return not uses_shared_host_layout(host_pool)
|
||||
|
||||
def _async_index_targets(self, op: CacheOperation):
|
||||
targets = []
|
||||
if self._needs_device_index_copy(
|
||||
self.mem_pool_host.anchor_entry.host_pool, op.device_indices
|
||||
):
|
||||
targets.append(((None, None), op.device_indices))
|
||||
for transfer in op.pool_transfers or []:
|
||||
if transfer.device_indices is None:
|
||||
continue
|
||||
entry = self.mem_pool_host.entry_map.get(transfer.name)
|
||||
if entry is None:
|
||||
continue
|
||||
if not self._needs_device_index_copy(
|
||||
entry.host_pool, transfer.device_indices
|
||||
):
|
||||
continue
|
||||
key = (transfer.name, transfer.indices_from_pool)
|
||||
targets.append((key, transfer.device_indices))
|
||||
return targets
|
||||
|
||||
def _start_async_index_copy(self, op: CacheOperation) -> None:
|
||||
"""Start the pinned index D2H copy now, off the scheduler stream.
|
||||
|
||||
Runs in the scheduler thread at backup-op creation time. The copy runs
|
||||
on a dedicated stream ordered after this instant, so it does not inherit
|
||||
later fences (the WAR barrier of a later forward) and the scheduler
|
||||
never blocks in ``.cpu()``.
|
||||
"""
|
||||
if not self.async_backup_enabled():
|
||||
return
|
||||
targets = self._async_index_targets(op)
|
||||
if not targets:
|
||||
return
|
||||
nbytes = sum(int(t.numel()) * int(t.element_size()) for _, t in targets)
|
||||
with self._write_submit_state_lock:
|
||||
if self._async_pinned_bytes + nbytes > _ASYNC_BACKUP_PINNED_BYTES_MAX:
|
||||
return
|
||||
self._async_pinned_bytes += nbytes
|
||||
try:
|
||||
stream = self._ensure_indices_stream()
|
||||
source_stream = device_module.current_stream()
|
||||
copies: List[AsyncIndexCopy] = []
|
||||
with device_module.StreamContext(stream):
|
||||
stream.wait_stream(source_stream)
|
||||
for key, src in targets:
|
||||
# Keep the device rows alive until the copy has run, and pin
|
||||
# the destination so copy_ is a real asynchronous D2H.
|
||||
src.record_stream(stream)
|
||||
dst = torch.empty(
|
||||
src.shape,
|
||||
dtype=src.dtype,
|
||||
device="cpu",
|
||||
pin_memory=True,
|
||||
)
|
||||
dst.copy_(src, non_blocking=True)
|
||||
copies.append(
|
||||
AsyncIndexCopy(
|
||||
key=key,
|
||||
src=src,
|
||||
dst=dst,
|
||||
nbytes=int(src.numel()) * int(src.element_size()),
|
||||
)
|
||||
)
|
||||
ready_event = device_module.Event()
|
||||
ready_event.record(stream)
|
||||
except BaseException:
|
||||
with self._write_submit_state_lock:
|
||||
self._async_pinned_bytes -= nbytes
|
||||
logger.exception("Async backup index copy failed; keeping sync path")
|
||||
return
|
||||
op.async_parts = AsyncIndexParts(
|
||||
copies=copies, ready_event=ready_event, nbytes=nbytes
|
||||
)
|
||||
|
||||
def _ensure_write_submit_thread(self) -> None:
|
||||
if self._write_submit_queue is None:
|
||||
self._write_submit_queue = Queue(maxsize=64)
|
||||
thread = self._write_submit_thread
|
||||
if thread is not None and thread.is_alive():
|
||||
return
|
||||
self._write_submit_stop.clear()
|
||||
thread = threading.Thread(
|
||||
target=self._write_submit_worker,
|
||||
name="hicache-async-backup",
|
||||
daemon=True,
|
||||
)
|
||||
self._write_submit_thread = thread
|
||||
thread.start()
|
||||
|
||||
def _enqueue_async_write(self, ops, tracer) -> bool:
|
||||
"""Enqueue one submission; False means "fall back to the sync path".
|
||||
|
||||
A full queue or a timeout is not fatal: the caller submits the same ops
|
||||
synchronously. Before returning False we wait for the worker to become
|
||||
idle so the fallback cannot race another submission on the shared
|
||||
device_to_host_stream.
|
||||
"""
|
||||
self._ensure_write_submit_thread()
|
||||
item = WriteSubmission(
|
||||
ops=ops, tracer=tracer, enqueue_mono_ns=time.monotonic_ns()
|
||||
)
|
||||
with self._write_submit_cond:
|
||||
self._write_submit_active += 1
|
||||
try:
|
||||
with batch_telemetry.backup_phase("enqueue"):
|
||||
self._write_submit_queue.put(
|
||||
item, timeout=_ASYNC_BACKUP_ENQUEUE_TIMEOUT
|
||||
)
|
||||
except Full:
|
||||
with self._write_submit_cond:
|
||||
self._write_submit_active -= 1
|
||||
self._write_submit_cond.notify_all()
|
||||
logger.warning(
|
||||
"Async backup submit queue full after %.1fs; falling back to "
|
||||
"the synchronous path",
|
||||
_ASYNC_BACKUP_ENQUEUE_TIMEOUT,
|
||||
)
|
||||
self.wait_write_submissions_idle()
|
||||
return False
|
||||
return True
|
||||
|
||||
def _materialize_async_indices(self, ops) -> None:
|
||||
"""Replace device index tensors with the pinned CPU copies (worker)."""
|
||||
for op in ops:
|
||||
parts = op.async_parts
|
||||
if parts is None:
|
||||
continue
|
||||
mapping = {copy.key: copy.dst for copy in parts.copies}
|
||||
anchor = mapping.get((None, None))
|
||||
if anchor is not None:
|
||||
op.device_indices = anchor
|
||||
for transfer in op.pool_transfers or []:
|
||||
dst = mapping.get((transfer.name, transfer.indices_from_pool))
|
||||
if dst is not None:
|
||||
transfer.device_indices = dst
|
||||
|
||||
def _write_submit_worker(self) -> None:
|
||||
while True:
|
||||
item = self._write_submit_queue.get()
|
||||
if item is None:
|
||||
break
|
||||
tracer = item.tracer
|
||||
previous = batch_telemetry.backup_flush_attach(tracer)
|
||||
try:
|
||||
with batch_telemetry.backup_phase("worker"):
|
||||
for op in item.ops:
|
||||
parts = op.async_parts
|
||||
if parts is not None:
|
||||
parts.ready_event.synchronize()
|
||||
self._materialize_async_indices(item.ops)
|
||||
merged = CacheOperation.merge_ops(item.ops)
|
||||
ack = self._submit_write_operation(merged)
|
||||
result = WriteResult(ack=ack, ops=item.ops, error=None)
|
||||
except BaseException as exc: # noqa: BLE001 - fail closed
|
||||
logger.exception("Async backup submission failed")
|
||||
result = WriteResult(ack=None, ops=item.ops, error=exc)
|
||||
finally:
|
||||
batch_telemetry.backup_flush_detach(previous)
|
||||
self._release_async_parts(item.ops)
|
||||
try:
|
||||
batch_telemetry.backup_flush_end(tracer)
|
||||
except Exception:
|
||||
pass
|
||||
# Publish the result before marking the submission done, so an idle
|
||||
# waiter cannot observe active==0 with the result still missing.
|
||||
self._write_result_queue.put(result)
|
||||
with self._write_submit_cond:
|
||||
self._write_submit_active -= 1
|
||||
self._write_submit_cond.notify_all()
|
||||
|
||||
def _release_async_parts(self, ops) -> None:
|
||||
released = 0
|
||||
for op in ops:
|
||||
parts = op.async_parts
|
||||
if parts is None:
|
||||
continue
|
||||
op.async_parts = None
|
||||
parts.copies.clear()
|
||||
released += parts.nbytes
|
||||
if released:
|
||||
with self._write_submit_state_lock:
|
||||
self._async_pinned_bytes -= released
|
||||
|
||||
def _retire_async_parts(self, ops) -> None:
|
||||
"""Release copies that will not be consumed (sync fallback)."""
|
||||
for op in ops:
|
||||
parts = op.async_parts
|
||||
if parts is not None:
|
||||
self._retire_parts.append(parts)
|
||||
op.async_parts = None
|
||||
self._reap_retired_parts()
|
||||
|
||||
def _reap_retired_parts(self) -> None:
|
||||
if not self._retire_parts:
|
||||
return
|
||||
pending = []
|
||||
for parts in self._retire_parts:
|
||||
try:
|
||||
done = parts.ready_event.query()
|
||||
except Exception:
|
||||
done = True
|
||||
if done:
|
||||
parts.copies.clear()
|
||||
with self._write_submit_state_lock:
|
||||
self._async_pinned_bytes -= parts.nbytes
|
||||
else:
|
||||
pending.append(parts)
|
||||
self._retire_parts = pending
|
||||
|
||||
def drain_write_submission_results(self) -> None:
|
||||
"""Move finished async ACKs into ack_write_queue (scheduler thread)."""
|
||||
while True:
|
||||
try:
|
||||
result = self._write_result_queue.get_nowait()
|
||||
except Empty:
|
||||
break
|
||||
if result.error is not None:
|
||||
raise RuntimeError("async backup submission failed") from result.error
|
||||
if result.ack is not None:
|
||||
self.ack_write_queue.append(result.ack)
|
||||
self._reap_retired_parts()
|
||||
|
||||
def wait_write_submissions_idle(
|
||||
self, timeout: float = _ASYNC_BACKUP_IDLE_TIMEOUT
|
||||
) -> None:
|
||||
deadline = time.monotonic() + timeout
|
||||
with self._write_submit_cond:
|
||||
while self._write_submit_active > 0:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
raise RuntimeError(
|
||||
"Timed out waiting for async backup submissions"
|
||||
)
|
||||
self._write_submit_cond.wait(remaining)
|
||||
|
||||
def _stop_write_submission_thread(self) -> None:
|
||||
if self._write_submit_thread is None:
|
||||
self.drain_write_submission_results()
|
||||
self._reap_retired_parts()
|
||||
if self._retire_parts:
|
||||
raise RuntimeError("async backup index copies still in flight")
|
||||
return
|
||||
try:
|
||||
self.wait_write_submissions_idle()
|
||||
finally:
|
||||
self._write_submit_stop.set()
|
||||
queue = self._write_submit_queue
|
||||
if queue is not None:
|
||||
try:
|
||||
queue.put(None, timeout=5.0)
|
||||
except Exception:
|
||||
logger.exception("Failed to wake the async backup worker")
|
||||
thread = self._write_submit_thread
|
||||
if thread is not None:
|
||||
thread.join(timeout=30.0)
|
||||
if thread.is_alive():
|
||||
raise RuntimeError("async backup submit worker did not stop")
|
||||
self.drain_write_submission_results()
|
||||
if self._retire_parts:
|
||||
raise RuntimeError("async backup index copies still in flight")
|
||||
self._write_submit_thread = None
|
||||
self._write_submit_queue = None
|
||||
|
||||
|
||||
def write(
|
||||
self,
|
||||
device_indices: torch.Tensor,
|
||||
@@ -883,9 +1256,9 @@ class HiCacheController:
|
||||
host_indices = self.mem_pool_host.alloc(len(device_indices))
|
||||
if host_indices is None:
|
||||
return None
|
||||
self.write_queue.append(
|
||||
CacheOperation(host_indices, device_indices, node_id, priority)
|
||||
)
|
||||
op = CacheOperation(host_indices, device_indices, node_id, priority)
|
||||
self._start_async_index_copy(op)
|
||||
self.write_queue.append(op)
|
||||
self.start_writing()
|
||||
return host_indices
|
||||
|
||||
@@ -898,98 +1271,130 @@ class HiCacheController:
|
||||
batch_telemetry.detail_count("flush_empty")
|
||||
return
|
||||
|
||||
ops = list(self.write_queue)
|
||||
tracer = batch_telemetry.backup_flush_begin()
|
||||
async_owned = False
|
||||
try:
|
||||
batch_telemetry.detail_count("flush_calls")
|
||||
batch_telemetry.detail_count("flush_nonempty")
|
||||
batch_telemetry.flush_note(
|
||||
"sglang.flush_write_queue_ops", len(self.write_queue)
|
||||
)
|
||||
batch_telemetry.flush_note("sglang.flush_io_backend", self.io_backend)
|
||||
batch_telemetry.flush_note(
|
||||
"sglang.flush_layout",
|
||||
str(getattr(self.mem_pool_host, "layout", None) or ""),
|
||||
)
|
||||
batch_telemetry.flush_note(
|
||||
"sglang.flush_ack_queue_before", len(self.ack_write_queue)
|
||||
)
|
||||
self._note_write_queue(ops, len(self.ack_write_queue))
|
||||
|
||||
if self._async_submit_applicable():
|
||||
if self._enqueue_async_write(ops, tracer):
|
||||
self.write_queue.clear()
|
||||
batch_telemetry.flush_note(
|
||||
"sglang.flush_submission_mode", "async"
|
||||
)
|
||||
# The worker owns the tracer from here on (it emits once).
|
||||
async_owned = True
|
||||
return
|
||||
# Queue full / enqueue timeout: retire the pinned copies and
|
||||
# submit synchronously on this thread. `_enqueue_async_write`
|
||||
# waited for the worker to go idle first, so the two paths
|
||||
# never submit to device_to_host_stream concurrently.
|
||||
batch_telemetry.flush_note("sglang.flush_fallback_sync", 1)
|
||||
self._retire_async_parts(ops)
|
||||
|
||||
# Any parts that were started for this queue are not consumed by the
|
||||
# sync path. Retire them unconditionally (they are released once
|
||||
# their events complete) so the pinned-byte budget never leaks.
|
||||
batch_telemetry.flush_note(
|
||||
"sglang.flush_async_skip_ops",
|
||||
sum(1 for op in ops if op.async_parts is None),
|
||||
)
|
||||
self._retire_async_parts(ops)
|
||||
batch_telemetry.flush_note("sglang.flush_submission_mode", "sync")
|
||||
with batch_telemetry.backup_phase("merge_ops"):
|
||||
op = CacheOperation.merge_ops(self.write_queue)
|
||||
tokens = len(op.device_indices)
|
||||
indices = op.device_indices
|
||||
batch_telemetry.flush_note("sglang.flush_tokens", int(tokens))
|
||||
batch_telemetry.flush_note(
|
||||
"sglang.flush_nodes_merged", len(op.node_ids)
|
||||
)
|
||||
batch_telemetry.flush_note(
|
||||
"sglang.flush_sidecar_pools", len(op.pool_transfers or [])
|
||||
)
|
||||
batch_telemetry.flush_note(
|
||||
"sglang.flush_indices_device", str(indices.device.type)
|
||||
)
|
||||
batch_telemetry.flush_note(
|
||||
"sglang.flush_indices_dtype", str(indices.dtype)
|
||||
)
|
||||
batch_telemetry.flush_note(
|
||||
"sglang.flush_indices_numel", int(indices.numel())
|
||||
)
|
||||
|
||||
with batch_telemetry.backup_phase("move_write"):
|
||||
host_indices, device_indices, pool_transfers = (
|
||||
self._move_write_operation(op)
|
||||
)
|
||||
self.write_queue.clear()
|
||||
batch_telemetry.flush_note(
|
||||
"sglang.flush_host_indices_on_cpu",
|
||||
bool(getattr(host_indices, "is_cuda", False) is False),
|
||||
)
|
||||
batch_telemetry.flush_note(
|
||||
"sglang.flush_device_indices_on_cpu",
|
||||
bool(getattr(device_indices, "is_cuda", False) is False),
|
||||
)
|
||||
|
||||
if self.write_fence_stream is not None:
|
||||
# The copy may read KV a forward is still writing: under overlap
|
||||
# scheduling a finished request is cached while the next forward
|
||||
# writes its last token, and an evicted SWA page can belong to an
|
||||
# unfinished checkpoint. Later forwards still overlap with the copy.
|
||||
with batch_telemetry.backup_phase("write_fence"):
|
||||
self.l2_transfer_engine.device_to_host_stream.wait_stream(
|
||||
self.write_fence_stream
|
||||
)
|
||||
|
||||
with batch_telemetry.backup_phase("build_l2"):
|
||||
transfers = self._l2_transfers(
|
||||
host_indices, device_indices, pool_transfers
|
||||
)
|
||||
batch_telemetry.flush_note("sglang.flush_l2_transfers", len(transfers))
|
||||
|
||||
with batch_telemetry.backup_phase("submit_d2h"):
|
||||
completion = self.l2_transfer_engine.submit_device_to_host(transfers)
|
||||
|
||||
with batch_telemetry.backup_phase("set_done_event"):
|
||||
self.mem_pool_device_allocator.set_hicache_transfer_done_event(
|
||||
(id(self), "write"), completion.finish_event
|
||||
)
|
||||
|
||||
ack = self._submit_write_operation(op)
|
||||
with batch_telemetry.backup_phase("ack_enqueue"):
|
||||
self.ack_write_queue.append(
|
||||
HiCacheAck(
|
||||
start_event=completion.start_event,
|
||||
finish_event=completion.finish_event,
|
||||
node_ids=op.node_ids,
|
||||
num_tokens=len(op.device_indices),
|
||||
timing_enabled=completion.timing_enabled,
|
||||
num_tokens_by_pool=self._num_tokens_by_pool(op),
|
||||
num_bytes=self._transfer_num_bytes(op),
|
||||
)
|
||||
)
|
||||
self.ack_write_queue.append(ack)
|
||||
batch_telemetry.flush_note(
|
||||
"sglang.flush_ack_queue_after", len(self.ack_write_queue)
|
||||
)
|
||||
finally:
|
||||
batch_telemetry.backup_flush_end(tracer)
|
||||
if not async_owned:
|
||||
batch_telemetry.backup_flush_end(tracer)
|
||||
|
||||
def _note_write_queue(self, ops: List[CacheOperation], ack_before: int) -> None:
|
||||
"""Queue-level flush dimensions (same for the sync and async paths)."""
|
||||
batch_telemetry.flush_note("sglang.flush_write_queue_ops", len(ops))
|
||||
batch_telemetry.flush_note("sglang.flush_io_backend", self.io_backend)
|
||||
batch_telemetry.flush_note(
|
||||
"sglang.flush_layout",
|
||||
str(getattr(self.mem_pool_host, "layout", None) or ""),
|
||||
)
|
||||
batch_telemetry.flush_note("sglang.flush_ack_queue_before", ack_before)
|
||||
|
||||
def _submit_write_operation(self, op: CacheOperation) -> HiCacheAck:
|
||||
"""Move indices, submit the D2H copy and build the ACK.
|
||||
|
||||
Runs on the scheduler thread (sync path) or on the async submit worker
|
||||
(async path, with CPU-resident indices). It must not touch
|
||||
``write_queue`` or ``ack_write_queue``; the caller owns those.
|
||||
"""
|
||||
tokens = len(op.device_indices)
|
||||
indices = op.device_indices
|
||||
batch_telemetry.flush_note("sglang.flush_tokens", int(tokens))
|
||||
batch_telemetry.flush_note("sglang.flush_nodes_merged", len(op.node_ids))
|
||||
batch_telemetry.flush_note(
|
||||
"sglang.flush_sidecar_pools", len(op.pool_transfers or [])
|
||||
)
|
||||
batch_telemetry.flush_note(
|
||||
"sglang.flush_indices_device", str(indices.device.type)
|
||||
)
|
||||
batch_telemetry.flush_note("sglang.flush_indices_dtype", str(indices.dtype))
|
||||
batch_telemetry.flush_note(
|
||||
"sglang.flush_indices_numel", int(indices.numel())
|
||||
)
|
||||
|
||||
with batch_telemetry.backup_phase("move_write"):
|
||||
host_indices, device_indices, pool_transfers = (
|
||||
self._move_write_operation(op)
|
||||
)
|
||||
batch_telemetry.flush_note(
|
||||
"sglang.flush_host_indices_on_cpu",
|
||||
bool(getattr(host_indices, "is_cuda", False) is False),
|
||||
)
|
||||
batch_telemetry.flush_note(
|
||||
"sglang.flush_device_indices_on_cpu",
|
||||
bool(getattr(device_indices, "is_cuda", False) is False),
|
||||
)
|
||||
|
||||
if self.write_fence_stream is not None:
|
||||
# The copy may read KV a forward is still writing: under overlap
|
||||
# scheduling a finished request is cached while the next forward
|
||||
# writes its last token, and an evicted SWA page can belong to an
|
||||
# unfinished checkpoint. Later forwards still overlap with the copy.
|
||||
with batch_telemetry.backup_phase("write_fence"):
|
||||
self.l2_transfer_engine.device_to_host_stream.wait_stream(
|
||||
self.write_fence_stream
|
||||
)
|
||||
|
||||
with batch_telemetry.backup_phase("build_l2"):
|
||||
transfers = self._l2_transfers(
|
||||
host_indices, device_indices, pool_transfers
|
||||
)
|
||||
batch_telemetry.flush_note("sglang.flush_l2_transfers", len(transfers))
|
||||
|
||||
with batch_telemetry.backup_phase("submit_d2h"):
|
||||
completion = self.l2_transfer_engine.submit_device_to_host(transfers)
|
||||
|
||||
with batch_telemetry.backup_phase("set_done_event"):
|
||||
self.mem_pool_device_allocator.set_hicache_transfer_done_event(
|
||||
(id(self), "write"), completion.finish_event
|
||||
)
|
||||
|
||||
return HiCacheAck(
|
||||
start_event=completion.start_event,
|
||||
finish_event=completion.finish_event,
|
||||
node_ids=op.node_ids,
|
||||
num_tokens=len(op.device_indices),
|
||||
timing_enabled=completion.timing_enabled,
|
||||
num_tokens_by_pool=self._num_tokens_by_pool(op),
|
||||
num_bytes=self._transfer_num_bytes(op),
|
||||
)
|
||||
|
||||
def _transfer_num_bytes(self, op: CacheOperation) -> int:
|
||||
return len(op.device_indices) * self.mem_pool_host.size_per_token
|
||||
|
||||
@@ -25,6 +25,7 @@ from __future__ import annotations
|
||||
import inspect
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from typing import (
|
||||
Callable,
|
||||
Dict,
|
||||
@@ -440,6 +441,9 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
# site; `_flush` materializes the write-set lazily, avoiding a sync here.
|
||||
self._inflight_forward: Optional[Tuple[torch.cuda.Event, torch.Tensor]] = None
|
||||
self._hicache_transfer_done_events: Dict[Hashable, torch.cuda.Event] = {}
|
||||
# The async backup submit worker records transfer events while the
|
||||
# scheduler may concurrently iterate/clear them for compaction gating.
|
||||
self._hicache_transfer_done_events_lock = threading.Lock()
|
||||
# HiCache reserves SWA physical pages during load-back, then binds them
|
||||
# into the tree before the H2D copy is submitted. Compaction must not
|
||||
# move those pages in that interval because the queued PoolTransfer still
|
||||
@@ -557,7 +561,8 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
self.live_page_count = 0
|
||||
self._inflight_forward = None
|
||||
self._latest_forward_done_event = None
|
||||
self._hicache_transfer_done_events.clear()
|
||||
with self._hicache_transfer_done_events_lock:
|
||||
self._hicache_transfer_done_events.clear()
|
||||
self._pending_hicache_load_pages = 0
|
||||
|
||||
def clear_inverse_history(self) -> None:
|
||||
@@ -2083,7 +2088,8 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
_pending_hicache_load_pages: _CapacityField[int] = _CapacityField()
|
||||
|
||||
def set_hicache_transfer_done_event(self, transfer_key: Hashable, event) -> None:
|
||||
self._hicache_transfer_done_events[transfer_key] = event
|
||||
with self._hicache_transfer_done_events_lock:
|
||||
self._hicache_transfer_done_events[transfer_key] = event
|
||||
if transfer_key == "load" or (
|
||||
isinstance(transfer_key, tuple) and transfer_key[-1] == "load"
|
||||
):
|
||||
@@ -2095,11 +2101,12 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
"""Order the current stream after outstanding HiCache copies. Only moves
|
||||
and frees need this: a copy's source and target are not free pages until
|
||||
its finish event is observed, so allocation never reuses them."""
|
||||
if not self._hicache_transfer_done_events:
|
||||
return
|
||||
with self._hicache_transfer_done_events_lock:
|
||||
if not self._hicache_transfer_done_events:
|
||||
return
|
||||
events = tuple(self._hicache_transfer_done_events.values())
|
||||
self._hicache_transfer_done_events.clear()
|
||||
current_stream = torch.cuda.current_stream()
|
||||
events = tuple(self._hicache_transfer_done_events.values())
|
||||
self._hicache_transfer_done_events.clear()
|
||||
for event in events:
|
||||
current_stream.wait_event(event)
|
||||
|
||||
|
||||
@@ -556,15 +556,17 @@ class HybridCacheController(BaseHiCacheController):
|
||||
return None
|
||||
host_indices, pool_transfers = allocation
|
||||
|
||||
self.write_queue.append(
|
||||
CacheOperation(
|
||||
host_indices,
|
||||
device_indices,
|
||||
node_id,
|
||||
priority,
|
||||
pool_transfers=pool_transfers or None,
|
||||
)
|
||||
op = CacheOperation(
|
||||
host_indices,
|
||||
device_indices,
|
||||
node_id,
|
||||
priority,
|
||||
pool_transfers=pool_transfers or None,
|
||||
)
|
||||
# Start the pinned index D2H before the (possibly deferred) flush, so
|
||||
# the copy can run while the scheduler keeps going.
|
||||
self._start_async_index_copy(op)
|
||||
self.write_queue.append(op)
|
||||
if flush:
|
||||
self.start_writing()
|
||||
return host_indices
|
||||
|
||||
@@ -3505,6 +3505,13 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
if write_back:
|
||||
# Blocking: submit what is still queued, then wait for every ack.
|
||||
cc.start_writing()
|
||||
# Async submissions cover ACKs asynchronously; wait for them before
|
||||
# asserting that every write-through node has been acknowledged.
|
||||
if hasattr(cc, "wait_write_submissions_idle"):
|
||||
cc.wait_write_submissions_idle()
|
||||
_drain = getattr(cc, "drain_write_submission_results", None)
|
||||
if _drain is not None:
|
||||
_drain()
|
||||
while self.ongoing_write_through:
|
||||
for ack in cc.ack_write_queue:
|
||||
ack.finish_event.synchronize()
|
||||
@@ -3517,6 +3524,9 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
return
|
||||
|
||||
if finish_count is None:
|
||||
_drain = getattr(cc, "drain_write_submission_results", None)
|
||||
if _drain is not None:
|
||||
_drain()
|
||||
# Every rank must enter the all_reduce below; ongoing_write_through can
|
||||
# diverge across ranks (e.g. write_backup returning 0 on a subset).
|
||||
finish_count = 0
|
||||
@@ -3694,9 +3704,16 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
|
||||
# Reap the previous round's PP-sync sends before issuing new ones.
|
||||
self._drain_async_work()
|
||||
# Collect ACKs finished by the async submit worker before counting them.
|
||||
# getattr keeps lightweight controller fakes (unit tests) working.
|
||||
_drain = getattr(self.cache_controller, "drain_write_submission_results", None)
|
||||
if _drain is not None:
|
||||
_drain()
|
||||
# Backups queued outside process_batch_result: the chunked-prefill stash
|
||||
# in get_next_batch_to_run, abort_request, and the PD prefill release.
|
||||
self.flush_pending_backups()
|
||||
if _drain is not None:
|
||||
_drain()
|
||||
|
||||
(
|
||||
write_finish_count,
|
||||
|
||||
@@ -1409,6 +1409,8 @@ class BackupFlushTracer:
|
||||
__slots__ = (
|
||||
"origin",
|
||||
"start_ns",
|
||||
"loop_seq",
|
||||
"loop_ctx",
|
||||
"phases",
|
||||
"attrs",
|
||||
"counts",
|
||||
@@ -1418,6 +1420,10 @@ class BackupFlushTracer:
|
||||
|
||||
def __init__(self, origin: str):
|
||||
self.origin = origin
|
||||
# Captured on the scheduler thread: the flushed span may be emitted by
|
||||
# the async submit worker, which has no scheduler loop TLS.
|
||||
self.loop_seq = current_loop_seq()
|
||||
self.loop_ctx = current_loop_context_for_spans()
|
||||
self.start_ns = time.monotonic_ns()
|
||||
self.phases: dict[str, int] = {}
|
||||
self.attrs: dict[str, Any] = {}
|
||||
@@ -1463,7 +1469,7 @@ class BackupFlushTracer:
|
||||
attrs = {
|
||||
"sglang.flush_origin": self.origin,
|
||||
"sglang.flush_total_ms": total_ns / 1e6,
|
||||
"sglang.loop_seq": current_loop_seq(),
|
||||
"sglang.loop_seq": int(self.loop_seq or 0),
|
||||
}
|
||||
for name, ns in self.phases.items():
|
||||
attrs["sglang.flush_" + name.replace(".", "_") + "_ms"] = ns / 1e6
|
||||
@@ -1476,7 +1482,7 @@ class BackupFlushTracer:
|
||||
try:
|
||||
from opentelemetry import trace as otel_trace
|
||||
|
||||
ctx = current_loop_context_for_spans()
|
||||
ctx = self.loop_ctx
|
||||
start_wall_ns = time.time_ns() - total_ns
|
||||
span = otel_trace.get_tracer("sglang.cache.flush").start_span(
|
||||
"scheduler.cache_flush_backup",
|
||||
@@ -1543,6 +1549,25 @@ def current_flush_tracer() -> Optional[BackupFlushTracer]:
|
||||
return getattr(_FLUSH_TLS, "tracer", None)
|
||||
|
||||
|
||||
def backup_flush_attach(tracer: Optional[BackupFlushTracer]):
|
||||
"""Bind ``tracer`` to this thread so sub-phases land on the same span.
|
||||
|
||||
Used by the async submit worker, which executes phases on a different
|
||||
thread than the one that created the tracer. Returns the previous binding.
|
||||
"""
|
||||
if not _BACKUP_DETAIL_ENABLED:
|
||||
return None
|
||||
previous = getattr(_FLUSH_TLS, "tracer", None)
|
||||
_FLUSH_TLS.tracer = tracer
|
||||
return previous
|
||||
|
||||
|
||||
def backup_flush_detach(previous) -> None:
|
||||
if not _BACKUP_DETAIL_ENABLED:
|
||||
return
|
||||
_FLUSH_TLS.tracer = previous
|
||||
|
||||
|
||||
def flush_note(key: str, value: Any) -> None:
|
||||
"""Attach a dimension to the active flush (no-op when disabled)."""
|
||||
if not _BACKUP_DETAIL_ENABLED:
|
||||
|
||||
@@ -0,0 +1,253 @@
|
||||
"""CUDA ordering/timeline tests for the per-op async backup index copy.
|
||||
|
||||
Plan: handoff/prefill-backup-overlap-next-20261107.md §3/§8.2.
|
||||
|
||||
Run inside the GPU container with the overlay source:
|
||||
|
||||
PYTHONPATH=/tmp/ovr python -m pytest \
|
||||
test/registered/cuda/test_async_backup_index_copy_cuda.py -q
|
||||
|
||||
What is proven on real hardware:
|
||||
|
||||
1. ``_start_async_index_copy`` returns while a long kernel is still queued on
|
||||
the scheduler stream, and the synchronous ``.cpu()`` on that stream blocks;
|
||||
2. the copy is only released after that fence, but the *next* forward (a
|
||||
marker kernel on another stream, ordered behind the same fence) runs
|
||||
concurrently with it: the copy end timestamp falls strictly inside the
|
||||
marker's [start, end] interval;
|
||||
3. the host can enqueue the next batch before ``ready_event`` completes and
|
||||
the event is not reported ready early;
|
||||
4. with three ops whose copies complete out of order, the worker still merges
|
||||
them in op order, identical to ``CacheOperation.merge_ops`` over the CPU
|
||||
reference tensors;
|
||||
5. the copy stream is distinct from the scheduler stream.
|
||||
"""
|
||||
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.managers import cache_controller as cc_mod
|
||||
|
||||
|
||||
class _FakeHostPool:
|
||||
"""Not a HostKVCache: uses_shared_host_layout() is false, i.e. the sync path
|
||||
would call device_indices.cpu()."""
|
||||
|
||||
shared_allocation_domain = None
|
||||
|
||||
|
||||
class _FakeEntry:
|
||||
def __init__(self):
|
||||
self.host_pool = _FakeHostPool()
|
||||
|
||||
|
||||
class _FakeMemPoolHost:
|
||||
layout = "page_first_direct"
|
||||
|
||||
def __init__(self):
|
||||
self.anchor_entry = _FakeEntry()
|
||||
self.entry_map = {}
|
||||
|
||||
|
||||
def _make_controller():
|
||||
c = cc_mod.HiCacheController.__new__(cc_mod.HiCacheController)
|
||||
c.io_backend = "direct"
|
||||
c.mem_pool_host = _FakeMemPoolHost()
|
||||
c.write_queue = []
|
||||
c.load_queue = []
|
||||
c.ack_write_queue = []
|
||||
c.ack_load_queue = []
|
||||
c._indices_stream = None
|
||||
c._write_submit_queue = None
|
||||
c._write_result_queue = cc_mod.Queue()
|
||||
c._write_submit_stop = threading.Event()
|
||||
c._write_submit_thread = None
|
||||
c._write_submit_state_lock = threading.Lock()
|
||||
c._write_submit_cond = threading.Condition(c._write_submit_state_lock)
|
||||
c._write_submit_active = 0
|
||||
c._async_pinned_bytes = 0
|
||||
c._retire_parts = []
|
||||
return c
|
||||
|
||||
|
||||
def _op(device_indices, node_id):
|
||||
host = torch.empty(device_indices.numel(), dtype=torch.int64)
|
||||
return cc_mod.CacheOperation(host, device_indices, node_id)
|
||||
|
||||
|
||||
class AsyncBackupIndexCopyCudaTest(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cc_mod._ASYNC_BACKUP_SUBMIT = True
|
||||
cls.saved_pin = cc_mod._ASYNC_BACKUP_PINNED_BYTES_MAX
|
||||
cc_mod._ASYNC_BACKUP_PINNED_BYTES_MAX = 1 << 30
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cc_mod._ASYNC_BACKUP_PINNED_BYTES_MAX = cls.saved_pin
|
||||
|
||||
def test_start_does_not_block_and_copy_is_async(self):
|
||||
torch.cuda.synchronize()
|
||||
sched = torch.cuda.current_stream()
|
||||
fwd = torch.cuda.Stream()
|
||||
indices = torch.arange(4096, dtype=torch.int64, device="cuda")
|
||||
|
||||
# Simulated in-flight forward plus the scheduler's WAR fence.
|
||||
with torch.cuda.stream(fwd):
|
||||
torch.cuda._sleep(2_000_000_000)
|
||||
sched.wait_stream(fwd)
|
||||
|
||||
c = _make_controller()
|
||||
op = _op(indices, 11)
|
||||
t0 = time.perf_counter()
|
||||
c._start_async_index_copy(op)
|
||||
start_elapsed = time.perf_counter() - t0
|
||||
self.assertIsNotNone(op.async_parts)
|
||||
# Starting the copy must not block on the simulated forward.
|
||||
self.assertLess(start_elapsed, 0.05, start_elapsed)
|
||||
self.assertFalse(op.async_parts.ready_event.query())
|
||||
|
||||
# The synchronous baseline on the same stream does block.
|
||||
t1 = time.perf_counter()
|
||||
reference = indices.cpu()
|
||||
sync_elapsed = time.perf_counter() - t1
|
||||
self.assertGreater(sync_elapsed, 0.1, sync_elapsed)
|
||||
|
||||
op.async_parts.ready_event.synchronize()
|
||||
self.assertTrue(op.async_parts.ready_event.query())
|
||||
c._materialize_async_indices([op])
|
||||
self.assertTrue(bool(torch.equal(op.device_indices, reference)))
|
||||
self.assertFalse(op.device_indices.is_cuda)
|
||||
c._release_async_parts([op])
|
||||
self.assertEqual(c._async_pinned_bytes, 0)
|
||||
|
||||
def test_next_forward_overlaps_the_index_copy(self):
|
||||
torch.cuda.synchronize()
|
||||
sched = torch.cuda.current_stream()
|
||||
fwd = torch.cuda.Stream()
|
||||
# Large enough that the copy takes long enough to observe (32 MB).
|
||||
indices = torch.arange(4 * 1024 * 1024, dtype=torch.int64, device="cuda")
|
||||
|
||||
# 1. A long fence on the scheduler stream (the in-flight forward).
|
||||
with torch.cuda.stream(sched):
|
||||
fence_done = torch.cuda.Event(enable_timing=True)
|
||||
fence_done.record()
|
||||
torch.cuda._sleep(1_000_000_000)
|
||||
|
||||
# 2. Per-op pinned copy started now; it inherits the fence.
|
||||
c = _make_controller()
|
||||
op = _op(indices, 3)
|
||||
c._start_async_index_copy(op)
|
||||
self.assertIsNotNone(op.async_parts)
|
||||
|
||||
# 3. The next batch launch waits for the scheduler stream (as the real
|
||||
# forward_stream does) and then runs a long marker kernel.
|
||||
copy_end = torch.cuda.Event(enable_timing=True)
|
||||
with torch.cuda.stream(c._indices_stream):
|
||||
copy_end.record()
|
||||
launch_host_ns = time.monotonic_ns()
|
||||
with torch.cuda.stream(fwd):
|
||||
fwd.wait_stream(sched)
|
||||
fwd_start = torch.cuda.Event(enable_timing=True)
|
||||
fwd_start.record()
|
||||
torch.cuda._sleep(1_500_000_000)
|
||||
fwd_end = torch.cuda.Event(enable_timing=True)
|
||||
fwd_end.record()
|
||||
|
||||
# 4. The event must not be ready while the fence is in flight, and the
|
||||
# host has already enqueued the next batch.
|
||||
ready_seen_ns = [None]
|
||||
|
||||
def _wait_ready():
|
||||
op.async_parts.ready_event.synchronize()
|
||||
ready_seen_ns[0] = time.monotonic_ns()
|
||||
|
||||
waiter = threading.Thread(target=_wait_ready)
|
||||
waiter.start()
|
||||
self.assertIsNone(ready_seen_ns[0])
|
||||
waiter.join(timeout=30)
|
||||
self.assertIsNotNone(ready_seen_ns[0])
|
||||
self.assertLess(launch_host_ns, ready_seen_ns[0])
|
||||
|
||||
torch.cuda.synchronize()
|
||||
# CUDA timestamps: copy end strictly inside the next forward.
|
||||
self.assertGreater(fwd_start.elapsed_time(copy_end), 0.0)
|
||||
self.assertGreater(copy_end.elapsed_time(fwd_end), 0.0)
|
||||
self.assertGreater(fwd_start.elapsed_time(fwd_end), 0.5)
|
||||
|
||||
c._materialize_async_indices([op])
|
||||
self.assertTrue(
|
||||
bool(torch.equal(op.device_indices, indices.cpu()))
|
||||
)
|
||||
c._release_async_parts([op])
|
||||
|
||||
def test_copy_stream_is_distinct_from_scheduler(self):
|
||||
torch.cuda.synchronize()
|
||||
c = _make_controller()
|
||||
op = _op(torch.arange(8, dtype=torch.int64, device="cuda"), 0)
|
||||
c._start_async_index_copy(op)
|
||||
self.assertIsNotNone(c._indices_stream)
|
||||
self.assertNotEqual(
|
||||
c._indices_stream.cuda_stream,
|
||||
torch.cuda.current_stream().cuda_stream,
|
||||
)
|
||||
op.async_parts.ready_event.synchronize()
|
||||
c._release_async_parts([op])
|
||||
|
||||
def test_out_of_order_completion_keeps_op_order(self):
|
||||
torch.cuda.synchronize()
|
||||
c = _make_controller()
|
||||
delays = {0: 600_000_000, 1: 150_000_000, 2: 300_000_000}
|
||||
ops = []
|
||||
references = []
|
||||
for i in range(3):
|
||||
src = torch.arange(8, dtype=torch.int64, device="cuda") + i * 100
|
||||
op = _op(src, node_id=i)
|
||||
buf = torch.empty(src.shape, dtype=src.dtype, device="cpu", pin_memory=True)
|
||||
stream = torch.cuda.Stream()
|
||||
with torch.cuda.stream(stream):
|
||||
torch.cuda._sleep(delays[i])
|
||||
buf.copy_(src, non_blocking=True)
|
||||
ev = torch.cuda.Event()
|
||||
ev.record(stream)
|
||||
op.async_parts = cc_mod.AsyncIndexParts(
|
||||
[cc_mod.AsyncIndexCopy((None, None), src, buf, buf.numel() * 8)],
|
||||
ev,
|
||||
buf.numel() * 8,
|
||||
)
|
||||
ops.append(op)
|
||||
references.append(src.cpu())
|
||||
|
||||
captured = []
|
||||
c._submit_write_operation = lambda merged: (
|
||||
captured.append(merged.device_indices.clone())
|
||||
or cc_mod.HiCacheAck(None, None, list(merged.node_ids), num_tokens=8)
|
||||
)
|
||||
c._write_submit_queue = cc_mod.Queue(maxsize=4)
|
||||
worker = threading.Thread(target=c._write_submit_worker, daemon=True)
|
||||
c._write_submit_thread = worker
|
||||
worker.start()
|
||||
try:
|
||||
c._write_submit_active = 1
|
||||
c._write_submit_queue.put(cc_mod.WriteSubmission(ops, None, 0))
|
||||
deadline = time.time() + 30
|
||||
while c._write_submit_active and time.time() < deadline:
|
||||
time.sleep(0.01)
|
||||
self.assertEqual(c._write_submit_active, 0)
|
||||
finally:
|
||||
c._write_submit_queue.put(None)
|
||||
worker.join(timeout=10)
|
||||
|
||||
self.assertEqual(len(captured), 1)
|
||||
expected = torch.cat(references)
|
||||
self.assertTrue(bool(torch.equal(captured[0], expected)))
|
||||
self.assertEqual(ops[0].node_ids, [0])
|
||||
self.assertEqual(ops[1].node_ids, [1])
|
||||
self.assertEqual(ops[2].node_ids, [2])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,550 @@
|
||||
"""Unit tests for the async HiCache backup submission state machine.
|
||||
|
||||
Design: handoff/prefill-backup-overlap-next-20261107.md §8.1.
|
||||
|
||||
These tests exercise the scheduler/worker split without a GPU: the index copy
|
||||
itself is covered by the CUDA test, and here the device stream/event and the
|
||||
submit path are faked. What is checked:
|
||||
|
||||
* disabled / inapplicable shapes keep the sync path;
|
||||
* per-op pinned parts are materialized per (pool, indices_from_pool) key;
|
||||
* the worker waits every ready event before touching host buffers;
|
||||
* exactly-once ACK, fail-closed errors, bounded queues, idle/stop lifecycle;
|
||||
* has_inflight_device_transfers reflects queued/running results.
|
||||
"""
|
||||
|
||||
import threading
|
||||
import time
|
||||
import types
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
from sglang.srt.managers import cache_controller as cc_mod
|
||||
from sglang.srt.mem_cache.hicache_storage import PoolName, PoolTransfer
|
||||
|
||||
|
||||
class _FakeEvent:
|
||||
def __init__(self, worker_log=None, name="ev"):
|
||||
self.worker_log = worker_log
|
||||
self.name = name
|
||||
self._set = False
|
||||
self.waited = 0
|
||||
|
||||
def record(self, stream=None):
|
||||
self._set = True
|
||||
|
||||
def synchronize(self):
|
||||
self.waited += 1
|
||||
if self.worker_log is not None:
|
||||
self.worker_log.append(("wait", self.name))
|
||||
self._set = True
|
||||
|
||||
def query(self):
|
||||
return self._set
|
||||
|
||||
|
||||
class _FakeStream:
|
||||
def __init__(self, name="stream", log=None):
|
||||
self.name = name
|
||||
self.log = log
|
||||
self.waited = 0
|
||||
|
||||
def wait_stream(self, other):
|
||||
self.waited += 1
|
||||
if self.log is not None:
|
||||
self.log.append(("wait_stream", self.name, getattr(other, "name", "?")))
|
||||
|
||||
def wait_event(self, event):
|
||||
pass
|
||||
|
||||
|
||||
class _FakeStreamCtx:
|
||||
def __init__(self, stream):
|
||||
self.stream = stream
|
||||
|
||||
def __enter__(self):
|
||||
return self.stream
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
|
||||
class _FakeCudaModule:
|
||||
def __init__(self, log=None):
|
||||
self.log = log
|
||||
self.current = _FakeStream("schedule", log)
|
||||
|
||||
def Stream(self, *a, **k):
|
||||
return _FakeStream("indices", self.log)
|
||||
|
||||
def StreamContext(self, stream):
|
||||
return _FakeStreamCtx(stream)
|
||||
|
||||
def Event(self, *a, **k):
|
||||
return _FakeEvent(self.log)
|
||||
|
||||
def current_stream(self):
|
||||
return self.current
|
||||
|
||||
|
||||
class _FakeTensor:
|
||||
def __init__(self, numel=8, element_size=8, is_cuda=True, tag="dev"):
|
||||
self._numel = numel
|
||||
self._element_size = element_size
|
||||
self.is_cuda = is_cuda
|
||||
self.tag = tag
|
||||
self.recorded = []
|
||||
self.cpu_calls = 0
|
||||
self.device = types.SimpleNamespace(type="cuda" if is_cuda else "cpu")
|
||||
self.dtype = "torch.int64"
|
||||
|
||||
def numel(self):
|
||||
return self._numel
|
||||
|
||||
def element_size(self):
|
||||
return self._element_size
|
||||
|
||||
def record_stream(self, stream):
|
||||
self.recorded.append(stream)
|
||||
|
||||
def copy_(self, src, non_blocking=False):
|
||||
self.copied_from = src
|
||||
|
||||
@property
|
||||
def shape(self):
|
||||
return (self._numel,)
|
||||
|
||||
def cpu(self):
|
||||
self.cpu_calls += 1
|
||||
return _FakeTensor(self._numel, self._element_size, False, tag=self.tag + "-cpu")
|
||||
|
||||
def __len__(self):
|
||||
return self._numel
|
||||
|
||||
|
||||
class _FakeHostPool:
|
||||
def __init__(self, shared=False):
|
||||
self.shared = shared
|
||||
|
||||
|
||||
class _FakeEntry:
|
||||
def __init__(self, host_pool):
|
||||
self.host_pool = host_pool
|
||||
|
||||
|
||||
class _FakeMemPoolHost:
|
||||
layout = "page_first_direct"
|
||||
|
||||
def __init__(self, anchor_shared=False, sidecars=None):
|
||||
self.anchor_entry = _FakeEntry(_FakeHostPool(anchor_shared))
|
||||
self.entry_map = {
|
||||
name: _FakeEntry(pool) for name, pool in (sidecars or {}).items()
|
||||
}
|
||||
|
||||
|
||||
def _make_controller(*, enabled=True, anchor_shared=False, sidecars=None, policy=""):
|
||||
c = cc_mod.HiCacheController.__new__(cc_mod.HiCacheController)
|
||||
c.io_backend = "direct"
|
||||
c.write_policy = policy
|
||||
c.mem_pool_host = _FakeMemPoolHost(anchor_shared, sidecars)
|
||||
c.write_queue = []
|
||||
c.load_queue = []
|
||||
c.ack_write_queue = []
|
||||
c.ack_load_queue = []
|
||||
c._indices_stream = None
|
||||
c._write_submit_queue = None
|
||||
c._write_result_queue = cc_mod.Queue()
|
||||
c._write_submit_stop = threading.Event()
|
||||
c._write_submit_thread = None
|
||||
c._write_submit_state_lock = threading.Lock()
|
||||
c._write_submit_cond = threading.Condition(c._write_submit_state_lock)
|
||||
c._write_submit_active = 0
|
||||
c._async_pinned_bytes = 0
|
||||
c._retire_parts = []
|
||||
return c
|
||||
|
||||
|
||||
def _op(controller, *, anchor=None, transfers=(), node_id=7):
|
||||
host = _FakeTensor(8, 8, is_cuda=False, tag="host")
|
||||
dev = anchor or _FakeTensor(8, 8, is_cuda=True, tag="anchor")
|
||||
op = cc_mod.CacheOperation(host, dev, node_id, pool_transfers=list(transfers))
|
||||
return op
|
||||
|
||||
|
||||
class AsyncBackupUnitTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
patcher = mock.patch.object(cc_mod, "_ASYNC_BACKUP_SUBMIT", True)
|
||||
patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
# The real predicate isinstance-checks HostKVCache; the fakes expose a
|
||||
# plain `shared` flag instead.
|
||||
shared_patcher = mock.patch.object(
|
||||
cc_mod, "uses_shared_host_layout", lambda pool: bool(getattr(pool, "shared", False))
|
||||
)
|
||||
shared_patcher.start()
|
||||
self.addCleanup(shared_patcher.stop)
|
||||
cat_patcher = mock.patch.object(
|
||||
cc_mod.torch,
|
||||
"cat",
|
||||
side_effect=lambda parts: _FakeTensor(
|
||||
sum(len(p) for p in parts), 8, False, "cat"
|
||||
),
|
||||
)
|
||||
cat_patcher.start()
|
||||
self.addCleanup(cat_patcher.stop)
|
||||
# Pinned destination buffers are faked; only the copy_ call is checked.
|
||||
self.pin_patch = mock.patch.object(
|
||||
cc_mod.torch, "empty", side_effect=lambda *a, **k: _FakeTensor(8, 8, False, "pin")
|
||||
)
|
||||
self.pin_patch.start()
|
||||
self.addCleanup(self.pin_patch.stop)
|
||||
|
||||
def test_disabled_keeps_sync_path(self):
|
||||
with mock.patch.object(cc_mod, "_ASYNC_BACKUP_SUBMIT", False):
|
||||
c = _make_controller()
|
||||
self.assertFalse(c.async_backup_enabled())
|
||||
self.assertFalse(c._async_submit_applicable())
|
||||
|
||||
def test_wrong_backend_or_layout_is_ineligible(self):
|
||||
c = _make_controller()
|
||||
c.io_backend = "kernel"
|
||||
self.assertFalse(c.async_backup_enabled())
|
||||
c = _make_controller()
|
||||
c.mem_pool_host.layout = "page_first"
|
||||
self.assertFalse(c.async_backup_enabled())
|
||||
|
||||
def test_index_targets_skip_shared_layout_and_cpu_tensors(self):
|
||||
shared = PoolName.SWA
|
||||
c = _make_controller(sidecars={shared: _FakeHostPool(shared=True)})
|
||||
transfer = PoolTransfer(
|
||||
name=shared, device_indices=_FakeTensor(4, 8, True, "swa")
|
||||
)
|
||||
cpu_anchor = _FakeTensor(8, 8, False, "anchor")
|
||||
op = _op(c, anchor=cpu_anchor, transfers=[transfer])
|
||||
self.assertEqual(c._async_index_targets(op), [])
|
||||
|
||||
c = _make_controller(sidecars={PoolName.SWA: _FakeHostPool(shared=False)})
|
||||
transfer = PoolTransfer(
|
||||
name=PoolName.SWA,
|
||||
device_indices=_FakeTensor(4, 8, True, "swa"),
|
||||
indices_from_pool=PoolName.KV,
|
||||
)
|
||||
anchor = _FakeTensor(8, 8, True, "anchor")
|
||||
op = _op(c, anchor=anchor, transfers=[transfer])
|
||||
targets = c._async_index_targets(op)
|
||||
self.assertEqual([k for k, _ in targets], [(None, None), (PoolName.SWA, PoolName.KV)])
|
||||
|
||||
def test_start_copy_orders_after_scheduler_and_records_event(self):
|
||||
log = []
|
||||
c = _make_controller()
|
||||
op = _op(c)
|
||||
with mock.patch.object(cc_mod, "device_module", _FakeCudaModule(log)):
|
||||
c._start_async_index_copy(op)
|
||||
self.assertIsNotNone(op.async_parts)
|
||||
self.assertEqual(op.async_parts.ready_event._set, True)
|
||||
self.assertEqual(op.device_indices.recorded[0].name, "indices")
|
||||
# The copy stream is ordered after the scheduler stream once.
|
||||
self.assertIn(("wait_stream", "indices", "schedule"), log)
|
||||
self.assertEqual(c._async_pinned_bytes, 64)
|
||||
|
||||
def test_pinned_cap_falls_back_to_sync(self):
|
||||
c = _make_controller()
|
||||
op = _op(c)
|
||||
with mock.patch.object(cc_mod, "_ASYNC_BACKUP_PINNED_BYTES_MAX", 1), \
|
||||
mock.patch.object(cc_mod, "device_module", _FakeCudaModule()):
|
||||
c._start_async_index_copy(op)
|
||||
self.assertIsNone(op.async_parts)
|
||||
self.assertEqual(c._async_pinned_bytes, 0)
|
||||
|
||||
def test_materialize_uses_pool_keys(self):
|
||||
c = _make_controller()
|
||||
anchor_copy = cc_mod.AsyncIndexCopy((None, None), None, _FakeTensor(8, 8, False, "a"), 64)
|
||||
deriv = PoolTransfer(
|
||||
name=PoolName.SWA,
|
||||
device_indices=_FakeTensor(4, 8, True, "swa"),
|
||||
indices_from_pool=PoolName.KV,
|
||||
)
|
||||
side_copy = cc_mod.AsyncIndexCopy(
|
||||
(PoolName.SWA, PoolName.KV), None, _FakeTensor(4, 8, False, "s"), 32
|
||||
)
|
||||
op = _op(c, transfers=[deriv])
|
||||
op.async_parts = cc_mod.AsyncIndexParts(
|
||||
copies=[anchor_copy, side_copy], ready_event=_FakeEvent(), nbytes=96
|
||||
)
|
||||
c._materialize_async_indices([op])
|
||||
self.assertIs(op.device_indices, anchor_copy.dst)
|
||||
self.assertIs(deriv.device_indices, side_copy.dst)
|
||||
|
||||
def test_worker_waits_events_then_submits_and_acks_once(self):
|
||||
log = []
|
||||
c = _make_controller()
|
||||
events = []
|
||||
|
||||
def _make_op(tag):
|
||||
op = _op(c, anchor=_FakeTensor(8, 8, True, tag))
|
||||
ev = _FakeEvent(log, tag)
|
||||
events.append(ev)
|
||||
op.async_parts = cc_mod.AsyncIndexParts(
|
||||
copies=[
|
||||
cc_mod.AsyncIndexCopy(
|
||||
(None, None), op.device_indices, _FakeTensor(8, 8, False, tag), 64
|
||||
)
|
||||
],
|
||||
ready_event=ev,
|
||||
nbytes=64,
|
||||
)
|
||||
return op
|
||||
|
||||
ops = [_make_op("a"), _make_op("b")]
|
||||
c._async_pinned_bytes = 128
|
||||
submitted = []
|
||||
|
||||
def _submit(op):
|
||||
# Every ready event must have been waited before the merge is used.
|
||||
submitted.append(list(op.node_ids))
|
||||
return cc_mod.HiCacheAck(None, None, list(op.node_ids), num_tokens=8)
|
||||
|
||||
c._submit_write_operation = _submit
|
||||
c._write_submit_queue = cc_mod.Queue(maxsize=4)
|
||||
c._ensure_write_submit_thread = lambda: None
|
||||
worker = threading.Thread(target=c._write_submit_worker, daemon=True)
|
||||
c._write_submit_thread = worker
|
||||
worker.start()
|
||||
try:
|
||||
item = cc_mod.WriteSubmission(ops=ops, tracer=None, enqueue_mono_ns=0)
|
||||
c._write_submit_active = 1
|
||||
c._write_submit_queue.put(item)
|
||||
deadline = time.time() + 5
|
||||
while c._write_submit_active and time.time() < deadline:
|
||||
time.sleep(0.005)
|
||||
self.assertEqual(c._write_submit_active, 0)
|
||||
finally:
|
||||
c._write_submit_queue.put(None)
|
||||
worker.join(timeout=5)
|
||||
self.assertEqual([e.waited for e in events], [1, 1])
|
||||
self.assertEqual(submitted, [[7, 7]])
|
||||
self.assertEqual(c._async_pinned_bytes, 0)
|
||||
c.drain_write_submission_results()
|
||||
self.assertEqual(len(c.ack_write_queue), 1)
|
||||
# Second drain is a no-op: exactly-once ACK.
|
||||
c.drain_write_submission_results()
|
||||
self.assertEqual(len(c.ack_write_queue), 1)
|
||||
|
||||
def test_worker_error_is_fail_closed(self):
|
||||
c = _make_controller()
|
||||
op = _op(c)
|
||||
op.async_parts = cc_mod.AsyncIndexParts([], _FakeEvent(), 0)
|
||||
c._submit_write_operation = mock.Mock(side_effect=RuntimeError("boom"))
|
||||
c._write_submit_queue = cc_mod.Queue(maxsize=4)
|
||||
worker = threading.Thread(target=c._write_submit_worker, daemon=True)
|
||||
c._write_submit_thread = worker
|
||||
worker.start()
|
||||
c._write_submit_active = 1
|
||||
c._write_submit_queue.put(cc_mod.WriteSubmission([op], None, 0))
|
||||
deadline = time.time() + 5
|
||||
while c._write_submit_active and time.time() < deadline:
|
||||
time.sleep(0.005)
|
||||
c._write_submit_queue.put(None)
|
||||
worker.join(timeout=5)
|
||||
with self.assertRaises(RuntimeError):
|
||||
c.drain_write_submission_results()
|
||||
self.assertEqual(c.ack_write_queue, [])
|
||||
|
||||
def test_enqueue_failure_keeps_active_consistent(self):
|
||||
c = _make_controller()
|
||||
c._write_submit_queue = cc_mod.Queue(maxsize=1)
|
||||
c._write_submit_queue.put("occupied")
|
||||
c._ensure_write_submit_thread = lambda: None
|
||||
with mock.patch.object(cc_mod, "_ASYNC_BACKUP_ENQUEUE_TIMEOUT", 0.05):
|
||||
ok = c._enqueue_async_write([_op(c)], None)
|
||||
self.assertFalse(ok)
|
||||
self.assertEqual(c._write_submit_active, 0)
|
||||
|
||||
def test_queue_full_falls_back_to_sync_submit(self):
|
||||
c = _make_controller()
|
||||
# A permanently-full submit queue forces the sync path.
|
||||
c._write_submit_queue = cc_mod.Queue(maxsize=1)
|
||||
c._write_submit_queue.put("occupied")
|
||||
c._ensure_write_submit_thread = lambda: None
|
||||
op = _op(c)
|
||||
retired_event = _FakeEvent()
|
||||
retired_event.record()
|
||||
op.async_parts = cc_mod.AsyncIndexParts(
|
||||
[cc_mod.AsyncIndexCopy((None, None), op.device_indices, _FakeTensor(8, 8, False), 64)],
|
||||
retired_event,
|
||||
64,
|
||||
)
|
||||
c._async_pinned_bytes = 64
|
||||
c.write_queue.append(op)
|
||||
submitted = []
|
||||
|
||||
def _submit(merged):
|
||||
# The fallback must hand over the original device indices, i.e. the
|
||||
# pinned parts are retired rather than materialized.
|
||||
submitted.append(merged.device_indices)
|
||||
return cc_mod.HiCacheAck(None, None, list(merged.node_ids), num_tokens=8)
|
||||
|
||||
c._submit_write_operation = _submit
|
||||
with mock.patch.object(cc_mod, "_ASYNC_BACKUP_ENQUEUE_TIMEOUT", 0.05):
|
||||
c.start_writing()
|
||||
self.assertEqual(len(submitted), 1)
|
||||
self.assertIs(submitted[0], op.device_indices)
|
||||
self.assertEqual(len(c.ack_write_queue), 1)
|
||||
self.assertEqual(c.write_queue, [])
|
||||
self.assertIsNone(op.async_parts)
|
||||
self.assertEqual(c._write_submit_active, 0)
|
||||
# The retired copy is released once its event completes.
|
||||
c._reap_retired_parts()
|
||||
self.assertEqual(c._async_pinned_bytes, 0)
|
||||
|
||||
def test_mixed_queue_sync_path_retires_unconsumed_parts(self):
|
||||
c = _make_controller()
|
||||
ev = _FakeEvent()
|
||||
ev.record()
|
||||
with_parts = _op(c, node_id=1)
|
||||
with_parts.async_parts = cc_mod.AsyncIndexParts(
|
||||
[
|
||||
cc_mod.AsyncIndexCopy(
|
||||
(None, None), with_parts.device_indices, _FakeTensor(8, 8, False), 64
|
||||
)
|
||||
],
|
||||
ev,
|
||||
64,
|
||||
)
|
||||
without_parts = _op(c, node_id=2)
|
||||
c.write_queue.extend([with_parts, without_parts])
|
||||
c._async_pinned_bytes = 64
|
||||
submitted = []
|
||||
c._submit_write_operation = lambda merged: (
|
||||
submitted.append(list(merged.node_ids))
|
||||
or cc_mod.HiCacheAck(None, None, list(merged.node_ids), num_tokens=8)
|
||||
)
|
||||
c.start_writing()
|
||||
# Mixed residency () forces the sync path, but the pinned copy started
|
||||
# for the first op must be retired and its budget released.
|
||||
self.assertEqual(submitted, [[1, 2]])
|
||||
self.assertIsNone(with_parts.async_parts)
|
||||
self.assertEqual(c._async_pinned_bytes, 0)
|
||||
self.assertEqual(len(c.ack_write_queue), 1)
|
||||
|
||||
def test_op_cap_falls_back_to_sync(self):
|
||||
c = _make_controller()
|
||||
with mock.patch.object(cc_mod, "_ASYNC_BACKUP_MAX_OPS", 1):
|
||||
for i in range(2):
|
||||
op = _op(c, node_id=i)
|
||||
op.async_parts = cc_mod.AsyncIndexParts([], _FakeEvent(), 0)
|
||||
c.write_queue.append(op)
|
||||
self.assertFalse(c._async_submit_applicable())
|
||||
|
||||
def test_inflight_reflects_async_state(self):
|
||||
c = _make_controller()
|
||||
self.assertFalse(c.has_inflight_device_transfers())
|
||||
c._write_submit_active = 1
|
||||
self.assertTrue(c.has_inflight_device_transfers())
|
||||
c._write_submit_active = 0
|
||||
c._async_pinned_bytes = 8
|
||||
self.assertTrue(c.has_inflight_device_transfers())
|
||||
c._async_pinned_bytes = 0
|
||||
c._write_result_queue.put(cc_mod.WriteResult(None, [], None))
|
||||
self.assertTrue(c.has_inflight_device_transfers())
|
||||
c._write_result_queue.get()
|
||||
self.assertFalse(c.has_inflight_device_transfers())
|
||||
|
||||
def test_idle_wait_times_out(self):
|
||||
c = _make_controller()
|
||||
c._write_submit_active = 1
|
||||
with self.assertRaises(RuntimeError):
|
||||
c.wait_write_submissions_idle(timeout=0.05)
|
||||
|
||||
def test_retired_parts_release_only_after_event(self):
|
||||
c = _make_controller()
|
||||
ev = _FakeEvent()
|
||||
ev._set = False
|
||||
op = _op(c)
|
||||
op.async_parts = cc_mod.AsyncIndexParts(
|
||||
[cc_mod.AsyncIndexCopy((None, None), None, _FakeTensor(8, 8, False), 64)],
|
||||
ev,
|
||||
64,
|
||||
)
|
||||
c._async_pinned_bytes = 64
|
||||
c._retire_async_parts([op])
|
||||
self.assertIsNone(op.async_parts)
|
||||
self.assertEqual(c._async_pinned_bytes, 64)
|
||||
ev._set = True
|
||||
c._reap_retired_parts()
|
||||
self.assertEqual(c._async_pinned_bytes, 0)
|
||||
self.assertEqual(c._retire_parts, [])
|
||||
|
||||
def test_merge_of_cpu_parts_matches_cpu_reference(self):
|
||||
c = _make_controller()
|
||||
ops = []
|
||||
for i in range(3):
|
||||
op = _op(c, anchor=_FakeTensor(4, 8, True, f"a{i}"), node_id=i)
|
||||
op.async_parts = cc_mod.AsyncIndexParts(
|
||||
[
|
||||
cc_mod.AsyncIndexCopy(
|
||||
(None, None),
|
||||
op.device_indices,
|
||||
_FakeTensor(4, 8, False, f"p{i}"),
|
||||
32,
|
||||
)
|
||||
],
|
||||
_FakeEvent(),
|
||||
32,
|
||||
)
|
||||
ops.append(op)
|
||||
c._materialize_async_indices(ops)
|
||||
merged = cc_mod.CacheOperation.merge_ops(ops)
|
||||
self.assertEqual(merged.node_ids, [0, 1, 2])
|
||||
# Single-op merge must not copy: it returns the operation itself.
|
||||
single = cc_mod.CacheOperation.merge_ops([ops[0]])
|
||||
self.assertIs(single, ops[0])
|
||||
|
||||
def test_stop_waits_for_inflight_submission_and_drains_ack(self):
|
||||
c = _make_controller()
|
||||
release = threading.Event()
|
||||
started = threading.Event()
|
||||
|
||||
def _submit(op):
|
||||
started.set()
|
||||
release.wait(10)
|
||||
return cc_mod.HiCacheAck(None, None, list(op.node_ids), num_tokens=8)
|
||||
|
||||
c._submit_write_operation = _submit
|
||||
op = _op(c)
|
||||
op.async_parts = cc_mod.AsyncIndexParts([], _FakeEvent(), 0)
|
||||
c._ensure_write_submit_thread()
|
||||
c._write_submit_active = 1
|
||||
c._write_submit_queue.put(cc_mod.WriteSubmission([op], None, 0))
|
||||
self.assertTrue(started.wait(5))
|
||||
|
||||
stopped = threading.Event()
|
||||
|
||||
def _stop():
|
||||
c._stop_write_submission_thread()
|
||||
stopped.set()
|
||||
|
||||
stopper = threading.Thread(target=_stop, daemon=True)
|
||||
stopper.start()
|
||||
# The stop must not return while a submission is still in flight.
|
||||
self.assertFalse(stopped.wait(0.05))
|
||||
release.set()
|
||||
self.assertTrue(stopped.wait(10))
|
||||
self.assertFalse(c._write_submit_thread is not None and c._write_submit_thread.is_alive())
|
||||
# The ACK produced by the in-flight submission was not dropped.
|
||||
self.assertEqual(len(c.ack_write_queue), 1)
|
||||
|
||||
def test_stop_joins_worker_and_allows_restart(self):
|
||||
c = _make_controller()
|
||||
c._ensure_write_submit_thread()
|
||||
self.assertTrue(c._write_submit_thread.is_alive())
|
||||
c._stop_write_submission_thread()
|
||||
self.assertIsNone(c._write_submit_thread)
|
||||
self.assertIsNone(c._write_submit_queue)
|
||||
c._ensure_write_submit_thread()
|
||||
self.assertTrue(c._write_submit_thread.is_alive())
|
||||
c._stop_write_submission_thread()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Print the CUDA timeline for one async index copy vs the next forward."""
|
||||
import importlib.util as _u
|
||||
import threading
|
||||
import time
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.managers import cache_controller as cc_mod
|
||||
|
||||
_spec = _u.spec_from_file_location(
|
||||
"t", "/data/ymk/wt-async/test/registered/cuda/test_async_backup_index_copy_cuda.py"
|
||||
)
|
||||
_m = _u.module_from_spec(_spec)
|
||||
_spec.loader.exec_module(_m)
|
||||
_make_controller, _op = _m._make_controller, _m._op
|
||||
|
||||
cc_mod._ASYNC_BACKUP_SUBMIT = True
|
||||
cc_mod._ASYNC_BACKUP_PINNED_BYTES_MAX = 1 << 30
|
||||
|
||||
torch.cuda.synchronize()
|
||||
sched = torch.cuda.current_stream()
|
||||
fwd = torch.cuda.Stream()
|
||||
indices = torch.arange(4 * 1024 * 1024, dtype=torch.int64, device="cuda")
|
||||
|
||||
origin = torch.cuda.Event(enable_timing=True)
|
||||
with torch.cuda.stream(sched):
|
||||
origin.record()
|
||||
torch.cuda._sleep(1_000_000_000)
|
||||
fence_done = torch.cuda.Event(enable_timing=True)
|
||||
fence_done.record()
|
||||
|
||||
c = _make_controller()
|
||||
op = _op(indices, 0)
|
||||
t0 = time.perf_counter()
|
||||
c._start_async_index_copy(op)
|
||||
host_ms = (time.perf_counter() - t0) * 1e3
|
||||
|
||||
copy_end = torch.cuda.Event(enable_timing=True)
|
||||
with torch.cuda.stream(c._indices_stream):
|
||||
copy_end.record()
|
||||
|
||||
t_launch = time.monotonic_ns()
|
||||
with torch.cuda.stream(fwd):
|
||||
fwd.wait_stream(sched)
|
||||
fwd_start = torch.cuda.Event(enable_timing=True)
|
||||
fwd_start.record()
|
||||
torch.cuda._sleep(1_500_000_000)
|
||||
fwd_end = torch.cuda.Event(enable_timing=True)
|
||||
fwd_end.record()
|
||||
|
||||
ready_ns = [None]
|
||||
|
||||
|
||||
def _w():
|
||||
op.async_parts.ready_event.synchronize()
|
||||
ready_ns[0] = time.monotonic_ns()
|
||||
|
||||
|
||||
th = threading.Thread(target=_w)
|
||||
th.start()
|
||||
th.join()
|
||||
torch.cuda.synchronize()
|
||||
|
||||
print(f"host start_async_index_copy: {host_ms:.2f} ms")
|
||||
print(f"host next-batch enqueue before ready: {t_launch < ready_ns[0]}")
|
||||
print(f"fence_done -> copy_end : {fence_done.elapsed_time(copy_end):8.2f} ms")
|
||||
print(f"origin -> fwd_start : {origin.elapsed_time(fwd_start):8.2f} ms")
|
||||
print(f"origin -> copy_end : {origin.elapsed_time(copy_end):8.2f} ms")
|
||||
print(f"origin -> fwd_end : {origin.elapsed_time(fwd_end):8.2f} ms")
|
||||
print(f"fwd duration : {fwd_start.elapsed_time(fwd_end):8.2f} ms")
|
||||
print(
|
||||
f"fwd_start -> copy_end : {fwd_start.elapsed_time(copy_end):8.2f} ms (>0 means overlap)"
|
||||
)
|
||||
print(
|
||||
f"copy_end -> fwd_end : {copy_end.elapsed_time(fwd_end):8.2f} ms (>0 means copy finished inside the forward)"
|
||||
)
|
||||
Reference in New Issue
Block a user