diff --git a/python/sglang/srt/managers/cache_controller.py b/python/sglang/srt/managers/cache_controller.py index b306db3880..9dd93d7008 100644 --- a/python/sglang/srt/managers/cache_controller.py +++ b/python/sglang/srt/managers/cache_controller.py @@ -18,7 +18,7 @@ import logging import threading import time from dataclasses import dataclass -from queue import Empty, Queue +from queue import Empty, Full, Queue from typing import TYPE_CHECKING, Any, Callable, List, NamedTuple, Optional import torch @@ -42,6 +42,7 @@ from sglang.srt.layers.dp_attention import ( ) from sglang.srt.mem_cache.l2_transfer import L2Transfer, L2TransferEngine from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool +from sglang.srt.mem_cache.pool_host.base import uses_shared_host_layout from sglang.srt.mem_cache.utils import get_storage_hash_str from sglang.srt.observability import batch_telemetry from sglang.srt.runtime_context import get_parallel @@ -69,6 +70,25 @@ _HC_PREFETCH_WORKERS = int(_hc_os.getenv("SGLANG_HICACHE_PREFETCH_IO_THREADS", " _HC_BACKUP_WORKERS = int(_hc_os.getenv("SGLANG_HICACHE_BACKUP_IO_THREADS", "1") or 1) _HC_TIMING = _hc_os.getenv("SGLANG_HICACHE_TIMING", "0") not in ("", "0", "false", "False") +# [ASYNC_BACKUP] Per-op pinned index copy + worker-side merge/submit. +# Only the direct + page_first_direct shape (the one that does a blocking +# device_indices.cpu() on the scheduler thread) is eligible. +_ASYNC_BACKUP_SUBMIT = _hc_os.getenv( + "SGLANG_ASYNC_BACKUP_SUBMIT", "0" +).strip().lower() not in ("", "0", "false", "no", "off") +_ASYNC_BACKUP_ENQUEUE_TIMEOUT = float( + _hc_os.getenv("SGLANG_ASYNC_BACKUP_ENQUEUE_TIMEOUT", "60") or 60 +) +_ASYNC_BACKUP_IDLE_TIMEOUT = float( + _hc_os.getenv("SGLANG_ASYNC_BACKUP_IDLE_TIMEOUT", "120") or 120 +) +_ASYNC_BACKUP_PINNED_BYTES_MAX = int( + _hc_os.getenv("SGLANG_ASYNC_BACKUP_PINNED_MB", "2048") or 2048 +) * 1024 * 1024 +_ASYNC_BACKUP_MAX_OPS = int( + _hc_os.getenv("SGLANG_ASYNC_BACKUP_MAX_OPS", "256") or 256 +) + class LayerLoadingEvent: def __init__(self, num_layers: int): self._num_layers = num_layers @@ -132,6 +152,9 @@ class CacheOperation: self.node_ids = [node_id] self.data = None self.pool_transfers = pool_transfers + # Set when SGLANG_ASYNC_BACKUP_SUBMIT copied this op's device indices + # into pinned host buffers (see _start_async_index_copy). + self.async_parts = None self.id = CacheOperation.counter CacheOperation.counter += 1 @@ -192,6 +215,39 @@ class CacheOperation: return self.priority < other.priority +@dataclass +class AsyncIndexCopy: + """One device->pinned-host index copy started where the backup op was made.""" + + key: tuple + src: torch.Tensor + dst: torch.Tensor + nbytes: int + + +@dataclass +class AsyncIndexParts: + """Per-op pinned index parts plus the event that guarantees they are final.""" + + copies: List[AsyncIndexCopy] + ready_event: object + nbytes: int + + +@dataclass +class WriteSubmission: + ops: List[CacheOperation] + tracer: object + enqueue_mono_ns: int + + +@dataclass +class WriteResult: + ack: Optional[HiCacheAck] + ops: List[CacheOperation] + error: Optional[BaseException] + + class HiCacheAck(NamedTuple): start_event: device_module.Event finish_event: device_module.Event @@ -383,6 +439,21 @@ class HiCacheController: self.write_fence_stream = None self.ack_write_queue: List[HiCacheAck] = [] + # [ASYNC_BACKUP] Pinned index copies are started where the backup op is + # created; a single worker waits the per-op events, merges the CPU + # index parts and submits the KV copy. Queues are bounded by the state + # lock/condition below; write_queue/ack_write_queue stay scheduler-only. + self._indices_stream = None + self._write_submit_queue: Optional[Queue] = None + self._write_result_queue: Queue = Queue() + self._write_submit_stop = threading.Event() + self._write_submit_thread: Optional[threading.Thread] = None + self._write_submit_state_lock = threading.Lock() + self._write_submit_cond = threading.Condition(self._write_submit_state_lock) + self._write_submit_active = 0 + self._async_pinned_bytes = 0 + self._retire_parts: List["AsyncIndexParts"] = [] + self.l2_transfer_engine = L2TransferEngine(io_backend) # If a storage backend is provided at startup, treat it as an implicit attach, @@ -708,6 +779,9 @@ class HiCacheController: Requirement: no in-flight requests. This will stop storage threads and release the backend instance (best-effort close). """ + # Async backup submissions own pinned index buffers and ACKs; stop them + # before any queue/pool teardown so no result is lost. + self._stop_write_submission_thread() # Idempotent cleanup: even if `enable_storage` is already False, # we may still have leftover resources (threads/backend/process group) from a # previous partial detach. We attempt cleanup whenever possible. @@ -822,6 +896,7 @@ class HiCacheController: def reset(self): # Reuse detach's queue wakeups and bounded joins, and fail if any # storage thread remains alive before clearing shared state. + self._stop_write_submission_thread() self._stop_storage_threads() self.write_queue.clear() @@ -861,13 +936,311 @@ class HiCacheController: def has_inflight_device_transfers(self) -> bool: """Whether queued or unacknowledged L2 transfers still use device rows.""" + with self._write_submit_state_lock: + async_pending = ( + self._write_submit_active > 0 + or self._async_pinned_bytes > 0 + or not self._write_result_queue.empty() + ) return bool( self.write_queue or self.load_queue or self.ack_write_queue or self.ack_load_queue + or async_pending ) + # ---- async backup submission ----------------------------------------- + def async_backup_enabled(self) -> bool: + return ( + _ASYNC_BACKUP_SUBMIT + and self.io_backend == "direct" + and getattr(self.mem_pool_host, "layout", None) == "page_first_direct" + ) + + def _async_submit_applicable(self) -> bool: + return ( + self.async_backup_enabled() + and len(self.write_queue) <= _ASYNC_BACKUP_MAX_OPS + and all(op.async_parts is not None for op in self.write_queue) + ) + + def _ensure_indices_stream(self): + if self._indices_stream is None: + self._indices_stream = device_module.Stream() + return self._indices_stream + + def _needs_device_index_copy(self, host_pool, indices) -> bool: + if indices is None or not getattr(indices, "is_cuda", False): + return False + # move_indices() returns shared-layout ids untouched; converting them + # here would hand the transfer a different interface than the sync path. + return not uses_shared_host_layout(host_pool) + + def _async_index_targets(self, op: CacheOperation): + targets = [] + if self._needs_device_index_copy( + self.mem_pool_host.anchor_entry.host_pool, op.device_indices + ): + targets.append(((None, None), op.device_indices)) + for transfer in op.pool_transfers or []: + if transfer.device_indices is None: + continue + entry = self.mem_pool_host.entry_map.get(transfer.name) + if entry is None: + continue + if not self._needs_device_index_copy( + entry.host_pool, transfer.device_indices + ): + continue + key = (transfer.name, transfer.indices_from_pool) + targets.append((key, transfer.device_indices)) + return targets + + def _start_async_index_copy(self, op: CacheOperation) -> None: + """Start the pinned index D2H copy now, off the scheduler stream. + + Runs in the scheduler thread at backup-op creation time. The copy runs + on a dedicated stream ordered after this instant, so it does not inherit + later fences (the WAR barrier of a later forward) and the scheduler + never blocks in ``.cpu()``. + """ + if not self.async_backup_enabled(): + return + targets = self._async_index_targets(op) + if not targets: + return + nbytes = sum(int(t.numel()) * int(t.element_size()) for _, t in targets) + with self._write_submit_state_lock: + if self._async_pinned_bytes + nbytes > _ASYNC_BACKUP_PINNED_BYTES_MAX: + return + self._async_pinned_bytes += nbytes + try: + stream = self._ensure_indices_stream() + source_stream = device_module.current_stream() + copies: List[AsyncIndexCopy] = [] + with device_module.StreamContext(stream): + stream.wait_stream(source_stream) + for key, src in targets: + # Keep the device rows alive until the copy has run, and pin + # the destination so copy_ is a real asynchronous D2H. + src.record_stream(stream) + dst = torch.empty( + src.shape, + dtype=src.dtype, + device="cpu", + pin_memory=True, + ) + dst.copy_(src, non_blocking=True) + copies.append( + AsyncIndexCopy( + key=key, + src=src, + dst=dst, + nbytes=int(src.numel()) * int(src.element_size()), + ) + ) + ready_event = device_module.Event() + ready_event.record(stream) + except BaseException: + with self._write_submit_state_lock: + self._async_pinned_bytes -= nbytes + logger.exception("Async backup index copy failed; keeping sync path") + return + op.async_parts = AsyncIndexParts( + copies=copies, ready_event=ready_event, nbytes=nbytes + ) + + def _ensure_write_submit_thread(self) -> None: + if self._write_submit_queue is None: + self._write_submit_queue = Queue(maxsize=64) + thread = self._write_submit_thread + if thread is not None and thread.is_alive(): + return + self._write_submit_stop.clear() + thread = threading.Thread( + target=self._write_submit_worker, + name="hicache-async-backup", + daemon=True, + ) + self._write_submit_thread = thread + thread.start() + + def _enqueue_async_write(self, ops, tracer) -> bool: + """Enqueue one submission; False means "fall back to the sync path". + + A full queue or a timeout is not fatal: the caller submits the same ops + synchronously. Before returning False we wait for the worker to become + idle so the fallback cannot race another submission on the shared + device_to_host_stream. + """ + self._ensure_write_submit_thread() + item = WriteSubmission( + ops=ops, tracer=tracer, enqueue_mono_ns=time.monotonic_ns() + ) + with self._write_submit_cond: + self._write_submit_active += 1 + try: + with batch_telemetry.backup_phase("enqueue"): + self._write_submit_queue.put( + item, timeout=_ASYNC_BACKUP_ENQUEUE_TIMEOUT + ) + except Full: + with self._write_submit_cond: + self._write_submit_active -= 1 + self._write_submit_cond.notify_all() + logger.warning( + "Async backup submit queue full after %.1fs; falling back to " + "the synchronous path", + _ASYNC_BACKUP_ENQUEUE_TIMEOUT, + ) + self.wait_write_submissions_idle() + return False + return True + + def _materialize_async_indices(self, ops) -> None: + """Replace device index tensors with the pinned CPU copies (worker).""" + for op in ops: + parts = op.async_parts + if parts is None: + continue + mapping = {copy.key: copy.dst for copy in parts.copies} + anchor = mapping.get((None, None)) + if anchor is not None: + op.device_indices = anchor + for transfer in op.pool_transfers or []: + dst = mapping.get((transfer.name, transfer.indices_from_pool)) + if dst is not None: + transfer.device_indices = dst + + def _write_submit_worker(self) -> None: + while True: + item = self._write_submit_queue.get() + if item is None: + break + tracer = item.tracer + previous = batch_telemetry.backup_flush_attach(tracer) + try: + with batch_telemetry.backup_phase("worker"): + for op in item.ops: + parts = op.async_parts + if parts is not None: + parts.ready_event.synchronize() + self._materialize_async_indices(item.ops) + merged = CacheOperation.merge_ops(item.ops) + ack = self._submit_write_operation(merged) + result = WriteResult(ack=ack, ops=item.ops, error=None) + except BaseException as exc: # noqa: BLE001 - fail closed + logger.exception("Async backup submission failed") + result = WriteResult(ack=None, ops=item.ops, error=exc) + finally: + batch_telemetry.backup_flush_detach(previous) + self._release_async_parts(item.ops) + try: + batch_telemetry.backup_flush_end(tracer) + except Exception: + pass + # Publish the result before marking the submission done, so an idle + # waiter cannot observe active==0 with the result still missing. + self._write_result_queue.put(result) + with self._write_submit_cond: + self._write_submit_active -= 1 + self._write_submit_cond.notify_all() + + def _release_async_parts(self, ops) -> None: + released = 0 + for op in ops: + parts = op.async_parts + if parts is None: + continue + op.async_parts = None + parts.copies.clear() + released += parts.nbytes + if released: + with self._write_submit_state_lock: + self._async_pinned_bytes -= released + + def _retire_async_parts(self, ops) -> None: + """Release copies that will not be consumed (sync fallback).""" + for op in ops: + parts = op.async_parts + if parts is not None: + self._retire_parts.append(parts) + op.async_parts = None + self._reap_retired_parts() + + def _reap_retired_parts(self) -> None: + if not self._retire_parts: + return + pending = [] + for parts in self._retire_parts: + try: + done = parts.ready_event.query() + except Exception: + done = True + if done: + parts.copies.clear() + with self._write_submit_state_lock: + self._async_pinned_bytes -= parts.nbytes + else: + pending.append(parts) + self._retire_parts = pending + + def drain_write_submission_results(self) -> None: + """Move finished async ACKs into ack_write_queue (scheduler thread).""" + while True: + try: + result = self._write_result_queue.get_nowait() + except Empty: + break + if result.error is not None: + raise RuntimeError("async backup submission failed") from result.error + if result.ack is not None: + self.ack_write_queue.append(result.ack) + self._reap_retired_parts() + + def wait_write_submissions_idle( + self, timeout: float = _ASYNC_BACKUP_IDLE_TIMEOUT + ) -> None: + deadline = time.monotonic() + timeout + with self._write_submit_cond: + while self._write_submit_active > 0: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise RuntimeError( + "Timed out waiting for async backup submissions" + ) + self._write_submit_cond.wait(remaining) + + def _stop_write_submission_thread(self) -> None: + if self._write_submit_thread is None: + self.drain_write_submission_results() + self._reap_retired_parts() + if self._retire_parts: + raise RuntimeError("async backup index copies still in flight") + return + try: + self.wait_write_submissions_idle() + finally: + self._write_submit_stop.set() + queue = self._write_submit_queue + if queue is not None: + try: + queue.put(None, timeout=5.0) + except Exception: + logger.exception("Failed to wake the async backup worker") + thread = self._write_submit_thread + if thread is not None: + thread.join(timeout=30.0) + if thread.is_alive(): + raise RuntimeError("async backup submit worker did not stop") + self.drain_write_submission_results() + if self._retire_parts: + raise RuntimeError("async backup index copies still in flight") + self._write_submit_thread = None + self._write_submit_queue = None + + def write( self, device_indices: torch.Tensor, @@ -883,9 +1256,9 @@ class HiCacheController: host_indices = self.mem_pool_host.alloc(len(device_indices)) if host_indices is None: return None - self.write_queue.append( - CacheOperation(host_indices, device_indices, node_id, priority) - ) + op = CacheOperation(host_indices, device_indices, node_id, priority) + self._start_async_index_copy(op) + self.write_queue.append(op) self.start_writing() return host_indices @@ -898,98 +1271,130 @@ class HiCacheController: batch_telemetry.detail_count("flush_empty") return + ops = list(self.write_queue) tracer = batch_telemetry.backup_flush_begin() + async_owned = False try: batch_telemetry.detail_count("flush_calls") batch_telemetry.detail_count("flush_nonempty") - batch_telemetry.flush_note( - "sglang.flush_write_queue_ops", len(self.write_queue) - ) - batch_telemetry.flush_note("sglang.flush_io_backend", self.io_backend) - batch_telemetry.flush_note( - "sglang.flush_layout", - str(getattr(self.mem_pool_host, "layout", None) or ""), - ) - batch_telemetry.flush_note( - "sglang.flush_ack_queue_before", len(self.ack_write_queue) - ) + self._note_write_queue(ops, len(self.ack_write_queue)) + if self._async_submit_applicable(): + if self._enqueue_async_write(ops, tracer): + self.write_queue.clear() + batch_telemetry.flush_note( + "sglang.flush_submission_mode", "async" + ) + # The worker owns the tracer from here on (it emits once). + async_owned = True + return + # Queue full / enqueue timeout: retire the pinned copies and + # submit synchronously on this thread. `_enqueue_async_write` + # waited for the worker to go idle first, so the two paths + # never submit to device_to_host_stream concurrently. + batch_telemetry.flush_note("sglang.flush_fallback_sync", 1) + self._retire_async_parts(ops) + + # Any parts that were started for this queue are not consumed by the + # sync path. Retire them unconditionally (they are released once + # their events complete) so the pinned-byte budget never leaks. + batch_telemetry.flush_note( + "sglang.flush_async_skip_ops", + sum(1 for op in ops if op.async_parts is None), + ) + self._retire_async_parts(ops) + batch_telemetry.flush_note("sglang.flush_submission_mode", "sync") with batch_telemetry.backup_phase("merge_ops"): op = CacheOperation.merge_ops(self.write_queue) - tokens = len(op.device_indices) - indices = op.device_indices - batch_telemetry.flush_note("sglang.flush_tokens", int(tokens)) - batch_telemetry.flush_note( - "sglang.flush_nodes_merged", len(op.node_ids) - ) - batch_telemetry.flush_note( - "sglang.flush_sidecar_pools", len(op.pool_transfers or []) - ) - batch_telemetry.flush_note( - "sglang.flush_indices_device", str(indices.device.type) - ) - batch_telemetry.flush_note( - "sglang.flush_indices_dtype", str(indices.dtype) - ) - batch_telemetry.flush_note( - "sglang.flush_indices_numel", int(indices.numel()) - ) - - with batch_telemetry.backup_phase("move_write"): - host_indices, device_indices, pool_transfers = ( - self._move_write_operation(op) - ) self.write_queue.clear() - batch_telemetry.flush_note( - "sglang.flush_host_indices_on_cpu", - bool(getattr(host_indices, "is_cuda", False) is False), - ) - batch_telemetry.flush_note( - "sglang.flush_device_indices_on_cpu", - bool(getattr(device_indices, "is_cuda", False) is False), - ) - - if self.write_fence_stream is not None: - # The copy may read KV a forward is still writing: under overlap - # scheduling a finished request is cached while the next forward - # writes its last token, and an evicted SWA page can belong to an - # unfinished checkpoint. Later forwards still overlap with the copy. - with batch_telemetry.backup_phase("write_fence"): - self.l2_transfer_engine.device_to_host_stream.wait_stream( - self.write_fence_stream - ) - - with batch_telemetry.backup_phase("build_l2"): - transfers = self._l2_transfers( - host_indices, device_indices, pool_transfers - ) - batch_telemetry.flush_note("sglang.flush_l2_transfers", len(transfers)) - - with batch_telemetry.backup_phase("submit_d2h"): - completion = self.l2_transfer_engine.submit_device_to_host(transfers) - - with batch_telemetry.backup_phase("set_done_event"): - self.mem_pool_device_allocator.set_hicache_transfer_done_event( - (id(self), "write"), completion.finish_event - ) - + ack = self._submit_write_operation(op) with batch_telemetry.backup_phase("ack_enqueue"): - self.ack_write_queue.append( - HiCacheAck( - start_event=completion.start_event, - finish_event=completion.finish_event, - node_ids=op.node_ids, - num_tokens=len(op.device_indices), - timing_enabled=completion.timing_enabled, - num_tokens_by_pool=self._num_tokens_by_pool(op), - num_bytes=self._transfer_num_bytes(op), - ) - ) + self.ack_write_queue.append(ack) batch_telemetry.flush_note( "sglang.flush_ack_queue_after", len(self.ack_write_queue) ) finally: - batch_telemetry.backup_flush_end(tracer) + if not async_owned: + batch_telemetry.backup_flush_end(tracer) + + def _note_write_queue(self, ops: List[CacheOperation], ack_before: int) -> None: + """Queue-level flush dimensions (same for the sync and async paths).""" + batch_telemetry.flush_note("sglang.flush_write_queue_ops", len(ops)) + batch_telemetry.flush_note("sglang.flush_io_backend", self.io_backend) + batch_telemetry.flush_note( + "sglang.flush_layout", + str(getattr(self.mem_pool_host, "layout", None) or ""), + ) + batch_telemetry.flush_note("sglang.flush_ack_queue_before", ack_before) + + def _submit_write_operation(self, op: CacheOperation) -> HiCacheAck: + """Move indices, submit the D2H copy and build the ACK. + + Runs on the scheduler thread (sync path) or on the async submit worker + (async path, with CPU-resident indices). It must not touch + ``write_queue`` or ``ack_write_queue``; the caller owns those. + """ + tokens = len(op.device_indices) + indices = op.device_indices + batch_telemetry.flush_note("sglang.flush_tokens", int(tokens)) + batch_telemetry.flush_note("sglang.flush_nodes_merged", len(op.node_ids)) + batch_telemetry.flush_note( + "sglang.flush_sidecar_pools", len(op.pool_transfers or []) + ) + batch_telemetry.flush_note( + "sglang.flush_indices_device", str(indices.device.type) + ) + batch_telemetry.flush_note("sglang.flush_indices_dtype", str(indices.dtype)) + batch_telemetry.flush_note( + "sglang.flush_indices_numel", int(indices.numel()) + ) + + with batch_telemetry.backup_phase("move_write"): + host_indices, device_indices, pool_transfers = ( + self._move_write_operation(op) + ) + batch_telemetry.flush_note( + "sglang.flush_host_indices_on_cpu", + bool(getattr(host_indices, "is_cuda", False) is False), + ) + batch_telemetry.flush_note( + "sglang.flush_device_indices_on_cpu", + bool(getattr(device_indices, "is_cuda", False) is False), + ) + + if self.write_fence_stream is not None: + # The copy may read KV a forward is still writing: under overlap + # scheduling a finished request is cached while the next forward + # writes its last token, and an evicted SWA page can belong to an + # unfinished checkpoint. Later forwards still overlap with the copy. + with batch_telemetry.backup_phase("write_fence"): + self.l2_transfer_engine.device_to_host_stream.wait_stream( + self.write_fence_stream + ) + + with batch_telemetry.backup_phase("build_l2"): + transfers = self._l2_transfers( + host_indices, device_indices, pool_transfers + ) + batch_telemetry.flush_note("sglang.flush_l2_transfers", len(transfers)) + + with batch_telemetry.backup_phase("submit_d2h"): + completion = self.l2_transfer_engine.submit_device_to_host(transfers) + + with batch_telemetry.backup_phase("set_done_event"): + self.mem_pool_device_allocator.set_hicache_transfer_done_event( + (id(self), "write"), completion.finish_event + ) + + return HiCacheAck( + start_event=completion.start_event, + finish_event=completion.finish_event, + node_ids=op.node_ids, + num_tokens=len(op.device_indices), + timing_enabled=completion.timing_enabled, + num_tokens_by_pool=self._num_tokens_by_pool(op), + num_bytes=self._transfer_num_bytes(op), + ) def _transfer_num_bytes(self, op: CacheOperation) -> int: return len(op.device_indices) * self.mem_pool_host.size_per_token diff --git a/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py b/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py index 90bd1fcc84..a7d82e5a09 100644 --- a/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py +++ b/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py @@ -25,6 +25,7 @@ from __future__ import annotations import inspect import logging import os +import threading from typing import ( Callable, Dict, @@ -440,6 +441,9 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator): # site; `_flush` materializes the write-set lazily, avoiding a sync here. self._inflight_forward: Optional[Tuple[torch.cuda.Event, torch.Tensor]] = None self._hicache_transfer_done_events: Dict[Hashable, torch.cuda.Event] = {} + # The async backup submit worker records transfer events while the + # scheduler may concurrently iterate/clear them for compaction gating. + self._hicache_transfer_done_events_lock = threading.Lock() # HiCache reserves SWA physical pages during load-back, then binds them # into the tree before the H2D copy is submitted. Compaction must not # move those pages in that interval because the queued PoolTransfer still @@ -557,7 +561,8 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator): self.live_page_count = 0 self._inflight_forward = None self._latest_forward_done_event = None - self._hicache_transfer_done_events.clear() + with self._hicache_transfer_done_events_lock: + self._hicache_transfer_done_events.clear() self._pending_hicache_load_pages = 0 def clear_inverse_history(self) -> None: @@ -2083,7 +2088,8 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator): _pending_hicache_load_pages: _CapacityField[int] = _CapacityField() def set_hicache_transfer_done_event(self, transfer_key: Hashable, event) -> None: - self._hicache_transfer_done_events[transfer_key] = event + with self._hicache_transfer_done_events_lock: + self._hicache_transfer_done_events[transfer_key] = event if transfer_key == "load" or ( isinstance(transfer_key, tuple) and transfer_key[-1] == "load" ): @@ -2095,11 +2101,12 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator): """Order the current stream after outstanding HiCache copies. Only moves and frees need this: a copy's source and target are not free pages until its finish event is observed, so allocation never reuses them.""" - if not self._hicache_transfer_done_events: - return + with self._hicache_transfer_done_events_lock: + if not self._hicache_transfer_done_events: + return + events = tuple(self._hicache_transfer_done_events.values()) + self._hicache_transfer_done_events.clear() current_stream = torch.cuda.current_stream() - events = tuple(self._hicache_transfer_done_events.values()) - self._hicache_transfer_done_events.clear() for event in events: current_stream.wait_event(event) diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py index e74880af6e..7d4389bb08 100644 --- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py +++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py @@ -556,15 +556,17 @@ class HybridCacheController(BaseHiCacheController): return None host_indices, pool_transfers = allocation - self.write_queue.append( - CacheOperation( - host_indices, - device_indices, - node_id, - priority, - pool_transfers=pool_transfers or None, - ) + op = CacheOperation( + host_indices, + device_indices, + node_id, + priority, + pool_transfers=pool_transfers or None, ) + # Start the pinned index D2H before the (possibly deferred) flush, so + # the copy can run while the scheduler keeps going. + self._start_async_index_copy(op) + self.write_queue.append(op) if flush: self.start_writing() return host_indices diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index f0da45c627..e2f5841d30 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -3505,6 +3505,13 @@ class UnifiedRadixCache(BasePrefixCache): if write_back: # Blocking: submit what is still queued, then wait for every ack. cc.start_writing() + # Async submissions cover ACKs asynchronously; wait for them before + # asserting that every write-through node has been acknowledged. + if hasattr(cc, "wait_write_submissions_idle"): + cc.wait_write_submissions_idle() + _drain = getattr(cc, "drain_write_submission_results", None) + if _drain is not None: + _drain() while self.ongoing_write_through: for ack in cc.ack_write_queue: ack.finish_event.synchronize() @@ -3517,6 +3524,9 @@ class UnifiedRadixCache(BasePrefixCache): return if finish_count is None: + _drain = getattr(cc, "drain_write_submission_results", None) + if _drain is not None: + _drain() # Every rank must enter the all_reduce below; ongoing_write_through can # diverge across ranks (e.g. write_backup returning 0 on a subset). finish_count = 0 @@ -3694,9 +3704,16 @@ class UnifiedRadixCache(BasePrefixCache): # Reap the previous round's PP-sync sends before issuing new ones. self._drain_async_work() + # Collect ACKs finished by the async submit worker before counting them. + # getattr keeps lightweight controller fakes (unit tests) working. + _drain = getattr(self.cache_controller, "drain_write_submission_results", None) + if _drain is not None: + _drain() # Backups queued outside process_batch_result: the chunked-prefill stash # in get_next_batch_to_run, abort_request, and the PD prefill release. self.flush_pending_backups() + if _drain is not None: + _drain() ( write_finish_count, diff --git a/python/sglang/srt/observability/batch_telemetry.py b/python/sglang/srt/observability/batch_telemetry.py index abee9f31ef..e3a4466924 100644 --- a/python/sglang/srt/observability/batch_telemetry.py +++ b/python/sglang/srt/observability/batch_telemetry.py @@ -1409,6 +1409,8 @@ class BackupFlushTracer: __slots__ = ( "origin", "start_ns", + "loop_seq", + "loop_ctx", "phases", "attrs", "counts", @@ -1418,6 +1420,10 @@ class BackupFlushTracer: def __init__(self, origin: str): self.origin = origin + # Captured on the scheduler thread: the flushed span may be emitted by + # the async submit worker, which has no scheduler loop TLS. + self.loop_seq = current_loop_seq() + self.loop_ctx = current_loop_context_for_spans() self.start_ns = time.monotonic_ns() self.phases: dict[str, int] = {} self.attrs: dict[str, Any] = {} @@ -1463,7 +1469,7 @@ class BackupFlushTracer: attrs = { "sglang.flush_origin": self.origin, "sglang.flush_total_ms": total_ns / 1e6, - "sglang.loop_seq": current_loop_seq(), + "sglang.loop_seq": int(self.loop_seq or 0), } for name, ns in self.phases.items(): attrs["sglang.flush_" + name.replace(".", "_") + "_ms"] = ns / 1e6 @@ -1476,7 +1482,7 @@ class BackupFlushTracer: try: from opentelemetry import trace as otel_trace - ctx = current_loop_context_for_spans() + ctx = self.loop_ctx start_wall_ns = time.time_ns() - total_ns span = otel_trace.get_tracer("sglang.cache.flush").start_span( "scheduler.cache_flush_backup", @@ -1543,6 +1549,25 @@ def current_flush_tracer() -> Optional[BackupFlushTracer]: return getattr(_FLUSH_TLS, "tracer", None) +def backup_flush_attach(tracer: Optional[BackupFlushTracer]): + """Bind ``tracer`` to this thread so sub-phases land on the same span. + + Used by the async submit worker, which executes phases on a different + thread than the one that created the tracer. Returns the previous binding. + """ + if not _BACKUP_DETAIL_ENABLED: + return None + previous = getattr(_FLUSH_TLS, "tracer", None) + _FLUSH_TLS.tracer = tracer + return previous + + +def backup_flush_detach(previous) -> None: + if not _BACKUP_DETAIL_ENABLED: + return + _FLUSH_TLS.tracer = previous + + def flush_note(key: str, value: Any) -> None: """Attach a dimension to the active flush (no-op when disabled).""" if not _BACKUP_DETAIL_ENABLED: diff --git a/test/registered/cuda/test_async_backup_index_copy_cuda.py b/test/registered/cuda/test_async_backup_index_copy_cuda.py new file mode 100644 index 0000000000..5b204b9975 --- /dev/null +++ b/test/registered/cuda/test_async_backup_index_copy_cuda.py @@ -0,0 +1,253 @@ +"""CUDA ordering/timeline tests for the per-op async backup index copy. + +Plan: handoff/prefill-backup-overlap-next-20261107.md §3/§8.2. + +Run inside the GPU container with the overlay source: + + PYTHONPATH=/tmp/ovr python -m pytest \ + test/registered/cuda/test_async_backup_index_copy_cuda.py -q + +What is proven on real hardware: + + 1. ``_start_async_index_copy`` returns while a long kernel is still queued on + the scheduler stream, and the synchronous ``.cpu()`` on that stream blocks; + 2. the copy is only released after that fence, but the *next* forward (a + marker kernel on another stream, ordered behind the same fence) runs + concurrently with it: the copy end timestamp falls strictly inside the + marker's [start, end] interval; + 3. the host can enqueue the next batch before ``ready_event`` completes and + the event is not reported ready early; + 4. with three ops whose copies complete out of order, the worker still merges + them in op order, identical to ``CacheOperation.merge_ops`` over the CPU + reference tensors; + 5. the copy stream is distinct from the scheduler stream. +""" + +import threading +import time +import unittest + +import torch + +from sglang.srt.managers import cache_controller as cc_mod + + +class _FakeHostPool: + """Not a HostKVCache: uses_shared_host_layout() is false, i.e. the sync path + would call device_indices.cpu().""" + + shared_allocation_domain = None + + +class _FakeEntry: + def __init__(self): + self.host_pool = _FakeHostPool() + + +class _FakeMemPoolHost: + layout = "page_first_direct" + + def __init__(self): + self.anchor_entry = _FakeEntry() + self.entry_map = {} + + +def _make_controller(): + c = cc_mod.HiCacheController.__new__(cc_mod.HiCacheController) + c.io_backend = "direct" + c.mem_pool_host = _FakeMemPoolHost() + c.write_queue = [] + c.load_queue = [] + c.ack_write_queue = [] + c.ack_load_queue = [] + c._indices_stream = None + c._write_submit_queue = None + c._write_result_queue = cc_mod.Queue() + c._write_submit_stop = threading.Event() + c._write_submit_thread = None + c._write_submit_state_lock = threading.Lock() + c._write_submit_cond = threading.Condition(c._write_submit_state_lock) + c._write_submit_active = 0 + c._async_pinned_bytes = 0 + c._retire_parts = [] + return c + + +def _op(device_indices, node_id): + host = torch.empty(device_indices.numel(), dtype=torch.int64) + return cc_mod.CacheOperation(host, device_indices, node_id) + + +class AsyncBackupIndexCopyCudaTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cc_mod._ASYNC_BACKUP_SUBMIT = True + cls.saved_pin = cc_mod._ASYNC_BACKUP_PINNED_BYTES_MAX + cc_mod._ASYNC_BACKUP_PINNED_BYTES_MAX = 1 << 30 + + @classmethod + def tearDownClass(cls): + cc_mod._ASYNC_BACKUP_PINNED_BYTES_MAX = cls.saved_pin + + def test_start_does_not_block_and_copy_is_async(self): + torch.cuda.synchronize() + sched = torch.cuda.current_stream() + fwd = torch.cuda.Stream() + indices = torch.arange(4096, dtype=torch.int64, device="cuda") + + # Simulated in-flight forward plus the scheduler's WAR fence. + with torch.cuda.stream(fwd): + torch.cuda._sleep(2_000_000_000) + sched.wait_stream(fwd) + + c = _make_controller() + op = _op(indices, 11) + t0 = time.perf_counter() + c._start_async_index_copy(op) + start_elapsed = time.perf_counter() - t0 + self.assertIsNotNone(op.async_parts) + # Starting the copy must not block on the simulated forward. + self.assertLess(start_elapsed, 0.05, start_elapsed) + self.assertFalse(op.async_parts.ready_event.query()) + + # The synchronous baseline on the same stream does block. + t1 = time.perf_counter() + reference = indices.cpu() + sync_elapsed = time.perf_counter() - t1 + self.assertGreater(sync_elapsed, 0.1, sync_elapsed) + + op.async_parts.ready_event.synchronize() + self.assertTrue(op.async_parts.ready_event.query()) + c._materialize_async_indices([op]) + self.assertTrue(bool(torch.equal(op.device_indices, reference))) + self.assertFalse(op.device_indices.is_cuda) + c._release_async_parts([op]) + self.assertEqual(c._async_pinned_bytes, 0) + + def test_next_forward_overlaps_the_index_copy(self): + torch.cuda.synchronize() + sched = torch.cuda.current_stream() + fwd = torch.cuda.Stream() + # Large enough that the copy takes long enough to observe (32 MB). + indices = torch.arange(4 * 1024 * 1024, dtype=torch.int64, device="cuda") + + # 1. A long fence on the scheduler stream (the in-flight forward). + with torch.cuda.stream(sched): + fence_done = torch.cuda.Event(enable_timing=True) + fence_done.record() + torch.cuda._sleep(1_000_000_000) + + # 2. Per-op pinned copy started now; it inherits the fence. + c = _make_controller() + op = _op(indices, 3) + c._start_async_index_copy(op) + self.assertIsNotNone(op.async_parts) + + # 3. The next batch launch waits for the scheduler stream (as the real + # forward_stream does) and then runs a long marker kernel. + copy_end = torch.cuda.Event(enable_timing=True) + with torch.cuda.stream(c._indices_stream): + copy_end.record() + launch_host_ns = time.monotonic_ns() + with torch.cuda.stream(fwd): + fwd.wait_stream(sched) + fwd_start = torch.cuda.Event(enable_timing=True) + fwd_start.record() + torch.cuda._sleep(1_500_000_000) + fwd_end = torch.cuda.Event(enable_timing=True) + fwd_end.record() + + # 4. The event must not be ready while the fence is in flight, and the + # host has already enqueued the next batch. + ready_seen_ns = [None] + + def _wait_ready(): + op.async_parts.ready_event.synchronize() + ready_seen_ns[0] = time.monotonic_ns() + + waiter = threading.Thread(target=_wait_ready) + waiter.start() + self.assertIsNone(ready_seen_ns[0]) + waiter.join(timeout=30) + self.assertIsNotNone(ready_seen_ns[0]) + self.assertLess(launch_host_ns, ready_seen_ns[0]) + + torch.cuda.synchronize() + # CUDA timestamps: copy end strictly inside the next forward. + self.assertGreater(fwd_start.elapsed_time(copy_end), 0.0) + self.assertGreater(copy_end.elapsed_time(fwd_end), 0.0) + self.assertGreater(fwd_start.elapsed_time(fwd_end), 0.5) + + c._materialize_async_indices([op]) + self.assertTrue( + bool(torch.equal(op.device_indices, indices.cpu())) + ) + c._release_async_parts([op]) + + def test_copy_stream_is_distinct_from_scheduler(self): + torch.cuda.synchronize() + c = _make_controller() + op = _op(torch.arange(8, dtype=torch.int64, device="cuda"), 0) + c._start_async_index_copy(op) + self.assertIsNotNone(c._indices_stream) + self.assertNotEqual( + c._indices_stream.cuda_stream, + torch.cuda.current_stream().cuda_stream, + ) + op.async_parts.ready_event.synchronize() + c._release_async_parts([op]) + + def test_out_of_order_completion_keeps_op_order(self): + torch.cuda.synchronize() + c = _make_controller() + delays = {0: 600_000_000, 1: 150_000_000, 2: 300_000_000} + ops = [] + references = [] + for i in range(3): + src = torch.arange(8, dtype=torch.int64, device="cuda") + i * 100 + op = _op(src, node_id=i) + buf = torch.empty(src.shape, dtype=src.dtype, device="cpu", pin_memory=True) + stream = torch.cuda.Stream() + with torch.cuda.stream(stream): + torch.cuda._sleep(delays[i]) + buf.copy_(src, non_blocking=True) + ev = torch.cuda.Event() + ev.record(stream) + op.async_parts = cc_mod.AsyncIndexParts( + [cc_mod.AsyncIndexCopy((None, None), src, buf, buf.numel() * 8)], + ev, + buf.numel() * 8, + ) + ops.append(op) + references.append(src.cpu()) + + captured = [] + c._submit_write_operation = lambda merged: ( + captured.append(merged.device_indices.clone()) + or cc_mod.HiCacheAck(None, None, list(merged.node_ids), num_tokens=8) + ) + c._write_submit_queue = cc_mod.Queue(maxsize=4) + worker = threading.Thread(target=c._write_submit_worker, daemon=True) + c._write_submit_thread = worker + worker.start() + try: + c._write_submit_active = 1 + c._write_submit_queue.put(cc_mod.WriteSubmission(ops, None, 0)) + deadline = time.time() + 30 + while c._write_submit_active and time.time() < deadline: + time.sleep(0.01) + self.assertEqual(c._write_submit_active, 0) + finally: + c._write_submit_queue.put(None) + worker.join(timeout=10) + + self.assertEqual(len(captured), 1) + expected = torch.cat(references) + self.assertTrue(bool(torch.equal(captured[0], expected))) + self.assertEqual(ops[0].node_ids, [0]) + self.assertEqual(ops[1].node_ids, [1]) + self.assertEqual(ops[2].node_ids, [2]) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/disaggregation/test_async_backup_submit.py b/test/registered/unit/disaggregation/test_async_backup_submit.py new file mode 100644 index 0000000000..2a576f912a --- /dev/null +++ b/test/registered/unit/disaggregation/test_async_backup_submit.py @@ -0,0 +1,550 @@ +"""Unit tests for the async HiCache backup submission state machine. + +Design: handoff/prefill-backup-overlap-next-20261107.md §8.1. + +These tests exercise the scheduler/worker split without a GPU: the index copy +itself is covered by the CUDA test, and here the device stream/event and the +submit path are faked. What is checked: + + * disabled / inapplicable shapes keep the sync path; + * per-op pinned parts are materialized per (pool, indices_from_pool) key; + * the worker waits every ready event before touching host buffers; + * exactly-once ACK, fail-closed errors, bounded queues, idle/stop lifecycle; + * has_inflight_device_transfers reflects queued/running results. +""" + +import threading +import time +import types +import unittest +from unittest import mock + +from sglang.srt.managers import cache_controller as cc_mod +from sglang.srt.mem_cache.hicache_storage import PoolName, PoolTransfer + + +class _FakeEvent: + def __init__(self, worker_log=None, name="ev"): + self.worker_log = worker_log + self.name = name + self._set = False + self.waited = 0 + + def record(self, stream=None): + self._set = True + + def synchronize(self): + self.waited += 1 + if self.worker_log is not None: + self.worker_log.append(("wait", self.name)) + self._set = True + + def query(self): + return self._set + + +class _FakeStream: + def __init__(self, name="stream", log=None): + self.name = name + self.log = log + self.waited = 0 + + def wait_stream(self, other): + self.waited += 1 + if self.log is not None: + self.log.append(("wait_stream", self.name, getattr(other, "name", "?"))) + + def wait_event(self, event): + pass + + +class _FakeStreamCtx: + def __init__(self, stream): + self.stream = stream + + def __enter__(self): + return self.stream + + def __exit__(self, *exc): + return False + + +class _FakeCudaModule: + def __init__(self, log=None): + self.log = log + self.current = _FakeStream("schedule", log) + + def Stream(self, *a, **k): + return _FakeStream("indices", self.log) + + def StreamContext(self, stream): + return _FakeStreamCtx(stream) + + def Event(self, *a, **k): + return _FakeEvent(self.log) + + def current_stream(self): + return self.current + + +class _FakeTensor: + def __init__(self, numel=8, element_size=8, is_cuda=True, tag="dev"): + self._numel = numel + self._element_size = element_size + self.is_cuda = is_cuda + self.tag = tag + self.recorded = [] + self.cpu_calls = 0 + self.device = types.SimpleNamespace(type="cuda" if is_cuda else "cpu") + self.dtype = "torch.int64" + + def numel(self): + return self._numel + + def element_size(self): + return self._element_size + + def record_stream(self, stream): + self.recorded.append(stream) + + def copy_(self, src, non_blocking=False): + self.copied_from = src + + @property + def shape(self): + return (self._numel,) + + def cpu(self): + self.cpu_calls += 1 + return _FakeTensor(self._numel, self._element_size, False, tag=self.tag + "-cpu") + + def __len__(self): + return self._numel + + +class _FakeHostPool: + def __init__(self, shared=False): + self.shared = shared + + +class _FakeEntry: + def __init__(self, host_pool): + self.host_pool = host_pool + + +class _FakeMemPoolHost: + layout = "page_first_direct" + + def __init__(self, anchor_shared=False, sidecars=None): + self.anchor_entry = _FakeEntry(_FakeHostPool(anchor_shared)) + self.entry_map = { + name: _FakeEntry(pool) for name, pool in (sidecars or {}).items() + } + + +def _make_controller(*, enabled=True, anchor_shared=False, sidecars=None, policy=""): + c = cc_mod.HiCacheController.__new__(cc_mod.HiCacheController) + c.io_backend = "direct" + c.write_policy = policy + c.mem_pool_host = _FakeMemPoolHost(anchor_shared, sidecars) + c.write_queue = [] + c.load_queue = [] + c.ack_write_queue = [] + c.ack_load_queue = [] + c._indices_stream = None + c._write_submit_queue = None + c._write_result_queue = cc_mod.Queue() + c._write_submit_stop = threading.Event() + c._write_submit_thread = None + c._write_submit_state_lock = threading.Lock() + c._write_submit_cond = threading.Condition(c._write_submit_state_lock) + c._write_submit_active = 0 + c._async_pinned_bytes = 0 + c._retire_parts = [] + return c + + +def _op(controller, *, anchor=None, transfers=(), node_id=7): + host = _FakeTensor(8, 8, is_cuda=False, tag="host") + dev = anchor or _FakeTensor(8, 8, is_cuda=True, tag="anchor") + op = cc_mod.CacheOperation(host, dev, node_id, pool_transfers=list(transfers)) + return op + + +class AsyncBackupUnitTest(unittest.TestCase): + def setUp(self): + patcher = mock.patch.object(cc_mod, "_ASYNC_BACKUP_SUBMIT", True) + patcher.start() + self.addCleanup(patcher.stop) + # The real predicate isinstance-checks HostKVCache; the fakes expose a + # plain `shared` flag instead. + shared_patcher = mock.patch.object( + cc_mod, "uses_shared_host_layout", lambda pool: bool(getattr(pool, "shared", False)) + ) + shared_patcher.start() + self.addCleanup(shared_patcher.stop) + cat_patcher = mock.patch.object( + cc_mod.torch, + "cat", + side_effect=lambda parts: _FakeTensor( + sum(len(p) for p in parts), 8, False, "cat" + ), + ) + cat_patcher.start() + self.addCleanup(cat_patcher.stop) + # Pinned destination buffers are faked; only the copy_ call is checked. + self.pin_patch = mock.patch.object( + cc_mod.torch, "empty", side_effect=lambda *a, **k: _FakeTensor(8, 8, False, "pin") + ) + self.pin_patch.start() + self.addCleanup(self.pin_patch.stop) + + def test_disabled_keeps_sync_path(self): + with mock.patch.object(cc_mod, "_ASYNC_BACKUP_SUBMIT", False): + c = _make_controller() + self.assertFalse(c.async_backup_enabled()) + self.assertFalse(c._async_submit_applicable()) + + def test_wrong_backend_or_layout_is_ineligible(self): + c = _make_controller() + c.io_backend = "kernel" + self.assertFalse(c.async_backup_enabled()) + c = _make_controller() + c.mem_pool_host.layout = "page_first" + self.assertFalse(c.async_backup_enabled()) + + def test_index_targets_skip_shared_layout_and_cpu_tensors(self): + shared = PoolName.SWA + c = _make_controller(sidecars={shared: _FakeHostPool(shared=True)}) + transfer = PoolTransfer( + name=shared, device_indices=_FakeTensor(4, 8, True, "swa") + ) + cpu_anchor = _FakeTensor(8, 8, False, "anchor") + op = _op(c, anchor=cpu_anchor, transfers=[transfer]) + self.assertEqual(c._async_index_targets(op), []) + + c = _make_controller(sidecars={PoolName.SWA: _FakeHostPool(shared=False)}) + transfer = PoolTransfer( + name=PoolName.SWA, + device_indices=_FakeTensor(4, 8, True, "swa"), + indices_from_pool=PoolName.KV, + ) + anchor = _FakeTensor(8, 8, True, "anchor") + op = _op(c, anchor=anchor, transfers=[transfer]) + targets = c._async_index_targets(op) + self.assertEqual([k for k, _ in targets], [(None, None), (PoolName.SWA, PoolName.KV)]) + + def test_start_copy_orders_after_scheduler_and_records_event(self): + log = [] + c = _make_controller() + op = _op(c) + with mock.patch.object(cc_mod, "device_module", _FakeCudaModule(log)): + c._start_async_index_copy(op) + self.assertIsNotNone(op.async_parts) + self.assertEqual(op.async_parts.ready_event._set, True) + self.assertEqual(op.device_indices.recorded[0].name, "indices") + # The copy stream is ordered after the scheduler stream once. + self.assertIn(("wait_stream", "indices", "schedule"), log) + self.assertEqual(c._async_pinned_bytes, 64) + + def test_pinned_cap_falls_back_to_sync(self): + c = _make_controller() + op = _op(c) + with mock.patch.object(cc_mod, "_ASYNC_BACKUP_PINNED_BYTES_MAX", 1), \ + mock.patch.object(cc_mod, "device_module", _FakeCudaModule()): + c._start_async_index_copy(op) + self.assertIsNone(op.async_parts) + self.assertEqual(c._async_pinned_bytes, 0) + + def test_materialize_uses_pool_keys(self): + c = _make_controller() + anchor_copy = cc_mod.AsyncIndexCopy((None, None), None, _FakeTensor(8, 8, False, "a"), 64) + deriv = PoolTransfer( + name=PoolName.SWA, + device_indices=_FakeTensor(4, 8, True, "swa"), + indices_from_pool=PoolName.KV, + ) + side_copy = cc_mod.AsyncIndexCopy( + (PoolName.SWA, PoolName.KV), None, _FakeTensor(4, 8, False, "s"), 32 + ) + op = _op(c, transfers=[deriv]) + op.async_parts = cc_mod.AsyncIndexParts( + copies=[anchor_copy, side_copy], ready_event=_FakeEvent(), nbytes=96 + ) + c._materialize_async_indices([op]) + self.assertIs(op.device_indices, anchor_copy.dst) + self.assertIs(deriv.device_indices, side_copy.dst) + + def test_worker_waits_events_then_submits_and_acks_once(self): + log = [] + c = _make_controller() + events = [] + + def _make_op(tag): + op = _op(c, anchor=_FakeTensor(8, 8, True, tag)) + ev = _FakeEvent(log, tag) + events.append(ev) + op.async_parts = cc_mod.AsyncIndexParts( + copies=[ + cc_mod.AsyncIndexCopy( + (None, None), op.device_indices, _FakeTensor(8, 8, False, tag), 64 + ) + ], + ready_event=ev, + nbytes=64, + ) + return op + + ops = [_make_op("a"), _make_op("b")] + c._async_pinned_bytes = 128 + submitted = [] + + def _submit(op): + # Every ready event must have been waited before the merge is used. + submitted.append(list(op.node_ids)) + return cc_mod.HiCacheAck(None, None, list(op.node_ids), num_tokens=8) + + c._submit_write_operation = _submit + c._write_submit_queue = cc_mod.Queue(maxsize=4) + c._ensure_write_submit_thread = lambda: None + worker = threading.Thread(target=c._write_submit_worker, daemon=True) + c._write_submit_thread = worker + worker.start() + try: + item = cc_mod.WriteSubmission(ops=ops, tracer=None, enqueue_mono_ns=0) + c._write_submit_active = 1 + c._write_submit_queue.put(item) + deadline = time.time() + 5 + while c._write_submit_active and time.time() < deadline: + time.sleep(0.005) + self.assertEqual(c._write_submit_active, 0) + finally: + c._write_submit_queue.put(None) + worker.join(timeout=5) + self.assertEqual([e.waited for e in events], [1, 1]) + self.assertEqual(submitted, [[7, 7]]) + self.assertEqual(c._async_pinned_bytes, 0) + c.drain_write_submission_results() + self.assertEqual(len(c.ack_write_queue), 1) + # Second drain is a no-op: exactly-once ACK. + c.drain_write_submission_results() + self.assertEqual(len(c.ack_write_queue), 1) + + def test_worker_error_is_fail_closed(self): + c = _make_controller() + op = _op(c) + op.async_parts = cc_mod.AsyncIndexParts([], _FakeEvent(), 0) + c._submit_write_operation = mock.Mock(side_effect=RuntimeError("boom")) + c._write_submit_queue = cc_mod.Queue(maxsize=4) + worker = threading.Thread(target=c._write_submit_worker, daemon=True) + c._write_submit_thread = worker + worker.start() + c._write_submit_active = 1 + c._write_submit_queue.put(cc_mod.WriteSubmission([op], None, 0)) + deadline = time.time() + 5 + while c._write_submit_active and time.time() < deadline: + time.sleep(0.005) + c._write_submit_queue.put(None) + worker.join(timeout=5) + with self.assertRaises(RuntimeError): + c.drain_write_submission_results() + self.assertEqual(c.ack_write_queue, []) + + def test_enqueue_failure_keeps_active_consistent(self): + c = _make_controller() + c._write_submit_queue = cc_mod.Queue(maxsize=1) + c._write_submit_queue.put("occupied") + c._ensure_write_submit_thread = lambda: None + with mock.patch.object(cc_mod, "_ASYNC_BACKUP_ENQUEUE_TIMEOUT", 0.05): + ok = c._enqueue_async_write([_op(c)], None) + self.assertFalse(ok) + self.assertEqual(c._write_submit_active, 0) + + def test_queue_full_falls_back_to_sync_submit(self): + c = _make_controller() + # A permanently-full submit queue forces the sync path. + c._write_submit_queue = cc_mod.Queue(maxsize=1) + c._write_submit_queue.put("occupied") + c._ensure_write_submit_thread = lambda: None + op = _op(c) + retired_event = _FakeEvent() + retired_event.record() + op.async_parts = cc_mod.AsyncIndexParts( + [cc_mod.AsyncIndexCopy((None, None), op.device_indices, _FakeTensor(8, 8, False), 64)], + retired_event, + 64, + ) + c._async_pinned_bytes = 64 + c.write_queue.append(op) + submitted = [] + + def _submit(merged): + # The fallback must hand over the original device indices, i.e. the + # pinned parts are retired rather than materialized. + submitted.append(merged.device_indices) + return cc_mod.HiCacheAck(None, None, list(merged.node_ids), num_tokens=8) + + c._submit_write_operation = _submit + with mock.patch.object(cc_mod, "_ASYNC_BACKUP_ENQUEUE_TIMEOUT", 0.05): + c.start_writing() + self.assertEqual(len(submitted), 1) + self.assertIs(submitted[0], op.device_indices) + self.assertEqual(len(c.ack_write_queue), 1) + self.assertEqual(c.write_queue, []) + self.assertIsNone(op.async_parts) + self.assertEqual(c._write_submit_active, 0) + # The retired copy is released once its event completes. + c._reap_retired_parts() + self.assertEqual(c._async_pinned_bytes, 0) + + def test_mixed_queue_sync_path_retires_unconsumed_parts(self): + c = _make_controller() + ev = _FakeEvent() + ev.record() + with_parts = _op(c, node_id=1) + with_parts.async_parts = cc_mod.AsyncIndexParts( + [ + cc_mod.AsyncIndexCopy( + (None, None), with_parts.device_indices, _FakeTensor(8, 8, False), 64 + ) + ], + ev, + 64, + ) + without_parts = _op(c, node_id=2) + c.write_queue.extend([with_parts, without_parts]) + c._async_pinned_bytes = 64 + submitted = [] + c._submit_write_operation = lambda merged: ( + submitted.append(list(merged.node_ids)) + or cc_mod.HiCacheAck(None, None, list(merged.node_ids), num_tokens=8) + ) + c.start_writing() + # Mixed residency () forces the sync path, but the pinned copy started + # for the first op must be retired and its budget released. + self.assertEqual(submitted, [[1, 2]]) + self.assertIsNone(with_parts.async_parts) + self.assertEqual(c._async_pinned_bytes, 0) + self.assertEqual(len(c.ack_write_queue), 1) + + def test_op_cap_falls_back_to_sync(self): + c = _make_controller() + with mock.patch.object(cc_mod, "_ASYNC_BACKUP_MAX_OPS", 1): + for i in range(2): + op = _op(c, node_id=i) + op.async_parts = cc_mod.AsyncIndexParts([], _FakeEvent(), 0) + c.write_queue.append(op) + self.assertFalse(c._async_submit_applicable()) + + def test_inflight_reflects_async_state(self): + c = _make_controller() + self.assertFalse(c.has_inflight_device_transfers()) + c._write_submit_active = 1 + self.assertTrue(c.has_inflight_device_transfers()) + c._write_submit_active = 0 + c._async_pinned_bytes = 8 + self.assertTrue(c.has_inflight_device_transfers()) + c._async_pinned_bytes = 0 + c._write_result_queue.put(cc_mod.WriteResult(None, [], None)) + self.assertTrue(c.has_inflight_device_transfers()) + c._write_result_queue.get() + self.assertFalse(c.has_inflight_device_transfers()) + + def test_idle_wait_times_out(self): + c = _make_controller() + c._write_submit_active = 1 + with self.assertRaises(RuntimeError): + c.wait_write_submissions_idle(timeout=0.05) + + def test_retired_parts_release_only_after_event(self): + c = _make_controller() + ev = _FakeEvent() + ev._set = False + op = _op(c) + op.async_parts = cc_mod.AsyncIndexParts( + [cc_mod.AsyncIndexCopy((None, None), None, _FakeTensor(8, 8, False), 64)], + ev, + 64, + ) + c._async_pinned_bytes = 64 + c._retire_async_parts([op]) + self.assertIsNone(op.async_parts) + self.assertEqual(c._async_pinned_bytes, 64) + ev._set = True + c._reap_retired_parts() + self.assertEqual(c._async_pinned_bytes, 0) + self.assertEqual(c._retire_parts, []) + + def test_merge_of_cpu_parts_matches_cpu_reference(self): + c = _make_controller() + ops = [] + for i in range(3): + op = _op(c, anchor=_FakeTensor(4, 8, True, f"a{i}"), node_id=i) + op.async_parts = cc_mod.AsyncIndexParts( + [ + cc_mod.AsyncIndexCopy( + (None, None), + op.device_indices, + _FakeTensor(4, 8, False, f"p{i}"), + 32, + ) + ], + _FakeEvent(), + 32, + ) + ops.append(op) + c._materialize_async_indices(ops) + merged = cc_mod.CacheOperation.merge_ops(ops) + self.assertEqual(merged.node_ids, [0, 1, 2]) + # Single-op merge must not copy: it returns the operation itself. + single = cc_mod.CacheOperation.merge_ops([ops[0]]) + self.assertIs(single, ops[0]) + + def test_stop_waits_for_inflight_submission_and_drains_ack(self): + c = _make_controller() + release = threading.Event() + started = threading.Event() + + def _submit(op): + started.set() + release.wait(10) + return cc_mod.HiCacheAck(None, None, list(op.node_ids), num_tokens=8) + + c._submit_write_operation = _submit + op = _op(c) + op.async_parts = cc_mod.AsyncIndexParts([], _FakeEvent(), 0) + c._ensure_write_submit_thread() + c._write_submit_active = 1 + c._write_submit_queue.put(cc_mod.WriteSubmission([op], None, 0)) + self.assertTrue(started.wait(5)) + + stopped = threading.Event() + + def _stop(): + c._stop_write_submission_thread() + stopped.set() + + stopper = threading.Thread(target=_stop, daemon=True) + stopper.start() + # The stop must not return while a submission is still in flight. + self.assertFalse(stopped.wait(0.05)) + release.set() + self.assertTrue(stopped.wait(10)) + self.assertFalse(c._write_submit_thread is not None and c._write_submit_thread.is_alive()) + # The ACK produced by the in-flight submission was not dropped. + self.assertEqual(len(c.ack_write_queue), 1) + + def test_stop_joins_worker_and_allows_restart(self): + c = _make_controller() + c._ensure_write_submit_thread() + self.assertTrue(c._write_submit_thread.is_alive()) + c._stop_write_submission_thread() + self.assertIsNone(c._write_submit_thread) + self.assertIsNone(c._write_submit_queue) + c._ensure_write_submit_thread() + self.assertTrue(c._write_submit_thread.is_alive()) + c._stop_write_submission_thread() + + +if __name__ == "__main__": + unittest.main() diff --git a/timeline_probe.py b/timeline_probe.py new file mode 100644 index 0000000000..978e6ff4d0 --- /dev/null +++ b/timeline_probe.py @@ -0,0 +1,76 @@ +"""Print the CUDA timeline for one async index copy vs the next forward.""" +import importlib.util as _u +import threading +import time + +import torch + +from sglang.srt.managers import cache_controller as cc_mod + +_spec = _u.spec_from_file_location( + "t", "/data/ymk/wt-async/test/registered/cuda/test_async_backup_index_copy_cuda.py" +) +_m = _u.module_from_spec(_spec) +_spec.loader.exec_module(_m) +_make_controller, _op = _m._make_controller, _m._op + +cc_mod._ASYNC_BACKUP_SUBMIT = True +cc_mod._ASYNC_BACKUP_PINNED_BYTES_MAX = 1 << 30 + +torch.cuda.synchronize() +sched = torch.cuda.current_stream() +fwd = torch.cuda.Stream() +indices = torch.arange(4 * 1024 * 1024, dtype=torch.int64, device="cuda") + +origin = torch.cuda.Event(enable_timing=True) +with torch.cuda.stream(sched): + origin.record() + torch.cuda._sleep(1_000_000_000) + fence_done = torch.cuda.Event(enable_timing=True) + fence_done.record() + +c = _make_controller() +op = _op(indices, 0) +t0 = time.perf_counter() +c._start_async_index_copy(op) +host_ms = (time.perf_counter() - t0) * 1e3 + +copy_end = torch.cuda.Event(enable_timing=True) +with torch.cuda.stream(c._indices_stream): + copy_end.record() + +t_launch = time.monotonic_ns() +with torch.cuda.stream(fwd): + fwd.wait_stream(sched) + fwd_start = torch.cuda.Event(enable_timing=True) + fwd_start.record() + torch.cuda._sleep(1_500_000_000) + fwd_end = torch.cuda.Event(enable_timing=True) + fwd_end.record() + +ready_ns = [None] + + +def _w(): + op.async_parts.ready_event.synchronize() + ready_ns[0] = time.monotonic_ns() + + +th = threading.Thread(target=_w) +th.start() +th.join() +torch.cuda.synchronize() + +print(f"host start_async_index_copy: {host_ms:.2f} ms") +print(f"host next-batch enqueue before ready: {t_launch < ready_ns[0]}") +print(f"fence_done -> copy_end : {fence_done.elapsed_time(copy_end):8.2f} ms") +print(f"origin -> fwd_start : {origin.elapsed_time(fwd_start):8.2f} ms") +print(f"origin -> copy_end : {origin.elapsed_time(copy_end):8.2f} ms") +print(f"origin -> fwd_end : {origin.elapsed_time(fwd_end):8.2f} ms") +print(f"fwd duration : {fwd_start.elapsed_time(fwd_end):8.2f} ms") +print( + f"fwd_start -> copy_end : {fwd_start.elapsed_time(copy_end):8.2f} ms (>0 means overlap)" +) +print( + f"copy_end -> fwd_end : {copy_end.elapsed_time(fwd_end):8.2f} ms (>0 means copy finished inside the forward)" +)