rebase(hicache): per-op async backup on top of #46884 + #48782

Net async change (from exp-async-backup-perop-20261107) applied onto
b300-pick-46884-20261107. Runtime files applied cleanly; the new tests and
the timeline probe are added as-is.

Interaction to validate on hardware: #46884 publishes prefill read-done before
the replay, so the scheduler-stream fence that the index copy waits on is
shorter; the async worker's wait should shrink and the copy may overlap more of
the in-flight forward. Re-baseline the 4-arm A/B on this branch.
This commit is contained in:
2026-11-07 22:04:33 +08:00
parent 520ca548ea
commit d6f1e1efbd
8 changed files with 1434 additions and 99 deletions
+488 -83
View File
@@ -18,7 +18,7 @@ import logging
import threading
import time
from dataclasses import dataclass
from queue import Empty, Queue
from queue import Empty, Full, Queue
from typing import TYPE_CHECKING, Any, Callable, List, NamedTuple, Optional
import torch
@@ -42,6 +42,7 @@ from sglang.srt.layers.dp_attention import (
)
from sglang.srt.mem_cache.l2_transfer import L2Transfer, L2TransferEngine
from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool
from sglang.srt.mem_cache.pool_host.base import uses_shared_host_layout
from sglang.srt.mem_cache.utils import get_storage_hash_str
from sglang.srt.observability import batch_telemetry
from sglang.srt.runtime_context import get_parallel
@@ -69,6 +70,25 @@ _HC_PREFETCH_WORKERS = int(_hc_os.getenv("SGLANG_HICACHE_PREFETCH_IO_THREADS", "
_HC_BACKUP_WORKERS = int(_hc_os.getenv("SGLANG_HICACHE_BACKUP_IO_THREADS", "1") or 1)
_HC_TIMING = _hc_os.getenv("SGLANG_HICACHE_TIMING", "0") not in ("", "0", "false", "False")
# [ASYNC_BACKUP] Per-op pinned index copy + worker-side merge/submit.
# Only the direct + page_first_direct shape (the one that does a blocking
# device_indices.cpu() on the scheduler thread) is eligible.
_ASYNC_BACKUP_SUBMIT = _hc_os.getenv(
"SGLANG_ASYNC_BACKUP_SUBMIT", "0"
).strip().lower() not in ("", "0", "false", "no", "off")
_ASYNC_BACKUP_ENQUEUE_TIMEOUT = float(
_hc_os.getenv("SGLANG_ASYNC_BACKUP_ENQUEUE_TIMEOUT", "60") or 60
)
_ASYNC_BACKUP_IDLE_TIMEOUT = float(
_hc_os.getenv("SGLANG_ASYNC_BACKUP_IDLE_TIMEOUT", "120") or 120
)
_ASYNC_BACKUP_PINNED_BYTES_MAX = int(
_hc_os.getenv("SGLANG_ASYNC_BACKUP_PINNED_MB", "2048") or 2048
) * 1024 * 1024
_ASYNC_BACKUP_MAX_OPS = int(
_hc_os.getenv("SGLANG_ASYNC_BACKUP_MAX_OPS", "256") or 256
)
class LayerLoadingEvent:
def __init__(self, num_layers: int):
self._num_layers = num_layers
@@ -132,6 +152,9 @@ class CacheOperation:
self.node_ids = [node_id]
self.data = None
self.pool_transfers = pool_transfers
# Set when SGLANG_ASYNC_BACKUP_SUBMIT copied this op's device indices
# into pinned host buffers (see _start_async_index_copy).
self.async_parts = None
self.id = CacheOperation.counter
CacheOperation.counter += 1
@@ -192,6 +215,39 @@ class CacheOperation:
return self.priority < other.priority
@dataclass
class AsyncIndexCopy:
"""One device->pinned-host index copy started where the backup op was made."""
key: tuple
src: torch.Tensor
dst: torch.Tensor
nbytes: int
@dataclass
class AsyncIndexParts:
"""Per-op pinned index parts plus the event that guarantees they are final."""
copies: List[AsyncIndexCopy]
ready_event: object
nbytes: int
@dataclass
class WriteSubmission:
ops: List[CacheOperation]
tracer: object
enqueue_mono_ns: int
@dataclass
class WriteResult:
ack: Optional[HiCacheAck]
ops: List[CacheOperation]
error: Optional[BaseException]
class HiCacheAck(NamedTuple):
start_event: device_module.Event
finish_event: device_module.Event
@@ -383,6 +439,21 @@ class HiCacheController:
self.write_fence_stream = None
self.ack_write_queue: List[HiCacheAck] = []
# [ASYNC_BACKUP] Pinned index copies are started where the backup op is
# created; a single worker waits the per-op events, merges the CPU
# index parts and submits the KV copy. Queues are bounded by the state
# lock/condition below; write_queue/ack_write_queue stay scheduler-only.
self._indices_stream = None
self._write_submit_queue: Optional[Queue] = None
self._write_result_queue: Queue = Queue()
self._write_submit_stop = threading.Event()
self._write_submit_thread: Optional[threading.Thread] = None
self._write_submit_state_lock = threading.Lock()
self._write_submit_cond = threading.Condition(self._write_submit_state_lock)
self._write_submit_active = 0
self._async_pinned_bytes = 0
self._retire_parts: List["AsyncIndexParts"] = []
self.l2_transfer_engine = L2TransferEngine(io_backend)
# If a storage backend is provided at startup, treat it as an implicit attach,
@@ -708,6 +779,9 @@ class HiCacheController:
Requirement: no in-flight requests. This will stop storage threads and release
the backend instance (best-effort close).
"""
# Async backup submissions own pinned index buffers and ACKs; stop them
# before any queue/pool teardown so no result is lost.
self._stop_write_submission_thread()
# Idempotent cleanup: even if `enable_storage` is already False,
# we may still have leftover resources (threads/backend/process group) from a
# previous partial detach. We attempt cleanup whenever possible.
@@ -822,6 +896,7 @@ class HiCacheController:
def reset(self):
# Reuse detach's queue wakeups and bounded joins, and fail if any
# storage thread remains alive before clearing shared state.
self._stop_write_submission_thread()
self._stop_storage_threads()
self.write_queue.clear()
@@ -861,13 +936,311 @@ class HiCacheController:
def has_inflight_device_transfers(self) -> bool:
"""Whether queued or unacknowledged L2 transfers still use device rows."""
with self._write_submit_state_lock:
async_pending = (
self._write_submit_active > 0
or self._async_pinned_bytes > 0
or not self._write_result_queue.empty()
)
return bool(
self.write_queue
or self.load_queue
or self.ack_write_queue
or self.ack_load_queue
or async_pending
)
# ---- async backup submission -----------------------------------------
def async_backup_enabled(self) -> bool:
return (
_ASYNC_BACKUP_SUBMIT
and self.io_backend == "direct"
and getattr(self.mem_pool_host, "layout", None) == "page_first_direct"
)
def _async_submit_applicable(self) -> bool:
return (
self.async_backup_enabled()
and len(self.write_queue) <= _ASYNC_BACKUP_MAX_OPS
and all(op.async_parts is not None for op in self.write_queue)
)
def _ensure_indices_stream(self):
if self._indices_stream is None:
self._indices_stream = device_module.Stream()
return self._indices_stream
def _needs_device_index_copy(self, host_pool, indices) -> bool:
if indices is None or not getattr(indices, "is_cuda", False):
return False
# move_indices() returns shared-layout ids untouched; converting them
# here would hand the transfer a different interface than the sync path.
return not uses_shared_host_layout(host_pool)
def _async_index_targets(self, op: CacheOperation):
targets = []
if self._needs_device_index_copy(
self.mem_pool_host.anchor_entry.host_pool, op.device_indices
):
targets.append(((None, None), op.device_indices))
for transfer in op.pool_transfers or []:
if transfer.device_indices is None:
continue
entry = self.mem_pool_host.entry_map.get(transfer.name)
if entry is None:
continue
if not self._needs_device_index_copy(
entry.host_pool, transfer.device_indices
):
continue
key = (transfer.name, transfer.indices_from_pool)
targets.append((key, transfer.device_indices))
return targets
def _start_async_index_copy(self, op: CacheOperation) -> None:
"""Start the pinned index D2H copy now, off the scheduler stream.
Runs in the scheduler thread at backup-op creation time. The copy runs
on a dedicated stream ordered after this instant, so it does not inherit
later fences (the WAR barrier of a later forward) and the scheduler
never blocks in ``.cpu()``.
"""
if not self.async_backup_enabled():
return
targets = self._async_index_targets(op)
if not targets:
return
nbytes = sum(int(t.numel()) * int(t.element_size()) for _, t in targets)
with self._write_submit_state_lock:
if self._async_pinned_bytes + nbytes > _ASYNC_BACKUP_PINNED_BYTES_MAX:
return
self._async_pinned_bytes += nbytes
try:
stream = self._ensure_indices_stream()
source_stream = device_module.current_stream()
copies: List[AsyncIndexCopy] = []
with device_module.StreamContext(stream):
stream.wait_stream(source_stream)
for key, src in targets:
# Keep the device rows alive until the copy has run, and pin
# the destination so copy_ is a real asynchronous D2H.
src.record_stream(stream)
dst = torch.empty(
src.shape,
dtype=src.dtype,
device="cpu",
pin_memory=True,
)
dst.copy_(src, non_blocking=True)
copies.append(
AsyncIndexCopy(
key=key,
src=src,
dst=dst,
nbytes=int(src.numel()) * int(src.element_size()),
)
)
ready_event = device_module.Event()
ready_event.record(stream)
except BaseException:
with self._write_submit_state_lock:
self._async_pinned_bytes -= nbytes
logger.exception("Async backup index copy failed; keeping sync path")
return
op.async_parts = AsyncIndexParts(
copies=copies, ready_event=ready_event, nbytes=nbytes
)
def _ensure_write_submit_thread(self) -> None:
if self._write_submit_queue is None:
self._write_submit_queue = Queue(maxsize=64)
thread = self._write_submit_thread
if thread is not None and thread.is_alive():
return
self._write_submit_stop.clear()
thread = threading.Thread(
target=self._write_submit_worker,
name="hicache-async-backup",
daemon=True,
)
self._write_submit_thread = thread
thread.start()
def _enqueue_async_write(self, ops, tracer) -> bool:
"""Enqueue one submission; False means "fall back to the sync path".
A full queue or a timeout is not fatal: the caller submits the same ops
synchronously. Before returning False we wait for the worker to become
idle so the fallback cannot race another submission on the shared
device_to_host_stream.
"""
self._ensure_write_submit_thread()
item = WriteSubmission(
ops=ops, tracer=tracer, enqueue_mono_ns=time.monotonic_ns()
)
with self._write_submit_cond:
self._write_submit_active += 1
try:
with batch_telemetry.backup_phase("enqueue"):
self._write_submit_queue.put(
item, timeout=_ASYNC_BACKUP_ENQUEUE_TIMEOUT
)
except Full:
with self._write_submit_cond:
self._write_submit_active -= 1
self._write_submit_cond.notify_all()
logger.warning(
"Async backup submit queue full after %.1fs; falling back to "
"the synchronous path",
_ASYNC_BACKUP_ENQUEUE_TIMEOUT,
)
self.wait_write_submissions_idle()
return False
return True
def _materialize_async_indices(self, ops) -> None:
"""Replace device index tensors with the pinned CPU copies (worker)."""
for op in ops:
parts = op.async_parts
if parts is None:
continue
mapping = {copy.key: copy.dst for copy in parts.copies}
anchor = mapping.get((None, None))
if anchor is not None:
op.device_indices = anchor
for transfer in op.pool_transfers or []:
dst = mapping.get((transfer.name, transfer.indices_from_pool))
if dst is not None:
transfer.device_indices = dst
def _write_submit_worker(self) -> None:
while True:
item = self._write_submit_queue.get()
if item is None:
break
tracer = item.tracer
previous = batch_telemetry.backup_flush_attach(tracer)
try:
with batch_telemetry.backup_phase("worker"):
for op in item.ops:
parts = op.async_parts
if parts is not None:
parts.ready_event.synchronize()
self._materialize_async_indices(item.ops)
merged = CacheOperation.merge_ops(item.ops)
ack = self._submit_write_operation(merged)
result = WriteResult(ack=ack, ops=item.ops, error=None)
except BaseException as exc: # noqa: BLE001 - fail closed
logger.exception("Async backup submission failed")
result = WriteResult(ack=None, ops=item.ops, error=exc)
finally:
batch_telemetry.backup_flush_detach(previous)
self._release_async_parts(item.ops)
try:
batch_telemetry.backup_flush_end(tracer)
except Exception:
pass
# Publish the result before marking the submission done, so an idle
# waiter cannot observe active==0 with the result still missing.
self._write_result_queue.put(result)
with self._write_submit_cond:
self._write_submit_active -= 1
self._write_submit_cond.notify_all()
def _release_async_parts(self, ops) -> None:
released = 0
for op in ops:
parts = op.async_parts
if parts is None:
continue
op.async_parts = None
parts.copies.clear()
released += parts.nbytes
if released:
with self._write_submit_state_lock:
self._async_pinned_bytes -= released
def _retire_async_parts(self, ops) -> None:
"""Release copies that will not be consumed (sync fallback)."""
for op in ops:
parts = op.async_parts
if parts is not None:
self._retire_parts.append(parts)
op.async_parts = None
self._reap_retired_parts()
def _reap_retired_parts(self) -> None:
if not self._retire_parts:
return
pending = []
for parts in self._retire_parts:
try:
done = parts.ready_event.query()
except Exception:
done = True
if done:
parts.copies.clear()
with self._write_submit_state_lock:
self._async_pinned_bytes -= parts.nbytes
else:
pending.append(parts)
self._retire_parts = pending
def drain_write_submission_results(self) -> None:
"""Move finished async ACKs into ack_write_queue (scheduler thread)."""
while True:
try:
result = self._write_result_queue.get_nowait()
except Empty:
break
if result.error is not None:
raise RuntimeError("async backup submission failed") from result.error
if result.ack is not None:
self.ack_write_queue.append(result.ack)
self._reap_retired_parts()
def wait_write_submissions_idle(
self, timeout: float = _ASYNC_BACKUP_IDLE_TIMEOUT
) -> None:
deadline = time.monotonic() + timeout
with self._write_submit_cond:
while self._write_submit_active > 0:
remaining = deadline - time.monotonic()
if remaining <= 0:
raise RuntimeError(
"Timed out waiting for async backup submissions"
)
self._write_submit_cond.wait(remaining)
def _stop_write_submission_thread(self) -> None:
if self._write_submit_thread is None:
self.drain_write_submission_results()
self._reap_retired_parts()
if self._retire_parts:
raise RuntimeError("async backup index copies still in flight")
return
try:
self.wait_write_submissions_idle()
finally:
self._write_submit_stop.set()
queue = self._write_submit_queue
if queue is not None:
try:
queue.put(None, timeout=5.0)
except Exception:
logger.exception("Failed to wake the async backup worker")
thread = self._write_submit_thread
if thread is not None:
thread.join(timeout=30.0)
if thread.is_alive():
raise RuntimeError("async backup submit worker did not stop")
self.drain_write_submission_results()
if self._retire_parts:
raise RuntimeError("async backup index copies still in flight")
self._write_submit_thread = None
self._write_submit_queue = None
def write(
self,
device_indices: torch.Tensor,
@@ -883,9 +1256,9 @@ class HiCacheController:
host_indices = self.mem_pool_host.alloc(len(device_indices))
if host_indices is None:
return None
self.write_queue.append(
CacheOperation(host_indices, device_indices, node_id, priority)
)
op = CacheOperation(host_indices, device_indices, node_id, priority)
self._start_async_index_copy(op)
self.write_queue.append(op)
self.start_writing()
return host_indices
@@ -898,98 +1271,130 @@ class HiCacheController:
batch_telemetry.detail_count("flush_empty")
return
ops = list(self.write_queue)
tracer = batch_telemetry.backup_flush_begin()
async_owned = False
try:
batch_telemetry.detail_count("flush_calls")
batch_telemetry.detail_count("flush_nonempty")
batch_telemetry.flush_note(
"sglang.flush_write_queue_ops", len(self.write_queue)
)
batch_telemetry.flush_note("sglang.flush_io_backend", self.io_backend)
batch_telemetry.flush_note(
"sglang.flush_layout",
str(getattr(self.mem_pool_host, "layout", None) or ""),
)
batch_telemetry.flush_note(
"sglang.flush_ack_queue_before", len(self.ack_write_queue)
)
self._note_write_queue(ops, len(self.ack_write_queue))
if self._async_submit_applicable():
if self._enqueue_async_write(ops, tracer):
self.write_queue.clear()
batch_telemetry.flush_note(
"sglang.flush_submission_mode", "async"
)
# The worker owns the tracer from here on (it emits once).
async_owned = True
return
# Queue full / enqueue timeout: retire the pinned copies and
# submit synchronously on this thread. `_enqueue_async_write`
# waited for the worker to go idle first, so the two paths
# never submit to device_to_host_stream concurrently.
batch_telemetry.flush_note("sglang.flush_fallback_sync", 1)
self._retire_async_parts(ops)
# Any parts that were started for this queue are not consumed by the
# sync path. Retire them unconditionally (they are released once
# their events complete) so the pinned-byte budget never leaks.
batch_telemetry.flush_note(
"sglang.flush_async_skip_ops",
sum(1 for op in ops if op.async_parts is None),
)
self._retire_async_parts(ops)
batch_telemetry.flush_note("sglang.flush_submission_mode", "sync")
with batch_telemetry.backup_phase("merge_ops"):
op = CacheOperation.merge_ops(self.write_queue)
tokens = len(op.device_indices)
indices = op.device_indices
batch_telemetry.flush_note("sglang.flush_tokens", int(tokens))
batch_telemetry.flush_note(
"sglang.flush_nodes_merged", len(op.node_ids)
)
batch_telemetry.flush_note(
"sglang.flush_sidecar_pools", len(op.pool_transfers or [])
)
batch_telemetry.flush_note(
"sglang.flush_indices_device", str(indices.device.type)
)
batch_telemetry.flush_note(
"sglang.flush_indices_dtype", str(indices.dtype)
)
batch_telemetry.flush_note(
"sglang.flush_indices_numel", int(indices.numel())
)
with batch_telemetry.backup_phase("move_write"):
host_indices, device_indices, pool_transfers = (
self._move_write_operation(op)
)
self.write_queue.clear()
batch_telemetry.flush_note(
"sglang.flush_host_indices_on_cpu",
bool(getattr(host_indices, "is_cuda", False) is False),
)
batch_telemetry.flush_note(
"sglang.flush_device_indices_on_cpu",
bool(getattr(device_indices, "is_cuda", False) is False),
)
if self.write_fence_stream is not None:
# The copy may read KV a forward is still writing: under overlap
# scheduling a finished request is cached while the next forward
# writes its last token, and an evicted SWA page can belong to an
# unfinished checkpoint. Later forwards still overlap with the copy.
with batch_telemetry.backup_phase("write_fence"):
self.l2_transfer_engine.device_to_host_stream.wait_stream(
self.write_fence_stream
)
with batch_telemetry.backup_phase("build_l2"):
transfers = self._l2_transfers(
host_indices, device_indices, pool_transfers
)
batch_telemetry.flush_note("sglang.flush_l2_transfers", len(transfers))
with batch_telemetry.backup_phase("submit_d2h"):
completion = self.l2_transfer_engine.submit_device_to_host(transfers)
with batch_telemetry.backup_phase("set_done_event"):
self.mem_pool_device_allocator.set_hicache_transfer_done_event(
(id(self), "write"), completion.finish_event
)
ack = self._submit_write_operation(op)
with batch_telemetry.backup_phase("ack_enqueue"):
self.ack_write_queue.append(
HiCacheAck(
start_event=completion.start_event,
finish_event=completion.finish_event,
node_ids=op.node_ids,
num_tokens=len(op.device_indices),
timing_enabled=completion.timing_enabled,
num_tokens_by_pool=self._num_tokens_by_pool(op),
num_bytes=self._transfer_num_bytes(op),
)
)
self.ack_write_queue.append(ack)
batch_telemetry.flush_note(
"sglang.flush_ack_queue_after", len(self.ack_write_queue)
)
finally:
batch_telemetry.backup_flush_end(tracer)
if not async_owned:
batch_telemetry.backup_flush_end(tracer)
def _note_write_queue(self, ops: List[CacheOperation], ack_before: int) -> None:
"""Queue-level flush dimensions (same for the sync and async paths)."""
batch_telemetry.flush_note("sglang.flush_write_queue_ops", len(ops))
batch_telemetry.flush_note("sglang.flush_io_backend", self.io_backend)
batch_telemetry.flush_note(
"sglang.flush_layout",
str(getattr(self.mem_pool_host, "layout", None) or ""),
)
batch_telemetry.flush_note("sglang.flush_ack_queue_before", ack_before)
def _submit_write_operation(self, op: CacheOperation) -> HiCacheAck:
"""Move indices, submit the D2H copy and build the ACK.
Runs on the scheduler thread (sync path) or on the async submit worker
(async path, with CPU-resident indices). It must not touch
``write_queue`` or ``ack_write_queue``; the caller owns those.
"""
tokens = len(op.device_indices)
indices = op.device_indices
batch_telemetry.flush_note("sglang.flush_tokens", int(tokens))
batch_telemetry.flush_note("sglang.flush_nodes_merged", len(op.node_ids))
batch_telemetry.flush_note(
"sglang.flush_sidecar_pools", len(op.pool_transfers or [])
)
batch_telemetry.flush_note(
"sglang.flush_indices_device", str(indices.device.type)
)
batch_telemetry.flush_note("sglang.flush_indices_dtype", str(indices.dtype))
batch_telemetry.flush_note(
"sglang.flush_indices_numel", int(indices.numel())
)
with batch_telemetry.backup_phase("move_write"):
host_indices, device_indices, pool_transfers = (
self._move_write_operation(op)
)
batch_telemetry.flush_note(
"sglang.flush_host_indices_on_cpu",
bool(getattr(host_indices, "is_cuda", False) is False),
)
batch_telemetry.flush_note(
"sglang.flush_device_indices_on_cpu",
bool(getattr(device_indices, "is_cuda", False) is False),
)
if self.write_fence_stream is not None:
# The copy may read KV a forward is still writing: under overlap
# scheduling a finished request is cached while the next forward
# writes its last token, and an evicted SWA page can belong to an
# unfinished checkpoint. Later forwards still overlap with the copy.
with batch_telemetry.backup_phase("write_fence"):
self.l2_transfer_engine.device_to_host_stream.wait_stream(
self.write_fence_stream
)
with batch_telemetry.backup_phase("build_l2"):
transfers = self._l2_transfers(
host_indices, device_indices, pool_transfers
)
batch_telemetry.flush_note("sglang.flush_l2_transfers", len(transfers))
with batch_telemetry.backup_phase("submit_d2h"):
completion = self.l2_transfer_engine.submit_device_to_host(transfers)
with batch_telemetry.backup_phase("set_done_event"):
self.mem_pool_device_allocator.set_hicache_transfer_done_event(
(id(self), "write"), completion.finish_event
)
return HiCacheAck(
start_event=completion.start_event,
finish_event=completion.finish_event,
node_ids=op.node_ids,
num_tokens=len(op.device_indices),
timing_enabled=completion.timing_enabled,
num_tokens_by_pool=self._num_tokens_by_pool(op),
num_bytes=self._transfer_num_bytes(op),
)
def _transfer_num_bytes(self, op: CacheOperation) -> int:
return len(op.device_indices) * self.mem_pool_host.size_per_token