hicache: pipeline prefetch/backup IO lanes + stage timing

- prefetch_io_aux_func: allow up to SGLANG_HICACHE_PREFETCH_IO_THREADS
  transfers in flight (default 1 = legacy behaviour); PrefetchAck is still
  emitted strictly FIFO so the per-rank ack stream (and the TP all_reduce in
  prefetch_sync_thread_func) is unchanged.
- backup_thread_func: same pipelining for L2->L3 writes via
  SGLANG_HICACHE_BACKUP_IO_THREADS.
- SGLANG_HICACHE_TIMING=1: emit [HC_TIMING] prefetch/backup queue, inflight,
  p50/p95 per 100 ops (the prefetch/backup lanes were previously single
  threaded; this is what caps L3<->L2 at ~1 op at a time).
This commit is contained in:
ymk
2026-11-06 13:18:07 +08:00
committed by minke.yu
parent c911d1b572
commit bf8cf7f1dc
3 changed files with 445 additions and 58 deletions
+178 -27
View File
@@ -62,6 +62,12 @@ def storage_model_name(
return f"{model_name}-{tag}" if model_name else tag
# [HC_PARALLEL_PATCH] knobs for the pipelined HiCache prefetch/backup lanes
import os as _hc_os
_HC_PREFETCH_WORKERS = int(_hc_os.getenv("SGLANG_HICACHE_PREFETCH_IO_THREADS", "1") or 1)
_HC_BACKUP_WORKERS = int(_hc_os.getenv("SGLANG_HICACHE_BACKUP_IO_THREADS", "1") or 1)
_HC_TIMING = _hc_os.getenv("SGLANG_HICACHE_TIMING", "0") not in ("", "0", "false", "False")
class LayerLoadingEvent:
def __init__(self, num_layers: int):
self._num_layers = num_layers
@@ -1152,7 +1158,11 @@ class HiCacheController:
count += 1
return count
def _page_transfer(self, operation: PrefetchOperation) -> int:
def _page_transfer(
self,
operation: PrefetchOperation,
ack_sink: Optional[Callable[[PrefetchAck], None]] = None,
) -> int:
# Transfer batch by batch
prefix_keys = operation.prefix_keys
kv_derived_transfers = [
@@ -1202,9 +1212,28 @@ class HiCacheController:
completed_tokens=completed_pages * self.page_size,
operation=operation,
)
self.prefetch_sync_queue.put(ack)
if ack_sink is None:
# Keep the direct-call behavior for existing users. The pipelined
# coordinator always supplies a sink so workers never share the queue.
self.prefetch_sync_queue.put(ack)
else:
ack_sink(ack)
return completed_pages
def _transfer_prefetch_and_collect_acks(
self, operation: PrefetchOperation
) -> List[PrefetchAck]:
"""Run one worker without publishing ACKs to the shared sync queue."""
acks = []
try:
self._page_transfer(operation, ack_sink=acks.append)
except Exception:
logger.exception(
"HiCache prefetch transfer failed for request %s",
operation.request_id,
)
return acks
def _page_transfer_kv_batch(
self,
operation: PrefetchOperation,
@@ -1247,24 +1276,86 @@ class HiCacheController:
def prefetch_io_aux_func(self):
"""
Auxiliary function conducting IO operations for prefetching.
# [HC_PARALLEL_PATCH] pipelined: up to SGLANG_HICACHE_PREFETCH_IO_THREADS
# transfers in flight. Every ACK of one operation is published before any
# ACK of the next operation, keeping per-rank collective pairing unchanged.
"""
while not self.storage_stop_event.is_set():
try:
operation = self.prefetch_buffer.get(block=True, timeout=1)
except Empty:
continue
if operation is None:
continue
try:
self._page_transfer(operation)
finally:
self.prefetch_sync_queue.put(
PrefetchAck(
rid=operation.request_id,
completed_req=True,
operation=operation,
import time as _time
from collections import deque as _deque
from concurrent.futures import ThreadPoolExecutor as _TPE
K = max(1, _HC_PREFETCH_WORKERS)
timing = _HC_TIMING
ex = _TPE(max_workers=K, thread_name_prefix="hc-prefetch-io")
inflight = _deque()
lat = []
pages = 0
try:
while not self.storage_stop_event.is_set():
while len(inflight) < K:
try:
operation = self.prefetch_buffer.get_nowait()
except Empty:
break
if operation is None:
continue
inflight.append(
(
operation,
_time.perf_counter(),
ex.submit(
self._transfer_prefetch_and_collect_acks, operation
),
)
)
)
if not inflight:
try:
operation = self.prefetch_buffer.get(block=True, timeout=1)
except Empty:
continue
if operation is None:
continue
inflight.append(
(
operation,
_time.perf_counter(),
ex.submit(
self._transfer_prefetch_and_collect_acks, operation
),
)
)
operation, t0, fut = inflight[0]
try:
acks = fut.result()
except Exception:
logger.exception("HiCache prefetch transfer failed")
acks = []
finally:
inflight.popleft()
if timing:
lat.append((_time.perf_counter() - t0) * 1000.0)
pages += len(getattr(operation, "hash_value", []) or [])
for ack in acks:
self.prefetch_sync_queue.put(ack)
self.prefetch_sync_queue.put(
PrefetchAck(
rid=operation.request_id,
completed_req=True,
operation=operation,
)
)
if timing and len(lat) >= 100:
lat.sort()
print("[HC_TIMING] prefetch queue=%d inflight=%d n=%d transfer_ms p50=%.1f p95=%.1f pages=%d"
% (self.prefetch_buffer.qsize(), len(inflight), len(lat),
lat[len(lat) // 2], lat[int(len(lat) * 0.95)], pages), flush=True)
lat = []
pages = 0
finally:
# Detach closes the backend after joining this outer thread. Do not
# let pool workers outlive that join and touch released state.
ex.shutdown(wait=True)
def prefetch_rate_limited(self) -> bool:
"""
@@ -1443,22 +1534,82 @@ class HiCacheController:
"HiCache backup failed after %d tokens", operation.completed_tokens
)
def _should_skip_backup(self) -> bool:
"""Whether this rank can ACK without doing any storage IO."""
return self.backup_skip
def backup_thread_func(self):
"""
Manage backup operations from host memory to storage backend.
# [HC_PARALLEL_PATCH] pipelined: up to SGLANG_HICACHE_BACKUP_IO_THREADS
# writes in flight, acked in FIFO order.
"""
while not self.storage_stop_event.is_set():
try:
operation = self.backup_queue.get(block=True, timeout=1)
if operation is None:
continue
import time as _time
from collections import deque as _deque
from concurrent.futures import ThreadPoolExecutor as _TPE
if not self.backup_skip:
self._page_backup_or_log(operation)
K = max(1, _HC_BACKUP_WORKERS)
timing = _HC_TIMING
ex = _TPE(max_workers=K, thread_name_prefix="hc-backup-io")
inflight = _deque()
lat = []
pages = 0
def submit(operation) -> bool:
if self._should_skip_backup():
self.ack_backup_queue.put(operation)
return False
inflight.append(
(
operation,
_time.perf_counter(),
ex.submit(self._page_backup_or_log, operation),
)
)
return True
except Empty:
continue
try:
while not self.storage_stop_event.is_set():
while len(inflight) < K:
try:
operation = self.backup_queue.get_nowait()
except Empty:
break
if operation is None:
continue
submit(operation)
if not inflight:
try:
operation = self.backup_queue.get(block=True, timeout=1)
except Empty:
continue
if operation is None:
continue
if not submit(operation):
continue
operation, t0, fut = inflight[0]
try:
fut.result()
except Exception:
logger.exception("HiCache backup failed")
finally:
inflight.popleft()
if timing:
lat.append((_time.perf_counter() - t0) * 1000.0)
pages += len(getattr(operation, "hash_value", []) or [])
self.ack_backup_queue.put(operation)
if timing and len(lat) >= 100:
lat.sort()
print("[HC_TIMING] backup queue=%d inflight=%d n=%d write_ms p50=%.1f p95=%.1f pages=%d"
% (self.backup_queue.qsize(), len(inflight), len(lat),
lat[len(lat) // 2], lat[int(len(lat) * 0.95)], pages), flush=True)
lat = []
pages = 0
finally:
# The outer thread is joined before backend/process-group teardown.
# Wait for every submitted write before reporting that thread stopped.
ex.shutdown(wait=True)
def prefetch_sync_thread_func(self):
"""Synchronize prefetch results across all PP and TP ranks."""
@@ -1547,19 +1547,31 @@ class HybridCacheController(BaseHiCacheController):
)
return host_indices, device_indices, resolved_pool_transfers
def _page_transfer(self, operation: PrefetchOperation) -> bool:
def _page_transfer(
self,
operation: PrefetchOperation,
ack_sink: Optional[Callable[[PrefetchAck], None]] = None,
) -> int:
with self.mem_pool_host.layout_lease():
return self._page_transfer_with_stable_layout(operation)
return self._page_transfer_with_stable_layout(operation, ack_sink)
def _page_transfer_with_stable_layout(self, operation: PrefetchOperation) -> None:
def _page_transfer_with_stable_layout(
self,
operation: PrefetchOperation,
ack_sink: Optional[Callable[[PrefetchAck], None]] = None,
) -> int:
# KV pools and KV-derived pools first — determines actual completed page count
kv_completed_pages = super()._page_transfer(operation)
kv_completed_pages = super()._page_transfer(operation, ack_sink)
# Read non-KV derived sidecar pool, e.g. SWA, Mamba.
self._page_transfer_sidecar(operation, kv_completed_pages)
self._page_transfer_sidecar(operation, kv_completed_pages, ack_sink)
return kv_completed_pages
def _page_transfer_sidecar(
self, operation: PrefetchOperation, kv_completed_pages: int
self,
operation: PrefetchOperation,
kv_completed_pages: int,
ack_sink: Optional[Callable[[PrefetchAck], None]] = None,
) -> None:
if operation.pool_transfers is None:
return
@@ -1604,17 +1616,19 @@ class HybridCacheController(BaseHiCacheController):
"HiCache sidecar prefetch failed for request %s",
operation.request_id,
)
# Emit PrefetchAck to prefetch_sync_queue, even the operation has been canceled by the
# scheduler thread. The prefetch sync thread expects the same number of PrefetchAck objects
# to perform all_reduce.
self.prefetch_sync_queue.put(
PrefetchAck(
rid=operation.request_id,
operation=operation,
pool_hits=pool_hits,
)
# Emit PrefetchAck even when the operation has been canceled by the
# scheduler thread. The sync thread expects the same number of ACKs on
# every rank. Pipelined workers buffer this ACK per operation instead of
# publishing it directly to the shared queue.
ack = PrefetchAck(
rid=operation.request_id,
operation=operation,
pool_hits=pool_hits,
)
return
if ack_sink is None:
self.prefetch_sync_queue.put(ack)
else:
ack_sink(ack)
def trim_prefetch_full_head(self, operation, trim_tokens: int) -> None:
"""Drop a device-resident FULL head after the availability query;
@@ -1704,22 +1718,15 @@ class HybridCacheController(BaseHiCacheController):
return False
def backup_thread_func(self):
"""Back up rank-sharded sidecars on every TP rank.
def _should_skip_backup(self) -> bool:
# Replicated primary MLA KV may be TP0-only, but rank-sharded sidecars
# still require IO on every rank. _page_backup applies should_backup()
# to each individual transfer, so the whole operation cannot be skipped.
return False
The base implementation skips the entire operation on non-zero MLA TP
ranks. That optimization is valid for replicated MLA KV, but not for
hybrid rank-sharded pools such as Kimi-K3 Mamba state.
"""
while not self.storage_stop_event.is_set():
try:
operation = self.backup_queue.get(block=True, timeout=1)
if operation is None:
continue
self._page_backup_or_log(operation)
self.ack_backup_queue.put(operation)
except Empty:
continue
def backup_thread_func(self):
"""Use the base pipeline while retaining per-pool writer selection."""
super().backup_thread_func()
def _resolve_sidecar_kv_derived_pool_transfers(self, operation):
for transfer in operation.pool_transfers:
@@ -0,0 +1,229 @@
"""CPU regressions for pipelined HiCache storage IO ordering and shutdown."""
import threading
import time
import unittest
from contextlib import nullcontext
from queue import Queue
from types import SimpleNamespace
from unittest.mock import Mock, patch
import torch
import sglang.srt.managers.cache_controller as cache_controller_module
from sglang.srt.managers.cache_controller import (
HiCacheController,
StorageOperation,
)
from sglang.srt.mem_cache.base_prefix_cache import CacheRequestHandle
from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import (
HybridCacheController,
PrefetchOperation,
)
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
def _drain(queue):
items = []
while not queue.empty():
items.append(queue.get_nowait())
return items
def _wait_for_queue_size(queue, size, timeout=5):
deadline = time.monotonic() + timeout
while queue.qsize() < size and time.monotonic() < deadline:
time.sleep(0.005)
return queue.qsize() >= size
class TestPrefetchPipelineOrdering(unittest.TestCase):
def _controller(self, wait_for, peer_done):
controller = HybridCacheController.__new__(HybridCacheController)
controller.page_size = 1
controller.prefetch_buffer = Queue()
controller.prefetch_sync_queue = Queue()
controller.storage_stop_event = threading.Event()
controller.mem_pool_host = SimpleNamespace(
layout_lease=lambda: nullcontext()
)
controller.storage_backend = Mock()
controller.storage_backend.batch_get_v2.return_value = {}
for rid in ("a", "b"):
operation = PrefetchOperation(
CacheRequestHandle(rid, 0),
list(range(2)),
pool_transfers=[],
)
operation.hash_value = [f"{rid}-0", f"{rid}-1"]
operation.host_indices = torch.arange(2)
controller.prefetch_buffer.put(operation)
def page_get(operation, batch_hashes, host_indices, extra_info):
if operation.request_id == wait_for:
if not peer_done.wait(timeout=5):
raise TimeoutError("peer prefetch did not finish")
return len(batch_hashes)
controller.page_get_func = page_get
collect = controller._transfer_prefetch_and_collect_acks
def collect_and_signal(operation):
acks = collect(operation)
if operation.request_id != wait_for:
# This rank's other operation has now completed. The delayed
# peer may resume, producing the opposite worker finish order.
peer_done.set()
return acks
controller._transfer_prefetch_and_collect_acks = collect_and_signal
return controller
def test_two_tp_ranks_keep_operation_ack_sequences_aligned(self):
# Rank 0 finishes A before B; rank 1 finishes B before A. Without
# per-operation buffering their progress/sidecar ACKs enter collectives
# in opposite orders even though completed_req itself remains FIFO.
rank0_done = threading.Event()
rank1_done = threading.Event()
rank0 = self._controller(wait_for="b", peer_done=rank0_done)
rank1 = self._controller(wait_for="a", peer_done=rank1_done)
threads = [
threading.Thread(target=rank.prefetch_io_aux_func, daemon=True)
for rank in (rank0, rank1)
]
with (
patch.object(cache_controller_module, "_HC_PREFETCH_WORKERS", 2),
patch.object(cache_controller_module, "STORAGE_BATCH_SIZE", 1),
):
for thread in threads:
thread.start()
try:
self.assertTrue(_wait_for_queue_size(rank0.prefetch_sync_queue, 8))
self.assertTrue(_wait_for_queue_size(rank1.prefetch_sync_queue, 8))
finally:
rank0.storage_stop_event.set()
rank1.storage_stop_event.set()
rank0_done.set()
rank1_done.set()
for thread in threads:
thread.join(timeout=5)
for thread in threads:
self.assertFalse(thread.is_alive())
rank0_acks = _drain(rank0.prefetch_sync_queue)
rank1_acks = _drain(rank1.prefetch_sync_queue)
def ack_kind(ack):
if ack.completed_req:
return "completed_req"
if ack.completed_tokens is not None:
return "completed_tokens"
if ack.pool_hits is not None:
return "pool_hits"
raise AssertionError(f"unknown ACK: {ack}")
self.assertEqual([ack.rid for ack in rank0_acks], ["a"] * 4 + ["b"] * 4)
self.assertEqual(
[(ack.rid, ack_kind(ack)) for ack in rank0_acks],
[(ack.rid, ack_kind(ack)) for ack in rank1_acks],
)
for acks in (rank0_acks, rank1_acks):
self.assertEqual(
[ack_kind(ack) for ack in acks[:4]],
["completed_tokens", "completed_tokens", "pool_hits", "completed_req"],
)
self.assertEqual(
[ack.completed_tokens for ack in acks[:2]],
[1, 2],
)
class TestBackupPipeline(unittest.TestCase):
@staticmethod
def _controller(controller_type, is_storage_writer, backup):
controller = controller_type.__new__(controller_type)
controller.backup_queue = Queue()
controller.ack_backup_queue = Queue()
controller.storage_stop_event = threading.Event()
controller.storage_config = SimpleNamespace(
is_storage_writer=is_storage_writer
)
controller._page_backup_or_log = backup
return controller
def _run_one(self, controller, operation):
thread = threading.Thread(target=controller.backup_thread_func, daemon=True)
with patch.object(cache_controller_module, "_HC_BACKUP_WORKERS", 1):
controller.backup_queue.put(operation)
thread.start()
try:
ack = controller.ack_backup_queue.get(timeout=5)
finally:
controller.storage_stop_event.set()
thread.join(timeout=5)
self.assertFalse(thread.is_alive())
self.assertIs(ack, operation)
def test_non_writer_never_submits_backup_io(self):
backup = Mock()
controller = self._controller(HiCacheController, False, backup)
operation = StorageOperation(None, [])
self._run_one(controller, operation)
backup.assert_not_called()
def test_hybrid_non_writer_still_runs_rank_sharded_backup(self):
backup = Mock()
controller = self._controller(HybridCacheController, False, backup)
operation = StorageOperation(None, [])
self._run_one(controller, operation)
backup.assert_called_once_with(operation)
def test_stop_waits_for_every_backup_worker(self):
started = [threading.Event(), threading.Event()]
release = [threading.Event(), threading.Event()]
finished = [threading.Event(), threading.Event()]
def backup(operation):
index = operation.token_ids[0]
started[index].set()
if not release[index].wait(timeout=5):
raise TimeoutError(f"backup {index} was not released")
finished[index].set()
controller = self._controller(HiCacheController, True, backup)
for index in range(2):
controller.backup_queue.put(StorageOperation(None, [index]))
thread = threading.Thread(target=controller.backup_thread_func, daemon=True)
with patch.object(cache_controller_module, "_HC_BACKUP_WORKERS", 2):
thread.start()
try:
self.assertTrue(started[0].wait(timeout=5))
self.assertTrue(started[1].wait(timeout=5))
controller.storage_stop_event.set()
release[0].set()
# The coordinator may finish operation 0, but shutdown cannot
# return while operation 1 still uses the storage backend.
thread.join(timeout=0.1)
self.assertTrue(thread.is_alive())
self.assertFalse(finished[1].is_set())
release[1].set()
thread.join(timeout=5)
self.assertFalse(thread.is_alive())
self.assertTrue(finished[0].is_set())
self.assertTrue(finished[1].is_set())
finally:
controller.storage_stop_event.set()
for event in release:
event.set()
thread.join(timeout=5)
if __name__ == "__main__":
unittest.main()