hicache: pipeline prefetch/backup IO lanes + stage timing

- prefetch_io_aux_func: allow up to SGLANG_HICACHE_PREFETCH_IO_THREADS
  transfers in flight (default 1 = legacy behaviour); PrefetchAck is still
  emitted strictly FIFO so the per-rank ack stream (and the TP all_reduce in
  prefetch_sync_thread_func) is unchanged.
- backup_thread_func: same pipelining for L2->L3 writes via
  SGLANG_HICACHE_BACKUP_IO_THREADS.
- SGLANG_HICACHE_TIMING=1: emit [HC_TIMING] prefetch/backup queue, inflight,
  p50/p95 per 100 ops (the prefetch/backup lanes were previously single
  threaded; this is what caps L3<->L2 at ~1 op at a time).
This commit is contained in:
ymk
2026-11-06 13:18:07 +08:00
committed by minke.yu
parent c911d1b572
commit bf8cf7f1dc
3 changed files with 445 additions and 58 deletions
+178 -27
View File
@@ -62,6 +62,12 @@ def storage_model_name(
return f"{model_name}-{tag}" if model_name else tag
# [HC_PARALLEL_PATCH] knobs for the pipelined HiCache prefetch/backup lanes
import os as _hc_os
_HC_PREFETCH_WORKERS = int(_hc_os.getenv("SGLANG_HICACHE_PREFETCH_IO_THREADS", "1") or 1)
_HC_BACKUP_WORKERS = int(_hc_os.getenv("SGLANG_HICACHE_BACKUP_IO_THREADS", "1") or 1)
_HC_TIMING = _hc_os.getenv("SGLANG_HICACHE_TIMING", "0") not in ("", "0", "false", "False")
class LayerLoadingEvent:
def __init__(self, num_layers: int):
self._num_layers = num_layers
@@ -1152,7 +1158,11 @@ class HiCacheController:
count += 1
return count
def _page_transfer(self, operation: PrefetchOperation) -> int:
def _page_transfer(
self,
operation: PrefetchOperation,
ack_sink: Optional[Callable[[PrefetchAck], None]] = None,
) -> int:
# Transfer batch by batch
prefix_keys = operation.prefix_keys
kv_derived_transfers = [
@@ -1202,9 +1212,28 @@ class HiCacheController:
completed_tokens=completed_pages * self.page_size,
operation=operation,
)
self.prefetch_sync_queue.put(ack)
if ack_sink is None:
# Keep the direct-call behavior for existing users. The pipelined
# coordinator always supplies a sink so workers never share the queue.
self.prefetch_sync_queue.put(ack)
else:
ack_sink(ack)
return completed_pages
def _transfer_prefetch_and_collect_acks(
self, operation: PrefetchOperation
) -> List[PrefetchAck]:
"""Run one worker without publishing ACKs to the shared sync queue."""
acks = []
try:
self._page_transfer(operation, ack_sink=acks.append)
except Exception:
logger.exception(
"HiCache prefetch transfer failed for request %s",
operation.request_id,
)
return acks
def _page_transfer_kv_batch(
self,
operation: PrefetchOperation,
@@ -1247,24 +1276,86 @@ class HiCacheController:
def prefetch_io_aux_func(self):
"""
Auxiliary function conducting IO operations for prefetching.
# [HC_PARALLEL_PATCH] pipelined: up to SGLANG_HICACHE_PREFETCH_IO_THREADS
# transfers in flight. Every ACK of one operation is published before any
# ACK of the next operation, keeping per-rank collective pairing unchanged.
"""
while not self.storage_stop_event.is_set():
try:
operation = self.prefetch_buffer.get(block=True, timeout=1)
except Empty:
continue
if operation is None:
continue
try:
self._page_transfer(operation)
finally:
self.prefetch_sync_queue.put(
PrefetchAck(
rid=operation.request_id,
completed_req=True,
operation=operation,
import time as _time
from collections import deque as _deque
from concurrent.futures import ThreadPoolExecutor as _TPE
K = max(1, _HC_PREFETCH_WORKERS)
timing = _HC_TIMING
ex = _TPE(max_workers=K, thread_name_prefix="hc-prefetch-io")
inflight = _deque()
lat = []
pages = 0
try:
while not self.storage_stop_event.is_set():
while len(inflight) < K:
try:
operation = self.prefetch_buffer.get_nowait()
except Empty:
break
if operation is None:
continue
inflight.append(
(
operation,
_time.perf_counter(),
ex.submit(
self._transfer_prefetch_and_collect_acks, operation
),
)
)
)
if not inflight:
try:
operation = self.prefetch_buffer.get(block=True, timeout=1)
except Empty:
continue
if operation is None:
continue
inflight.append(
(
operation,
_time.perf_counter(),
ex.submit(
self._transfer_prefetch_and_collect_acks, operation
),
)
)
operation, t0, fut = inflight[0]
try:
acks = fut.result()
except Exception:
logger.exception("HiCache prefetch transfer failed")
acks = []
finally:
inflight.popleft()
if timing:
lat.append((_time.perf_counter() - t0) * 1000.0)
pages += len(getattr(operation, "hash_value", []) or [])
for ack in acks:
self.prefetch_sync_queue.put(ack)
self.prefetch_sync_queue.put(
PrefetchAck(
rid=operation.request_id,
completed_req=True,
operation=operation,
)
)
if timing and len(lat) >= 100:
lat.sort()
print("[HC_TIMING] prefetch queue=%d inflight=%d n=%d transfer_ms p50=%.1f p95=%.1f pages=%d"
% (self.prefetch_buffer.qsize(), len(inflight), len(lat),
lat[len(lat) // 2], lat[int(len(lat) * 0.95)], pages), flush=True)
lat = []
pages = 0
finally:
# Detach closes the backend after joining this outer thread. Do not
# let pool workers outlive that join and touch released state.
ex.shutdown(wait=True)
def prefetch_rate_limited(self) -> bool:
"""
@@ -1443,22 +1534,82 @@ class HiCacheController:
"HiCache backup failed after %d tokens", operation.completed_tokens
)
def _should_skip_backup(self) -> bool:
"""Whether this rank can ACK without doing any storage IO."""
return self.backup_skip
def backup_thread_func(self):
"""
Manage backup operations from host memory to storage backend.
# [HC_PARALLEL_PATCH] pipelined: up to SGLANG_HICACHE_BACKUP_IO_THREADS
# writes in flight, acked in FIFO order.
"""
while not self.storage_stop_event.is_set():
try:
operation = self.backup_queue.get(block=True, timeout=1)
if operation is None:
continue
import time as _time
from collections import deque as _deque
from concurrent.futures import ThreadPoolExecutor as _TPE
if not self.backup_skip:
self._page_backup_or_log(operation)
K = max(1, _HC_BACKUP_WORKERS)
timing = _HC_TIMING
ex = _TPE(max_workers=K, thread_name_prefix="hc-backup-io")
inflight = _deque()
lat = []
pages = 0
def submit(operation) -> bool:
if self._should_skip_backup():
self.ack_backup_queue.put(operation)
return False
inflight.append(
(
operation,
_time.perf_counter(),
ex.submit(self._page_backup_or_log, operation),
)
)
return True
except Empty:
continue
try:
while not self.storage_stop_event.is_set():
while len(inflight) < K:
try:
operation = self.backup_queue.get_nowait()
except Empty:
break
if operation is None:
continue
submit(operation)
if not inflight:
try:
operation = self.backup_queue.get(block=True, timeout=1)
except Empty:
continue
if operation is None:
continue
if not submit(operation):
continue
operation, t0, fut = inflight[0]
try:
fut.result()
except Exception:
logger.exception("HiCache backup failed")
finally:
inflight.popleft()
if timing:
lat.append((_time.perf_counter() - t0) * 1000.0)
pages += len(getattr(operation, "hash_value", []) or [])
self.ack_backup_queue.put(operation)
if timing and len(lat) >= 100:
lat.sort()
print("[HC_TIMING] backup queue=%d inflight=%d n=%d write_ms p50=%.1f p95=%.1f pages=%d"
% (self.backup_queue.qsize(), len(inflight), len(lat),
lat[len(lat) // 2], lat[int(len(lat) * 0.95)], pages), flush=True)
lat = []
pages = 0
finally:
# The outer thread is joined before backend/process-group teardown.
# Wait for every submitted write before reporting that thread stopped.
ex.shutdown(wait=True)
def prefetch_sync_thread_func(self):
"""Synchronize prefetch results across all PP and TP ranks."""
@@ -1547,19 +1547,31 @@ class HybridCacheController(BaseHiCacheController):
)
return host_indices, device_indices, resolved_pool_transfers
def _page_transfer(self, operation: PrefetchOperation) -> bool:
def _page_transfer(
self,
operation: PrefetchOperation,
ack_sink: Optional[Callable[[PrefetchAck], None]] = None,
) -> int:
with self.mem_pool_host.layout_lease():
return self._page_transfer_with_stable_layout(operation)
return self._page_transfer_with_stable_layout(operation, ack_sink)
def _page_transfer_with_stable_layout(self, operation: PrefetchOperation) -> None:
def _page_transfer_with_stable_layout(
self,
operation: PrefetchOperation,
ack_sink: Optional[Callable[[PrefetchAck], None]] = None,
) -> int:
# KV pools and KV-derived pools first — determines actual completed page count
kv_completed_pages = super()._page_transfer(operation)
kv_completed_pages = super()._page_transfer(operation, ack_sink)
# Read non-KV derived sidecar pool, e.g. SWA, Mamba.
self._page_transfer_sidecar(operation, kv_completed_pages)
self._page_transfer_sidecar(operation, kv_completed_pages, ack_sink)
return kv_completed_pages
def _page_transfer_sidecar(
self, operation: PrefetchOperation, kv_completed_pages: int
self,
operation: PrefetchOperation,
kv_completed_pages: int,
ack_sink: Optional[Callable[[PrefetchAck], None]] = None,
) -> None:
if operation.pool_transfers is None:
return
@@ -1604,17 +1616,19 @@ class HybridCacheController(BaseHiCacheController):
"HiCache sidecar prefetch failed for request %s",
operation.request_id,
)
# Emit PrefetchAck to prefetch_sync_queue, even the operation has been canceled by the
# scheduler thread. The prefetch sync thread expects the same number of PrefetchAck objects
# to perform all_reduce.
self.prefetch_sync_queue.put(
PrefetchAck(
rid=operation.request_id,
operation=operation,
pool_hits=pool_hits,
)
# Emit PrefetchAck even when the operation has been canceled by the
# scheduler thread. The sync thread expects the same number of ACKs on
# every rank. Pipelined workers buffer this ACK per operation instead of
# publishing it directly to the shared queue.
ack = PrefetchAck(
rid=operation.request_id,
operation=operation,
pool_hits=pool_hits,
)
return
if ack_sink is None:
self.prefetch_sync_queue.put(ack)
else:
ack_sink(ack)
def trim_prefetch_full_head(self, operation, trim_tokens: int) -> None:
"""Drop a device-resident FULL head after the availability query;
@@ -1704,22 +1718,15 @@ class HybridCacheController(BaseHiCacheController):
return False
def backup_thread_func(self):
"""Back up rank-sharded sidecars on every TP rank.
def _should_skip_backup(self) -> bool:
# Replicated primary MLA KV may be TP0-only, but rank-sharded sidecars
# still require IO on every rank. _page_backup applies should_backup()
# to each individual transfer, so the whole operation cannot be skipped.
return False
The base implementation skips the entire operation on non-zero MLA TP
ranks. That optimization is valid for replicated MLA KV, but not for
hybrid rank-sharded pools such as Kimi-K3 Mamba state.
"""
while not self.storage_stop_event.is_set():
try:
operation = self.backup_queue.get(block=True, timeout=1)
if operation is None:
continue
self._page_backup_or_log(operation)
self.ack_backup_queue.put(operation)
except Empty:
continue
def backup_thread_func(self):
"""Use the base pipeline while retaining per-pool writer selection."""
super().backup_thread_func()
def _resolve_sidecar_kv_derived_pool_transfers(self, operation):
for transfer in operation.pool_transfers: