hicache: pipeline prefetch/backup IO lanes + stage timing
- prefetch_io_aux_func: allow up to SGLANG_HICACHE_PREFETCH_IO_THREADS transfers in flight (default 1 = legacy behaviour); PrefetchAck is still emitted strictly FIFO so the per-rank ack stream (and the TP all_reduce in prefetch_sync_thread_func) is unchanged. - backup_thread_func: same pipelining for L2->L3 writes via SGLANG_HICACHE_BACKUP_IO_THREADS. - SGLANG_HICACHE_TIMING=1: emit [HC_TIMING] prefetch/backup queue, inflight, p50/p95 per 100 ops (the prefetch/backup lanes were previously single threaded; this is what caps L3<->L2 at ~1 op at a time).
This commit is contained in:
@@ -62,6 +62,12 @@ def storage_model_name(
|
||||
return f"{model_name}-{tag}" if model_name else tag
|
||||
|
||||
|
||||
# [HC_PARALLEL_PATCH] knobs for the pipelined HiCache prefetch/backup lanes
|
||||
import os as _hc_os
|
||||
_HC_PREFETCH_WORKERS = int(_hc_os.getenv("SGLANG_HICACHE_PREFETCH_IO_THREADS", "1") or 1)
|
||||
_HC_BACKUP_WORKERS = int(_hc_os.getenv("SGLANG_HICACHE_BACKUP_IO_THREADS", "1") or 1)
|
||||
_HC_TIMING = _hc_os.getenv("SGLANG_HICACHE_TIMING", "0") not in ("", "0", "false", "False")
|
||||
|
||||
class LayerLoadingEvent:
|
||||
def __init__(self, num_layers: int):
|
||||
self._num_layers = num_layers
|
||||
@@ -1152,7 +1158,11 @@ class HiCacheController:
|
||||
count += 1
|
||||
return count
|
||||
|
||||
def _page_transfer(self, operation: PrefetchOperation) -> int:
|
||||
def _page_transfer(
|
||||
self,
|
||||
operation: PrefetchOperation,
|
||||
ack_sink: Optional[Callable[[PrefetchAck], None]] = None,
|
||||
) -> int:
|
||||
# Transfer batch by batch
|
||||
prefix_keys = operation.prefix_keys
|
||||
kv_derived_transfers = [
|
||||
@@ -1202,9 +1212,28 @@ class HiCacheController:
|
||||
completed_tokens=completed_pages * self.page_size,
|
||||
operation=operation,
|
||||
)
|
||||
self.prefetch_sync_queue.put(ack)
|
||||
if ack_sink is None:
|
||||
# Keep the direct-call behavior for existing users. The pipelined
|
||||
# coordinator always supplies a sink so workers never share the queue.
|
||||
self.prefetch_sync_queue.put(ack)
|
||||
else:
|
||||
ack_sink(ack)
|
||||
return completed_pages
|
||||
|
||||
def _transfer_prefetch_and_collect_acks(
|
||||
self, operation: PrefetchOperation
|
||||
) -> List[PrefetchAck]:
|
||||
"""Run one worker without publishing ACKs to the shared sync queue."""
|
||||
acks = []
|
||||
try:
|
||||
self._page_transfer(operation, ack_sink=acks.append)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"HiCache prefetch transfer failed for request %s",
|
||||
operation.request_id,
|
||||
)
|
||||
return acks
|
||||
|
||||
def _page_transfer_kv_batch(
|
||||
self,
|
||||
operation: PrefetchOperation,
|
||||
@@ -1247,24 +1276,86 @@ class HiCacheController:
|
||||
def prefetch_io_aux_func(self):
|
||||
"""
|
||||
Auxiliary function conducting IO operations for prefetching.
|
||||
|
||||
# [HC_PARALLEL_PATCH] pipelined: up to SGLANG_HICACHE_PREFETCH_IO_THREADS
|
||||
# transfers in flight. Every ACK of one operation is published before any
|
||||
# ACK of the next operation, keeping per-rank collective pairing unchanged.
|
||||
"""
|
||||
while not self.storage_stop_event.is_set():
|
||||
try:
|
||||
operation = self.prefetch_buffer.get(block=True, timeout=1)
|
||||
except Empty:
|
||||
continue
|
||||
if operation is None:
|
||||
continue
|
||||
try:
|
||||
self._page_transfer(operation)
|
||||
finally:
|
||||
self.prefetch_sync_queue.put(
|
||||
PrefetchAck(
|
||||
rid=operation.request_id,
|
||||
completed_req=True,
|
||||
operation=operation,
|
||||
import time as _time
|
||||
from collections import deque as _deque
|
||||
from concurrent.futures import ThreadPoolExecutor as _TPE
|
||||
|
||||
K = max(1, _HC_PREFETCH_WORKERS)
|
||||
timing = _HC_TIMING
|
||||
ex = _TPE(max_workers=K, thread_name_prefix="hc-prefetch-io")
|
||||
inflight = _deque()
|
||||
lat = []
|
||||
pages = 0
|
||||
try:
|
||||
while not self.storage_stop_event.is_set():
|
||||
while len(inflight) < K:
|
||||
try:
|
||||
operation = self.prefetch_buffer.get_nowait()
|
||||
except Empty:
|
||||
break
|
||||
if operation is None:
|
||||
continue
|
||||
inflight.append(
|
||||
(
|
||||
operation,
|
||||
_time.perf_counter(),
|
||||
ex.submit(
|
||||
self._transfer_prefetch_and_collect_acks, operation
|
||||
),
|
||||
)
|
||||
)
|
||||
)
|
||||
if not inflight:
|
||||
try:
|
||||
operation = self.prefetch_buffer.get(block=True, timeout=1)
|
||||
except Empty:
|
||||
continue
|
||||
if operation is None:
|
||||
continue
|
||||
inflight.append(
|
||||
(
|
||||
operation,
|
||||
_time.perf_counter(),
|
||||
ex.submit(
|
||||
self._transfer_prefetch_and_collect_acks, operation
|
||||
),
|
||||
)
|
||||
)
|
||||
operation, t0, fut = inflight[0]
|
||||
try:
|
||||
acks = fut.result()
|
||||
except Exception:
|
||||
logger.exception("HiCache prefetch transfer failed")
|
||||
acks = []
|
||||
finally:
|
||||
inflight.popleft()
|
||||
if timing:
|
||||
lat.append((_time.perf_counter() - t0) * 1000.0)
|
||||
pages += len(getattr(operation, "hash_value", []) or [])
|
||||
for ack in acks:
|
||||
self.prefetch_sync_queue.put(ack)
|
||||
self.prefetch_sync_queue.put(
|
||||
PrefetchAck(
|
||||
rid=operation.request_id,
|
||||
completed_req=True,
|
||||
operation=operation,
|
||||
)
|
||||
)
|
||||
if timing and len(lat) >= 100:
|
||||
lat.sort()
|
||||
print("[HC_TIMING] prefetch queue=%d inflight=%d n=%d transfer_ms p50=%.1f p95=%.1f pages=%d"
|
||||
% (self.prefetch_buffer.qsize(), len(inflight), len(lat),
|
||||
lat[len(lat) // 2], lat[int(len(lat) * 0.95)], pages), flush=True)
|
||||
lat = []
|
||||
pages = 0
|
||||
finally:
|
||||
# Detach closes the backend after joining this outer thread. Do not
|
||||
# let pool workers outlive that join and touch released state.
|
||||
ex.shutdown(wait=True)
|
||||
|
||||
def prefetch_rate_limited(self) -> bool:
|
||||
"""
|
||||
@@ -1443,22 +1534,82 @@ class HiCacheController:
|
||||
"HiCache backup failed after %d tokens", operation.completed_tokens
|
||||
)
|
||||
|
||||
def _should_skip_backup(self) -> bool:
|
||||
"""Whether this rank can ACK without doing any storage IO."""
|
||||
return self.backup_skip
|
||||
|
||||
def backup_thread_func(self):
|
||||
"""
|
||||
Manage backup operations from host memory to storage backend.
|
||||
|
||||
# [HC_PARALLEL_PATCH] pipelined: up to SGLANG_HICACHE_BACKUP_IO_THREADS
|
||||
# writes in flight, acked in FIFO order.
|
||||
"""
|
||||
while not self.storage_stop_event.is_set():
|
||||
try:
|
||||
operation = self.backup_queue.get(block=True, timeout=1)
|
||||
if operation is None:
|
||||
continue
|
||||
import time as _time
|
||||
from collections import deque as _deque
|
||||
from concurrent.futures import ThreadPoolExecutor as _TPE
|
||||
|
||||
if not self.backup_skip:
|
||||
self._page_backup_or_log(operation)
|
||||
K = max(1, _HC_BACKUP_WORKERS)
|
||||
timing = _HC_TIMING
|
||||
ex = _TPE(max_workers=K, thread_name_prefix="hc-backup-io")
|
||||
inflight = _deque()
|
||||
lat = []
|
||||
pages = 0
|
||||
|
||||
def submit(operation) -> bool:
|
||||
if self._should_skip_backup():
|
||||
self.ack_backup_queue.put(operation)
|
||||
return False
|
||||
inflight.append(
|
||||
(
|
||||
operation,
|
||||
_time.perf_counter(),
|
||||
ex.submit(self._page_backup_or_log, operation),
|
||||
)
|
||||
)
|
||||
return True
|
||||
|
||||
except Empty:
|
||||
continue
|
||||
try:
|
||||
while not self.storage_stop_event.is_set():
|
||||
while len(inflight) < K:
|
||||
try:
|
||||
operation = self.backup_queue.get_nowait()
|
||||
except Empty:
|
||||
break
|
||||
if operation is None:
|
||||
continue
|
||||
submit(operation)
|
||||
if not inflight:
|
||||
try:
|
||||
operation = self.backup_queue.get(block=True, timeout=1)
|
||||
except Empty:
|
||||
continue
|
||||
if operation is None:
|
||||
continue
|
||||
if not submit(operation):
|
||||
continue
|
||||
operation, t0, fut = inflight[0]
|
||||
try:
|
||||
fut.result()
|
||||
except Exception:
|
||||
logger.exception("HiCache backup failed")
|
||||
finally:
|
||||
inflight.popleft()
|
||||
if timing:
|
||||
lat.append((_time.perf_counter() - t0) * 1000.0)
|
||||
pages += len(getattr(operation, "hash_value", []) or [])
|
||||
self.ack_backup_queue.put(operation)
|
||||
if timing and len(lat) >= 100:
|
||||
lat.sort()
|
||||
print("[HC_TIMING] backup queue=%d inflight=%d n=%d write_ms p50=%.1f p95=%.1f pages=%d"
|
||||
% (self.backup_queue.qsize(), len(inflight), len(lat),
|
||||
lat[len(lat) // 2], lat[int(len(lat) * 0.95)], pages), flush=True)
|
||||
lat = []
|
||||
pages = 0
|
||||
finally:
|
||||
# The outer thread is joined before backend/process-group teardown.
|
||||
# Wait for every submitted write before reporting that thread stopped.
|
||||
ex.shutdown(wait=True)
|
||||
|
||||
def prefetch_sync_thread_func(self):
|
||||
"""Synchronize prefetch results across all PP and TP ranks."""
|
||||
|
||||
@@ -1547,19 +1547,31 @@ class HybridCacheController(BaseHiCacheController):
|
||||
)
|
||||
return host_indices, device_indices, resolved_pool_transfers
|
||||
|
||||
def _page_transfer(self, operation: PrefetchOperation) -> bool:
|
||||
def _page_transfer(
|
||||
self,
|
||||
operation: PrefetchOperation,
|
||||
ack_sink: Optional[Callable[[PrefetchAck], None]] = None,
|
||||
) -> int:
|
||||
with self.mem_pool_host.layout_lease():
|
||||
return self._page_transfer_with_stable_layout(operation)
|
||||
return self._page_transfer_with_stable_layout(operation, ack_sink)
|
||||
|
||||
def _page_transfer_with_stable_layout(self, operation: PrefetchOperation) -> None:
|
||||
def _page_transfer_with_stable_layout(
|
||||
self,
|
||||
operation: PrefetchOperation,
|
||||
ack_sink: Optional[Callable[[PrefetchAck], None]] = None,
|
||||
) -> int:
|
||||
# KV pools and KV-derived pools first — determines actual completed page count
|
||||
kv_completed_pages = super()._page_transfer(operation)
|
||||
kv_completed_pages = super()._page_transfer(operation, ack_sink)
|
||||
|
||||
# Read non-KV derived sidecar pool, e.g. SWA, Mamba.
|
||||
self._page_transfer_sidecar(operation, kv_completed_pages)
|
||||
self._page_transfer_sidecar(operation, kv_completed_pages, ack_sink)
|
||||
return kv_completed_pages
|
||||
|
||||
def _page_transfer_sidecar(
|
||||
self, operation: PrefetchOperation, kv_completed_pages: int
|
||||
self,
|
||||
operation: PrefetchOperation,
|
||||
kv_completed_pages: int,
|
||||
ack_sink: Optional[Callable[[PrefetchAck], None]] = None,
|
||||
) -> None:
|
||||
if operation.pool_transfers is None:
|
||||
return
|
||||
@@ -1604,17 +1616,19 @@ class HybridCacheController(BaseHiCacheController):
|
||||
"HiCache sidecar prefetch failed for request %s",
|
||||
operation.request_id,
|
||||
)
|
||||
# Emit PrefetchAck to prefetch_sync_queue, even the operation has been canceled by the
|
||||
# scheduler thread. The prefetch sync thread expects the same number of PrefetchAck objects
|
||||
# to perform all_reduce.
|
||||
self.prefetch_sync_queue.put(
|
||||
PrefetchAck(
|
||||
rid=operation.request_id,
|
||||
operation=operation,
|
||||
pool_hits=pool_hits,
|
||||
)
|
||||
# Emit PrefetchAck even when the operation has been canceled by the
|
||||
# scheduler thread. The sync thread expects the same number of ACKs on
|
||||
# every rank. Pipelined workers buffer this ACK per operation instead of
|
||||
# publishing it directly to the shared queue.
|
||||
ack = PrefetchAck(
|
||||
rid=operation.request_id,
|
||||
operation=operation,
|
||||
pool_hits=pool_hits,
|
||||
)
|
||||
return
|
||||
if ack_sink is None:
|
||||
self.prefetch_sync_queue.put(ack)
|
||||
else:
|
||||
ack_sink(ack)
|
||||
|
||||
def trim_prefetch_full_head(self, operation, trim_tokens: int) -> None:
|
||||
"""Drop a device-resident FULL head after the availability query;
|
||||
@@ -1704,22 +1718,15 @@ class HybridCacheController(BaseHiCacheController):
|
||||
|
||||
return False
|
||||
|
||||
def backup_thread_func(self):
|
||||
"""Back up rank-sharded sidecars on every TP rank.
|
||||
def _should_skip_backup(self) -> bool:
|
||||
# Replicated primary MLA KV may be TP0-only, but rank-sharded sidecars
|
||||
# still require IO on every rank. _page_backup applies should_backup()
|
||||
# to each individual transfer, so the whole operation cannot be skipped.
|
||||
return False
|
||||
|
||||
The base implementation skips the entire operation on non-zero MLA TP
|
||||
ranks. That optimization is valid for replicated MLA KV, but not for
|
||||
hybrid rank-sharded pools such as Kimi-K3 Mamba state.
|
||||
"""
|
||||
while not self.storage_stop_event.is_set():
|
||||
try:
|
||||
operation = self.backup_queue.get(block=True, timeout=1)
|
||||
if operation is None:
|
||||
continue
|
||||
self._page_backup_or_log(operation)
|
||||
self.ack_backup_queue.put(operation)
|
||||
except Empty:
|
||||
continue
|
||||
def backup_thread_func(self):
|
||||
"""Use the base pipeline while retaining per-pool writer selection."""
|
||||
super().backup_thread_func()
|
||||
|
||||
def _resolve_sidecar_kv_derived_pool_transfers(self, operation):
|
||||
for transfer in operation.pool_transfers:
|
||||
|
||||
Reference in New Issue
Block a user