hicache L3 prefetch: io_aux 传输并行化(按 op 分 lane)
build-sglang-image / build (push) Failing after 48m34s

打点实测单条 prefetch_io_aux_func 线程串行传输是所有请求的瓶颈:
mc8 下每请求 xfer 排队 0->1.9s 线性增长(每请求 5 批 x ~68ms 固定开销,
介质无关)。本提交把传输层拆成 N 条并行 lane(默认 4,
SGLANG_HICACHE_PREFETCH_IO_THREADS 可调):

- dispatch_prefetch() 用 crc32(request_id) % N 分 lane(内置 hash 对 str
  每进程加盐,跨 rank 不一致会导致 gloo 集合通信错位,禁用)
- 同一 op 的所有 chunk 保持在同一 lane,保序 reduce 语义不变
- 每 lane 独立 io 队列 + io_aux 线程 + sync 队列 + sync 线程 +
  独立 gloo completion 组(集合通信按 lane 过滤后的分发顺序保序)
- query 阶段的 hit_reduce 仍在单 prefetch_thread,不动
- 生产点(hiradix/unified/hybrid pp-ticket)改调 dispatch_prefetch
- 模拟器与两个单测文件同步适配

注意:SGLANG_HICACHE_PREFETCH_IO_THREADS 必须在实例内所有
TP/CP/PP rank 保持一致,否则 lane 组上的集合通信会错位挂死。
This commit is contained in:
2026-10-10 19:04:49 +08:00
parent 0afcb334a4
commit baba18bc7d
8 changed files with 181 additions and 97 deletions
+5
View File
@@ -763,6 +763,11 @@ class Envs:
SGLANG_HICACHE_TIMING = EnvBool(False)
# Pages per batched storage IO call (query / get / put chunking).
SGLANG_HICACHE_STORAGE_BATCH_SIZE = EnvInt(128)
# Number of parallel prefetch IO lanes (worker threads) per controller.
# Operations are sharded by crc32(request_id) so every rank maps a request
# to the same lane; each lane owns dedicated completion sync groups.
# MUST be identical across all TP/CP/PP ranks of an instance.
SGLANG_HICACHE_PREFETCH_IO_THREADS = EnvInt(4)
SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR = EnvStr(None)
# File-backend LRU eviction (opt-in; sizes accept SI/IEC suffixes, "0" disables).
SGLANG_HICACHE_FILE_BACKEND_MAX_SIZE = EnvStr(None)
+121 -54
View File
@@ -17,6 +17,7 @@ limitations under the License.
import logging
import threading
import time
import zlib
from dataclasses import dataclass
from queue import Empty, Queue
from typing import TYPE_CHECKING, Callable, List, NamedTuple, Optional
@@ -317,7 +318,18 @@ class HiCacheController:
self.attn_tp_group = attn_tp_group
self.pp_group = pp_group
self.prefetch_hits_sync_groups: List[torch.distributed.ProcessGroup] = []
self.prefetch_completion_sync_groups: List[torch.distributed.ProcessGroup] = []
# One set of completion sync groups per prefetch IO lane. Each lane's
# worker/sync threads form a serial pipeline, so the collective
# sequence on a lane's groups is identical across ranks as long as
# lane assignment is deterministic (see dispatch_prefetch).
self.prefetch_completion_sync_groups_by_lane: List[
List[torch.distributed.ProcessGroup]
] = []
# Read once at construction; attach/reset reuse the stored value so
# lane count never changes underneath live sync groups.
self.num_prefetch_io_lanes = max(
1, _envs.SGLANG_HICACHE_PREFETCH_IO_THREADS.get()
)
self.mem_pool_device_allocator = token_to_kv_pool_allocator
mem_pool_device = token_to_kv_pool_allocator.get_kvcache()
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool
@@ -346,8 +358,8 @@ class HiCacheController:
self.storage_stop_event = threading.Event()
# Storage control queues, (re)created whenever the storage threads start.
self.prefetch_buffer: Optional[Queue[PrefetchOperation]] = None
self.prefetch_sync_queue: Optional[Queue[PrefetchAck]] = None
self.prefetch_io_queues: List[Queue[PrefetchOperation]] = []
self.prefetch_sync_queues: List[Queue[PrefetchAck]] = []
self.prefetch_hit_queue: Optional[Queue[StorageOperation]] = None
self.ack_prefetch_queue = Queue[PrefetchAck]()
self.ack_backup_queue: Optional[Queue[StorageOperation]] = None
@@ -455,27 +467,39 @@ class HiCacheController:
self.prefetch_thread = threading.Thread(
target=self.prefetch_thread_func, daemon=True
)
self.prefetch_io_aux_thread = threading.Thread(
target=self.prefetch_io_aux_func, daemon=True
)
self.prefetch_sync_thread = threading.Thread(
target=self.prefetch_sync_thread_func, daemon=True
)
self.prefetch_io_aux_threads = [
threading.Thread(
target=self.prefetch_io_aux_func, args=(lane,), daemon=True
)
for lane in range(self.num_prefetch_io_lanes)
]
self.prefetch_sync_threads = [
threading.Thread(
target=self.prefetch_sync_thread_func, args=(lane,), daemon=True
)
for lane in range(self.num_prefetch_io_lanes)
]
self.backup_thread = threading.Thread(
target=self.backup_thread_func, daemon=True
)
self.prefetch_queue = Queue()
self.backup_queue = Queue()
self.prefetch_buffer = Queue()
self.prefetch_sync_queue = Queue()
self.prefetch_io_queues = [
Queue() for _ in range(self.num_prefetch_io_lanes)
]
self.prefetch_sync_queues = [
Queue() for _ in range(self.num_prefetch_io_lanes)
]
self.prefetch_hit_queue = Queue()
self.ack_prefetch_queue = Queue()
self.ack_backup_queue = Queue()
self.host_mem_release_queue = Queue()
self.prefetch_thread.start()
self.prefetch_io_aux_thread.start()
self.prefetch_sync_thread.start()
for thread in self.prefetch_io_aux_threads:
thread.start()
for thread in self.prefetch_sync_threads:
thread.start()
self.backup_thread.start()
def _stop_storage_threads(self):
@@ -496,10 +520,10 @@ class HiCacheController:
self.prefetch_queue.put_nowait(None)
if hasattr(self, "backup_queue"):
self.backup_queue.put_nowait(None)
if hasattr(self, "prefetch_buffer"):
self.prefetch_buffer.put_nowait(None)
if hasattr(self, "prefetch_sync_queue"):
self.prefetch_sync_queue.put_nowait(None)
for queue in getattr(self, "prefetch_io_queues", []) or []:
queue.put_nowait(None)
for queue in getattr(self, "prefetch_sync_queues", []) or []:
queue.put_nowait(None)
except Exception:
pass
@@ -509,10 +533,8 @@ class HiCacheController:
threads.append(self.prefetch_thread)
if hasattr(self, "backup_thread"):
threads.append(self.backup_thread)
if hasattr(self, "prefetch_io_aux_thread"):
threads.append(self.prefetch_io_aux_thread)
if hasattr(self, "prefetch_sync_thread"):
threads.append(self.prefetch_sync_thread)
threads.extend(getattr(self, "prefetch_io_aux_threads", []) or [])
threads.extend(getattr(self, "prefetch_sync_threads", []) or [])
for t in threads:
try:
@@ -596,7 +618,13 @@ class HiCacheController:
# Use dedicated gloo groups so storage prefetch sync is isolated
# from other collectives and consistent across CPxTP participants.
self.prefetch_hits_sync_groups = self._create_sync_groups()
self.prefetch_completion_sync_groups = self._create_sync_groups()
# Each IO lane gets its own completion groups: per-lane collectives
# stay ordered because a lane is a serial pipeline and dispatch is
# deterministic (crc32(request_id) % num_prefetch_io_lanes).
self.prefetch_completion_sync_groups_by_lane = [
self._create_sync_groups()
for _ in range(self.num_prefetch_io_lanes)
]
# Select the get and set functions
self.page_get_func = self._generic_page_get
@@ -631,9 +659,10 @@ class HiCacheController:
except Exception:
pass
self._destroy_sync_groups(self.prefetch_hits_sync_groups)
self._destroy_sync_groups(self.prefetch_completion_sync_groups)
for lane_groups in self.prefetch_completion_sync_groups_by_lane:
self._destroy_sync_groups(lane_groups)
self.prefetch_hits_sync_groups = []
self.prefetch_completion_sync_groups = []
self.prefetch_completion_sync_groups_by_lane = []
try:
if (
hasattr(self, "storage_backend")
@@ -671,11 +700,11 @@ class HiCacheController:
raise RuntimeError("Stop storage threads failed; detach aborted.") from e
# Best-effort destroy process groups created for storage ops.
self._destroy_sync_groups(
self.prefetch_hits_sync_groups + self.prefetch_completion_sync_groups
)
self._destroy_sync_groups(self.prefetch_hits_sync_groups)
for lane_groups in self.prefetch_completion_sync_groups_by_lane:
self._destroy_sync_groups(lane_groups)
self.prefetch_hits_sync_groups = []
self.prefetch_completion_sync_groups = []
self.prefetch_completion_sync_groups_by_lane = []
# Best-effort close (some backends rely on GC/destructor).
try:
@@ -769,13 +798,17 @@ class HiCacheController:
self.ack_load_queue.clear()
if self.enable_storage:
self.prefetch_thread.join()
self.prefetch_io_aux_thread.join()
self.prefetch_sync_thread.join()
for thread in self.prefetch_io_aux_threads:
thread.join()
for thread in self.prefetch_sync_threads:
thread.join()
self.backup_thread.join()
self.prefetch_queue.queue.clear()
self.backup_queue.queue.clear()
self.prefetch_buffer.queue.clear()
self.prefetch_sync_queue.queue.clear()
for queue in self.prefetch_io_queues:
queue.queue.clear()
for queue in self.prefetch_sync_queues:
queue.queue.clear()
self.prefetch_hit_queue.queue.clear()
self.ack_prefetch_queue.queue.clear()
self.ack_backup_queue.queue.clear()
@@ -788,18 +821,26 @@ class HiCacheController:
self.prefetch_thread = threading.Thread(
target=self.prefetch_thread_func, daemon=True
)
self.prefetch_io_aux_thread = threading.Thread(
target=self.prefetch_io_aux_func, daemon=True
)
self.prefetch_sync_thread = threading.Thread(
target=self.prefetch_sync_thread_func, daemon=True
)
self.prefetch_io_aux_threads = [
threading.Thread(
target=self.prefetch_io_aux_func, args=(lane,), daemon=True
)
for lane in range(self.num_prefetch_io_lanes)
]
self.prefetch_sync_threads = [
threading.Thread(
target=self.prefetch_sync_thread_func, args=(lane,), daemon=True
)
for lane in range(self.num_prefetch_io_lanes)
]
self.backup_thread = threading.Thread(
target=self.backup_thread_func, daemon=True
)
self.prefetch_thread.start()
self.prefetch_io_aux_thread.start()
self.prefetch_sync_thread.start()
for thread in self.prefetch_io_aux_threads:
thread.start()
for thread in self.prefetch_sync_threads:
thread.start()
self.backup_thread.start()
def has_inflight_device_transfers(self) -> bool:
@@ -1086,7 +1127,23 @@ class HiCacheController:
count += 1
return count
def _page_transfer(self, operation: PrefetchOperation) -> int:
def dispatch_prefetch(self, operation: PrefetchOperation) -> None:
"""Route a prefetch operation to its IO lane.
Lane assignment must be deterministic and identical on every rank:
each lane owns dedicated completion sync groups, and the collective
sequence on a lane is the filtered dispatch order. crc32 (not the
per-process salted builtin hash) keeps the mapping stable across
processes. All chunks of one operation stay in one lane, preserving
the per-operation ack order the sync thread reduces over.
"""
lane = (
zlib.crc32(str(operation.request_id).encode())
% self.num_prefetch_io_lanes
)
self.prefetch_io_queues[lane].put(operation)
def _page_transfer(self, operation: PrefetchOperation, sync_queue: Queue) -> int:
# Transfer batch by batch
prefix_keys = operation.prefix_keys
kv_derived_transfers = [
@@ -1098,8 +1155,8 @@ class HiCacheController:
completed_pages = 0
for i in range(0, len(operation.hash_value), STORAGE_BATCH_SIZE):
# When an error is occurred, we should keep looping and produce the same number of
# PrefetchAck as other ranks do, because prefetch_sync_thread (i.e. consumer of
# prefetch_sync_queue) perform reduce on the results. This is so tricky.
# PrefetchAck as other ranks do, because the prefetch sync thread (i.e. consumer of
# the lane sync queue) perform reduce on the results. This is so tricky.
if all_success and operation.is_terminated():
all_success = False
if all_success:
@@ -1138,7 +1195,7 @@ class HiCacheController:
completed_tokens=completed_pages * self.page_size,
operation=operation,
)
self.prefetch_sync_queue.put(ack)
sync_queue.put(ack)
return completed_pages
def _page_transfer_kv_batch(
@@ -1180,27 +1237,33 @@ class HiCacheController:
# Clamp to minimal number of hits.
return min([kv_hits, *sidecar_hits.values()])
def prefetch_io_aux_func(self):
def prefetch_io_aux_func(self, lane: int = 0):
"""
Auxiliary function conducting IO operations for prefetching.
Each lane has its own worker thread and queues; an operation's whole
chunk sequence stays on one lane so the sync thread sees it in order.
"""
io_queue = self.prefetch_io_queues[lane]
sync_queue = self.prefetch_sync_queues[lane]
while not self.storage_stop_event.is_set():
try:
operation = self.prefetch_buffer.get(block=True, timeout=1)
operation = io_queue.get(block=True, timeout=1)
if operation is None:
continue
_hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0
self._page_transfer(operation)
self._page_transfer(operation, sync_queue)
if _hc_timing_enabled():
logger.info(
"HC_TIMING stage=xfer req=%s wait_ms=%.1f xfer_ms=%.1f",
"HC_TIMING stage=xfer req=%s lane=%d wait_ms=%.1f xfer_ms=%.1f",
operation.request_id,
lane,
(_hc_t0 - getattr(operation, "query_done_ts", operation.start_time))
* 1e3,
(time.monotonic() - _hc_t0) * 1e3,
)
self.prefetch_sync_queue.put(
sync_queue.put(
PrefetchAck(
rid=operation.request_id,
completed_req=True,
@@ -1381,21 +1444,25 @@ class HiCacheController:
except Empty:
continue
def prefetch_sync_thread_func(self):
def prefetch_sync_thread_func(self, lane: int = 0):
"""Synchronize prefetch results across all PP and TP ranks."""
sync_queue = self.prefetch_sync_queues[lane]
groups = self.prefetch_completion_sync_groups_by_lane[lane]
while not self.storage_stop_event.is_set():
try:
ack = self.prefetch_sync_queue.get(block=True, timeout=1)
ack = sync_queue.get(block=True, timeout=1)
if ack is None:
continue
self._reduce_prefetch_ack(ack)
self._reduce_prefetch_ack(ack, groups)
if _hc_timing_enabled() and ack.completed_req:
ack.operation.final_ack_enqueue_ts = time.monotonic()
self.ack_prefetch_queue.put(ack)
except Empty:
continue
def _reduce_prefetch_ack(self, ack: PrefetchAck) -> None:
def _reduce_prefetch_ack(
self, ack: PrefetchAck, groups: List[torch.distributed.ProcessGroup]
) -> None:
"""Synchronize all ranks to agree on a PrefetchAck."""
if ack.completed_tokens is not None:
# Determine the minimal successful prefix of tokens.
@@ -1406,7 +1473,7 @@ class HiCacheController:
self._all_reduce(
completed_tokens_tensor,
torch.distributed.ReduceOp.MIN,
self.prefetch_completion_sync_groups,
groups,
)
ack.completed_tokens = completed_tokens_tensor.item()
if _hc_timing_enabled():
+1 -1
View File
@@ -653,7 +653,7 @@ class HiRadixCache(RadixCache):
: alloc_len // self.page_size
]
operation.host_indices = host_indices
cc.prefetch_buffer.put(operation)
cc.dispatch_prefetch(operation)
def _drain_ack_prefetch():
for ack in _drain_queue(cc.ack_prefetch_queue, n_ack_prefetch):
@@ -1006,7 +1006,7 @@ class HybridCacheController(BaseHiCacheController):
if not self._allocate_pp_prefetch_buffers(ticket, operation):
operation.host_indices = torch.empty(0, dtype=torch.int64)
operation.mark_terminate()
self.prefetch_buffer.put(operation)
self.dispatch_prefetch(operation)
except Exception:
logger.exception("PP HiCache ticket processing failed.")
if operation is None or not operation.hash_value:
@@ -1016,7 +1016,7 @@ class HybridCacheController(BaseHiCacheController):
operation.host_indices = torch.empty(0, dtype=torch.int64)
operation.mark_terminate()
# Preserve every KV/sidecar ACK even when local allocation fails.
self.prefetch_buffer.put(operation)
self.dispatch_prefetch(operation)
finally:
if is_source and objects:
self.pp_prefetch_command_queue.task_done()
@@ -1104,15 +1104,15 @@ class HybridCacheController(BaseHiCacheController):
)
return host_indices, device_indices, resolved_pool_transfers
def _page_transfer(self, operation: PrefetchOperation) -> bool:
def _page_transfer(self, operation: PrefetchOperation, sync_queue: Queue) -> bool:
# KV pools and KV-derived pools first — determines actual completed page count
kv_completed_pages = super()._page_transfer(operation)
kv_completed_pages = super()._page_transfer(operation, sync_queue)
# Read non-KV derived sidecar pool, e.g. SWA, Mamba.
self._page_transfer_sidecar(operation, kv_completed_pages)
self._page_transfer_sidecar(operation, kv_completed_pages, sync_queue)
def _page_transfer_sidecar(
self, operation: PrefetchOperation, kv_completed_pages: int
self, operation: PrefetchOperation, kv_completed_pages: int, sync_queue: Queue
) -> None:
if operation.pool_transfers is None:
return
@@ -1144,10 +1144,10 @@ class HybridCacheController(BaseHiCacheController):
transfers_nonkv, extra_info=extra_info
)
pool_hits = count_pool_hits(results)
# Emit PrefetchAck to prefetch_sync_queue, even the operation has been canceled by the
# Emit PrefetchAck to the lane sync queue, even the operation has been canceled by the
# scheduler thread. The prefetch sync thread expects the same number of PrefetchAck objects
# to perform all_reduce.
self.prefetch_sync_queue.put(
sync_queue.put(
PrefetchAck(
rid=operation.request_id,
operation=operation,
@@ -1421,9 +1421,11 @@ class HybridCacheController(BaseHiCacheController):
pool.device_indices = source.device_indices
return extra_pools
def _reduce_prefetch_ack(self, ack: PrefetchAck) -> None:
def _reduce_prefetch_ack(
self, ack: PrefetchAck, groups: List[torch.distributed.ProcessGroup]
) -> None:
# Handle KV-derived pool.
super()._reduce_prefetch_ack(ack)
super()._reduce_prefetch_ack(ack, groups)
# Handle other sidecar pools, e.g. SWA, Mamba.
if ack.pool_hits is not None:
@@ -1436,19 +1438,21 @@ class HybridCacheController(BaseHiCacheController):
self._all_reduce(
packed,
torch.distributed.ReduceOp.MIN,
self.prefetch_completion_sync_groups,
groups,
)
for i, pool in enumerate(PoolName):
ack.pool_hits[pool.value] = packed[i].item()
def prefetch_sync_thread_func(self) -> None:
def prefetch_sync_thread_func(self, lane: int = 0) -> None:
"""Synchronize progressive ACKs and publish PP ticket completion."""
sync_queue = self.prefetch_sync_queues[lane]
groups = self.prefetch_completion_sync_groups_by_lane[lane]
while not self.storage_stop_event.is_set():
try:
ack = self.prefetch_sync_queue.get(block=True, timeout=1)
ack = sync_queue.get(block=True, timeout=1)
if ack is None:
continue
self._reduce_prefetch_ack(ack)
self._reduce_prefetch_ack(ack, groups)
if not getattr(ack.operation, "is_pp_broadcast", False):
if _hc_timing_enabled() and ack.completed_req:
ack.operation.final_ack_enqueue_ts = time.monotonic()
@@ -3001,7 +3001,7 @@ class UnifiedRadixCache(BasePrefixCache):
self.ongoing_prefetch[request] = info._replace(host_indices=host_indices)
if buffer_mode:
cc.prefetch_tokens_occupied += alloc_len
cc.prefetch_buffer.put(operation)
cc.dispatch_prefetch(operation)
return True
def _drain_and_alloc_storage_hit():