hicache L3 prefetch: io_aux 传输并行化(按 op 分 lane)
build-sglang-image / build (push) Failing after 48m34s

打点实测单条 prefetch_io_aux_func 线程串行传输是所有请求的瓶颈:
mc8 下每请求 xfer 排队 0->1.9s 线性增长(每请求 5 批 x ~68ms 固定开销,
介质无关)。本提交把传输层拆成 N 条并行 lane(默认 4,
SGLANG_HICACHE_PREFETCH_IO_THREADS 可调):

- dispatch_prefetch() 用 crc32(request_id) % N 分 lane(内置 hash 对 str
  每进程加盐,跨 rank 不一致会导致 gloo 集合通信错位,禁用)
- 同一 op 的所有 chunk 保持在同一 lane,保序 reduce 语义不变
- 每 lane 独立 io 队列 + io_aux 线程 + sync 队列 + sync 线程 +
  独立 gloo completion 组(集合通信按 lane 过滤后的分发顺序保序)
- query 阶段的 hit_reduce 仍在单 prefetch_thread,不动
- 生产点(hiradix/unified/hybrid pp-ticket)改调 dispatch_prefetch
- 模拟器与两个单测文件同步适配

注意:SGLANG_HICACHE_PREFETCH_IO_THREADS 必须在实例内所有
TP/CP/PP rank 保持一致,否则 lane 组上的集合通信会错位挂死。
This commit is contained in:
2026-10-10 19:04:49 +08:00
parent 0afcb334a4
commit baba18bc7d
8 changed files with 181 additions and 97 deletions
+5
View File
@@ -763,6 +763,11 @@ class Envs:
SGLANG_HICACHE_TIMING = EnvBool(False) SGLANG_HICACHE_TIMING = EnvBool(False)
# Pages per batched storage IO call (query / get / put chunking). # Pages per batched storage IO call (query / get / put chunking).
SGLANG_HICACHE_STORAGE_BATCH_SIZE = EnvInt(128) SGLANG_HICACHE_STORAGE_BATCH_SIZE = EnvInt(128)
# Number of parallel prefetch IO lanes (worker threads) per controller.
# Operations are sharded by crc32(request_id) so every rank maps a request
# to the same lane; each lane owns dedicated completion sync groups.
# MUST be identical across all TP/CP/PP ranks of an instance.
SGLANG_HICACHE_PREFETCH_IO_THREADS = EnvInt(4)
SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR = EnvStr(None) SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR = EnvStr(None)
# File-backend LRU eviction (opt-in; sizes accept SI/IEC suffixes, "0" disables). # File-backend LRU eviction (opt-in; sizes accept SI/IEC suffixes, "0" disables).
SGLANG_HICACHE_FILE_BACKEND_MAX_SIZE = EnvStr(None) SGLANG_HICACHE_FILE_BACKEND_MAX_SIZE = EnvStr(None)
+121 -54
View File
@@ -17,6 +17,7 @@ limitations under the License.
import logging import logging
import threading import threading
import time import time
import zlib
from dataclasses import dataclass from dataclasses import dataclass
from queue import Empty, Queue from queue import Empty, Queue
from typing import TYPE_CHECKING, Callable, List, NamedTuple, Optional from typing import TYPE_CHECKING, Callable, List, NamedTuple, Optional
@@ -317,7 +318,18 @@ class HiCacheController:
self.attn_tp_group = attn_tp_group self.attn_tp_group = attn_tp_group
self.pp_group = pp_group self.pp_group = pp_group
self.prefetch_hits_sync_groups: List[torch.distributed.ProcessGroup] = [] self.prefetch_hits_sync_groups: List[torch.distributed.ProcessGroup] = []
self.prefetch_completion_sync_groups: List[torch.distributed.ProcessGroup] = [] # One set of completion sync groups per prefetch IO lane. Each lane's
# worker/sync threads form a serial pipeline, so the collective
# sequence on a lane's groups is identical across ranks as long as
# lane assignment is deterministic (see dispatch_prefetch).
self.prefetch_completion_sync_groups_by_lane: List[
List[torch.distributed.ProcessGroup]
] = []
# Read once at construction; attach/reset reuse the stored value so
# lane count never changes underneath live sync groups.
self.num_prefetch_io_lanes = max(
1, _envs.SGLANG_HICACHE_PREFETCH_IO_THREADS.get()
)
self.mem_pool_device_allocator = token_to_kv_pool_allocator self.mem_pool_device_allocator = token_to_kv_pool_allocator
mem_pool_device = token_to_kv_pool_allocator.get_kvcache() mem_pool_device = token_to_kv_pool_allocator.get_kvcache()
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool
@@ -346,8 +358,8 @@ class HiCacheController:
self.storage_stop_event = threading.Event() self.storage_stop_event = threading.Event()
# Storage control queues, (re)created whenever the storage threads start. # Storage control queues, (re)created whenever the storage threads start.
self.prefetch_buffer: Optional[Queue[PrefetchOperation]] = None self.prefetch_io_queues: List[Queue[PrefetchOperation]] = []
self.prefetch_sync_queue: Optional[Queue[PrefetchAck]] = None self.prefetch_sync_queues: List[Queue[PrefetchAck]] = []
self.prefetch_hit_queue: Optional[Queue[StorageOperation]] = None self.prefetch_hit_queue: Optional[Queue[StorageOperation]] = None
self.ack_prefetch_queue = Queue[PrefetchAck]() self.ack_prefetch_queue = Queue[PrefetchAck]()
self.ack_backup_queue: Optional[Queue[StorageOperation]] = None self.ack_backup_queue: Optional[Queue[StorageOperation]] = None
@@ -455,27 +467,39 @@ class HiCacheController:
self.prefetch_thread = threading.Thread( self.prefetch_thread = threading.Thread(
target=self.prefetch_thread_func, daemon=True target=self.prefetch_thread_func, daemon=True
) )
self.prefetch_io_aux_thread = threading.Thread( self.prefetch_io_aux_threads = [
target=self.prefetch_io_aux_func, daemon=True threading.Thread(
) target=self.prefetch_io_aux_func, args=(lane,), daemon=True
self.prefetch_sync_thread = threading.Thread( )
target=self.prefetch_sync_thread_func, daemon=True for lane in range(self.num_prefetch_io_lanes)
) ]
self.prefetch_sync_threads = [
threading.Thread(
target=self.prefetch_sync_thread_func, args=(lane,), daemon=True
)
for lane in range(self.num_prefetch_io_lanes)
]
self.backup_thread = threading.Thread( self.backup_thread = threading.Thread(
target=self.backup_thread_func, daemon=True target=self.backup_thread_func, daemon=True
) )
self.prefetch_queue = Queue() self.prefetch_queue = Queue()
self.backup_queue = Queue() self.backup_queue = Queue()
self.prefetch_buffer = Queue() self.prefetch_io_queues = [
self.prefetch_sync_queue = Queue() Queue() for _ in range(self.num_prefetch_io_lanes)
]
self.prefetch_sync_queues = [
Queue() for _ in range(self.num_prefetch_io_lanes)
]
self.prefetch_hit_queue = Queue() self.prefetch_hit_queue = Queue()
self.ack_prefetch_queue = Queue() self.ack_prefetch_queue = Queue()
self.ack_backup_queue = Queue() self.ack_backup_queue = Queue()
self.host_mem_release_queue = Queue() self.host_mem_release_queue = Queue()
self.prefetch_thread.start() self.prefetch_thread.start()
self.prefetch_io_aux_thread.start() for thread in self.prefetch_io_aux_threads:
self.prefetch_sync_thread.start() thread.start()
for thread in self.prefetch_sync_threads:
thread.start()
self.backup_thread.start() self.backup_thread.start()
def _stop_storage_threads(self): def _stop_storage_threads(self):
@@ -496,10 +520,10 @@ class HiCacheController:
self.prefetch_queue.put_nowait(None) self.prefetch_queue.put_nowait(None)
if hasattr(self, "backup_queue"): if hasattr(self, "backup_queue"):
self.backup_queue.put_nowait(None) self.backup_queue.put_nowait(None)
if hasattr(self, "prefetch_buffer"): for queue in getattr(self, "prefetch_io_queues", []) or []:
self.prefetch_buffer.put_nowait(None) queue.put_nowait(None)
if hasattr(self, "prefetch_sync_queue"): for queue in getattr(self, "prefetch_sync_queues", []) or []:
self.prefetch_sync_queue.put_nowait(None) queue.put_nowait(None)
except Exception: except Exception:
pass pass
@@ -509,10 +533,8 @@ class HiCacheController:
threads.append(self.prefetch_thread) threads.append(self.prefetch_thread)
if hasattr(self, "backup_thread"): if hasattr(self, "backup_thread"):
threads.append(self.backup_thread) threads.append(self.backup_thread)
if hasattr(self, "prefetch_io_aux_thread"): threads.extend(getattr(self, "prefetch_io_aux_threads", []) or [])
threads.append(self.prefetch_io_aux_thread) threads.extend(getattr(self, "prefetch_sync_threads", []) or [])
if hasattr(self, "prefetch_sync_thread"):
threads.append(self.prefetch_sync_thread)
for t in threads: for t in threads:
try: try:
@@ -596,7 +618,13 @@ class HiCacheController:
# Use dedicated gloo groups so storage prefetch sync is isolated # Use dedicated gloo groups so storage prefetch sync is isolated
# from other collectives and consistent across CPxTP participants. # from other collectives and consistent across CPxTP participants.
self.prefetch_hits_sync_groups = self._create_sync_groups() self.prefetch_hits_sync_groups = self._create_sync_groups()
self.prefetch_completion_sync_groups = self._create_sync_groups() # Each IO lane gets its own completion groups: per-lane collectives
# stay ordered because a lane is a serial pipeline and dispatch is
# deterministic (crc32(request_id) % num_prefetch_io_lanes).
self.prefetch_completion_sync_groups_by_lane = [
self._create_sync_groups()
for _ in range(self.num_prefetch_io_lanes)
]
# Select the get and set functions # Select the get and set functions
self.page_get_func = self._generic_page_get self.page_get_func = self._generic_page_get
@@ -631,9 +659,10 @@ class HiCacheController:
except Exception: except Exception:
pass pass
self._destroy_sync_groups(self.prefetch_hits_sync_groups) self._destroy_sync_groups(self.prefetch_hits_sync_groups)
self._destroy_sync_groups(self.prefetch_completion_sync_groups) for lane_groups in self.prefetch_completion_sync_groups_by_lane:
self._destroy_sync_groups(lane_groups)
self.prefetch_hits_sync_groups = [] self.prefetch_hits_sync_groups = []
self.prefetch_completion_sync_groups = [] self.prefetch_completion_sync_groups_by_lane = []
try: try:
if ( if (
hasattr(self, "storage_backend") hasattr(self, "storage_backend")
@@ -671,11 +700,11 @@ class HiCacheController:
raise RuntimeError("Stop storage threads failed; detach aborted.") from e raise RuntimeError("Stop storage threads failed; detach aborted.") from e
# Best-effort destroy process groups created for storage ops. # Best-effort destroy process groups created for storage ops.
self._destroy_sync_groups( self._destroy_sync_groups(self.prefetch_hits_sync_groups)
self.prefetch_hits_sync_groups + self.prefetch_completion_sync_groups for lane_groups in self.prefetch_completion_sync_groups_by_lane:
) self._destroy_sync_groups(lane_groups)
self.prefetch_hits_sync_groups = [] self.prefetch_hits_sync_groups = []
self.prefetch_completion_sync_groups = [] self.prefetch_completion_sync_groups_by_lane = []
# Best-effort close (some backends rely on GC/destructor). # Best-effort close (some backends rely on GC/destructor).
try: try:
@@ -769,13 +798,17 @@ class HiCacheController:
self.ack_load_queue.clear() self.ack_load_queue.clear()
if self.enable_storage: if self.enable_storage:
self.prefetch_thread.join() self.prefetch_thread.join()
self.prefetch_io_aux_thread.join() for thread in self.prefetch_io_aux_threads:
self.prefetch_sync_thread.join() thread.join()
for thread in self.prefetch_sync_threads:
thread.join()
self.backup_thread.join() self.backup_thread.join()
self.prefetch_queue.queue.clear() self.prefetch_queue.queue.clear()
self.backup_queue.queue.clear() self.backup_queue.queue.clear()
self.prefetch_buffer.queue.clear() for queue in self.prefetch_io_queues:
self.prefetch_sync_queue.queue.clear() queue.queue.clear()
for queue in self.prefetch_sync_queues:
queue.queue.clear()
self.prefetch_hit_queue.queue.clear() self.prefetch_hit_queue.queue.clear()
self.ack_prefetch_queue.queue.clear() self.ack_prefetch_queue.queue.clear()
self.ack_backup_queue.queue.clear() self.ack_backup_queue.queue.clear()
@@ -788,18 +821,26 @@ class HiCacheController:
self.prefetch_thread = threading.Thread( self.prefetch_thread = threading.Thread(
target=self.prefetch_thread_func, daemon=True target=self.prefetch_thread_func, daemon=True
) )
self.prefetch_io_aux_thread = threading.Thread( self.prefetch_io_aux_threads = [
target=self.prefetch_io_aux_func, daemon=True threading.Thread(
) target=self.prefetch_io_aux_func, args=(lane,), daemon=True
self.prefetch_sync_thread = threading.Thread( )
target=self.prefetch_sync_thread_func, daemon=True for lane in range(self.num_prefetch_io_lanes)
) ]
self.prefetch_sync_threads = [
threading.Thread(
target=self.prefetch_sync_thread_func, args=(lane,), daemon=True
)
for lane in range(self.num_prefetch_io_lanes)
]
self.backup_thread = threading.Thread( self.backup_thread = threading.Thread(
target=self.backup_thread_func, daemon=True target=self.backup_thread_func, daemon=True
) )
self.prefetch_thread.start() self.prefetch_thread.start()
self.prefetch_io_aux_thread.start() for thread in self.prefetch_io_aux_threads:
self.prefetch_sync_thread.start() thread.start()
for thread in self.prefetch_sync_threads:
thread.start()
self.backup_thread.start() self.backup_thread.start()
def has_inflight_device_transfers(self) -> bool: def has_inflight_device_transfers(self) -> bool:
@@ -1086,7 +1127,23 @@ class HiCacheController:
count += 1 count += 1
return count return count
def _page_transfer(self, operation: PrefetchOperation) -> int: def dispatch_prefetch(self, operation: PrefetchOperation) -> None:
"""Route a prefetch operation to its IO lane.
Lane assignment must be deterministic and identical on every rank:
each lane owns dedicated completion sync groups, and the collective
sequence on a lane is the filtered dispatch order. crc32 (not the
per-process salted builtin hash) keeps the mapping stable across
processes. All chunks of one operation stay in one lane, preserving
the per-operation ack order the sync thread reduces over.
"""
lane = (
zlib.crc32(str(operation.request_id).encode())
% self.num_prefetch_io_lanes
)
self.prefetch_io_queues[lane].put(operation)
def _page_transfer(self, operation: PrefetchOperation, sync_queue: Queue) -> int:
# Transfer batch by batch # Transfer batch by batch
prefix_keys = operation.prefix_keys prefix_keys = operation.prefix_keys
kv_derived_transfers = [ kv_derived_transfers = [
@@ -1098,8 +1155,8 @@ class HiCacheController:
completed_pages = 0 completed_pages = 0
for i in range(0, len(operation.hash_value), STORAGE_BATCH_SIZE): for i in range(0, len(operation.hash_value), STORAGE_BATCH_SIZE):
# When an error is occurred, we should keep looping and produce the same number of # When an error is occurred, we should keep looping and produce the same number of
# PrefetchAck as other ranks do, because prefetch_sync_thread (i.e. consumer of # PrefetchAck as other ranks do, because the prefetch sync thread (i.e. consumer of
# prefetch_sync_queue) perform reduce on the results. This is so tricky. # the lane sync queue) perform reduce on the results. This is so tricky.
if all_success and operation.is_terminated(): if all_success and operation.is_terminated():
all_success = False all_success = False
if all_success: if all_success:
@@ -1138,7 +1195,7 @@ class HiCacheController:
completed_tokens=completed_pages * self.page_size, completed_tokens=completed_pages * self.page_size,
operation=operation, operation=operation,
) )
self.prefetch_sync_queue.put(ack) sync_queue.put(ack)
return completed_pages return completed_pages
def _page_transfer_kv_batch( def _page_transfer_kv_batch(
@@ -1180,27 +1237,33 @@ class HiCacheController:
# Clamp to minimal number of hits. # Clamp to minimal number of hits.
return min([kv_hits, *sidecar_hits.values()]) return min([kv_hits, *sidecar_hits.values()])
def prefetch_io_aux_func(self): def prefetch_io_aux_func(self, lane: int = 0):
""" """
Auxiliary function conducting IO operations for prefetching. Auxiliary function conducting IO operations for prefetching.
Each lane has its own worker thread and queues; an operation's whole
chunk sequence stays on one lane so the sync thread sees it in order.
""" """
io_queue = self.prefetch_io_queues[lane]
sync_queue = self.prefetch_sync_queues[lane]
while not self.storage_stop_event.is_set(): while not self.storage_stop_event.is_set():
try: try:
operation = self.prefetch_buffer.get(block=True, timeout=1) operation = io_queue.get(block=True, timeout=1)
if operation is None: if operation is None:
continue continue
_hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0 _hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0
self._page_transfer(operation) self._page_transfer(operation, sync_queue)
if _hc_timing_enabled(): if _hc_timing_enabled():
logger.info( logger.info(
"HC_TIMING stage=xfer req=%s wait_ms=%.1f xfer_ms=%.1f", "HC_TIMING stage=xfer req=%s lane=%d wait_ms=%.1f xfer_ms=%.1f",
operation.request_id, operation.request_id,
lane,
(_hc_t0 - getattr(operation, "query_done_ts", operation.start_time)) (_hc_t0 - getattr(operation, "query_done_ts", operation.start_time))
* 1e3, * 1e3,
(time.monotonic() - _hc_t0) * 1e3, (time.monotonic() - _hc_t0) * 1e3,
) )
self.prefetch_sync_queue.put( sync_queue.put(
PrefetchAck( PrefetchAck(
rid=operation.request_id, rid=operation.request_id,
completed_req=True, completed_req=True,
@@ -1381,21 +1444,25 @@ class HiCacheController:
except Empty: except Empty:
continue continue
def prefetch_sync_thread_func(self): def prefetch_sync_thread_func(self, lane: int = 0):
"""Synchronize prefetch results across all PP and TP ranks.""" """Synchronize prefetch results across all PP and TP ranks."""
sync_queue = self.prefetch_sync_queues[lane]
groups = self.prefetch_completion_sync_groups_by_lane[lane]
while not self.storage_stop_event.is_set(): while not self.storage_stop_event.is_set():
try: try:
ack = self.prefetch_sync_queue.get(block=True, timeout=1) ack = sync_queue.get(block=True, timeout=1)
if ack is None: if ack is None:
continue continue
self._reduce_prefetch_ack(ack) self._reduce_prefetch_ack(ack, groups)
if _hc_timing_enabled() and ack.completed_req: if _hc_timing_enabled() and ack.completed_req:
ack.operation.final_ack_enqueue_ts = time.monotonic() ack.operation.final_ack_enqueue_ts = time.monotonic()
self.ack_prefetch_queue.put(ack) self.ack_prefetch_queue.put(ack)
except Empty: except Empty:
continue continue
def _reduce_prefetch_ack(self, ack: PrefetchAck) -> None: def _reduce_prefetch_ack(
self, ack: PrefetchAck, groups: List[torch.distributed.ProcessGroup]
) -> None:
"""Synchronize all ranks to agree on a PrefetchAck.""" """Synchronize all ranks to agree on a PrefetchAck."""
if ack.completed_tokens is not None: if ack.completed_tokens is not None:
# Determine the minimal successful prefix of tokens. # Determine the minimal successful prefix of tokens.
@@ -1406,7 +1473,7 @@ class HiCacheController:
self._all_reduce( self._all_reduce(
completed_tokens_tensor, completed_tokens_tensor,
torch.distributed.ReduceOp.MIN, torch.distributed.ReduceOp.MIN,
self.prefetch_completion_sync_groups, groups,
) )
ack.completed_tokens = completed_tokens_tensor.item() ack.completed_tokens = completed_tokens_tensor.item()
if _hc_timing_enabled(): if _hc_timing_enabled():
+1 -1
View File
@@ -653,7 +653,7 @@ class HiRadixCache(RadixCache):
: alloc_len // self.page_size : alloc_len // self.page_size
] ]
operation.host_indices = host_indices operation.host_indices = host_indices
cc.prefetch_buffer.put(operation) cc.dispatch_prefetch(operation)
def _drain_ack_prefetch(): def _drain_ack_prefetch():
for ack in _drain_queue(cc.ack_prefetch_queue, n_ack_prefetch): for ack in _drain_queue(cc.ack_prefetch_queue, n_ack_prefetch):
@@ -1006,7 +1006,7 @@ class HybridCacheController(BaseHiCacheController):
if not self._allocate_pp_prefetch_buffers(ticket, operation): if not self._allocate_pp_prefetch_buffers(ticket, operation):
operation.host_indices = torch.empty(0, dtype=torch.int64) operation.host_indices = torch.empty(0, dtype=torch.int64)
operation.mark_terminate() operation.mark_terminate()
self.prefetch_buffer.put(operation) self.dispatch_prefetch(operation)
except Exception: except Exception:
logger.exception("PP HiCache ticket processing failed.") logger.exception("PP HiCache ticket processing failed.")
if operation is None or not operation.hash_value: if operation is None or not operation.hash_value:
@@ -1016,7 +1016,7 @@ class HybridCacheController(BaseHiCacheController):
operation.host_indices = torch.empty(0, dtype=torch.int64) operation.host_indices = torch.empty(0, dtype=torch.int64)
operation.mark_terminate() operation.mark_terminate()
# Preserve every KV/sidecar ACK even when local allocation fails. # Preserve every KV/sidecar ACK even when local allocation fails.
self.prefetch_buffer.put(operation) self.dispatch_prefetch(operation)
finally: finally:
if is_source and objects: if is_source and objects:
self.pp_prefetch_command_queue.task_done() self.pp_prefetch_command_queue.task_done()
@@ -1104,15 +1104,15 @@ class HybridCacheController(BaseHiCacheController):
) )
return host_indices, device_indices, resolved_pool_transfers return host_indices, device_indices, resolved_pool_transfers
def _page_transfer(self, operation: PrefetchOperation) -> bool: def _page_transfer(self, operation: PrefetchOperation, sync_queue: Queue) -> bool:
# KV pools and KV-derived pools first — determines actual completed page count # KV pools and KV-derived pools first — determines actual completed page count
kv_completed_pages = super()._page_transfer(operation) kv_completed_pages = super()._page_transfer(operation, sync_queue)
# Read non-KV derived sidecar pool, e.g. SWA, Mamba. # Read non-KV derived sidecar pool, e.g. SWA, Mamba.
self._page_transfer_sidecar(operation, kv_completed_pages) self._page_transfer_sidecar(operation, kv_completed_pages, sync_queue)
def _page_transfer_sidecar( def _page_transfer_sidecar(
self, operation: PrefetchOperation, kv_completed_pages: int self, operation: PrefetchOperation, kv_completed_pages: int, sync_queue: Queue
) -> None: ) -> None:
if operation.pool_transfers is None: if operation.pool_transfers is None:
return return
@@ -1144,10 +1144,10 @@ class HybridCacheController(BaseHiCacheController):
transfers_nonkv, extra_info=extra_info transfers_nonkv, extra_info=extra_info
) )
pool_hits = count_pool_hits(results) pool_hits = count_pool_hits(results)
# Emit PrefetchAck to prefetch_sync_queue, even the operation has been canceled by the # Emit PrefetchAck to the lane sync queue, even the operation has been canceled by the
# scheduler thread. The prefetch sync thread expects the same number of PrefetchAck objects # scheduler thread. The prefetch sync thread expects the same number of PrefetchAck objects
# to perform all_reduce. # to perform all_reduce.
self.prefetch_sync_queue.put( sync_queue.put(
PrefetchAck( PrefetchAck(
rid=operation.request_id, rid=operation.request_id,
operation=operation, operation=operation,
@@ -1421,9 +1421,11 @@ class HybridCacheController(BaseHiCacheController):
pool.device_indices = source.device_indices pool.device_indices = source.device_indices
return extra_pools return extra_pools
def _reduce_prefetch_ack(self, ack: PrefetchAck) -> None: def _reduce_prefetch_ack(
self, ack: PrefetchAck, groups: List[torch.distributed.ProcessGroup]
) -> None:
# Handle KV-derived pool. # Handle KV-derived pool.
super()._reduce_prefetch_ack(ack) super()._reduce_prefetch_ack(ack, groups)
# Handle other sidecar pools, e.g. SWA, Mamba. # Handle other sidecar pools, e.g. SWA, Mamba.
if ack.pool_hits is not None: if ack.pool_hits is not None:
@@ -1436,19 +1438,21 @@ class HybridCacheController(BaseHiCacheController):
self._all_reduce( self._all_reduce(
packed, packed,
torch.distributed.ReduceOp.MIN, torch.distributed.ReduceOp.MIN,
self.prefetch_completion_sync_groups, groups,
) )
for i, pool in enumerate(PoolName): for i, pool in enumerate(PoolName):
ack.pool_hits[pool.value] = packed[i].item() ack.pool_hits[pool.value] = packed[i].item()
def prefetch_sync_thread_func(self) -> None: def prefetch_sync_thread_func(self, lane: int = 0) -> None:
"""Synchronize progressive ACKs and publish PP ticket completion.""" """Synchronize progressive ACKs and publish PP ticket completion."""
sync_queue = self.prefetch_sync_queues[lane]
groups = self.prefetch_completion_sync_groups_by_lane[lane]
while not self.storage_stop_event.is_set(): while not self.storage_stop_event.is_set():
try: try:
ack = self.prefetch_sync_queue.get(block=True, timeout=1) ack = sync_queue.get(block=True, timeout=1)
if ack is None: if ack is None:
continue continue
self._reduce_prefetch_ack(ack) self._reduce_prefetch_ack(ack, groups)
if not getattr(ack.operation, "is_pp_broadcast", False): if not getattr(ack.operation, "is_pp_broadcast", False):
if _hc_timing_enabled() and ack.completed_req: if _hc_timing_enabled() and ack.completed_req:
ack.operation.final_ack_enqueue_ts = time.monotonic() ack.operation.final_ack_enqueue_ts = time.monotonic()
@@ -3001,7 +3001,7 @@ class UnifiedRadixCache(BasePrefixCache):
self.ongoing_prefetch[request] = info._replace(host_indices=host_indices) self.ongoing_prefetch[request] = info._replace(host_indices=host_indices)
if buffer_mode: if buffer_mode:
cc.prefetch_tokens_occupied += alloc_len cc.prefetch_tokens_occupied += alloc_len
cc.prefetch_buffer.put(operation) cc.dispatch_prefetch(operation)
return True return True
def _drain_and_alloc_storage_hit(): def _drain_and_alloc_storage_hit():
@@ -37,15 +37,17 @@ class TestPPPrefetchTicket(unittest.TestCase):
c.pp_size, c.tp_size = 4, 2 c.pp_size, c.tp_size = 4, 2
c.pp_group, c.pp_prefetch_command_group = "pp", "command" c.pp_group, c.pp_prefetch_command_group = "pp", "command"
c.pp_prefetch_command_thread = None c.pp_prefetch_command_thread = None
c.prefetch_hits_sync_groups = c.prefetch_completion_sync_groups = ["pp", "tp"] c.prefetch_hits_sync_groups = ["pp", "tp"]
c.num_prefetch_io_lanes = 1
c.prefetch_completion_sync_groups_by_lane = [["pp", "tp"]]
c.pp_prefetch_states, c.pp_prefetch_decisions = {}, {} c.pp_prefetch_states, c.pp_prefetch_decisions = {}, {}
c.pp_prefetch_state_lock = threading.Lock() c.pp_prefetch_state_lock = threading.Lock()
c.pp_prefetch_command_queue = Queue() c.pp_prefetch_command_queue = Queue()
c.prefetch_queue = Queue() c.prefetch_queue = Queue()
c.prefetch_sync_queue = Queue() c.prefetch_sync_queues = [Queue()]
c.ack_prefetch_queue = Queue() c.ack_prefetch_queue = Queue()
c.host_mem_release_queue = Queue() c.host_mem_release_queue = Queue()
c.prefetch_buffer = Queue() c.prefetch_io_queues = [Queue()]
c.storage_stop_event = threading.Event() c.storage_stop_event = threading.Event()
c.prefetch_tokens_occupied = 0 c.prefetch_tokens_occupied = 0
c.mem_pool_host = Mock(page_size=4) c.mem_pool_host = Mock(page_size=4)
@@ -103,7 +105,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
def sync_acks(self, *acks): def sync_acks(self, *acks):
for ack in acks: for ack in acks:
self.c.prefetch_sync_queue.put(ack) self.c.prefetch_sync_queues[0].put(ack)
with patch.object( with patch.object(
self.c.storage_stop_event, self.c.storage_stop_event,
"is_set", "is_set",
@@ -183,7 +185,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
self.assertFalse(cache.check_prefetch_progress(handle)) self.assertFalse(cache.check_prefetch_progress(handle))
c._storage_hit_query.reset_mock() c._storage_hit_query.reset_mock()
self.run_commands(ticket) self.run_commands(ticket)
operation = c.prefetch_buffer.get_nowait() operation = c.prefetch_io_queues[0].get_nowait()
self.assertEqual(operation.handle, handle) self.assertEqual(operation.handle, handle)
self.assertEqual( self.assertEqual(
operation.hash_value, get_storage_hash_str(ticket.prefetch_key, page_size=4) operation.hash_value, get_storage_hash_str(ticket.prefetch_key, page_size=4)
@@ -237,7 +239,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
), ),
): ):
c.prefetch_io_aux_func() c.prefetch_io_aux_func()
acks = [c.prefetch_sync_queue.get_nowait() for _ in range(8)] acks = [c.prefetch_sync_queues[0].get_nowait() for _ in range(8)]
self.assertEqual([a.rid for a in acks], ["hit"] * 4 + ["next"] * 4) self.assertEqual([a.rid for a in acks], ["hit"] * 4 + ["next"] * 4)
self.assertEqual( self.assertEqual(
[a.completed_tokens for a in acks], [0, 0, None, None, 4, 8, None, None] [a.completed_tokens for a in acks], [0, 0, None, None, 4, 8, None, None]
@@ -261,7 +263,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
cache.finish(ticket.handle, CacheRequestOutcome.ABORT) cache.finish(ticket.handle, CacheRequestOutcome.ABORT)
self.assertIs(c.pp_prefetch_decisions["hit"], PPPrefetchDecision.CANCELLED) self.assertIs(c.pp_prefetch_decisions["hit"], PPPrefetchDecision.CANCELLED)
self.run_commands(ticket) self.run_commands(ticket)
operation = c.prefetch_buffer.get_nowait() operation = c.prefetch_io_queues[0].get_nowait()
self.sync_acks(PrefetchAck("hit", operation, completed_tokens=4)) self.sync_acks(PrefetchAck("hit", operation, completed_tokens=4))
c.mem_pool_host.free.assert_not_called() c.mem_pool_host.free.assert_not_called()
self.sync_acks(PrefetchAck("hit", operation, completed_req=True)) self.sync_acks(PrefetchAck("hit", operation, completed_req=True))
@@ -293,7 +295,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
c.pp_prefetch_states.clear() c.pp_prefetch_states.clear()
c.mem_pool_host.alloc.reset_mock() c.mem_pool_host.alloc.reset_mock()
self.run_commands(ticket) self.run_commands(ticket)
operation = c.prefetch_buffer.get_nowait() operation = c.prefetch_io_queues[0].get_nowait()
self.assertEqual( self.assertEqual(
c.mem_pool_host.alloc.call_args_list, c.mem_pool_host.alloc.call_args_list,
[ [
@@ -311,7 +313,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
c.mem_pool_host.get_pool.side_effect = KeyError(PoolName.SWA) c.mem_pool_host.get_pool.side_effect = KeyError(PoolName.SWA)
with self.assertLogs(level="ERROR"): with self.assertLogs(level="ERROR"):
self.run_commands(ticket) self.run_commands(ticket)
self.assertTrue(c.prefetch_buffer.get_nowait().is_terminated()) self.assertTrue(c.prefetch_io_queues[0].get_nowait().is_terminated())
self.assertEqual(c.mem_pool_host.free.call_count, 1) self.assertEqual(c.mem_pool_host.free.call_count, 1)
self.assertEqual(c.mem_pool_host.free.call_args.kwargs["pool"], PoolName.KV) self.assertEqual(c.mem_pool_host.free.call_args.kwargs["pool"], PoolName.KV)
@@ -387,8 +389,8 @@ class TestPPPrefetchTicket(unittest.TestCase):
self.assertEqual(invocation.kwargs, {"src": 1}) self.assertEqual(invocation.kwargs, {"src": 1})
self.assertEqual(c.pp_prefetch_command_queue.unfinished_tasks, 0) self.assertEqual(c.pp_prefetch_command_queue.unfinished_tasks, 0)
self.assertTrue(c.pp_prefetch_command_queue.empty()) self.assertTrue(c.pp_prefetch_command_queue.empty())
self.assertIs(c.prefetch_buffer.get_nowait(), operation) self.assertIs(c.prefetch_io_queues[0].get_nowait(), operation)
self.assertTrue(c.prefetch_buffer.empty()) self.assertTrue(c.prefetch_io_queues[0].empty())
c.mem_pool_host.alloc.assert_called_once() c.mem_pool_host.alloc.assert_called_once()
def test_idle_downstream_skips_empty_broadcasts_before_ticket_and_stop(self): def test_idle_downstream_skips_empty_broadcasts_before_ticket_and_stop(self):
@@ -404,8 +406,8 @@ class TestPPPrefetchTicket(unittest.TestCase):
patch.object(c.pp_prefetch_command_queue, "task_done") as task_done, patch.object(c.pp_prefetch_command_queue, "task_done") as task_done,
): ):
c.pp_prefetch_command_thread_func() c.pp_prefetch_command_thread_func()
self.assertEqual(c.prefetch_buffer.get_nowait().request_id, "hit") self.assertEqual(c.prefetch_io_queues[0].get_nowait().request_id, "hit")
self.assertTrue(c.prefetch_buffer.empty()) self.assertTrue(c.prefetch_io_queues[0].empty())
c.mem_pool_host.alloc.assert_called_once() c.mem_pool_host.alloc.assert_called_once()
task_done.assert_not_called() task_done.assert_not_called()
@@ -3635,14 +3635,14 @@ class UnifiedRadixCacheSuite:
gate = threading.Event() gate = threading.Event()
swa_release_q = cc.extra_host_mem_release_queues.get(PoolName.SWA) swa_release_q = cc.extra_host_mem_release_queues.get(PoolName.SWA)
def _page_transfer_hook(_self, op): def _page_transfer_hook(_self, op, sync_queue):
# Signal that the worker has reached the sidecar step, then block # Signal that the worker has reached the sidecar step, then block
# until the main thread releases the gate. # until the main thread releases the gate.
entered.set() entered.set()
gate.wait(timeout=10.0) gate.wait(timeout=10.0)
# Forward to the real implementation so the worker runs the # Forward to the real implementation so the worker runs the
# terminate-aware release path itself. # terminate-aware release path itself.
original_sidecar(_self, op) original_sidecar(_self, op, sync_queue)
# Simulate a slow prefetch IO. Hook on _page_transfer. # Simulate a slow prefetch IO. Hook on _page_transfer.
with mock.patch.object( with mock.patch.object(
@@ -5290,7 +5290,7 @@ class UnifiedRadixCacheSuite:
cc = cache.cache_controller cc = cache.cache_controller
# Fake _reduce_prefetch_ack to drop SWA pool_hits when drop_swa is True. # Fake _reduce_prefetch_ack to drop SWA pool_hits when drop_swa is True.
def fake_reduce(ack): def fake_reduce(ack, groups=None):
if drop_swa and ack.pool_hits is not None: if drop_swa and ack.pool_hits is not None:
if PoolName.SWA.value in ack.pool_hits: if PoolName.SWA.value in ack.pool_hits:
ack.pool_hits[PoolName.SWA.value] = 0 ack.pool_hits[PoolName.SWA.value] = 0
@@ -37,10 +37,15 @@ class C_HiCacheController(BaseHook):
def wrapped_init(self, *args, **kwargs): def wrapped_init(self, *args, **kwargs):
self.sim_prefetch_buffer = Queue() self.sim_prefetch_buffer = Queue()
result = original_init(self, *args, **kwargs) result = original_init(self, *args, **kwargs)
# The real IO thread normally creates this queue. The simulator # The real IO threads normally create the handoff queues. The
# replaces that thread, so initialize the handoff queue here. # simulator replaces those threads, so initialize the per-lane
if hasattr(self, "prefetch_hit_queue"): # handoff queues here.
self.prefetch_buffer = Queue() if hasattr(self, "prefetch_hit_queue") and not getattr(
self, "prefetch_io_queues", None
):
self.prefetch_io_queues = [
Queue() for _ in range(getattr(self, "num_prefetch_io_lanes", 1))
]
return result return result
def wrapped_append_host_mem_release(self, host_indices): def wrapped_append_host_mem_release(self, host_indices):
@@ -182,11 +187,10 @@ class C_HiCacheController(BaseHook):
# Feed operations whose host pages were allocated by the scheduler # Feed operations whose host pages were allocated by the scheduler
# into the virtual-time transfer loop. # into the virtual-time transfer loop.
prefetch_buffer = getattr(self, "prefetch_buffer", None) for io_queue in getattr(self, "prefetch_io_queues", None) or []:
if prefetch_buffer is not None: while True:
while not prefetch_buffer.empty():
try: try:
operation = prefetch_buffer.get(block=False) operation = io_queue.get(block=False)
if operation is not None: if operation is not None:
self.sim_prefetch_buffer.put(operation) self.sim_prefetch_buffer.put(operation)
except Empty: except Empty:
@@ -295,10 +299,12 @@ class C_HybridCacheController(BaseHook):
def wrapped_init(self, *args, **kwargs): def wrapped_init(self, *args, **kwargs):
result = original_init(self, *args, **kwargs) result = original_init(self, *args, **kwargs)
if hasattr(self, "prefetch_hit_queue") and not hasattr( if hasattr(self, "prefetch_hit_queue") and not getattr(
self, "prefetch_buffer" self, "prefetch_io_queues", None
): ):
self.prefetch_buffer = Queue() self.prefetch_io_queues = [
Queue() for _ in range(getattr(self, "num_prefetch_io_lanes", 1))
]
return result return result
def wrapped_storage_hit_query(self, operator): def wrapped_storage_hit_query(self, operator):