hicache L3 prefetch: io_aux 传输并行化(按 op 分 lane)
build-sglang-image / build (push) Failing after 48m34s
build-sglang-image / build (push) Failing after 48m34s
打点实测单条 prefetch_io_aux_func 线程串行传输是所有请求的瓶颈: mc8 下每请求 xfer 排队 0->1.9s 线性增长(每请求 5 批 x ~68ms 固定开销, 介质无关)。本提交把传输层拆成 N 条并行 lane(默认 4, SGLANG_HICACHE_PREFETCH_IO_THREADS 可调): - dispatch_prefetch() 用 crc32(request_id) % N 分 lane(内置 hash 对 str 每进程加盐,跨 rank 不一致会导致 gloo 集合通信错位,禁用) - 同一 op 的所有 chunk 保持在同一 lane,保序 reduce 语义不变 - 每 lane 独立 io 队列 + io_aux 线程 + sync 队列 + sync 线程 + 独立 gloo completion 组(集合通信按 lane 过滤后的分发顺序保序) - query 阶段的 hit_reduce 仍在单 prefetch_thread,不动 - 生产点(hiradix/unified/hybrid pp-ticket)改调 dispatch_prefetch - 模拟器与两个单测文件同步适配 注意:SGLANG_HICACHE_PREFETCH_IO_THREADS 必须在实例内所有 TP/CP/PP rank 保持一致,否则 lane 组上的集合通信会错位挂死。
This commit is contained in:
@@ -763,6 +763,11 @@ class Envs:
|
|||||||
SGLANG_HICACHE_TIMING = EnvBool(False)
|
SGLANG_HICACHE_TIMING = EnvBool(False)
|
||||||
# Pages per batched storage IO call (query / get / put chunking).
|
# Pages per batched storage IO call (query / get / put chunking).
|
||||||
SGLANG_HICACHE_STORAGE_BATCH_SIZE = EnvInt(128)
|
SGLANG_HICACHE_STORAGE_BATCH_SIZE = EnvInt(128)
|
||||||
|
# Number of parallel prefetch IO lanes (worker threads) per controller.
|
||||||
|
# Operations are sharded by crc32(request_id) so every rank maps a request
|
||||||
|
# to the same lane; each lane owns dedicated completion sync groups.
|
||||||
|
# MUST be identical across all TP/CP/PP ranks of an instance.
|
||||||
|
SGLANG_HICACHE_PREFETCH_IO_THREADS = EnvInt(4)
|
||||||
SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR = EnvStr(None)
|
SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR = EnvStr(None)
|
||||||
# File-backend LRU eviction (opt-in; sizes accept SI/IEC suffixes, "0" disables).
|
# File-backend LRU eviction (opt-in; sizes accept SI/IEC suffixes, "0" disables).
|
||||||
SGLANG_HICACHE_FILE_BACKEND_MAX_SIZE = EnvStr(None)
|
SGLANG_HICACHE_FILE_BACKEND_MAX_SIZE = EnvStr(None)
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ limitations under the License.
|
|||||||
import logging
|
import logging
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
|
import zlib
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from queue import Empty, Queue
|
from queue import Empty, Queue
|
||||||
from typing import TYPE_CHECKING, Callable, List, NamedTuple, Optional
|
from typing import TYPE_CHECKING, Callable, List, NamedTuple, Optional
|
||||||
@@ -317,7 +318,18 @@ class HiCacheController:
|
|||||||
self.attn_tp_group = attn_tp_group
|
self.attn_tp_group = attn_tp_group
|
||||||
self.pp_group = pp_group
|
self.pp_group = pp_group
|
||||||
self.prefetch_hits_sync_groups: List[torch.distributed.ProcessGroup] = []
|
self.prefetch_hits_sync_groups: List[torch.distributed.ProcessGroup] = []
|
||||||
self.prefetch_completion_sync_groups: List[torch.distributed.ProcessGroup] = []
|
# One set of completion sync groups per prefetch IO lane. Each lane's
|
||||||
|
# worker/sync threads form a serial pipeline, so the collective
|
||||||
|
# sequence on a lane's groups is identical across ranks as long as
|
||||||
|
# lane assignment is deterministic (see dispatch_prefetch).
|
||||||
|
self.prefetch_completion_sync_groups_by_lane: List[
|
||||||
|
List[torch.distributed.ProcessGroup]
|
||||||
|
] = []
|
||||||
|
# Read once at construction; attach/reset reuse the stored value so
|
||||||
|
# lane count never changes underneath live sync groups.
|
||||||
|
self.num_prefetch_io_lanes = max(
|
||||||
|
1, _envs.SGLANG_HICACHE_PREFETCH_IO_THREADS.get()
|
||||||
|
)
|
||||||
self.mem_pool_device_allocator = token_to_kv_pool_allocator
|
self.mem_pool_device_allocator = token_to_kv_pool_allocator
|
||||||
mem_pool_device = token_to_kv_pool_allocator.get_kvcache()
|
mem_pool_device = token_to_kv_pool_allocator.get_kvcache()
|
||||||
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool
|
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool
|
||||||
@@ -346,8 +358,8 @@ class HiCacheController:
|
|||||||
self.storage_stop_event = threading.Event()
|
self.storage_stop_event = threading.Event()
|
||||||
|
|
||||||
# Storage control queues, (re)created whenever the storage threads start.
|
# Storage control queues, (re)created whenever the storage threads start.
|
||||||
self.prefetch_buffer: Optional[Queue[PrefetchOperation]] = None
|
self.prefetch_io_queues: List[Queue[PrefetchOperation]] = []
|
||||||
self.prefetch_sync_queue: Optional[Queue[PrefetchAck]] = None
|
self.prefetch_sync_queues: List[Queue[PrefetchAck]] = []
|
||||||
self.prefetch_hit_queue: Optional[Queue[StorageOperation]] = None
|
self.prefetch_hit_queue: Optional[Queue[StorageOperation]] = None
|
||||||
self.ack_prefetch_queue = Queue[PrefetchAck]()
|
self.ack_prefetch_queue = Queue[PrefetchAck]()
|
||||||
self.ack_backup_queue: Optional[Queue[StorageOperation]] = None
|
self.ack_backup_queue: Optional[Queue[StorageOperation]] = None
|
||||||
@@ -455,27 +467,39 @@ class HiCacheController:
|
|||||||
self.prefetch_thread = threading.Thread(
|
self.prefetch_thread = threading.Thread(
|
||||||
target=self.prefetch_thread_func, daemon=True
|
target=self.prefetch_thread_func, daemon=True
|
||||||
)
|
)
|
||||||
self.prefetch_io_aux_thread = threading.Thread(
|
self.prefetch_io_aux_threads = [
|
||||||
target=self.prefetch_io_aux_func, daemon=True
|
threading.Thread(
|
||||||
)
|
target=self.prefetch_io_aux_func, args=(lane,), daemon=True
|
||||||
self.prefetch_sync_thread = threading.Thread(
|
)
|
||||||
target=self.prefetch_sync_thread_func, daemon=True
|
for lane in range(self.num_prefetch_io_lanes)
|
||||||
)
|
]
|
||||||
|
self.prefetch_sync_threads = [
|
||||||
|
threading.Thread(
|
||||||
|
target=self.prefetch_sync_thread_func, args=(lane,), daemon=True
|
||||||
|
)
|
||||||
|
for lane in range(self.num_prefetch_io_lanes)
|
||||||
|
]
|
||||||
self.backup_thread = threading.Thread(
|
self.backup_thread = threading.Thread(
|
||||||
target=self.backup_thread_func, daemon=True
|
target=self.backup_thread_func, daemon=True
|
||||||
)
|
)
|
||||||
self.prefetch_queue = Queue()
|
self.prefetch_queue = Queue()
|
||||||
self.backup_queue = Queue()
|
self.backup_queue = Queue()
|
||||||
self.prefetch_buffer = Queue()
|
self.prefetch_io_queues = [
|
||||||
self.prefetch_sync_queue = Queue()
|
Queue() for _ in range(self.num_prefetch_io_lanes)
|
||||||
|
]
|
||||||
|
self.prefetch_sync_queues = [
|
||||||
|
Queue() for _ in range(self.num_prefetch_io_lanes)
|
||||||
|
]
|
||||||
self.prefetch_hit_queue = Queue()
|
self.prefetch_hit_queue = Queue()
|
||||||
self.ack_prefetch_queue = Queue()
|
self.ack_prefetch_queue = Queue()
|
||||||
self.ack_backup_queue = Queue()
|
self.ack_backup_queue = Queue()
|
||||||
self.host_mem_release_queue = Queue()
|
self.host_mem_release_queue = Queue()
|
||||||
|
|
||||||
self.prefetch_thread.start()
|
self.prefetch_thread.start()
|
||||||
self.prefetch_io_aux_thread.start()
|
for thread in self.prefetch_io_aux_threads:
|
||||||
self.prefetch_sync_thread.start()
|
thread.start()
|
||||||
|
for thread in self.prefetch_sync_threads:
|
||||||
|
thread.start()
|
||||||
self.backup_thread.start()
|
self.backup_thread.start()
|
||||||
|
|
||||||
def _stop_storage_threads(self):
|
def _stop_storage_threads(self):
|
||||||
@@ -496,10 +520,10 @@ class HiCacheController:
|
|||||||
self.prefetch_queue.put_nowait(None)
|
self.prefetch_queue.put_nowait(None)
|
||||||
if hasattr(self, "backup_queue"):
|
if hasattr(self, "backup_queue"):
|
||||||
self.backup_queue.put_nowait(None)
|
self.backup_queue.put_nowait(None)
|
||||||
if hasattr(self, "prefetch_buffer"):
|
for queue in getattr(self, "prefetch_io_queues", []) or []:
|
||||||
self.prefetch_buffer.put_nowait(None)
|
queue.put_nowait(None)
|
||||||
if hasattr(self, "prefetch_sync_queue"):
|
for queue in getattr(self, "prefetch_sync_queues", []) or []:
|
||||||
self.prefetch_sync_queue.put_nowait(None)
|
queue.put_nowait(None)
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@@ -509,10 +533,8 @@ class HiCacheController:
|
|||||||
threads.append(self.prefetch_thread)
|
threads.append(self.prefetch_thread)
|
||||||
if hasattr(self, "backup_thread"):
|
if hasattr(self, "backup_thread"):
|
||||||
threads.append(self.backup_thread)
|
threads.append(self.backup_thread)
|
||||||
if hasattr(self, "prefetch_io_aux_thread"):
|
threads.extend(getattr(self, "prefetch_io_aux_threads", []) or [])
|
||||||
threads.append(self.prefetch_io_aux_thread)
|
threads.extend(getattr(self, "prefetch_sync_threads", []) or [])
|
||||||
if hasattr(self, "prefetch_sync_thread"):
|
|
||||||
threads.append(self.prefetch_sync_thread)
|
|
||||||
|
|
||||||
for t in threads:
|
for t in threads:
|
||||||
try:
|
try:
|
||||||
@@ -596,7 +618,13 @@ class HiCacheController:
|
|||||||
# Use dedicated gloo groups so storage prefetch sync is isolated
|
# Use dedicated gloo groups so storage prefetch sync is isolated
|
||||||
# from other collectives and consistent across CPxTP participants.
|
# from other collectives and consistent across CPxTP participants.
|
||||||
self.prefetch_hits_sync_groups = self._create_sync_groups()
|
self.prefetch_hits_sync_groups = self._create_sync_groups()
|
||||||
self.prefetch_completion_sync_groups = self._create_sync_groups()
|
# Each IO lane gets its own completion groups: per-lane collectives
|
||||||
|
# stay ordered because a lane is a serial pipeline and dispatch is
|
||||||
|
# deterministic (crc32(request_id) % num_prefetch_io_lanes).
|
||||||
|
self.prefetch_completion_sync_groups_by_lane = [
|
||||||
|
self._create_sync_groups()
|
||||||
|
for _ in range(self.num_prefetch_io_lanes)
|
||||||
|
]
|
||||||
|
|
||||||
# Select the get and set functions
|
# Select the get and set functions
|
||||||
self.page_get_func = self._generic_page_get
|
self.page_get_func = self._generic_page_get
|
||||||
@@ -631,9 +659,10 @@ class HiCacheController:
|
|||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
self._destroy_sync_groups(self.prefetch_hits_sync_groups)
|
self._destroy_sync_groups(self.prefetch_hits_sync_groups)
|
||||||
self._destroy_sync_groups(self.prefetch_completion_sync_groups)
|
for lane_groups in self.prefetch_completion_sync_groups_by_lane:
|
||||||
|
self._destroy_sync_groups(lane_groups)
|
||||||
self.prefetch_hits_sync_groups = []
|
self.prefetch_hits_sync_groups = []
|
||||||
self.prefetch_completion_sync_groups = []
|
self.prefetch_completion_sync_groups_by_lane = []
|
||||||
try:
|
try:
|
||||||
if (
|
if (
|
||||||
hasattr(self, "storage_backend")
|
hasattr(self, "storage_backend")
|
||||||
@@ -671,11 +700,11 @@ class HiCacheController:
|
|||||||
raise RuntimeError("Stop storage threads failed; detach aborted.") from e
|
raise RuntimeError("Stop storage threads failed; detach aborted.") from e
|
||||||
|
|
||||||
# Best-effort destroy process groups created for storage ops.
|
# Best-effort destroy process groups created for storage ops.
|
||||||
self._destroy_sync_groups(
|
self._destroy_sync_groups(self.prefetch_hits_sync_groups)
|
||||||
self.prefetch_hits_sync_groups + self.prefetch_completion_sync_groups
|
for lane_groups in self.prefetch_completion_sync_groups_by_lane:
|
||||||
)
|
self._destroy_sync_groups(lane_groups)
|
||||||
self.prefetch_hits_sync_groups = []
|
self.prefetch_hits_sync_groups = []
|
||||||
self.prefetch_completion_sync_groups = []
|
self.prefetch_completion_sync_groups_by_lane = []
|
||||||
|
|
||||||
# Best-effort close (some backends rely on GC/destructor).
|
# Best-effort close (some backends rely on GC/destructor).
|
||||||
try:
|
try:
|
||||||
@@ -769,13 +798,17 @@ class HiCacheController:
|
|||||||
self.ack_load_queue.clear()
|
self.ack_load_queue.clear()
|
||||||
if self.enable_storage:
|
if self.enable_storage:
|
||||||
self.prefetch_thread.join()
|
self.prefetch_thread.join()
|
||||||
self.prefetch_io_aux_thread.join()
|
for thread in self.prefetch_io_aux_threads:
|
||||||
self.prefetch_sync_thread.join()
|
thread.join()
|
||||||
|
for thread in self.prefetch_sync_threads:
|
||||||
|
thread.join()
|
||||||
self.backup_thread.join()
|
self.backup_thread.join()
|
||||||
self.prefetch_queue.queue.clear()
|
self.prefetch_queue.queue.clear()
|
||||||
self.backup_queue.queue.clear()
|
self.backup_queue.queue.clear()
|
||||||
self.prefetch_buffer.queue.clear()
|
for queue in self.prefetch_io_queues:
|
||||||
self.prefetch_sync_queue.queue.clear()
|
queue.queue.clear()
|
||||||
|
for queue in self.prefetch_sync_queues:
|
||||||
|
queue.queue.clear()
|
||||||
self.prefetch_hit_queue.queue.clear()
|
self.prefetch_hit_queue.queue.clear()
|
||||||
self.ack_prefetch_queue.queue.clear()
|
self.ack_prefetch_queue.queue.clear()
|
||||||
self.ack_backup_queue.queue.clear()
|
self.ack_backup_queue.queue.clear()
|
||||||
@@ -788,18 +821,26 @@ class HiCacheController:
|
|||||||
self.prefetch_thread = threading.Thread(
|
self.prefetch_thread = threading.Thread(
|
||||||
target=self.prefetch_thread_func, daemon=True
|
target=self.prefetch_thread_func, daemon=True
|
||||||
)
|
)
|
||||||
self.prefetch_io_aux_thread = threading.Thread(
|
self.prefetch_io_aux_threads = [
|
||||||
target=self.prefetch_io_aux_func, daemon=True
|
threading.Thread(
|
||||||
)
|
target=self.prefetch_io_aux_func, args=(lane,), daemon=True
|
||||||
self.prefetch_sync_thread = threading.Thread(
|
)
|
||||||
target=self.prefetch_sync_thread_func, daemon=True
|
for lane in range(self.num_prefetch_io_lanes)
|
||||||
)
|
]
|
||||||
|
self.prefetch_sync_threads = [
|
||||||
|
threading.Thread(
|
||||||
|
target=self.prefetch_sync_thread_func, args=(lane,), daemon=True
|
||||||
|
)
|
||||||
|
for lane in range(self.num_prefetch_io_lanes)
|
||||||
|
]
|
||||||
self.backup_thread = threading.Thread(
|
self.backup_thread = threading.Thread(
|
||||||
target=self.backup_thread_func, daemon=True
|
target=self.backup_thread_func, daemon=True
|
||||||
)
|
)
|
||||||
self.prefetch_thread.start()
|
self.prefetch_thread.start()
|
||||||
self.prefetch_io_aux_thread.start()
|
for thread in self.prefetch_io_aux_threads:
|
||||||
self.prefetch_sync_thread.start()
|
thread.start()
|
||||||
|
for thread in self.prefetch_sync_threads:
|
||||||
|
thread.start()
|
||||||
self.backup_thread.start()
|
self.backup_thread.start()
|
||||||
|
|
||||||
def has_inflight_device_transfers(self) -> bool:
|
def has_inflight_device_transfers(self) -> bool:
|
||||||
@@ -1086,7 +1127,23 @@ class HiCacheController:
|
|||||||
count += 1
|
count += 1
|
||||||
return count
|
return count
|
||||||
|
|
||||||
def _page_transfer(self, operation: PrefetchOperation) -> int:
|
def dispatch_prefetch(self, operation: PrefetchOperation) -> None:
|
||||||
|
"""Route a prefetch operation to its IO lane.
|
||||||
|
|
||||||
|
Lane assignment must be deterministic and identical on every rank:
|
||||||
|
each lane owns dedicated completion sync groups, and the collective
|
||||||
|
sequence on a lane is the filtered dispatch order. crc32 (not the
|
||||||
|
per-process salted builtin hash) keeps the mapping stable across
|
||||||
|
processes. All chunks of one operation stay in one lane, preserving
|
||||||
|
the per-operation ack order the sync thread reduces over.
|
||||||
|
"""
|
||||||
|
lane = (
|
||||||
|
zlib.crc32(str(operation.request_id).encode())
|
||||||
|
% self.num_prefetch_io_lanes
|
||||||
|
)
|
||||||
|
self.prefetch_io_queues[lane].put(operation)
|
||||||
|
|
||||||
|
def _page_transfer(self, operation: PrefetchOperation, sync_queue: Queue) -> int:
|
||||||
# Transfer batch by batch
|
# Transfer batch by batch
|
||||||
prefix_keys = operation.prefix_keys
|
prefix_keys = operation.prefix_keys
|
||||||
kv_derived_transfers = [
|
kv_derived_transfers = [
|
||||||
@@ -1098,8 +1155,8 @@ class HiCacheController:
|
|||||||
completed_pages = 0
|
completed_pages = 0
|
||||||
for i in range(0, len(operation.hash_value), STORAGE_BATCH_SIZE):
|
for i in range(0, len(operation.hash_value), STORAGE_BATCH_SIZE):
|
||||||
# When an error is occurred, we should keep looping and produce the same number of
|
# When an error is occurred, we should keep looping and produce the same number of
|
||||||
# PrefetchAck as other ranks do, because prefetch_sync_thread (i.e. consumer of
|
# PrefetchAck as other ranks do, because the prefetch sync thread (i.e. consumer of
|
||||||
# prefetch_sync_queue) perform reduce on the results. This is so tricky.
|
# the lane sync queue) perform reduce on the results. This is so tricky.
|
||||||
if all_success and operation.is_terminated():
|
if all_success and operation.is_terminated():
|
||||||
all_success = False
|
all_success = False
|
||||||
if all_success:
|
if all_success:
|
||||||
@@ -1138,7 +1195,7 @@ class HiCacheController:
|
|||||||
completed_tokens=completed_pages * self.page_size,
|
completed_tokens=completed_pages * self.page_size,
|
||||||
operation=operation,
|
operation=operation,
|
||||||
)
|
)
|
||||||
self.prefetch_sync_queue.put(ack)
|
sync_queue.put(ack)
|
||||||
return completed_pages
|
return completed_pages
|
||||||
|
|
||||||
def _page_transfer_kv_batch(
|
def _page_transfer_kv_batch(
|
||||||
@@ -1180,27 +1237,33 @@ class HiCacheController:
|
|||||||
# Clamp to minimal number of hits.
|
# Clamp to minimal number of hits.
|
||||||
return min([kv_hits, *sidecar_hits.values()])
|
return min([kv_hits, *sidecar_hits.values()])
|
||||||
|
|
||||||
def prefetch_io_aux_func(self):
|
def prefetch_io_aux_func(self, lane: int = 0):
|
||||||
"""
|
"""
|
||||||
Auxiliary function conducting IO operations for prefetching.
|
Auxiliary function conducting IO operations for prefetching.
|
||||||
|
|
||||||
|
Each lane has its own worker thread and queues; an operation's whole
|
||||||
|
chunk sequence stays on one lane so the sync thread sees it in order.
|
||||||
"""
|
"""
|
||||||
|
io_queue = self.prefetch_io_queues[lane]
|
||||||
|
sync_queue = self.prefetch_sync_queues[lane]
|
||||||
while not self.storage_stop_event.is_set():
|
while not self.storage_stop_event.is_set():
|
||||||
try:
|
try:
|
||||||
operation = self.prefetch_buffer.get(block=True, timeout=1)
|
operation = io_queue.get(block=True, timeout=1)
|
||||||
if operation is None:
|
if operation is None:
|
||||||
continue
|
continue
|
||||||
_hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0
|
_hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||||
self._page_transfer(operation)
|
self._page_transfer(operation, sync_queue)
|
||||||
if _hc_timing_enabled():
|
if _hc_timing_enabled():
|
||||||
logger.info(
|
logger.info(
|
||||||
"HC_TIMING stage=xfer req=%s wait_ms=%.1f xfer_ms=%.1f",
|
"HC_TIMING stage=xfer req=%s lane=%d wait_ms=%.1f xfer_ms=%.1f",
|
||||||
operation.request_id,
|
operation.request_id,
|
||||||
|
lane,
|
||||||
(_hc_t0 - getattr(operation, "query_done_ts", operation.start_time))
|
(_hc_t0 - getattr(operation, "query_done_ts", operation.start_time))
|
||||||
* 1e3,
|
* 1e3,
|
||||||
(time.monotonic() - _hc_t0) * 1e3,
|
(time.monotonic() - _hc_t0) * 1e3,
|
||||||
)
|
)
|
||||||
|
|
||||||
self.prefetch_sync_queue.put(
|
sync_queue.put(
|
||||||
PrefetchAck(
|
PrefetchAck(
|
||||||
rid=operation.request_id,
|
rid=operation.request_id,
|
||||||
completed_req=True,
|
completed_req=True,
|
||||||
@@ -1381,21 +1444,25 @@ class HiCacheController:
|
|||||||
except Empty:
|
except Empty:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
def prefetch_sync_thread_func(self):
|
def prefetch_sync_thread_func(self, lane: int = 0):
|
||||||
"""Synchronize prefetch results across all PP and TP ranks."""
|
"""Synchronize prefetch results across all PP and TP ranks."""
|
||||||
|
sync_queue = self.prefetch_sync_queues[lane]
|
||||||
|
groups = self.prefetch_completion_sync_groups_by_lane[lane]
|
||||||
while not self.storage_stop_event.is_set():
|
while not self.storage_stop_event.is_set():
|
||||||
try:
|
try:
|
||||||
ack = self.prefetch_sync_queue.get(block=True, timeout=1)
|
ack = sync_queue.get(block=True, timeout=1)
|
||||||
if ack is None:
|
if ack is None:
|
||||||
continue
|
continue
|
||||||
self._reduce_prefetch_ack(ack)
|
self._reduce_prefetch_ack(ack, groups)
|
||||||
if _hc_timing_enabled() and ack.completed_req:
|
if _hc_timing_enabled() and ack.completed_req:
|
||||||
ack.operation.final_ack_enqueue_ts = time.monotonic()
|
ack.operation.final_ack_enqueue_ts = time.monotonic()
|
||||||
self.ack_prefetch_queue.put(ack)
|
self.ack_prefetch_queue.put(ack)
|
||||||
except Empty:
|
except Empty:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
def _reduce_prefetch_ack(self, ack: PrefetchAck) -> None:
|
def _reduce_prefetch_ack(
|
||||||
|
self, ack: PrefetchAck, groups: List[torch.distributed.ProcessGroup]
|
||||||
|
) -> None:
|
||||||
"""Synchronize all ranks to agree on a PrefetchAck."""
|
"""Synchronize all ranks to agree on a PrefetchAck."""
|
||||||
if ack.completed_tokens is not None:
|
if ack.completed_tokens is not None:
|
||||||
# Determine the minimal successful prefix of tokens.
|
# Determine the minimal successful prefix of tokens.
|
||||||
@@ -1406,7 +1473,7 @@ class HiCacheController:
|
|||||||
self._all_reduce(
|
self._all_reduce(
|
||||||
completed_tokens_tensor,
|
completed_tokens_tensor,
|
||||||
torch.distributed.ReduceOp.MIN,
|
torch.distributed.ReduceOp.MIN,
|
||||||
self.prefetch_completion_sync_groups,
|
groups,
|
||||||
)
|
)
|
||||||
ack.completed_tokens = completed_tokens_tensor.item()
|
ack.completed_tokens = completed_tokens_tensor.item()
|
||||||
if _hc_timing_enabled():
|
if _hc_timing_enabled():
|
||||||
|
|||||||
@@ -653,7 +653,7 @@ class HiRadixCache(RadixCache):
|
|||||||
: alloc_len // self.page_size
|
: alloc_len // self.page_size
|
||||||
]
|
]
|
||||||
operation.host_indices = host_indices
|
operation.host_indices = host_indices
|
||||||
cc.prefetch_buffer.put(operation)
|
cc.dispatch_prefetch(operation)
|
||||||
|
|
||||||
def _drain_ack_prefetch():
|
def _drain_ack_prefetch():
|
||||||
for ack in _drain_queue(cc.ack_prefetch_queue, n_ack_prefetch):
|
for ack in _drain_queue(cc.ack_prefetch_queue, n_ack_prefetch):
|
||||||
|
|||||||
@@ -1006,7 +1006,7 @@ class HybridCacheController(BaseHiCacheController):
|
|||||||
if not self._allocate_pp_prefetch_buffers(ticket, operation):
|
if not self._allocate_pp_prefetch_buffers(ticket, operation):
|
||||||
operation.host_indices = torch.empty(0, dtype=torch.int64)
|
operation.host_indices = torch.empty(0, dtype=torch.int64)
|
||||||
operation.mark_terminate()
|
operation.mark_terminate()
|
||||||
self.prefetch_buffer.put(operation)
|
self.dispatch_prefetch(operation)
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.exception("PP HiCache ticket processing failed.")
|
logger.exception("PP HiCache ticket processing failed.")
|
||||||
if operation is None or not operation.hash_value:
|
if operation is None or not operation.hash_value:
|
||||||
@@ -1016,7 +1016,7 @@ class HybridCacheController(BaseHiCacheController):
|
|||||||
operation.host_indices = torch.empty(0, dtype=torch.int64)
|
operation.host_indices = torch.empty(0, dtype=torch.int64)
|
||||||
operation.mark_terminate()
|
operation.mark_terminate()
|
||||||
# Preserve every KV/sidecar ACK even when local allocation fails.
|
# Preserve every KV/sidecar ACK even when local allocation fails.
|
||||||
self.prefetch_buffer.put(operation)
|
self.dispatch_prefetch(operation)
|
||||||
finally:
|
finally:
|
||||||
if is_source and objects:
|
if is_source and objects:
|
||||||
self.pp_prefetch_command_queue.task_done()
|
self.pp_prefetch_command_queue.task_done()
|
||||||
@@ -1104,15 +1104,15 @@ class HybridCacheController(BaseHiCacheController):
|
|||||||
)
|
)
|
||||||
return host_indices, device_indices, resolved_pool_transfers
|
return host_indices, device_indices, resolved_pool_transfers
|
||||||
|
|
||||||
def _page_transfer(self, operation: PrefetchOperation) -> bool:
|
def _page_transfer(self, operation: PrefetchOperation, sync_queue: Queue) -> bool:
|
||||||
# KV pools and KV-derived pools first — determines actual completed page count
|
# KV pools and KV-derived pools first — determines actual completed page count
|
||||||
kv_completed_pages = super()._page_transfer(operation)
|
kv_completed_pages = super()._page_transfer(operation, sync_queue)
|
||||||
|
|
||||||
# Read non-KV derived sidecar pool, e.g. SWA, Mamba.
|
# Read non-KV derived sidecar pool, e.g. SWA, Mamba.
|
||||||
self._page_transfer_sidecar(operation, kv_completed_pages)
|
self._page_transfer_sidecar(operation, kv_completed_pages, sync_queue)
|
||||||
|
|
||||||
def _page_transfer_sidecar(
|
def _page_transfer_sidecar(
|
||||||
self, operation: PrefetchOperation, kv_completed_pages: int
|
self, operation: PrefetchOperation, kv_completed_pages: int, sync_queue: Queue
|
||||||
) -> None:
|
) -> None:
|
||||||
if operation.pool_transfers is None:
|
if operation.pool_transfers is None:
|
||||||
return
|
return
|
||||||
@@ -1144,10 +1144,10 @@ class HybridCacheController(BaseHiCacheController):
|
|||||||
transfers_nonkv, extra_info=extra_info
|
transfers_nonkv, extra_info=extra_info
|
||||||
)
|
)
|
||||||
pool_hits = count_pool_hits(results)
|
pool_hits = count_pool_hits(results)
|
||||||
# Emit PrefetchAck to prefetch_sync_queue, even the operation has been canceled by the
|
# Emit PrefetchAck to the lane sync queue, even the operation has been canceled by the
|
||||||
# scheduler thread. The prefetch sync thread expects the same number of PrefetchAck objects
|
# scheduler thread. The prefetch sync thread expects the same number of PrefetchAck objects
|
||||||
# to perform all_reduce.
|
# to perform all_reduce.
|
||||||
self.prefetch_sync_queue.put(
|
sync_queue.put(
|
||||||
PrefetchAck(
|
PrefetchAck(
|
||||||
rid=operation.request_id,
|
rid=operation.request_id,
|
||||||
operation=operation,
|
operation=operation,
|
||||||
@@ -1421,9 +1421,11 @@ class HybridCacheController(BaseHiCacheController):
|
|||||||
pool.device_indices = source.device_indices
|
pool.device_indices = source.device_indices
|
||||||
return extra_pools
|
return extra_pools
|
||||||
|
|
||||||
def _reduce_prefetch_ack(self, ack: PrefetchAck) -> None:
|
def _reduce_prefetch_ack(
|
||||||
|
self, ack: PrefetchAck, groups: List[torch.distributed.ProcessGroup]
|
||||||
|
) -> None:
|
||||||
# Handle KV-derived pool.
|
# Handle KV-derived pool.
|
||||||
super()._reduce_prefetch_ack(ack)
|
super()._reduce_prefetch_ack(ack, groups)
|
||||||
|
|
||||||
# Handle other sidecar pools, e.g. SWA, Mamba.
|
# Handle other sidecar pools, e.g. SWA, Mamba.
|
||||||
if ack.pool_hits is not None:
|
if ack.pool_hits is not None:
|
||||||
@@ -1436,19 +1438,21 @@ class HybridCacheController(BaseHiCacheController):
|
|||||||
self._all_reduce(
|
self._all_reduce(
|
||||||
packed,
|
packed,
|
||||||
torch.distributed.ReduceOp.MIN,
|
torch.distributed.ReduceOp.MIN,
|
||||||
self.prefetch_completion_sync_groups,
|
groups,
|
||||||
)
|
)
|
||||||
for i, pool in enumerate(PoolName):
|
for i, pool in enumerate(PoolName):
|
||||||
ack.pool_hits[pool.value] = packed[i].item()
|
ack.pool_hits[pool.value] = packed[i].item()
|
||||||
|
|
||||||
def prefetch_sync_thread_func(self) -> None:
|
def prefetch_sync_thread_func(self, lane: int = 0) -> None:
|
||||||
"""Synchronize progressive ACKs and publish PP ticket completion."""
|
"""Synchronize progressive ACKs and publish PP ticket completion."""
|
||||||
|
sync_queue = self.prefetch_sync_queues[lane]
|
||||||
|
groups = self.prefetch_completion_sync_groups_by_lane[lane]
|
||||||
while not self.storage_stop_event.is_set():
|
while not self.storage_stop_event.is_set():
|
||||||
try:
|
try:
|
||||||
ack = self.prefetch_sync_queue.get(block=True, timeout=1)
|
ack = sync_queue.get(block=True, timeout=1)
|
||||||
if ack is None:
|
if ack is None:
|
||||||
continue
|
continue
|
||||||
self._reduce_prefetch_ack(ack)
|
self._reduce_prefetch_ack(ack, groups)
|
||||||
if not getattr(ack.operation, "is_pp_broadcast", False):
|
if not getattr(ack.operation, "is_pp_broadcast", False):
|
||||||
if _hc_timing_enabled() and ack.completed_req:
|
if _hc_timing_enabled() and ack.completed_req:
|
||||||
ack.operation.final_ack_enqueue_ts = time.monotonic()
|
ack.operation.final_ack_enqueue_ts = time.monotonic()
|
||||||
|
|||||||
@@ -3001,7 +3001,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
|||||||
self.ongoing_prefetch[request] = info._replace(host_indices=host_indices)
|
self.ongoing_prefetch[request] = info._replace(host_indices=host_indices)
|
||||||
if buffer_mode:
|
if buffer_mode:
|
||||||
cc.prefetch_tokens_occupied += alloc_len
|
cc.prefetch_tokens_occupied += alloc_len
|
||||||
cc.prefetch_buffer.put(operation)
|
cc.dispatch_prefetch(operation)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def _drain_and_alloc_storage_hit():
|
def _drain_and_alloc_storage_hit():
|
||||||
|
|||||||
@@ -37,15 +37,17 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
|||||||
c.pp_size, c.tp_size = 4, 2
|
c.pp_size, c.tp_size = 4, 2
|
||||||
c.pp_group, c.pp_prefetch_command_group = "pp", "command"
|
c.pp_group, c.pp_prefetch_command_group = "pp", "command"
|
||||||
c.pp_prefetch_command_thread = None
|
c.pp_prefetch_command_thread = None
|
||||||
c.prefetch_hits_sync_groups = c.prefetch_completion_sync_groups = ["pp", "tp"]
|
c.prefetch_hits_sync_groups = ["pp", "tp"]
|
||||||
|
c.num_prefetch_io_lanes = 1
|
||||||
|
c.prefetch_completion_sync_groups_by_lane = [["pp", "tp"]]
|
||||||
c.pp_prefetch_states, c.pp_prefetch_decisions = {}, {}
|
c.pp_prefetch_states, c.pp_prefetch_decisions = {}, {}
|
||||||
c.pp_prefetch_state_lock = threading.Lock()
|
c.pp_prefetch_state_lock = threading.Lock()
|
||||||
c.pp_prefetch_command_queue = Queue()
|
c.pp_prefetch_command_queue = Queue()
|
||||||
c.prefetch_queue = Queue()
|
c.prefetch_queue = Queue()
|
||||||
c.prefetch_sync_queue = Queue()
|
c.prefetch_sync_queues = [Queue()]
|
||||||
c.ack_prefetch_queue = Queue()
|
c.ack_prefetch_queue = Queue()
|
||||||
c.host_mem_release_queue = Queue()
|
c.host_mem_release_queue = Queue()
|
||||||
c.prefetch_buffer = Queue()
|
c.prefetch_io_queues = [Queue()]
|
||||||
c.storage_stop_event = threading.Event()
|
c.storage_stop_event = threading.Event()
|
||||||
c.prefetch_tokens_occupied = 0
|
c.prefetch_tokens_occupied = 0
|
||||||
c.mem_pool_host = Mock(page_size=4)
|
c.mem_pool_host = Mock(page_size=4)
|
||||||
@@ -103,7 +105,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
|||||||
|
|
||||||
def sync_acks(self, *acks):
|
def sync_acks(self, *acks):
|
||||||
for ack in acks:
|
for ack in acks:
|
||||||
self.c.prefetch_sync_queue.put(ack)
|
self.c.prefetch_sync_queues[0].put(ack)
|
||||||
with patch.object(
|
with patch.object(
|
||||||
self.c.storage_stop_event,
|
self.c.storage_stop_event,
|
||||||
"is_set",
|
"is_set",
|
||||||
@@ -183,7 +185,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
|||||||
self.assertFalse(cache.check_prefetch_progress(handle))
|
self.assertFalse(cache.check_prefetch_progress(handle))
|
||||||
c._storage_hit_query.reset_mock()
|
c._storage_hit_query.reset_mock()
|
||||||
self.run_commands(ticket)
|
self.run_commands(ticket)
|
||||||
operation = c.prefetch_buffer.get_nowait()
|
operation = c.prefetch_io_queues[0].get_nowait()
|
||||||
self.assertEqual(operation.handle, handle)
|
self.assertEqual(operation.handle, handle)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
operation.hash_value, get_storage_hash_str(ticket.prefetch_key, page_size=4)
|
operation.hash_value, get_storage_hash_str(ticket.prefetch_key, page_size=4)
|
||||||
@@ -237,7 +239,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
|||||||
),
|
),
|
||||||
):
|
):
|
||||||
c.prefetch_io_aux_func()
|
c.prefetch_io_aux_func()
|
||||||
acks = [c.prefetch_sync_queue.get_nowait() for _ in range(8)]
|
acks = [c.prefetch_sync_queues[0].get_nowait() for _ in range(8)]
|
||||||
self.assertEqual([a.rid for a in acks], ["hit"] * 4 + ["next"] * 4)
|
self.assertEqual([a.rid for a in acks], ["hit"] * 4 + ["next"] * 4)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
[a.completed_tokens for a in acks], [0, 0, None, None, 4, 8, None, None]
|
[a.completed_tokens for a in acks], [0, 0, None, None, 4, 8, None, None]
|
||||||
@@ -261,7 +263,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
|||||||
cache.finish(ticket.handle, CacheRequestOutcome.ABORT)
|
cache.finish(ticket.handle, CacheRequestOutcome.ABORT)
|
||||||
self.assertIs(c.pp_prefetch_decisions["hit"], PPPrefetchDecision.CANCELLED)
|
self.assertIs(c.pp_prefetch_decisions["hit"], PPPrefetchDecision.CANCELLED)
|
||||||
self.run_commands(ticket)
|
self.run_commands(ticket)
|
||||||
operation = c.prefetch_buffer.get_nowait()
|
operation = c.prefetch_io_queues[0].get_nowait()
|
||||||
self.sync_acks(PrefetchAck("hit", operation, completed_tokens=4))
|
self.sync_acks(PrefetchAck("hit", operation, completed_tokens=4))
|
||||||
c.mem_pool_host.free.assert_not_called()
|
c.mem_pool_host.free.assert_not_called()
|
||||||
self.sync_acks(PrefetchAck("hit", operation, completed_req=True))
|
self.sync_acks(PrefetchAck("hit", operation, completed_req=True))
|
||||||
@@ -293,7 +295,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
|||||||
c.pp_prefetch_states.clear()
|
c.pp_prefetch_states.clear()
|
||||||
c.mem_pool_host.alloc.reset_mock()
|
c.mem_pool_host.alloc.reset_mock()
|
||||||
self.run_commands(ticket)
|
self.run_commands(ticket)
|
||||||
operation = c.prefetch_buffer.get_nowait()
|
operation = c.prefetch_io_queues[0].get_nowait()
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
c.mem_pool_host.alloc.call_args_list,
|
c.mem_pool_host.alloc.call_args_list,
|
||||||
[
|
[
|
||||||
@@ -311,7 +313,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
|||||||
c.mem_pool_host.get_pool.side_effect = KeyError(PoolName.SWA)
|
c.mem_pool_host.get_pool.side_effect = KeyError(PoolName.SWA)
|
||||||
with self.assertLogs(level="ERROR"):
|
with self.assertLogs(level="ERROR"):
|
||||||
self.run_commands(ticket)
|
self.run_commands(ticket)
|
||||||
self.assertTrue(c.prefetch_buffer.get_nowait().is_terminated())
|
self.assertTrue(c.prefetch_io_queues[0].get_nowait().is_terminated())
|
||||||
self.assertEqual(c.mem_pool_host.free.call_count, 1)
|
self.assertEqual(c.mem_pool_host.free.call_count, 1)
|
||||||
self.assertEqual(c.mem_pool_host.free.call_args.kwargs["pool"], PoolName.KV)
|
self.assertEqual(c.mem_pool_host.free.call_args.kwargs["pool"], PoolName.KV)
|
||||||
|
|
||||||
@@ -387,8 +389,8 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
|||||||
self.assertEqual(invocation.kwargs, {"src": 1})
|
self.assertEqual(invocation.kwargs, {"src": 1})
|
||||||
self.assertEqual(c.pp_prefetch_command_queue.unfinished_tasks, 0)
|
self.assertEqual(c.pp_prefetch_command_queue.unfinished_tasks, 0)
|
||||||
self.assertTrue(c.pp_prefetch_command_queue.empty())
|
self.assertTrue(c.pp_prefetch_command_queue.empty())
|
||||||
self.assertIs(c.prefetch_buffer.get_nowait(), operation)
|
self.assertIs(c.prefetch_io_queues[0].get_nowait(), operation)
|
||||||
self.assertTrue(c.prefetch_buffer.empty())
|
self.assertTrue(c.prefetch_io_queues[0].empty())
|
||||||
c.mem_pool_host.alloc.assert_called_once()
|
c.mem_pool_host.alloc.assert_called_once()
|
||||||
|
|
||||||
def test_idle_downstream_skips_empty_broadcasts_before_ticket_and_stop(self):
|
def test_idle_downstream_skips_empty_broadcasts_before_ticket_and_stop(self):
|
||||||
@@ -404,8 +406,8 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
|||||||
patch.object(c.pp_prefetch_command_queue, "task_done") as task_done,
|
patch.object(c.pp_prefetch_command_queue, "task_done") as task_done,
|
||||||
):
|
):
|
||||||
c.pp_prefetch_command_thread_func()
|
c.pp_prefetch_command_thread_func()
|
||||||
self.assertEqual(c.prefetch_buffer.get_nowait().request_id, "hit")
|
self.assertEqual(c.prefetch_io_queues[0].get_nowait().request_id, "hit")
|
||||||
self.assertTrue(c.prefetch_buffer.empty())
|
self.assertTrue(c.prefetch_io_queues[0].empty())
|
||||||
c.mem_pool_host.alloc.assert_called_once()
|
c.mem_pool_host.alloc.assert_called_once()
|
||||||
task_done.assert_not_called()
|
task_done.assert_not_called()
|
||||||
|
|
||||||
|
|||||||
@@ -3635,14 +3635,14 @@ class UnifiedRadixCacheSuite:
|
|||||||
gate = threading.Event()
|
gate = threading.Event()
|
||||||
swa_release_q = cc.extra_host_mem_release_queues.get(PoolName.SWA)
|
swa_release_q = cc.extra_host_mem_release_queues.get(PoolName.SWA)
|
||||||
|
|
||||||
def _page_transfer_hook(_self, op):
|
def _page_transfer_hook(_self, op, sync_queue):
|
||||||
# Signal that the worker has reached the sidecar step, then block
|
# Signal that the worker has reached the sidecar step, then block
|
||||||
# until the main thread releases the gate.
|
# until the main thread releases the gate.
|
||||||
entered.set()
|
entered.set()
|
||||||
gate.wait(timeout=10.0)
|
gate.wait(timeout=10.0)
|
||||||
# Forward to the real implementation so the worker runs the
|
# Forward to the real implementation so the worker runs the
|
||||||
# terminate-aware release path itself.
|
# terminate-aware release path itself.
|
||||||
original_sidecar(_self, op)
|
original_sidecar(_self, op, sync_queue)
|
||||||
|
|
||||||
# Simulate a slow prefetch IO. Hook on _page_transfer.
|
# Simulate a slow prefetch IO. Hook on _page_transfer.
|
||||||
with mock.patch.object(
|
with mock.patch.object(
|
||||||
@@ -5290,7 +5290,7 @@ class UnifiedRadixCacheSuite:
|
|||||||
cc = cache.cache_controller
|
cc = cache.cache_controller
|
||||||
|
|
||||||
# Fake _reduce_prefetch_ack to drop SWA pool_hits when drop_swa is True.
|
# Fake _reduce_prefetch_ack to drop SWA pool_hits when drop_swa is True.
|
||||||
def fake_reduce(ack):
|
def fake_reduce(ack, groups=None):
|
||||||
if drop_swa and ack.pool_hits is not None:
|
if drop_swa and ack.pool_hits is not None:
|
||||||
if PoolName.SWA.value in ack.pool_hits:
|
if PoolName.SWA.value in ack.pool_hits:
|
||||||
ack.pool_hits[PoolName.SWA.value] = 0
|
ack.pool_hits[PoolName.SWA.value] = 0
|
||||||
|
|||||||
@@ -37,10 +37,15 @@ class C_HiCacheController(BaseHook):
|
|||||||
def wrapped_init(self, *args, **kwargs):
|
def wrapped_init(self, *args, **kwargs):
|
||||||
self.sim_prefetch_buffer = Queue()
|
self.sim_prefetch_buffer = Queue()
|
||||||
result = original_init(self, *args, **kwargs)
|
result = original_init(self, *args, **kwargs)
|
||||||
# The real IO thread normally creates this queue. The simulator
|
# The real IO threads normally create the handoff queues. The
|
||||||
# replaces that thread, so initialize the handoff queue here.
|
# simulator replaces those threads, so initialize the per-lane
|
||||||
if hasattr(self, "prefetch_hit_queue"):
|
# handoff queues here.
|
||||||
self.prefetch_buffer = Queue()
|
if hasattr(self, "prefetch_hit_queue") and not getattr(
|
||||||
|
self, "prefetch_io_queues", None
|
||||||
|
):
|
||||||
|
self.prefetch_io_queues = [
|
||||||
|
Queue() for _ in range(getattr(self, "num_prefetch_io_lanes", 1))
|
||||||
|
]
|
||||||
return result
|
return result
|
||||||
|
|
||||||
def wrapped_append_host_mem_release(self, host_indices):
|
def wrapped_append_host_mem_release(self, host_indices):
|
||||||
@@ -182,11 +187,10 @@ class C_HiCacheController(BaseHook):
|
|||||||
|
|
||||||
# Feed operations whose host pages were allocated by the scheduler
|
# Feed operations whose host pages were allocated by the scheduler
|
||||||
# into the virtual-time transfer loop.
|
# into the virtual-time transfer loop.
|
||||||
prefetch_buffer = getattr(self, "prefetch_buffer", None)
|
for io_queue in getattr(self, "prefetch_io_queues", None) or []:
|
||||||
if prefetch_buffer is not None:
|
while True:
|
||||||
while not prefetch_buffer.empty():
|
|
||||||
try:
|
try:
|
||||||
operation = prefetch_buffer.get(block=False)
|
operation = io_queue.get(block=False)
|
||||||
if operation is not None:
|
if operation is not None:
|
||||||
self.sim_prefetch_buffer.put(operation)
|
self.sim_prefetch_buffer.put(operation)
|
||||||
except Empty:
|
except Empty:
|
||||||
@@ -295,10 +299,12 @@ class C_HybridCacheController(BaseHook):
|
|||||||
|
|
||||||
def wrapped_init(self, *args, **kwargs):
|
def wrapped_init(self, *args, **kwargs):
|
||||||
result = original_init(self, *args, **kwargs)
|
result = original_init(self, *args, **kwargs)
|
||||||
if hasattr(self, "prefetch_hit_queue") and not hasattr(
|
if hasattr(self, "prefetch_hit_queue") and not getattr(
|
||||||
self, "prefetch_buffer"
|
self, "prefetch_io_queues", None
|
||||||
):
|
):
|
||||||
self.prefetch_buffer = Queue()
|
self.prefetch_io_queues = [
|
||||||
|
Queue() for _ in range(getattr(self, "num_prefetch_io_lanes", 1))
|
||||||
|
]
|
||||||
return result
|
return result
|
||||||
|
|
||||||
def wrapped_storage_hit_query(self, operator):
|
def wrapped_storage_hit_query(self, operator):
|
||||||
|
|||||||
Reference in New Issue
Block a user