hicache L3 prefetch: io_aux 传输并行化(按 op 分 lane)
build-sglang-image / build (push) Failing after 48m34s
build-sglang-image / build (push) Failing after 48m34s
打点实测单条 prefetch_io_aux_func 线程串行传输是所有请求的瓶颈: mc8 下每请求 xfer 排队 0->1.9s 线性增长(每请求 5 批 x ~68ms 固定开销, 介质无关)。本提交把传输层拆成 N 条并行 lane(默认 4, SGLANG_HICACHE_PREFETCH_IO_THREADS 可调): - dispatch_prefetch() 用 crc32(request_id) % N 分 lane(内置 hash 对 str 每进程加盐,跨 rank 不一致会导致 gloo 集合通信错位,禁用) - 同一 op 的所有 chunk 保持在同一 lane,保序 reduce 语义不变 - 每 lane 独立 io 队列 + io_aux 线程 + sync 队列 + sync 线程 + 独立 gloo completion 组(集合通信按 lane 过滤后的分发顺序保序) - query 阶段的 hit_reduce 仍在单 prefetch_thread,不动 - 生产点(hiradix/unified/hybrid pp-ticket)改调 dispatch_prefetch - 模拟器与两个单测文件同步适配 注意:SGLANG_HICACHE_PREFETCH_IO_THREADS 必须在实例内所有 TP/CP/PP rank 保持一致,否则 lane 组上的集合通信会错位挂死。
This commit is contained in:
@@ -763,6 +763,11 @@ class Envs:
|
||||
SGLANG_HICACHE_TIMING = EnvBool(False)
|
||||
# Pages per batched storage IO call (query / get / put chunking).
|
||||
SGLANG_HICACHE_STORAGE_BATCH_SIZE = EnvInt(128)
|
||||
# Number of parallel prefetch IO lanes (worker threads) per controller.
|
||||
# Operations are sharded by crc32(request_id) so every rank maps a request
|
||||
# to the same lane; each lane owns dedicated completion sync groups.
|
||||
# MUST be identical across all TP/CP/PP ranks of an instance.
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS = EnvInt(4)
|
||||
SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR = EnvStr(None)
|
||||
# File-backend LRU eviction (opt-in; sizes accept SI/IEC suffixes, "0" disables).
|
||||
SGLANG_HICACHE_FILE_BACKEND_MAX_SIZE = EnvStr(None)
|
||||
|
||||
@@ -17,6 +17,7 @@ limitations under the License.
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import zlib
|
||||
from dataclasses import dataclass
|
||||
from queue import Empty, Queue
|
||||
from typing import TYPE_CHECKING, Callable, List, NamedTuple, Optional
|
||||
@@ -317,7 +318,18 @@ class HiCacheController:
|
||||
self.attn_tp_group = attn_tp_group
|
||||
self.pp_group = pp_group
|
||||
self.prefetch_hits_sync_groups: List[torch.distributed.ProcessGroup] = []
|
||||
self.prefetch_completion_sync_groups: List[torch.distributed.ProcessGroup] = []
|
||||
# One set of completion sync groups per prefetch IO lane. Each lane's
|
||||
# worker/sync threads form a serial pipeline, so the collective
|
||||
# sequence on a lane's groups is identical across ranks as long as
|
||||
# lane assignment is deterministic (see dispatch_prefetch).
|
||||
self.prefetch_completion_sync_groups_by_lane: List[
|
||||
List[torch.distributed.ProcessGroup]
|
||||
] = []
|
||||
# Read once at construction; attach/reset reuse the stored value so
|
||||
# lane count never changes underneath live sync groups.
|
||||
self.num_prefetch_io_lanes = max(
|
||||
1, _envs.SGLANG_HICACHE_PREFETCH_IO_THREADS.get()
|
||||
)
|
||||
self.mem_pool_device_allocator = token_to_kv_pool_allocator
|
||||
mem_pool_device = token_to_kv_pool_allocator.get_kvcache()
|
||||
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool
|
||||
@@ -346,8 +358,8 @@ class HiCacheController:
|
||||
self.storage_stop_event = threading.Event()
|
||||
|
||||
# Storage control queues, (re)created whenever the storage threads start.
|
||||
self.prefetch_buffer: Optional[Queue[PrefetchOperation]] = None
|
||||
self.prefetch_sync_queue: Optional[Queue[PrefetchAck]] = None
|
||||
self.prefetch_io_queues: List[Queue[PrefetchOperation]] = []
|
||||
self.prefetch_sync_queues: List[Queue[PrefetchAck]] = []
|
||||
self.prefetch_hit_queue: Optional[Queue[StorageOperation]] = None
|
||||
self.ack_prefetch_queue = Queue[PrefetchAck]()
|
||||
self.ack_backup_queue: Optional[Queue[StorageOperation]] = None
|
||||
@@ -455,27 +467,39 @@ class HiCacheController:
|
||||
self.prefetch_thread = threading.Thread(
|
||||
target=self.prefetch_thread_func, daemon=True
|
||||
)
|
||||
self.prefetch_io_aux_thread = threading.Thread(
|
||||
target=self.prefetch_io_aux_func, daemon=True
|
||||
)
|
||||
self.prefetch_sync_thread = threading.Thread(
|
||||
target=self.prefetch_sync_thread_func, daemon=True
|
||||
)
|
||||
self.prefetch_io_aux_threads = [
|
||||
threading.Thread(
|
||||
target=self.prefetch_io_aux_func, args=(lane,), daemon=True
|
||||
)
|
||||
for lane in range(self.num_prefetch_io_lanes)
|
||||
]
|
||||
self.prefetch_sync_threads = [
|
||||
threading.Thread(
|
||||
target=self.prefetch_sync_thread_func, args=(lane,), daemon=True
|
||||
)
|
||||
for lane in range(self.num_prefetch_io_lanes)
|
||||
]
|
||||
self.backup_thread = threading.Thread(
|
||||
target=self.backup_thread_func, daemon=True
|
||||
)
|
||||
self.prefetch_queue = Queue()
|
||||
self.backup_queue = Queue()
|
||||
self.prefetch_buffer = Queue()
|
||||
self.prefetch_sync_queue = Queue()
|
||||
self.prefetch_io_queues = [
|
||||
Queue() for _ in range(self.num_prefetch_io_lanes)
|
||||
]
|
||||
self.prefetch_sync_queues = [
|
||||
Queue() for _ in range(self.num_prefetch_io_lanes)
|
||||
]
|
||||
self.prefetch_hit_queue = Queue()
|
||||
self.ack_prefetch_queue = Queue()
|
||||
self.ack_backup_queue = Queue()
|
||||
self.host_mem_release_queue = Queue()
|
||||
|
||||
self.prefetch_thread.start()
|
||||
self.prefetch_io_aux_thread.start()
|
||||
self.prefetch_sync_thread.start()
|
||||
for thread in self.prefetch_io_aux_threads:
|
||||
thread.start()
|
||||
for thread in self.prefetch_sync_threads:
|
||||
thread.start()
|
||||
self.backup_thread.start()
|
||||
|
||||
def _stop_storage_threads(self):
|
||||
@@ -496,10 +520,10 @@ class HiCacheController:
|
||||
self.prefetch_queue.put_nowait(None)
|
||||
if hasattr(self, "backup_queue"):
|
||||
self.backup_queue.put_nowait(None)
|
||||
if hasattr(self, "prefetch_buffer"):
|
||||
self.prefetch_buffer.put_nowait(None)
|
||||
if hasattr(self, "prefetch_sync_queue"):
|
||||
self.prefetch_sync_queue.put_nowait(None)
|
||||
for queue in getattr(self, "prefetch_io_queues", []) or []:
|
||||
queue.put_nowait(None)
|
||||
for queue in getattr(self, "prefetch_sync_queues", []) or []:
|
||||
queue.put_nowait(None)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -509,10 +533,8 @@ class HiCacheController:
|
||||
threads.append(self.prefetch_thread)
|
||||
if hasattr(self, "backup_thread"):
|
||||
threads.append(self.backup_thread)
|
||||
if hasattr(self, "prefetch_io_aux_thread"):
|
||||
threads.append(self.prefetch_io_aux_thread)
|
||||
if hasattr(self, "prefetch_sync_thread"):
|
||||
threads.append(self.prefetch_sync_thread)
|
||||
threads.extend(getattr(self, "prefetch_io_aux_threads", []) or [])
|
||||
threads.extend(getattr(self, "prefetch_sync_threads", []) or [])
|
||||
|
||||
for t in threads:
|
||||
try:
|
||||
@@ -596,7 +618,13 @@ class HiCacheController:
|
||||
# Use dedicated gloo groups so storage prefetch sync is isolated
|
||||
# from other collectives and consistent across CPxTP participants.
|
||||
self.prefetch_hits_sync_groups = self._create_sync_groups()
|
||||
self.prefetch_completion_sync_groups = self._create_sync_groups()
|
||||
# Each IO lane gets its own completion groups: per-lane collectives
|
||||
# stay ordered because a lane is a serial pipeline and dispatch is
|
||||
# deterministic (crc32(request_id) % num_prefetch_io_lanes).
|
||||
self.prefetch_completion_sync_groups_by_lane = [
|
||||
self._create_sync_groups()
|
||||
for _ in range(self.num_prefetch_io_lanes)
|
||||
]
|
||||
|
||||
# Select the get and set functions
|
||||
self.page_get_func = self._generic_page_get
|
||||
@@ -631,9 +659,10 @@ class HiCacheController:
|
||||
except Exception:
|
||||
pass
|
||||
self._destroy_sync_groups(self.prefetch_hits_sync_groups)
|
||||
self._destroy_sync_groups(self.prefetch_completion_sync_groups)
|
||||
for lane_groups in self.prefetch_completion_sync_groups_by_lane:
|
||||
self._destroy_sync_groups(lane_groups)
|
||||
self.prefetch_hits_sync_groups = []
|
||||
self.prefetch_completion_sync_groups = []
|
||||
self.prefetch_completion_sync_groups_by_lane = []
|
||||
try:
|
||||
if (
|
||||
hasattr(self, "storage_backend")
|
||||
@@ -671,11 +700,11 @@ class HiCacheController:
|
||||
raise RuntimeError("Stop storage threads failed; detach aborted.") from e
|
||||
|
||||
# Best-effort destroy process groups created for storage ops.
|
||||
self._destroy_sync_groups(
|
||||
self.prefetch_hits_sync_groups + self.prefetch_completion_sync_groups
|
||||
)
|
||||
self._destroy_sync_groups(self.prefetch_hits_sync_groups)
|
||||
for lane_groups in self.prefetch_completion_sync_groups_by_lane:
|
||||
self._destroy_sync_groups(lane_groups)
|
||||
self.prefetch_hits_sync_groups = []
|
||||
self.prefetch_completion_sync_groups = []
|
||||
self.prefetch_completion_sync_groups_by_lane = []
|
||||
|
||||
# Best-effort close (some backends rely on GC/destructor).
|
||||
try:
|
||||
@@ -769,13 +798,17 @@ class HiCacheController:
|
||||
self.ack_load_queue.clear()
|
||||
if self.enable_storage:
|
||||
self.prefetch_thread.join()
|
||||
self.prefetch_io_aux_thread.join()
|
||||
self.prefetch_sync_thread.join()
|
||||
for thread in self.prefetch_io_aux_threads:
|
||||
thread.join()
|
||||
for thread in self.prefetch_sync_threads:
|
||||
thread.join()
|
||||
self.backup_thread.join()
|
||||
self.prefetch_queue.queue.clear()
|
||||
self.backup_queue.queue.clear()
|
||||
self.prefetch_buffer.queue.clear()
|
||||
self.prefetch_sync_queue.queue.clear()
|
||||
for queue in self.prefetch_io_queues:
|
||||
queue.queue.clear()
|
||||
for queue in self.prefetch_sync_queues:
|
||||
queue.queue.clear()
|
||||
self.prefetch_hit_queue.queue.clear()
|
||||
self.ack_prefetch_queue.queue.clear()
|
||||
self.ack_backup_queue.queue.clear()
|
||||
@@ -788,18 +821,26 @@ class HiCacheController:
|
||||
self.prefetch_thread = threading.Thread(
|
||||
target=self.prefetch_thread_func, daemon=True
|
||||
)
|
||||
self.prefetch_io_aux_thread = threading.Thread(
|
||||
target=self.prefetch_io_aux_func, daemon=True
|
||||
)
|
||||
self.prefetch_sync_thread = threading.Thread(
|
||||
target=self.prefetch_sync_thread_func, daemon=True
|
||||
)
|
||||
self.prefetch_io_aux_threads = [
|
||||
threading.Thread(
|
||||
target=self.prefetch_io_aux_func, args=(lane,), daemon=True
|
||||
)
|
||||
for lane in range(self.num_prefetch_io_lanes)
|
||||
]
|
||||
self.prefetch_sync_threads = [
|
||||
threading.Thread(
|
||||
target=self.prefetch_sync_thread_func, args=(lane,), daemon=True
|
||||
)
|
||||
for lane in range(self.num_prefetch_io_lanes)
|
||||
]
|
||||
self.backup_thread = threading.Thread(
|
||||
target=self.backup_thread_func, daemon=True
|
||||
)
|
||||
self.prefetch_thread.start()
|
||||
self.prefetch_io_aux_thread.start()
|
||||
self.prefetch_sync_thread.start()
|
||||
for thread in self.prefetch_io_aux_threads:
|
||||
thread.start()
|
||||
for thread in self.prefetch_sync_threads:
|
||||
thread.start()
|
||||
self.backup_thread.start()
|
||||
|
||||
def has_inflight_device_transfers(self) -> bool:
|
||||
@@ -1086,7 +1127,23 @@ class HiCacheController:
|
||||
count += 1
|
||||
return count
|
||||
|
||||
def _page_transfer(self, operation: PrefetchOperation) -> int:
|
||||
def dispatch_prefetch(self, operation: PrefetchOperation) -> None:
|
||||
"""Route a prefetch operation to its IO lane.
|
||||
|
||||
Lane assignment must be deterministic and identical on every rank:
|
||||
each lane owns dedicated completion sync groups, and the collective
|
||||
sequence on a lane is the filtered dispatch order. crc32 (not the
|
||||
per-process salted builtin hash) keeps the mapping stable across
|
||||
processes. All chunks of one operation stay in one lane, preserving
|
||||
the per-operation ack order the sync thread reduces over.
|
||||
"""
|
||||
lane = (
|
||||
zlib.crc32(str(operation.request_id).encode())
|
||||
% self.num_prefetch_io_lanes
|
||||
)
|
||||
self.prefetch_io_queues[lane].put(operation)
|
||||
|
||||
def _page_transfer(self, operation: PrefetchOperation, sync_queue: Queue) -> int:
|
||||
# Transfer batch by batch
|
||||
prefix_keys = operation.prefix_keys
|
||||
kv_derived_transfers = [
|
||||
@@ -1098,8 +1155,8 @@ class HiCacheController:
|
||||
completed_pages = 0
|
||||
for i in range(0, len(operation.hash_value), STORAGE_BATCH_SIZE):
|
||||
# When an error is occurred, we should keep looping and produce the same number of
|
||||
# PrefetchAck as other ranks do, because prefetch_sync_thread (i.e. consumer of
|
||||
# prefetch_sync_queue) perform reduce on the results. This is so tricky.
|
||||
# PrefetchAck as other ranks do, because the prefetch sync thread (i.e. consumer of
|
||||
# the lane sync queue) perform reduce on the results. This is so tricky.
|
||||
if all_success and operation.is_terminated():
|
||||
all_success = False
|
||||
if all_success:
|
||||
@@ -1138,7 +1195,7 @@ class HiCacheController:
|
||||
completed_tokens=completed_pages * self.page_size,
|
||||
operation=operation,
|
||||
)
|
||||
self.prefetch_sync_queue.put(ack)
|
||||
sync_queue.put(ack)
|
||||
return completed_pages
|
||||
|
||||
def _page_transfer_kv_batch(
|
||||
@@ -1180,27 +1237,33 @@ class HiCacheController:
|
||||
# Clamp to minimal number of hits.
|
||||
return min([kv_hits, *sidecar_hits.values()])
|
||||
|
||||
def prefetch_io_aux_func(self):
|
||||
def prefetch_io_aux_func(self, lane: int = 0):
|
||||
"""
|
||||
Auxiliary function conducting IO operations for prefetching.
|
||||
|
||||
Each lane has its own worker thread and queues; an operation's whole
|
||||
chunk sequence stays on one lane so the sync thread sees it in order.
|
||||
"""
|
||||
io_queue = self.prefetch_io_queues[lane]
|
||||
sync_queue = self.prefetch_sync_queues[lane]
|
||||
while not self.storage_stop_event.is_set():
|
||||
try:
|
||||
operation = self.prefetch_buffer.get(block=True, timeout=1)
|
||||
operation = io_queue.get(block=True, timeout=1)
|
||||
if operation is None:
|
||||
continue
|
||||
_hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
self._page_transfer(operation)
|
||||
self._page_transfer(operation, sync_queue)
|
||||
if _hc_timing_enabled():
|
||||
logger.info(
|
||||
"HC_TIMING stage=xfer req=%s wait_ms=%.1f xfer_ms=%.1f",
|
||||
"HC_TIMING stage=xfer req=%s lane=%d wait_ms=%.1f xfer_ms=%.1f",
|
||||
operation.request_id,
|
||||
lane,
|
||||
(_hc_t0 - getattr(operation, "query_done_ts", operation.start_time))
|
||||
* 1e3,
|
||||
(time.monotonic() - _hc_t0) * 1e3,
|
||||
)
|
||||
|
||||
self.prefetch_sync_queue.put(
|
||||
sync_queue.put(
|
||||
PrefetchAck(
|
||||
rid=operation.request_id,
|
||||
completed_req=True,
|
||||
@@ -1381,21 +1444,25 @@ class HiCacheController:
|
||||
except Empty:
|
||||
continue
|
||||
|
||||
def prefetch_sync_thread_func(self):
|
||||
def prefetch_sync_thread_func(self, lane: int = 0):
|
||||
"""Synchronize prefetch results across all PP and TP ranks."""
|
||||
sync_queue = self.prefetch_sync_queues[lane]
|
||||
groups = self.prefetch_completion_sync_groups_by_lane[lane]
|
||||
while not self.storage_stop_event.is_set():
|
||||
try:
|
||||
ack = self.prefetch_sync_queue.get(block=True, timeout=1)
|
||||
ack = sync_queue.get(block=True, timeout=1)
|
||||
if ack is None:
|
||||
continue
|
||||
self._reduce_prefetch_ack(ack)
|
||||
self._reduce_prefetch_ack(ack, groups)
|
||||
if _hc_timing_enabled() and ack.completed_req:
|
||||
ack.operation.final_ack_enqueue_ts = time.monotonic()
|
||||
self.ack_prefetch_queue.put(ack)
|
||||
except Empty:
|
||||
continue
|
||||
|
||||
def _reduce_prefetch_ack(self, ack: PrefetchAck) -> None:
|
||||
def _reduce_prefetch_ack(
|
||||
self, ack: PrefetchAck, groups: List[torch.distributed.ProcessGroup]
|
||||
) -> None:
|
||||
"""Synchronize all ranks to agree on a PrefetchAck."""
|
||||
if ack.completed_tokens is not None:
|
||||
# Determine the minimal successful prefix of tokens.
|
||||
@@ -1406,7 +1473,7 @@ class HiCacheController:
|
||||
self._all_reduce(
|
||||
completed_tokens_tensor,
|
||||
torch.distributed.ReduceOp.MIN,
|
||||
self.prefetch_completion_sync_groups,
|
||||
groups,
|
||||
)
|
||||
ack.completed_tokens = completed_tokens_tensor.item()
|
||||
if _hc_timing_enabled():
|
||||
|
||||
@@ -653,7 +653,7 @@ class HiRadixCache(RadixCache):
|
||||
: alloc_len // self.page_size
|
||||
]
|
||||
operation.host_indices = host_indices
|
||||
cc.prefetch_buffer.put(operation)
|
||||
cc.dispatch_prefetch(operation)
|
||||
|
||||
def _drain_ack_prefetch():
|
||||
for ack in _drain_queue(cc.ack_prefetch_queue, n_ack_prefetch):
|
||||
|
||||
@@ -1006,7 +1006,7 @@ class HybridCacheController(BaseHiCacheController):
|
||||
if not self._allocate_pp_prefetch_buffers(ticket, operation):
|
||||
operation.host_indices = torch.empty(0, dtype=torch.int64)
|
||||
operation.mark_terminate()
|
||||
self.prefetch_buffer.put(operation)
|
||||
self.dispatch_prefetch(operation)
|
||||
except Exception:
|
||||
logger.exception("PP HiCache ticket processing failed.")
|
||||
if operation is None or not operation.hash_value:
|
||||
@@ -1016,7 +1016,7 @@ class HybridCacheController(BaseHiCacheController):
|
||||
operation.host_indices = torch.empty(0, dtype=torch.int64)
|
||||
operation.mark_terminate()
|
||||
# Preserve every KV/sidecar ACK even when local allocation fails.
|
||||
self.prefetch_buffer.put(operation)
|
||||
self.dispatch_prefetch(operation)
|
||||
finally:
|
||||
if is_source and objects:
|
||||
self.pp_prefetch_command_queue.task_done()
|
||||
@@ -1104,15 +1104,15 @@ class HybridCacheController(BaseHiCacheController):
|
||||
)
|
||||
return host_indices, device_indices, resolved_pool_transfers
|
||||
|
||||
def _page_transfer(self, operation: PrefetchOperation) -> bool:
|
||||
def _page_transfer(self, operation: PrefetchOperation, sync_queue: Queue) -> bool:
|
||||
# KV pools and KV-derived pools first — determines actual completed page count
|
||||
kv_completed_pages = super()._page_transfer(operation)
|
||||
kv_completed_pages = super()._page_transfer(operation, sync_queue)
|
||||
|
||||
# Read non-KV derived sidecar pool, e.g. SWA, Mamba.
|
||||
self._page_transfer_sidecar(operation, kv_completed_pages)
|
||||
self._page_transfer_sidecar(operation, kv_completed_pages, sync_queue)
|
||||
|
||||
def _page_transfer_sidecar(
|
||||
self, operation: PrefetchOperation, kv_completed_pages: int
|
||||
self, operation: PrefetchOperation, kv_completed_pages: int, sync_queue: Queue
|
||||
) -> None:
|
||||
if operation.pool_transfers is None:
|
||||
return
|
||||
@@ -1144,10 +1144,10 @@ class HybridCacheController(BaseHiCacheController):
|
||||
transfers_nonkv, extra_info=extra_info
|
||||
)
|
||||
pool_hits = count_pool_hits(results)
|
||||
# Emit PrefetchAck to prefetch_sync_queue, even the operation has been canceled by the
|
||||
# Emit PrefetchAck to the lane sync queue, even the operation has been canceled by the
|
||||
# scheduler thread. The prefetch sync thread expects the same number of PrefetchAck objects
|
||||
# to perform all_reduce.
|
||||
self.prefetch_sync_queue.put(
|
||||
sync_queue.put(
|
||||
PrefetchAck(
|
||||
rid=operation.request_id,
|
||||
operation=operation,
|
||||
@@ -1421,9 +1421,11 @@ class HybridCacheController(BaseHiCacheController):
|
||||
pool.device_indices = source.device_indices
|
||||
return extra_pools
|
||||
|
||||
def _reduce_prefetch_ack(self, ack: PrefetchAck) -> None:
|
||||
def _reduce_prefetch_ack(
|
||||
self, ack: PrefetchAck, groups: List[torch.distributed.ProcessGroup]
|
||||
) -> None:
|
||||
# Handle KV-derived pool.
|
||||
super()._reduce_prefetch_ack(ack)
|
||||
super()._reduce_prefetch_ack(ack, groups)
|
||||
|
||||
# Handle other sidecar pools, e.g. SWA, Mamba.
|
||||
if ack.pool_hits is not None:
|
||||
@@ -1436,19 +1438,21 @@ class HybridCacheController(BaseHiCacheController):
|
||||
self._all_reduce(
|
||||
packed,
|
||||
torch.distributed.ReduceOp.MIN,
|
||||
self.prefetch_completion_sync_groups,
|
||||
groups,
|
||||
)
|
||||
for i, pool in enumerate(PoolName):
|
||||
ack.pool_hits[pool.value] = packed[i].item()
|
||||
|
||||
def prefetch_sync_thread_func(self) -> None:
|
||||
def prefetch_sync_thread_func(self, lane: int = 0) -> None:
|
||||
"""Synchronize progressive ACKs and publish PP ticket completion."""
|
||||
sync_queue = self.prefetch_sync_queues[lane]
|
||||
groups = self.prefetch_completion_sync_groups_by_lane[lane]
|
||||
while not self.storage_stop_event.is_set():
|
||||
try:
|
||||
ack = self.prefetch_sync_queue.get(block=True, timeout=1)
|
||||
ack = sync_queue.get(block=True, timeout=1)
|
||||
if ack is None:
|
||||
continue
|
||||
self._reduce_prefetch_ack(ack)
|
||||
self._reduce_prefetch_ack(ack, groups)
|
||||
if not getattr(ack.operation, "is_pp_broadcast", False):
|
||||
if _hc_timing_enabled() and ack.completed_req:
|
||||
ack.operation.final_ack_enqueue_ts = time.monotonic()
|
||||
|
||||
@@ -3001,7 +3001,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
self.ongoing_prefetch[request] = info._replace(host_indices=host_indices)
|
||||
if buffer_mode:
|
||||
cc.prefetch_tokens_occupied += alloc_len
|
||||
cc.prefetch_buffer.put(operation)
|
||||
cc.dispatch_prefetch(operation)
|
||||
return True
|
||||
|
||||
def _drain_and_alloc_storage_hit():
|
||||
|
||||
Reference in New Issue
Block a user