From baba18bc7dddebddb9080e7ae8123cf0002ddaab Mon Sep 17 00:00:00 2001 From: "minke.yu" Date: Sat, 10 Oct 2026 19:04:49 +0800 Subject: [PATCH] =?UTF-8?q?hicache=20L3=20prefetch:=20io=5Faux=20=E4=BC=A0?= =?UTF-8?q?=E8=BE=93=E5=B9=B6=E8=A1=8C=E5=8C=96=EF=BC=88=E6=8C=89=20op=20?= =?UTF-8?q?=E5=88=86=20lane=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 打点实测单条 prefetch_io_aux_func 线程串行传输是所有请求的瓶颈: mc8 下每请求 xfer 排队 0->1.9s 线性增长(每请求 5 批 x ~68ms 固定开销, 介质无关)。本提交把传输层拆成 N 条并行 lane(默认 4, SGLANG_HICACHE_PREFETCH_IO_THREADS 可调): - dispatch_prefetch() 用 crc32(request_id) % N 分 lane(内置 hash 对 str 每进程加盐,跨 rank 不一致会导致 gloo 集合通信错位,禁用) - 同一 op 的所有 chunk 保持在同一 lane,保序 reduce 语义不变 - 每 lane 独立 io 队列 + io_aux 线程 + sync 队列 + sync 线程 + 独立 gloo completion 组(集合通信按 lane 过滤后的分发顺序保序) - query 阶段的 hit_reduce 仍在单 prefetch_thread,不动 - 生产点(hiradix/unified/hybrid pp-ticket)改调 dispatch_prefetch - 模拟器与两个单测文件同步适配 注意:SGLANG_HICACHE_PREFETCH_IO_THREADS 必须在实例内所有 TP/CP/PP rank 保持一致,否则 lane 组上的集合通信会错位挂死。 --- python/sglang/srt/environ.py | 5 + .../sglang/srt/managers/cache_controller.py | 175 ++++++++++++------ python/sglang/srt/mem_cache/hiradix_cache.py | 2 +- .../hybrid_cache/hybrid_cache_controller.py | 32 ++-- .../srt/mem_cache/unified_radix_cache.py | 2 +- .../unit/mem_cache/test_pp_prefetch_ticket.py | 28 +-- .../test_unified_radix_cache_unittest.py | 6 +- .../simulation/sglang/cache_controller.py | 28 +-- 8 files changed, 181 insertions(+), 97 deletions(-) diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 06a691994e..bc92c0c537 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -763,6 +763,11 @@ class Envs: SGLANG_HICACHE_TIMING = EnvBool(False) # Pages per batched storage IO call (query / get / put chunking). SGLANG_HICACHE_STORAGE_BATCH_SIZE = EnvInt(128) + # Number of parallel prefetch IO lanes (worker threads) per controller. + # Operations are sharded by crc32(request_id) so every rank maps a request + # to the same lane; each lane owns dedicated completion sync groups. + # MUST be identical across all TP/CP/PP ranks of an instance. + SGLANG_HICACHE_PREFETCH_IO_THREADS = EnvInt(4) SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR = EnvStr(None) # File-backend LRU eviction (opt-in; sizes accept SI/IEC suffixes, "0" disables). SGLANG_HICACHE_FILE_BACKEND_MAX_SIZE = EnvStr(None) diff --git a/python/sglang/srt/managers/cache_controller.py b/python/sglang/srt/managers/cache_controller.py index d9db7f7337..e8468ae9b3 100644 --- a/python/sglang/srt/managers/cache_controller.py +++ b/python/sglang/srt/managers/cache_controller.py @@ -17,6 +17,7 @@ limitations under the License. import logging import threading import time +import zlib from dataclasses import dataclass from queue import Empty, Queue from typing import TYPE_CHECKING, Callable, List, NamedTuple, Optional @@ -317,7 +318,18 @@ class HiCacheController: self.attn_tp_group = attn_tp_group self.pp_group = pp_group self.prefetch_hits_sync_groups: List[torch.distributed.ProcessGroup] = [] - self.prefetch_completion_sync_groups: List[torch.distributed.ProcessGroup] = [] + # One set of completion sync groups per prefetch IO lane. Each lane's + # worker/sync threads form a serial pipeline, so the collective + # sequence on a lane's groups is identical across ranks as long as + # lane assignment is deterministic (see dispatch_prefetch). + self.prefetch_completion_sync_groups_by_lane: List[ + List[torch.distributed.ProcessGroup] + ] = [] + # Read once at construction; attach/reset reuse the stored value so + # lane count never changes underneath live sync groups. + self.num_prefetch_io_lanes = max( + 1, _envs.SGLANG_HICACHE_PREFETCH_IO_THREADS.get() + ) self.mem_pool_device_allocator = token_to_kv_pool_allocator mem_pool_device = token_to_kv_pool_allocator.get_kvcache() from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool @@ -346,8 +358,8 @@ class HiCacheController: self.storage_stop_event = threading.Event() # Storage control queues, (re)created whenever the storage threads start. - self.prefetch_buffer: Optional[Queue[PrefetchOperation]] = None - self.prefetch_sync_queue: Optional[Queue[PrefetchAck]] = None + self.prefetch_io_queues: List[Queue[PrefetchOperation]] = [] + self.prefetch_sync_queues: List[Queue[PrefetchAck]] = [] self.prefetch_hit_queue: Optional[Queue[StorageOperation]] = None self.ack_prefetch_queue = Queue[PrefetchAck]() self.ack_backup_queue: Optional[Queue[StorageOperation]] = None @@ -455,27 +467,39 @@ class HiCacheController: self.prefetch_thread = threading.Thread( target=self.prefetch_thread_func, daemon=True ) - self.prefetch_io_aux_thread = threading.Thread( - target=self.prefetch_io_aux_func, daemon=True - ) - self.prefetch_sync_thread = threading.Thread( - target=self.prefetch_sync_thread_func, daemon=True - ) + self.prefetch_io_aux_threads = [ + threading.Thread( + target=self.prefetch_io_aux_func, args=(lane,), daemon=True + ) + for lane in range(self.num_prefetch_io_lanes) + ] + self.prefetch_sync_threads = [ + threading.Thread( + target=self.prefetch_sync_thread_func, args=(lane,), daemon=True + ) + for lane in range(self.num_prefetch_io_lanes) + ] self.backup_thread = threading.Thread( target=self.backup_thread_func, daemon=True ) self.prefetch_queue = Queue() self.backup_queue = Queue() - self.prefetch_buffer = Queue() - self.prefetch_sync_queue = Queue() + self.prefetch_io_queues = [ + Queue() for _ in range(self.num_prefetch_io_lanes) + ] + self.prefetch_sync_queues = [ + Queue() for _ in range(self.num_prefetch_io_lanes) + ] self.prefetch_hit_queue = Queue() self.ack_prefetch_queue = Queue() self.ack_backup_queue = Queue() self.host_mem_release_queue = Queue() self.prefetch_thread.start() - self.prefetch_io_aux_thread.start() - self.prefetch_sync_thread.start() + for thread in self.prefetch_io_aux_threads: + thread.start() + for thread in self.prefetch_sync_threads: + thread.start() self.backup_thread.start() def _stop_storage_threads(self): @@ -496,10 +520,10 @@ class HiCacheController: self.prefetch_queue.put_nowait(None) if hasattr(self, "backup_queue"): self.backup_queue.put_nowait(None) - if hasattr(self, "prefetch_buffer"): - self.prefetch_buffer.put_nowait(None) - if hasattr(self, "prefetch_sync_queue"): - self.prefetch_sync_queue.put_nowait(None) + for queue in getattr(self, "prefetch_io_queues", []) or []: + queue.put_nowait(None) + for queue in getattr(self, "prefetch_sync_queues", []) or []: + queue.put_nowait(None) except Exception: pass @@ -509,10 +533,8 @@ class HiCacheController: threads.append(self.prefetch_thread) if hasattr(self, "backup_thread"): threads.append(self.backup_thread) - if hasattr(self, "prefetch_io_aux_thread"): - threads.append(self.prefetch_io_aux_thread) - if hasattr(self, "prefetch_sync_thread"): - threads.append(self.prefetch_sync_thread) + threads.extend(getattr(self, "prefetch_io_aux_threads", []) or []) + threads.extend(getattr(self, "prefetch_sync_threads", []) or []) for t in threads: try: @@ -596,7 +618,13 @@ class HiCacheController: # Use dedicated gloo groups so storage prefetch sync is isolated # from other collectives and consistent across CPxTP participants. self.prefetch_hits_sync_groups = self._create_sync_groups() - self.prefetch_completion_sync_groups = self._create_sync_groups() + # Each IO lane gets its own completion groups: per-lane collectives + # stay ordered because a lane is a serial pipeline and dispatch is + # deterministic (crc32(request_id) % num_prefetch_io_lanes). + self.prefetch_completion_sync_groups_by_lane = [ + self._create_sync_groups() + for _ in range(self.num_prefetch_io_lanes) + ] # Select the get and set functions self.page_get_func = self._generic_page_get @@ -631,9 +659,10 @@ class HiCacheController: except Exception: pass self._destroy_sync_groups(self.prefetch_hits_sync_groups) - self._destroy_sync_groups(self.prefetch_completion_sync_groups) + for lane_groups in self.prefetch_completion_sync_groups_by_lane: + self._destroy_sync_groups(lane_groups) self.prefetch_hits_sync_groups = [] - self.prefetch_completion_sync_groups = [] + self.prefetch_completion_sync_groups_by_lane = [] try: if ( hasattr(self, "storage_backend") @@ -671,11 +700,11 @@ class HiCacheController: raise RuntimeError("Stop storage threads failed; detach aborted.") from e # Best-effort destroy process groups created for storage ops. - self._destroy_sync_groups( - self.prefetch_hits_sync_groups + self.prefetch_completion_sync_groups - ) + self._destroy_sync_groups(self.prefetch_hits_sync_groups) + for lane_groups in self.prefetch_completion_sync_groups_by_lane: + self._destroy_sync_groups(lane_groups) self.prefetch_hits_sync_groups = [] - self.prefetch_completion_sync_groups = [] + self.prefetch_completion_sync_groups_by_lane = [] # Best-effort close (some backends rely on GC/destructor). try: @@ -769,13 +798,17 @@ class HiCacheController: self.ack_load_queue.clear() if self.enable_storage: self.prefetch_thread.join() - self.prefetch_io_aux_thread.join() - self.prefetch_sync_thread.join() + for thread in self.prefetch_io_aux_threads: + thread.join() + for thread in self.prefetch_sync_threads: + thread.join() self.backup_thread.join() self.prefetch_queue.queue.clear() self.backup_queue.queue.clear() - self.prefetch_buffer.queue.clear() - self.prefetch_sync_queue.queue.clear() + for queue in self.prefetch_io_queues: + queue.queue.clear() + for queue in self.prefetch_sync_queues: + queue.queue.clear() self.prefetch_hit_queue.queue.clear() self.ack_prefetch_queue.queue.clear() self.ack_backup_queue.queue.clear() @@ -788,18 +821,26 @@ class HiCacheController: self.prefetch_thread = threading.Thread( target=self.prefetch_thread_func, daemon=True ) - self.prefetch_io_aux_thread = threading.Thread( - target=self.prefetch_io_aux_func, daemon=True - ) - self.prefetch_sync_thread = threading.Thread( - target=self.prefetch_sync_thread_func, daemon=True - ) + self.prefetch_io_aux_threads = [ + threading.Thread( + target=self.prefetch_io_aux_func, args=(lane,), daemon=True + ) + for lane in range(self.num_prefetch_io_lanes) + ] + self.prefetch_sync_threads = [ + threading.Thread( + target=self.prefetch_sync_thread_func, args=(lane,), daemon=True + ) + for lane in range(self.num_prefetch_io_lanes) + ] self.backup_thread = threading.Thread( target=self.backup_thread_func, daemon=True ) self.prefetch_thread.start() - self.prefetch_io_aux_thread.start() - self.prefetch_sync_thread.start() + for thread in self.prefetch_io_aux_threads: + thread.start() + for thread in self.prefetch_sync_threads: + thread.start() self.backup_thread.start() def has_inflight_device_transfers(self) -> bool: @@ -1086,7 +1127,23 @@ class HiCacheController: count += 1 return count - def _page_transfer(self, operation: PrefetchOperation) -> int: + def dispatch_prefetch(self, operation: PrefetchOperation) -> None: + """Route a prefetch operation to its IO lane. + + Lane assignment must be deterministic and identical on every rank: + each lane owns dedicated completion sync groups, and the collective + sequence on a lane is the filtered dispatch order. crc32 (not the + per-process salted builtin hash) keeps the mapping stable across + processes. All chunks of one operation stay in one lane, preserving + the per-operation ack order the sync thread reduces over. + """ + lane = ( + zlib.crc32(str(operation.request_id).encode()) + % self.num_prefetch_io_lanes + ) + self.prefetch_io_queues[lane].put(operation) + + def _page_transfer(self, operation: PrefetchOperation, sync_queue: Queue) -> int: # Transfer batch by batch prefix_keys = operation.prefix_keys kv_derived_transfers = [ @@ -1098,8 +1155,8 @@ class HiCacheController: completed_pages = 0 for i in range(0, len(operation.hash_value), STORAGE_BATCH_SIZE): # When an error is occurred, we should keep looping and produce the same number of - # PrefetchAck as other ranks do, because prefetch_sync_thread (i.e. consumer of - # prefetch_sync_queue) perform reduce on the results. This is so tricky. + # PrefetchAck as other ranks do, because the prefetch sync thread (i.e. consumer of + # the lane sync queue) perform reduce on the results. This is so tricky. if all_success and operation.is_terminated(): all_success = False if all_success: @@ -1138,7 +1195,7 @@ class HiCacheController: completed_tokens=completed_pages * self.page_size, operation=operation, ) - self.prefetch_sync_queue.put(ack) + sync_queue.put(ack) return completed_pages def _page_transfer_kv_batch( @@ -1180,27 +1237,33 @@ class HiCacheController: # Clamp to minimal number of hits. return min([kv_hits, *sidecar_hits.values()]) - def prefetch_io_aux_func(self): + def prefetch_io_aux_func(self, lane: int = 0): """ Auxiliary function conducting IO operations for prefetching. + + Each lane has its own worker thread and queues; an operation's whole + chunk sequence stays on one lane so the sync thread sees it in order. """ + io_queue = self.prefetch_io_queues[lane] + sync_queue = self.prefetch_sync_queues[lane] while not self.storage_stop_event.is_set(): try: - operation = self.prefetch_buffer.get(block=True, timeout=1) + operation = io_queue.get(block=True, timeout=1) if operation is None: continue _hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0 - self._page_transfer(operation) + self._page_transfer(operation, sync_queue) if _hc_timing_enabled(): logger.info( - "HC_TIMING stage=xfer req=%s wait_ms=%.1f xfer_ms=%.1f", + "HC_TIMING stage=xfer req=%s lane=%d wait_ms=%.1f xfer_ms=%.1f", operation.request_id, + lane, (_hc_t0 - getattr(operation, "query_done_ts", operation.start_time)) * 1e3, (time.monotonic() - _hc_t0) * 1e3, ) - self.prefetch_sync_queue.put( + sync_queue.put( PrefetchAck( rid=operation.request_id, completed_req=True, @@ -1381,21 +1444,25 @@ class HiCacheController: except Empty: continue - def prefetch_sync_thread_func(self): + def prefetch_sync_thread_func(self, lane: int = 0): """Synchronize prefetch results across all PP and TP ranks.""" + sync_queue = self.prefetch_sync_queues[lane] + groups = self.prefetch_completion_sync_groups_by_lane[lane] while not self.storage_stop_event.is_set(): try: - ack = self.prefetch_sync_queue.get(block=True, timeout=1) + ack = sync_queue.get(block=True, timeout=1) if ack is None: continue - self._reduce_prefetch_ack(ack) + self._reduce_prefetch_ack(ack, groups) if _hc_timing_enabled() and ack.completed_req: ack.operation.final_ack_enqueue_ts = time.monotonic() self.ack_prefetch_queue.put(ack) except Empty: continue - def _reduce_prefetch_ack(self, ack: PrefetchAck) -> None: + def _reduce_prefetch_ack( + self, ack: PrefetchAck, groups: List[torch.distributed.ProcessGroup] + ) -> None: """Synchronize all ranks to agree on a PrefetchAck.""" if ack.completed_tokens is not None: # Determine the minimal successful prefix of tokens. @@ -1406,7 +1473,7 @@ class HiCacheController: self._all_reduce( completed_tokens_tensor, torch.distributed.ReduceOp.MIN, - self.prefetch_completion_sync_groups, + groups, ) ack.completed_tokens = completed_tokens_tensor.item() if _hc_timing_enabled(): diff --git a/python/sglang/srt/mem_cache/hiradix_cache.py b/python/sglang/srt/mem_cache/hiradix_cache.py index 9c2e95b568..7d35a0605b 100644 --- a/python/sglang/srt/mem_cache/hiradix_cache.py +++ b/python/sglang/srt/mem_cache/hiradix_cache.py @@ -653,7 +653,7 @@ class HiRadixCache(RadixCache): : alloc_len // self.page_size ] operation.host_indices = host_indices - cc.prefetch_buffer.put(operation) + cc.dispatch_prefetch(operation) def _drain_ack_prefetch(): for ack in _drain_queue(cc.ack_prefetch_queue, n_ack_prefetch): diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py index b58bb5c413..8a8157ebbf 100644 --- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py +++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py @@ -1006,7 +1006,7 @@ class HybridCacheController(BaseHiCacheController): if not self._allocate_pp_prefetch_buffers(ticket, operation): operation.host_indices = torch.empty(0, dtype=torch.int64) operation.mark_terminate() - self.prefetch_buffer.put(operation) + self.dispatch_prefetch(operation) except Exception: logger.exception("PP HiCache ticket processing failed.") if operation is None or not operation.hash_value: @@ -1016,7 +1016,7 @@ class HybridCacheController(BaseHiCacheController): operation.host_indices = torch.empty(0, dtype=torch.int64) operation.mark_terminate() # Preserve every KV/sidecar ACK even when local allocation fails. - self.prefetch_buffer.put(operation) + self.dispatch_prefetch(operation) finally: if is_source and objects: self.pp_prefetch_command_queue.task_done() @@ -1104,15 +1104,15 @@ class HybridCacheController(BaseHiCacheController): ) return host_indices, device_indices, resolved_pool_transfers - def _page_transfer(self, operation: PrefetchOperation) -> bool: + def _page_transfer(self, operation: PrefetchOperation, sync_queue: Queue) -> bool: # KV pools and KV-derived pools first — determines actual completed page count - kv_completed_pages = super()._page_transfer(operation) + kv_completed_pages = super()._page_transfer(operation, sync_queue) # Read non-KV derived sidecar pool, e.g. SWA, Mamba. - self._page_transfer_sidecar(operation, kv_completed_pages) + self._page_transfer_sidecar(operation, kv_completed_pages, sync_queue) def _page_transfer_sidecar( - self, operation: PrefetchOperation, kv_completed_pages: int + self, operation: PrefetchOperation, kv_completed_pages: int, sync_queue: Queue ) -> None: if operation.pool_transfers is None: return @@ -1144,10 +1144,10 @@ class HybridCacheController(BaseHiCacheController): transfers_nonkv, extra_info=extra_info ) pool_hits = count_pool_hits(results) - # Emit PrefetchAck to prefetch_sync_queue, even the operation has been canceled by the + # Emit PrefetchAck to the lane sync queue, even the operation has been canceled by the # scheduler thread. The prefetch sync thread expects the same number of PrefetchAck objects # to perform all_reduce. - self.prefetch_sync_queue.put( + sync_queue.put( PrefetchAck( rid=operation.request_id, operation=operation, @@ -1421,9 +1421,11 @@ class HybridCacheController(BaseHiCacheController): pool.device_indices = source.device_indices return extra_pools - def _reduce_prefetch_ack(self, ack: PrefetchAck) -> None: + def _reduce_prefetch_ack( + self, ack: PrefetchAck, groups: List[torch.distributed.ProcessGroup] + ) -> None: # Handle KV-derived pool. - super()._reduce_prefetch_ack(ack) + super()._reduce_prefetch_ack(ack, groups) # Handle other sidecar pools, e.g. SWA, Mamba. if ack.pool_hits is not None: @@ -1436,19 +1438,21 @@ class HybridCacheController(BaseHiCacheController): self._all_reduce( packed, torch.distributed.ReduceOp.MIN, - self.prefetch_completion_sync_groups, + groups, ) for i, pool in enumerate(PoolName): ack.pool_hits[pool.value] = packed[i].item() - def prefetch_sync_thread_func(self) -> None: + def prefetch_sync_thread_func(self, lane: int = 0) -> None: """Synchronize progressive ACKs and publish PP ticket completion.""" + sync_queue = self.prefetch_sync_queues[lane] + groups = self.prefetch_completion_sync_groups_by_lane[lane] while not self.storage_stop_event.is_set(): try: - ack = self.prefetch_sync_queue.get(block=True, timeout=1) + ack = sync_queue.get(block=True, timeout=1) if ack is None: continue - self._reduce_prefetch_ack(ack) + self._reduce_prefetch_ack(ack, groups) if not getattr(ack.operation, "is_pp_broadcast", False): if _hc_timing_enabled() and ack.completed_req: ack.operation.final_ack_enqueue_ts = time.monotonic() diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index 930d7854f4..d62147fb26 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -3001,7 +3001,7 @@ class UnifiedRadixCache(BasePrefixCache): self.ongoing_prefetch[request] = info._replace(host_indices=host_indices) if buffer_mode: cc.prefetch_tokens_occupied += alloc_len - cc.prefetch_buffer.put(operation) + cc.dispatch_prefetch(operation) return True def _drain_and_alloc_storage_hit(): diff --git a/test/registered/unit/mem_cache/test_pp_prefetch_ticket.py b/test/registered/unit/mem_cache/test_pp_prefetch_ticket.py index 6802af04e6..d75c6cf0f0 100644 --- a/test/registered/unit/mem_cache/test_pp_prefetch_ticket.py +++ b/test/registered/unit/mem_cache/test_pp_prefetch_ticket.py @@ -37,15 +37,17 @@ class TestPPPrefetchTicket(unittest.TestCase): c.pp_size, c.tp_size = 4, 2 c.pp_group, c.pp_prefetch_command_group = "pp", "command" c.pp_prefetch_command_thread = None - c.prefetch_hits_sync_groups = c.prefetch_completion_sync_groups = ["pp", "tp"] + c.prefetch_hits_sync_groups = ["pp", "tp"] + c.num_prefetch_io_lanes = 1 + c.prefetch_completion_sync_groups_by_lane = [["pp", "tp"]] c.pp_prefetch_states, c.pp_prefetch_decisions = {}, {} c.pp_prefetch_state_lock = threading.Lock() c.pp_prefetch_command_queue = Queue() c.prefetch_queue = Queue() - c.prefetch_sync_queue = Queue() + c.prefetch_sync_queues = [Queue()] c.ack_prefetch_queue = Queue() c.host_mem_release_queue = Queue() - c.prefetch_buffer = Queue() + c.prefetch_io_queues = [Queue()] c.storage_stop_event = threading.Event() c.prefetch_tokens_occupied = 0 c.mem_pool_host = Mock(page_size=4) @@ -103,7 +105,7 @@ class TestPPPrefetchTicket(unittest.TestCase): def sync_acks(self, *acks): for ack in acks: - self.c.prefetch_sync_queue.put(ack) + self.c.prefetch_sync_queues[0].put(ack) with patch.object( self.c.storage_stop_event, "is_set", @@ -183,7 +185,7 @@ class TestPPPrefetchTicket(unittest.TestCase): self.assertFalse(cache.check_prefetch_progress(handle)) c._storage_hit_query.reset_mock() self.run_commands(ticket) - operation = c.prefetch_buffer.get_nowait() + operation = c.prefetch_io_queues[0].get_nowait() self.assertEqual(operation.handle, handle) self.assertEqual( operation.hash_value, get_storage_hash_str(ticket.prefetch_key, page_size=4) @@ -237,7 +239,7 @@ class TestPPPrefetchTicket(unittest.TestCase): ), ): c.prefetch_io_aux_func() - acks = [c.prefetch_sync_queue.get_nowait() for _ in range(8)] + acks = [c.prefetch_sync_queues[0].get_nowait() for _ in range(8)] self.assertEqual([a.rid for a in acks], ["hit"] * 4 + ["next"] * 4) self.assertEqual( [a.completed_tokens for a in acks], [0, 0, None, None, 4, 8, None, None] @@ -261,7 +263,7 @@ class TestPPPrefetchTicket(unittest.TestCase): cache.finish(ticket.handle, CacheRequestOutcome.ABORT) self.assertIs(c.pp_prefetch_decisions["hit"], PPPrefetchDecision.CANCELLED) self.run_commands(ticket) - operation = c.prefetch_buffer.get_nowait() + operation = c.prefetch_io_queues[0].get_nowait() self.sync_acks(PrefetchAck("hit", operation, completed_tokens=4)) c.mem_pool_host.free.assert_not_called() self.sync_acks(PrefetchAck("hit", operation, completed_req=True)) @@ -293,7 +295,7 @@ class TestPPPrefetchTicket(unittest.TestCase): c.pp_prefetch_states.clear() c.mem_pool_host.alloc.reset_mock() self.run_commands(ticket) - operation = c.prefetch_buffer.get_nowait() + operation = c.prefetch_io_queues[0].get_nowait() self.assertEqual( c.mem_pool_host.alloc.call_args_list, [ @@ -311,7 +313,7 @@ class TestPPPrefetchTicket(unittest.TestCase): c.mem_pool_host.get_pool.side_effect = KeyError(PoolName.SWA) with self.assertLogs(level="ERROR"): self.run_commands(ticket) - self.assertTrue(c.prefetch_buffer.get_nowait().is_terminated()) + self.assertTrue(c.prefetch_io_queues[0].get_nowait().is_terminated()) self.assertEqual(c.mem_pool_host.free.call_count, 1) self.assertEqual(c.mem_pool_host.free.call_args.kwargs["pool"], PoolName.KV) @@ -387,8 +389,8 @@ class TestPPPrefetchTicket(unittest.TestCase): self.assertEqual(invocation.kwargs, {"src": 1}) self.assertEqual(c.pp_prefetch_command_queue.unfinished_tasks, 0) self.assertTrue(c.pp_prefetch_command_queue.empty()) - self.assertIs(c.prefetch_buffer.get_nowait(), operation) - self.assertTrue(c.prefetch_buffer.empty()) + self.assertIs(c.prefetch_io_queues[0].get_nowait(), operation) + self.assertTrue(c.prefetch_io_queues[0].empty()) c.mem_pool_host.alloc.assert_called_once() def test_idle_downstream_skips_empty_broadcasts_before_ticket_and_stop(self): @@ -404,8 +406,8 @@ class TestPPPrefetchTicket(unittest.TestCase): patch.object(c.pp_prefetch_command_queue, "task_done") as task_done, ): c.pp_prefetch_command_thread_func() - self.assertEqual(c.prefetch_buffer.get_nowait().request_id, "hit") - self.assertTrue(c.prefetch_buffer.empty()) + self.assertEqual(c.prefetch_io_queues[0].get_nowait().request_id, "hit") + self.assertTrue(c.prefetch_io_queues[0].empty()) c.mem_pool_host.alloc.assert_called_once() task_done.assert_not_called() diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py index 51ec4297fa..a0e9a02d3f 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py +++ b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py @@ -3635,14 +3635,14 @@ class UnifiedRadixCacheSuite: gate = threading.Event() swa_release_q = cc.extra_host_mem_release_queues.get(PoolName.SWA) - def _page_transfer_hook(_self, op): + def _page_transfer_hook(_self, op, sync_queue): # Signal that the worker has reached the sidecar step, then block # until the main thread releases the gate. entered.set() gate.wait(timeout=10.0) # Forward to the real implementation so the worker runs the # terminate-aware release path itself. - original_sidecar(_self, op) + original_sidecar(_self, op, sync_queue) # Simulate a slow prefetch IO. Hook on _page_transfer. with mock.patch.object( @@ -5290,7 +5290,7 @@ class UnifiedRadixCacheSuite: cc = cache.cache_controller # Fake _reduce_prefetch_ack to drop SWA pool_hits when drop_swa is True. - def fake_reduce(ack): + def fake_reduce(ack, groups=None): if drop_swa and ack.pool_hits is not None: if PoolName.SWA.value in ack.pool_hits: ack.pool_hits[PoolName.SWA.value] = 0 diff --git a/tools/sglang-simulator/src/sglang_simulator/simulation/sglang/cache_controller.py b/tools/sglang-simulator/src/sglang_simulator/simulation/sglang/cache_controller.py index 8421caf10e..29cae9a3af 100644 --- a/tools/sglang-simulator/src/sglang_simulator/simulation/sglang/cache_controller.py +++ b/tools/sglang-simulator/src/sglang_simulator/simulation/sglang/cache_controller.py @@ -37,10 +37,15 @@ class C_HiCacheController(BaseHook): def wrapped_init(self, *args, **kwargs): self.sim_prefetch_buffer = Queue() result = original_init(self, *args, **kwargs) - # The real IO thread normally creates this queue. The simulator - # replaces that thread, so initialize the handoff queue here. - if hasattr(self, "prefetch_hit_queue"): - self.prefetch_buffer = Queue() + # The real IO threads normally create the handoff queues. The + # simulator replaces those threads, so initialize the per-lane + # handoff queues here. + if hasattr(self, "prefetch_hit_queue") and not getattr( + self, "prefetch_io_queues", None + ): + self.prefetch_io_queues = [ + Queue() for _ in range(getattr(self, "num_prefetch_io_lanes", 1)) + ] return result def wrapped_append_host_mem_release(self, host_indices): @@ -182,11 +187,10 @@ class C_HiCacheController(BaseHook): # Feed operations whose host pages were allocated by the scheduler # into the virtual-time transfer loop. - prefetch_buffer = getattr(self, "prefetch_buffer", None) - if prefetch_buffer is not None: - while not prefetch_buffer.empty(): + for io_queue in getattr(self, "prefetch_io_queues", None) or []: + while True: try: - operation = prefetch_buffer.get(block=False) + operation = io_queue.get(block=False) if operation is not None: self.sim_prefetch_buffer.put(operation) except Empty: @@ -295,10 +299,12 @@ class C_HybridCacheController(BaseHook): def wrapped_init(self, *args, **kwargs): result = original_init(self, *args, **kwargs) - if hasattr(self, "prefetch_hit_queue") and not hasattr( - self, "prefetch_buffer" + if hasattr(self, "prefetch_hit_queue") and not getattr( + self, "prefetch_io_queues", None ): - self.prefetch_buffer = Queue() + self.prefetch_io_queues = [ + Queue() for _ in range(getattr(self, "num_prefetch_io_lanes", 1)) + ] return result def wrapped_storage_hit_query(self, operator):