diff --git a/python/sglang/srt/managers/cache_controller.py b/python/sglang/srt/managers/cache_controller.py index 08932ece2d..256ae63311 100644 --- a/python/sglang/srt/managers/cache_controller.py +++ b/python/sglang/srt/managers/cache_controller.py @@ -62,6 +62,12 @@ def storage_model_name( return f"{model_name}-{tag}" if model_name else tag +# [HC_PARALLEL_PATCH] knobs for the pipelined HiCache prefetch/backup lanes +import os as _hc_os +_HC_PREFETCH_WORKERS = int(_hc_os.getenv("SGLANG_HICACHE_PREFETCH_IO_THREADS", "1") or 1) +_HC_BACKUP_WORKERS = int(_hc_os.getenv("SGLANG_HICACHE_BACKUP_IO_THREADS", "1") or 1) +_HC_TIMING = _hc_os.getenv("SGLANG_HICACHE_TIMING", "0") not in ("", "0", "false", "False") + class LayerLoadingEvent: def __init__(self, num_layers: int): self._num_layers = num_layers @@ -1152,7 +1158,11 @@ class HiCacheController: count += 1 return count - def _page_transfer(self, operation: PrefetchOperation) -> int: + def _page_transfer( + self, + operation: PrefetchOperation, + ack_sink: Optional[Callable[[PrefetchAck], None]] = None, + ) -> int: # Transfer batch by batch prefix_keys = operation.prefix_keys kv_derived_transfers = [ @@ -1202,9 +1212,28 @@ class HiCacheController: completed_tokens=completed_pages * self.page_size, operation=operation, ) - self.prefetch_sync_queue.put(ack) + if ack_sink is None: + # Keep the direct-call behavior for existing users. The pipelined + # coordinator always supplies a sink so workers never share the queue. + self.prefetch_sync_queue.put(ack) + else: + ack_sink(ack) return completed_pages + def _transfer_prefetch_and_collect_acks( + self, operation: PrefetchOperation + ) -> List[PrefetchAck]: + """Run one worker without publishing ACKs to the shared sync queue.""" + acks = [] + try: + self._page_transfer(operation, ack_sink=acks.append) + except Exception: + logger.exception( + "HiCache prefetch transfer failed for request %s", + operation.request_id, + ) + return acks + def _page_transfer_kv_batch( self, operation: PrefetchOperation, @@ -1247,24 +1276,86 @@ class HiCacheController: def prefetch_io_aux_func(self): """ Auxiliary function conducting IO operations for prefetching. + + # [HC_PARALLEL_PATCH] pipelined: up to SGLANG_HICACHE_PREFETCH_IO_THREADS + # transfers in flight. Every ACK of one operation is published before any + # ACK of the next operation, keeping per-rank collective pairing unchanged. """ - while not self.storage_stop_event.is_set(): - try: - operation = self.prefetch_buffer.get(block=True, timeout=1) - except Empty: - continue - if operation is None: - continue - try: - self._page_transfer(operation) - finally: - self.prefetch_sync_queue.put( - PrefetchAck( - rid=operation.request_id, - completed_req=True, - operation=operation, + import time as _time + from collections import deque as _deque + from concurrent.futures import ThreadPoolExecutor as _TPE + + K = max(1, _HC_PREFETCH_WORKERS) + timing = _HC_TIMING + ex = _TPE(max_workers=K, thread_name_prefix="hc-prefetch-io") + inflight = _deque() + lat = [] + pages = 0 + try: + while not self.storage_stop_event.is_set(): + while len(inflight) < K: + try: + operation = self.prefetch_buffer.get_nowait() + except Empty: + break + if operation is None: + continue + inflight.append( + ( + operation, + _time.perf_counter(), + ex.submit( + self._transfer_prefetch_and_collect_acks, operation + ), + ) ) - ) + if not inflight: + try: + operation = self.prefetch_buffer.get(block=True, timeout=1) + except Empty: + continue + if operation is None: + continue + inflight.append( + ( + operation, + _time.perf_counter(), + ex.submit( + self._transfer_prefetch_and_collect_acks, operation + ), + ) + ) + operation, t0, fut = inflight[0] + try: + acks = fut.result() + except Exception: + logger.exception("HiCache prefetch transfer failed") + acks = [] + finally: + inflight.popleft() + if timing: + lat.append((_time.perf_counter() - t0) * 1000.0) + pages += len(getattr(operation, "hash_value", []) or []) + for ack in acks: + self.prefetch_sync_queue.put(ack) + self.prefetch_sync_queue.put( + PrefetchAck( + rid=operation.request_id, + completed_req=True, + operation=operation, + ) + ) + if timing and len(lat) >= 100: + lat.sort() + print("[HC_TIMING] prefetch queue=%d inflight=%d n=%d transfer_ms p50=%.1f p95=%.1f pages=%d" + % (self.prefetch_buffer.qsize(), len(inflight), len(lat), + lat[len(lat) // 2], lat[int(len(lat) * 0.95)], pages), flush=True) + lat = [] + pages = 0 + finally: + # Detach closes the backend after joining this outer thread. Do not + # let pool workers outlive that join and touch released state. + ex.shutdown(wait=True) def prefetch_rate_limited(self) -> bool: """ @@ -1443,22 +1534,82 @@ class HiCacheController: "HiCache backup failed after %d tokens", operation.completed_tokens ) + def _should_skip_backup(self) -> bool: + """Whether this rank can ACK without doing any storage IO.""" + return self.backup_skip + def backup_thread_func(self): """ Manage backup operations from host memory to storage backend. + + # [HC_PARALLEL_PATCH] pipelined: up to SGLANG_HICACHE_BACKUP_IO_THREADS + # writes in flight, acked in FIFO order. """ - while not self.storage_stop_event.is_set(): - try: - operation = self.backup_queue.get(block=True, timeout=1) - if operation is None: - continue + import time as _time + from collections import deque as _deque + from concurrent.futures import ThreadPoolExecutor as _TPE - if not self.backup_skip: - self._page_backup_or_log(operation) + K = max(1, _HC_BACKUP_WORKERS) + timing = _HC_TIMING + ex = _TPE(max_workers=K, thread_name_prefix="hc-backup-io") + inflight = _deque() + lat = [] + pages = 0 + + def submit(operation) -> bool: + if self._should_skip_backup(): self.ack_backup_queue.put(operation) + return False + inflight.append( + ( + operation, + _time.perf_counter(), + ex.submit(self._page_backup_or_log, operation), + ) + ) + return True - except Empty: - continue + try: + while not self.storage_stop_event.is_set(): + while len(inflight) < K: + try: + operation = self.backup_queue.get_nowait() + except Empty: + break + if operation is None: + continue + submit(operation) + if not inflight: + try: + operation = self.backup_queue.get(block=True, timeout=1) + except Empty: + continue + if operation is None: + continue + if not submit(operation): + continue + operation, t0, fut = inflight[0] + try: + fut.result() + except Exception: + logger.exception("HiCache backup failed") + finally: + inflight.popleft() + if timing: + lat.append((_time.perf_counter() - t0) * 1000.0) + pages += len(getattr(operation, "hash_value", []) or []) + self.ack_backup_queue.put(operation) + if timing and len(lat) >= 100: + lat.sort() + print("[HC_TIMING] backup queue=%d inflight=%d n=%d write_ms p50=%.1f p95=%.1f pages=%d" + % (self.backup_queue.qsize(), len(inflight), len(lat), + lat[len(lat) // 2], lat[int(len(lat) * 0.95)], pages), flush=True) + lat = [] + pages = 0 + finally: + # The outer thread is joined before backend/process-group teardown. + # Wait for every submitted write before reporting that thread stopped. + ex.shutdown(wait=True) def prefetch_sync_thread_func(self): """Synchronize prefetch results across all PP and TP ranks.""" diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py index 4710617f9a..dde54ef69d 100644 --- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py +++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py @@ -1547,19 +1547,31 @@ class HybridCacheController(BaseHiCacheController): ) return host_indices, device_indices, resolved_pool_transfers - def _page_transfer(self, operation: PrefetchOperation) -> bool: + def _page_transfer( + self, + operation: PrefetchOperation, + ack_sink: Optional[Callable[[PrefetchAck], None]] = None, + ) -> int: with self.mem_pool_host.layout_lease(): - return self._page_transfer_with_stable_layout(operation) + return self._page_transfer_with_stable_layout(operation, ack_sink) - def _page_transfer_with_stable_layout(self, operation: PrefetchOperation) -> None: + def _page_transfer_with_stable_layout( + self, + operation: PrefetchOperation, + ack_sink: Optional[Callable[[PrefetchAck], None]] = None, + ) -> int: # KV pools and KV-derived pools first — determines actual completed page count - kv_completed_pages = super()._page_transfer(operation) + kv_completed_pages = super()._page_transfer(operation, ack_sink) # Read non-KV derived sidecar pool, e.g. SWA, Mamba. - self._page_transfer_sidecar(operation, kv_completed_pages) + self._page_transfer_sidecar(operation, kv_completed_pages, ack_sink) + return kv_completed_pages def _page_transfer_sidecar( - self, operation: PrefetchOperation, kv_completed_pages: int + self, + operation: PrefetchOperation, + kv_completed_pages: int, + ack_sink: Optional[Callable[[PrefetchAck], None]] = None, ) -> None: if operation.pool_transfers is None: return @@ -1604,17 +1616,19 @@ class HybridCacheController(BaseHiCacheController): "HiCache sidecar prefetch failed for request %s", operation.request_id, ) - # Emit PrefetchAck to prefetch_sync_queue, even the operation has been canceled by the - # scheduler thread. The prefetch sync thread expects the same number of PrefetchAck objects - # to perform all_reduce. - self.prefetch_sync_queue.put( - PrefetchAck( - rid=operation.request_id, - operation=operation, - pool_hits=pool_hits, - ) + # Emit PrefetchAck even when the operation has been canceled by the + # scheduler thread. The sync thread expects the same number of ACKs on + # every rank. Pipelined workers buffer this ACK per operation instead of + # publishing it directly to the shared queue. + ack = PrefetchAck( + rid=operation.request_id, + operation=operation, + pool_hits=pool_hits, ) - return + if ack_sink is None: + self.prefetch_sync_queue.put(ack) + else: + ack_sink(ack) def trim_prefetch_full_head(self, operation, trim_tokens: int) -> None: """Drop a device-resident FULL head after the availability query; @@ -1704,22 +1718,15 @@ class HybridCacheController(BaseHiCacheController): return False - def backup_thread_func(self): - """Back up rank-sharded sidecars on every TP rank. + def _should_skip_backup(self) -> bool: + # Replicated primary MLA KV may be TP0-only, but rank-sharded sidecars + # still require IO on every rank. _page_backup applies should_backup() + # to each individual transfer, so the whole operation cannot be skipped. + return False - The base implementation skips the entire operation on non-zero MLA TP - ranks. That optimization is valid for replicated MLA KV, but not for - hybrid rank-sharded pools such as Kimi-K3 Mamba state. - """ - while not self.storage_stop_event.is_set(): - try: - operation = self.backup_queue.get(block=True, timeout=1) - if operation is None: - continue - self._page_backup_or_log(operation) - self.ack_backup_queue.put(operation) - except Empty: - continue + def backup_thread_func(self): + """Use the base pipeline while retaining per-pool writer selection.""" + super().backup_thread_func() def _resolve_sidecar_kv_derived_pool_transfers(self, operation): for transfer in operation.pool_transfers: diff --git a/test/registered/unit/mem_cache/test_hicache_io_pipeline.py b/test/registered/unit/mem_cache/test_hicache_io_pipeline.py new file mode 100644 index 0000000000..c498eba7ea --- /dev/null +++ b/test/registered/unit/mem_cache/test_hicache_io_pipeline.py @@ -0,0 +1,229 @@ +"""CPU regressions for pipelined HiCache storage IO ordering and shutdown.""" + +import threading +import time +import unittest +from contextlib import nullcontext +from queue import Queue +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import torch + +import sglang.srt.managers.cache_controller as cache_controller_module +from sglang.srt.managers.cache_controller import ( + HiCacheController, + StorageOperation, +) +from sglang.srt.mem_cache.base_prefix_cache import CacheRequestHandle +from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import ( + HybridCacheController, + PrefetchOperation, +) +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +def _drain(queue): + items = [] + while not queue.empty(): + items.append(queue.get_nowait()) + return items + + +def _wait_for_queue_size(queue, size, timeout=5): + deadline = time.monotonic() + timeout + while queue.qsize() < size and time.monotonic() < deadline: + time.sleep(0.005) + return queue.qsize() >= size + + +class TestPrefetchPipelineOrdering(unittest.TestCase): + def _controller(self, wait_for, peer_done): + controller = HybridCacheController.__new__(HybridCacheController) + controller.page_size = 1 + controller.prefetch_buffer = Queue() + controller.prefetch_sync_queue = Queue() + controller.storage_stop_event = threading.Event() + controller.mem_pool_host = SimpleNamespace( + layout_lease=lambda: nullcontext() + ) + controller.storage_backend = Mock() + controller.storage_backend.batch_get_v2.return_value = {} + + for rid in ("a", "b"): + operation = PrefetchOperation( + CacheRequestHandle(rid, 0), + list(range(2)), + pool_transfers=[], + ) + operation.hash_value = [f"{rid}-0", f"{rid}-1"] + operation.host_indices = torch.arange(2) + controller.prefetch_buffer.put(operation) + + def page_get(operation, batch_hashes, host_indices, extra_info): + if operation.request_id == wait_for: + if not peer_done.wait(timeout=5): + raise TimeoutError("peer prefetch did not finish") + return len(batch_hashes) + + controller.page_get_func = page_get + collect = controller._transfer_prefetch_and_collect_acks + + def collect_and_signal(operation): + acks = collect(operation) + if operation.request_id != wait_for: + # This rank's other operation has now completed. The delayed + # peer may resume, producing the opposite worker finish order. + peer_done.set() + return acks + + controller._transfer_prefetch_and_collect_acks = collect_and_signal + return controller + + def test_two_tp_ranks_keep_operation_ack_sequences_aligned(self): + # Rank 0 finishes A before B; rank 1 finishes B before A. Without + # per-operation buffering their progress/sidecar ACKs enter collectives + # in opposite orders even though completed_req itself remains FIFO. + rank0_done = threading.Event() + rank1_done = threading.Event() + rank0 = self._controller(wait_for="b", peer_done=rank0_done) + rank1 = self._controller(wait_for="a", peer_done=rank1_done) + threads = [ + threading.Thread(target=rank.prefetch_io_aux_func, daemon=True) + for rank in (rank0, rank1) + ] + + with ( + patch.object(cache_controller_module, "_HC_PREFETCH_WORKERS", 2), + patch.object(cache_controller_module, "STORAGE_BATCH_SIZE", 1), + ): + for thread in threads: + thread.start() + try: + self.assertTrue(_wait_for_queue_size(rank0.prefetch_sync_queue, 8)) + self.assertTrue(_wait_for_queue_size(rank1.prefetch_sync_queue, 8)) + finally: + rank0.storage_stop_event.set() + rank1.storage_stop_event.set() + rank0_done.set() + rank1_done.set() + for thread in threads: + thread.join(timeout=5) + for thread in threads: + self.assertFalse(thread.is_alive()) + + rank0_acks = _drain(rank0.prefetch_sync_queue) + rank1_acks = _drain(rank1.prefetch_sync_queue) + + def ack_kind(ack): + if ack.completed_req: + return "completed_req" + if ack.completed_tokens is not None: + return "completed_tokens" + if ack.pool_hits is not None: + return "pool_hits" + raise AssertionError(f"unknown ACK: {ack}") + + self.assertEqual([ack.rid for ack in rank0_acks], ["a"] * 4 + ["b"] * 4) + self.assertEqual( + [(ack.rid, ack_kind(ack)) for ack in rank0_acks], + [(ack.rid, ack_kind(ack)) for ack in rank1_acks], + ) + for acks in (rank0_acks, rank1_acks): + self.assertEqual( + [ack_kind(ack) for ack in acks[:4]], + ["completed_tokens", "completed_tokens", "pool_hits", "completed_req"], + ) + self.assertEqual( + [ack.completed_tokens for ack in acks[:2]], + [1, 2], + ) + + +class TestBackupPipeline(unittest.TestCase): + @staticmethod + def _controller(controller_type, is_storage_writer, backup): + controller = controller_type.__new__(controller_type) + controller.backup_queue = Queue() + controller.ack_backup_queue = Queue() + controller.storage_stop_event = threading.Event() + controller.storage_config = SimpleNamespace( + is_storage_writer=is_storage_writer + ) + controller._page_backup_or_log = backup + return controller + + def _run_one(self, controller, operation): + thread = threading.Thread(target=controller.backup_thread_func, daemon=True) + with patch.object(cache_controller_module, "_HC_BACKUP_WORKERS", 1): + controller.backup_queue.put(operation) + thread.start() + try: + ack = controller.ack_backup_queue.get(timeout=5) + finally: + controller.storage_stop_event.set() + thread.join(timeout=5) + self.assertFalse(thread.is_alive()) + self.assertIs(ack, operation) + + def test_non_writer_never_submits_backup_io(self): + backup = Mock() + controller = self._controller(HiCacheController, False, backup) + operation = StorageOperation(None, []) + self._run_one(controller, operation) + backup.assert_not_called() + + def test_hybrid_non_writer_still_runs_rank_sharded_backup(self): + backup = Mock() + controller = self._controller(HybridCacheController, False, backup) + operation = StorageOperation(None, []) + self._run_one(controller, operation) + backup.assert_called_once_with(operation) + + def test_stop_waits_for_every_backup_worker(self): + started = [threading.Event(), threading.Event()] + release = [threading.Event(), threading.Event()] + finished = [threading.Event(), threading.Event()] + + def backup(operation): + index = operation.token_ids[0] + started[index].set() + if not release[index].wait(timeout=5): + raise TimeoutError(f"backup {index} was not released") + finished[index].set() + + controller = self._controller(HiCacheController, True, backup) + for index in range(2): + controller.backup_queue.put(StorageOperation(None, [index])) + thread = threading.Thread(target=controller.backup_thread_func, daemon=True) + + with patch.object(cache_controller_module, "_HC_BACKUP_WORKERS", 2): + thread.start() + try: + self.assertTrue(started[0].wait(timeout=5)) + self.assertTrue(started[1].wait(timeout=5)) + controller.storage_stop_event.set() + release[0].set() + + # The coordinator may finish operation 0, but shutdown cannot + # return while operation 1 still uses the storage backend. + thread.join(timeout=0.1) + self.assertTrue(thread.is_alive()) + self.assertFalse(finished[1].is_set()) + + release[1].set() + thread.join(timeout=5) + self.assertFalse(thread.is_alive()) + self.assertTrue(finished[0].is_set()) + self.assertTrue(finished[1].is_set()) + finally: + controller.storage_stop_event.set() + for event in release: + event.set() + thread.join(timeout=5) + + +if __name__ == "__main__": + unittest.main()