From 453b251dbd0f9ca6e3de2ddec99a4ce7fe723e0f Mon Sep 17 00:00:00 2001 From: metamergebot Date: Fri, 16 Oct 2026 10:37:11 -0700 Subject: [PATCH] [Fix] hicache: Bound the storage prefetch timeout from the first blocked admission poll (#44614) Co-authored-by: metamergebot Co-authored-by: xiezhq-hermann --- python/sglang/srt/managers/schedule_policy.py | 3 + .../hybrid_cache/hybrid_cache_controller.py | 2 + .../unified_cache/storage_attachment.py | 1 + .../srt/mem_cache/unified_radix_cache.py | 126 ++++++++---- .../srt/observability/metrics_collector.py | 4 +- .../srt/observability/req_time_stats.py | 53 ++--- python/sglang/srt/utils/poll_timer.py | 26 +++ .../unit/managers/test_prefill_adder.py | 4 +- .../unit/mem_cache/test_pp_prefetch_ticket.py | 1 + .../test_storage_prefetch_lifecycle.py | 187 +++++++++++++++++- 10 files changed, 333 insertions(+), 74 deletions(-) create mode 100644 python/sglang/srt/utils/poll_timer.py diff --git a/python/sglang/srt/managers/schedule_policy.py b/python/sglang/srt/managers/schedule_policy.py index ce0dffb826..8b31d88d0f 100644 --- a/python/sglang/srt/managers/schedule_policy.py +++ b/python/sglang/srt/managers/schedule_policy.py @@ -923,6 +923,9 @@ class PrefillAdder: self.tree_cache.discard_storage_prefetch_accounting( req.cache_request_handle ) + self.tree_cache.finish_storage_prefetch_admission( + req.cache_request_handle, fulfilled_tokens=0, reason=None + ) return if prefix_len > 0: diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py index 11cb913517..da8eedb1a9 100644 --- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py +++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py @@ -184,6 +184,8 @@ class PrefetchOperation(StorageOperation): # query. Trailing sidecars still use the untrimmed hit endpoint. self.sidecar_hash_values: Optional[list[str]] = None self.sidecar_hit_pages = 0 + # Cut by the stop policy, not a storage shortfall: the span is still stored. + self.deadline_expired = False def mark_terminate(self): with self._lock: diff --git a/python/sglang/srt/mem_cache/unified_cache/storage_attachment.py b/python/sglang/srt/mem_cache/unified_cache/storage_attachment.py index 6854a4ef91..b98cdd627a 100644 --- a/python/sglang/srt/mem_cache/unified_cache/storage_attachment.py +++ b/python/sglang/srt/mem_cache/unified_cache/storage_attachment.py @@ -451,3 +451,4 @@ class StorageAttachment: cache.prefetch_loaded_tokens_by_reqid.clear() cache.prefetch_loaded_storage_start_by_reqid.clear() cache.storage_prefetch_retries.clear() + cache._storage_prefetch_wait.clear() diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index d3473f744b..cf098c884c 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -10,6 +10,7 @@ from http import HTTPStatus from queue import Queue from typing import TYPE_CHECKING, Iterator, NamedTuple, Optional, Sequence, TypeVar +import msgspec import torch from sglang.srt.distributed.communication_tags import P2PTag @@ -105,6 +106,7 @@ from sglang.srt.observability.metrics_collector import ( from sglang.srt.runtime_context import get_memory, get_model, get_observability from sglang.srt.session.streaming_session import StreamingSession, is_virtual_node from sglang.srt.utils.common import ceil_align +from sglang.srt.utils.poll_timer import PollTimer if TYPE_CHECKING: from sglang.srt.managers.cache_controller import HiCacheAck @@ -161,6 +163,12 @@ class _OngoingLoadBack(NamedTuple): host_lock_params: DecLockRefParams +class _StoragePrefetchWait(msgspec.Struct): + timer: PollTimer = msgspec.field(default_factory=PollTimer) + timeout_s: float = 0.0 + expired: bool = False + + class _OngoingPrefetch(NamedTuple): """Tracks an in-flight storage→host prefetch operation.""" @@ -404,6 +412,7 @@ class UnifiedRadixCache(BasePrefixCache): self.prefetch_loaded_tokens_by_reqid: dict[CacheRequestHandle, int] = {} self.prefetch_loaded_storage_start_by_reqid: dict[CacheRequestHandle, int] = {} self.ongoing_prefetch: dict[CacheRequestHandle, _OngoingPrefetch] = {} + self._storage_prefetch_wait: dict[CacheRequestHandle, _StoragePrefetchWait] = {} # Rank-agreed L3-hit tokens not yet resolved as usable or unfulfilled. # Cache-mode entries survive L3->L2 until H2D succeeds or admission # fails; buffer-mode entries survive staging until the H2D ack. @@ -2009,6 +2018,9 @@ class UnifiedRadixCache(BasePrefixCache): req_id = request.rid self.storage_prefetch_retries.cancel(req_id) + wait = self._storage_prefetch_wait.get(request) + if wait is not None and wait.expired: + return submission = self.cache_controller.get_prefetch_submission(req_id) if submission is not None: return submission.decision @@ -2138,8 +2150,8 @@ class UnifiedRadixCache(BasePrefixCache): stats["issued"] += 1 if operation.assume_stored: stats["issued_assumed_stored"] += 1 - # Snapshot the requested span for L3 miss-token accounting at the - # rank-synchronized query outcome. + # Keep the requested span stable for the wait budget and miss accounting; + # the hit hash chain can shrink as device coverage changes. operation.stats_requested_tokens = prefetch_length operation.storage_start = storage_start if submission.decision is not None: @@ -2167,13 +2179,41 @@ class UnifiedRadixCache(BasePrefixCache): # grants occupancy later at hit-alloc time, sized to the hit. self.cache_controller.prefetch_tokens_occupied += len(prefetch_key) - def _prefetch_timeout_check_linear_func(self, operation: PrefetchOperation) -> bool: - return ( - time.monotonic() - operation.start_time - > self.prefetch_timeout_base - + len(operation.hash_value) * self.prefetch_timeout_per_page + def _start_storage_prefetch_wait(self, operation: PrefetchOperation) -> None: + if ( + self.prefetch_stop_policy == "timeout" + and operation.handle not in self._storage_prefetch_wait + ): + # Start at the first blocked admission poll; refetches keep this budget. + pages = operation.stats_requested_tokens // self.page_size + wait = _StoragePrefetchWait( + timeout_s=self.prefetch_timeout_base + + pages * self.prefetch_timeout_per_page, + ) + wait.timer.observe(ready=False) + self._storage_prefetch_wait[operation.handle] = wait + + def _prefetch_deadline_expired(self, operation: PrefetchOperation) -> bool: + if operation.host_indices is not None and ( + operation.completed_tokens >= len(operation.hash_value) * self.page_size + ): + # KV landed: cutting the pending sidecar read would salvage nothing. + return False + wait = self._storage_prefetch_wait.get(operation.handle) + return wait is not None and ( + wait.expired + or (wait.timer.running and wait.timer.elapsed() > wait.timeout_s) ) + def _mark_prefetch_deadline_expired(self, operation: PrefetchOperation) -> None: + operation.deadline_expired = True + self.storage_prefetch_retries.cancel(operation.handle.rid) + # Only a rank-agreed verdict can suppress retries, including on peers + # that have not reached their conditional admission poll yet. + self._storage_prefetch_wait.setdefault( + operation.handle, _StoragePrefetchWait() + ).expired = True + @rank_consensus(same_results=True) def _can_terminate_prefetch(self, operation: PrefetchOperation) -> bool: if self.prefetch_stop_policy == "best_effort": @@ -2187,7 +2227,7 @@ class UnifiedRadixCache(BasePrefixCache): # rank timed out) and _all_reduce broadcasts the verdict along PP. should_terminate = False if self.pp_rank == 0: - should_terminate = self._prefetch_timeout_check_linear_func(operation) + should_terminate = self._prefetch_deadline_expired(operation) should_terminate_tensor = torch.tensor( int(should_terminate), dtype=torch.int, device="cpu" ) @@ -2267,17 +2307,19 @@ class UnifiedRadixCache(BasePrefixCache): operation = self.ongoing_prefetch[request].operation - # Determine whether or not we should terminate this prefetch request. - should_terminate = operation.is_terminated() or self._can_terminate_prefetch( - operation - ) - - if not should_terminate: - return False + if not operation.is_terminated(): + if not self._can_terminate_prefetch(operation): + self._start_storage_prefetch_wait(operation) + return False + if self.prefetch_stop_policy == "timeout": + self._mark_prefetch_deadline_expired(operation) self.cache_controller.terminate_prefetch(operation) if operation.host_indices is None: - self.storage_prefetch_retries.poll_miss(request.rid) + if operation.deadline_expired: + self._finish_storage_prefetch(request, 0, reason="deadline") + else: + self.storage_prefetch_retries.poll_miss(request.rid) self.revoke_pending_prefetch(request) else: self._handle_prefetch_result(operation) @@ -2317,7 +2359,7 @@ class UnifiedRadixCache(BasePrefixCache): self._resolve_storage_prefetch_tokens( request, allocated_tokens - completed_tokens, - reason="storage_transfer", + reason="deadline" if operation.deadline_expired else "storage_transfer", ) if ( completed_tokens > 0 @@ -2445,7 +2487,7 @@ class UnifiedRadixCache(BasePrefixCache): if pool_transfers and not all_succeeded: # Drop each pool's beliefs beyond what it served (KV and its # sidecars beyond the completed span); the next insert re-writes - # only the missing pools. + # only the missing pools. A deadline is not evidence of absent data. invalidation_hashes = ( operation.sidecar_hash_values if operation.sidecar_hash_values is not None @@ -2454,7 +2496,7 @@ class UnifiedRadixCache(BasePrefixCache): trimmed_pages = len(invalidation_hashes) - len(hash_value) keep_pages = trimmed_pages + completed_tokens // self.page_size beliefs = self.storage_existence_cache - if keep_pages < len(invalidation_hashes): + if not operation.deadline_expired and keep_pages < len(invalidation_hashes): self._belief_policy( PoolName.KV, PoolHitPolicy.ALL_PAGES ).heal_from_transfer( @@ -2470,15 +2512,18 @@ class UnifiedRadixCache(BasePrefixCache): keep_pages, ) for transfer, count in zip(pool_transfers, pool_hit_pages): - if transfer.keys is not None and count < len(transfer.keys): + if ( + not operation.deadline_expired + and transfer.keys is not None + and count < len(transfer.keys) + ): self._belief_policy( transfer.name, transfer.hit_policy ).heal_from_transfer( beliefs.pool(transfer.name), transfer.keys, count ) - self._finish_storage_prefetch( - request, fulfilled_tokens=0, reason="storage_transfer" - ) + reason = "deadline" if operation.deadline_expired else "storage_transfer" + self._finish_storage_prefetch(request, fulfilled_tokens=0, reason=reason) # The controller's prefetch IO thread already releases the untransferred # tail (host_indices[completed_tokens:]) self._retire_ongoing_prefetch( @@ -2489,12 +2534,13 @@ class UnifiedRadixCache(BasePrefixCache): self.prefetch_loaded_tokens_by_reqid[request] = 0 self.prefetch_loaded_storage_start_by_reqid.pop(request, None) logger.warning( - "HiCache hybrid prefetch discarded req=%s completed=%d requested=%d " - "kv_beliefs_kept_pages=%d aux_pool_hit_pages=%s", + "HiCache hybrid prefetch discarded req=%s reason=%s completed=%d " + "requested=%d kv_beliefs_kept_pages=%s aux_pool_hit_pages=%s", request.rid, + reason, completed_tokens, expected_tokens, - keep_pages, + None if operation.deadline_expired else keep_pages, { getattr(transfer.name, "value", transfer.name): count for transfer, count in zip(pool_transfers, pool_hit_pages) @@ -2586,11 +2632,12 @@ class UnifiedRadixCache(BasePrefixCache): def finish_storage_prefetch_admission( self, request: CacheRequestHandle, fulfilled_tokens: int, reason: Optional[str] ) -> None: - """Resolve a cache-mode L3 hit after request admission. + """Close the admission wait budget and resolve a cache-mode L3 hit. Buffer mode resolves at the H2D completion ack instead, because its host allocation is transport staging rather than a resident L2 hit. """ + self._storage_prefetch_wait.pop(request, None) if self.host_memory_mode == "cache": self._finish_storage_prefetch(request, fulfilled_tokens, reason) @@ -2627,6 +2674,7 @@ class UnifiedRadixCache(BasePrefixCache): @rank_consensus(same_params=True) def release_aborted_request(self, request: CacheRequestHandle) -> None: rid = request.rid + self._storage_prefetch_wait.pop(request, None) if self.linker is not None: self.linker.release_request(rid) self.prefetch_loaded_tokens_by_reqid.pop(request, None) @@ -2702,7 +2750,8 @@ class UnifiedRadixCache(BasePrefixCache): """Heal each pool's beliefs beyond its rank-agreed boundary (a chain-page pool also learns presence up to it); a pool without a verdict heals from the folded cut.""" - if self.host_memory_mode != "buffer_only": + if self.host_memory_mode != "buffer_only" or operation.deadline_expired: + # A cut query's agreed hit may be a MIN over skipped lookups, not a verdict. return chain = operation.all_hash_values if chain is None: @@ -2736,7 +2785,7 @@ class UnifiedRadixCache(BasePrefixCache): """Feed the cumulative prefetch-outcome counters at the (rank-synced) query outcome: T = prompt tokens, L = requested, m = L3-miss.""" requested = operation.stats_requested_tokens - if requested <= 0: + if requested <= 0 or operation.deadline_expired: return stats = self._prefetch_outcome_stats hit = max(0, min(operation.storage_hit_count, requested)) @@ -3027,22 +3076,21 @@ class UnifiedRadixCache(BasePrefixCache): if hit_tokens > 0: self._record_storage_prefetch_hit(request, hit_tokens) if operation.is_terminated(): - # Controller-side miss termination (retryable) or an abort - # race (abort cleanup discards the marker). + # A storage miss can retry; a policy deadline cannot. if hit_tokens > 0: + reason = None + if operation.deadline_expired: + reason = "deadline" + elif hit_tokens < self.prefetch_threshold: + reason = "below_threshold" self._finish_storage_prefetch( - request, - fulfilled_tokens=0, - reason=( - "below_threshold" - if hit_tokens < self.prefetch_threshold - else None - ), + request, fulfilled_tokens=0, reason=reason ) # The published (rank-agreed) hit count is the L3 verdict; # without this, store misses never reach l3_miss_tokens. self._account_prefetch_outcome(operation, revoked=True) - self.storage_prefetch_retries.poll_miss(request.rid) + if not operation.deadline_expired: + self.storage_prefetch_retries.poll_miss(request.rid) self.revoke_pending_prefetch(request) continue if hit_tokens < self.prefetch_threshold: diff --git a/python/sglang/srt/observability/metrics_collector.py b/python/sglang/srt/observability/metrics_collector.py index 7f74a5a203..508feed7ca 100644 --- a/python/sglang/srt/observability/metrics_collector.py +++ b/python/sglang/srt/observability/metrics_collector.py @@ -2175,7 +2175,8 @@ class StorageMetricsCollector(_StatLoggerDIMixin): "device_overlap means the live device prefix diverged from the " "request's view at splice time, so the span beyond it was released; " "cache_admission_shortfall means an L3-loaded L2 span was no longer " - "reusable when cache-mode admission ran.", + "reusable when cache-mode admission ran; deadline means the " + "storage prefetch stop policy ended the read before it completed.", labelnames=list(labels.keys()) + ["reason"], ) for reason in ( @@ -2184,6 +2185,7 @@ class StorageMetricsCollector(_StatLoggerDIMixin): "device_capacity", "device_covered", "storage_transfer", + "deadline", "anchor_lost", "aux_window_trim", "device_overlap", diff --git a/python/sglang/srt/observability/req_time_stats.py b/python/sglang/srt/observability/req_time_stats.py index 068059ecb2..8cf4785366 100644 --- a/python/sglang/srt/observability/req_time_stats.py +++ b/python/sglang/srt/observability/req_time_stats.py @@ -42,6 +42,7 @@ from sglang.srt.observability.trace_async import ( is_async_tracing_available, ) from sglang.srt.utils import get_bool_env_var +from sglang.srt.utils.poll_timer import PollTimer if TYPE_CHECKING: from sglang.srt.disaggregation.base.conn import KVTransferMetric @@ -668,8 +669,8 @@ class SchedulerReqTimeStats(ReqTimeStatsBase): # None means unmeasured (including stats received from an older scheduler). storage_prefetch_wait_s: Optional[float] = None storage_staging_wait_s: Optional[float] = None - _storage_prefetch_wait_start: Optional[float] = None - _storage_staging_wait_start: Optional[float] = None + _storage_prefetch_wait: Optional[PollTimer] = None + _storage_staging_wait: Optional[PollTimer] = None has_timing_data: bool = False @@ -687,12 +688,16 @@ class SchedulerReqTimeStats(ReqTimeStatsBase): "queue_duration_s": self.queue_duration_s, "storage_prefetch_wait_s": ( self.storage_prefetch_wait_s - if self._storage_prefetch_wait_start is None + if not ( + self._storage_prefetch_wait and self._storage_prefetch_wait.running + ) else None ), "storage_staging_wait_s": ( self.storage_staging_wait_s - if self._storage_staging_wait_start is None + if not ( + self._storage_staging_wait and self._storage_staging_wait.running + ) else None ), "diff_realtime_monotonic": global_diff_realtime_monotonic, @@ -756,33 +761,17 @@ class SchedulerReqTimeStats(ReqTimeStatsBase): self.queue_duration_s = 0.0 self.storage_prefetch_wait_s = None self.storage_staging_wait_s = None - self._storage_prefetch_wait_start = None - self._storage_staging_wait_start = None + self._storage_prefetch_wait = None + self._storage_staging_wait = None def observe_storage_prefetch_ready(self, ready: bool): """Accumulate first blocked poll to the subsequent ready poll, per episode.""" - if self.storage_prefetch_wait_s is None: - return - if ready: - if self._storage_prefetch_wait_start is not None: - self.storage_prefetch_wait_s += ( - time.perf_counter() - self._storage_prefetch_wait_start - ) - self._storage_prefetch_wait_start = None - elif self._storage_prefetch_wait_start is None: - self._storage_prefetch_wait_start = time.perf_counter() + if self._storage_prefetch_wait is not None: + self.storage_prefetch_wait_s += self._storage_prefetch_wait.observe(ready) def observe_storage_staging_ready(self, ready: bool): - if self.storage_staging_wait_s is None: - return - if ready: - if self._storage_staging_wait_start is not None: - self.storage_staging_wait_s += ( - time.perf_counter() - self._storage_staging_wait_start - ) - self._storage_staging_wait_start = None - elif self._storage_staging_wait_start is None: - self._storage_staging_wait_start = time.perf_counter() + if self._storage_staging_wait is not None: + self.storage_staging_wait_s += self._storage_staging_wait.observe(ready) def set_wait_queue_entry_time(self, ts=None): ts = ts or time.perf_counter() @@ -790,6 +779,8 @@ class SchedulerReqTimeStats(ReqTimeStatsBase): if self.enable_metrics and self.disagg_mode != DisaggregationMode.DECODE: self.storage_prefetch_wait_s = 0.0 self.storage_staging_wait_s = 0.0 + self._storage_prefetch_wait = PollTimer() + self._storage_staging_wait = PollTimer() if self.enable_metrics or self.trace_ctx.tracing_enable: if self.disagg_mode == DisaggregationMode.PREFILL: stage = RequestStage.PREFILL_BOOTSTRAP @@ -1221,14 +1212,12 @@ class SchedulerReqTimeStats(ReqTimeStatsBase): def convert_to_output_meta_info(self): meta_data = {} - if ( - self.storage_prefetch_wait_s is not None - and self._storage_prefetch_wait_start is None + if self.storage_prefetch_wait_s is not None and not ( + self._storage_prefetch_wait and self._storage_prefetch_wait.running ): meta_data["storage_prefetch_wait"] = self.storage_prefetch_wait_s - if ( - self.storage_staging_wait_s is not None - and self._storage_staging_wait_start is None + if self.storage_staging_wait_s is not None and not ( + self._storage_staging_wait and self._storage_staging_wait.running ): meta_data["storage_staging_wait"] = self.storage_staging_wait_s if self.forward_entry_time > 0.0: diff --git a/python/sglang/srt/utils/poll_timer.py b/python/sglang/srt/utils/poll_timer.py new file mode 100644 index 0000000000..2abb62bb52 --- /dev/null +++ b/python/sglang/srt/utils/poll_timer.py @@ -0,0 +1,26 @@ +import time +from typing import Optional + +import msgspec + + +class PollTimer(msgspec.Struct): + """Time an interval from its first blocked poll to its first ready poll.""" + + started_at: Optional[float] = None + + @property + def running(self) -> bool: + return self.started_at is not None + + def elapsed(self) -> float: + return time.perf_counter() - self.started_at if self.running else 0.0 + + def observe(self, ready: bool) -> float: + if ready: + elapsed = self.elapsed() + self.started_at = None + return elapsed + if not self.running: + self.started_at = time.perf_counter() + return 0.0 diff --git a/test/registered/unit/managers/test_prefill_adder.py b/test/registered/unit/managers/test_prefill_adder.py index df62e9a032..0d90203997 100644 --- a/test/registered/unit/managers/test_prefill_adder.py +++ b/test/registered/unit/managers/test_prefill_adder.py @@ -497,7 +497,9 @@ class TestPrefillAdder(CustomTestCase): self.mock_tree_cache.discard_storage_prefetch_accounting.assert_called_once_with( req.cache_request_handle ) - self.mock_tree_cache.finish_storage_prefetch_admission.assert_not_called() + self.mock_tree_cache.finish_storage_prefetch_admission.assert_called_once_with( + req.cache_request_handle, fulfilled_tokens=0, reason=None + ) def test_preempt_success_high_priority_values_first(self): params = [ diff --git a/test/registered/unit/mem_cache/test_pp_prefetch_ticket.py b/test/registered/unit/mem_cache/test_pp_prefetch_ticket.py index 4bd1669d63..4314040a70 100644 --- a/test/registered/unit/mem_cache/test_pp_prefetch_ticket.py +++ b/test/registered/unit/mem_cache/test_pp_prefetch_ticket.py @@ -90,6 +90,7 @@ class TestPPPrefetchTicket(unittest.TestCase): cache.prefetch_loaded_tokens_by_reqid = {} cache.prefetch_loaded_storage_start_by_reqid = {} cache.storage_prefetch_retries = StoragePrefetchRetries() + cache._storage_prefetch_wait = {} cache.linker = None cache.root_node_handle = Mock(return_value=0) cache.buffer_pipeline = Mock(spec=BufferModePipeline) diff --git a/test/registered/unit/mem_cache/test_storage_prefetch_lifecycle.py b/test/registered/unit/mem_cache/test_storage_prefetch_lifecycle.py index c49f4446fc..1ddfe4b5df 100644 --- a/test/registered/unit/mem_cache/test_storage_prefetch_lifecycle.py +++ b/test/registered/unit/mem_cache/test_storage_prefetch_lifecycle.py @@ -7,7 +7,7 @@ from collections import defaultdict, deque from datetime import timedelta from queue import Queue from types import SimpleNamespace -from unittest.mock import Mock +from unittest.mock import Mock, patch import torch @@ -32,6 +32,7 @@ from sglang.srt.mem_cache.unified_radix_cache import ( _OngoingPrefetch, ) from sglang.srt.mem_cache.utils import get_hash_str +from sglang.srt.observability.req_time_stats import SchedulerReqTimeStats from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci(est_time=20, suite="base-a-test-cpu") @@ -59,6 +60,7 @@ def _staged_fixture(full_match=2): cache.host_memory_mode = "buffer_only" cache.linker = None cache.storage_prefetch_retries = StoragePrefetchRetries() + cache._storage_prefetch_wait = {} cache.prefetch_loaded_tokens_by_reqid = {_REQ: 6} cache.prefetch_loaded_storage_start_by_reqid = {_REQ: 2} cache._storage_prefetch_hit_remaining_by_reqid = {} @@ -144,6 +146,7 @@ def _hit_drain_fixture(): cache.enable_storage_metrics = False cache.storage_metrics_collector = None cache.storage_prefetch_retries = StoragePrefetchRetries() + cache._storage_prefetch_wait = {} cache._prefetch_outcome_stats = defaultdict(int) cache._storage_prefetch_hit_remaining_by_reqid = {} cache._record_storage_prefetch_hit = Mock() @@ -168,6 +171,7 @@ def _terminated_query(cache, rid, hit_tokens): storage_hit_count=hit_tokens, stats_requested_tokens=8, all_hash_values=None, + deadline_expired=False, is_terminated=lambda: True, ) cache.ongoing_prefetch[handle] = _OngoingPrefetch( @@ -176,6 +180,186 @@ def _terminated_query(cache, rid, hit_tokens): cache.cache_controller.prefetch_hit_queue.put(operation) +def _timeout_fixture(mode="buffer_only", policy="timeout"): + cache, pipeline, req = _staged_fixture() + pipeline.staged_prefetches.clear() + cache.host_memory_mode = mode + if mode == "cache": + cache.buffer_pipeline = None + cache.prefetch_stop_policy = policy + cache.prefetch_timeout_base = 1.0 + cache.prefetch_timeout_per_page = 0.25 + cache.pp_rank = 0 + cache._all_reduce = lambda tensor, op: None + cache.inc_host_lock_ref = Mock() + cache.dec_host_lock_ref = Mock() + cache.cache_controller.pp_prefetch_decisions = {} + cache.cache_controller.prefetch_tokens_occupied = 0 + cache.prefetch_from_storage(_REQ, 0, list(range(8))) + return cache, cache.ongoing_prefetch[_REQ].operation + + +def _two_rank_prefetch_deadline(rank, rendezvous): + torch.distributed.init_process_group( + "gloo", + init_method=f"file://{rendezvous}", + rank=rank, + world_size=2, + timeout=timedelta(seconds=30), + ) + try: + cache, operation = _timeout_fixture() + cache._all_reduce = torch.distributed.all_reduce + # Only one rank has reached admission; its expiry must close both budgets. + if rank == 0: + with patch("time.perf_counter", return_value=100.0): + cache._start_storage_prefetch_wait(operation) + with patch("time.perf_counter", return_value=103.0): + assert cache.check_prefetch_progress(_REQ) + assert operation.deadline_expired + assert cache._storage_prefetch_wait[_REQ].expired + cache.prefetch_from_storage(_REQ, 0, list(range(8))) + assert not cache.ongoing_prefetch + assert not cache.storage_prefetch_retries._pending + finally: + torch.distributed.destroy_process_group() + + +class TestStoragePrefetchTimeout(unittest.TestCase): + def test_metric_episodes_do_not_restart_the_timeout(self): + """Closing a wait-metric episode on readiness must not restart the + timeout budget: a refetch keeps the budget until admission.""" + for enable_metrics in (False, True): + with self.subTest(enable_metrics=enable_metrics): + cache, _ = _timeout_fixture() + stats = SchedulerReqTimeStats( + enable_metrics=enable_metrics, metrics_collector=Mock() + ) + stats.set_wait_queue_entry_time(99.0) + + def poll(now): + with patch("time.perf_counter", return_value=now): + ready = cache.check_prefetch_progress(_REQ) + stats.observe_storage_prefetch_ready(ready) + return ready + + self.assertFalse(poll(100.0)) + self.assertFalse(poll(100.5)) + # Readiness closes a metric episode; a subsequent refetch must + # retain the timeout budget until the request is admitted. + cache.revoke_pending_prefetch(_REQ) + self.assertTrue(poll(101.0)) + cache.prefetch_from_storage(_REQ, 0, list(range(8))) + self.assertFalse(poll(101.5)) + self.assertTrue(poll(102.1)) + self.assertTrue(cache._storage_prefetch_wait[_REQ].expired) + meta = stats.convert_to_output_meta_info() + if enable_metrics: + self.assertAlmostEqual(meta["storage_prefetch_wait"], 1.6) + else: + self.assertNotIn("storage_prefetch_wait", meta) + + def test_budget_starts_at_first_wait_and_survives_refetch(self): + """The budget is sized from the first requested span at the first + blocked poll and is kept across re-issues until admission or abort.""" + for mode in ("buffer_only", "cache"): + with self.subTest(mode=mode): + cache, operation = _timeout_fixture(mode) + operation.start_time = 0.0 + with patch("time.perf_counter", return_value=100.0): + self.assertFalse(cache.check_prefetch_progress(_REQ)) + self.assertEqual( + cache._storage_prefetch_wait[_REQ].timer.started_at, 100.0 + ) + self.assertEqual(cache._storage_prefetch_wait[_REQ].timeout_s, 2.0) + # A shorter hit cannot reduce the original four-page allowance. + operation.hash_value = ["hit"] + with patch("time.perf_counter", return_value=101.5): + self.assertFalse(cache.check_prefetch_progress(_REQ)) + cache.revoke_pending_prefetch(_REQ) + cache.prefetch_from_storage(_REQ, 0, list(range(4))) + refetch = cache.ongoing_prefetch[_REQ].operation + self.assertIsNot(refetch, operation) + self.assertFalse(cache.check_prefetch_progress(_REQ)) + self.assertEqual( + cache._storage_prefetch_wait[_REQ].timer.started_at, 100.0 + ) + self.assertEqual(cache._storage_prefetch_wait[_REQ].timeout_s, 2.0) + with patch("time.perf_counter", return_value=102.1): + self.assertTrue(cache.check_prefetch_progress(_REQ)) + self.assertTrue(refetch.deadline_expired) + self.assertNotIn(_REQ, cache.ongoing_prefetch) + cache.storage_prefetch_retries.refetch(_REQ.rid) + cache.prefetch_from_storage(_REQ, 0, list(range(8))) + self.assertNotIn(_REQ, cache.ongoing_prefetch) + self.assertFalse(cache.storage_prefetch_retries._pending) + cache.finish_storage_prefetch_admission(_REQ, 0, None) + self.assertFalse(cache._storage_prefetch_wait) + cache.prefetch_from_storage(_REQ, 0, list(range(8))) + with patch("time.perf_counter", return_value=200.0): + self.assertFalse(cache.check_prefetch_progress(_REQ)) + self.assertEqual( + cache._storage_prefetch_wait[_REQ].timer.started_at, 200.0 + ) + self.assertEqual(cache._storage_prefetch_wait[_REQ].timeout_s, 2.0) + cache.release_aborted_request(_REQ) + self.assertFalse(cache._storage_prefetch_wait) + + def test_timeout_preserves_complete_kv_with_pending_sidecar(self): + """A lapsed budget must not cut a read whose KV already landed and + only waits on a sidecar: cutting it would salvage nothing.""" + for mode in ("buffer_only", "cache"): + for remaining_pages in (0, 2): + with self.subTest(mode=mode, remaining_pages=remaining_pages): + cache, operation = _timeout_fixture(mode) + with patch("time.perf_counter", return_value=100.0): + self.assertFalse(cache.check_prefetch_progress(_REQ)) + operation.hash_value = ["hit"] * remaining_pages + operation.host_indices = torch.arange(remaining_pages * 2) + operation.completed_tokens = remaining_pages * 2 + operation.pool_transfers = [PoolTransfer(name=PoolName.SWA)] + operation.pool_transfers_done = False + with patch("time.perf_counter", return_value=200.0): + self.assertFalse(cache.check_prefetch_progress(_REQ)) + self.assertFalse(operation.deadline_expired) + + def test_stop_policies_distinguish_deadline_from_storage_failure(self): + """Only the timeout policy marks a deadline cut; a storage failure + under any policy terminates without the deadline flag.""" + for policy in ("timeout", "wait_complete", "best_effort"): + for failed in (False, True): + with self.subTest(policy=policy, failed=failed): + cache, operation = _timeout_fixture(policy=policy) + operation.hash_value = ["a", "b", "c", "d"] + operation.host_indices = torch.arange(8) + operation.completed_tokens = 2 + cache._handle_prefetch_result = Mock() + with patch("time.perf_counter", return_value=100.0): + cache.check_prefetch_progress(_REQ) + if failed: + operation.mark_terminate() + with patch("time.perf_counter", return_value=200.0): + self.assertEqual( + cache.check_prefetch_progress(_REQ), + failed or policy != "wait_complete", + ) + self.assertEqual( + operation.deadline_expired, policy == "timeout" and not failed + ) + if policy != "timeout": + self.assertFalse(cache._storage_prefetch_wait) + + def test_expiry_is_agreed_even_before_a_peer_starts_waiting(self): + """A rank that has not reached its admission poll must still adopt + the agreed expiry, or it would re-issue the prefetch alone.""" + with tempfile.TemporaryDirectory() as tmpdir: + torch.multiprocessing.spawn( + _two_rank_prefetch_deadline, + args=(f"{tmpdir}/gloo",), + nprocs=2, + ) + + def _two_rank_retry_trace(rank, rendezvous): torch.distributed.init_process_group( "gloo", @@ -577,6 +761,7 @@ class TestStagedPrefetchLifecycle(unittest.TestCase): stats_requested_tokens=8, is_terminated=lambda: False, all_hash_values=None, + deadline_expired=False, ) cache.ongoing_prefetch[handle] = _OngoingPrefetch( 0, RadixKey(array("q", range(8))), None, operation, None, {}