[Fix] hicache: Bound the storage prefetch timeout from the first blocked admission poll (#44614)

Co-authored-by: metamergebot <metamergebot@users.noreply.github.com>
Co-authored-by: xiezhq-hermann <xiezhq-hermann@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-16 10:37:11 -07:00
committed by GitHub
co-authored by metamergebot xiezhq-hermann
parent 989ccf4863
commit 453b251dbd
10 changed files with 333 additions and 74 deletions
@@ -923,6 +923,9 @@ class PrefillAdder:
self.tree_cache.discard_storage_prefetch_accounting(
req.cache_request_handle
)
self.tree_cache.finish_storage_prefetch_admission(
req.cache_request_handle, fulfilled_tokens=0, reason=None
)
return
if prefix_len > 0:
@@ -184,6 +184,8 @@ class PrefetchOperation(StorageOperation):
# query. Trailing sidecars still use the untrimmed hit endpoint.
self.sidecar_hash_values: Optional[list[str]] = None
self.sidecar_hit_pages = 0
# Cut by the stop policy, not a storage shortfall: the span is still stored.
self.deadline_expired = False
def mark_terminate(self):
with self._lock:
@@ -451,3 +451,4 @@ class StorageAttachment:
cache.prefetch_loaded_tokens_by_reqid.clear()
cache.prefetch_loaded_storage_start_by_reqid.clear()
cache.storage_prefetch_retries.clear()
cache._storage_prefetch_wait.clear()
@@ -10,6 +10,7 @@ from http import HTTPStatus
from queue import Queue
from typing import TYPE_CHECKING, Iterator, NamedTuple, Optional, Sequence, TypeVar
import msgspec
import torch
from sglang.srt.distributed.communication_tags import P2PTag
@@ -105,6 +106,7 @@ from sglang.srt.observability.metrics_collector import (
from sglang.srt.runtime_context import get_memory, get_model, get_observability
from sglang.srt.session.streaming_session import StreamingSession, is_virtual_node
from sglang.srt.utils.common import ceil_align
from sglang.srt.utils.poll_timer import PollTimer
if TYPE_CHECKING:
from sglang.srt.managers.cache_controller import HiCacheAck
@@ -161,6 +163,12 @@ class _OngoingLoadBack(NamedTuple):
host_lock_params: DecLockRefParams
class _StoragePrefetchWait(msgspec.Struct):
timer: PollTimer = msgspec.field(default_factory=PollTimer)
timeout_s: float = 0.0
expired: bool = False
class _OngoingPrefetch(NamedTuple):
"""Tracks an in-flight storage→host prefetch operation."""
@@ -404,6 +412,7 @@ class UnifiedRadixCache(BasePrefixCache):
self.prefetch_loaded_tokens_by_reqid: dict[CacheRequestHandle, int] = {}
self.prefetch_loaded_storage_start_by_reqid: dict[CacheRequestHandle, int] = {}
self.ongoing_prefetch: dict[CacheRequestHandle, _OngoingPrefetch] = {}
self._storage_prefetch_wait: dict[CacheRequestHandle, _StoragePrefetchWait] = {}
# Rank-agreed L3-hit tokens not yet resolved as usable or unfulfilled.
# Cache-mode entries survive L3->L2 until H2D succeeds or admission
# fails; buffer-mode entries survive staging until the H2D ack.
@@ -2009,6 +2018,9 @@ class UnifiedRadixCache(BasePrefixCache):
req_id = request.rid
self.storage_prefetch_retries.cancel(req_id)
wait = self._storage_prefetch_wait.get(request)
if wait is not None and wait.expired:
return
submission = self.cache_controller.get_prefetch_submission(req_id)
if submission is not None:
return submission.decision
@@ -2138,8 +2150,8 @@ class UnifiedRadixCache(BasePrefixCache):
stats["issued"] += 1
if operation.assume_stored:
stats["issued_assumed_stored"] += 1
# Snapshot the requested span for L3 miss-token accounting at the
# rank-synchronized query outcome.
# Keep the requested span stable for the wait budget and miss accounting;
# the hit hash chain can shrink as device coverage changes.
operation.stats_requested_tokens = prefetch_length
operation.storage_start = storage_start
if submission.decision is not None:
@@ -2167,13 +2179,41 @@ class UnifiedRadixCache(BasePrefixCache):
# grants occupancy later at hit-alloc time, sized to the hit.
self.cache_controller.prefetch_tokens_occupied += len(prefetch_key)
def _prefetch_timeout_check_linear_func(self, operation: PrefetchOperation) -> bool:
return (
time.monotonic() - operation.start_time
> self.prefetch_timeout_base
+ len(operation.hash_value) * self.prefetch_timeout_per_page
def _start_storage_prefetch_wait(self, operation: PrefetchOperation) -> None:
if (
self.prefetch_stop_policy == "timeout"
and operation.handle not in self._storage_prefetch_wait
):
# Start at the first blocked admission poll; refetches keep this budget.
pages = operation.stats_requested_tokens // self.page_size
wait = _StoragePrefetchWait(
timeout_s=self.prefetch_timeout_base
+ pages * self.prefetch_timeout_per_page,
)
wait.timer.observe(ready=False)
self._storage_prefetch_wait[operation.handle] = wait
def _prefetch_deadline_expired(self, operation: PrefetchOperation) -> bool:
if operation.host_indices is not None and (
operation.completed_tokens >= len(operation.hash_value) * self.page_size
):
# KV landed: cutting the pending sidecar read would salvage nothing.
return False
wait = self._storage_prefetch_wait.get(operation.handle)
return wait is not None and (
wait.expired
or (wait.timer.running and wait.timer.elapsed() > wait.timeout_s)
)
def _mark_prefetch_deadline_expired(self, operation: PrefetchOperation) -> None:
operation.deadline_expired = True
self.storage_prefetch_retries.cancel(operation.handle.rid)
# Only a rank-agreed verdict can suppress retries, including on peers
# that have not reached their conditional admission poll yet.
self._storage_prefetch_wait.setdefault(
operation.handle, _StoragePrefetchWait()
).expired = True
@rank_consensus(same_results=True)
def _can_terminate_prefetch(self, operation: PrefetchOperation) -> bool:
if self.prefetch_stop_policy == "best_effort":
@@ -2187,7 +2227,7 @@ class UnifiedRadixCache(BasePrefixCache):
# rank timed out) and _all_reduce broadcasts the verdict along PP.
should_terminate = False
if self.pp_rank == 0:
should_terminate = self._prefetch_timeout_check_linear_func(operation)
should_terminate = self._prefetch_deadline_expired(operation)
should_terminate_tensor = torch.tensor(
int(should_terminate), dtype=torch.int, device="cpu"
)
@@ -2267,17 +2307,19 @@ class UnifiedRadixCache(BasePrefixCache):
operation = self.ongoing_prefetch[request].operation
# Determine whether or not we should terminate this prefetch request.
should_terminate = operation.is_terminated() or self._can_terminate_prefetch(
operation
)
if not should_terminate:
return False
if not operation.is_terminated():
if not self._can_terminate_prefetch(operation):
self._start_storage_prefetch_wait(operation)
return False
if self.prefetch_stop_policy == "timeout":
self._mark_prefetch_deadline_expired(operation)
self.cache_controller.terminate_prefetch(operation)
if operation.host_indices is None:
self.storage_prefetch_retries.poll_miss(request.rid)
if operation.deadline_expired:
self._finish_storage_prefetch(request, 0, reason="deadline")
else:
self.storage_prefetch_retries.poll_miss(request.rid)
self.revoke_pending_prefetch(request)
else:
self._handle_prefetch_result(operation)
@@ -2317,7 +2359,7 @@ class UnifiedRadixCache(BasePrefixCache):
self._resolve_storage_prefetch_tokens(
request,
allocated_tokens - completed_tokens,
reason="storage_transfer",
reason="deadline" if operation.deadline_expired else "storage_transfer",
)
if (
completed_tokens > 0
@@ -2445,7 +2487,7 @@ class UnifiedRadixCache(BasePrefixCache):
if pool_transfers and not all_succeeded:
# Drop each pool's beliefs beyond what it served (KV and its
# sidecars beyond the completed span); the next insert re-writes
# only the missing pools.
# only the missing pools. A deadline is not evidence of absent data.
invalidation_hashes = (
operation.sidecar_hash_values
if operation.sidecar_hash_values is not None
@@ -2454,7 +2496,7 @@ class UnifiedRadixCache(BasePrefixCache):
trimmed_pages = len(invalidation_hashes) - len(hash_value)
keep_pages = trimmed_pages + completed_tokens // self.page_size
beliefs = self.storage_existence_cache
if keep_pages < len(invalidation_hashes):
if not operation.deadline_expired and keep_pages < len(invalidation_hashes):
self._belief_policy(
PoolName.KV, PoolHitPolicy.ALL_PAGES
).heal_from_transfer(
@@ -2470,15 +2512,18 @@ class UnifiedRadixCache(BasePrefixCache):
keep_pages,
)
for transfer, count in zip(pool_transfers, pool_hit_pages):
if transfer.keys is not None and count < len(transfer.keys):
if (
not operation.deadline_expired
and transfer.keys is not None
and count < len(transfer.keys)
):
self._belief_policy(
transfer.name, transfer.hit_policy
).heal_from_transfer(
beliefs.pool(transfer.name), transfer.keys, count
)
self._finish_storage_prefetch(
request, fulfilled_tokens=0, reason="storage_transfer"
)
reason = "deadline" if operation.deadline_expired else "storage_transfer"
self._finish_storage_prefetch(request, fulfilled_tokens=0, reason=reason)
# The controller's prefetch IO thread already releases the untransferred
# tail (host_indices[completed_tokens:])
self._retire_ongoing_prefetch(
@@ -2489,12 +2534,13 @@ class UnifiedRadixCache(BasePrefixCache):
self.prefetch_loaded_tokens_by_reqid[request] = 0
self.prefetch_loaded_storage_start_by_reqid.pop(request, None)
logger.warning(
"HiCache hybrid prefetch discarded req=%s completed=%d requested=%d "
"kv_beliefs_kept_pages=%d aux_pool_hit_pages=%s",
"HiCache hybrid prefetch discarded req=%s reason=%s completed=%d "
"requested=%d kv_beliefs_kept_pages=%s aux_pool_hit_pages=%s",
request.rid,
reason,
completed_tokens,
expected_tokens,
keep_pages,
None if operation.deadline_expired else keep_pages,
{
getattr(transfer.name, "value", transfer.name): count
for transfer, count in zip(pool_transfers, pool_hit_pages)
@@ -2586,11 +2632,12 @@ class UnifiedRadixCache(BasePrefixCache):
def finish_storage_prefetch_admission(
self, request: CacheRequestHandle, fulfilled_tokens: int, reason: Optional[str]
) -> None:
"""Resolve a cache-mode L3 hit after request admission.
"""Close the admission wait budget and resolve a cache-mode L3 hit.
Buffer mode resolves at the H2D completion ack instead, because its
host allocation is transport staging rather than a resident L2 hit.
"""
self._storage_prefetch_wait.pop(request, None)
if self.host_memory_mode == "cache":
self._finish_storage_prefetch(request, fulfilled_tokens, reason)
@@ -2627,6 +2674,7 @@ class UnifiedRadixCache(BasePrefixCache):
@rank_consensus(same_params=True)
def release_aborted_request(self, request: CacheRequestHandle) -> None:
rid = request.rid
self._storage_prefetch_wait.pop(request, None)
if self.linker is not None:
self.linker.release_request(rid)
self.prefetch_loaded_tokens_by_reqid.pop(request, None)
@@ -2702,7 +2750,8 @@ class UnifiedRadixCache(BasePrefixCache):
"""Heal each pool's beliefs beyond its rank-agreed boundary (a
chain-page pool also learns presence up to it); a pool without a
verdict heals from the folded cut."""
if self.host_memory_mode != "buffer_only":
if self.host_memory_mode != "buffer_only" or operation.deadline_expired:
# A cut query's agreed hit may be a MIN over skipped lookups, not a verdict.
return
chain = operation.all_hash_values
if chain is None:
@@ -2736,7 +2785,7 @@ class UnifiedRadixCache(BasePrefixCache):
"""Feed the cumulative prefetch-outcome counters at the (rank-synced)
query outcome: T = prompt tokens, L = requested, m = L3-miss."""
requested = operation.stats_requested_tokens
if requested <= 0:
if requested <= 0 or operation.deadline_expired:
return
stats = self._prefetch_outcome_stats
hit = max(0, min(operation.storage_hit_count, requested))
@@ -3027,22 +3076,21 @@ class UnifiedRadixCache(BasePrefixCache):
if hit_tokens > 0:
self._record_storage_prefetch_hit(request, hit_tokens)
if operation.is_terminated():
# Controller-side miss termination (retryable) or an abort
# race (abort cleanup discards the marker).
# A storage miss can retry; a policy deadline cannot.
if hit_tokens > 0:
reason = None
if operation.deadline_expired:
reason = "deadline"
elif hit_tokens < self.prefetch_threshold:
reason = "below_threshold"
self._finish_storage_prefetch(
request,
fulfilled_tokens=0,
reason=(
"below_threshold"
if hit_tokens < self.prefetch_threshold
else None
),
request, fulfilled_tokens=0, reason=reason
)
# The published (rank-agreed) hit count is the L3 verdict;
# without this, store misses never reach l3_miss_tokens.
self._account_prefetch_outcome(operation, revoked=True)
self.storage_prefetch_retries.poll_miss(request.rid)
if not operation.deadline_expired:
self.storage_prefetch_retries.poll_miss(request.rid)
self.revoke_pending_prefetch(request)
continue
if hit_tokens < self.prefetch_threshold:
@@ -2175,7 +2175,8 @@ class StorageMetricsCollector(_StatLoggerDIMixin):
"device_overlap means the live device prefix diverged from the "
"request's view at splice time, so the span beyond it was released; "
"cache_admission_shortfall means an L3-loaded L2 span was no longer "
"reusable when cache-mode admission ran.",
"reusable when cache-mode admission ran; deadline means the "
"storage prefetch stop policy ended the read before it completed.",
labelnames=list(labels.keys()) + ["reason"],
)
for reason in (
@@ -2184,6 +2185,7 @@ class StorageMetricsCollector(_StatLoggerDIMixin):
"device_capacity",
"device_covered",
"storage_transfer",
"deadline",
"anchor_lost",
"aux_window_trim",
"device_overlap",
@@ -42,6 +42,7 @@ from sglang.srt.observability.trace_async import (
is_async_tracing_available,
)
from sglang.srt.utils import get_bool_env_var
from sglang.srt.utils.poll_timer import PollTimer
if TYPE_CHECKING:
from sglang.srt.disaggregation.base.conn import KVTransferMetric
@@ -668,8 +669,8 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
# None means unmeasured (including stats received from an older scheduler).
storage_prefetch_wait_s: Optional[float] = None
storage_staging_wait_s: Optional[float] = None
_storage_prefetch_wait_start: Optional[float] = None
_storage_staging_wait_start: Optional[float] = None
_storage_prefetch_wait: Optional[PollTimer] = None
_storage_staging_wait: Optional[PollTimer] = None
has_timing_data: bool = False
@@ -687,12 +688,16 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
"queue_duration_s": self.queue_duration_s,
"storage_prefetch_wait_s": (
self.storage_prefetch_wait_s
if self._storage_prefetch_wait_start is None
if not (
self._storage_prefetch_wait and self._storage_prefetch_wait.running
)
else None
),
"storage_staging_wait_s": (
self.storage_staging_wait_s
if self._storage_staging_wait_start is None
if not (
self._storage_staging_wait and self._storage_staging_wait.running
)
else None
),
"diff_realtime_monotonic": global_diff_realtime_monotonic,
@@ -756,33 +761,17 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
self.queue_duration_s = 0.0
self.storage_prefetch_wait_s = None
self.storage_staging_wait_s = None
self._storage_prefetch_wait_start = None
self._storage_staging_wait_start = None
self._storage_prefetch_wait = None
self._storage_staging_wait = None
def observe_storage_prefetch_ready(self, ready: bool):
"""Accumulate first blocked poll to the subsequent ready poll, per episode."""
if self.storage_prefetch_wait_s is None:
return
if ready:
if self._storage_prefetch_wait_start is not None:
self.storage_prefetch_wait_s += (
time.perf_counter() - self._storage_prefetch_wait_start
)
self._storage_prefetch_wait_start = None
elif self._storage_prefetch_wait_start is None:
self._storage_prefetch_wait_start = time.perf_counter()
if self._storage_prefetch_wait is not None:
self.storage_prefetch_wait_s += self._storage_prefetch_wait.observe(ready)
def observe_storage_staging_ready(self, ready: bool):
if self.storage_staging_wait_s is None:
return
if ready:
if self._storage_staging_wait_start is not None:
self.storage_staging_wait_s += (
time.perf_counter() - self._storage_staging_wait_start
)
self._storage_staging_wait_start = None
elif self._storage_staging_wait_start is None:
self._storage_staging_wait_start = time.perf_counter()
if self._storage_staging_wait is not None:
self.storage_staging_wait_s += self._storage_staging_wait.observe(ready)
def set_wait_queue_entry_time(self, ts=None):
ts = ts or time.perf_counter()
@@ -790,6 +779,8 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
if self.enable_metrics and self.disagg_mode != DisaggregationMode.DECODE:
self.storage_prefetch_wait_s = 0.0
self.storage_staging_wait_s = 0.0
self._storage_prefetch_wait = PollTimer()
self._storage_staging_wait = PollTimer()
if self.enable_metrics or self.trace_ctx.tracing_enable:
if self.disagg_mode == DisaggregationMode.PREFILL:
stage = RequestStage.PREFILL_BOOTSTRAP
@@ -1221,14 +1212,12 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
def convert_to_output_meta_info(self):
meta_data = {}
if (
self.storage_prefetch_wait_s is not None
and self._storage_prefetch_wait_start is None
if self.storage_prefetch_wait_s is not None and not (
self._storage_prefetch_wait and self._storage_prefetch_wait.running
):
meta_data["storage_prefetch_wait"] = self.storage_prefetch_wait_s
if (
self.storage_staging_wait_s is not None
and self._storage_staging_wait_start is None
if self.storage_staging_wait_s is not None and not (
self._storage_staging_wait and self._storage_staging_wait.running
):
meta_data["storage_staging_wait"] = self.storage_staging_wait_s
if self.forward_entry_time > 0.0:
+26
View File
@@ -0,0 +1,26 @@
import time
from typing import Optional
import msgspec
class PollTimer(msgspec.Struct):
"""Time an interval from its first blocked poll to its first ready poll."""
started_at: Optional[float] = None
@property
def running(self) -> bool:
return self.started_at is not None
def elapsed(self) -> float:
return time.perf_counter() - self.started_at if self.running else 0.0
def observe(self, ready: bool) -> float:
if ready:
elapsed = self.elapsed()
self.started_at = None
return elapsed
if not self.running:
self.started_at = time.perf_counter()
return 0.0
@@ -497,7 +497,9 @@ class TestPrefillAdder(CustomTestCase):
self.mock_tree_cache.discard_storage_prefetch_accounting.assert_called_once_with(
req.cache_request_handle
)
self.mock_tree_cache.finish_storage_prefetch_admission.assert_not_called()
self.mock_tree_cache.finish_storage_prefetch_admission.assert_called_once_with(
req.cache_request_handle, fulfilled_tokens=0, reason=None
)
def test_preempt_success_high_priority_values_first(self):
params = [
@@ -90,6 +90,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
cache.prefetch_loaded_tokens_by_reqid = {}
cache.prefetch_loaded_storage_start_by_reqid = {}
cache.storage_prefetch_retries = StoragePrefetchRetries()
cache._storage_prefetch_wait = {}
cache.linker = None
cache.root_node_handle = Mock(return_value=0)
cache.buffer_pipeline = Mock(spec=BufferModePipeline)
@@ -7,7 +7,7 @@ from collections import defaultdict, deque
from datetime import timedelta
from queue import Queue
from types import SimpleNamespace
from unittest.mock import Mock
from unittest.mock import Mock, patch
import torch
@@ -32,6 +32,7 @@ from sglang.srt.mem_cache.unified_radix_cache import (
_OngoingPrefetch,
)
from sglang.srt.mem_cache.utils import get_hash_str
from sglang.srt.observability.req_time_stats import SchedulerReqTimeStats
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=20, suite="base-a-test-cpu")
@@ -59,6 +60,7 @@ def _staged_fixture(full_match=2):
cache.host_memory_mode = "buffer_only"
cache.linker = None
cache.storage_prefetch_retries = StoragePrefetchRetries()
cache._storage_prefetch_wait = {}
cache.prefetch_loaded_tokens_by_reqid = {_REQ: 6}
cache.prefetch_loaded_storage_start_by_reqid = {_REQ: 2}
cache._storage_prefetch_hit_remaining_by_reqid = {}
@@ -144,6 +146,7 @@ def _hit_drain_fixture():
cache.enable_storage_metrics = False
cache.storage_metrics_collector = None
cache.storage_prefetch_retries = StoragePrefetchRetries()
cache._storage_prefetch_wait = {}
cache._prefetch_outcome_stats = defaultdict(int)
cache._storage_prefetch_hit_remaining_by_reqid = {}
cache._record_storage_prefetch_hit = Mock()
@@ -168,6 +171,7 @@ def _terminated_query(cache, rid, hit_tokens):
storage_hit_count=hit_tokens,
stats_requested_tokens=8,
all_hash_values=None,
deadline_expired=False,
is_terminated=lambda: True,
)
cache.ongoing_prefetch[handle] = _OngoingPrefetch(
@@ -176,6 +180,186 @@ def _terminated_query(cache, rid, hit_tokens):
cache.cache_controller.prefetch_hit_queue.put(operation)
def _timeout_fixture(mode="buffer_only", policy="timeout"):
cache, pipeline, req = _staged_fixture()
pipeline.staged_prefetches.clear()
cache.host_memory_mode = mode
if mode == "cache":
cache.buffer_pipeline = None
cache.prefetch_stop_policy = policy
cache.prefetch_timeout_base = 1.0
cache.prefetch_timeout_per_page = 0.25
cache.pp_rank = 0
cache._all_reduce = lambda tensor, op: None
cache.inc_host_lock_ref = Mock()
cache.dec_host_lock_ref = Mock()
cache.cache_controller.pp_prefetch_decisions = {}
cache.cache_controller.prefetch_tokens_occupied = 0
cache.prefetch_from_storage(_REQ, 0, list(range(8)))
return cache, cache.ongoing_prefetch[_REQ].operation
def _two_rank_prefetch_deadline(rank, rendezvous):
torch.distributed.init_process_group(
"gloo",
init_method=f"file://{rendezvous}",
rank=rank,
world_size=2,
timeout=timedelta(seconds=30),
)
try:
cache, operation = _timeout_fixture()
cache._all_reduce = torch.distributed.all_reduce
# Only one rank has reached admission; its expiry must close both budgets.
if rank == 0:
with patch("time.perf_counter", return_value=100.0):
cache._start_storage_prefetch_wait(operation)
with patch("time.perf_counter", return_value=103.0):
assert cache.check_prefetch_progress(_REQ)
assert operation.deadline_expired
assert cache._storage_prefetch_wait[_REQ].expired
cache.prefetch_from_storage(_REQ, 0, list(range(8)))
assert not cache.ongoing_prefetch
assert not cache.storage_prefetch_retries._pending
finally:
torch.distributed.destroy_process_group()
class TestStoragePrefetchTimeout(unittest.TestCase):
def test_metric_episodes_do_not_restart_the_timeout(self):
"""Closing a wait-metric episode on readiness must not restart the
timeout budget: a refetch keeps the budget until admission."""
for enable_metrics in (False, True):
with self.subTest(enable_metrics=enable_metrics):
cache, _ = _timeout_fixture()
stats = SchedulerReqTimeStats(
enable_metrics=enable_metrics, metrics_collector=Mock()
)
stats.set_wait_queue_entry_time(99.0)
def poll(now):
with patch("time.perf_counter", return_value=now):
ready = cache.check_prefetch_progress(_REQ)
stats.observe_storage_prefetch_ready(ready)
return ready
self.assertFalse(poll(100.0))
self.assertFalse(poll(100.5))
# Readiness closes a metric episode; a subsequent refetch must
# retain the timeout budget until the request is admitted.
cache.revoke_pending_prefetch(_REQ)
self.assertTrue(poll(101.0))
cache.prefetch_from_storage(_REQ, 0, list(range(8)))
self.assertFalse(poll(101.5))
self.assertTrue(poll(102.1))
self.assertTrue(cache._storage_prefetch_wait[_REQ].expired)
meta = stats.convert_to_output_meta_info()
if enable_metrics:
self.assertAlmostEqual(meta["storage_prefetch_wait"], 1.6)
else:
self.assertNotIn("storage_prefetch_wait", meta)
def test_budget_starts_at_first_wait_and_survives_refetch(self):
"""The budget is sized from the first requested span at the first
blocked poll and is kept across re-issues until admission or abort."""
for mode in ("buffer_only", "cache"):
with self.subTest(mode=mode):
cache, operation = _timeout_fixture(mode)
operation.start_time = 0.0
with patch("time.perf_counter", return_value=100.0):
self.assertFalse(cache.check_prefetch_progress(_REQ))
self.assertEqual(
cache._storage_prefetch_wait[_REQ].timer.started_at, 100.0
)
self.assertEqual(cache._storage_prefetch_wait[_REQ].timeout_s, 2.0)
# A shorter hit cannot reduce the original four-page allowance.
operation.hash_value = ["hit"]
with patch("time.perf_counter", return_value=101.5):
self.assertFalse(cache.check_prefetch_progress(_REQ))
cache.revoke_pending_prefetch(_REQ)
cache.prefetch_from_storage(_REQ, 0, list(range(4)))
refetch = cache.ongoing_prefetch[_REQ].operation
self.assertIsNot(refetch, operation)
self.assertFalse(cache.check_prefetch_progress(_REQ))
self.assertEqual(
cache._storage_prefetch_wait[_REQ].timer.started_at, 100.0
)
self.assertEqual(cache._storage_prefetch_wait[_REQ].timeout_s, 2.0)
with patch("time.perf_counter", return_value=102.1):
self.assertTrue(cache.check_prefetch_progress(_REQ))
self.assertTrue(refetch.deadline_expired)
self.assertNotIn(_REQ, cache.ongoing_prefetch)
cache.storage_prefetch_retries.refetch(_REQ.rid)
cache.prefetch_from_storage(_REQ, 0, list(range(8)))
self.assertNotIn(_REQ, cache.ongoing_prefetch)
self.assertFalse(cache.storage_prefetch_retries._pending)
cache.finish_storage_prefetch_admission(_REQ, 0, None)
self.assertFalse(cache._storage_prefetch_wait)
cache.prefetch_from_storage(_REQ, 0, list(range(8)))
with patch("time.perf_counter", return_value=200.0):
self.assertFalse(cache.check_prefetch_progress(_REQ))
self.assertEqual(
cache._storage_prefetch_wait[_REQ].timer.started_at, 200.0
)
self.assertEqual(cache._storage_prefetch_wait[_REQ].timeout_s, 2.0)
cache.release_aborted_request(_REQ)
self.assertFalse(cache._storage_prefetch_wait)
def test_timeout_preserves_complete_kv_with_pending_sidecar(self):
"""A lapsed budget must not cut a read whose KV already landed and
only waits on a sidecar: cutting it would salvage nothing."""
for mode in ("buffer_only", "cache"):
for remaining_pages in (0, 2):
with self.subTest(mode=mode, remaining_pages=remaining_pages):
cache, operation = _timeout_fixture(mode)
with patch("time.perf_counter", return_value=100.0):
self.assertFalse(cache.check_prefetch_progress(_REQ))
operation.hash_value = ["hit"] * remaining_pages
operation.host_indices = torch.arange(remaining_pages * 2)
operation.completed_tokens = remaining_pages * 2
operation.pool_transfers = [PoolTransfer(name=PoolName.SWA)]
operation.pool_transfers_done = False
with patch("time.perf_counter", return_value=200.0):
self.assertFalse(cache.check_prefetch_progress(_REQ))
self.assertFalse(operation.deadline_expired)
def test_stop_policies_distinguish_deadline_from_storage_failure(self):
"""Only the timeout policy marks a deadline cut; a storage failure
under any policy terminates without the deadline flag."""
for policy in ("timeout", "wait_complete", "best_effort"):
for failed in (False, True):
with self.subTest(policy=policy, failed=failed):
cache, operation = _timeout_fixture(policy=policy)
operation.hash_value = ["a", "b", "c", "d"]
operation.host_indices = torch.arange(8)
operation.completed_tokens = 2
cache._handle_prefetch_result = Mock()
with patch("time.perf_counter", return_value=100.0):
cache.check_prefetch_progress(_REQ)
if failed:
operation.mark_terminate()
with patch("time.perf_counter", return_value=200.0):
self.assertEqual(
cache.check_prefetch_progress(_REQ),
failed or policy != "wait_complete",
)
self.assertEqual(
operation.deadline_expired, policy == "timeout" and not failed
)
if policy != "timeout":
self.assertFalse(cache._storage_prefetch_wait)
def test_expiry_is_agreed_even_before_a_peer_starts_waiting(self):
"""A rank that has not reached its admission poll must still adopt
the agreed expiry, or it would re-issue the prefetch alone."""
with tempfile.TemporaryDirectory() as tmpdir:
torch.multiprocessing.spawn(
_two_rank_prefetch_deadline,
args=(f"{tmpdir}/gloo",),
nprocs=2,
)
def _two_rank_retry_trace(rank, rendezvous):
torch.distributed.init_process_group(
"gloo",
@@ -577,6 +761,7 @@ class TestStagedPrefetchLifecycle(unittest.TestCase):
stats_requested_tokens=8,
is_terminated=lambda: False,
all_hash_values=None,
deadline_expired=False,
)
cache.ongoing_prefetch[handle] = _OngoingPrefetch(
0, RadixKey(array("q", range(8))), None, operation, None, {}