[Fix] hicache: Bound the storage prefetch timeout from the first blocked admission poll (#44614)

Co-authored-by: metamergebot <metamergebot@users.noreply.github.com>
Co-authored-by: xiezhq-hermann <xiezhq-hermann@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-16 10:37:11 -07:00
committed by GitHub
co-authored by metamergebot xiezhq-hermann
parent 989ccf4863
commit 453b251dbd
10 changed files with 333 additions and 74 deletions
@@ -923,6 +923,9 @@ class PrefillAdder:
self.tree_cache.discard_storage_prefetch_accounting(
req.cache_request_handle
)
self.tree_cache.finish_storage_prefetch_admission(
req.cache_request_handle, fulfilled_tokens=0, reason=None
)
return
if prefix_len > 0:
@@ -184,6 +184,8 @@ class PrefetchOperation(StorageOperation):
# query. Trailing sidecars still use the untrimmed hit endpoint.
self.sidecar_hash_values: Optional[list[str]] = None
self.sidecar_hit_pages = 0
# Cut by the stop policy, not a storage shortfall: the span is still stored.
self.deadline_expired = False
def mark_terminate(self):
with self._lock:
@@ -451,3 +451,4 @@ class StorageAttachment:
cache.prefetch_loaded_tokens_by_reqid.clear()
cache.prefetch_loaded_storage_start_by_reqid.clear()
cache.storage_prefetch_retries.clear()
cache._storage_prefetch_wait.clear()
@@ -10,6 +10,7 @@ from http import HTTPStatus
from queue import Queue
from typing import TYPE_CHECKING, Iterator, NamedTuple, Optional, Sequence, TypeVar
import msgspec
import torch
from sglang.srt.distributed.communication_tags import P2PTag
@@ -105,6 +106,7 @@ from sglang.srt.observability.metrics_collector import (
from sglang.srt.runtime_context import get_memory, get_model, get_observability
from sglang.srt.session.streaming_session import StreamingSession, is_virtual_node
from sglang.srt.utils.common import ceil_align
from sglang.srt.utils.poll_timer import PollTimer
if TYPE_CHECKING:
from sglang.srt.managers.cache_controller import HiCacheAck
@@ -161,6 +163,12 @@ class _OngoingLoadBack(NamedTuple):
host_lock_params: DecLockRefParams
class _StoragePrefetchWait(msgspec.Struct):
timer: PollTimer = msgspec.field(default_factory=PollTimer)
timeout_s: float = 0.0
expired: bool = False
class _OngoingPrefetch(NamedTuple):
"""Tracks an in-flight storage→host prefetch operation."""
@@ -404,6 +412,7 @@ class UnifiedRadixCache(BasePrefixCache):
self.prefetch_loaded_tokens_by_reqid: dict[CacheRequestHandle, int] = {}
self.prefetch_loaded_storage_start_by_reqid: dict[CacheRequestHandle, int] = {}
self.ongoing_prefetch: dict[CacheRequestHandle, _OngoingPrefetch] = {}
self._storage_prefetch_wait: dict[CacheRequestHandle, _StoragePrefetchWait] = {}
# Rank-agreed L3-hit tokens not yet resolved as usable or unfulfilled.
# Cache-mode entries survive L3->L2 until H2D succeeds or admission
# fails; buffer-mode entries survive staging until the H2D ack.
@@ -2009,6 +2018,9 @@ class UnifiedRadixCache(BasePrefixCache):
req_id = request.rid
self.storage_prefetch_retries.cancel(req_id)
wait = self._storage_prefetch_wait.get(request)
if wait is not None and wait.expired:
return
submission = self.cache_controller.get_prefetch_submission(req_id)
if submission is not None:
return submission.decision
@@ -2138,8 +2150,8 @@ class UnifiedRadixCache(BasePrefixCache):
stats["issued"] += 1
if operation.assume_stored:
stats["issued_assumed_stored"] += 1
# Snapshot the requested span for L3 miss-token accounting at the
# rank-synchronized query outcome.
# Keep the requested span stable for the wait budget and miss accounting;
# the hit hash chain can shrink as device coverage changes.
operation.stats_requested_tokens = prefetch_length
operation.storage_start = storage_start
if submission.decision is not None:
@@ -2167,13 +2179,41 @@ class UnifiedRadixCache(BasePrefixCache):
# grants occupancy later at hit-alloc time, sized to the hit.
self.cache_controller.prefetch_tokens_occupied += len(prefetch_key)
def _prefetch_timeout_check_linear_func(self, operation: PrefetchOperation) -> bool:
return (
time.monotonic() - operation.start_time
> self.prefetch_timeout_base
+ len(operation.hash_value) * self.prefetch_timeout_per_page
def _start_storage_prefetch_wait(self, operation: PrefetchOperation) -> None:
if (
self.prefetch_stop_policy == "timeout"
and operation.handle not in self._storage_prefetch_wait
):
# Start at the first blocked admission poll; refetches keep this budget.
pages = operation.stats_requested_tokens // self.page_size
wait = _StoragePrefetchWait(
timeout_s=self.prefetch_timeout_base
+ pages * self.prefetch_timeout_per_page,
)
wait.timer.observe(ready=False)
self._storage_prefetch_wait[operation.handle] = wait
def _prefetch_deadline_expired(self, operation: PrefetchOperation) -> bool:
if operation.host_indices is not None and (
operation.completed_tokens >= len(operation.hash_value) * self.page_size
):
# KV landed: cutting the pending sidecar read would salvage nothing.
return False
wait = self._storage_prefetch_wait.get(operation.handle)
return wait is not None and (
wait.expired
or (wait.timer.running and wait.timer.elapsed() > wait.timeout_s)
)
def _mark_prefetch_deadline_expired(self, operation: PrefetchOperation) -> None:
operation.deadline_expired = True
self.storage_prefetch_retries.cancel(operation.handle.rid)
# Only a rank-agreed verdict can suppress retries, including on peers
# that have not reached their conditional admission poll yet.
self._storage_prefetch_wait.setdefault(
operation.handle, _StoragePrefetchWait()
).expired = True
@rank_consensus(same_results=True)
def _can_terminate_prefetch(self, operation: PrefetchOperation) -> bool:
if self.prefetch_stop_policy == "best_effort":
@@ -2187,7 +2227,7 @@ class UnifiedRadixCache(BasePrefixCache):
# rank timed out) and _all_reduce broadcasts the verdict along PP.
should_terminate = False
if self.pp_rank == 0:
should_terminate = self._prefetch_timeout_check_linear_func(operation)
should_terminate = self._prefetch_deadline_expired(operation)
should_terminate_tensor = torch.tensor(
int(should_terminate), dtype=torch.int, device="cpu"
)
@@ -2267,17 +2307,19 @@ class UnifiedRadixCache(BasePrefixCache):
operation = self.ongoing_prefetch[request].operation
# Determine whether or not we should terminate this prefetch request.
should_terminate = operation.is_terminated() or self._can_terminate_prefetch(
operation
)
if not should_terminate:
return False
if not operation.is_terminated():
if not self._can_terminate_prefetch(operation):
self._start_storage_prefetch_wait(operation)
return False
if self.prefetch_stop_policy == "timeout":
self._mark_prefetch_deadline_expired(operation)
self.cache_controller.terminate_prefetch(operation)
if operation.host_indices is None:
self.storage_prefetch_retries.poll_miss(request.rid)
if operation.deadline_expired:
self._finish_storage_prefetch(request, 0, reason="deadline")
else:
self.storage_prefetch_retries.poll_miss(request.rid)
self.revoke_pending_prefetch(request)
else:
self._handle_prefetch_result(operation)
@@ -2317,7 +2359,7 @@ class UnifiedRadixCache(BasePrefixCache):
self._resolve_storage_prefetch_tokens(
request,
allocated_tokens - completed_tokens,
reason="storage_transfer",
reason="deadline" if operation.deadline_expired else "storage_transfer",
)
if (
completed_tokens > 0
@@ -2445,7 +2487,7 @@ class UnifiedRadixCache(BasePrefixCache):
if pool_transfers and not all_succeeded:
# Drop each pool's beliefs beyond what it served (KV and its
# sidecars beyond the completed span); the next insert re-writes
# only the missing pools.
# only the missing pools. A deadline is not evidence of absent data.
invalidation_hashes = (
operation.sidecar_hash_values
if operation.sidecar_hash_values is not None
@@ -2454,7 +2496,7 @@ class UnifiedRadixCache(BasePrefixCache):
trimmed_pages = len(invalidation_hashes) - len(hash_value)
keep_pages = trimmed_pages + completed_tokens // self.page_size
beliefs = self.storage_existence_cache
if keep_pages < len(invalidation_hashes):
if not operation.deadline_expired and keep_pages < len(invalidation_hashes):
self._belief_policy(
PoolName.KV, PoolHitPolicy.ALL_PAGES
).heal_from_transfer(
@@ -2470,15 +2512,18 @@ class UnifiedRadixCache(BasePrefixCache):
keep_pages,
)
for transfer, count in zip(pool_transfers, pool_hit_pages):
if transfer.keys is not None and count < len(transfer.keys):
if (
not operation.deadline_expired
and transfer.keys is not None
and count < len(transfer.keys)
):
self._belief_policy(
transfer.name, transfer.hit_policy
).heal_from_transfer(
beliefs.pool(transfer.name), transfer.keys, count
)
self._finish_storage_prefetch(
request, fulfilled_tokens=0, reason="storage_transfer"
)
reason = "deadline" if operation.deadline_expired else "storage_transfer"
self._finish_storage_prefetch(request, fulfilled_tokens=0, reason=reason)
# The controller's prefetch IO thread already releases the untransferred
# tail (host_indices[completed_tokens:])
self._retire_ongoing_prefetch(
@@ -2489,12 +2534,13 @@ class UnifiedRadixCache(BasePrefixCache):
self.prefetch_loaded_tokens_by_reqid[request] = 0
self.prefetch_loaded_storage_start_by_reqid.pop(request, None)
logger.warning(
"HiCache hybrid prefetch discarded req=%s completed=%d requested=%d "
"kv_beliefs_kept_pages=%d aux_pool_hit_pages=%s",
"HiCache hybrid prefetch discarded req=%s reason=%s completed=%d "
"requested=%d kv_beliefs_kept_pages=%s aux_pool_hit_pages=%s",
request.rid,
reason,
completed_tokens,
expected_tokens,
keep_pages,
None if operation.deadline_expired else keep_pages,
{
getattr(transfer.name, "value", transfer.name): count
for transfer, count in zip(pool_transfers, pool_hit_pages)
@@ -2586,11 +2632,12 @@ class UnifiedRadixCache(BasePrefixCache):
def finish_storage_prefetch_admission(
self, request: CacheRequestHandle, fulfilled_tokens: int, reason: Optional[str]
) -> None:
"""Resolve a cache-mode L3 hit after request admission.
"""Close the admission wait budget and resolve a cache-mode L3 hit.
Buffer mode resolves at the H2D completion ack instead, because its
host allocation is transport staging rather than a resident L2 hit.
"""
self._storage_prefetch_wait.pop(request, None)
if self.host_memory_mode == "cache":
self._finish_storage_prefetch(request, fulfilled_tokens, reason)
@@ -2627,6 +2674,7 @@ class UnifiedRadixCache(BasePrefixCache):
@rank_consensus(same_params=True)
def release_aborted_request(self, request: CacheRequestHandle) -> None:
rid = request.rid
self._storage_prefetch_wait.pop(request, None)
if self.linker is not None:
self.linker.release_request(rid)
self.prefetch_loaded_tokens_by_reqid.pop(request, None)
@@ -2702,7 +2750,8 @@ class UnifiedRadixCache(BasePrefixCache):
"""Heal each pool's beliefs beyond its rank-agreed boundary (a
chain-page pool also learns presence up to it); a pool without a
verdict heals from the folded cut."""
if self.host_memory_mode != "buffer_only":
if self.host_memory_mode != "buffer_only" or operation.deadline_expired:
# A cut query's agreed hit may be a MIN over skipped lookups, not a verdict.
return
chain = operation.all_hash_values
if chain is None:
@@ -2736,7 +2785,7 @@ class UnifiedRadixCache(BasePrefixCache):
"""Feed the cumulative prefetch-outcome counters at the (rank-synced)
query outcome: T = prompt tokens, L = requested, m = L3-miss."""
requested = operation.stats_requested_tokens
if requested <= 0:
if requested <= 0 or operation.deadline_expired:
return
stats = self._prefetch_outcome_stats
hit = max(0, min(operation.storage_hit_count, requested))
@@ -3027,22 +3076,21 @@ class UnifiedRadixCache(BasePrefixCache):
if hit_tokens > 0:
self._record_storage_prefetch_hit(request, hit_tokens)
if operation.is_terminated():
# Controller-side miss termination (retryable) or an abort
# race (abort cleanup discards the marker).
# A storage miss can retry; a policy deadline cannot.
if hit_tokens > 0:
reason = None
if operation.deadline_expired:
reason = "deadline"
elif hit_tokens < self.prefetch_threshold:
reason = "below_threshold"
self._finish_storage_prefetch(
request,
fulfilled_tokens=0,
reason=(
"below_threshold"
if hit_tokens < self.prefetch_threshold
else None
),
request, fulfilled_tokens=0, reason=reason
)
# The published (rank-agreed) hit count is the L3 verdict;
# without this, store misses never reach l3_miss_tokens.
self._account_prefetch_outcome(operation, revoked=True)
self.storage_prefetch_retries.poll_miss(request.rid)
if not operation.deadline_expired:
self.storage_prefetch_retries.poll_miss(request.rid)
self.revoke_pending_prefetch(request)
continue
if hit_tokens < self.prefetch_threshold:
@@ -2175,7 +2175,8 @@ class StorageMetricsCollector(_StatLoggerDIMixin):
"device_overlap means the live device prefix diverged from the "
"request's view at splice time, so the span beyond it was released; "
"cache_admission_shortfall means an L3-loaded L2 span was no longer "
"reusable when cache-mode admission ran.",
"reusable when cache-mode admission ran; deadline means the "
"storage prefetch stop policy ended the read before it completed.",
labelnames=list(labels.keys()) + ["reason"],
)
for reason in (
@@ -2184,6 +2185,7 @@ class StorageMetricsCollector(_StatLoggerDIMixin):
"device_capacity",
"device_covered",
"storage_transfer",
"deadline",
"anchor_lost",
"aux_window_trim",
"device_overlap",
@@ -42,6 +42,7 @@ from sglang.srt.observability.trace_async import (
is_async_tracing_available,
)
from sglang.srt.utils import get_bool_env_var
from sglang.srt.utils.poll_timer import PollTimer
if TYPE_CHECKING:
from sglang.srt.disaggregation.base.conn import KVTransferMetric
@@ -668,8 +669,8 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
# None means unmeasured (including stats received from an older scheduler).
storage_prefetch_wait_s: Optional[float] = None
storage_staging_wait_s: Optional[float] = None
_storage_prefetch_wait_start: Optional[float] = None
_storage_staging_wait_start: Optional[float] = None
_storage_prefetch_wait: Optional[PollTimer] = None
_storage_staging_wait: Optional[PollTimer] = None
has_timing_data: bool = False
@@ -687,12 +688,16 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
"queue_duration_s": self.queue_duration_s,
"storage_prefetch_wait_s": (
self.storage_prefetch_wait_s
if self._storage_prefetch_wait_start is None
if not (
self._storage_prefetch_wait and self._storage_prefetch_wait.running
)
else None
),
"storage_staging_wait_s": (
self.storage_staging_wait_s
if self._storage_staging_wait_start is None
if not (
self._storage_staging_wait and self._storage_staging_wait.running
)
else None
),
"diff_realtime_monotonic": global_diff_realtime_monotonic,
@@ -756,33 +761,17 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
self.queue_duration_s = 0.0
self.storage_prefetch_wait_s = None
self.storage_staging_wait_s = None
self._storage_prefetch_wait_start = None
self._storage_staging_wait_start = None
self._storage_prefetch_wait = None
self._storage_staging_wait = None
def observe_storage_prefetch_ready(self, ready: bool):
"""Accumulate first blocked poll to the subsequent ready poll, per episode."""
if self.storage_prefetch_wait_s is None:
return
if ready:
if self._storage_prefetch_wait_start is not None:
self.storage_prefetch_wait_s += (
time.perf_counter() - self._storage_prefetch_wait_start
)
self._storage_prefetch_wait_start = None
elif self._storage_prefetch_wait_start is None:
self._storage_prefetch_wait_start = time.perf_counter()
if self._storage_prefetch_wait is not None:
self.storage_prefetch_wait_s += self._storage_prefetch_wait.observe(ready)
def observe_storage_staging_ready(self, ready: bool):
if self.storage_staging_wait_s is None:
return
if ready:
if self._storage_staging_wait_start is not None:
self.storage_staging_wait_s += (
time.perf_counter() - self._storage_staging_wait_start
)
self._storage_staging_wait_start = None
elif self._storage_staging_wait_start is None:
self._storage_staging_wait_start = time.perf_counter()
if self._storage_staging_wait is not None:
self.storage_staging_wait_s += self._storage_staging_wait.observe(ready)
def set_wait_queue_entry_time(self, ts=None):
ts = ts or time.perf_counter()
@@ -790,6 +779,8 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
if self.enable_metrics and self.disagg_mode != DisaggregationMode.DECODE:
self.storage_prefetch_wait_s = 0.0
self.storage_staging_wait_s = 0.0
self._storage_prefetch_wait = PollTimer()
self._storage_staging_wait = PollTimer()
if self.enable_metrics or self.trace_ctx.tracing_enable:
if self.disagg_mode == DisaggregationMode.PREFILL:
stage = RequestStage.PREFILL_BOOTSTRAP
@@ -1221,14 +1212,12 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
def convert_to_output_meta_info(self):
meta_data = {}
if (
self.storage_prefetch_wait_s is not None
and self._storage_prefetch_wait_start is None
if self.storage_prefetch_wait_s is not None and not (
self._storage_prefetch_wait and self._storage_prefetch_wait.running
):
meta_data["storage_prefetch_wait"] = self.storage_prefetch_wait_s
if (
self.storage_staging_wait_s is not None
and self._storage_staging_wait_start is None
if self.storage_staging_wait_s is not None and not (
self._storage_staging_wait and self._storage_staging_wait.running
):
meta_data["storage_staging_wait"] = self.storage_staging_wait_s
if self.forward_entry_time > 0.0:
+26
View File
@@ -0,0 +1,26 @@
import time
from typing import Optional
import msgspec
class PollTimer(msgspec.Struct):
"""Time an interval from its first blocked poll to its first ready poll."""
started_at: Optional[float] = None
@property
def running(self) -> bool:
return self.started_at is not None
def elapsed(self) -> float:
return time.perf_counter() - self.started_at if self.running else 0.0
def observe(self, ready: bool) -> float:
if ready:
elapsed = self.elapsed()
self.started_at = None
return elapsed
if not self.running:
self.started_at = time.perf_counter()
return 0.0