[Fix] hicache: Bound the storage prefetch timeout from the first blocked admission poll (#44614)
Co-authored-by: metamergebot <metamergebot@users.noreply.github.com> Co-authored-by: xiezhq-hermann <xiezhq-hermann@users.noreply.github.com>
This commit is contained in:
co-authored by
metamergebot
xiezhq-hermann
parent
989ccf4863
commit
453b251dbd
@@ -923,6 +923,9 @@ class PrefillAdder:
|
||||
self.tree_cache.discard_storage_prefetch_accounting(
|
||||
req.cache_request_handle
|
||||
)
|
||||
self.tree_cache.finish_storage_prefetch_admission(
|
||||
req.cache_request_handle, fulfilled_tokens=0, reason=None
|
||||
)
|
||||
return
|
||||
|
||||
if prefix_len > 0:
|
||||
|
||||
@@ -184,6 +184,8 @@ class PrefetchOperation(StorageOperation):
|
||||
# query. Trailing sidecars still use the untrimmed hit endpoint.
|
||||
self.sidecar_hash_values: Optional[list[str]] = None
|
||||
self.sidecar_hit_pages = 0
|
||||
# Cut by the stop policy, not a storage shortfall: the span is still stored.
|
||||
self.deadline_expired = False
|
||||
|
||||
def mark_terminate(self):
|
||||
with self._lock:
|
||||
|
||||
@@ -451,3 +451,4 @@ class StorageAttachment:
|
||||
cache.prefetch_loaded_tokens_by_reqid.clear()
|
||||
cache.prefetch_loaded_storage_start_by_reqid.clear()
|
||||
cache.storage_prefetch_retries.clear()
|
||||
cache._storage_prefetch_wait.clear()
|
||||
|
||||
@@ -10,6 +10,7 @@ from http import HTTPStatus
|
||||
from queue import Queue
|
||||
from typing import TYPE_CHECKING, Iterator, NamedTuple, Optional, Sequence, TypeVar
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.distributed.communication_tags import P2PTag
|
||||
@@ -105,6 +106,7 @@ from sglang.srt.observability.metrics_collector import (
|
||||
from sglang.srt.runtime_context import get_memory, get_model, get_observability
|
||||
from sglang.srt.session.streaming_session import StreamingSession, is_virtual_node
|
||||
from sglang.srt.utils.common import ceil_align
|
||||
from sglang.srt.utils.poll_timer import PollTimer
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.cache_controller import HiCacheAck
|
||||
@@ -161,6 +163,12 @@ class _OngoingLoadBack(NamedTuple):
|
||||
host_lock_params: DecLockRefParams
|
||||
|
||||
|
||||
class _StoragePrefetchWait(msgspec.Struct):
|
||||
timer: PollTimer = msgspec.field(default_factory=PollTimer)
|
||||
timeout_s: float = 0.0
|
||||
expired: bool = False
|
||||
|
||||
|
||||
class _OngoingPrefetch(NamedTuple):
|
||||
"""Tracks an in-flight storage→host prefetch operation."""
|
||||
|
||||
@@ -404,6 +412,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
self.prefetch_loaded_tokens_by_reqid: dict[CacheRequestHandle, int] = {}
|
||||
self.prefetch_loaded_storage_start_by_reqid: dict[CacheRequestHandle, int] = {}
|
||||
self.ongoing_prefetch: dict[CacheRequestHandle, _OngoingPrefetch] = {}
|
||||
self._storage_prefetch_wait: dict[CacheRequestHandle, _StoragePrefetchWait] = {}
|
||||
# Rank-agreed L3-hit tokens not yet resolved as usable or unfulfilled.
|
||||
# Cache-mode entries survive L3->L2 until H2D succeeds or admission
|
||||
# fails; buffer-mode entries survive staging until the H2D ack.
|
||||
@@ -2009,6 +2018,9 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
|
||||
req_id = request.rid
|
||||
self.storage_prefetch_retries.cancel(req_id)
|
||||
wait = self._storage_prefetch_wait.get(request)
|
||||
if wait is not None and wait.expired:
|
||||
return
|
||||
submission = self.cache_controller.get_prefetch_submission(req_id)
|
||||
if submission is not None:
|
||||
return submission.decision
|
||||
@@ -2138,8 +2150,8 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
stats["issued"] += 1
|
||||
if operation.assume_stored:
|
||||
stats["issued_assumed_stored"] += 1
|
||||
# Snapshot the requested span for L3 miss-token accounting at the
|
||||
# rank-synchronized query outcome.
|
||||
# Keep the requested span stable for the wait budget and miss accounting;
|
||||
# the hit hash chain can shrink as device coverage changes.
|
||||
operation.stats_requested_tokens = prefetch_length
|
||||
operation.storage_start = storage_start
|
||||
if submission.decision is not None:
|
||||
@@ -2167,13 +2179,41 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
# grants occupancy later at hit-alloc time, sized to the hit.
|
||||
self.cache_controller.prefetch_tokens_occupied += len(prefetch_key)
|
||||
|
||||
def _prefetch_timeout_check_linear_func(self, operation: PrefetchOperation) -> bool:
|
||||
return (
|
||||
time.monotonic() - operation.start_time
|
||||
> self.prefetch_timeout_base
|
||||
+ len(operation.hash_value) * self.prefetch_timeout_per_page
|
||||
def _start_storage_prefetch_wait(self, operation: PrefetchOperation) -> None:
|
||||
if (
|
||||
self.prefetch_stop_policy == "timeout"
|
||||
and operation.handle not in self._storage_prefetch_wait
|
||||
):
|
||||
# Start at the first blocked admission poll; refetches keep this budget.
|
||||
pages = operation.stats_requested_tokens // self.page_size
|
||||
wait = _StoragePrefetchWait(
|
||||
timeout_s=self.prefetch_timeout_base
|
||||
+ pages * self.prefetch_timeout_per_page,
|
||||
)
|
||||
wait.timer.observe(ready=False)
|
||||
self._storage_prefetch_wait[operation.handle] = wait
|
||||
|
||||
def _prefetch_deadline_expired(self, operation: PrefetchOperation) -> bool:
|
||||
if operation.host_indices is not None and (
|
||||
operation.completed_tokens >= len(operation.hash_value) * self.page_size
|
||||
):
|
||||
# KV landed: cutting the pending sidecar read would salvage nothing.
|
||||
return False
|
||||
wait = self._storage_prefetch_wait.get(operation.handle)
|
||||
return wait is not None and (
|
||||
wait.expired
|
||||
or (wait.timer.running and wait.timer.elapsed() > wait.timeout_s)
|
||||
)
|
||||
|
||||
def _mark_prefetch_deadline_expired(self, operation: PrefetchOperation) -> None:
|
||||
operation.deadline_expired = True
|
||||
self.storage_prefetch_retries.cancel(operation.handle.rid)
|
||||
# Only a rank-agreed verdict can suppress retries, including on peers
|
||||
# that have not reached their conditional admission poll yet.
|
||||
self._storage_prefetch_wait.setdefault(
|
||||
operation.handle, _StoragePrefetchWait()
|
||||
).expired = True
|
||||
|
||||
@rank_consensus(same_results=True)
|
||||
def _can_terminate_prefetch(self, operation: PrefetchOperation) -> bool:
|
||||
if self.prefetch_stop_policy == "best_effort":
|
||||
@@ -2187,7 +2227,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
# rank timed out) and _all_reduce broadcasts the verdict along PP.
|
||||
should_terminate = False
|
||||
if self.pp_rank == 0:
|
||||
should_terminate = self._prefetch_timeout_check_linear_func(operation)
|
||||
should_terminate = self._prefetch_deadline_expired(operation)
|
||||
should_terminate_tensor = torch.tensor(
|
||||
int(should_terminate), dtype=torch.int, device="cpu"
|
||||
)
|
||||
@@ -2267,17 +2307,19 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
|
||||
operation = self.ongoing_prefetch[request].operation
|
||||
|
||||
# Determine whether or not we should terminate this prefetch request.
|
||||
should_terminate = operation.is_terminated() or self._can_terminate_prefetch(
|
||||
operation
|
||||
)
|
||||
|
||||
if not should_terminate:
|
||||
return False
|
||||
if not operation.is_terminated():
|
||||
if not self._can_terminate_prefetch(operation):
|
||||
self._start_storage_prefetch_wait(operation)
|
||||
return False
|
||||
if self.prefetch_stop_policy == "timeout":
|
||||
self._mark_prefetch_deadline_expired(operation)
|
||||
|
||||
self.cache_controller.terminate_prefetch(operation)
|
||||
if operation.host_indices is None:
|
||||
self.storage_prefetch_retries.poll_miss(request.rid)
|
||||
if operation.deadline_expired:
|
||||
self._finish_storage_prefetch(request, 0, reason="deadline")
|
||||
else:
|
||||
self.storage_prefetch_retries.poll_miss(request.rid)
|
||||
self.revoke_pending_prefetch(request)
|
||||
else:
|
||||
self._handle_prefetch_result(operation)
|
||||
@@ -2317,7 +2359,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
self._resolve_storage_prefetch_tokens(
|
||||
request,
|
||||
allocated_tokens - completed_tokens,
|
||||
reason="storage_transfer",
|
||||
reason="deadline" if operation.deadline_expired else "storage_transfer",
|
||||
)
|
||||
if (
|
||||
completed_tokens > 0
|
||||
@@ -2445,7 +2487,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
if pool_transfers and not all_succeeded:
|
||||
# Drop each pool's beliefs beyond what it served (KV and its
|
||||
# sidecars beyond the completed span); the next insert re-writes
|
||||
# only the missing pools.
|
||||
# only the missing pools. A deadline is not evidence of absent data.
|
||||
invalidation_hashes = (
|
||||
operation.sidecar_hash_values
|
||||
if operation.sidecar_hash_values is not None
|
||||
@@ -2454,7 +2496,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
trimmed_pages = len(invalidation_hashes) - len(hash_value)
|
||||
keep_pages = trimmed_pages + completed_tokens // self.page_size
|
||||
beliefs = self.storage_existence_cache
|
||||
if keep_pages < len(invalidation_hashes):
|
||||
if not operation.deadline_expired and keep_pages < len(invalidation_hashes):
|
||||
self._belief_policy(
|
||||
PoolName.KV, PoolHitPolicy.ALL_PAGES
|
||||
).heal_from_transfer(
|
||||
@@ -2470,15 +2512,18 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
keep_pages,
|
||||
)
|
||||
for transfer, count in zip(pool_transfers, pool_hit_pages):
|
||||
if transfer.keys is not None and count < len(transfer.keys):
|
||||
if (
|
||||
not operation.deadline_expired
|
||||
and transfer.keys is not None
|
||||
and count < len(transfer.keys)
|
||||
):
|
||||
self._belief_policy(
|
||||
transfer.name, transfer.hit_policy
|
||||
).heal_from_transfer(
|
||||
beliefs.pool(transfer.name), transfer.keys, count
|
||||
)
|
||||
self._finish_storage_prefetch(
|
||||
request, fulfilled_tokens=0, reason="storage_transfer"
|
||||
)
|
||||
reason = "deadline" if operation.deadline_expired else "storage_transfer"
|
||||
self._finish_storage_prefetch(request, fulfilled_tokens=0, reason=reason)
|
||||
# The controller's prefetch IO thread already releases the untransferred
|
||||
# tail (host_indices[completed_tokens:])
|
||||
self._retire_ongoing_prefetch(
|
||||
@@ -2489,12 +2534,13 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
self.prefetch_loaded_tokens_by_reqid[request] = 0
|
||||
self.prefetch_loaded_storage_start_by_reqid.pop(request, None)
|
||||
logger.warning(
|
||||
"HiCache hybrid prefetch discarded req=%s completed=%d requested=%d "
|
||||
"kv_beliefs_kept_pages=%d aux_pool_hit_pages=%s",
|
||||
"HiCache hybrid prefetch discarded req=%s reason=%s completed=%d "
|
||||
"requested=%d kv_beliefs_kept_pages=%s aux_pool_hit_pages=%s",
|
||||
request.rid,
|
||||
reason,
|
||||
completed_tokens,
|
||||
expected_tokens,
|
||||
keep_pages,
|
||||
None if operation.deadline_expired else keep_pages,
|
||||
{
|
||||
getattr(transfer.name, "value", transfer.name): count
|
||||
for transfer, count in zip(pool_transfers, pool_hit_pages)
|
||||
@@ -2586,11 +2632,12 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
def finish_storage_prefetch_admission(
|
||||
self, request: CacheRequestHandle, fulfilled_tokens: int, reason: Optional[str]
|
||||
) -> None:
|
||||
"""Resolve a cache-mode L3 hit after request admission.
|
||||
"""Close the admission wait budget and resolve a cache-mode L3 hit.
|
||||
|
||||
Buffer mode resolves at the H2D completion ack instead, because its
|
||||
host allocation is transport staging rather than a resident L2 hit.
|
||||
"""
|
||||
self._storage_prefetch_wait.pop(request, None)
|
||||
if self.host_memory_mode == "cache":
|
||||
self._finish_storage_prefetch(request, fulfilled_tokens, reason)
|
||||
|
||||
@@ -2627,6 +2674,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
@rank_consensus(same_params=True)
|
||||
def release_aborted_request(self, request: CacheRequestHandle) -> None:
|
||||
rid = request.rid
|
||||
self._storage_prefetch_wait.pop(request, None)
|
||||
if self.linker is not None:
|
||||
self.linker.release_request(rid)
|
||||
self.prefetch_loaded_tokens_by_reqid.pop(request, None)
|
||||
@@ -2702,7 +2750,8 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
"""Heal each pool's beliefs beyond its rank-agreed boundary (a
|
||||
chain-page pool also learns presence up to it); a pool without a
|
||||
verdict heals from the folded cut."""
|
||||
if self.host_memory_mode != "buffer_only":
|
||||
if self.host_memory_mode != "buffer_only" or operation.deadline_expired:
|
||||
# A cut query's agreed hit may be a MIN over skipped lookups, not a verdict.
|
||||
return
|
||||
chain = operation.all_hash_values
|
||||
if chain is None:
|
||||
@@ -2736,7 +2785,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
"""Feed the cumulative prefetch-outcome counters at the (rank-synced)
|
||||
query outcome: T = prompt tokens, L = requested, m = L3-miss."""
|
||||
requested = operation.stats_requested_tokens
|
||||
if requested <= 0:
|
||||
if requested <= 0 or operation.deadline_expired:
|
||||
return
|
||||
stats = self._prefetch_outcome_stats
|
||||
hit = max(0, min(operation.storage_hit_count, requested))
|
||||
@@ -3027,22 +3076,21 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
if hit_tokens > 0:
|
||||
self._record_storage_prefetch_hit(request, hit_tokens)
|
||||
if operation.is_terminated():
|
||||
# Controller-side miss termination (retryable) or an abort
|
||||
# race (abort cleanup discards the marker).
|
||||
# A storage miss can retry; a policy deadline cannot.
|
||||
if hit_tokens > 0:
|
||||
reason = None
|
||||
if operation.deadline_expired:
|
||||
reason = "deadline"
|
||||
elif hit_tokens < self.prefetch_threshold:
|
||||
reason = "below_threshold"
|
||||
self._finish_storage_prefetch(
|
||||
request,
|
||||
fulfilled_tokens=0,
|
||||
reason=(
|
||||
"below_threshold"
|
||||
if hit_tokens < self.prefetch_threshold
|
||||
else None
|
||||
),
|
||||
request, fulfilled_tokens=0, reason=reason
|
||||
)
|
||||
# The published (rank-agreed) hit count is the L3 verdict;
|
||||
# without this, store misses never reach l3_miss_tokens.
|
||||
self._account_prefetch_outcome(operation, revoked=True)
|
||||
self.storage_prefetch_retries.poll_miss(request.rid)
|
||||
if not operation.deadline_expired:
|
||||
self.storage_prefetch_retries.poll_miss(request.rid)
|
||||
self.revoke_pending_prefetch(request)
|
||||
continue
|
||||
if hit_tokens < self.prefetch_threshold:
|
||||
|
||||
@@ -2175,7 +2175,8 @@ class StorageMetricsCollector(_StatLoggerDIMixin):
|
||||
"device_overlap means the live device prefix diverged from the "
|
||||
"request's view at splice time, so the span beyond it was released; "
|
||||
"cache_admission_shortfall means an L3-loaded L2 span was no longer "
|
||||
"reusable when cache-mode admission ran.",
|
||||
"reusable when cache-mode admission ran; deadline means the "
|
||||
"storage prefetch stop policy ended the read before it completed.",
|
||||
labelnames=list(labels.keys()) + ["reason"],
|
||||
)
|
||||
for reason in (
|
||||
@@ -2184,6 +2185,7 @@ class StorageMetricsCollector(_StatLoggerDIMixin):
|
||||
"device_capacity",
|
||||
"device_covered",
|
||||
"storage_transfer",
|
||||
"deadline",
|
||||
"anchor_lost",
|
||||
"aux_window_trim",
|
||||
"device_overlap",
|
||||
|
||||
@@ -42,6 +42,7 @@ from sglang.srt.observability.trace_async import (
|
||||
is_async_tracing_available,
|
||||
)
|
||||
from sglang.srt.utils import get_bool_env_var
|
||||
from sglang.srt.utils.poll_timer import PollTimer
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.disaggregation.base.conn import KVTransferMetric
|
||||
@@ -668,8 +669,8 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
|
||||
# None means unmeasured (including stats received from an older scheduler).
|
||||
storage_prefetch_wait_s: Optional[float] = None
|
||||
storage_staging_wait_s: Optional[float] = None
|
||||
_storage_prefetch_wait_start: Optional[float] = None
|
||||
_storage_staging_wait_start: Optional[float] = None
|
||||
_storage_prefetch_wait: Optional[PollTimer] = None
|
||||
_storage_staging_wait: Optional[PollTimer] = None
|
||||
|
||||
has_timing_data: bool = False
|
||||
|
||||
@@ -687,12 +688,16 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
|
||||
"queue_duration_s": self.queue_duration_s,
|
||||
"storage_prefetch_wait_s": (
|
||||
self.storage_prefetch_wait_s
|
||||
if self._storage_prefetch_wait_start is None
|
||||
if not (
|
||||
self._storage_prefetch_wait and self._storage_prefetch_wait.running
|
||||
)
|
||||
else None
|
||||
),
|
||||
"storage_staging_wait_s": (
|
||||
self.storage_staging_wait_s
|
||||
if self._storage_staging_wait_start is None
|
||||
if not (
|
||||
self._storage_staging_wait and self._storage_staging_wait.running
|
||||
)
|
||||
else None
|
||||
),
|
||||
"diff_realtime_monotonic": global_diff_realtime_monotonic,
|
||||
@@ -756,33 +761,17 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
|
||||
self.queue_duration_s = 0.0
|
||||
self.storage_prefetch_wait_s = None
|
||||
self.storage_staging_wait_s = None
|
||||
self._storage_prefetch_wait_start = None
|
||||
self._storage_staging_wait_start = None
|
||||
self._storage_prefetch_wait = None
|
||||
self._storage_staging_wait = None
|
||||
|
||||
def observe_storage_prefetch_ready(self, ready: bool):
|
||||
"""Accumulate first blocked poll to the subsequent ready poll, per episode."""
|
||||
if self.storage_prefetch_wait_s is None:
|
||||
return
|
||||
if ready:
|
||||
if self._storage_prefetch_wait_start is not None:
|
||||
self.storage_prefetch_wait_s += (
|
||||
time.perf_counter() - self._storage_prefetch_wait_start
|
||||
)
|
||||
self._storage_prefetch_wait_start = None
|
||||
elif self._storage_prefetch_wait_start is None:
|
||||
self._storage_prefetch_wait_start = time.perf_counter()
|
||||
if self._storage_prefetch_wait is not None:
|
||||
self.storage_prefetch_wait_s += self._storage_prefetch_wait.observe(ready)
|
||||
|
||||
def observe_storage_staging_ready(self, ready: bool):
|
||||
if self.storage_staging_wait_s is None:
|
||||
return
|
||||
if ready:
|
||||
if self._storage_staging_wait_start is not None:
|
||||
self.storage_staging_wait_s += (
|
||||
time.perf_counter() - self._storage_staging_wait_start
|
||||
)
|
||||
self._storage_staging_wait_start = None
|
||||
elif self._storage_staging_wait_start is None:
|
||||
self._storage_staging_wait_start = time.perf_counter()
|
||||
if self._storage_staging_wait is not None:
|
||||
self.storage_staging_wait_s += self._storage_staging_wait.observe(ready)
|
||||
|
||||
def set_wait_queue_entry_time(self, ts=None):
|
||||
ts = ts or time.perf_counter()
|
||||
@@ -790,6 +779,8 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
|
||||
if self.enable_metrics and self.disagg_mode != DisaggregationMode.DECODE:
|
||||
self.storage_prefetch_wait_s = 0.0
|
||||
self.storage_staging_wait_s = 0.0
|
||||
self._storage_prefetch_wait = PollTimer()
|
||||
self._storage_staging_wait = PollTimer()
|
||||
if self.enable_metrics or self.trace_ctx.tracing_enable:
|
||||
if self.disagg_mode == DisaggregationMode.PREFILL:
|
||||
stage = RequestStage.PREFILL_BOOTSTRAP
|
||||
@@ -1221,14 +1212,12 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
|
||||
|
||||
def convert_to_output_meta_info(self):
|
||||
meta_data = {}
|
||||
if (
|
||||
self.storage_prefetch_wait_s is not None
|
||||
and self._storage_prefetch_wait_start is None
|
||||
if self.storage_prefetch_wait_s is not None and not (
|
||||
self._storage_prefetch_wait and self._storage_prefetch_wait.running
|
||||
):
|
||||
meta_data["storage_prefetch_wait"] = self.storage_prefetch_wait_s
|
||||
if (
|
||||
self.storage_staging_wait_s is not None
|
||||
and self._storage_staging_wait_start is None
|
||||
if self.storage_staging_wait_s is not None and not (
|
||||
self._storage_staging_wait and self._storage_staging_wait.running
|
||||
):
|
||||
meta_data["storage_staging_wait"] = self.storage_staging_wait_s
|
||||
if self.forward_entry_time > 0.0:
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
import msgspec
|
||||
|
||||
|
||||
class PollTimer(msgspec.Struct):
|
||||
"""Time an interval from its first blocked poll to its first ready poll."""
|
||||
|
||||
started_at: Optional[float] = None
|
||||
|
||||
@property
|
||||
def running(self) -> bool:
|
||||
return self.started_at is not None
|
||||
|
||||
def elapsed(self) -> float:
|
||||
return time.perf_counter() - self.started_at if self.running else 0.0
|
||||
|
||||
def observe(self, ready: bool) -> float:
|
||||
if ready:
|
||||
elapsed = self.elapsed()
|
||||
self.started_at = None
|
||||
return elapsed
|
||||
if not self.running:
|
||||
self.started_at = time.perf_counter()
|
||||
return 0.0
|
||||
Reference in New Issue
Block a user