[Fix] hicache: Bound the storage prefetch timeout from the first blocked admission poll (#44614)
Co-authored-by: metamergebot <metamergebot@users.noreply.github.com> Co-authored-by: xiezhq-hermann <xiezhq-hermann@users.noreply.github.com>
This commit is contained in:
co-authored by
metamergebot
xiezhq-hermann
parent
989ccf4863
commit
453b251dbd
@@ -923,6 +923,9 @@ class PrefillAdder:
|
||||
self.tree_cache.discard_storage_prefetch_accounting(
|
||||
req.cache_request_handle
|
||||
)
|
||||
self.tree_cache.finish_storage_prefetch_admission(
|
||||
req.cache_request_handle, fulfilled_tokens=0, reason=None
|
||||
)
|
||||
return
|
||||
|
||||
if prefix_len > 0:
|
||||
|
||||
@@ -184,6 +184,8 @@ class PrefetchOperation(StorageOperation):
|
||||
# query. Trailing sidecars still use the untrimmed hit endpoint.
|
||||
self.sidecar_hash_values: Optional[list[str]] = None
|
||||
self.sidecar_hit_pages = 0
|
||||
# Cut by the stop policy, not a storage shortfall: the span is still stored.
|
||||
self.deadline_expired = False
|
||||
|
||||
def mark_terminate(self):
|
||||
with self._lock:
|
||||
|
||||
@@ -451,3 +451,4 @@ class StorageAttachment:
|
||||
cache.prefetch_loaded_tokens_by_reqid.clear()
|
||||
cache.prefetch_loaded_storage_start_by_reqid.clear()
|
||||
cache.storage_prefetch_retries.clear()
|
||||
cache._storage_prefetch_wait.clear()
|
||||
|
||||
@@ -10,6 +10,7 @@ from http import HTTPStatus
|
||||
from queue import Queue
|
||||
from typing import TYPE_CHECKING, Iterator, NamedTuple, Optional, Sequence, TypeVar
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.distributed.communication_tags import P2PTag
|
||||
@@ -105,6 +106,7 @@ from sglang.srt.observability.metrics_collector import (
|
||||
from sglang.srt.runtime_context import get_memory, get_model, get_observability
|
||||
from sglang.srt.session.streaming_session import StreamingSession, is_virtual_node
|
||||
from sglang.srt.utils.common import ceil_align
|
||||
from sglang.srt.utils.poll_timer import PollTimer
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.cache_controller import HiCacheAck
|
||||
@@ -161,6 +163,12 @@ class _OngoingLoadBack(NamedTuple):
|
||||
host_lock_params: DecLockRefParams
|
||||
|
||||
|
||||
class _StoragePrefetchWait(msgspec.Struct):
|
||||
timer: PollTimer = msgspec.field(default_factory=PollTimer)
|
||||
timeout_s: float = 0.0
|
||||
expired: bool = False
|
||||
|
||||
|
||||
class _OngoingPrefetch(NamedTuple):
|
||||
"""Tracks an in-flight storage→host prefetch operation."""
|
||||
|
||||
@@ -404,6 +412,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
self.prefetch_loaded_tokens_by_reqid: dict[CacheRequestHandle, int] = {}
|
||||
self.prefetch_loaded_storage_start_by_reqid: dict[CacheRequestHandle, int] = {}
|
||||
self.ongoing_prefetch: dict[CacheRequestHandle, _OngoingPrefetch] = {}
|
||||
self._storage_prefetch_wait: dict[CacheRequestHandle, _StoragePrefetchWait] = {}
|
||||
# Rank-agreed L3-hit tokens not yet resolved as usable or unfulfilled.
|
||||
# Cache-mode entries survive L3->L2 until H2D succeeds or admission
|
||||
# fails; buffer-mode entries survive staging until the H2D ack.
|
||||
@@ -2009,6 +2018,9 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
|
||||
req_id = request.rid
|
||||
self.storage_prefetch_retries.cancel(req_id)
|
||||
wait = self._storage_prefetch_wait.get(request)
|
||||
if wait is not None and wait.expired:
|
||||
return
|
||||
submission = self.cache_controller.get_prefetch_submission(req_id)
|
||||
if submission is not None:
|
||||
return submission.decision
|
||||
@@ -2138,8 +2150,8 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
stats["issued"] += 1
|
||||
if operation.assume_stored:
|
||||
stats["issued_assumed_stored"] += 1
|
||||
# Snapshot the requested span for L3 miss-token accounting at the
|
||||
# rank-synchronized query outcome.
|
||||
# Keep the requested span stable for the wait budget and miss accounting;
|
||||
# the hit hash chain can shrink as device coverage changes.
|
||||
operation.stats_requested_tokens = prefetch_length
|
||||
operation.storage_start = storage_start
|
||||
if submission.decision is not None:
|
||||
@@ -2167,13 +2179,41 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
# grants occupancy later at hit-alloc time, sized to the hit.
|
||||
self.cache_controller.prefetch_tokens_occupied += len(prefetch_key)
|
||||
|
||||
def _prefetch_timeout_check_linear_func(self, operation: PrefetchOperation) -> bool:
|
||||
return (
|
||||
time.monotonic() - operation.start_time
|
||||
> self.prefetch_timeout_base
|
||||
+ len(operation.hash_value) * self.prefetch_timeout_per_page
|
||||
def _start_storage_prefetch_wait(self, operation: PrefetchOperation) -> None:
|
||||
if (
|
||||
self.prefetch_stop_policy == "timeout"
|
||||
and operation.handle not in self._storage_prefetch_wait
|
||||
):
|
||||
# Start at the first blocked admission poll; refetches keep this budget.
|
||||
pages = operation.stats_requested_tokens // self.page_size
|
||||
wait = _StoragePrefetchWait(
|
||||
timeout_s=self.prefetch_timeout_base
|
||||
+ pages * self.prefetch_timeout_per_page,
|
||||
)
|
||||
wait.timer.observe(ready=False)
|
||||
self._storage_prefetch_wait[operation.handle] = wait
|
||||
|
||||
def _prefetch_deadline_expired(self, operation: PrefetchOperation) -> bool:
|
||||
if operation.host_indices is not None and (
|
||||
operation.completed_tokens >= len(operation.hash_value) * self.page_size
|
||||
):
|
||||
# KV landed: cutting the pending sidecar read would salvage nothing.
|
||||
return False
|
||||
wait = self._storage_prefetch_wait.get(operation.handle)
|
||||
return wait is not None and (
|
||||
wait.expired
|
||||
or (wait.timer.running and wait.timer.elapsed() > wait.timeout_s)
|
||||
)
|
||||
|
||||
def _mark_prefetch_deadline_expired(self, operation: PrefetchOperation) -> None:
|
||||
operation.deadline_expired = True
|
||||
self.storage_prefetch_retries.cancel(operation.handle.rid)
|
||||
# Only a rank-agreed verdict can suppress retries, including on peers
|
||||
# that have not reached their conditional admission poll yet.
|
||||
self._storage_prefetch_wait.setdefault(
|
||||
operation.handle, _StoragePrefetchWait()
|
||||
).expired = True
|
||||
|
||||
@rank_consensus(same_results=True)
|
||||
def _can_terminate_prefetch(self, operation: PrefetchOperation) -> bool:
|
||||
if self.prefetch_stop_policy == "best_effort":
|
||||
@@ -2187,7 +2227,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
# rank timed out) and _all_reduce broadcasts the verdict along PP.
|
||||
should_terminate = False
|
||||
if self.pp_rank == 0:
|
||||
should_terminate = self._prefetch_timeout_check_linear_func(operation)
|
||||
should_terminate = self._prefetch_deadline_expired(operation)
|
||||
should_terminate_tensor = torch.tensor(
|
||||
int(should_terminate), dtype=torch.int, device="cpu"
|
||||
)
|
||||
@@ -2267,17 +2307,19 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
|
||||
operation = self.ongoing_prefetch[request].operation
|
||||
|
||||
# Determine whether or not we should terminate this prefetch request.
|
||||
should_terminate = operation.is_terminated() or self._can_terminate_prefetch(
|
||||
operation
|
||||
)
|
||||
|
||||
if not should_terminate:
|
||||
return False
|
||||
if not operation.is_terminated():
|
||||
if not self._can_terminate_prefetch(operation):
|
||||
self._start_storage_prefetch_wait(operation)
|
||||
return False
|
||||
if self.prefetch_stop_policy == "timeout":
|
||||
self._mark_prefetch_deadline_expired(operation)
|
||||
|
||||
self.cache_controller.terminate_prefetch(operation)
|
||||
if operation.host_indices is None:
|
||||
self.storage_prefetch_retries.poll_miss(request.rid)
|
||||
if operation.deadline_expired:
|
||||
self._finish_storage_prefetch(request, 0, reason="deadline")
|
||||
else:
|
||||
self.storage_prefetch_retries.poll_miss(request.rid)
|
||||
self.revoke_pending_prefetch(request)
|
||||
else:
|
||||
self._handle_prefetch_result(operation)
|
||||
@@ -2317,7 +2359,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
self._resolve_storage_prefetch_tokens(
|
||||
request,
|
||||
allocated_tokens - completed_tokens,
|
||||
reason="storage_transfer",
|
||||
reason="deadline" if operation.deadline_expired else "storage_transfer",
|
||||
)
|
||||
if (
|
||||
completed_tokens > 0
|
||||
@@ -2445,7 +2487,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
if pool_transfers and not all_succeeded:
|
||||
# Drop each pool's beliefs beyond what it served (KV and its
|
||||
# sidecars beyond the completed span); the next insert re-writes
|
||||
# only the missing pools.
|
||||
# only the missing pools. A deadline is not evidence of absent data.
|
||||
invalidation_hashes = (
|
||||
operation.sidecar_hash_values
|
||||
if operation.sidecar_hash_values is not None
|
||||
@@ -2454,7 +2496,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
trimmed_pages = len(invalidation_hashes) - len(hash_value)
|
||||
keep_pages = trimmed_pages + completed_tokens // self.page_size
|
||||
beliefs = self.storage_existence_cache
|
||||
if keep_pages < len(invalidation_hashes):
|
||||
if not operation.deadline_expired and keep_pages < len(invalidation_hashes):
|
||||
self._belief_policy(
|
||||
PoolName.KV, PoolHitPolicy.ALL_PAGES
|
||||
).heal_from_transfer(
|
||||
@@ -2470,15 +2512,18 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
keep_pages,
|
||||
)
|
||||
for transfer, count in zip(pool_transfers, pool_hit_pages):
|
||||
if transfer.keys is not None and count < len(transfer.keys):
|
||||
if (
|
||||
not operation.deadline_expired
|
||||
and transfer.keys is not None
|
||||
and count < len(transfer.keys)
|
||||
):
|
||||
self._belief_policy(
|
||||
transfer.name, transfer.hit_policy
|
||||
).heal_from_transfer(
|
||||
beliefs.pool(transfer.name), transfer.keys, count
|
||||
)
|
||||
self._finish_storage_prefetch(
|
||||
request, fulfilled_tokens=0, reason="storage_transfer"
|
||||
)
|
||||
reason = "deadline" if operation.deadline_expired else "storage_transfer"
|
||||
self._finish_storage_prefetch(request, fulfilled_tokens=0, reason=reason)
|
||||
# The controller's prefetch IO thread already releases the untransferred
|
||||
# tail (host_indices[completed_tokens:])
|
||||
self._retire_ongoing_prefetch(
|
||||
@@ -2489,12 +2534,13 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
self.prefetch_loaded_tokens_by_reqid[request] = 0
|
||||
self.prefetch_loaded_storage_start_by_reqid.pop(request, None)
|
||||
logger.warning(
|
||||
"HiCache hybrid prefetch discarded req=%s completed=%d requested=%d "
|
||||
"kv_beliefs_kept_pages=%d aux_pool_hit_pages=%s",
|
||||
"HiCache hybrid prefetch discarded req=%s reason=%s completed=%d "
|
||||
"requested=%d kv_beliefs_kept_pages=%s aux_pool_hit_pages=%s",
|
||||
request.rid,
|
||||
reason,
|
||||
completed_tokens,
|
||||
expected_tokens,
|
||||
keep_pages,
|
||||
None if operation.deadline_expired else keep_pages,
|
||||
{
|
||||
getattr(transfer.name, "value", transfer.name): count
|
||||
for transfer, count in zip(pool_transfers, pool_hit_pages)
|
||||
@@ -2586,11 +2632,12 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
def finish_storage_prefetch_admission(
|
||||
self, request: CacheRequestHandle, fulfilled_tokens: int, reason: Optional[str]
|
||||
) -> None:
|
||||
"""Resolve a cache-mode L3 hit after request admission.
|
||||
"""Close the admission wait budget and resolve a cache-mode L3 hit.
|
||||
|
||||
Buffer mode resolves at the H2D completion ack instead, because its
|
||||
host allocation is transport staging rather than a resident L2 hit.
|
||||
"""
|
||||
self._storage_prefetch_wait.pop(request, None)
|
||||
if self.host_memory_mode == "cache":
|
||||
self._finish_storage_prefetch(request, fulfilled_tokens, reason)
|
||||
|
||||
@@ -2627,6 +2674,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
@rank_consensus(same_params=True)
|
||||
def release_aborted_request(self, request: CacheRequestHandle) -> None:
|
||||
rid = request.rid
|
||||
self._storage_prefetch_wait.pop(request, None)
|
||||
if self.linker is not None:
|
||||
self.linker.release_request(rid)
|
||||
self.prefetch_loaded_tokens_by_reqid.pop(request, None)
|
||||
@@ -2702,7 +2750,8 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
"""Heal each pool's beliefs beyond its rank-agreed boundary (a
|
||||
chain-page pool also learns presence up to it); a pool without a
|
||||
verdict heals from the folded cut."""
|
||||
if self.host_memory_mode != "buffer_only":
|
||||
if self.host_memory_mode != "buffer_only" or operation.deadline_expired:
|
||||
# A cut query's agreed hit may be a MIN over skipped lookups, not a verdict.
|
||||
return
|
||||
chain = operation.all_hash_values
|
||||
if chain is None:
|
||||
@@ -2736,7 +2785,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
"""Feed the cumulative prefetch-outcome counters at the (rank-synced)
|
||||
query outcome: T = prompt tokens, L = requested, m = L3-miss."""
|
||||
requested = operation.stats_requested_tokens
|
||||
if requested <= 0:
|
||||
if requested <= 0 or operation.deadline_expired:
|
||||
return
|
||||
stats = self._prefetch_outcome_stats
|
||||
hit = max(0, min(operation.storage_hit_count, requested))
|
||||
@@ -3027,22 +3076,21 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
if hit_tokens > 0:
|
||||
self._record_storage_prefetch_hit(request, hit_tokens)
|
||||
if operation.is_terminated():
|
||||
# Controller-side miss termination (retryable) or an abort
|
||||
# race (abort cleanup discards the marker).
|
||||
# A storage miss can retry; a policy deadline cannot.
|
||||
if hit_tokens > 0:
|
||||
reason = None
|
||||
if operation.deadline_expired:
|
||||
reason = "deadline"
|
||||
elif hit_tokens < self.prefetch_threshold:
|
||||
reason = "below_threshold"
|
||||
self._finish_storage_prefetch(
|
||||
request,
|
||||
fulfilled_tokens=0,
|
||||
reason=(
|
||||
"below_threshold"
|
||||
if hit_tokens < self.prefetch_threshold
|
||||
else None
|
||||
),
|
||||
request, fulfilled_tokens=0, reason=reason
|
||||
)
|
||||
# The published (rank-agreed) hit count is the L3 verdict;
|
||||
# without this, store misses never reach l3_miss_tokens.
|
||||
self._account_prefetch_outcome(operation, revoked=True)
|
||||
self.storage_prefetch_retries.poll_miss(request.rid)
|
||||
if not operation.deadline_expired:
|
||||
self.storage_prefetch_retries.poll_miss(request.rid)
|
||||
self.revoke_pending_prefetch(request)
|
||||
continue
|
||||
if hit_tokens < self.prefetch_threshold:
|
||||
|
||||
@@ -2175,7 +2175,8 @@ class StorageMetricsCollector(_StatLoggerDIMixin):
|
||||
"device_overlap means the live device prefix diverged from the "
|
||||
"request's view at splice time, so the span beyond it was released; "
|
||||
"cache_admission_shortfall means an L3-loaded L2 span was no longer "
|
||||
"reusable when cache-mode admission ran.",
|
||||
"reusable when cache-mode admission ran; deadline means the "
|
||||
"storage prefetch stop policy ended the read before it completed.",
|
||||
labelnames=list(labels.keys()) + ["reason"],
|
||||
)
|
||||
for reason in (
|
||||
@@ -2184,6 +2185,7 @@ class StorageMetricsCollector(_StatLoggerDIMixin):
|
||||
"device_capacity",
|
||||
"device_covered",
|
||||
"storage_transfer",
|
||||
"deadline",
|
||||
"anchor_lost",
|
||||
"aux_window_trim",
|
||||
"device_overlap",
|
||||
|
||||
@@ -42,6 +42,7 @@ from sglang.srt.observability.trace_async import (
|
||||
is_async_tracing_available,
|
||||
)
|
||||
from sglang.srt.utils import get_bool_env_var
|
||||
from sglang.srt.utils.poll_timer import PollTimer
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.disaggregation.base.conn import KVTransferMetric
|
||||
@@ -668,8 +669,8 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
|
||||
# None means unmeasured (including stats received from an older scheduler).
|
||||
storage_prefetch_wait_s: Optional[float] = None
|
||||
storage_staging_wait_s: Optional[float] = None
|
||||
_storage_prefetch_wait_start: Optional[float] = None
|
||||
_storage_staging_wait_start: Optional[float] = None
|
||||
_storage_prefetch_wait: Optional[PollTimer] = None
|
||||
_storage_staging_wait: Optional[PollTimer] = None
|
||||
|
||||
has_timing_data: bool = False
|
||||
|
||||
@@ -687,12 +688,16 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
|
||||
"queue_duration_s": self.queue_duration_s,
|
||||
"storage_prefetch_wait_s": (
|
||||
self.storage_prefetch_wait_s
|
||||
if self._storage_prefetch_wait_start is None
|
||||
if not (
|
||||
self._storage_prefetch_wait and self._storage_prefetch_wait.running
|
||||
)
|
||||
else None
|
||||
),
|
||||
"storage_staging_wait_s": (
|
||||
self.storage_staging_wait_s
|
||||
if self._storage_staging_wait_start is None
|
||||
if not (
|
||||
self._storage_staging_wait and self._storage_staging_wait.running
|
||||
)
|
||||
else None
|
||||
),
|
||||
"diff_realtime_monotonic": global_diff_realtime_monotonic,
|
||||
@@ -756,33 +761,17 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
|
||||
self.queue_duration_s = 0.0
|
||||
self.storage_prefetch_wait_s = None
|
||||
self.storage_staging_wait_s = None
|
||||
self._storage_prefetch_wait_start = None
|
||||
self._storage_staging_wait_start = None
|
||||
self._storage_prefetch_wait = None
|
||||
self._storage_staging_wait = None
|
||||
|
||||
def observe_storage_prefetch_ready(self, ready: bool):
|
||||
"""Accumulate first blocked poll to the subsequent ready poll, per episode."""
|
||||
if self.storage_prefetch_wait_s is None:
|
||||
return
|
||||
if ready:
|
||||
if self._storage_prefetch_wait_start is not None:
|
||||
self.storage_prefetch_wait_s += (
|
||||
time.perf_counter() - self._storage_prefetch_wait_start
|
||||
)
|
||||
self._storage_prefetch_wait_start = None
|
||||
elif self._storage_prefetch_wait_start is None:
|
||||
self._storage_prefetch_wait_start = time.perf_counter()
|
||||
if self._storage_prefetch_wait is not None:
|
||||
self.storage_prefetch_wait_s += self._storage_prefetch_wait.observe(ready)
|
||||
|
||||
def observe_storage_staging_ready(self, ready: bool):
|
||||
if self.storage_staging_wait_s is None:
|
||||
return
|
||||
if ready:
|
||||
if self._storage_staging_wait_start is not None:
|
||||
self.storage_staging_wait_s += (
|
||||
time.perf_counter() - self._storage_staging_wait_start
|
||||
)
|
||||
self._storage_staging_wait_start = None
|
||||
elif self._storage_staging_wait_start is None:
|
||||
self._storage_staging_wait_start = time.perf_counter()
|
||||
if self._storage_staging_wait is not None:
|
||||
self.storage_staging_wait_s += self._storage_staging_wait.observe(ready)
|
||||
|
||||
def set_wait_queue_entry_time(self, ts=None):
|
||||
ts = ts or time.perf_counter()
|
||||
@@ -790,6 +779,8 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
|
||||
if self.enable_metrics and self.disagg_mode != DisaggregationMode.DECODE:
|
||||
self.storage_prefetch_wait_s = 0.0
|
||||
self.storage_staging_wait_s = 0.0
|
||||
self._storage_prefetch_wait = PollTimer()
|
||||
self._storage_staging_wait = PollTimer()
|
||||
if self.enable_metrics or self.trace_ctx.tracing_enable:
|
||||
if self.disagg_mode == DisaggregationMode.PREFILL:
|
||||
stage = RequestStage.PREFILL_BOOTSTRAP
|
||||
@@ -1221,14 +1212,12 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
|
||||
|
||||
def convert_to_output_meta_info(self):
|
||||
meta_data = {}
|
||||
if (
|
||||
self.storage_prefetch_wait_s is not None
|
||||
and self._storage_prefetch_wait_start is None
|
||||
if self.storage_prefetch_wait_s is not None and not (
|
||||
self._storage_prefetch_wait and self._storage_prefetch_wait.running
|
||||
):
|
||||
meta_data["storage_prefetch_wait"] = self.storage_prefetch_wait_s
|
||||
if (
|
||||
self.storage_staging_wait_s is not None
|
||||
and self._storage_staging_wait_start is None
|
||||
if self.storage_staging_wait_s is not None and not (
|
||||
self._storage_staging_wait and self._storage_staging_wait.running
|
||||
):
|
||||
meta_data["storage_staging_wait"] = self.storage_staging_wait_s
|
||||
if self.forward_entry_time > 0.0:
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
import msgspec
|
||||
|
||||
|
||||
class PollTimer(msgspec.Struct):
|
||||
"""Time an interval from its first blocked poll to its first ready poll."""
|
||||
|
||||
started_at: Optional[float] = None
|
||||
|
||||
@property
|
||||
def running(self) -> bool:
|
||||
return self.started_at is not None
|
||||
|
||||
def elapsed(self) -> float:
|
||||
return time.perf_counter() - self.started_at if self.running else 0.0
|
||||
|
||||
def observe(self, ready: bool) -> float:
|
||||
if ready:
|
||||
elapsed = self.elapsed()
|
||||
self.started_at = None
|
||||
return elapsed
|
||||
if not self.running:
|
||||
self.started_at = time.perf_counter()
|
||||
return 0.0
|
||||
@@ -497,7 +497,9 @@ class TestPrefillAdder(CustomTestCase):
|
||||
self.mock_tree_cache.discard_storage_prefetch_accounting.assert_called_once_with(
|
||||
req.cache_request_handle
|
||||
)
|
||||
self.mock_tree_cache.finish_storage_prefetch_admission.assert_not_called()
|
||||
self.mock_tree_cache.finish_storage_prefetch_admission.assert_called_once_with(
|
||||
req.cache_request_handle, fulfilled_tokens=0, reason=None
|
||||
)
|
||||
|
||||
def test_preempt_success_high_priority_values_first(self):
|
||||
params = [
|
||||
|
||||
@@ -90,6 +90,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
||||
cache.prefetch_loaded_tokens_by_reqid = {}
|
||||
cache.prefetch_loaded_storage_start_by_reqid = {}
|
||||
cache.storage_prefetch_retries = StoragePrefetchRetries()
|
||||
cache._storage_prefetch_wait = {}
|
||||
cache.linker = None
|
||||
cache.root_node_handle = Mock(return_value=0)
|
||||
cache.buffer_pipeline = Mock(spec=BufferModePipeline)
|
||||
|
||||
@@ -7,7 +7,7 @@ from collections import defaultdict, deque
|
||||
from datetime import timedelta
|
||||
from queue import Queue
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import torch
|
||||
|
||||
@@ -32,6 +32,7 @@ from sglang.srt.mem_cache.unified_radix_cache import (
|
||||
_OngoingPrefetch,
|
||||
)
|
||||
from sglang.srt.mem_cache.utils import get_hash_str
|
||||
from sglang.srt.observability.req_time_stats import SchedulerReqTimeStats
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=20, suite="base-a-test-cpu")
|
||||
@@ -59,6 +60,7 @@ def _staged_fixture(full_match=2):
|
||||
cache.host_memory_mode = "buffer_only"
|
||||
cache.linker = None
|
||||
cache.storage_prefetch_retries = StoragePrefetchRetries()
|
||||
cache._storage_prefetch_wait = {}
|
||||
cache.prefetch_loaded_tokens_by_reqid = {_REQ: 6}
|
||||
cache.prefetch_loaded_storage_start_by_reqid = {_REQ: 2}
|
||||
cache._storage_prefetch_hit_remaining_by_reqid = {}
|
||||
@@ -144,6 +146,7 @@ def _hit_drain_fixture():
|
||||
cache.enable_storage_metrics = False
|
||||
cache.storage_metrics_collector = None
|
||||
cache.storage_prefetch_retries = StoragePrefetchRetries()
|
||||
cache._storage_prefetch_wait = {}
|
||||
cache._prefetch_outcome_stats = defaultdict(int)
|
||||
cache._storage_prefetch_hit_remaining_by_reqid = {}
|
||||
cache._record_storage_prefetch_hit = Mock()
|
||||
@@ -168,6 +171,7 @@ def _terminated_query(cache, rid, hit_tokens):
|
||||
storage_hit_count=hit_tokens,
|
||||
stats_requested_tokens=8,
|
||||
all_hash_values=None,
|
||||
deadline_expired=False,
|
||||
is_terminated=lambda: True,
|
||||
)
|
||||
cache.ongoing_prefetch[handle] = _OngoingPrefetch(
|
||||
@@ -176,6 +180,186 @@ def _terminated_query(cache, rid, hit_tokens):
|
||||
cache.cache_controller.prefetch_hit_queue.put(operation)
|
||||
|
||||
|
||||
def _timeout_fixture(mode="buffer_only", policy="timeout"):
|
||||
cache, pipeline, req = _staged_fixture()
|
||||
pipeline.staged_prefetches.clear()
|
||||
cache.host_memory_mode = mode
|
||||
if mode == "cache":
|
||||
cache.buffer_pipeline = None
|
||||
cache.prefetch_stop_policy = policy
|
||||
cache.prefetch_timeout_base = 1.0
|
||||
cache.prefetch_timeout_per_page = 0.25
|
||||
cache.pp_rank = 0
|
||||
cache._all_reduce = lambda tensor, op: None
|
||||
cache.inc_host_lock_ref = Mock()
|
||||
cache.dec_host_lock_ref = Mock()
|
||||
cache.cache_controller.pp_prefetch_decisions = {}
|
||||
cache.cache_controller.prefetch_tokens_occupied = 0
|
||||
cache.prefetch_from_storage(_REQ, 0, list(range(8)))
|
||||
return cache, cache.ongoing_prefetch[_REQ].operation
|
||||
|
||||
|
||||
def _two_rank_prefetch_deadline(rank, rendezvous):
|
||||
torch.distributed.init_process_group(
|
||||
"gloo",
|
||||
init_method=f"file://{rendezvous}",
|
||||
rank=rank,
|
||||
world_size=2,
|
||||
timeout=timedelta(seconds=30),
|
||||
)
|
||||
try:
|
||||
cache, operation = _timeout_fixture()
|
||||
cache._all_reduce = torch.distributed.all_reduce
|
||||
# Only one rank has reached admission; its expiry must close both budgets.
|
||||
if rank == 0:
|
||||
with patch("time.perf_counter", return_value=100.0):
|
||||
cache._start_storage_prefetch_wait(operation)
|
||||
with patch("time.perf_counter", return_value=103.0):
|
||||
assert cache.check_prefetch_progress(_REQ)
|
||||
assert operation.deadline_expired
|
||||
assert cache._storage_prefetch_wait[_REQ].expired
|
||||
cache.prefetch_from_storage(_REQ, 0, list(range(8)))
|
||||
assert not cache.ongoing_prefetch
|
||||
assert not cache.storage_prefetch_retries._pending
|
||||
finally:
|
||||
torch.distributed.destroy_process_group()
|
||||
|
||||
|
||||
class TestStoragePrefetchTimeout(unittest.TestCase):
|
||||
def test_metric_episodes_do_not_restart_the_timeout(self):
|
||||
"""Closing a wait-metric episode on readiness must not restart the
|
||||
timeout budget: a refetch keeps the budget until admission."""
|
||||
for enable_metrics in (False, True):
|
||||
with self.subTest(enable_metrics=enable_metrics):
|
||||
cache, _ = _timeout_fixture()
|
||||
stats = SchedulerReqTimeStats(
|
||||
enable_metrics=enable_metrics, metrics_collector=Mock()
|
||||
)
|
||||
stats.set_wait_queue_entry_time(99.0)
|
||||
|
||||
def poll(now):
|
||||
with patch("time.perf_counter", return_value=now):
|
||||
ready = cache.check_prefetch_progress(_REQ)
|
||||
stats.observe_storage_prefetch_ready(ready)
|
||||
return ready
|
||||
|
||||
self.assertFalse(poll(100.0))
|
||||
self.assertFalse(poll(100.5))
|
||||
# Readiness closes a metric episode; a subsequent refetch must
|
||||
# retain the timeout budget until the request is admitted.
|
||||
cache.revoke_pending_prefetch(_REQ)
|
||||
self.assertTrue(poll(101.0))
|
||||
cache.prefetch_from_storage(_REQ, 0, list(range(8)))
|
||||
self.assertFalse(poll(101.5))
|
||||
self.assertTrue(poll(102.1))
|
||||
self.assertTrue(cache._storage_prefetch_wait[_REQ].expired)
|
||||
meta = stats.convert_to_output_meta_info()
|
||||
if enable_metrics:
|
||||
self.assertAlmostEqual(meta["storage_prefetch_wait"], 1.6)
|
||||
else:
|
||||
self.assertNotIn("storage_prefetch_wait", meta)
|
||||
|
||||
def test_budget_starts_at_first_wait_and_survives_refetch(self):
|
||||
"""The budget is sized from the first requested span at the first
|
||||
blocked poll and is kept across re-issues until admission or abort."""
|
||||
for mode in ("buffer_only", "cache"):
|
||||
with self.subTest(mode=mode):
|
||||
cache, operation = _timeout_fixture(mode)
|
||||
operation.start_time = 0.0
|
||||
with patch("time.perf_counter", return_value=100.0):
|
||||
self.assertFalse(cache.check_prefetch_progress(_REQ))
|
||||
self.assertEqual(
|
||||
cache._storage_prefetch_wait[_REQ].timer.started_at, 100.0
|
||||
)
|
||||
self.assertEqual(cache._storage_prefetch_wait[_REQ].timeout_s, 2.0)
|
||||
# A shorter hit cannot reduce the original four-page allowance.
|
||||
operation.hash_value = ["hit"]
|
||||
with patch("time.perf_counter", return_value=101.5):
|
||||
self.assertFalse(cache.check_prefetch_progress(_REQ))
|
||||
cache.revoke_pending_prefetch(_REQ)
|
||||
cache.prefetch_from_storage(_REQ, 0, list(range(4)))
|
||||
refetch = cache.ongoing_prefetch[_REQ].operation
|
||||
self.assertIsNot(refetch, operation)
|
||||
self.assertFalse(cache.check_prefetch_progress(_REQ))
|
||||
self.assertEqual(
|
||||
cache._storage_prefetch_wait[_REQ].timer.started_at, 100.0
|
||||
)
|
||||
self.assertEqual(cache._storage_prefetch_wait[_REQ].timeout_s, 2.0)
|
||||
with patch("time.perf_counter", return_value=102.1):
|
||||
self.assertTrue(cache.check_prefetch_progress(_REQ))
|
||||
self.assertTrue(refetch.deadline_expired)
|
||||
self.assertNotIn(_REQ, cache.ongoing_prefetch)
|
||||
cache.storage_prefetch_retries.refetch(_REQ.rid)
|
||||
cache.prefetch_from_storage(_REQ, 0, list(range(8)))
|
||||
self.assertNotIn(_REQ, cache.ongoing_prefetch)
|
||||
self.assertFalse(cache.storage_prefetch_retries._pending)
|
||||
cache.finish_storage_prefetch_admission(_REQ, 0, None)
|
||||
self.assertFalse(cache._storage_prefetch_wait)
|
||||
cache.prefetch_from_storage(_REQ, 0, list(range(8)))
|
||||
with patch("time.perf_counter", return_value=200.0):
|
||||
self.assertFalse(cache.check_prefetch_progress(_REQ))
|
||||
self.assertEqual(
|
||||
cache._storage_prefetch_wait[_REQ].timer.started_at, 200.0
|
||||
)
|
||||
self.assertEqual(cache._storage_prefetch_wait[_REQ].timeout_s, 2.0)
|
||||
cache.release_aborted_request(_REQ)
|
||||
self.assertFalse(cache._storage_prefetch_wait)
|
||||
|
||||
def test_timeout_preserves_complete_kv_with_pending_sidecar(self):
|
||||
"""A lapsed budget must not cut a read whose KV already landed and
|
||||
only waits on a sidecar: cutting it would salvage nothing."""
|
||||
for mode in ("buffer_only", "cache"):
|
||||
for remaining_pages in (0, 2):
|
||||
with self.subTest(mode=mode, remaining_pages=remaining_pages):
|
||||
cache, operation = _timeout_fixture(mode)
|
||||
with patch("time.perf_counter", return_value=100.0):
|
||||
self.assertFalse(cache.check_prefetch_progress(_REQ))
|
||||
operation.hash_value = ["hit"] * remaining_pages
|
||||
operation.host_indices = torch.arange(remaining_pages * 2)
|
||||
operation.completed_tokens = remaining_pages * 2
|
||||
operation.pool_transfers = [PoolTransfer(name=PoolName.SWA)]
|
||||
operation.pool_transfers_done = False
|
||||
with patch("time.perf_counter", return_value=200.0):
|
||||
self.assertFalse(cache.check_prefetch_progress(_REQ))
|
||||
self.assertFalse(operation.deadline_expired)
|
||||
|
||||
def test_stop_policies_distinguish_deadline_from_storage_failure(self):
|
||||
"""Only the timeout policy marks a deadline cut; a storage failure
|
||||
under any policy terminates without the deadline flag."""
|
||||
for policy in ("timeout", "wait_complete", "best_effort"):
|
||||
for failed in (False, True):
|
||||
with self.subTest(policy=policy, failed=failed):
|
||||
cache, operation = _timeout_fixture(policy=policy)
|
||||
operation.hash_value = ["a", "b", "c", "d"]
|
||||
operation.host_indices = torch.arange(8)
|
||||
operation.completed_tokens = 2
|
||||
cache._handle_prefetch_result = Mock()
|
||||
with patch("time.perf_counter", return_value=100.0):
|
||||
cache.check_prefetch_progress(_REQ)
|
||||
if failed:
|
||||
operation.mark_terminate()
|
||||
with patch("time.perf_counter", return_value=200.0):
|
||||
self.assertEqual(
|
||||
cache.check_prefetch_progress(_REQ),
|
||||
failed or policy != "wait_complete",
|
||||
)
|
||||
self.assertEqual(
|
||||
operation.deadline_expired, policy == "timeout" and not failed
|
||||
)
|
||||
if policy != "timeout":
|
||||
self.assertFalse(cache._storage_prefetch_wait)
|
||||
|
||||
def test_expiry_is_agreed_even_before_a_peer_starts_waiting(self):
|
||||
"""A rank that has not reached its admission poll must still adopt
|
||||
the agreed expiry, or it would re-issue the prefetch alone."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
torch.multiprocessing.spawn(
|
||||
_two_rank_prefetch_deadline,
|
||||
args=(f"{tmpdir}/gloo",),
|
||||
nprocs=2,
|
||||
)
|
||||
|
||||
|
||||
def _two_rank_retry_trace(rank, rendezvous):
|
||||
torch.distributed.init_process_group(
|
||||
"gloo",
|
||||
@@ -577,6 +761,7 @@ class TestStagedPrefetchLifecycle(unittest.TestCase):
|
||||
stats_requested_tokens=8,
|
||||
is_terminated=lambda: False,
|
||||
all_hash_values=None,
|
||||
deadline_expired=False,
|
||||
)
|
||||
cache.ongoing_prefetch[handle] = _OngoingPrefetch(
|
||||
0, RadixKey(array("q", range(8))), None, operation, None, {}
|
||||
|
||||
Reference in New Issue
Block a user