observability: measure storage prefetch ready latency (hit/miss)
Under --hicache-storage-prefetch-policy wait_complete a request is not
admitted until its L3 lookup resolves, and a miss pays the same query round
trip as a hit. The scheduler only exposed a skip counter, so the actual wait
was unknown.
Add sglang:storage_prefetch_ready_seconds{dp_rank, result=hit|miss}:
* stamp cache_request_handle on its first in-flight check,
* observe at the moment check_prefetch_progress() turns usable, split by
whether prefetch_loaded_tokens_by_reqid ended up > 0,
* clear the stamp on revoke / abort / accounting discard.
Buckets cover 1 ms .. 30 s. Gated by SGLANG_BATCH_STAGE_TIMING (off by
default). The measurement is ~2 dict ops + one observe per request, not per
pass.
Tests: test/registered/unit/observability/test_batch_telemetry_stage_timing.py
adds the hit/miss split assertion (9 tests, all passing).
This commit is contained in:
@@ -111,11 +111,13 @@ from sglang.srt.observability.metrics_collector import (
|
||||
StorageMetrics,
|
||||
StorageMetricsCollector,
|
||||
)
|
||||
from sglang.srt.observability import batch_telemetry
|
||||
from sglang.srt.runtime_context import (
|
||||
get_disagg,
|
||||
get_memory,
|
||||
get_model,
|
||||
get_observability,
|
||||
get_parallel,
|
||||
)
|
||||
from sglang.srt.session.streaming_session import StreamingSession, is_virtual_node
|
||||
from sglang.srt.utils.common import ceil_align
|
||||
@@ -437,6 +439,8 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
CacheRequestHandle, int
|
||||
] = {}
|
||||
self.storage_prefetch_retries = StoragePrefetchRetries()
|
||||
# request -> monotonic stamp of the first in-flight check (ready latency)
|
||||
self._prefetch_issue_ts: dict[CacheRequestHandle, float] = {}
|
||||
self.ongoing_backup: dict[int, tuple[NodeId, DecLockRefParams]] = {}
|
||||
if self.buffer_pipeline is not None:
|
||||
self.buffer_pipeline.reset()
|
||||
@@ -2383,8 +2387,11 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
return self._check_pp_prefetch_progress(request)
|
||||
|
||||
if request not in self.ongoing_prefetch:
|
||||
self._observe_prefetch_ready(request)
|
||||
return True
|
||||
|
||||
if request not in self._prefetch_issue_ts:
|
||||
self._prefetch_issue_ts[request] = time.monotonic()
|
||||
operation = self.ongoing_prefetch[request].operation
|
||||
|
||||
if not operation.is_terminated():
|
||||
@@ -2403,8 +2410,24 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
self.revoke_pending_prefetch(request)
|
||||
else:
|
||||
self._handle_prefetch_result(operation)
|
||||
self._observe_prefetch_ready(request)
|
||||
return True
|
||||
|
||||
def _observe_prefetch_ready(self, request: CacheRequestHandle) -> None:
|
||||
started = self._prefetch_issue_ts.pop(request, None)
|
||||
if started is None:
|
||||
return
|
||||
loaded = self.prefetch_loaded_tokens_by_reqid.get(request, 0)
|
||||
try:
|
||||
dp_rank = get_parallel().attn_dp_rank
|
||||
except Exception:
|
||||
dp_rank = 0
|
||||
batch_telemetry.observe_prefetch_ready(
|
||||
dp_rank=dp_rank,
|
||||
result="hit" if loaded > 0 else "miss",
|
||||
seconds=time.monotonic() - started,
|
||||
)
|
||||
|
||||
def _handle_prefetch_result(self, operation: PrefetchOperation) -> None:
|
||||
# This function **owns**:
|
||||
# - host_indices[0 : completed_tokens]
|
||||
@@ -2753,6 +2776,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
|
||||
@rank_consensus(same_params=True)
|
||||
def release_aborted_request(self, request: CacheRequestHandle) -> None:
|
||||
self._prefetch_issue_ts.pop(request, None)
|
||||
rid = request.rid
|
||||
self._storage_prefetch_wait.pop(request, None)
|
||||
if self.linker is not None:
|
||||
@@ -2981,6 +3005,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
return info, hit_tokens - trim_tokens, original_hit_tokens
|
||||
|
||||
def revoke_pending_prefetch(self, request: CacheRequestHandle) -> None:
|
||||
self._prefetch_issue_ts.pop(request, None)
|
||||
self._finish_storage_prefetch(request, fulfilled_tokens=0, reason="dropped")
|
||||
info = self.ongoing_prefetch.get(request)
|
||||
if info is None:
|
||||
|
||||
@@ -845,3 +845,36 @@ def observe_recv(
|
||||
except Exception:
|
||||
# Telemetry must never break the scheduler loop.
|
||||
pass
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Storage prefetch ready latency (same SGLANG_BATCH_STAGE_TIMING gate).
|
||||
#
|
||||
# Request-level wait from "prefetch in flight" to "usable" (the scheduler
|
||||
# check returns True), split by L3 hit/miss. Under wait_complete this is the
|
||||
# admission delay a request pays for its L3 lookup.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_PREFETCH_READY_HIST = None
|
||||
|
||||
|
||||
def observe_prefetch_ready(*, dp_rank: int, result: str, seconds: float) -> None:
|
||||
if not _STAGE_TIMING:
|
||||
return
|
||||
global _PREFETCH_READY_HIST
|
||||
try:
|
||||
if _PREFETCH_READY_HIST is None:
|
||||
from prometheus_client import Histogram
|
||||
|
||||
_PREFETCH_READY_HIST = Histogram(
|
||||
"sglang:storage_prefetch_ready_seconds",
|
||||
"Prefetch in-flight -> usable latency, split by L3 hit/miss",
|
||||
["dp_rank", "result"],
|
||||
buckets=(1e-3, 3e-3, 1e-2, 3e-2, 1e-1, 3e-1, 1.0, 3.0, 10.0, 30.0),
|
||||
)
|
||||
_PREFETCH_READY_HIST.labels(dp_rank=str(dp_rank), result=result).observe(
|
||||
seconds
|
||||
)
|
||||
except Exception:
|
||||
# Telemetry must never break the scheduler loop.
|
||||
pass
|
||||
|
||||
Reference in New Issue
Block a user