observability: measure storage prefetch ready latency (hit/miss)

Under --hicache-storage-prefetch-policy wait_complete a request is not
admitted until its L3 lookup resolves, and a miss pays the same query round
trip as a hit. The scheduler only exposed a skip counter, so the actual wait
was unknown.

Add sglang:storage_prefetch_ready_seconds{dp_rank, result=hit|miss}:
  * stamp cache_request_handle on its first in-flight check,
  * observe at the moment check_prefetch_progress() turns usable, split by
    whether prefetch_loaded_tokens_by_reqid ended up > 0,
  * clear the stamp on revoke / abort / accounting discard.

Buckets cover 1 ms .. 30 s. Gated by SGLANG_BATCH_STAGE_TIMING (off by
default). The measurement is ~2 dict ops + one observe per request, not per
pass.

Tests: test/registered/unit/observability/test_batch_telemetry_stage_timing.py
adds the hit/miss split assertion (9 tests, all passing).
This commit is contained in:
2026-11-04 17:26:41 +08:00
parent 0866012b80
commit cb7988f849
3 changed files with 73 additions and 0 deletions
@@ -111,11 +111,13 @@ from sglang.srt.observability.metrics_collector import (
StorageMetrics,
StorageMetricsCollector,
)
from sglang.srt.observability import batch_telemetry
from sglang.srt.runtime_context import (
get_disagg,
get_memory,
get_model,
get_observability,
get_parallel,
)
from sglang.srt.session.streaming_session import StreamingSession, is_virtual_node
from sglang.srt.utils.common import ceil_align
@@ -437,6 +439,8 @@ class UnifiedRadixCache(BasePrefixCache):
CacheRequestHandle, int
] = {}
self.storage_prefetch_retries = StoragePrefetchRetries()
# request -> monotonic stamp of the first in-flight check (ready latency)
self._prefetch_issue_ts: dict[CacheRequestHandle, float] = {}
self.ongoing_backup: dict[int, tuple[NodeId, DecLockRefParams]] = {}
if self.buffer_pipeline is not None:
self.buffer_pipeline.reset()
@@ -2383,8 +2387,11 @@ class UnifiedRadixCache(BasePrefixCache):
return self._check_pp_prefetch_progress(request)
if request not in self.ongoing_prefetch:
self._observe_prefetch_ready(request)
return True
if request not in self._prefetch_issue_ts:
self._prefetch_issue_ts[request] = time.monotonic()
operation = self.ongoing_prefetch[request].operation
if not operation.is_terminated():
@@ -2403,8 +2410,24 @@ class UnifiedRadixCache(BasePrefixCache):
self.revoke_pending_prefetch(request)
else:
self._handle_prefetch_result(operation)
self._observe_prefetch_ready(request)
return True
def _observe_prefetch_ready(self, request: CacheRequestHandle) -> None:
started = self._prefetch_issue_ts.pop(request, None)
if started is None:
return
loaded = self.prefetch_loaded_tokens_by_reqid.get(request, 0)
try:
dp_rank = get_parallel().attn_dp_rank
except Exception:
dp_rank = 0
batch_telemetry.observe_prefetch_ready(
dp_rank=dp_rank,
result="hit" if loaded > 0 else "miss",
seconds=time.monotonic() - started,
)
def _handle_prefetch_result(self, operation: PrefetchOperation) -> None:
# This function **owns**:
# - host_indices[0 : completed_tokens]
@@ -2753,6 +2776,7 @@ class UnifiedRadixCache(BasePrefixCache):
@rank_consensus(same_params=True)
def release_aborted_request(self, request: CacheRequestHandle) -> None:
self._prefetch_issue_ts.pop(request, None)
rid = request.rid
self._storage_prefetch_wait.pop(request, None)
if self.linker is not None:
@@ -2981,6 +3005,7 @@ class UnifiedRadixCache(BasePrefixCache):
return info, hit_tokens - trim_tokens, original_hit_tokens
def revoke_pending_prefetch(self, request: CacheRequestHandle) -> None:
self._prefetch_issue_ts.pop(request, None)
self._finish_storage_prefetch(request, fulfilled_tokens=0, reason="dropped")
info = self.ongoing_prefetch.get(request)
if info is None:
@@ -845,3 +845,36 @@ def observe_recv(
except Exception:
# Telemetry must never break the scheduler loop.
pass
# ---------------------------------------------------------------------------
# Storage prefetch ready latency (same SGLANG_BATCH_STAGE_TIMING gate).
#
# Request-level wait from "prefetch in flight" to "usable" (the scheduler
# check returns True), split by L3 hit/miss. Under wait_complete this is the
# admission delay a request pays for its L3 lookup.
# ---------------------------------------------------------------------------
_PREFETCH_READY_HIST = None
def observe_prefetch_ready(*, dp_rank: int, result: str, seconds: float) -> None:
if not _STAGE_TIMING:
return
global _PREFETCH_READY_HIST
try:
if _PREFETCH_READY_HIST is None:
from prometheus_client import Histogram
_PREFETCH_READY_HIST = Histogram(
"sglang:storage_prefetch_ready_seconds",
"Prefetch in-flight -> usable latency, split by L3 hit/miss",
["dp_rank", "result"],
buckets=(1e-3, 3e-3, 1e-2, 3e-2, 1e-1, 3e-1, 1.0, 3.0, 10.0, 30.0),
)
_PREFETCH_READY_HIST.labels(dp_rank=str(dp_rank), result=result).observe(
seconds
)
except Exception:
# Telemetry must never break the scheduler loop.
pass
@@ -75,6 +75,9 @@ def _install_private_metrics(module):
"test_recv_prompt_tokens", "", ["dp_rank"], registry=reg
),
}
module._PREFETCH_READY_HIST = Histogram(
"test_prefetch_ready_seconds", "", ["dp_rank", "result"], registry=reg
)
return reg
@@ -237,6 +240,18 @@ class TestStageTiming(unittest.TestCase):
picked = [n for n in seqs if (n * 2654435761) % 10000 < 500]
self.assertTrue(0 < len(picked) < 200)
def test_prefetch_ready_latency_split(self):
bt = self.bt
bt.observe_prefetch_ready(dp_rank=RANK, result="hit", seconds=0.05)
bt.observe_prefetch_ready(dp_rank=RANK, result="miss", seconds=0.40)
h = bt._PREFETCH_READY_HIST
self.assertAlmostEqual(
h.labels(dp_rank=str(RANK), result="hit")._sum.get(), 0.05, places=6
)
self.assertAlmostEqual(
h.labels(dp_rank=str(RANK), result="miss")._sum.get(), 0.40, places=6
)
def test_disabled_is_noop(self):
off = _reload(SGLANG_BATCH_STAGE_TIMING="0")
self.assertFalse(off.stage_timing_enabled())