diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index f67d4ed85a..465fa0d7b3 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -111,11 +111,13 @@ from sglang.srt.observability.metrics_collector import ( StorageMetrics, StorageMetricsCollector, ) +from sglang.srt.observability import batch_telemetry from sglang.srt.runtime_context import ( get_disagg, get_memory, get_model, get_observability, + get_parallel, ) from sglang.srt.session.streaming_session import StreamingSession, is_virtual_node from sglang.srt.utils.common import ceil_align @@ -437,6 +439,8 @@ class UnifiedRadixCache(BasePrefixCache): CacheRequestHandle, int ] = {} self.storage_prefetch_retries = StoragePrefetchRetries() + # request -> monotonic stamp of the first in-flight check (ready latency) + self._prefetch_issue_ts: dict[CacheRequestHandle, float] = {} self.ongoing_backup: dict[int, tuple[NodeId, DecLockRefParams]] = {} if self.buffer_pipeline is not None: self.buffer_pipeline.reset() @@ -2383,8 +2387,11 @@ class UnifiedRadixCache(BasePrefixCache): return self._check_pp_prefetch_progress(request) if request not in self.ongoing_prefetch: + self._observe_prefetch_ready(request) return True + if request not in self._prefetch_issue_ts: + self._prefetch_issue_ts[request] = time.monotonic() operation = self.ongoing_prefetch[request].operation if not operation.is_terminated(): @@ -2403,8 +2410,24 @@ class UnifiedRadixCache(BasePrefixCache): self.revoke_pending_prefetch(request) else: self._handle_prefetch_result(operation) + self._observe_prefetch_ready(request) return True + def _observe_prefetch_ready(self, request: CacheRequestHandle) -> None: + started = self._prefetch_issue_ts.pop(request, None) + if started is None: + return + loaded = self.prefetch_loaded_tokens_by_reqid.get(request, 0) + try: + dp_rank = get_parallel().attn_dp_rank + except Exception: + dp_rank = 0 + batch_telemetry.observe_prefetch_ready( + dp_rank=dp_rank, + result="hit" if loaded > 0 else "miss", + seconds=time.monotonic() - started, + ) + def _handle_prefetch_result(self, operation: PrefetchOperation) -> None: # This function **owns**: # - host_indices[0 : completed_tokens] @@ -2753,6 +2776,7 @@ class UnifiedRadixCache(BasePrefixCache): @rank_consensus(same_params=True) def release_aborted_request(self, request: CacheRequestHandle) -> None: + self._prefetch_issue_ts.pop(request, None) rid = request.rid self._storage_prefetch_wait.pop(request, None) if self.linker is not None: @@ -2981,6 +3005,7 @@ class UnifiedRadixCache(BasePrefixCache): return info, hit_tokens - trim_tokens, original_hit_tokens def revoke_pending_prefetch(self, request: CacheRequestHandle) -> None: + self._prefetch_issue_ts.pop(request, None) self._finish_storage_prefetch(request, fulfilled_tokens=0, reason="dropped") info = self.ongoing_prefetch.get(request) if info is None: diff --git a/python/sglang/srt/observability/batch_telemetry.py b/python/sglang/srt/observability/batch_telemetry.py index a56a647071..191f38c325 100644 --- a/python/sglang/srt/observability/batch_telemetry.py +++ b/python/sglang/srt/observability/batch_telemetry.py @@ -845,3 +845,36 @@ def observe_recv( except Exception: # Telemetry must never break the scheduler loop. pass + + +# --------------------------------------------------------------------------- +# Storage prefetch ready latency (same SGLANG_BATCH_STAGE_TIMING gate). +# +# Request-level wait from "prefetch in flight" to "usable" (the scheduler +# check returns True), split by L3 hit/miss. Under wait_complete this is the +# admission delay a request pays for its L3 lookup. +# --------------------------------------------------------------------------- + +_PREFETCH_READY_HIST = None + + +def observe_prefetch_ready(*, dp_rank: int, result: str, seconds: float) -> None: + if not _STAGE_TIMING: + return + global _PREFETCH_READY_HIST + try: + if _PREFETCH_READY_HIST is None: + from prometheus_client import Histogram + + _PREFETCH_READY_HIST = Histogram( + "sglang:storage_prefetch_ready_seconds", + "Prefetch in-flight -> usable latency, split by L3 hit/miss", + ["dp_rank", "result"], + buckets=(1e-3, 3e-3, 1e-2, 3e-2, 1e-1, 3e-1, 1.0, 3.0, 10.0, 30.0), + ) + _PREFETCH_READY_HIST.labels(dp_rank=str(dp_rank), result=result).observe( + seconds + ) + except Exception: + # Telemetry must never break the scheduler loop. + pass diff --git a/test/registered/unit/observability/test_batch_telemetry_stage_timing.py b/test/registered/unit/observability/test_batch_telemetry_stage_timing.py index af47dea867..a33a347753 100644 --- a/test/registered/unit/observability/test_batch_telemetry_stage_timing.py +++ b/test/registered/unit/observability/test_batch_telemetry_stage_timing.py @@ -75,6 +75,9 @@ def _install_private_metrics(module): "test_recv_prompt_tokens", "", ["dp_rank"], registry=reg ), } + module._PREFETCH_READY_HIST = Histogram( + "test_prefetch_ready_seconds", "", ["dp_rank", "result"], registry=reg + ) return reg @@ -237,6 +240,18 @@ class TestStageTiming(unittest.TestCase): picked = [n for n in seqs if (n * 2654435761) % 10000 < 500] self.assertTrue(0 < len(picked) < 200) + def test_prefetch_ready_latency_split(self): + bt = self.bt + bt.observe_prefetch_ready(dp_rank=RANK, result="hit", seconds=0.05) + bt.observe_prefetch_ready(dp_rank=RANK, result="miss", seconds=0.40) + h = bt._PREFETCH_READY_HIST + self.assertAlmostEqual( + h.labels(dp_rank=str(RANK), result="hit")._sum.get(), 0.05, places=6 + ) + self.assertAlmostEqual( + h.labels(dp_rank=str(RANK), result="miss")._sum.get(), 0.40, places=6 + ) + def test_disabled_is_noop(self): off = _reload(SGLANG_BATCH_STAGE_TIMING="0") self.assertFalse(off.stage_timing_enabled())