observability: fix prefetch ready latency to cover fast lookups

The first version stamped the request on its first in-flight
check_prefetch_progress() call. A fast L3 query can be drained by
_process_hicache_events before that check runs (the op is already gone
from ongoing_prefetch), so those requests produced no sample -- exactly
the common case.

Stamp at issue time in prefetch_from_storage() and observe once at
_retire_ongoing_prefetch(), classifying by operation.storage_hit_count.
Retire is the single exit for hit completion, below-threshold revoke,
anchor loss and host-capacity fallback, so hit and miss are both covered.

No behavior change; the hook is still behind SGLANG_BATCH_STAGE_TIMING.
This commit is contained in:
2026-11-04 18:04:34 +08:00
parent cb7988f849
commit 1a274df27e
@@ -2232,6 +2232,8 @@ class UnifiedRadixCache(BasePrefixCache):
anchor_lock_params,
comp_xfers,
)
if batch_telemetry.stage_timing_enabled():
self._prefetch_issue_ts.setdefault(request, time.monotonic())
if buffer_mode:
# The query reads no tree state; pinning across it would hold the
# anchor cap for a storage round trip, so the pin waits for IO commit.
@@ -2387,11 +2389,8 @@ class UnifiedRadixCache(BasePrefixCache):
return self._check_pp_prefetch_progress(request)
if request not in self.ongoing_prefetch:
self._observe_prefetch_ready(request)
return True
if request not in self._prefetch_issue_ts:
self._prefetch_issue_ts[request] = time.monotonic()
operation = self.ongoing_prefetch[request].operation
if not operation.is_terminated():
@@ -2410,21 +2409,21 @@ class UnifiedRadixCache(BasePrefixCache):
self.revoke_pending_prefetch(request)
else:
self._handle_prefetch_result(operation)
self._observe_prefetch_ready(request)
return True
def _observe_prefetch_ready(self, request: CacheRequestHandle) -> None:
def _observe_prefetch_ready(
self, request: CacheRequestHandle, hit: bool
) -> None:
started = self._prefetch_issue_ts.pop(request, None)
if started is None:
return
loaded = self.prefetch_loaded_tokens_by_reqid.get(request, 0)
try:
dp_rank = get_parallel().attn_dp_rank
except Exception:
dp_rank = 0
batch_telemetry.observe_prefetch_ready(
dp_rank=dp_rank,
result="hit" if loaded > 0 else "miss",
result="hit" if hit else "miss",
seconds=time.monotonic() - started,
)
@@ -2824,6 +2823,9 @@ class UnifiedRadixCache(BasePrefixCache):
) -> None:
"""Drop an in-flight prefetch, releasing ``host_indices`` and ``extra_pools``."""
info = self.ongoing_prefetch.pop(request)
self._observe_prefetch_ready(
request, bool(info.operation.storage_hit_count)
)
if info.anchor_lock_params is not None:
self.dec_host_lock_ref(info.anchor_node_id, info.anchor_lock_params)
if self.buffer_pipeline is not None:
@@ -3005,7 +3007,6 @@ class UnifiedRadixCache(BasePrefixCache):
return info, hit_tokens - trim_tokens, original_hit_tokens
def revoke_pending_prefetch(self, request: CacheRequestHandle) -> None:
self._prefetch_issue_ts.pop(request, None)
self._finish_storage_prefetch(request, fulfilled_tokens=0, reason="dropped")
info = self.ongoing_prefetch.get(request)
if info is None: