observability: fix prefetch ready latency to cover fast lookups
The first version stamped the request on its first in-flight check_prefetch_progress() call. A fast L3 query can be drained by _process_hicache_events before that check runs (the op is already gone from ongoing_prefetch), so those requests produced no sample -- exactly the common case. Stamp at issue time in prefetch_from_storage() and observe once at _retire_ongoing_prefetch(), classifying by operation.storage_hit_count. Retire is the single exit for hit completion, below-threshold revoke, anchor loss and host-capacity fallback, so hit and miss are both covered. No behavior change; the hook is still behind SGLANG_BATCH_STAGE_TIMING.
This commit is contained in:
@@ -2232,6 +2232,8 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
anchor_lock_params,
|
||||
comp_xfers,
|
||||
)
|
||||
if batch_telemetry.stage_timing_enabled():
|
||||
self._prefetch_issue_ts.setdefault(request, time.monotonic())
|
||||
if buffer_mode:
|
||||
# The query reads no tree state; pinning across it would hold the
|
||||
# anchor cap for a storage round trip, so the pin waits for IO commit.
|
||||
@@ -2387,11 +2389,8 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
return self._check_pp_prefetch_progress(request)
|
||||
|
||||
if request not in self.ongoing_prefetch:
|
||||
self._observe_prefetch_ready(request)
|
||||
return True
|
||||
|
||||
if request not in self._prefetch_issue_ts:
|
||||
self._prefetch_issue_ts[request] = time.monotonic()
|
||||
operation = self.ongoing_prefetch[request].operation
|
||||
|
||||
if not operation.is_terminated():
|
||||
@@ -2410,21 +2409,21 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
self.revoke_pending_prefetch(request)
|
||||
else:
|
||||
self._handle_prefetch_result(operation)
|
||||
self._observe_prefetch_ready(request)
|
||||
return True
|
||||
|
||||
def _observe_prefetch_ready(self, request: CacheRequestHandle) -> None:
|
||||
def _observe_prefetch_ready(
|
||||
self, request: CacheRequestHandle, hit: bool
|
||||
) -> None:
|
||||
started = self._prefetch_issue_ts.pop(request, None)
|
||||
if started is None:
|
||||
return
|
||||
loaded = self.prefetch_loaded_tokens_by_reqid.get(request, 0)
|
||||
try:
|
||||
dp_rank = get_parallel().attn_dp_rank
|
||||
except Exception:
|
||||
dp_rank = 0
|
||||
batch_telemetry.observe_prefetch_ready(
|
||||
dp_rank=dp_rank,
|
||||
result="hit" if loaded > 0 else "miss",
|
||||
result="hit" if hit else "miss",
|
||||
seconds=time.monotonic() - started,
|
||||
)
|
||||
|
||||
@@ -2824,6 +2823,9 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
) -> None:
|
||||
"""Drop an in-flight prefetch, releasing ``host_indices`` and ``extra_pools``."""
|
||||
info = self.ongoing_prefetch.pop(request)
|
||||
self._observe_prefetch_ready(
|
||||
request, bool(info.operation.storage_hit_count)
|
||||
)
|
||||
if info.anchor_lock_params is not None:
|
||||
self.dec_host_lock_ref(info.anchor_node_id, info.anchor_lock_params)
|
||||
if self.buffer_pipeline is not None:
|
||||
@@ -3005,7 +3007,6 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
return info, hit_tokens - trim_tokens, original_hit_tokens
|
||||
|
||||
def revoke_pending_prefetch(self, request: CacheRequestHandle) -> None:
|
||||
self._prefetch_issue_ts.pop(request, None)
|
||||
self._finish_storage_prefetch(request, fulfilled_tokens=0, reason="dropped")
|
||||
info = self.ongoing_prefetch.get(request)
|
||||
if info is None:
|
||||
|
||||
Reference in New Issue
Block a user