diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index 465fa0d7b3..3fb717ee18 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -2232,6 +2232,8 @@ class UnifiedRadixCache(BasePrefixCache): anchor_lock_params, comp_xfers, ) + if batch_telemetry.stage_timing_enabled(): + self._prefetch_issue_ts.setdefault(request, time.monotonic()) if buffer_mode: # The query reads no tree state; pinning across it would hold the # anchor cap for a storage round trip, so the pin waits for IO commit. @@ -2387,11 +2389,8 @@ class UnifiedRadixCache(BasePrefixCache): return self._check_pp_prefetch_progress(request) if request not in self.ongoing_prefetch: - self._observe_prefetch_ready(request) return True - if request not in self._prefetch_issue_ts: - self._prefetch_issue_ts[request] = time.monotonic() operation = self.ongoing_prefetch[request].operation if not operation.is_terminated(): @@ -2410,21 +2409,21 @@ class UnifiedRadixCache(BasePrefixCache): self.revoke_pending_prefetch(request) else: self._handle_prefetch_result(operation) - self._observe_prefetch_ready(request) return True - def _observe_prefetch_ready(self, request: CacheRequestHandle) -> None: + def _observe_prefetch_ready( + self, request: CacheRequestHandle, hit: bool + ) -> None: started = self._prefetch_issue_ts.pop(request, None) if started is None: return - loaded = self.prefetch_loaded_tokens_by_reqid.get(request, 0) try: dp_rank = get_parallel().attn_dp_rank except Exception: dp_rank = 0 batch_telemetry.observe_prefetch_ready( dp_rank=dp_rank, - result="hit" if loaded > 0 else "miss", + result="hit" if hit else "miss", seconds=time.monotonic() - started, ) @@ -2824,6 +2823,9 @@ class UnifiedRadixCache(BasePrefixCache): ) -> None: """Drop an in-flight prefetch, releasing ``host_indices`` and ``extra_pools``.""" info = self.ongoing_prefetch.pop(request) + self._observe_prefetch_ready( + request, bool(info.operation.storage_hit_count) + ) if info.anchor_lock_params is not None: self.dec_host_lock_ref(info.anchor_node_id, info.anchor_lock_params) if self.buffer_pipeline is not None: @@ -3005,7 +3007,6 @@ class UnifiedRadixCache(BasePrefixCache): return info, hit_tokens - trim_tokens, original_hit_tokens def revoke_pending_prefetch(self, request: CacheRequestHandle) -> None: - self._prefetch_issue_ts.pop(request, None) self._finish_storage_prefetch(request, fulfilled_tokens=0, reason="dropped") info = self.ongoing_prefetch.get(request) if info is None: