[Observability] Report per-request storage prefetch and staging waits in meta_info (#43946)

Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com>
Co-authored-by: xiezhq-hermann <xiezhq-hermann@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-13 11:13:01 -07:00
committed by GitHub
co-authored by Hanming Lu xiezhq-hermann
parent e98c6d7b25
commit ff3f8c348c
3 changed files with 177 additions and 6 deletions
+6 -6
View File
@@ -4102,6 +4102,7 @@ class Scheduler(
prefetch_done = self.tree_cache.check_prefetch_progress(
req.cache_request_handle
)
req.time_stats.observe_storage_prefetch_ready(prefetch_done)
if not prefetch_done:
# skip staging requests that are ongoing prefetch
continue
@@ -4122,12 +4123,11 @@ class Scheduler(
self._prefetch_after_device_hit_loss(req)
):
continue
if (
self.enable_hicache_storage
and buffer_pipeline is not None
and not buffer_pipeline.prepare_staged_prefetch(req)
):
continue
if self.enable_hicache_storage and buffer_pipeline is not None:
staging_ready = buffer_pipeline.prepare_staged_prefetch(req)
req.time_stats.observe_storage_staging_ready(staging_ready)
if not staging_ready:
continue
res = adder.add_one_req(
req,
has_chunked_req=(self.chunked_req is not None),
@@ -664,6 +664,13 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
# re-enters it). Must not end in "time": __setstate__ clock-rebases those.
queue_duration_s: float = 0.0
# Poll-observed waits; staging can include a re-prefetch and overlap prefetch.
# None means unmeasured (including stats received from an older scheduler).
storage_prefetch_wait_s: Optional[float] = None
storage_staging_wait_s: Optional[float] = None
_storage_prefetch_wait_start: Optional[float] = None
_storage_staging_wait_start: Optional[float] = None
has_timing_data: bool = False
def __getstate__(self) -> object:
@@ -678,6 +685,16 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
"prefill_finished_time": self.prefill_finished_time,
# Read by meta_info["queue_time"] on the tokenizer; timestamps cannot rebuild it.
"queue_duration_s": self.queue_duration_s,
"storage_prefetch_wait_s": (
self.storage_prefetch_wait_s
if self._storage_prefetch_wait_start is None
else None
),
"storage_staging_wait_s": (
self.storage_staging_wait_s
if self._storage_staging_wait_start is None
else None
),
"diff_realtime_monotonic": global_diff_realtime_monotonic,
}
return state
@@ -737,10 +754,42 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
self.last_prefill_finished_time = 0.0
self.last_chunked_prefill_finish_time = 0.0
self.queue_duration_s = 0.0
self.storage_prefetch_wait_s = None
self.storage_staging_wait_s = None
self._storage_prefetch_wait_start = None
self._storage_staging_wait_start = None
def observe_storage_prefetch_ready(self, ready: bool):
"""Accumulate first blocked poll to the subsequent ready poll, per episode."""
if self.storage_prefetch_wait_s is None:
return
if ready:
if self._storage_prefetch_wait_start is not None:
self.storage_prefetch_wait_s += (
time.perf_counter() - self._storage_prefetch_wait_start
)
self._storage_prefetch_wait_start = None
elif self._storage_prefetch_wait_start is None:
self._storage_prefetch_wait_start = time.perf_counter()
def observe_storage_staging_ready(self, ready: bool):
if self.storage_staging_wait_s is None:
return
if ready:
if self._storage_staging_wait_start is not None:
self.storage_staging_wait_s += (
time.perf_counter() - self._storage_staging_wait_start
)
self._storage_staging_wait_start = None
elif self._storage_staging_wait_start is None:
self._storage_staging_wait_start = time.perf_counter()
def set_wait_queue_entry_time(self, ts=None):
ts = ts or time.perf_counter()
if self.wait_queue_entry_time == 0.0:
if self.enable_metrics and self.disagg_mode != DisaggregationMode.DECODE:
self.storage_prefetch_wait_s = 0.0
self.storage_staging_wait_s = 0.0
if self.enable_metrics or self.trace_ctx.tracing_enable:
if self.disagg_mode == DisaggregationMode.PREFILL:
stage = RequestStage.PREFILL_BOOTSTRAP
@@ -1172,6 +1221,16 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
def convert_to_output_meta_info(self):
meta_data = {}
if (
self.storage_prefetch_wait_s is not None
and self._storage_prefetch_wait_start is None
):
meta_data["storage_prefetch_wait"] = self.storage_prefetch_wait_s
if (
self.storage_staging_wait_s is not None
and self._storage_staging_wait_start is None
):
meta_data["storage_staging_wait"] = self.storage_staging_wait_s
if self.forward_entry_time > 0.0:
meta_data["forward_entry_time"] = convert_time_to_realtime(
self.forward_entry_time