[Observability] Report per-request storage prefetch and staging waits in meta_info (#43946)
Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com> Co-authored-by: xiezhq-hermann <xiezhq-hermann@users.noreply.github.com>
This commit is contained in:
co-authored by
Hanming Lu
xiezhq-hermann
parent
e98c6d7b25
commit
ff3f8c348c
@@ -4102,6 +4102,7 @@ class Scheduler(
|
||||
prefetch_done = self.tree_cache.check_prefetch_progress(
|
||||
req.cache_request_handle
|
||||
)
|
||||
req.time_stats.observe_storage_prefetch_ready(prefetch_done)
|
||||
if not prefetch_done:
|
||||
# skip staging requests that are ongoing prefetch
|
||||
continue
|
||||
@@ -4122,12 +4123,11 @@ class Scheduler(
|
||||
self._prefetch_after_device_hit_loss(req)
|
||||
):
|
||||
continue
|
||||
if (
|
||||
self.enable_hicache_storage
|
||||
and buffer_pipeline is not None
|
||||
and not buffer_pipeline.prepare_staged_prefetch(req)
|
||||
):
|
||||
continue
|
||||
if self.enable_hicache_storage and buffer_pipeline is not None:
|
||||
staging_ready = buffer_pipeline.prepare_staged_prefetch(req)
|
||||
req.time_stats.observe_storage_staging_ready(staging_ready)
|
||||
if not staging_ready:
|
||||
continue
|
||||
res = adder.add_one_req(
|
||||
req,
|
||||
has_chunked_req=(self.chunked_req is not None),
|
||||
|
||||
@@ -664,6 +664,13 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
|
||||
# re-enters it). Must not end in "time": __setstate__ clock-rebases those.
|
||||
queue_duration_s: float = 0.0
|
||||
|
||||
# Poll-observed waits; staging can include a re-prefetch and overlap prefetch.
|
||||
# None means unmeasured (including stats received from an older scheduler).
|
||||
storage_prefetch_wait_s: Optional[float] = None
|
||||
storage_staging_wait_s: Optional[float] = None
|
||||
_storage_prefetch_wait_start: Optional[float] = None
|
||||
_storage_staging_wait_start: Optional[float] = None
|
||||
|
||||
has_timing_data: bool = False
|
||||
|
||||
def __getstate__(self) -> object:
|
||||
@@ -678,6 +685,16 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
|
||||
"prefill_finished_time": self.prefill_finished_time,
|
||||
# Read by meta_info["queue_time"] on the tokenizer; timestamps cannot rebuild it.
|
||||
"queue_duration_s": self.queue_duration_s,
|
||||
"storage_prefetch_wait_s": (
|
||||
self.storage_prefetch_wait_s
|
||||
if self._storage_prefetch_wait_start is None
|
||||
else None
|
||||
),
|
||||
"storage_staging_wait_s": (
|
||||
self.storage_staging_wait_s
|
||||
if self._storage_staging_wait_start is None
|
||||
else None
|
||||
),
|
||||
"diff_realtime_monotonic": global_diff_realtime_monotonic,
|
||||
}
|
||||
return state
|
||||
@@ -737,10 +754,42 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
|
||||
self.last_prefill_finished_time = 0.0
|
||||
self.last_chunked_prefill_finish_time = 0.0
|
||||
self.queue_duration_s = 0.0
|
||||
self.storage_prefetch_wait_s = None
|
||||
self.storage_staging_wait_s = None
|
||||
self._storage_prefetch_wait_start = None
|
||||
self._storage_staging_wait_start = None
|
||||
|
||||
def observe_storage_prefetch_ready(self, ready: bool):
|
||||
"""Accumulate first blocked poll to the subsequent ready poll, per episode."""
|
||||
if self.storage_prefetch_wait_s is None:
|
||||
return
|
||||
if ready:
|
||||
if self._storage_prefetch_wait_start is not None:
|
||||
self.storage_prefetch_wait_s += (
|
||||
time.perf_counter() - self._storage_prefetch_wait_start
|
||||
)
|
||||
self._storage_prefetch_wait_start = None
|
||||
elif self._storage_prefetch_wait_start is None:
|
||||
self._storage_prefetch_wait_start = time.perf_counter()
|
||||
|
||||
def observe_storage_staging_ready(self, ready: bool):
|
||||
if self.storage_staging_wait_s is None:
|
||||
return
|
||||
if ready:
|
||||
if self._storage_staging_wait_start is not None:
|
||||
self.storage_staging_wait_s += (
|
||||
time.perf_counter() - self._storage_staging_wait_start
|
||||
)
|
||||
self._storage_staging_wait_start = None
|
||||
elif self._storage_staging_wait_start is None:
|
||||
self._storage_staging_wait_start = time.perf_counter()
|
||||
|
||||
def set_wait_queue_entry_time(self, ts=None):
|
||||
ts = ts or time.perf_counter()
|
||||
if self.wait_queue_entry_time == 0.0:
|
||||
if self.enable_metrics and self.disagg_mode != DisaggregationMode.DECODE:
|
||||
self.storage_prefetch_wait_s = 0.0
|
||||
self.storage_staging_wait_s = 0.0
|
||||
if self.enable_metrics or self.trace_ctx.tracing_enable:
|
||||
if self.disagg_mode == DisaggregationMode.PREFILL:
|
||||
stage = RequestStage.PREFILL_BOOTSTRAP
|
||||
@@ -1172,6 +1221,16 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
|
||||
|
||||
def convert_to_output_meta_info(self):
|
||||
meta_data = {}
|
||||
if (
|
||||
self.storage_prefetch_wait_s is not None
|
||||
and self._storage_prefetch_wait_start is None
|
||||
):
|
||||
meta_data["storage_prefetch_wait"] = self.storage_prefetch_wait_s
|
||||
if (
|
||||
self.storage_staging_wait_s is not None
|
||||
and self._storage_staging_wait_start is None
|
||||
):
|
||||
meta_data["storage_staging_wait"] = self.storage_staging_wait_s
|
||||
if self.forward_entry_time > 0.0:
|
||||
meta_data["forward_entry_time"] = convert_time_to_realtime(
|
||||
self.forward_entry_time
|
||||
|
||||
Reference in New Issue
Block a user