[observability] Fix negative queue_time for retracted requests (#39312)

This commit is contained in:
Suraj
2026-09-22 15:08:01 +08:00
committed by GitHub
parent c948114d58
commit 1815e490de
2 changed files with 103 additions and 12 deletions
@@ -648,6 +648,10 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
transfer_speed_gb_s: float = 0.0
transfer_total_mb: float = 0.0
# Seconds spent in the waiting queue over every entry (a retracted request
# re-enters it). Must not end in "time": __setstate__ clock-rebases those.
queue_duration_s: float = 0.0
has_timing_data: bool = False
def __getstate__(self) -> object:
@@ -660,6 +664,8 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
"wait_queue_entry_time": self.wait_queue_entry_time,
"forward_entry_time": self.forward_entry_time,
"prefill_finished_time": self.prefill_finished_time,
# Read by meta_info["queue_time"] on the tokenizer; timestamps cannot rebuild it.
"queue_duration_s": self.queue_duration_s,
"diff_realtime_monotonic": global_diff_realtime_monotonic,
}
return state
@@ -742,6 +748,7 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
self.last_forward_entry_time = 0.0
self.last_prefill_finished_time = 0.0
self.last_chunked_prefill_finish_time = 0.0
self.queue_duration_s = 0.0
def set_wait_queue_entry_time(self, ts=None):
ts = ts or time.perf_counter()
@@ -769,9 +776,13 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
if self.forward_entry_time == 0.0:
self.forward_entry_time = ts
self.last_forward_entry_time = ts
self.queue_duration_s += ts - self.wait_queue_entry_time
if self.enable_metrics:
self.metrics_collector.observe_queue_time(self.get_queueing_time())
# One sample per request: the wait before the first forward.
self.metrics_collector.observe_queue_time(
ts - self.wait_queue_entry_time
)
if self.enable_metrics or self.trace_ctx.tracing_enable:
if self.disagg_mode == DisaggregationMode.DECODE:
@@ -796,7 +807,9 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
convert_time_to_realtime_ns(ts),
)
elif self.last_forward_entry_time == 0.0:
# First forward after a retraction; later prefill chunks skip this branch.
self.last_forward_entry_time = ts
self.queue_duration_s += ts - self.wait_queue_entry_time
def set_last_chunked_prefill_finish_time(self, ts=None):
ts = ts or time.perf_counter()
@@ -1053,13 +1066,11 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
self.trace_slice(stage, self.last_forward_entry_time, ts)
def get_queueing_time(self) -> float:
return self.forward_entry_time - self.wait_queue_entry_time
return self.queue_duration_s
def convert_to_duration(self) -> str:
if self.disagg_mode == DisaggregationMode.NULL:
queue_duration = self.duration_between(
self.wait_queue_entry_time, self.forward_entry_time
)
queue_duration = self.get_queueing_time()
forward_duration = self.duration_between(
self.forward_entry_time, self.completion_time
)
@@ -1074,9 +1085,7 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
bootstrap_queue_duration = self.duration_between(
self.prefill_bootstrap_queue_entry_time, self.wait_queue_entry_time
)
queue_duration = self.duration_between(
self.wait_queue_entry_time, self.forward_entry_time
)
queue_duration = self.get_queueing_time()
forward_duration = self.duration_between(
self.forward_entry_time, self.completion_time
)
@@ -1127,10 +1136,7 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
self.decode_transfer_queue_entry_time,
self.wait_queue_entry_time,
)
queue_duration = self.duration_between(
self.wait_queue_entry_time,
self.forward_entry_time,
)
queue_duration = self.get_queueing_time()
forward_duration = self.duration_between(
self.forward_entry_time,
self.completion_time,