[observability] Fix negative queue_time for retracted requests (#39312)
This commit is contained in:
@@ -648,6 +648,10 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
|
||||
transfer_speed_gb_s: float = 0.0
|
||||
transfer_total_mb: float = 0.0
|
||||
|
||||
# Seconds spent in the waiting queue over every entry (a retracted request
|
||||
# re-enters it). Must not end in "time": __setstate__ clock-rebases those.
|
||||
queue_duration_s: float = 0.0
|
||||
|
||||
has_timing_data: bool = False
|
||||
|
||||
def __getstate__(self) -> object:
|
||||
@@ -660,6 +664,8 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
|
||||
"wait_queue_entry_time": self.wait_queue_entry_time,
|
||||
"forward_entry_time": self.forward_entry_time,
|
||||
"prefill_finished_time": self.prefill_finished_time,
|
||||
# Read by meta_info["queue_time"] on the tokenizer; timestamps cannot rebuild it.
|
||||
"queue_duration_s": self.queue_duration_s,
|
||||
"diff_realtime_monotonic": global_diff_realtime_monotonic,
|
||||
}
|
||||
return state
|
||||
@@ -742,6 +748,7 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
|
||||
self.last_forward_entry_time = 0.0
|
||||
self.last_prefill_finished_time = 0.0
|
||||
self.last_chunked_prefill_finish_time = 0.0
|
||||
self.queue_duration_s = 0.0
|
||||
|
||||
def set_wait_queue_entry_time(self, ts=None):
|
||||
ts = ts or time.perf_counter()
|
||||
@@ -769,9 +776,13 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
|
||||
if self.forward_entry_time == 0.0:
|
||||
self.forward_entry_time = ts
|
||||
self.last_forward_entry_time = ts
|
||||
self.queue_duration_s += ts - self.wait_queue_entry_time
|
||||
|
||||
if self.enable_metrics:
|
||||
self.metrics_collector.observe_queue_time(self.get_queueing_time())
|
||||
# One sample per request: the wait before the first forward.
|
||||
self.metrics_collector.observe_queue_time(
|
||||
ts - self.wait_queue_entry_time
|
||||
)
|
||||
|
||||
if self.enable_metrics or self.trace_ctx.tracing_enable:
|
||||
if self.disagg_mode == DisaggregationMode.DECODE:
|
||||
@@ -796,7 +807,9 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
|
||||
convert_time_to_realtime_ns(ts),
|
||||
)
|
||||
elif self.last_forward_entry_time == 0.0:
|
||||
# First forward after a retraction; later prefill chunks skip this branch.
|
||||
self.last_forward_entry_time = ts
|
||||
self.queue_duration_s += ts - self.wait_queue_entry_time
|
||||
|
||||
def set_last_chunked_prefill_finish_time(self, ts=None):
|
||||
ts = ts or time.perf_counter()
|
||||
@@ -1053,13 +1066,11 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
|
||||
self.trace_slice(stage, self.last_forward_entry_time, ts)
|
||||
|
||||
def get_queueing_time(self) -> float:
|
||||
return self.forward_entry_time - self.wait_queue_entry_time
|
||||
return self.queue_duration_s
|
||||
|
||||
def convert_to_duration(self) -> str:
|
||||
if self.disagg_mode == DisaggregationMode.NULL:
|
||||
queue_duration = self.duration_between(
|
||||
self.wait_queue_entry_time, self.forward_entry_time
|
||||
)
|
||||
queue_duration = self.get_queueing_time()
|
||||
forward_duration = self.duration_between(
|
||||
self.forward_entry_time, self.completion_time
|
||||
)
|
||||
@@ -1074,9 +1085,7 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
|
||||
bootstrap_queue_duration = self.duration_between(
|
||||
self.prefill_bootstrap_queue_entry_time, self.wait_queue_entry_time
|
||||
)
|
||||
queue_duration = self.duration_between(
|
||||
self.wait_queue_entry_time, self.forward_entry_time
|
||||
)
|
||||
queue_duration = self.get_queueing_time()
|
||||
forward_duration = self.duration_between(
|
||||
self.forward_entry_time, self.completion_time
|
||||
)
|
||||
@@ -1127,10 +1136,7 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
|
||||
self.decode_transfer_queue_entry_time,
|
||||
self.wait_queue_entry_time,
|
||||
)
|
||||
queue_duration = self.duration_between(
|
||||
self.wait_queue_entry_time,
|
||||
self.forward_entry_time,
|
||||
)
|
||||
queue_duration = self.get_queueing_time()
|
||||
forward_duration = self.duration_between(
|
||||
self.forward_entry_time,
|
||||
self.completion_time,
|
||||
|
||||
Reference in New Issue
Block a user