[observability] Fix negative queue_time for retracted requests (#39312)

This commit is contained in:
Suraj
2026-09-22 15:08:01 +08:00
committed by GitHub
parent c948114d58
commit 1815e490de
2 changed files with 103 additions and 12 deletions
@@ -648,6 +648,10 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
transfer_speed_gb_s: float = 0.0
transfer_total_mb: float = 0.0
# Seconds spent in the waiting queue over every entry (a retracted request
# re-enters it). Must not end in "time": __setstate__ clock-rebases those.
queue_duration_s: float = 0.0
has_timing_data: bool = False
def __getstate__(self) -> object:
@@ -660,6 +664,8 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
"wait_queue_entry_time": self.wait_queue_entry_time,
"forward_entry_time": self.forward_entry_time,
"prefill_finished_time": self.prefill_finished_time,
# Read by meta_info["queue_time"] on the tokenizer; timestamps cannot rebuild it.
"queue_duration_s": self.queue_duration_s,
"diff_realtime_monotonic": global_diff_realtime_monotonic,
}
return state
@@ -742,6 +748,7 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
self.last_forward_entry_time = 0.0
self.last_prefill_finished_time = 0.0
self.last_chunked_prefill_finish_time = 0.0
self.queue_duration_s = 0.0
def set_wait_queue_entry_time(self, ts=None):
ts = ts or time.perf_counter()
@@ -769,9 +776,13 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
if self.forward_entry_time == 0.0:
self.forward_entry_time = ts
self.last_forward_entry_time = ts
self.queue_duration_s += ts - self.wait_queue_entry_time
if self.enable_metrics:
self.metrics_collector.observe_queue_time(self.get_queueing_time())
# One sample per request: the wait before the first forward.
self.metrics_collector.observe_queue_time(
ts - self.wait_queue_entry_time
)
if self.enable_metrics or self.trace_ctx.tracing_enable:
if self.disagg_mode == DisaggregationMode.DECODE:
@@ -796,7 +807,9 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
convert_time_to_realtime_ns(ts),
)
elif self.last_forward_entry_time == 0.0:
# First forward after a retraction; later prefill chunks skip this branch.
self.last_forward_entry_time = ts
self.queue_duration_s += ts - self.wait_queue_entry_time
def set_last_chunked_prefill_finish_time(self, ts=None):
ts = ts or time.perf_counter()
@@ -1053,13 +1066,11 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
self.trace_slice(stage, self.last_forward_entry_time, ts)
def get_queueing_time(self) -> float:
return self.forward_entry_time - self.wait_queue_entry_time
return self.queue_duration_s
def convert_to_duration(self) -> str:
if self.disagg_mode == DisaggregationMode.NULL:
queue_duration = self.duration_between(
self.wait_queue_entry_time, self.forward_entry_time
)
queue_duration = self.get_queueing_time()
forward_duration = self.duration_between(
self.forward_entry_time, self.completion_time
)
@@ -1074,9 +1085,7 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
bootstrap_queue_duration = self.duration_between(
self.prefill_bootstrap_queue_entry_time, self.wait_queue_entry_time
)
queue_duration = self.duration_between(
self.wait_queue_entry_time, self.forward_entry_time
)
queue_duration = self.get_queueing_time()
forward_duration = self.duration_between(
self.forward_entry_time, self.completion_time
)
@@ -1127,10 +1136,7 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
self.decode_transfer_queue_entry_time,
self.wait_queue_entry_time,
)
queue_duration = self.duration_between(
self.wait_queue_entry_time,
self.forward_entry_time,
)
queue_duration = self.get_queueing_time()
forward_duration = self.duration_between(
self.forward_entry_time,
self.completion_time,
@@ -146,5 +146,90 @@ class TestSetFinishedTimeSpanAttrs(CustomTestCase):
self.assertEqual(caller_attrs, {"gen_ai.request.id": "rid-1"})
class TestQueueTimeAcrossRetraction(CustomTestCase):
"""A retracted request re-enters the waiting queue after its first forward.
queue_time must be the total time spent waiting across every entry. It used
to be first-forward minus the latest queue entry, which went negative.
"""
T0 = 1000.0
def _stats(self, disagg_mode=rts.DisaggregationMode.NULL):
stats = rts.SchedulerReqTimeStats(disagg_mode=disagg_mode)
stats.enable_metrics = True
stats.metrics_collector = mock.MagicMock()
return stats
def _retract_once(self, stats, forward_calls=(0.10,), readmit_calls=(1.50,)):
# waits: [0.00, 0.10) before the first forward, [1.00, 1.50) after retraction
stats.set_wait_queue_entry_time(self.T0 + 0.00)
for dt in forward_calls:
stats.set_forward_entry_time(self.T0 + dt)
stats.set_wait_queue_entry_time(self.T0 + 1.00)
for dt in readmit_calls:
stats.set_forward_entry_time(self.T0 + dt)
stats.set_completion_time(self.T0 + 2.00)
def test_retracted_request_reports_total_queue_time(self):
stats = self._stats()
self._retract_once(stats)
self.assertAlmostEqual(stats.convert_to_output_meta_info()["queue_time"], 0.6)
with mock.patch.object(rts, "SGLANG_TEST_REQUEST_TIME_STATS", True):
self.assertIn("queue_duration=600.00ms", stats.convert_to_duration())
def test_queue_time_survives_clock_rebase_across_hops(self):
stats = self._stats()
self._retract_once(stats)
with mock.patch.object(rts, "global_diff_realtime_monotonic", 1_000_000.0):
blob = pickle.dumps(stats)
with mock.patch.object(rts, "global_diff_realtime_monotonic", 1_000_005.0):
blob = pickle.dumps(pickle.loads(blob))
with mock.patch.object(rts, "global_diff_realtime_monotonic", 1_000_009.0):
tokenizer_side = pickle.loads(blob)
self.assertAlmostEqual(tokenizer_side.get_queueing_time(), 0.6)
def test_prefill_chunks_do_not_extend_queue_time(self):
stats = self._stats()
self._retract_once(
stats, forward_calls=(0.10, 0.30, 0.50), readmit_calls=(1.50, 1.70)
)
self.assertAlmostEqual(stats.get_queueing_time(), 0.6)
def test_queue_time_histogram_keeps_one_first_wait_sample(self):
stats = self._stats()
self._retract_once(stats)
stats.metrics_collector.observe_queue_time.assert_called_once()
(sample,) = stats.metrics_collector.observe_queue_time.call_args.args
self.assertAlmostEqual(sample, 0.1)
def test_decode_rebootstrap_counts_both_waits(self):
stats = self._stats(disagg_mode=rts.DisaggregationMode.DECODE)
stats.set_wait_queue_entry_time(self.T0 + 0.00)
stats.set_forward_entry_time(self.T0 + 0.10)
# held for re-bootstrap: not in the waiting queue until [1.00, 1.25)
stats.set_retract_time(self.T0 + 0.50)
stats.set_wait_queue_entry_time(self.T0 + 1.00)
stats.set_forward_entry_time(self.T0 + 1.25)
stats.set_quick_finish_time(self.T0 + 1.30)
self.assertAlmostEqual(stats.get_queueing_time(), 0.35)
def test_prefill_retry_reset_discards_earlier_attempt_waits(self):
stats = self._stats(disagg_mode=rts.DisaggregationMode.PREFILL)
stats.set_wait_queue_entry_time(self.T0 + 0.00)
stats.set_forward_entry_time(self.T0 + 0.30)
stats.reset_prefill_retry_time()
stats.set_wait_queue_entry_time(self.T0 + 1.00)
stats.set_forward_entry_time(self.T0 + 1.20)
self.assertAlmostEqual(stats.get_queueing_time(), 0.2)
if __name__ == "__main__":
unittest.main()