[observability] Fix negative queue_time for retracted requests (#39312)
This commit is contained in:
@@ -648,6 +648,10 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
|
||||
transfer_speed_gb_s: float = 0.0
|
||||
transfer_total_mb: float = 0.0
|
||||
|
||||
# Seconds spent in the waiting queue over every entry (a retracted request
|
||||
# re-enters it). Must not end in "time": __setstate__ clock-rebases those.
|
||||
queue_duration_s: float = 0.0
|
||||
|
||||
has_timing_data: bool = False
|
||||
|
||||
def __getstate__(self) -> object:
|
||||
@@ -660,6 +664,8 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
|
||||
"wait_queue_entry_time": self.wait_queue_entry_time,
|
||||
"forward_entry_time": self.forward_entry_time,
|
||||
"prefill_finished_time": self.prefill_finished_time,
|
||||
# Read by meta_info["queue_time"] on the tokenizer; timestamps cannot rebuild it.
|
||||
"queue_duration_s": self.queue_duration_s,
|
||||
"diff_realtime_monotonic": global_diff_realtime_monotonic,
|
||||
}
|
||||
return state
|
||||
@@ -742,6 +748,7 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
|
||||
self.last_forward_entry_time = 0.0
|
||||
self.last_prefill_finished_time = 0.0
|
||||
self.last_chunked_prefill_finish_time = 0.0
|
||||
self.queue_duration_s = 0.0
|
||||
|
||||
def set_wait_queue_entry_time(self, ts=None):
|
||||
ts = ts or time.perf_counter()
|
||||
@@ -769,9 +776,13 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
|
||||
if self.forward_entry_time == 0.0:
|
||||
self.forward_entry_time = ts
|
||||
self.last_forward_entry_time = ts
|
||||
self.queue_duration_s += ts - self.wait_queue_entry_time
|
||||
|
||||
if self.enable_metrics:
|
||||
self.metrics_collector.observe_queue_time(self.get_queueing_time())
|
||||
# One sample per request: the wait before the first forward.
|
||||
self.metrics_collector.observe_queue_time(
|
||||
ts - self.wait_queue_entry_time
|
||||
)
|
||||
|
||||
if self.enable_metrics or self.trace_ctx.tracing_enable:
|
||||
if self.disagg_mode == DisaggregationMode.DECODE:
|
||||
@@ -796,7 +807,9 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
|
||||
convert_time_to_realtime_ns(ts),
|
||||
)
|
||||
elif self.last_forward_entry_time == 0.0:
|
||||
# First forward after a retraction; later prefill chunks skip this branch.
|
||||
self.last_forward_entry_time = ts
|
||||
self.queue_duration_s += ts - self.wait_queue_entry_time
|
||||
|
||||
def set_last_chunked_prefill_finish_time(self, ts=None):
|
||||
ts = ts or time.perf_counter()
|
||||
@@ -1053,13 +1066,11 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
|
||||
self.trace_slice(stage, self.last_forward_entry_time, ts)
|
||||
|
||||
def get_queueing_time(self) -> float:
|
||||
return self.forward_entry_time - self.wait_queue_entry_time
|
||||
return self.queue_duration_s
|
||||
|
||||
def convert_to_duration(self) -> str:
|
||||
if self.disagg_mode == DisaggregationMode.NULL:
|
||||
queue_duration = self.duration_between(
|
||||
self.wait_queue_entry_time, self.forward_entry_time
|
||||
)
|
||||
queue_duration = self.get_queueing_time()
|
||||
forward_duration = self.duration_between(
|
||||
self.forward_entry_time, self.completion_time
|
||||
)
|
||||
@@ -1074,9 +1085,7 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
|
||||
bootstrap_queue_duration = self.duration_between(
|
||||
self.prefill_bootstrap_queue_entry_time, self.wait_queue_entry_time
|
||||
)
|
||||
queue_duration = self.duration_between(
|
||||
self.wait_queue_entry_time, self.forward_entry_time
|
||||
)
|
||||
queue_duration = self.get_queueing_time()
|
||||
forward_duration = self.duration_between(
|
||||
self.forward_entry_time, self.completion_time
|
||||
)
|
||||
@@ -1127,10 +1136,7 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
|
||||
self.decode_transfer_queue_entry_time,
|
||||
self.wait_queue_entry_time,
|
||||
)
|
||||
queue_duration = self.duration_between(
|
||||
self.wait_queue_entry_time,
|
||||
self.forward_entry_time,
|
||||
)
|
||||
queue_duration = self.get_queueing_time()
|
||||
forward_duration = self.duration_between(
|
||||
self.forward_entry_time,
|
||||
self.completion_time,
|
||||
|
||||
@@ -146,5 +146,90 @@ class TestSetFinishedTimeSpanAttrs(CustomTestCase):
|
||||
self.assertEqual(caller_attrs, {"gen_ai.request.id": "rid-1"})
|
||||
|
||||
|
||||
class TestQueueTimeAcrossRetraction(CustomTestCase):
|
||||
"""A retracted request re-enters the waiting queue after its first forward.
|
||||
|
||||
queue_time must be the total time spent waiting across every entry. It used
|
||||
to be first-forward minus the latest queue entry, which went negative.
|
||||
"""
|
||||
|
||||
T0 = 1000.0
|
||||
|
||||
def _stats(self, disagg_mode=rts.DisaggregationMode.NULL):
|
||||
stats = rts.SchedulerReqTimeStats(disagg_mode=disagg_mode)
|
||||
stats.enable_metrics = True
|
||||
stats.metrics_collector = mock.MagicMock()
|
||||
return stats
|
||||
|
||||
def _retract_once(self, stats, forward_calls=(0.10,), readmit_calls=(1.50,)):
|
||||
# waits: [0.00, 0.10) before the first forward, [1.00, 1.50) after retraction
|
||||
stats.set_wait_queue_entry_time(self.T0 + 0.00)
|
||||
for dt in forward_calls:
|
||||
stats.set_forward_entry_time(self.T0 + dt)
|
||||
stats.set_wait_queue_entry_time(self.T0 + 1.00)
|
||||
for dt in readmit_calls:
|
||||
stats.set_forward_entry_time(self.T0 + dt)
|
||||
stats.set_completion_time(self.T0 + 2.00)
|
||||
|
||||
def test_retracted_request_reports_total_queue_time(self):
|
||||
stats = self._stats()
|
||||
self._retract_once(stats)
|
||||
|
||||
self.assertAlmostEqual(stats.convert_to_output_meta_info()["queue_time"], 0.6)
|
||||
with mock.patch.object(rts, "SGLANG_TEST_REQUEST_TIME_STATS", True):
|
||||
self.assertIn("queue_duration=600.00ms", stats.convert_to_duration())
|
||||
|
||||
def test_queue_time_survives_clock_rebase_across_hops(self):
|
||||
stats = self._stats()
|
||||
self._retract_once(stats)
|
||||
|
||||
with mock.patch.object(rts, "global_diff_realtime_monotonic", 1_000_000.0):
|
||||
blob = pickle.dumps(stats)
|
||||
with mock.patch.object(rts, "global_diff_realtime_monotonic", 1_000_005.0):
|
||||
blob = pickle.dumps(pickle.loads(blob))
|
||||
with mock.patch.object(rts, "global_diff_realtime_monotonic", 1_000_009.0):
|
||||
tokenizer_side = pickle.loads(blob)
|
||||
|
||||
self.assertAlmostEqual(tokenizer_side.get_queueing_time(), 0.6)
|
||||
|
||||
def test_prefill_chunks_do_not_extend_queue_time(self):
|
||||
stats = self._stats()
|
||||
self._retract_once(
|
||||
stats, forward_calls=(0.10, 0.30, 0.50), readmit_calls=(1.50, 1.70)
|
||||
)
|
||||
|
||||
self.assertAlmostEqual(stats.get_queueing_time(), 0.6)
|
||||
|
||||
def test_queue_time_histogram_keeps_one_first_wait_sample(self):
|
||||
stats = self._stats()
|
||||
self._retract_once(stats)
|
||||
|
||||
stats.metrics_collector.observe_queue_time.assert_called_once()
|
||||
(sample,) = stats.metrics_collector.observe_queue_time.call_args.args
|
||||
self.assertAlmostEqual(sample, 0.1)
|
||||
|
||||
def test_decode_rebootstrap_counts_both_waits(self):
|
||||
stats = self._stats(disagg_mode=rts.DisaggregationMode.DECODE)
|
||||
stats.set_wait_queue_entry_time(self.T0 + 0.00)
|
||||
stats.set_forward_entry_time(self.T0 + 0.10)
|
||||
# held for re-bootstrap: not in the waiting queue until [1.00, 1.25)
|
||||
stats.set_retract_time(self.T0 + 0.50)
|
||||
stats.set_wait_queue_entry_time(self.T0 + 1.00)
|
||||
stats.set_forward_entry_time(self.T0 + 1.25)
|
||||
stats.set_quick_finish_time(self.T0 + 1.30)
|
||||
|
||||
self.assertAlmostEqual(stats.get_queueing_time(), 0.35)
|
||||
|
||||
def test_prefill_retry_reset_discards_earlier_attempt_waits(self):
|
||||
stats = self._stats(disagg_mode=rts.DisaggregationMode.PREFILL)
|
||||
stats.set_wait_queue_entry_time(self.T0 + 0.00)
|
||||
stats.set_forward_entry_time(self.T0 + 0.30)
|
||||
stats.reset_prefill_retry_time()
|
||||
stats.set_wait_queue_entry_time(self.T0 + 1.00)
|
||||
stats.set_forward_entry_time(self.T0 + 1.20)
|
||||
|
||||
self.assertAlmostEqual(stats.get_queueing_time(), 0.2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user