diff --git a/python/sglang/srt/observability/req_time_stats.py b/python/sglang/srt/observability/req_time_stats.py index 4afd9d73b3..6f88ce189f 100644 --- a/python/sglang/srt/observability/req_time_stats.py +++ b/python/sglang/srt/observability/req_time_stats.py @@ -648,6 +648,10 @@ class SchedulerReqTimeStats(ReqTimeStatsBase): transfer_speed_gb_s: float = 0.0 transfer_total_mb: float = 0.0 + # Seconds spent in the waiting queue over every entry (a retracted request + # re-enters it). Must not end in "time": __setstate__ clock-rebases those. + queue_duration_s: float = 0.0 + has_timing_data: bool = False def __getstate__(self) -> object: @@ -660,6 +664,8 @@ class SchedulerReqTimeStats(ReqTimeStatsBase): "wait_queue_entry_time": self.wait_queue_entry_time, "forward_entry_time": self.forward_entry_time, "prefill_finished_time": self.prefill_finished_time, + # Read by meta_info["queue_time"] on the tokenizer; timestamps cannot rebuild it. + "queue_duration_s": self.queue_duration_s, "diff_realtime_monotonic": global_diff_realtime_monotonic, } return state @@ -742,6 +748,7 @@ class SchedulerReqTimeStats(ReqTimeStatsBase): self.last_forward_entry_time = 0.0 self.last_prefill_finished_time = 0.0 self.last_chunked_prefill_finish_time = 0.0 + self.queue_duration_s = 0.0 def set_wait_queue_entry_time(self, ts=None): ts = ts or time.perf_counter() @@ -769,9 +776,13 @@ class SchedulerReqTimeStats(ReqTimeStatsBase): if self.forward_entry_time == 0.0: self.forward_entry_time = ts self.last_forward_entry_time = ts + self.queue_duration_s += ts - self.wait_queue_entry_time if self.enable_metrics: - self.metrics_collector.observe_queue_time(self.get_queueing_time()) + # One sample per request: the wait before the first forward. + self.metrics_collector.observe_queue_time( + ts - self.wait_queue_entry_time + ) if self.enable_metrics or self.trace_ctx.tracing_enable: if self.disagg_mode == DisaggregationMode.DECODE: @@ -796,7 +807,9 @@ class SchedulerReqTimeStats(ReqTimeStatsBase): convert_time_to_realtime_ns(ts), ) elif self.last_forward_entry_time == 0.0: + # First forward after a retraction; later prefill chunks skip this branch. self.last_forward_entry_time = ts + self.queue_duration_s += ts - self.wait_queue_entry_time def set_last_chunked_prefill_finish_time(self, ts=None): ts = ts or time.perf_counter() @@ -1053,13 +1066,11 @@ class SchedulerReqTimeStats(ReqTimeStatsBase): self.trace_slice(stage, self.last_forward_entry_time, ts) def get_queueing_time(self) -> float: - return self.forward_entry_time - self.wait_queue_entry_time + return self.queue_duration_s def convert_to_duration(self) -> str: if self.disagg_mode == DisaggregationMode.NULL: - queue_duration = self.duration_between( - self.wait_queue_entry_time, self.forward_entry_time - ) + queue_duration = self.get_queueing_time() forward_duration = self.duration_between( self.forward_entry_time, self.completion_time ) @@ -1074,9 +1085,7 @@ class SchedulerReqTimeStats(ReqTimeStatsBase): bootstrap_queue_duration = self.duration_between( self.prefill_bootstrap_queue_entry_time, self.wait_queue_entry_time ) - queue_duration = self.duration_between( - self.wait_queue_entry_time, self.forward_entry_time - ) + queue_duration = self.get_queueing_time() forward_duration = self.duration_between( self.forward_entry_time, self.completion_time ) @@ -1127,10 +1136,7 @@ class SchedulerReqTimeStats(ReqTimeStatsBase): self.decode_transfer_queue_entry_time, self.wait_queue_entry_time, ) - queue_duration = self.duration_between( - self.wait_queue_entry_time, - self.forward_entry_time, - ) + queue_duration = self.get_queueing_time() forward_duration = self.duration_between( self.forward_entry_time, self.completion_time, diff --git a/test/registered/unit/observability/test_req_time_stats.py b/test/registered/unit/observability/test_req_time_stats.py index bfb5ddf4f2..c6dd7dac4b 100644 --- a/test/registered/unit/observability/test_req_time_stats.py +++ b/test/registered/unit/observability/test_req_time_stats.py @@ -146,5 +146,90 @@ class TestSetFinishedTimeSpanAttrs(CustomTestCase): self.assertEqual(caller_attrs, {"gen_ai.request.id": "rid-1"}) +class TestQueueTimeAcrossRetraction(CustomTestCase): + """A retracted request re-enters the waiting queue after its first forward. + + queue_time must be the total time spent waiting across every entry. It used + to be first-forward minus the latest queue entry, which went negative. + """ + + T0 = 1000.0 + + def _stats(self, disagg_mode=rts.DisaggregationMode.NULL): + stats = rts.SchedulerReqTimeStats(disagg_mode=disagg_mode) + stats.enable_metrics = True + stats.metrics_collector = mock.MagicMock() + return stats + + def _retract_once(self, stats, forward_calls=(0.10,), readmit_calls=(1.50,)): + # waits: [0.00, 0.10) before the first forward, [1.00, 1.50) after retraction + stats.set_wait_queue_entry_time(self.T0 + 0.00) + for dt in forward_calls: + stats.set_forward_entry_time(self.T0 + dt) + stats.set_wait_queue_entry_time(self.T0 + 1.00) + for dt in readmit_calls: + stats.set_forward_entry_time(self.T0 + dt) + stats.set_completion_time(self.T0 + 2.00) + + def test_retracted_request_reports_total_queue_time(self): + stats = self._stats() + self._retract_once(stats) + + self.assertAlmostEqual(stats.convert_to_output_meta_info()["queue_time"], 0.6) + with mock.patch.object(rts, "SGLANG_TEST_REQUEST_TIME_STATS", True): + self.assertIn("queue_duration=600.00ms", stats.convert_to_duration()) + + def test_queue_time_survives_clock_rebase_across_hops(self): + stats = self._stats() + self._retract_once(stats) + + with mock.patch.object(rts, "global_diff_realtime_monotonic", 1_000_000.0): + blob = pickle.dumps(stats) + with mock.patch.object(rts, "global_diff_realtime_monotonic", 1_000_005.0): + blob = pickle.dumps(pickle.loads(blob)) + with mock.patch.object(rts, "global_diff_realtime_monotonic", 1_000_009.0): + tokenizer_side = pickle.loads(blob) + + self.assertAlmostEqual(tokenizer_side.get_queueing_time(), 0.6) + + def test_prefill_chunks_do_not_extend_queue_time(self): + stats = self._stats() + self._retract_once( + stats, forward_calls=(0.10, 0.30, 0.50), readmit_calls=(1.50, 1.70) + ) + + self.assertAlmostEqual(stats.get_queueing_time(), 0.6) + + def test_queue_time_histogram_keeps_one_first_wait_sample(self): + stats = self._stats() + self._retract_once(stats) + + stats.metrics_collector.observe_queue_time.assert_called_once() + (sample,) = stats.metrics_collector.observe_queue_time.call_args.args + self.assertAlmostEqual(sample, 0.1) + + def test_decode_rebootstrap_counts_both_waits(self): + stats = self._stats(disagg_mode=rts.DisaggregationMode.DECODE) + stats.set_wait_queue_entry_time(self.T0 + 0.00) + stats.set_forward_entry_time(self.T0 + 0.10) + # held for re-bootstrap: not in the waiting queue until [1.00, 1.25) + stats.set_retract_time(self.T0 + 0.50) + stats.set_wait_queue_entry_time(self.T0 + 1.00) + stats.set_forward_entry_time(self.T0 + 1.25) + stats.set_quick_finish_time(self.T0 + 1.30) + + self.assertAlmostEqual(stats.get_queueing_time(), 0.35) + + def test_prefill_retry_reset_discards_earlier_attempt_waits(self): + stats = self._stats(disagg_mode=rts.DisaggregationMode.PREFILL) + stats.set_wait_queue_entry_time(self.T0 + 0.00) + stats.set_forward_entry_time(self.T0 + 0.30) + stats.reset_prefill_retry_time() + stats.set_wait_queue_entry_time(self.T0 + 1.00) + stats.set_forward_entry_time(self.T0 + 1.20) + + self.assertAlmostEqual(stats.get_queueing_time(), 0.2) + + if __name__ == "__main__": unittest.main()