diff --git a/python/sglang/srt/observability/req_time_stats.py b/python/sglang/srt/observability/req_time_stats.py index c9053c9eb..6d32a2591 100644 --- a/python/sglang/srt/observability/req_time_stats.py +++ b/python/sglang/srt/observability/req_time_stats.py @@ -363,7 +363,7 @@ class ReqTimeStatsBase: state["trace_ctx"] = TraceNullContext() for key in state.keys(): - if key.endswith("time"): + if key.endswith("time") and state[key]: state[key] = convert_time_cross_thread( state[key], state["diff_realtime_monotonic"], diff --git a/test/registered/unit/observability/test_req_time_stats.py b/test/registered/unit/observability/test_req_time_stats.py new file mode 100644 index 000000000..82c8e1f47 --- /dev/null +++ b/test/registered/unit/observability/test_req_time_stats.py @@ -0,0 +1,44 @@ +"""Unit tests for ReqTimeStats IPC serialization. + +ReqTimeStatsBase.__setstate__ rebases perf_counter fields onto the receiving +process's clock anchor. Rebasing a field that was never stamped (0.0) turns +the sentinel into a tiny epsilon (sender_diff - receiver_diff), which defeats +== 0.0 / > 0.0 "was this stamped?" checks downstream. Concretely, a PD decode +server never stamps prefill_finished_time locally; if the sentinel arrives at +the tokenizer as an epsilon, first-token bookkeeping mistakes it for a real +stamp and the TTFT / inter-token-latency histograms record ~node-uptime-sized +garbage samples. +""" + +import pickle +import unittest +from unittest import mock + +import sglang.srt.observability.req_time_stats as rts +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +class TestSetstatePreservesUnsetTimeSentinels(CustomTestCase): + def test_two_hop_round_trip(self): + src = rts.SchedulerReqTimeStats() + src.enable_metrics = True + src.wait_queue_entry_time = 123.456 + src.prefill_finished_time = 0.0 + + with mock.patch.object(rts, "global_diff_realtime_monotonic", 1_000_000.0): + blob = pickle.dumps(src) + with mock.patch.object(rts, "global_diff_realtime_monotonic", 1_000_005.0): + hop1 = pickle.loads(blob) + blob2 = pickle.dumps(hop1) + with mock.patch.object(rts, "global_diff_realtime_monotonic", 1_000_009.0): + hop2 = pickle.loads(blob2) + + self.assertEqual(hop2.prefill_finished_time, 0.0) + self.assertAlmostEqual(hop2.wait_queue_entry_time, 123.456 - 9.0) + + +if __name__ == "__main__": + unittest.main()