metrics: don't clock-rebase unset time sentinels in ReqTimeStats deserialization (#34335)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Sam Shleifer
2026-08-12 14:27:47 +08:00
committed by GitHub
co-authored by Claude Fable 5
parent 793a11dd27
commit a2e88279c2
2 changed files with 45 additions and 1 deletions
@@ -363,7 +363,7 @@ class ReqTimeStatsBase:
state["trace_ctx"] = TraceNullContext()
for key in state.keys():
if key.endswith("time"):
if key.endswith("time") and state[key]:
state[key] = convert_time_cross_thread(
state[key],
state["diff_realtime_monotonic"],
@@ -0,0 +1,44 @@
"""Unit tests for ReqTimeStats IPC serialization.
ReqTimeStatsBase.__setstate__ rebases perf_counter fields onto the receiving
process's clock anchor. Rebasing a field that was never stamped (0.0) turns
the sentinel into a tiny epsilon (sender_diff - receiver_diff), which defeats
== 0.0 / > 0.0 "was this stamped?" checks downstream. Concretely, a PD decode
server never stamps prefill_finished_time locally; if the sentinel arrives at
the tokenizer as an epsilon, first-token bookkeeping mistakes it for a real
stamp and the TTFT / inter-token-latency histograms record ~node-uptime-sized
garbage samples.
"""
import pickle
import unittest
from unittest import mock
import sglang.srt.observability.req_time_stats as rts
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
class TestSetstatePreservesUnsetTimeSentinels(CustomTestCase):
def test_two_hop_round_trip(self):
src = rts.SchedulerReqTimeStats()
src.enable_metrics = True
src.wait_queue_entry_time = 123.456
src.prefill_finished_time = 0.0
with mock.patch.object(rts, "global_diff_realtime_monotonic", 1_000_000.0):
blob = pickle.dumps(src)
with mock.patch.object(rts, "global_diff_realtime_monotonic", 1_000_005.0):
hop1 = pickle.loads(blob)
blob2 = pickle.dumps(hop1)
with mock.patch.object(rts, "global_diff_realtime_monotonic", 1_000_009.0):
hop2 = pickle.loads(blob2)
self.assertEqual(hop2.prefill_finished_time, 0.0)
self.assertAlmostEqual(hop2.wait_queue_entry_time, 123.456 - 9.0)
if __name__ == "__main__":
unittest.main()