Co-authored-by: Gilford Ting <gilford@modal.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
271 lines
11 KiB
Python
271 lines
11 KiB
Python
"""Unit tests for ReqTimeStats.
|
|
|
|
ReqTimeStatsBase.__setstate__ rebases perf_counter fields onto the receiving
|
|
process's clock anchor. Rebasing a field that was never stamped (0.0) turns
|
|
the sentinel into a tiny epsilon (sender_diff - receiver_diff), which defeats
|
|
== 0.0 / > 0.0 "was this stamped?" checks downstream. Concretely, a PD decode
|
|
server never stamps prefill_finished_time locally; if the sentinel arrives at
|
|
the tokenizer as an epsilon, first-token bookkeeping mistakes it for a real
|
|
stamp and the TTFT / inter-token-latency histograms record ~node-uptime-sized
|
|
garbage samples.
|
|
|
|
The same "was this stamped?" rule governs the gen_ai.latency.* span attrs: the
|
|
ones derived from finished_time are only emitted once it is stamped, and
|
|
set_finished_time() closes the trace root span, so they have to be derived
|
|
inside that call rather than by the caller.
|
|
"""
|
|
|
|
import pickle
|
|
import unittest
|
|
from unittest import mock
|
|
|
|
import sglang.srt.observability.req_time_stats as rts
|
|
from sglang.srt.observability.trace import SpanAttributes
|
|
from sglang.test.ci.ci_register import register_cpu_ci
|
|
from sglang.test.test_utils import CustomTestCase
|
|
|
|
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
|
|
|
|
|
|
class TestSetstatePreservesUnsetTimeSentinels(CustomTestCase):
|
|
def test_two_hop_round_trip(self):
|
|
src = rts.SchedulerReqTimeStats()
|
|
src.enable_metrics = True
|
|
src.wait_queue_entry_time = 123.456
|
|
src.prefill_finished_time = 0.0
|
|
|
|
with mock.patch.object(rts, "global_diff_realtime_monotonic", 1_000_000.0):
|
|
blob = pickle.dumps(src)
|
|
with mock.patch.object(rts, "global_diff_realtime_monotonic", 1_000_005.0):
|
|
hop1 = pickle.loads(blob)
|
|
blob2 = pickle.dumps(hop1)
|
|
with mock.patch.object(rts, "global_diff_realtime_monotonic", 1_000_009.0):
|
|
hop2 = pickle.loads(blob2)
|
|
|
|
self.assertEqual(hop2.prefill_finished_time, 0.0)
|
|
self.assertAlmostEqual(hop2.wait_queue_entry_time, 123.456 - 9.0)
|
|
|
|
|
|
class TestOutputMetaInfo(CustomTestCase):
|
|
def test_first_token_latency_requires_valid_timestamps(self):
|
|
for created, first, expected in (
|
|
(1.0, 1.5, 0.5),
|
|
(0.0, 1.5, None),
|
|
(1.0, 0.0, None),
|
|
(1.0, 1.0, None),
|
|
(1.0, 0.5, None),
|
|
):
|
|
with self.subTest(created=created, first=first):
|
|
stats = rts.APIServerReqTimeStats()
|
|
stats.created_time = created
|
|
stats.first_token_time = first
|
|
meta_info = stats.convert_to_output_meta_info()
|
|
if expected is None:
|
|
self.assertNotIn("first_token_latency", meta_info)
|
|
else:
|
|
self.assertAlmostEqual(meta_info["first_token_latency"], expected)
|
|
|
|
def test_tpot_excludes_first_token_from_denominator(self):
|
|
stats = rts.APIServerReqTimeStats()
|
|
stats.created_time = 1.0
|
|
stats.first_token_time = 1.5
|
|
stats.finished_time = 3.5
|
|
# 101 completion tokens, 100 of them in the 2.0s decode interval.
|
|
self.assertAlmostEqual(stats.get_time_per_output_token(101), 0.02)
|
|
self.assertAlmostEqual(stats.get_decode_throughput(101), 50.0)
|
|
meta_info = stats.convert_to_output_meta_info(completion_tokens=101)
|
|
self.assertAlmostEqual(meta_info["decode_throughput"], 50.0)
|
|
|
|
def test_tpot_undefined_cases(self):
|
|
for first, finished, completion_tokens in (
|
|
# Single-batch finish: first token stamped in the same batch.
|
|
(3.5, 3.5, 40),
|
|
# First token never stamped.
|
|
(0.0, 3.5, 40),
|
|
# One or zero output tokens: no decode interval to measure.
|
|
(1.5, 3.5, 1),
|
|
(1.5, 3.5, 0),
|
|
):
|
|
with self.subTest(
|
|
first=first, finished=finished, completion_tokens=completion_tokens
|
|
):
|
|
stats = rts.APIServerReqTimeStats()
|
|
stats.created_time = 1.0
|
|
stats.first_token_time = first
|
|
stats.finished_time = finished
|
|
self.assertIsNone(stats.get_time_per_output_token(completion_tokens))
|
|
self.assertIsNone(stats.get_decode_throughput(completion_tokens))
|
|
meta_info = stats.convert_to_output_meta_info(
|
|
completion_tokens=completion_tokens
|
|
)
|
|
self.assertNotIn("decode_throughput", meta_info)
|
|
|
|
|
|
class TestConvertToGenAiSpanAttrs(CustomTestCase):
|
|
def _stats_after_first_token(self) -> rts.APIServerReqTimeStats:
|
|
stats = rts.APIServerReqTimeStats()
|
|
stats.created_time = 1.0
|
|
stats.api_server_dispatch_finish_time = 1.1
|
|
stats.first_token_time = 1.5
|
|
return stats
|
|
|
|
def test_prefill_without_finished_time_omits_e2e_and_decode(self):
|
|
attrs = self._stats_after_first_token().convert_to_gen_ai_span_attrs()
|
|
self.assertIn(SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_PREFILL, attrs)
|
|
self.assertIn(SpanAttributes.GEN_AI_LATENCY_TIME_TO_FIRST_TOKEN, attrs)
|
|
self.assertNotIn(SpanAttributes.GEN_AI_LATENCY_E2E, attrs)
|
|
self.assertNotIn(SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_DECODE, attrs)
|
|
self.assertNotIn(SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_INFERENCE, attrs)
|
|
|
|
def test_finished_time_populates_e2e_and_decode(self):
|
|
stats = self._stats_after_first_token()
|
|
stats.finished_time = 2.0
|
|
attrs = stats.convert_to_gen_ai_span_attrs()
|
|
self.assertAlmostEqual(attrs[SpanAttributes.GEN_AI_LATENCY_E2E], 1.0)
|
|
self.assertAlmostEqual(
|
|
attrs[SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_DECODE], 0.5
|
|
)
|
|
self.assertAlmostEqual(
|
|
attrs[SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_PREFILL], 0.4
|
|
)
|
|
self.assertAlmostEqual(
|
|
attrs[SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_INFERENCE], 0.9
|
|
)
|
|
|
|
|
|
class TestSetFinishedTimeSpanAttrs(CustomTestCase):
|
|
def _tracing_stats(self) -> rts.APIServerReqTimeStats:
|
|
stats = rts.APIServerReqTimeStats()
|
|
stats.trace_ctx = mock.MagicMock()
|
|
stats.trace_ctx.tracing_enable = True
|
|
return stats
|
|
|
|
def test_passes_caller_attrs_into_trace_req_finish(self):
|
|
stats = self._tracing_stats()
|
|
|
|
stats.set_finished_time(ts=1.25, span_attrs={"gen_ai.request.id": "rid-1"})
|
|
|
|
self.assertEqual(stats.finished_time, 1.25)
|
|
stats.trace_ctx.trace_req_finish.assert_called_once_with(
|
|
mock.ANY, attrs={"gen_ai.request.id": "rid-1"}
|
|
)
|
|
|
|
def test_merges_latency_attrs_derived_from_finished_time(self):
|
|
stats = self._tracing_stats()
|
|
stats.created_time = 1.0
|
|
stats.api_server_dispatch_finish_time = 1.1
|
|
stats.first_token_time = 1.5
|
|
|
|
stats.set_finished_time(ts=2.0, span_attrs={"gen_ai.request.id": "rid-1"})
|
|
|
|
attrs = stats.trace_ctx.trace_req_finish.call_args.kwargs["attrs"]
|
|
self.assertEqual(attrs["gen_ai.request.id"], "rid-1")
|
|
self.assertAlmostEqual(attrs[SpanAttributes.GEN_AI_LATENCY_E2E], 1.0)
|
|
self.assertAlmostEqual(
|
|
attrs[SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_DECODE], 0.5
|
|
)
|
|
self.assertAlmostEqual(
|
|
attrs[SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_INFERENCE], 0.9
|
|
)
|
|
self.assertAlmostEqual(
|
|
attrs[SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_PREFILL], 0.4
|
|
)
|
|
|
|
def test_caller_attrs_not_mutated(self):
|
|
stats = self._tracing_stats()
|
|
stats.created_time = 1.0
|
|
caller_attrs = {"gen_ai.request.id": "rid-1"}
|
|
|
|
stats.set_finished_time(ts=2.0, span_attrs=caller_attrs)
|
|
|
|
self.assertEqual(caller_attrs, {"gen_ai.request.id": "rid-1"})
|
|
|
|
|
|
class TestQueueTimeAcrossRetraction(CustomTestCase):
|
|
"""A retracted request re-enters the waiting queue after its first forward.
|
|
|
|
queue_time must be the total time spent waiting across every entry. It used
|
|
to be first-forward minus the latest queue entry, which went negative.
|
|
"""
|
|
|
|
T0 = 1000.0
|
|
|
|
def _stats(self, disagg_mode=rts.DisaggregationMode.NULL):
|
|
stats = rts.SchedulerReqTimeStats(disagg_mode=disagg_mode)
|
|
stats.enable_metrics = True
|
|
stats.metrics_collector = mock.MagicMock()
|
|
return stats
|
|
|
|
def _retract_once(self, stats, forward_calls=(0.10,), readmit_calls=(1.50,)):
|
|
# waits: [0.00, 0.10) before the first forward, [1.00, 1.50) after retraction
|
|
stats.set_wait_queue_entry_time(self.T0 + 0.00)
|
|
for dt in forward_calls:
|
|
stats.set_forward_entry_time(self.T0 + dt)
|
|
stats.set_wait_queue_entry_time(self.T0 + 1.00)
|
|
for dt in readmit_calls:
|
|
stats.set_forward_entry_time(self.T0 + dt)
|
|
stats.set_completion_time(self.T0 + 2.00)
|
|
|
|
def test_retracted_request_reports_total_queue_time(self):
|
|
stats = self._stats()
|
|
self._retract_once(stats)
|
|
|
|
self.assertAlmostEqual(stats.convert_to_output_meta_info()["queue_time"], 0.6)
|
|
with mock.patch.object(rts, "SGLANG_TEST_REQUEST_TIME_STATS", True):
|
|
self.assertIn("queue_duration=600.00ms", stats.convert_to_duration())
|
|
|
|
def test_queue_time_survives_clock_rebase_across_hops(self):
|
|
stats = self._stats()
|
|
self._retract_once(stats)
|
|
|
|
with mock.patch.object(rts, "global_diff_realtime_monotonic", 1_000_000.0):
|
|
blob = pickle.dumps(stats)
|
|
with mock.patch.object(rts, "global_diff_realtime_monotonic", 1_000_005.0):
|
|
blob = pickle.dumps(pickle.loads(blob))
|
|
with mock.patch.object(rts, "global_diff_realtime_monotonic", 1_000_009.0):
|
|
tokenizer_side = pickle.loads(blob)
|
|
|
|
self.assertAlmostEqual(tokenizer_side.get_queueing_time(), 0.6)
|
|
|
|
def test_prefill_chunks_do_not_extend_queue_time(self):
|
|
stats = self._stats()
|
|
self._retract_once(
|
|
stats, forward_calls=(0.10, 0.30, 0.50), readmit_calls=(1.50, 1.70)
|
|
)
|
|
|
|
self.assertAlmostEqual(stats.get_queueing_time(), 0.6)
|
|
|
|
def test_queue_time_histogram_keeps_one_first_wait_sample(self):
|
|
stats = self._stats()
|
|
self._retract_once(stats)
|
|
|
|
stats.metrics_collector.observe_queue_time.assert_called_once()
|
|
(sample,) = stats.metrics_collector.observe_queue_time.call_args.args
|
|
self.assertAlmostEqual(sample, 0.1)
|
|
|
|
def test_decode_rebootstrap_counts_both_waits(self):
|
|
stats = self._stats(disagg_mode=rts.DisaggregationMode.DECODE)
|
|
stats.set_wait_queue_entry_time(self.T0 + 0.00)
|
|
stats.set_forward_entry_time(self.T0 + 0.10)
|
|
# held for re-bootstrap: not in the waiting queue until [1.00, 1.25)
|
|
stats.set_retract_time(self.T0 + 0.50)
|
|
stats.set_wait_queue_entry_time(self.T0 + 1.00)
|
|
stats.set_forward_entry_time(self.T0 + 1.25)
|
|
stats.set_quick_finish_time(self.T0 + 1.30)
|
|
|
|
self.assertAlmostEqual(stats.get_queueing_time(), 0.35)
|
|
|
|
def test_prefill_retry_reset_discards_earlier_attempt_waits(self):
|
|
stats = self._stats(disagg_mode=rts.DisaggregationMode.PREFILL)
|
|
stats.set_wait_queue_entry_time(self.T0 + 0.00)
|
|
stats.set_forward_entry_time(self.T0 + 0.30)
|
|
stats.reset_prefill_retry_time()
|
|
stats.set_wait_queue_entry_time(self.T0 + 1.00)
|
|
stats.set_forward_entry_time(self.T0 + 1.20)
|
|
|
|
self.assertAlmostEqual(stats.get_queueing_time(), 0.2)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|