diff --git a/docs/docs/references/production_metrics.mdx b/docs/docs/references/production_metrics.mdx index 12fa8557ad..edf96a17de 100644 --- a/docs/docs/references/production_metrics.mdx +++ b/docs/docs/references/production_metrics.mdx @@ -72,29 +72,6 @@ sglang:e2e_request_latency_seconds_bucket{le="50.0",model_name="meta-llama/Llama sglang:e2e_request_latency_seconds_bucket{le="60.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 1513.0 sglang:e2e_request_latency_seconds_bucket{le="+Inf",model_name="meta-llama/Llama-3.1-8B-Instruct"} 11228.0 sglang:e2e_request_latency_seconds_count{model_name="meta-llama/Llama-3.1-8B-Instruct"} 11228.0 -# HELP sglang:time_per_output_token_seconds Histogram of time per output token in seconds. -# TYPE sglang:time_per_output_token_seconds histogram -sglang:time_per_output_token_seconds_sum{model_name="meta-llama/Llama-3.1-8B-Instruct"} 866964.5791549598 -sglang:time_per_output_token_seconds_bucket{le="0.005",model_name="meta-llama/Llama-3.1-8B-Instruct"} 1.0 -sglang:time_per_output_token_seconds_bucket{le="0.01",model_name="meta-llama/Llama-3.1-8B-Instruct"} 73.0 -sglang:time_per_output_token_seconds_bucket{le="0.015",model_name="meta-llama/Llama-3.1-8B-Instruct"} 382.0 -sglang:time_per_output_token_seconds_bucket{le="0.02",model_name="meta-llama/Llama-3.1-8B-Instruct"} 593.0 -sglang:time_per_output_token_seconds_bucket{le="0.025",model_name="meta-llama/Llama-3.1-8B-Instruct"} 855.0 -sglang:time_per_output_token_seconds_bucket{le="0.03",model_name="meta-llama/Llama-3.1-8B-Instruct"} 1035.0 -sglang:time_per_output_token_seconds_bucket{le="0.04",model_name="meta-llama/Llama-3.1-8B-Instruct"} 1815.0 -sglang:time_per_output_token_seconds_bucket{le="0.05",model_name="meta-llama/Llama-3.1-8B-Instruct"} 11685.0 -sglang:time_per_output_token_seconds_bucket{le="0.075",model_name="meta-llama/Llama-3.1-8B-Instruct"} 433413.0 -sglang:time_per_output_token_seconds_bucket{le="0.1",model_name="meta-llama/Llama-3.1-8B-Instruct"} 4.950195e+06 -sglang:time_per_output_token_seconds_bucket{le="0.15",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.039435e+06 -sglang:time_per_output_token_seconds_bucket{le="0.2",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.171662e+06 -sglang:time_per_output_token_seconds_bucket{le="0.3",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.266055e+06 -sglang:time_per_output_token_seconds_bucket{le="0.4",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.296752e+06 -sglang:time_per_output_token_seconds_bucket{le="0.5",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.312226e+06 -sglang:time_per_output_token_seconds_bucket{le="0.75",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.339675e+06 -sglang:time_per_output_token_seconds_bucket{le="1.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.357747e+06 -sglang:time_per_output_token_seconds_bucket{le="2.5",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.389414e+06 -sglang:time_per_output_token_seconds_bucket{le="+Inf",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.400757e+06 -sglang:time_per_output_token_seconds_count{model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.400757e+06 # HELP sglang:func_latency_seconds Function latency in seconds # TYPE sglang:func_latency_seconds histogram sglang:func_latency_seconds_sum{name="generate_request"} 4.514771912145079 @@ -138,6 +115,32 @@ sglang:spec_num_steps{model_name="meta-llama/Llama-3.1-8B-Instruct"} 3.0 sglang:spec_num_draft_tokens{model_name="meta-llama/Llama-3.1-8B-Instruct"} 4.0 ``` +## Request-level time per output token (TPOT) + +`sglang:request_time_per_output_token_seconds` records TPOT once per successfully finished request: + +```text +TPOT = (finished_time - first_token_time) / (completion_tokens - 1) + = 1 / meta_info["decode_throughput"] +``` + +The first token is excluded because its latency belongs to TTFT. Requests with fewer than two output tokens, streaming requests that finish in their first output, aborted requests, and PD prefill workers do not contribute. The histogram is split by `is_streaming` and reuses the `--bucket-inter-token-latency` buckets. + +Unlike the token-weighted `sglang:inter_token_latency_seconds`, this metric gives every qualifying request equal weight. + +p95 TPOT per model: + +```promql +histogram_quantile( + 0.95, + sum by (model_name, is_streaming, le) ( + rate(sglang:request_time_per_output_token_seconds_bucket[5m]) + ) +) +``` + +Mean TPOT is `rate(..._sum[5m]) / rate(..._count[5m])`. Per-request decode tokens/s is its reciprocal, so p5 TPS is `1 / p95 TPOT`; the reciprocal of mean TPOT is not mean TPS. + ## Diffusion metrics SGLang Diffusion exposes request, queue, stage and LoRA metrics with diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index 3e6a0d9f21..3c11205b9f 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -2594,11 +2594,14 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): # Set first_token_time on the first output batch. # This is the single write point for first_token_time. + first_output_time = None if state.time_stats.first_token_time == 0.0: + first_output_time = time.perf_counter() state.time_stats.set_first_token_time( ts=self._get_scheduler_first_token_time(state, recv_obj, i) + or first_output_time ) - state.time_stats.set_last_time() + state.time_stats.set_last_time(ts=first_output_time) if state.finished: span_attrs = ( @@ -2606,7 +2609,11 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): if state.time_stats.trace_ctx.tracing_enable else None ) - state.time_stats.set_finished_time(span_attrs=span_attrs) + # Reuse the first batch's arrival time: a streaming request that + # finishes in its first output has no decode interval. + state.time_stats.set_finished_time( + ts=first_output_time, span_attrs=span_attrs + ) meta_info["e2e_latency"] = state.time_stats.get_e2e_latency() if get_spec().speculative_algorithm: @@ -3097,6 +3104,15 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): else 0 ) + # Aborts have a truncated decode period; PD prefill workers have none. + finish_reason = recv_obj.finished_reasons[i] or {} + time_per_output_token = ( + state.time_stats.get_time_per_output_token(completion_tokens) + if self.disaggregation_mode != DisaggregationMode.PREFILL + and finish_reason.get("type") in ("stop", "length") + else None + ) + self.metrics_collector.observe_one_finished_request( labels, recv_obj.prompt_tokens[i], @@ -3107,6 +3123,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): cached_tokens_details, spec_verify_ct=spec_verify_ct, is_streaming=getattr(state.obj, "stream", False), + time_per_output_token=time_per_output_token, ) def dump_requests(self, state: ReqState, out_dict: dict): diff --git a/python/sglang/srt/observability/metrics_collector.py b/python/sglang/srt/observability/metrics_collector.py index bab9530251..101f05af8f 100644 --- a/python/sglang/srt/observability/metrics_collector.py +++ b/python/sglang/srt/observability/metrics_collector.py @@ -1752,6 +1752,16 @@ class TokenizerMetricsCollector(_StatLoggerDIMixin): buckets=bucket_inter_token_latency, ) + self.histogram_request_time_per_output_token = Histogram( + name="sglang:request_time_per_output_token_seconds", + documentation=( + "Per-request TPOT in seconds: (finished_time - " + "first_token_time) / (completion_tokens - 1)." + ), + labelnames=[*labels.keys(), "is_streaming"], + buckets=bucket_inter_token_latency, + ) + self.histogram_e2e_request_latency = Histogram( name="sglang:e2e_request_latency_seconds", documentation="Histogram of End-to-end request latency in seconds", @@ -1788,6 +1798,7 @@ class TokenizerMetricsCollector(_StatLoggerDIMixin): cached_tokens_details: Optional[Dict[str, Any]] = None, spec_verify_ct: int = 0, is_streaming: bool = False, + time_per_output_token: Optional[float] = None, ): stream_labels = { **labels, @@ -1826,6 +1837,10 @@ class TokenizerMetricsCollector(_StatLoggerDIMixin): self.histogram_e2e_request_latency.labels(**stream_labels).observe( float(e2e_latency) ) + if time_per_output_token is not None: + self.histogram_request_time_per_output_token.labels( + **stream_labels + ).observe(float(time_per_output_token)) self.prompt_tokens_histogram.labels(**labels).observe(float(prompt_tokens)) self.uncached_prompt_tokens_histogram.labels(**labels).observe( float(prompt_tokens - cached_tokens) diff --git a/python/sglang/srt/observability/req_time_stats.py b/python/sglang/srt/observability/req_time_stats.py index 5af9cbc404..e79470e030 100644 --- a/python/sglang/srt/observability/req_time_stats.py +++ b/python/sglang/srt/observability/req_time_stats.py @@ -488,6 +488,18 @@ class APIServerReqTimeStats(ReqTimeStatsBase): def get_decode_latency(self): return self.finished_time - self.first_token_time + def get_time_per_output_token(self, completion_tokens: int) -> Optional[float]: + if self.first_token_time <= 0.0 or completion_tokens <= 1: + return None + decode_latency = self.get_decode_latency() + if decode_latency <= 0.0: + return None + return decode_latency / (completion_tokens - 1) + + def get_decode_throughput(self, completion_tokens: int) -> Optional[float]: + tpot = self.get_time_per_output_token(completion_tokens) + return None if tpot is None else 1.0 / tpot + def get_response_sent_to_client_realtime(self): return convert_time_to_realtime(self.response_sent_to_client_time) @@ -514,9 +526,9 @@ class APIServerReqTimeStats(ReqTimeStatsBase): self.finished_time ) - decode_latency = self.get_decode_latency() - if decode_latency > 0.0 and completion_tokens > 1: - meta_info["decode_throughput"] = (completion_tokens - 1) / decode_latency + decode_throughput = self.get_decode_throughput(completion_tokens) + if decode_throughput is not None: + meta_info["decode_throughput"] = decode_throughput return meta_info def convert_to_gen_ai_span_attrs(self): diff --git a/test/registered/observability/test_metrics.py b/test/registered/observability/test_metrics.py index 5105e54d40..4013c63bd8 100644 --- a/test/registered/observability/test_metrics.py +++ b/test/registered/observability/test_metrics.py @@ -246,6 +246,10 @@ class TestEnableMetrics(CustomTestCase): ("sglang:startup_time_seconds", {"phase": "scheduler_e2e"}), ("sglang:startup_time_seconds", {"phase": "tokenizer_e2e"}), ("sglang:startup_cuda_graph_time_seconds", {"phase": "decode"}), + ( + "sglang:request_time_per_output_token_seconds_count", + {"is_streaming": "true"}, + ), ] _check_metrics_positive(self, metrics, metrics_to_check) diff --git a/test/registered/unit/managers/test_tokenizer_manager_rid_cleanup.py b/test/registered/unit/managers/test_tokenizer_manager_rid_cleanup.py index eb28f38f0a..0b1e2eb54e 100644 --- a/test/registered/unit/managers/test_tokenizer_manager_rid_cleanup.py +++ b/test/registered/unit/managers/test_tokenizer_manager_rid_cleanup.py @@ -26,6 +26,7 @@ from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel maybe_stub_sgl_kernel() +from sglang.srt.disaggregation.utils import DisaggregationMode # noqa: E402 from sglang.srt.managers.io_struct import ( # noqa: E402 AbortReq, BatchStrOutput, @@ -457,6 +458,76 @@ class TestRidToStateCleanupOnBatchOutput(CustomTestCase): self.assertIn(rid, tm.rid_to_state) +class TestRequestTpotGating(CustomTestCase): + """Request TPOT is observed once, only for requests with a real decode + interval that ran to completion on a non-prefill worker.""" + + def _run( + self, + batches, + disaggregation_mode=DisaggregationMode.NULL, + *, + stream=False, + prefill_finished_time=0.0, + ): + tm = _make_tokenizer_manager(self) + tm.enable_metrics = True + tm.enable_priority_scheduling = False + tm.disaggregation_mode = disaggregation_mode + tm.metrics_collector = MagicMock(labels={}) + rid = "tpot_rid" + state = _make_req_state(rid) + state.obj.log_metrics = True + state.obj.sampling_params = {} + state.obj.custom_labels = None + state.obj.stream = stream + state.time_stats.created_time = time.perf_counter() - 1.0 + tm.rid_to_state[rid] = state + sched_stats = SchedulerReqTimeStats( + enable_metrics=True, prefill_finished_time=prefill_finished_time + ) + for completion_tokens, finished_reason in batches: + batch_output = _make_batch_str_output(rid, finished_reason) + batch_output.completion_tokens = [completion_tokens] + batch_output.time_stats = wrap_as_pickle([sched_stats]) + asyncio.run(tm._handle_batch_output(batch_output)) + (call,) = tm.metrics_collector.observe_one_finished_request.call_args_list + return call.kwargs["time_per_output_token"] + + def test_single_batch_finish(self): + produced = time.perf_counter() - 0.5 + # Streaming: first and final output in one batch leave no decode interval. + self.assertIsNone( + self._run( + [(40, {"type": "stop"})], + stream=True, + prefill_finished_time=produced, + ) + ) + # Non-streaming: decode runs from the scheduler's first-token time. + self.assertGreater( + self._run([(40, {"type": "stop"})], prefill_finished_time=produced), + 0.0, + ) + + def test_multi_batch_records_only_completed_decode(self): + cases = [ + ("stop", {"type": "stop"}, DisaggregationMode.NULL, True), + ("abort", {"type": "abort"}, DisaggregationMode.NULL, False), + ("prefill", {"type": "stop"}, DisaggregationMode.PREFILL, False), + ] + for name, finished_reason, mode, recorded in cases: + with self.subTest(name): + tpot = self._run( + [(1, _NOT_FINISHED), (3, finished_reason)], + disaggregation_mode=mode, + ) + if recorded: + self.assertGreater(tpot, 0.0) + else: + self.assertIsNone(tpot) + + class TestNonStreamingFirstTokenTime(CustomTestCase): def _run(self, *, stream: bool, prefill_finished_time: float): tm = _make_tokenizer_manager(self) diff --git a/test/registered/unit/observability/test_req_time_stats.py b/test/registered/unit/observability/test_req_time_stats.py index c6dd7dac4b..61fd94dd69 100644 --- a/test/registered/unit/observability/test_req_time_stats.py +++ b/test/registered/unit/observability/test_req_time_stats.py @@ -65,6 +65,41 @@ class TestOutputMetaInfo(CustomTestCase): else: self.assertAlmostEqual(meta_info["first_token_latency"], expected) + def test_tpot_excludes_first_token_from_denominator(self): + stats = rts.APIServerReqTimeStats() + stats.created_time = 1.0 + stats.first_token_time = 1.5 + stats.finished_time = 3.5 + # 101 completion tokens, 100 of them in the 2.0s decode interval. + self.assertAlmostEqual(stats.get_time_per_output_token(101), 0.02) + self.assertAlmostEqual(stats.get_decode_throughput(101), 50.0) + meta_info = stats.convert_to_output_meta_info(completion_tokens=101) + self.assertAlmostEqual(meta_info["decode_throughput"], 50.0) + + def test_tpot_undefined_cases(self): + for first, finished, completion_tokens in ( + # Single-batch finish: first token stamped in the same batch. + (3.5, 3.5, 40), + # First token never stamped. + (0.0, 3.5, 40), + # One or zero output tokens: no decode interval to measure. + (1.5, 3.5, 1), + (1.5, 3.5, 0), + ): + with self.subTest( + first=first, finished=finished, completion_tokens=completion_tokens + ): + stats = rts.APIServerReqTimeStats() + stats.created_time = 1.0 + stats.first_token_time = first + stats.finished_time = finished + self.assertIsNone(stats.get_time_per_output_token(completion_tokens)) + self.assertIsNone(stats.get_decode_throughput(completion_tokens)) + meta_info = stats.convert_to_output_meta_info( + completion_tokens=completion_tokens + ) + self.assertNotIn("decode_throughput", meta_info) + class TestConvertToGenAiSpanAttrs(CustomTestCase): def _stats_after_first_token(self) -> rts.APIServerReqTimeStats: diff --git a/test/registered/unit/observability/test_stat_loggers_di.py b/test/registered/unit/observability/test_stat_loggers_di.py index 0d3883f141..512e303152 100644 --- a/test/registered/unit/observability/test_stat_loggers_di.py +++ b/test/registered/unit/observability/test_stat_loggers_di.py @@ -38,6 +38,7 @@ from sglang.srt.observability.metrics_collector import ( resolve_collector_class, ) from sglang.srt.runtime_context import get_context, reset_context +from sglang.test.test_utils import CustomTestCase class _BoundRecordingMetric: @@ -224,5 +225,49 @@ class TestHiCacheMetrics(unittest.TestCase): ) +class TestRequestTimePerOutputToken(CustomTestCase): + def _collector(self, labels): + with get_context().override_server_args( + prompt_tokens_buckets=None, generation_tokens_buckets=None + ): + return _RecordingTokenizerMetricsCollector(labels=labels) + + def _finish(self, collector, labels, **kwargs): + collector.observe_one_finished_request( + labels=labels, + prompt_tokens=20, + generation_tokens=101, + cached_tokens=0, + e2e_latency=2.5, + has_grammar=False, + **kwargs, + ) + + def test_observed_with_streaming_label(self): + labels = {"model_name": "test"} + collector = self._collector(labels) + + self._finish(collector, labels, is_streaming=True, time_per_output_token=0.02) + self._finish(collector, labels, is_streaming=False, time_per_output_token=0.08) + + self.assertEqual( + collector.histogram_request_time_per_output_token.observations, + [ + ({**labels, "is_streaming": "true"}, 0.02), + ({**labels, "is_streaming": "false"}, 0.08), + ], + ) + + def test_not_observed_when_undefined(self): + labels = {"model_name": "test"} + collector = self._collector(labels) + + self._finish(collector, labels, time_per_output_token=None) + + self.assertEqual( + collector.histogram_request_time_per_output_token.observations, [] + ) + + if __name__ == "__main__": unittest.main()