[Metrics] Add request-level TPOT histogram (sglang:request_time_per_output_token_seconds) (#40275)

Co-authored-by: Gilford Ting <gilford@modal.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
James Liu
2026-09-29 12:20:22 -07:00
committed by GitHub
co-authored by Gilford Ting Claude Fable 5.1
parent 9a16e47437
commit 51ace48387
8 changed files with 230 additions and 28 deletions
@@ -2594,11 +2594,14 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
# Set first_token_time on the first output batch.
# This is the single write point for first_token_time.
first_output_time = None
if state.time_stats.first_token_time == 0.0:
first_output_time = time.perf_counter()
state.time_stats.set_first_token_time(
ts=self._get_scheduler_first_token_time(state, recv_obj, i)
or first_output_time
)
state.time_stats.set_last_time()
state.time_stats.set_last_time(ts=first_output_time)
if state.finished:
span_attrs = (
@@ -2606,7 +2609,11 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
if state.time_stats.trace_ctx.tracing_enable
else None
)
state.time_stats.set_finished_time(span_attrs=span_attrs)
# Reuse the first batch's arrival time: a streaming request that
# finishes in its first output has no decode interval.
state.time_stats.set_finished_time(
ts=first_output_time, span_attrs=span_attrs
)
meta_info["e2e_latency"] = state.time_stats.get_e2e_latency()
if get_spec().speculative_algorithm:
@@ -3097,6 +3104,15 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
else 0
)
# Aborts have a truncated decode period; PD prefill workers have none.
finish_reason = recv_obj.finished_reasons[i] or {}
time_per_output_token = (
state.time_stats.get_time_per_output_token(completion_tokens)
if self.disaggregation_mode != DisaggregationMode.PREFILL
and finish_reason.get("type") in ("stop", "length")
else None
)
self.metrics_collector.observe_one_finished_request(
labels,
recv_obj.prompt_tokens[i],
@@ -3107,6 +3123,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
cached_tokens_details,
spec_verify_ct=spec_verify_ct,
is_streaming=getattr(state.obj, "stream", False),
time_per_output_token=time_per_output_token,
)
def dump_requests(self, state: ReqState, out_dict: dict):
@@ -1752,6 +1752,16 @@ class TokenizerMetricsCollector(_StatLoggerDIMixin):
buckets=bucket_inter_token_latency,
)
self.histogram_request_time_per_output_token = Histogram(
name="sglang:request_time_per_output_token_seconds",
documentation=(
"Per-request TPOT in seconds: (finished_time - "
"first_token_time) / (completion_tokens - 1)."
),
labelnames=[*labels.keys(), "is_streaming"],
buckets=bucket_inter_token_latency,
)
self.histogram_e2e_request_latency = Histogram(
name="sglang:e2e_request_latency_seconds",
documentation="Histogram of End-to-end request latency in seconds",
@@ -1788,6 +1798,7 @@ class TokenizerMetricsCollector(_StatLoggerDIMixin):
cached_tokens_details: Optional[Dict[str, Any]] = None,
spec_verify_ct: int = 0,
is_streaming: bool = False,
time_per_output_token: Optional[float] = None,
):
stream_labels = {
**labels,
@@ -1826,6 +1837,10 @@ class TokenizerMetricsCollector(_StatLoggerDIMixin):
self.histogram_e2e_request_latency.labels(**stream_labels).observe(
float(e2e_latency)
)
if time_per_output_token is not None:
self.histogram_request_time_per_output_token.labels(
**stream_labels
).observe(float(time_per_output_token))
self.prompt_tokens_histogram.labels(**labels).observe(float(prompt_tokens))
self.uncached_prompt_tokens_histogram.labels(**labels).observe(
float(prompt_tokens - cached_tokens)
@@ -488,6 +488,18 @@ class APIServerReqTimeStats(ReqTimeStatsBase):
def get_decode_latency(self):
return self.finished_time - self.first_token_time
def get_time_per_output_token(self, completion_tokens: int) -> Optional[float]:
if self.first_token_time <= 0.0 or completion_tokens <= 1:
return None
decode_latency = self.get_decode_latency()
if decode_latency <= 0.0:
return None
return decode_latency / (completion_tokens - 1)
def get_decode_throughput(self, completion_tokens: int) -> Optional[float]:
tpot = self.get_time_per_output_token(completion_tokens)
return None if tpot is None else 1.0 / tpot
def get_response_sent_to_client_realtime(self):
return convert_time_to_realtime(self.response_sent_to_client_time)
@@ -514,9 +526,9 @@ class APIServerReqTimeStats(ReqTimeStatsBase):
self.finished_time
)
decode_latency = self.get_decode_latency()
if decode_latency > 0.0 and completion_tokens > 1:
meta_info["decode_throughput"] = (completion_tokens - 1) / decode_latency
decode_throughput = self.get_decode_throughput(completion_tokens)
if decode_throughput is not None:
meta_info["decode_throughput"] = decode_throughput
return meta_info
def convert_to_gen_ai_span_attrs(self):