[Metrics] Add request-level TPOT histogram (sglang:request_time_per_output_token_seconds) (#40275)
Co-authored-by: Gilford Ting <gilford@modal.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Gilford Ting
Claude Fable 5.1
parent
9a16e47437
commit
51ace48387
@@ -2594,11 +2594,14 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
|
||||
# Set first_token_time on the first output batch.
|
||||
# This is the single write point for first_token_time.
|
||||
first_output_time = None
|
||||
if state.time_stats.first_token_time == 0.0:
|
||||
first_output_time = time.perf_counter()
|
||||
state.time_stats.set_first_token_time(
|
||||
ts=self._get_scheduler_first_token_time(state, recv_obj, i)
|
||||
or first_output_time
|
||||
)
|
||||
state.time_stats.set_last_time()
|
||||
state.time_stats.set_last_time(ts=first_output_time)
|
||||
|
||||
if state.finished:
|
||||
span_attrs = (
|
||||
@@ -2606,7 +2609,11 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
if state.time_stats.trace_ctx.tracing_enable
|
||||
else None
|
||||
)
|
||||
state.time_stats.set_finished_time(span_attrs=span_attrs)
|
||||
# Reuse the first batch's arrival time: a streaming request that
|
||||
# finishes in its first output has no decode interval.
|
||||
state.time_stats.set_finished_time(
|
||||
ts=first_output_time, span_attrs=span_attrs
|
||||
)
|
||||
meta_info["e2e_latency"] = state.time_stats.get_e2e_latency()
|
||||
|
||||
if get_spec().speculative_algorithm:
|
||||
@@ -3097,6 +3104,15 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
else 0
|
||||
)
|
||||
|
||||
# Aborts have a truncated decode period; PD prefill workers have none.
|
||||
finish_reason = recv_obj.finished_reasons[i] or {}
|
||||
time_per_output_token = (
|
||||
state.time_stats.get_time_per_output_token(completion_tokens)
|
||||
if self.disaggregation_mode != DisaggregationMode.PREFILL
|
||||
and finish_reason.get("type") in ("stop", "length")
|
||||
else None
|
||||
)
|
||||
|
||||
self.metrics_collector.observe_one_finished_request(
|
||||
labels,
|
||||
recv_obj.prompt_tokens[i],
|
||||
@@ -3107,6 +3123,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
cached_tokens_details,
|
||||
spec_verify_ct=spec_verify_ct,
|
||||
is_streaming=getattr(state.obj, "stream", False),
|
||||
time_per_output_token=time_per_output_token,
|
||||
)
|
||||
|
||||
def dump_requests(self, state: ReqState, out_dict: dict):
|
||||
|
||||
@@ -1752,6 +1752,16 @@ class TokenizerMetricsCollector(_StatLoggerDIMixin):
|
||||
buckets=bucket_inter_token_latency,
|
||||
)
|
||||
|
||||
self.histogram_request_time_per_output_token = Histogram(
|
||||
name="sglang:request_time_per_output_token_seconds",
|
||||
documentation=(
|
||||
"Per-request TPOT in seconds: (finished_time - "
|
||||
"first_token_time) / (completion_tokens - 1)."
|
||||
),
|
||||
labelnames=[*labels.keys(), "is_streaming"],
|
||||
buckets=bucket_inter_token_latency,
|
||||
)
|
||||
|
||||
self.histogram_e2e_request_latency = Histogram(
|
||||
name="sglang:e2e_request_latency_seconds",
|
||||
documentation="Histogram of End-to-end request latency in seconds",
|
||||
@@ -1788,6 +1798,7 @@ class TokenizerMetricsCollector(_StatLoggerDIMixin):
|
||||
cached_tokens_details: Optional[Dict[str, Any]] = None,
|
||||
spec_verify_ct: int = 0,
|
||||
is_streaming: bool = False,
|
||||
time_per_output_token: Optional[float] = None,
|
||||
):
|
||||
stream_labels = {
|
||||
**labels,
|
||||
@@ -1826,6 +1837,10 @@ class TokenizerMetricsCollector(_StatLoggerDIMixin):
|
||||
self.histogram_e2e_request_latency.labels(**stream_labels).observe(
|
||||
float(e2e_latency)
|
||||
)
|
||||
if time_per_output_token is not None:
|
||||
self.histogram_request_time_per_output_token.labels(
|
||||
**stream_labels
|
||||
).observe(float(time_per_output_token))
|
||||
self.prompt_tokens_histogram.labels(**labels).observe(float(prompt_tokens))
|
||||
self.uncached_prompt_tokens_histogram.labels(**labels).observe(
|
||||
float(prompt_tokens - cached_tokens)
|
||||
|
||||
@@ -488,6 +488,18 @@ class APIServerReqTimeStats(ReqTimeStatsBase):
|
||||
def get_decode_latency(self):
|
||||
return self.finished_time - self.first_token_time
|
||||
|
||||
def get_time_per_output_token(self, completion_tokens: int) -> Optional[float]:
|
||||
if self.first_token_time <= 0.0 or completion_tokens <= 1:
|
||||
return None
|
||||
decode_latency = self.get_decode_latency()
|
||||
if decode_latency <= 0.0:
|
||||
return None
|
||||
return decode_latency / (completion_tokens - 1)
|
||||
|
||||
def get_decode_throughput(self, completion_tokens: int) -> Optional[float]:
|
||||
tpot = self.get_time_per_output_token(completion_tokens)
|
||||
return None if tpot is None else 1.0 / tpot
|
||||
|
||||
def get_response_sent_to_client_realtime(self):
|
||||
return convert_time_to_realtime(self.response_sent_to_client_time)
|
||||
|
||||
@@ -514,9 +526,9 @@ class APIServerReqTimeStats(ReqTimeStatsBase):
|
||||
self.finished_time
|
||||
)
|
||||
|
||||
decode_latency = self.get_decode_latency()
|
||||
if decode_latency > 0.0 and completion_tokens > 1:
|
||||
meta_info["decode_throughput"] = (completion_tokens - 1) / decode_latency
|
||||
decode_throughput = self.get_decode_throughput(completion_tokens)
|
||||
if decode_throughput is not None:
|
||||
meta_info["decode_throughput"] = decode_throughput
|
||||
return meta_info
|
||||
|
||||
def convert_to_gen_ai_span_attrs(self):
|
||||
|
||||
Reference in New Issue
Block a user