From a690e5e0b383ebaf827f399366cb8402d88b4926 Mon Sep 17 00:00:00 2001 From: cctry Date: Fri, 24 Jul 2026 17:44:04 -0700 Subject: [PATCH] Add stream label to TTFT metrics (#32363) Co-authored-by: Jialin Ouyang --- python/sglang/srt/managers/tokenizer_manager.py | 4 +++- .../sglang/srt/observability/metrics_collector.py | 13 +++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index 7a1eb61c6..7fe6e9491 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -2501,7 +2501,9 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): state.ttft_observed = True state.last_completion_tokens = completion_tokens self.metrics_collector.observe_time_to_first_token( - labels, state.time_stats.get_first_token_latency() + labels, + state.time_stats.get_first_token_latency(), + stream=getattr(state.obj, "stream", False), ) else: num_new_tokens = completion_tokens - state.last_completion_tokens diff --git a/python/sglang/srt/observability/metrics_collector.py b/python/sglang/srt/observability/metrics_collector.py index 2e6759bb9..0552c46a5 100644 --- a/python/sglang/srt/observability/metrics_collector.py +++ b/python/sglang/srt/observability/metrics_collector.py @@ -1630,7 +1630,8 @@ class TokenizerMetricsCollector(_StatLoggerDIMixin): self.histogram_time_to_first_token = Histogram( name="sglang:time_to_first_token_seconds", documentation="Histogram of time to first token in seconds.", - labelnames=labels.keys(), + # "stream" splits streaming vs non-streaming requests. + labelnames=[*labels.keys(), "stream"], buckets=bucket_time_to_first_token, ) @@ -1701,11 +1702,15 @@ class TokenizerMetricsCollector(_StatLoggerDIMixin): float(generation_tokens) ) - def observe_time_to_first_token(self, labels: Dict[str, str], value: float): - self.histogram_time_to_first_token.labels(**labels).observe(value) + def observe_time_to_first_token( + self, labels: Dict[str, str], value: float, *, stream: bool + ): + self.histogram_time_to_first_token.labels( + **labels, stream="true" if stream else "false" + ).observe(value) def check_time_to_first_token_straggler(self, value: float) -> bool: - his = self.histogram_time_to_first_token.labels(**self.labels) + his = self.histogram_time_to_first_token.labels(**self.labels, stream="true") total_observations = sum(bucket._value for bucket in his._buckets) if total_observations < 100: return False