diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index 7a1eb61c6..7fe6e9491 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -2501,7 +2501,9 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): state.ttft_observed = True state.last_completion_tokens = completion_tokens self.metrics_collector.observe_time_to_first_token( - labels, state.time_stats.get_first_token_latency() + labels, + state.time_stats.get_first_token_latency(), + stream=getattr(state.obj, "stream", False), ) else: num_new_tokens = completion_tokens - state.last_completion_tokens diff --git a/python/sglang/srt/observability/metrics_collector.py b/python/sglang/srt/observability/metrics_collector.py index 2e6759bb9..0552c46a5 100644 --- a/python/sglang/srt/observability/metrics_collector.py +++ b/python/sglang/srt/observability/metrics_collector.py @@ -1630,7 +1630,8 @@ class TokenizerMetricsCollector(_StatLoggerDIMixin): self.histogram_time_to_first_token = Histogram( name="sglang:time_to_first_token_seconds", documentation="Histogram of time to first token in seconds.", - labelnames=labels.keys(), + # "stream" splits streaming vs non-streaming requests. + labelnames=[*labels.keys(), "stream"], buckets=bucket_time_to_first_token, ) @@ -1701,11 +1702,15 @@ class TokenizerMetricsCollector(_StatLoggerDIMixin): float(generation_tokens) ) - def observe_time_to_first_token(self, labels: Dict[str, str], value: float): - self.histogram_time_to_first_token.labels(**labels).observe(value) + def observe_time_to_first_token( + self, labels: Dict[str, str], value: float, *, stream: bool + ): + self.histogram_time_to_first_token.labels( + **labels, stream="true" if stream else "false" + ).observe(value) def check_time_to_first_token_straggler(self, value: float) -> bool: - his = self.histogram_time_to_first_token.labels(**self.labels) + his = self.histogram_time_to_first_token.labels(**self.labels, stream="true") total_observations = sum(bucket._value for bucket in his._buckets) if total_observations < 100: return False