diff --git a/python/sglang/srt/entrypoints/v1_loads.py b/python/sglang/srt/entrypoints/v1_loads.py index 0b5bb6f5c..086ba559e 100644 --- a/python/sglang/srt/entrypoints/v1_loads.py +++ b/python/sglang/srt/entrypoints/v1_loads.py @@ -19,6 +19,7 @@ metrics for load balancing, monitoring, and capacity planning. """ import dataclasses +import time from datetime import datetime, timezone from typing import Optional @@ -153,6 +154,7 @@ async def get_loads( """ include_list = [s.strip() for s in include.split(",")] if include else None + start = time.perf_counter() try: load_results = await tokenizer_manager.get_loads( include=include_list, @@ -160,6 +162,12 @@ async def get_loads( ) except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) + finally: + mc = getattr(tokenizer_manager, "metrics_collector", None) + if mc is not None: + mc.get_loads_duration_seconds.labels(**mc.labels).observe( + time.perf_counter() - start + ) if format == "prometheus": return _format_loads_prometheus(load_results) diff --git a/python/sglang/srt/observability/metrics_collector.py b/python/sglang/srt/observability/metrics_collector.py index 4b0edd32e..4fe3b6f37 100644 --- a/python/sglang/srt/observability/metrics_collector.py +++ b/python/sglang/srt/observability/metrics_collector.py @@ -1352,6 +1352,13 @@ class TokenizerMetricsCollector: labelnames=labels.keys(), ) + self.get_loads_duration_seconds = Histogram( + name="sglang:get_loads_duration_seconds", + documentation="Time spent serving /v1/loads requests (seconds).", + labelnames=labels.keys(), + buckets=(0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0), + ) + self.num_so_requests_total = Counter( name="sglang:num_so_requests_total", documentation="Number of structured output requests processed.", diff --git a/test/registered/language/test_srt_backend.py b/test/registered/language/test_srt_backend.py index 849331ae8..8b775bea9 100644 --- a/test/registered/language/test_srt_backend.py +++ b/test/registered/language/test_srt_backend.py @@ -35,6 +35,7 @@ class TestSRTBackend(CustomTestCase): mem_fraction_static=0.7, incremental_streaming_output=True, log_level="info", + enable_metrics=True, ) sgl.set_default_backend(cls.backend)