From 89553ff82bfc1b4c6bb960ef9151ebf2bc398368 Mon Sep 17 00:00:00 2001 From: Kangyan-Zhou Date: Thu, 9 Apr 2026 20:04:54 -0700 Subject: [PATCH] [Observability] Add Prometheus metrics endpoint for gRPC mode (#20801) Co-authored-by: Claude Opus 4.6 (1M context) --- python/sglang/srt/entrypoints/grpc_server.py | 105 ++++++++++++++++++- python/sglang/srt/server_args.py | 9 ++ 2 files changed, 113 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/entrypoints/grpc_server.py b/python/sglang/srt/entrypoints/grpc_server.py index b0188a137..848fc520b 100644 --- a/python/sglang/srt/entrypoints/grpc_server.py +++ b/python/sglang/srt/entrypoints/grpc_server.py @@ -1,7 +1,67 @@ """ Thin gRPC server wrapper — delegates to smg-grpc-servicer package. + +When --enable-metrics is set, a lightweight HTTP server is started on +--metrics-http-port (default: --port + 1) to expose Prometheus /metrics. """ +import logging + +logger = logging.getLogger(__name__) + + +async def _start_metrics_server(host: str, port: int): + """Start an HTTP server exposing Prometheus /metrics. + + The caller is responsible for calling ``runner.cleanup()`` on the returned + AppRunner when shutting down. The server begins accepting requests before + this function returns. + """ + from aiohttp import web + from prometheus_client import ( + CollectorRegistry, + multiprocess, + ) + from prometheus_client.openmetrics.exposition import ( + CONTENT_TYPE_LATEST, + generate_latest, + ) + + async def metrics_handler(request): + try: + # Create a fresh registry and attach a MultiProcessCollector + # on each request. This is the recommended pattern from the + # prometheus_client multiprocess docs to ensure up-to-date + # data from PROMETHEUS_MULTIPROC_DIR. + # + # Use OpenMetrics format to match what the HTTP-mode endpoint + # returns when Prometheus scrapes it with an OpenMetrics Accept + # header (make_asgi_app performs content negotiation). + registry = CollectorRegistry() + multiprocess.MultiProcessCollector(registry) + data = generate_latest(registry) + return web.Response( + body=data, + headers={"Content-Type": CONTENT_TYPE_LATEST}, + ) + except Exception: + logger.exception("Failed to generate Prometheus metrics") + return web.Response(status=500, text="Failed to generate metrics") + + app = web.Application() + app.router.add_get("/metrics", metrics_handler) + + runner = web.AppRunner(app) + await runner.setup() + try: + site = web.TCPSite(runner, host, port) + await site.start() + except BaseException: + await runner.cleanup() + raise + logger.info("Prometheus metrics server started on http://%s:%d/metrics", host, port) + return runner + async def serve_grpc(server_args, model_info=None): """Start the standalone gRPC server with integrated scheduler.""" @@ -14,4 +74,47 @@ async def serve_grpc(server_args, model_info=None): "If already installed, there may be a broken import due to a " "version mismatch — see the chained exception above for details." ) from e - await _serve_grpc(server_args, model_info) + + metrics_runner = None + if server_args.enable_metrics: + try: + from sglang.srt.observability.func_timer import enable_func_timer + from sglang.srt.utils import set_prometheus_multiproc_dir + + # Must set PROMETHEUS_MULTIPROC_DIR env var before any + # prometheus_client import. The env var is inherited by child + # processes (schedulers) that import prometheus_client later. + set_prometheus_multiproc_dir() + enable_func_timer() + + metrics_port = ( + server_args.metrics_http_port + if server_args.metrics_http_port is not None + else server_args.port + 1 + ) + metrics_runner = await _start_metrics_server(server_args.host, metrics_port) + except OSError as e: + logger.error( + "Failed to start metrics server: %s. " "Continuing without metrics.", + e, + exc_info=True, + ) + except Exception as e: + logger.error( + "Unexpected error starting metrics server: %s. " + "Continuing without metrics.", + e, + exc_info=True, + ) + + try: + await _serve_grpc(server_args, model_info) + finally: + if metrics_runner is not None: + try: + await metrics_runner.cleanup() + except Exception as e: + logger.exception( + "Failed to cleanly shut down Prometheus metrics server: %s", + e, + ) diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index fe00f97df..753d18c21 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -405,6 +405,7 @@ class ServerArgs: crash_dump_folder: Optional[str] = None show_time_cost: bool = False enable_metrics: bool = False + metrics_http_port: Optional[int] = None enable_mfu_metrics: bool = False enable_metrics_for_all_schedulers: bool = False tokenizer_metrics_custom_labels_header: str = "x-custom-labels" @@ -4515,6 +4516,14 @@ class ServerArgs: action="store_true", help="Enable log prometheus metrics.", ) + parser.add_argument( + "--metrics-http-port", + type=int, + default=ServerArgs.metrics_http_port, + help="Port for the Prometheus metrics HTTP server. " + "Only used in gRPC mode (--grpc-mode); in HTTP mode, metrics are served on the main --port. " + "Defaults to --port + 1 when --enable-metrics is set.", + ) parser.add_argument( "--enable-mfu-metrics", action="store_true",