[Observability] Add Prometheus metrics endpoint for gRPC mode (#20801)

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

python/sglang/srt/entrypoints/grpc_server.py

@@ -1,7 +1,67 @@
 """
 Thin gRPC server wrapper — delegates to smg-grpc-servicer package.
+
+When --enable-metrics is set, a lightweight HTTP server is started on
+--metrics-http-port (default: --port + 1) to expose Prometheus /metrics.
 """
 
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+async def _start_metrics_server(host: str, port: int):
+    """Start an HTTP server exposing Prometheus /metrics.
+
+    The caller is responsible for calling ``runner.cleanup()`` on the returned
+    AppRunner when shutting down.  The server begins accepting requests before
+    this function returns.
+    """
+    from aiohttp import web
+    from prometheus_client import (
+        CollectorRegistry,
+        multiprocess,
+    )
+    from prometheus_client.openmetrics.exposition import (
+        CONTENT_TYPE_LATEST,
+        generate_latest,
+    )
+
+    async def metrics_handler(request):
+        try:
+            # Create a fresh registry and attach a MultiProcessCollector
+            # on each request.  This is the recommended pattern from the
+            # prometheus_client multiprocess docs to ensure up-to-date
+            # data from PROMETHEUS_MULTIPROC_DIR.
+            #
+            # Use OpenMetrics format to match what the HTTP-mode endpoint
+            # returns when Prometheus scrapes it with an OpenMetrics Accept
+            # header (make_asgi_app performs content negotiation).
+            registry = CollectorRegistry()
+            multiprocess.MultiProcessCollector(registry)
+            data = generate_latest(registry)
+            return web.Response(
+                body=data,
+                headers={"Content-Type": CONTENT_TYPE_LATEST},
+            )
+        except Exception:
+            logger.exception("Failed to generate Prometheus metrics")
+            return web.Response(status=500, text="Failed to generate metrics")
+
+    app = web.Application()
+    app.router.add_get("/metrics", metrics_handler)
+
+    runner = web.AppRunner(app)
+    await runner.setup()
+    try:
+        site = web.TCPSite(runner, host, port)
+        await site.start()
+    except BaseException:
+        await runner.cleanup()
+        raise
+    logger.info("Prometheus metrics server started on http://%s:%d/metrics", host, port)
+    return runner
+
 
 async def serve_grpc(server_args, model_info=None):
     """Start the standalone gRPC server with integrated scheduler."""
@@ -14,4 +74,47 @@ async def serve_grpc(server_args, model_info=None):
             "If already installed, there may be a broken import due to a "
             "version mismatch — see the chained exception above for details."
         ) from e
-    await _serve_grpc(server_args, model_info)
+
+    metrics_runner = None
+    if server_args.enable_metrics:
+        try:
+            from sglang.srt.observability.func_timer import enable_func_timer
+            from sglang.srt.utils import set_prometheus_multiproc_dir
+
+            # Must set PROMETHEUS_MULTIPROC_DIR env var before any
+            # prometheus_client import.  The env var is inherited by child
+            # processes (schedulers) that import prometheus_client later.
+            set_prometheus_multiproc_dir()
+            enable_func_timer()
+
+            metrics_port = (
+                server_args.metrics_http_port
+                if server_args.metrics_http_port is not None
+                else server_args.port + 1
+            )
+            metrics_runner = await _start_metrics_server(server_args.host, metrics_port)
+        except OSError as e:
+            logger.error(
+                "Failed to start metrics server: %s. " "Continuing without metrics.",
+                e,
+                exc_info=True,
+            )
+        except Exception as e:
+            logger.error(
+                "Unexpected error starting metrics server: %s. "
+                "Continuing without metrics.",
+                e,
+                exc_info=True,
+            )
+
+    try:
+        await _serve_grpc(server_args, model_info)
+    finally:
+        if metrics_runner is not None:
+            try:
+                await metrics_runner.cleanup()
+            except Exception as e:
+                logger.exception(
+                    "Failed to cleanly shut down Prometheus metrics server: %s",
+                    e,
+                )

python/sglang/srt/server_args.py

@@ -405,6 +405,7 @@ class ServerArgs:
     crash_dump_folder: Optional[str] = None
     show_time_cost: bool = False
     enable_metrics: bool = False
+    metrics_http_port: Optional[int] = None
     enable_mfu_metrics: bool = False
     enable_metrics_for_all_schedulers: bool = False
     tokenizer_metrics_custom_labels_header: str = "x-custom-labels"
@@ -4515,6 +4516,14 @@ class ServerArgs:
             action="store_true",
             help="Enable log prometheus metrics.",
         )
+        parser.add_argument(
+            "--metrics-http-port",
+            type=int,
+            default=ServerArgs.metrics_http_port,
+            help="Port for the Prometheus metrics HTTP server. "
+            "Only used in gRPC mode (--grpc-mode); in HTTP mode, metrics are served on the main --port. "
+            "Defaults to --port + 1 when --enable-metrics is set.",
+        )
         parser.add_argument(
             "--enable-mfu-metrics",
             action="store_true",