From 7e5256d1ba3aaa628bb8b756fafb904a76a1123a Mon Sep 17 00:00:00 2001 From: metamergebot Date: Wed, 21 Oct 2026 11:57:26 -0700 Subject: [PATCH] [Feat] srt: Balance uvicorn tokenizer workers with per-worker SO_REUSEPORT listeners (#45482) Co-authored-by: luccafong <116399278+luccafong@users.noreply.github.com> Co-authored-by: joshuadeng --- .../advanced_features/server_arguments.mdx | 6 + python/pyproject.toml | 2 +- python/sglang/srt/arg_groups/field_order.py | 1 + .../sglang/srt/arg_groups/fields/serving.py | 4 + python/sglang/srt/arg_groups/serving_hook.py | 7 + python/sglang/srt/entrypoints/http_server.py | 152 ++++++++++++-- .../entrypoints/test_http_server_reuseport.py | 198 ++++++++++++++++++ .../unit/server_args/test_server_args.py | 7 + .../unit/test_server_args_cli_metadata.py | 12 ++ 9 files changed, 373 insertions(+), 16 deletions(-) create mode 100644 test/registered/unit/entrypoints/test_http_server_reuseport.py diff --git a/docs/docs/advanced_features/server_arguments.mdx b/docs/docs/advanced_features/server_arguments.mdx index ed260f7751..e202df203e 100644 --- a/docs/docs/advanced_features/server_arguments.mdx +++ b/docs/docs/advanced_features/server_arguments.mdx @@ -306,6 +306,12 @@ Please consult the documentation below and [server_args.py](https://github.com/s `False` bool flag (set to enable) + + `--enable-http-worker-reuseport` + Give each Uvicorn HTTP worker its own SO_REUSEPORT listener. + `False` + bool flag (set to enable) + `--enable-http2` Use Granian instead of Uvicorn as the ASGI server, enabling HTTP/1.1 and HTTP/2 auto-negotiation. Clients may use h2c (cleartext HTTP/2) or plain HTTP/1.1. Requires 'pip install sglang[http2]'. diff --git a/python/pyproject.toml b/python/pyproject.toml index 618ca6a809..51f18793f3 100755 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -90,7 +90,7 @@ dependencies = [ "torchvision", "tqdm", "transformers==5.19.0", - "uvicorn", + "uvicorn>=0.51", "uvloop", "watchfiles", "xgrammar==0.2.7", diff --git a/python/sglang/srt/arg_groups/field_order.py b/python/sglang/srt/arg_groups/field_order.py index 03bb58d317..9df5aa674f 100644 --- a/python/sglang/srt/arg_groups/field_order.py +++ b/python/sglang/srt/arg_groups/field_order.py @@ -546,5 +546,6 @@ POSITIONAL_FIELD_ORDER = ( "paged_experts_num_resident", "enable_nvidia_megakernel", "enable_megamoe_for_layernorm_sp", + "enable_http_worker_reuseport", ) # fmt: on diff --git a/python/sglang/srt/arg_groups/fields/serving.py b/python/sglang/srt/arg_groups/fields/serving.py index f1b828ee05..a3580a1f68 100644 --- a/python/sglang/srt/arg_groups/fields/serving.py +++ b/python/sglang/srt/arg_groups/fields/serving.py @@ -68,6 +68,10 @@ class Serving(msgspec.Struct): str, "App is behind a path based routing proxy.", ] = "" + enable_http_worker_reuseport: A[ + bool, + "Give each Uvicorn HTTP worker its own SO_REUSEPORT listener.", + ] = False smg_grpc_mode: A[ bool, "Use the legacy SMG gRPC server (smg-grpc-servicer) instead of the HTTP " diff --git a/python/sglang/srt/arg_groups/serving_hook.py b/python/sglang/srt/arg_groups/serving_hook.py index f7583dd010..12ae08985b 100644 --- a/python/sglang/srt/arg_groups/serving_hook.py +++ b/python/sglang/srt/arg_groups/serving_hook.py @@ -85,6 +85,13 @@ def handle_ssl_validation(server_args: Any): "--enable-ssl-refresh requires --ssl-certfile and --ssl-keyfile " "to be specified." ) + if cfg.enable_http_worker_reuseport and ( + cfg.tokenizer_worker_num == 1 or cfg.enable_http2 + ): + raise ValueError( + "--enable-http-worker-reuseport requires --tokenizer-worker-num > 1 " + "and has no effect with --enable-http2." + ) if cfg.enable_http2: if not 0 < cfg.http2_max_concurrent_streams < 2**32: diff --git a/python/sglang/srt/entrypoints/http_server.py b/python/sglang/srt/entrypoints/http_server.py index 430d86b9b6..0abcc2924d 100644 --- a/python/sglang/srt/entrypoints/http_server.py +++ b/python/sglang/srt/entrypoints/http_server.py @@ -19,8 +19,10 @@ This file implements HTTP APIs for the inference engine via fastapi. import asyncio import dataclasses +import errno import logging import os +import socket import ssl import tempfile import threading @@ -64,6 +66,7 @@ from fastapi.exceptions import RequestValidationError from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import ORJSONResponse, Response, StreamingResponse from fastapi.routing import APIRoute +from uvicorn.supervisors.multiprocess import Multiprocess from sglang.srt.arg_groups.overrides import resolving_view from sglang.srt.configs.embedding_model_spec import resolved_embedding_plan @@ -2562,6 +2565,115 @@ def _wait_weights_ready(): ) +class _ReusePortMultiprocess(Multiprocess): + """Give each uvicorn worker a private SO_REUSEPORT listener. + + uvicorn's supervisor reads ``self.sockets`` once per worker it spawns, so a + fresh listener is handed out on every read and the parent's copy is closed + once the child holds it; the lifecycle (health checks, restarts, SIGTTIN) + stays uvicorn's own. Requires uvicorn >= 0.51, whose Process builds its + own Server. + """ + + def __init__(self, config: uvicorn.Config): + self._handed_out: List[socket.socket] = [] + self._reserved_socket: Optional[socket.socket] = _reserve_reuseport_socket( + config + ) + self._socket_family = self._reserved_socket.family + self._socket_address = self._reserved_socket.getsockname() + super().__init__(config, sockets=[]) + + @property + def sockets(self) -> List[socket.socket]: + self._close_handed_out() + sock = self._reserved_socket + self._reserved_socket = None + if sock is None: + sock = _bind_reuseport_socket( + self.config, self._socket_family, self._socket_address + ) + self._handed_out.append(sock) + return [sock] + + @sockets.setter + def sockets(self, value: List[socket.socket]) -> None: + if value: + raise ValueError("_ReusePortMultiprocess binds its own listeners") + + def _close_handed_out(self) -> None: + # Each spawn site starts its Process before the next read, so every + # listener handed out so far is already held by its child. + for sock in self._handed_out: + sock.close() + self._handed_out.clear() + + def init_processes(self) -> None: + try: + super().init_processes() + except BaseException: + self.terminate_all() + self.join_all() + raise + finally: + self._close_handed_out() + + def restart_all(self) -> None: + try: + super().restart_all() + finally: + self._close_handed_out() + + def keep_subprocess_alive(self) -> None: + try: + super().keep_subprocess_alive() + finally: + self._close_handed_out() + + def handle_ttin(self) -> None: + try: + super().handle_ttin() + finally: + self._close_handed_out() + + +def _bind_reuseport_socket(config: uvicorn.Config, family, address) -> socket.socket: + sock = socket.socket(family=family) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + sock.bind(address) + sock.listen(config.backlog) + sock.set_inheritable(True) + return sock + + +def _reserve_reuseport_socket(config: uvicorn.Config) -> socket.socket: + try: + family = socket.AF_INET6 if ":" in config.host else socket.AF_INET + address = socket.getaddrinfo(config.host, config.port, family)[0][4] + if config.port == 0: + sock = _bind_reuseport_socket(config, family, address) + config.port = sock.getsockname()[1] + return sock + name = f"\0sglang-reuseport-{os.geteuid()}-{config.port}" + with socket.socket(socket.AF_UNIX) as reservation: + while True: + try: + reservation.bind(name) + break + except OSError as error: + if error.errno != errno.EADDRINUSE: + raise + time.sleep(0.01) + with socket.socket(family=family) as probe: + probe.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + probe.bind(address) + return _bind_reuseport_socket(config, family, address) + except OSError as error: + logger.error(error) + raise SystemExit(1) from error + + def _run_granian_server( host, port, @@ -2858,22 +2970,32 @@ def _setup_and_run_http_server( ssl_keyfile_password=get_serving().ssl_keyfile_password, ) else: - uvicorn.run( - envs.SGLANG_HTTP_SERVER_APP_PATH.get(), - host=get_serving().host, - port=get_serving().port, - root_path=get_serving().fastapi_root_path, - log_level=get_observability().log_level_http + app_path = envs.SGLANG_HTTP_SERVER_APP_PATH.get() + uvicorn_options = { + "host": get_serving().host, + "port": get_serving().port, + "root_path": get_serving().fastapi_root_path, + "log_level": get_observability().log_level_http or get_observability().log_level, - timeout_keep_alive=envs.SGLANG_TIMEOUT_KEEP_ALIVE.get(), - timeout_worker_healthcheck=envs.SGLANG_UVICORN_WORKER_HEALTHCHECK_TIMEOUT.get(), - loop="uvloop", - workers=get_serving().tokenizer_worker_num, - ssl_keyfile=get_serving().ssl_keyfile, - ssl_certfile=get_serving().ssl_certfile, - ssl_ca_certs=get_serving().ssl_ca_certs, - ssl_keyfile_password=get_serving().ssl_keyfile_password, - ) + "timeout_keep_alive": envs.SGLANG_TIMEOUT_KEEP_ALIVE.get(), + "timeout_worker_healthcheck": envs.SGLANG_UVICORN_WORKER_HEALTHCHECK_TIMEOUT.get(), + "loop": "uvloop", + "workers": get_serving().tokenizer_worker_num, + "ssl_keyfile": get_serving().ssl_keyfile, + "ssl_certfile": get_serving().ssl_certfile, + "ssl_ca_certs": get_serving().ssl_ca_certs, + "ssl_keyfile_password": get_serving().ssl_keyfile_password, + } + if get_serving().enable_http_worker_reuseport: + config = uvicorn.Config(app_path, **uvicorn_options) + supervisor = _ReusePortMultiprocess(config) + logger.info( + f"Uvicorn running on {config.host}:{config.port} " + f"with {config.workers} SO_REUSEPORT workers" + ) + supervisor.run() + else: + uvicorn.run(app_path, **uvicorn_options) finally: if get_serving().tokenizer_worker_num > 1: if multi_tokenizer_args_shm is not None: diff --git a/test/registered/unit/entrypoints/test_http_server_reuseport.py b/test/registered/unit/entrypoints/test_http_server_reuseport.py new file mode 100644 index 0000000000..8b231eb44e --- /dev/null +++ b/test/registered/unit/entrypoints/test_http_server_reuseport.py @@ -0,0 +1,198 @@ +"""With --enable-http-worker-reuseport every uvicorn worker must own a private +SO_REUSEPORT listener on the configured address, a replaced worker must get a +fresh one, and a second server on the same fixed host:port must still fail to +start instead of silently joining the first server's reuseport group. +""" + +import copy +import signal +import socket +import sys +import threading +from concurrent.futures import ThreadPoolExecutor +from contextlib import suppress +from unittest.mock import Mock + +import pytest +import uvicorn +from uvicorn.supervisors import multiprocess +from uvicorn.supervisors.multiprocess import SIGNALS + +from sglang.srt.entrypoints import http_server +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=2, suite="base-a-test-cpu") + + +@pytest.fixture(autouse=True) +def restore_signal_handlers(): + # Multiprocess.__init__ replaces process-wide signal handlers. + handlers = {sig: signal.getsignal(sig) for sig in SIGNALS} + yield + for sig, handler in handlers.items(): + signal.signal(sig, handler) + + +@pytest.fixture +def config(free_tcp_port): + return uvicorn.Config( + "starlette.applications:Starlette", + host="127.0.0.1", + port=free_tcp_port, + workers=4, + factory=True, + lifespan="off", + log_level="error", + ) + + +def _start(config, monkeypatch): + children = [] + + class FakeProcess: + # The uvicorn supervisor spawns through the module-level Process name. + def __init__(self, _config, sockets): + self.sockets = sockets + self.alive = True + self.exitcode = None + self.pid = len(children) + 1 + + def start(self): + self.listener = self.sockets[0].dup() + children.append(self.listener) + + def is_alive(self, timeout): + return self.alive + + def wait_until_ready(self, timeout, should_exit=None): + return True + + def kill(self): + self.alive = False + self.listener.close() + + terminate = kill + + def join(self): + return None + + monkeypatch.setattr(multiprocess, "Process", FakeProcess) + supervisor = http_server._ReusePortMultiprocess(config) + supervisor.init_processes() + return supervisor, children + + +def test_worker_listener_lifecycle(monkeypatch, config): + resolve = Mock(wraps=socket.getaddrinfo) + monkeypatch.setattr(socket, "getaddrinfo", resolve) + supervisor, children = _start(config, monkeypatch) + assert resolve.call_count == 1 + try: + assert all(p.sockets[0].fileno() == -1 for p in supervisor.processes) + assert len({sock.fileno() for sock in children}) == config.workers + assert all( + sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT) for sock in children + ) + + old = supervisor.processes[0] + old.alive = False + supervisor.keep_subprocess_alive() + new = supervisor.processes[0] + assert new is not old and old.listener.fileno() == -1 + assert new.sockets[0].fileno() == -1 and new.listener.fileno() != -1 + + before_restart = list(supervisor.processes) + supervisor.restart_all() + assert all(process.listener.fileno() == -1 for process in before_restart) + assert all(p.sockets[0].fileno() == -1 for p in supervisor.processes) + + supervisor.handle_ttin() + assert len(supervisor.processes) == config.workers + 1 + assert supervisor.processes[-1].sockets[0].fileno() == -1 + finally: + for sock in children: + sock.close() + + +def test_partial_spawn_failure_cleans_up_workers(monkeypatch, config): + config.workers = 3 + workers = [Mock(), Mock()] + monkeypatch.setattr( + multiprocess, "Process", Mock(side_effect=[*workers, RuntimeError("failed")]) + ) + supervisor = http_server._ReusePortMultiprocess(config) + with pytest.raises(RuntimeError): + supervisor.init_processes() + for worker in workers: + worker.terminate.assert_called_once() + worker.join.assert_called_once() + assert supervisor._reserved_socket is None and not supervisor._handed_out + + +def test_unresolvable_host_exits_cleanly(monkeypatch, caplog, config): + resolve = Mock(side_effect=socket.gaierror("bad host")) + monkeypatch.setattr(socket, "getaddrinfo", resolve) + with pytest.raises(SystemExit): + http_server._ReusePortMultiprocess(config) + assert "bad host" in caplog.text + + +@pytest.mark.parametrize( + ("hosts", "ephemeral", "expected_winners"), + [ + (("127.0.0.1",) * 2, False, 1), + (("127.0.0.1",) * 2, True, 2), + (("127.0.0.1", "127.0.0.2"), False, 2), + ], +) +def test_concurrent_supervisor_reservation( + monkeypatch, config, hosts, ephemeral, expected_winners +): + barrier = threading.Barrier(2) + bind = http_server._bind_reuseport_socket + + def synchronized_bind(*args): + with suppress(threading.BrokenBarrierError): + barrier.wait(timeout=1) + return bind(*args) + + def create_supervisor(index): + worker_config = copy.copy(config) + worker_config.host = hosts[index] + worker_config.port = 0 if ephemeral else config.port + try: + return http_server._ReusePortMultiprocess(worker_config) + except SystemExit: + return None + + monkeypatch.setattr(http_server.Multiprocess, "__init__", Mock(return_value=None)) + monkeypatch.setattr(http_server, "_bind_reuseport_socket", synchronized_bind) + with ThreadPoolExecutor(max_workers=2) as pool: + supervisors = list(pool.map(create_supervisor, range(2))) + + winners = [supervisor for supervisor in supervisors if supervisor is not None] + for supervisor in winners: + supervisor._reserved_socket.close() + assert len(winners) == expected_winners + + +def test_real_uvicorn_worker_accepts_on_private_listener(config): + config.workers = 1 + supervisor = http_server._ReusePortMultiprocess(config) + supervisor.init_processes() + try: + assert supervisor._reserved_socket is None and not supervisor._handed_out + # The listener accepts at once; the reply waits for the spawned worker, + # which re-imports this module and its server imports before serving. + with socket.create_connection( + (config.host, config.port), timeout=120 + ) as connection: + connection.sendall(b"GET /openapi.json HTTP/1.0\r\n\r\n") + assert b"404 Not Found" in connection.recv(1024) + finally: + supervisor.terminate_all() + supervisor.join_all() + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py index 023fb5b8b5..6ecb9ebd04 100644 --- a/test/registered/unit/server_args/test_server_args.py +++ b/test/registered/unit/server_args/test_server_args.py @@ -2229,6 +2229,13 @@ class TestSSLArgs(unittest.TestCase): self.assertIn("--enable-ssl-refresh", str(context.exception)) self.assertIn("--ssl-certfile", str(context.exception)) + def test_http_worker_reuseport_requires_multiple_uvicorn_workers(self): + for kwargs in ({}, {"tokenizer_worker_num": 2, "enable_http2": True}): + with self.subTest(**kwargs), self.assertRaises(ValueError) as context: + self._validate_ssl(enable_http_worker_reuseport=True, **kwargs) + self.assertIn("--enable-http-worker-reuseport", str(context.exception)) + self._validate_ssl(enable_http_worker_reuseport=True, tokenizer_worker_num=2) + @patch("os.path.isfile", return_value=True) def test_enable_ssl_refresh_with_ssl_accepted(self, _mock_isfile): server_args = self._validate_ssl( diff --git a/test/registered/unit/test_server_args_cli_metadata.py b/test/registered/unit/test_server_args_cli_metadata.py index b7a03063f8..46c09bda6e 100644 --- a/test/registered/unit/test_server_args_cli_metadata.py +++ b/test/registered/unit/test_server_args_cli_metadata.py @@ -31,6 +31,18 @@ class TestServerArgsMigratedCliMetadata(CustomTestCase): ServerArgs.from_cli_args(args).trust_request_chat_template, expected ) + def test_http_worker_reuseport_requires_explicit_opt_in(self): + for flags, expected in ( + ([], False), + (["--enable-http-worker-reuseport"], True), + ): + with self.subTest(flags=flags): + args = self.parser.parse_args(["--model", "dummy", *flags]) + self.assertIs( + ServerArgs.from_cli_args(args).enable_http_worker_reuseport, + expected, + ) + def test_prefill_max_context_accepts_human_readable_values(self): for option in ( "--cuda-graph-prefill-max-context",