diff --git a/docs/docs/advanced_features/server_arguments.mdx b/docs/docs/advanced_features/server_arguments.mdx
index ed260f7751..e202df203e 100644
--- a/docs/docs/advanced_features/server_arguments.mdx
+++ b/docs/docs/advanced_features/server_arguments.mdx
@@ -306,6 +306,12 @@ Please consult the documentation below and [server_args.py](https://github.com/s
`False` |
bool flag (set to enable) |
+
+ | `--enable-http-worker-reuseport` |
+ Give each Uvicorn HTTP worker its own SO_REUSEPORT listener. |
+ `False` |
+ bool flag (set to enable) |
+
| `--enable-http2` |
Use Granian instead of Uvicorn as the ASGI server, enabling HTTP/1.1 and HTTP/2 auto-negotiation. Clients may use h2c (cleartext HTTP/2) or plain HTTP/1.1. Requires 'pip install sglang[http2]'. |
diff --git a/python/pyproject.toml b/python/pyproject.toml
index 618ca6a809..51f18793f3 100755
--- a/python/pyproject.toml
+++ b/python/pyproject.toml
@@ -90,7 +90,7 @@ dependencies = [
"torchvision",
"tqdm",
"transformers==5.19.0",
- "uvicorn",
+ "uvicorn>=0.51",
"uvloop",
"watchfiles",
"xgrammar==0.2.7",
diff --git a/python/sglang/srt/arg_groups/field_order.py b/python/sglang/srt/arg_groups/field_order.py
index 03bb58d317..9df5aa674f 100644
--- a/python/sglang/srt/arg_groups/field_order.py
+++ b/python/sglang/srt/arg_groups/field_order.py
@@ -546,5 +546,6 @@ POSITIONAL_FIELD_ORDER = (
"paged_experts_num_resident",
"enable_nvidia_megakernel",
"enable_megamoe_for_layernorm_sp",
+ "enable_http_worker_reuseport",
)
# fmt: on
diff --git a/python/sglang/srt/arg_groups/fields/serving.py b/python/sglang/srt/arg_groups/fields/serving.py
index f1b828ee05..a3580a1f68 100644
--- a/python/sglang/srt/arg_groups/fields/serving.py
+++ b/python/sglang/srt/arg_groups/fields/serving.py
@@ -68,6 +68,10 @@ class Serving(msgspec.Struct):
str,
"App is behind a path based routing proxy.",
] = ""
+ enable_http_worker_reuseport: A[
+ bool,
+ "Give each Uvicorn HTTP worker its own SO_REUSEPORT listener.",
+ ] = False
smg_grpc_mode: A[
bool,
"Use the legacy SMG gRPC server (smg-grpc-servicer) instead of the HTTP "
diff --git a/python/sglang/srt/arg_groups/serving_hook.py b/python/sglang/srt/arg_groups/serving_hook.py
index f7583dd010..12ae08985b 100644
--- a/python/sglang/srt/arg_groups/serving_hook.py
+++ b/python/sglang/srt/arg_groups/serving_hook.py
@@ -85,6 +85,13 @@ def handle_ssl_validation(server_args: Any):
"--enable-ssl-refresh requires --ssl-certfile and --ssl-keyfile "
"to be specified."
)
+ if cfg.enable_http_worker_reuseport and (
+ cfg.tokenizer_worker_num == 1 or cfg.enable_http2
+ ):
+ raise ValueError(
+ "--enable-http-worker-reuseport requires --tokenizer-worker-num > 1 "
+ "and has no effect with --enable-http2."
+ )
if cfg.enable_http2:
if not 0 < cfg.http2_max_concurrent_streams < 2**32:
diff --git a/python/sglang/srt/entrypoints/http_server.py b/python/sglang/srt/entrypoints/http_server.py
index 430d86b9b6..0abcc2924d 100644
--- a/python/sglang/srt/entrypoints/http_server.py
+++ b/python/sglang/srt/entrypoints/http_server.py
@@ -19,8 +19,10 @@ This file implements HTTP APIs for the inference engine via fastapi.
import asyncio
import dataclasses
+import errno
import logging
import os
+import socket
import ssl
import tempfile
import threading
@@ -64,6 +66,7 @@ from fastapi.exceptions import RequestValidationError
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import ORJSONResponse, Response, StreamingResponse
from fastapi.routing import APIRoute
+from uvicorn.supervisors.multiprocess import Multiprocess
from sglang.srt.arg_groups.overrides import resolving_view
from sglang.srt.configs.embedding_model_spec import resolved_embedding_plan
@@ -2562,6 +2565,115 @@ def _wait_weights_ready():
)
+class _ReusePortMultiprocess(Multiprocess):
+ """Give each uvicorn worker a private SO_REUSEPORT listener.
+
+ uvicorn's supervisor reads ``self.sockets`` once per worker it spawns, so a
+ fresh listener is handed out on every read and the parent's copy is closed
+ once the child holds it; the lifecycle (health checks, restarts, SIGTTIN)
+ stays uvicorn's own. Requires uvicorn >= 0.51, whose Process builds its
+ own Server.
+ """
+
+ def __init__(self, config: uvicorn.Config):
+ self._handed_out: List[socket.socket] = []
+ self._reserved_socket: Optional[socket.socket] = _reserve_reuseport_socket(
+ config
+ )
+ self._socket_family = self._reserved_socket.family
+ self._socket_address = self._reserved_socket.getsockname()
+ super().__init__(config, sockets=[])
+
+ @property
+ def sockets(self) -> List[socket.socket]:
+ self._close_handed_out()
+ sock = self._reserved_socket
+ self._reserved_socket = None
+ if sock is None:
+ sock = _bind_reuseport_socket(
+ self.config, self._socket_family, self._socket_address
+ )
+ self._handed_out.append(sock)
+ return [sock]
+
+ @sockets.setter
+ def sockets(self, value: List[socket.socket]) -> None:
+ if value:
+ raise ValueError("_ReusePortMultiprocess binds its own listeners")
+
+ def _close_handed_out(self) -> None:
+ # Each spawn site starts its Process before the next read, so every
+ # listener handed out so far is already held by its child.
+ for sock in self._handed_out:
+ sock.close()
+ self._handed_out.clear()
+
+ def init_processes(self) -> None:
+ try:
+ super().init_processes()
+ except BaseException:
+ self.terminate_all()
+ self.join_all()
+ raise
+ finally:
+ self._close_handed_out()
+
+ def restart_all(self) -> None:
+ try:
+ super().restart_all()
+ finally:
+ self._close_handed_out()
+
+ def keep_subprocess_alive(self) -> None:
+ try:
+ super().keep_subprocess_alive()
+ finally:
+ self._close_handed_out()
+
+ def handle_ttin(self) -> None:
+ try:
+ super().handle_ttin()
+ finally:
+ self._close_handed_out()
+
+
+def _bind_reuseport_socket(config: uvicorn.Config, family, address) -> socket.socket:
+ sock = socket.socket(family=family)
+ sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
+ sock.bind(address)
+ sock.listen(config.backlog)
+ sock.set_inheritable(True)
+ return sock
+
+
+def _reserve_reuseport_socket(config: uvicorn.Config) -> socket.socket:
+ try:
+ family = socket.AF_INET6 if ":" in config.host else socket.AF_INET
+ address = socket.getaddrinfo(config.host, config.port, family)[0][4]
+ if config.port == 0:
+ sock = _bind_reuseport_socket(config, family, address)
+ config.port = sock.getsockname()[1]
+ return sock
+ name = f"\0sglang-reuseport-{os.geteuid()}-{config.port}"
+ with socket.socket(socket.AF_UNIX) as reservation:
+ while True:
+ try:
+ reservation.bind(name)
+ break
+ except OSError as error:
+ if error.errno != errno.EADDRINUSE:
+ raise
+ time.sleep(0.01)
+ with socket.socket(family=family) as probe:
+ probe.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ probe.bind(address)
+ return _bind_reuseport_socket(config, family, address)
+ except OSError as error:
+ logger.error(error)
+ raise SystemExit(1) from error
+
+
def _run_granian_server(
host,
port,
@@ -2858,22 +2970,32 @@ def _setup_and_run_http_server(
ssl_keyfile_password=get_serving().ssl_keyfile_password,
)
else:
- uvicorn.run(
- envs.SGLANG_HTTP_SERVER_APP_PATH.get(),
- host=get_serving().host,
- port=get_serving().port,
- root_path=get_serving().fastapi_root_path,
- log_level=get_observability().log_level_http
+ app_path = envs.SGLANG_HTTP_SERVER_APP_PATH.get()
+ uvicorn_options = {
+ "host": get_serving().host,
+ "port": get_serving().port,
+ "root_path": get_serving().fastapi_root_path,
+ "log_level": get_observability().log_level_http
or get_observability().log_level,
- timeout_keep_alive=envs.SGLANG_TIMEOUT_KEEP_ALIVE.get(),
- timeout_worker_healthcheck=envs.SGLANG_UVICORN_WORKER_HEALTHCHECK_TIMEOUT.get(),
- loop="uvloop",
- workers=get_serving().tokenizer_worker_num,
- ssl_keyfile=get_serving().ssl_keyfile,
- ssl_certfile=get_serving().ssl_certfile,
- ssl_ca_certs=get_serving().ssl_ca_certs,
- ssl_keyfile_password=get_serving().ssl_keyfile_password,
- )
+ "timeout_keep_alive": envs.SGLANG_TIMEOUT_KEEP_ALIVE.get(),
+ "timeout_worker_healthcheck": envs.SGLANG_UVICORN_WORKER_HEALTHCHECK_TIMEOUT.get(),
+ "loop": "uvloop",
+ "workers": get_serving().tokenizer_worker_num,
+ "ssl_keyfile": get_serving().ssl_keyfile,
+ "ssl_certfile": get_serving().ssl_certfile,
+ "ssl_ca_certs": get_serving().ssl_ca_certs,
+ "ssl_keyfile_password": get_serving().ssl_keyfile_password,
+ }
+ if get_serving().enable_http_worker_reuseport:
+ config = uvicorn.Config(app_path, **uvicorn_options)
+ supervisor = _ReusePortMultiprocess(config)
+ logger.info(
+ f"Uvicorn running on {config.host}:{config.port} "
+ f"with {config.workers} SO_REUSEPORT workers"
+ )
+ supervisor.run()
+ else:
+ uvicorn.run(app_path, **uvicorn_options)
finally:
if get_serving().tokenizer_worker_num > 1:
if multi_tokenizer_args_shm is not None:
diff --git a/test/registered/unit/entrypoints/test_http_server_reuseport.py b/test/registered/unit/entrypoints/test_http_server_reuseport.py
new file mode 100644
index 0000000000..8b231eb44e
--- /dev/null
+++ b/test/registered/unit/entrypoints/test_http_server_reuseport.py
@@ -0,0 +1,198 @@
+"""With --enable-http-worker-reuseport every uvicorn worker must own a private
+SO_REUSEPORT listener on the configured address, a replaced worker must get a
+fresh one, and a second server on the same fixed host:port must still fail to
+start instead of silently joining the first server's reuseport group.
+"""
+
+import copy
+import signal
+import socket
+import sys
+import threading
+from concurrent.futures import ThreadPoolExecutor
+from contextlib import suppress
+from unittest.mock import Mock
+
+import pytest
+import uvicorn
+from uvicorn.supervisors import multiprocess
+from uvicorn.supervisors.multiprocess import SIGNALS
+
+from sglang.srt.entrypoints import http_server
+from sglang.test.ci.ci_register import register_cpu_ci
+
+register_cpu_ci(est_time=2, suite="base-a-test-cpu")
+
+
+@pytest.fixture(autouse=True)
+def restore_signal_handlers():
+ # Multiprocess.__init__ replaces process-wide signal handlers.
+ handlers = {sig: signal.getsignal(sig) for sig in SIGNALS}
+ yield
+ for sig, handler in handlers.items():
+ signal.signal(sig, handler)
+
+
+@pytest.fixture
+def config(free_tcp_port):
+ return uvicorn.Config(
+ "starlette.applications:Starlette",
+ host="127.0.0.1",
+ port=free_tcp_port,
+ workers=4,
+ factory=True,
+ lifespan="off",
+ log_level="error",
+ )
+
+
+def _start(config, monkeypatch):
+ children = []
+
+ class FakeProcess:
+ # The uvicorn supervisor spawns through the module-level Process name.
+ def __init__(self, _config, sockets):
+ self.sockets = sockets
+ self.alive = True
+ self.exitcode = None
+ self.pid = len(children) + 1
+
+ def start(self):
+ self.listener = self.sockets[0].dup()
+ children.append(self.listener)
+
+ def is_alive(self, timeout):
+ return self.alive
+
+ def wait_until_ready(self, timeout, should_exit=None):
+ return True
+
+ def kill(self):
+ self.alive = False
+ self.listener.close()
+
+ terminate = kill
+
+ def join(self):
+ return None
+
+ monkeypatch.setattr(multiprocess, "Process", FakeProcess)
+ supervisor = http_server._ReusePortMultiprocess(config)
+ supervisor.init_processes()
+ return supervisor, children
+
+
+def test_worker_listener_lifecycle(monkeypatch, config):
+ resolve = Mock(wraps=socket.getaddrinfo)
+ monkeypatch.setattr(socket, "getaddrinfo", resolve)
+ supervisor, children = _start(config, monkeypatch)
+ assert resolve.call_count == 1
+ try:
+ assert all(p.sockets[0].fileno() == -1 for p in supervisor.processes)
+ assert len({sock.fileno() for sock in children}) == config.workers
+ assert all(
+ sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT) for sock in children
+ )
+
+ old = supervisor.processes[0]
+ old.alive = False
+ supervisor.keep_subprocess_alive()
+ new = supervisor.processes[0]
+ assert new is not old and old.listener.fileno() == -1
+ assert new.sockets[0].fileno() == -1 and new.listener.fileno() != -1
+
+ before_restart = list(supervisor.processes)
+ supervisor.restart_all()
+ assert all(process.listener.fileno() == -1 for process in before_restart)
+ assert all(p.sockets[0].fileno() == -1 for p in supervisor.processes)
+
+ supervisor.handle_ttin()
+ assert len(supervisor.processes) == config.workers + 1
+ assert supervisor.processes[-1].sockets[0].fileno() == -1
+ finally:
+ for sock in children:
+ sock.close()
+
+
+def test_partial_spawn_failure_cleans_up_workers(monkeypatch, config):
+ config.workers = 3
+ workers = [Mock(), Mock()]
+ monkeypatch.setattr(
+ multiprocess, "Process", Mock(side_effect=[*workers, RuntimeError("failed")])
+ )
+ supervisor = http_server._ReusePortMultiprocess(config)
+ with pytest.raises(RuntimeError):
+ supervisor.init_processes()
+ for worker in workers:
+ worker.terminate.assert_called_once()
+ worker.join.assert_called_once()
+ assert supervisor._reserved_socket is None and not supervisor._handed_out
+
+
+def test_unresolvable_host_exits_cleanly(monkeypatch, caplog, config):
+ resolve = Mock(side_effect=socket.gaierror("bad host"))
+ monkeypatch.setattr(socket, "getaddrinfo", resolve)
+ with pytest.raises(SystemExit):
+ http_server._ReusePortMultiprocess(config)
+ assert "bad host" in caplog.text
+
+
+@pytest.mark.parametrize(
+ ("hosts", "ephemeral", "expected_winners"),
+ [
+ (("127.0.0.1",) * 2, False, 1),
+ (("127.0.0.1",) * 2, True, 2),
+ (("127.0.0.1", "127.0.0.2"), False, 2),
+ ],
+)
+def test_concurrent_supervisor_reservation(
+ monkeypatch, config, hosts, ephemeral, expected_winners
+):
+ barrier = threading.Barrier(2)
+ bind = http_server._bind_reuseport_socket
+
+ def synchronized_bind(*args):
+ with suppress(threading.BrokenBarrierError):
+ barrier.wait(timeout=1)
+ return bind(*args)
+
+ def create_supervisor(index):
+ worker_config = copy.copy(config)
+ worker_config.host = hosts[index]
+ worker_config.port = 0 if ephemeral else config.port
+ try:
+ return http_server._ReusePortMultiprocess(worker_config)
+ except SystemExit:
+ return None
+
+ monkeypatch.setattr(http_server.Multiprocess, "__init__", Mock(return_value=None))
+ monkeypatch.setattr(http_server, "_bind_reuseport_socket", synchronized_bind)
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ supervisors = list(pool.map(create_supervisor, range(2)))
+
+ winners = [supervisor for supervisor in supervisors if supervisor is not None]
+ for supervisor in winners:
+ supervisor._reserved_socket.close()
+ assert len(winners) == expected_winners
+
+
+def test_real_uvicorn_worker_accepts_on_private_listener(config):
+ config.workers = 1
+ supervisor = http_server._ReusePortMultiprocess(config)
+ supervisor.init_processes()
+ try:
+ assert supervisor._reserved_socket is None and not supervisor._handed_out
+ # The listener accepts at once; the reply waits for the spawned worker,
+ # which re-imports this module and its server imports before serving.
+ with socket.create_connection(
+ (config.host, config.port), timeout=120
+ ) as connection:
+ connection.sendall(b"GET /openapi.json HTTP/1.0\r\n\r\n")
+ assert b"404 Not Found" in connection.recv(1024)
+ finally:
+ supervisor.terminate_all()
+ supervisor.join_all()
+
+
+if __name__ == "__main__":
+ sys.exit(pytest.main([__file__, "-v"]))
diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py
index 023fb5b8b5..6ecb9ebd04 100644
--- a/test/registered/unit/server_args/test_server_args.py
+++ b/test/registered/unit/server_args/test_server_args.py
@@ -2229,6 +2229,13 @@ class TestSSLArgs(unittest.TestCase):
self.assertIn("--enable-ssl-refresh", str(context.exception))
self.assertIn("--ssl-certfile", str(context.exception))
+ def test_http_worker_reuseport_requires_multiple_uvicorn_workers(self):
+ for kwargs in ({}, {"tokenizer_worker_num": 2, "enable_http2": True}):
+ with self.subTest(**kwargs), self.assertRaises(ValueError) as context:
+ self._validate_ssl(enable_http_worker_reuseport=True, **kwargs)
+ self.assertIn("--enable-http-worker-reuseport", str(context.exception))
+ self._validate_ssl(enable_http_worker_reuseport=True, tokenizer_worker_num=2)
+
@patch("os.path.isfile", return_value=True)
def test_enable_ssl_refresh_with_ssl_accepted(self, _mock_isfile):
server_args = self._validate_ssl(
diff --git a/test/registered/unit/test_server_args_cli_metadata.py b/test/registered/unit/test_server_args_cli_metadata.py
index b7a03063f8..46c09bda6e 100644
--- a/test/registered/unit/test_server_args_cli_metadata.py
+++ b/test/registered/unit/test_server_args_cli_metadata.py
@@ -31,6 +31,18 @@ class TestServerArgsMigratedCliMetadata(CustomTestCase):
ServerArgs.from_cli_args(args).trust_request_chat_template, expected
)
+ def test_http_worker_reuseport_requires_explicit_opt_in(self):
+ for flags, expected in (
+ ([], False),
+ (["--enable-http-worker-reuseport"], True),
+ ):
+ with self.subTest(flags=flags):
+ args = self.parser.parse_args(["--model", "dummy", *flags])
+ self.assertIs(
+ ServerArgs.from_cli_args(args).enable_http_worker_reuseport,
+ expected,
+ )
+
def test_prefill_max_context_accepts_human_readable_values(self):
for option in (
"--cuda-graph-prefill-max-context",