diff --git a/python/pyproject.toml b/python/pyproject.toml index 8e96b44af..99c6e1b3c 100755 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -129,6 +129,10 @@ tracing = [ "opentelemetry-sdk", ] +http2 = [ + "granian>=2.6.0", +] + test = [ "accelerate", "addict", @@ -146,6 +150,7 @@ test = [ "diff-cover", "sentence_transformers", "tabulate", + "granian>=2.6.0", ] dev = ["sglang[test]"] @@ -153,6 +158,7 @@ dev = ["sglang[test]"] all = [ "sglang[diffusion]", "sglang[tracing]", + "sglang[http2]", ] [tool.uv.extra-build-dependencies] diff --git a/python/sglang/srt/entrypoints/http_server.py b/python/sglang/srt/entrypoints/http_server.py index 61f7513f5..43afc1577 100644 --- a/python/sglang/srt/entrypoints/http_server.py +++ b/python/sglang/srt/entrypoints/http_server.py @@ -206,6 +206,32 @@ def get_global_state() -> _GlobalState: return _global_state +async def _init_granian_worker() -> ServerArgs: + main_pid = get_main_process_id() + port_args, server_args, scheduler_info = read_from_shared_memory( + f"multi_tokenizer_args_{main_pid}" + ) + + tokenizer_manager = TokenizerManager(server_args, port_args) + template_manager = TemplateManager() + template_manager.initialize_templates( + tokenizer_manager=tokenizer_manager, + model_path=server_args.model_path, + chat_template=server_args.chat_template, + completion_template=server_args.completion_template, + ) + tokenizer_manager.max_req_input_len = scheduler_info["max_req_input_len"] + + set_global_state( + _GlobalState( + tokenizer_manager=tokenizer_manager, + template_manager=template_manager, + scheduler_info=scheduler_info, + ) + ) + return server_args + + async def init_multi_tokenizer() -> ServerArgs: """ Initialization function for multi-process tokenizer mode. @@ -263,6 +289,10 @@ async def lifespan(fast_api_app: FastAPI): server_args = fast_api_app.server_args warmup_thread_kwargs = fast_api_app.warmup_thread_kwargs thread_label = "Tokenizer" + elif envs.SGLANG_GRANIAN_PARENT_PID.get() is not None: + server_args = await _init_granian_worker() + warmup_thread_kwargs = dict(server_args=server_args) + thread_label = "Tokenizer" else: # Initialize multi-tokenizer support for worker processes server_args = await init_multi_tokenizer() @@ -2017,6 +2047,53 @@ def _wait_weights_ready(): ) +def _close_main_process_sockets(): + """Close the main process's ZMQ sockets before spawning Granian workers. + + Granian workers create their own TokenizerManager with fresh ZMQ sockets. + The main process must release its sockets first to avoid binding conflicts + on the same IPC addresses. + """ + if _global_state is None or _global_state.tokenizer_manager is None: + return + tm = _global_state.tokenizer_manager + for attr in ("recv_from_detokenizer", "send_to_scheduler"): + sock = getattr(tm, attr, None) + if sock is None: + continue + inner = getattr(sock, "socket", None) + if inner is not None: + inner.close() + elif hasattr(sock, "close"): + sock.close() + setattr(tm, attr, None) + + +def _run_granian_server(server_args: ServerArgs): + """Launch Granian with HTTP/2 support""" + from granian import Granian + from granian.constants import HTTPModes, Interfaces, Loops + + granian_kwargs = dict( + target="sglang.srt.entrypoints.http_server:app", + address=server_args.host, + port=server_args.port, + interface=Interfaces.ASGI, + http=HTTPModes.auto, + loop=Loops.uvloop, + log_level=server_args.log_level_http or server_args.log_level or "info", + workers=1, + ) + + ssl_enabled = server_args.ssl_certfile and server_args.ssl_keyfile + if ssl_enabled: + granian_kwargs["ssl_cert"] = server_args.ssl_certfile + granian_kwargs["ssl_key"] = server_args.ssl_keyfile + + server = Granian(**granian_kwargs) + server.serve() + + def _setup_and_run_http_server( server_args: ServerArgs, tokenizer_manager, @@ -2047,6 +2124,35 @@ def _setup_and_run_http_server( if server_args.enable_metrics: add_prometheus_track_response_middleware(app) + # Use Granian for HTTP/2 server + if server_args.enable_http2: + # Reuse the multi-tokenizer shared memory mechanism to pass + # init args (port_args, server_args, scheduler_info) to + # Granian workers, which are independent processes. + multi_tokenizer_args_shm = write_data_for_multi_tokenizer( + port_args, server_args, scheduler_infos[0] + ) + try: + if server_args.ssl_certfile: + logger.info( + f"SSL enabled: certfile={server_args.ssl_certfile}, " + f"keyfile={server_args.ssl_keyfile}" + ) + logger.info( + f"Starting Granian HTTP/2 server on " + f"{server_args.host}:{server_args.port}" + ) + # Propagate the main process PID via os.environ so Granian + # workers (forked or spawned) can locate the shared memory + # segment created above. + envs.SGLANG_GRANIAN_PARENT_PID.set(os.getpid()) + _close_main_process_sockets() + _run_granian_server(server_args) + finally: + if multi_tokenizer_args_shm is not None: + multi_tokenizer_args_shm.unlink() + return + # Pass additional arguments to the lifespan function. # They will be used for additional initialization setups. if server_args.tokenizer_worker_num == 1: diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index dfc5507de..e6eb48212 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -489,6 +489,9 @@ class Envs: # HTTP Server SGLANG_TIMEOUT_KEEP_ALIVE = EnvInt(5) + # HTTP/2 Server + SGLANG_GRANIAN_PARENT_PID = EnvInt(None) + # Health Check SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION = EnvBool(True) diff --git a/python/sglang/srt/managers/multi_tokenizer_mixin.py b/python/sglang/srt/managers/multi_tokenizer_mixin.py index e0a1669fb..8da3d3b0d 100644 --- a/python/sglang/srt/managers/multi_tokenizer_mixin.py +++ b/python/sglang/srt/managers/multi_tokenizer_mixin.py @@ -433,7 +433,16 @@ async def print_exception_wrapper(func): def get_main_process_id() -> int: - """Get the main process ID""" + """Get the main process ID. + + Supports override via SGLANG_GRANIAN_PARENT_PID for workers whose + multiprocessing parent PID differs from the shared-memory owner. + """ + from sglang.srt.environ import envs + + override = envs.SGLANG_GRANIAN_PARENT_PID.get() + if override is not None: + return override return multiprocessing.current_process()._parent_pid diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index f6757a1bd..86de1b327 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -326,6 +326,7 @@ class ServerArgs: ssl_ca_certs: Optional[str] = None ssl_keyfile_password: Optional[str] = None enable_ssl_refresh: bool = False + enable_http2: bool = False # Quantization and data type dtype: str = "auto" @@ -923,6 +924,26 @@ class ServerArgs: "to be specified." ) + if self.enable_http2: + try: + import granian # noqa: F401 + except ImportError: + raise ValueError( + "--enable-http2 requires the 'granian' package. " + 'Install it with: pip install "sglang[http2]"' + ) + if self.enable_ssl_refresh: + raise ValueError( + "--enable-ssl-refresh is not supported with --enable-http2. " + "Granian does not support SSL certificate hot-reloading. " + "Use Uvicorn (the default) or handle certificate rotation externally." + ) + if self.tokenizer_worker_num > 1: + raise ValueError( + "--enable-http2 does not yet support --tokenizer-worker-num > 1. " + "Multi-worker HTTP/2 support will be added in a future release." + ) + def _handle_deprecated_args(self): # Handle deprecated tool call parsers deprecated_tool_call_parsers = {"qwen25": "qwen", "glm45": "glm"} @@ -3865,6 +3886,14 @@ class ServerArgs: help="Enable automatic SSL certificate hot-reloading when cert/key " "files change on disk. Requires --ssl-certfile and --ssl-keyfile.", ) + parser.add_argument( + "--enable-http2", + action="store_true", + default=ServerArgs.enable_http2, + help="Use Granian instead of Uvicorn as the ASGI server, enabling HTTP/1.1 and " + "HTTP/2 auto-negotiation. Clients may use h2c (cleartext HTTP/2) or plain HTTP/1.1. " + "Requires 'pip install sglang[http2]'.", + ) # Quantization and data type parser.add_argument( diff --git a/test/registered/openai_server/basic/test_http2_server.py b/test/registered/openai_server/basic/test_http2_server.py new file mode 100644 index 000000000..6cfc3ee7e --- /dev/null +++ b/test/registered/openai_server/basic/test_http2_server.py @@ -0,0 +1,112 @@ +""" +Test HTTP/2 server (Granian) with basic OpenAI-compatible endpoints. + +Verifies that --enable-http2 launches successfully and serves requests +via both HTTP/1.1 and HTTP/2 (h2c). +""" + +import subprocess +import unittest + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +try: + import granian # noqa: F401 + + _HAS_GRANIAN = True +except ImportError: + _HAS_GRANIAN = False + +register_cuda_ci(est_time=120, suite="stage-b-test-1-gpu-small") + + +@unittest.skipUnless(_HAS_GRANIAN, "granian not installed (pip install sglang[http2])") +class TestHTTP2Server(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--enable-http2"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_health(self): + resp = requests.get(f"{self.base_url}/health") + self.assertEqual(resp.status_code, 200) + + def test_get_model_info(self): + resp = requests.get(f"{self.base_url}/get_model_info") + self.assertEqual(resp.status_code, 200) + self.assertIn("model_path", resp.json()) + + def test_completion(self): + resp = requests.post( + f"{self.base_url}/v1/completions", + json={ + "model": self.model, + "prompt": "The capital of France is", + "max_tokens": 8, + "temperature": 0, + }, + ) + self.assertEqual(resp.status_code, 200) + data = resp.json() + self.assertIn("choices", data) + self.assertGreater(len(data["choices"][0]["text"]), 0) + + def test_chat_completion(self): + resp = requests.post( + f"{self.base_url}/v1/chat/completions", + json={ + "model": self.model, + "messages": [{"role": "user", "content": "Say hello"}], + "max_tokens": 16, + "temperature": 0, + }, + ) + self.assertEqual(resp.status_code, 200) + data = resp.json() + self.assertIn("choices", data) + self.assertGreater(len(data["choices"][0]["message"]["content"]), 0) + + def test_h2c_with_curl(self): + """Verify the server actually speaks HTTP/2 via h2c.""" + result = subprocess.run( + [ + "curl", + "--http2-prior-knowledge", + "-s", + "-o", + "/dev/null", + "-w", + "%{http_version}", + f"{self.base_url}/health", + ], + capture_output=True, + text=True, + timeout=10, + ) + self.assertEqual( + result.stdout.strip(), "2", "Server should respond with HTTP/2" + ) + + +if __name__ == "__main__": + unittest.main(verbosity=3)