From e5b3a487515a8d76672f87c5d959eeb857394661 Mon Sep 17 00:00:00 2001 From: cctry Date: Sat, 15 Aug 2026 10:34:49 -0700 Subject: [PATCH] Add --http2-max-concurrent-streams server arg (#34796) Co-authored-by: Yilong Zhao <74357408+happierpig@users.noreply.github.com> --- .../advanced_features/server_arguments.mdx | 6 +++ python/sglang/srt/entrypoints/http_server.py | 11 +++++ python/sglang/srt/server_args.py | 12 ++++++ .../openai_server/basic/test_http2_server.py | 14 ++++++- .../entrypoints/test_http2_server_config.py | 40 +++++++++++++++++++ 5 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 test/registered/unit/entrypoints/test_http2_server_config.py diff --git a/docs/docs/advanced_features/server_arguments.mdx b/docs/docs/advanced_features/server_arguments.mdx index 698ef417d..eea2a0768 100644 --- a/docs/docs/advanced_features/server_arguments.mdx +++ b/docs/docs/advanced_features/server_arguments.mdx @@ -312,6 +312,12 @@ Please consult the documentation below and [server_args.py](https://github.com/s `False` bool flag (set to enable) + + `--http2-max-concurrent-streams` + Maximum number of concurrent streams advertised on each HTTP/2 connection (1 to 2^32 - 1). Only applies with --enable-http2. + `200` + Type: int + diff --git a/python/sglang/srt/entrypoints/http_server.py b/python/sglang/srt/entrypoints/http_server.py index 5aa6fcca1..3371fcf0a 100644 --- a/python/sglang/srt/entrypoints/http_server.py +++ b/python/sglang/srt/entrypoints/http_server.py @@ -2392,6 +2392,7 @@ def _run_granian_server( host, port, log_level, + http2_max_concurrent_streams, tokenizer_worker_num=1, ssl_certfile=None, ssl_keyfile=None, @@ -2415,6 +2416,7 @@ def _run_granian_server( from granian import Granian from granian.constants import HTTPModes, Interfaces, Loops + from granian.http import HTTP2Settings from granian.server.embed import Server as GranianEmbeddedServer Server = GranianEmbeddedServer if tokenizer_worker_num == 1 else Granian @@ -2427,6 +2429,9 @@ def _run_granian_server( port=port, interface=Interfaces.ASGI, http=HTTPModes.auto, + http2_settings=HTTP2Settings( + max_concurrent_streams=http2_max_concurrent_streams + ), log_level=log_level, ssl_cert=ssl_certfile, ssl_key=ssl_keyfile, @@ -2556,6 +2561,9 @@ def _setup_and_run_http_server( host=server_args.host, port=server_args.port, log_level=server_args.log_level_http or server_args.log_level, + http2_max_concurrent_streams=( + server_args.http2_max_concurrent_streams + ), ssl_certfile=server_args.ssl_certfile, ssl_keyfile=server_args.ssl_keyfile, ssl_ca_certs=server_args.ssl_ca_certs, @@ -2640,6 +2648,9 @@ def _setup_and_run_http_server( host=server_args.host, port=server_args.port, log_level=server_args.log_level_http or server_args.log_level, + http2_max_concurrent_streams=( + server_args.http2_max_concurrent_streams + ), tokenizer_worker_num=server_args.tokenizer_worker_num, ssl_certfile=server_args.ssl_certfile, ssl_keyfile=server_args.ssl_keyfile, diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 0eca8eee2..d7c0c15ab 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -1306,6 +1306,12 @@ class ServerArgs: "Use Granian instead of Uvicorn as the ASGI server, enabling HTTP/1.1 and HTTP/2 auto-negotiation. Clients may use h2c (cleartext HTTP/2) or plain HTTP/1.1. Requires 'pip install sglang[http2]'.", NS("serving"), ] = False + http2_max_concurrent_streams: A[ + int, + "Maximum number of concurrent streams advertised on each HTTP/2 " + "connection (1 to 2^32 - 1). Only applies with --enable-http2.", + NS("serving"), + ] = 200 # ------------------------------------------------------------------------- # SSL/TLS @@ -4029,6 +4035,12 @@ class ServerArgs: ) if self.enable_http2: + if not 0 < self.http2_max_concurrent_streams < 2**32: + raise ValueError( + "--http2-max-concurrent-streams must be between 1 and " + "4294967295." + ) + try: import granian # noqa: F401 except ImportError: diff --git a/test/registered/openai_server/basic/test_http2_server.py b/test/registered/openai_server/basic/test_http2_server.py index e90580b49..db0ceddfb 100644 --- a/test/registered/openai_server/basic/test_http2_server.py +++ b/test/registered/openai_server/basic/test_http2_server.py @@ -41,7 +41,11 @@ class TestHTTP2Server(CustomTestCase): cls.model, cls.base_url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=["--enable-http2"], + other_args=[ + "--enable-http2", + "--http2-max-concurrent-streams", + "64", + ], ) @classmethod @@ -126,7 +130,13 @@ class TestHTTP2ServerMultiTokenizer(TestHTTP2Server): cls.model, cls.base_url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=["--enable-http2", "--tokenizer-worker-num", "2"], + other_args=[ + "--enable-http2", + "--http2-max-concurrent-streams", + "64", + "--tokenizer-worker-num", + "2", + ], ) diff --git a/test/registered/unit/entrypoints/test_http2_server_config.py b/test/registered/unit/entrypoints/test_http2_server_config.py new file mode 100644 index 000000000..b3881a686 --- /dev/null +++ b/test/registered/unit/entrypoints/test_http2_server_config.py @@ -0,0 +1,40 @@ +import importlib.util +import unittest +from unittest.mock import patch + +from sglang.srt.entrypoints.http_server import _run_granian_server +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=2, suite="base-a-test-cpu") + + +@unittest.skipUnless( + importlib.util.find_spec("granian"), "granian is required for HTTP/2" +) +class TestGranianHTTP2Config(unittest.TestCase): + def test_passes_explicit_max_concurrent_streams(self): + configured = {} + + class FakeEmbeddedServer: + def __init__(self, **kwargs): + configured.update(kwargs) + + async def serve(self): + return None + + def stop(self): + return None + + with patch("granian.server.embed.Server", FakeEmbeddedServer): + _run_granian_server( + host="127.0.0.1", + port=30000, + log_level="info", + http2_max_concurrent_streams=37, + ) + + self.assertEqual(configured["http2_settings"].max_concurrent_streams, 37) + + +if __name__ == "__main__": + unittest.main()