Add --http2-max-concurrent-streams server arg (#34796)

Co-authored-by: Yilong Zhao <74357408+happierpig@users.noreply.github.com>

docs/docs/advanced_features/server_arguments.mdx

@@ -312,6 +312,12 @@ Please consult the documentation below and [server_args.py](https://github.com/s
       <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>`False`</td>
       <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>bool flag (set to enable)</td>
     </tr>
+    <tr>
+      <td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--http2-max-concurrent-streams`</td>
+      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Maximum number of concurrent streams advertised on each HTTP/2 connection (1 to 2^32 - 1). Only applies with --enable-http2.</td>
+      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>`200`</td>
+      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Type: int</td>
+    </tr>
 </tbody>
 </table>
 

python/sglang/srt/entrypoints/http_server.py

@@ -2392,6 +2392,7 @@ def _run_granian_server(
     host,
     port,
     log_level,
+    http2_max_concurrent_streams,
     tokenizer_worker_num=1,
     ssl_certfile=None,
     ssl_keyfile=None,
@@ -2415,6 +2416,7 @@ def _run_granian_server(
 
     from granian import Granian
     from granian.constants import HTTPModes, Interfaces, Loops
+    from granian.http import HTTP2Settings
     from granian.server.embed import Server as GranianEmbeddedServer
 
     Server = GranianEmbeddedServer if tokenizer_worker_num == 1 else Granian
@@ -2427,6 +2429,9 @@ def _run_granian_server(
         port=port,
         interface=Interfaces.ASGI,
         http=HTTPModes.auto,
+        http2_settings=HTTP2Settings(
+            max_concurrent_streams=http2_max_concurrent_streams
+        ),
         log_level=log_level,
         ssl_cert=ssl_certfile,
         ssl_key=ssl_keyfile,
@@ -2556,6 +2561,9 @@ def _setup_and_run_http_server(
                     host=server_args.host,
                     port=server_args.port,
                     log_level=server_args.log_level_http or server_args.log_level,
+                    http2_max_concurrent_streams=(
+                        server_args.http2_max_concurrent_streams
+                    ),
                     ssl_certfile=server_args.ssl_certfile,
                     ssl_keyfile=server_args.ssl_keyfile,
                     ssl_ca_certs=server_args.ssl_ca_certs,
@@ -2640,6 +2648,9 @@ def _setup_and_run_http_server(
                     host=server_args.host,
                     port=server_args.port,
                     log_level=server_args.log_level_http or server_args.log_level,
+                    http2_max_concurrent_streams=(
+                        server_args.http2_max_concurrent_streams
+                    ),
                     tokenizer_worker_num=server_args.tokenizer_worker_num,
                     ssl_certfile=server_args.ssl_certfile,
                     ssl_keyfile=server_args.ssl_keyfile,

python/sglang/srt/server_args.py

@@ -1306,6 +1306,12 @@ class ServerArgs:
         "Use Granian instead of Uvicorn as the ASGI server, enabling HTTP/1.1 and HTTP/2 auto-negotiation. Clients may use h2c (cleartext HTTP/2) or plain HTTP/1.1. Requires 'pip install sglang[http2]'.",
         NS("serving"),
     ] = False
+    http2_max_concurrent_streams: A[
+        int,
+        "Maximum number of concurrent streams advertised on each HTTP/2 "
+        "connection (1 to 2^32 - 1). Only applies with --enable-http2.",
+        NS("serving"),
+    ] = 200
 
     # -------------------------------------------------------------------------
     # SSL/TLS
@@ -4029,6 +4035,12 @@ class ServerArgs:
             )
 
         if self.enable_http2:
+            if not 0 < self.http2_max_concurrent_streams < 2**32:
+                raise ValueError(
+                    "--http2-max-concurrent-streams must be between 1 and "
+                    "4294967295."
+                )
+
             try:
                 import granian  # noqa: F401
             except ImportError:

test/registered/openai_server/basic/test_http2_server.py

@@ -41,7 +41,11 @@ class TestHTTP2Server(CustomTestCase):
             cls.model,
             cls.base_url,
             timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-            other_args=["--enable-http2"],
+            other_args=[
+                "--enable-http2",
+                "--http2-max-concurrent-streams",
+                "64",
+            ],
         )
 
     @classmethod
@@ -126,7 +130,13 @@ class TestHTTP2ServerMultiTokenizer(TestHTTP2Server):
             cls.model,
             cls.base_url,
             timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-            other_args=["--enable-http2", "--tokenizer-worker-num", "2"],
+            other_args=[
+                "--enable-http2",
+                "--http2-max-concurrent-streams",
+                "64",
+                "--tokenizer-worker-num",
+                "2",
+            ],
         )
 
 

test/registered/unit/entrypoints/test_http2_server_config.py

@@ -0,0 +1,40 @@
+import importlib.util
+import unittest
+from unittest.mock import patch
+
+from sglang.srt.entrypoints.http_server import _run_granian_server
+from sglang.test.ci.ci_register import register_cpu_ci
+
+register_cpu_ci(est_time=2, suite="base-a-test-cpu")
+
+
+@unittest.skipUnless(
+    importlib.util.find_spec("granian"), "granian is required for HTTP/2"
+)
+class TestGranianHTTP2Config(unittest.TestCase):
+    def test_passes_explicit_max_concurrent_streams(self):
+        configured = {}
+
+        class FakeEmbeddedServer:
+            def __init__(self, **kwargs):
+                configured.update(kwargs)
+
+            async def serve(self):
+                return None
+
+            def stop(self):
+                return None
+
+        with patch("granian.server.embed.Server", FakeEmbeddedServer):
+            _run_granian_server(
+                host="127.0.0.1",
+                port=30000,
+                log_level="info",
+                http2_max_concurrent_streams=37,
+            )
+
+        self.assertEqual(configured["http2_settings"].max_concurrent_streams, 37)
+
+
+if __name__ == "__main__":
+    unittest.main()