minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Add --http2-max-concurrent-streams server arg (#34796)

Browse Source 
Co-authored-by: Yilong Zhao <74357408+happierpig@users.noreply.github.com>
This commit is contained in:
cctry
2026-08-15 10:34:49 -07:00
committed by GitHub
co-authored by Yilong Zhao
parent cef8a32b9d
commit e5b3a48751
5 changed files with 81 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/docs/advanced_features/server_arguments.mdx
+6
View File
@@ -312,6 +312,12 @@ Please consult the documentation below and [server_args.py](https://github.com/s
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>`False`</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>bool flag (set to enable)</td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--http2-max-concurrent-streams`</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Maximum number of concurrent streams advertised on each HTTP/2 connection (1 to 2^32 - 1). Only applies with --enable-http2.</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>`200`</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Type: int</td>
</tr>
</tbody>
</table>
python/sglang/srt/entrypoints/http_server.py
+11
View File
@@ -2392,6 +2392,7 @@ def _run_granian_server(
host,
port,
log_level,
http2_max_concurrent_streams,
tokenizer_worker_num=1,
ssl_certfile=None,
ssl_keyfile=None,
@@ -2415,6 +2416,7 @@ def _run_granian_server(
from granian import Granian
from granian.constants import HTTPModes, Interfaces, Loops
from granian.http import HTTP2Settings
from granian.server.embed import Server as GranianEmbeddedServer
Server = GranianEmbeddedServer if tokenizer_worker_num == 1 else Granian
@@ -2427,6 +2429,9 @@ def _run_granian_server(
port=port,
interface=Interfaces.ASGI,
http=HTTPModes.auto,
http2_settings=HTTP2Settings(
max_concurrent_streams=http2_max_concurrent_streams
),
log_level=log_level,
ssl_cert=ssl_certfile,
ssl_key=ssl_keyfile,
@@ -2556,6 +2561,9 @@ def _setup_and_run_http_server(
host=server_args.host,
port=server_args.port,
log_level=server_args.log_level_http or server_args.log_level,
http2_max_concurrent_streams=(
server_args.http2_max_concurrent_streams
),
ssl_certfile=server_args.ssl_certfile,
ssl_keyfile=server_args.ssl_keyfile,
ssl_ca_certs=server_args.ssl_ca_certs,
@@ -2640,6 +2648,9 @@ def _setup_and_run_http_server(
host=server_args.host,
port=server_args.port,
log_level=server_args.log_level_http or server_args.log_level,
http2_max_concurrent_streams=(
server_args.http2_max_concurrent_streams
),
tokenizer_worker_num=server_args.tokenizer_worker_num,
ssl_certfile=server_args.ssl_certfile,
ssl_keyfile=server_args.ssl_keyfile,
python/sglang/srt/server_args.py
+12
View File
@@ -1306,6 +1306,12 @@ class ServerArgs:
"Use Granian instead of Uvicorn as the ASGI server, enabling HTTP/1.1 and HTTP/2 auto-negotiation. Clients may use h2c (cleartext HTTP/2) or plain HTTP/1.1. Requires 'pip install sglang[http2]'.",
NS("serving"),
] = False
http2_max_concurrent_streams: A[
int,
"Maximum number of concurrent streams advertised on each HTTP/2 "
"connection (1 to 2^32 - 1). Only applies with --enable-http2.",
NS("serving"),
] = 200
# -------------------------------------------------------------------------
# SSL/TLS
@@ -4029,6 +4035,12 @@ class ServerArgs:
)
if self.enable_http2:
if not 0 < self.http2_max_concurrent_streams < 2**32:
raise ValueError(
"--http2-max-concurrent-streams must be between 1 and "
"4294967295."
)
try:
import granian # noqa: F401
except ImportError:
test/registered/openai_server/basic/test_http2_server.py
+12 -2
View File
@@ -41,7 +41,11 @@ class TestHTTP2Server(CustomTestCase):
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=["--enable-http2"],
other_args=[
"--enable-http2",
"--http2-max-concurrent-streams",
"64",
],
)
@classmethod
@@ -126,7 +130,13 @@ class TestHTTP2ServerMultiTokenizer(TestHTTP2Server):
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=["--enable-http2", "--tokenizer-worker-num", "2"],
other_args=[
"--enable-http2",
"--http2-max-concurrent-streams",
"64",
"--tokenizer-worker-num",
"2",
],
)
test/registered/unit/entrypoints/test_http2_server_config.py
+40
View File
@@ -0,0 +1,40 @@
import importlib.util
import unittest
from unittest.mock import patch
from sglang.srt.entrypoints.http_server import _run_granian_server
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=2, suite="base-a-test-cpu")
@unittest.skipUnless(
importlib.util.find_spec("granian"), "granian is required for HTTP/2"
)
class TestGranianHTTP2Config(unittest.TestCase):
def test_passes_explicit_max_concurrent_streams(self):
configured = {}
class FakeEmbeddedServer:
def __init__(self, **kwargs):
configured.update(kwargs)
async def serve(self):
return None
def stop(self):
return None
with patch("granian.server.embed.Server", FakeEmbeddedServer):
_run_granian_server(
host="127.0.0.1",
port=30000,
log_level="info",
http2_max_concurrent_streams=37,
)
self.assertEqual(configured["http2_settings"].max_concurrent_streams, 37)
if __name__ == "__main__":
unittest.main()