minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[router] Speak cleartext h2c on both edges: serve it inbound, forward it outbound (#39006)

Browse Source 
Co-authored-by: Kangyan Zhou <kangyan.zhou@radixark.ai>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Kangyan-Zhou
2026-09-15 22:30:31 -07:00
committed by GitHub
co-authored by Kangyan Zhou Claude Opus 5
parent c1f5b4736a
commit afde31a2f5
17 changed files with 1009 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
experimental/sgl-router/tests/e2e/k8s_integration/Dockerfile.fake_worker
+1 -1
View File
@@ -1,6 +1,6 @@
FROM python:3.12-slim
WORKDIR /app
RUN pip install --no-cache-dir fastapi uvicorn
RUN pip install --no-cache-dir fastapi uvicorn granian
COPY fake_worker.py .
EXPOSE 30000
CMD ["python", "fake_worker.py"]
experimental/sgl-router/tests/e2e/k8s_integration/fake_worker.py
+52 -5
View File
@@ -2,9 +2,21 @@
Responds to:
GET /health -> {"status": "ok"}
GET /server_info -> {"served_model_name": MODEL_ID}
GET /server_info -> {"served_model_name": MODEL_ID, ...}
GET /v1/models -> list with a single MODEL_ID model entry
POST /v1/chat/completions -> echoes the last user message back
POST /v1/chat/completions -> echoes the last user message back, plus
the HTTP version the request arrived on
Set `FAKE_WORKER_HTTP2=1` to imitate an engine launched with
`--enable-http2`: `/server_info` advertises the flag and the app is served by
Granian in `HTTPModes.auto`, which is what the real engine runs, so one port
serves cleartext h2c alongside HTTP/1.1. The default is uvicorn, which speaks
HTTP/1.1 only.
`x_http_version` on the chat response is the load-bearing part for
`test_h2c_forwarding.py`: a chat completion returns 200 over either protocol,
so without the worker reporting what it actually received, an h2c test passes
whether or not h2c was used.
"""
from __future__ import annotations
@@ -17,6 +29,16 @@ from fastapi import FastAPI, Request
app = FastAPI()
MODEL_ID = os.environ.get("MODEL_ID", "tiny")
# Normalised before comparing: a manifest that writes the Python-idiomatic
# "False" must not silently turn this worker into a Granian/h2c one, which
# would fail test_h2c_forwarding with a message about a router bug.
ENABLE_HTTP2 = os.environ.get("FAKE_WORKER_HTTP2", "").strip().lower() not in (
"",
"0",
"false",
"no",
"off",
)
@app.get("/health")
@@ -27,8 +49,12 @@ async def health():
@app.get("/server_info")
async def server_info():
# The sgl-router worker manager fetches this on every Added event and
# uses `served_model_name` to populate the registry's model index.
return {"served_model_name": MODEL_ID}
# uses `served_model_name` to populate the registry's model index, and
# `enable_http2` to resolve the worker's forwarding protocol.
info = {"served_model_name": MODEL_ID}
if ENABLE_HTTP2:
info["enable_http2"] = True
return info
@app.get("/v1/models")
@@ -55,6 +81,11 @@ async def chat_completions(request: Request):
"id": "chatcmpl-mock",
"object": "chat.completion",
"model": payload.get("model", MODEL_ID),
# Non-standard, and deliberately so: the router returns the upstream
# body verbatim (`proxy::forward_*` hands back `resp.bytes()`), so this
# is how a test on the other side of the router learns which protocol
# the forward leg actually used. "1.1" or "2".
"x_http_version": request.scope.get("http_version"),
"choices": [
{
"index": 0,
@@ -70,4 +101,20 @@ async def chat_completions(request: Request):
if __name__ == "__main__":
uvicorn.run(app, host="0.0.0.0", port=30000)
if ENABLE_HTTP2:
# Mirrors the engine's own server (`_run_granian_server` in
# sglang/srt/entrypoints/http_server.py): HTTPModes.auto dispatches per
# connection on the first bytes, so h2c prior-knowledge and HTTP/1.1
# share one cleartext port.
from granian import Granian
from granian.constants import HTTPModes, Interfaces
Granian(
target="fake_worker:app",
address="0.0.0.0",
port=30000,
interface=Interfaces.ASGI,
http=HTTPModes.auto,
).serve()
else:
uvicorn.run(app, host="0.0.0.0", port=30000)
experimental/sgl-router/tests/e2e/k8s_integration/test_h2c_forwarding.py
+198
View File
@@ -0,0 +1,198 @@
"""E2E: the router forwards to an h2c-capable worker over cleartext HTTP/2.
The in-process tests (`tests/proxy/h2c_forward.rs`, `inbound_h2c.rs`) already
drive real HTTP/2 sockets, and dropping reqwest's `http2` feature fails the
build outright, so neither the client nor the framing needs covering again
here. What no in-process test can assemble is the *chain*: a worker discovered
through a real EndpointSlice, introspected over the network, resolved to
`WireProtocol::H2c` from its own `/server_info`, and then actually forwarded to
over h2c.
The fleet is deliberately mixed. `setup.sh` leaves three uvicorn workers
(HTTP/1.1 only) behind the `app=sglang` Service; this module adds one Granian
worker reporting `enable_http2: true` to the same Service, so both protocols
must be in use simultaneously. That is the e2e form of the per-worker-protocol
property: a router that resolved one protocol fleet-wide would either fail
against the h2c worker or break the three HTTP/1.1 ones, and either way this
test fails.
"""
from __future__ import annotations
import httpx
import pytest
from conftest import (
NAMESPACE,
_apply_from_stdin,
_kubectl,
_poll_until,
_wait_for_deployment_ready,
logger,
)
H2C_DEPLOYMENT = "fake-worker-h2c"
# Round-robin over a 4-worker pool: 12 requests give every worker ~3 turns, so
# a miss means a routing or resolution failure rather than an unlucky draw.
_PROBE_REQUESTS = 12
# Kept low enough that a whole probe round (12 x 5 s worst case) fits inside the
# 90 s poll budget below. A fake worker answers instantly; a request that needs
# more than 5 s is already a failure, and letting a round outlast its own poll
# would make that timeout non-binding.
_CHAT_TIMEOUT = 5.0
_CONVERGE_TIMEOUT = 90
_H2C_WORKER_MANIFEST = f"""
apiVersion: apps/v1
kind: Deployment
metadata:
name: {H2C_DEPLOYMENT}
namespace: {NAMESPACE}
spec:
replicas: 1
selector:
matchLabels:
app: sglang-h2c
template:
metadata:
labels:
app: sglang-h2c
spec:
containers:
- name: worker
image: sgl-router-fake-worker:e2e
imagePullPolicy: Never
env:
- name: FAKE_WORKER_HTTP2
value: "1"
- name: MODEL_ID
value: "tiny"
ports:
- containerPort: 30000
readinessProbe:
httpGet:
path: /health
port: 30000
initialDelaySeconds: 2
periodSeconds: 3
---
apiVersion: v1
kind: Service
metadata:
name: {H2C_DEPLOYMENT}
namespace: {NAMESPACE}
# The router watches ENDPOINTSLICES whose labels match `--selector
# app=sglang`, and Kubernetes mirrors a Service's labels onto the slices it
# manages -- so this label, not the pods', is what puts these workers in the
# router's view.
labels:
app: sglang
spec:
# Pods are labelled `app: sglang-h2c`, deliberately NOT `app: sglang`: the
# fake-worker Deployment's selector is a bare `app=sglang`, so sharing that
# label would put these pods inside another controller's selector and into
# the HTTP/1.1 Service as well.
selector:
app: sglang-h2c
ports:
- port: 30000
targetPort: 30000
"""
@pytest.fixture(scope="module")
def h2c_worker(k8s_cluster):
"""Add one Granian/h2c worker, behind its own Service, to the router's view.
Its own Service rather than the existing one: the router selects
EndpointSlices, so a second Service labelled `app: sglang` is watched just
the same, while its pods stay out of the `fake-worker` Deployment's bare
`app=sglang` selector. Torn down afterwards so the suite's other modules
see the three-worker fleet they expect.
"""
_apply_from_stdin(_H2C_WORKER_MANIFEST)
try:
_wait_for_deployment_ready(H2C_DEPLOYMENT)
yield
finally:
for kind in ("deployment", "service"):
_kubectl(
"delete",
kind,
H2C_DEPLOYMENT,
"-n",
NAMESPACE,
"--ignore-not-found",
"--wait=true",
check=False,
)
def _chat(router_url: str, content: str) -> httpx.Response:
return httpx.post(
f"{router_url}/v1/chat/completions",
json={
"model": "tiny",
"messages": [{"role": "user", "content": content}],
"stream": False,
},
timeout=_CHAT_TIMEOUT,
)
def _observed_protocols(router_url: str, *, strict: bool) -> set[str]:
"""Fan out round-robin and collect the HTTP version each worker saw.
`x_http_version` is reported by the worker itself, not inferred from the
client side: the test's own connection to the router is a separate hop, so
only the worker can say what the forward leg used. Distinct content per
request keeps any content-derived routing from collapsing onto one worker.
`strict=False` while converging. The router runs `--cb-threshold 1`
(manifests/router.yaml), so one refused connection to the still-starting h2c
pod opens its breaker and round-robin hands back a 502 for that turn. That
is precisely what the poll is meant to wait out — and `conftest._poll_until`
retries only transport-level errors, so an `AssertionError` raised here
would escape the retry budget and fail the test on the first blip. Skip
non-200s while converging; assert on them once converged.
"""
seen: set[str] = set()
for i in range(_PROBE_REQUESTS):
r = _chat(router_url, f"h2c-probe-{i}")
if r.status_code != 200:
if strict:
raise AssertionError(f"request {i} failed {r.status_code}: {r.text}")
continue
version = r.json().get("x_http_version")
# Fatal either way: this is a stale fake-worker image or a router that
# stopped returning the upstream body verbatim, neither of which a retry
# fixes, and without it the test cannot tell h2c from HTTP/1.1 at all.
assert version is not None, (
"worker did not report `x_http_version` — the fake-worker image is "
"stale, or the router stopped returning the upstream body verbatim; "
"either way this test cannot tell h2c from HTTP/1.1"
)
seen.add(version)
logger.info("protocols observed across %d requests: %s", _PROBE_REQUESTS, seen)
return seen
def test_router_forwards_over_h2c_to_an_http2_worker(router_url, h2c_worker):
"""A worker advertising `enable_http2` is reached over HTTP/2, and the
HTTP/1.1 workers alongside it keep their own protocol."""
# The router must first see the new pod's EndpointSlice entry and
# introspect it; until then every response comes back "1.1".
_poll_until(
lambda: "2" in _observed_protocols(router_url, strict=False),
"router forwards to the h2c worker over HTTP/2",
timeout=_CONVERGE_TIMEOUT,
interval=5,
)
seen = _observed_protocols(router_url, strict=True)
assert "2" in seen, f"expected an HTTP/2 forward, saw {seen}"
assert "1.1" in seen, (
f"expected the three uvicorn workers to stay on HTTP/1.1, saw {seen}; "
"a fleet-wide protocol would have taken them with it"
)
experimental/sgl-router/tests/proxy/chat_routing.rs
+7 -1
View File
@@ -12,7 +12,7 @@ use sgl_router::server::app::build_router;
use sgl_router::server::app_context::AppContext;
use sgl_router::server::routes::chat::MAX_CHAT_BODY_BYTES;
use sgl_router::tokenizer::TokenizerRegistry;
use sgl_router::workers::{Worker, WorkerRegistry};
use sgl_router::workers::{WireProtocol, Worker, WorkerRegistry};
use axum::body::Body;
use axum::http::{Request, StatusCode};
@@ -839,6 +839,7 @@ async fn forward_json_to_records_failure_on_body_drop() {
let res: Result<_, ApiError> = proxy
.forward_json_to(
&worker.url,
WireProtocol::Http1,
&breaker,
"/v1/chat/completions",
&headers,
@@ -895,6 +896,7 @@ async fn forward_json_to_records_success_only_after_body_completes() {
let res: Result<_, ApiError> = proxy
.forward_json_to(
&ok_worker.url,
WireProtocol::Http1,
&breaker,
"/v1/chat/completions",
&headers,
@@ -945,6 +947,7 @@ async fn forward_streaming_to_records_failure_on_mid_stream_drop() {
let res: Result<_, ApiError> = proxy
.forward_streaming_to(
&worker.url,
WireProtocol::Http1,
&breaker,
"/v1/chat/completions",
&headers,
@@ -995,6 +998,7 @@ async fn forward_json_to_records_failure_on_5xx() {
let _: Result<_, ApiError> = proxy
.forward_json_to(
&worker.url,
WireProtocol::Http1,
&breaker,
"/v1/chat/completions",
&headers,
@@ -1028,6 +1032,7 @@ async fn forward_json_to_rejects_when_breaker_open() {
let res = proxy
.forward_json_to(
&worker.url,
WireProtocol::Http1,
&breaker,
"/v1/chat/completions",
&headers,
@@ -1065,6 +1070,7 @@ async fn forward_json_to_malformed_url_returns_worker_misconfigured_and_trips_br
let res = proxy
.forward_json_to(
"not-a-url",
WireProtocol::Http1,
&breaker,
"/v1/chat/completions",
&headers,
experimental/sgl-router/tests/proxy/h2c_forward.rs
+195
View File
@@ -0,0 +1,195 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors
// SPDX-License-Identifier: Apache-2.0
//! Cleartext-HTTP/2 (h2c) forwarding tests for [`Proxy`].
//!
//! These spin up an **HTTP/2-only** server (hyper's `http2::Builder`, no
//! HTTP/1.1 path) and drive `forward_json_to` against it with an explicit
//! per-request [`WireProtocol`]. Together the two tests prove what a value-only
//! unit test cannot: that the proxy's h2c client genuinely speaks HTTP/2 on the
//! wire when a request selects [`WireProtocol::H2c`] (not just that the value
//! was threaded through), and that the HTTP/1.1 client cannot reach an h2c
//! worker — so a regression that dropped the `http2` Cargo feature, removed
//! `http2_prior_knowledge()`, or mismatched `build_client`'s arms would fail
//! here rather than slip through. They also pin the per-worker design: the
//! protocol is chosen per `forward_json_to` call, not committed fleet-wide.
use std::convert::Infallible;
use std::sync::Arc;
use std::time::Duration;
use bytes::Bytes;
use http_body_util::{BodyExt, Full, StreamBody};
use hyper::body::Frame;
use hyper::server::conn::http2;
use hyper::service::service_fn;
use hyper::{Request, Response};
use hyper_util::rt::{TokioExecutor, TokioIo};
use sgl_router::health::circuit_breaker::CircuitBreaker;
use sgl_router::proxy::Proxy;
use sgl_router::workers::WireProtocol;
use tokio::net::TcpListener;
/// Serve HTTP/2 only. `http2::Builder::serve_connection` speaks the HTTP/2
/// framing protocol with no HTTP/1.1 fallback, so a client that does not
/// send the HTTP/2 connection preface cannot complete a request. Returns the
/// base URL; the accept loop is aborted when the test runtime shuts down.
async fn spawn_h2c_only_server() -> String {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move {
while let Ok((stream, _)) = listener.accept().await {
tokio::spawn(async move {
let _ = http2::Builder::new(TokioExecutor::new())
.serve_connection(
TokioIo::new(stream),
service_fn(|_req: Request<hyper::body::Incoming>| async {
Ok::<_, Infallible>(Response::new(Full::new(Bytes::from_static(
b"{\"ok\":true}",
))))
}),
)
.await;
});
}
});
format!("http://{addr}")
}
#[tokio::test]
async fn h2c_client_reaches_http2_only_worker() {
let url = spawn_h2c_only_server().await;
let proxy = Proxy::new(Duration::from_secs(5)).unwrap();
let breaker = CircuitBreaker::new();
// Select h2c per request, as the chat handler does for a worker whose
// `/server_info` reported --enable-http2 on a cleartext URL.
let resp = proxy
.forward_json_to(
&url,
WireProtocol::H2c,
&breaker,
"/v1/chat/completions",
&axum::http::HeaderMap::new(),
Bytes::from_static(b"{}"),
)
.await
.expect("h2c client must reach an HTTP/2-only worker");
assert_eq!(resp.status(), 200);
}
#[tokio::test]
async fn http1_client_cannot_reach_http2_only_worker() {
let url = spawn_h2c_only_server().await;
let proxy = Proxy::new(Duration::from_secs(5)).unwrap();
let breaker = CircuitBreaker::new();
// The HTTP/1.1 client never sends the HTTP/2 preface, so the h2c-only
// server cannot serve it. This is what makes selecting the protocol
// meaningful: Http1 and H2c are different protocols on the wire, not the
// same client pointed at the same endpoint.
let res = proxy
.forward_json_to(
&url,
WireProtocol::Http1,
&breaker,
"/v1/chat/completions",
&axum::http::HeaderMap::new(),
Bytes::from_static(b"{}"),
)
.await;
assert!(
res.is_err(),
"HTTP/1.1 client must not complete a request against an HTTP/2-only worker, got {res:?}",
);
}
/// Serve HTTP/2 only, answering with a multi-frame SSE body. Each `data:`
/// chunk is its own HTTP/2 DATA frame, which is the framing the real engine
/// produces for a streaming generation.
async fn spawn_h2c_only_sse_server() -> String {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move {
while let Ok((stream, _)) = listener.accept().await {
tokio::spawn(async move {
let _ = http2::Builder::new(TokioExecutor::new())
.serve_connection(
TokioIo::new(stream),
service_fn(|_req: Request<hyper::body::Incoming>| async {
let chunks: Vec<Result<Frame<Bytes>, Infallible>> = vec![
Ok(Frame::data(Bytes::from_static(b"data: {\"i\":0}\n\n"))),
Ok(Frame::data(Bytes::from_static(b"data: {\"i\":1}\n\n"))),
Ok(Frame::data(Bytes::from_static(b"data: [DONE]\n\n"))),
];
let body = StreamBody::new(futures::stream::iter(chunks));
Ok::<_, Infallible>(
Response::builder()
.header("content-type", "text/event-stream")
.body(body)
.unwrap(),
)
}),
)
.await;
});
}
});
format!("http://{addr}")
}
/// The streaming forward — the path every chat generation actually takes — must
/// work over h2c, and must deliver the whole multi-frame body.
///
/// `forward_streaming_to` differs from `forward_json_to` in ways HTTP/2 reaches
/// differently: it omits the request timeout, and it hands `bytes_stream()` to
/// the SSE pump rather than buffering. A break confined to SSE-over-h2c — a
/// truncated body, a stall, a mid-stream reset surfacing as success — leaves
/// the buffered tests green, so cover it directly.
#[tokio::test]
async fn h2c_client_streams_sse_from_http2_only_worker() {
let url = spawn_h2c_only_sse_server().await;
let proxy = Proxy::new(Duration::from_secs(5)).unwrap();
let breaker = Arc::new(CircuitBreaker::new());
let first_byte_seen = Arc::new(std::sync::atomic::AtomicBool::new(false));
let flag = Arc::clone(&first_byte_seen);
let resp = proxy
.forward_streaming_to(
&url,
WireProtocol::H2c,
&breaker,
"/v1/chat/completions",
&axum::http::HeaderMap::new(),
Bytes::from_static(b"{}"),
None,
Some(Box::new(move || {
flag.store(true, std::sync::atomic::Ordering::SeqCst);
})),
None,
)
.await
.expect("h2c client must stream from an HTTP/2-only worker");
assert_eq!(resp.status(), 200);
let body = resp
.into_body()
.collect()
.await
.expect("streaming body must complete over h2c")
.to_bytes();
let text = String::from_utf8(body.to_vec()).unwrap();
// Every frame arrives, in order — not just the first.
assert!(text.contains("\"i\":0"), "missing first chunk: {text}");
assert!(text.contains("\"i\":1"), "missing second chunk: {text}");
assert!(
text.contains("[DONE]"),
"stream truncated before DONE: {text}"
);
assert!(
first_byte_seen.load(std::sync::atomic::Ordering::SeqCst),
"on_first_byte must fire for an h2c stream (TTFT accounting depends on it)",
);
}
experimental/sgl-router/tests/proxy/inbound_h2c.rs
+116
View File
@@ -0,0 +1,116 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors
// SPDX-License-Identifier: Apache-2.0
//! Inbound cleartext-HTTP/2 (h2c) listener tests for the router's own server.
//!
//! Where `h2c_forward.rs` proves the router's *outbound* client speaks h2c to
//! an HTTP/2-only worker, these prove the *inbound* side: the router's
//! `axum::serve` listener accepts an h2c prior-knowledge client on the same
//! cleartext port it serves HTTP/1.1 on, auto-negotiating per connection.
//!
//! Protocol negotiation happens at the connection layer, below the tower
//! service — so unlike the `oneshot` tests in `chat_routing.rs`, these must
//! drive a real socket via `axum::serve` (mirroring `main.rs`).
//!
//! What actually compiles the listener's h2 path is `hyper-util/http2`, reached
//! through `server-auto`, and several dependencies enable it. So these tests do
//! NOT fail if axum's own `http2` feature is dropped — verified by removing it.
//! Their job is the property an operator depends on, whatever the feature graph
//! happens to look like: one cleartext port serving both h2c prior-knowledge
//! and HTTP/1.1. That is what would break if a dependency change silently took
//! `hyper-util/http2` away.
use std::sync::Arc;
use std::time::Duration;
use sgl_router::config::PolicyKind;
use sgl_router::policies::factory::build_registry_with_defaults as build_policy_registry;
use sgl_router::proxy::Proxy;
use sgl_router::server::app::build_router;
use sgl_router::server::app_context::AppContext;
use sgl_router::tokenizer::TokenizerRegistry;
use sgl_router::workers::WorkerRegistry;
use axum::http::Version;
const TEST_TIMEOUT: Duration = Duration::from_secs(5);
fn build_ctx() -> Arc<AppContext> {
// Reuse the shared fixture rather than restating the whole `Config`: this
// test is about the listener, not the routing policy, and a local literal
// would be one more site to edit every time `ModelConfig` gains a field.
let mut cfg = crate::common::cache_aware_fixture::config();
cfg.model.policy = PolicyKind::RoundRobin;
cfg.model.cache_aware = None;
let tokenizers = Arc::new(TokenizerRegistry::load_from_config(&cfg).unwrap());
let registry = Arc::new(WorkerRegistry::default());
let policies = Arc::new(build_policy_registry(&cfg).unwrap());
let proxy = Arc::new(Proxy::new(TEST_TIMEOUT).unwrap());
Arc::new(AppContext::new(cfg, tokenizers, proxy, registry, policies))
}
/// Spawn the real router (`build_router`) behind `axum::serve` on an ephemeral
/// port — the exact serve path used in `main.rs`. `/healthz` returns 200
/// unconditionally, so no worker is needed. Returns the base URL; the accept
/// loop is dropped when the test runtime shuts down.
async fn spawn_router() -> String {
let ctx = build_ctx();
ctx.mark_ready();
let app = build_router(ctx);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move {
let _ = axum::serve(listener, app).await;
});
format!("http://{addr}")
}
#[tokio::test]
async fn inbound_accepts_h2c_prior_knowledge() {
let base = spawn_router().await;
// `http2_prior_knowledge()` sends the HTTP/2 connection preface directly
// over cleartext (no ALPN, no h1 upgrade) — the same way a service-mesh
// sidecar dials h2c. The listener must recognize the preface and serve h2.
let client = reqwest::Client::builder()
.http2_prior_knowledge()
.build()
.unwrap();
let resp = client
.get(format!("{base}/healthz"))
.send()
.await
.expect("h2c prior-knowledge client must reach the router listener");
assert_eq!(resp.status(), 200);
assert_eq!(
resp.version(),
Version::HTTP_2,
"listener must negotiate HTTP/2 for an h2c prior-knowledge client",
);
}
#[tokio::test]
async fn inbound_still_accepts_http1() {
let base = spawn_router().await;
// The default reqwest client speaks HTTP/1.1 over cleartext. Enabling h2c
// must not break existing HTTP/1.1 callers (load balancers, probes, curl)
// — the auto builder serves both on the same port.
let client = reqwest::Client::new();
let resp = client
.get(format!("{base}/healthz"))
.send()
.await
.expect("HTTP/1.1 client must still reach the router listener");
assert_eq!(resp.status(), 200);
assert_eq!(
resp.version(),
Version::HTTP_11,
"an HTTP/1.1 client must keep negotiating HTTP/1.1 on the same port",
);
}
experimental/sgl-router/tests/proxy/main.rs
+3
View File
@@ -16,9 +16,12 @@ mod chat_routing;
mod external_indexer_routing;
mod failover;
mod graceful_shutdown;
mod h2c_forward;
mod header_forwarding;
mod inbound_h2c;
mod pd_bootstrap_injection;
mod pd_pool_isolation;
mod pd_protocol_binding;
mod radix_tree_routing;
mod roundrobin_input_ids;
mod shared_prefill_admission;
experimental/sgl-router/tests/proxy/pd_protocol_binding.rs
+260
View File
@@ -0,0 +1,260 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors
// SPDX-License-Identifier: Apache-2.0
//! Binds protocol *resolution* to protocol *use*, across the PD split.
//!
//! Everywhere else the two halves are tested apart: the manager tests assert
//! what lands on the registry, and `h2c_forward.rs` passes a `WireProtocol` to
//! the proxy by hand. Nothing asserts that the protocol a worker resolved to is
//! the one its own forward actually uses — and in the PD arm of
//! `chat_completions` that join is three separate expressions, a
//! `prefill_protocol` captured before a `tokio::spawn` plus two live
//! `decode_worker.protocol()` reads. Passing the wrong one of those compiles,
//! and every other test in the suite stays green.
//!
//! So make the two workers disagree and let the wire enforce it: the prefill
//! mock speaks **HTTP/2 only**, the decode mock speaks **HTTP/1.1 only**, and
//! each is registered with the matching protocol. Any mix-up sends a client at
//! a server that cannot answer it. This is also why neither mock can be the
//! shared `MockWorker` — that one runs on `axum::serve`, which answers both
//! protocols and would pass no matter which was selected.
use std::convert::Infallible;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use axum::body::Body;
use axum::http::{Request, StatusCode};
use bytes::Bytes;
use http_body_util::{BodyExt, Full};
use hyper::server::conn::{http1, http2};
use hyper::service::service_fn;
use hyper::Response as HyperResponse;
use hyper_util::rt::{TokioExecutor, TokioIo};
use sgl_router::config::{
ActiveLoadConfig, Config, DiscoveryBackend, ModelConfig, ObservabilityConfig, PolicyKind,
ProxyConfig, ServerConfig, StaticUrlsDiscoveryConfig,
};
use sgl_router::discovery::{ModelId, WorkerId, WorkerMode, WorkerSpec};
use sgl_router::policies::factory::build_registry_with_defaults;
use sgl_router::proxy::Proxy;
use sgl_router::server::app::build_router;
use sgl_router::server::app_context::AppContext;
use sgl_router::tokenizer::TokenizerRegistry;
use sgl_router::workers::{WireProtocol, WorkerRegistry};
use tokio::net::TcpListener;
use tower::ServiceExt;
/// A chat-completions response shaped enough for the handler to return 200.
fn chat_completion_json() -> Bytes {
Bytes::from(
serde_json::json!({
"id": "chatcmpl-test",
"object": "chat.completion",
"created": 0,
"model": "tiny",
"choices": [{
"index": 0,
"message": {"role": "assistant", "content": "ok"},
"finish_reason": "stop"
}],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}
})
.to_string(),
)
}
/// Serve HTTP/2 only, recording that a request body arrived. An HTTP/1.1
/// client cannot complete a request here — it never sends the HTTP/2 preface.
async fn spawn_h2_only_worker(hits: Arc<Mutex<usize>>) -> String {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move {
while let Ok((stream, _)) = listener.accept().await {
let hits = Arc::clone(&hits);
tokio::spawn(async move {
let _ = http2::Builder::new(TokioExecutor::new())
.serve_connection(
TokioIo::new(stream),
service_fn(move |_req| {
let hits = Arc::clone(&hits);
async move {
*hits.lock().unwrap() += 1;
Ok::<_, Infallible>(HyperResponse::new(Full::new(
chat_completion_json(),
)))
}
}),
)
.await;
});
}
});
format!("http://{addr}")
}
/// Serve HTTP/1.1 only. `http1::Builder` speaks no HTTP/2, so an h2c client
/// sending the preface cannot be served here.
async fn spawn_http1_only_worker(hits: Arc<Mutex<usize>>) -> String {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move {
while let Ok((stream, _)) = listener.accept().await {
let hits = Arc::clone(&hits);
tokio::spawn(async move {
let _ = http1::Builder::new()
.serve_connection(
TokioIo::new(stream),
service_fn(move |_req| {
let hits = Arc::clone(&hits);
async move {
*hits.lock().unwrap() += 1;
Ok::<_, Infallible>(HyperResponse::new(Full::new(
chat_completion_json(),
)))
}
}),
)
.await;
});
}
});
format!("http://{addr}")
}
fn config() -> Config {
Config {
server: ServerConfig {
host: "0".into(),
port: 0,
},
observability: ObservabilityConfig::default(),
model: ModelConfig {
id: "tiny".into(),
tokenizer_path: "tests/fixtures/tiny_tokenizer.json".into(),
policy: PolicyKind::RoundRobin,
decode_policy: Default::default(),
bucket_config: None,
circuit_breaker: None,
cache_aware: None,
sticky: None,
affinity: None,
fused: None,
eligibility: None,
},
discovery: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig {
urls: vec!["http://placeholder:0".into()],
}),
proxy: ProxyConfig::default(),
active_load: ActiveLoadConfig::default(),
}
}
/// Register both workers with their own resolved protocol, the way
/// `manager::register_one` does after introspecting `/server_info`.
fn build_ctx(prefill_url: String, decode_url: String) -> Arc<AppContext> {
let cfg = config();
let tokenizers = Arc::new(TokenizerRegistry::load_from_config(&cfg).unwrap());
let registry = Arc::new(WorkerRegistry::default());
let prefill_id = WorkerId("p1".into());
let decode_id = WorkerId("d1".into());
// The two disagree on purpose. This is the state a mixed fleet reaches
// when only some engines run with --enable-http2.
registry
.add_with_cb(
WorkerSpec {
id: prefill_id.clone(),
url: prefill_url,
mode: WorkerMode::Prefill,
model_ids: vec![ModelId("tiny".into())],
bootstrap_port: Some(8997),
},
None,
WireProtocol::H2c,
)
.unwrap();
registry
.add_with_cb(
WorkerSpec {
id: decode_id.clone(),
url: decode_url,
mode: WorkerMode::Decode,
model_ids: vec![ModelId("tiny".into())],
bootstrap_port: None,
},
None,
WireProtocol::Http1,
)
.unwrap();
let policies = Arc::new(build_registry_with_defaults(&cfg).unwrap());
let proxy = Arc::new(Proxy::new(Duration::from_secs(5)).unwrap());
Arc::new(AppContext::new(cfg, tokenizers, proxy, registry, policies))
}
fn chat_request() -> Request<Body> {
Request::builder()
.method("POST")
.uri("/v1/chat/completions")
.header("content-type", "application/json")
.body(Body::from(
serde_json::to_vec(&serde_json::json!({
"model": "tiny",
"messages": [{"role": "user", "content": "hi"}],
"stream": false
}))
.unwrap(),
))
.unwrap()
}
/// Each PD leg must forward over the protocol *its own* worker resolved.
///
/// The decode leg is awaited, so a protocol mix-up there fails the response
/// outright. The prefill leg is detached, so its mix-up shows up as a body that
/// never arrives — poll for it rather than reading once.
#[tokio::test]
async fn pd_legs_each_use_their_own_workers_protocol() {
let prefill_hits = Arc::new(Mutex::new(0usize));
let decode_hits = Arc::new(Mutex::new(0usize));
let prefill_url = spawn_h2_only_worker(Arc::clone(&prefill_hits)).await;
let decode_url = spawn_http1_only_worker(Arc::clone(&decode_hits)).await;
let app = build_router(build_ctx(prefill_url, decode_url));
let res = app.oneshot(chat_request()).await.unwrap();
// Decode answered, so the decode leg used HTTP/1.1 — had it been handed the
// prefill worker's H2c, this HTTP/1.1-only server could not have replied.
assert_eq!(
res.status(),
StatusCode::OK,
"decode leg must forward over the decode worker's own protocol (HTTP/1.1)",
);
let body = res.into_body().collect().await.unwrap().to_bytes();
assert!(
!body.is_empty(),
"decode response body must reach the client",
);
// Prefill is spawn-and-forget; it races the response back to the client.
let reached = tokio::time::timeout(Duration::from_secs(3), async {
loop {
if *prefill_hits.lock().unwrap() > 0 {
return true;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await;
assert!(
reached.is_ok(),
"prefill leg must forward over the prefill worker's own protocol (h2c); \
an HTTP/1.1 client cannot reach this HTTP/2-only worker",
);
assert_eq!(
*decode_hits.lock().unwrap(),
1,
"decode worker should be hit exactly once",
);
}