minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[router] Configure experimental sgl-router via CLI flags instead of a config file (#27073)

Browse Source 
Signed-off-by: Kangyan Zhou <zky314343421@gmail.com>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Kangyan-Zhou
2026-06-05 10:02:10 +08:00
committed by GitHub
co-authored by Claude Opus 4.8
parent 631db6c757
commit bcf89928b4
43 changed files with 2355 additions and 1237 deletions
Download Patch File Download Diff File Expand all files Collapse all files
experimental/sgl-router/tests/e2e/conftest.py
+34 -51
View File
@@ -166,81 +166,64 @@ def _find_tokenizer_path(model: str) -> str:
return model
def build_smoke_router_config(
def build_smoke_router_args(
*,
host: str,
port: int,
model: str,
tokenizer_path: str,
sglang_url: str,
) -> str:
"""Build the TOML the smoke `router` fixture writes to disk.
) -> list[str]:
"""Build the sgl-router CLI flags the smoke ``router`` fixture launches.
Returns ``main_config_text`` carrying ``[server]``, ``[[models]]``,
and ``[discovery] backend = "static_urls"`` with the worker URL
inline. The Rust ``Config`` struct requires a ``[discovery]``
section (``DiscoveryConfig`` has no ``#[serde(default)]``) and has
no top-level ``workers`` field. The previous ``static_file``
backend was replaced by ``static_urls`` (which holds the URL list
inline rather than via a side-car file).
Static single-worker discovery (``--worker-urls``) pointed at the one
SGLang worker, serving exactly one model.
"""
return f"""\
[server]
host = "{host}"
port = {port}
[[models]]
id = "{model}"
tokenizer_path = "{tokenizer_path}"
[discovery]
backend = "static_urls"
[discovery.static_urls]
urls = ["{sglang_url}"]
"""
return [
"--host",
host,
"--port",
str(port),
"--model-id",
model,
"--tokenizer-path",
tokenizer_path,
"--worker-urls",
sglang_url,
]
@pytest.fixture(scope="session")
def router(sglang_server): # noqa: ARG001 (sglang_server must start first)
"""Launch sgl-router on port 8090 pointed at the SGLang worker."""
tok_path = _find_tokenizer_path(MODEL)
cfg_handle = tempfile.NamedTemporaryFile(mode="w", suffix=".toml", delete=False)
cfg_path = Path(cfg_handle.name)
main_text = build_smoke_router_config(
args = build_smoke_router_args(
host="0.0.0.0",
port=ROUTER_PORT,
model=MODEL,
tokenizer_path=tok_path,
sglang_url=f"http://localhost:{SGLANG_PORT}",
)
cfg_handle.write(main_text)
cfg_handle.close()
proc = subprocess.Popen(
[str(_BINARY), *args],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
# try/finally so the router is always reaped — on a readiness-probe
# failure, a test-body error, or a session-teardown exception alike.
try:
proc = subprocess.Popen(
[str(_BINARY), "--config", str(cfg_path)],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
try:
_wait_http(f"http://localhost:{ROUTER_PORT}/readyz", timeout=60)
except Exception:
proc.send_signal(signal.SIGTERM)
proc.wait(timeout=30)
raise
_wait_http(f"http://localhost:{ROUTER_PORT}/readyz", timeout=60)
yield f"http://localhost:{ROUTER_PORT}"
proc.send_signal(signal.SIGTERM)
try:
proc.wait(timeout=30)
except subprocess.TimeoutExpired:
proc.kill()
proc.wait()
finally:
cfg_path.unlink(missing_ok=True)
if proc.poll() is None:
proc.send_signal(signal.SIGTERM)
try:
proc.wait(timeout=30)
except subprocess.TimeoutExpired:
proc.kill()
proc.wait()
# ---------------------------------------------------------------------------
experimental/sgl-router/tests/e2e/infra/gateway.py
+54 -81
View File
@@ -1,23 +1,22 @@
"""Minimal sgl-router Gateway class — adapted from SMG's e2e_test/infra/gateway.py.
"""Minimal sgl-router Gateway class for e2e tests.
Differences from SMG:
- SMG drives a Python launcher (`python3 -m sglang_router.launch_router`)
with worker URLs on the CLI.
- sgl-router uses a Rust binary (`experimental/sgl-router/target/release/sgl-router`)
with a TOML config file. Worker discovery is config-file-based; this
Gateway writes a TOML to a tempfile and execs the binary with
`--config <tempfile>`.
sgl-router is a Rust binary
(`experimental/sgl-router/target/release/sgl-router`) configured entirely
through CLI flags. This Gateway execs the binary with `--worker-urls <...>`
(static discovery) plus the model + policy flags.
Supported lifecycles:
- Regular mode: one model, N worker URLs, single policy.
- PD mode: one model, prefill_workers + decode_workers (lists of URLs),
discovery emits separate `WorkerMode::Prefill` / `WorkerMode::Decode`
entries. The router resolves PD pool isolation at request time.
- PD mode: one model; prefill + decode URLs all go into one
`--worker-urls` static list. Each worker is seeded as
`WorkerMode::Plain` and its actual prefill/decode role + bootstrap
port are resolved from `/server_info` introspection, after which the
router isolates the PD pools at request time.
Use as a context manager:
with Gateway() as gw:
gw.start_regular(model_path="...", worker_urls=[...])
gw.start_regular(model_id="...", tokenizer_path="...", worker_urls=[...])
resp = httpx.post(f"{gw.base_url}/v1/chat/completions", json=...)
or pytest fixture style (see e2e_test/conftest.py).
@@ -30,7 +29,6 @@ import os
import signal
import socket
import subprocess
import tempfile
import time
from dataclasses import dataclass, field
from pathlib import Path
@@ -93,8 +91,11 @@ def _resolve_tokenizer_path(tokenizer_path: str) -> str:
cached = try_to_load_from_cache(tokenizer_path, "tokenizer.json")
if cached and Path(cached).is_file():
return str(cached)
except Exception: # noqa: BLE001
pass
except Exception as exc: # noqa: BLE001
# A cache miss is normal; log other failures (corrupt cache,
# signature change) so a later tokenizer-load error is traceable
# rather than mysterious.
logger.debug("HF tokenizer cache lookup failed for %r: %s", tokenizer_path, exc)
return tokenizer_path
@@ -150,7 +151,6 @@ class Gateway:
self.stale_request_timeout_secs = stale_request_timeout_secs
self.process: subprocess.Popen | None = None
self._config_path: Path | None = None
self._started: bool = False
# Track child workers we spawned so __exit__ can tear them down.
self._owned_workers: list[subprocess.Popen] = []
@@ -172,7 +172,6 @@ class Gateway:
tokenizer_path: str,
worker_urls: list[str],
policy: str = "round_robin",
extra_models: list[dict] | None = None,
timeout: float = 60.0,
) -> None:
"""Start the router in regular (non-PD) mode.
@@ -190,12 +189,11 @@ class Gateway:
timeout: How long to wait for ``/readyz`` before giving up.
"""
self._launch(
self._build_config(
self._build_args(
model_id=model_id,
tokenizer_path=tokenizer_path,
urls=list(worker_urls),
policy=policy,
extra_models=extra_models or [],
),
timeout=timeout,
)
@@ -222,12 +220,11 @@ class Gateway:
assumed.
"""
self._launch(
self._build_config(
self._build_args(
model_id=model_id,
tokenizer_path=tokenizer_path,
urls=list(prefill_urls) + list(decode_urls),
policy=policy,
extra_models=[],
),
timeout=timeout,
)
@@ -247,9 +244,6 @@ class Gateway:
except ProcessLookupError:
pass
self.process = None
if self._config_path and self._config_path.exists():
self._config_path.unlink(missing_ok=True)
self._config_path = None
self._started = False
# Tear down any owned upstream workers.
for w in self._owned_workers:
@@ -292,77 +286,53 @@ class Gateway:
# ----- internals ------------------------------------------------------
def _build_config(
def _build_args(
self,
*,
model_id: str,
tokenizer_path: str,
urls: list[str],
policy: str,
extra_models: list[dict],
) -> str:
) -> list[str]:
resolved_tokenizer = _resolve_tokenizer_path(tokenizer_path)
extra_model_toml = ""
for em in extra_models:
extra_model_toml += (
f'\n[[models]]\nid = "{em["id"]}"\n'
f'tokenizer_path = "{_resolve_tokenizer_path(em["tokenizer_path"])}"\n'
f'policy = "{em.get("policy", policy)}"\n'
)
# Optional tunables — only emit the [proxy] and [active_load]
# sections if a test has overridden them, so production defaults
# apply otherwise.
proxy_section = ""
args = [
"--host",
self.host,
"--port",
str(self.port),
"--model-id",
model_id,
"--tokenizer-path",
resolved_tokenizer,
"--policy",
policy,
]
# Optional tunables — only pass them if a test overrode them, so
# the router's production defaults apply otherwise.
if self.proxy_request_timeout_secs is not None:
proxy_section = (
f"\n[proxy]\nrequest_timeout_secs = {self.proxy_request_timeout_secs}\n"
)
active_load_section = ""
args += ["--request-timeout-secs", str(self.proxy_request_timeout_secs)]
if self.stale_request_timeout_secs is not None:
active_load_section = (
f"\n[active_load]\nstale_request_timeout_secs = "
f"{self.stale_request_timeout_secs}\n"
)
args += [
"--stale-request-timeout-secs",
str(self.stale_request_timeout_secs),
]
# `--worker-urls` is multi-valued; keep it last so clap doesn't
# absorb a following flag as a URL.
args += ["--worker-urls", *urls]
return args
urls_toml = ", ".join(f'"{u}"' for u in urls)
return f"""\
[server]
host = "{self.host}"
port = {self.port}
[[models]]
id = "{model_id}"
tokenizer_path = "{resolved_tokenizer}"
policy = "{policy}"
{extra_model_toml}
[discovery]
backend = "static_urls"
[discovery.static_urls]
urls = [{urls_toml}]
{proxy_section}{active_load_section}"""
def _launch(self, config_text: str, *, timeout: float) -> None:
def _launch(self, args: list[str], *, timeout: float) -> None:
if not self.binary.exists():
raise RuntimeError(
f"sgl-router binary not found at {self.binary}. "
"Build it first: `cd experimental/sgl-router && cargo build --release` "
"or set SGL_ROUTER_BINARY to the binary path."
)
# Write the main config.
fd, path = tempfile.mkstemp(suffix=".toml", prefix="sgl-router-")
os.close(fd)
self._config_path = Path(path)
self._config_path.write_text(config_text, encoding="utf-8")
logger.info("sgl-router config: %s", self._config_path)
logger.debug("sgl-router config text:\n%s", config_text)
logger.info("sgl-router args: %s", args)
self.process = subprocess.Popen(
[str(self.binary), "--config", str(self._config_path)],
[str(self.binary), *args],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
start_new_session=True,
@@ -380,16 +350,19 @@ urls = [{urls_toml}]
last_exc: Exception | None = None
while time.time() < deadline:
if self.process is not None and self.process.poll() is not None:
# Process exited early — surface stdout/stderr.
out = b""
# Process exited early — surface stdout/stderr. This is the
# primary startup-failure diagnostic, so if the read itself
# fails, report that instead of blanking the output.
try:
out = b""
if self.process.stdout is not None:
out = self.process.stdout.read() or b""
except Exception: # noqa: BLE001
pass
output = out.decode(errors="replace")
except Exception as read_exc: # noqa: BLE001
output = f"<failed to read router stdout: {read_exc}>"
raise RuntimeError(
f"sgl-router exited during startup with code "
f"{self.process.returncode}. output:\n{out.decode(errors='replace')}",
f"{self.process.returncode}. output:\n{output}",
)
try:
resp = httpx.get(f"{self.base_url}/readyz", timeout=2.0)
experimental/sgl-router/tests/e2e/k8s_integration/manifests/router-cluster-scoped.yaml
+16 -9
View File
@@ -20,9 +20,23 @@ spec:
- name: router
image: sgl-router:e2e
imagePullPolicy: Never
# Configured entirely via CLI flags. No --service-discovery-namespace
# means a cluster-wide EndpointSlice watch (all namespaces); the
# `cross-ns-test=true` selector term scopes it to this test's workers.
args:
- "--config"
- "/etc/config/router-cluster.toml"
- "--host"
- "0.0.0.0"
- "--port"
- "8091"
- "--model-id"
- "tiny"
- "--tokenizer-path"
- "/etc/tokenizer/tiny.json"
- "--policy"
- "round_robin"
- "--service-discovery"
- "--selector"
- "app=sglang,cross-ns-test=true"
ports:
- containerPort: 8091
name: http
@@ -38,13 +52,6 @@ spec:
port: 8091
initialDelaySeconds: 5
periodSeconds: 10
volumeMounts:
- name: config
mountPath: /etc/config
volumes:
- name: config
configMap:
name: sgl-router-cluster-config
---
apiVersion: v1
kind: Service
experimental/sgl-router/tests/e2e/k8s_integration/manifests/router.yaml
+25 -9
View File
@@ -18,9 +18,32 @@ spec:
- name: router
image: sgl-router:e2e
imagePullPolicy: Never
# Configured entirely via CLI flags. K8s EndpointSlice discovery
# watches `app=sglang` pods in this namespace. The aggressive
# circuit breaker (threshold 1, 5s cool-down) lets a terminating
# pod's connection-refused immediately drop it from the candidate
# set — the reconciliation tests scale workers rapidly and depend
# on fast eviction to absorb the churn.
args:
- "--config"
- "/etc/config/router.toml"
- "--host"
- "0.0.0.0"
- "--port"
- "8090"
- "--model-id"
- "tiny"
- "--tokenizer-path"
- "/etc/tokenizer/tiny.json"
- "--policy"
- "round_robin"
- "--cb-threshold"
- "1"
- "--cb-cool-down-secs"
- "5"
- "--service-discovery"
- "--service-discovery-namespace"
- "sgl-router-test"
- "--selector"
- "app=sglang"
ports:
- containerPort: 8090
name: http
@@ -36,13 +59,6 @@ spec:
port: 8090
initialDelaySeconds: 5
periodSeconds: 10
volumeMounts:
- name: config
mountPath: /etc/config
volumes:
- name: config
configMap:
name: sgl-router-config
---
apiVersion: v1
kind: Service
experimental/sgl-router/tests/e2e/k8s_integration/setup.sh
+3 -32
View File
@@ -141,38 +141,9 @@ log "Waiting for fake-worker rollout..."
kubectl --context "${CONTEXT}" -n "${NAMESPACE}" rollout status deployment/fake-worker --timeout=120s
# ---------------------------------------------------------------------------
# Step 6: Create sgl-router ConfigMap with k8s discovery pointing at the
# namespace where fake-worker pods live.
# ---------------------------------------------------------------------------
log "Creating sgl-router-config ConfigMap..."
ROUTER_CONFIG="[server]
host = \"0.0.0.0\"
port = 8090
[[models]]
id = \"tiny\"
tokenizer_path = \"/etc/tokenizer/tiny.json\"
policy = \"round_robin\"
# Aggressive breaker so a terminating pod's connection-refused
# immediately excludes it from the next request's candidate set —
# the reconciliation tests scale workers rapidly and depend on
# fast worker eviction to absorb the churn.
circuit_breaker = { threshold = 1, cool_down_secs = 5 }
[discovery]
backend = \"k8s\"
[discovery.k8s]
namespace = \"${NAMESPACE}\"
label_selector = \"app=sglang\""
kubectl --context "${CONTEXT}" -n "${NAMESPACE}" create configmap sgl-router-config \
--from-literal=router.toml="${ROUTER_CONFIG}" \
--dry-run=client -o yaml \
| kubectl --context "${CONTEXT}" apply -f -
# ---------------------------------------------------------------------------
# Step 7: Deploy sgl-router
# Step 6: Deploy sgl-router. It is configured entirely via CLI flags in
# router.yaml — k8s EndpointSlice discovery watches `app=sglang`
# pods in the sgl-router-test namespace (where fake-worker lives).
# ---------------------------------------------------------------------------
log "Deploying sgl-router..."
kubectl --context "${CONTEXT}" apply -f "${MANIFESTS_DIR}/router.yaml"
experimental/sgl-router/tests/e2e/k8s_integration/test_cross_namespace.py
+3 -43
View File
@@ -149,49 +149,9 @@ def cluster_scoped_router(k8s_cluster):
_ensure_namespace(EXTRA_NAMESPACE)
_ensure_service_in_ns(EXTRA_NAMESPACE)
# ConfigMap for the cluster-scoped router: empty namespace = watch all
cluster_config = """[server]
host = "0.0.0.0"
port = 8091
[[models]]
id = "tiny"
tokenizer_path = "/etc/tokenizer/tiny.json"
policy = "round_robin"
[discovery]
backend = "k8s"
[discovery.k8s]
namespace = ""
label_selector = "app=sglang,cross-ns-test=true"
"""
_kubectl(
"create",
"configmap",
"sgl-router-cluster-config",
f"--from-literal=router-cluster.toml={cluster_config}",
"-n",
NAMESPACE,
"--dry-run=client",
"-o",
"yaml",
check=True,
)
# pipe through apply
proc = _kubectl(
"create",
"configmap",
"sgl-router-cluster-config",
f"--from-literal=router-cluster.toml={cluster_config}",
"-n",
NAMESPACE,
"--dry-run=client",
"-o",
"yaml",
)
_apply_from_stdin(proc.stdout)
# The cluster-scoped router is configured via CLI flags in
# router-cluster-scoped.yaml: no --service-discovery-namespace (watch
# all namespaces) and --selector app=sglang,cross-ns-test=true.
_kubectl("apply", "-f", str(router_manifest))
# The cluster-scoped router's /readyz blocks on registry-not-empty, so