sgl-router: experimental Rust HTTP router for SGLang worker pools (#25851)
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Kangyan-Zhou
co-authored by
Claude Opus 4.7
Claude Opus 4.7
parent
aae04b1241
commit
6e8fe176be
@@ -0,0 +1,6 @@
|
||||
FROM python:3.12-slim
|
||||
WORKDIR /app
|
||||
RUN pip install --no-cache-dir fastapi uvicorn
|
||||
COPY fake_worker.py .
|
||||
EXPOSE 30000
|
||||
CMD ["python", "fake_worker.py"]
|
||||
@@ -0,0 +1,39 @@
|
||||
# syntax=docker/dockerfile:1.6
|
||||
# Build sgl-router binary for k8s integration E2E.
|
||||
# Context root: repo root (one level above experimental/sgl-router/).
|
||||
|
||||
# Matches rust-toolchain.toml's pinned channel, avoiding an in-build rustup channel-sync.
|
||||
FROM rust:1.90-bookworm AS builder
|
||||
|
||||
# Pin to the exact toolchain pre-installed in the base image so rustup
|
||||
# doesn't try to sync the channel manifest when it sees rust-toolchain.toml's
|
||||
# `channel = "1.90"`.
|
||||
ENV RUSTUP_TOOLCHAIN=1.90.0
|
||||
|
||||
# libssl-dev + pkg-config ship with rust:1.90-bookworm already; no apt-get needed.
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
# Copy just the sgl-router crate (context is the repo root)
|
||||
COPY experimental/sgl-router /build/experimental/sgl-router
|
||||
|
||||
RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||
--mount=type=cache,target=/usr/local/cargo/git \
|
||||
--mount=type=cache,target=/build/experimental/sgl-router/target \
|
||||
cd /build/experimental/sgl-router \
|
||||
&& cargo build --release --bin sgl-router \
|
||||
&& cp target/release/sgl-router /usr/local/bin/sgl-router
|
||||
|
||||
FROM debian:bookworm-slim
|
||||
|
||||
RUN apt-get update && apt-get install -y ca-certificates \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY --from=builder /usr/local/bin/sgl-router /usr/local/bin/sgl-router
|
||||
|
||||
# Tiny tokenizer fixture used by the E2E config
|
||||
COPY experimental/sgl-router/tests/fixtures/tiny_tokenizer.json /etc/tokenizer/tiny.json
|
||||
|
||||
EXPOSE 8090
|
||||
|
||||
ENTRYPOINT ["sgl-router"]
|
||||
@@ -0,0 +1,251 @@
|
||||
"""Pytest configuration for sgl-router K8s integration tests.
|
||||
|
||||
These tests require:
|
||||
- A kind cluster named 'sgl-router-kind'
|
||||
- The sgl-router:e2e and sgl-router-fake-worker:e2e images loaded into kind
|
||||
- kubectl configured to use the kind-sgl-router-kind context
|
||||
|
||||
Setup: ./tests/e2e/k8s_integration/setup.sh
|
||||
Teardown: ./tests/e2e/k8s_integration/setup.sh teardown
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import socket
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
NAMESPACE = "sgl-router-test"
|
||||
CLUSTER_NAME = "sgl-router-kind"
|
||||
KUBECTL_CONTEXT = f"kind-{CLUSTER_NAME}"
|
||||
|
||||
# sgl-router discovery reconciliation: if the watcher misses an event the
|
||||
# reconciler fires within ~60s. Tests that exercise removal wait up to 90s.
|
||||
RECONCILIATION_WAIT_SECS = 90
|
||||
|
||||
# Errors safe to retry while polling (transport-level only — HTTP 4xx/5xx
|
||||
# are intentionally NOT included so real regressions surface immediately).
|
||||
_TRANSIENT_ERRORS = (
|
||||
httpx.TransportError,
|
||||
httpx.TimeoutException,
|
||||
ConnectionError,
|
||||
OSError,
|
||||
)
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
config.addinivalue_line(
|
||||
"markers",
|
||||
"slow: marks tests that wait for multiple reconciliation cycles "
|
||||
"(deselect with '-m \"not slow\"')",
|
||||
)
|
||||
|
||||
|
||||
def _kubectl(
|
||||
*args: str,
|
||||
check: bool = True,
|
||||
capture: bool = True,
|
||||
) -> subprocess.CompletedProcess:
|
||||
cmd = ["kubectl", "--context", KUBECTL_CONTEXT, *args]
|
||||
logger.debug("Running: %s", " ".join(cmd))
|
||||
return subprocess.run(cmd, capture_output=capture, text=True, check=check)
|
||||
|
||||
|
||||
def _apply_from_stdin(yaml_content: str) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(
|
||||
["kubectl", "--context", KUBECTL_CONTEXT, "apply", "-f", "-"],
|
||||
input=yaml_content,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
def _wait_for_deployment_ready(
|
||||
name: str,
|
||||
namespace: str = NAMESPACE,
|
||||
timeout: int = 180,
|
||||
) -> None:
|
||||
_kubectl(
|
||||
"rollout",
|
||||
"status",
|
||||
f"deployment/{name}",
|
||||
"-n",
|
||||
namespace,
|
||||
f"--timeout={timeout}s",
|
||||
)
|
||||
|
||||
|
||||
def _wait_for_pod_ready(
|
||||
name: str,
|
||||
namespace: str = NAMESPACE,
|
||||
timeout: int = 120,
|
||||
) -> None:
|
||||
_kubectl(
|
||||
"wait",
|
||||
"--for=condition=Ready",
|
||||
f"pod/{name}",
|
||||
"-n",
|
||||
namespace,
|
||||
f"--timeout={timeout}s",
|
||||
)
|
||||
|
||||
|
||||
def _wait_for_port(port: int, proc: subprocess.Popen, timeout: int = 15) -> None:
|
||||
"""Poll until a TCP connection to localhost:port succeeds."""
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
if proc.poll() is not None:
|
||||
stderr = proc.stderr.read().decode() if proc.stderr else ""
|
||||
raise RuntimeError(f"port-forward process exited early: {stderr}")
|
||||
try:
|
||||
with socket.create_connection(("127.0.0.1", port), timeout=1):
|
||||
return
|
||||
except OSError:
|
||||
time.sleep(0.5)
|
||||
raise TimeoutError(f"Port {port} not ready after {timeout}s")
|
||||
|
||||
|
||||
def _port_forward_start(
|
||||
namespace: str,
|
||||
service: str,
|
||||
local_port: int,
|
||||
remote_port: int,
|
||||
) -> subprocess.Popen:
|
||||
"""Start kubectl port-forward and wait until the port is reachable."""
|
||||
cmd = [
|
||||
"kubectl",
|
||||
"--context",
|
||||
KUBECTL_CONTEXT,
|
||||
"port-forward",
|
||||
f"svc/{service}",
|
||||
f"{local_port}:{remote_port}",
|
||||
"-n",
|
||||
namespace,
|
||||
]
|
||||
logger.info("Starting port-forward: %s", " ".join(cmd))
|
||||
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
_wait_for_port(local_port, proc)
|
||||
return proc
|
||||
|
||||
|
||||
def _cleanup_port_forward(name: str, pf: subprocess.Popen) -> None:
|
||||
try:
|
||||
pf.terminate()
|
||||
pf.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning(
|
||||
"Port-forward %s did not exit on SIGTERM after 10s; killing", name
|
||||
)
|
||||
pf.kill()
|
||||
try:
|
||||
pf.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning("Port-forward %s still running after SIGKILL", name)
|
||||
except Exception as exc:
|
||||
logger.warning("Error cleaning up %s port-forward: %s", name, exc)
|
||||
|
||||
rc = pf.returncode
|
||||
stderr = pf.stderr.read().decode() if pf.stderr else ""
|
||||
if rc != -15:
|
||||
suffix = f": {stderr.strip()}" if stderr.strip() else ""
|
||||
logger.warning("Port-forward %s exited rc=%s%s", name, rc, suffix)
|
||||
else:
|
||||
logger.debug("Port-forward %s exited cleanly (rc=%s)", name, rc)
|
||||
|
||||
|
||||
def _poll_until(
|
||||
predicate,
|
||||
description: str,
|
||||
timeout: int,
|
||||
interval: float = 5,
|
||||
) -> bool:
|
||||
"""Poll predicate until True, or raise TimeoutError.
|
||||
|
||||
Only transient network errors are retried; HTTP status errors and
|
||||
programming errors propagate immediately.
|
||||
"""
|
||||
deadline = time.time() + timeout
|
||||
last_error = None
|
||||
attempts = 0
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
attempts += 1
|
||||
if predicate():
|
||||
logger.info(
|
||||
"Condition met: %s (after %d attempts)", description, attempts
|
||||
)
|
||||
return True
|
||||
except _TRANSIENT_ERRORS as exc:
|
||||
last_error = exc
|
||||
logger.debug("Transient error on attempt %d: %s", attempts, exc)
|
||||
time.sleep(interval)
|
||||
msg = f"Timeout waiting for: {description} (after {timeout}s, {attempts} attempts)"
|
||||
if last_error:
|
||||
msg += f" — last error: {last_error}"
|
||||
raise TimeoutError(msg)
|
||||
|
||||
|
||||
def _get_router_url(router_base: str) -> str:
|
||||
return router_base
|
||||
|
||||
|
||||
def _router_is_healthy(router_base: str) -> bool:
|
||||
try:
|
||||
r = httpx.get(f"{router_base}/healthz", timeout=3.0)
|
||||
return r.status_code == 200
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def k8s_cluster():
|
||||
"""Assert the kind cluster exists and kubectl context is reachable."""
|
||||
result = subprocess.run(
|
||||
["kind", "get", "clusters"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
if CLUSTER_NAME not in result.stdout.splitlines():
|
||||
pytest.skip(
|
||||
f"kind cluster '{CLUSTER_NAME}' not found — run "
|
||||
f"./tests/e2e/k8s_integration/setup.sh first"
|
||||
)
|
||||
_kubectl("cluster-info")
|
||||
return True
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def router_port_forward(k8s_cluster):
|
||||
"""Per-test port-forward to sgl-router service.
|
||||
|
||||
Function-scoped because some tests (notably
|
||||
test_lifecycle.TestRouterRestart) force-delete the router pod;
|
||||
a session-scoped port-forward would be bound to the deleted pod's
|
||||
network namespace and stay dead for all subsequent tests in the
|
||||
suite. Per-test setup costs ~1-2s.
|
||||
"""
|
||||
_wait_for_deployment_ready("sgl-router")
|
||||
pf = _port_forward_start(NAMESPACE, "sgl-router", 8090, 8090)
|
||||
try:
|
||||
_poll_until(
|
||||
lambda: _router_is_healthy("http://127.0.0.1:8090"),
|
||||
"sgl-router /healthz returns 200",
|
||||
timeout=30,
|
||||
interval=1,
|
||||
)
|
||||
yield "http://127.0.0.1:8090"
|
||||
finally:
|
||||
_cleanup_port_forward("sgl-router", pf)
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def router_url(router_port_forward):
|
||||
return router_port_forward
|
||||
@@ -0,0 +1,73 @@
|
||||
"""Minimal fake SGLang worker for kind E2E integration testing.
|
||||
|
||||
Responds to:
|
||||
GET /health -> {"status": "ok"}
|
||||
GET /server_info -> {"served_model_name": MODEL_ID}
|
||||
GET /v1/models -> list with a single MODEL_ID model entry
|
||||
POST /v1/chat/completions -> echoes the last user message back
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
import uvicorn
|
||||
from fastapi import FastAPI, Request
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
MODEL_ID = os.environ.get("MODEL_ID", "tiny")
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
async def health():
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get("/server_info")
|
||||
async def server_info():
|
||||
# The sgl-router worker manager fetches this on every Added event and
|
||||
# uses `served_model_name` to populate the registry's model index.
|
||||
return {"served_model_name": MODEL_ID}
|
||||
|
||||
|
||||
@app.get("/v1/models")
|
||||
async def models():
|
||||
return {
|
||||
"object": "list",
|
||||
"data": [
|
||||
{
|
||||
"id": MODEL_ID,
|
||||
"object": "model",
|
||||
"created": 0,
|
||||
"owned_by": "sglang",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@app.post("/v1/chat/completions")
|
||||
async def chat_completions(request: Request):
|
||||
payload = await request.json()
|
||||
messages = payload.get("messages", [])
|
||||
last_content = messages[-1]["content"] if messages else ""
|
||||
return {
|
||||
"id": "chatcmpl-mock",
|
||||
"object": "chat.completion",
|
||||
"model": payload.get("model", MODEL_ID),
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": f"echo: {last_content}",
|
||||
},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
uvicorn.run(app, host="0.0.0.0", port=30000)
|
||||
@@ -0,0 +1,4 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: sgl-router-test
|
||||
@@ -0,0 +1,33 @@
|
||||
# Cluster-wide RBAC for the cross-namespace discovery test.
|
||||
# Distinct ServiceAccount/ClusterRole names to avoid collision with
|
||||
# the namespace-scoped Role in rbac.yaml used by the default router.
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: sgl-router-cluster
|
||||
namespace: sgl-router-test
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: sgl-router-cluster
|
||||
rules:
|
||||
- apiGroups: ["discovery.k8s.io"]
|
||||
resources: ["endpointslices"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["services", "pods"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: sgl-router-cluster
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: sgl-router-cluster
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: sgl-router-cluster
|
||||
namespace: sgl-router-test
|
||||
@@ -0,0 +1,34 @@
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: sgl-router
|
||||
namespace: sgl-router-test
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: sgl-router
|
||||
namespace: sgl-router-test
|
||||
rules:
|
||||
# EndpointSlice watch (k8s discovery backend)
|
||||
- apiGroups: ["discovery.k8s.io"]
|
||||
resources: ["endpointslices"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
# Service list/watch (needed to resolve EndpointSlice owner)
|
||||
- apiGroups: [""]
|
||||
resources: ["services", "pods"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: sgl-router
|
||||
namespace: sgl-router-test
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: sgl-router
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: sgl-router
|
||||
namespace: sgl-router-test
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
# sgl-router deployment with ClusterRole for cross-namespace discovery test.
|
||||
# Watches workers in ALL namespaces via cluster-scoped EndpointSlice access.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: sgl-router-cluster
|
||||
namespace: sgl-router-test
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: sgl-router-cluster
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: sgl-router-cluster
|
||||
spec:
|
||||
serviceAccountName: sgl-router-cluster
|
||||
containers:
|
||||
- name: router
|
||||
image: sgl-router:e2e
|
||||
imagePullPolicy: Never
|
||||
args:
|
||||
- "--config"
|
||||
- "/etc/config/router-cluster.toml"
|
||||
ports:
|
||||
- containerPort: 8091
|
||||
name: http
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /readyz
|
||||
port: 8091
|
||||
initialDelaySeconds: 3
|
||||
periodSeconds: 3
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: 8091
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /etc/config
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: sgl-router-cluster-config
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: sgl-router-cluster
|
||||
namespace: sgl-router-test
|
||||
spec:
|
||||
selector:
|
||||
app: sgl-router-cluster
|
||||
ports:
|
||||
- name: http
|
||||
port: 8091
|
||||
targetPort: 8091
|
||||
@@ -0,0 +1,58 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: sgl-router
|
||||
namespace: sgl-router-test
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: sgl-router
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: sgl-router
|
||||
spec:
|
||||
serviceAccountName: sgl-router
|
||||
containers:
|
||||
- name: router
|
||||
image: sgl-router:e2e
|
||||
imagePullPolicy: Never
|
||||
args:
|
||||
- "--config"
|
||||
- "/etc/config/router.toml"
|
||||
ports:
|
||||
- containerPort: 8090
|
||||
name: http
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /readyz
|
||||
port: 8090
|
||||
initialDelaySeconds: 3
|
||||
periodSeconds: 3
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: 8090
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /etc/config
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: sgl-router-config
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: sgl-router
|
||||
namespace: sgl-router-test
|
||||
spec:
|
||||
selector:
|
||||
app: sgl-router
|
||||
ports:
|
||||
- name: http
|
||||
port: 8090
|
||||
targetPort: 8090
|
||||
@@ -0,0 +1,2 @@
|
||||
httpx==0.27.2
|
||||
pytest==8.3.3
|
||||
+191
@@ -0,0 +1,191 @@
|
||||
#!/usr/bin/env bash
|
||||
# Bootstrap a kind cluster for sgl-router K8s integration E2E tests.
|
||||
#
|
||||
# Prerequisites: Docker, kind, kubectl
|
||||
#
|
||||
# Usage:
|
||||
# ./tests/e2e/k8s_integration/setup.sh # full setup
|
||||
# ./tests/e2e/k8s_integration/setup.sh teardown # delete the cluster
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../../.." && pwd)" # repo root (above experimental/)
|
||||
SGL_ROUTER_DIR="${REPO_ROOT}/experimental/sgl-router"
|
||||
CLUSTER_NAME="${CLUSTER:-sgl-router-kind}"
|
||||
NAMESPACE="${NAMESPACE:-sgl-router-test}"
|
||||
CONTEXT="kind-${CLUSTER_NAME}"
|
||||
MANIFESTS_DIR="${SCRIPT_DIR}/manifests"
|
||||
|
||||
log() { echo "==> $*"; }
|
||||
|
||||
teardown() {
|
||||
log "Tearing down cluster '${CLUSTER_NAME}'..."
|
||||
if kind get clusters 2>/dev/null | grep -q "^${CLUSTER_NAME}$"; then
|
||||
kind delete cluster --name "${CLUSTER_NAME}"
|
||||
else
|
||||
log "Cluster '${CLUSTER_NAME}' not found, nothing to tear down."
|
||||
fi
|
||||
log "Done."
|
||||
}
|
||||
|
||||
if [[ "${1:-}" == "teardown" ]]; then
|
||||
teardown
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Step 1: Create kind cluster (idempotent)
|
||||
# ---------------------------------------------------------------------------
|
||||
if kind get clusters 2>/dev/null | grep -q "^${CLUSTER_NAME}$"; then
|
||||
log "Kind cluster '${CLUSTER_NAME}' already exists — reusing."
|
||||
else
|
||||
log "Creating kind cluster '${CLUSTER_NAME}'..."
|
||||
kind create cluster --name "${CLUSTER_NAME}" --wait 60s
|
||||
fi
|
||||
|
||||
kubectl config use-context "${CONTEXT}"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Step 2: Build Docker images (unless SKIP_DOCKER_BUILD=1)
|
||||
# ---------------------------------------------------------------------------
|
||||
if [[ "${SKIP_DOCKER_BUILD:-}" == "1" ]]; then
|
||||
log "SKIP_DOCKER_BUILD=1 — skipping docker build; expecting images to exist locally."
|
||||
for img in sgl-router:e2e sgl-router-fake-worker:e2e; do
|
||||
if ! docker image inspect "${img}" >/dev/null 2>&1; then
|
||||
log "ERROR: ${img} not found locally; cannot continue without building."
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
else
|
||||
log "Building sgl-router:e2e from ${REPO_ROOT} ..."
|
||||
docker build \
|
||||
-f "${SCRIPT_DIR}/Dockerfile.router" \
|
||||
-t sgl-router:e2e \
|
||||
"${REPO_ROOT}"
|
||||
|
||||
log "Building sgl-router-fake-worker:e2e ..."
|
||||
docker build \
|
||||
-f "${SCRIPT_DIR}/Dockerfile.fake_worker" \
|
||||
-t sgl-router-fake-worker:e2e \
|
||||
"${SCRIPT_DIR}"
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Step 3: Load images into kind
|
||||
# ---------------------------------------------------------------------------
|
||||
log "Loading images into kind cluster '${CLUSTER_NAME}'..."
|
||||
kind load docker-image sgl-router:e2e --name "${CLUSTER_NAME}"
|
||||
kind load docker-image sgl-router-fake-worker:e2e --name "${CLUSTER_NAME}"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Step 4: Apply namespace and RBAC
|
||||
# ---------------------------------------------------------------------------
|
||||
log "Applying namespace and RBAC..."
|
||||
kubectl --context "${CONTEXT}" apply -f "${MANIFESTS_DIR}/namespace.yaml"
|
||||
kubectl --context "${CONTEXT}" apply -f "${MANIFESTS_DIR}/rbac.yaml"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Step 5: Deploy 3 fake-worker replicas behind a Service
|
||||
# The Service causes K8s to auto-create an EndpointSlice, which
|
||||
# the sgl-router K8s discovery backend watches.
|
||||
# ---------------------------------------------------------------------------
|
||||
log "Deploying fake-worker Deployment + Service (3 replicas, app=sglang)..."
|
||||
kubectl --context "${CONTEXT}" -n "${NAMESPACE}" apply -f - <<EOF
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: fake-worker
|
||||
namespace: ${NAMESPACE}
|
||||
labels:
|
||||
app: sglang
|
||||
spec:
|
||||
replicas: 3
|
||||
selector:
|
||||
matchLabels:
|
||||
app: sglang
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: sglang
|
||||
spec:
|
||||
containers:
|
||||
- name: worker
|
||||
image: sgl-router-fake-worker:e2e
|
||||
imagePullPolicy: Never
|
||||
ports:
|
||||
- containerPort: 30000
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 30000
|
||||
initialDelaySeconds: 2
|
||||
periodSeconds: 3
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: fake-worker
|
||||
namespace: ${NAMESPACE}
|
||||
labels:
|
||||
app: sglang
|
||||
spec:
|
||||
selector:
|
||||
app: sglang
|
||||
ports:
|
||||
- port: 30000
|
||||
targetPort: 30000
|
||||
EOF
|
||||
|
||||
log "Waiting for fake-worker rollout..."
|
||||
kubectl --context "${CONTEXT}" -n "${NAMESPACE}" rollout status deployment/fake-worker --timeout=120s
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Step 6: Create sgl-router ConfigMap with k8s discovery pointing at the
|
||||
# namespace where fake-worker pods live.
|
||||
# ---------------------------------------------------------------------------
|
||||
log "Creating sgl-router-config ConfigMap..."
|
||||
ROUTER_CONFIG="[server]
|
||||
host = \"0.0.0.0\"
|
||||
port = 8090
|
||||
|
||||
[[models]]
|
||||
id = \"tiny\"
|
||||
tokenizer_path = \"/etc/tokenizer/tiny.json\"
|
||||
policy = \"round_robin\"
|
||||
# Aggressive breaker so a terminating pod's connection-refused
|
||||
# immediately excludes it from the next request's candidate set —
|
||||
# the reconciliation tests scale workers rapidly and depend on
|
||||
# fast worker eviction to absorb the churn.
|
||||
circuit_breaker = { threshold = 1, cool_down_secs = 5 }
|
||||
|
||||
[discovery]
|
||||
backend = \"k8s\"
|
||||
|
||||
[discovery.k8s]
|
||||
namespace = \"${NAMESPACE}\"
|
||||
label_selector = \"app=sglang\""
|
||||
|
||||
kubectl --context "${CONTEXT}" -n "${NAMESPACE}" create configmap sgl-router-config \
|
||||
--from-literal=router.toml="${ROUTER_CONFIG}" \
|
||||
--dry-run=client -o yaml \
|
||||
| kubectl --context "${CONTEXT}" apply -f -
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Step 7: Deploy sgl-router
|
||||
# ---------------------------------------------------------------------------
|
||||
log "Deploying sgl-router..."
|
||||
kubectl --context "${CONTEXT}" apply -f "${MANIFESTS_DIR}/router.yaml"
|
||||
|
||||
log "Waiting for sgl-router rollout..."
|
||||
kubectl --context "${CONTEXT}" -n "${NAMESPACE}" rollout status deployment/sgl-router --timeout=300s
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Done
|
||||
# ---------------------------------------------------------------------------
|
||||
log ""
|
||||
log "Setup complete! Run the integration tests with:"
|
||||
log " pytest tests/e2e/k8s_integration/ -v -s"
|
||||
log ""
|
||||
log "To tear down:"
|
||||
log " ./tests/e2e/k8s_integration/setup.sh teardown"
|
||||
@@ -0,0 +1,265 @@
|
||||
"""Cross-namespace service discovery integration test.
|
||||
|
||||
Validates that a sgl-router instance with cluster-wide RBAC and no namespace
|
||||
filter in its k8s discovery config watches EndpointSlices in all namespaces.
|
||||
Workers deployed in a second namespace (sgl-router-test-extra) must be
|
||||
discovered alongside those in the primary namespace.
|
||||
|
||||
This test deploys a separate router Deployment (sgl-router-cluster) with a
|
||||
ClusterRole that grants EndpointSlice access across all namespaces.
|
||||
|
||||
Run with:
|
||||
pytest tests/e2e/k8s_integration/test_cross_namespace.py -v -s
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from conftest import (
|
||||
KUBECTL_CONTEXT,
|
||||
NAMESPACE,
|
||||
_apply_from_stdin,
|
||||
_cleanup_port_forward,
|
||||
_kubectl,
|
||||
_poll_until,
|
||||
_port_forward_start,
|
||||
_wait_for_deployment_ready,
|
||||
logger,
|
||||
)
|
||||
|
||||
MANIFESTS_DIR = Path(__file__).parent / "manifests"
|
||||
EXTRA_NAMESPACE = "sgl-router-test-extra"
|
||||
CLUSTER_ROUTER_PORT = 8093
|
||||
|
||||
|
||||
def _deploy_fake_worker_in_ns(name: str, namespace: str) -> None:
|
||||
"""Deploy a fake-worker pod with imagePullPolicy=Never in the given namespace."""
|
||||
pod_manifest = {
|
||||
"apiVersion": "v1",
|
||||
"kind": "Pod",
|
||||
"metadata": {
|
||||
"name": name,
|
||||
"namespace": namespace,
|
||||
"labels": {"app": "sglang", "cross-ns-test": "true"},
|
||||
},
|
||||
"spec": {
|
||||
"containers": [
|
||||
{
|
||||
"name": "worker",
|
||||
"image": "sgl-router-fake-worker:e2e",
|
||||
"imagePullPolicy": "Never",
|
||||
"ports": [{"containerPort": 30000}],
|
||||
"readinessProbe": {
|
||||
"httpGet": {"path": "/health", "port": 30000},
|
||||
"initialDelaySeconds": 2,
|
||||
"periodSeconds": 3,
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
}
|
||||
proc = subprocess.run(
|
||||
["kubectl", "--context", KUBECTL_CONTEXT, "apply", "-f", "-"],
|
||||
input=json.dumps(pod_manifest),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"Failed to deploy pod {name} in namespace {namespace} "
|
||||
f"(rc={proc.returncode}): {proc.stderr.strip()!r}"
|
||||
)
|
||||
logger.info("Deployed worker %s in namespace %s", name, namespace)
|
||||
|
||||
|
||||
def _safe_delete_pod(name: str, namespace: str) -> None:
|
||||
try:
|
||||
_kubectl(
|
||||
"delete",
|
||||
"pod",
|
||||
name,
|
||||
"-n",
|
||||
namespace,
|
||||
"--ignore-not-found",
|
||||
"--force",
|
||||
"--grace-period=0",
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("Cleanup failed for pod %s in ns %s: %s", name, namespace, exc)
|
||||
|
||||
|
||||
def _ensure_namespace(name: str) -> None:
|
||||
manifest = {"apiVersion": "v1", "kind": "Namespace", "metadata": {"name": name}}
|
||||
_apply_from_stdin(json.dumps(manifest))
|
||||
|
||||
|
||||
def _ensure_service_in_ns(namespace: str, selector: str = "app=sglang") -> None:
|
||||
"""Create a Service so K8s auto-creates an EndpointSlice for cross-ns workers.
|
||||
|
||||
Service `metadata.labels` propagates to the auto-created EndpointSlice's
|
||||
labels — and the cluster-scoped router filters slices server-side by
|
||||
`app=sglang,cross-ns-test=true`. Without those labels on the Service,
|
||||
its EndpointSlice gets filtered out and the cross-ns worker is invisible.
|
||||
"""
|
||||
svc_manifest = {
|
||||
"apiVersion": "v1",
|
||||
"kind": "Service",
|
||||
"metadata": {
|
||||
"name": "fake-worker",
|
||||
"namespace": namespace,
|
||||
"labels": {"app": "sglang", "cross-ns-test": "true"},
|
||||
},
|
||||
"spec": {
|
||||
"selector": {"app": "sglang", "cross-ns-test": "true"},
|
||||
"ports": [{"port": 30000, "targetPort": 30000}],
|
||||
},
|
||||
}
|
||||
_apply_from_stdin(json.dumps(svc_manifest))
|
||||
|
||||
|
||||
def _can_route(router_url: str) -> bool:
|
||||
try:
|
||||
r = httpx.post(
|
||||
f"{router_url}/v1/chat/completions",
|
||||
json={
|
||||
"model": "tiny",
|
||||
"messages": [{"role": "user", "content": "cross-ns"}],
|
||||
},
|
||||
timeout=8.0,
|
||||
)
|
||||
return r.status_code == 200
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def cluster_scoped_router(k8s_cluster):
|
||||
"""Deploy the cluster-scoped RBAC + router, plus a second namespace."""
|
||||
rbac_manifest = MANIFESTS_DIR / "rbac-cluster-scoped.yaml"
|
||||
router_manifest = MANIFESTS_DIR / "router-cluster-scoped.yaml"
|
||||
|
||||
_kubectl("apply", "-f", str(rbac_manifest))
|
||||
_ensure_namespace(EXTRA_NAMESPACE)
|
||||
_ensure_service_in_ns(EXTRA_NAMESPACE)
|
||||
|
||||
# ConfigMap for the cluster-scoped router: empty namespace = watch all
|
||||
cluster_config = """[server]
|
||||
host = "0.0.0.0"
|
||||
port = 8091
|
||||
|
||||
[[models]]
|
||||
id = "tiny"
|
||||
tokenizer_path = "/etc/tokenizer/tiny.json"
|
||||
policy = "round_robin"
|
||||
|
||||
[discovery]
|
||||
backend = "k8s"
|
||||
|
||||
[discovery.k8s]
|
||||
namespace = ""
|
||||
label_selector = "app=sglang,cross-ns-test=true"
|
||||
"""
|
||||
_kubectl(
|
||||
"create",
|
||||
"configmap",
|
||||
"sgl-router-cluster-config",
|
||||
f"--from-literal=router-cluster.toml={cluster_config}",
|
||||
"-n",
|
||||
NAMESPACE,
|
||||
"--dry-run=client",
|
||||
"-o",
|
||||
"yaml",
|
||||
check=True,
|
||||
)
|
||||
# pipe through apply
|
||||
proc = _kubectl(
|
||||
"create",
|
||||
"configmap",
|
||||
"sgl-router-cluster-config",
|
||||
f"--from-literal=router-cluster.toml={cluster_config}",
|
||||
"-n",
|
||||
NAMESPACE,
|
||||
"--dry-run=client",
|
||||
"-o",
|
||||
"yaml",
|
||||
)
|
||||
_apply_from_stdin(proc.stdout)
|
||||
|
||||
_kubectl("apply", "-f", str(router_manifest))
|
||||
|
||||
# The cluster-scoped router's /readyz blocks on registry-not-empty, so
|
||||
# without at least one matching worker the rollout-status check below
|
||||
# would hang for 180s. Deploy a "bootstrap" worker in EXTRA_NAMESPACE
|
||||
# with the label_selector match (app=sglang,cross-ns-test=true) so the
|
||||
# router's k8s discovery picks it up before the readiness probe runs.
|
||||
# The test body adds a SECOND worker later to verify dynamic discovery.
|
||||
bootstrap_worker = "cross-ns-worker-bootstrap"
|
||||
_deploy_fake_worker_in_ns(bootstrap_worker, EXTRA_NAMESPACE)
|
||||
|
||||
pf = None
|
||||
try:
|
||||
_wait_for_deployment_ready("sgl-router-cluster")
|
||||
pf = _port_forward_start(
|
||||
NAMESPACE, "sgl-router-cluster", CLUSTER_ROUTER_PORT, 8091
|
||||
)
|
||||
yield f"http://127.0.0.1:{CLUSTER_ROUTER_PORT}"
|
||||
finally:
|
||||
if pf is not None:
|
||||
_cleanup_port_forward("cluster_router", pf)
|
||||
_safe_delete_pod(bootstrap_worker, EXTRA_NAMESPACE)
|
||||
_kubectl(
|
||||
"delete", "-f", str(router_manifest), "--ignore-not-found", check=False
|
||||
)
|
||||
_kubectl("delete", "-f", str(rbac_manifest), "--ignore-not-found", check=False)
|
||||
_kubectl(
|
||||
"delete",
|
||||
"namespace",
|
||||
EXTRA_NAMESPACE,
|
||||
"--ignore-not-found",
|
||||
"--wait=true",
|
||||
"--timeout=60s",
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
class TestClusterWideDiscovery:
|
||||
"""Router with ClusterRole and no namespace filter sees workers in every namespace."""
|
||||
|
||||
def test_router_routes_to_worker_in_extra_namespace(self, cluster_scoped_router):
|
||||
"""Deploy one fake-worker pod in the extra namespace behind a Service;
|
||||
the cluster-scoped router must discover it (via its EndpointSlice) and
|
||||
successfully route a chat completion to it."""
|
||||
router_url = cluster_scoped_router
|
||||
worker_name = "cross-ns-worker-extra"
|
||||
|
||||
try:
|
||||
_deploy_fake_worker_in_ns(worker_name, EXTRA_NAMESPACE)
|
||||
|
||||
_poll_until(
|
||||
lambda: _can_route(router_url),
|
||||
"cluster-scoped router routes to worker in extra namespace",
|
||||
timeout=60,
|
||||
interval=3,
|
||||
)
|
||||
|
||||
r = httpx.post(
|
||||
f"{router_url}/v1/chat/completions",
|
||||
json={
|
||||
"model": "tiny",
|
||||
"messages": [
|
||||
{"role": "user", "content": "cross-namespace routing"}
|
||||
],
|
||||
},
|
||||
timeout=15.0,
|
||||
)
|
||||
assert r.status_code == 200, f"expected 200, got {r.status_code}: {r.text}"
|
||||
assert "echo:" in r.json()["choices"][0]["message"]["content"]
|
||||
finally:
|
||||
_safe_delete_pod(worker_name, EXTRA_NAMESPACE)
|
||||
@@ -0,0 +1,84 @@
|
||||
"""E2E: sgl-router K8s discovery — basic routing.
|
||||
|
||||
Verifies that sgl-router, configured with the k8s EndpointSlice backend,
|
||||
discovers the 3 fake-worker replicas deployed by setup.sh and successfully
|
||||
routes chat-completion requests to them.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from conftest import NAMESPACE, _kubectl, _poll_until, logger
|
||||
|
||||
|
||||
def _scale_fake_worker(replicas: int) -> None:
|
||||
_kubectl(
|
||||
"scale",
|
||||
"deployment/fake-worker",
|
||||
f"--replicas={replicas}",
|
||||
"-n",
|
||||
NAMESPACE,
|
||||
)
|
||||
|
||||
|
||||
def test_router_routes_chat_to_a_worker(router_url):
|
||||
"""A /v1/chat/completions request through the router returns 200 with the
|
||||
fake-worker echo payload, proving end-to-end routing works."""
|
||||
r = httpx.post(
|
||||
f"{router_url}/v1/chat/completions",
|
||||
json={
|
||||
"model": "tiny",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"stream": False,
|
||||
},
|
||||
timeout=15.0,
|
||||
)
|
||||
assert r.status_code == 200, f"expected 200, got {r.status_code}: {r.text}"
|
||||
body = r.json()
|
||||
assert "echo:" in body["choices"][0]["message"]["content"]
|
||||
|
||||
|
||||
def test_router_lists_model(router_url):
|
||||
"""GET /v1/models returns the 'tiny' model entry from the router config."""
|
||||
r = httpx.get(f"{router_url}/v1/models", timeout=10.0)
|
||||
assert r.status_code == 200, f"expected 200, got {r.status_code}: {r.text}"
|
||||
body = r.json()
|
||||
ids = [m["id"] for m in body["data"]]
|
||||
assert "tiny" in ids, f"expected 'tiny' in model list, got {ids}"
|
||||
|
||||
|
||||
def test_router_discovers_multiple_workers(router_url):
|
||||
"""Scale down from 3 to 1 and back to 3 replicas; router must continue
|
||||
routing successfully after each transition (EndpointSlice watch reflects
|
||||
the change)."""
|
||||
# First confirm baseline routing
|
||||
r = httpx.post(
|
||||
f"{router_url}/v1/chat/completions",
|
||||
json={
|
||||
"model": "tiny",
|
||||
"messages": [{"role": "user", "content": "scale-test"}],
|
||||
},
|
||||
timeout=15.0,
|
||||
)
|
||||
assert r.status_code == 200
|
||||
|
||||
# Scale down to 1 — router should still route after reconverging
|
||||
_scale_fake_worker(1)
|
||||
_poll_until(
|
||||
lambda: httpx.post(
|
||||
f"{router_url}/v1/chat/completions",
|
||||
json={
|
||||
"model": "tiny",
|
||||
"messages": [{"role": "user", "content": "post-scale-down"}],
|
||||
},
|
||||
timeout=10.0,
|
||||
).status_code
|
||||
== 200,
|
||||
"router routes after scale-down to 1",
|
||||
timeout=60,
|
||||
interval=3,
|
||||
)
|
||||
|
||||
# Restore to 3
|
||||
_scale_fake_worker(3)
|
||||
@@ -0,0 +1,150 @@
|
||||
"""Worker lifecycle integration tests.
|
||||
|
||||
Covers:
|
||||
1. Scaling replicas up — new EndpointSlice entries are discovered.
|
||||
2. Scaling replicas down — removed endpoints are deregistered.
|
||||
3. Router restart — after the router pod is killed, the Deployment restarts
|
||||
it and it re-lists the existing EndpointSlice entries without duplicates.
|
||||
|
||||
These tests DO NOT use a /workers admin API (sgl-router does not expose
|
||||
one). They verify behaviour through /v1/chat/completions responses and
|
||||
by driving the deployment scale.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from conftest import (
|
||||
NAMESPACE,
|
||||
_cleanup_port_forward,
|
||||
_kubectl,
|
||||
_poll_until,
|
||||
_port_forward_start,
|
||||
_wait_for_deployment_ready,
|
||||
logger,
|
||||
)
|
||||
|
||||
ROUTER_RESTART_PORT = 8092
|
||||
|
||||
|
||||
def _scale(deployment: str, replicas: int) -> None:
|
||||
_kubectl(
|
||||
"scale", f"deployment/{deployment}", f"--replicas={replicas}", "-n", NAMESPACE
|
||||
)
|
||||
|
||||
|
||||
def _can_route(router_url: str) -> bool:
|
||||
try:
|
||||
r = httpx.post(
|
||||
f"{router_url}/v1/chat/completions",
|
||||
json={"model": "tiny", "messages": [{"role": "user", "content": "ping"}]},
|
||||
timeout=8.0,
|
||||
)
|
||||
return r.status_code == 200
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class TestScaleUp:
|
||||
"""Scaling fake-worker replicas up must not break routing."""
|
||||
|
||||
def test_router_routes_after_scale_up(self, router_url):
|
||||
"""Restore 3 replicas (in case a prior test left 1), verify routing."""
|
||||
_scale("fake-worker", 3)
|
||||
_poll_until(
|
||||
lambda: _can_route(router_url),
|
||||
"router routes after scale-up to 3",
|
||||
timeout=60,
|
||||
interval=3,
|
||||
)
|
||||
|
||||
|
||||
class TestScaleDown:
|
||||
"""Scaling to 0 then back up must restore routing."""
|
||||
|
||||
def test_router_recovers_after_scale_to_zero_and_back(self, router_url):
|
||||
try:
|
||||
_scale("fake-worker", 0)
|
||||
# After scale-to-0 the router may return 503 (no healthy workers)
|
||||
# That is expected behaviour — assert it transitions back on scale-up.
|
||||
_scale("fake-worker", 2)
|
||||
_poll_until(
|
||||
lambda: _can_route(router_url),
|
||||
"router routes again after scale-up from 0",
|
||||
timeout=90,
|
||||
interval=3,
|
||||
)
|
||||
finally:
|
||||
_scale("fake-worker", 3)
|
||||
|
||||
|
||||
class TestRouterRestart:
|
||||
"""Killing the router pod forces a Deployment restart; the new pod must
|
||||
re-discover workers via the EndpointSlice watch without duplicates."""
|
||||
|
||||
def test_router_rediscovers_workers_after_restart(self, k8s_cluster):
|
||||
# Use a dedicated port to avoid clashing with the session fixture
|
||||
pf_holder: list = [None]
|
||||
try:
|
||||
_wait_for_deployment_ready("sgl-router")
|
||||
pf_holder[0] = _port_forward_start(
|
||||
NAMESPACE, "sgl-router", ROUTER_RESTART_PORT, 8090
|
||||
)
|
||||
restart_url = f"http://127.0.0.1:{ROUTER_RESTART_PORT}"
|
||||
|
||||
# Baseline: routing works pre-restart
|
||||
_poll_until(
|
||||
lambda: _can_route(restart_url),
|
||||
"baseline routing works pre-restart",
|
||||
timeout=30,
|
||||
interval=2,
|
||||
)
|
||||
|
||||
# Kill the router pod — the Deployment ReplicaSet will restart it
|
||||
res = _kubectl(
|
||||
"get",
|
||||
"pod",
|
||||
"-n",
|
||||
NAMESPACE,
|
||||
"-l",
|
||||
"app=sgl-router",
|
||||
"-o",
|
||||
"jsonpath={.items[0].metadata.name}",
|
||||
check=False,
|
||||
)
|
||||
old_pod = res.stdout.strip()
|
||||
if old_pod:
|
||||
_kubectl(
|
||||
"delete",
|
||||
"pod",
|
||||
old_pod,
|
||||
"-n",
|
||||
NAMESPACE,
|
||||
"--force",
|
||||
"--grace-period=0",
|
||||
)
|
||||
|
||||
# Tear down the old port-forward before waiting for the new pod
|
||||
if pf_holder[0] is not None:
|
||||
_cleanup_port_forward("router-restart-pre-kill", pf_holder[0])
|
||||
pf_holder[0] = None
|
||||
|
||||
_wait_for_deployment_ready("sgl-router")
|
||||
|
||||
pf_holder[0] = _port_forward_start(
|
||||
NAMESPACE, "sgl-router", ROUTER_RESTART_PORT, 8090
|
||||
)
|
||||
|
||||
# After restart, routing must come back (EndpointSlice re-watch)
|
||||
_poll_until(
|
||||
lambda: _can_route(restart_url),
|
||||
"routing restored after router restart",
|
||||
timeout=60,
|
||||
interval=3,
|
||||
)
|
||||
finally:
|
||||
if pf_holder[0] is not None:
|
||||
_cleanup_port_forward("router-restart", pf_holder[0])
|
||||
@@ -0,0 +1,163 @@
|
||||
"""K8s discovery reconciliation integration tests.
|
||||
|
||||
Tests verify that:
|
||||
1. The K8s EndpointSlice watcher correctly discovers new workers as Services
|
||||
and backing Deployments are updated.
|
||||
2. Workers are removed from the router's registry after the backing EndpointSlice
|
||||
entries disappear (pod deleted / deployment scaled to 0).
|
||||
3. After a simulated watch-connection interruption (router restarted), the
|
||||
registry converges back to the correct worker set.
|
||||
|
||||
Note: sgl-router does not currently expose a Prometheus /metrics endpoint,
|
||||
so the SMG-style metric assertions are not used here. Disconnect/reconnect
|
||||
coverage is provided by test_lifecycle.TestRouterRestart.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from conftest import (
|
||||
NAMESPACE,
|
||||
RECONCILIATION_WAIT_SECS,
|
||||
_kubectl,
|
||||
_poll_until,
|
||||
logger,
|
||||
)
|
||||
|
||||
|
||||
def _scale_fake_worker(replicas: int) -> None:
|
||||
_kubectl(
|
||||
"scale", "deployment/fake-worker", f"--replicas={replicas}", "-n", NAMESPACE
|
||||
)
|
||||
|
||||
|
||||
def _can_route(router_url: str) -> bool:
|
||||
try:
|
||||
r = httpx.post(
|
||||
f"{router_url}/v1/chat/completions",
|
||||
json={
|
||||
"model": "tiny",
|
||||
"messages": [{"role": "user", "content": "reconcile"}],
|
||||
},
|
||||
timeout=8.0,
|
||||
)
|
||||
return r.status_code == 200
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class TestWatcherDiscovery:
|
||||
"""The EndpointSlice watcher discovers new endpoints on Deployment scale-up."""
|
||||
|
||||
def test_watcher_discovers_new_endpoints_on_scale_up(self, router_url):
|
||||
"""Scale from 1 to 3 replicas; router must continue routing successfully."""
|
||||
_scale_fake_worker(1)
|
||||
# Wait for scale-down to propagate and routing to stabilise
|
||||
_poll_until(
|
||||
lambda: _can_route(router_url),
|
||||
"router routes with 1 replica",
|
||||
timeout=60,
|
||||
interval=3,
|
||||
)
|
||||
|
||||
_scale_fake_worker(3)
|
||||
_poll_until(
|
||||
lambda: _can_route(router_url),
|
||||
"router routes with 3 replicas (after scale-up)",
|
||||
timeout=60,
|
||||
interval=3,
|
||||
)
|
||||
|
||||
|
||||
class TestStaleEndpointRemoval:
|
||||
"""When fake-worker replicas drop, the router must stop routing to the
|
||||
removed endpoints.
|
||||
|
||||
Because sgl-router has no /workers admin API, we verify removal
|
||||
indirectly: scale to 0, assert the router returns non-200 (or at least
|
||||
that scaling back to 2 restores routing), then restore.
|
||||
"""
|
||||
|
||||
def test_routing_restores_after_scale_down_and_back_up(self, router_url):
|
||||
"""Scale to 0 (no workers → expect non-200), then restore to 2.
|
||||
After restore the router must route again within the reconciliation window.
|
||||
"""
|
||||
try:
|
||||
_scale_fake_worker(0)
|
||||
|
||||
# Expect routing to fail eventually (503 or connection error)
|
||||
deadline = time.time() + RECONCILIATION_WAIT_SECS
|
||||
routing_failed = False
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
r = httpx.post(
|
||||
f"{router_url}/v1/chat/completions",
|
||||
json={
|
||||
"model": "tiny",
|
||||
"messages": [{"role": "user", "content": "no-workers"}],
|
||||
},
|
||||
timeout=5.0,
|
||||
)
|
||||
if r.status_code != 200:
|
||||
routing_failed = True
|
||||
break
|
||||
except Exception:
|
||||
routing_failed = True
|
||||
break
|
||||
time.sleep(3)
|
||||
|
||||
# If after RECONCILIATION_WAIT_SECS the router is still routing,
|
||||
# that means old endpoints are cached — not necessarily wrong for
|
||||
# a watcher that hasn't ticked yet, but log a warning.
|
||||
if not routing_failed:
|
||||
logger.warning(
|
||||
"Router still returning 200 after scale-to-0; "
|
||||
"EndpointSlice event may be delayed — continuing test."
|
||||
)
|
||||
|
||||
# Restore workers and verify routing comes back
|
||||
_scale_fake_worker(2)
|
||||
_poll_until(
|
||||
lambda: _can_route(router_url),
|
||||
"routing restored after scale back up to 2",
|
||||
timeout=RECONCILIATION_WAIT_SECS,
|
||||
interval=3,
|
||||
)
|
||||
finally:
|
||||
_scale_fake_worker(3)
|
||||
|
||||
|
||||
class TestReconciliationConsistency:
|
||||
"""Routing remains stable over multiple reconciliation windows with steady
|
||||
worker state — no spurious deregistrations or duplicate registrations."""
|
||||
|
||||
@pytest.mark.slow
|
||||
def test_routing_stable_over_multiple_reconciliation_cycles(self, router_url):
|
||||
"""Deploy 3 workers, sample routing success over ~150s (2 reconciliation
|
||||
cycles + margin), assert no interruptions."""
|
||||
_scale_fake_worker(3)
|
||||
_poll_until(
|
||||
lambda: _can_route(router_url),
|
||||
"baseline routing with 3 workers",
|
||||
timeout=30,
|
||||
interval=2,
|
||||
)
|
||||
|
||||
# Sample every 15s for 150s
|
||||
wait_secs = RECONCILIATION_WAIT_SECS + 60
|
||||
end_time = time.time() + wait_secs
|
||||
failures = []
|
||||
while time.time() < end_time:
|
||||
ok = _can_route(router_url)
|
||||
if not ok:
|
||||
failures.append(time.time())
|
||||
time.sleep(15)
|
||||
|
||||
assert not failures, (
|
||||
f"Routing failed at {len(failures)} sample(s) during stability window; "
|
||||
f"timestamps: {failures}"
|
||||
)
|
||||
Reference in New Issue
Block a user