[Fix] sgl-router: Resend on a closed pooled connection; add --disable-circuit-breaker (#44075)

This commit is contained in:
AMD-yanfeiwang
2026-10-18 18:23:18 +08:00
committed by GitHub
parent 3b932cde26
commit a3abea3560
17 changed files with 306 additions and 87 deletions
+2
View File
@@ -53,6 +53,8 @@ http-body = "1"
tower = { version = "0.5", features = ["full"] }
tower-http = { version = "0.6", features = ["trace", "compression-gzip", "cors", "timeout", "request-id", "catch-panic"] }
reqwest = { version = "0.12", features = ["stream", "json", "rustls-tls", "http2"], default-features = false }
# Named directly for `Error::is_incomplete_message`, which reqwest does not expose.
hyper = "1"
sgl-kv-indexer = { path = "sgl-kv-indexer" }
# Native SglangService gRPC types, shared with the engine (proto/sglang/runtime/v1).
sglang-api-types = { path = "../../rust/sglang-api-types" }
+3
View File
@@ -325,6 +325,9 @@ unset fields inherit the CLI flags. See
`session_aware`); see [optional tokenizer](REFERENCE.md#optional-tokenizer-for-load-only-routing).
- **Retries**: `--retry-max-attempts N` resends a request that failed before
any response to another worker; see [retries](REFERENCE.md#retries).
- **Circuit breaker**: a worker leaves routing after `--cb-threshold`
consecutive failures; `--disable-circuit-breaker` turns this off; see
[circuit breaker](REFERENCE.md#circuit-breaker).
- **DP-rank routing**: `--dp-aware` also picks the DP rank inside multi-rank
engines, and `--dp-rank-policy round_robin` cycles ranks that no routing key
or cached prefix decides; see [DP-rank routing](REFERENCE.md#dp-rank-routing).
+16
View File
@@ -218,6 +218,22 @@ doubles per retry, and is capped at `--retry-max-backoff-ms` (default 2000).
Set the initial backoff to 0 to retry immediately. A backoff never waits past
the request's stale deadline.
Separately, a send whose pooled HTTP/1.1 connection the worker closed before
answering (reset, broken pipe or EOF) is repeated once, immediately, on the same
worker: an engine closing an idle connection never read the request. Idle
connections leave the pool after 3 s, under SGLang's default 5 s keep-alive.
### Circuit breaker
Each worker has a circuit breaker: after `--cb-threshold` (default 3)
consecutive failures the worker is taken out of routing until a request already
sent to it succeeds or, once `--cb-cool-down-secs` (default 30) has passed, a
probe request does. Under `--worker-urls` it is the only thing that takes a
dead or hung worker out of routing, since there is no active health check.
`--disable-circuit-breaker` turns it off, for example when one worker serves a
role and an open breaker would reject all of that role's requests; a worker
that stops answering then keeps its share of traffic.
### Engines with `--api-key`
The router reads each worker's `/server_info` and `/model_info` to learn its
+57 -42
View File
@@ -11,15 +11,15 @@ use std::num::NonZeroU32;
use crate::config::sampling::{parse_sampling_overrides, ConflictPolicy};
use crate::config::types::is_selector_empty;
use crate::config::{
default_cb_cool_down, default_host, default_port, default_proxy_request_timeout_secs,
default_shutdown_drain_secs, default_stale_request_timeout_secs, resolve_mode, AffinityConfig,
AffinityMode, BalancedBy, CacheAwareConfig, CachePrefixProvider, ChatRoutingKind,
CircuitBreakerConfig, Config, DecodePolicyKind, DiscoveryBackend, DpRankPolicy,
EligibilityConfig, FilterKind, FusedTerm, InflightLoadConfig, K8sDiscoveryConfig,
K8sDiscoveryMode, KvIndexerEndpointConfig, LogFormat, ModelConfig, ObservabilityConfig,
PolicyKind, ProxyConfig, ServerConfig, SessionAffinityMode, StaticUrlsDiscoveryConfig,
StickyConfig, StickyFallbackKind, TokenizerBackend, TokenizerConfig, DEFAULT_FUSE,
DEFAULT_KV_BOOTSTRAP_FETCH_TIMEOUT_CAP_MS, DEFAULT_KV_BOOTSTRAP_TIMEOUT_MS,
default_cb_cool_down, default_cb_threshold, default_host, default_port,
default_proxy_request_timeout_secs, default_shutdown_drain_secs,
default_stale_request_timeout_secs, resolve_mode, AffinityConfig, AffinityMode, BalancedBy,
CacheAwareConfig, CachePrefixProvider, ChatRoutingKind, CircuitBreakerConfig, Config,
DecodePolicyKind, DiscoveryBackend, DpRankPolicy, EligibilityConfig, FilterKind, FusedTerm,
InflightLoadConfig, K8sDiscoveryConfig, K8sDiscoveryMode, KvIndexerEndpointConfig, LogFormat,
ModelConfig, ObservabilityConfig, PolicyKind, ProxyConfig, ServerConfig, SessionAffinityMode,
StaticUrlsDiscoveryConfig, StickyConfig, StickyFallbackKind, TokenizerBackend, TokenizerConfig,
DEFAULT_FUSE, DEFAULT_KV_BOOTSTRAP_FETCH_TIMEOUT_CAP_MS, DEFAULT_KV_BOOTSTRAP_TIMEOUT_MS,
};
use crate::policies_reorg::admission::AdmissionLimits;
use crate::policies_reorg::factory::AffinitySpec;
@@ -297,14 +297,18 @@ pub struct RoutingArgs {
#[arg(long)]
pub prefix_cache_min_share: Option<f32>,
/// Consecutive upstream failures before opening the breaker. Must be positive; enables the breaker.
#[arg(long)]
pub cb_threshold: Option<NonZeroU32>,
/// Consecutive upstream failures before a worker's circuit breaker opens; must be positive.
#[arg(long, default_value_t = default_cb_threshold())]
pub cb_threshold: NonZeroU32,
/// Circuit-breaker cool-down in seconds. Only meaningful with
/// `--cb-threshold`; defaults to 30 when the breaker is enabled.
#[arg(long)]
pub cb_cool_down_secs: Option<u64>,
/// Seconds an open circuit breaker waits before letting a probe request through.
#[arg(long, default_value_t = default_cb_cool_down())]
pub cb_cool_down_secs: u64,
/// Turn off the per-worker circuit breaker. Under `--worker-urls` nothing else takes
/// a dead or hung worker out of routing.
#[arg(long, conflicts_with_all = ["cb_threshold", "cb_cool_down_secs"])]
pub disable_circuit_breaker: bool,
}
#[derive(clap::Args, Debug)]
@@ -541,7 +545,7 @@ impl Cli {
Some(path) => (Some(load_bucket_config(path)?), None),
None => (None, None),
};
let circuit_breaker = self.routing.build_circuit_breaker()?;
let circuit_breaker = self.routing.build_circuit_breaker();
let kv_bootstrap_timeout_ms = self.cache.kv_bootstrap_timeout_ms;
let kv_bootstrap_fetch_timeout_cap_ms = self.cache.kv_bootstrap_fetch_timeout_cap_ms;
let kv_bootstrap_seed_required = self.cache.kv_bootstrap_seed_required;
@@ -737,18 +741,11 @@ impl DiscoveryArgs {
}
impl RoutingArgs {
fn build_circuit_breaker(&self) -> Result<Option<CircuitBreakerConfig>> {
ensure!(
self.cb_cool_down_secs.is_none() || self.cb_threshold.is_some(),
"--cb-cool-down-secs requires --cb-threshold (the circuit breaker is \
enabled by --cb-threshold)"
);
let circuit_breaker = self.cb_threshold.map(|threshold| CircuitBreakerConfig {
threshold,
cool_down_secs: self.cb_cool_down_secs.unwrap_or_else(default_cb_cool_down),
});
Ok(circuit_breaker)
fn build_circuit_breaker(&self) -> Option<CircuitBreakerConfig> {
(!self.disable_circuit_breaker).then_some(CircuitBreakerConfig {
threshold: self.cb_threshold,
cool_down_secs: self.cb_cool_down_secs,
})
}
fn build_fused(&self) -> Result<Option<Vec<FusedTerm>>> {
@@ -2085,7 +2082,15 @@ mod tests {
}
#[test]
fn cb_threshold_enables_circuit_breaker_with_default_cool_down() {
fn circuit_breaker_is_on_by_default() {
let c = into_config_owned(with_model(&["--worker-urls", "http://x:30000"])).unwrap();
let cb = c.model.circuit_breaker.expect("cb enabled");
assert_eq!(cb.threshold.get(), 3);
assert_eq!(cb.cool_down_secs, 30);
}
#[test]
fn cb_threshold_keeps_default_cool_down() {
let c = into_config_owned(with_model(&[
"--worker-urls",
"http://x:30000",
@@ -2103,30 +2108,40 @@ mod tests {
let c = into_config_owned(with_model(&[
"--worker-urls",
"http://x:30000",
"--cb-threshold",
"3",
"--cb-cool-down-secs",
"10",
]))
.unwrap();
let cb = c.model.circuit_breaker.expect("cb enabled");
assert_eq!(cb.threshold.get(), 3);
assert_eq!(cb.cool_down_secs, 10);
}
#[test]
fn rejects_cb_cool_down_without_threshold() {
let err = into_config_owned(with_model(&[
fn disable_circuit_breaker_turns_it_off() {
let c = into_config_owned(with_model(&[
"--worker-urls",
"http://x:30000",
"--cb-cool-down-secs",
"10",
"--disable-circuit-breaker",
]))
.unwrap_err()
.to_string();
assert!(
err.contains("--cb-cool-down-secs requires --cb-threshold"),
"got: {err}"
);
.unwrap();
assert!(c.model.circuit_breaker.is_none());
}
#[test]
fn disable_circuit_breaker_rejects_breaker_settings() {
for flag in ["--cb-threshold", "--cb-cool-down-secs"] {
let err = into_config_owned(with_model(&[
"--worker-urls",
"http://x:30000",
"--disable-circuit-breaker",
flag,
"5",
]))
.unwrap_err()
.to_string();
assert!(err.contains("cannot be used with"), "{flag}: {err}");
}
}
#[test]
+7 -2
View File
@@ -397,6 +397,7 @@ pub struct ModelConfig {
/// Reorg admission for groups that leave a limit unset: `--max-inflight-requests`
/// and `--max-kv-usage`.
pub reorg_admission: crate::policies_reorg::admission::AdmissionLimits,
/// Per-worker circuit breaker; `None` under `--disable-circuit-breaker`.
pub circuit_breaker: Option<CircuitBreakerConfig>,
/// Cache-Aware prefix configuration.
pub cache_aware: Option<CacheAwareConfig>,
@@ -730,8 +731,12 @@ pub struct CircuitBreakerConfig {
pub cool_down_secs: u64,
}
/// Default circuit-breaker cool-down, applied when `--cb-threshold` is
/// set without an explicit `--cb-cool-down-secs`.
/// Default circuit-breaker threshold, without `--cb-threshold`.
pub fn default_cb_threshold() -> NonZeroU32 {
NonZeroU32::new(3).expect("3 is non-zero")
}
/// Default circuit-breaker cool-down, without `--cb-cool-down-secs`.
pub fn default_cb_cool_down() -> u64 {
30
}
@@ -56,6 +56,8 @@ struct Inner {
pub struct CircuitBreaker {
inner: Mutex<Inner>,
config: CircuitBreakerConfig,
/// A disabled breaker ignores failures, so it never opens.
enabled: bool,
}
impl CircuitBreaker {
@@ -71,11 +73,21 @@ impl CircuitBreaker {
probe_generation: 0,
}),
config,
enabled: true,
}
}
pub fn config(&self) -> &CircuitBreakerConfig {
&self.config
/// The breaker of every worker under `--disable-circuit-breaker`.
pub fn disabled() -> Self {
Self {
enabled: false,
..Self::new()
}
}
/// `None` for a [`disabled`](Self::disabled) breaker.
pub fn config(&self) -> Option<&CircuitBreakerConfig> {
self.enabled.then_some(&self.config)
}
/// Non-mutating predicate: would [`allow`] return `true` if called
@@ -162,6 +174,9 @@ impl CircuitBreaker {
}
pub fn record_failure(&self) {
if !self.enabled {
return;
}
let mut g = self.inner.lock().unwrap();
match g.state {
State::Closed | State::HalfOpen { .. } => {
@@ -344,6 +344,7 @@ fn host_of(worker_url: &str) -> Option<String> {
mod tests {
use super::*;
use crate::discovery::{ModelId, WorkerId, WorkerSpec};
use crate::workers::WireProtocol;
fn spec(id: &str, mode: WorkerMode, model: &str) -> WorkerSpec {
WorkerSpec {
@@ -359,7 +360,7 @@ mod tests {
fn registry(specs: &[WorkerSpec]) -> Arc<WorkerRegistry> {
let r = Arc::new(WorkerRegistry::default());
for s in specs {
let _ = r.add(s.clone());
let _ = r.add_with_cb(s.clone(), Some(Default::default()), WireProtocol::default());
}
r
}
+124 -6
View File
@@ -17,7 +17,7 @@ use anyhow::Context;
use axum::body::Body;
use axum::http::{HeaderMap, HeaderName, HeaderValue, Response};
use bytes::Bytes;
use reqwest::{Client, Url};
use reqwest::{Client, RequestBuilder, Url};
use std::sync::Arc;
use std::time::Duration;
use tokio_util::sync::CancellationToken;
@@ -136,6 +136,8 @@ pub struct Proxy {
fn build_client(protocol: WireProtocol) -> Result<Client, anyhow::Error> {
let builder = Client::builder()
.pool_max_idle_per_host(64)
// Under SGLang's default 5 s keep-alive, so idle connections close here first.
.pool_idle_timeout(Duration::from_secs(3))
.connect_timeout(Duration::from_secs(5));
match protocol {
WireProtocol::Http1 => builder,
@@ -145,6 +147,39 @@ fn build_client(protocol: WireProtocol) -> Result<Client, anyhow::Error> {
.context("build reqwest client")
}
/// Sends `req`, and once more if the worker closed an HTTP/1.1 connection before
/// answering: an engine closing an idle keep-alive connection never read the request.
/// Not for h2c, where one connection carries many requests.
async fn send_retrying_closed_connection(
req: RequestBuilder,
protocol: WireProtocol,
) -> reqwest::Result<reqwest::Response> {
let retry = match protocol {
WireProtocol::Http1 => req.try_clone(),
WireProtocol::H2c => None,
};
match (req.send().await, retry) {
(Err(e), Some(retry)) if closed_before_response(&e) => {
tracing::debug!(error = ?e, "worker closed the connection before answering; resending");
retry.send().await
}
(result, _) => result,
}
}
/// A reset, broken pipe or EOF on an open connection, before any response.
fn closed_before_response(e: &reqwest::Error) -> bool {
use std::io::ErrorKind::{BrokenPipe, ConnectionReset, UnexpectedEof};
!e.is_connect()
&& std::iter::successors(std::error::Error::source(e), |c| c.source()).any(|c| {
c.downcast_ref::<hyper::Error>()
.is_some_and(hyper::Error::is_incomplete_message)
|| c.downcast_ref::<std::io::Error>().is_some_and(|io| {
matches!(io.kind(), ConnectionReset | BrokenPipe | UnexpectedEof)
})
})
}
impl Proxy {
/// Build a proxy. `request_timeout` is the per-request wall-clock budget for
/// non-streaming forwards and for streaming response headers. Connect timeout is hard-coded to 5 s — even a
@@ -248,10 +283,12 @@ impl Proxy {
.timeout(self.request_timeout);
let mut abort =
AbortOnDrop::new(self.client_for(protocol), &worker_url, headers, abort_rid);
let resp = req.send().await.map_err(|e| {
breaker.record_failure();
Self::classify_reqwest_error_for(worker_url.clone(), e, path)
})?;
let resp = send_retrying_closed_connection(req, protocol)
.await
.map_err(|e| {
breaker.record_failure();
Self::classify_reqwest_error_for(worker_url.clone(), e, path)
})?;
let status = resp.status();
// Defer breaker recording until after the body completes — a
// worker that returns 2xx headers and then drops mid-body is
@@ -348,7 +385,12 @@ impl Proxy {
let mut abort =
AbortOnDrop::new(self.client_for(protocol), &worker_url, headers, abort_rid);
// A worker that never answers is a fault, unlike a long stream once it answers.
let resp = match tokio::time::timeout(self.request_timeout, req.send()).await {
let resp = match tokio::time::timeout(
self.request_timeout,
send_retrying_closed_connection(req, protocol),
)
.await
{
Ok(Ok(resp)) => resp,
Ok(Err(e)) => {
breaker.record_failure();
@@ -456,6 +498,7 @@ mod tests {
use axum::Router;
use reqwest::StatusCode;
use std::num::NonZeroU32;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
use tokio::net::TcpListener;
use tokio::sync::oneshot;
@@ -885,4 +928,79 @@ mod tests {
);
}
}
#[derive(Debug, Clone, Copy)]
enum Failure {
/// Close with the request unread, so the kernel resets the connection.
Reset,
/// Read the request, then close.
Close,
/// Read the request and never answer.
Hang,
}
/// A raw HTTP/1.1 worker failing its first `failing` connections; also
/// returns how many connections it accepted.
async fn spawn_failing_worker(failing: usize, failure: Failure) -> (String, Arc<AtomicUsize>) {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let url = format!("http://{}", listener.local_addr().unwrap());
let accepted = Arc::new(AtomicUsize::new(0));
let count = Arc::clone(&accepted);
tokio::spawn(async move {
let mut hung = Vec::new();
while let Ok((mut socket, _)) = listener.accept().await {
let fail = count.fetch_add(1, Ordering::SeqCst) < failing;
if fail && matches!(failure, Failure::Reset) {
let _ = socket.peek(&mut [0]).await;
continue;
}
let mut head = Vec::new();
let mut buf = [0; 1024];
while !head.windows(4).any(|w| w == b"\r\n\r\n") {
match socket.read(&mut buf).await {
Ok(0) | Err(_) => break,
Ok(n) => head.extend_from_slice(&buf[..n]),
}
}
if !fail {
let _ = socket
.write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 2\r\n\r\n{}")
.await;
} else if matches!(failure, Failure::Hang) {
hung.push(socket);
}
}
});
(url, accepted)
}
/// A closed connection is resent once on the same worker; a timeout is not.
#[tokio::test]
async fn closed_connection_is_retried_once() {
for (failing, failure, status, connections) in [
(1, Failure::Reset, StatusCode::OK, 2),
(1, Failure::Close, StatusCode::OK, 2),
(usize::MAX, Failure::Close, StatusCode::BAD_GATEWAY, 2),
(1, Failure::Hang, StatusCode::GATEWAY_TIMEOUT, 1),
] {
let (url, accepted) = spawn_failing_worker(failing, failure).await;
let proxy = Proxy::new(Duration::from_millis(500)).unwrap();
let result = proxy
.forward_json_to(
&url,
WireProtocol::Http1,
&CircuitBreaker::new(),
"/generate",
&HeaderMap::new(),
Bytes::new(),
None,
)
.await;
let got = result.map_or_else(|e| e.status_code(), |r| r.status());
let conns = accepted.load(Ordering::SeqCst);
assert_eq!((got, conns), (status, connections), "{failure:?}");
}
}
}
+21 -5
View File
@@ -34,8 +34,9 @@ const BOOTSTRAP_PORT_RETRY_INTERVAL: Duration = Duration::from_secs(3);
/// Resolve the circuit-breaker config for all model IDs carried by a spec.
///
/// The router serves a single configured model; apply its circuit-breaker
/// config when this worker advertises that model id. Falls back to `None`
/// (default config) otherwise.
/// config when this worker advertises that model id, and the default config
/// otherwise. Under `--disable-circuit-breaker` there is none, so the worker's
/// breaker never opens.
fn cb_config_for_spec(spec: &WorkerSpec, cfg: &Config) -> Option<CircuitBreakerConfig> {
let model = &cfg.model;
let cbc = model.circuit_breaker.as_ref()?;
@@ -45,7 +46,7 @@ fn cb_config_for_spec(spec: &WorkerSpec, cfg: &Config) -> Option<CircuitBreakerC
cool_down: Duration::from_secs(cbc.cool_down_secs),
});
}
None
Some(CircuitBreakerConfig::default())
}
/// Whether `worker_url` is one the router will dial in cleartext.
@@ -119,8 +120,7 @@ pub async fn run(rx: mpsc::Receiver<DiscoveryEvent>, registry: Arc<WorkerRegistr
/// `router_inflight_load` is `None` the active-load bookkeeping is not pruned
/// on worker removal (leaks one `WorkerCounters` slot per departed
/// worker — fine for tests, but production passes `Some(...)`); when
/// `cfg` is `None` the default CB config is used for every worker
/// (threshold = 3).
/// `cfg` is `None` no worker's circuit breaker ever opens.
///
/// Introspects workers with a 2-second timeout, sending `--worker-api-key`
/// when set. Tests that want a tighter timeout call
@@ -683,6 +683,22 @@ mod tests {
assert_eq!(cb.cool_down, Duration::from_secs(60));
}
#[test]
fn cb_config_for_spec_is_none_only_when_disabled() {
let mut cfg = cfg_with_model_cb("m", 5, 60);
let spec = WorkerSpec {
id: WorkerId("w".into()),
url: "http://x".into(),
mode: WorkerMode::Plain,
model_ids: vec![ModelId("other".into())],
..Default::default()
};
let cb = cb_config_for_spec(&spec, &cfg).expect("default breaker for another model");
assert_eq!(cb.threshold.get(), 3);
cfg.model.circuit_breaker = None;
assert!(cb_config_for_spec(&spec, &cfg).is_none());
}
#[test]
fn resolve_protocol_upgrades_to_h2c_only_for_cleartext_http2_engine() {
// The one case that gets h2c: engine self-reports --enable-http2 and
@@ -55,8 +55,8 @@ impl WorkerRegistry {
}
/// Add a worker, optionally supplying a circuit-breaker config, and with
/// the engine profile resolved for it. Pass `None` to use the
/// circuit-breaker default (threshold = 3).
/// the engine profile resolved for it. Pass `None` for a breaker that
/// never opens.
///
/// Re-adding an existing `WorkerId` is an upsert: the prior entry's
/// `by_model` memberships are cleared first so a model that the new
@@ -307,7 +307,21 @@ mod tests {
WireProtocol::default(),
)
.unwrap();
assert_eq!(r.get(&WorkerId("p".into())).unwrap().breaker.config(), &cb);
assert_eq!(
r.get(&WorkerId("p".into())).unwrap().breaker.config(),
Some(&cb)
);
}
#[test]
fn breaker_without_config_never_opens() {
let r = WorkerRegistry::default();
r.add(spec("p", WorkerMode::Prefill, &["m"])).unwrap();
let worker = r.get(&WorkerId("p".into())).unwrap();
for _ in 0..10 {
worker.breaker.record_failure();
}
assert!(worker.breaker.would_allow());
}
#[test]
@@ -264,8 +264,8 @@ impl Worker {
}
/// Construct a worker with an explicit circuit-breaker configuration and
/// engine profile (a bare [`WireProtocol`] means one DP rank). Pass `None`
/// for the default breaker config (threshold = 3, cool_down = 30 s).
/// engine profile (a bare [`WireProtocol`] means one DP rank). Pass `None`,
/// as under `--disable-circuit-breaker`, for a breaker that never opens.
pub fn with_cb_config(
spec: crate::discovery::WorkerSpec,
cb: Option<CircuitBreakerConfig>,
@@ -279,7 +279,7 @@ impl Worker {
} = profile.into();
let breaker = match cb {
Some(cfg) => Arc::new(CircuitBreaker::with_config(cfg)),
None => Arc::new(CircuitBreaker::new()),
None => Arc::new(CircuitBreaker::disabled()),
};
let bootstrap_host = parse_bootstrap_host(&spec.url);
let grpc = grpc_port.and_then(|port| grpc_client(&bootstrap_host, port));
@@ -13,7 +13,7 @@ use sgl_router::policies_reorg::admission::{
};
use sgl_router::policies_reorg::{Pick, PickError, PickRequest, Policy, Rejection, Stage};
use sgl_router::state::kv_events::CacheNamespace;
use sgl_router::workers::{Worker, WorkerRegistry};
use sgl_router::workers::{WireProtocol, Worker, WorkerRegistry};
#[derive(Debug)]
struct TestPolicy {
@@ -104,7 +104,10 @@ fn registry() -> Arc<WorkerRegistry> {
("p", Stage::Prefill, "pd"),
("d", Stage::Decode, "pd"),
] {
workers.add(spec(id, mode, model)).unwrap();
let cb = Some(Default::default());
workers
.add_with_cb(spec(id, mode, model), cb, WireProtocol::default())
.unwrap();
}
let unhealthy = workers.get(&WorkerId("unhealthy".into())).unwrap();
for _ in 0..3 {
@@ -2,7 +2,9 @@
// SPDX-License-Identifier: Apache-2.0
use axum::{routing::get, Json, Router};
use clap::Parser;
use serde_json::{json, Value};
use sgl_router::config::Cli;
use sgl_router::discovery::{DiscoveryEvent, ModelId, WorkerId, WorkerMode, WorkerSpec};
use sgl_router::workers::{manager, WireProtocol, WorkerRegistry};
use std::sync::atomic::{AtomicUsize, Ordering};
@@ -401,7 +403,11 @@ async fn mode_changed_preserves_active_requests_and_breaker() {
let (tx, rx) = mpsc::channel(16);
let registry = Arc::new(WorkerRegistry::default());
let h = tokio::spawn(manager::run(rx, registry.clone()));
let argv = format!("sgl-router --model-id m --worker-urls {url} --cb-threshold 3");
let cli = Cli::try_parse_from(argv.split(' ')).unwrap();
let cfg = Some(Arc::new(cli.into_config().unwrap()));
let run = manager::run_with_config(rx, registry.clone(), cfg, None, None);
let h = tokio::spawn(run);
tx.send(DiscoveryEvent::Added(spec_for(
"w1",
@@ -415,7 +421,7 @@ async fn mode_changed_preserves_active_requests_and_breaker() {
// Grab a handle, bump active_requests, and open the breaker.
let w = registry.get(&WorkerId("w1".into())).unwrap();
w.active_requests.fetch_add(5, Ordering::Relaxed);
// Default threshold is 3 — record 10 failures to guarantee Open state.
// Threshold is 3 — record 10 failures to guarantee Open state.
for _ in 0..10 {
w.breaker.record_failure();
}
@@ -70,13 +70,14 @@ fn build_ctx_with_worker(url: &str) -> Arc<AppContext> {
let cfg = config_for(url);
let tokenizers = Arc::new(TokenizerRegistry::load_from_config(&cfg).unwrap());
let registry = Arc::new(WorkerRegistry::default());
let _ = registry.add(WorkerSpec {
let spec = WorkerSpec {
id: WorkerId("w1".into()),
url: url.to_string(),
mode: WorkerMode::Plain,
model_ids: vec![ModelId("tiny".into())],
..Default::default()
});
};
let _ = registry.add_with_cb(spec, Some(Default::default()), WireProtocol::default());
let policies = Arc::new(build_policy_registry(&cfg).unwrap());
// Per-request worker URLs flow from the registry through
// `forward_*_to(&worker.url, ...)`; the proxy itself is URL-less.
@@ -245,7 +245,9 @@ fn router_ctx_with(
dp_ranks: 1,
grpc_port,
};
registry.add_with_cb(spec, None, profile).unwrap();
registry
.add_with_cb(spec, Some(Default::default()), profile)
.unwrap();
}
if reorg {
cfg.model.policy = PolicyKind::PowerOfTwo;
@@ -29,7 +29,7 @@ use sgl_router::proxy::Proxy;
use sgl_router::server::app::build_router;
use sgl_router::server::app_context::AppContext;
use sgl_router::tokenizer::TokenizerRegistry;
use sgl_router::workers::WorkerRegistry;
use sgl_router::workers::{WireProtocol, WorkerRegistry};
use std::sync::Arc;
use std::time::Duration;
use tower::ServiceExt;
@@ -124,14 +124,15 @@ async fn pd_decode_stream_expires_after_prefill_completes() {
..Default::default()
})
.unwrap();
let spec = WorkerSpec {
id: WorkerId("d1".into()),
url: decode.url.clone(),
mode: WorkerMode::Decode,
model_ids: vec![ModelId("tiny".into())],
..Default::default()
};
registry
.add(WorkerSpec {
id: WorkerId("d1".into()),
url: decode.url.clone(),
mode: WorkerMode::Decode,
model_ids: vec![ModelId("tiny".into())],
..Default::default()
})
.add_with_cb(spec, Some(Default::default()), WireProtocol::default())
.unwrap();
let prefill_worker = registry.get(&WorkerId("p1".into())).unwrap();
let decode_worker = registry.get(&WorkerId("d1".into())).unwrap();
@@ -23,7 +23,7 @@ use sgl_router::proxy::Proxy;
use sgl_router::server::app::build_router;
use sgl_router::server::app_context::AppContext;
use sgl_router::tokenizer::TokenizerRegistry;
use sgl_router::workers::WorkerRegistry;
use sgl_router::workers::{WireProtocol, WorkerRegistry};
use std::sync::Arc;
use std::time::Duration;
use tower::ServiceExt;
@@ -148,14 +148,15 @@ fn hanging_ctx(urls: &[&str], max_attempts: u32) -> Arc<AppContext> {
let tokenizers = Arc::new(TokenizerRegistry::load_from_config(&cfg).unwrap());
let registry = Arc::new(WorkerRegistry::default());
for (i, url) in urls.iter().enumerate() {
let spec = WorkerSpec {
id: WorkerId(format!("w{i}")),
url: (*url).into(),
mode: WorkerMode::Plain,
model_ids: vec![ModelId("tiny".into())],
..Default::default()
};
registry
.add(WorkerSpec {
id: WorkerId(format!("w{i}")),
url: (*url).into(),
mode: WorkerMode::Plain,
model_ids: vec![ModelId("tiny".into())],
..Default::default()
})
.add_with_cb(spec, Some(Default::default()), WireProtocol::default())
.unwrap();
}
let policies = Arc::new(build_policy_registry(&cfg).unwrap());