[Fix] sgl-router: Resend on a closed pooled connection; add --disable-circuit-breaker (#44075)
This commit is contained in:
@@ -53,6 +53,8 @@ http-body = "1"
|
||||
tower = { version = "0.5", features = ["full"] }
|
||||
tower-http = { version = "0.6", features = ["trace", "compression-gzip", "cors", "timeout", "request-id", "catch-panic"] }
|
||||
reqwest = { version = "0.12", features = ["stream", "json", "rustls-tls", "http2"], default-features = false }
|
||||
# Named directly for `Error::is_incomplete_message`, which reqwest does not expose.
|
||||
hyper = "1"
|
||||
sgl-kv-indexer = { path = "sgl-kv-indexer" }
|
||||
# Native SglangService gRPC types, shared with the engine (proto/sglang/runtime/v1).
|
||||
sglang-api-types = { path = "../../rust/sglang-api-types" }
|
||||
|
||||
@@ -325,6 +325,9 @@ unset fields inherit the CLI flags. See
|
||||
`session_aware`); see [optional tokenizer](REFERENCE.md#optional-tokenizer-for-load-only-routing).
|
||||
- **Retries**: `--retry-max-attempts N` resends a request that failed before
|
||||
any response to another worker; see [retries](REFERENCE.md#retries).
|
||||
- **Circuit breaker**: a worker leaves routing after `--cb-threshold`
|
||||
consecutive failures; `--disable-circuit-breaker` turns this off; see
|
||||
[circuit breaker](REFERENCE.md#circuit-breaker).
|
||||
- **DP-rank routing**: `--dp-aware` also picks the DP rank inside multi-rank
|
||||
engines, and `--dp-rank-policy round_robin` cycles ranks that no routing key
|
||||
or cached prefix decides; see [DP-rank routing](REFERENCE.md#dp-rank-routing).
|
||||
|
||||
@@ -218,6 +218,22 @@ doubles per retry, and is capped at `--retry-max-backoff-ms` (default 2000).
|
||||
Set the initial backoff to 0 to retry immediately. A backoff never waits past
|
||||
the request's stale deadline.
|
||||
|
||||
Separately, a send whose pooled HTTP/1.1 connection the worker closed before
|
||||
answering (reset, broken pipe or EOF) is repeated once, immediately, on the same
|
||||
worker: an engine closing an idle connection never read the request. Idle
|
||||
connections leave the pool after 3 s, under SGLang's default 5 s keep-alive.
|
||||
|
||||
### Circuit breaker
|
||||
|
||||
Each worker has a circuit breaker: after `--cb-threshold` (default 3)
|
||||
consecutive failures the worker is taken out of routing until a request already
|
||||
sent to it succeeds or, once `--cb-cool-down-secs` (default 30) has passed, a
|
||||
probe request does. Under `--worker-urls` it is the only thing that takes a
|
||||
dead or hung worker out of routing, since there is no active health check.
|
||||
`--disable-circuit-breaker` turns it off, for example when one worker serves a
|
||||
role and an open breaker would reject all of that role's requests; a worker
|
||||
that stops answering then keeps its share of traffic.
|
||||
|
||||
### Engines with `--api-key`
|
||||
|
||||
The router reads each worker's `/server_info` and `/model_info` to learn its
|
||||
|
||||
@@ -11,15 +11,15 @@ use std::num::NonZeroU32;
|
||||
use crate::config::sampling::{parse_sampling_overrides, ConflictPolicy};
|
||||
use crate::config::types::is_selector_empty;
|
||||
use crate::config::{
|
||||
default_cb_cool_down, default_host, default_port, default_proxy_request_timeout_secs,
|
||||
default_shutdown_drain_secs, default_stale_request_timeout_secs, resolve_mode, AffinityConfig,
|
||||
AffinityMode, BalancedBy, CacheAwareConfig, CachePrefixProvider, ChatRoutingKind,
|
||||
CircuitBreakerConfig, Config, DecodePolicyKind, DiscoveryBackend, DpRankPolicy,
|
||||
EligibilityConfig, FilterKind, FusedTerm, InflightLoadConfig, K8sDiscoveryConfig,
|
||||
K8sDiscoveryMode, KvIndexerEndpointConfig, LogFormat, ModelConfig, ObservabilityConfig,
|
||||
PolicyKind, ProxyConfig, ServerConfig, SessionAffinityMode, StaticUrlsDiscoveryConfig,
|
||||
StickyConfig, StickyFallbackKind, TokenizerBackend, TokenizerConfig, DEFAULT_FUSE,
|
||||
DEFAULT_KV_BOOTSTRAP_FETCH_TIMEOUT_CAP_MS, DEFAULT_KV_BOOTSTRAP_TIMEOUT_MS,
|
||||
default_cb_cool_down, default_cb_threshold, default_host, default_port,
|
||||
default_proxy_request_timeout_secs, default_shutdown_drain_secs,
|
||||
default_stale_request_timeout_secs, resolve_mode, AffinityConfig, AffinityMode, BalancedBy,
|
||||
CacheAwareConfig, CachePrefixProvider, ChatRoutingKind, CircuitBreakerConfig, Config,
|
||||
DecodePolicyKind, DiscoveryBackend, DpRankPolicy, EligibilityConfig, FilterKind, FusedTerm,
|
||||
InflightLoadConfig, K8sDiscoveryConfig, K8sDiscoveryMode, KvIndexerEndpointConfig, LogFormat,
|
||||
ModelConfig, ObservabilityConfig, PolicyKind, ProxyConfig, ServerConfig, SessionAffinityMode,
|
||||
StaticUrlsDiscoveryConfig, StickyConfig, StickyFallbackKind, TokenizerBackend, TokenizerConfig,
|
||||
DEFAULT_FUSE, DEFAULT_KV_BOOTSTRAP_FETCH_TIMEOUT_CAP_MS, DEFAULT_KV_BOOTSTRAP_TIMEOUT_MS,
|
||||
};
|
||||
use crate::policies_reorg::admission::AdmissionLimits;
|
||||
use crate::policies_reorg::factory::AffinitySpec;
|
||||
@@ -297,14 +297,18 @@ pub struct RoutingArgs {
|
||||
#[arg(long)]
|
||||
pub prefix_cache_min_share: Option<f32>,
|
||||
|
||||
/// Consecutive upstream failures before opening the breaker. Must be positive; enables the breaker.
|
||||
#[arg(long)]
|
||||
pub cb_threshold: Option<NonZeroU32>,
|
||||
/// Consecutive upstream failures before a worker's circuit breaker opens; must be positive.
|
||||
#[arg(long, default_value_t = default_cb_threshold())]
|
||||
pub cb_threshold: NonZeroU32,
|
||||
|
||||
/// Circuit-breaker cool-down in seconds. Only meaningful with
|
||||
/// `--cb-threshold`; defaults to 30 when the breaker is enabled.
|
||||
#[arg(long)]
|
||||
pub cb_cool_down_secs: Option<u64>,
|
||||
/// Seconds an open circuit breaker waits before letting a probe request through.
|
||||
#[arg(long, default_value_t = default_cb_cool_down())]
|
||||
pub cb_cool_down_secs: u64,
|
||||
|
||||
/// Turn off the per-worker circuit breaker. Under `--worker-urls` nothing else takes
|
||||
/// a dead or hung worker out of routing.
|
||||
#[arg(long, conflicts_with_all = ["cb_threshold", "cb_cool_down_secs"])]
|
||||
pub disable_circuit_breaker: bool,
|
||||
}
|
||||
|
||||
#[derive(clap::Args, Debug)]
|
||||
@@ -541,7 +545,7 @@ impl Cli {
|
||||
Some(path) => (Some(load_bucket_config(path)?), None),
|
||||
None => (None, None),
|
||||
};
|
||||
let circuit_breaker = self.routing.build_circuit_breaker()?;
|
||||
let circuit_breaker = self.routing.build_circuit_breaker();
|
||||
let kv_bootstrap_timeout_ms = self.cache.kv_bootstrap_timeout_ms;
|
||||
let kv_bootstrap_fetch_timeout_cap_ms = self.cache.kv_bootstrap_fetch_timeout_cap_ms;
|
||||
let kv_bootstrap_seed_required = self.cache.kv_bootstrap_seed_required;
|
||||
@@ -737,18 +741,11 @@ impl DiscoveryArgs {
|
||||
}
|
||||
|
||||
impl RoutingArgs {
|
||||
fn build_circuit_breaker(&self) -> Result<Option<CircuitBreakerConfig>> {
|
||||
ensure!(
|
||||
self.cb_cool_down_secs.is_none() || self.cb_threshold.is_some(),
|
||||
"--cb-cool-down-secs requires --cb-threshold (the circuit breaker is \
|
||||
enabled by --cb-threshold)"
|
||||
);
|
||||
let circuit_breaker = self.cb_threshold.map(|threshold| CircuitBreakerConfig {
|
||||
threshold,
|
||||
cool_down_secs: self.cb_cool_down_secs.unwrap_or_else(default_cb_cool_down),
|
||||
});
|
||||
|
||||
Ok(circuit_breaker)
|
||||
fn build_circuit_breaker(&self) -> Option<CircuitBreakerConfig> {
|
||||
(!self.disable_circuit_breaker).then_some(CircuitBreakerConfig {
|
||||
threshold: self.cb_threshold,
|
||||
cool_down_secs: self.cb_cool_down_secs,
|
||||
})
|
||||
}
|
||||
|
||||
fn build_fused(&self) -> Result<Option<Vec<FusedTerm>>> {
|
||||
@@ -2085,7 +2082,15 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cb_threshold_enables_circuit_breaker_with_default_cool_down() {
|
||||
fn circuit_breaker_is_on_by_default() {
|
||||
let c = into_config_owned(with_model(&["--worker-urls", "http://x:30000"])).unwrap();
|
||||
let cb = c.model.circuit_breaker.expect("cb enabled");
|
||||
assert_eq!(cb.threshold.get(), 3);
|
||||
assert_eq!(cb.cool_down_secs, 30);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cb_threshold_keeps_default_cool_down() {
|
||||
let c = into_config_owned(with_model(&[
|
||||
"--worker-urls",
|
||||
"http://x:30000",
|
||||
@@ -2103,30 +2108,40 @@ mod tests {
|
||||
let c = into_config_owned(with_model(&[
|
||||
"--worker-urls",
|
||||
"http://x:30000",
|
||||
"--cb-threshold",
|
||||
"3",
|
||||
"--cb-cool-down-secs",
|
||||
"10",
|
||||
]))
|
||||
.unwrap();
|
||||
let cb = c.model.circuit_breaker.expect("cb enabled");
|
||||
assert_eq!(cb.threshold.get(), 3);
|
||||
assert_eq!(cb.cool_down_secs, 10);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_cb_cool_down_without_threshold() {
|
||||
let err = into_config_owned(with_model(&[
|
||||
fn disable_circuit_breaker_turns_it_off() {
|
||||
let c = into_config_owned(with_model(&[
|
||||
"--worker-urls",
|
||||
"http://x:30000",
|
||||
"--cb-cool-down-secs",
|
||||
"10",
|
||||
"--disable-circuit-breaker",
|
||||
]))
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(
|
||||
err.contains("--cb-cool-down-secs requires --cb-threshold"),
|
||||
"got: {err}"
|
||||
);
|
||||
.unwrap();
|
||||
assert!(c.model.circuit_breaker.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disable_circuit_breaker_rejects_breaker_settings() {
|
||||
for flag in ["--cb-threshold", "--cb-cool-down-secs"] {
|
||||
let err = into_config_owned(with_model(&[
|
||||
"--worker-urls",
|
||||
"http://x:30000",
|
||||
"--disable-circuit-breaker",
|
||||
flag,
|
||||
"5",
|
||||
]))
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(err.contains("cannot be used with"), "{flag}: {err}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -397,6 +397,7 @@ pub struct ModelConfig {
|
||||
/// Reorg admission for groups that leave a limit unset: `--max-inflight-requests`
|
||||
/// and `--max-kv-usage`.
|
||||
pub reorg_admission: crate::policies_reorg::admission::AdmissionLimits,
|
||||
/// Per-worker circuit breaker; `None` under `--disable-circuit-breaker`.
|
||||
pub circuit_breaker: Option<CircuitBreakerConfig>,
|
||||
/// Cache-Aware prefix configuration.
|
||||
pub cache_aware: Option<CacheAwareConfig>,
|
||||
@@ -730,8 +731,12 @@ pub struct CircuitBreakerConfig {
|
||||
pub cool_down_secs: u64,
|
||||
}
|
||||
|
||||
/// Default circuit-breaker cool-down, applied when `--cb-threshold` is
|
||||
/// set without an explicit `--cb-cool-down-secs`.
|
||||
/// Default circuit-breaker threshold, without `--cb-threshold`.
|
||||
pub fn default_cb_threshold() -> NonZeroU32 {
|
||||
NonZeroU32::new(3).expect("3 is non-zero")
|
||||
}
|
||||
|
||||
/// Default circuit-breaker cool-down, without `--cb-cool-down-secs`.
|
||||
pub fn default_cb_cool_down() -> u64 {
|
||||
30
|
||||
}
|
||||
|
||||
@@ -56,6 +56,8 @@ struct Inner {
|
||||
pub struct CircuitBreaker {
|
||||
inner: Mutex<Inner>,
|
||||
config: CircuitBreakerConfig,
|
||||
/// A disabled breaker ignores failures, so it never opens.
|
||||
enabled: bool,
|
||||
}
|
||||
|
||||
impl CircuitBreaker {
|
||||
@@ -71,11 +73,21 @@ impl CircuitBreaker {
|
||||
probe_generation: 0,
|
||||
}),
|
||||
config,
|
||||
enabled: true,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn config(&self) -> &CircuitBreakerConfig {
|
||||
&self.config
|
||||
/// The breaker of every worker under `--disable-circuit-breaker`.
|
||||
pub fn disabled() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
..Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// `None` for a [`disabled`](Self::disabled) breaker.
|
||||
pub fn config(&self) -> Option<&CircuitBreakerConfig> {
|
||||
self.enabled.then_some(&self.config)
|
||||
}
|
||||
|
||||
/// Non-mutating predicate: would [`allow`] return `true` if called
|
||||
@@ -162,6 +174,9 @@ impl CircuitBreaker {
|
||||
}
|
||||
|
||||
pub fn record_failure(&self) {
|
||||
if !self.enabled {
|
||||
return;
|
||||
}
|
||||
let mut g = self.inner.lock().unwrap();
|
||||
match g.state {
|
||||
State::Closed | State::HalfOpen { .. } => {
|
||||
|
||||
@@ -344,6 +344,7 @@ fn host_of(worker_url: &str) -> Option<String> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::discovery::{ModelId, WorkerId, WorkerSpec};
|
||||
use crate::workers::WireProtocol;
|
||||
|
||||
fn spec(id: &str, mode: WorkerMode, model: &str) -> WorkerSpec {
|
||||
WorkerSpec {
|
||||
@@ -359,7 +360,7 @@ mod tests {
|
||||
fn registry(specs: &[WorkerSpec]) -> Arc<WorkerRegistry> {
|
||||
let r = Arc::new(WorkerRegistry::default());
|
||||
for s in specs {
|
||||
let _ = r.add(s.clone());
|
||||
let _ = r.add_with_cb(s.clone(), Some(Default::default()), WireProtocol::default());
|
||||
}
|
||||
r
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ use anyhow::Context;
|
||||
use axum::body::Body;
|
||||
use axum::http::{HeaderMap, HeaderName, HeaderValue, Response};
|
||||
use bytes::Bytes;
|
||||
use reqwest::{Client, Url};
|
||||
use reqwest::{Client, RequestBuilder, Url};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
@@ -136,6 +136,8 @@ pub struct Proxy {
|
||||
fn build_client(protocol: WireProtocol) -> Result<Client, anyhow::Error> {
|
||||
let builder = Client::builder()
|
||||
.pool_max_idle_per_host(64)
|
||||
// Under SGLang's default 5 s keep-alive, so idle connections close here first.
|
||||
.pool_idle_timeout(Duration::from_secs(3))
|
||||
.connect_timeout(Duration::from_secs(5));
|
||||
match protocol {
|
||||
WireProtocol::Http1 => builder,
|
||||
@@ -145,6 +147,39 @@ fn build_client(protocol: WireProtocol) -> Result<Client, anyhow::Error> {
|
||||
.context("build reqwest client")
|
||||
}
|
||||
|
||||
/// Sends `req`, and once more if the worker closed an HTTP/1.1 connection before
|
||||
/// answering: an engine closing an idle keep-alive connection never read the request.
|
||||
/// Not for h2c, where one connection carries many requests.
|
||||
async fn send_retrying_closed_connection(
|
||||
req: RequestBuilder,
|
||||
protocol: WireProtocol,
|
||||
) -> reqwest::Result<reqwest::Response> {
|
||||
let retry = match protocol {
|
||||
WireProtocol::Http1 => req.try_clone(),
|
||||
WireProtocol::H2c => None,
|
||||
};
|
||||
match (req.send().await, retry) {
|
||||
(Err(e), Some(retry)) if closed_before_response(&e) => {
|
||||
tracing::debug!(error = ?e, "worker closed the connection before answering; resending");
|
||||
retry.send().await
|
||||
}
|
||||
(result, _) => result,
|
||||
}
|
||||
}
|
||||
|
||||
/// A reset, broken pipe or EOF on an open connection, before any response.
|
||||
fn closed_before_response(e: &reqwest::Error) -> bool {
|
||||
use std::io::ErrorKind::{BrokenPipe, ConnectionReset, UnexpectedEof};
|
||||
!e.is_connect()
|
||||
&& std::iter::successors(std::error::Error::source(e), |c| c.source()).any(|c| {
|
||||
c.downcast_ref::<hyper::Error>()
|
||||
.is_some_and(hyper::Error::is_incomplete_message)
|
||||
|| c.downcast_ref::<std::io::Error>().is_some_and(|io| {
|
||||
matches!(io.kind(), ConnectionReset | BrokenPipe | UnexpectedEof)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
impl Proxy {
|
||||
/// Build a proxy. `request_timeout` is the per-request wall-clock budget for
|
||||
/// non-streaming forwards and for streaming response headers. Connect timeout is hard-coded to 5 s — even a
|
||||
@@ -248,10 +283,12 @@ impl Proxy {
|
||||
.timeout(self.request_timeout);
|
||||
let mut abort =
|
||||
AbortOnDrop::new(self.client_for(protocol), &worker_url, headers, abort_rid);
|
||||
let resp = req.send().await.map_err(|e| {
|
||||
breaker.record_failure();
|
||||
Self::classify_reqwest_error_for(worker_url.clone(), e, path)
|
||||
})?;
|
||||
let resp = send_retrying_closed_connection(req, protocol)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
breaker.record_failure();
|
||||
Self::classify_reqwest_error_for(worker_url.clone(), e, path)
|
||||
})?;
|
||||
let status = resp.status();
|
||||
// Defer breaker recording until after the body completes — a
|
||||
// worker that returns 2xx headers and then drops mid-body is
|
||||
@@ -348,7 +385,12 @@ impl Proxy {
|
||||
let mut abort =
|
||||
AbortOnDrop::new(self.client_for(protocol), &worker_url, headers, abort_rid);
|
||||
// A worker that never answers is a fault, unlike a long stream once it answers.
|
||||
let resp = match tokio::time::timeout(self.request_timeout, req.send()).await {
|
||||
let resp = match tokio::time::timeout(
|
||||
self.request_timeout,
|
||||
send_retrying_closed_connection(req, protocol),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Ok(resp)) => resp,
|
||||
Ok(Err(e)) => {
|
||||
breaker.record_failure();
|
||||
@@ -456,6 +498,7 @@ mod tests {
|
||||
use axum::Router;
|
||||
use reqwest::StatusCode;
|
||||
use std::num::NonZeroU32;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::sync::oneshot;
|
||||
@@ -885,4 +928,79 @@ mod tests {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
enum Failure {
|
||||
/// Close with the request unread, so the kernel resets the connection.
|
||||
Reset,
|
||||
/// Read the request, then close.
|
||||
Close,
|
||||
/// Read the request and never answer.
|
||||
Hang,
|
||||
}
|
||||
|
||||
/// A raw HTTP/1.1 worker failing its first `failing` connections; also
|
||||
/// returns how many connections it accepted.
|
||||
async fn spawn_failing_worker(failing: usize, failure: Failure) -> (String, Arc<AtomicUsize>) {
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let url = format!("http://{}", listener.local_addr().unwrap());
|
||||
let accepted = Arc::new(AtomicUsize::new(0));
|
||||
let count = Arc::clone(&accepted);
|
||||
tokio::spawn(async move {
|
||||
let mut hung = Vec::new();
|
||||
while let Ok((mut socket, _)) = listener.accept().await {
|
||||
let fail = count.fetch_add(1, Ordering::SeqCst) < failing;
|
||||
if fail && matches!(failure, Failure::Reset) {
|
||||
let _ = socket.peek(&mut [0]).await;
|
||||
continue;
|
||||
}
|
||||
let mut head = Vec::new();
|
||||
let mut buf = [0; 1024];
|
||||
while !head.windows(4).any(|w| w == b"\r\n\r\n") {
|
||||
match socket.read(&mut buf).await {
|
||||
Ok(0) | Err(_) => break,
|
||||
Ok(n) => head.extend_from_slice(&buf[..n]),
|
||||
}
|
||||
}
|
||||
if !fail {
|
||||
let _ = socket
|
||||
.write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 2\r\n\r\n{}")
|
||||
.await;
|
||||
} else if matches!(failure, Failure::Hang) {
|
||||
hung.push(socket);
|
||||
}
|
||||
}
|
||||
});
|
||||
(url, accepted)
|
||||
}
|
||||
|
||||
/// A closed connection is resent once on the same worker; a timeout is not.
|
||||
#[tokio::test]
|
||||
async fn closed_connection_is_retried_once() {
|
||||
for (failing, failure, status, connections) in [
|
||||
(1, Failure::Reset, StatusCode::OK, 2),
|
||||
(1, Failure::Close, StatusCode::OK, 2),
|
||||
(usize::MAX, Failure::Close, StatusCode::BAD_GATEWAY, 2),
|
||||
(1, Failure::Hang, StatusCode::GATEWAY_TIMEOUT, 1),
|
||||
] {
|
||||
let (url, accepted) = spawn_failing_worker(failing, failure).await;
|
||||
let proxy = Proxy::new(Duration::from_millis(500)).unwrap();
|
||||
let result = proxy
|
||||
.forward_json_to(
|
||||
&url,
|
||||
WireProtocol::Http1,
|
||||
&CircuitBreaker::new(),
|
||||
"/generate",
|
||||
&HeaderMap::new(),
|
||||
Bytes::new(),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
let got = result.map_or_else(|e| e.status_code(), |r| r.status());
|
||||
let conns = accepted.load(Ordering::SeqCst);
|
||||
assert_eq!((got, conns), (status, connections), "{failure:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,8 +34,9 @@ const BOOTSTRAP_PORT_RETRY_INTERVAL: Duration = Duration::from_secs(3);
|
||||
/// Resolve the circuit-breaker config for all model IDs carried by a spec.
|
||||
///
|
||||
/// The router serves a single configured model; apply its circuit-breaker
|
||||
/// config when this worker advertises that model id. Falls back to `None`
|
||||
/// (default config) otherwise.
|
||||
/// config when this worker advertises that model id, and the default config
|
||||
/// otherwise. Under `--disable-circuit-breaker` there is none, so the worker's
|
||||
/// breaker never opens.
|
||||
fn cb_config_for_spec(spec: &WorkerSpec, cfg: &Config) -> Option<CircuitBreakerConfig> {
|
||||
let model = &cfg.model;
|
||||
let cbc = model.circuit_breaker.as_ref()?;
|
||||
@@ -45,7 +46,7 @@ fn cb_config_for_spec(spec: &WorkerSpec, cfg: &Config) -> Option<CircuitBreakerC
|
||||
cool_down: Duration::from_secs(cbc.cool_down_secs),
|
||||
});
|
||||
}
|
||||
None
|
||||
Some(CircuitBreakerConfig::default())
|
||||
}
|
||||
|
||||
/// Whether `worker_url` is one the router will dial in cleartext.
|
||||
@@ -119,8 +120,7 @@ pub async fn run(rx: mpsc::Receiver<DiscoveryEvent>, registry: Arc<WorkerRegistr
|
||||
/// `router_inflight_load` is `None` the active-load bookkeeping is not pruned
|
||||
/// on worker removal (leaks one `WorkerCounters` slot per departed
|
||||
/// worker — fine for tests, but production passes `Some(...)`); when
|
||||
/// `cfg` is `None` the default CB config is used for every worker
|
||||
/// (threshold = 3).
|
||||
/// `cfg` is `None` no worker's circuit breaker ever opens.
|
||||
///
|
||||
/// Introspects workers with a 2-second timeout, sending `--worker-api-key`
|
||||
/// when set. Tests that want a tighter timeout call
|
||||
@@ -683,6 +683,22 @@ mod tests {
|
||||
assert_eq!(cb.cool_down, Duration::from_secs(60));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cb_config_for_spec_is_none_only_when_disabled() {
|
||||
let mut cfg = cfg_with_model_cb("m", 5, 60);
|
||||
let spec = WorkerSpec {
|
||||
id: WorkerId("w".into()),
|
||||
url: "http://x".into(),
|
||||
mode: WorkerMode::Plain,
|
||||
model_ids: vec![ModelId("other".into())],
|
||||
..Default::default()
|
||||
};
|
||||
let cb = cb_config_for_spec(&spec, &cfg).expect("default breaker for another model");
|
||||
assert_eq!(cb.threshold.get(), 3);
|
||||
cfg.model.circuit_breaker = None;
|
||||
assert!(cb_config_for_spec(&spec, &cfg).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_protocol_upgrades_to_h2c_only_for_cleartext_http2_engine() {
|
||||
// The one case that gets h2c: engine self-reports --enable-http2 and
|
||||
|
||||
@@ -55,8 +55,8 @@ impl WorkerRegistry {
|
||||
}
|
||||
|
||||
/// Add a worker, optionally supplying a circuit-breaker config, and with
|
||||
/// the engine profile resolved for it. Pass `None` to use the
|
||||
/// circuit-breaker default (threshold = 3).
|
||||
/// the engine profile resolved for it. Pass `None` for a breaker that
|
||||
/// never opens.
|
||||
///
|
||||
/// Re-adding an existing `WorkerId` is an upsert: the prior entry's
|
||||
/// `by_model` memberships are cleared first so a model that the new
|
||||
@@ -307,7 +307,21 @@ mod tests {
|
||||
WireProtocol::default(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(r.get(&WorkerId("p".into())).unwrap().breaker.config(), &cb);
|
||||
assert_eq!(
|
||||
r.get(&WorkerId("p".into())).unwrap().breaker.config(),
|
||||
Some(&cb)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn breaker_without_config_never_opens() {
|
||||
let r = WorkerRegistry::default();
|
||||
r.add(spec("p", WorkerMode::Prefill, &["m"])).unwrap();
|
||||
let worker = r.get(&WorkerId("p".into())).unwrap();
|
||||
for _ in 0..10 {
|
||||
worker.breaker.record_failure();
|
||||
}
|
||||
assert!(worker.breaker.would_allow());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -264,8 +264,8 @@ impl Worker {
|
||||
}
|
||||
|
||||
/// Construct a worker with an explicit circuit-breaker configuration and
|
||||
/// engine profile (a bare [`WireProtocol`] means one DP rank). Pass `None`
|
||||
/// for the default breaker config (threshold = 3, cool_down = 30 s).
|
||||
/// engine profile (a bare [`WireProtocol`] means one DP rank). Pass `None`,
|
||||
/// as under `--disable-circuit-breaker`, for a breaker that never opens.
|
||||
pub fn with_cb_config(
|
||||
spec: crate::discovery::WorkerSpec,
|
||||
cb: Option<CircuitBreakerConfig>,
|
||||
@@ -279,7 +279,7 @@ impl Worker {
|
||||
} = profile.into();
|
||||
let breaker = match cb {
|
||||
Some(cfg) => Arc::new(CircuitBreaker::with_config(cfg)),
|
||||
None => Arc::new(CircuitBreaker::new()),
|
||||
None => Arc::new(CircuitBreaker::disabled()),
|
||||
};
|
||||
let bootstrap_host = parse_bootstrap_host(&spec.url);
|
||||
let grpc = grpc_port.and_then(|port| grpc_client(&bootstrap_host, port));
|
||||
|
||||
@@ -13,7 +13,7 @@ use sgl_router::policies_reorg::admission::{
|
||||
};
|
||||
use sgl_router::policies_reorg::{Pick, PickError, PickRequest, Policy, Rejection, Stage};
|
||||
use sgl_router::state::kv_events::CacheNamespace;
|
||||
use sgl_router::workers::{Worker, WorkerRegistry};
|
||||
use sgl_router::workers::{WireProtocol, Worker, WorkerRegistry};
|
||||
|
||||
#[derive(Debug)]
|
||||
struct TestPolicy {
|
||||
@@ -104,7 +104,10 @@ fn registry() -> Arc<WorkerRegistry> {
|
||||
("p", Stage::Prefill, "pd"),
|
||||
("d", Stage::Decode, "pd"),
|
||||
] {
|
||||
workers.add(spec(id, mode, model)).unwrap();
|
||||
let cb = Some(Default::default());
|
||||
workers
|
||||
.add_with_cb(spec(id, mode, model), cb, WireProtocol::default())
|
||||
.unwrap();
|
||||
}
|
||||
let unhealthy = workers.get(&WorkerId("unhealthy".into())).unwrap();
|
||||
for _ in 0..3 {
|
||||
|
||||
@@ -2,7 +2,9 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use axum::{routing::get, Json, Router};
|
||||
use clap::Parser;
|
||||
use serde_json::{json, Value};
|
||||
use sgl_router::config::Cli;
|
||||
use sgl_router::discovery::{DiscoveryEvent, ModelId, WorkerId, WorkerMode, WorkerSpec};
|
||||
use sgl_router::workers::{manager, WireProtocol, WorkerRegistry};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
@@ -401,7 +403,11 @@ async fn mode_changed_preserves_active_requests_and_breaker() {
|
||||
|
||||
let (tx, rx) = mpsc::channel(16);
|
||||
let registry = Arc::new(WorkerRegistry::default());
|
||||
let h = tokio::spawn(manager::run(rx, registry.clone()));
|
||||
let argv = format!("sgl-router --model-id m --worker-urls {url} --cb-threshold 3");
|
||||
let cli = Cli::try_parse_from(argv.split(' ')).unwrap();
|
||||
let cfg = Some(Arc::new(cli.into_config().unwrap()));
|
||||
let run = manager::run_with_config(rx, registry.clone(), cfg, None, None);
|
||||
let h = tokio::spawn(run);
|
||||
|
||||
tx.send(DiscoveryEvent::Added(spec_for(
|
||||
"w1",
|
||||
@@ -415,7 +421,7 @@ async fn mode_changed_preserves_active_requests_and_breaker() {
|
||||
// Grab a handle, bump active_requests, and open the breaker.
|
||||
let w = registry.get(&WorkerId("w1".into())).unwrap();
|
||||
w.active_requests.fetch_add(5, Ordering::Relaxed);
|
||||
// Default threshold is 3 — record 10 failures to guarantee Open state.
|
||||
// Threshold is 3 — record 10 failures to guarantee Open state.
|
||||
for _ in 0..10 {
|
||||
w.breaker.record_failure();
|
||||
}
|
||||
|
||||
@@ -70,13 +70,14 @@ fn build_ctx_with_worker(url: &str) -> Arc<AppContext> {
|
||||
let cfg = config_for(url);
|
||||
let tokenizers = Arc::new(TokenizerRegistry::load_from_config(&cfg).unwrap());
|
||||
let registry = Arc::new(WorkerRegistry::default());
|
||||
let _ = registry.add(WorkerSpec {
|
||||
let spec = WorkerSpec {
|
||||
id: WorkerId("w1".into()),
|
||||
url: url.to_string(),
|
||||
mode: WorkerMode::Plain,
|
||||
model_ids: vec![ModelId("tiny".into())],
|
||||
..Default::default()
|
||||
});
|
||||
};
|
||||
let _ = registry.add_with_cb(spec, Some(Default::default()), WireProtocol::default());
|
||||
let policies = Arc::new(build_policy_registry(&cfg).unwrap());
|
||||
// Per-request worker URLs flow from the registry through
|
||||
// `forward_*_to(&worker.url, ...)`; the proxy itself is URL-less.
|
||||
|
||||
@@ -245,7 +245,9 @@ fn router_ctx_with(
|
||||
dp_ranks: 1,
|
||||
grpc_port,
|
||||
};
|
||||
registry.add_with_cb(spec, None, profile).unwrap();
|
||||
registry
|
||||
.add_with_cb(spec, Some(Default::default()), profile)
|
||||
.unwrap();
|
||||
}
|
||||
if reorg {
|
||||
cfg.model.policy = PolicyKind::PowerOfTwo;
|
||||
|
||||
@@ -29,7 +29,7 @@ use sgl_router::proxy::Proxy;
|
||||
use sgl_router::server::app::build_router;
|
||||
use sgl_router::server::app_context::AppContext;
|
||||
use sgl_router::tokenizer::TokenizerRegistry;
|
||||
use sgl_router::workers::WorkerRegistry;
|
||||
use sgl_router::workers::{WireProtocol, WorkerRegistry};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tower::ServiceExt;
|
||||
@@ -124,14 +124,15 @@ async fn pd_decode_stream_expires_after_prefill_completes() {
|
||||
..Default::default()
|
||||
})
|
||||
.unwrap();
|
||||
let spec = WorkerSpec {
|
||||
id: WorkerId("d1".into()),
|
||||
url: decode.url.clone(),
|
||||
mode: WorkerMode::Decode,
|
||||
model_ids: vec![ModelId("tiny".into())],
|
||||
..Default::default()
|
||||
};
|
||||
registry
|
||||
.add(WorkerSpec {
|
||||
id: WorkerId("d1".into()),
|
||||
url: decode.url.clone(),
|
||||
mode: WorkerMode::Decode,
|
||||
model_ids: vec![ModelId("tiny".into())],
|
||||
..Default::default()
|
||||
})
|
||||
.add_with_cb(spec, Some(Default::default()), WireProtocol::default())
|
||||
.unwrap();
|
||||
let prefill_worker = registry.get(&WorkerId("p1".into())).unwrap();
|
||||
let decode_worker = registry.get(&WorkerId("d1".into())).unwrap();
|
||||
|
||||
@@ -23,7 +23,7 @@ use sgl_router::proxy::Proxy;
|
||||
use sgl_router::server::app::build_router;
|
||||
use sgl_router::server::app_context::AppContext;
|
||||
use sgl_router::tokenizer::TokenizerRegistry;
|
||||
use sgl_router::workers::WorkerRegistry;
|
||||
use sgl_router::workers::{WireProtocol, WorkerRegistry};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tower::ServiceExt;
|
||||
@@ -148,14 +148,15 @@ fn hanging_ctx(urls: &[&str], max_attempts: u32) -> Arc<AppContext> {
|
||||
let tokenizers = Arc::new(TokenizerRegistry::load_from_config(&cfg).unwrap());
|
||||
let registry = Arc::new(WorkerRegistry::default());
|
||||
for (i, url) in urls.iter().enumerate() {
|
||||
let spec = WorkerSpec {
|
||||
id: WorkerId(format!("w{i}")),
|
||||
url: (*url).into(),
|
||||
mode: WorkerMode::Plain,
|
||||
model_ids: vec![ModelId("tiny".into())],
|
||||
..Default::default()
|
||||
};
|
||||
registry
|
||||
.add(WorkerSpec {
|
||||
id: WorkerId(format!("w{i}")),
|
||||
url: (*url).into(),
|
||||
mode: WorkerMode::Plain,
|
||||
model_ids: vec![ModelId("tiny".into())],
|
||||
..Default::default()
|
||||
})
|
||||
.add_with_cb(spec, Some(Default::default()), WireProtocol::default())
|
||||
.unwrap();
|
||||
}
|
||||
let policies = Arc::new(build_policy_registry(&cfg).unwrap());
|
||||
|
||||
Reference in New Issue
Block a user