diff --git a/experimental/sgl-router/Cargo.toml b/experimental/sgl-router/Cargo.toml index 11c8ecf45f..67b15055bc 100644 --- a/experimental/sgl-router/Cargo.toml +++ b/experimental/sgl-router/Cargo.toml @@ -53,6 +53,8 @@ http-body = "1" tower = { version = "0.5", features = ["full"] } tower-http = { version = "0.6", features = ["trace", "compression-gzip", "cors", "timeout", "request-id", "catch-panic"] } reqwest = { version = "0.12", features = ["stream", "json", "rustls-tls", "http2"], default-features = false } +# Named directly for `Error::is_incomplete_message`, which reqwest does not expose. +hyper = "1" sgl-kv-indexer = { path = "sgl-kv-indexer" } # Native SglangService gRPC types, shared with the engine (proto/sglang/runtime/v1). sglang-api-types = { path = "../../rust/sglang-api-types" } diff --git a/experimental/sgl-router/README.md b/experimental/sgl-router/README.md index afdda387f9..52540915e8 100644 --- a/experimental/sgl-router/README.md +++ b/experimental/sgl-router/README.md @@ -325,6 +325,9 @@ unset fields inherit the CLI flags. See `session_aware`); see [optional tokenizer](REFERENCE.md#optional-tokenizer-for-load-only-routing). - **Retries**: `--retry-max-attempts N` resends a request that failed before any response to another worker; see [retries](REFERENCE.md#retries). +- **Circuit breaker**: a worker leaves routing after `--cb-threshold` + consecutive failures; `--disable-circuit-breaker` turns this off; see + [circuit breaker](REFERENCE.md#circuit-breaker). - **DP-rank routing**: `--dp-aware` also picks the DP rank inside multi-rank engines, and `--dp-rank-policy round_robin` cycles ranks that no routing key or cached prefix decides; see [DP-rank routing](REFERENCE.md#dp-rank-routing). diff --git a/experimental/sgl-router/REFERENCE.md b/experimental/sgl-router/REFERENCE.md index b4a5d50214..45caa891df 100644 --- a/experimental/sgl-router/REFERENCE.md +++ b/experimental/sgl-router/REFERENCE.md @@ -218,6 +218,22 @@ doubles per retry, and is capped at `--retry-max-backoff-ms` (default 2000). Set the initial backoff to 0 to retry immediately. A backoff never waits past the request's stale deadline. +Separately, a send whose pooled HTTP/1.1 connection the worker closed before +answering (reset, broken pipe or EOF) is repeated once, immediately, on the same +worker: an engine closing an idle connection never read the request. Idle +connections leave the pool after 3 s, under SGLang's default 5 s keep-alive. + +### Circuit breaker + +Each worker has a circuit breaker: after `--cb-threshold` (default 3) +consecutive failures the worker is taken out of routing until a request already +sent to it succeeds or, once `--cb-cool-down-secs` (default 30) has passed, a +probe request does. Under `--worker-urls` it is the only thing that takes a +dead or hung worker out of routing, since there is no active health check. +`--disable-circuit-breaker` turns it off, for example when one worker serves a +role and an open breaker would reject all of that role's requests; a worker +that stops answering then keeps its share of traffic. + ### Engines with `--api-key` The router reads each worker's `/server_info` and `/model_info` to learn its diff --git a/experimental/sgl-router/src/config/cli.rs b/experimental/sgl-router/src/config/cli.rs index 363502de23..bd0f772627 100644 --- a/experimental/sgl-router/src/config/cli.rs +++ b/experimental/sgl-router/src/config/cli.rs @@ -11,15 +11,15 @@ use std::num::NonZeroU32; use crate::config::sampling::{parse_sampling_overrides, ConflictPolicy}; use crate::config::types::is_selector_empty; use crate::config::{ - default_cb_cool_down, default_host, default_port, default_proxy_request_timeout_secs, - default_shutdown_drain_secs, default_stale_request_timeout_secs, resolve_mode, AffinityConfig, - AffinityMode, BalancedBy, CacheAwareConfig, CachePrefixProvider, ChatRoutingKind, - CircuitBreakerConfig, Config, DecodePolicyKind, DiscoveryBackend, DpRankPolicy, - EligibilityConfig, FilterKind, FusedTerm, InflightLoadConfig, K8sDiscoveryConfig, - K8sDiscoveryMode, KvIndexerEndpointConfig, LogFormat, ModelConfig, ObservabilityConfig, - PolicyKind, ProxyConfig, ServerConfig, SessionAffinityMode, StaticUrlsDiscoveryConfig, - StickyConfig, StickyFallbackKind, TokenizerBackend, TokenizerConfig, DEFAULT_FUSE, - DEFAULT_KV_BOOTSTRAP_FETCH_TIMEOUT_CAP_MS, DEFAULT_KV_BOOTSTRAP_TIMEOUT_MS, + default_cb_cool_down, default_cb_threshold, default_host, default_port, + default_proxy_request_timeout_secs, default_shutdown_drain_secs, + default_stale_request_timeout_secs, resolve_mode, AffinityConfig, AffinityMode, BalancedBy, + CacheAwareConfig, CachePrefixProvider, ChatRoutingKind, CircuitBreakerConfig, Config, + DecodePolicyKind, DiscoveryBackend, DpRankPolicy, EligibilityConfig, FilterKind, FusedTerm, + InflightLoadConfig, K8sDiscoveryConfig, K8sDiscoveryMode, KvIndexerEndpointConfig, LogFormat, + ModelConfig, ObservabilityConfig, PolicyKind, ProxyConfig, ServerConfig, SessionAffinityMode, + StaticUrlsDiscoveryConfig, StickyConfig, StickyFallbackKind, TokenizerBackend, TokenizerConfig, + DEFAULT_FUSE, DEFAULT_KV_BOOTSTRAP_FETCH_TIMEOUT_CAP_MS, DEFAULT_KV_BOOTSTRAP_TIMEOUT_MS, }; use crate::policies_reorg::admission::AdmissionLimits; use crate::policies_reorg::factory::AffinitySpec; @@ -297,14 +297,18 @@ pub struct RoutingArgs { #[arg(long)] pub prefix_cache_min_share: Option, - /// Consecutive upstream failures before opening the breaker. Must be positive; enables the breaker. - #[arg(long)] - pub cb_threshold: Option, + /// Consecutive upstream failures before a worker's circuit breaker opens; must be positive. + #[arg(long, default_value_t = default_cb_threshold())] + pub cb_threshold: NonZeroU32, - /// Circuit-breaker cool-down in seconds. Only meaningful with - /// `--cb-threshold`; defaults to 30 when the breaker is enabled. - #[arg(long)] - pub cb_cool_down_secs: Option, + /// Seconds an open circuit breaker waits before letting a probe request through. + #[arg(long, default_value_t = default_cb_cool_down())] + pub cb_cool_down_secs: u64, + + /// Turn off the per-worker circuit breaker. Under `--worker-urls` nothing else takes + /// a dead or hung worker out of routing. + #[arg(long, conflicts_with_all = ["cb_threshold", "cb_cool_down_secs"])] + pub disable_circuit_breaker: bool, } #[derive(clap::Args, Debug)] @@ -541,7 +545,7 @@ impl Cli { Some(path) => (Some(load_bucket_config(path)?), None), None => (None, None), }; - let circuit_breaker = self.routing.build_circuit_breaker()?; + let circuit_breaker = self.routing.build_circuit_breaker(); let kv_bootstrap_timeout_ms = self.cache.kv_bootstrap_timeout_ms; let kv_bootstrap_fetch_timeout_cap_ms = self.cache.kv_bootstrap_fetch_timeout_cap_ms; let kv_bootstrap_seed_required = self.cache.kv_bootstrap_seed_required; @@ -737,18 +741,11 @@ impl DiscoveryArgs { } impl RoutingArgs { - fn build_circuit_breaker(&self) -> Result> { - ensure!( - self.cb_cool_down_secs.is_none() || self.cb_threshold.is_some(), - "--cb-cool-down-secs requires --cb-threshold (the circuit breaker is \ - enabled by --cb-threshold)" - ); - let circuit_breaker = self.cb_threshold.map(|threshold| CircuitBreakerConfig { - threshold, - cool_down_secs: self.cb_cool_down_secs.unwrap_or_else(default_cb_cool_down), - }); - - Ok(circuit_breaker) + fn build_circuit_breaker(&self) -> Option { + (!self.disable_circuit_breaker).then_some(CircuitBreakerConfig { + threshold: self.cb_threshold, + cool_down_secs: self.cb_cool_down_secs, + }) } fn build_fused(&self) -> Result>> { @@ -2085,7 +2082,15 @@ mod tests { } #[test] - fn cb_threshold_enables_circuit_breaker_with_default_cool_down() { + fn circuit_breaker_is_on_by_default() { + let c = into_config_owned(with_model(&["--worker-urls", "http://x:30000"])).unwrap(); + let cb = c.model.circuit_breaker.expect("cb enabled"); + assert_eq!(cb.threshold.get(), 3); + assert_eq!(cb.cool_down_secs, 30); + } + + #[test] + fn cb_threshold_keeps_default_cool_down() { let c = into_config_owned(with_model(&[ "--worker-urls", "http://x:30000", @@ -2103,30 +2108,40 @@ mod tests { let c = into_config_owned(with_model(&[ "--worker-urls", "http://x:30000", - "--cb-threshold", - "3", "--cb-cool-down-secs", "10", ])) .unwrap(); let cb = c.model.circuit_breaker.expect("cb enabled"); + assert_eq!(cb.threshold.get(), 3); assert_eq!(cb.cool_down_secs, 10); } #[test] - fn rejects_cb_cool_down_without_threshold() { - let err = into_config_owned(with_model(&[ + fn disable_circuit_breaker_turns_it_off() { + let c = into_config_owned(with_model(&[ "--worker-urls", "http://x:30000", - "--cb-cool-down-secs", - "10", + "--disable-circuit-breaker", ])) - .unwrap_err() - .to_string(); - assert!( - err.contains("--cb-cool-down-secs requires --cb-threshold"), - "got: {err}" - ); + .unwrap(); + assert!(c.model.circuit_breaker.is_none()); + } + + #[test] + fn disable_circuit_breaker_rejects_breaker_settings() { + for flag in ["--cb-threshold", "--cb-cool-down-secs"] { + let err = into_config_owned(with_model(&[ + "--worker-urls", + "http://x:30000", + "--disable-circuit-breaker", + flag, + "5", + ])) + .unwrap_err() + .to_string(); + assert!(err.contains("cannot be used with"), "{flag}: {err}"); + } } #[test] diff --git a/experimental/sgl-router/src/config/types.rs b/experimental/sgl-router/src/config/types.rs index 671c62c841..8d92f79070 100644 --- a/experimental/sgl-router/src/config/types.rs +++ b/experimental/sgl-router/src/config/types.rs @@ -397,6 +397,7 @@ pub struct ModelConfig { /// Reorg admission for groups that leave a limit unset: `--max-inflight-requests` /// and `--max-kv-usage`. pub reorg_admission: crate::policies_reorg::admission::AdmissionLimits, + /// Per-worker circuit breaker; `None` under `--disable-circuit-breaker`. pub circuit_breaker: Option, /// Cache-Aware prefix configuration. pub cache_aware: Option, @@ -730,8 +731,12 @@ pub struct CircuitBreakerConfig { pub cool_down_secs: u64, } -/// Default circuit-breaker cool-down, applied when `--cb-threshold` is -/// set without an explicit `--cb-cool-down-secs`. +/// Default circuit-breaker threshold, without `--cb-threshold`. +pub fn default_cb_threshold() -> NonZeroU32 { + NonZeroU32::new(3).expect("3 is non-zero") +} + +/// Default circuit-breaker cool-down, without `--cb-cool-down-secs`. pub fn default_cb_cool_down() -> u64 { 30 } diff --git a/experimental/sgl-router/src/health/circuit_breaker.rs b/experimental/sgl-router/src/health/circuit_breaker.rs index 5181144e83..12eccf4982 100644 --- a/experimental/sgl-router/src/health/circuit_breaker.rs +++ b/experimental/sgl-router/src/health/circuit_breaker.rs @@ -56,6 +56,8 @@ struct Inner { pub struct CircuitBreaker { inner: Mutex, config: CircuitBreakerConfig, + /// A disabled breaker ignores failures, so it never opens. + enabled: bool, } impl CircuitBreaker { @@ -71,11 +73,21 @@ impl CircuitBreaker { probe_generation: 0, }), config, + enabled: true, } } - pub fn config(&self) -> &CircuitBreakerConfig { - &self.config + /// The breaker of every worker under `--disable-circuit-breaker`. + pub fn disabled() -> Self { + Self { + enabled: false, + ..Self::new() + } + } + + /// `None` for a [`disabled`](Self::disabled) breaker. + pub fn config(&self) -> Option<&CircuitBreakerConfig> { + self.enabled.then_some(&self.config) } /// Non-mutating predicate: would [`allow`] return `true` if called @@ -162,6 +174,9 @@ impl CircuitBreaker { } pub fn record_failure(&self) { + if !self.enabled { + return; + } let mut g = self.inner.lock().unwrap(); match g.state { State::Closed | State::HalfOpen { .. } => { diff --git a/experimental/sgl-router/src/policies/registry.rs b/experimental/sgl-router/src/policies/registry.rs index 958af215c3..601a17ea25 100644 --- a/experimental/sgl-router/src/policies/registry.rs +++ b/experimental/sgl-router/src/policies/registry.rs @@ -344,6 +344,7 @@ fn host_of(worker_url: &str) -> Option { mod tests { use super::*; use crate::discovery::{ModelId, WorkerId, WorkerSpec}; + use crate::workers::WireProtocol; fn spec(id: &str, mode: WorkerMode, model: &str) -> WorkerSpec { WorkerSpec { @@ -359,7 +360,7 @@ mod tests { fn registry(specs: &[WorkerSpec]) -> Arc { let r = Arc::new(WorkerRegistry::default()); for s in specs { - let _ = r.add(s.clone()); + let _ = r.add_with_cb(s.clone(), Some(Default::default()), WireProtocol::default()); } r } diff --git a/experimental/sgl-router/src/proxy/mod.rs b/experimental/sgl-router/src/proxy/mod.rs index 21e2ab34b2..2a26002238 100644 --- a/experimental/sgl-router/src/proxy/mod.rs +++ b/experimental/sgl-router/src/proxy/mod.rs @@ -17,7 +17,7 @@ use anyhow::Context; use axum::body::Body; use axum::http::{HeaderMap, HeaderName, HeaderValue, Response}; use bytes::Bytes; -use reqwest::{Client, Url}; +use reqwest::{Client, RequestBuilder, Url}; use std::sync::Arc; use std::time::Duration; use tokio_util::sync::CancellationToken; @@ -136,6 +136,8 @@ pub struct Proxy { fn build_client(protocol: WireProtocol) -> Result { let builder = Client::builder() .pool_max_idle_per_host(64) + // Under SGLang's default 5 s keep-alive, so idle connections close here first. + .pool_idle_timeout(Duration::from_secs(3)) .connect_timeout(Duration::from_secs(5)); match protocol { WireProtocol::Http1 => builder, @@ -145,6 +147,39 @@ fn build_client(protocol: WireProtocol) -> Result { .context("build reqwest client") } +/// Sends `req`, and once more if the worker closed an HTTP/1.1 connection before +/// answering: an engine closing an idle keep-alive connection never read the request. +/// Not for h2c, where one connection carries many requests. +async fn send_retrying_closed_connection( + req: RequestBuilder, + protocol: WireProtocol, +) -> reqwest::Result { + let retry = match protocol { + WireProtocol::Http1 => req.try_clone(), + WireProtocol::H2c => None, + }; + match (req.send().await, retry) { + (Err(e), Some(retry)) if closed_before_response(&e) => { + tracing::debug!(error = ?e, "worker closed the connection before answering; resending"); + retry.send().await + } + (result, _) => result, + } +} + +/// A reset, broken pipe or EOF on an open connection, before any response. +fn closed_before_response(e: &reqwest::Error) -> bool { + use std::io::ErrorKind::{BrokenPipe, ConnectionReset, UnexpectedEof}; + !e.is_connect() + && std::iter::successors(std::error::Error::source(e), |c| c.source()).any(|c| { + c.downcast_ref::() + .is_some_and(hyper::Error::is_incomplete_message) + || c.downcast_ref::().is_some_and(|io| { + matches!(io.kind(), ConnectionReset | BrokenPipe | UnexpectedEof) + }) + }) +} + impl Proxy { /// Build a proxy. `request_timeout` is the per-request wall-clock budget for /// non-streaming forwards and for streaming response headers. Connect timeout is hard-coded to 5 s — even a @@ -248,10 +283,12 @@ impl Proxy { .timeout(self.request_timeout); let mut abort = AbortOnDrop::new(self.client_for(protocol), &worker_url, headers, abort_rid); - let resp = req.send().await.map_err(|e| { - breaker.record_failure(); - Self::classify_reqwest_error_for(worker_url.clone(), e, path) - })?; + let resp = send_retrying_closed_connection(req, protocol) + .await + .map_err(|e| { + breaker.record_failure(); + Self::classify_reqwest_error_for(worker_url.clone(), e, path) + })?; let status = resp.status(); // Defer breaker recording until after the body completes — a // worker that returns 2xx headers and then drops mid-body is @@ -348,7 +385,12 @@ impl Proxy { let mut abort = AbortOnDrop::new(self.client_for(protocol), &worker_url, headers, abort_rid); // A worker that never answers is a fault, unlike a long stream once it answers. - let resp = match tokio::time::timeout(self.request_timeout, req.send()).await { + let resp = match tokio::time::timeout( + self.request_timeout, + send_retrying_closed_connection(req, protocol), + ) + .await + { Ok(Ok(resp)) => resp, Ok(Err(e)) => { breaker.record_failure(); @@ -456,6 +498,7 @@ mod tests { use axum::Router; use reqwest::StatusCode; use std::num::NonZeroU32; + use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Duration; use tokio::net::TcpListener; use tokio::sync::oneshot; @@ -885,4 +928,79 @@ mod tests { ); } } + + #[derive(Debug, Clone, Copy)] + enum Failure { + /// Close with the request unread, so the kernel resets the connection. + Reset, + /// Read the request, then close. + Close, + /// Read the request and never answer. + Hang, + } + + /// A raw HTTP/1.1 worker failing its first `failing` connections; also + /// returns how many connections it accepted. + async fn spawn_failing_worker(failing: usize, failure: Failure) -> (String, Arc) { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let accepted = Arc::new(AtomicUsize::new(0)); + let count = Arc::clone(&accepted); + tokio::spawn(async move { + let mut hung = Vec::new(); + while let Ok((mut socket, _)) = listener.accept().await { + let fail = count.fetch_add(1, Ordering::SeqCst) < failing; + if fail && matches!(failure, Failure::Reset) { + let _ = socket.peek(&mut [0]).await; + continue; + } + let mut head = Vec::new(); + let mut buf = [0; 1024]; + while !head.windows(4).any(|w| w == b"\r\n\r\n") { + match socket.read(&mut buf).await { + Ok(0) | Err(_) => break, + Ok(n) => head.extend_from_slice(&buf[..n]), + } + } + if !fail { + let _ = socket + .write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 2\r\n\r\n{}") + .await; + } else if matches!(failure, Failure::Hang) { + hung.push(socket); + } + } + }); + (url, accepted) + } + + /// A closed connection is resent once on the same worker; a timeout is not. + #[tokio::test] + async fn closed_connection_is_retried_once() { + for (failing, failure, status, connections) in [ + (1, Failure::Reset, StatusCode::OK, 2), + (1, Failure::Close, StatusCode::OK, 2), + (usize::MAX, Failure::Close, StatusCode::BAD_GATEWAY, 2), + (1, Failure::Hang, StatusCode::GATEWAY_TIMEOUT, 1), + ] { + let (url, accepted) = spawn_failing_worker(failing, failure).await; + let proxy = Proxy::new(Duration::from_millis(500)).unwrap(); + let result = proxy + .forward_json_to( + &url, + WireProtocol::Http1, + &CircuitBreaker::new(), + "/generate", + &HeaderMap::new(), + Bytes::new(), + None, + ) + .await; + let got = result.map_or_else(|e| e.status_code(), |r| r.status()); + let conns = accepted.load(Ordering::SeqCst); + assert_eq!((got, conns), (status, connections), "{failure:?}"); + } + } } diff --git a/experimental/sgl-router/src/workers/manager.rs b/experimental/sgl-router/src/workers/manager.rs index 01cacbf9d6..f45f3c697e 100644 --- a/experimental/sgl-router/src/workers/manager.rs +++ b/experimental/sgl-router/src/workers/manager.rs @@ -34,8 +34,9 @@ const BOOTSTRAP_PORT_RETRY_INTERVAL: Duration = Duration::from_secs(3); /// Resolve the circuit-breaker config for all model IDs carried by a spec. /// /// The router serves a single configured model; apply its circuit-breaker -/// config when this worker advertises that model id. Falls back to `None` -/// (default config) otherwise. +/// config when this worker advertises that model id, and the default config +/// otherwise. Under `--disable-circuit-breaker` there is none, so the worker's +/// breaker never opens. fn cb_config_for_spec(spec: &WorkerSpec, cfg: &Config) -> Option { let model = &cfg.model; let cbc = model.circuit_breaker.as_ref()?; @@ -45,7 +46,7 @@ fn cb_config_for_spec(spec: &WorkerSpec, cfg: &Config) -> Option, registry: Arc, @@ -279,7 +279,7 @@ impl Worker { } = profile.into(); let breaker = match cb { Some(cfg) => Arc::new(CircuitBreaker::with_config(cfg)), - None => Arc::new(CircuitBreaker::new()), + None => Arc::new(CircuitBreaker::disabled()), }; let bootstrap_host = parse_bootstrap_host(&spec.url); let grpc = grpc_port.and_then(|port| grpc_client(&bootstrap_host, port)); diff --git a/experimental/sgl-router/tests/component/policies_reorg.rs b/experimental/sgl-router/tests/component/policies_reorg.rs index e138760b17..6b23533261 100644 --- a/experimental/sgl-router/tests/component/policies_reorg.rs +++ b/experimental/sgl-router/tests/component/policies_reorg.rs @@ -13,7 +13,7 @@ use sgl_router::policies_reorg::admission::{ }; use sgl_router::policies_reorg::{Pick, PickError, PickRequest, Policy, Rejection, Stage}; use sgl_router::state::kv_events::CacheNamespace; -use sgl_router::workers::{Worker, WorkerRegistry}; +use sgl_router::workers::{WireProtocol, Worker, WorkerRegistry}; #[derive(Debug)] struct TestPolicy { @@ -104,7 +104,10 @@ fn registry() -> Arc { ("p", Stage::Prefill, "pd"), ("d", Stage::Decode, "pd"), ] { - workers.add(spec(id, mode, model)).unwrap(); + let cb = Some(Default::default()); + workers + .add_with_cb(spec(id, mode, model), cb, WireProtocol::default()) + .unwrap(); } let unhealthy = workers.get(&WorkerId("unhealthy".into())).unwrap(); for _ in 0..3 { diff --git a/experimental/sgl-router/tests/component/workers/manager.rs b/experimental/sgl-router/tests/component/workers/manager.rs index 470f770c33..28387aebae 100644 --- a/experimental/sgl-router/tests/component/workers/manager.rs +++ b/experimental/sgl-router/tests/component/workers/manager.rs @@ -2,7 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 use axum::{routing::get, Json, Router}; +use clap::Parser; use serde_json::{json, Value}; +use sgl_router::config::Cli; use sgl_router::discovery::{DiscoveryEvent, ModelId, WorkerId, WorkerMode, WorkerSpec}; use sgl_router::workers::{manager, WireProtocol, WorkerRegistry}; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -401,7 +403,11 @@ async fn mode_changed_preserves_active_requests_and_breaker() { let (tx, rx) = mpsc::channel(16); let registry = Arc::new(WorkerRegistry::default()); - let h = tokio::spawn(manager::run(rx, registry.clone())); + let argv = format!("sgl-router --model-id m --worker-urls {url} --cb-threshold 3"); + let cli = Cli::try_parse_from(argv.split(' ')).unwrap(); + let cfg = Some(Arc::new(cli.into_config().unwrap())); + let run = manager::run_with_config(rx, registry.clone(), cfg, None, None); + let h = tokio::spawn(run); tx.send(DiscoveryEvent::Added(spec_for( "w1", @@ -415,7 +421,7 @@ async fn mode_changed_preserves_active_requests_and_breaker() { // Grab a handle, bump active_requests, and open the breaker. let w = registry.get(&WorkerId("w1".into())).unwrap(); w.active_requests.fetch_add(5, Ordering::Relaxed); - // Default threshold is 3 — record 10 failures to guarantee Open state. + // Threshold is 3 — record 10 failures to guarantee Open state. for _ in 0..10 { w.breaker.record_failure(); } diff --git a/experimental/sgl-router/tests/proxy/chat_routing.rs b/experimental/sgl-router/tests/proxy/chat_routing.rs index 5bb7f25c8c..71da1cfb09 100644 --- a/experimental/sgl-router/tests/proxy/chat_routing.rs +++ b/experimental/sgl-router/tests/proxy/chat_routing.rs @@ -70,13 +70,14 @@ fn build_ctx_with_worker(url: &str) -> Arc { let cfg = config_for(url); let tokenizers = Arc::new(TokenizerRegistry::load_from_config(&cfg).unwrap()); let registry = Arc::new(WorkerRegistry::default()); - let _ = registry.add(WorkerSpec { + let spec = WorkerSpec { id: WorkerId("w1".into()), url: url.to_string(), mode: WorkerMode::Plain, model_ids: vec![ModelId("tiny".into())], ..Default::default() - }); + }; + let _ = registry.add_with_cb(spec, Some(Default::default()), WireProtocol::default()); let policies = Arc::new(build_policy_registry(&cfg).unwrap()); // Per-request worker URLs flow from the registry through // `forward_*_to(&worker.url, ...)`; the proxy itself is URL-less. diff --git a/experimental/sgl-router/tests/proxy/grpc_routing.rs b/experimental/sgl-router/tests/proxy/grpc_routing.rs index f4fc490d7c..205c3fefba 100644 --- a/experimental/sgl-router/tests/proxy/grpc_routing.rs +++ b/experimental/sgl-router/tests/proxy/grpc_routing.rs @@ -245,7 +245,9 @@ fn router_ctx_with( dp_ranks: 1, grpc_port, }; - registry.add_with_cb(spec, None, profile).unwrap(); + registry + .add_with_cb(spec, Some(Default::default()), profile) + .unwrap(); } if reorg { cfg.model.policy = PolicyKind::PowerOfTwo; diff --git a/experimental/sgl-router/tests/proxy/pd_pool_isolation.rs b/experimental/sgl-router/tests/proxy/pd_pool_isolation.rs index 7b52008988..2e6fa38d57 100644 --- a/experimental/sgl-router/tests/proxy/pd_pool_isolation.rs +++ b/experimental/sgl-router/tests/proxy/pd_pool_isolation.rs @@ -29,7 +29,7 @@ use sgl_router::proxy::Proxy; use sgl_router::server::app::build_router; use sgl_router::server::app_context::AppContext; use sgl_router::tokenizer::TokenizerRegistry; -use sgl_router::workers::WorkerRegistry; +use sgl_router::workers::{WireProtocol, WorkerRegistry}; use std::sync::Arc; use std::time::Duration; use tower::ServiceExt; @@ -124,14 +124,15 @@ async fn pd_decode_stream_expires_after_prefill_completes() { ..Default::default() }) .unwrap(); + let spec = WorkerSpec { + id: WorkerId("d1".into()), + url: decode.url.clone(), + mode: WorkerMode::Decode, + model_ids: vec![ModelId("tiny".into())], + ..Default::default() + }; registry - .add(WorkerSpec { - id: WorkerId("d1".into()), - url: decode.url.clone(), - mode: WorkerMode::Decode, - model_ids: vec![ModelId("tiny".into())], - ..Default::default() - }) + .add_with_cb(spec, Some(Default::default()), WireProtocol::default()) .unwrap(); let prefill_worker = registry.get(&WorkerId("p1".into())).unwrap(); let decode_worker = registry.get(&WorkerId("d1".into())).unwrap(); diff --git a/experimental/sgl-router/tests/proxy/timeout.rs b/experimental/sgl-router/tests/proxy/timeout.rs index b62a0a8ec8..70e919b7d3 100644 --- a/experimental/sgl-router/tests/proxy/timeout.rs +++ b/experimental/sgl-router/tests/proxy/timeout.rs @@ -23,7 +23,7 @@ use sgl_router::proxy::Proxy; use sgl_router::server::app::build_router; use sgl_router::server::app_context::AppContext; use sgl_router::tokenizer::TokenizerRegistry; -use sgl_router::workers::WorkerRegistry; +use sgl_router::workers::{WireProtocol, WorkerRegistry}; use std::sync::Arc; use std::time::Duration; use tower::ServiceExt; @@ -148,14 +148,15 @@ fn hanging_ctx(urls: &[&str], max_attempts: u32) -> Arc { let tokenizers = Arc::new(TokenizerRegistry::load_from_config(&cfg).unwrap()); let registry = Arc::new(WorkerRegistry::default()); for (i, url) in urls.iter().enumerate() { + let spec = WorkerSpec { + id: WorkerId(format!("w{i}")), + url: (*url).into(), + mode: WorkerMode::Plain, + model_ids: vec![ModelId("tiny".into())], + ..Default::default() + }; registry - .add(WorkerSpec { - id: WorkerId(format!("w{i}")), - url: (*url).into(), - mode: WorkerMode::Plain, - model_ids: vec![ModelId("tiny".into())], - ..Default::default() - }) + .add_with_cb(spec, Some(Default::default()), WireProtocol::default()) .unwrap(); } let policies = Arc::new(build_policy_registry(&cfg).unwrap());