diff --git a/docs/advanced_features/sgl_model_gateway.md b/docs/advanced_features/sgl_model_gateway.md index 8aa1b2d1b..0f2da5b47 100644 --- a/docs/advanced_features/sgl_model_gateway.md +++ b/docs/advanced_features/sgl_model_gateway.md @@ -593,13 +593,16 @@ Response: ## Reliability and Flow Control -### HTTP Client Pool +### HTTP Client -Configure the idle timeout for pooled upstream HTTP connections: +Configure upstream HTTP client connection settings: | Parameter | Default | Description | |-----------|---------|-------------| | `--pool-idle-timeout-secs` | 50 | Idle timeout in seconds for pooled upstream HTTP connections. Can also be set with `SMG_POOL_IDLE_TIMEOUT_SECS`. | +| `--connect-timeout-secs` | 10 | Timeout in seconds for new upstream HTTP connections. Can also be set with `SMG_CONNECT_TIMEOUT_SECS`. | +| `--pool-max-idle-per-host` | 500 | Maximum idle upstream HTTP connections to keep per host. Can also be set with `SMG_POOL_MAX_IDLE_PER_HOST`. | +| `--tcp-keepalive-secs` | 30 | TCP keepalive idle time in seconds for upstream HTTP connections. Can also be set with `SMG_TCP_KEEPALIVE_SECS`. | ### Retries diff --git a/docs_new/docs/advanced_features/sgl_model_gateway.mdx b/docs_new/docs/advanced_features/sgl_model_gateway.mdx index f6867cda2..049e6d408 100644 --- a/docs_new/docs/advanced_features/sgl_model_gateway.mdx +++ b/docs_new/docs/advanced_features/sgl_model_gateway.mdx @@ -944,9 +944,9 @@ Response: *** ## Reliability and Flow Control -### HTTP Client Pool +### HTTP Client -Configure the idle timeout for pooled upstream HTTP connections: +Configure upstream HTTP client connection settings: @@ -967,6 +967,21 @@ Configure the idle timeout for pooled upstream HTTP connections: + + + + + + + + + + + + + + +
50 Idle timeout in seconds for pooled upstream HTTP connections. Can also be set with `SMG_POOL_IDLE_TIMEOUT_SECS`.
`--connect-timeout-secs`10Timeout in seconds for new upstream HTTP connections. Can also be set with `SMG_CONNECT_TIMEOUT_SECS`.
`--pool-max-idle-per-host`500Maximum idle upstream HTTP connections to keep per host. Can also be set with `SMG_POOL_MAX_IDLE_PER_HOST`.
`--tcp-keepalive-secs`30TCP keepalive idle time in seconds for upstream HTTP connections. Can also be set with `SMG_TCP_KEEPALIVE_SECS`.
diff --git a/sgl-model-gateway/README.md b/sgl-model-gateway/README.md index c221a8f90..bffeae39f 100644 --- a/sgl-model-gateway/README.md +++ b/sgl-model-gateway/README.md @@ -726,7 +726,7 @@ Router flags map to these values: - `--redis-retention-days` (env: `REDIS_RETENTION_DAYS`). Set to `-1` for persistent storage (default: 30 days). ## Reliability & Flow Control -- **HTTP Client Pool**: Upstream HTTP connection pool idle timeout defaults to 50 seconds. Configure via `--pool-idle-timeout-secs` or `SMG_POOL_IDLE_TIMEOUT_SECS`. +- **HTTP Client**: Upstream HTTP client connection settings default to pool idle timeout 50s, connect timeout 10s, max idle connections per host 500, and TCP keepalive 30s. Configure via `--pool-idle-timeout-secs`, `--connect-timeout-secs`, `--pool-max-idle-per-host`, `--tcp-keepalive-secs`, or the corresponding `SMG_*` env vars. - **Retries**: Default max retries = 5 with exponential backoff (`--retry-max-retries`, `--retry-initial-backoff-ms`, `--retry-max-backoff-ms`, `--retry-backoff-multiplier`, `--retry-jitter-factor`). Retries trigger on 408/429/500/502/503/504. - **Circuit Breakers**: Per worker thresholds (`--cb-failure-threshold`, `--cb-success-threshold`, `--cb-timeout-duration-secs`, `--cb-window-duration-secs`). Disable via `--disable-circuit-breaker`. - **Rate Limiting**: Token bucket driven by `--max-concurrent-requests`. Set `--rate-limit-tokens-per-second` to override refill rate. Configure request queue via `--queue-size` and `--queue-timeout-secs`; queued requests observe FIFO order and respect cancellation. diff --git a/sgl-model-gateway/src/app_context.rs b/sgl-model-gateway/src/app_context.rs index 0254ff222..3ec742298 100644 --- a/sgl-model-gateway/src/app_context.rs +++ b/sgl-model-gateway/src/app_context.rs @@ -330,11 +330,11 @@ impl AppContextBuilder { let mut client_builder = Client::builder() .pool_idle_timeout(Some(Duration::from_secs(config.pool_idle_timeout_secs))) - .pool_max_idle_per_host(500) + .pool_max_idle_per_host(config.pool_max_idle_per_host) .timeout(Duration::from_secs(timeout_secs)) - .connect_timeout(Duration::from_secs(10)) + .connect_timeout(Duration::from_secs(config.connect_timeout_secs)) .tcp_nodelay(true) - .tcp_keepalive(Some(Duration::from_secs(30))); + .tcp_keepalive(Some(Duration::from_secs(config.tcp_keepalive_secs))); // Force rustls backend when TLS is configured if has_tls_config { diff --git a/sgl-model-gateway/src/config/builder.rs b/sgl-model-gateway/src/config/builder.rs index 70091180a..5d4891ca6 100644 --- a/sgl-model-gateway/src/config/builder.rs +++ b/sgl-model-gateway/src/config/builder.rs @@ -192,6 +192,21 @@ impl RouterConfigBuilder { self } + pub fn connect_timeout_secs(mut self, timeout: u64) -> Self { + self.config.connect_timeout_secs = timeout; + self + } + + pub fn pool_max_idle_per_host(mut self, max: usize) -> Self { + self.config.pool_max_idle_per_host = max; + self + } + + pub fn tcp_keepalive_secs(mut self, timeout: u64) -> Self { + self.config.tcp_keepalive_secs = timeout; + self + } + // ==================== Rate Limiting ==================== pub fn max_concurrent_requests(mut self, max: i32) -> Self { diff --git a/sgl-model-gateway/src/config/types.rs b/sgl-model-gateway/src/config/types.rs index a8f93a1e7..b19802642 100644 --- a/sgl-model-gateway/src/config/types.rs +++ b/sgl-model-gateway/src/config/types.rs @@ -8,6 +8,9 @@ use super::ConfigResult; use crate::core::ConnectionMode; pub const DEFAULT_POOL_IDLE_TIMEOUT_SECS: u64 = 50; +pub const DEFAULT_CONNECT_TIMEOUT_SECS: u64 = 10; +pub const DEFAULT_POOL_MAX_IDLE_PER_HOST: usize = 500; +pub const DEFAULT_TCP_KEEPALIVE_SECS: u64 = 30; /// Main router configuration #[derive(Debug, Clone, Serialize, Deserialize)] @@ -32,6 +35,12 @@ pub struct RouterConfig { pub request_id_headers: Option>, #[serde(default = "default_pool_idle_timeout_secs")] pub pool_idle_timeout_secs: u64, + #[serde(default = "default_connect_timeout_secs")] + pub connect_timeout_secs: u64, + #[serde(default = "default_pool_max_idle_per_host")] + pub pool_max_idle_per_host: usize, + #[serde(default = "default_tcp_keepalive_secs")] + pub tcp_keepalive_secs: u64, /// Set to -1 to disable rate limiting pub max_concurrent_requests: i32, pub queue_size: usize, @@ -127,6 +136,18 @@ fn default_pool_idle_timeout_secs() -> u64 { DEFAULT_POOL_IDLE_TIMEOUT_SECS } +fn default_connect_timeout_secs() -> u64 { + DEFAULT_CONNECT_TIMEOUT_SECS +} + +fn default_pool_max_idle_per_host() -> usize { + DEFAULT_POOL_MAX_IDLE_PER_HOST +} + +fn default_tcp_keepalive_secs() -> u64 { + DEFAULT_TCP_KEEPALIVE_SECS +} + impl TokenizerCacheConfig { /// Returns Some(self) if any caching is enabled, None otherwise. /// Use this when passing cache config to tokenizer registration workflow. @@ -501,6 +522,9 @@ impl Default for RouterConfig { log_level: None, request_id_headers: None, pool_idle_timeout_secs: default_pool_idle_timeout_secs(), + connect_timeout_secs: default_connect_timeout_secs(), + pool_max_idle_per_host: default_pool_max_idle_per_host(), + tcp_keepalive_secs: default_tcp_keepalive_secs(), max_concurrent_requests: -1, queue_size: 100, queue_timeout_secs: 60, @@ -626,6 +650,12 @@ mod tests { config.pool_idle_timeout_secs, DEFAULT_POOL_IDLE_TIMEOUT_SECS ); + assert_eq!(config.connect_timeout_secs, DEFAULT_CONNECT_TIMEOUT_SECS); + assert_eq!( + config.pool_max_idle_per_host, + DEFAULT_POOL_MAX_IDLE_PER_HOST + ); + assert_eq!(config.tcp_keepalive_secs, DEFAULT_TCP_KEEPALIVE_SECS); } #[test] @@ -676,19 +706,30 @@ mod tests { } #[test] - fn test_router_config_pool_idle_timeout_deserialization_default() { + fn test_router_config_http_client_deserialization_defaults() { let config = RouterConfig::default(); let mut json = serde_json::to_value(&config).unwrap(); - json.as_object_mut() - .unwrap() - .remove("pool_idle_timeout_secs"); + let json_object = json.as_object_mut().unwrap(); + json_object.remove("pool_idle_timeout_secs"); + json_object.remove("connect_timeout_secs"); + json_object.remove("pool_max_idle_per_host"); + json_object.remove("tcp_keepalive_secs"); let deserialized: RouterConfig = serde_json::from_value(json).unwrap(); assert_eq!( deserialized.pool_idle_timeout_secs, - default_pool_idle_timeout_secs() + DEFAULT_POOL_IDLE_TIMEOUT_SECS ); + assert_eq!( + deserialized.connect_timeout_secs, + DEFAULT_CONNECT_TIMEOUT_SECS + ); + assert_eq!( + deserialized.pool_max_idle_per_host, + DEFAULT_POOL_MAX_IDLE_PER_HOST + ); + assert_eq!(deserialized.tcp_keepalive_secs, DEFAULT_TCP_KEEPALIVE_SECS); } #[test] diff --git a/sgl-model-gateway/src/config/validation.rs b/sgl-model-gateway/src/config/validation.rs index c85ebe13e..534aa8a4f 100644 --- a/sgl-model-gateway/src/config/validation.rs +++ b/sgl-model-gateway/src/config/validation.rs @@ -313,6 +313,22 @@ impl ConfigValidator { }); } + if config.connect_timeout_secs == 0 { + return Err(ConfigError::InvalidValue { + field: "connect_timeout_secs".to_string(), + value: config.connect_timeout_secs.to_string(), + reason: "Must be > 0".to_string(), + }); + } + + if config.tcp_keepalive_secs == 0 { + return Err(ConfigError::InvalidValue { + field: "tcp_keepalive_secs".to_string(), + value: config.tcp_keepalive_secs.to_string(), + reason: "Must be > 0".to_string(), + }); + } + Ok(()) } diff --git a/sgl-model-gateway/src/main.rs b/sgl-model-gateway/src/main.rs index 6d6126403..fe1bcf4d3 100644 --- a/sgl-model-gateway/src/main.rs +++ b/sgl-model-gateway/src/main.rs @@ -8,7 +8,8 @@ use smg::{ CircuitBreakerConfig, ConfigError, ConfigResult, DiscoveryConfig, HealthCheckConfig, HistoryBackend, ManualAssignmentMode, MetricsConfig, OracleConfig, PolicyConfig, PostgresConfig, RedisConfig, RetryConfig, RouterConfig, RoutingMode, TokenizerCacheConfig, - TraceConfig, DEFAULT_POOL_IDLE_TIMEOUT_SECS, + TraceConfig, DEFAULT_CONNECT_TIMEOUT_SECS, DEFAULT_POOL_IDLE_TIMEOUT_SECS, + DEFAULT_POOL_MAX_IDLE_PER_HOST, DEFAULT_TCP_KEEPALIVE_SECS, }, core::ConnectionMode, observability::{ @@ -298,16 +299,43 @@ struct CliArgs { #[arg(long, num_args = 0.., help_heading = "Request Handling")] cors_allowed_origins: Vec, - // ==================== HTTP Client Pool ==================== + // ==================== HTTP Client ==================== /// Idle timeout in seconds for pooled upstream HTTP connections #[arg( long, env = "SMG_POOL_IDLE_TIMEOUT_SECS", default_value_t = DEFAULT_POOL_IDLE_TIMEOUT_SECS, - help_heading = "HTTP Client Pool" + help_heading = "HTTP Client" )] pool_idle_timeout_secs: u64, + /// Timeout in seconds for new upstream HTTP connections + #[arg( + long, + env = "SMG_CONNECT_TIMEOUT_SECS", + default_value_t = DEFAULT_CONNECT_TIMEOUT_SECS, + help_heading = "HTTP Client" + )] + connect_timeout_secs: u64, + + /// Maximum idle upstream HTTP connections to keep per host + #[arg( + long, + env = "SMG_POOL_MAX_IDLE_PER_HOST", + default_value_t = DEFAULT_POOL_MAX_IDLE_PER_HOST, + help_heading = "HTTP Client" + )] + pool_max_idle_per_host: usize, + + /// TCP keepalive idle time in seconds for upstream HTTP connections + #[arg( + long, + env = "SMG_TCP_KEEPALIVE_SECS", + default_value_t = DEFAULT_TCP_KEEPALIVE_SECS, + help_heading = "HTTP Client" + )] + tcp_keepalive_secs: u64, + // ==================== Rate Limiting ==================== /// Maximum concurrent requests (-1 to disable) #[arg(long, default_value_t = -1, help_heading = "Rate Limiting")] @@ -983,6 +1011,9 @@ impl CliArgs { .worker_startup_timeout_secs(self.worker_startup_timeout_secs) .worker_startup_check_interval_secs(self.worker_startup_check_interval) .pool_idle_timeout_secs(self.pool_idle_timeout_secs) + .connect_timeout_secs(self.connect_timeout_secs) + .pool_max_idle_per_host(self.pool_max_idle_per_host) + .tcp_keepalive_secs(self.tcp_keepalive_secs) .max_concurrent_requests(self.max_concurrent_requests) .queue_size(self.queue_size) .queue_timeout_secs(self.queue_timeout_secs)