fix(router): configure HTTP client connection settings (#24330)

Signed-off-by: Revanth Reddy Airre <revanthreddy@hippocraticai.com>

docs/advanced_features/sgl_model_gateway.md

@@ -593,13 +593,16 @@ Response:
 
 ## Reliability and Flow Control
 
-### HTTP Client Pool
+### HTTP Client
 
-Configure the idle timeout for pooled upstream HTTP connections:
+Configure upstream HTTP client connection settings:
 
 | Parameter | Default | Description |
 |-----------|---------|-------------|
 | `--pool-idle-timeout-secs` | 50 | Idle timeout in seconds for pooled upstream HTTP connections. Can also be set with `SMG_POOL_IDLE_TIMEOUT_SECS`. |
+| `--connect-timeout-secs` | 10 | Timeout in seconds for new upstream HTTP connections. Can also be set with `SMG_CONNECT_TIMEOUT_SECS`. |
+| `--pool-max-idle-per-host` | 500 | Maximum idle upstream HTTP connections to keep per host. Can also be set with `SMG_POOL_MAX_IDLE_PER_HOST`. |
+| `--tcp-keepalive-secs` | 30 | TCP keepalive idle time in seconds for upstream HTTP connections. Can also be set with `SMG_TCP_KEEPALIVE_SECS`. |
 
 ### Retries
 

docs_new/docs/advanced_features/sgl_model_gateway.mdx

@@ -944,9 +944,9 @@ Response:
 ***
 ## Reliability and Flow Control
 
-### HTTP Client Pool
+### HTTP Client
 
-Configure the idle timeout for pooled upstream HTTP connections:
+Configure upstream HTTP client connection settings:
 
 <table style={{width: "100%", borderCollapse: "collapse", tableLayout: "fixed"}}>
   <colgroup>
@@ -967,6 +967,21 @@ Configure the idle timeout for pooled upstream HTTP connections:
       <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>50</td>
       <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>Idle timeout in seconds for pooled upstream HTTP connections. Can also be set with `SMG_POOL_IDLE_TIMEOUT_SECS`.</td>
     </tr>
+    <tr>
+      <td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--connect-timeout-secs`</td>
+      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>10</td>
+      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>Timeout in seconds for new upstream HTTP connections. Can also be set with `SMG_CONNECT_TIMEOUT_SECS`.</td>
+    </tr>
+    <tr>
+      <td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--pool-max-idle-per-host`</td>
+      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>500</td>
+      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>Maximum idle upstream HTTP connections to keep per host. Can also be set with `SMG_POOL_MAX_IDLE_PER_HOST`.</td>
+    </tr>
+    <tr>
+      <td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--tcp-keepalive-secs`</td>
+      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>30</td>
+      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>TCP keepalive idle time in seconds for upstream HTTP connections. Can also be set with `SMG_TCP_KEEPALIVE_SECS`.</td>
+    </tr>
   </tbody>
 </table>
 

sgl-model-gateway/README.md

@@ -726,7 +726,7 @@ Router flags map to these values:
 - `--redis-retention-days` (env: `REDIS_RETENTION_DAYS`). Set to `-1` for persistent storage (default: 30 days).
 
 ## Reliability & Flow Control
-- **HTTP Client Pool**: Upstream HTTP connection pool idle timeout defaults to 50 seconds. Configure via `--pool-idle-timeout-secs` or `SMG_POOL_IDLE_TIMEOUT_SECS`.
+- **HTTP Client**: Upstream HTTP client connection settings default to pool idle timeout 50s, connect timeout 10s, max idle connections per host 500, and TCP keepalive 30s. Configure via `--pool-idle-timeout-secs`, `--connect-timeout-secs`, `--pool-max-idle-per-host`, `--tcp-keepalive-secs`, or the corresponding `SMG_*` env vars.
 - **Retries**: Default max retries = 5 with exponential backoff (`--retry-max-retries`, `--retry-initial-backoff-ms`, `--retry-max-backoff-ms`, `--retry-backoff-multiplier`, `--retry-jitter-factor`). Retries trigger on 408/429/500/502/503/504.
 - **Circuit Breakers**: Per worker thresholds (`--cb-failure-threshold`, `--cb-success-threshold`, `--cb-timeout-duration-secs`, `--cb-window-duration-secs`). Disable via `--disable-circuit-breaker`.
 - **Rate Limiting**: Token bucket driven by `--max-concurrent-requests`. Set `--rate-limit-tokens-per-second` to override refill rate. Configure request queue via `--queue-size` and `--queue-timeout-secs`; queued requests observe FIFO order and respect cancellation.

sgl-model-gateway/src/app_context.rs

@@ -330,11 +330,11 @@ impl AppContextBuilder {
 
         let mut client_builder = Client::builder()
             .pool_idle_timeout(Some(Duration::from_secs(config.pool_idle_timeout_secs)))
-            .pool_max_idle_per_host(500)
+            .pool_max_idle_per_host(config.pool_max_idle_per_host)
             .timeout(Duration::from_secs(timeout_secs))
-            .connect_timeout(Duration::from_secs(10))
+            .connect_timeout(Duration::from_secs(config.connect_timeout_secs))
             .tcp_nodelay(true)
-            .tcp_keepalive(Some(Duration::from_secs(30)));
+            .tcp_keepalive(Some(Duration::from_secs(config.tcp_keepalive_secs)));
 
         // Force rustls backend when TLS is configured
         if has_tls_config {

sgl-model-gateway/src/config/builder.rs

@@ -192,6 +192,21 @@ impl RouterConfigBuilder {
         self
     }
 
+    pub fn connect_timeout_secs(mut self, timeout: u64) -> Self {
+        self.config.connect_timeout_secs = timeout;
+        self
+    }
+
+    pub fn pool_max_idle_per_host(mut self, max: usize) -> Self {
+        self.config.pool_max_idle_per_host = max;
+        self
+    }
+
+    pub fn tcp_keepalive_secs(mut self, timeout: u64) -> Self {
+        self.config.tcp_keepalive_secs = timeout;
+        self
+    }
+
     // ==================== Rate Limiting ====================
 
     pub fn max_concurrent_requests(mut self, max: i32) -> Self {

sgl-model-gateway/src/config/types.rs

@@ -8,6 +8,9 @@ use super::ConfigResult;
 use crate::core::ConnectionMode;
 
 pub const DEFAULT_POOL_IDLE_TIMEOUT_SECS: u64 = 50;
+pub const DEFAULT_CONNECT_TIMEOUT_SECS: u64 = 10;
+pub const DEFAULT_POOL_MAX_IDLE_PER_HOST: usize = 500;
+pub const DEFAULT_TCP_KEEPALIVE_SECS: u64 = 30;
 
 /// Main router configuration
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -32,6 +35,12 @@ pub struct RouterConfig {
     pub request_id_headers: Option<Vec<String>>,
     #[serde(default = "default_pool_idle_timeout_secs")]
     pub pool_idle_timeout_secs: u64,
+    #[serde(default = "default_connect_timeout_secs")]
+    pub connect_timeout_secs: u64,
+    #[serde(default = "default_pool_max_idle_per_host")]
+    pub pool_max_idle_per_host: usize,
+    #[serde(default = "default_tcp_keepalive_secs")]
+    pub tcp_keepalive_secs: u64,
     /// Set to -1 to disable rate limiting
     pub max_concurrent_requests: i32,
     pub queue_size: usize,
@@ -127,6 +136,18 @@ fn default_pool_idle_timeout_secs() -> u64 {
     DEFAULT_POOL_IDLE_TIMEOUT_SECS
 }
 
+fn default_connect_timeout_secs() -> u64 {
+    DEFAULT_CONNECT_TIMEOUT_SECS
+}
+
+fn default_pool_max_idle_per_host() -> usize {
+    DEFAULT_POOL_MAX_IDLE_PER_HOST
+}
+
+fn default_tcp_keepalive_secs() -> u64 {
+    DEFAULT_TCP_KEEPALIVE_SECS
+}
+
 impl TokenizerCacheConfig {
     /// Returns Some(self) if any caching is enabled, None otherwise.
     /// Use this when passing cache config to tokenizer registration workflow.
@@ -501,6 +522,9 @@ impl Default for RouterConfig {
             log_level: None,
             request_id_headers: None,
             pool_idle_timeout_secs: default_pool_idle_timeout_secs(),
+            connect_timeout_secs: default_connect_timeout_secs(),
+            pool_max_idle_per_host: default_pool_max_idle_per_host(),
+            tcp_keepalive_secs: default_tcp_keepalive_secs(),
             max_concurrent_requests: -1,
             queue_size: 100,
             queue_timeout_secs: 60,
@@ -626,6 +650,12 @@ mod tests {
             config.pool_idle_timeout_secs,
             DEFAULT_POOL_IDLE_TIMEOUT_SECS
         );
+        assert_eq!(config.connect_timeout_secs, DEFAULT_CONNECT_TIMEOUT_SECS);
+        assert_eq!(
+            config.pool_max_idle_per_host,
+            DEFAULT_POOL_MAX_IDLE_PER_HOST
+        );
+        assert_eq!(config.tcp_keepalive_secs, DEFAULT_TCP_KEEPALIVE_SECS);
     }
 
     #[test]
@@ -676,19 +706,30 @@ mod tests {
     }
 
     #[test]
-    fn test_router_config_pool_idle_timeout_deserialization_default() {
+    fn test_router_config_http_client_deserialization_defaults() {
         let config = RouterConfig::default();
         let mut json = serde_json::to_value(&config).unwrap();
-        json.as_object_mut()
+        let json_object = json.as_object_mut().unwrap();
-            .unwrap()
+        json_object.remove("pool_idle_timeout_secs");
-            .remove("pool_idle_timeout_secs");
+        json_object.remove("connect_timeout_secs");
+        json_object.remove("pool_max_idle_per_host");
+        json_object.remove("tcp_keepalive_secs");
 
         let deserialized: RouterConfig = serde_json::from_value(json).unwrap();
 
         assert_eq!(
             deserialized.pool_idle_timeout_secs,
-            default_pool_idle_timeout_secs()
+            DEFAULT_POOL_IDLE_TIMEOUT_SECS
         );
+        assert_eq!(
+            deserialized.connect_timeout_secs,
+            DEFAULT_CONNECT_TIMEOUT_SECS
+        );
+        assert_eq!(
+            deserialized.pool_max_idle_per_host,
+            DEFAULT_POOL_MAX_IDLE_PER_HOST
+        );
+        assert_eq!(deserialized.tcp_keepalive_secs, DEFAULT_TCP_KEEPALIVE_SECS);
     }
 
     #[test]

sgl-model-gateway/src/config/validation.rs

@@ -313,6 +313,22 @@ impl ConfigValidator {
             });
         }
 
+        if config.connect_timeout_secs == 0 {
+            return Err(ConfigError::InvalidValue {
+                field: "connect_timeout_secs".to_string(),
+                value: config.connect_timeout_secs.to_string(),
+                reason: "Must be > 0".to_string(),
+            });
+        }
+
+        if config.tcp_keepalive_secs == 0 {
+            return Err(ConfigError::InvalidValue {
+                field: "tcp_keepalive_secs".to_string(),
+                value: config.tcp_keepalive_secs.to_string(),
+                reason: "Must be > 0".to_string(),
+            });
+        }
+
         Ok(())
     }
 

sgl-model-gateway/src/main.rs

@@ -8,7 +8,8 @@ use smg::{
         CircuitBreakerConfig, ConfigError, ConfigResult, DiscoveryConfig, HealthCheckConfig,
         HistoryBackend, ManualAssignmentMode, MetricsConfig, OracleConfig, PolicyConfig,
         PostgresConfig, RedisConfig, RetryConfig, RouterConfig, RoutingMode, TokenizerCacheConfig,
-        TraceConfig, DEFAULT_POOL_IDLE_TIMEOUT_SECS,
+        TraceConfig, DEFAULT_CONNECT_TIMEOUT_SECS, DEFAULT_POOL_IDLE_TIMEOUT_SECS,
+        DEFAULT_POOL_MAX_IDLE_PER_HOST, DEFAULT_TCP_KEEPALIVE_SECS,
     },
     core::ConnectionMode,
     observability::{
@@ -298,16 +299,43 @@ struct CliArgs {
     #[arg(long, num_args = 0.., help_heading = "Request Handling")]
     cors_allowed_origins: Vec<String>,
 
-    // ==================== HTTP Client Pool ====================
+    // ==================== HTTP Client ====================
     /// Idle timeout in seconds for pooled upstream HTTP connections
     #[arg(
         long,
         env = "SMG_POOL_IDLE_TIMEOUT_SECS",
         default_value_t = DEFAULT_POOL_IDLE_TIMEOUT_SECS,
-        help_heading = "HTTP Client Pool"
+        help_heading = "HTTP Client"
     )]
     pool_idle_timeout_secs: u64,
 
+    /// Timeout in seconds for new upstream HTTP connections
+    #[arg(
+        long,
+        env = "SMG_CONNECT_TIMEOUT_SECS",
+        default_value_t = DEFAULT_CONNECT_TIMEOUT_SECS,
+        help_heading = "HTTP Client"
+    )]
+    connect_timeout_secs: u64,
+
+    /// Maximum idle upstream HTTP connections to keep per host
+    #[arg(
+        long,
+        env = "SMG_POOL_MAX_IDLE_PER_HOST",
+        default_value_t = DEFAULT_POOL_MAX_IDLE_PER_HOST,
+        help_heading = "HTTP Client"
+    )]
+    pool_max_idle_per_host: usize,
+
+    /// TCP keepalive idle time in seconds for upstream HTTP connections
+    #[arg(
+        long,
+        env = "SMG_TCP_KEEPALIVE_SECS",
+        default_value_t = DEFAULT_TCP_KEEPALIVE_SECS,
+        help_heading = "HTTP Client"
+    )]
+    tcp_keepalive_secs: u64,
+
     // ==================== Rate Limiting ====================
     /// Maximum concurrent requests (-1 to disable)
     #[arg(long, default_value_t = -1, help_heading = "Rate Limiting")]
@@ -983,6 +1011,9 @@ impl CliArgs {
             .worker_startup_timeout_secs(self.worker_startup_timeout_secs)
             .worker_startup_check_interval_secs(self.worker_startup_check_interval)
             .pool_idle_timeout_secs(self.pool_idle_timeout_secs)
+            .connect_timeout_secs(self.connect_timeout_secs)
+            .pool_max_idle_per_host(self.pool_max_idle_per_host)
+            .tcp_keepalive_secs(self.tcp_keepalive_secs)
             .max_concurrent_requests(self.max_concurrent_requests)
             .queue_size(self.queue_size)
             .queue_timeout_secs(self.queue_timeout_secs)