diff --git a/sgl-model-gateway/src/routers/http/g9i.rs b/sgl-model-gateway/src/routers/http/g9i.rs index b9433a7aeb..b829092537 100644 --- a/sgl-model-gateway/src/routers/http/g9i.rs +++ b/sgl-model-gateway/src/routers/http/g9i.rs @@ -822,18 +822,41 @@ impl G9iState { body: &Value, ) -> Result, String> { let url = format!("{}/query_cache_hit", normalize_url(&worker.url().to_string())); - let mut req = client - .post(&url) - .timeout(Duration::from_millis( - self.config.cache_hit_query_timeout_ms.max(200), - )) - .json(body); - if let Some(key) = worker.api_key() { - req = req.bearer_auth(key); - } + let build_req = || { + let mut req = client + .post(&url) + .timeout(Duration::from_millis( + self.config.cache_hit_query_timeout_ms.max(200), + )) + .json(body); + if let Some(key) = worker.api_key() { + req = req.bearer_auth(key); + } + req + }; let start = Instant::now(); - let resp = req.send().await.map_err(|e| e.to_string())?; + let resp = match build_req().send().await { + Ok(r) => r, + Err(e) => { + // Retry once on transport-level failures only. Stale pooled + // keep-alive connections surface here as "error sending + // request" (server closed the idle conn between reuse) — + // observed twice on 2026-10-12 (idle-first-request and + // under-load), each drowning hit-class traffic in + // prefill_probe_failed 429s. A reconnect is milliseconds. + // Timeouts are deliberately NOT retried: a wedged engine + // won't recover within 2s, and the probe sits on the TTFT + // path (worst case must stay ~6s, not 12s). + if e.is_connect() || e.is_request() { + metrics::counter!(METRIC_DECISIONS, "path" => "probe_retry").increment(1); + debug!("G9i probe transport error on {}: {} — retrying once", url, e); + build_req().send().await.map_err(|e2| e2.to_string())? + } else { + return Err(e.to_string()); + } + } + }; metrics::histogram!(METRIC_QUERY_LATENCY).record(start.elapsed().as_secs_f64()); if !resp.status().is_success() {