gateway g9i: retry probe once on transport errors (stale keep-alive pool)

Observed twice on 2026-10-12: reqwest reused server-closed pooled connections,
all-candidate probes failed with connect-level errors, fail-closed 429 drowned
hit-class traffic. Retry once on is_connect/is_request (ms-scale reconnect);
timeouts stay non-retried (wedged engine + 6s worst-case TTFT budget).
This commit is contained in:
2026-10-12 12:02:49 +00:00
parent 57db2a4ae9
commit 0308920bdb
+33 -10
View File
@@ -822,18 +822,41 @@ impl G9iState {
body: &Value,
) -> Result<Option<ProbeHit>, String> {
let url = format!("{}/query_cache_hit", normalize_url(&worker.url().to_string()));
let mut req = client
.post(&url)
.timeout(Duration::from_millis(
self.config.cache_hit_query_timeout_ms.max(200),
))
.json(body);
if let Some(key) = worker.api_key() {
req = req.bearer_auth(key);
}
let build_req = || {
let mut req = client
.post(&url)
.timeout(Duration::from_millis(
self.config.cache_hit_query_timeout_ms.max(200),
))
.json(body);
if let Some(key) = worker.api_key() {
req = req.bearer_auth(key);
}
req
};
let start = Instant::now();
let resp = req.send().await.map_err(|e| e.to_string())?;
let resp = match build_req().send().await {
Ok(r) => r,
Err(e) => {
// Retry once on transport-level failures only. Stale pooled
// keep-alive connections surface here as "error sending
// request" (server closed the idle conn between reuse) —
// observed twice on 2026-10-12 (idle-first-request and
// under-load), each drowning hit-class traffic in
// prefill_probe_failed 429s. A reconnect is milliseconds.
// Timeouts are deliberately NOT retried: a wedged engine
// won't recover within 2s, and the probe sits on the TTFT
// path (worst case must stay ~6s, not 12s).
if e.is_connect() || e.is_request() {
metrics::counter!(METRIC_DECISIONS, "path" => "probe_retry").increment(1);
debug!("G9i probe transport error on {}: {} — retrying once", url, e);
build_req().send().await.map_err(|e2| e2.to_string())?
} else {
return Err(e.to_string());
}
}
};
metrics::histogram!(METRIC_QUERY_LATENCY).record(start.elapsed().as_secs_f64());
if !resp.status().is_success() {