gateway g9i: retry probe once on transport errors (stale keep-alive pool)
Observed twice on 2026-10-12: reqwest reused server-closed pooled connections, all-candidate probes failed with connect-level errors, fail-closed 429 drowned hit-class traffic. Retry once on is_connect/is_request (ms-scale reconnect); timeouts stay non-retried (wedged engine + 6s worst-case TTFT budget).
This commit is contained in:
@@ -822,18 +822,41 @@ impl G9iState {
|
||||
body: &Value,
|
||||
) -> Result<Option<ProbeHit>, String> {
|
||||
let url = format!("{}/query_cache_hit", normalize_url(&worker.url().to_string()));
|
||||
let mut req = client
|
||||
.post(&url)
|
||||
.timeout(Duration::from_millis(
|
||||
self.config.cache_hit_query_timeout_ms.max(200),
|
||||
))
|
||||
.json(body);
|
||||
if let Some(key) = worker.api_key() {
|
||||
req = req.bearer_auth(key);
|
||||
}
|
||||
let build_req = || {
|
||||
let mut req = client
|
||||
.post(&url)
|
||||
.timeout(Duration::from_millis(
|
||||
self.config.cache_hit_query_timeout_ms.max(200),
|
||||
))
|
||||
.json(body);
|
||||
if let Some(key) = worker.api_key() {
|
||||
req = req.bearer_auth(key);
|
||||
}
|
||||
req
|
||||
};
|
||||
|
||||
let start = Instant::now();
|
||||
let resp = req.send().await.map_err(|e| e.to_string())?;
|
||||
let resp = match build_req().send().await {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
// Retry once on transport-level failures only. Stale pooled
|
||||
// keep-alive connections surface here as "error sending
|
||||
// request" (server closed the idle conn between reuse) —
|
||||
// observed twice on 2026-10-12 (idle-first-request and
|
||||
// under-load), each drowning hit-class traffic in
|
||||
// prefill_probe_failed 429s. A reconnect is milliseconds.
|
||||
// Timeouts are deliberately NOT retried: a wedged engine
|
||||
// won't recover within 2s, and the probe sits on the TTFT
|
||||
// path (worst case must stay ~6s, not 12s).
|
||||
if e.is_connect() || e.is_request() {
|
||||
metrics::counter!(METRIC_DECISIONS, "path" => "probe_retry").increment(1);
|
||||
debug!("G9i probe transport error on {}: {} — retrying once", url, e);
|
||||
build_req().send().await.map_err(|e2| e2.to_string())?
|
||||
} else {
|
||||
return Err(e.to_string());
|
||||
}
|
||||
}
|
||||
};
|
||||
metrics::histogram!(METRIC_QUERY_LATENCY).record(start.elapsed().as_secs_f64());
|
||||
|
||||
if !resp.status().is_success() {
|
||||
|
||||
Reference in New Issue
Block a user