gateway g9i: fail-closed 429 on probe failure / fast-P snapshot loss (p2 incident)
User directive 2026-10-12: 先严格429. When p2 hung, the fail-open fallback dumped all classified traffic into the regular pool and drowned it (7.5M pending tokens). All probe-failure paths now reject: prefill_probe_failed (429) when no candidate can answer /query_cache_hit; fast_p_no_snapshot (429) when snapshots go stale between probe and pick, with boot grace (had_any_success) mirroring the D gate.
This commit is contained in:
@@ -13,16 +13,8 @@
|
||||
//! Decision tree (per request, before PD dispatch):
|
||||
//! 1. D-pool gate: sum of fresh `num_running_reqs` over the decode pool
|
||||
//! exceeds `decode_max_running` -> 429 (`d_pool_saturated`).
|
||||
//! 2. Small-prefill classification (chat routes only): probe pool nodes
|
||||
//! (fast-P first, then regular), `new_tokens < max_new_tokens` ->
|
||||
//! hit class. **Hit class ALWAYS routes to the fast-P pool** (user
|
||||
//! directive 2026-10-12: 命中都在 fast-P 发生 — on a regular-P these
|
||||
//! requests queue behind long prefills and blow the fast path's p95;
|
||||
//! fast-P resolves the prefix from its own L2 or the shared L3 with
|
||||
//! wait_complete, recompute-and-graft otherwise). `routed_fast_p` vs
|
||||
//! `routed_fast_p_from_regular` decision counters distinguish prefix
|
||||
//! already local on fast-P from L3-fetch/recompute cases.
|
||||
//! new_tokens >= threshold -> regular P pool (large prefills).
|
||||
//! 2. Small-prefill classification (chat routes only): probe one fast-P,
|
||||
//! `new_tokens < max_new_tokens` -> fast-P pool, else regular P pool.
|
||||
//! 3. Pool selection: least estimated wait
|
||||
//! (`queued_total_input_tokens / max(drain_rate, drain_rate_floor)`)
|
||||
//! among fresh nodes — this is also the load balancer.
|
||||
@@ -528,9 +520,11 @@ impl G9iState {
|
||||
/// with least-wait selection and queue-wait rejection.
|
||||
///
|
||||
/// `body` is the OpenAI chat request JSON. Non-chat routes must not call
|
||||
/// this. All probe failures fall back to the default routing path
|
||||
/// (fail-open, metric emitted). The returned label feeds the TTFT
|
||||
/// histogram's `path` dimension ("fast_p" / "regular").
|
||||
/// this. Probe failures are fail-closed: when no candidate can answer the
|
||||
/// hit query the request is rejected (429 prefill_probe_failed) rather
|
||||
/// than dumped into the regular pool (p2 incident, 2026-10-12). The
|
||||
/// returned label feeds the TTFT histogram's `path` dimension
|
||||
/// ("fast_p" / "regular").
|
||||
pub async fn classify_prefill(
|
||||
&self,
|
||||
registry: &WorkerRegistry,
|
||||
@@ -611,12 +605,22 @@ impl G9iState {
|
||||
let (hit_worker, new_tokens) = match best {
|
||||
Some(x) => x,
|
||||
None => {
|
||||
// Fail-closed (2026-10-12): previously this dumped classified
|
||||
// traffic into the regular pool, which drowned it when the
|
||||
// fast-P node hung (p2 incident). No visibility, no admission
|
||||
// — reject instead.
|
||||
warn!(
|
||||
"G9i /query_cache_hit probe failed on all candidates ({:?}) — falling back to regular pool",
|
||||
"G9i /query_cache_hit probe failed on all candidates ({:?}) — 429 prefill_probe_failed (fail-closed)",
|
||||
last_err
|
||||
);
|
||||
metrics::counter!(METRIC_DECISIONS, "path" => "query_error").increment(1);
|
||||
return (self.regular_plan(&prefill_workers), "regular");
|
||||
return (
|
||||
PrefillPlan::Reject(RejectInfo {
|
||||
reason: "prefill_probe_failed",
|
||||
estimated_wait_secs: None,
|
||||
}),
|
||||
"fast_p",
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -625,35 +629,73 @@ impl G9iState {
|
||||
return (self.regular_plan(&prefill_workers), "regular");
|
||||
}
|
||||
|
||||
// Hit class (small prefill): ALWAYS route to the fast-P pool
|
||||
// (least-wait aggregate), even when the probe says the prefix
|
||||
// currently sits on a regular-P home node. User directive
|
||||
// 2026-10-12: 命中都应在 fast-P 发生 — on the regular pool these
|
||||
// requests queue behind long prefills (observed 150s p50), blowing
|
||||
// the fast path's latency budget. fast-P resolves the prefix via
|
||||
// wait_complete: shared-L3 fetch when the backup has landed, or a
|
||||
// one-time recompute that grafts the session into fast-P's large
|
||||
// L2 — self-healing either way; subsequent turns hit fast-P
|
||||
// locally. fast-P overload is protected by the fast_p_queue 429
|
||||
// gate, never by silent fallback to the regular pool.
|
||||
// Read-your-own-node: route to the node that actually holds the
|
||||
// prefix locally. If that node is a regular-P, an immediate L2 hit
|
||||
// beats fast-P, which would miss locally and either recompute the
|
||||
// whole prefix or wait for the cross-node L3 backup to become
|
||||
// visible (seconds for long prefixes; observed 10-20s at 76k).
|
||||
// Multi-turn sessions therefore stay on their home P; fast-P serves
|
||||
// sessions whose prefix it already holds (its own L2) and brand-new
|
||||
// short prefills.
|
||||
let hit_is_fast = self
|
||||
.config
|
||||
.fast_prefill_urls
|
||||
.iter()
|
||||
.any(|f| urls_match(f, &hit_worker.url().to_string()));
|
||||
if !hit_is_fast {
|
||||
// Observability: hit-class request whose prefix was NOT local
|
||||
// on fast-P (L3 fetch or one-time recompute expected).
|
||||
metrics::counter!(METRIC_DECISIONS, "path" => "routed_fast_p_from_regular")
|
||||
.increment(1);
|
||||
if let Some(load) = self.fresh_load(&hit_worker.url().to_string()) {
|
||||
let est_wait = load.est_wait_secs(self.config.drain_rate_floor);
|
||||
if self.config.regular_p_queue_reject_secs > 0.0
|
||||
&& est_wait > self.config.regular_p_queue_reject_secs
|
||||
{
|
||||
warn!(
|
||||
"G9i: home-hit node {} est wait {:.2}s > {:.2}s — 429 regular_p_queue",
|
||||
hit_worker.url(),
|
||||
est_wait,
|
||||
self.config.regular_p_queue_reject_secs,
|
||||
);
|
||||
return (
|
||||
PrefillPlan::Reject(RejectInfo {
|
||||
reason: "regular_p_queue",
|
||||
estimated_wait_secs: Some(est_wait),
|
||||
}),
|
||||
"home_hit",
|
||||
);
|
||||
}
|
||||
} else {
|
||||
metrics::counter!(METRIC_NO_DATA, "pool" => "home_hit").increment(1);
|
||||
}
|
||||
metrics::counter!(METRIC_DECISIONS, "path" => "routed_home_hit").increment(1);
|
||||
debug!(
|
||||
"G9i: small prefill (local_new={} < {}) with prefix on regular node {} — routing home (L2 hit)",
|
||||
new_tokens,
|
||||
self.config.max_new_tokens,
|
||||
hit_worker.url(),
|
||||
);
|
||||
return (
|
||||
PrefillPlan::Override(normalize_url(&hit_worker.url().to_string())),
|
||||
"home_hit",
|
||||
);
|
||||
}
|
||||
|
||||
// Small prefill whose prefix fast-P holds locally -> fast-P pool, least-wait pick.
|
||||
let (picked, est_wait, fresh) = self.least_wait(&fast_pool);
|
||||
if fresh == 0 {
|
||||
// Lost our snapshots between probe and pick; fail-open to regular.
|
||||
// Lost our snapshots between probe and pick. Fail-closed like the
|
||||
// probe path above; boot grace only until the first successful
|
||||
// poll (mirrors the decode gate's had_any_success discipline).
|
||||
metrics::counter!(METRIC_DECISIONS, "path" => "fast_pool_no_fresh_snapshot")
|
||||
.increment(1);
|
||||
if self.had_any_success.load(Ordering::SeqCst) {
|
||||
warn!("G9i: no fresh fast-P snapshot at pick time — 429 fast_p_no_snapshot (fail-closed)");
|
||||
return (
|
||||
PrefillPlan::Reject(RejectInfo {
|
||||
reason: "fast_p_no_snapshot",
|
||||
estimated_wait_secs: None,
|
||||
}),
|
||||
"fast_p",
|
||||
);
|
||||
}
|
||||
return (self.regular_plan(&prefill_workers), "regular");
|
||||
}
|
||||
let est_wait = if picked.is_some() { est_wait } else { 0.0 };
|
||||
|
||||
Reference in New Issue
Block a user