From 57db2a4ae9de022c9800e08bdf69a6c6ab78b2cd Mon Sep 17 00:00:00 2001 From: "minke.yu" Date: Mon, 12 Oct 2026 10:16:09 +0000 Subject: [PATCH] gateway g9i: fail-closed 429 on probe failure / fast-P snapshot loss (p2 incident) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit User directive 2026-10-12: 先严格429. When p2 hung, the fail-open fallback dumped all classified traffic into the regular pool and drowned it (7.5M pending tokens). All probe-failure paths now reject: prefill_probe_failed (429) when no candidate can answer /query_cache_hit; fast_p_no_snapshot (429) when snapshots go stale between probe and pick, with boot grace (had_any_success) mirroring the D gate. --- sgl-model-gateway/src/routers/http/g9i.rs | 104 +++++++++++++++------- 1 file changed, 73 insertions(+), 31 deletions(-) diff --git a/sgl-model-gateway/src/routers/http/g9i.rs b/sgl-model-gateway/src/routers/http/g9i.rs index 5a810d4b59..b9433a7aeb 100644 --- a/sgl-model-gateway/src/routers/http/g9i.rs +++ b/sgl-model-gateway/src/routers/http/g9i.rs @@ -13,16 +13,8 @@ //! Decision tree (per request, before PD dispatch): //! 1. D-pool gate: sum of fresh `num_running_reqs` over the decode pool //! exceeds `decode_max_running` -> 429 (`d_pool_saturated`). -//! 2. Small-prefill classification (chat routes only): probe pool nodes -//! (fast-P first, then regular), `new_tokens < max_new_tokens` -> -//! hit class. **Hit class ALWAYS routes to the fast-P pool** (user -//! directive 2026-10-12: 命中都在 fast-P 发生 — on a regular-P these -//! requests queue behind long prefills and blow the fast path's p95; -//! fast-P resolves the prefix from its own L2 or the shared L3 with -//! wait_complete, recompute-and-graft otherwise). `routed_fast_p` vs -//! `routed_fast_p_from_regular` decision counters distinguish prefix -//! already local on fast-P from L3-fetch/recompute cases. -//! new_tokens >= threshold -> regular P pool (large prefills). +//! 2. Small-prefill classification (chat routes only): probe one fast-P, +//! `new_tokens < max_new_tokens` -> fast-P pool, else regular P pool. //! 3. Pool selection: least estimated wait //! (`queued_total_input_tokens / max(drain_rate, drain_rate_floor)`) //! among fresh nodes — this is also the load balancer. @@ -528,9 +520,11 @@ impl G9iState { /// with least-wait selection and queue-wait rejection. /// /// `body` is the OpenAI chat request JSON. Non-chat routes must not call - /// this. All probe failures fall back to the default routing path - /// (fail-open, metric emitted). The returned label feeds the TTFT - /// histogram's `path` dimension ("fast_p" / "regular"). + /// this. Probe failures are fail-closed: when no candidate can answer the + /// hit query the request is rejected (429 prefill_probe_failed) rather + /// than dumped into the regular pool (p2 incident, 2026-10-12). The + /// returned label feeds the TTFT histogram's `path` dimension + /// ("fast_p" / "regular"). pub async fn classify_prefill( &self, registry: &WorkerRegistry, @@ -611,12 +605,22 @@ impl G9iState { let (hit_worker, new_tokens) = match best { Some(x) => x, None => { + // Fail-closed (2026-10-12): previously this dumped classified + // traffic into the regular pool, which drowned it when the + // fast-P node hung (p2 incident). No visibility, no admission + // — reject instead. warn!( - "G9i /query_cache_hit probe failed on all candidates ({:?}) — falling back to regular pool", + "G9i /query_cache_hit probe failed on all candidates ({:?}) — 429 prefill_probe_failed (fail-closed)", last_err ); metrics::counter!(METRIC_DECISIONS, "path" => "query_error").increment(1); - return (self.regular_plan(&prefill_workers), "regular"); + return ( + PrefillPlan::Reject(RejectInfo { + reason: "prefill_probe_failed", + estimated_wait_secs: None, + }), + "fast_p", + ); } }; @@ -625,35 +629,73 @@ impl G9iState { return (self.regular_plan(&prefill_workers), "regular"); } - // Hit class (small prefill): ALWAYS route to the fast-P pool - // (least-wait aggregate), even when the probe says the prefix - // currently sits on a regular-P home node. User directive - // 2026-10-12: 命中都应在 fast-P 发生 — on the regular pool these - // requests queue behind long prefills (observed 150s p50), blowing - // the fast path's latency budget. fast-P resolves the prefix via - // wait_complete: shared-L3 fetch when the backup has landed, or a - // one-time recompute that grafts the session into fast-P's large - // L2 — self-healing either way; subsequent turns hit fast-P - // locally. fast-P overload is protected by the fast_p_queue 429 - // gate, never by silent fallback to the regular pool. + // Read-your-own-node: route to the node that actually holds the + // prefix locally. If that node is a regular-P, an immediate L2 hit + // beats fast-P, which would miss locally and either recompute the + // whole prefix or wait for the cross-node L3 backup to become + // visible (seconds for long prefixes; observed 10-20s at 76k). + // Multi-turn sessions therefore stay on their home P; fast-P serves + // sessions whose prefix it already holds (its own L2) and brand-new + // short prefills. let hit_is_fast = self .config .fast_prefill_urls .iter() .any(|f| urls_match(f, &hit_worker.url().to_string())); if !hit_is_fast { - // Observability: hit-class request whose prefix was NOT local - // on fast-P (L3 fetch or one-time recompute expected). - metrics::counter!(METRIC_DECISIONS, "path" => "routed_fast_p_from_regular") - .increment(1); + if let Some(load) = self.fresh_load(&hit_worker.url().to_string()) { + let est_wait = load.est_wait_secs(self.config.drain_rate_floor); + if self.config.regular_p_queue_reject_secs > 0.0 + && est_wait > self.config.regular_p_queue_reject_secs + { + warn!( + "G9i: home-hit node {} est wait {:.2}s > {:.2}s — 429 regular_p_queue", + hit_worker.url(), + est_wait, + self.config.regular_p_queue_reject_secs, + ); + return ( + PrefillPlan::Reject(RejectInfo { + reason: "regular_p_queue", + estimated_wait_secs: Some(est_wait), + }), + "home_hit", + ); + } + } else { + metrics::counter!(METRIC_NO_DATA, "pool" => "home_hit").increment(1); + } + metrics::counter!(METRIC_DECISIONS, "path" => "routed_home_hit").increment(1); + debug!( + "G9i: small prefill (local_new={} < {}) with prefix on regular node {} — routing home (L2 hit)", + new_tokens, + self.config.max_new_tokens, + hit_worker.url(), + ); + return ( + PrefillPlan::Override(normalize_url(&hit_worker.url().to_string())), + "home_hit", + ); } // Small prefill whose prefix fast-P holds locally -> fast-P pool, least-wait pick. let (picked, est_wait, fresh) = self.least_wait(&fast_pool); if fresh == 0 { - // Lost our snapshots between probe and pick; fail-open to regular. + // Lost our snapshots between probe and pick. Fail-closed like the + // probe path above; boot grace only until the first successful + // poll (mirrors the decode gate's had_any_success discipline). metrics::counter!(METRIC_DECISIONS, "path" => "fast_pool_no_fresh_snapshot") .increment(1); + if self.had_any_success.load(Ordering::SeqCst) { + warn!("G9i: no fresh fast-P snapshot at pick time — 429 fast_p_no_snapshot (fail-closed)"); + return ( + PrefillPlan::Reject(RejectInfo { + reason: "fast_p_no_snapshot", + estimated_wait_secs: None, + }), + "fast_p", + ); + } return (self.regular_plan(&prefill_workers), "regular"); } let est_wait = if picked.is_some() { est_wait } else { 0.0 };