From 6217b9de46ec2efe997d83fc9f1b157ead070fa7 Mon Sep 17 00:00:00 2001 From: "minke.yu" Date: Fri, 16 Oct 2026 15:51:40 +0800 Subject: [PATCH] gateway g9i: TTL reaper for wedged D-ledger bookings (pipeline 120s / streaming 600s) (P2 T19 P0-1) --- sgl-model-gateway/src/main.rs | 10 + sgl-model-gateway/src/routers/http/g9i.rs | 57 +++++ .../src/routers/http/g9i_shadow.rs | 239 ++++++++++++++++-- 3 files changed, 290 insertions(+), 16 deletions(-) diff --git a/sgl-model-gateway/src/main.rs b/sgl-model-gateway/src/main.rs index 7b8dde8f1c..bc97485575 100644 --- a/sgl-model-gateway/src/main.rs +++ b/sgl-model-gateway/src/main.rs @@ -320,6 +320,14 @@ struct CliArgs { #[arg(long, default_value_t = 512, help_heading = "G9i Pool Admission")] g9i_admit_hold_max: u64, + /// D-ledger pipeline booking TTL (seconds): force-release bookings that never saw a first token (wedge backstop; 0 = disabled) + #[arg(long, default_value_t = 120, help_heading = "G9i Pool Admission")] + g9i_pipeline_ttl_secs: u64, + + /// D-ledger streaming booking TTL (seconds): force-release bookings stuck mid-decode (0 = disabled) + #[arg(long, default_value_t = 600, help_heading = "G9i Pool Admission")] + g9i_streaming_ttl_secs: u64, + // ==================== Service Discovery (Kubernetes) ==================== /// Enable Kubernetes service discovery #[arg( @@ -1208,6 +1216,8 @@ impl CliArgs { p_queue_reqs_cap: self.g9i_p_queue_reqs_cap, admit_hold_secs: self.g9i_admit_hold_secs, admit_hold_max: self.g9i_admit_hold_max, + pipeline_ttl_secs: self.g9i_pipeline_ttl_secs, + streaming_ttl_secs: self.g9i_streaming_ttl_secs, }; config.validate().map_err(|reason| ConfigError::ValidationFailed { reason })?; Ok(Some(config)) diff --git a/sgl-model-gateway/src/routers/http/g9i.rs b/sgl-model-gateway/src/routers/http/g9i.rs index a1f1e47b7d..4982b250d9 100644 --- a/sgl-model-gateway/src/routers/http/g9i.rs +++ b/sgl-model-gateway/src/routers/http/g9i.rs @@ -230,6 +230,18 @@ pub struct G9iConfig { /// queue rejects immediately — the last true 429. Default 512. #[serde(default = "default_admit_hold_max")] pub admit_hold_max: u64, + /// T19 (P0) pipeline-booking TTL (seconds): a D-ledger booking that + /// never saw a first token within this window is force-released by the + /// poller's reaper — the wedge兜底 (2026-10-16: 6 engine-side wedged + /// transfers pinned the admission window for 50+ min). Default 120 + /// (4x the TTFT p99 SLO — 宁可错放不可错杀); 0 = reaper disabled. + #[serde(default = "default_pipeline_ttl_secs")] + pub pipeline_ttl_secs: u64, + /// T19 (P0) streaming-booking TTL (seconds): same兜底 for bookings + /// stuck mid-decode (first token seen, completion never arrives). + /// Default 600; 0 = disabled. + #[serde(default = "default_streaming_ttl_secs")] + pub streaming_ttl_secs: u64, } fn default_max_new_tokens() -> u64 { @@ -292,6 +304,12 @@ fn default_admit_hold_secs() -> f64 { fn default_admit_hold_max() -> u64 { 512 } +fn default_pipeline_ttl_secs() -> u64 { + 120 +} +fn default_streaming_ttl_secs() -> u64 { + 600 +} impl Default for G9iConfig { fn default() -> Self { @@ -321,6 +339,8 @@ impl Default for G9iConfig { p_queue_reqs_cap: default_p_queue_reqs_cap(), admit_hold_secs: default_admit_hold_secs(), admit_hold_max: default_admit_hold_max(), + pipeline_ttl_secs: default_pipeline_ttl_secs(), + streaming_ttl_secs: default_streaming_ttl_secs(), } } } @@ -865,6 +885,43 @@ impl G9iState { if let Ok(mut p) = self.p_nodes.write() { *p = p_results; } + // T19 (P0) TTL reaper: force-release D-ledger bookings whose + // engine-side request wedged and will never promote/complete + // (2026-10-16 incident: 6 wedged KV transfers -> 6 phantom + // bookings -> AIMD cut to WINDOW_MIN -> pipeline >= W forever + // -> d_window_full starved the pool for 50+ min). Runs on the + // 1s poll cadence; a freed booking is room -> hold_wake. + if let Some(q) = &self.shadow_queue { + let reaped = if let Ok(mut q) = q.write() { + q.d_reap_expired( + Duration::from_secs(self.config.pipeline_ttl_secs), + Duration::from_secs(self.config.streaming_ttl_secs), + Instant::now(), + ) + } else { + Vec::new() + }; + let mut freed = false; + for (node, p_exp, s_exp, oldest_age) in reaped { + if p_exp > 0 { + metrics::counter!("smg_g9i_pipeline_expire_total", "worker" => node.clone()) + .increment(p_exp); + } + if s_exp > 0 { + metrics::counter!("smg_g9i_streaming_expire_total", "worker" => node.clone()) + .increment(s_exp); + } + warn!( + "G9i TTL reaper: {} force-released {} pipeline + {} streaming booking(s), oldest age {:.0}s (ttl {}s/{}s)", + node, p_exp, s_exp, oldest_age, + self.config.pipeline_ttl_secs, self.config.streaming_ttl_secs, + ); + freed = true; + } + if freed { + self.hold_wake(); + } + } // T14: create/refresh AIMD window entries as soon as a node's // capacity is known (the window gate is live from the first // poll — no startup blind spot like the T13 snapshot race). diff --git a/sgl-model-gateway/src/routers/http/g9i_shadow.rs b/sgl-model-gateway/src/routers/http/g9i_shadow.rs index 111a503fc7..32198ae19b 100644 --- a/sgl-model-gateway/src/routers/http/g9i_shadow.rs +++ b/sgl-model-gateway/src/routers/http/g9i_shadow.rs @@ -49,7 +49,7 @@ //! against the probe ground truth via the agreement metrics. use std::{ - collections::HashMap, + collections::{HashMap, VecDeque}, sync::{ atomic::{AtomicBool, Ordering}, Arc, @@ -560,6 +560,16 @@ struct DPoolEntry { /// physically. pipeline: u64, streaming: u64, + /// T19: per-booking timestamps, in lockstep with the counters above + /// (len == counter). The TTL reaper (`d_reap_expired`) force-releases + /// bookings whose engine-side request wedged and will never + /// promote/complete — without a TTL any engine wedge permanently + /// chokes the admission window (2026-10-16 transfer-wedge incident: + /// 6 phantom bookings pinned the pool at d_window_full). + /// FIFO order is advisory only (rekey may insert an older timestamp); + /// the reaper scans, it never assumes sorted order. + pipeline_admits: VecDeque, + streaming_since: VecDeque, /// Admit time of the oldest ledger resident (set on the 0→1 transition, /// cleared at 0). Live lower bound on current decode dwell — visible /// even when NOTHING completes (the stall case). @@ -1111,20 +1121,33 @@ impl QueueModel { self.d_pipeline_admit_at(node, Instant::now()); } - fn d_pipeline_admit_at(&mut self, node: &str, now: Instant) { + /// pub(crate) for G9iState-level TTL tests (explicit clock). + pub(crate) fn d_pipeline_admit_at(&mut self, node: &str, now: Instant) { let e = self.d_pools.entry(node.to_string()).or_default(); if e.pipeline + e.streaming == 0 { e.oldest_inflight_since = Some(now); } e.pipeline = e.pipeline.saturating_add(1); + e.pipeline_admits.push_back(now); } /// T16 state machine: first token seen -> pipeline to streaming - /// (conserving move; saturating on the source). + /// (conserving move; saturating on the source). The streaming clock + /// starts at promote time (T19: the streaming TTL is independent of + /// how long the request queued in the pipeline). pub fn d_pipeline_promote(&mut self, node: &str) { + self.d_pipeline_promote_at(node, Instant::now()); + } + + /// pub(crate) for G9iState-level TTL tests (explicit clock). + pub(crate) fn d_pipeline_promote_at(&mut self, node: &str, now: Instant) { if let Some(e) = self.d_pools.get_mut(node) { - e.pipeline = e.pipeline.saturating_sub(1); - e.streaming = e.streaming.saturating_add(1); + if e.pipeline > 0 { + e.pipeline -= 1; + e.streaming += 1; + e.pipeline_admits.pop_front(); + e.streaming_since.push_back(now); + } } } @@ -1133,23 +1156,24 @@ impl QueueModel { /// never dispatched). Saturating — a stray release never underflows. pub fn d_pipeline_release(&mut self, node: &str) { if let Some(e) = self.d_pools.get_mut(node) { - e.pipeline = e.pipeline.saturating_sub(1); + if e.pipeline > 0 { + e.pipeline -= 1; + e.pipeline_admits.pop_front(); + } if e.pipeline + e.streaming == 0 { e.oldest_inflight_since = None; } } } - #[cfg(test)] - fn d_pipeline_release_at(&mut self, node: &str, _now: Instant) { - self.d_pipeline_release(node); - } - /// Ledger −1 for a request that was decoding (abort mid-stream, /// streaming completion). pub fn d_streaming_release(&mut self, node: &str) { if let Some(e) = self.d_pools.get_mut(node) { - e.streaming = e.streaming.saturating_sub(1); + if e.streaming > 0 { + e.streaming -= 1; + e.streaming_since.pop_front(); + } if e.pipeline + e.streaming == 0 { e.oldest_inflight_since = None; } @@ -1158,13 +1182,104 @@ impl QueueModel { /// Move a pipeline booking between D nodes (decision-time pick vs /// actual landing, mirrors the P-side rekey; rekeys happen at response - /// headers — always pre-first-token, hence pipeline-side). + /// headers — always pre-first-token, hence pipeline-side). T19: the + /// booking travels WITH its original admit timestamp — a rekeyed + /// booking must not look fresh to the TTL reaper. pub fn d_ledger_rekey(&mut self, from: &str, to: &str) { if from == to { return; } - self.d_pipeline_release(from); - self.d_pipeline_admit(to); + let ts = self.d_pools.get_mut(from).and_then(|e| { + if e.pipeline > 0 { + e.pipeline -= 1; + if e.pipeline + e.streaming == 0 { + e.oldest_inflight_since = None; + } + Some(e.pipeline_admits.pop_front()) + } else { + None + } + }); + let Some(ts) = ts else { + return; // nothing booked on `from` — nothing to move + }; + let now = Instant::now(); + let e = self.d_pools.entry(to.to_string()).or_default(); + if e.pipeline + e.streaming == 0 { + e.oldest_inflight_since = Some(now); + } + e.pipeline = e.pipeline.saturating_add(1); + e.pipeline_admits.push_back(ts.unwrap_or(now)); + } + + /// T19 TTL reaper (楔死兜底): force-release bookings older than their + /// TTL — pipeline bookings that never saw a first token and streaming + /// bookings stuck mid-decode. An engine-side wedge (the 2026-10-16 + /// transfer-queue incident: 6 requests wedged 50+ min, kv_speed=0) + /// otherwise occupies the admission window FOREVER: AIMD cuts to + /// WINDOW_MIN, pipeline stays >= W, and d_window_full starves the + /// pool. 宁可错放不可错杀: defaults are 4x/20x the TTFT SLO p99, so a + /// false release is far cheaper than a false keep. A zero TTL disables + /// that sub-ledger's reaper. Scans (no FIFO-order assumption — rekey + /// may insert older timestamps). Returns (node, pipeline_expired, + /// streaming_expired, oldest_expired_age_secs) per affected node. + pub fn d_reap_expired( + &mut self, + pipeline_ttl: Duration, + streaming_ttl: Duration, + now: Instant, + ) -> Vec<(String, u64, u64, f64)> { + let mut out = Vec::new(); + for (node, e) in self.d_pools.iter_mut() { + let mut oldest: Option = None; + let mut mark = |t: Instant, oldest: &mut Option| { + if oldest.is_none_or(|o| t < o) { + *oldest = Some(t); + } + }; + let p_exp = if pipeline_ttl.is_zero() { + 0 + } else { + let before = e.pipeline_admits.len(); + e.pipeline_admits.retain(|t| { + let expired = now.saturating_duration_since(*t) >= pipeline_ttl; + if expired { + mark(*t, &mut oldest); + } + !expired + }); + (before - e.pipeline_admits.len()) as u64 + }; + e.pipeline = e.pipeline.saturating_sub(p_exp); + let s_exp = if streaming_ttl.is_zero() { + 0 + } else { + let before = e.streaming_since.len(); + e.streaming_since.retain(|t| { + let expired = now.saturating_duration_since(*t) >= streaming_ttl; + if expired { + mark(*t, &mut oldest); + } + !expired + }); + (before - e.streaming_since.len()) as u64 + }; + e.streaming = e.streaming.saturating_sub(s_exp); + if p_exp + s_exp > 0 { + if e.pipeline + e.streaming == 0 { + e.oldest_inflight_since = None; + } + out.push(( + node.clone(), + p_exp, + s_exp, + oldest + .map(|t| now.saturating_duration_since(t).as_secs_f64()) + .unwrap_or(0.0), + )); + } + } + out } /// Pipeline depth (admitted, awaiting first token) — the T16 admission @@ -2401,7 +2516,7 @@ mod tests { // Completions pair their ledger bookings (production does this // in shadow_record; the QueueModel completion hook deliberately // does NOT touch the ledger). - q.d_pipeline_release_at("d1", t); + q.d_pipeline_release("d1"); } // 236 still resident, nothing completing (long tail). At +120s the // oldest age dominates: rate = 236 / max(0.5, 120) ≈ 1.97 req/s — @@ -2948,6 +3063,98 @@ mod tests { assert!((rate - 0.01).abs() < 1e-9, "1/100s -> 0.01 req/s floor-ish"); } + /// T19 (P0) TTL reaper: pipeline bookings older than the TTL are + /// force-released (the wedged-transfer incident shape); younger bookings + /// are untouched. The oldest expired age is reported for the warn log. + #[test] + fn t19_pipeline_ttl_reaps_expired() { + let mut q = QueueModel::new(); + let t0 = Instant::now(); + q.d_pipeline_admit_at("d1", t0 - Duration::from_secs(300)); + q.d_pipeline_admit_at("d1", t0 - Duration::from_secs(200)); + q.d_pipeline_admit_at("d1", t0 - Duration::from_secs(10)); + let out = q.d_reap_expired( + Duration::from_secs(120), + Duration::ZERO, + t0, + ); + assert_eq!(out.len(), 1); + let (node, p, s, age) = &out[0]; + assert_eq!(node, "d1"); + assert_eq!(*p, 2, "two expired pipeline bookings reaped"); + assert_eq!(*s, 0); + assert!((*age - 300.0).abs() < 1.0, "oldest expired age ~300s, got {age}"); + assert_eq!(q.d_pipeline("d1"), 1, "the fresh booking survives"); + assert_eq!(q.d_inflight("d1"), 1); + } + + /// T19: bookings under the TTL are never touched (宁可错放不可错杀). + #[test] + fn t19_pipeline_ttl_under_ttl_untouched() { + let mut q = QueueModel::new(); + let t0 = Instant::now(); + q.d_pipeline_admit_at("d1", t0 - Duration::from_secs(60)); + q.d_pipeline_admit_at("d1", t0 - Duration::from_secs(119)); + let out = q.d_reap_expired(Duration::from_secs(120), Duration::ZERO, t0); + assert!(out.is_empty(), "nothing at/under the TTL is reaped"); + assert_eq!(q.d_pipeline("d1"), 2); + } + + /// T19: a PROMOTED booking leaves the pipeline TTL's reach — it is + /// judged by the streaming TTL from promote time (the pipeline TTL must + /// not kill a healthy decoding request that queued long). + #[test] + fn t19_promoted_booking_uses_streaming_ttl() { + let mut q = QueueModel::new(); + let t0 = Instant::now(); + // Admitted 300s ago (way past the pipeline TTL), promoted 100s ago. + q.d_pipeline_admit_at("d1", t0 - Duration::from_secs(300)); + q.d_pipeline_promote_at("d1", t0 - Duration::from_secs(100)); + let out = q.d_reap_expired(Duration::from_secs(120), Duration::from_secs(600), t0); + assert!(out.is_empty(), "promoted: pipeline TTL no longer applies"); + assert_eq!(q.d_streaming("d1"), 1); + // But the streaming TTL (from promote time) does reap it. + let out = q.d_reap_expired(Duration::from_secs(120), Duration::from_secs(60), t0); + assert_eq!(out.len(), 1); + assert_eq!(out[0].2, 1, "streaming TTL reaps the promoted booking"); + assert_eq!(q.d_inflight("d1"), 0); + } + + /// T19: a rekeyed booking travels WITH its original admit timestamp — a + /// moved booking must not look fresh to the reaper. + #[test] + fn t19_rekey_preserves_admit_time() { + let mut q = QueueModel::new(); + let t0 = Instant::now(); + q.d_pipeline_admit_at("d1", t0 - Duration::from_secs(300)); + q.d_ledger_rekey("d1", "d2"); + assert_eq!(q.d_pipeline("d1"), 0); + assert_eq!(q.d_pipeline("d2"), 1); + let out = q.d_reap_expired(Duration::from_secs(120), Duration::ZERO, t0); + assert_eq!(out.len(), 1, "the old timestamp still expires after rekey"); + assert_eq!(out[0].0, "d2", "the reaper reports the CURRENT node"); + assert_eq!(q.d_pipeline("d2"), 0); + // Rekey with no source booking is a no-op (never manufactures one). + q.d_ledger_rekey("d1", "d3"); + assert_eq!(q.d_pipeline("d3"), 0); + } + + /// T19: a zero TTL disables that sub-ledger's reaper entirely. + #[test] + fn t19_zero_ttl_disables_reaper() { + let mut q = QueueModel::new(); + let t0 = Instant::now(); + q.d_pipeline_admit_at("d1", t0 - Duration::from_secs(10_000)); + q.d_pipeline_promote_at("d1", t0 - Duration::from_secs(9_000)); + let out = q.d_reap_expired(Duration::ZERO, Duration::ZERO, t0); + assert!(out.is_empty(), "zero TTL = reaper off on both sub-ledgers"); + assert_eq!(q.d_inflight("d1"), 1); + // Pipeline off, streaming on: only the streaming side is judged. + let out = q.d_reap_expired(Duration::ZERO, Duration::from_secs(60), t0); + assert_eq!(out.len(), 1); + assert_eq!(out[0].2, 1); + } + /// T17 修正版: the engine-real queue verdict (real_over predicate) /// excludes candidates exactly like the token cap — holder skipped in /// enforce, pick unchanged in shadow with the signal computed.