gateway g9i: TTL reaper for wedged D-ledger bookings (pipeline 120s / streaming 600s) (P2 T19 P0-1)

This commit is contained in:
2026-10-16 15:51:40 +08:00
parent 9dd446b5c1
commit 6217b9de46
3 changed files with 290 additions and 16 deletions
+10
View File
@@ -320,6 +320,14 @@ struct CliArgs {
#[arg(long, default_value_t = 512, help_heading = "G9i Pool Admission")]
g9i_admit_hold_max: u64,
/// D-ledger pipeline booking TTL (seconds): force-release bookings that never saw a first token (wedge backstop; 0 = disabled)
#[arg(long, default_value_t = 120, help_heading = "G9i Pool Admission")]
g9i_pipeline_ttl_secs: u64,
/// D-ledger streaming booking TTL (seconds): force-release bookings stuck mid-decode (0 = disabled)
#[arg(long, default_value_t = 600, help_heading = "G9i Pool Admission")]
g9i_streaming_ttl_secs: u64,
// ==================== Service Discovery (Kubernetes) ====================
/// Enable Kubernetes service discovery
#[arg(
@@ -1208,6 +1216,8 @@ impl CliArgs {
p_queue_reqs_cap: self.g9i_p_queue_reqs_cap,
admit_hold_secs: self.g9i_admit_hold_secs,
admit_hold_max: self.g9i_admit_hold_max,
pipeline_ttl_secs: self.g9i_pipeline_ttl_secs,
streaming_ttl_secs: self.g9i_streaming_ttl_secs,
};
config.validate().map_err(|reason| ConfigError::ValidationFailed { reason })?;
Ok(Some(config))
+57
View File
@@ -230,6 +230,18 @@ pub struct G9iConfig {
/// queue rejects immediately — the last true 429. Default 512.
#[serde(default = "default_admit_hold_max")]
pub admit_hold_max: u64,
/// T19 (P0) pipeline-booking TTL (seconds): a D-ledger booking that
/// never saw a first token within this window is force-released by the
/// poller's reaper — the wedge兜底 (2026-10-16: 6 engine-side wedged
/// transfers pinned the admission window for 50+ min). Default 120
/// (4x the TTFT p99 SLO — 宁可错放不可错杀); 0 = reaper disabled.
#[serde(default = "default_pipeline_ttl_secs")]
pub pipeline_ttl_secs: u64,
/// T19 (P0) streaming-booking TTL (seconds): same兜底 for bookings
/// stuck mid-decode (first token seen, completion never arrives).
/// Default 600; 0 = disabled.
#[serde(default = "default_streaming_ttl_secs")]
pub streaming_ttl_secs: u64,
}
fn default_max_new_tokens() -> u64 {
@@ -292,6 +304,12 @@ fn default_admit_hold_secs() -> f64 {
fn default_admit_hold_max() -> u64 {
512
}
fn default_pipeline_ttl_secs() -> u64 {
120
}
fn default_streaming_ttl_secs() -> u64 {
600
}
impl Default for G9iConfig {
fn default() -> Self {
@@ -321,6 +339,8 @@ impl Default for G9iConfig {
p_queue_reqs_cap: default_p_queue_reqs_cap(),
admit_hold_secs: default_admit_hold_secs(),
admit_hold_max: default_admit_hold_max(),
pipeline_ttl_secs: default_pipeline_ttl_secs(),
streaming_ttl_secs: default_streaming_ttl_secs(),
}
}
}
@@ -865,6 +885,43 @@ impl G9iState {
if let Ok(mut p) = self.p_nodes.write() {
*p = p_results;
}
// T19 (P0) TTL reaper: force-release D-ledger bookings whose
// engine-side request wedged and will never promote/complete
// (2026-10-16 incident: 6 wedged KV transfers -> 6 phantom
// bookings -> AIMD cut to WINDOW_MIN -> pipeline >= W forever
// -> d_window_full starved the pool for 50+ min). Runs on the
// 1s poll cadence; a freed booking is room -> hold_wake.
if let Some(q) = &self.shadow_queue {
let reaped = if let Ok(mut q) = q.write() {
q.d_reap_expired(
Duration::from_secs(self.config.pipeline_ttl_secs),
Duration::from_secs(self.config.streaming_ttl_secs),
Instant::now(),
)
} else {
Vec::new()
};
let mut freed = false;
for (node, p_exp, s_exp, oldest_age) in reaped {
if p_exp > 0 {
metrics::counter!("smg_g9i_pipeline_expire_total", "worker" => node.clone())
.increment(p_exp);
}
if s_exp > 0 {
metrics::counter!("smg_g9i_streaming_expire_total", "worker" => node.clone())
.increment(s_exp);
}
warn!(
"G9i TTL reaper: {} force-released {} pipeline + {} streaming booking(s), oldest age {:.0}s (ttl {}s/{}s)",
node, p_exp, s_exp, oldest_age,
self.config.pipeline_ttl_secs, self.config.streaming_ttl_secs,
);
freed = true;
}
if freed {
self.hold_wake();
}
}
// T14: create/refresh AIMD window entries as soon as a node's
// capacity is known (the window gate is live from the first
// poll — no startup blind spot like the T13 snapshot race).
+223 -16
View File
@@ -49,7 +49,7 @@
//! against the probe ground truth via the agreement metrics.
use std::{
collections::HashMap,
collections::{HashMap, VecDeque},
sync::{
atomic::{AtomicBool, Ordering},
Arc,
@@ -560,6 +560,16 @@ struct DPoolEntry {
/// physically.
pipeline: u64,
streaming: u64,
/// T19: per-booking timestamps, in lockstep with the counters above
/// (len == counter). The TTL reaper (`d_reap_expired`) force-releases
/// bookings whose engine-side request wedged and will never
/// promote/complete — without a TTL any engine wedge permanently
/// chokes the admission window (2026-10-16 transfer-wedge incident:
/// 6 phantom bookings pinned the pool at d_window_full).
/// FIFO order is advisory only (rekey may insert an older timestamp);
/// the reaper scans, it never assumes sorted order.
pipeline_admits: VecDeque<Instant>,
streaming_since: VecDeque<Instant>,
/// Admit time of the oldest ledger resident (set on the 0→1 transition,
/// cleared at 0). Live lower bound on current decode dwell — visible
/// even when NOTHING completes (the stall case).
@@ -1111,20 +1121,33 @@ impl QueueModel {
self.d_pipeline_admit_at(node, Instant::now());
}
fn d_pipeline_admit_at(&mut self, node: &str, now: Instant) {
/// pub(crate) for G9iState-level TTL tests (explicit clock).
pub(crate) fn d_pipeline_admit_at(&mut self, node: &str, now: Instant) {
let e = self.d_pools.entry(node.to_string()).or_default();
if e.pipeline + e.streaming == 0 {
e.oldest_inflight_since = Some(now);
}
e.pipeline = e.pipeline.saturating_add(1);
e.pipeline_admits.push_back(now);
}
/// T16 state machine: first token seen -> pipeline to streaming
/// (conserving move; saturating on the source).
/// (conserving move; saturating on the source). The streaming clock
/// starts at promote time (T19: the streaming TTL is independent of
/// how long the request queued in the pipeline).
pub fn d_pipeline_promote(&mut self, node: &str) {
self.d_pipeline_promote_at(node, Instant::now());
}
/// pub(crate) for G9iState-level TTL tests (explicit clock).
pub(crate) fn d_pipeline_promote_at(&mut self, node: &str, now: Instant) {
if let Some(e) = self.d_pools.get_mut(node) {
e.pipeline = e.pipeline.saturating_sub(1);
e.streaming = e.streaming.saturating_add(1);
if e.pipeline > 0 {
e.pipeline -= 1;
e.streaming += 1;
e.pipeline_admits.pop_front();
e.streaming_since.push_back(now);
}
}
}
@@ -1133,23 +1156,24 @@ impl QueueModel {
/// never dispatched). Saturating — a stray release never underflows.
pub fn d_pipeline_release(&mut self, node: &str) {
if let Some(e) = self.d_pools.get_mut(node) {
e.pipeline = e.pipeline.saturating_sub(1);
if e.pipeline > 0 {
e.pipeline -= 1;
e.pipeline_admits.pop_front();
}
if e.pipeline + e.streaming == 0 {
e.oldest_inflight_since = None;
}
}
}
#[cfg(test)]
fn d_pipeline_release_at(&mut self, node: &str, _now: Instant) {
self.d_pipeline_release(node);
}
/// Ledger −1 for a request that was decoding (abort mid-stream,
/// streaming completion).
pub fn d_streaming_release(&mut self, node: &str) {
if let Some(e) = self.d_pools.get_mut(node) {
e.streaming = e.streaming.saturating_sub(1);
if e.streaming > 0 {
e.streaming -= 1;
e.streaming_since.pop_front();
}
if e.pipeline + e.streaming == 0 {
e.oldest_inflight_since = None;
}
@@ -1158,13 +1182,104 @@ impl QueueModel {
/// Move a pipeline booking between D nodes (decision-time pick vs
/// actual landing, mirrors the P-side rekey; rekeys happen at response
/// headers — always pre-first-token, hence pipeline-side).
/// headers — always pre-first-token, hence pipeline-side). T19: the
/// booking travels WITH its original admit timestamp — a rekeyed
/// booking must not look fresh to the TTL reaper.
pub fn d_ledger_rekey(&mut self, from: &str, to: &str) {
if from == to {
return;
}
self.d_pipeline_release(from);
self.d_pipeline_admit(to);
let ts = self.d_pools.get_mut(from).and_then(|e| {
if e.pipeline > 0 {
e.pipeline -= 1;
if e.pipeline + e.streaming == 0 {
e.oldest_inflight_since = None;
}
Some(e.pipeline_admits.pop_front())
} else {
None
}
});
let Some(ts) = ts else {
return; // nothing booked on `from` — nothing to move
};
let now = Instant::now();
let e = self.d_pools.entry(to.to_string()).or_default();
if e.pipeline + e.streaming == 0 {
e.oldest_inflight_since = Some(now);
}
e.pipeline = e.pipeline.saturating_add(1);
e.pipeline_admits.push_back(ts.unwrap_or(now));
}
/// T19 TTL reaper (楔死兜底): force-release bookings older than their
/// TTL — pipeline bookings that never saw a first token and streaming
/// bookings stuck mid-decode. An engine-side wedge (the 2026-10-16
/// transfer-queue incident: 6 requests wedged 50+ min, kv_speed=0)
/// otherwise occupies the admission window FOREVER: AIMD cuts to
/// WINDOW_MIN, pipeline stays >= W, and d_window_full starves the
/// pool. 宁可错放不可错杀: defaults are 4x/20x the TTFT SLO p99, so a
/// false release is far cheaper than a false keep. A zero TTL disables
/// that sub-ledger's reaper. Scans (no FIFO-order assumption — rekey
/// may insert older timestamps). Returns (node, pipeline_expired,
/// streaming_expired, oldest_expired_age_secs) per affected node.
pub fn d_reap_expired(
&mut self,
pipeline_ttl: Duration,
streaming_ttl: Duration,
now: Instant,
) -> Vec<(String, u64, u64, f64)> {
let mut out = Vec::new();
for (node, e) in self.d_pools.iter_mut() {
let mut oldest: Option<Instant> = None;
let mut mark = |t: Instant, oldest: &mut Option<Instant>| {
if oldest.is_none_or(|o| t < o) {
*oldest = Some(t);
}
};
let p_exp = if pipeline_ttl.is_zero() {
0
} else {
let before = e.pipeline_admits.len();
e.pipeline_admits.retain(|t| {
let expired = now.saturating_duration_since(*t) >= pipeline_ttl;
if expired {
mark(*t, &mut oldest);
}
!expired
});
(before - e.pipeline_admits.len()) as u64
};
e.pipeline = e.pipeline.saturating_sub(p_exp);
let s_exp = if streaming_ttl.is_zero() {
0
} else {
let before = e.streaming_since.len();
e.streaming_since.retain(|t| {
let expired = now.saturating_duration_since(*t) >= streaming_ttl;
if expired {
mark(*t, &mut oldest);
}
!expired
});
(before - e.streaming_since.len()) as u64
};
e.streaming = e.streaming.saturating_sub(s_exp);
if p_exp + s_exp > 0 {
if e.pipeline + e.streaming == 0 {
e.oldest_inflight_since = None;
}
out.push((
node.clone(),
p_exp,
s_exp,
oldest
.map(|t| now.saturating_duration_since(t).as_secs_f64())
.unwrap_or(0.0),
));
}
}
out
}
/// Pipeline depth (admitted, awaiting first token) — the T16 admission
@@ -2401,7 +2516,7 @@ mod tests {
// Completions pair their ledger bookings (production does this
// in shadow_record; the QueueModel completion hook deliberately
// does NOT touch the ledger).
q.d_pipeline_release_at("d1", t);
q.d_pipeline_release("d1");
}
// 236 still resident, nothing completing (long tail). At +120s the
// oldest age dominates: rate = 236 / max(0.5, 120) ≈ 1.97 req/s —
@@ -2948,6 +3063,98 @@ mod tests {
assert!((rate - 0.01).abs() < 1e-9, "1/100s -> 0.01 req/s floor-ish");
}
/// T19 (P0) TTL reaper: pipeline bookings older than the TTL are
/// force-released (the wedged-transfer incident shape); younger bookings
/// are untouched. The oldest expired age is reported for the warn log.
#[test]
fn t19_pipeline_ttl_reaps_expired() {
let mut q = QueueModel::new();
let t0 = Instant::now();
q.d_pipeline_admit_at("d1", t0 - Duration::from_secs(300));
q.d_pipeline_admit_at("d1", t0 - Duration::from_secs(200));
q.d_pipeline_admit_at("d1", t0 - Duration::from_secs(10));
let out = q.d_reap_expired(
Duration::from_secs(120),
Duration::ZERO,
t0,
);
assert_eq!(out.len(), 1);
let (node, p, s, age) = &out[0];
assert_eq!(node, "d1");
assert_eq!(*p, 2, "two expired pipeline bookings reaped");
assert_eq!(*s, 0);
assert!((*age - 300.0).abs() < 1.0, "oldest expired age ~300s, got {age}");
assert_eq!(q.d_pipeline("d1"), 1, "the fresh booking survives");
assert_eq!(q.d_inflight("d1"), 1);
}
/// T19: bookings under the TTL are never touched (宁可错放不可错杀).
#[test]
fn t19_pipeline_ttl_under_ttl_untouched() {
let mut q = QueueModel::new();
let t0 = Instant::now();
q.d_pipeline_admit_at("d1", t0 - Duration::from_secs(60));
q.d_pipeline_admit_at("d1", t0 - Duration::from_secs(119));
let out = q.d_reap_expired(Duration::from_secs(120), Duration::ZERO, t0);
assert!(out.is_empty(), "nothing at/under the TTL is reaped");
assert_eq!(q.d_pipeline("d1"), 2);
}
/// T19: a PROMOTED booking leaves the pipeline TTL's reach — it is
/// judged by the streaming TTL from promote time (the pipeline TTL must
/// not kill a healthy decoding request that queued long).
#[test]
fn t19_promoted_booking_uses_streaming_ttl() {
let mut q = QueueModel::new();
let t0 = Instant::now();
// Admitted 300s ago (way past the pipeline TTL), promoted 100s ago.
q.d_pipeline_admit_at("d1", t0 - Duration::from_secs(300));
q.d_pipeline_promote_at("d1", t0 - Duration::from_secs(100));
let out = q.d_reap_expired(Duration::from_secs(120), Duration::from_secs(600), t0);
assert!(out.is_empty(), "promoted: pipeline TTL no longer applies");
assert_eq!(q.d_streaming("d1"), 1);
// But the streaming TTL (from promote time) does reap it.
let out = q.d_reap_expired(Duration::from_secs(120), Duration::from_secs(60), t0);
assert_eq!(out.len(), 1);
assert_eq!(out[0].2, 1, "streaming TTL reaps the promoted booking");
assert_eq!(q.d_inflight("d1"), 0);
}
/// T19: a rekeyed booking travels WITH its original admit timestamp — a
/// moved booking must not look fresh to the reaper.
#[test]
fn t19_rekey_preserves_admit_time() {
let mut q = QueueModel::new();
let t0 = Instant::now();
q.d_pipeline_admit_at("d1", t0 - Duration::from_secs(300));
q.d_ledger_rekey("d1", "d2");
assert_eq!(q.d_pipeline("d1"), 0);
assert_eq!(q.d_pipeline("d2"), 1);
let out = q.d_reap_expired(Duration::from_secs(120), Duration::ZERO, t0);
assert_eq!(out.len(), 1, "the old timestamp still expires after rekey");
assert_eq!(out[0].0, "d2", "the reaper reports the CURRENT node");
assert_eq!(q.d_pipeline("d2"), 0);
// Rekey with no source booking is a no-op (never manufactures one).
q.d_ledger_rekey("d1", "d3");
assert_eq!(q.d_pipeline("d3"), 0);
}
/// T19: a zero TTL disables that sub-ledger's reaper entirely.
#[test]
fn t19_zero_ttl_disables_reaper() {
let mut q = QueueModel::new();
let t0 = Instant::now();
q.d_pipeline_admit_at("d1", t0 - Duration::from_secs(10_000));
q.d_pipeline_promote_at("d1", t0 - Duration::from_secs(9_000));
let out = q.d_reap_expired(Duration::ZERO, Duration::ZERO, t0);
assert!(out.is_empty(), "zero TTL = reaper off on both sub-ledgers");
assert_eq!(q.d_inflight("d1"), 1);
// Pipeline off, streaming on: only the streaming side is judged.
let out = q.d_reap_expired(Duration::ZERO, Duration::from_secs(60), t0);
assert_eq!(out.len(), 1);
assert_eq!(out[0].2, 1);
}
/// T17 修正版: the engine-real queue verdict (real_over predicate)
/// excludes candidates exactly like the token cap — holder skipped in
/// enforce, pick unchanged in shadow with the signal computed.