gateway g9i: TTL reaper for wedged D-ledger bookings (pipeline 120s / streaming 600s) (P2 T19 P0-1)
This commit is contained in:
@@ -320,6 +320,14 @@ struct CliArgs {
|
||||
#[arg(long, default_value_t = 512, help_heading = "G9i Pool Admission")]
|
||||
g9i_admit_hold_max: u64,
|
||||
|
||||
/// D-ledger pipeline booking TTL (seconds): force-release bookings that never saw a first token (wedge backstop; 0 = disabled)
|
||||
#[arg(long, default_value_t = 120, help_heading = "G9i Pool Admission")]
|
||||
g9i_pipeline_ttl_secs: u64,
|
||||
|
||||
/// D-ledger streaming booking TTL (seconds): force-release bookings stuck mid-decode (0 = disabled)
|
||||
#[arg(long, default_value_t = 600, help_heading = "G9i Pool Admission")]
|
||||
g9i_streaming_ttl_secs: u64,
|
||||
|
||||
// ==================== Service Discovery (Kubernetes) ====================
|
||||
/// Enable Kubernetes service discovery
|
||||
#[arg(
|
||||
@@ -1208,6 +1216,8 @@ impl CliArgs {
|
||||
p_queue_reqs_cap: self.g9i_p_queue_reqs_cap,
|
||||
admit_hold_secs: self.g9i_admit_hold_secs,
|
||||
admit_hold_max: self.g9i_admit_hold_max,
|
||||
pipeline_ttl_secs: self.g9i_pipeline_ttl_secs,
|
||||
streaming_ttl_secs: self.g9i_streaming_ttl_secs,
|
||||
};
|
||||
config.validate().map_err(|reason| ConfigError::ValidationFailed { reason })?;
|
||||
Ok(Some(config))
|
||||
|
||||
@@ -230,6 +230,18 @@ pub struct G9iConfig {
|
||||
/// queue rejects immediately — the last true 429. Default 512.
|
||||
#[serde(default = "default_admit_hold_max")]
|
||||
pub admit_hold_max: u64,
|
||||
/// T19 (P0) pipeline-booking TTL (seconds): a D-ledger booking that
|
||||
/// never saw a first token within this window is force-released by the
|
||||
/// poller's reaper — the wedge兜底 (2026-10-16: 6 engine-side wedged
|
||||
/// transfers pinned the admission window for 50+ min). Default 120
|
||||
/// (4x the TTFT p99 SLO — 宁可错放不可错杀); 0 = reaper disabled.
|
||||
#[serde(default = "default_pipeline_ttl_secs")]
|
||||
pub pipeline_ttl_secs: u64,
|
||||
/// T19 (P0) streaming-booking TTL (seconds): same兜底 for bookings
|
||||
/// stuck mid-decode (first token seen, completion never arrives).
|
||||
/// Default 600; 0 = disabled.
|
||||
#[serde(default = "default_streaming_ttl_secs")]
|
||||
pub streaming_ttl_secs: u64,
|
||||
}
|
||||
|
||||
fn default_max_new_tokens() -> u64 {
|
||||
@@ -292,6 +304,12 @@ fn default_admit_hold_secs() -> f64 {
|
||||
fn default_admit_hold_max() -> u64 {
|
||||
512
|
||||
}
|
||||
fn default_pipeline_ttl_secs() -> u64 {
|
||||
120
|
||||
}
|
||||
fn default_streaming_ttl_secs() -> u64 {
|
||||
600
|
||||
}
|
||||
|
||||
impl Default for G9iConfig {
|
||||
fn default() -> Self {
|
||||
@@ -321,6 +339,8 @@ impl Default for G9iConfig {
|
||||
p_queue_reqs_cap: default_p_queue_reqs_cap(),
|
||||
admit_hold_secs: default_admit_hold_secs(),
|
||||
admit_hold_max: default_admit_hold_max(),
|
||||
pipeline_ttl_secs: default_pipeline_ttl_secs(),
|
||||
streaming_ttl_secs: default_streaming_ttl_secs(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -865,6 +885,43 @@ impl G9iState {
|
||||
if let Ok(mut p) = self.p_nodes.write() {
|
||||
*p = p_results;
|
||||
}
|
||||
// T19 (P0) TTL reaper: force-release D-ledger bookings whose
|
||||
// engine-side request wedged and will never promote/complete
|
||||
// (2026-10-16 incident: 6 wedged KV transfers -> 6 phantom
|
||||
// bookings -> AIMD cut to WINDOW_MIN -> pipeline >= W forever
|
||||
// -> d_window_full starved the pool for 50+ min). Runs on the
|
||||
// 1s poll cadence; a freed booking is room -> hold_wake.
|
||||
if let Some(q) = &self.shadow_queue {
|
||||
let reaped = if let Ok(mut q) = q.write() {
|
||||
q.d_reap_expired(
|
||||
Duration::from_secs(self.config.pipeline_ttl_secs),
|
||||
Duration::from_secs(self.config.streaming_ttl_secs),
|
||||
Instant::now(),
|
||||
)
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
let mut freed = false;
|
||||
for (node, p_exp, s_exp, oldest_age) in reaped {
|
||||
if p_exp > 0 {
|
||||
metrics::counter!("smg_g9i_pipeline_expire_total", "worker" => node.clone())
|
||||
.increment(p_exp);
|
||||
}
|
||||
if s_exp > 0 {
|
||||
metrics::counter!("smg_g9i_streaming_expire_total", "worker" => node.clone())
|
||||
.increment(s_exp);
|
||||
}
|
||||
warn!(
|
||||
"G9i TTL reaper: {} force-released {} pipeline + {} streaming booking(s), oldest age {:.0}s (ttl {}s/{}s)",
|
||||
node, p_exp, s_exp, oldest_age,
|
||||
self.config.pipeline_ttl_secs, self.config.streaming_ttl_secs,
|
||||
);
|
||||
freed = true;
|
||||
}
|
||||
if freed {
|
||||
self.hold_wake();
|
||||
}
|
||||
}
|
||||
// T14: create/refresh AIMD window entries as soon as a node's
|
||||
// capacity is known (the window gate is live from the first
|
||||
// poll — no startup blind spot like the T13 snapshot race).
|
||||
|
||||
@@ -49,7 +49,7 @@
|
||||
//! against the probe ground truth via the agreement metrics.
|
||||
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
collections::{HashMap, VecDeque},
|
||||
sync::{
|
||||
atomic::{AtomicBool, Ordering},
|
||||
Arc,
|
||||
@@ -560,6 +560,16 @@ struct DPoolEntry {
|
||||
/// physically.
|
||||
pipeline: u64,
|
||||
streaming: u64,
|
||||
/// T19: per-booking timestamps, in lockstep with the counters above
|
||||
/// (len == counter). The TTL reaper (`d_reap_expired`) force-releases
|
||||
/// bookings whose engine-side request wedged and will never
|
||||
/// promote/complete — without a TTL any engine wedge permanently
|
||||
/// chokes the admission window (2026-10-16 transfer-wedge incident:
|
||||
/// 6 phantom bookings pinned the pool at d_window_full).
|
||||
/// FIFO order is advisory only (rekey may insert an older timestamp);
|
||||
/// the reaper scans, it never assumes sorted order.
|
||||
pipeline_admits: VecDeque<Instant>,
|
||||
streaming_since: VecDeque<Instant>,
|
||||
/// Admit time of the oldest ledger resident (set on the 0→1 transition,
|
||||
/// cleared at 0). Live lower bound on current decode dwell — visible
|
||||
/// even when NOTHING completes (the stall case).
|
||||
@@ -1111,20 +1121,33 @@ impl QueueModel {
|
||||
self.d_pipeline_admit_at(node, Instant::now());
|
||||
}
|
||||
|
||||
fn d_pipeline_admit_at(&mut self, node: &str, now: Instant) {
|
||||
/// pub(crate) for G9iState-level TTL tests (explicit clock).
|
||||
pub(crate) fn d_pipeline_admit_at(&mut self, node: &str, now: Instant) {
|
||||
let e = self.d_pools.entry(node.to_string()).or_default();
|
||||
if e.pipeline + e.streaming == 0 {
|
||||
e.oldest_inflight_since = Some(now);
|
||||
}
|
||||
e.pipeline = e.pipeline.saturating_add(1);
|
||||
e.pipeline_admits.push_back(now);
|
||||
}
|
||||
|
||||
/// T16 state machine: first token seen -> pipeline to streaming
|
||||
/// (conserving move; saturating on the source).
|
||||
/// (conserving move; saturating on the source). The streaming clock
|
||||
/// starts at promote time (T19: the streaming TTL is independent of
|
||||
/// how long the request queued in the pipeline).
|
||||
pub fn d_pipeline_promote(&mut self, node: &str) {
|
||||
self.d_pipeline_promote_at(node, Instant::now());
|
||||
}
|
||||
|
||||
/// pub(crate) for G9iState-level TTL tests (explicit clock).
|
||||
pub(crate) fn d_pipeline_promote_at(&mut self, node: &str, now: Instant) {
|
||||
if let Some(e) = self.d_pools.get_mut(node) {
|
||||
e.pipeline = e.pipeline.saturating_sub(1);
|
||||
e.streaming = e.streaming.saturating_add(1);
|
||||
if e.pipeline > 0 {
|
||||
e.pipeline -= 1;
|
||||
e.streaming += 1;
|
||||
e.pipeline_admits.pop_front();
|
||||
e.streaming_since.push_back(now);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1133,23 +1156,24 @@ impl QueueModel {
|
||||
/// never dispatched). Saturating — a stray release never underflows.
|
||||
pub fn d_pipeline_release(&mut self, node: &str) {
|
||||
if let Some(e) = self.d_pools.get_mut(node) {
|
||||
e.pipeline = e.pipeline.saturating_sub(1);
|
||||
if e.pipeline > 0 {
|
||||
e.pipeline -= 1;
|
||||
e.pipeline_admits.pop_front();
|
||||
}
|
||||
if e.pipeline + e.streaming == 0 {
|
||||
e.oldest_inflight_since = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn d_pipeline_release_at(&mut self, node: &str, _now: Instant) {
|
||||
self.d_pipeline_release(node);
|
||||
}
|
||||
|
||||
/// Ledger −1 for a request that was decoding (abort mid-stream,
|
||||
/// streaming completion).
|
||||
pub fn d_streaming_release(&mut self, node: &str) {
|
||||
if let Some(e) = self.d_pools.get_mut(node) {
|
||||
e.streaming = e.streaming.saturating_sub(1);
|
||||
if e.streaming > 0 {
|
||||
e.streaming -= 1;
|
||||
e.streaming_since.pop_front();
|
||||
}
|
||||
if e.pipeline + e.streaming == 0 {
|
||||
e.oldest_inflight_since = None;
|
||||
}
|
||||
@@ -1158,13 +1182,104 @@ impl QueueModel {
|
||||
|
||||
/// Move a pipeline booking between D nodes (decision-time pick vs
|
||||
/// actual landing, mirrors the P-side rekey; rekeys happen at response
|
||||
/// headers — always pre-first-token, hence pipeline-side).
|
||||
/// headers — always pre-first-token, hence pipeline-side). T19: the
|
||||
/// booking travels WITH its original admit timestamp — a rekeyed
|
||||
/// booking must not look fresh to the TTL reaper.
|
||||
pub fn d_ledger_rekey(&mut self, from: &str, to: &str) {
|
||||
if from == to {
|
||||
return;
|
||||
}
|
||||
self.d_pipeline_release(from);
|
||||
self.d_pipeline_admit(to);
|
||||
let ts = self.d_pools.get_mut(from).and_then(|e| {
|
||||
if e.pipeline > 0 {
|
||||
e.pipeline -= 1;
|
||||
if e.pipeline + e.streaming == 0 {
|
||||
e.oldest_inflight_since = None;
|
||||
}
|
||||
Some(e.pipeline_admits.pop_front())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
});
|
||||
let Some(ts) = ts else {
|
||||
return; // nothing booked on `from` — nothing to move
|
||||
};
|
||||
let now = Instant::now();
|
||||
let e = self.d_pools.entry(to.to_string()).or_default();
|
||||
if e.pipeline + e.streaming == 0 {
|
||||
e.oldest_inflight_since = Some(now);
|
||||
}
|
||||
e.pipeline = e.pipeline.saturating_add(1);
|
||||
e.pipeline_admits.push_back(ts.unwrap_or(now));
|
||||
}
|
||||
|
||||
/// T19 TTL reaper (楔死兜底): force-release bookings older than their
|
||||
/// TTL — pipeline bookings that never saw a first token and streaming
|
||||
/// bookings stuck mid-decode. An engine-side wedge (the 2026-10-16
|
||||
/// transfer-queue incident: 6 requests wedged 50+ min, kv_speed=0)
|
||||
/// otherwise occupies the admission window FOREVER: AIMD cuts to
|
||||
/// WINDOW_MIN, pipeline stays >= W, and d_window_full starves the
|
||||
/// pool. 宁可错放不可错杀: defaults are 4x/20x the TTFT SLO p99, so a
|
||||
/// false release is far cheaper than a false keep. A zero TTL disables
|
||||
/// that sub-ledger's reaper. Scans (no FIFO-order assumption — rekey
|
||||
/// may insert older timestamps). Returns (node, pipeline_expired,
|
||||
/// streaming_expired, oldest_expired_age_secs) per affected node.
|
||||
pub fn d_reap_expired(
|
||||
&mut self,
|
||||
pipeline_ttl: Duration,
|
||||
streaming_ttl: Duration,
|
||||
now: Instant,
|
||||
) -> Vec<(String, u64, u64, f64)> {
|
||||
let mut out = Vec::new();
|
||||
for (node, e) in self.d_pools.iter_mut() {
|
||||
let mut oldest: Option<Instant> = None;
|
||||
let mut mark = |t: Instant, oldest: &mut Option<Instant>| {
|
||||
if oldest.is_none_or(|o| t < o) {
|
||||
*oldest = Some(t);
|
||||
}
|
||||
};
|
||||
let p_exp = if pipeline_ttl.is_zero() {
|
||||
0
|
||||
} else {
|
||||
let before = e.pipeline_admits.len();
|
||||
e.pipeline_admits.retain(|t| {
|
||||
let expired = now.saturating_duration_since(*t) >= pipeline_ttl;
|
||||
if expired {
|
||||
mark(*t, &mut oldest);
|
||||
}
|
||||
!expired
|
||||
});
|
||||
(before - e.pipeline_admits.len()) as u64
|
||||
};
|
||||
e.pipeline = e.pipeline.saturating_sub(p_exp);
|
||||
let s_exp = if streaming_ttl.is_zero() {
|
||||
0
|
||||
} else {
|
||||
let before = e.streaming_since.len();
|
||||
e.streaming_since.retain(|t| {
|
||||
let expired = now.saturating_duration_since(*t) >= streaming_ttl;
|
||||
if expired {
|
||||
mark(*t, &mut oldest);
|
||||
}
|
||||
!expired
|
||||
});
|
||||
(before - e.streaming_since.len()) as u64
|
||||
};
|
||||
e.streaming = e.streaming.saturating_sub(s_exp);
|
||||
if p_exp + s_exp > 0 {
|
||||
if e.pipeline + e.streaming == 0 {
|
||||
e.oldest_inflight_since = None;
|
||||
}
|
||||
out.push((
|
||||
node.clone(),
|
||||
p_exp,
|
||||
s_exp,
|
||||
oldest
|
||||
.map(|t| now.saturating_duration_since(t).as_secs_f64())
|
||||
.unwrap_or(0.0),
|
||||
));
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Pipeline depth (admitted, awaiting first token) — the T16 admission
|
||||
@@ -2401,7 +2516,7 @@ mod tests {
|
||||
// Completions pair their ledger bookings (production does this
|
||||
// in shadow_record; the QueueModel completion hook deliberately
|
||||
// does NOT touch the ledger).
|
||||
q.d_pipeline_release_at("d1", t);
|
||||
q.d_pipeline_release("d1");
|
||||
}
|
||||
// 236 still resident, nothing completing (long tail). At +120s the
|
||||
// oldest age dominates: rate = 236 / max(0.5, 120) ≈ 1.97 req/s —
|
||||
@@ -2948,6 +3063,98 @@ mod tests {
|
||||
assert!((rate - 0.01).abs() < 1e-9, "1/100s -> 0.01 req/s floor-ish");
|
||||
}
|
||||
|
||||
/// T19 (P0) TTL reaper: pipeline bookings older than the TTL are
|
||||
/// force-released (the wedged-transfer incident shape); younger bookings
|
||||
/// are untouched. The oldest expired age is reported for the warn log.
|
||||
#[test]
|
||||
fn t19_pipeline_ttl_reaps_expired() {
|
||||
let mut q = QueueModel::new();
|
||||
let t0 = Instant::now();
|
||||
q.d_pipeline_admit_at("d1", t0 - Duration::from_secs(300));
|
||||
q.d_pipeline_admit_at("d1", t0 - Duration::from_secs(200));
|
||||
q.d_pipeline_admit_at("d1", t0 - Duration::from_secs(10));
|
||||
let out = q.d_reap_expired(
|
||||
Duration::from_secs(120),
|
||||
Duration::ZERO,
|
||||
t0,
|
||||
);
|
||||
assert_eq!(out.len(), 1);
|
||||
let (node, p, s, age) = &out[0];
|
||||
assert_eq!(node, "d1");
|
||||
assert_eq!(*p, 2, "two expired pipeline bookings reaped");
|
||||
assert_eq!(*s, 0);
|
||||
assert!((*age - 300.0).abs() < 1.0, "oldest expired age ~300s, got {age}");
|
||||
assert_eq!(q.d_pipeline("d1"), 1, "the fresh booking survives");
|
||||
assert_eq!(q.d_inflight("d1"), 1);
|
||||
}
|
||||
|
||||
/// T19: bookings under the TTL are never touched (宁可错放不可错杀).
|
||||
#[test]
|
||||
fn t19_pipeline_ttl_under_ttl_untouched() {
|
||||
let mut q = QueueModel::new();
|
||||
let t0 = Instant::now();
|
||||
q.d_pipeline_admit_at("d1", t0 - Duration::from_secs(60));
|
||||
q.d_pipeline_admit_at("d1", t0 - Duration::from_secs(119));
|
||||
let out = q.d_reap_expired(Duration::from_secs(120), Duration::ZERO, t0);
|
||||
assert!(out.is_empty(), "nothing at/under the TTL is reaped");
|
||||
assert_eq!(q.d_pipeline("d1"), 2);
|
||||
}
|
||||
|
||||
/// T19: a PROMOTED booking leaves the pipeline TTL's reach — it is
|
||||
/// judged by the streaming TTL from promote time (the pipeline TTL must
|
||||
/// not kill a healthy decoding request that queued long).
|
||||
#[test]
|
||||
fn t19_promoted_booking_uses_streaming_ttl() {
|
||||
let mut q = QueueModel::new();
|
||||
let t0 = Instant::now();
|
||||
// Admitted 300s ago (way past the pipeline TTL), promoted 100s ago.
|
||||
q.d_pipeline_admit_at("d1", t0 - Duration::from_secs(300));
|
||||
q.d_pipeline_promote_at("d1", t0 - Duration::from_secs(100));
|
||||
let out = q.d_reap_expired(Duration::from_secs(120), Duration::from_secs(600), t0);
|
||||
assert!(out.is_empty(), "promoted: pipeline TTL no longer applies");
|
||||
assert_eq!(q.d_streaming("d1"), 1);
|
||||
// But the streaming TTL (from promote time) does reap it.
|
||||
let out = q.d_reap_expired(Duration::from_secs(120), Duration::from_secs(60), t0);
|
||||
assert_eq!(out.len(), 1);
|
||||
assert_eq!(out[0].2, 1, "streaming TTL reaps the promoted booking");
|
||||
assert_eq!(q.d_inflight("d1"), 0);
|
||||
}
|
||||
|
||||
/// T19: a rekeyed booking travels WITH its original admit timestamp — a
|
||||
/// moved booking must not look fresh to the reaper.
|
||||
#[test]
|
||||
fn t19_rekey_preserves_admit_time() {
|
||||
let mut q = QueueModel::new();
|
||||
let t0 = Instant::now();
|
||||
q.d_pipeline_admit_at("d1", t0 - Duration::from_secs(300));
|
||||
q.d_ledger_rekey("d1", "d2");
|
||||
assert_eq!(q.d_pipeline("d1"), 0);
|
||||
assert_eq!(q.d_pipeline("d2"), 1);
|
||||
let out = q.d_reap_expired(Duration::from_secs(120), Duration::ZERO, t0);
|
||||
assert_eq!(out.len(), 1, "the old timestamp still expires after rekey");
|
||||
assert_eq!(out[0].0, "d2", "the reaper reports the CURRENT node");
|
||||
assert_eq!(q.d_pipeline("d2"), 0);
|
||||
// Rekey with no source booking is a no-op (never manufactures one).
|
||||
q.d_ledger_rekey("d1", "d3");
|
||||
assert_eq!(q.d_pipeline("d3"), 0);
|
||||
}
|
||||
|
||||
/// T19: a zero TTL disables that sub-ledger's reaper entirely.
|
||||
#[test]
|
||||
fn t19_zero_ttl_disables_reaper() {
|
||||
let mut q = QueueModel::new();
|
||||
let t0 = Instant::now();
|
||||
q.d_pipeline_admit_at("d1", t0 - Duration::from_secs(10_000));
|
||||
q.d_pipeline_promote_at("d1", t0 - Duration::from_secs(9_000));
|
||||
let out = q.d_reap_expired(Duration::ZERO, Duration::ZERO, t0);
|
||||
assert!(out.is_empty(), "zero TTL = reaper off on both sub-ledgers");
|
||||
assert_eq!(q.d_inflight("d1"), 1);
|
||||
// Pipeline off, streaming on: only the streaming side is judged.
|
||||
let out = q.d_reap_expired(Duration::ZERO, Duration::from_secs(60), t0);
|
||||
assert_eq!(out.len(), 1);
|
||||
assert_eq!(out[0].2, 1);
|
||||
}
|
||||
|
||||
/// T17 修正版: the engine-real queue verdict (real_over predicate)
|
||||
/// excludes candidates exactly like the token cap — holder skipped in
|
||||
/// enforce, pick unchanged in shadow with the signal computed.
|
||||
|
||||
Reference in New Issue
Block a user