diff --git a/sgl-model-gateway/src/main.rs b/sgl-model-gateway/src/main.rs index bc97485575..a3498c95b8 100644 --- a/sgl-model-gateway/src/main.rs +++ b/sgl-model-gateway/src/main.rs @@ -328,6 +328,10 @@ struct CliArgs { #[arg(long, default_value_t = 600, help_heading = "G9i Pool Admission")] g9i_streaming_ttl_secs: u64, + /// Ledger reconcile: clamp the D ledger down to the engine-side in-flight when the excess is sustained >60s (subtract-only) + #[arg(long, default_value_t = true, help_heading = "G9i Pool Admission")] + g9i_ledger_reconcile: bool, + // ==================== Service Discovery (Kubernetes) ==================== /// Enable Kubernetes service discovery #[arg( @@ -1218,6 +1222,7 @@ impl CliArgs { admit_hold_max: self.g9i_admit_hold_max, pipeline_ttl_secs: self.g9i_pipeline_ttl_secs, streaming_ttl_secs: self.g9i_streaming_ttl_secs, + ledger_reconcile: self.g9i_ledger_reconcile, }; config.validate().map_err(|reason| ConfigError::ValidationFailed { reason })?; Ok(Some(config)) diff --git a/sgl-model-gateway/src/routers/http/g9i.rs b/sgl-model-gateway/src/routers/http/g9i.rs index 4982b250d9..3ada8fa241 100644 --- a/sgl-model-gateway/src/routers/http/g9i.rs +++ b/sgl-model-gateway/src/routers/http/g9i.rs @@ -242,6 +242,13 @@ pub struct G9iConfig { /// Default 600; 0 = disabled. #[serde(default = "default_streaming_ttl_secs")] pub streaming_ttl_secs: u64, + /// T19 (P0) ledger reconcile: when the gateway-side D ledger exceeds the + /// engine-side in-flight (running + prealloc + transfer) by more than + /// max(8, 25%) for a sustained 60s, clamp the ledger DOWN to the engine + /// value (subtract-only; drift self-corrects instead of wedging the + /// admission window forever). Default on. + #[serde(default = "default_ledger_reconcile")] + pub ledger_reconcile: bool, } fn default_max_new_tokens() -> u64 { @@ -310,6 +317,9 @@ fn default_pipeline_ttl_secs() -> u64 { fn default_streaming_ttl_secs() -> u64 { 600 } +fn default_ledger_reconcile() -> bool { + true +} impl Default for G9iConfig { fn default() -> Self { @@ -341,6 +351,7 @@ impl Default for G9iConfig { admit_hold_max: default_admit_hold_max(), pipeline_ttl_secs: default_pipeline_ttl_secs(), streaming_ttl_secs: default_streaming_ttl_secs(), + ledger_reconcile: default_ledger_reconcile(), } } } @@ -523,6 +534,10 @@ pub struct G9iState { /// underlying cache_aware policy, which pinned same-prefix sessions /// onto a single node. rr_cursor: AtomicUsize, + /// T19 (P0) ledger reconcile: per-decode-node start Instant of the + /// current sustained ledger>>engine excess episode; 60s sustained -> + /// clamp. Cleared as soon as the excess disappears. + reconcile_since: RwLock>, } /// T14 AIMD congestion-window state for one decode node. @@ -633,6 +648,11 @@ const AIMD_MIN_SAMPLES: usize = 20; /// T18: bootstrap /metrics scrape cadence (own slow loop, off the snapshot /// round). const BOOTSTRAP_SCRAPE_INTERVAL_SECS: u64 = 10; +/// T19 (P0) ledger reconcile: clamp only when the gateway ledger exceeds +/// the engine-side in-flight (running + prealloc + transfer) by more than +/// max(8, 25%) for this long — transient poll-lag drift must never trigger +/// a clamp (只减不加, and only after sustained divergence). +const RECONCILE_SUSTAIN_SECS: u64 = 60; /// TTFT SLO lines (p50/p95/p99 seconds). const SLO_P50_SECS: f64 = 5.0; const SLO_P95_SECS: f64 = 10.0; @@ -738,6 +758,7 @@ impl G9iState { hold_timeouts_total: AtomicU64::new(0), last_tick_timeouts: AtomicU64::new(0), rr_cursor: AtomicUsize::new(0), + reconcile_since: RwLock::new(HashMap::new()), } } @@ -922,6 +943,53 @@ impl G9iState { self.hold_wake(); } } + // T19 (P0) ledger reconcile (对账纠偏): the T18 ledger is a + // faithful bookkeeper — when the ENGINE wedges (booking never + // promotes/completes), the ledger is right but useless, and the + // phantom bookings pin the AIMD window shut. If the ledger + // exceeds the engine-side in-flight (running + prealloc + + // transfer) by > max(8, 25%) for a sustained 60s, clamp the + // ledger DOWN to the engine value. Subtract-only; stale engine + // views never justify a clamp. Runs after the TTL reaper (the + // reaper is the precise instrument; this is the drift fuse). + if self.config.ledger_reconcile { + let snap_ttl = Duration::from_secs(self.config.snapshot_ttl_secs); + let mut clamps: Vec<(String, u64, u64, u64)> = Vec::new(); + if let (Some(q), Ok(d)) = (self.shadow_queue.as_ref(), self.d_snapshots.read()) { + if let Ok(mut q) = q.write() { + let now = Instant::now(); + for (url, load) in d.iter() { + let engine = load.running_reqs + + load.decode_prealloc_queue_reqs + + load.decode_transfer_queue_reqs; + let ledger = q.d_inflight(url); + if let Some(target) = self.reconcile_target( + url, + ledger, + engine, + load.is_fresh(snap_ttl), + now, + ) { + let released = q.d_clamp_to(url, target); + if released > 0 { + clamps.push((url.clone(), released, ledger, engine)); + } + } + } + } + } + for (node, released, ledger, engine) in &clamps { + metrics::counter!("smg_g9i_ledger_reconcile_total", "worker" => node.clone()) + .increment(*released); + warn!( + "G9i ledger reconcile: {} ledger={} vs engine={} sustained >{}s — clamped, {} phantom booking(s) released", + node, ledger, engine, RECONCILE_SUSTAIN_SECS, released, + ); + } + if !clamps.is_empty() { + self.hold_wake(); + } + } // T14: create/refresh AIMD window entries as soon as a node's // capacity is known (the window gate is live from the first // poll — no startup blind spot like the T13 snapshot race). @@ -2060,6 +2128,39 @@ impl G9iState { .unwrap_or_default() } + /// T19 (P0) ledger reconcile verdict: Some(engine) when the gateway + /// ledger has exceeded the engine-side in-flight by more than + /// max(8, 25%) for RECONCILE_SUSTAIN_SECS — the caller then clamps the + /// ledger down to the engine value. Subtract-only, fresh snapshots only + /// (a stale engine view never justifies a clamp, and a stale stretch + /// does not count toward the sustain window — the marker is cleared so + /// the 60s must be CONTINUOUS fresh excess). The excess disappearing + /// also clears the marker immediately. + fn reconcile_target( + &self, + node: &str, + ledger: u64, + engine: u64, + fresh: bool, + now: Instant, + ) -> Option { + let margin = (engine / 4).max(8); + let over = fresh && ledger > engine.saturating_add(margin); + let Ok(mut since) = self.reconcile_since.write() else { + return None; + }; + if !over { + since.remove(node); + return None; + } + let start = *since.entry(node.to_string()).or_insert(now); + if now.saturating_duration_since(start) >= Duration::from_secs(RECONCILE_SUSTAIN_SECS) { + Some(engine) + } else { + None + } + } + /// T11: shadow classify + T_PREFER/cap candidate pick + admission in a /// SINGLE queue-model critical section (atomic check-and-act). The /// pre-T11 flow admitted only when response headers arrived — after the @@ -3456,6 +3557,93 @@ sglang:token_usage{dp_rank=\"0\"} 0.5 ); } + /// T19 (P0) ledger reconcile: the clamp verdict requires a SUSTAINED + /// (60s) fresh excess of ledger over engine by > max(8, 25%); transient + /// drift never clamps. The clamp itself is subtract-only (QueueModel + /// level covered by t19_clamp_subtract_only). + #[test] + fn t19_reconcile_requires_60s_sustain() { + let st = p2_state("enforce"); + let node = "http://d1:30001"; + set_d_snapshots(&st, &[(node, 0, 10_000_000, true, false)]); + book_ledger(&st, node, 20); // engine = 0, margin = max(8, 0) = 8 -> over + let t0 = Instant::now(); + assert_eq!( + st.reconcile_target(node, 20, 0, true, t0), + None, + "first sight starts the clock, no clamp" + ); + assert_eq!( + st.reconcile_target(node, 20, 0, true, t0 + Duration::from_secs(30)), + None, + "30s sustained — still under the 60s fuse" + ); + assert_eq!( + st.reconcile_target(node, 20, 0, true, t0 + Duration::from_secs(61)), + Some(0), + "61s sustained -> clamp to the engine value" + ); + // Applying the verdict releases exactly the phantom bookings. + let q = st.shadow_queue.as_ref().unwrap(); + assert_eq!(q.write().unwrap().d_clamp_to(node, 0), 20); + assert_eq!(q.read().unwrap().d_inflight(node), 0); + // Excess disappears -> marker cleared; a later excess restarts at 0. + assert_eq!( + st.reconcile_target(node, 5, 0, true, t0 + Duration::from_secs(62)), + None + ); + assert!( + st.reconcile_since.read().unwrap().is_empty(), + "marker cleared when the excess disappears" + ); + assert_eq!( + st.reconcile_target(node, 20, 0, true, t0 + Duration::from_secs(200)), + None, + "a reappearing excess restarts the clock from zero" + ); + } + + /// T19: margin and freshness rules — ledger <= engine + max(8, 25%) is + /// never clamped; stale snapshots never clamp and clear the marker (a + /// stale stretch does not count toward the 60s); ledger < engine never. + #[test] + fn t19_reconcile_margin_and_freshness() { + let st = p2_state("enforce"); + let node = "http://d1:30001"; + let t0 = Instant::now(); + // engine 100 -> margin max(8, 25) = 25: ledger 125 NOT over, 126 over. + assert_eq!(st.reconcile_target(node, 125, 100, true, t0), None); + assert_eq!( + st.reconcile_target(node, 126, 100, true, t0), + None, + "over the margin, but the clock just started" + ); + // Stale snapshot: never clamps and clears the marker. + assert_eq!( + st.reconcile_target(node, 1000, 100, false, t0 + Duration::from_secs(120)), + None, + "stale engine view never justifies a clamp" + ); + assert!( + !st.reconcile_since.read().unwrap().contains_key(node), + "the stale stretch cleared the marker" + ); + // Small engine: margin floor 8. ledger == engine+8 -> NOT over. + let st2 = p2_state("enforce"); + assert_eq!(st2.reconcile_target(node, 8, 0, true, t0), None); + assert_eq!(st2.reconcile_target(node, 3, 10, true, t0), None, "ledger < engine: never"); + // Config flag defaults on, explicit off is honored (the poll_once + // call site gates on it). + assert!(st.config.ledger_reconcile); + let cfg = G9iConfig { + enabled: true, + shadow_enabled: true, + ledger_reconcile: false, + ..Default::default() + }; + assert!(!G9iState::new(cfg).config.ledger_reconcile); + } + /// Window admission: ledger < W admits; ledger >= W (all D) -> 429 /// d_window_full in enforce; shadow mode only counts. #[test] diff --git a/sgl-model-gateway/src/routers/http/g9i_shadow.rs b/sgl-model-gateway/src/routers/http/g9i_shadow.rs index 32198ae19b..07cf3b342f 100644 --- a/sgl-model-gateway/src/routers/http/g9i_shadow.rs +++ b/sgl-model-gateway/src/routers/http/g9i_shadow.rs @@ -1282,6 +1282,37 @@ impl QueueModel { out } + /// T19 (P0) ledger reconcile: force the node's ledger DOWN to `target` + /// (subtract-only — the reconcile never invents bookings). Pipeline + /// bookings are cut first (a wedge before the first token is the common + /// shape — the transfer-queue incident), then streaming. Returns the + /// number of phantom bookings released; 0 when already at/under target. + pub fn d_clamp_to(&mut self, node: &str, target: u64) -> u64 { + let Some(e) = self.d_pools.get_mut(node) else { + return 0; + }; + let current = e.pipeline + e.streaming; + if current <= target { + return 0; + } + let mut excess = current - target; + let p_cut = e.pipeline.min(excess); + e.pipeline -= p_cut; + for _ in 0..p_cut { + e.pipeline_admits.pop_front(); + } + excess -= p_cut; + let s_cut = e.streaming.min(excess); + e.streaming -= s_cut; + for _ in 0..s_cut { + e.streaming_since.pop_front(); + } + if e.pipeline + e.streaming == 0 { + e.oldest_inflight_since = None; + } + p_cut + s_cut + } + /// Pipeline depth (admitted, awaiting first token) — the T16 admission /// window's denominator-side quantity. pub fn d_pipeline(&self, node: &str) -> u64 { @@ -3155,6 +3186,36 @@ mod tests { assert_eq!(out[0].2, 1); } + /// T19 (P0) reconcile clamp: subtract-only (never invents bookings), + /// pipeline cut before streaming (the pre-first-token wedge is the + /// common shape), deques stay in lockstep with the counters. + #[test] + fn t19_clamp_subtract_only() { + let mut q = QueueModel::new(); + for _ in 0..3 { + q.d_pipeline_admit("d1"); + } + q.d_pipeline_promote("d1"); + q.d_pipeline_promote("d1"); // pipeline 1, streaming 2 + // Already under target -> no-op (never adds). + assert_eq!(q.d_clamp_to("d1", 10), 0); + assert_eq!(q.d_inflight("d1"), 3); + // Clamp 3 -> 2: the pipeline booking goes first. + assert_eq!(q.d_clamp_to("d1", 2), 1); + assert_eq!(q.d_pipeline("d1"), 0, "pipeline cut first"); + assert_eq!(q.d_streaming("d1"), 2); + // Clamp to 0 releases the rest; unknown node is a no-op. + assert_eq!(q.d_clamp_to("d1", 0), 2); + assert_eq!(q.d_inflight("d1"), 0); + assert_eq!(q.d_clamp_to("d-nope", 0), 0); + // Lockstep invariant: a post-clamp admit/promote/release cycle + // still conserves (no deque/counter drift). + q.d_pipeline_admit("d1"); + q.d_pipeline_promote("d1"); + q.d_streaming_release("d1"); + assert_eq!(q.d_inflight("d1"), 0); + } + /// T17 修正版: the engine-real queue verdict (real_over predicate) /// excludes candidates exactly like the token cap — holder skipped in /// enforce, pick unchanged in shadow with the signal computed.