gateway g9i: ledger reconcile clamp on sustained ledger>>engine divergence (subtract-only, 60s fuse) (P2 T19 P0-2)

This commit is contained in:
2026-10-16 15:56:56 +08:00
parent 6217b9de46
commit 1910eccb8e
3 changed files with 254 additions and 0 deletions
+5
View File
@@ -328,6 +328,10 @@ struct CliArgs {
#[arg(long, default_value_t = 600, help_heading = "G9i Pool Admission")]
g9i_streaming_ttl_secs: u64,
/// Ledger reconcile: clamp the D ledger down to the engine-side in-flight when the excess is sustained >60s (subtract-only)
#[arg(long, default_value_t = true, help_heading = "G9i Pool Admission")]
g9i_ledger_reconcile: bool,
// ==================== Service Discovery (Kubernetes) ====================
/// Enable Kubernetes service discovery
#[arg(
@@ -1218,6 +1222,7 @@ impl CliArgs {
admit_hold_max: self.g9i_admit_hold_max,
pipeline_ttl_secs: self.g9i_pipeline_ttl_secs,
streaming_ttl_secs: self.g9i_streaming_ttl_secs,
ledger_reconcile: self.g9i_ledger_reconcile,
};
config.validate().map_err(|reason| ConfigError::ValidationFailed { reason })?;
Ok(Some(config))
+188
View File
@@ -242,6 +242,13 @@ pub struct G9iConfig {
/// Default 600; 0 = disabled.
#[serde(default = "default_streaming_ttl_secs")]
pub streaming_ttl_secs: u64,
/// T19 (P0) ledger reconcile: when the gateway-side D ledger exceeds the
/// engine-side in-flight (running + prealloc + transfer) by more than
/// max(8, 25%) for a sustained 60s, clamp the ledger DOWN to the engine
/// value (subtract-only; drift self-corrects instead of wedging the
/// admission window forever). Default on.
#[serde(default = "default_ledger_reconcile")]
pub ledger_reconcile: bool,
}
fn default_max_new_tokens() -> u64 {
@@ -310,6 +317,9 @@ fn default_pipeline_ttl_secs() -> u64 {
fn default_streaming_ttl_secs() -> u64 {
600
}
fn default_ledger_reconcile() -> bool {
true
}
impl Default for G9iConfig {
fn default() -> Self {
@@ -341,6 +351,7 @@ impl Default for G9iConfig {
admit_hold_max: default_admit_hold_max(),
pipeline_ttl_secs: default_pipeline_ttl_secs(),
streaming_ttl_secs: default_streaming_ttl_secs(),
ledger_reconcile: default_ledger_reconcile(),
}
}
}
@@ -523,6 +534,10 @@ pub struct G9iState {
/// underlying cache_aware policy, which pinned same-prefix sessions
/// onto a single node.
rr_cursor: AtomicUsize,
/// T19 (P0) ledger reconcile: per-decode-node start Instant of the
/// current sustained ledger>>engine excess episode; 60s sustained ->
/// clamp. Cleared as soon as the excess disappears.
reconcile_since: RwLock<HashMap<String, Instant>>,
}
/// T14 AIMD congestion-window state for one decode node.
@@ -633,6 +648,11 @@ const AIMD_MIN_SAMPLES: usize = 20;
/// T18: bootstrap /metrics scrape cadence (own slow loop, off the snapshot
/// round).
const BOOTSTRAP_SCRAPE_INTERVAL_SECS: u64 = 10;
/// T19 (P0) ledger reconcile: clamp only when the gateway ledger exceeds
/// the engine-side in-flight (running + prealloc + transfer) by more than
/// max(8, 25%) for this long — transient poll-lag drift must never trigger
/// a clamp (只减不加, and only after sustained divergence).
const RECONCILE_SUSTAIN_SECS: u64 = 60;
/// TTFT SLO lines (p50/p95/p99 seconds).
const SLO_P50_SECS: f64 = 5.0;
const SLO_P95_SECS: f64 = 10.0;
@@ -738,6 +758,7 @@ impl G9iState {
hold_timeouts_total: AtomicU64::new(0),
last_tick_timeouts: AtomicU64::new(0),
rr_cursor: AtomicUsize::new(0),
reconcile_since: RwLock::new(HashMap::new()),
}
}
@@ -922,6 +943,53 @@ impl G9iState {
self.hold_wake();
}
}
// T19 (P0) ledger reconcile (对账纠偏): the T18 ledger is a
// faithful bookkeeper — when the ENGINE wedges (booking never
// promotes/completes), the ledger is right but useless, and the
// phantom bookings pin the AIMD window shut. If the ledger
// exceeds the engine-side in-flight (running + prealloc +
// transfer) by > max(8, 25%) for a sustained 60s, clamp the
// ledger DOWN to the engine value. Subtract-only; stale engine
// views never justify a clamp. Runs after the TTL reaper (the
// reaper is the precise instrument; this is the drift fuse).
if self.config.ledger_reconcile {
let snap_ttl = Duration::from_secs(self.config.snapshot_ttl_secs);
let mut clamps: Vec<(String, u64, u64, u64)> = Vec::new();
if let (Some(q), Ok(d)) = (self.shadow_queue.as_ref(), self.d_snapshots.read()) {
if let Ok(mut q) = q.write() {
let now = Instant::now();
for (url, load) in d.iter() {
let engine = load.running_reqs
+ load.decode_prealloc_queue_reqs
+ load.decode_transfer_queue_reqs;
let ledger = q.d_inflight(url);
if let Some(target) = self.reconcile_target(
url,
ledger,
engine,
load.is_fresh(snap_ttl),
now,
) {
let released = q.d_clamp_to(url, target);
if released > 0 {
clamps.push((url.clone(), released, ledger, engine));
}
}
}
}
}
for (node, released, ledger, engine) in &clamps {
metrics::counter!("smg_g9i_ledger_reconcile_total", "worker" => node.clone())
.increment(*released);
warn!(
"G9i ledger reconcile: {} ledger={} vs engine={} sustained >{}s — clamped, {} phantom booking(s) released",
node, ledger, engine, RECONCILE_SUSTAIN_SECS, released,
);
}
if !clamps.is_empty() {
self.hold_wake();
}
}
// T14: create/refresh AIMD window entries as soon as a node's
// capacity is known (the window gate is live from the first
// poll — no startup blind spot like the T13 snapshot race).
@@ -2060,6 +2128,39 @@ impl G9iState {
.unwrap_or_default()
}
/// T19 (P0) ledger reconcile verdict: Some(engine) when the gateway
/// ledger has exceeded the engine-side in-flight by more than
/// max(8, 25%) for RECONCILE_SUSTAIN_SECS — the caller then clamps the
/// ledger down to the engine value. Subtract-only, fresh snapshots only
/// (a stale engine view never justifies a clamp, and a stale stretch
/// does not count toward the sustain window — the marker is cleared so
/// the 60s must be CONTINUOUS fresh excess). The excess disappearing
/// also clears the marker immediately.
fn reconcile_target(
&self,
node: &str,
ledger: u64,
engine: u64,
fresh: bool,
now: Instant,
) -> Option<u64> {
let margin = (engine / 4).max(8);
let over = fresh && ledger > engine.saturating_add(margin);
let Ok(mut since) = self.reconcile_since.write() else {
return None;
};
if !over {
since.remove(node);
return None;
}
let start = *since.entry(node.to_string()).or_insert(now);
if now.saturating_duration_since(start) >= Duration::from_secs(RECONCILE_SUSTAIN_SECS) {
Some(engine)
} else {
None
}
}
/// T11: shadow classify + T_PREFER/cap candidate pick + admission in a
/// SINGLE queue-model critical section (atomic check-and-act). The
/// pre-T11 flow admitted only when response headers arrived — after the
@@ -3456,6 +3557,93 @@ sglang:token_usage{dp_rank=\"0\"} 0.5
);
}
/// T19 (P0) ledger reconcile: the clamp verdict requires a SUSTAINED
/// (60s) fresh excess of ledger over engine by > max(8, 25%); transient
/// drift never clamps. The clamp itself is subtract-only (QueueModel
/// level covered by t19_clamp_subtract_only).
#[test]
fn t19_reconcile_requires_60s_sustain() {
let st = p2_state("enforce");
let node = "http://d1:30001";
set_d_snapshots(&st, &[(node, 0, 10_000_000, true, false)]);
book_ledger(&st, node, 20); // engine = 0, margin = max(8, 0) = 8 -> over
let t0 = Instant::now();
assert_eq!(
st.reconcile_target(node, 20, 0, true, t0),
None,
"first sight starts the clock, no clamp"
);
assert_eq!(
st.reconcile_target(node, 20, 0, true, t0 + Duration::from_secs(30)),
None,
"30s sustained — still under the 60s fuse"
);
assert_eq!(
st.reconcile_target(node, 20, 0, true, t0 + Duration::from_secs(61)),
Some(0),
"61s sustained -> clamp to the engine value"
);
// Applying the verdict releases exactly the phantom bookings.
let q = st.shadow_queue.as_ref().unwrap();
assert_eq!(q.write().unwrap().d_clamp_to(node, 0), 20);
assert_eq!(q.read().unwrap().d_inflight(node), 0);
// Excess disappears -> marker cleared; a later excess restarts at 0.
assert_eq!(
st.reconcile_target(node, 5, 0, true, t0 + Duration::from_secs(62)),
None
);
assert!(
st.reconcile_since.read().unwrap().is_empty(),
"marker cleared when the excess disappears"
);
assert_eq!(
st.reconcile_target(node, 20, 0, true, t0 + Duration::from_secs(200)),
None,
"a reappearing excess restarts the clock from zero"
);
}
/// T19: margin and freshness rules — ledger <= engine + max(8, 25%) is
/// never clamped; stale snapshots never clamp and clear the marker (a
/// stale stretch does not count toward the 60s); ledger < engine never.
#[test]
fn t19_reconcile_margin_and_freshness() {
let st = p2_state("enforce");
let node = "http://d1:30001";
let t0 = Instant::now();
// engine 100 -> margin max(8, 25) = 25: ledger 125 NOT over, 126 over.
assert_eq!(st.reconcile_target(node, 125, 100, true, t0), None);
assert_eq!(
st.reconcile_target(node, 126, 100, true, t0),
None,
"over the margin, but the clock just started"
);
// Stale snapshot: never clamps and clears the marker.
assert_eq!(
st.reconcile_target(node, 1000, 100, false, t0 + Duration::from_secs(120)),
None,
"stale engine view never justifies a clamp"
);
assert!(
!st.reconcile_since.read().unwrap().contains_key(node),
"the stale stretch cleared the marker"
);
// Small engine: margin floor 8. ledger == engine+8 -> NOT over.
let st2 = p2_state("enforce");
assert_eq!(st2.reconcile_target(node, 8, 0, true, t0), None);
assert_eq!(st2.reconcile_target(node, 3, 10, true, t0), None, "ledger < engine: never");
// Config flag defaults on, explicit off is honored (the poll_once
// call site gates on it).
assert!(st.config.ledger_reconcile);
let cfg = G9iConfig {
enabled: true,
shadow_enabled: true,
ledger_reconcile: false,
..Default::default()
};
assert!(!G9iState::new(cfg).config.ledger_reconcile);
}
/// Window admission: ledger < W admits; ledger >= W (all D) -> 429
/// d_window_full in enforce; shadow mode only counts.
#[test]
@@ -1282,6 +1282,37 @@ impl QueueModel {
out
}
/// T19 (P0) ledger reconcile: force the node's ledger DOWN to `target`
/// (subtract-only — the reconcile never invents bookings). Pipeline
/// bookings are cut first (a wedge before the first token is the common
/// shape — the transfer-queue incident), then streaming. Returns the
/// number of phantom bookings released; 0 when already at/under target.
pub fn d_clamp_to(&mut self, node: &str, target: u64) -> u64 {
let Some(e) = self.d_pools.get_mut(node) else {
return 0;
};
let current = e.pipeline + e.streaming;
if current <= target {
return 0;
}
let mut excess = current - target;
let p_cut = e.pipeline.min(excess);
e.pipeline -= p_cut;
for _ in 0..p_cut {
e.pipeline_admits.pop_front();
}
excess -= p_cut;
let s_cut = e.streaming.min(excess);
e.streaming -= s_cut;
for _ in 0..s_cut {
e.streaming_since.pop_front();
}
if e.pipeline + e.streaming == 0 {
e.oldest_inflight_since = None;
}
p_cut + s_cut
}
/// Pipeline depth (admitted, awaiting first token) — the T16 admission
/// window's denominator-side quantity.
pub fn d_pipeline(&self, node: &str) -> u64 {
@@ -3155,6 +3186,36 @@ mod tests {
assert_eq!(out[0].2, 1);
}
/// T19 (P0) reconcile clamp: subtract-only (never invents bookings),
/// pipeline cut before streaming (the pre-first-token wedge is the
/// common shape), deques stay in lockstep with the counters.
#[test]
fn t19_clamp_subtract_only() {
let mut q = QueueModel::new();
for _ in 0..3 {
q.d_pipeline_admit("d1");
}
q.d_pipeline_promote("d1");
q.d_pipeline_promote("d1"); // pipeline 1, streaming 2
// Already under target -> no-op (never adds).
assert_eq!(q.d_clamp_to("d1", 10), 0);
assert_eq!(q.d_inflight("d1"), 3);
// Clamp 3 -> 2: the pipeline booking goes first.
assert_eq!(q.d_clamp_to("d1", 2), 1);
assert_eq!(q.d_pipeline("d1"), 0, "pipeline cut first");
assert_eq!(q.d_streaming("d1"), 2);
// Clamp to 0 releases the rest; unknown node is a no-op.
assert_eq!(q.d_clamp_to("d1", 0), 2);
assert_eq!(q.d_inflight("d1"), 0);
assert_eq!(q.d_clamp_to("d-nope", 0), 0);
// Lockstep invariant: a post-clamp admit/promote/release cycle
// still conserves (no deque/counter drift).
q.d_pipeline_admit("d1");
q.d_pipeline_promote("d1");
q.d_streaming_release("d1");
assert_eq!(q.d_inflight("d1"), 0);
}
/// T17 修正版: the engine-real queue verdict (real_over predicate)
/// excludes candidates exactly like the token cap — holder skipped in
/// enforce, pick unchanged in shadow with the signal computed.