gateway g9i: ledger reconcile clamp on sustained ledger>>engine divergence (subtract-only, 60s fuse) (P2 T19 P0-2)
This commit is contained in:
@@ -328,6 +328,10 @@ struct CliArgs {
|
||||
#[arg(long, default_value_t = 600, help_heading = "G9i Pool Admission")]
|
||||
g9i_streaming_ttl_secs: u64,
|
||||
|
||||
/// Ledger reconcile: clamp the D ledger down to the engine-side in-flight when the excess is sustained >60s (subtract-only)
|
||||
#[arg(long, default_value_t = true, help_heading = "G9i Pool Admission")]
|
||||
g9i_ledger_reconcile: bool,
|
||||
|
||||
// ==================== Service Discovery (Kubernetes) ====================
|
||||
/// Enable Kubernetes service discovery
|
||||
#[arg(
|
||||
@@ -1218,6 +1222,7 @@ impl CliArgs {
|
||||
admit_hold_max: self.g9i_admit_hold_max,
|
||||
pipeline_ttl_secs: self.g9i_pipeline_ttl_secs,
|
||||
streaming_ttl_secs: self.g9i_streaming_ttl_secs,
|
||||
ledger_reconcile: self.g9i_ledger_reconcile,
|
||||
};
|
||||
config.validate().map_err(|reason| ConfigError::ValidationFailed { reason })?;
|
||||
Ok(Some(config))
|
||||
|
||||
@@ -242,6 +242,13 @@ pub struct G9iConfig {
|
||||
/// Default 600; 0 = disabled.
|
||||
#[serde(default = "default_streaming_ttl_secs")]
|
||||
pub streaming_ttl_secs: u64,
|
||||
/// T19 (P0) ledger reconcile: when the gateway-side D ledger exceeds the
|
||||
/// engine-side in-flight (running + prealloc + transfer) by more than
|
||||
/// max(8, 25%) for a sustained 60s, clamp the ledger DOWN to the engine
|
||||
/// value (subtract-only; drift self-corrects instead of wedging the
|
||||
/// admission window forever). Default on.
|
||||
#[serde(default = "default_ledger_reconcile")]
|
||||
pub ledger_reconcile: bool,
|
||||
}
|
||||
|
||||
fn default_max_new_tokens() -> u64 {
|
||||
@@ -310,6 +317,9 @@ fn default_pipeline_ttl_secs() -> u64 {
|
||||
fn default_streaming_ttl_secs() -> u64 {
|
||||
600
|
||||
}
|
||||
fn default_ledger_reconcile() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
impl Default for G9iConfig {
|
||||
fn default() -> Self {
|
||||
@@ -341,6 +351,7 @@ impl Default for G9iConfig {
|
||||
admit_hold_max: default_admit_hold_max(),
|
||||
pipeline_ttl_secs: default_pipeline_ttl_secs(),
|
||||
streaming_ttl_secs: default_streaming_ttl_secs(),
|
||||
ledger_reconcile: default_ledger_reconcile(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -523,6 +534,10 @@ pub struct G9iState {
|
||||
/// underlying cache_aware policy, which pinned same-prefix sessions
|
||||
/// onto a single node.
|
||||
rr_cursor: AtomicUsize,
|
||||
/// T19 (P0) ledger reconcile: per-decode-node start Instant of the
|
||||
/// current sustained ledger>>engine excess episode; 60s sustained ->
|
||||
/// clamp. Cleared as soon as the excess disappears.
|
||||
reconcile_since: RwLock<HashMap<String, Instant>>,
|
||||
}
|
||||
|
||||
/// T14 AIMD congestion-window state for one decode node.
|
||||
@@ -633,6 +648,11 @@ const AIMD_MIN_SAMPLES: usize = 20;
|
||||
/// T18: bootstrap /metrics scrape cadence (own slow loop, off the snapshot
|
||||
/// round).
|
||||
const BOOTSTRAP_SCRAPE_INTERVAL_SECS: u64 = 10;
|
||||
/// T19 (P0) ledger reconcile: clamp only when the gateway ledger exceeds
|
||||
/// the engine-side in-flight (running + prealloc + transfer) by more than
|
||||
/// max(8, 25%) for this long — transient poll-lag drift must never trigger
|
||||
/// a clamp (只减不加, and only after sustained divergence).
|
||||
const RECONCILE_SUSTAIN_SECS: u64 = 60;
|
||||
/// TTFT SLO lines (p50/p95/p99 seconds).
|
||||
const SLO_P50_SECS: f64 = 5.0;
|
||||
const SLO_P95_SECS: f64 = 10.0;
|
||||
@@ -738,6 +758,7 @@ impl G9iState {
|
||||
hold_timeouts_total: AtomicU64::new(0),
|
||||
last_tick_timeouts: AtomicU64::new(0),
|
||||
rr_cursor: AtomicUsize::new(0),
|
||||
reconcile_since: RwLock::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -922,6 +943,53 @@ impl G9iState {
|
||||
self.hold_wake();
|
||||
}
|
||||
}
|
||||
// T19 (P0) ledger reconcile (对账纠偏): the T18 ledger is a
|
||||
// faithful bookkeeper — when the ENGINE wedges (booking never
|
||||
// promotes/completes), the ledger is right but useless, and the
|
||||
// phantom bookings pin the AIMD window shut. If the ledger
|
||||
// exceeds the engine-side in-flight (running + prealloc +
|
||||
// transfer) by > max(8, 25%) for a sustained 60s, clamp the
|
||||
// ledger DOWN to the engine value. Subtract-only; stale engine
|
||||
// views never justify a clamp. Runs after the TTL reaper (the
|
||||
// reaper is the precise instrument; this is the drift fuse).
|
||||
if self.config.ledger_reconcile {
|
||||
let snap_ttl = Duration::from_secs(self.config.snapshot_ttl_secs);
|
||||
let mut clamps: Vec<(String, u64, u64, u64)> = Vec::new();
|
||||
if let (Some(q), Ok(d)) = (self.shadow_queue.as_ref(), self.d_snapshots.read()) {
|
||||
if let Ok(mut q) = q.write() {
|
||||
let now = Instant::now();
|
||||
for (url, load) in d.iter() {
|
||||
let engine = load.running_reqs
|
||||
+ load.decode_prealloc_queue_reqs
|
||||
+ load.decode_transfer_queue_reqs;
|
||||
let ledger = q.d_inflight(url);
|
||||
if let Some(target) = self.reconcile_target(
|
||||
url,
|
||||
ledger,
|
||||
engine,
|
||||
load.is_fresh(snap_ttl),
|
||||
now,
|
||||
) {
|
||||
let released = q.d_clamp_to(url, target);
|
||||
if released > 0 {
|
||||
clamps.push((url.clone(), released, ledger, engine));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (node, released, ledger, engine) in &clamps {
|
||||
metrics::counter!("smg_g9i_ledger_reconcile_total", "worker" => node.clone())
|
||||
.increment(*released);
|
||||
warn!(
|
||||
"G9i ledger reconcile: {} ledger={} vs engine={} sustained >{}s — clamped, {} phantom booking(s) released",
|
||||
node, ledger, engine, RECONCILE_SUSTAIN_SECS, released,
|
||||
);
|
||||
}
|
||||
if !clamps.is_empty() {
|
||||
self.hold_wake();
|
||||
}
|
||||
}
|
||||
// T14: create/refresh AIMD window entries as soon as a node's
|
||||
// capacity is known (the window gate is live from the first
|
||||
// poll — no startup blind spot like the T13 snapshot race).
|
||||
@@ -2060,6 +2128,39 @@ impl G9iState {
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// T19 (P0) ledger reconcile verdict: Some(engine) when the gateway
|
||||
/// ledger has exceeded the engine-side in-flight by more than
|
||||
/// max(8, 25%) for RECONCILE_SUSTAIN_SECS — the caller then clamps the
|
||||
/// ledger down to the engine value. Subtract-only, fresh snapshots only
|
||||
/// (a stale engine view never justifies a clamp, and a stale stretch
|
||||
/// does not count toward the sustain window — the marker is cleared so
|
||||
/// the 60s must be CONTINUOUS fresh excess). The excess disappearing
|
||||
/// also clears the marker immediately.
|
||||
fn reconcile_target(
|
||||
&self,
|
||||
node: &str,
|
||||
ledger: u64,
|
||||
engine: u64,
|
||||
fresh: bool,
|
||||
now: Instant,
|
||||
) -> Option<u64> {
|
||||
let margin = (engine / 4).max(8);
|
||||
let over = fresh && ledger > engine.saturating_add(margin);
|
||||
let Ok(mut since) = self.reconcile_since.write() else {
|
||||
return None;
|
||||
};
|
||||
if !over {
|
||||
since.remove(node);
|
||||
return None;
|
||||
}
|
||||
let start = *since.entry(node.to_string()).or_insert(now);
|
||||
if now.saturating_duration_since(start) >= Duration::from_secs(RECONCILE_SUSTAIN_SECS) {
|
||||
Some(engine)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// T11: shadow classify + T_PREFER/cap candidate pick + admission in a
|
||||
/// SINGLE queue-model critical section (atomic check-and-act). The
|
||||
/// pre-T11 flow admitted only when response headers arrived — after the
|
||||
@@ -3456,6 +3557,93 @@ sglang:token_usage{dp_rank=\"0\"} 0.5
|
||||
);
|
||||
}
|
||||
|
||||
/// T19 (P0) ledger reconcile: the clamp verdict requires a SUSTAINED
|
||||
/// (60s) fresh excess of ledger over engine by > max(8, 25%); transient
|
||||
/// drift never clamps. The clamp itself is subtract-only (QueueModel
|
||||
/// level covered by t19_clamp_subtract_only).
|
||||
#[test]
|
||||
fn t19_reconcile_requires_60s_sustain() {
|
||||
let st = p2_state("enforce");
|
||||
let node = "http://d1:30001";
|
||||
set_d_snapshots(&st, &[(node, 0, 10_000_000, true, false)]);
|
||||
book_ledger(&st, node, 20); // engine = 0, margin = max(8, 0) = 8 -> over
|
||||
let t0 = Instant::now();
|
||||
assert_eq!(
|
||||
st.reconcile_target(node, 20, 0, true, t0),
|
||||
None,
|
||||
"first sight starts the clock, no clamp"
|
||||
);
|
||||
assert_eq!(
|
||||
st.reconcile_target(node, 20, 0, true, t0 + Duration::from_secs(30)),
|
||||
None,
|
||||
"30s sustained — still under the 60s fuse"
|
||||
);
|
||||
assert_eq!(
|
||||
st.reconcile_target(node, 20, 0, true, t0 + Duration::from_secs(61)),
|
||||
Some(0),
|
||||
"61s sustained -> clamp to the engine value"
|
||||
);
|
||||
// Applying the verdict releases exactly the phantom bookings.
|
||||
let q = st.shadow_queue.as_ref().unwrap();
|
||||
assert_eq!(q.write().unwrap().d_clamp_to(node, 0), 20);
|
||||
assert_eq!(q.read().unwrap().d_inflight(node), 0);
|
||||
// Excess disappears -> marker cleared; a later excess restarts at 0.
|
||||
assert_eq!(
|
||||
st.reconcile_target(node, 5, 0, true, t0 + Duration::from_secs(62)),
|
||||
None
|
||||
);
|
||||
assert!(
|
||||
st.reconcile_since.read().unwrap().is_empty(),
|
||||
"marker cleared when the excess disappears"
|
||||
);
|
||||
assert_eq!(
|
||||
st.reconcile_target(node, 20, 0, true, t0 + Duration::from_secs(200)),
|
||||
None,
|
||||
"a reappearing excess restarts the clock from zero"
|
||||
);
|
||||
}
|
||||
|
||||
/// T19: margin and freshness rules — ledger <= engine + max(8, 25%) is
|
||||
/// never clamped; stale snapshots never clamp and clear the marker (a
|
||||
/// stale stretch does not count toward the 60s); ledger < engine never.
|
||||
#[test]
|
||||
fn t19_reconcile_margin_and_freshness() {
|
||||
let st = p2_state("enforce");
|
||||
let node = "http://d1:30001";
|
||||
let t0 = Instant::now();
|
||||
// engine 100 -> margin max(8, 25) = 25: ledger 125 NOT over, 126 over.
|
||||
assert_eq!(st.reconcile_target(node, 125, 100, true, t0), None);
|
||||
assert_eq!(
|
||||
st.reconcile_target(node, 126, 100, true, t0),
|
||||
None,
|
||||
"over the margin, but the clock just started"
|
||||
);
|
||||
// Stale snapshot: never clamps and clears the marker.
|
||||
assert_eq!(
|
||||
st.reconcile_target(node, 1000, 100, false, t0 + Duration::from_secs(120)),
|
||||
None,
|
||||
"stale engine view never justifies a clamp"
|
||||
);
|
||||
assert!(
|
||||
!st.reconcile_since.read().unwrap().contains_key(node),
|
||||
"the stale stretch cleared the marker"
|
||||
);
|
||||
// Small engine: margin floor 8. ledger == engine+8 -> NOT over.
|
||||
let st2 = p2_state("enforce");
|
||||
assert_eq!(st2.reconcile_target(node, 8, 0, true, t0), None);
|
||||
assert_eq!(st2.reconcile_target(node, 3, 10, true, t0), None, "ledger < engine: never");
|
||||
// Config flag defaults on, explicit off is honored (the poll_once
|
||||
// call site gates on it).
|
||||
assert!(st.config.ledger_reconcile);
|
||||
let cfg = G9iConfig {
|
||||
enabled: true,
|
||||
shadow_enabled: true,
|
||||
ledger_reconcile: false,
|
||||
..Default::default()
|
||||
};
|
||||
assert!(!G9iState::new(cfg).config.ledger_reconcile);
|
||||
}
|
||||
|
||||
/// Window admission: ledger < W admits; ledger >= W (all D) -> 429
|
||||
/// d_window_full in enforce; shadow mode only counts.
|
||||
#[test]
|
||||
|
||||
@@ -1282,6 +1282,37 @@ impl QueueModel {
|
||||
out
|
||||
}
|
||||
|
||||
/// T19 (P0) ledger reconcile: force the node's ledger DOWN to `target`
|
||||
/// (subtract-only — the reconcile never invents bookings). Pipeline
|
||||
/// bookings are cut first (a wedge before the first token is the common
|
||||
/// shape — the transfer-queue incident), then streaming. Returns the
|
||||
/// number of phantom bookings released; 0 when already at/under target.
|
||||
pub fn d_clamp_to(&mut self, node: &str, target: u64) -> u64 {
|
||||
let Some(e) = self.d_pools.get_mut(node) else {
|
||||
return 0;
|
||||
};
|
||||
let current = e.pipeline + e.streaming;
|
||||
if current <= target {
|
||||
return 0;
|
||||
}
|
||||
let mut excess = current - target;
|
||||
let p_cut = e.pipeline.min(excess);
|
||||
e.pipeline -= p_cut;
|
||||
for _ in 0..p_cut {
|
||||
e.pipeline_admits.pop_front();
|
||||
}
|
||||
excess -= p_cut;
|
||||
let s_cut = e.streaming.min(excess);
|
||||
e.streaming -= s_cut;
|
||||
for _ in 0..s_cut {
|
||||
e.streaming_since.pop_front();
|
||||
}
|
||||
if e.pipeline + e.streaming == 0 {
|
||||
e.oldest_inflight_since = None;
|
||||
}
|
||||
p_cut + s_cut
|
||||
}
|
||||
|
||||
/// Pipeline depth (admitted, awaiting first token) — the T16 admission
|
||||
/// window's denominator-side quantity.
|
||||
pub fn d_pipeline(&self, node: &str) -> u64 {
|
||||
@@ -3155,6 +3186,36 @@ mod tests {
|
||||
assert_eq!(out[0].2, 1);
|
||||
}
|
||||
|
||||
/// T19 (P0) reconcile clamp: subtract-only (never invents bookings),
|
||||
/// pipeline cut before streaming (the pre-first-token wedge is the
|
||||
/// common shape), deques stay in lockstep with the counters.
|
||||
#[test]
|
||||
fn t19_clamp_subtract_only() {
|
||||
let mut q = QueueModel::new();
|
||||
for _ in 0..3 {
|
||||
q.d_pipeline_admit("d1");
|
||||
}
|
||||
q.d_pipeline_promote("d1");
|
||||
q.d_pipeline_promote("d1"); // pipeline 1, streaming 2
|
||||
// Already under target -> no-op (never adds).
|
||||
assert_eq!(q.d_clamp_to("d1", 10), 0);
|
||||
assert_eq!(q.d_inflight("d1"), 3);
|
||||
// Clamp 3 -> 2: the pipeline booking goes first.
|
||||
assert_eq!(q.d_clamp_to("d1", 2), 1);
|
||||
assert_eq!(q.d_pipeline("d1"), 0, "pipeline cut first");
|
||||
assert_eq!(q.d_streaming("d1"), 2);
|
||||
// Clamp to 0 releases the rest; unknown node is a no-op.
|
||||
assert_eq!(q.d_clamp_to("d1", 0), 2);
|
||||
assert_eq!(q.d_inflight("d1"), 0);
|
||||
assert_eq!(q.d_clamp_to("d-nope", 0), 0);
|
||||
// Lockstep invariant: a post-clamp admit/promote/release cycle
|
||||
// still conserves (no deque/counter drift).
|
||||
q.d_pipeline_admit("d1");
|
||||
q.d_pipeline_promote("d1");
|
||||
q.d_streaming_release("d1");
|
||||
assert_eq!(q.d_inflight("d1"), 0);
|
||||
}
|
||||
|
||||
/// T17 修正版: the engine-real queue verdict (real_over predicate)
|
||||
/// excludes candidates exactly like the token cap — holder skipped in
|
||||
/// enforce, pick unchanged in shadow with the signal computed.
|
||||
|
||||
Reference in New Issue
Block a user