diff --git a/sgl-model-gateway/src/routers/http/g9i.rs b/sgl-model-gateway/src/routers/http/g9i.rs index 5ad2c13ed1..3734aee034 100644 --- a/sgl-model-gateway/src/routers/http/g9i.rs +++ b/sgl-model-gateway/src/routers/http/g9i.rs @@ -334,6 +334,9 @@ pub struct NodeLoad { /// summed (T12: the slot dimension of T_D — c512 实测 d1 槽位 128/128 /// 打满而内存仅 36%,内存口径完全看不见). pub free_slots: u64, + /// Slot capacity: sum of max_running_requests across dp ranks (T13 + /// ledger口径 needs capacity, not just headroom). + pub max_running_reqs: u64, /// True when any rank answered with a disaggregation section. False on /// old engines (no disagg include support / non-PD deployment) — the /// est_T_D path then degrades to the P-only model exactly (C13). @@ -604,6 +607,14 @@ impl G9iState { "worker" => url.clone(), ) .set(q.slot_release_rate(url)); + // T13: the gateway-side in-flight ledger (vs the + // snapshot's running+prealloc — divergence shows + // poll-lag / bookkeeping drift). + metrics::gauge!( + "smg_g9i_d_inflight_ledger", + "worker" => url.clone(), + ) + .set(q.d_inflight(url) as f64); } } } @@ -846,7 +857,7 @@ impl G9iState { let lock = self.shadow_queue.as_ref()?; let mut q = lock.write().ok()?; q.est_td_min(&d_snaps, own_ctx_tokens, self.config.est_td_enabled) - .map(|(t_d, _slot)| t_d) + .map(|(t_d, _slot, _node)| t_d) } /// P2 T_D hard gate (design §2.4-2, user-ruled): when min(T_D) over ALL @@ -1071,20 +1082,36 @@ impl G9iState { let actual = super::g9i_shadow::extract_prompt_tokens(resp_json) .unwrap_or(ctx.est_new_tokens); q.complete(&ctx.node, class, ctx.est_new_tokens, actual); + // T13: pair the D-side ledger booking (admit→complete). + // Booked iff the decision-time admission carried a + // d_node, or the deferred admit booked at response time. + let d_booked = match &ctx.admission { + Some(adm) => adm.d_node.is_some(), + None => true, + }; + if d_booked { + q.d_ledger_release(&ctx.decode_node); + } } // est_T_D P1 (design 2026-10-14 §2.2): a successful response // frees its ctx tokens of KV pool on the decode node — fold // the freed amount into that node's pool_free_rate EWMA. // Only responses with real usage count: the release path // (errors/aborts) freed nothing we can measure. + // T13: dwell = admit→complete wall time (Little's-law + // residency sample behind the slot release rate). if let Some(ctx_tokens) = super::g9i_shadow::extract_prompt_tokens(resp_json) { - q.record_decode_completion(&ctx.decode_node, ctx_tokens); + let dwell = ctx + .admission + .as_ref() + .map(|a| a.created.elapsed().as_secs_f64()); + q.record_decode_completion(&ctx.decode_node, ctx_tokens, dwell); metrics::gauge!( super::g9i_shadow::METRIC_POOL_FREE_RATE, "worker" => ctx.decode_node.clone(), ) .set(q.pool_free_rate(&ctx.decode_node)); - // T12: the same completion is one slot release. + // T13: Little's-law slot release rate beside it. metrics::gauge!( super::g9i_shadow::METRIC_SLOT_RELEASE_RATE, "worker" => ctx.decode_node.clone(), @@ -1111,11 +1138,19 @@ impl G9iState { /// Undo a queue admission whose response will never carry usage /// (non-200, aborted stream, unparseable body). Keeps outstanding /// conservative without polluting the drain EWMA with failure samples. + /// T13: pairs the D-side ledger booking too (same bookkeeping scope). pub fn shadow_release(&self, ctx: &ShadowResponseContext) { if let Some(class) = ctx.class { if let Some(lock) = &self.shadow_queue { if let Ok(mut q) = lock.write() { q.release(&ctx.node, class, ctx.est_new_tokens); + let d_booked = match &ctx.admission { + Some(adm) => adm.d_node.is_some(), + None => true, // deferred admit booked the ledger + }; + if d_booked { + q.d_ledger_release(&ctx.decode_node); + } } } } @@ -1147,9 +1182,17 @@ impl G9iState { return; } q.rekey(&adm.node, &ctx.node, class, adm.est_tokens); + // T13: same for the D-side ledger (decision-time pick vs + // actual decode landing). + if let Some(dn) = &adm.d_node { + q.d_ledger_rekey(dn, &ctx.decode_node); + } } // Deferred admission (decision had no node to book). - None => q.admit(&ctx.node, class, ctx.est_new_tokens), + None => { + q.admit(&ctx.node, class, ctx.est_new_tokens); + q.d_ledger_admit(&ctx.decode_node); + } } } @@ -1171,6 +1214,11 @@ impl G9iState { if let Some(lock) = &self.shadow_queue { if let Ok(mut q) = lock.write() { q.release(&adm.node, adm.class, adm.est_tokens); + // T13: pair the D-side ledger booking (never dispatched — + // the booking was intent-only, released here). + if let Some(dn) = &adm.d_node { + q.d_ledger_release(dn); + } } } } @@ -1189,7 +1237,8 @@ impl G9iState { awaiting_tokens: load.queued_total_input_tokens, free_tokens: load.free_tokens, prealloc_queue_reqs: load.decode_prealloc_queue_reqs, - free_slots: load.free_slots, + running_reqs: load.running_reqs, + max_running_total: load.max_running_reqs, usable: load.has_disagg && load.is_fresh(ttl), }) .collect() @@ -1275,10 +1324,21 @@ impl G9iState { out.est.total, )?; q.admit(&out.node, decision.class, decision.est_new_tokens); + // T13: book the D-side in-flight ledger on the + // min-T_D node — same write lock, so the slot + // gate's check→act is atomic too (快照竞态消灭). + // The booking reads the ledger BEFORE this request. + let d_node = q + .est_td_min(&d_snaps, own_ctx, self.config.est_td_enabled) + .map(|(_, _, n)| n); + if let Some(dn) = &d_node { + q.d_ledger_admit(dn); + } ctx.admission = Some(Arc::new(super::g9i_shadow::AdmissionMark::new( out.node, decision.class, decision.est_new_tokens, + d_node, ))); out.est } @@ -1822,6 +1882,7 @@ fn parse_node_load(json: &Value) -> Result { .get("max_running_requests") .and_then(|v| v.as_u64()) .unwrap_or(0); + node.max_running_reqs += max_running; node.free_slots += max_running.saturating_sub(running); node.queued_total_input_tokens += rank .get("queued_total_input_tokens") @@ -2022,6 +2083,7 @@ mod tests { assert_eq!(node.free_tokens, 2_000_000); // (128-3) + (128-5) = 248 free slots (per-rank, then summed). assert_eq!(node.free_slots, 248); + assert_eq!(node.max_running_reqs, 256); assert!(node.has_disagg); } @@ -2199,8 +2261,8 @@ sglang:token_usage{dp_rank=\"0\"} 0.5 } /// Install decode snapshots: (url, awaiting_tokens, free_tokens, - /// has_disagg, stale). free_slots pinned at 128 with an empty prealloc - /// queue so the T12 slot dimension stays inert in the pre-T12 tests. + /// has_disagg, stale). Slot view pinned at empty+128 capacity so the + /// slot dimension stays inert in the pre-T13 tests. fn set_d_snapshots(st: &G9iState, entries: &[(&str, u64, u64, bool, bool)]) { let mapped: Vec<(String, NodeLoad)> = entries .iter() @@ -2210,7 +2272,8 @@ sglang:token_usage{dp_rank=\"0\"} 0.5 NodeLoad { queued_total_input_tokens: *awaiting, free_tokens: *free, - free_slots: 128, + running_reqs: 0, + max_running_reqs: 128, has_disagg: *has_disagg, fetch_ok: true, fetched_at: Some(if *stale { @@ -2320,18 +2383,21 @@ sglang:token_usage{dp_rank=\"0\"} 0.5 assert!((err.estimated_wait_secs.unwrap() - 1000.0).abs() < 1e-6); } - /// T12: the slot dimension drives the admission gate even with memory - /// wide open — the c512 blind spot (slots 128/128, memory 36%), covered. + /// T12/T13: the slot dimension drives the admission gate even with + /// memory wide open — the c512 blind spot (slots 128/128 + 224 queued, + /// memory 36%), covered. #[test] fn td_gate_slot_bound_rejects_in_enforce() { let st = p2_state("enforce"); - // Memory idle (10M free tokens), slots full: 224 queued, 0 free. - // Cold-start slot rate floor 0.01 req/s -> (224+1)/0.01 = 22500s. + // Memory idle (10M free tokens), slots full: running 128/128, + // 224 queued. Cold-start slot rate floor 0.01 req/s -> + // (352+1-128)/0.01 = 22500s. let load = NodeLoad { queued_total_input_tokens: 0, free_tokens: 10_000_000, decode_prealloc_queue_reqs: 224, - free_slots: 0, + running_reqs: 128, + max_running_reqs: 128, has_disagg: true, fetch_ok: true, fetched_at: Some(Instant::now()), @@ -2678,4 +2744,66 @@ sglang:token_usage{dp_rank=\"0\"} 0.5 assert!(!d.all_over_cap, "cap inert when td_admit=off (C13)"); assert!(ctx.admission.is_some(), "bookkeeping still happens"); } + + // ============ T13: D-side in-flight ledger wiring ============ + + fn d_inflight_of(st: &G9iState, node: &str) -> u64 { + st.shadow_queue + .as_ref() + .unwrap() + .read() + .unwrap() + .d_inflight(node) + } + + /// The D ledger is booked at DECISION time (same critical section as + /// the gates — the snapshot race is closed), released by the unconsumed + /// fallback, and rekeyed to the actual decode node on consume. + #[test] + fn t13_d_ledger_booked_at_decision_and_paired() { + let st = t11_state("enforce", 4_000_000); + *st.p_nodes.write().unwrap() = vec!["http://p1:30000".to_string()]; + set_d_snapshots(&st, &[("http://d1:30001", 0, 10_000_000, true, false)]); + let seed = chat_ctx_turn1(); + seed_table(&st, &seed.chain, "http://p1:30000", 1000); + let mut ctx = chat_ctx(); + st.shadow_classify_and_admit(&mut ctx).unwrap().unwrap(); + assert_eq!( + d_inflight_of(&st, "http://d1:30001"), + 1, + "ledger booked at decision time, before dispatch" + ); + // Never dispatched -> fallback release pairs it (P and D both). + st.shadow_release_unconsumed(ctx.admission.clone()); + assert_eq!(d_inflight_of(&st, "http://d1:30001"), 0); + assert_eq!(outstanding_of(&st, "http://p1:30000"), 0); + // Exactly-once. + st.shadow_release_unconsumed(ctx.admission.clone()); + assert_eq!(d_inflight_of(&st, "http://d1:30001"), 0); + } + + /// Consume rekeys the D booking when the actual decode node differs. + #[test] + fn t13_d_ledger_rekey_on_consume() { + let st = t11_state("enforce", 4_000_000); + *st.p_nodes.write().unwrap() = vec!["http://p1:30000".to_string()]; + set_d_snapshots(&st, &[("http://d1:30001", 0, 10_000_000, true, false)]); + let seed = chat_ctx_turn1(); + seed_table(&st, &seed.chain, "http://p1:30000", 1000); + let mut ctx = chat_ctx(); + st.shadow_classify_and_admit(&mut ctx).unwrap().unwrap(); + assert_eq!(d_inflight_of(&st, "http://d1:30001"), 1); + // Actual decode landing differs (multi-D future shape). + let resp_ctx = ShadowResponseContext::from_request( + ctx, + "http://p1:30000".to_string(), + "http://d2:30002".to_string(), + ); + st.shadow_admit(&resp_ctx); + assert_eq!(d_inflight_of(&st, "http://d1:30001"), 0, "rekeyed off d1"); + assert_eq!(d_inflight_of(&st, "http://d2:30002"), 1, "rekeyed onto d2"); + // Abort-path release pairs the rekeyed booking. + st.shadow_release(&resp_ctx); + assert_eq!(d_inflight_of(&st, "http://d2:30002"), 0); + } } diff --git a/sgl-model-gateway/src/routers/http/g9i_shadow.rs b/sgl-model-gateway/src/routers/http/g9i_shadow.rs index fc42b5a9c4..e17015e4c8 100644 --- a/sgl-model-gateway/src/routers/http/g9i_shadow.rs +++ b/sgl-model-gateway/src/routers/http/g9i_shadow.rs @@ -508,11 +508,15 @@ pub const POOL_FREE_RATE_MIN: f64 = 1_000.0; pub const POOL_FREE_RATE_MAX: f64 = 2_000_000.0; /// Fixed EWMA weight per completion event (design §2.2). const POOL_FREE_RATE_ALPHA: f64 = 0.3; -/// T12 slot_release_rate EWMA hard bounds in req/s: [0.01, 50] — one -/// release per 100s at the floor (unknown = conservative), 50 req/s cap -/// against burst skew. +/// T13 slot_release_rate bounds in req/s: [0.01, 50]. The rate itself is +/// Little's law (ledger / dwell_ewma); dwell samples are clamped to +/// [0.5, 500]s before the EWMA. Cold start (no dwell sample yet) reads the +/// slow floor — unknown is conservative. pub const SLOT_RELEASE_RATE_MIN: f64 = 0.01; pub const SLOT_RELEASE_RATE_MAX: f64 = 50.0; +/// Decode residency (dwell) sample clamp, seconds. +const DWELL_MIN_SECS: f64 = 0.5; +const DWELL_MAX_SECS: f64 = 500.0; /// est_T_D shadow metrics (design §2.5). Emitted from G9iState; the names /// live here next to the model that produces the values. @@ -539,18 +543,32 @@ pub const METRIC_BOOTSTRAP_ACTUAL: &str = "smg_g9i_bootstrap_actual_secs"; struct DPoolEntry { /// Completion-driven pool free-rate EWMA (tokens/s), clamped on write. free_rate_ewma: f64, - /// T12: completion-driven request-rate EWMA (req/s) — the release rate - /// of the max_running slot dimension. Same event stream, same α. - slot_rate_ewma: f64, + /// T13: decode-residency EWMA (seconds per request, admit→complete as + /// seen by the gateway — directly measured, no right-censoring). The + /// slot release rate is Little's law: ledger_inflight / dwell_ewma, + /// so a short-request burst cannot pump the rate (the T12 incident: + /// EWMA of req/s read 8.6 while true long-tail release was ~2). + dwell_ewma: f64, + /// T13 gateway-side in-flight ledger: requests admitted at decision + /// time minus completions/releases. Kills the snapshot race (the first + /// poll interval after a burst start reads an empty pool). + inflight_ledger: u64, + /// Admit time of the oldest ledger resident (set on the 0→1 transition, + /// cleared at 0). Live lower bound on current decode dwell — visible + /// even when NOTHING completes (the stall case). + oldest_inflight_since: Option, last_complete: Option, /// C4 high-water: max(0, awaiting - free) tokens at the last FRESH /// disagg snapshot; reused when the snapshot goes stale or loses its /// disagg section (misjudgment is asymmetric — overestimating the wait /// is the safe direction). last_deficit_tokens: Option, - /// C4 high-water for the slot dimension (T12): max(0, - /// prealloc_queue_reqs + 1 - free_slots) at the last fresh snapshot. - last_slot_numerator: Option, + /// C4 high-water for the slot dimension (T13): snapshot-side pressure + /// (num_running + prealloc_queue) and the pool's max_running total at + /// the last fresh snapshot. On stale reads the LIVE ledger still wins + /// via max(). + last_pressure_snap: Option, + last_max_running_total: Option, } /// Minimal in-flight queue model, per (node, class) bucket. Metrics-only in @@ -649,11 +667,14 @@ pub struct DSnapshot { /// Free KV-pool tokens: per-rank (1 - token_usage) * max_total, summed. pub free_tokens: u64, /// Requests queued at the D-side prealloc gate (disagg section), summed - /// over dp ranks — the queue length of the SLOT dimension (T12). + /// over dp ranks — the engine-side queue of the SLOT dimension (T12/T13). pub prealloc_queue_reqs: u64, - /// Free running slots: per-rank max_running_requests - num_running_reqs, - /// summed (core section). - pub free_slots: u64, + /// Requests in processing: Σ num_running_reqs (core section). With + /// prealloc_queue_reqs this is the engine-side slot occupancy. + pub running_reqs: u64, + /// Slot capacity: Σ max_running_requests (core section). 0 = unknown + /// (old engine) — the slot dimension then degrades out (C13). + pub max_running_total: u64, /// Snapshot carries disagg data AND is fresh. Otherwise the C4 /// high-water fallback applies (last known deficit, never 0). pub usable: bool, @@ -712,7 +733,14 @@ impl QueueModel { /// T12: per node, T_D = max(memory wait, slot wait) — the prealloc /// memory gate and the max_running slot gate are in series, either one /// stalls the request (c512 实测盲区:槽位 128/128 打满而内存 36%). - /// Returns (t_d_min, slot_wait_of_the_min_node). + /// Returns (t_d_min, slot_wait_of_the_min_node, min_node). + /// + /// T13 slot口径: pressure = max(台账 inflight_ledger, 快照 + /// num_running + prealloc_queue) — both sides are LOWER bounds of the + /// true occupancy (the ledger misses what the gateway never saw, the + /// snapshot misses in-flight-to-engine and lags by a poll interval), + /// so the max is the conservative join (C4). Slot wait = max(0, + /// pressure + 1 − max_running_total) / slot_release_rate. /// /// None = est_T_D inapplicable (est_td_enabled false, no D snapshots, /// or no node has EVER reported disagg data): the C13 "behave exactly @@ -723,55 +751,78 @@ impl QueueModel { d_snaps: &[DSnapshot], own_ctx_tokens: u64, est_td_enabled: bool, - ) -> Option<(f64, f64)> { + ) -> Option<(f64, f64, String)> { if !est_td_enabled || d_snaps.is_empty() { return None; } let mut saw_d = false; let mut min_td = f64::MAX; let mut min_slot = 0.0f64; + let mut min_node = String::new(); for snap in d_snaps { - let rate = self.pool_free_rate(&snap.node); - let slot_rate = self.slot_release_rate(&snap.node); let (mem_wait, slot_wait) = if snap.usable { saw_d = true; // C4 high-water: remember the request-independent deficit // for reuse when this node's snapshot later goes stale. let deficit_base = snap.awaiting_tokens.saturating_sub(snap.free_tokens); - // T12 slot numerator: queued prealloc reqs + this request - // − free slots (a full queue with free slots waits ~0). - let slot_numer = snap - .prealloc_queue_reqs - .saturating_add(1) - .saturating_sub(snap.free_slots); + // T13: engine-side slot pressure (running + queued). + let pressure_snap = snap + .running_reqs + .saturating_add(snap.prealloc_queue_reqs); { let e = self.d_pools.entry(snap.node.clone()).or_default(); e.last_deficit_tokens = Some(deficit_base); - e.last_slot_numerator = Some(slot_numer); + e.last_pressure_snap = Some(pressure_snap); + if snap.max_running_total > 0 { + e.last_max_running_total = Some(snap.max_running_total); + } } + let rate = self.pool_free_rate(&snap.node); + let slot_rate = self.slot_release_rate(&snap.node); + let ledger = self.d_inflight(&snap.node); + let pressure = pressure_snap.max(ledger); // C4: 冲突取大 // T_D(mem) = max(0, awaiting + own_ctx - free) / pool_free_rate. let cur = snap .awaiting_tokens .saturating_add(own_ctx_tokens) .saturating_sub(snap.free_tokens); - (cur as f64 / rate, slot_numer as f64 / slot_rate) + let slot = if snap.max_running_total > 0 { + pressure.saturating_add(1).saturating_sub(snap.max_running_total) as f64 + / slot_rate + } else { + 0.0 // capacity unknown -> slot dimension degrades out (C13) + }; + (cur as f64 / rate, slot) } else { match self.d_pools.get(&snap.node) { // C4: stale snapshot -> last known high-water deficits, - // both dimensions (mem still charges this request's ctx). + // both dimensions; the LIVE ledger still wins via max(). Some(e) if e.last_deficit_tokens.is_some() - || e.last_slot_numerator.is_some() => + || e.last_pressure_snap.is_some() => { saw_d = true; + let rate = e + .free_rate_ewma + .clamp(POOL_FREE_RATE_MIN, POOL_FREE_RATE_MAX); + let dwell = self.dwell_for_rate(e, Instant::now()); + let slot_rate = if e.inflight_ledger > 0 && dwell > 0.0 { + (e.inflight_ledger as f64 / dwell) + .clamp(SLOT_RELEASE_RATE_MIN, SLOT_RELEASE_RATE_MAX) + } else { + SLOT_RELEASE_RATE_MIN + }; let mem = e .last_deficit_tokens .map(|d| d.saturating_add(own_ctx_tokens) as f64 / rate) .unwrap_or(0.0); - let slot = e - .last_slot_numerator - .map(|n| n as f64 / slot_rate) - .unwrap_or(0.0); + let slot = match (e.last_pressure_snap, e.last_max_running_total) { + (Some(p_snap), Some(mr)) if mr > 0 => { + let pressure = p_snap.max(e.inflight_ledger); + pressure.saturating_add(1).saturating_sub(mr) as f64 / slot_rate + } + _ => 0.0, + }; (mem, slot) } // Never reported disagg (old engine / non-PD): not a D @@ -783,9 +834,10 @@ impl QueueModel { if td < min_td { min_td = td; min_slot = slot_wait; + min_node = snap.node.clone(); } } - saw_d.then_some((min_td, min_slot)) + saw_d.then_some((min_td, min_slot, min_node)) } /// est_TTFT with the D-side prealloc wait folded in (design §2.1). @@ -804,7 +856,8 @@ impl QueueModel { est_td_enabled: bool, ) -> EstTtft { let t_p = self.t_p_of(node, class); - let Some((t_d, t_d_slot)) = self.est_td_min(d_snaps, own_ctx_tokens, est_td_enabled) + let Some((t_d, t_d_slot, _d_node)) = + self.est_td_min(d_snaps, own_ctx_tokens, est_td_enabled) else { return EstTtft { total: t_p, @@ -1015,12 +1068,19 @@ impl QueueModel { /// the decode node. Rate sample = ctx / dt since the previous completion /// on the same node; fixed α=0.3 per event, hard-clamped to /// [POOL_FREE_RATE_MIN, POOL_FREE_RATE_MAX]. The first completion only - /// arms the clock (no dt to divide by). - pub fn record_decode_completion(&mut self, node: &str, ctx_tokens: u64) { - self.record_decode_completion_at(node, ctx_tokens, Instant::now()); + /// arms the clock (no dt to divide by). `dwell_secs` (T13) feeds the + /// decode-residency EWMA behind the Little's-law slot release rate. + pub fn record_decode_completion(&mut self, node: &str, ctx_tokens: u64, dwell_secs: Option) { + self.record_decode_completion_at(node, ctx_tokens, dwell_secs, Instant::now()); } - fn record_decode_completion_at(&mut self, node: &str, ctx_tokens: u64, now: Instant) { + fn record_decode_completion_at( + &mut self, + node: &str, + ctx_tokens: u64, + dwell_secs: Option, + now: Instant, + ) { let e = self.d_pools.entry(node.to_string()).or_default(); if let Some(last) = e.last_complete { let dt = now.duration_since(last).as_secs_f64().max(0.001); @@ -1028,12 +1088,17 @@ impl QueueModel { e.free_rate_ewma = ((1.0 - POOL_FREE_RATE_ALPHA) * e.free_rate_ewma + POOL_FREE_RATE_ALPHA * sample) .clamp(POOL_FREE_RATE_MIN, POOL_FREE_RATE_MAX); - // T12: the same event is one slot release — req/s EWMA beside - // the tokens/s one (α shared, own hard bounds). - let slot_sample = 1.0 / dt; - e.slot_rate_ewma = ((1.0 - POOL_FREE_RATE_ALPHA) * e.slot_rate_ewma - + POOL_FREE_RATE_ALPHA * slot_sample) - .clamp(SLOT_RELEASE_RATE_MIN, SLOT_RELEASE_RATE_MAX); + } + // T13: decode-residency sample (admit→complete wall time, directly + // measured by the gateway — no right-censoring, unlike a completion + // -rate EWMA which only sees the requests that finished EARLY). + if let Some(dwell) = dwell_secs { + let dwell = dwell.clamp(DWELL_MIN_SECS, DWELL_MAX_SECS); + e.dwell_ewma = if e.dwell_ewma <= 0.0 { + dwell + } else { + (1.0 - POOL_FREE_RATE_ALPHA) * e.dwell_ewma + POOL_FREE_RATE_ALPHA * dwell + }; } e.last_complete = Some(now); } @@ -1049,15 +1114,96 @@ impl QueueModel { .clamp(POOL_FREE_RATE_MIN, POOL_FREE_RATE_MAX) } - /// T12: current slot release rate (req/s) for a decode node — the - /// completion rate EWMA. Floor doubles as cold start (0.01 req/s = - /// one release per 100s: unknown is conservative). + /// T12/T13: current slot release rate (req/s) for a decode node — + /// Little's law over gateway-measured state: ledger_inflight / + /// max(dwell_ewma, oldest resident age). Both factors are + /// gateway-observed and live; a completion-only rate estimator is + /// right-censored during stalls (the T12 incident: EWMA of req/s read + /// 8.6 from the opening short-request burst while true long-tail + /// release was ~2 req/s). Cold start (no dwell, empty ledger) reads + /// the slow floor — unknown is conservative. pub fn slot_release_rate(&self, node: &str) -> f64 { + self.slot_release_rate_at(node, Instant::now()) + } + + fn slot_release_rate_at(&self, node: &str, now: Instant) -> f64 { + let Some(e) = self.d_pools.get(node) else { + return SLOT_RELEASE_RATE_MIN; + }; + if e.inflight_ledger == 0 { + return SLOT_RELEASE_RATE_MIN; + } + let dwell = self.dwell_for_rate(e, now); + if dwell <= 0.0 { + return SLOT_RELEASE_RATE_MIN; + } + (e.inflight_ledger as f64 / dwell).clamp(SLOT_RELEASE_RATE_MIN, SLOT_RELEASE_RATE_MAX) + } + + /// T13 D-side in-flight ledger: +1 at admission-decision time (same + /// queue-model write lock as the P booking — the check→act race is + /// closed on both sides). Tracks the oldest resident's admit time + /// (0→1 transition) for the live dwell lower bound. + pub fn d_ledger_admit(&mut self, node: &str) { + self.d_ledger_admit_at(node, Instant::now()); + } + + fn d_ledger_admit_at(&mut self, node: &str, now: Instant) { + let e = self.d_pools.entry(node.to_string()).or_default(); + if e.inflight_ledger == 0 { + e.oldest_inflight_since = Some(now); + } + e.inflight_ledger = e.inflight_ledger.saturating_add(1); + } + + /// Ledger −1 (completion/release paths; saturating — a stray release + /// must never underflow the ledger). Hitting 0 clears the oldest-age + /// tracker; otherwise the oldest timestamp stays sticky (after the + /// oldest completes we over-estimate age briefly — the conservative + /// direction). + pub fn d_ledger_release(&mut self, node: &str) { + self.d_ledger_release_at(node, Instant::now()); + } + + fn d_ledger_release_at(&mut self, node: &str, _now: Instant) { + if let Some(e) = self.d_pools.get_mut(node) { + e.inflight_ledger = e.inflight_ledger.saturating_sub(1); + if e.inflight_ledger == 0 { + e.oldest_inflight_since = None; + } + } + } + + /// Move a ledger booking between D nodes (decision-time pick vs actual + /// landing, mirrors the P-side rekey). + pub fn d_ledger_rekey(&mut self, from: &str, to: &str) { + if from == to { + return; + } + self.d_ledger_release(from); + self.d_ledger_admit(to); + } + + /// Current ledger count for a decode node. + pub fn d_inflight(&self, node: &str) -> u64 { self.d_pools .get(node) - .map(|e| e.slot_rate_ewma) - .unwrap_or(0.0) - .clamp(SLOT_RELEASE_RATE_MIN, SLOT_RELEASE_RATE_MAX) + .map(|e| e.inflight_ledger) + .unwrap_or(0) + } + + /// Effective decode residency (s) behind the slot release rate: + /// max(dwell_ewma from completions, oldest in-flight resident's age). + /// The second term is what a completion-only estimator cannot see: + /// during a long-request stall NOTHING completes, and the oldest + /// resident's age is a live, conservative lower bound on dwell (kills + /// the T12 right-censoring pump — 短请求潮灌高速率). + fn dwell_for_rate(&self, e: &DPoolEntry, now: Instant) -> f64 { + let oldest_age = e + .oldest_inflight_since + .map(|t| now.duration_since(t).as_secs_f64()) + .unwrap_or(0.0); + e.dwell_ewma.max(oldest_age) } } @@ -1103,15 +1249,24 @@ pub struct AdmissionMark { pub node: String, pub class: ShadowClass, pub est_tokens: u64, + /// T13: the D node this request was ledger-booked against at decision + /// time (the min-T_D node). None when no D view existed (C13) — no + /// ledger ops then. Rekeyed to the actual decode node at response time. + pub d_node: Option, + /// Decision time; the admit→complete span is the decode-residency + /// (dwell) sample behind the Little's-law slot release rate. + pub created: Instant, consumed: AtomicBool, } impl AdmissionMark { - pub fn new(node: String, class: ShadowClass, est_tokens: u64) -> Self { + pub fn new(node: String, class: ShadowClass, est_tokens: u64, d_node: Option) -> Self { Self { node, class, est_tokens, + d_node, + created: Instant::now(), consumed: AtomicBool::new(false), } } @@ -2045,16 +2200,22 @@ mod tests { // ============ est_T_D P1 (T3): T_D estimation + max() composition ============ fn d_snap(node: &str, awaiting: u64, free: u64, usable: bool) -> DSnapshot { - // Slots never constrain in the pre-T12 tests: 128 free, empty queue. - d_snap_slots(node, awaiting, free, 0, 128, usable) + // Slots never constrain in the pre-T12 tests: empty engine view, + // capacity 128 -> pressure 0, wait 0. + d_snap_slots(node, awaiting, free, 0, 0, 128, usable) } + /// (node, awaiting_tokens, free_tokens, prealloc_queue, running, + /// max_running_total, usable). T13口径: slot pressure = max(ledger, + /// running + prealloc_queue); slot wait = max(0, pressure + 1 - + /// max_running_total) / slot_release_rate. fn d_snap_slots( node: &str, awaiting: u64, free: u64, prealloc_queue_reqs: u64, - free_slots: u64, + running_reqs: u64, + max_running_total: u64, usable: bool, ) -> DSnapshot { DSnapshot { @@ -2062,7 +2223,8 @@ mod tests { awaiting_tokens: awaiting, free_tokens: free, prealloc_queue_reqs, - free_slots, + running_reqs, + max_running_total, usable, } } @@ -2136,8 +2298,8 @@ mod tests { let mut q = QueueModel::new(); let t0 = Instant::now(); // 200k tok/s measured release rate on d1. - q.record_decode_completion_at("d1", 200_000, t0); - q.record_decode_completion_at("d1", 200_000, t0 + Duration::from_secs(1)); + q.record_decode_completion_at("d1", 200_000, None, t0); + q.record_decode_completion_at("d1", 200_000, None, t0 + Duration::from_secs(1)); let snaps = [d_snap("d1", 900_000, 100_000, true)]; // deficit 800k let est = q.est_ttft(Some("p1"), ShadowClass::Hot, 0, &snaps, true); // rate = 0.3*200k (first real sample after arming) = 60k tok/s. @@ -2148,17 +2310,17 @@ mod tests { /// Slot-bound blind spot (the c512 shape): memory wide idle, slots full /// → T_D is slot-dominated and grows linearly with the prealloc queue. + /// Engine view: running 128/128 (full), queue 10; cold-start slot rate + /// floor 0.01 req/s -> (138+1-128)/0.01 = 1100s. #[test] fn est_td_slot_bound_dominates_and_scales() { let mut q = QueueModel::new(); - // Memory: 10M free (mem wait 0). Slots: 0 free, 10 queued; cold-start - // slot rate floor 0.01 req/s -> (10+1)/0.01 = 1100s. - let snaps = [d_snap_slots("d1", 0, 10_000_000, 10, 0, true)]; + let snaps = [d_snap_slots("d1", 0, 10_000_000, 10, 128, 128, true)]; let est = q.est_ttft(Some("p1"), ShadowClass::Hot, 0, &snaps, true); assert_eq!(est.t_d, 1100.0, "slot wait dominates: {}", est.t_d); assert_eq!(est.t_d_slot, 1100.0); - // Linear in the queue: 20 queued -> 2100s. - let snaps = [d_snap_slots("d1", 0, 10_000_000, 20, 0, true)]; + // Linear in the queue: 20 queued -> (148+1-128)/0.01 = 2100s. + let snaps = [d_snap_slots("d1", 0, 10_000_000, 20, 128, 128, true)]; let est = q.est_ttft(Some("p1"), ShadowClass::Hot, 0, &snaps, true); assert_eq!(est.t_d, 2100.0); } @@ -2168,71 +2330,212 @@ mod tests { #[test] fn est_td_takes_max_of_memory_and_slot() { let mut q = QueueModel::new(); - // Memory: deficit 100k/1k floor = 100s. Slots: full + 10 queued = - // 1100s -> slot wins. - let snaps = [d_snap_slots("d1", 100_000, 0, 10, 0, true)]; + // Memory: deficit 100k/1k floor = 100s. Slots: full (128/128) + + // 10 queued = (138+1-128)/0.01 = 1100s -> slot wins. + let snaps = [d_snap_slots("d1", 100_000, 0, 10, 128, 128, true)]; let est = q.est_ttft(Some("p1"), ShadowClass::Hot, 0, &snaps, true); assert_eq!(est.t_d, 1100.0, "max picks the slot side"); // Same memory deficit, slots wide open -> memory wins (unchanged // pre-T12 behavior: 100s). - let snaps = [d_snap_slots("d1", 100_000, 0, 0, 128, true)]; + let snaps = [d_snap_slots("d1", 100_000, 0, 0, 0, 128, true)]; let est = q.est_ttft(Some("p1"), ShadowClass::Hot, 0, &snaps, true); assert_eq!(est.t_d, 100.0, "max picks the memory side"); assert_eq!(est.t_d_slot, 0.0); } - /// Slot release rate EWMA: convergence, both clamps, cold-start floor. + /// T13 Little's-law slot release rate: ledger / max(dwell_ewma, + /// oldest resident age). The short-burst pump is impossible: during a + /// long-request stall nothing completes, and the oldest resident's age + /// drives the rate down live. #[test] - fn slot_release_rate_ewma_converges_and_clamps() { + fn slot_release_rate_littles_law_not_pumped_by_short_burst() { let mut q = QueueModel::new(); let t0 = Instant::now(); - // Cold start: the floor. - assert_eq!(q.slot_release_rate("d1"), SLOT_RELEASE_RATE_MIN); - q.record_decode_completion_at("d1", 100_000, t0); // arms the clock - assert_eq!(q.slot_release_rate("d1"), SLOT_RELEASE_RATE_MIN); - // 10 completions at 1/s -> sample 1 req/s; EWMA α=0.3 -> 1-0.7^10. - for i in 1..=10u64 { - q.record_decode_completion_at("d1", 100_000, t0 + Duration::from_secs(i)); + // Cold start: floor. + assert_eq!(q.slot_release_rate_at("d1", t0), SLOT_RELEASE_RATE_MIN); + // Opening burst: 256 shorts admitted over 10s, 20 complete fast + // (dwell 0.5s). Rate right after the burst: dwell_ewma 0.5s, + // oldest age ~0 -> high rate (the burst IS fast, that's true). + for i in 0..256u64 { + q.d_ledger_admit_at("d1", t0 + Duration::from_millis(i * 40)); } - let rate = q.slot_release_rate("d1"); - assert!((rate - (1.0 - 0.7f64.powi(10))).abs() < 1e-6, "rate={rate}"); - // Burst: 1ms apart -> sample 1000 req/s clamps to 50. - let mut t = t0 + Duration::from_secs(20); - for _ in 0..20 { - t += Duration::from_millis(1); - q.record_decode_completion_at("d2", 1000, t); + for i in 0..20u64 { + let t = t0 + Duration::from_millis(256 * 40 + i * 25); + q.record_decode_completion_at("d1", 10_000, Some(0.5), t); + // Completions pair their ledger bookings (production does this + // in shadow_record; the QueueModel completion hook deliberately + // does NOT touch the ledger). + q.d_ledger_release_at("d1", t); } - assert_eq!(q.slot_release_rate("d2"), SLOT_RELEASE_RATE_MAX); + // 236 still resident, nothing completing (long tail). At +120s the + // oldest age dominates: rate = 236 / max(0.5, 120) ≈ 1.97 req/s — + // NOT the burst's implied hundreds. + let t_later = t0 + Duration::from_secs(120); + let rate = q.slot_release_rate_at("d1", t_later); + assert!( + (rate - 236.0 / 120.0).abs() < 0.05, + "rate={rate} must collapse with resident age, not stay pumped" + ); + // Healthy steady state: dwell_ewma 30s from completions dominates + // over a young oldest age. + let mut q2 = QueueModel::new(); + for i in 0..10u64 { + q2.d_ledger_admit_at("d1", t0 + Duration::from_secs(i)); + } + for i in 0..5u64 { + q2.record_decode_completion_at( + "d1", + 10_000, + Some(30.0), + t0 + Duration::from_secs(10 + i), + ); + } + // ledger 10, dwell_ewma 30 (first sample sets), oldest age ~5s. + let rate2 = q2.slot_release_rate_at("d1", t0 + Duration::from_secs(15)); + assert!((rate2 - 10.0 / 30.0).abs() < 1e-6, "rate={rate2}"); + // Ledger empty -> floor (unknown = conservative). + let q3 = QueueModel::new(); + assert_eq!(q3.slot_release_rate_at("d1", t0), SLOT_RELEASE_RATE_MIN); + } + + /// Rate clamps: absurdly fast turnover caps at 50 req/s; a permanent + /// stall sinks to the 0.01 floor. + #[test] + fn slot_release_rate_hard_bounds() { + let mut q = QueueModel::new(); + let t0 = Instant::now(); + for i in 0..100u64 { + q.d_ledger_admit_at("d1", t0 + Duration::from_millis(i)); + } + // 100 residents aged 100ms -> 100/0.1 = 1000 req/s -> clamp 50. + assert_eq!( + q.slot_release_rate_at("d1", t0 + Duration::from_millis(100)), + SLOT_RELEASE_RATE_MAX + ); + // Same ledger a year later -> 100/31.5M s -> floor 0.01. + assert_eq!( + q.slot_release_rate_at("d1", t0 + Duration::from_secs(31_536_000)), + SLOT_RELEASE_RATE_MIN + ); } /// C4 for the slot dimension: a stale snapshot keeps the last known - /// slot-queue high-water — it must NOT clear the gate. + /// slot-pressure high-water — it must NOT clear the gate; the LIVE + /// ledger still wins via max(). #[test] fn est_td_slot_stale_uses_high_water() { let mut q = QueueModel::new(); - // Fresh: 10 queued, 0 free slots -> slot numerator 11 stored. - let snaps = [d_snap_slots("d1", 0, 10_000_000, 10, 0, true)]; + // Fresh: full (128/128) + 10 queued -> pressure_snap 138 stored. + let snaps = [d_snap_slots("d1", 0, 10_000_000, 10, 128, 128, true)]; let est = q.est_ttft(Some("p1"), ShadowClass::Hot, 0, &snaps, true); assert_eq!(est.t_d, 1100.0); - // Stale: snapshot now claims empty queue + free slots — ignored, - // the stored high-water still drives T_D. - let snaps = [d_snap_slots("d1", 0, 10_000_000, 0, 128, false)]; + // Stale: snapshot now claims empty + idle — ignored, stored 138 + // still drives T_D. + let snaps = [d_snap_slots("d1", 0, 10_000_000, 0, 0, 128, false)]; let est = q.est_ttft(Some("p1"), ShadowClass::Hot, 0, &snaps, true); assert_eq!(est.t_d, 1100.0, "stale slot view falls back to high-water"); assert_eq!(est.t_d_slot, 1100.0); + // T13: a live ledger ABOVE the stored snapshot wins even stale. + for _ in 0..200 { + q.d_ledger_admit("d1"); + } + let snaps = [d_snap_slots("d1", 0, 10_000_000, 0, 0, 128, false)]; + let est = q.est_ttft(Some("p1"), ShadowClass::Hot, 0, &snaps, true); + // pressure = max(200, 138) = 200 -> (200+1-128)/rate. rate: ledger + // 200, no dwell, oldest age ~0 -> dwell ~0 -> floor... read the + // actual rate to keep the assertion honest. + let rate = q.slot_release_rate("d1"); + let expect = 73.0 / rate; + assert!((est.t_d - expect).abs() < 1e-6, "t_d={} expect={expect}", est.t_d); } /// C13 regression with slot fields: a node without disagg data (old /// engine) is not a D source — est stays exactly the P-only model. + /// Capacity unknown (max_running 0) also degrades the slot term to 0. #[test] fn est_td_slot_c13_unchanged() { let mut q = QueueModel::new(); q.admit("p1", ShadowClass::Hot, 2000); // t_p = 2.5 let base = q.est_ttft(Some("p1"), ShadowClass::Hot, 0, &[], true); - let snaps = [d_snap_slots("d1", 0, 0, 224, 0, false)]; // unusable + let snaps = [d_snap_slots("d1", 0, 0, 224, 128, 128, false)]; // unusable let degraded = q.est_ttft(Some("p1"), ShadowClass::Hot, 0, &snaps, true); assert_eq!(degraded, base); assert_eq!(degraded.t_d_slot, 0.0); + // max_running_total unknown (0): slot term degrades out, memory + // dimension still works. + let snaps = [d_snap_slots("d1", 100_000, 0, 224, 128, 0, true)]; + let est = q.est_ttft(Some("p1"), ShadowClass::Hot, 0, &snaps, true); + assert_eq!(est.t_d, 100.0, "memory only; slot degraded"); + assert_eq!(est.t_d_slot, 0.0); + } + + // ============ T13: D-side in-flight ledger ============ + + /// Ledger balance: decision-time admit, completion/release pairing, + /// rekey between D nodes, saturating release. + #[test] + fn t13_d_ledger_balance_and_rekey() { + let mut q = QueueModel::new(); + q.d_ledger_admit("d1"); + q.d_ledger_admit("d1"); + q.d_ledger_admit("d2"); + assert_eq!(q.d_inflight("d1"), 2); + assert_eq!(q.d_inflight("d2"), 1); + q.d_ledger_rekey("d1", "d2"); + assert_eq!(q.d_inflight("d1"), 1); + assert_eq!(q.d_inflight("d2"), 2); + q.d_ledger_release("d1"); + q.d_ledger_release("d1"); // saturating at 0, no underflow + q.d_ledger_release("d1"); + assert_eq!(q.d_inflight("d1"), 0); + assert_eq!(q.d_inflight("d2"), 2); + } + + /// Burst oversell test: 300 concurrent check-and-admit at the slot + /// boundary. mr=256, dwell pinned 100s -> rate = ledger/100, gate + /// wait = (ledger+1-256)/rate > 5s iff ledger > 268.4 -> exactly 268 + /// admitted, deterministically. + #[test] + fn t13_slot_gate_no_oversell_multithreaded() { + use std::sync::atomic::AtomicUsize; + let q = Arc::new(std::sync::RwLock::new(QueueModel::new())); + let t0 = Instant::now(); + // Pin dwell at 100s via completion samples (before any ledger). + { + let mut qq = q.write().unwrap(); + qq.record_decode_completion_at("d1", 10_000, Some(100.0), t0); + qq.record_decode_completion_at("d1", 10_000, Some(100.0), t0 + Duration::from_secs(1)); + } + let t_gate = t0 + Duration::from_secs(100); // oldest_age = 100s too + let admitted = Arc::new(AtomicUsize::new(0)); + let barrier = Arc::new(std::sync::Barrier::new(300)); + let mut handles = Vec::new(); + for _ in 0..300 { + let q = Arc::clone(&q); + let admitted = Arc::clone(&admitted); + let barrier = Arc::clone(&barrier); + handles.push(std::thread::spawn(move || { + barrier.wait(); + // Mirrors the slot gate inside shadow_classify_and_admit's + // write lock: pressure = ledger (engine view absent here). + let mut q = q.write().unwrap(); + let ledger = q.d_inflight("d1"); + let rate = q.slot_release_rate_at("d1", t_gate); + let wait = ledger.saturating_add(1).saturating_sub(256) as f64 / rate; + if wait <= 5.0 { + q.d_ledger_admit_at("d1", t0); + admitted.fetch_add(1, Ordering::SeqCst); + } + })); + } + for h in handles { + h.join().unwrap(); + } + // wait > 5 iff (l+1-256)*100/l > 5 iff l > 268.42 -> admits while + // ledger l <= 268, so exactly 269 bookings land (the 270th is the + // first rejection). + assert_eq!(admitted.load(Ordering::SeqCst), 269, "deterministic gate boundary"); + assert_eq!(q.read().unwrap().d_inflight("d1"), 269); } #[test] @@ -2270,13 +2573,14 @@ mod tests { let mut q = QueueModel::new(); let t0 = Instant::now(); // First completion only arms the clock: cold start reads the floor. - q.record_decode_completion_at("http://d1:30001", 100_000, t0); + q.record_decode_completion_at("http://d1:30001", 100_000, None, t0); assert_eq!(q.pool_free_rate("http://d1:30001"), POOL_FREE_RATE_MIN); // Ten completions of 100k ctx at 1s spacing -> 100k tok/s samples. for i in 1..=10u64 { q.record_decode_completion_at( "http://d1:30001", 100_000, + None, t0 + Duration::from_secs(i), ); } @@ -2296,12 +2600,12 @@ mod tests { let mut q = QueueModel::new(); let t0 = Instant::now(); // Enormous sample (100M tokens in 1ms) clamps to the 2M upper bound. - q.record_decode_completion_at("d1", 1_000, t0); - q.record_decode_completion_at("d1", 100_000_000, t0 + Duration::from_millis(1)); + q.record_decode_completion_at("d1", 1_000, None, t0); + q.record_decode_completion_at("d1", 100_000_000, None, t0 + Duration::from_millis(1)); assert_eq!(q.pool_free_rate("d1"), POOL_FREE_RATE_MAX); // Tiny sample on a fresh node clamps to the 1k floor. - q.record_decode_completion_at("d2", 1_000, t0); - q.record_decode_completion_at("d2", 1, t0 + Duration::from_secs(100)); + q.record_decode_completion_at("d2", 1_000, None, t0); + q.record_decode_completion_at("d2", 1, None, t0 + Duration::from_secs(100)); assert_eq!(q.pool_free_rate("d2"), POOL_FREE_RATE_MIN); }