gateway g9i: AIMD blind-state recovery on real engine signals + real-queue P gate + nonstream e2e metric (P2 T17)

This commit is contained in:
2026-10-16 11:16:32 +08:00
parent 59e3aee6d9
commit e4bfdc949b
4 changed files with 329 additions and 129 deletions
+7 -2
View File
@@ -304,10 +304,14 @@ struct CliArgs {
#[arg(long, default_value_t = 2.0, help_heading = "G9i Pool Admission")]
g9i_t_prefer_secs: f64,
/// Hard P-queue token cap (physical backstop): skip prefill nodes whose in-flight tokens exceed this; 429 p_queue_cap only when ALL are over (0 = disabled; follows --g9i-td-admit mode)
#[arg(long, default_value_t = 4_000_000, help_heading = "G9i Pool Admission")]
/// Hard P-queue token cap (physical backstop; effective cap = min(this, drain_ewma x budget)) — 429 p_queue_cap only when ALL nodes over (0 = disabled; follows --g9i-td-admit mode)
#[arg(long, default_value_t = 16_000_000, help_heading = "G9i Pool Admission")]
g9i_p_queue_token_cap: u64,
/// P-queue real-queue gate: skip prefill nodes whose engine-reported waiting queue (num_waiting_reqs) exceeds this (0 = disabled; follows --g9i-td-admit mode)
#[arg(long, default_value_t = 10, help_heading = "G9i Pool Admission")]
g9i_p_queue_reqs_cap: u64,
/// Admission hold queue: park holdable rejections (d_window_full / p_queue_cap) up to this many seconds instead of an immediate 429 (0 = disabled; enforce mode only; hold time counts into TTFT)
#[arg(long, default_value_t = 1.0, help_heading = "G9i Pool Admission")]
g9i_admit_hold_secs: f64,
@@ -1201,6 +1205,7 @@ impl CliArgs {
td_gate_secs: self.g9i_td_gate_secs,
t_prefer_secs: self.g9i_t_prefer_secs,
p_queue_token_cap: self.g9i_p_queue_token_cap,
p_queue_reqs_cap: self.g9i_p_queue_reqs_cap,
admit_hold_secs: self.g9i_admit_hold_secs,
admit_hold_max: self.g9i_admit_hold_max,
};
+222 -48
View File
@@ -45,7 +45,7 @@
use std::{
collections::{HashMap, VecDeque},
sync::{
atomic::{AtomicBool, Ordering},
atomic::{AtomicBool, AtomicU64, Ordering},
Arc, RwLock,
},
time::{Duration, Instant},
@@ -197,12 +197,21 @@ pub struct G9iConfig {
/// a prefill node whose shadow-bookkept in-flight tokens + the
/// request's own ctx exceed this is skipped (candidate fallback first);
/// only when EVERY prefill node is over the cap does the request 429
/// (p_queue_cap). Token-based, not request-count (昨天 p1 积压 64M
/// tokens ≈ 450s of work). Default 4M ≈ 30s of queue at the measured
/// 141k tok/s drain. 0 = disabled. Mode follows td_admit
/// (off/shadow/enforce); off or 0 ≡ current behavior (C13).
/// (p_queue_cap). T17 修正版: this is now the loose BACKSTOP — the
/// primary口径 is the engine's REAL waiting-queue length
/// (p_queue_reqs_cap below; 估算不可信). Raised to 16M (the T10 4M
/// choked all-cold 200k-ctx traffic at ~20-50 requests/node). 0 =
/// disabled. Mode follows td_admit; off or 0 ≡ current behavior (C13).
#[serde(default = "default_p_queue_token_cap")]
pub p_queue_token_cap: u64,
/// T17 (修正版) primary P-queue gate: a node whose ENGINE-REPORTED
/// waiting queue (num_waiting_reqs, /v1/loads core) exceeds this is
/// excluded from candidates; all over -> hold/429 (p_queue_cap).
/// Real measurement, not an estimate. Default 10 (待实测调).
/// Stale/missing snapshots are never judged (不冤枉); all-missing ->
/// C13 degrade. 0 = disabled.
#[serde(default = "default_p_queue_reqs_cap")]
pub p_queue_reqs_cap: u64,
/// T15 admission hold queue: holdable rejections (d_window_full /
/// p_queue_cap) park the request up to this many seconds — TOTAL budget
/// across hold retries — waiting for room before the 429 fires
@@ -267,7 +276,10 @@ fn default_t_prefer_secs() -> f64 {
2.0
}
fn default_p_queue_token_cap() -> u64 {
4_000_000
16_000_000
}
fn default_p_queue_reqs_cap() -> u64 {
10
}
fn default_admit_hold_secs() -> f64 {
1.0
@@ -301,6 +313,7 @@ impl Default for G9iConfig {
td_gate_secs: default_td_gate_secs(),
t_prefer_secs: default_t_prefer_secs(),
p_queue_token_cap: default_p_queue_token_cap(),
p_queue_reqs_cap: default_p_queue_reqs_cap(),
admit_hold_secs: default_admit_hold_secs(),
admit_hold_max: default_admit_hold_max(),
}
@@ -349,6 +362,10 @@ pub struct NodeLoad {
pub queued_total_input_tokens: u64,
/// Sum of drain_rate_tokens_per_s across dp ranks.
pub drain_rate_tokens_per_s: f64,
/// Sum of num_waiting_reqs across dp ranks — the engine's REAL queue
/// length, the primary P-admission口径 (T17 修正版: 估算管路由,
/// 真实队列管准入).
pub waiting_reqs: u64,
/// Sum of disaggregation.decode_prealloc_queue_reqs across dp ranks.
pub decode_prealloc_queue_reqs: u64,
/// Sum of disaggregation.decode_transfer_queue_reqs across dp ranks.
@@ -471,6 +488,11 @@ pub struct G9iState {
/// only used when a holdable rejection fires in enforce mode with a
/// nonzero hold budget.
hold: HoldQueue,
/// T17: cumulative hold timeouts (blind-state AIMD input — a timeout
/// this tick means someone is hurting even without TTFT samples).
hold_timeouts_total: AtomicU64,
/// Timeouts seen at the previous AIMD tick (for the diff).
last_tick_timeouts: AtomicU64,
}
/// T14 AIMD congestion-window state for one decode node.
@@ -680,6 +702,8 @@ impl G9iState {
d_windows: RwLock::new(HashMap::new()),
last_aimd: RwLock::new(Instant::now()),
hold: HoldQueue::default(),
hold_timeouts_total: AtomicU64::new(0),
last_tick_timeouts: AtomicU64::new(0),
}
}
@@ -1068,9 +1092,19 @@ impl G9iState {
fn aimd_tick(&self) {
let (count, p50, p95, p99) = self.ttft_slo_view();
// T17 (修正版) blind-state inputs (non-streaming traffic leaves no
// TTFT samples): hold timeouts this tick, and per-node REAL engine
// signals — D prealloc queue (disagg 段真实值) + pipeline depth.
let timeouts_now = self.hold_timeouts_total.load(Ordering::Relaxed);
let timed_out = {
let prev = self.last_tick_timeouts.swap(timeouts_now, Ordering::Relaxed);
timeouts_now > prev
};
let d_snaps = self.current_d_snapshots();
let Ok(mut windows) = self.d_windows.write() else {
return;
};
let q = self.shadow_queue.as_ref().and_then(|l| l.read().ok());
let mut grew = false;
for (node, e) in windows.iter_mut() {
if e.max_running_total == 0 {
@@ -1081,19 +1115,37 @@ impl G9iState {
let w_min = WINDOW_MIN;
let w_max = WINDOW_MAX;
e.w = e.w.clamp(w_min, w_max);
if count < AIMD_MIN_SAMPLES {
continue; // 样本不足/无新流量: hold
}
let old = e.w;
let why: &str;
if p95 > SLO_P95_SECS || p99 > SLO_P99_SECS {
e.w = (e.w * AIMD_CUT).max(w_min);
why = "slo_violation";
} else if p50 < SLO_P50_SECS && p95 < SLO_P95_SECS {
e.w = (e.w + AIMD_ADD).min(w_max);
why = "slo_ok";
if count >= AIMD_MIN_SAMPLES {
// SLO 判定优先(流式样本充足时照旧).
if p95 > SLO_P95_SECS || p99 > SLO_P99_SECS {
e.w = (e.w * AIMD_CUT).max(w_min);
why = "slo_violation";
} else if p50 < SLO_P50_SECS && p95 < SLO_P95_SECS {
e.w = (e.w + AIMD_ADD).min(w_max);
why = "slo_ok";
} else {
why = "hold";
}
} else {
why = "hold";
// T17 (修正版) 盲态降级(非流式流量没有 TTFT 样本——窗口
// 必须不靠它们恢复): 本拍有持有超时 -> hold(有人在受伤);
// D prealloc 队列清空(==0) 且管线宽裕 (<0.8W) -> 慢加恢复;
// prealloc 持续 >0 或管线顶满 -> hold(窗口本身已在限流)。
let pipe = q.as_ref().map(|q| q.d_pipeline(node)).unwrap_or(0) as f64;
let prealloc_busy = d_snaps
.iter()
.find(|s| &s.node == node)
.is_some_and(|s| s.prealloc_queue_reqs > 0);
if timed_out {
why = "blind_hold_timeout";
} else if !prealloc_busy && pipe < 0.8 * e.w {
e.w = (e.w + AIMD_ADD).min(w_max);
why = "blind_recover";
} else {
why = "blind_hold_full";
}
}
if e.w != old {
info!(
@@ -1278,6 +1330,8 @@ impl G9iState {
self.hold.remove(id, body_bytes);
guard.disarm();
let held = parked_at.elapsed().as_secs_f64();
// T17: AIMD blind-state input.
self.hold_timeouts_total.fetch_add(1, Ordering::Relaxed);
metrics::counter!(METRIC_ADMIT_HELD, "outcome" => "timeout", "class" => class_label)
.increment(1);
metrics::histogram!(METRIC_ADMIT_HOLD_SECS).record(held);
@@ -1513,17 +1567,40 @@ impl G9iState {
})
}
/// v1-path cap judgment (T10): over cap only when a FRESH /v1/loads
/// snapshot's queued_total_input_tokens says so. Missing or stale
/// snapshots are NOT judged (fresh_load already enforces the TTL) — a
/// blind node must never trip the physical backstop (避免误伤).
/// v1-path cap judgment (T17 修正版): over cap when a FRESH /v1/loads
/// snapshot says so — PRIMARY: engine-reported waiting queue
/// (num_waiting_reqs > p_queue_reqs_cap); BACKSTOP: queued tokens over
/// p_queue_token_cap (16M, loose). Missing or stale snapshots are NOT
/// judged (fresh_load already enforces the TTL) — a blind node must
/// never trip the physical backstop (不冤枉), and all-blind means the
/// gate stays open (C13).
/// The poll interval (default 1s) keeps this real-time enough for the
/// gate; no faster polling is needed.
fn v1_over_cap(&self, url: &str) -> bool {
let cap = self.p_cap();
if !self.td_admit_active() {
return false;
}
let Some(l) = self.fresh_load(url) else {
return false;
};
// Primary: real waiting-queue length.
if self.config.p_queue_reqs_cap > 0 && l.waiting_reqs > self.config.p_queue_reqs_cap {
return true;
}
// Loose token backstop (16M): catches drain/queue-metric failure.
let cap = self.config.p_queue_token_cap;
cap > 0 && l.queued_total_input_tokens > cap
}
/// T17 (修正版) shadow-path P gate: the engine's REAL waiting queue
/// length over the reqs cap (same data as v1_over_cap's primary rule).
/// Missing/stale snapshot -> not judged.
fn p_real_queue_over(&self, url: &str) -> bool {
let cap = self.config.p_queue_reqs_cap;
if cap == 0 {
return false;
}
self.fresh_load(url)
.is_some_and(|l| l.queued_total_input_tokens > cap)
self.fresh_load(url).is_some_and(|l| l.waiting_reqs > cap)
}
/// T10 v1-path candidate split: (kept, over_cap). Inactive
@@ -1829,6 +1906,16 @@ impl G9iState {
// T10: token cap skips over-cap nodes first
// (enforce); shadow mode computes all_over_cap
// without altering the pick.
// T17 (修正版): the ENGINE-REAL waiting queue is the
// primary over-cap verdict (估算管路由,真实队列管
// 准入); inactive -> always-false closure.
let real_gate_on =
self.td_admit_active() && self.config.p_queue_reqs_cap > 0;
// (type annotation forces HRTB generalization for
// the capturing closure)
let real_over: &dyn Fn(&str) -> bool = &|node: &str| {
real_gate_on && self.p_real_queue_over(node)
};
let candidates = self
.p_nodes
.read()
@@ -1844,6 +1931,7 @@ impl G9iState {
self.config.t_prefer_secs,
self.p_cap(),
self.config.td_admit == "enforce",
&real_over,
);
if out.node != holder {
metrics::counter!(METRIC_DECISIONS, "path" => "shadow_t_prefer_fallback")
@@ -1932,6 +2020,14 @@ impl G9iState {
}
}
/// T17: non-streaming requests carry no observable first-token moment;
/// record their end-to-end latency instead (observability ONLY — never
/// feeds AIMD, which degrades to local blind-state signals when TTFT
/// samples are absent).
pub fn record_nonstream_e2e(&self, seconds: f64) {
metrics::histogram!("smg_g9i_nonstream_e2e_secs").record(seconds);
}
/// Publish observed TTFT percentiles as gauges (called from the snapshot
/// poller — cheap, piggybacks an existing periodic task). Statistics
/// only: never rejects. Reference budgets (p95 10s / p99 30s) are an
@@ -2439,6 +2535,10 @@ fn parse_node_load(json: &Value) -> Result<NodeLoad, String> {
.get("drain_rate_tokens_per_s")
.and_then(|v| v.as_f64())
.unwrap_or(0.0);
node.waiting_reqs += rank
.get("num_waiting_reqs")
.and_then(|v| v.as_u64())
.unwrap_or(0);
let max_total = rank
.get("max_total_num_tokens")
.and_then(|v| v.as_u64())
@@ -2585,6 +2685,7 @@ mod tests {
{
"dp_rank": 0,
"num_running_reqs": 3,
"num_waiting_reqs": 7,
"max_running_requests": 128,
"queued_total_input_tokens": 100_000,
"drain_rate_tokens_per_s": 20_000.0,
@@ -2631,6 +2732,7 @@ mod tests {
// (128-3) + (128-5) = 248 free slots (per-rank, then summed).
assert_eq!(node.free_slots, 248);
assert_eq!(node.max_running_reqs, 256);
assert_eq!(node.waiting_reqs, 7, "T17: engine real queue parsed");
assert!(node.has_disagg);
}
@@ -3034,21 +3136,61 @@ sglang:token_usage{dp_rank=\"0\"} 0.5
);
}
/// Insufficient samples (< 20) -> the window never drifts.
/// T17 blind-state degradation: without TTFT samples the window
/// RECOVERS on local signals (pipeline under 0.8W + empty hold queue +
/// no timeouts), holds on timeouts, holds when pipeline-pinned.
#[test]
fn aimd_insufficient_samples_hold() {
fn aimd_blind_state_recovery_and_holds() {
let st = p2_state("enforce");
set_d_snapshots(&st, &[("http://d1:30001", 0, 10_000_000, true, false)]);
set_window(&st, "http://d1:30001", 32.0, 256);
// Blind + idle pipeline + empty hold queue -> slow-add recovery,
// no TTFT samples needed (非流式盲态也能爬回来).
st.aimd_tick();
assert_eq!(window_of(&st, "http://d1:30001"), 48.0, "blind recover +16");
st.aimd_tick();
assert_eq!(window_of(&st, "http://d1:30001"), 64.0, "recovery continues");
// A hold timeout this tick -> hold (someone is hurting).
st.hold_timeouts_total.store(1, Ordering::Relaxed);
st.aimd_tick();
assert_eq!(window_of(&st, "http://d1:30001"), 64.0, "timeout -> hold");
// Pipeline pinned at the window -> hold (the window IS the throttle).
book_ledger(&st, "http://d1:30001", 64);
st.aimd_tick();
assert_eq!(window_of(&st, "http://d1:30001"), 64.0, "pipeline pinned -> hold");
}
/// T17 修正版: D prealloc 队列 >0(引擎真实值)时盲态不恢复。
#[test]
fn aimd_blind_holds_while_prealloc_busy() {
let st = p2_state("enforce");
set_d_snapshots(&st, &[("http://d1:30001", 0, 10_000_000, true, false)]);
set_window(&st, "http://d1:30001", 32.0, 256);
st.aimd_tick();
assert_eq!(window_of(&st, "http://d1:30001"), 48.0, "idle+prealloc 0 -> recover");
// prealloc > 0 -> hold even with an idle pipeline.
let mut d = st.d_snapshots.write().unwrap();
d[0].1.decode_prealloc_queue_reqs = 3;
drop(d);
st.aimd_tick();
assert_eq!(window_of(&st, "http://d1:30001"), 48.0, "prealloc>0 -> hold");
}
/// SLO judgment stays preferred when streaming samples suffice — even
/// with the pipeline pinned (which would hold in blind mode).
#[test]
fn aimd_slo_preferred_over_blind_signals() {
let st = p2_state("enforce");
set_d_snapshots(&st, &[("http://d1:30001", 0, 10_000_000, true, false)]);
set_window(&st, "http://d1:30001", 100.0, 256);
feed_ttft(&st, 5, 2.0); // would be "ok" with enough samples
book_ledger(&st, "http://d1:30001", 100); // pipeline pinned at W
feed_ttft(&st, 20, 2.0); // SLO ok
st.aimd_tick();
assert_eq!(window_of(&st, "http://d1:30001"), 100.0);
// Zero samples at all: also hold.
let st2 = p2_state("enforce");
set_d_snapshots(&st2, &[("http://d1:30001", 0, 10_000_000, true, false)]);
set_window(&st2, "http://d1:30001", 100.0, 256);
st2.aimd_tick();
assert_eq!(window_of(&st2, "http://d1:30001"), 100.0);
assert_eq!(
window_of(&st, "http://d1:30001"),
116.0,
"SLO path wins over the blind pipeline-pinned hold"
);
}
/// Window admission: ledger < W admits; ledger >= W (all D) -> 429
@@ -3197,10 +3339,16 @@ sglang:token_usage{dp_rank=\"0\"} 0.5
}
fn set_load(st: &G9iState, url: &str, queued: u64, stale: bool) {
set_load_full(st, url, queued, 0, stale);
}
/// T17 修正版: waiting 参数进快照(真实队列口径测试).
fn set_load_full(st: &G9iState, url: &str, queued: u64, waiting: u64, stale: bool) {
st.snapshots.write().unwrap().insert(
normalize_url(url),
NodeLoad {
queued_total_input_tokens: queued,
waiting_reqs: waiting,
fetch_ok: true,
fetched_at: Some(if stale {
Instant::now()
@@ -3214,37 +3362,63 @@ sglang:token_usage{dp_rank=\"0\"} 0.5
);
}
/// v1 路径快照口径: only a FRESH snapshot's queued_total_input_tokens
/// can trip the cap; stale/missing snapshots are never judged; cap=0
/// disables entirely.
/// v1 路径闸门(T17 修正版): PRIMARY = 引擎真实 waiting 队列
/// (> reqs_cap); BACKSTOP = 排队 token (> 16M 宽松兜底);
/// stale/missing 快照不判定; off / 双禁 -> 惰性 (C13)。
#[test]
fn v1_over_cap_snapshot_semantics() {
let st = p2_state("enforce"); // cap default 4M
set_load(&st, "http://p1:30000", 5_000_000, false);
assert!(st.v1_over_cap("http://p1:30000"), "fresh over-cap judged");
set_load(&st, "http://p2:30000", 1_000, false);
assert!(!st.v1_over_cap("http://p2:30000"), "fresh under-cap passes");
set_load(&st, "http://p3:30000", 50_000_000, true);
assert!(!st.v1_over_cap("http://p3:30000"), "stale never judged");
let st = p2_state("enforce"); // reqs_cap default 10, token cap 16M
// waiting 超阈(token 远未超)-> 判定(真实队列口径).
set_load_full(&st, "http://p1:30000", 1_000_000, 50, false);
assert!(st.v1_over_cap("http://p1:30000"), "waiting 50 > 10 -> over");
// waiting 未超、token 也未超 -> 放.
set_load_full(&st, "http://p2:30000", 1_000_000, 3, false);
assert!(!st.v1_over_cap("http://p2:30000"), "waiting 3 <= 10 -> pass");
// waiting 低但 token 超 16M 兜底 -> 判定.
set_load_full(&st, "http://p3:30000", 20_000_000, 2, false);
assert!(st.v1_over_cap("http://p3:30000"), "token backstop binds");
// stale / missing -> 不判定.
set_load_full(&st, "http://p4:30000", 50_000_000, 500, true);
assert!(!st.v1_over_cap("http://p4:30000"), "stale never judged");
assert!(!st.v1_over_cap("http://p9:30000"), "missing never judged");
// cap=0 disables the judgment outright (C13).
// cap=0 双禁 -> 惰性 (C13).
let cfg = G9iConfig {
enabled: true,
shadow_enabled: true,
td_admit: "enforce".to_string(),
p_queue_token_cap: 0,
p_queue_reqs_cap: 0,
..Default::default()
};
let st2 = G9iState::new(cfg);
set_load(&st2, "http://p1:30000", 5_000_000, false);
assert!(!st2.v1_over_cap("http://p1:30000"), "cap=0 inert");
// td_admit=off also inert (p_cap() gates on the mode).
set_load_full(&st2, "http://p1:30000", 50_000_000, 500, false);
assert!(!st2.v1_over_cap("http://p1:30000"), "both disabled -> inert");
// td_admit=off -> 惰性.
let st3 = p2_state("off");
set_load(&st3, "http://p1:30000", 5_000_000, false);
set_load_full(&st3, "http://p1:30000", 50_000_000, 500, false);
assert!(!st3.v1_over_cap("http://p1:30000"), "td_admit=off inert");
}
/// T17 修正版 shadow 路径: pick_prefer 的 real_over 谓词(引擎真实
/// waiting 队列)在 enforce 档真实剔除候选, 超阈节点被跳过。
#[test]
fn shadow_path_real_queue_gate_filters() {
let st = p2_state("enforce");
set_load_full(&st, "http://p1:30000", 100_000, 50, false); // over
set_load_full(&st, "http://p2:30000", 100_000, 3, false); // fine
assert!(st.p_real_queue_over("http://p1:30000"));
assert!(!st.p_real_queue_over("http://p2:30000"));
assert!(!st.p_real_queue_over("http://p9:30000"), "missing not judged");
// off 档惰性.
let st_off = p2_state("off");
set_load_full(&st_off, "http://p1:30000", 100_000, 500, false);
// p_real_queue_over itself is mode-independent (callers gate);
// the shadow caller wraps it with real_gate_on. Verify the wrap:
// td_admit off -> the closure in shadow_classify_and_admit is inert.
assert!(!st_off.td_admit_active());
}
// ============ T11: atomic decision-time admission ============
fn t11_state(mode: &str, cap: u64) -> G9iState {
@@ -897,11 +897,14 @@ impl QueueModel {
///
/// `t_prefer_secs = +∞` degenerates to "always the holder" — the
/// pre-T_PREFER behavior exactly.
/// Token cap layering (T10, `token_cap` > 0, evaluated only while
/// td_admit != off): a node whose shadow-bookkept in-flight tokens +
/// this request's own_ctx exceed the cap is SKIPPED (回退优先于拒绝).
/// Cap layering (T10 token backstop + T17 engine-real queue verdict):
/// a node is over cap when its shadow-bookkept in-flight tokens + this
/// request's own_ctx exceed token_cap (T10) OR the engine's REAL
/// waiting-queue verdict says so (`real_over`, T17 — 估算管路由,
/// 真实队列管准入). Over-cap nodes are SKIPPED (回退优先于拒绝).
/// When `cap_enforce` is on the skip is real; in shadow mode the pick
/// is unchanged and only `all_over_cap` is computed.
#[allow(clippy::too_many_arguments)]
pub fn pick_prefer(
&mut self,
holder: &str,
@@ -913,17 +916,17 @@ impl QueueModel {
t_prefer_secs: f64,
token_cap: u64,
cap_enforce: bool,
real_over: &dyn Fn(&str) -> bool,
) -> PreferOutcome {
let holder_est =
self.est_ttft(Some(holder), class, own_ctx_tokens, d_snaps, est_td_enabled);
// Fast path: cap inactive -> the pure T_PREFER logic, no scans.
if token_cap == 0 {
return self.prefer_by_threshold(holder, holder_est, class, own_ctx_tokens, candidates, d_snaps, est_td_enabled, t_prefer_secs, false);
}
let filter = cap_enforce;
let holder_over_cap =
self.node_outstanding(holder).saturating_add(own_ctx_tokens) > token_cap;
if !(filter && holder_over_cap) && holder_est.total <= t_prefer_secs {
let over = |q: &QueueModel, node: &str| {
(token_cap > 0
&& q.node_outstanding(node).saturating_add(own_ctx_tokens) > token_cap)
|| real_over(node)
};
let holder_over_cap = over(self, holder);
if !(cap_enforce && holder_over_cap) && holder_est.total <= t_prefer_secs {
// Holder picked on threshold. The all-over signal: holder under
// cap settles it (false) without a scan; holder over cap in
// SHADOW mode still needs the candidate scan (the pick stays
@@ -932,9 +935,7 @@ impl QueueModel {
&& candidates
.iter()
.filter(|c| c.as_str() != holder)
.all(|c| {
self.node_outstanding(c).saturating_add(own_ctx_tokens) > token_cap
});
.all(|c| over(self, c));
return PreferOutcome {
node: holder.to_string(),
est: holder_est,
@@ -947,10 +948,9 @@ impl QueueModel {
let mut rest: Vec<(String, EstTtft)> = Vec::new();
let mut any_under_cap = !holder_over_cap;
for c in candidates.iter().filter(|c| c.as_str() != holder) {
let c_over =
self.node_outstanding(c).saturating_add(own_ctx_tokens) > token_cap;
let c_over = over(self, c);
any_under_cap = any_under_cap || !c_over;
if filter && c_over {
if cap_enforce && c_over {
continue;
}
let est = self.est_ttft(Some(c), class, own_ctx_tokens, d_snaps, est_td_enabled);
@@ -974,7 +974,7 @@ impl QueueModel {
// All over the T_PREFER threshold: the minimum-est ELIGIBLE node
// (holder participates unless cap-excluded in enforce mode).
let mut best: Option<(String, EstTtft)> = None;
if !(filter && holder_over_cap) {
if !(cap_enforce && holder_over_cap) {
best = Some((holder.to_string(), holder_est));
}
for (node, est) in rest {
@@ -998,59 +998,6 @@ impl QueueModel {
}
}
/// The pre-cap T_PREFER logic (threshold ordering only), shared by the
/// token_cap==0 fast path. `all_over_cap` is always false here.
#[allow(clippy::too_many_arguments)]
fn prefer_by_threshold(
&mut self,
holder: &str,
holder_est: EstTtft,
class: ShadowClass,
own_ctx_tokens: u64,
candidates: &[String],
d_snaps: &[DSnapshot],
est_td_enabled: bool,
t_prefer_secs: f64,
_cap_active: bool,
) -> PreferOutcome {
if holder_est.total <= t_prefer_secs {
return PreferOutcome {
node: holder.to_string(),
est: holder_est,
all_over_cap: false,
};
}
let mut rest: Vec<(String, EstTtft)> = Vec::new();
for c in candidates.iter().filter(|c| c.as_str() != holder) {
let est = self.est_ttft(Some(c), class, own_ctx_tokens, d_snaps, est_td_enabled);
rest.push((c.clone(), est));
}
rest.sort_by(|a, b| {
a.1.total
.partial_cmp(&b.1.total)
.unwrap_or(std::cmp::Ordering::Equal)
});
for (node, est) in &rest {
if est.total <= t_prefer_secs {
return PreferOutcome {
node: node.clone(),
est: *est,
all_over_cap: false,
};
}
}
let mut best = (holder.to_string(), holder_est);
for (node, est) in rest {
if est.total < best.1.total {
best = (node, est);
}
}
PreferOutcome {
node: best.0,
est: best.1,
all_over_cap: false,
}
}
/// Per-node total in-flight estimated tokens (all classes summed) —
/// the shadow bookkeeping view behind the P-queue hard cap (T10).
@@ -2694,6 +2641,7 @@ mod tests {
3.0,
0,
false,
&|_| false,
);
assert_eq!(out.node, "http://p1:30000");
assert!((out.est.total - 2.5).abs() < 1e-9);
@@ -2716,6 +2664,7 @@ mod tests {
3.0,
0,
false,
&|_| false,
);
assert_eq!(out.node, "http://p2:30000");
assert!((out.est.total - 0.5).abs() < 1e-9);
@@ -2739,6 +2688,7 @@ mod tests {
1.0,
0,
false,
&|_| false,
);
assert_eq!(out.node, "http://p2:30000", "global minimum est wins");
assert!((out.est.total - 2.5).abs() < 1e-9);
@@ -2760,6 +2710,7 @@ mod tests {
f64::INFINITY,
0,
false,
&|_| false,
);
assert_eq!(out.node, "http://p1:30000");
}
@@ -2783,6 +2734,7 @@ mod tests {
f64::INFINITY, // T_PREFER out of the way: cap is the actor here
4_000_000,
true,
&|_| false,
);
assert_eq!(out.node, "http://p2:30000", "over-cap holder skipped");
assert!(!out.all_over_cap, "p2 is under the cap");
@@ -2804,6 +2756,7 @@ mod tests {
f64::INFINITY,
4_000_000,
true,
&|_| false,
);
assert_eq!(out.node, "http://p2:30000");
}
@@ -2825,6 +2778,7 @@ mod tests {
f64::INFINITY,
4_000_000,
true,
&|_| false,
);
assert!(out.all_over_cap, "every candidate over -> 429 signal");
assert_eq!(out.node, "http://p1:30000", "holder kept for the record");
@@ -2847,6 +2801,7 @@ mod tests {
f64::INFINITY,
4_000_000,
false, // shadow: measure only
&|_| false,
);
assert!(out.all_over_cap, "the signal is computed");
assert_eq!(out.node, "http://p1:30000", "but the pick is unchanged");
@@ -2966,4 +2921,59 @@ mod tests {
let rate = q.slot_release_rate_at("d1", t0 + Duration::from_secs(100));
assert!((rate - 0.01).abs() < 1e-9, "1/100s -> 0.01 req/s floor-ish");
}
/// T17 修正版: the engine-real queue verdict (real_over predicate)
/// excludes candidates exactly like the token cap — holder skipped in
/// enforce, pick unchanged in shadow with the signal computed.
#[test]
fn pick_prefer_real_over_predicate_filters() {
let mut q = QueueModel::new();
let cands = candidates(&["http://p1:30000", "http://p2:30000"]);
// Real verdict: p1's waiting queue is over the cap.
let real_over: &dyn Fn(&str) -> bool = &|n: &str| n == "http://p1:30000";
let out = q.pick_prefer(
"http://p1:30000",
ShadowClass::Hot,
0,
&cands,
&[],
true,
f64::INFINITY, // T_PREFER out of the way
0, // token cap off: only the real verdict acts
true,
&real_over,
);
assert_eq!(out.node, "http://p2:30000", "real-over holder skipped");
assert!(!out.all_over_cap);
// Both real-over -> all_over_cap signals (holder kept for record).
let all_over: &dyn Fn(&str) -> bool = &|_| true;
let out = q.pick_prefer(
"http://p1:30000",
ShadowClass::Hot,
0,
&cands,
&[],
true,
f64::INFINITY,
0,
true,
&all_over,
);
assert!(out.all_over_cap);
// Shadow mode: pick unchanged, signal still computed.
let out = q.pick_prefer(
"http://p1:30000",
ShadowClass::Hot,
0,
&cands,
&[],
true,
f64::INFINITY,
0,
false,
&real_over,
);
assert_eq!(out.node, "http://p1:30000", "shadow: pick unchanged");
assert!(!out.all_over_cap, "p2 is not over");
}
}
@@ -1118,14 +1118,21 @@ impl PDRouter {
g9i.shadow_admit(s);
}
if context.return_logprob {
self.process_non_streaming_response(
res,
status,
context.return_logprob,
prefill_body,
shadow,
)
.await
let resp = self
.process_non_streaming_response(
res,
status,
context.return_logprob,
prefill_body,
shadow,
)
.await;
// T17: non-streaming e2e (observability only — the
// AIMD blind state reads local signals instead).
if let Some(g9i) = &self.g9i {
g9i.record_nonstream_e2e(start_time.elapsed().as_secs_f64());
}
resp
} else {
// Direct passthrough when no logprobs needed
let response_headers =
@@ -1138,6 +1145,10 @@ impl PDRouter {
let mut response = Response::new(Body::from(decode_body));
*response.status_mut() = status;
*response.headers_mut() = response_headers;
// T17: non-streaming e2e (observability only).
if let Some(g9i) = &self.g9i {
g9i.record_nonstream_e2e(start_time.elapsed().as_secs_f64());
}
response
}
Err(e) => {