gateway g9i: AIMD blind-state recovery on real engine signals + real-queue P gate + nonstream e2e metric (P2 T17)
This commit is contained in:
@@ -304,10 +304,14 @@ struct CliArgs {
|
||||
#[arg(long, default_value_t = 2.0, help_heading = "G9i Pool Admission")]
|
||||
g9i_t_prefer_secs: f64,
|
||||
|
||||
/// Hard P-queue token cap (physical backstop): skip prefill nodes whose in-flight tokens exceed this; 429 p_queue_cap only when ALL are over (0 = disabled; follows --g9i-td-admit mode)
|
||||
#[arg(long, default_value_t = 4_000_000, help_heading = "G9i Pool Admission")]
|
||||
/// Hard P-queue token cap (physical backstop; effective cap = min(this, drain_ewma x budget)) — 429 p_queue_cap only when ALL nodes over (0 = disabled; follows --g9i-td-admit mode)
|
||||
#[arg(long, default_value_t = 16_000_000, help_heading = "G9i Pool Admission")]
|
||||
g9i_p_queue_token_cap: u64,
|
||||
|
||||
/// P-queue real-queue gate: skip prefill nodes whose engine-reported waiting queue (num_waiting_reqs) exceeds this (0 = disabled; follows --g9i-td-admit mode)
|
||||
#[arg(long, default_value_t = 10, help_heading = "G9i Pool Admission")]
|
||||
g9i_p_queue_reqs_cap: u64,
|
||||
|
||||
/// Admission hold queue: park holdable rejections (d_window_full / p_queue_cap) up to this many seconds instead of an immediate 429 (0 = disabled; enforce mode only; hold time counts into TTFT)
|
||||
#[arg(long, default_value_t = 1.0, help_heading = "G9i Pool Admission")]
|
||||
g9i_admit_hold_secs: f64,
|
||||
@@ -1201,6 +1205,7 @@ impl CliArgs {
|
||||
td_gate_secs: self.g9i_td_gate_secs,
|
||||
t_prefer_secs: self.g9i_t_prefer_secs,
|
||||
p_queue_token_cap: self.g9i_p_queue_token_cap,
|
||||
p_queue_reqs_cap: self.g9i_p_queue_reqs_cap,
|
||||
admit_hold_secs: self.g9i_admit_hold_secs,
|
||||
admit_hold_max: self.g9i_admit_hold_max,
|
||||
};
|
||||
|
||||
@@ -45,7 +45,7 @@
|
||||
use std::{
|
||||
collections::{HashMap, VecDeque},
|
||||
sync::{
|
||||
atomic::{AtomicBool, Ordering},
|
||||
atomic::{AtomicBool, AtomicU64, Ordering},
|
||||
Arc, RwLock,
|
||||
},
|
||||
time::{Duration, Instant},
|
||||
@@ -197,12 +197,21 @@ pub struct G9iConfig {
|
||||
/// a prefill node whose shadow-bookkept in-flight tokens + the
|
||||
/// request's own ctx exceed this is skipped (candidate fallback first);
|
||||
/// only when EVERY prefill node is over the cap does the request 429
|
||||
/// (p_queue_cap). Token-based, not request-count (昨天 p1 积压 64M
|
||||
/// tokens ≈ 450s of work). Default 4M ≈ 30s of queue at the measured
|
||||
/// 141k tok/s drain. 0 = disabled. Mode follows td_admit
|
||||
/// (off/shadow/enforce); off or 0 ≡ current behavior (C13).
|
||||
/// (p_queue_cap). T17 修正版: this is now the loose BACKSTOP — the
|
||||
/// primary口径 is the engine's REAL waiting-queue length
|
||||
/// (p_queue_reqs_cap below; 估算不可信). Raised to 16M (the T10 4M
|
||||
/// choked all-cold 200k-ctx traffic at ~20-50 requests/node). 0 =
|
||||
/// disabled. Mode follows td_admit; off or 0 ≡ current behavior (C13).
|
||||
#[serde(default = "default_p_queue_token_cap")]
|
||||
pub p_queue_token_cap: u64,
|
||||
/// T17 (修正版) primary P-queue gate: a node whose ENGINE-REPORTED
|
||||
/// waiting queue (num_waiting_reqs, /v1/loads core) exceeds this is
|
||||
/// excluded from candidates; all over -> hold/429 (p_queue_cap).
|
||||
/// Real measurement, not an estimate. Default 10 (待实测调).
|
||||
/// Stale/missing snapshots are never judged (不冤枉); all-missing ->
|
||||
/// C13 degrade. 0 = disabled.
|
||||
#[serde(default = "default_p_queue_reqs_cap")]
|
||||
pub p_queue_reqs_cap: u64,
|
||||
/// T15 admission hold queue: holdable rejections (d_window_full /
|
||||
/// p_queue_cap) park the request up to this many seconds — TOTAL budget
|
||||
/// across hold retries — waiting for room before the 429 fires
|
||||
@@ -267,7 +276,10 @@ fn default_t_prefer_secs() -> f64 {
|
||||
2.0
|
||||
}
|
||||
fn default_p_queue_token_cap() -> u64 {
|
||||
4_000_000
|
||||
16_000_000
|
||||
}
|
||||
fn default_p_queue_reqs_cap() -> u64 {
|
||||
10
|
||||
}
|
||||
fn default_admit_hold_secs() -> f64 {
|
||||
1.0
|
||||
@@ -301,6 +313,7 @@ impl Default for G9iConfig {
|
||||
td_gate_secs: default_td_gate_secs(),
|
||||
t_prefer_secs: default_t_prefer_secs(),
|
||||
p_queue_token_cap: default_p_queue_token_cap(),
|
||||
p_queue_reqs_cap: default_p_queue_reqs_cap(),
|
||||
admit_hold_secs: default_admit_hold_secs(),
|
||||
admit_hold_max: default_admit_hold_max(),
|
||||
}
|
||||
@@ -349,6 +362,10 @@ pub struct NodeLoad {
|
||||
pub queued_total_input_tokens: u64,
|
||||
/// Sum of drain_rate_tokens_per_s across dp ranks.
|
||||
pub drain_rate_tokens_per_s: f64,
|
||||
/// Sum of num_waiting_reqs across dp ranks — the engine's REAL queue
|
||||
/// length, the primary P-admission口径 (T17 修正版: 估算管路由,
|
||||
/// 真实队列管准入).
|
||||
pub waiting_reqs: u64,
|
||||
/// Sum of disaggregation.decode_prealloc_queue_reqs across dp ranks.
|
||||
pub decode_prealloc_queue_reqs: u64,
|
||||
/// Sum of disaggregation.decode_transfer_queue_reqs across dp ranks.
|
||||
@@ -471,6 +488,11 @@ pub struct G9iState {
|
||||
/// only used when a holdable rejection fires in enforce mode with a
|
||||
/// nonzero hold budget.
|
||||
hold: HoldQueue,
|
||||
/// T17: cumulative hold timeouts (blind-state AIMD input — a timeout
|
||||
/// this tick means someone is hurting even without TTFT samples).
|
||||
hold_timeouts_total: AtomicU64,
|
||||
/// Timeouts seen at the previous AIMD tick (for the diff).
|
||||
last_tick_timeouts: AtomicU64,
|
||||
}
|
||||
|
||||
/// T14 AIMD congestion-window state for one decode node.
|
||||
@@ -680,6 +702,8 @@ impl G9iState {
|
||||
d_windows: RwLock::new(HashMap::new()),
|
||||
last_aimd: RwLock::new(Instant::now()),
|
||||
hold: HoldQueue::default(),
|
||||
hold_timeouts_total: AtomicU64::new(0),
|
||||
last_tick_timeouts: AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1068,9 +1092,19 @@ impl G9iState {
|
||||
|
||||
fn aimd_tick(&self) {
|
||||
let (count, p50, p95, p99) = self.ttft_slo_view();
|
||||
// T17 (修正版) blind-state inputs (non-streaming traffic leaves no
|
||||
// TTFT samples): hold timeouts this tick, and per-node REAL engine
|
||||
// signals — D prealloc queue (disagg 段真实值) + pipeline depth.
|
||||
let timeouts_now = self.hold_timeouts_total.load(Ordering::Relaxed);
|
||||
let timed_out = {
|
||||
let prev = self.last_tick_timeouts.swap(timeouts_now, Ordering::Relaxed);
|
||||
timeouts_now > prev
|
||||
};
|
||||
let d_snaps = self.current_d_snapshots();
|
||||
let Ok(mut windows) = self.d_windows.write() else {
|
||||
return;
|
||||
};
|
||||
let q = self.shadow_queue.as_ref().and_then(|l| l.read().ok());
|
||||
let mut grew = false;
|
||||
for (node, e) in windows.iter_mut() {
|
||||
if e.max_running_total == 0 {
|
||||
@@ -1081,19 +1115,37 @@ impl G9iState {
|
||||
let w_min = WINDOW_MIN;
|
||||
let w_max = WINDOW_MAX;
|
||||
e.w = e.w.clamp(w_min, w_max);
|
||||
if count < AIMD_MIN_SAMPLES {
|
||||
continue; // 样本不足/无新流量: hold
|
||||
}
|
||||
let old = e.w;
|
||||
let why: &str;
|
||||
if p95 > SLO_P95_SECS || p99 > SLO_P99_SECS {
|
||||
e.w = (e.w * AIMD_CUT).max(w_min);
|
||||
why = "slo_violation";
|
||||
} else if p50 < SLO_P50_SECS && p95 < SLO_P95_SECS {
|
||||
e.w = (e.w + AIMD_ADD).min(w_max);
|
||||
why = "slo_ok";
|
||||
if count >= AIMD_MIN_SAMPLES {
|
||||
// SLO 判定优先(流式样本充足时照旧).
|
||||
if p95 > SLO_P95_SECS || p99 > SLO_P99_SECS {
|
||||
e.w = (e.w * AIMD_CUT).max(w_min);
|
||||
why = "slo_violation";
|
||||
} else if p50 < SLO_P50_SECS && p95 < SLO_P95_SECS {
|
||||
e.w = (e.w + AIMD_ADD).min(w_max);
|
||||
why = "slo_ok";
|
||||
} else {
|
||||
why = "hold";
|
||||
}
|
||||
} else {
|
||||
why = "hold";
|
||||
// T17 (修正版) 盲态降级(非流式流量没有 TTFT 样本——窗口
|
||||
// 必须不靠它们恢复): 本拍有持有超时 -> hold(有人在受伤);
|
||||
// D prealloc 队列清空(==0) 且管线宽裕 (<0.8W) -> 慢加恢复;
|
||||
// prealloc 持续 >0 或管线顶满 -> hold(窗口本身已在限流)。
|
||||
let pipe = q.as_ref().map(|q| q.d_pipeline(node)).unwrap_or(0) as f64;
|
||||
let prealloc_busy = d_snaps
|
||||
.iter()
|
||||
.find(|s| &s.node == node)
|
||||
.is_some_and(|s| s.prealloc_queue_reqs > 0);
|
||||
if timed_out {
|
||||
why = "blind_hold_timeout";
|
||||
} else if !prealloc_busy && pipe < 0.8 * e.w {
|
||||
e.w = (e.w + AIMD_ADD).min(w_max);
|
||||
why = "blind_recover";
|
||||
} else {
|
||||
why = "blind_hold_full";
|
||||
}
|
||||
}
|
||||
if e.w != old {
|
||||
info!(
|
||||
@@ -1278,6 +1330,8 @@ impl G9iState {
|
||||
self.hold.remove(id, body_bytes);
|
||||
guard.disarm();
|
||||
let held = parked_at.elapsed().as_secs_f64();
|
||||
// T17: AIMD blind-state input.
|
||||
self.hold_timeouts_total.fetch_add(1, Ordering::Relaxed);
|
||||
metrics::counter!(METRIC_ADMIT_HELD, "outcome" => "timeout", "class" => class_label)
|
||||
.increment(1);
|
||||
metrics::histogram!(METRIC_ADMIT_HOLD_SECS).record(held);
|
||||
@@ -1513,17 +1567,40 @@ impl G9iState {
|
||||
})
|
||||
}
|
||||
|
||||
/// v1-path cap judgment (T10): over cap only when a FRESH /v1/loads
|
||||
/// snapshot's queued_total_input_tokens says so. Missing or stale
|
||||
/// snapshots are NOT judged (fresh_load already enforces the TTL) — a
|
||||
/// blind node must never trip the physical backstop (避免误伤).
|
||||
/// v1-path cap judgment (T17 修正版): over cap when a FRESH /v1/loads
|
||||
/// snapshot says so — PRIMARY: engine-reported waiting queue
|
||||
/// (num_waiting_reqs > p_queue_reqs_cap); BACKSTOP: queued tokens over
|
||||
/// p_queue_token_cap (16M, loose). Missing or stale snapshots are NOT
|
||||
/// judged (fresh_load already enforces the TTL) — a blind node must
|
||||
/// never trip the physical backstop (不冤枉), and all-blind means the
|
||||
/// gate stays open (C13).
|
||||
/// The poll interval (default 1s) keeps this real-time enough for the
|
||||
/// gate; no faster polling is needed.
|
||||
fn v1_over_cap(&self, url: &str) -> bool {
|
||||
let cap = self.p_cap();
|
||||
if !self.td_admit_active() {
|
||||
return false;
|
||||
}
|
||||
let Some(l) = self.fresh_load(url) else {
|
||||
return false;
|
||||
};
|
||||
// Primary: real waiting-queue length.
|
||||
if self.config.p_queue_reqs_cap > 0 && l.waiting_reqs > self.config.p_queue_reqs_cap {
|
||||
return true;
|
||||
}
|
||||
// Loose token backstop (16M): catches drain/queue-metric failure.
|
||||
let cap = self.config.p_queue_token_cap;
|
||||
cap > 0 && l.queued_total_input_tokens > cap
|
||||
}
|
||||
|
||||
/// T17 (修正版) shadow-path P gate: the engine's REAL waiting queue
|
||||
/// length over the reqs cap (same data as v1_over_cap's primary rule).
|
||||
/// Missing/stale snapshot -> not judged.
|
||||
fn p_real_queue_over(&self, url: &str) -> bool {
|
||||
let cap = self.config.p_queue_reqs_cap;
|
||||
if cap == 0 {
|
||||
return false;
|
||||
}
|
||||
self.fresh_load(url)
|
||||
.is_some_and(|l| l.queued_total_input_tokens > cap)
|
||||
self.fresh_load(url).is_some_and(|l| l.waiting_reqs > cap)
|
||||
}
|
||||
|
||||
/// T10 v1-path candidate split: (kept, over_cap). Inactive
|
||||
@@ -1829,6 +1906,16 @@ impl G9iState {
|
||||
// T10: token cap skips over-cap nodes first
|
||||
// (enforce); shadow mode computes all_over_cap
|
||||
// without altering the pick.
|
||||
// T17 (修正版): the ENGINE-REAL waiting queue is the
|
||||
// primary over-cap verdict (估算管路由,真实队列管
|
||||
// 准入); inactive -> always-false closure.
|
||||
let real_gate_on =
|
||||
self.td_admit_active() && self.config.p_queue_reqs_cap > 0;
|
||||
// (type annotation forces HRTB generalization for
|
||||
// the capturing closure)
|
||||
let real_over: &dyn Fn(&str) -> bool = &|node: &str| {
|
||||
real_gate_on && self.p_real_queue_over(node)
|
||||
};
|
||||
let candidates = self
|
||||
.p_nodes
|
||||
.read()
|
||||
@@ -1844,6 +1931,7 @@ impl G9iState {
|
||||
self.config.t_prefer_secs,
|
||||
self.p_cap(),
|
||||
self.config.td_admit == "enforce",
|
||||
&real_over,
|
||||
);
|
||||
if out.node != holder {
|
||||
metrics::counter!(METRIC_DECISIONS, "path" => "shadow_t_prefer_fallback")
|
||||
@@ -1932,6 +2020,14 @@ impl G9iState {
|
||||
}
|
||||
}
|
||||
|
||||
/// T17: non-streaming requests carry no observable first-token moment;
|
||||
/// record their end-to-end latency instead (observability ONLY — never
|
||||
/// feeds AIMD, which degrades to local blind-state signals when TTFT
|
||||
/// samples are absent).
|
||||
pub fn record_nonstream_e2e(&self, seconds: f64) {
|
||||
metrics::histogram!("smg_g9i_nonstream_e2e_secs").record(seconds);
|
||||
}
|
||||
|
||||
/// Publish observed TTFT percentiles as gauges (called from the snapshot
|
||||
/// poller — cheap, piggybacks an existing periodic task). Statistics
|
||||
/// only: never rejects. Reference budgets (p95 10s / p99 30s) are an
|
||||
@@ -2439,6 +2535,10 @@ fn parse_node_load(json: &Value) -> Result<NodeLoad, String> {
|
||||
.get("drain_rate_tokens_per_s")
|
||||
.and_then(|v| v.as_f64())
|
||||
.unwrap_or(0.0);
|
||||
node.waiting_reqs += rank
|
||||
.get("num_waiting_reqs")
|
||||
.and_then(|v| v.as_u64())
|
||||
.unwrap_or(0);
|
||||
let max_total = rank
|
||||
.get("max_total_num_tokens")
|
||||
.and_then(|v| v.as_u64())
|
||||
@@ -2585,6 +2685,7 @@ mod tests {
|
||||
{
|
||||
"dp_rank": 0,
|
||||
"num_running_reqs": 3,
|
||||
"num_waiting_reqs": 7,
|
||||
"max_running_requests": 128,
|
||||
"queued_total_input_tokens": 100_000,
|
||||
"drain_rate_tokens_per_s": 20_000.0,
|
||||
@@ -2631,6 +2732,7 @@ mod tests {
|
||||
// (128-3) + (128-5) = 248 free slots (per-rank, then summed).
|
||||
assert_eq!(node.free_slots, 248);
|
||||
assert_eq!(node.max_running_reqs, 256);
|
||||
assert_eq!(node.waiting_reqs, 7, "T17: engine real queue parsed");
|
||||
assert!(node.has_disagg);
|
||||
}
|
||||
|
||||
@@ -3034,21 +3136,61 @@ sglang:token_usage{dp_rank=\"0\"} 0.5
|
||||
);
|
||||
}
|
||||
|
||||
/// Insufficient samples (< 20) -> the window never drifts.
|
||||
/// T17 blind-state degradation: without TTFT samples the window
|
||||
/// RECOVERS on local signals (pipeline under 0.8W + empty hold queue +
|
||||
/// no timeouts), holds on timeouts, holds when pipeline-pinned.
|
||||
#[test]
|
||||
fn aimd_insufficient_samples_hold() {
|
||||
fn aimd_blind_state_recovery_and_holds() {
|
||||
let st = p2_state("enforce");
|
||||
set_d_snapshots(&st, &[("http://d1:30001", 0, 10_000_000, true, false)]);
|
||||
set_window(&st, "http://d1:30001", 32.0, 256);
|
||||
// Blind + idle pipeline + empty hold queue -> slow-add recovery,
|
||||
// no TTFT samples needed (非流式盲态也能爬回来).
|
||||
st.aimd_tick();
|
||||
assert_eq!(window_of(&st, "http://d1:30001"), 48.0, "blind recover +16");
|
||||
st.aimd_tick();
|
||||
assert_eq!(window_of(&st, "http://d1:30001"), 64.0, "recovery continues");
|
||||
// A hold timeout this tick -> hold (someone is hurting).
|
||||
st.hold_timeouts_total.store(1, Ordering::Relaxed);
|
||||
st.aimd_tick();
|
||||
assert_eq!(window_of(&st, "http://d1:30001"), 64.0, "timeout -> hold");
|
||||
// Pipeline pinned at the window -> hold (the window IS the throttle).
|
||||
book_ledger(&st, "http://d1:30001", 64);
|
||||
st.aimd_tick();
|
||||
assert_eq!(window_of(&st, "http://d1:30001"), 64.0, "pipeline pinned -> hold");
|
||||
}
|
||||
|
||||
/// T17 修正版: D prealloc 队列 >0(引擎真实值)时盲态不恢复。
|
||||
#[test]
|
||||
fn aimd_blind_holds_while_prealloc_busy() {
|
||||
let st = p2_state("enforce");
|
||||
set_d_snapshots(&st, &[("http://d1:30001", 0, 10_000_000, true, false)]);
|
||||
set_window(&st, "http://d1:30001", 32.0, 256);
|
||||
st.aimd_tick();
|
||||
assert_eq!(window_of(&st, "http://d1:30001"), 48.0, "idle+prealloc 0 -> recover");
|
||||
// prealloc > 0 -> hold even with an idle pipeline.
|
||||
let mut d = st.d_snapshots.write().unwrap();
|
||||
d[0].1.decode_prealloc_queue_reqs = 3;
|
||||
drop(d);
|
||||
st.aimd_tick();
|
||||
assert_eq!(window_of(&st, "http://d1:30001"), 48.0, "prealloc>0 -> hold");
|
||||
}
|
||||
|
||||
/// SLO judgment stays preferred when streaming samples suffice — even
|
||||
/// with the pipeline pinned (which would hold in blind mode).
|
||||
#[test]
|
||||
fn aimd_slo_preferred_over_blind_signals() {
|
||||
let st = p2_state("enforce");
|
||||
set_d_snapshots(&st, &[("http://d1:30001", 0, 10_000_000, true, false)]);
|
||||
set_window(&st, "http://d1:30001", 100.0, 256);
|
||||
feed_ttft(&st, 5, 2.0); // would be "ok" with enough samples
|
||||
book_ledger(&st, "http://d1:30001", 100); // pipeline pinned at W
|
||||
feed_ttft(&st, 20, 2.0); // SLO ok
|
||||
st.aimd_tick();
|
||||
assert_eq!(window_of(&st, "http://d1:30001"), 100.0);
|
||||
// Zero samples at all: also hold.
|
||||
let st2 = p2_state("enforce");
|
||||
set_d_snapshots(&st2, &[("http://d1:30001", 0, 10_000_000, true, false)]);
|
||||
set_window(&st2, "http://d1:30001", 100.0, 256);
|
||||
st2.aimd_tick();
|
||||
assert_eq!(window_of(&st2, "http://d1:30001"), 100.0);
|
||||
assert_eq!(
|
||||
window_of(&st, "http://d1:30001"),
|
||||
116.0,
|
||||
"SLO path wins over the blind pipeline-pinned hold"
|
||||
);
|
||||
}
|
||||
|
||||
/// Window admission: ledger < W admits; ledger >= W (all D) -> 429
|
||||
@@ -3197,10 +3339,16 @@ sglang:token_usage{dp_rank=\"0\"} 0.5
|
||||
}
|
||||
|
||||
fn set_load(st: &G9iState, url: &str, queued: u64, stale: bool) {
|
||||
set_load_full(st, url, queued, 0, stale);
|
||||
}
|
||||
|
||||
/// T17 修正版: waiting 参数进快照(真实队列口径测试).
|
||||
fn set_load_full(st: &G9iState, url: &str, queued: u64, waiting: u64, stale: bool) {
|
||||
st.snapshots.write().unwrap().insert(
|
||||
normalize_url(url),
|
||||
NodeLoad {
|
||||
queued_total_input_tokens: queued,
|
||||
waiting_reqs: waiting,
|
||||
fetch_ok: true,
|
||||
fetched_at: Some(if stale {
|
||||
Instant::now()
|
||||
@@ -3214,37 +3362,63 @@ sglang:token_usage{dp_rank=\"0\"} 0.5
|
||||
);
|
||||
}
|
||||
|
||||
/// v1 路径快照口径: only a FRESH snapshot's queued_total_input_tokens
|
||||
/// can trip the cap; stale/missing snapshots are never judged; cap=0
|
||||
/// disables entirely.
|
||||
/// v1 路径闸门(T17 修正版): PRIMARY = 引擎真实 waiting 队列
|
||||
/// (> reqs_cap); BACKSTOP = 排队 token (> 16M 宽松兜底);
|
||||
/// stale/missing 快照不判定; off / 双禁 -> 惰性 (C13)。
|
||||
#[test]
|
||||
fn v1_over_cap_snapshot_semantics() {
|
||||
let st = p2_state("enforce"); // cap default 4M
|
||||
set_load(&st, "http://p1:30000", 5_000_000, false);
|
||||
assert!(st.v1_over_cap("http://p1:30000"), "fresh over-cap judged");
|
||||
set_load(&st, "http://p2:30000", 1_000, false);
|
||||
assert!(!st.v1_over_cap("http://p2:30000"), "fresh under-cap passes");
|
||||
set_load(&st, "http://p3:30000", 50_000_000, true);
|
||||
assert!(!st.v1_over_cap("http://p3:30000"), "stale never judged");
|
||||
let st = p2_state("enforce"); // reqs_cap default 10, token cap 16M
|
||||
// waiting 超阈(token 远未超)-> 判定(真实队列口径).
|
||||
set_load_full(&st, "http://p1:30000", 1_000_000, 50, false);
|
||||
assert!(st.v1_over_cap("http://p1:30000"), "waiting 50 > 10 -> over");
|
||||
// waiting 未超、token 也未超 -> 放.
|
||||
set_load_full(&st, "http://p2:30000", 1_000_000, 3, false);
|
||||
assert!(!st.v1_over_cap("http://p2:30000"), "waiting 3 <= 10 -> pass");
|
||||
// waiting 低但 token 超 16M 兜底 -> 判定.
|
||||
set_load_full(&st, "http://p3:30000", 20_000_000, 2, false);
|
||||
assert!(st.v1_over_cap("http://p3:30000"), "token backstop binds");
|
||||
// stale / missing -> 不判定.
|
||||
set_load_full(&st, "http://p4:30000", 50_000_000, 500, true);
|
||||
assert!(!st.v1_over_cap("http://p4:30000"), "stale never judged");
|
||||
assert!(!st.v1_over_cap("http://p9:30000"), "missing never judged");
|
||||
|
||||
// cap=0 disables the judgment outright (C13).
|
||||
// cap=0 双禁 -> 惰性 (C13).
|
||||
let cfg = G9iConfig {
|
||||
enabled: true,
|
||||
shadow_enabled: true,
|
||||
td_admit: "enforce".to_string(),
|
||||
p_queue_token_cap: 0,
|
||||
p_queue_reqs_cap: 0,
|
||||
..Default::default()
|
||||
};
|
||||
let st2 = G9iState::new(cfg);
|
||||
set_load(&st2, "http://p1:30000", 5_000_000, false);
|
||||
assert!(!st2.v1_over_cap("http://p1:30000"), "cap=0 inert");
|
||||
// td_admit=off also inert (p_cap() gates on the mode).
|
||||
set_load_full(&st2, "http://p1:30000", 50_000_000, 500, false);
|
||||
assert!(!st2.v1_over_cap("http://p1:30000"), "both disabled -> inert");
|
||||
// td_admit=off -> 惰性.
|
||||
let st3 = p2_state("off");
|
||||
set_load(&st3, "http://p1:30000", 5_000_000, false);
|
||||
set_load_full(&st3, "http://p1:30000", 50_000_000, 500, false);
|
||||
assert!(!st3.v1_over_cap("http://p1:30000"), "td_admit=off inert");
|
||||
}
|
||||
|
||||
/// T17 修正版 shadow 路径: pick_prefer 的 real_over 谓词(引擎真实
|
||||
/// waiting 队列)在 enforce 档真实剔除候选, 超阈节点被跳过。
|
||||
#[test]
|
||||
fn shadow_path_real_queue_gate_filters() {
|
||||
let st = p2_state("enforce");
|
||||
set_load_full(&st, "http://p1:30000", 100_000, 50, false); // over
|
||||
set_load_full(&st, "http://p2:30000", 100_000, 3, false); // fine
|
||||
assert!(st.p_real_queue_over("http://p1:30000"));
|
||||
assert!(!st.p_real_queue_over("http://p2:30000"));
|
||||
assert!(!st.p_real_queue_over("http://p9:30000"), "missing not judged");
|
||||
// off 档惰性.
|
||||
let st_off = p2_state("off");
|
||||
set_load_full(&st_off, "http://p1:30000", 100_000, 500, false);
|
||||
// p_real_queue_over itself is mode-independent (callers gate);
|
||||
// the shadow caller wraps it with real_gate_on. Verify the wrap:
|
||||
// td_admit off -> the closure in shadow_classify_and_admit is inert.
|
||||
assert!(!st_off.td_admit_active());
|
||||
}
|
||||
|
||||
// ============ T11: atomic decision-time admission ============
|
||||
|
||||
fn t11_state(mode: &str, cap: u64) -> G9iState {
|
||||
|
||||
@@ -897,11 +897,14 @@ impl QueueModel {
|
||||
///
|
||||
/// `t_prefer_secs = +∞` degenerates to "always the holder" — the
|
||||
/// pre-T_PREFER behavior exactly.
|
||||
/// Token cap layering (T10, `token_cap` > 0, evaluated only while
|
||||
/// td_admit != off): a node whose shadow-bookkept in-flight tokens +
|
||||
/// this request's own_ctx exceed the cap is SKIPPED (回退优先于拒绝).
|
||||
/// Cap layering (T10 token backstop + T17 engine-real queue verdict):
|
||||
/// a node is over cap when its shadow-bookkept in-flight tokens + this
|
||||
/// request's own_ctx exceed token_cap (T10) OR the engine's REAL
|
||||
/// waiting-queue verdict says so (`real_over`, T17 — 估算管路由,
|
||||
/// 真实队列管准入). Over-cap nodes are SKIPPED (回退优先于拒绝).
|
||||
/// When `cap_enforce` is on the skip is real; in shadow mode the pick
|
||||
/// is unchanged and only `all_over_cap` is computed.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn pick_prefer(
|
||||
&mut self,
|
||||
holder: &str,
|
||||
@@ -913,17 +916,17 @@ impl QueueModel {
|
||||
t_prefer_secs: f64,
|
||||
token_cap: u64,
|
||||
cap_enforce: bool,
|
||||
real_over: &dyn Fn(&str) -> bool,
|
||||
) -> PreferOutcome {
|
||||
let holder_est =
|
||||
self.est_ttft(Some(holder), class, own_ctx_tokens, d_snaps, est_td_enabled);
|
||||
// Fast path: cap inactive -> the pure T_PREFER logic, no scans.
|
||||
if token_cap == 0 {
|
||||
return self.prefer_by_threshold(holder, holder_est, class, own_ctx_tokens, candidates, d_snaps, est_td_enabled, t_prefer_secs, false);
|
||||
}
|
||||
let filter = cap_enforce;
|
||||
let holder_over_cap =
|
||||
self.node_outstanding(holder).saturating_add(own_ctx_tokens) > token_cap;
|
||||
if !(filter && holder_over_cap) && holder_est.total <= t_prefer_secs {
|
||||
let over = |q: &QueueModel, node: &str| {
|
||||
(token_cap > 0
|
||||
&& q.node_outstanding(node).saturating_add(own_ctx_tokens) > token_cap)
|
||||
|| real_over(node)
|
||||
};
|
||||
let holder_over_cap = over(self, holder);
|
||||
if !(cap_enforce && holder_over_cap) && holder_est.total <= t_prefer_secs {
|
||||
// Holder picked on threshold. The all-over signal: holder under
|
||||
// cap settles it (false) without a scan; holder over cap in
|
||||
// SHADOW mode still needs the candidate scan (the pick stays
|
||||
@@ -932,9 +935,7 @@ impl QueueModel {
|
||||
&& candidates
|
||||
.iter()
|
||||
.filter(|c| c.as_str() != holder)
|
||||
.all(|c| {
|
||||
self.node_outstanding(c).saturating_add(own_ctx_tokens) > token_cap
|
||||
});
|
||||
.all(|c| over(self, c));
|
||||
return PreferOutcome {
|
||||
node: holder.to_string(),
|
||||
est: holder_est,
|
||||
@@ -947,10 +948,9 @@ impl QueueModel {
|
||||
let mut rest: Vec<(String, EstTtft)> = Vec::new();
|
||||
let mut any_under_cap = !holder_over_cap;
|
||||
for c in candidates.iter().filter(|c| c.as_str() != holder) {
|
||||
let c_over =
|
||||
self.node_outstanding(c).saturating_add(own_ctx_tokens) > token_cap;
|
||||
let c_over = over(self, c);
|
||||
any_under_cap = any_under_cap || !c_over;
|
||||
if filter && c_over {
|
||||
if cap_enforce && c_over {
|
||||
continue;
|
||||
}
|
||||
let est = self.est_ttft(Some(c), class, own_ctx_tokens, d_snaps, est_td_enabled);
|
||||
@@ -974,7 +974,7 @@ impl QueueModel {
|
||||
// All over the T_PREFER threshold: the minimum-est ELIGIBLE node
|
||||
// (holder participates unless cap-excluded in enforce mode).
|
||||
let mut best: Option<(String, EstTtft)> = None;
|
||||
if !(filter && holder_over_cap) {
|
||||
if !(cap_enforce && holder_over_cap) {
|
||||
best = Some((holder.to_string(), holder_est));
|
||||
}
|
||||
for (node, est) in rest {
|
||||
@@ -998,59 +998,6 @@ impl QueueModel {
|
||||
}
|
||||
}
|
||||
|
||||
/// The pre-cap T_PREFER logic (threshold ordering only), shared by the
|
||||
/// token_cap==0 fast path. `all_over_cap` is always false here.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn prefer_by_threshold(
|
||||
&mut self,
|
||||
holder: &str,
|
||||
holder_est: EstTtft,
|
||||
class: ShadowClass,
|
||||
own_ctx_tokens: u64,
|
||||
candidates: &[String],
|
||||
d_snaps: &[DSnapshot],
|
||||
est_td_enabled: bool,
|
||||
t_prefer_secs: f64,
|
||||
_cap_active: bool,
|
||||
) -> PreferOutcome {
|
||||
if holder_est.total <= t_prefer_secs {
|
||||
return PreferOutcome {
|
||||
node: holder.to_string(),
|
||||
est: holder_est,
|
||||
all_over_cap: false,
|
||||
};
|
||||
}
|
||||
let mut rest: Vec<(String, EstTtft)> = Vec::new();
|
||||
for c in candidates.iter().filter(|c| c.as_str() != holder) {
|
||||
let est = self.est_ttft(Some(c), class, own_ctx_tokens, d_snaps, est_td_enabled);
|
||||
rest.push((c.clone(), est));
|
||||
}
|
||||
rest.sort_by(|a, b| {
|
||||
a.1.total
|
||||
.partial_cmp(&b.1.total)
|
||||
.unwrap_or(std::cmp::Ordering::Equal)
|
||||
});
|
||||
for (node, est) in &rest {
|
||||
if est.total <= t_prefer_secs {
|
||||
return PreferOutcome {
|
||||
node: node.clone(),
|
||||
est: *est,
|
||||
all_over_cap: false,
|
||||
};
|
||||
}
|
||||
}
|
||||
let mut best = (holder.to_string(), holder_est);
|
||||
for (node, est) in rest {
|
||||
if est.total < best.1.total {
|
||||
best = (node, est);
|
||||
}
|
||||
}
|
||||
PreferOutcome {
|
||||
node: best.0,
|
||||
est: best.1,
|
||||
all_over_cap: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-node total in-flight estimated tokens (all classes summed) —
|
||||
/// the shadow bookkeeping view behind the P-queue hard cap (T10).
|
||||
@@ -2694,6 +2641,7 @@ mod tests {
|
||||
3.0,
|
||||
0,
|
||||
false,
|
||||
&|_| false,
|
||||
);
|
||||
assert_eq!(out.node, "http://p1:30000");
|
||||
assert!((out.est.total - 2.5).abs() < 1e-9);
|
||||
@@ -2716,6 +2664,7 @@ mod tests {
|
||||
3.0,
|
||||
0,
|
||||
false,
|
||||
&|_| false,
|
||||
);
|
||||
assert_eq!(out.node, "http://p2:30000");
|
||||
assert!((out.est.total - 0.5).abs() < 1e-9);
|
||||
@@ -2739,6 +2688,7 @@ mod tests {
|
||||
1.0,
|
||||
0,
|
||||
false,
|
||||
&|_| false,
|
||||
);
|
||||
assert_eq!(out.node, "http://p2:30000", "global minimum est wins");
|
||||
assert!((out.est.total - 2.5).abs() < 1e-9);
|
||||
@@ -2760,6 +2710,7 @@ mod tests {
|
||||
f64::INFINITY,
|
||||
0,
|
||||
false,
|
||||
&|_| false,
|
||||
);
|
||||
assert_eq!(out.node, "http://p1:30000");
|
||||
}
|
||||
@@ -2783,6 +2734,7 @@ mod tests {
|
||||
f64::INFINITY, // T_PREFER out of the way: cap is the actor here
|
||||
4_000_000,
|
||||
true,
|
||||
&|_| false,
|
||||
);
|
||||
assert_eq!(out.node, "http://p2:30000", "over-cap holder skipped");
|
||||
assert!(!out.all_over_cap, "p2 is under the cap");
|
||||
@@ -2804,6 +2756,7 @@ mod tests {
|
||||
f64::INFINITY,
|
||||
4_000_000,
|
||||
true,
|
||||
&|_| false,
|
||||
);
|
||||
assert_eq!(out.node, "http://p2:30000");
|
||||
}
|
||||
@@ -2825,6 +2778,7 @@ mod tests {
|
||||
f64::INFINITY,
|
||||
4_000_000,
|
||||
true,
|
||||
&|_| false,
|
||||
);
|
||||
assert!(out.all_over_cap, "every candidate over -> 429 signal");
|
||||
assert_eq!(out.node, "http://p1:30000", "holder kept for the record");
|
||||
@@ -2847,6 +2801,7 @@ mod tests {
|
||||
f64::INFINITY,
|
||||
4_000_000,
|
||||
false, // shadow: measure only
|
||||
&|_| false,
|
||||
);
|
||||
assert!(out.all_over_cap, "the signal is computed");
|
||||
assert_eq!(out.node, "http://p1:30000", "but the pick is unchanged");
|
||||
@@ -2966,4 +2921,59 @@ mod tests {
|
||||
let rate = q.slot_release_rate_at("d1", t0 + Duration::from_secs(100));
|
||||
assert!((rate - 0.01).abs() < 1e-9, "1/100s -> 0.01 req/s floor-ish");
|
||||
}
|
||||
|
||||
/// T17 修正版: the engine-real queue verdict (real_over predicate)
|
||||
/// excludes candidates exactly like the token cap — holder skipped in
|
||||
/// enforce, pick unchanged in shadow with the signal computed.
|
||||
#[test]
|
||||
fn pick_prefer_real_over_predicate_filters() {
|
||||
let mut q = QueueModel::new();
|
||||
let cands = candidates(&["http://p1:30000", "http://p2:30000"]);
|
||||
// Real verdict: p1's waiting queue is over the cap.
|
||||
let real_over: &dyn Fn(&str) -> bool = &|n: &str| n == "http://p1:30000";
|
||||
let out = q.pick_prefer(
|
||||
"http://p1:30000",
|
||||
ShadowClass::Hot,
|
||||
0,
|
||||
&cands,
|
||||
&[],
|
||||
true,
|
||||
f64::INFINITY, // T_PREFER out of the way
|
||||
0, // token cap off: only the real verdict acts
|
||||
true,
|
||||
&real_over,
|
||||
);
|
||||
assert_eq!(out.node, "http://p2:30000", "real-over holder skipped");
|
||||
assert!(!out.all_over_cap);
|
||||
// Both real-over -> all_over_cap signals (holder kept for record).
|
||||
let all_over: &dyn Fn(&str) -> bool = &|_| true;
|
||||
let out = q.pick_prefer(
|
||||
"http://p1:30000",
|
||||
ShadowClass::Hot,
|
||||
0,
|
||||
&cands,
|
||||
&[],
|
||||
true,
|
||||
f64::INFINITY,
|
||||
0,
|
||||
true,
|
||||
&all_over,
|
||||
);
|
||||
assert!(out.all_over_cap);
|
||||
// Shadow mode: pick unchanged, signal still computed.
|
||||
let out = q.pick_prefer(
|
||||
"http://p1:30000",
|
||||
ShadowClass::Hot,
|
||||
0,
|
||||
&cands,
|
||||
&[],
|
||||
true,
|
||||
f64::INFINITY,
|
||||
0,
|
||||
false,
|
||||
&real_over,
|
||||
);
|
||||
assert_eq!(out.node, "http://p1:30000", "shadow: pick unchanged");
|
||||
assert!(!out.all_over_cap, "p2 is not over");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1118,14 +1118,21 @@ impl PDRouter {
|
||||
g9i.shadow_admit(s);
|
||||
}
|
||||
if context.return_logprob {
|
||||
self.process_non_streaming_response(
|
||||
res,
|
||||
status,
|
||||
context.return_logprob,
|
||||
prefill_body,
|
||||
shadow,
|
||||
)
|
||||
.await
|
||||
let resp = self
|
||||
.process_non_streaming_response(
|
||||
res,
|
||||
status,
|
||||
context.return_logprob,
|
||||
prefill_body,
|
||||
shadow,
|
||||
)
|
||||
.await;
|
||||
// T17: non-streaming e2e (observability only — the
|
||||
// AIMD blind state reads local signals instead).
|
||||
if let Some(g9i) = &self.g9i {
|
||||
g9i.record_nonstream_e2e(start_time.elapsed().as_secs_f64());
|
||||
}
|
||||
resp
|
||||
} else {
|
||||
// Direct passthrough when no logprobs needed
|
||||
let response_headers =
|
||||
@@ -1138,6 +1145,10 @@ impl PDRouter {
|
||||
let mut response = Response::new(Body::from(decode_body));
|
||||
*response.status_mut() = status;
|
||||
*response.headers_mut() = response_headers;
|
||||
// T17: non-streaming e2e (observability only).
|
||||
if let Some(g9i) = &self.g9i {
|
||||
g9i.record_nonstream_e2e(start_time.elapsed().as_secs_f64());
|
||||
}
|
||||
response
|
||||
}
|
||||
Err(e) => {
|
||||
|
||||
Reference in New Issue
Block a user