diff --git a/sgl-model-gateway/src/main.rs b/sgl-model-gateway/src/main.rs index ab4135ff51..7b8dde8f1c 100644 --- a/sgl-model-gateway/src/main.rs +++ b/sgl-model-gateway/src/main.rs @@ -304,10 +304,14 @@ struct CliArgs { #[arg(long, default_value_t = 2.0, help_heading = "G9i Pool Admission")] g9i_t_prefer_secs: f64, - /// Hard P-queue token cap (physical backstop): skip prefill nodes whose in-flight tokens exceed this; 429 p_queue_cap only when ALL are over (0 = disabled; follows --g9i-td-admit mode) - #[arg(long, default_value_t = 4_000_000, help_heading = "G9i Pool Admission")] + /// Hard P-queue token cap (physical backstop; effective cap = min(this, drain_ewma x budget)) — 429 p_queue_cap only when ALL nodes over (0 = disabled; follows --g9i-td-admit mode) + #[arg(long, default_value_t = 16_000_000, help_heading = "G9i Pool Admission")] g9i_p_queue_token_cap: u64, + /// P-queue real-queue gate: skip prefill nodes whose engine-reported waiting queue (num_waiting_reqs) exceeds this (0 = disabled; follows --g9i-td-admit mode) + #[arg(long, default_value_t = 10, help_heading = "G9i Pool Admission")] + g9i_p_queue_reqs_cap: u64, + /// Admission hold queue: park holdable rejections (d_window_full / p_queue_cap) up to this many seconds instead of an immediate 429 (0 = disabled; enforce mode only; hold time counts into TTFT) #[arg(long, default_value_t = 1.0, help_heading = "G9i Pool Admission")] g9i_admit_hold_secs: f64, @@ -1201,6 +1205,7 @@ impl CliArgs { td_gate_secs: self.g9i_td_gate_secs, t_prefer_secs: self.g9i_t_prefer_secs, p_queue_token_cap: self.g9i_p_queue_token_cap, + p_queue_reqs_cap: self.g9i_p_queue_reqs_cap, admit_hold_secs: self.g9i_admit_hold_secs, admit_hold_max: self.g9i_admit_hold_max, }; diff --git a/sgl-model-gateway/src/routers/http/g9i.rs b/sgl-model-gateway/src/routers/http/g9i.rs index 5f4a5264c0..c34336727b 100644 --- a/sgl-model-gateway/src/routers/http/g9i.rs +++ b/sgl-model-gateway/src/routers/http/g9i.rs @@ -45,7 +45,7 @@ use std::{ collections::{HashMap, VecDeque}, sync::{ - atomic::{AtomicBool, Ordering}, + atomic::{AtomicBool, AtomicU64, Ordering}, Arc, RwLock, }, time::{Duration, Instant}, @@ -197,12 +197,21 @@ pub struct G9iConfig { /// a prefill node whose shadow-bookkept in-flight tokens + the /// request's own ctx exceed this is skipped (candidate fallback first); /// only when EVERY prefill node is over the cap does the request 429 - /// (p_queue_cap). Token-based, not request-count (昨天 p1 积压 64M - /// tokens ≈ 450s of work). Default 4M ≈ 30s of queue at the measured - /// 141k tok/s drain. 0 = disabled. Mode follows td_admit - /// (off/shadow/enforce); off or 0 ≡ current behavior (C13). + /// (p_queue_cap). T17 修正版: this is now the loose BACKSTOP — the + /// primary口径 is the engine's REAL waiting-queue length + /// (p_queue_reqs_cap below; 估算不可信). Raised to 16M (the T10 4M + /// choked all-cold 200k-ctx traffic at ~20-50 requests/node). 0 = + /// disabled. Mode follows td_admit; off or 0 ≡ current behavior (C13). #[serde(default = "default_p_queue_token_cap")] pub p_queue_token_cap: u64, + /// T17 (修正版) primary P-queue gate: a node whose ENGINE-REPORTED + /// waiting queue (num_waiting_reqs, /v1/loads core) exceeds this is + /// excluded from candidates; all over -> hold/429 (p_queue_cap). + /// Real measurement, not an estimate. Default 10 (待实测调). + /// Stale/missing snapshots are never judged (不冤枉); all-missing -> + /// C13 degrade. 0 = disabled. + #[serde(default = "default_p_queue_reqs_cap")] + pub p_queue_reqs_cap: u64, /// T15 admission hold queue: holdable rejections (d_window_full / /// p_queue_cap) park the request up to this many seconds — TOTAL budget /// across hold retries — waiting for room before the 429 fires @@ -267,7 +276,10 @@ fn default_t_prefer_secs() -> f64 { 2.0 } fn default_p_queue_token_cap() -> u64 { - 4_000_000 + 16_000_000 +} +fn default_p_queue_reqs_cap() -> u64 { + 10 } fn default_admit_hold_secs() -> f64 { 1.0 @@ -301,6 +313,7 @@ impl Default for G9iConfig { td_gate_secs: default_td_gate_secs(), t_prefer_secs: default_t_prefer_secs(), p_queue_token_cap: default_p_queue_token_cap(), + p_queue_reqs_cap: default_p_queue_reqs_cap(), admit_hold_secs: default_admit_hold_secs(), admit_hold_max: default_admit_hold_max(), } @@ -349,6 +362,10 @@ pub struct NodeLoad { pub queued_total_input_tokens: u64, /// Sum of drain_rate_tokens_per_s across dp ranks. pub drain_rate_tokens_per_s: f64, + /// Sum of num_waiting_reqs across dp ranks — the engine's REAL queue + /// length, the primary P-admission口径 (T17 修正版: 估算管路由, + /// 真实队列管准入). + pub waiting_reqs: u64, /// Sum of disaggregation.decode_prealloc_queue_reqs across dp ranks. pub decode_prealloc_queue_reqs: u64, /// Sum of disaggregation.decode_transfer_queue_reqs across dp ranks. @@ -471,6 +488,11 @@ pub struct G9iState { /// only used when a holdable rejection fires in enforce mode with a /// nonzero hold budget. hold: HoldQueue, + /// T17: cumulative hold timeouts (blind-state AIMD input — a timeout + /// this tick means someone is hurting even without TTFT samples). + hold_timeouts_total: AtomicU64, + /// Timeouts seen at the previous AIMD tick (for the diff). + last_tick_timeouts: AtomicU64, } /// T14 AIMD congestion-window state for one decode node. @@ -680,6 +702,8 @@ impl G9iState { d_windows: RwLock::new(HashMap::new()), last_aimd: RwLock::new(Instant::now()), hold: HoldQueue::default(), + hold_timeouts_total: AtomicU64::new(0), + last_tick_timeouts: AtomicU64::new(0), } } @@ -1068,9 +1092,19 @@ impl G9iState { fn aimd_tick(&self) { let (count, p50, p95, p99) = self.ttft_slo_view(); + // T17 (修正版) blind-state inputs (non-streaming traffic leaves no + // TTFT samples): hold timeouts this tick, and per-node REAL engine + // signals — D prealloc queue (disagg 段真实值) + pipeline depth. + let timeouts_now = self.hold_timeouts_total.load(Ordering::Relaxed); + let timed_out = { + let prev = self.last_tick_timeouts.swap(timeouts_now, Ordering::Relaxed); + timeouts_now > prev + }; + let d_snaps = self.current_d_snapshots(); let Ok(mut windows) = self.d_windows.write() else { return; }; + let q = self.shadow_queue.as_ref().and_then(|l| l.read().ok()); let mut grew = false; for (node, e) in windows.iter_mut() { if e.max_running_total == 0 { @@ -1081,19 +1115,37 @@ impl G9iState { let w_min = WINDOW_MIN; let w_max = WINDOW_MAX; e.w = e.w.clamp(w_min, w_max); - if count < AIMD_MIN_SAMPLES { - continue; // 样本不足/无新流量: hold - } let old = e.w; let why: &str; - if p95 > SLO_P95_SECS || p99 > SLO_P99_SECS { - e.w = (e.w * AIMD_CUT).max(w_min); - why = "slo_violation"; - } else if p50 < SLO_P50_SECS && p95 < SLO_P95_SECS { - e.w = (e.w + AIMD_ADD).min(w_max); - why = "slo_ok"; + if count >= AIMD_MIN_SAMPLES { + // SLO 判定优先(流式样本充足时照旧). + if p95 > SLO_P95_SECS || p99 > SLO_P99_SECS { + e.w = (e.w * AIMD_CUT).max(w_min); + why = "slo_violation"; + } else if p50 < SLO_P50_SECS && p95 < SLO_P95_SECS { + e.w = (e.w + AIMD_ADD).min(w_max); + why = "slo_ok"; + } else { + why = "hold"; + } } else { - why = "hold"; + // T17 (修正版) 盲态降级(非流式流量没有 TTFT 样本——窗口 + // 必须不靠它们恢复): 本拍有持有超时 -> hold(有人在受伤); + // D prealloc 队列清空(==0) 且管线宽裕 (<0.8W) -> 慢加恢复; + // prealloc 持续 >0 或管线顶满 -> hold(窗口本身已在限流)。 + let pipe = q.as_ref().map(|q| q.d_pipeline(node)).unwrap_or(0) as f64; + let prealloc_busy = d_snaps + .iter() + .find(|s| &s.node == node) + .is_some_and(|s| s.prealloc_queue_reqs > 0); + if timed_out { + why = "blind_hold_timeout"; + } else if !prealloc_busy && pipe < 0.8 * e.w { + e.w = (e.w + AIMD_ADD).min(w_max); + why = "blind_recover"; + } else { + why = "blind_hold_full"; + } } if e.w != old { info!( @@ -1278,6 +1330,8 @@ impl G9iState { self.hold.remove(id, body_bytes); guard.disarm(); let held = parked_at.elapsed().as_secs_f64(); + // T17: AIMD blind-state input. + self.hold_timeouts_total.fetch_add(1, Ordering::Relaxed); metrics::counter!(METRIC_ADMIT_HELD, "outcome" => "timeout", "class" => class_label) .increment(1); metrics::histogram!(METRIC_ADMIT_HOLD_SECS).record(held); @@ -1513,17 +1567,40 @@ impl G9iState { }) } - /// v1-path cap judgment (T10): over cap only when a FRESH /v1/loads - /// snapshot's queued_total_input_tokens says so. Missing or stale - /// snapshots are NOT judged (fresh_load already enforces the TTL) — a - /// blind node must never trip the physical backstop (避免误伤). + /// v1-path cap judgment (T17 修正版): over cap when a FRESH /v1/loads + /// snapshot says so — PRIMARY: engine-reported waiting queue + /// (num_waiting_reqs > p_queue_reqs_cap); BACKSTOP: queued tokens over + /// p_queue_token_cap (16M, loose). Missing or stale snapshots are NOT + /// judged (fresh_load already enforces the TTL) — a blind node must + /// never trip the physical backstop (不冤枉), and all-blind means the + /// gate stays open (C13). + /// The poll interval (default 1s) keeps this real-time enough for the + /// gate; no faster polling is needed. fn v1_over_cap(&self, url: &str) -> bool { - let cap = self.p_cap(); + if !self.td_admit_active() { + return false; + } + let Some(l) = self.fresh_load(url) else { + return false; + }; + // Primary: real waiting-queue length. + if self.config.p_queue_reqs_cap > 0 && l.waiting_reqs > self.config.p_queue_reqs_cap { + return true; + } + // Loose token backstop (16M): catches drain/queue-metric failure. + let cap = self.config.p_queue_token_cap; + cap > 0 && l.queued_total_input_tokens > cap + } + + /// T17 (修正版) shadow-path P gate: the engine's REAL waiting queue + /// length over the reqs cap (same data as v1_over_cap's primary rule). + /// Missing/stale snapshot -> not judged. + fn p_real_queue_over(&self, url: &str) -> bool { + let cap = self.config.p_queue_reqs_cap; if cap == 0 { return false; } - self.fresh_load(url) - .is_some_and(|l| l.queued_total_input_tokens > cap) + self.fresh_load(url).is_some_and(|l| l.waiting_reqs > cap) } /// T10 v1-path candidate split: (kept, over_cap). Inactive @@ -1829,6 +1906,16 @@ impl G9iState { // T10: token cap skips over-cap nodes first // (enforce); shadow mode computes all_over_cap // without altering the pick. + // T17 (修正版): the ENGINE-REAL waiting queue is the + // primary over-cap verdict (估算管路由,真实队列管 + // 准入); inactive -> always-false closure. + let real_gate_on = + self.td_admit_active() && self.config.p_queue_reqs_cap > 0; + // (type annotation forces HRTB generalization for + // the capturing closure) + let real_over: &dyn Fn(&str) -> bool = &|node: &str| { + real_gate_on && self.p_real_queue_over(node) + }; let candidates = self .p_nodes .read() @@ -1844,6 +1931,7 @@ impl G9iState { self.config.t_prefer_secs, self.p_cap(), self.config.td_admit == "enforce", + &real_over, ); if out.node != holder { metrics::counter!(METRIC_DECISIONS, "path" => "shadow_t_prefer_fallback") @@ -1932,6 +2020,14 @@ impl G9iState { } } + /// T17: non-streaming requests carry no observable first-token moment; + /// record their end-to-end latency instead (observability ONLY — never + /// feeds AIMD, which degrades to local blind-state signals when TTFT + /// samples are absent). + pub fn record_nonstream_e2e(&self, seconds: f64) { + metrics::histogram!("smg_g9i_nonstream_e2e_secs").record(seconds); + } + /// Publish observed TTFT percentiles as gauges (called from the snapshot /// poller — cheap, piggybacks an existing periodic task). Statistics /// only: never rejects. Reference budgets (p95 10s / p99 30s) are an @@ -2439,6 +2535,10 @@ fn parse_node_load(json: &Value) -> Result { .get("drain_rate_tokens_per_s") .and_then(|v| v.as_f64()) .unwrap_or(0.0); + node.waiting_reqs += rank + .get("num_waiting_reqs") + .and_then(|v| v.as_u64()) + .unwrap_or(0); let max_total = rank .get("max_total_num_tokens") .and_then(|v| v.as_u64()) @@ -2585,6 +2685,7 @@ mod tests { { "dp_rank": 0, "num_running_reqs": 3, + "num_waiting_reqs": 7, "max_running_requests": 128, "queued_total_input_tokens": 100_000, "drain_rate_tokens_per_s": 20_000.0, @@ -2631,6 +2732,7 @@ mod tests { // (128-3) + (128-5) = 248 free slots (per-rank, then summed). assert_eq!(node.free_slots, 248); assert_eq!(node.max_running_reqs, 256); + assert_eq!(node.waiting_reqs, 7, "T17: engine real queue parsed"); assert!(node.has_disagg); } @@ -3034,21 +3136,61 @@ sglang:token_usage{dp_rank=\"0\"} 0.5 ); } - /// Insufficient samples (< 20) -> the window never drifts. + /// T17 blind-state degradation: without TTFT samples the window + /// RECOVERS on local signals (pipeline under 0.8W + empty hold queue + + /// no timeouts), holds on timeouts, holds when pipeline-pinned. #[test] - fn aimd_insufficient_samples_hold() { + fn aimd_blind_state_recovery_and_holds() { + let st = p2_state("enforce"); + set_d_snapshots(&st, &[("http://d1:30001", 0, 10_000_000, true, false)]); + set_window(&st, "http://d1:30001", 32.0, 256); + // Blind + idle pipeline + empty hold queue -> slow-add recovery, + // no TTFT samples needed (非流式盲态也能爬回来). + st.aimd_tick(); + assert_eq!(window_of(&st, "http://d1:30001"), 48.0, "blind recover +16"); + st.aimd_tick(); + assert_eq!(window_of(&st, "http://d1:30001"), 64.0, "recovery continues"); + // A hold timeout this tick -> hold (someone is hurting). + st.hold_timeouts_total.store(1, Ordering::Relaxed); + st.aimd_tick(); + assert_eq!(window_of(&st, "http://d1:30001"), 64.0, "timeout -> hold"); + // Pipeline pinned at the window -> hold (the window IS the throttle). + book_ledger(&st, "http://d1:30001", 64); + st.aimd_tick(); + assert_eq!(window_of(&st, "http://d1:30001"), 64.0, "pipeline pinned -> hold"); + } + + /// T17 修正版: D prealloc 队列 >0(引擎真实值)时盲态不恢复。 + #[test] + fn aimd_blind_holds_while_prealloc_busy() { + let st = p2_state("enforce"); + set_d_snapshots(&st, &[("http://d1:30001", 0, 10_000_000, true, false)]); + set_window(&st, "http://d1:30001", 32.0, 256); + st.aimd_tick(); + assert_eq!(window_of(&st, "http://d1:30001"), 48.0, "idle+prealloc 0 -> recover"); + // prealloc > 0 -> hold even with an idle pipeline. + let mut d = st.d_snapshots.write().unwrap(); + d[0].1.decode_prealloc_queue_reqs = 3; + drop(d); + st.aimd_tick(); + assert_eq!(window_of(&st, "http://d1:30001"), 48.0, "prealloc>0 -> hold"); + } + + /// SLO judgment stays preferred when streaming samples suffice — even + /// with the pipeline pinned (which would hold in blind mode). + #[test] + fn aimd_slo_preferred_over_blind_signals() { let st = p2_state("enforce"); set_d_snapshots(&st, &[("http://d1:30001", 0, 10_000_000, true, false)]); set_window(&st, "http://d1:30001", 100.0, 256); - feed_ttft(&st, 5, 2.0); // would be "ok" with enough samples + book_ledger(&st, "http://d1:30001", 100); // pipeline pinned at W + feed_ttft(&st, 20, 2.0); // SLO ok st.aimd_tick(); - assert_eq!(window_of(&st, "http://d1:30001"), 100.0); - // Zero samples at all: also hold. - let st2 = p2_state("enforce"); - set_d_snapshots(&st2, &[("http://d1:30001", 0, 10_000_000, true, false)]); - set_window(&st2, "http://d1:30001", 100.0, 256); - st2.aimd_tick(); - assert_eq!(window_of(&st2, "http://d1:30001"), 100.0); + assert_eq!( + window_of(&st, "http://d1:30001"), + 116.0, + "SLO path wins over the blind pipeline-pinned hold" + ); } /// Window admission: ledger < W admits; ledger >= W (all D) -> 429 @@ -3197,10 +3339,16 @@ sglang:token_usage{dp_rank=\"0\"} 0.5 } fn set_load(st: &G9iState, url: &str, queued: u64, stale: bool) { + set_load_full(st, url, queued, 0, stale); + } + + /// T17 修正版: waiting 参数进快照(真实队列口径测试). + fn set_load_full(st: &G9iState, url: &str, queued: u64, waiting: u64, stale: bool) { st.snapshots.write().unwrap().insert( normalize_url(url), NodeLoad { queued_total_input_tokens: queued, + waiting_reqs: waiting, fetch_ok: true, fetched_at: Some(if stale { Instant::now() @@ -3214,37 +3362,63 @@ sglang:token_usage{dp_rank=\"0\"} 0.5 ); } - /// v1 路径快照口径: only a FRESH snapshot's queued_total_input_tokens - /// can trip the cap; stale/missing snapshots are never judged; cap=0 - /// disables entirely. + /// v1 路径闸门(T17 修正版): PRIMARY = 引擎真实 waiting 队列 + /// (> reqs_cap); BACKSTOP = 排队 token (> 16M 宽松兜底); + /// stale/missing 快照不判定; off / 双禁 -> 惰性 (C13)。 #[test] fn v1_over_cap_snapshot_semantics() { - let st = p2_state("enforce"); // cap default 4M - set_load(&st, "http://p1:30000", 5_000_000, false); - assert!(st.v1_over_cap("http://p1:30000"), "fresh over-cap judged"); - set_load(&st, "http://p2:30000", 1_000, false); - assert!(!st.v1_over_cap("http://p2:30000"), "fresh under-cap passes"); - set_load(&st, "http://p3:30000", 50_000_000, true); - assert!(!st.v1_over_cap("http://p3:30000"), "stale never judged"); + let st = p2_state("enforce"); // reqs_cap default 10, token cap 16M + // waiting 超阈(token 远未超)-> 判定(真实队列口径). + set_load_full(&st, "http://p1:30000", 1_000_000, 50, false); + assert!(st.v1_over_cap("http://p1:30000"), "waiting 50 > 10 -> over"); + // waiting 未超、token 也未超 -> 放. + set_load_full(&st, "http://p2:30000", 1_000_000, 3, false); + assert!(!st.v1_over_cap("http://p2:30000"), "waiting 3 <= 10 -> pass"); + // waiting 低但 token 超 16M 兜底 -> 判定. + set_load_full(&st, "http://p3:30000", 20_000_000, 2, false); + assert!(st.v1_over_cap("http://p3:30000"), "token backstop binds"); + // stale / missing -> 不判定. + set_load_full(&st, "http://p4:30000", 50_000_000, 500, true); + assert!(!st.v1_over_cap("http://p4:30000"), "stale never judged"); assert!(!st.v1_over_cap("http://p9:30000"), "missing never judged"); - // cap=0 disables the judgment outright (C13). + // cap=0 双禁 -> 惰性 (C13). let cfg = G9iConfig { enabled: true, shadow_enabled: true, td_admit: "enforce".to_string(), p_queue_token_cap: 0, + p_queue_reqs_cap: 0, ..Default::default() }; let st2 = G9iState::new(cfg); - set_load(&st2, "http://p1:30000", 5_000_000, false); - assert!(!st2.v1_over_cap("http://p1:30000"), "cap=0 inert"); - // td_admit=off also inert (p_cap() gates on the mode). + set_load_full(&st2, "http://p1:30000", 50_000_000, 500, false); + assert!(!st2.v1_over_cap("http://p1:30000"), "both disabled -> inert"); + // td_admit=off -> 惰性. let st3 = p2_state("off"); - set_load(&st3, "http://p1:30000", 5_000_000, false); + set_load_full(&st3, "http://p1:30000", 50_000_000, 500, false); assert!(!st3.v1_over_cap("http://p1:30000"), "td_admit=off inert"); } + /// T17 修正版 shadow 路径: pick_prefer 的 real_over 谓词(引擎真实 + /// waiting 队列)在 enforce 档真实剔除候选, 超阈节点被跳过。 + #[test] + fn shadow_path_real_queue_gate_filters() { + let st = p2_state("enforce"); + set_load_full(&st, "http://p1:30000", 100_000, 50, false); // over + set_load_full(&st, "http://p2:30000", 100_000, 3, false); // fine + assert!(st.p_real_queue_over("http://p1:30000")); + assert!(!st.p_real_queue_over("http://p2:30000")); + assert!(!st.p_real_queue_over("http://p9:30000"), "missing not judged"); + // off 档惰性. + let st_off = p2_state("off"); + set_load_full(&st_off, "http://p1:30000", 100_000, 500, false); + // p_real_queue_over itself is mode-independent (callers gate); + // the shadow caller wraps it with real_gate_on. Verify the wrap: + // td_admit off -> the closure in shadow_classify_and_admit is inert. + assert!(!st_off.td_admit_active()); + } + // ============ T11: atomic decision-time admission ============ fn t11_state(mode: &str, cap: u64) -> G9iState { diff --git a/sgl-model-gateway/src/routers/http/g9i_shadow.rs b/sgl-model-gateway/src/routers/http/g9i_shadow.rs index ef51a80afb..6e153ec46f 100644 --- a/sgl-model-gateway/src/routers/http/g9i_shadow.rs +++ b/sgl-model-gateway/src/routers/http/g9i_shadow.rs @@ -897,11 +897,14 @@ impl QueueModel { /// /// `t_prefer_secs = +∞` degenerates to "always the holder" — the /// pre-T_PREFER behavior exactly. - /// Token cap layering (T10, `token_cap` > 0, evaluated only while - /// td_admit != off): a node whose shadow-bookkept in-flight tokens + - /// this request's own_ctx exceed the cap is SKIPPED (回退优先于拒绝). + /// Cap layering (T10 token backstop + T17 engine-real queue verdict): + /// a node is over cap when its shadow-bookkept in-flight tokens + this + /// request's own_ctx exceed token_cap (T10) OR the engine's REAL + /// waiting-queue verdict says so (`real_over`, T17 — 估算管路由, + /// 真实队列管准入). Over-cap nodes are SKIPPED (回退优先于拒绝). /// When `cap_enforce` is on the skip is real; in shadow mode the pick /// is unchanged and only `all_over_cap` is computed. + #[allow(clippy::too_many_arguments)] pub fn pick_prefer( &mut self, holder: &str, @@ -913,17 +916,17 @@ impl QueueModel { t_prefer_secs: f64, token_cap: u64, cap_enforce: bool, + real_over: &dyn Fn(&str) -> bool, ) -> PreferOutcome { let holder_est = self.est_ttft(Some(holder), class, own_ctx_tokens, d_snaps, est_td_enabled); - // Fast path: cap inactive -> the pure T_PREFER logic, no scans. - if token_cap == 0 { - return self.prefer_by_threshold(holder, holder_est, class, own_ctx_tokens, candidates, d_snaps, est_td_enabled, t_prefer_secs, false); - } - let filter = cap_enforce; - let holder_over_cap = - self.node_outstanding(holder).saturating_add(own_ctx_tokens) > token_cap; - if !(filter && holder_over_cap) && holder_est.total <= t_prefer_secs { + let over = |q: &QueueModel, node: &str| { + (token_cap > 0 + && q.node_outstanding(node).saturating_add(own_ctx_tokens) > token_cap) + || real_over(node) + }; + let holder_over_cap = over(self, holder); + if !(cap_enforce && holder_over_cap) && holder_est.total <= t_prefer_secs { // Holder picked on threshold. The all-over signal: holder under // cap settles it (false) without a scan; holder over cap in // SHADOW mode still needs the candidate scan (the pick stays @@ -932,9 +935,7 @@ impl QueueModel { && candidates .iter() .filter(|c| c.as_str() != holder) - .all(|c| { - self.node_outstanding(c).saturating_add(own_ctx_tokens) > token_cap - }); + .all(|c| over(self, c)); return PreferOutcome { node: holder.to_string(), est: holder_est, @@ -947,10 +948,9 @@ impl QueueModel { let mut rest: Vec<(String, EstTtft)> = Vec::new(); let mut any_under_cap = !holder_over_cap; for c in candidates.iter().filter(|c| c.as_str() != holder) { - let c_over = - self.node_outstanding(c).saturating_add(own_ctx_tokens) > token_cap; + let c_over = over(self, c); any_under_cap = any_under_cap || !c_over; - if filter && c_over { + if cap_enforce && c_over { continue; } let est = self.est_ttft(Some(c), class, own_ctx_tokens, d_snaps, est_td_enabled); @@ -974,7 +974,7 @@ impl QueueModel { // All over the T_PREFER threshold: the minimum-est ELIGIBLE node // (holder participates unless cap-excluded in enforce mode). let mut best: Option<(String, EstTtft)> = None; - if !(filter && holder_over_cap) { + if !(cap_enforce && holder_over_cap) { best = Some((holder.to_string(), holder_est)); } for (node, est) in rest { @@ -998,59 +998,6 @@ impl QueueModel { } } - /// The pre-cap T_PREFER logic (threshold ordering only), shared by the - /// token_cap==0 fast path. `all_over_cap` is always false here. - #[allow(clippy::too_many_arguments)] - fn prefer_by_threshold( - &mut self, - holder: &str, - holder_est: EstTtft, - class: ShadowClass, - own_ctx_tokens: u64, - candidates: &[String], - d_snaps: &[DSnapshot], - est_td_enabled: bool, - t_prefer_secs: f64, - _cap_active: bool, - ) -> PreferOutcome { - if holder_est.total <= t_prefer_secs { - return PreferOutcome { - node: holder.to_string(), - est: holder_est, - all_over_cap: false, - }; - } - let mut rest: Vec<(String, EstTtft)> = Vec::new(); - for c in candidates.iter().filter(|c| c.as_str() != holder) { - let est = self.est_ttft(Some(c), class, own_ctx_tokens, d_snaps, est_td_enabled); - rest.push((c.clone(), est)); - } - rest.sort_by(|a, b| { - a.1.total - .partial_cmp(&b.1.total) - .unwrap_or(std::cmp::Ordering::Equal) - }); - for (node, est) in &rest { - if est.total <= t_prefer_secs { - return PreferOutcome { - node: node.clone(), - est: *est, - all_over_cap: false, - }; - } - } - let mut best = (holder.to_string(), holder_est); - for (node, est) in rest { - if est.total < best.1.total { - best = (node, est); - } - } - PreferOutcome { - node: best.0, - est: best.1, - all_over_cap: false, - } - } /// Per-node total in-flight estimated tokens (all classes summed) — /// the shadow bookkeeping view behind the P-queue hard cap (T10). @@ -2694,6 +2641,7 @@ mod tests { 3.0, 0, false, + &|_| false, ); assert_eq!(out.node, "http://p1:30000"); assert!((out.est.total - 2.5).abs() < 1e-9); @@ -2716,6 +2664,7 @@ mod tests { 3.0, 0, false, + &|_| false, ); assert_eq!(out.node, "http://p2:30000"); assert!((out.est.total - 0.5).abs() < 1e-9); @@ -2739,6 +2688,7 @@ mod tests { 1.0, 0, false, + &|_| false, ); assert_eq!(out.node, "http://p2:30000", "global minimum est wins"); assert!((out.est.total - 2.5).abs() < 1e-9); @@ -2760,6 +2710,7 @@ mod tests { f64::INFINITY, 0, false, + &|_| false, ); assert_eq!(out.node, "http://p1:30000"); } @@ -2783,6 +2734,7 @@ mod tests { f64::INFINITY, // T_PREFER out of the way: cap is the actor here 4_000_000, true, + &|_| false, ); assert_eq!(out.node, "http://p2:30000", "over-cap holder skipped"); assert!(!out.all_over_cap, "p2 is under the cap"); @@ -2804,6 +2756,7 @@ mod tests { f64::INFINITY, 4_000_000, true, + &|_| false, ); assert_eq!(out.node, "http://p2:30000"); } @@ -2825,6 +2778,7 @@ mod tests { f64::INFINITY, 4_000_000, true, + &|_| false, ); assert!(out.all_over_cap, "every candidate over -> 429 signal"); assert_eq!(out.node, "http://p1:30000", "holder kept for the record"); @@ -2847,6 +2801,7 @@ mod tests { f64::INFINITY, 4_000_000, false, // shadow: measure only + &|_| false, ); assert!(out.all_over_cap, "the signal is computed"); assert_eq!(out.node, "http://p1:30000", "but the pick is unchanged"); @@ -2966,4 +2921,59 @@ mod tests { let rate = q.slot_release_rate_at("d1", t0 + Duration::from_secs(100)); assert!((rate - 0.01).abs() < 1e-9, "1/100s -> 0.01 req/s floor-ish"); } + + /// T17 修正版: the engine-real queue verdict (real_over predicate) + /// excludes candidates exactly like the token cap — holder skipped in + /// enforce, pick unchanged in shadow with the signal computed. + #[test] + fn pick_prefer_real_over_predicate_filters() { + let mut q = QueueModel::new(); + let cands = candidates(&["http://p1:30000", "http://p2:30000"]); + // Real verdict: p1's waiting queue is over the cap. + let real_over: &dyn Fn(&str) -> bool = &|n: &str| n == "http://p1:30000"; + let out = q.pick_prefer( + "http://p1:30000", + ShadowClass::Hot, + 0, + &cands, + &[], + true, + f64::INFINITY, // T_PREFER out of the way + 0, // token cap off: only the real verdict acts + true, + &real_over, + ); + assert_eq!(out.node, "http://p2:30000", "real-over holder skipped"); + assert!(!out.all_over_cap); + // Both real-over -> all_over_cap signals (holder kept for record). + let all_over: &dyn Fn(&str) -> bool = &|_| true; + let out = q.pick_prefer( + "http://p1:30000", + ShadowClass::Hot, + 0, + &cands, + &[], + true, + f64::INFINITY, + 0, + true, + &all_over, + ); + assert!(out.all_over_cap); + // Shadow mode: pick unchanged, signal still computed. + let out = q.pick_prefer( + "http://p1:30000", + ShadowClass::Hot, + 0, + &cands, + &[], + true, + f64::INFINITY, + 0, + false, + &real_over, + ); + assert_eq!(out.node, "http://p1:30000", "shadow: pick unchanged"); + assert!(!out.all_over_cap, "p2 is not over"); + } } diff --git a/sgl-model-gateway/src/routers/http/pd_router.rs b/sgl-model-gateway/src/routers/http/pd_router.rs index 833ffccebf..a506c69621 100644 --- a/sgl-model-gateway/src/routers/http/pd_router.rs +++ b/sgl-model-gateway/src/routers/http/pd_router.rs @@ -1118,14 +1118,21 @@ impl PDRouter { g9i.shadow_admit(s); } if context.return_logprob { - self.process_non_streaming_response( - res, - status, - context.return_logprob, - prefill_body, - shadow, - ) - .await + let resp = self + .process_non_streaming_response( + res, + status, + context.return_logprob, + prefill_body, + shadow, + ) + .await; + // T17: non-streaming e2e (observability only — the + // AIMD blind state reads local signals instead). + if let Some(g9i) = &self.g9i { + g9i.record_nonstream_e2e(start_time.elapsed().as_secs_f64()); + } + resp } else { // Direct passthrough when no logprobs needed let response_headers = @@ -1138,6 +1145,10 @@ impl PDRouter { let mut response = Response::new(Body::from(decode_body)); *response.status_mut() = status; *response.headers_mut() = response_headers; + // T17: non-streaming e2e (observability only). + if let Some(g9i) = &self.g9i { + g9i.record_nonstream_e2e(start_time.elapsed().as_secs_f64()); + } response } Err(e) => {