diff --git a/sgl-model-gateway/src/config/builder.rs b/sgl-model-gateway/src/config/builder.rs index 5d4891ca60..ba06d8d79c 100644 --- a/sgl-model-gateway/src/config/builder.rs +++ b/sgl-model-gateway/src/config/builder.rs @@ -378,6 +378,12 @@ impl RouterConfigBuilder { self } + /// G9i pool-aware admission control + fast-P routing config (PD mode). + pub fn g9i(mut self, g9i: Option) -> Self { + self.config.g9i = g9i; + self + } + pub fn model_path>(mut self, path: S) -> Self { self.config.model_path = Some(path.into()); self diff --git a/sgl-model-gateway/src/config/types.rs b/sgl-model-gateway/src/config/types.rs index b19802642c..900b627261 100644 --- a/sgl-model-gateway/src/config/types.rs +++ b/sgl-model-gateway/src/config/types.rs @@ -99,6 +99,9 @@ pub struct RouterConfig { /// Enable WASM support #[serde(default)] pub enable_wasm: bool, + /// G9i pool-aware admission control + fast-P routing (PD mode, HTTP only). + #[serde(default)] + pub g9i: Option, } /// Tokenizer cache configuration @@ -551,6 +554,7 @@ impl Default for RouterConfig { ca_certificates: vec![], mcp_config: None, enable_wasm: false, + g9i: None, server_cert: None, server_key: None, } diff --git a/sgl-model-gateway/src/main.rs b/sgl-model-gateway/src/main.rs index b389bdad19..360a111e8c 100644 --- a/sgl-model-gateway/src/main.rs +++ b/sgl-model-gateway/src/main.rs @@ -219,6 +219,51 @@ struct CliArgs { #[arg(long, default_value_t = 30, help_heading = "PD Disaggregation")] worker_startup_check_interval: u64, + // ==================== G9i: Pool Admission + Fast-P Routing ==================== + /// Enable G9i pool-aware admission control + fast-P routing (PD HTTP mode) + #[arg(long, default_value_t = false, help_heading = "G9i Pool Admission")] + g9i_enabled: bool, + + /// Fast-P pool member base URLs (repeatable; router-side role, engine unaware) + #[arg(long, action = ArgAction::Append, help_heading = "G9i Pool Admission")] + g9i_fast_prefill_url: Vec, + + /// Small-prefill cutoff on NEW tokens (total - prefix hit). Tentative; tune by bench. + #[arg(long, default_value_t = 4096, help_heading = "G9i Pool Admission")] + g9i_max_new_tokens: u64, + + /// 429 when the least-wait fast-P queue estimate exceeds this many seconds. Tentative. + #[arg(long, default_value_t = 5.0, help_heading = "G9i Pool Admission")] + g9i_fast_p_queue_reject_secs: f64, + + /// D-pool admission ceiling: sum of in-processing requests across decode nodes. Tentative. + #[arg(long, default_value_t = 256, help_heading = "G9i Pool Admission")] + g9i_decode_max_running: u64, + + /// Drain-rate floor (tok/s) for queue-wait estimates (div-by-zero / cold-start guard) + #[arg(long, default_value_t = 1000.0, help_heading = "G9i Pool Admission")] + g9i_drain_rate_floor: f64, + + /// Snapshot poll interval (seconds) + #[arg(long, default_value_t = 1, help_heading = "G9i Pool Admission")] + g9i_poll_interval_secs: u64, + + /// Snapshot TTL — stale nodes leave pool decisions (seconds) + #[arg(long, default_value_t = 5, help_heading = "G9i Pool Admission")] + g9i_snapshot_ttl_secs: u64, + + /// No fresh D-pool snapshot: "allow" (default, fail-open + warn) or "reject" + #[arg(long, default_value = "allow", value_parser = ["allow", "reject"], help_heading = "G9i Pool Admission")] + g9i_d_no_snapshot: String, + + /// /query_cache_hit probe timeout per request (ms) + #[arg(long, default_value_t = 2000, help_heading = "G9i Pool Admission")] + g9i_cache_hit_query_timeout_ms: u64, + + /// Also least-wait route the regular P pool (default: keep configured policy) + #[arg(long, default_value_t = false, help_heading = "G9i Pool Admission")] + g9i_regular_pool_least_wait: bool, + // ==================== Service Discovery (Kubernetes) ==================== /// Enable Kubernetes service discovery #[arg( @@ -1070,11 +1115,34 @@ impl CliArgs { .circuit_breaker(!self.disable_circuit_breaker) .enable_wasm(self.enable_wasm) .igw(self.enable_igw) + .g9i(self.build_g9i_config()?) .maybe_server_cert_and_key(self.tls_cert_path.as_ref(), self.tls_key_path.as_ref()); builder.build() } + /// G9i pool admission + fast-P routing config; None when the feature is off. + fn build_g9i_config(&self) -> ConfigResult> { + if !self.g9i_enabled && self.g9i_fast_prefill_url.is_empty() { + return Ok(None); + } + let config = smg::routers::http::g9i::G9iConfig { + enabled: self.g9i_enabled, + fast_prefill_urls: self.g9i_fast_prefill_url.clone(), + max_new_tokens: self.g9i_max_new_tokens, + fast_p_queue_reject_secs: self.g9i_fast_p_queue_reject_secs, + decode_max_running: self.g9i_decode_max_running, + drain_rate_floor: self.g9i_drain_rate_floor, + poll_interval_secs: self.g9i_poll_interval_secs, + snapshot_ttl_secs: self.g9i_snapshot_ttl_secs, + d_no_snapshot: self.g9i_d_no_snapshot.clone(), + cache_hit_query_timeout_ms: self.g9i_cache_hit_query_timeout_ms, + regular_pool_least_wait: self.g9i_regular_pool_least_wait, + }; + config.validate().map_err(|reason| ConfigError::ValidationFailed { reason })?; + Ok(Some(config)) + } + fn to_server_config(&self, router_config: RouterConfig) -> ServerConfig { let service_discovery_config = if self.service_discovery { // Get router discovery config from router_config.discovery if available diff --git a/sgl-model-gateway/src/routers/http/g9i.rs b/sgl-model-gateway/src/routers/http/g9i.rs new file mode 100644 index 0000000000..9a797d92d9 --- /dev/null +++ b/sgl-model-gateway/src/routers/http/g9i.rs @@ -0,0 +1,628 @@ +//! G9i: pool-aware admission control + fast-P routing for PD mode. +//! +//! Router-side only — zero engine changes. Consumes two engine surfaces that +//! already exist (G9 P0): +//! * `GET /v1/loads?include=core` — per-DP-rank `LoadSnapshot` list; we sum +//! `num_running_reqs` / `queued_total_input_tokens` / +//! `drain_rate_tokens_per_s` across ranks into per-node, pool-aggregated +//! views (poll loop below, independent of `LoadMonitor`, whose +//! `aggregate.total_tokens` parse target does not exist on this engine). +//! * `POST /query_cache_hit` — chat-body prefix-hit probe used to classify +//! small-prefill (new tokens < threshold) requests. +//! +//! Decision tree (per request, before PD dispatch): +//! 1. D-pool gate: sum of fresh `num_running_reqs` over the decode pool +//! exceeds `decode_max_running` -> 429 (`d_pool_saturated`). +//! 2. Small-prefill classification (chat routes only): probe one fast-P, +//! `new_tokens < max_new_tokens` -> fast-P pool, else regular P pool. +//! 3. Pool selection: least estimated wait +//! (`queued_total_input_tokens / max(drain_rate, drain_rate_floor)`) +//! among fresh nodes — this is also the load balancer. +//! 4. Fast-P queue gate: least-wait fast-P estimate > `fast_p_queue_reject_secs` +//! -> 429 (`fast_p_queue`) with `x-estimated-wait-s`. +//! +//! Fail-open discipline: missing/stale snapshots never hard-fail a request +//! (the engine's own max-running-requests is the hard backstop); every +//! fail-open path emits a warn log + metric. All numeric thresholds are +//! tentative (measured values to be tuned in bench) and config-driven. + +use std::{ + collections::HashMap, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, RwLock, + }, + time::{Duration, Instant}, +}; + +use reqwest::Client; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use tracing::{debug, info, warn}; + +use crate::core::{ConnectionMode, Worker, WorkerRegistry, WorkerType}; + +const METRIC_DECISIONS: &str = "smg_g9i_decisions_total"; +const METRIC_REJECTS: &str = "smg_g9i_rejects_total"; +const METRIC_QUERY_LATENCY: &str = "smg_g9i_cache_hit_query_duration_seconds"; +const METRIC_NO_DATA: &str = "smg_g9i_no_fresh_snapshot_total"; + +/// G9i admission / fast-P routing configuration. +/// +/// All thresholds are **tentative** (user directive 2026-10-11: numbers get +/// fixed by measurement after the feature lands) and centralized here. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct G9iConfig { + /// Master switch; when false the router behaves exactly as before. + #[serde(default)] + pub enabled: bool, + /// Fast-P pool membership: prefill worker base URLs (e.g. + /// "http://127.0.0.1:30000"). Router-side role only — the engine does not + /// know about it. Empty = feature inert (everything routes as before). + #[serde(default)] + pub fast_prefill_urls: Vec, + /// Small-prefill cutoff on NEW tokens (total - prefix hit). Tentative 4096. + #[serde(default = "default_max_new_tokens")] + pub max_new_tokens: u64, + /// Reject with 429 when the least-wait fast-P node's estimated queue wait + /// exceeds this many seconds. Tentative 5.0. + #[serde(default = "default_fast_p_queue_reject_secs")] + pub fast_p_queue_reject_secs: f64, + /// D-pool admission ceiling: sum of instantaneous in-processing requests + /// across all decode nodes. Tentative 256 (instance-level; per-worker + /// semantics to be confirmed against /v1/loads dp-rank reporting). + #[serde(default = "default_decode_max_running")] + pub decode_max_running: u64, + /// Lower bound for drain rate in the est-wait denominator (tok/s). + /// Protects against div-by-zero and cold-start sliding-window noise. + #[serde(default = "default_drain_rate_floor")] + pub drain_rate_floor: f64, + /// Snapshot poll interval. + #[serde(default = "default_poll_interval_secs")] + pub poll_interval_secs: u64, + /// Snapshots older than this are stale: node leaves its pool's decision + /// set until refreshed. + #[serde(default = "default_snapshot_ttl_secs")] + pub snapshot_ttl_secs: u64, + /// Behavior when the D-pool has no fresh snapshot (endpoint broken): + /// "allow" (default, fail-open + warn; engine max-running is the hard + /// backstop) or "reject" (fail-closed). + #[serde(default = "default_d_no_snapshot")] + pub d_no_snapshot: String, + /// Timeout for the per-request /query_cache_hit probe. + #[serde(default = "default_cache_hit_query_timeout_ms")] + pub cache_hit_query_timeout_ms: u64, + /// Also route the regular P pool by least-wait (overriding the configured + /// prefill policy). Default false: regular pool keeps existing policy. + #[serde(default)] + pub regular_pool_least_wait: bool, +} + +fn default_max_new_tokens() -> u64 { + 4096 +} +fn default_fast_p_queue_reject_secs() -> f64 { + 5.0 +} +fn default_decode_max_running() -> u64 { + 256 +} +fn default_drain_rate_floor() -> f64 { + 1000.0 +} +fn default_poll_interval_secs() -> u64 { + 1 +} +fn default_snapshot_ttl_secs() -> u64 { + 5 +} +fn default_d_no_snapshot() -> String { + "allow".to_string() +} +fn default_cache_hit_query_timeout_ms() -> u64 { + 2000 +} + +impl Default for G9iConfig { + fn default() -> Self { + Self { + enabled: false, + fast_prefill_urls: Vec::new(), + max_new_tokens: default_max_new_tokens(), + fast_p_queue_reject_secs: default_fast_p_queue_reject_secs(), + decode_max_running: default_decode_max_running(), + drain_rate_floor: default_drain_rate_floor(), + poll_interval_secs: default_poll_interval_secs(), + snapshot_ttl_secs: default_snapshot_ttl_secs(), + d_no_snapshot: default_d_no_snapshot(), + cache_hit_query_timeout_ms: default_cache_hit_query_timeout_ms(), + regular_pool_least_wait: false, + } + } +} + +impl G9iConfig { + pub fn validate(&self) -> Result<(), String> { + if self.enabled && self.fast_prefill_urls.is_empty() { + return Err( + "g9i.enabled requires at least one --g9i-fast-prefill-url".to_string(), + ); + } + if !matches!(self.d_no_snapshot.as_str(), "allow" | "reject") { + return Err(format!( + "g9i.d_no_snapshot must be 'allow' or 'reject', got '{}'", + self.d_no_snapshot + )); + } + Ok(()) + } +} + +/// Aggregated per-node view (summed over DP ranks), refreshed by the poller. +#[derive(Debug, Clone, Default)] +pub struct NodeLoad { + /// Sum of num_running_reqs across dp ranks (instance-level). + pub running_reqs: u64, + /// Sum of queued_total_input_tokens across dp ranks. + pub queued_total_input_tokens: u64, + /// Sum of drain_rate_tokens_per_s across dp ranks. + pub drain_rate_tokens_per_s: f64, + fetched_at: Option, + fetch_ok: bool, +} + +impl NodeLoad { + fn est_wait_secs(&self, floor: f64) -> f64 { + self.queued_total_input_tokens as f64 / self.drain_rate_tokens_per_s.max(floor) + } + + fn is_fresh(&self, ttl: Duration) -> bool { + self.fetch_ok && self.fetched_at.is_some_and(|t| t.elapsed() <= ttl) + } +} + +/// Why a request was rejected (429). +#[derive(Debug, Clone, Copy)] +pub struct RejectInfo { + pub reason: &'static str, + pub estimated_wait_secs: Option, +} + +/// Per-request prefill routing outcome of the G9i classifier. +pub enum PrefillPlan { + /// Let the configured prefill policy decide (regular pool). + Default, + /// Force this worker base URL as the prefill target (least-wait pick). + Override(String), + /// Reject with 429 (fast-P queue over threshold). + Reject(RejectInfo), +} + +#[derive(Debug)] +pub struct G9iState { + config: G9iConfig, + /// Keyed by worker base_url (normalized, no trailing slash). + snapshots: RwLock>, + /// Whether any /v1/loads fetch ever succeeded (startup grace for D-gate). + had_any_success: AtomicBool, + monitor_started: AtomicBool, +} + +impl G9iState { + pub fn new(config: G9iConfig) -> Self { + Self { + config, + snapshots: RwLock::new(HashMap::new()), + had_any_success: AtomicBool::new(false), + monitor_started: AtomicBool::new(false), + } + } + + /// Spawn the background snapshot poller. Idempotent. + pub fn start_monitor(self: &Arc, registry: Arc, client: Client) { + if self + .monitor_started + .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) + .is_err() + { + return; + } + let state = Arc::clone(self); + let interval = Duration::from_secs(self.config.poll_interval_secs.max(1)); + info!( + "G9i monitor started: interval={:?} ttl={}s fast_p={:?} max_new={} d_max_running={}", + interval, + self.config.snapshot_ttl_secs, + self.config.fast_prefill_urls, + self.config.max_new_tokens, + self.config.decode_max_running, + ); + tokio::spawn(async move { + let mut timer = tokio::time::interval(interval); + loop { + timer.tick().await; + state.poll_once(®istry, &client).await; + } + }); + } + + async fn poll_once(&self, registry: &WorkerRegistry, client: &Client) { + let workers: Vec> = registry + .get_all() + .into_iter() + .filter(|w| { + matches!(w.worker_type(), WorkerType::Prefill { .. } | WorkerType::Decode) + && matches!(w.connection_mode(), ConnectionMode::Http) + }) + .collect(); + + let mut results: HashMap = HashMap::new(); + for worker in &workers { + let url = normalize_url(&worker.url().to_string()); + let load = match fetch_node_load(client, worker.as_ref()).await { + Ok(load) => { + self.had_any_success.store(true, Ordering::SeqCst); + load + } + Err(e) => { + debug!("G9i loads fetch failed for {}: {}", url, e); + NodeLoad::default() + } + }; + metrics::gauge!( + "smg_g9i_node_running_reqs", + "worker" => url.clone(), + ) + .set(load.running_reqs as f64); + results.insert(url, load); + } + + if let Ok(mut snap) = self.snapshots.write() { + for (url, load) in results { + snap.insert(url, load); + } + } + } + + fn snapshot(&self, url: &str) -> Option { + self.snapshots + .read() + .ok() + .and_then(|snap| snap.get(&normalize_url(url)).cloned()) + } + + /// Fresh, usable load for a worker, if any. + fn fresh_load(&self, url: &str) -> Option { + let ttl = Duration::from_secs(self.config.snapshot_ttl_secs); + let load = self.snapshot(url)?; + load.is_fresh(ttl).then_some(load) + } + + fn fast_pool<'a>(&self, prefill_workers: &'a [Arc]) -> Vec<&'a Arc> { + prefill_workers + .iter() + .filter(|w| { + self.config + .fast_prefill_urls + .iter() + .any(|f| urls_match(f, &w.url().to_string())) + }) + .collect() + } + + /// Least estimated wait among workers with fresh snapshots. + /// Returns (worker, est_wait_secs, fresh_count). + fn least_wait<'a>( + &self, + workers: &[&'a Arc], + ) -> (Option<&'a Arc>, f64, usize) { + let mut best: Option<(&Arc, f64)> = None; + let mut fresh_count = 0usize; + for w in workers { + if let Some(load) = self.fresh_load(&w.url().to_string()) { + fresh_count += 1; + let wait = load.est_wait_secs(self.config.drain_rate_floor); + if best.is_none() || wait < best.unwrap().1 { + best = Some((w, wait)); + } + } + } + (best.map(|(w, _)| w), best.map(|(_, t)| t).unwrap_or(0.0), fresh_count) + } + + /// Gate 1: D-pool aggregate admission. + /// + /// Returns `Err(RejectInfo)` when the decode pool must shed load. + /// Fail-open when no fresh snapshot exists (configurable), since the + /// engine-side max-running-requests remains the hard backstop. + pub async fn check_decode_pool(&self, registry: &WorkerRegistry) -> Result<(), RejectInfo> { + let ttl = Duration::from_secs(self.config.snapshot_ttl_secs); + let decode_workers: Vec> = registry + .get_decode_workers() + .into_iter() + .filter(|w| matches!(w.connection_mode(), ConnectionMode::Http)) + .collect(); + + let mut total_running = 0u64; + let mut fresh = 0usize; + for w in &decode_workers { + if let Some(load) = self.snapshot(&w.url().to_string()) { + if load.is_fresh(ttl) { + fresh += 1; + total_running += load.running_reqs; + } + } + } + + if fresh == 0 { + metrics::counter!(METRIC_NO_DATA, "pool" => "decode").increment(1); + if self.had_any_success.load(Ordering::SeqCst) + && self.config.d_no_snapshot == "reject" + { + warn!("G9i: no fresh decode snapshots, d_no_snapshot=reject -> 429"); + return Err(RejectInfo { + reason: "d_pool_no_snapshot", + estimated_wait_secs: None, + }); + } + // Startup grace or fail-open: nothing to enforce yet. + return Ok(()); + } + + metrics::gauge!("smg_g9i_d_pool_running_reqs").set(total_running as f64); + if total_running > self.config.decode_max_running { + return Err(RejectInfo { + reason: "d_pool_saturated", + estimated_wait_secs: None, + }); + } + Ok(()) + } + + /// Gate 2 + 3 + 4: classify small-prefill requests onto the fast-P pool + /// with least-wait selection and queue-wait rejection. + /// + /// `body` is the OpenAI chat request JSON. Non-chat routes must not call + /// this. All probe failures fall back to the default routing path + /// (fail-open, metric emitted). + pub async fn classify_prefill( + &self, + registry: &WorkerRegistry, + client: &Client, + body: &Value, + ) -> PrefillPlan { + let prefill_workers = registry.get_prefill_workers(); + let fast_pool = self.fast_pool(&prefill_workers); + if fast_pool.is_empty() { + metrics::counter!(METRIC_DECISIONS, "path" => "fast_pool_empty").increment(1); + return PrefillPlan::Default; + } + + // Probe the least-wait healthy fast-P for prefix-hit classification + // (L3 is shared, so hit lengths are comparable across fast-P nodes). + let (probe_target, _, fast_fresh) = self.least_wait(&fast_pool); + if fast_fresh == 0 { + metrics::counter!(METRIC_DECISIONS, "path" => "fast_pool_no_fresh_snapshot") + .increment(1); + return self.regular_plan(&prefill_workers); + } + let probe_target = match probe_target { + Some(w) => w, + None => return self.regular_plan(&prefill_workers), + }; + + let new_tokens = match self + .query_new_tokens(client, probe_target.as_ref(), body) + .await + { + Ok(Some(n)) => n, + Ok(None) => { + // Not classifiable (multimodal 400, engine error): regular path. + metrics::counter!(METRIC_DECISIONS, "path" => "query_unclassifiable") + .increment(1); + return self.regular_plan(&prefill_workers); + } + Err(e) => { + warn!( + "G9i /query_cache_hit probe failed on {}: {} — falling back to regular pool", + probe_target.url(), + e + ); + metrics::counter!(METRIC_DECISIONS, "path" => "query_error").increment(1); + return self.regular_plan(&prefill_workers); + } + }; + + if new_tokens >= self.config.max_new_tokens { + metrics::counter!(METRIC_DECISIONS, "path" => "regular_large_prefill").increment(1); + return self.regular_plan(&prefill_workers); + } + + // Small prefill -> fast-P pool, least-wait pick. + let (picked, est_wait, fresh) = self.least_wait(&fast_pool); + if fresh == 0 { + // Lost our snapshots between probe and pick; fail-open to regular. + metrics::counter!(METRIC_DECISIONS, "path" => "fast_pool_no_fresh_snapshot") + .increment(1); + return self.regular_plan(&prefill_workers); + } + let est_wait = if picked.is_some() { est_wait } else { 0.0 }; + if est_wait > self.config.fast_p_queue_reject_secs { + metrics::counter!(METRIC_REJECTS, "reason" => "fast_p_queue").increment(1); + return PrefillPlan::Reject(RejectInfo { + reason: "fast_p_queue", + estimated_wait_secs: Some(est_wait), + }); + } + match picked { + Some(w) => { + metrics::counter!(METRIC_DECISIONS, "path" => "routed_fast_p").increment(1); + debug!( + "G9i: small prefill (new={} < {}) -> fast-P {} est_wait={:.2}s", + new_tokens, + self.config.max_new_tokens, + w.url(), + est_wait, + ); + PrefillPlan::Override(normalize_url(&w.url().to_string())) + } + None => self.regular_plan(&prefill_workers), + } + } + + fn regular_plan(&self, prefill_workers: &[Arc]) -> PrefillPlan { + if !self.config.regular_pool_least_wait { + return PrefillPlan::Default; + } + let all: Vec<&Arc> = prefill_workers.iter().collect(); + let (picked, _, fresh) = self.least_wait(&all); + if fresh == 0 { + metrics::counter!(METRIC_DECISIONS, "path" => "regular_no_fresh_snapshot") + .increment(1); + return PrefillPlan::Default; + } + match picked { + Some(w) => { + metrics::counter!(METRIC_DECISIONS, "path" => "routed_regular_least_wait") + .increment(1); + PrefillPlan::Override(normalize_url(&w.url().to_string())) + } + None => PrefillPlan::Default, + } + } + + /// Probe a fast-P's /query_cache_hit; Ok(None) = "not classifiable". + async fn query_new_tokens( + &self, + client: &Client, + worker: &dyn Worker, + body: &Value, + ) -> Result, String> { + let url = format!("{}/query_cache_hit", normalize_url(&worker.url().to_string())); + let mut req = client + .post(&url) + .timeout(Duration::from_millis( + self.config.cache_hit_query_timeout_ms.max(200), + )) + .json(body); + if let Some(key) = worker.api_key() { + req = req.bearer_auth(key); + } + + let start = Instant::now(); + let resp = req.send().await.map_err(|e| e.to_string())?; + metrics::histogram!(METRIC_QUERY_LATENCY).record(start.elapsed().as_secs_f64()); + + if !resp.status().is_success() { + // 400 = multimodal/unsupported; anything else = engine-side issue. + // Either way: not classifiable, caller falls back. + return Ok(None); + } + let json: Value = resp.json().await.map_err(|e| e.to_string())?; + Ok(json + .get("new_tokens") + .and_then(|v| v.as_u64())) + } +} + +impl PrefillPlan { + pub fn into_override(self) -> Result, RejectInfo> { + match self { + PrefillPlan::Default => Ok(None), + PrefillPlan::Override(url) => Ok(Some(url)), + PrefillPlan::Reject(info) => Err(info), + } + } +} + +/// Fetch and aggregate one node's /v1/loads over its DP ranks. +async fn fetch_node_load(client: &Client, worker: &dyn Worker) -> Result { + let url = format!( + "{}/v1/loads?include=core", + normalize_url(&worker.url().to_string()) + ); + let mut req = client.get(&url).timeout(Duration::from_secs(3)); + if let Some(key) = worker.api_key() { + req = req.bearer_auth(key); + } + let resp = req.send().await.map_err(|e| e.to_string())?; + if !resp.status().is_success() { + return Err(format!("HTTP {}", resp.status())); + } + let json: Value = resp.json().await.map_err(|e| e.to_string())?; + let ranks = json + .get("loads") + .and_then(|l| l.as_array()) + .ok_or_else(|| "missing 'loads' array".to_string())?; + + let mut node = NodeLoad { + fetched_at: Some(Instant::now()), + fetch_ok: true, + ..Default::default() + }; + for rank in ranks { + node.running_reqs += rank + .get("num_running_reqs") + .and_then(|v| v.as_u64()) + .unwrap_or(0); + node.queued_total_input_tokens += rank + .get("queued_total_input_tokens") + .and_then(|v| v.as_u64()) + .unwrap_or(0); + node.drain_rate_tokens_per_s += rank + .get("drain_rate_tokens_per_s") + .and_then(|v| v.as_f64()) + .unwrap_or(0.0); + } + Ok(node) +} + +fn normalize_url(url: &str) -> String { + url.trim_end_matches('/').to_string() +} + +fn urls_match(configured: &str, worker_url: &str) -> bool { + normalize_url(configured) == normalize_url(worker_url) +} + +/// Build the G9i 429 admission response: reason + estimated wait in headers +/// (downstream reroutes on 429 without retrying — decision ①), JSON body +/// shaped like the router's standard error envelope. +pub fn reject_response(info: &RejectInfo) -> axum::response::Response { + use axum::http::{HeaderMap, HeaderValue, StatusCode}; + use axum::response::IntoResponse; + + let mut headers = HeaderMap::with_capacity(3); + headers.insert( + crate::routers::error::HEADER_X_SMG_ERROR_CODE, + HeaderValue::from_static(info.reason), + ); + headers.insert( + "x-reject-reason", + HeaderValue::from_static(info.reason), + ); + if let Some(wait) = info.estimated_wait_secs { + if let Ok(v) = HeaderValue::from_str(&format!("{wait:.2}")) { + headers.insert("x-estimated-wait-s", v); + } + } + let message = match info.estimated_wait_secs { + Some(wait) => format!( + "G9i admission rejected ({}): estimated wait {wait:.2}s exceeds the configured threshold", + info.reason + ), + None => format!("G9i admission rejected ({})", info.reason), + }; + ( + StatusCode::TOO_MANY_REQUESTS, + headers, + axum::Json(serde_json::json!({ + "error": { + "type": "Too Many Requests", + "code": info.reason, + "message": message, + } + })), + ) + .into_response() +} diff --git a/sgl-model-gateway/src/routers/http/mod.rs b/sgl-model-gateway/src/routers/http/mod.rs index 3f31b6f869..635f31b60b 100644 --- a/sgl-model-gateway/src/routers/http/mod.rs +++ b/sgl-model-gateway/src/routers/http/mod.rs @@ -1,5 +1,6 @@ //! HTTP router implementations +pub mod g9i; pub mod pd_router; pub mod pd_types; pub mod router; diff --git a/sgl-model-gateway/src/routers/http/pd_router.rs b/sgl-model-gateway/src/routers/http/pd_router.rs index d5ca6557fe..6889b10db2 100644 --- a/sgl-model-gateway/src/routers/http/pd_router.rs +++ b/sgl-model-gateway/src/routers/http/pd_router.rs @@ -58,6 +58,8 @@ pub struct PDRouter { pub retry_config: RetryConfig, pub api_key: Option, pub enable_igw: bool, + /// G9i pool admission + fast-P routing state; None = feature off. + pub g9i: Option>, } struct PreparedWorkerRequest<'a> { @@ -182,6 +184,14 @@ impl PDRouter { } pub async fn new(ctx: &Arc) -> Result { + let g9i = match &ctx.router_config.g9i { + Some(config) if config.enabled => { + let state = Arc::new(super::g9i::G9iState::new(config.clone())); + state.start_monitor(Arc::clone(&ctx.worker_registry), ctx.client.clone()); + Some(state) + } + _ => None, + }; Ok(PDRouter { worker_registry: Arc::clone(&ctx.worker_registry), policy_registry: Arc::clone(&ctx.policy_registry), @@ -189,6 +199,7 @@ impl PDRouter { retry_config: ctx.router_config.effective_retry_config(), api_key: ctx.router_config.api_key.clone(), enable_igw: ctx.router_config.enable_igw, + g9i, }) } @@ -390,6 +401,43 @@ impl PDRouter { // Clone request once outside the retry loop, then use Arc to share across attempts // This avoids O(retries) clones by sharing the same data let shared_request = Arc::new(original_request.clone()); + + // G9i admission (router-side, zero engine changes). Computed once per + // request, outside the retry loop: our own 429s must never be retried. + let g9i_prefill_override: Option = if let Some(g9i) = &self.g9i { + if let Err(info) = g9i.check_decode_pool(&self.worker_registry).await { + metrics::counter!("smg_g9i_rejects_total", "reason" => info.reason) + .increment(1); + return super::g9i::reject_response(&info); + } + if context.route == "/v1/chat/completions" { + match serde_json::to_value(original_request) { + Ok(body) => { + match g9i + .classify_prefill(&self.worker_registry, &self.client, &body) + .await + .into_override() + { + Ok(override_url) => override_url, + Err(info) => { + metrics::counter!( + "smg_g9i_rejects_total", + "reason" => info.reason + ) + .increment(1); + return super::g9i::reject_response(&info); + } + } + } + Err(_) => None, + } + } else { + None + } + } else { + None + }; + let response = RetryExecutor::execute_response_with_retry( &self.retry_config, { @@ -397,6 +445,7 @@ impl PDRouter { // Clone Arc (cheap reference count increment) instead of cloning the entire request let shared_request = Arc::clone(&shared_request); let context = context.clone(); + let g9i_prefill_override = g9i_prefill_override.clone(); async move { let (prefill, decode) = match self .select_pd_pair( @@ -412,6 +461,38 @@ impl PDRouter { } }; + // G9i least-wait override: route onto the classified + // fast-P node instead of the policy pick. + let (prefill, decode) = + if let Some(override_url) = g9i_prefill_override.as_ref() { + let matched = self + .worker_registry + .get_prefill_workers() + .into_iter() + .find(|w| { + w.url().trim_end_matches('/') == override_url.as_str() + && w.is_available() + }); + match matched { + Some(w) => { + debug!( + "G9i: overriding prefill selection with {}", + override_url + ); + (w, decode) + } + None => { + warn!( + "G9i: override target {} unavailable, keeping policy pick", + override_url + ); + (prefill, decode) + } + } + } else { + (prefill, decode) + }; + debug!( "PD retry attempt {} using prefill={} decode={}", attempt, @@ -1780,6 +1861,7 @@ mod tests { retry_config: RetryConfig::default(), api_key: Some("test_api_key".to_string()), enable_igw: false, + g9i: None, } }