G9i: router-side pool admission + fast-P routing (gateway)

Pool-aware admission control and small-prefill routing, entirely in
sgl-model-gateway (zero engine changes). Consumes G9 P0 engine surfaces:
/v1/loads (per-dp-rank snapshots, summed to per-node instance-level
views) and /query_cache_hit (new-token classification).

Decision tree per request, before PD dispatch and outside the retry
loop (own 429s are never retried):
  1. D-pool gate: sum of fresh num_running_reqs over decode pool >
     threshold -> 429 d_pool_saturated (fail-open on missing snapshots;
     engine max-running-requests is the hard backstop).
  2. Chat routes: probe least-wait fast-P /query_cache_hit; new_tokens
     < max_new_tokens (tentative 4096) -> fast-P pool.
  3. Pool selection: least estimated wait (queued_total_input_tokens /
     max(drain_rate, floor)) — doubles as the load balancer for the
     multi-fast-P / multi-D future; pool membership is config-only.
  4. Fast-P queue gate: least-wait estimate > threshold (tentative 5s)
     -> 429 fast_p_queue with x-estimated-wait-s.

CLI: --g9i-enabled/--g9i-fast-prefill-url/--g9i-max-new-tokens/
--g9i-fast-p-queue-reject-secs/--g9i-decode-max-running/... all numbers
tentative and centralized (user directive: fixed by measurement later).
Metrics: smg_g9i_decisions_total/rejects_total/no_fresh_snapshot_total,
per-node running gauge, probe latency histogram.
This commit is contained in:
2026-10-11 19:14:40 +08:00
parent b9d259441c
commit 39ceba8a3d
6 changed files with 789 additions and 0 deletions
+6
View File
@@ -378,6 +378,12 @@ impl RouterConfigBuilder {
self
}
/// G9i pool-aware admission control + fast-P routing config (PD mode).
pub fn g9i(mut self, g9i: Option<crate::routers::http::g9i::G9iConfig>) -> Self {
self.config.g9i = g9i;
self
}
pub fn model_path<S: Into<String>>(mut self, path: S) -> Self {
self.config.model_path = Some(path.into());
self
+4
View File
@@ -99,6 +99,9 @@ pub struct RouterConfig {
/// Enable WASM support
#[serde(default)]
pub enable_wasm: bool,
/// G9i pool-aware admission control + fast-P routing (PD mode, HTTP only).
#[serde(default)]
pub g9i: Option<crate::routers::http::g9i::G9iConfig>,
}
/// Tokenizer cache configuration
@@ -551,6 +554,7 @@ impl Default for RouterConfig {
ca_certificates: vec![],
mcp_config: None,
enable_wasm: false,
g9i: None,
server_cert: None,
server_key: None,
}
+68
View File
@@ -219,6 +219,51 @@ struct CliArgs {
#[arg(long, default_value_t = 30, help_heading = "PD Disaggregation")]
worker_startup_check_interval: u64,
// ==================== G9i: Pool Admission + Fast-P Routing ====================
/// Enable G9i pool-aware admission control + fast-P routing (PD HTTP mode)
#[arg(long, default_value_t = false, help_heading = "G9i Pool Admission")]
g9i_enabled: bool,
/// Fast-P pool member base URLs (repeatable; router-side role, engine unaware)
#[arg(long, action = ArgAction::Append, help_heading = "G9i Pool Admission")]
g9i_fast_prefill_url: Vec<String>,
/// Small-prefill cutoff on NEW tokens (total - prefix hit). Tentative; tune by bench.
#[arg(long, default_value_t = 4096, help_heading = "G9i Pool Admission")]
g9i_max_new_tokens: u64,
/// 429 when the least-wait fast-P queue estimate exceeds this many seconds. Tentative.
#[arg(long, default_value_t = 5.0, help_heading = "G9i Pool Admission")]
g9i_fast_p_queue_reject_secs: f64,
/// D-pool admission ceiling: sum of in-processing requests across decode nodes. Tentative.
#[arg(long, default_value_t = 256, help_heading = "G9i Pool Admission")]
g9i_decode_max_running: u64,
/// Drain-rate floor (tok/s) for queue-wait estimates (div-by-zero / cold-start guard)
#[arg(long, default_value_t = 1000.0, help_heading = "G9i Pool Admission")]
g9i_drain_rate_floor: f64,
/// Snapshot poll interval (seconds)
#[arg(long, default_value_t = 1, help_heading = "G9i Pool Admission")]
g9i_poll_interval_secs: u64,
/// Snapshot TTL — stale nodes leave pool decisions (seconds)
#[arg(long, default_value_t = 5, help_heading = "G9i Pool Admission")]
g9i_snapshot_ttl_secs: u64,
/// No fresh D-pool snapshot: "allow" (default, fail-open + warn) or "reject"
#[arg(long, default_value = "allow", value_parser = ["allow", "reject"], help_heading = "G9i Pool Admission")]
g9i_d_no_snapshot: String,
/// /query_cache_hit probe timeout per request (ms)
#[arg(long, default_value_t = 2000, help_heading = "G9i Pool Admission")]
g9i_cache_hit_query_timeout_ms: u64,
/// Also least-wait route the regular P pool (default: keep configured policy)
#[arg(long, default_value_t = false, help_heading = "G9i Pool Admission")]
g9i_regular_pool_least_wait: bool,
// ==================== Service Discovery (Kubernetes) ====================
/// Enable Kubernetes service discovery
#[arg(
@@ -1070,11 +1115,34 @@ impl CliArgs {
.circuit_breaker(!self.disable_circuit_breaker)
.enable_wasm(self.enable_wasm)
.igw(self.enable_igw)
.g9i(self.build_g9i_config()?)
.maybe_server_cert_and_key(self.tls_cert_path.as_ref(), self.tls_key_path.as_ref());
builder.build()
}
/// G9i pool admission + fast-P routing config; None when the feature is off.
fn build_g9i_config(&self) -> ConfigResult<Option<smg::routers::http::g9i::G9iConfig>> {
if !self.g9i_enabled && self.g9i_fast_prefill_url.is_empty() {
return Ok(None);
}
let config = smg::routers::http::g9i::G9iConfig {
enabled: self.g9i_enabled,
fast_prefill_urls: self.g9i_fast_prefill_url.clone(),
max_new_tokens: self.g9i_max_new_tokens,
fast_p_queue_reject_secs: self.g9i_fast_p_queue_reject_secs,
decode_max_running: self.g9i_decode_max_running,
drain_rate_floor: self.g9i_drain_rate_floor,
poll_interval_secs: self.g9i_poll_interval_secs,
snapshot_ttl_secs: self.g9i_snapshot_ttl_secs,
d_no_snapshot: self.g9i_d_no_snapshot.clone(),
cache_hit_query_timeout_ms: self.g9i_cache_hit_query_timeout_ms,
regular_pool_least_wait: self.g9i_regular_pool_least_wait,
};
config.validate().map_err(|reason| ConfigError::ValidationFailed { reason })?;
Ok(Some(config))
}
fn to_server_config(&self, router_config: RouterConfig) -> ServerConfig {
let service_discovery_config = if self.service_discovery {
// Get router discovery config from router_config.discovery if available
+628
View File
@@ -0,0 +1,628 @@
//! G9i: pool-aware admission control + fast-P routing for PD mode.
//!
//! Router-side only — zero engine changes. Consumes two engine surfaces that
//! already exist (G9 P0):
//! * `GET /v1/loads?include=core` — per-DP-rank `LoadSnapshot` list; we sum
//! `num_running_reqs` / `queued_total_input_tokens` /
//! `drain_rate_tokens_per_s` across ranks into per-node, pool-aggregated
//! views (poll loop below, independent of `LoadMonitor`, whose
//! `aggregate.total_tokens` parse target does not exist on this engine).
//! * `POST /query_cache_hit` — chat-body prefix-hit probe used to classify
//! small-prefill (new tokens < threshold) requests.
//!
//! Decision tree (per request, before PD dispatch):
//! 1. D-pool gate: sum of fresh `num_running_reqs` over the decode pool
//! exceeds `decode_max_running` -> 429 (`d_pool_saturated`).
//! 2. Small-prefill classification (chat routes only): probe one fast-P,
//! `new_tokens < max_new_tokens` -> fast-P pool, else regular P pool.
//! 3. Pool selection: least estimated wait
//! (`queued_total_input_tokens / max(drain_rate, drain_rate_floor)`)
//! among fresh nodes — this is also the load balancer.
//! 4. Fast-P queue gate: least-wait fast-P estimate > `fast_p_queue_reject_secs`
//! -> 429 (`fast_p_queue`) with `x-estimated-wait-s`.
//!
//! Fail-open discipline: missing/stale snapshots never hard-fail a request
//! (the engine's own max-running-requests is the hard backstop); every
//! fail-open path emits a warn log + metric. All numeric thresholds are
//! tentative (measured values to be tuned in bench) and config-driven.
use std::{
collections::HashMap,
sync::{
atomic::{AtomicBool, Ordering},
Arc, RwLock,
},
time::{Duration, Instant},
};
use reqwest::Client;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use tracing::{debug, info, warn};
use crate::core::{ConnectionMode, Worker, WorkerRegistry, WorkerType};
const METRIC_DECISIONS: &str = "smg_g9i_decisions_total";
const METRIC_REJECTS: &str = "smg_g9i_rejects_total";
const METRIC_QUERY_LATENCY: &str = "smg_g9i_cache_hit_query_duration_seconds";
const METRIC_NO_DATA: &str = "smg_g9i_no_fresh_snapshot_total";
/// G9i admission / fast-P routing configuration.
///
/// All thresholds are **tentative** (user directive 2026-10-11: numbers get
/// fixed by measurement after the feature lands) and centralized here.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct G9iConfig {
/// Master switch; when false the router behaves exactly as before.
#[serde(default)]
pub enabled: bool,
/// Fast-P pool membership: prefill worker base URLs (e.g.
/// "http://127.0.0.1:30000"). Router-side role only — the engine does not
/// know about it. Empty = feature inert (everything routes as before).
#[serde(default)]
pub fast_prefill_urls: Vec<String>,
/// Small-prefill cutoff on NEW tokens (total - prefix hit). Tentative 4096.
#[serde(default = "default_max_new_tokens")]
pub max_new_tokens: u64,
/// Reject with 429 when the least-wait fast-P node's estimated queue wait
/// exceeds this many seconds. Tentative 5.0.
#[serde(default = "default_fast_p_queue_reject_secs")]
pub fast_p_queue_reject_secs: f64,
/// D-pool admission ceiling: sum of instantaneous in-processing requests
/// across all decode nodes. Tentative 256 (instance-level; per-worker
/// semantics to be confirmed against /v1/loads dp-rank reporting).
#[serde(default = "default_decode_max_running")]
pub decode_max_running: u64,
/// Lower bound for drain rate in the est-wait denominator (tok/s).
/// Protects against div-by-zero and cold-start sliding-window noise.
#[serde(default = "default_drain_rate_floor")]
pub drain_rate_floor: f64,
/// Snapshot poll interval.
#[serde(default = "default_poll_interval_secs")]
pub poll_interval_secs: u64,
/// Snapshots older than this are stale: node leaves its pool's decision
/// set until refreshed.
#[serde(default = "default_snapshot_ttl_secs")]
pub snapshot_ttl_secs: u64,
/// Behavior when the D-pool has no fresh snapshot (endpoint broken):
/// "allow" (default, fail-open + warn; engine max-running is the hard
/// backstop) or "reject" (fail-closed).
#[serde(default = "default_d_no_snapshot")]
pub d_no_snapshot: String,
/// Timeout for the per-request /query_cache_hit probe.
#[serde(default = "default_cache_hit_query_timeout_ms")]
pub cache_hit_query_timeout_ms: u64,
/// Also route the regular P pool by least-wait (overriding the configured
/// prefill policy). Default false: regular pool keeps existing policy.
#[serde(default)]
pub regular_pool_least_wait: bool,
}
fn default_max_new_tokens() -> u64 {
4096
}
fn default_fast_p_queue_reject_secs() -> f64 {
5.0
}
fn default_decode_max_running() -> u64 {
256
}
fn default_drain_rate_floor() -> f64 {
1000.0
}
fn default_poll_interval_secs() -> u64 {
1
}
fn default_snapshot_ttl_secs() -> u64 {
5
}
fn default_d_no_snapshot() -> String {
"allow".to_string()
}
fn default_cache_hit_query_timeout_ms() -> u64 {
2000
}
impl Default for G9iConfig {
fn default() -> Self {
Self {
enabled: false,
fast_prefill_urls: Vec::new(),
max_new_tokens: default_max_new_tokens(),
fast_p_queue_reject_secs: default_fast_p_queue_reject_secs(),
decode_max_running: default_decode_max_running(),
drain_rate_floor: default_drain_rate_floor(),
poll_interval_secs: default_poll_interval_secs(),
snapshot_ttl_secs: default_snapshot_ttl_secs(),
d_no_snapshot: default_d_no_snapshot(),
cache_hit_query_timeout_ms: default_cache_hit_query_timeout_ms(),
regular_pool_least_wait: false,
}
}
}
impl G9iConfig {
pub fn validate(&self) -> Result<(), String> {
if self.enabled && self.fast_prefill_urls.is_empty() {
return Err(
"g9i.enabled requires at least one --g9i-fast-prefill-url".to_string(),
);
}
if !matches!(self.d_no_snapshot.as_str(), "allow" | "reject") {
return Err(format!(
"g9i.d_no_snapshot must be 'allow' or 'reject', got '{}'",
self.d_no_snapshot
));
}
Ok(())
}
}
/// Aggregated per-node view (summed over DP ranks), refreshed by the poller.
#[derive(Debug, Clone, Default)]
pub struct NodeLoad {
/// Sum of num_running_reqs across dp ranks (instance-level).
pub running_reqs: u64,
/// Sum of queued_total_input_tokens across dp ranks.
pub queued_total_input_tokens: u64,
/// Sum of drain_rate_tokens_per_s across dp ranks.
pub drain_rate_tokens_per_s: f64,
fetched_at: Option<Instant>,
fetch_ok: bool,
}
impl NodeLoad {
fn est_wait_secs(&self, floor: f64) -> f64 {
self.queued_total_input_tokens as f64 / self.drain_rate_tokens_per_s.max(floor)
}
fn is_fresh(&self, ttl: Duration) -> bool {
self.fetch_ok && self.fetched_at.is_some_and(|t| t.elapsed() <= ttl)
}
}
/// Why a request was rejected (429).
#[derive(Debug, Clone, Copy)]
pub struct RejectInfo {
pub reason: &'static str,
pub estimated_wait_secs: Option<f64>,
}
/// Per-request prefill routing outcome of the G9i classifier.
pub enum PrefillPlan {
/// Let the configured prefill policy decide (regular pool).
Default,
/// Force this worker base URL as the prefill target (least-wait pick).
Override(String),
/// Reject with 429 (fast-P queue over threshold).
Reject(RejectInfo),
}
#[derive(Debug)]
pub struct G9iState {
config: G9iConfig,
/// Keyed by worker base_url (normalized, no trailing slash).
snapshots: RwLock<HashMap<String, NodeLoad>>,
/// Whether any /v1/loads fetch ever succeeded (startup grace for D-gate).
had_any_success: AtomicBool,
monitor_started: AtomicBool,
}
impl G9iState {
pub fn new(config: G9iConfig) -> Self {
Self {
config,
snapshots: RwLock::new(HashMap::new()),
had_any_success: AtomicBool::new(false),
monitor_started: AtomicBool::new(false),
}
}
/// Spawn the background snapshot poller. Idempotent.
pub fn start_monitor(self: &Arc<Self>, registry: Arc<WorkerRegistry>, client: Client) {
if self
.monitor_started
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
.is_err()
{
return;
}
let state = Arc::clone(self);
let interval = Duration::from_secs(self.config.poll_interval_secs.max(1));
info!(
"G9i monitor started: interval={:?} ttl={}s fast_p={:?} max_new={} d_max_running={}",
interval,
self.config.snapshot_ttl_secs,
self.config.fast_prefill_urls,
self.config.max_new_tokens,
self.config.decode_max_running,
);
tokio::spawn(async move {
let mut timer = tokio::time::interval(interval);
loop {
timer.tick().await;
state.poll_once(&registry, &client).await;
}
});
}
async fn poll_once(&self, registry: &WorkerRegistry, client: &Client) {
let workers: Vec<Arc<dyn Worker>> = registry
.get_all()
.into_iter()
.filter(|w| {
matches!(w.worker_type(), WorkerType::Prefill { .. } | WorkerType::Decode)
&& matches!(w.connection_mode(), ConnectionMode::Http)
})
.collect();
let mut results: HashMap<String, NodeLoad> = HashMap::new();
for worker in &workers {
let url = normalize_url(&worker.url().to_string());
let load = match fetch_node_load(client, worker.as_ref()).await {
Ok(load) => {
self.had_any_success.store(true, Ordering::SeqCst);
load
}
Err(e) => {
debug!("G9i loads fetch failed for {}: {}", url, e);
NodeLoad::default()
}
};
metrics::gauge!(
"smg_g9i_node_running_reqs",
"worker" => url.clone(),
)
.set(load.running_reqs as f64);
results.insert(url, load);
}
if let Ok(mut snap) = self.snapshots.write() {
for (url, load) in results {
snap.insert(url, load);
}
}
}
fn snapshot(&self, url: &str) -> Option<NodeLoad> {
self.snapshots
.read()
.ok()
.and_then(|snap| snap.get(&normalize_url(url)).cloned())
}
/// Fresh, usable load for a worker, if any.
fn fresh_load(&self, url: &str) -> Option<NodeLoad> {
let ttl = Duration::from_secs(self.config.snapshot_ttl_secs);
let load = self.snapshot(url)?;
load.is_fresh(ttl).then_some(load)
}
fn fast_pool<'a>(&self, prefill_workers: &'a [Arc<dyn Worker>]) -> Vec<&'a Arc<dyn Worker>> {
prefill_workers
.iter()
.filter(|w| {
self.config
.fast_prefill_urls
.iter()
.any(|f| urls_match(f, &w.url().to_string()))
})
.collect()
}
/// Least estimated wait among workers with fresh snapshots.
/// Returns (worker, est_wait_secs, fresh_count).
fn least_wait<'a>(
&self,
workers: &[&'a Arc<dyn Worker>],
) -> (Option<&'a Arc<dyn Worker>>, f64, usize) {
let mut best: Option<(&Arc<dyn Worker>, f64)> = None;
let mut fresh_count = 0usize;
for w in workers {
if let Some(load) = self.fresh_load(&w.url().to_string()) {
fresh_count += 1;
let wait = load.est_wait_secs(self.config.drain_rate_floor);
if best.is_none() || wait < best.unwrap().1 {
best = Some((w, wait));
}
}
}
(best.map(|(w, _)| w), best.map(|(_, t)| t).unwrap_or(0.0), fresh_count)
}
/// Gate 1: D-pool aggregate admission.
///
/// Returns `Err(RejectInfo)` when the decode pool must shed load.
/// Fail-open when no fresh snapshot exists (configurable), since the
/// engine-side max-running-requests remains the hard backstop.
pub async fn check_decode_pool(&self, registry: &WorkerRegistry) -> Result<(), RejectInfo> {
let ttl = Duration::from_secs(self.config.snapshot_ttl_secs);
let decode_workers: Vec<Arc<dyn Worker>> = registry
.get_decode_workers()
.into_iter()
.filter(|w| matches!(w.connection_mode(), ConnectionMode::Http))
.collect();
let mut total_running = 0u64;
let mut fresh = 0usize;
for w in &decode_workers {
if let Some(load) = self.snapshot(&w.url().to_string()) {
if load.is_fresh(ttl) {
fresh += 1;
total_running += load.running_reqs;
}
}
}
if fresh == 0 {
metrics::counter!(METRIC_NO_DATA, "pool" => "decode").increment(1);
if self.had_any_success.load(Ordering::SeqCst)
&& self.config.d_no_snapshot == "reject"
{
warn!("G9i: no fresh decode snapshots, d_no_snapshot=reject -> 429");
return Err(RejectInfo {
reason: "d_pool_no_snapshot",
estimated_wait_secs: None,
});
}
// Startup grace or fail-open: nothing to enforce yet.
return Ok(());
}
metrics::gauge!("smg_g9i_d_pool_running_reqs").set(total_running as f64);
if total_running > self.config.decode_max_running {
return Err(RejectInfo {
reason: "d_pool_saturated",
estimated_wait_secs: None,
});
}
Ok(())
}
/// Gate 2 + 3 + 4: classify small-prefill requests onto the fast-P pool
/// with least-wait selection and queue-wait rejection.
///
/// `body` is the OpenAI chat request JSON. Non-chat routes must not call
/// this. All probe failures fall back to the default routing path
/// (fail-open, metric emitted).
pub async fn classify_prefill(
&self,
registry: &WorkerRegistry,
client: &Client,
body: &Value,
) -> PrefillPlan {
let prefill_workers = registry.get_prefill_workers();
let fast_pool = self.fast_pool(&prefill_workers);
if fast_pool.is_empty() {
metrics::counter!(METRIC_DECISIONS, "path" => "fast_pool_empty").increment(1);
return PrefillPlan::Default;
}
// Probe the least-wait healthy fast-P for prefix-hit classification
// (L3 is shared, so hit lengths are comparable across fast-P nodes).
let (probe_target, _, fast_fresh) = self.least_wait(&fast_pool);
if fast_fresh == 0 {
metrics::counter!(METRIC_DECISIONS, "path" => "fast_pool_no_fresh_snapshot")
.increment(1);
return self.regular_plan(&prefill_workers);
}
let probe_target = match probe_target {
Some(w) => w,
None => return self.regular_plan(&prefill_workers),
};
let new_tokens = match self
.query_new_tokens(client, probe_target.as_ref(), body)
.await
{
Ok(Some(n)) => n,
Ok(None) => {
// Not classifiable (multimodal 400, engine error): regular path.
metrics::counter!(METRIC_DECISIONS, "path" => "query_unclassifiable")
.increment(1);
return self.regular_plan(&prefill_workers);
}
Err(e) => {
warn!(
"G9i /query_cache_hit probe failed on {}: {} — falling back to regular pool",
probe_target.url(),
e
);
metrics::counter!(METRIC_DECISIONS, "path" => "query_error").increment(1);
return self.regular_plan(&prefill_workers);
}
};
if new_tokens >= self.config.max_new_tokens {
metrics::counter!(METRIC_DECISIONS, "path" => "regular_large_prefill").increment(1);
return self.regular_plan(&prefill_workers);
}
// Small prefill -> fast-P pool, least-wait pick.
let (picked, est_wait, fresh) = self.least_wait(&fast_pool);
if fresh == 0 {
// Lost our snapshots between probe and pick; fail-open to regular.
metrics::counter!(METRIC_DECISIONS, "path" => "fast_pool_no_fresh_snapshot")
.increment(1);
return self.regular_plan(&prefill_workers);
}
let est_wait = if picked.is_some() { est_wait } else { 0.0 };
if est_wait > self.config.fast_p_queue_reject_secs {
metrics::counter!(METRIC_REJECTS, "reason" => "fast_p_queue").increment(1);
return PrefillPlan::Reject(RejectInfo {
reason: "fast_p_queue",
estimated_wait_secs: Some(est_wait),
});
}
match picked {
Some(w) => {
metrics::counter!(METRIC_DECISIONS, "path" => "routed_fast_p").increment(1);
debug!(
"G9i: small prefill (new={} < {}) -> fast-P {} est_wait={:.2}s",
new_tokens,
self.config.max_new_tokens,
w.url(),
est_wait,
);
PrefillPlan::Override(normalize_url(&w.url().to_string()))
}
None => self.regular_plan(&prefill_workers),
}
}
fn regular_plan(&self, prefill_workers: &[Arc<dyn Worker>]) -> PrefillPlan {
if !self.config.regular_pool_least_wait {
return PrefillPlan::Default;
}
let all: Vec<&Arc<dyn Worker>> = prefill_workers.iter().collect();
let (picked, _, fresh) = self.least_wait(&all);
if fresh == 0 {
metrics::counter!(METRIC_DECISIONS, "path" => "regular_no_fresh_snapshot")
.increment(1);
return PrefillPlan::Default;
}
match picked {
Some(w) => {
metrics::counter!(METRIC_DECISIONS, "path" => "routed_regular_least_wait")
.increment(1);
PrefillPlan::Override(normalize_url(&w.url().to_string()))
}
None => PrefillPlan::Default,
}
}
/// Probe a fast-P's /query_cache_hit; Ok(None) = "not classifiable".
async fn query_new_tokens(
&self,
client: &Client,
worker: &dyn Worker,
body: &Value,
) -> Result<Option<u64>, String> {
let url = format!("{}/query_cache_hit", normalize_url(&worker.url().to_string()));
let mut req = client
.post(&url)
.timeout(Duration::from_millis(
self.config.cache_hit_query_timeout_ms.max(200),
))
.json(body);
if let Some(key) = worker.api_key() {
req = req.bearer_auth(key);
}
let start = Instant::now();
let resp = req.send().await.map_err(|e| e.to_string())?;
metrics::histogram!(METRIC_QUERY_LATENCY).record(start.elapsed().as_secs_f64());
if !resp.status().is_success() {
// 400 = multimodal/unsupported; anything else = engine-side issue.
// Either way: not classifiable, caller falls back.
return Ok(None);
}
let json: Value = resp.json().await.map_err(|e| e.to_string())?;
Ok(json
.get("new_tokens")
.and_then(|v| v.as_u64()))
}
}
impl PrefillPlan {
pub fn into_override(self) -> Result<Option<String>, RejectInfo> {
match self {
PrefillPlan::Default => Ok(None),
PrefillPlan::Override(url) => Ok(Some(url)),
PrefillPlan::Reject(info) => Err(info),
}
}
}
/// Fetch and aggregate one node's /v1/loads over its DP ranks.
async fn fetch_node_load(client: &Client, worker: &dyn Worker) -> Result<NodeLoad, String> {
let url = format!(
"{}/v1/loads?include=core",
normalize_url(&worker.url().to_string())
);
let mut req = client.get(&url).timeout(Duration::from_secs(3));
if let Some(key) = worker.api_key() {
req = req.bearer_auth(key);
}
let resp = req.send().await.map_err(|e| e.to_string())?;
if !resp.status().is_success() {
return Err(format!("HTTP {}", resp.status()));
}
let json: Value = resp.json().await.map_err(|e| e.to_string())?;
let ranks = json
.get("loads")
.and_then(|l| l.as_array())
.ok_or_else(|| "missing 'loads' array".to_string())?;
let mut node = NodeLoad {
fetched_at: Some(Instant::now()),
fetch_ok: true,
..Default::default()
};
for rank in ranks {
node.running_reqs += rank
.get("num_running_reqs")
.and_then(|v| v.as_u64())
.unwrap_or(0);
node.queued_total_input_tokens += rank
.get("queued_total_input_tokens")
.and_then(|v| v.as_u64())
.unwrap_or(0);
node.drain_rate_tokens_per_s += rank
.get("drain_rate_tokens_per_s")
.and_then(|v| v.as_f64())
.unwrap_or(0.0);
}
Ok(node)
}
fn normalize_url(url: &str) -> String {
url.trim_end_matches('/').to_string()
}
fn urls_match(configured: &str, worker_url: &str) -> bool {
normalize_url(configured) == normalize_url(worker_url)
}
/// Build the G9i 429 admission response: reason + estimated wait in headers
/// (downstream reroutes on 429 without retrying — decision ①), JSON body
/// shaped like the router's standard error envelope.
pub fn reject_response(info: &RejectInfo) -> axum::response::Response {
use axum::http::{HeaderMap, HeaderValue, StatusCode};
use axum::response::IntoResponse;
let mut headers = HeaderMap::with_capacity(3);
headers.insert(
crate::routers::error::HEADER_X_SMG_ERROR_CODE,
HeaderValue::from_static(info.reason),
);
headers.insert(
"x-reject-reason",
HeaderValue::from_static(info.reason),
);
if let Some(wait) = info.estimated_wait_secs {
if let Ok(v) = HeaderValue::from_str(&format!("{wait:.2}")) {
headers.insert("x-estimated-wait-s", v);
}
}
let message = match info.estimated_wait_secs {
Some(wait) => format!(
"G9i admission rejected ({}): estimated wait {wait:.2}s exceeds the configured threshold",
info.reason
),
None => format!("G9i admission rejected ({})", info.reason),
};
(
StatusCode::TOO_MANY_REQUESTS,
headers,
axum::Json(serde_json::json!({
"error": {
"type": "Too Many Requests",
"code": info.reason,
"message": message,
}
})),
)
.into_response()
}
@@ -1,5 +1,6 @@
//! HTTP router implementations
pub mod g9i;
pub mod pd_router;
pub mod pd_types;
pub mod router;
@@ -58,6 +58,8 @@ pub struct PDRouter {
pub retry_config: RetryConfig,
pub api_key: Option<String>,
pub enable_igw: bool,
/// G9i pool admission + fast-P routing state; None = feature off.
pub g9i: Option<Arc<super::g9i::G9iState>>,
}
struct PreparedWorkerRequest<'a> {
@@ -182,6 +184,14 @@ impl PDRouter {
}
pub async fn new(ctx: &Arc<crate::app_context::AppContext>) -> Result<Self, String> {
let g9i = match &ctx.router_config.g9i {
Some(config) if config.enabled => {
let state = Arc::new(super::g9i::G9iState::new(config.clone()));
state.start_monitor(Arc::clone(&ctx.worker_registry), ctx.client.clone());
Some(state)
}
_ => None,
};
Ok(PDRouter {
worker_registry: Arc::clone(&ctx.worker_registry),
policy_registry: Arc::clone(&ctx.policy_registry),
@@ -189,6 +199,7 @@ impl PDRouter {
retry_config: ctx.router_config.effective_retry_config(),
api_key: ctx.router_config.api_key.clone(),
enable_igw: ctx.router_config.enable_igw,
g9i,
})
}
@@ -390,6 +401,43 @@ impl PDRouter {
// Clone request once outside the retry loop, then use Arc to share across attempts
// This avoids O(retries) clones by sharing the same data
let shared_request = Arc::new(original_request.clone());
// G9i admission (router-side, zero engine changes). Computed once per
// request, outside the retry loop: our own 429s must never be retried.
let g9i_prefill_override: Option<String> = if let Some(g9i) = &self.g9i {
if let Err(info) = g9i.check_decode_pool(&self.worker_registry).await {
metrics::counter!("smg_g9i_rejects_total", "reason" => info.reason)
.increment(1);
return super::g9i::reject_response(&info);
}
if context.route == "/v1/chat/completions" {
match serde_json::to_value(original_request) {
Ok(body) => {
match g9i
.classify_prefill(&self.worker_registry, &self.client, &body)
.await
.into_override()
{
Ok(override_url) => override_url,
Err(info) => {
metrics::counter!(
"smg_g9i_rejects_total",
"reason" => info.reason
)
.increment(1);
return super::g9i::reject_response(&info);
}
}
}
Err(_) => None,
}
} else {
None
}
} else {
None
};
let response = RetryExecutor::execute_response_with_retry(
&self.retry_config,
{
@@ -397,6 +445,7 @@ impl PDRouter {
// Clone Arc (cheap reference count increment) instead of cloning the entire request
let shared_request = Arc::clone(&shared_request);
let context = context.clone();
let g9i_prefill_override = g9i_prefill_override.clone();
async move {
let (prefill, decode) = match self
.select_pd_pair(
@@ -412,6 +461,38 @@ impl PDRouter {
}
};
// G9i least-wait override: route onto the classified
// fast-P node instead of the policy pick.
let (prefill, decode) =
if let Some(override_url) = g9i_prefill_override.as_ref() {
let matched = self
.worker_registry
.get_prefill_workers()
.into_iter()
.find(|w| {
w.url().trim_end_matches('/') == override_url.as_str()
&& w.is_available()
});
match matched {
Some(w) => {
debug!(
"G9i: overriding prefill selection with {}",
override_url
);
(w, decode)
}
None => {
warn!(
"G9i: override target {} unavailable, keeping policy pick",
override_url
);
(prefill, decode)
}
}
} else {
(prefill, decode)
};
debug!(
"PD retry attempt {} using prefill={} decode={}",
attempt,
@@ -1780,6 +1861,7 @@ mod tests {
retry_config: RetryConfig::default(),
api_key: Some("test_api_key".to_string()),
enable_igw: false,
g9i: None,
}
}