G9i: router-side pool admission + fast-P routing (gateway)
Pool-aware admission control and small-prefill routing, entirely in
sgl-model-gateway (zero engine changes). Consumes G9 P0 engine surfaces:
/v1/loads (per-dp-rank snapshots, summed to per-node instance-level
views) and /query_cache_hit (new-token classification).
Decision tree per request, before PD dispatch and outside the retry
loop (own 429s are never retried):
1. D-pool gate: sum of fresh num_running_reqs over decode pool >
threshold -> 429 d_pool_saturated (fail-open on missing snapshots;
engine max-running-requests is the hard backstop).
2. Chat routes: probe least-wait fast-P /query_cache_hit; new_tokens
< max_new_tokens (tentative 4096) -> fast-P pool.
3. Pool selection: least estimated wait (queued_total_input_tokens /
max(drain_rate, floor)) — doubles as the load balancer for the
multi-fast-P / multi-D future; pool membership is config-only.
4. Fast-P queue gate: least-wait estimate > threshold (tentative 5s)
-> 429 fast_p_queue with x-estimated-wait-s.
CLI: --g9i-enabled/--g9i-fast-prefill-url/--g9i-max-new-tokens/
--g9i-fast-p-queue-reject-secs/--g9i-decode-max-running/... all numbers
tentative and centralized (user directive: fixed by measurement later).
Metrics: smg_g9i_decisions_total/rejects_total/no_fresh_snapshot_total,
per-node running gauge, probe latency histogram.
This commit is contained in:
@@ -378,6 +378,12 @@ impl RouterConfigBuilder {
|
||||
self
|
||||
}
|
||||
|
||||
/// G9i pool-aware admission control + fast-P routing config (PD mode).
|
||||
pub fn g9i(mut self, g9i: Option<crate::routers::http::g9i::G9iConfig>) -> Self {
|
||||
self.config.g9i = g9i;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn model_path<S: Into<String>>(mut self, path: S) -> Self {
|
||||
self.config.model_path = Some(path.into());
|
||||
self
|
||||
|
||||
@@ -99,6 +99,9 @@ pub struct RouterConfig {
|
||||
/// Enable WASM support
|
||||
#[serde(default)]
|
||||
pub enable_wasm: bool,
|
||||
/// G9i pool-aware admission control + fast-P routing (PD mode, HTTP only).
|
||||
#[serde(default)]
|
||||
pub g9i: Option<crate::routers::http::g9i::G9iConfig>,
|
||||
}
|
||||
|
||||
/// Tokenizer cache configuration
|
||||
@@ -551,6 +554,7 @@ impl Default for RouterConfig {
|
||||
ca_certificates: vec![],
|
||||
mcp_config: None,
|
||||
enable_wasm: false,
|
||||
g9i: None,
|
||||
server_cert: None,
|
||||
server_key: None,
|
||||
}
|
||||
|
||||
@@ -219,6 +219,51 @@ struct CliArgs {
|
||||
#[arg(long, default_value_t = 30, help_heading = "PD Disaggregation")]
|
||||
worker_startup_check_interval: u64,
|
||||
|
||||
// ==================== G9i: Pool Admission + Fast-P Routing ====================
|
||||
/// Enable G9i pool-aware admission control + fast-P routing (PD HTTP mode)
|
||||
#[arg(long, default_value_t = false, help_heading = "G9i Pool Admission")]
|
||||
g9i_enabled: bool,
|
||||
|
||||
/// Fast-P pool member base URLs (repeatable; router-side role, engine unaware)
|
||||
#[arg(long, action = ArgAction::Append, help_heading = "G9i Pool Admission")]
|
||||
g9i_fast_prefill_url: Vec<String>,
|
||||
|
||||
/// Small-prefill cutoff on NEW tokens (total - prefix hit). Tentative; tune by bench.
|
||||
#[arg(long, default_value_t = 4096, help_heading = "G9i Pool Admission")]
|
||||
g9i_max_new_tokens: u64,
|
||||
|
||||
/// 429 when the least-wait fast-P queue estimate exceeds this many seconds. Tentative.
|
||||
#[arg(long, default_value_t = 5.0, help_heading = "G9i Pool Admission")]
|
||||
g9i_fast_p_queue_reject_secs: f64,
|
||||
|
||||
/// D-pool admission ceiling: sum of in-processing requests across decode nodes. Tentative.
|
||||
#[arg(long, default_value_t = 256, help_heading = "G9i Pool Admission")]
|
||||
g9i_decode_max_running: u64,
|
||||
|
||||
/// Drain-rate floor (tok/s) for queue-wait estimates (div-by-zero / cold-start guard)
|
||||
#[arg(long, default_value_t = 1000.0, help_heading = "G9i Pool Admission")]
|
||||
g9i_drain_rate_floor: f64,
|
||||
|
||||
/// Snapshot poll interval (seconds)
|
||||
#[arg(long, default_value_t = 1, help_heading = "G9i Pool Admission")]
|
||||
g9i_poll_interval_secs: u64,
|
||||
|
||||
/// Snapshot TTL — stale nodes leave pool decisions (seconds)
|
||||
#[arg(long, default_value_t = 5, help_heading = "G9i Pool Admission")]
|
||||
g9i_snapshot_ttl_secs: u64,
|
||||
|
||||
/// No fresh D-pool snapshot: "allow" (default, fail-open + warn) or "reject"
|
||||
#[arg(long, default_value = "allow", value_parser = ["allow", "reject"], help_heading = "G9i Pool Admission")]
|
||||
g9i_d_no_snapshot: String,
|
||||
|
||||
/// /query_cache_hit probe timeout per request (ms)
|
||||
#[arg(long, default_value_t = 2000, help_heading = "G9i Pool Admission")]
|
||||
g9i_cache_hit_query_timeout_ms: u64,
|
||||
|
||||
/// Also least-wait route the regular P pool (default: keep configured policy)
|
||||
#[arg(long, default_value_t = false, help_heading = "G9i Pool Admission")]
|
||||
g9i_regular_pool_least_wait: bool,
|
||||
|
||||
// ==================== Service Discovery (Kubernetes) ====================
|
||||
/// Enable Kubernetes service discovery
|
||||
#[arg(
|
||||
@@ -1070,11 +1115,34 @@ impl CliArgs {
|
||||
.circuit_breaker(!self.disable_circuit_breaker)
|
||||
.enable_wasm(self.enable_wasm)
|
||||
.igw(self.enable_igw)
|
||||
.g9i(self.build_g9i_config()?)
|
||||
.maybe_server_cert_and_key(self.tls_cert_path.as_ref(), self.tls_key_path.as_ref());
|
||||
|
||||
builder.build()
|
||||
}
|
||||
|
||||
/// G9i pool admission + fast-P routing config; None when the feature is off.
|
||||
fn build_g9i_config(&self) -> ConfigResult<Option<smg::routers::http::g9i::G9iConfig>> {
|
||||
if !self.g9i_enabled && self.g9i_fast_prefill_url.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
let config = smg::routers::http::g9i::G9iConfig {
|
||||
enabled: self.g9i_enabled,
|
||||
fast_prefill_urls: self.g9i_fast_prefill_url.clone(),
|
||||
max_new_tokens: self.g9i_max_new_tokens,
|
||||
fast_p_queue_reject_secs: self.g9i_fast_p_queue_reject_secs,
|
||||
decode_max_running: self.g9i_decode_max_running,
|
||||
drain_rate_floor: self.g9i_drain_rate_floor,
|
||||
poll_interval_secs: self.g9i_poll_interval_secs,
|
||||
snapshot_ttl_secs: self.g9i_snapshot_ttl_secs,
|
||||
d_no_snapshot: self.g9i_d_no_snapshot.clone(),
|
||||
cache_hit_query_timeout_ms: self.g9i_cache_hit_query_timeout_ms,
|
||||
regular_pool_least_wait: self.g9i_regular_pool_least_wait,
|
||||
};
|
||||
config.validate().map_err(|reason| ConfigError::ValidationFailed { reason })?;
|
||||
Ok(Some(config))
|
||||
}
|
||||
|
||||
fn to_server_config(&self, router_config: RouterConfig) -> ServerConfig {
|
||||
let service_discovery_config = if self.service_discovery {
|
||||
// Get router discovery config from router_config.discovery if available
|
||||
|
||||
@@ -0,0 +1,628 @@
|
||||
//! G9i: pool-aware admission control + fast-P routing for PD mode.
|
||||
//!
|
||||
//! Router-side only — zero engine changes. Consumes two engine surfaces that
|
||||
//! already exist (G9 P0):
|
||||
//! * `GET /v1/loads?include=core` — per-DP-rank `LoadSnapshot` list; we sum
|
||||
//! `num_running_reqs` / `queued_total_input_tokens` /
|
||||
//! `drain_rate_tokens_per_s` across ranks into per-node, pool-aggregated
|
||||
//! views (poll loop below, independent of `LoadMonitor`, whose
|
||||
//! `aggregate.total_tokens` parse target does not exist on this engine).
|
||||
//! * `POST /query_cache_hit` — chat-body prefix-hit probe used to classify
|
||||
//! small-prefill (new tokens < threshold) requests.
|
||||
//!
|
||||
//! Decision tree (per request, before PD dispatch):
|
||||
//! 1. D-pool gate: sum of fresh `num_running_reqs` over the decode pool
|
||||
//! exceeds `decode_max_running` -> 429 (`d_pool_saturated`).
|
||||
//! 2. Small-prefill classification (chat routes only): probe one fast-P,
|
||||
//! `new_tokens < max_new_tokens` -> fast-P pool, else regular P pool.
|
||||
//! 3. Pool selection: least estimated wait
|
||||
//! (`queued_total_input_tokens / max(drain_rate, drain_rate_floor)`)
|
||||
//! among fresh nodes — this is also the load balancer.
|
||||
//! 4. Fast-P queue gate: least-wait fast-P estimate > `fast_p_queue_reject_secs`
|
||||
//! -> 429 (`fast_p_queue`) with `x-estimated-wait-s`.
|
||||
//!
|
||||
//! Fail-open discipline: missing/stale snapshots never hard-fail a request
|
||||
//! (the engine's own max-running-requests is the hard backstop); every
|
||||
//! fail-open path emits a warn log + metric. All numeric thresholds are
|
||||
//! tentative (measured values to be tuned in bench) and config-driven.
|
||||
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
sync::{
|
||||
atomic::{AtomicBool, Ordering},
|
||||
Arc, RwLock,
|
||||
},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
use reqwest::Client;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use crate::core::{ConnectionMode, Worker, WorkerRegistry, WorkerType};
|
||||
|
||||
const METRIC_DECISIONS: &str = "smg_g9i_decisions_total";
|
||||
const METRIC_REJECTS: &str = "smg_g9i_rejects_total";
|
||||
const METRIC_QUERY_LATENCY: &str = "smg_g9i_cache_hit_query_duration_seconds";
|
||||
const METRIC_NO_DATA: &str = "smg_g9i_no_fresh_snapshot_total";
|
||||
|
||||
/// G9i admission / fast-P routing configuration.
|
||||
///
|
||||
/// All thresholds are **tentative** (user directive 2026-10-11: numbers get
|
||||
/// fixed by measurement after the feature lands) and centralized here.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
pub struct G9iConfig {
|
||||
/// Master switch; when false the router behaves exactly as before.
|
||||
#[serde(default)]
|
||||
pub enabled: bool,
|
||||
/// Fast-P pool membership: prefill worker base URLs (e.g.
|
||||
/// "http://127.0.0.1:30000"). Router-side role only — the engine does not
|
||||
/// know about it. Empty = feature inert (everything routes as before).
|
||||
#[serde(default)]
|
||||
pub fast_prefill_urls: Vec<String>,
|
||||
/// Small-prefill cutoff on NEW tokens (total - prefix hit). Tentative 4096.
|
||||
#[serde(default = "default_max_new_tokens")]
|
||||
pub max_new_tokens: u64,
|
||||
/// Reject with 429 when the least-wait fast-P node's estimated queue wait
|
||||
/// exceeds this many seconds. Tentative 5.0.
|
||||
#[serde(default = "default_fast_p_queue_reject_secs")]
|
||||
pub fast_p_queue_reject_secs: f64,
|
||||
/// D-pool admission ceiling: sum of instantaneous in-processing requests
|
||||
/// across all decode nodes. Tentative 256 (instance-level; per-worker
|
||||
/// semantics to be confirmed against /v1/loads dp-rank reporting).
|
||||
#[serde(default = "default_decode_max_running")]
|
||||
pub decode_max_running: u64,
|
||||
/// Lower bound for drain rate in the est-wait denominator (tok/s).
|
||||
/// Protects against div-by-zero and cold-start sliding-window noise.
|
||||
#[serde(default = "default_drain_rate_floor")]
|
||||
pub drain_rate_floor: f64,
|
||||
/// Snapshot poll interval.
|
||||
#[serde(default = "default_poll_interval_secs")]
|
||||
pub poll_interval_secs: u64,
|
||||
/// Snapshots older than this are stale: node leaves its pool's decision
|
||||
/// set until refreshed.
|
||||
#[serde(default = "default_snapshot_ttl_secs")]
|
||||
pub snapshot_ttl_secs: u64,
|
||||
/// Behavior when the D-pool has no fresh snapshot (endpoint broken):
|
||||
/// "allow" (default, fail-open + warn; engine max-running is the hard
|
||||
/// backstop) or "reject" (fail-closed).
|
||||
#[serde(default = "default_d_no_snapshot")]
|
||||
pub d_no_snapshot: String,
|
||||
/// Timeout for the per-request /query_cache_hit probe.
|
||||
#[serde(default = "default_cache_hit_query_timeout_ms")]
|
||||
pub cache_hit_query_timeout_ms: u64,
|
||||
/// Also route the regular P pool by least-wait (overriding the configured
|
||||
/// prefill policy). Default false: regular pool keeps existing policy.
|
||||
#[serde(default)]
|
||||
pub regular_pool_least_wait: bool,
|
||||
}
|
||||
|
||||
fn default_max_new_tokens() -> u64 {
|
||||
4096
|
||||
}
|
||||
fn default_fast_p_queue_reject_secs() -> f64 {
|
||||
5.0
|
||||
}
|
||||
fn default_decode_max_running() -> u64 {
|
||||
256
|
||||
}
|
||||
fn default_drain_rate_floor() -> f64 {
|
||||
1000.0
|
||||
}
|
||||
fn default_poll_interval_secs() -> u64 {
|
||||
1
|
||||
}
|
||||
fn default_snapshot_ttl_secs() -> u64 {
|
||||
5
|
||||
}
|
||||
fn default_d_no_snapshot() -> String {
|
||||
"allow".to_string()
|
||||
}
|
||||
fn default_cache_hit_query_timeout_ms() -> u64 {
|
||||
2000
|
||||
}
|
||||
|
||||
impl Default for G9iConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
fast_prefill_urls: Vec::new(),
|
||||
max_new_tokens: default_max_new_tokens(),
|
||||
fast_p_queue_reject_secs: default_fast_p_queue_reject_secs(),
|
||||
decode_max_running: default_decode_max_running(),
|
||||
drain_rate_floor: default_drain_rate_floor(),
|
||||
poll_interval_secs: default_poll_interval_secs(),
|
||||
snapshot_ttl_secs: default_snapshot_ttl_secs(),
|
||||
d_no_snapshot: default_d_no_snapshot(),
|
||||
cache_hit_query_timeout_ms: default_cache_hit_query_timeout_ms(),
|
||||
regular_pool_least_wait: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl G9iConfig {
|
||||
pub fn validate(&self) -> Result<(), String> {
|
||||
if self.enabled && self.fast_prefill_urls.is_empty() {
|
||||
return Err(
|
||||
"g9i.enabled requires at least one --g9i-fast-prefill-url".to_string(),
|
||||
);
|
||||
}
|
||||
if !matches!(self.d_no_snapshot.as_str(), "allow" | "reject") {
|
||||
return Err(format!(
|
||||
"g9i.d_no_snapshot must be 'allow' or 'reject', got '{}'",
|
||||
self.d_no_snapshot
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Aggregated per-node view (summed over DP ranks), refreshed by the poller.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct NodeLoad {
|
||||
/// Sum of num_running_reqs across dp ranks (instance-level).
|
||||
pub running_reqs: u64,
|
||||
/// Sum of queued_total_input_tokens across dp ranks.
|
||||
pub queued_total_input_tokens: u64,
|
||||
/// Sum of drain_rate_tokens_per_s across dp ranks.
|
||||
pub drain_rate_tokens_per_s: f64,
|
||||
fetched_at: Option<Instant>,
|
||||
fetch_ok: bool,
|
||||
}
|
||||
|
||||
impl NodeLoad {
|
||||
fn est_wait_secs(&self, floor: f64) -> f64 {
|
||||
self.queued_total_input_tokens as f64 / self.drain_rate_tokens_per_s.max(floor)
|
||||
}
|
||||
|
||||
fn is_fresh(&self, ttl: Duration) -> bool {
|
||||
self.fetch_ok && self.fetched_at.is_some_and(|t| t.elapsed() <= ttl)
|
||||
}
|
||||
}
|
||||
|
||||
/// Why a request was rejected (429).
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct RejectInfo {
|
||||
pub reason: &'static str,
|
||||
pub estimated_wait_secs: Option<f64>,
|
||||
}
|
||||
|
||||
/// Per-request prefill routing outcome of the G9i classifier.
|
||||
pub enum PrefillPlan {
|
||||
/// Let the configured prefill policy decide (regular pool).
|
||||
Default,
|
||||
/// Force this worker base URL as the prefill target (least-wait pick).
|
||||
Override(String),
|
||||
/// Reject with 429 (fast-P queue over threshold).
|
||||
Reject(RejectInfo),
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct G9iState {
|
||||
config: G9iConfig,
|
||||
/// Keyed by worker base_url (normalized, no trailing slash).
|
||||
snapshots: RwLock<HashMap<String, NodeLoad>>,
|
||||
/// Whether any /v1/loads fetch ever succeeded (startup grace for D-gate).
|
||||
had_any_success: AtomicBool,
|
||||
monitor_started: AtomicBool,
|
||||
}
|
||||
|
||||
impl G9iState {
|
||||
pub fn new(config: G9iConfig) -> Self {
|
||||
Self {
|
||||
config,
|
||||
snapshots: RwLock::new(HashMap::new()),
|
||||
had_any_success: AtomicBool::new(false),
|
||||
monitor_started: AtomicBool::new(false),
|
||||
}
|
||||
}
|
||||
|
||||
/// Spawn the background snapshot poller. Idempotent.
|
||||
pub fn start_monitor(self: &Arc<Self>, registry: Arc<WorkerRegistry>, client: Client) {
|
||||
if self
|
||||
.monitor_started
|
||||
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
|
||||
.is_err()
|
||||
{
|
||||
return;
|
||||
}
|
||||
let state = Arc::clone(self);
|
||||
let interval = Duration::from_secs(self.config.poll_interval_secs.max(1));
|
||||
info!(
|
||||
"G9i monitor started: interval={:?} ttl={}s fast_p={:?} max_new={} d_max_running={}",
|
||||
interval,
|
||||
self.config.snapshot_ttl_secs,
|
||||
self.config.fast_prefill_urls,
|
||||
self.config.max_new_tokens,
|
||||
self.config.decode_max_running,
|
||||
);
|
||||
tokio::spawn(async move {
|
||||
let mut timer = tokio::time::interval(interval);
|
||||
loop {
|
||||
timer.tick().await;
|
||||
state.poll_once(®istry, &client).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async fn poll_once(&self, registry: &WorkerRegistry, client: &Client) {
|
||||
let workers: Vec<Arc<dyn Worker>> = registry
|
||||
.get_all()
|
||||
.into_iter()
|
||||
.filter(|w| {
|
||||
matches!(w.worker_type(), WorkerType::Prefill { .. } | WorkerType::Decode)
|
||||
&& matches!(w.connection_mode(), ConnectionMode::Http)
|
||||
})
|
||||
.collect();
|
||||
|
||||
let mut results: HashMap<String, NodeLoad> = HashMap::new();
|
||||
for worker in &workers {
|
||||
let url = normalize_url(&worker.url().to_string());
|
||||
let load = match fetch_node_load(client, worker.as_ref()).await {
|
||||
Ok(load) => {
|
||||
self.had_any_success.store(true, Ordering::SeqCst);
|
||||
load
|
||||
}
|
||||
Err(e) => {
|
||||
debug!("G9i loads fetch failed for {}: {}", url, e);
|
||||
NodeLoad::default()
|
||||
}
|
||||
};
|
||||
metrics::gauge!(
|
||||
"smg_g9i_node_running_reqs",
|
||||
"worker" => url.clone(),
|
||||
)
|
||||
.set(load.running_reqs as f64);
|
||||
results.insert(url, load);
|
||||
}
|
||||
|
||||
if let Ok(mut snap) = self.snapshots.write() {
|
||||
for (url, load) in results {
|
||||
snap.insert(url, load);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn snapshot(&self, url: &str) -> Option<NodeLoad> {
|
||||
self.snapshots
|
||||
.read()
|
||||
.ok()
|
||||
.and_then(|snap| snap.get(&normalize_url(url)).cloned())
|
||||
}
|
||||
|
||||
/// Fresh, usable load for a worker, if any.
|
||||
fn fresh_load(&self, url: &str) -> Option<NodeLoad> {
|
||||
let ttl = Duration::from_secs(self.config.snapshot_ttl_secs);
|
||||
let load = self.snapshot(url)?;
|
||||
load.is_fresh(ttl).then_some(load)
|
||||
}
|
||||
|
||||
fn fast_pool<'a>(&self, prefill_workers: &'a [Arc<dyn Worker>]) -> Vec<&'a Arc<dyn Worker>> {
|
||||
prefill_workers
|
||||
.iter()
|
||||
.filter(|w| {
|
||||
self.config
|
||||
.fast_prefill_urls
|
||||
.iter()
|
||||
.any(|f| urls_match(f, &w.url().to_string()))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Least estimated wait among workers with fresh snapshots.
|
||||
/// Returns (worker, est_wait_secs, fresh_count).
|
||||
fn least_wait<'a>(
|
||||
&self,
|
||||
workers: &[&'a Arc<dyn Worker>],
|
||||
) -> (Option<&'a Arc<dyn Worker>>, f64, usize) {
|
||||
let mut best: Option<(&Arc<dyn Worker>, f64)> = None;
|
||||
let mut fresh_count = 0usize;
|
||||
for w in workers {
|
||||
if let Some(load) = self.fresh_load(&w.url().to_string()) {
|
||||
fresh_count += 1;
|
||||
let wait = load.est_wait_secs(self.config.drain_rate_floor);
|
||||
if best.is_none() || wait < best.unwrap().1 {
|
||||
best = Some((w, wait));
|
||||
}
|
||||
}
|
||||
}
|
||||
(best.map(|(w, _)| w), best.map(|(_, t)| t).unwrap_or(0.0), fresh_count)
|
||||
}
|
||||
|
||||
/// Gate 1: D-pool aggregate admission.
|
||||
///
|
||||
/// Returns `Err(RejectInfo)` when the decode pool must shed load.
|
||||
/// Fail-open when no fresh snapshot exists (configurable), since the
|
||||
/// engine-side max-running-requests remains the hard backstop.
|
||||
pub async fn check_decode_pool(&self, registry: &WorkerRegistry) -> Result<(), RejectInfo> {
|
||||
let ttl = Duration::from_secs(self.config.snapshot_ttl_secs);
|
||||
let decode_workers: Vec<Arc<dyn Worker>> = registry
|
||||
.get_decode_workers()
|
||||
.into_iter()
|
||||
.filter(|w| matches!(w.connection_mode(), ConnectionMode::Http))
|
||||
.collect();
|
||||
|
||||
let mut total_running = 0u64;
|
||||
let mut fresh = 0usize;
|
||||
for w in &decode_workers {
|
||||
if let Some(load) = self.snapshot(&w.url().to_string()) {
|
||||
if load.is_fresh(ttl) {
|
||||
fresh += 1;
|
||||
total_running += load.running_reqs;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if fresh == 0 {
|
||||
metrics::counter!(METRIC_NO_DATA, "pool" => "decode").increment(1);
|
||||
if self.had_any_success.load(Ordering::SeqCst)
|
||||
&& self.config.d_no_snapshot == "reject"
|
||||
{
|
||||
warn!("G9i: no fresh decode snapshots, d_no_snapshot=reject -> 429");
|
||||
return Err(RejectInfo {
|
||||
reason: "d_pool_no_snapshot",
|
||||
estimated_wait_secs: None,
|
||||
});
|
||||
}
|
||||
// Startup grace or fail-open: nothing to enforce yet.
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
metrics::gauge!("smg_g9i_d_pool_running_reqs").set(total_running as f64);
|
||||
if total_running > self.config.decode_max_running {
|
||||
return Err(RejectInfo {
|
||||
reason: "d_pool_saturated",
|
||||
estimated_wait_secs: None,
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Gate 2 + 3 + 4: classify small-prefill requests onto the fast-P pool
|
||||
/// with least-wait selection and queue-wait rejection.
|
||||
///
|
||||
/// `body` is the OpenAI chat request JSON. Non-chat routes must not call
|
||||
/// this. All probe failures fall back to the default routing path
|
||||
/// (fail-open, metric emitted).
|
||||
pub async fn classify_prefill(
|
||||
&self,
|
||||
registry: &WorkerRegistry,
|
||||
client: &Client,
|
||||
body: &Value,
|
||||
) -> PrefillPlan {
|
||||
let prefill_workers = registry.get_prefill_workers();
|
||||
let fast_pool = self.fast_pool(&prefill_workers);
|
||||
if fast_pool.is_empty() {
|
||||
metrics::counter!(METRIC_DECISIONS, "path" => "fast_pool_empty").increment(1);
|
||||
return PrefillPlan::Default;
|
||||
}
|
||||
|
||||
// Probe the least-wait healthy fast-P for prefix-hit classification
|
||||
// (L3 is shared, so hit lengths are comparable across fast-P nodes).
|
||||
let (probe_target, _, fast_fresh) = self.least_wait(&fast_pool);
|
||||
if fast_fresh == 0 {
|
||||
metrics::counter!(METRIC_DECISIONS, "path" => "fast_pool_no_fresh_snapshot")
|
||||
.increment(1);
|
||||
return self.regular_plan(&prefill_workers);
|
||||
}
|
||||
let probe_target = match probe_target {
|
||||
Some(w) => w,
|
||||
None => return self.regular_plan(&prefill_workers),
|
||||
};
|
||||
|
||||
let new_tokens = match self
|
||||
.query_new_tokens(client, probe_target.as_ref(), body)
|
||||
.await
|
||||
{
|
||||
Ok(Some(n)) => n,
|
||||
Ok(None) => {
|
||||
// Not classifiable (multimodal 400, engine error): regular path.
|
||||
metrics::counter!(METRIC_DECISIONS, "path" => "query_unclassifiable")
|
||||
.increment(1);
|
||||
return self.regular_plan(&prefill_workers);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
"G9i /query_cache_hit probe failed on {}: {} — falling back to regular pool",
|
||||
probe_target.url(),
|
||||
e
|
||||
);
|
||||
metrics::counter!(METRIC_DECISIONS, "path" => "query_error").increment(1);
|
||||
return self.regular_plan(&prefill_workers);
|
||||
}
|
||||
};
|
||||
|
||||
if new_tokens >= self.config.max_new_tokens {
|
||||
metrics::counter!(METRIC_DECISIONS, "path" => "regular_large_prefill").increment(1);
|
||||
return self.regular_plan(&prefill_workers);
|
||||
}
|
||||
|
||||
// Small prefill -> fast-P pool, least-wait pick.
|
||||
let (picked, est_wait, fresh) = self.least_wait(&fast_pool);
|
||||
if fresh == 0 {
|
||||
// Lost our snapshots between probe and pick; fail-open to regular.
|
||||
metrics::counter!(METRIC_DECISIONS, "path" => "fast_pool_no_fresh_snapshot")
|
||||
.increment(1);
|
||||
return self.regular_plan(&prefill_workers);
|
||||
}
|
||||
let est_wait = if picked.is_some() { est_wait } else { 0.0 };
|
||||
if est_wait > self.config.fast_p_queue_reject_secs {
|
||||
metrics::counter!(METRIC_REJECTS, "reason" => "fast_p_queue").increment(1);
|
||||
return PrefillPlan::Reject(RejectInfo {
|
||||
reason: "fast_p_queue",
|
||||
estimated_wait_secs: Some(est_wait),
|
||||
});
|
||||
}
|
||||
match picked {
|
||||
Some(w) => {
|
||||
metrics::counter!(METRIC_DECISIONS, "path" => "routed_fast_p").increment(1);
|
||||
debug!(
|
||||
"G9i: small prefill (new={} < {}) -> fast-P {} est_wait={:.2}s",
|
||||
new_tokens,
|
||||
self.config.max_new_tokens,
|
||||
w.url(),
|
||||
est_wait,
|
||||
);
|
||||
PrefillPlan::Override(normalize_url(&w.url().to_string()))
|
||||
}
|
||||
None => self.regular_plan(&prefill_workers),
|
||||
}
|
||||
}
|
||||
|
||||
fn regular_plan(&self, prefill_workers: &[Arc<dyn Worker>]) -> PrefillPlan {
|
||||
if !self.config.regular_pool_least_wait {
|
||||
return PrefillPlan::Default;
|
||||
}
|
||||
let all: Vec<&Arc<dyn Worker>> = prefill_workers.iter().collect();
|
||||
let (picked, _, fresh) = self.least_wait(&all);
|
||||
if fresh == 0 {
|
||||
metrics::counter!(METRIC_DECISIONS, "path" => "regular_no_fresh_snapshot")
|
||||
.increment(1);
|
||||
return PrefillPlan::Default;
|
||||
}
|
||||
match picked {
|
||||
Some(w) => {
|
||||
metrics::counter!(METRIC_DECISIONS, "path" => "routed_regular_least_wait")
|
||||
.increment(1);
|
||||
PrefillPlan::Override(normalize_url(&w.url().to_string()))
|
||||
}
|
||||
None => PrefillPlan::Default,
|
||||
}
|
||||
}
|
||||
|
||||
/// Probe a fast-P's /query_cache_hit; Ok(None) = "not classifiable".
|
||||
async fn query_new_tokens(
|
||||
&self,
|
||||
client: &Client,
|
||||
worker: &dyn Worker,
|
||||
body: &Value,
|
||||
) -> Result<Option<u64>, String> {
|
||||
let url = format!("{}/query_cache_hit", normalize_url(&worker.url().to_string()));
|
||||
let mut req = client
|
||||
.post(&url)
|
||||
.timeout(Duration::from_millis(
|
||||
self.config.cache_hit_query_timeout_ms.max(200),
|
||||
))
|
||||
.json(body);
|
||||
if let Some(key) = worker.api_key() {
|
||||
req = req.bearer_auth(key);
|
||||
}
|
||||
|
||||
let start = Instant::now();
|
||||
let resp = req.send().await.map_err(|e| e.to_string())?;
|
||||
metrics::histogram!(METRIC_QUERY_LATENCY).record(start.elapsed().as_secs_f64());
|
||||
|
||||
if !resp.status().is_success() {
|
||||
// 400 = multimodal/unsupported; anything else = engine-side issue.
|
||||
// Either way: not classifiable, caller falls back.
|
||||
return Ok(None);
|
||||
}
|
||||
let json: Value = resp.json().await.map_err(|e| e.to_string())?;
|
||||
Ok(json
|
||||
.get("new_tokens")
|
||||
.and_then(|v| v.as_u64()))
|
||||
}
|
||||
}
|
||||
|
||||
impl PrefillPlan {
|
||||
pub fn into_override(self) -> Result<Option<String>, RejectInfo> {
|
||||
match self {
|
||||
PrefillPlan::Default => Ok(None),
|
||||
PrefillPlan::Override(url) => Ok(Some(url)),
|
||||
PrefillPlan::Reject(info) => Err(info),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetch and aggregate one node's /v1/loads over its DP ranks.
|
||||
async fn fetch_node_load(client: &Client, worker: &dyn Worker) -> Result<NodeLoad, String> {
|
||||
let url = format!(
|
||||
"{}/v1/loads?include=core",
|
||||
normalize_url(&worker.url().to_string())
|
||||
);
|
||||
let mut req = client.get(&url).timeout(Duration::from_secs(3));
|
||||
if let Some(key) = worker.api_key() {
|
||||
req = req.bearer_auth(key);
|
||||
}
|
||||
let resp = req.send().await.map_err(|e| e.to_string())?;
|
||||
if !resp.status().is_success() {
|
||||
return Err(format!("HTTP {}", resp.status()));
|
||||
}
|
||||
let json: Value = resp.json().await.map_err(|e| e.to_string())?;
|
||||
let ranks = json
|
||||
.get("loads")
|
||||
.and_then(|l| l.as_array())
|
||||
.ok_or_else(|| "missing 'loads' array".to_string())?;
|
||||
|
||||
let mut node = NodeLoad {
|
||||
fetched_at: Some(Instant::now()),
|
||||
fetch_ok: true,
|
||||
..Default::default()
|
||||
};
|
||||
for rank in ranks {
|
||||
node.running_reqs += rank
|
||||
.get("num_running_reqs")
|
||||
.and_then(|v| v.as_u64())
|
||||
.unwrap_or(0);
|
||||
node.queued_total_input_tokens += rank
|
||||
.get("queued_total_input_tokens")
|
||||
.and_then(|v| v.as_u64())
|
||||
.unwrap_or(0);
|
||||
node.drain_rate_tokens_per_s += rank
|
||||
.get("drain_rate_tokens_per_s")
|
||||
.and_then(|v| v.as_f64())
|
||||
.unwrap_or(0.0);
|
||||
}
|
||||
Ok(node)
|
||||
}
|
||||
|
||||
fn normalize_url(url: &str) -> String {
|
||||
url.trim_end_matches('/').to_string()
|
||||
}
|
||||
|
||||
fn urls_match(configured: &str, worker_url: &str) -> bool {
|
||||
normalize_url(configured) == normalize_url(worker_url)
|
||||
}
|
||||
|
||||
/// Build the G9i 429 admission response: reason + estimated wait in headers
|
||||
/// (downstream reroutes on 429 without retrying — decision ①), JSON body
|
||||
/// shaped like the router's standard error envelope.
|
||||
pub fn reject_response(info: &RejectInfo) -> axum::response::Response {
|
||||
use axum::http::{HeaderMap, HeaderValue, StatusCode};
|
||||
use axum::response::IntoResponse;
|
||||
|
||||
let mut headers = HeaderMap::with_capacity(3);
|
||||
headers.insert(
|
||||
crate::routers::error::HEADER_X_SMG_ERROR_CODE,
|
||||
HeaderValue::from_static(info.reason),
|
||||
);
|
||||
headers.insert(
|
||||
"x-reject-reason",
|
||||
HeaderValue::from_static(info.reason),
|
||||
);
|
||||
if let Some(wait) = info.estimated_wait_secs {
|
||||
if let Ok(v) = HeaderValue::from_str(&format!("{wait:.2}")) {
|
||||
headers.insert("x-estimated-wait-s", v);
|
||||
}
|
||||
}
|
||||
let message = match info.estimated_wait_secs {
|
||||
Some(wait) => format!(
|
||||
"G9i admission rejected ({}): estimated wait {wait:.2}s exceeds the configured threshold",
|
||||
info.reason
|
||||
),
|
||||
None => format!("G9i admission rejected ({})", info.reason),
|
||||
};
|
||||
(
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
headers,
|
||||
axum::Json(serde_json::json!({
|
||||
"error": {
|
||||
"type": "Too Many Requests",
|
||||
"code": info.reason,
|
||||
"message": message,
|
||||
}
|
||||
})),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
//! HTTP router implementations
|
||||
|
||||
pub mod g9i;
|
||||
pub mod pd_router;
|
||||
pub mod pd_types;
|
||||
pub mod router;
|
||||
|
||||
@@ -58,6 +58,8 @@ pub struct PDRouter {
|
||||
pub retry_config: RetryConfig,
|
||||
pub api_key: Option<String>,
|
||||
pub enable_igw: bool,
|
||||
/// G9i pool admission + fast-P routing state; None = feature off.
|
||||
pub g9i: Option<Arc<super::g9i::G9iState>>,
|
||||
}
|
||||
|
||||
struct PreparedWorkerRequest<'a> {
|
||||
@@ -182,6 +184,14 @@ impl PDRouter {
|
||||
}
|
||||
|
||||
pub async fn new(ctx: &Arc<crate::app_context::AppContext>) -> Result<Self, String> {
|
||||
let g9i = match &ctx.router_config.g9i {
|
||||
Some(config) if config.enabled => {
|
||||
let state = Arc::new(super::g9i::G9iState::new(config.clone()));
|
||||
state.start_monitor(Arc::clone(&ctx.worker_registry), ctx.client.clone());
|
||||
Some(state)
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
Ok(PDRouter {
|
||||
worker_registry: Arc::clone(&ctx.worker_registry),
|
||||
policy_registry: Arc::clone(&ctx.policy_registry),
|
||||
@@ -189,6 +199,7 @@ impl PDRouter {
|
||||
retry_config: ctx.router_config.effective_retry_config(),
|
||||
api_key: ctx.router_config.api_key.clone(),
|
||||
enable_igw: ctx.router_config.enable_igw,
|
||||
g9i,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -390,6 +401,43 @@ impl PDRouter {
|
||||
// Clone request once outside the retry loop, then use Arc to share across attempts
|
||||
// This avoids O(retries) clones by sharing the same data
|
||||
let shared_request = Arc::new(original_request.clone());
|
||||
|
||||
// G9i admission (router-side, zero engine changes). Computed once per
|
||||
// request, outside the retry loop: our own 429s must never be retried.
|
||||
let g9i_prefill_override: Option<String> = if let Some(g9i) = &self.g9i {
|
||||
if let Err(info) = g9i.check_decode_pool(&self.worker_registry).await {
|
||||
metrics::counter!("smg_g9i_rejects_total", "reason" => info.reason)
|
||||
.increment(1);
|
||||
return super::g9i::reject_response(&info);
|
||||
}
|
||||
if context.route == "/v1/chat/completions" {
|
||||
match serde_json::to_value(original_request) {
|
||||
Ok(body) => {
|
||||
match g9i
|
||||
.classify_prefill(&self.worker_registry, &self.client, &body)
|
||||
.await
|
||||
.into_override()
|
||||
{
|
||||
Ok(override_url) => override_url,
|
||||
Err(info) => {
|
||||
metrics::counter!(
|
||||
"smg_g9i_rejects_total",
|
||||
"reason" => info.reason
|
||||
)
|
||||
.increment(1);
|
||||
return super::g9i::reject_response(&info);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(_) => None,
|
||||
}
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let response = RetryExecutor::execute_response_with_retry(
|
||||
&self.retry_config,
|
||||
{
|
||||
@@ -397,6 +445,7 @@ impl PDRouter {
|
||||
// Clone Arc (cheap reference count increment) instead of cloning the entire request
|
||||
let shared_request = Arc::clone(&shared_request);
|
||||
let context = context.clone();
|
||||
let g9i_prefill_override = g9i_prefill_override.clone();
|
||||
async move {
|
||||
let (prefill, decode) = match self
|
||||
.select_pd_pair(
|
||||
@@ -412,6 +461,38 @@ impl PDRouter {
|
||||
}
|
||||
};
|
||||
|
||||
// G9i least-wait override: route onto the classified
|
||||
// fast-P node instead of the policy pick.
|
||||
let (prefill, decode) =
|
||||
if let Some(override_url) = g9i_prefill_override.as_ref() {
|
||||
let matched = self
|
||||
.worker_registry
|
||||
.get_prefill_workers()
|
||||
.into_iter()
|
||||
.find(|w| {
|
||||
w.url().trim_end_matches('/') == override_url.as_str()
|
||||
&& w.is_available()
|
||||
});
|
||||
match matched {
|
||||
Some(w) => {
|
||||
debug!(
|
||||
"G9i: overriding prefill selection with {}",
|
||||
override_url
|
||||
);
|
||||
(w, decode)
|
||||
}
|
||||
None => {
|
||||
warn!(
|
||||
"G9i: override target {} unavailable, keeping policy pick",
|
||||
override_url
|
||||
);
|
||||
(prefill, decode)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
(prefill, decode)
|
||||
};
|
||||
|
||||
debug!(
|
||||
"PD retry attempt {} using prefill={} decode={}",
|
||||
attempt,
|
||||
@@ -1780,6 +1861,7 @@ mod tests {
|
||||
retry_config: RetryConfig::default(),
|
||||
api_key: Some("test_api_key".to_string()),
|
||||
enable_igw: false,
|
||||
g9i: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user