From 09ca4957eec76cb23d6c7ec3bf771ce8c141f382 Mon Sep 17 00:00:00 2001 From: "minke.yu" Date: Wed, 14 Oct 2026 21:50:11 +0800 Subject: [PATCH] gateway g9i: scrape P /metrics kv_transfer_bootstrap_ms as est_T_D ground truth (P1 shadow T4) --- sgl-model-gateway/src/routers/http/g9i.rs | 149 ++++++++++++++++++++++ 1 file changed, 149 insertions(+) diff --git a/sgl-model-gateway/src/routers/http/g9i.rs b/sgl-model-gateway/src/routers/http/g9i.rs index b98d8d27aa..e12b078073 100644 --- a/sgl-model-gateway/src/routers/http/g9i.rs +++ b/sgl-model-gateway/src/routers/http/g9i.rs @@ -309,6 +309,11 @@ pub struct G9iState { /// Input of the est_T_D shadow estimate (design 2026-10-14 §2.1); /// populated only while shadow mode is on. d_snapshots: RwLock>, + /// Per-prefill-node cumulative (sum_ms, count) of the engine's + /// `sglang:kv_transfer_bootstrap_ms` Prometheus histogram — the est_T_D + /// ground truth (design §2.3-2). Windowed means come from diffing + /// successive scrapes. + bootstrap_stats: RwLock>, } /// Freshness of a cached probe answer. Short on purpose: an eviction on the @@ -398,6 +403,7 @@ impl G9iState { shadow, shadow_queue, d_snapshots: RwLock::new(Vec::new()), + bootstrap_stats: RwLock::new(HashMap::new()), } } @@ -477,6 +483,16 @@ impl G9iState { if let Ok(mut d) = self.d_snapshots.write() { *d = d_results; } + // est_T_D P1 ground truth (design 2026-10-14 §2.3-2): scrape + // each prefill node's /metrics for the per-request + // kv_transfer_bootstrap_ms histogram and publish the windowed + // mean next to the estimate. P1 only measures side by side — + // the correction loop is P3. + for worker in &workers { + if matches!(worker.worker_type(), WorkerType::Prefill { .. }) { + self.scrape_bootstrap_metric(client, worker.as_ref()).await; + } + } } if let Ok(mut snap) = self.snapshots.write() { @@ -488,6 +504,51 @@ impl G9iState { self.publish_ttft_percentiles(); } + /// Scrape one prefill node's /metrics for `sglang:kv_transfer_bootstrap_ms` + /// and publish the windowed mean (seconds) since the previous scrape. + /// All failures are debug-logged and skipped — this is a shadow-side + /// observability path, never worth a warn. + async fn scrape_bootstrap_metric(&self, client: &Client, worker: &dyn Worker) { + let url = normalize_url(&worker.url().to_string()); + let mut req = client + .get(format!("{url}/metrics")) + .timeout(Duration::from_secs(3)); + if let Some(key) = worker.api_key() { + req = req.bearer_auth(key); + } + let text = match req.send().await { + Ok(r) if r.status().is_success() => match r.text().await { + Ok(t) => t, + Err(e) => { + debug!("G9i /metrics read failed for {}: {}", url, e); + return; + } + }, + Ok(r) => { + debug!("G9i /metrics fetch HTTP {} for {}", r.status(), url); + return; + } + Err(e) => { + debug!("G9i /metrics fetch failed for {}: {}", url, e); + return; + } + }; + let Some(cur) = parse_bootstrap_histogram(&text) else { + return; + }; + let prev = match self.bootstrap_stats.write() { + Ok(mut s) => s.insert(url.clone(), cur), + Err(_) => None, + }; + if let Some(mean_secs) = prev.and_then(|p| bootstrap_window_mean(p, cur)) { + metrics::gauge!( + super::g9i_shadow::METRIC_BOOTSTRAP_ACTUAL, + "worker" => url, + ) + .set(mean_secs); + } + } + fn snapshot(&self, url: &str) -> Option { self.snapshots .read() @@ -1239,6 +1300,54 @@ fn parse_node_load(json: &Value) -> Result { Ok(node) } +/// Aggregate `sglang:kv_transfer_bootstrap_ms` histogram totals (sum_ms, +/// count) from a Prometheus text exposition, summing over every label +/// variant (per-dp-rank series). None when the metric is absent entirely. +fn parse_bootstrap_histogram(text: &str) -> Option<(f64, u64)> { + const SUM: &str = "sglang:kv_transfer_bootstrap_ms_sum"; + const COUNT: &str = "sglang:kv_transfer_bootstrap_ms_count"; + let mut sum_ms = 0.0f64; + let mut count = 0u64; + let mut seen = false; + for line in text.lines() { + // Series name ends at the first '{' (label block) or whitespace. + let name_end = line + .find(|c: char| c == '{' || c == ' ' || c == '\t') + .unwrap_or(line.len()); + let name = &line[..name_end]; + if name != SUM && name != COUNT { + continue; + } + // The sample value is the last whitespace-separated token (label + // values of this metric — dp_rank & friends — never contain spaces). + let Some(val_idx) = line.rfind([' ', '\t']) else { + continue; + }; + let Ok(v) = line[val_idx + 1..].parse::() else { + continue; + }; + seen = true; + if name == SUM { + sum_ms += v; + } else { + count += v as u64; + } + } + seen.then_some((sum_ms, count)) +} + +/// Windowed mean bootstrap wait (seconds) between two cumulative scrapes. +/// None on the first scrape, when no bootstrap completed in the window, or +/// on a counter reset (engine restart — the caller rebases onto `cur`). +fn bootstrap_window_mean(prev: (f64, u64), cur: (f64, u64)) -> Option { + let dc = cur.1.checked_sub(prev.1)?; + let ds = cur.0 - prev.0; + if dc == 0 || ds < 0.0 { + return None; + } + Some(ds / dc as f64 / 1000.0) +} + /// Normalize a worker base URL (trailing slashes removed). pub(crate) so /// shadow bookkeeping (pd_router) keys the prefix table identically. pub(crate) fn normalize_url(url: &str) -> String { @@ -1383,4 +1492,44 @@ mod tests { assert!(parse_node_load(&json!({})).is_err()); assert!(parse_node_load(&json!({"loads": {}})).is_err()); } + + /// Prometheus text parsing (T4): multi-rank sum/count aggregation, + /// bucket lines and unrelated metrics ignored, absent metric -> None. + #[test] + fn parse_bootstrap_histogram_aggregates_series() { + let text = "\ +# HELP sglang:kv_transfer_bootstrap_ms KV transfer bootstrap latency +# TYPE sglang:kv_transfer_bootstrap_ms histogram +sglang:kv_transfer_bootstrap_ms_bucket{dp_rank=\"0\",le=\"10.0\"} 3 +sglang:kv_transfer_bootstrap_ms_bucket{dp_rank=\"0\",le=\"+Inf\"} 5 +sglang:kv_transfer_bootstrap_ms_sum{dp_rank=\"0\"} 250.5 +sglang:kv_transfer_bootstrap_ms_count{dp_rank=\"0\"} 5 +sglang:kv_transfer_bootstrap_ms_sum{dp_rank=\"1\"} 149.5 +sglang:kv_transfer_bootstrap_ms_count{dp_rank=\"1\"} 3 +sglang:token_usage{dp_rank=\"0\"} 0.5 +"; + let (sum, count) = parse_bootstrap_histogram(text).expect("metric present"); + assert!((sum - 400.0).abs() < 1e-9, "sums aggregate over ranks"); + assert_eq!(count, 8, "counts aggregate over ranks"); + + // Metric absent -> None (engine without the histogram, non-PD, ...). + assert!(parse_bootstrap_histogram("sglang:token_usage 0.5\n").is_none()); + // A sum-only / count-only exposition still parses (partial data is + // better than none; the window mean just uses what is there). + assert_eq!( + parse_bootstrap_histogram("sglang:kv_transfer_bootstrap_ms_count 4\n"), + Some((0.0, 4)) + ); + } + + #[test] + fn bootstrap_window_mean_diffs_cumulative_scrapes() { + // 400ms over 8 bootstraps in the window -> 50ms mean. + let mean = bootstrap_window_mean((1000.0, 10), (1400.0, 18)).unwrap(); + assert!((mean - 0.05).abs() < 1e-9); + // No new samples in the window -> None. + assert_eq!(bootstrap_window_mean((1000.0, 10), (1000.0, 10)), None); + // Counter reset (engine restart) -> None; caller rebases. + assert_eq!(bootstrap_window_mean((1000.0, 10), (50.0, 1)), None); + } }