gateway g9i: scrape P /metrics kv_transfer_bootstrap_ms as est_T_D ground truth (P1 shadow T4)

This commit is contained in:
2026-10-14 21:50:11 +08:00
parent 7921442a5c
commit 09ca4957ee
+149
View File
@@ -309,6 +309,11 @@ pub struct G9iState {
/// Input of the est_T_D shadow estimate (design 2026-10-14 §2.1);
/// populated only while shadow mode is on.
d_snapshots: RwLock<Vec<(String, NodeLoad)>>,
/// Per-prefill-node cumulative (sum_ms, count) of the engine's
/// `sglang:kv_transfer_bootstrap_ms` Prometheus histogram — the est_T_D
/// ground truth (design §2.3-2). Windowed means come from diffing
/// successive scrapes.
bootstrap_stats: RwLock<HashMap<String, (f64, u64)>>,
}
/// Freshness of a cached probe answer. Short on purpose: an eviction on the
@@ -398,6 +403,7 @@ impl G9iState {
shadow,
shadow_queue,
d_snapshots: RwLock::new(Vec::new()),
bootstrap_stats: RwLock::new(HashMap::new()),
}
}
@@ -477,6 +483,16 @@ impl G9iState {
if let Ok(mut d) = self.d_snapshots.write() {
*d = d_results;
}
// est_T_D P1 ground truth (design 2026-10-14 §2.3-2): scrape
// each prefill node's /metrics for the per-request
// kv_transfer_bootstrap_ms histogram and publish the windowed
// mean next to the estimate. P1 only measures side by side —
// the correction loop is P3.
for worker in &workers {
if matches!(worker.worker_type(), WorkerType::Prefill { .. }) {
self.scrape_bootstrap_metric(client, worker.as_ref()).await;
}
}
}
if let Ok(mut snap) = self.snapshots.write() {
@@ -488,6 +504,51 @@ impl G9iState {
self.publish_ttft_percentiles();
}
/// Scrape one prefill node's /metrics for `sglang:kv_transfer_bootstrap_ms`
/// and publish the windowed mean (seconds) since the previous scrape.
/// All failures are debug-logged and skipped — this is a shadow-side
/// observability path, never worth a warn.
async fn scrape_bootstrap_metric(&self, client: &Client, worker: &dyn Worker) {
let url = normalize_url(&worker.url().to_string());
let mut req = client
.get(format!("{url}/metrics"))
.timeout(Duration::from_secs(3));
if let Some(key) = worker.api_key() {
req = req.bearer_auth(key);
}
let text = match req.send().await {
Ok(r) if r.status().is_success() => match r.text().await {
Ok(t) => t,
Err(e) => {
debug!("G9i /metrics read failed for {}: {}", url, e);
return;
}
},
Ok(r) => {
debug!("G9i /metrics fetch HTTP {} for {}", r.status(), url);
return;
}
Err(e) => {
debug!("G9i /metrics fetch failed for {}: {}", url, e);
return;
}
};
let Some(cur) = parse_bootstrap_histogram(&text) else {
return;
};
let prev = match self.bootstrap_stats.write() {
Ok(mut s) => s.insert(url.clone(), cur),
Err(_) => None,
};
if let Some(mean_secs) = prev.and_then(|p| bootstrap_window_mean(p, cur)) {
metrics::gauge!(
super::g9i_shadow::METRIC_BOOTSTRAP_ACTUAL,
"worker" => url,
)
.set(mean_secs);
}
}
fn snapshot(&self, url: &str) -> Option<NodeLoad> {
self.snapshots
.read()
@@ -1239,6 +1300,54 @@ fn parse_node_load(json: &Value) -> Result<NodeLoad, String> {
Ok(node)
}
/// Aggregate `sglang:kv_transfer_bootstrap_ms` histogram totals (sum_ms,
/// count) from a Prometheus text exposition, summing over every label
/// variant (per-dp-rank series). None when the metric is absent entirely.
fn parse_bootstrap_histogram(text: &str) -> Option<(f64, u64)> {
const SUM: &str = "sglang:kv_transfer_bootstrap_ms_sum";
const COUNT: &str = "sglang:kv_transfer_bootstrap_ms_count";
let mut sum_ms = 0.0f64;
let mut count = 0u64;
let mut seen = false;
for line in text.lines() {
// Series name ends at the first '{' (label block) or whitespace.
let name_end = line
.find(|c: char| c == '{' || c == ' ' || c == '\t')
.unwrap_or(line.len());
let name = &line[..name_end];
if name != SUM && name != COUNT {
continue;
}
// The sample value is the last whitespace-separated token (label
// values of this metric — dp_rank & friends — never contain spaces).
let Some(val_idx) = line.rfind([' ', '\t']) else {
continue;
};
let Ok(v) = line[val_idx + 1..].parse::<f64>() else {
continue;
};
seen = true;
if name == SUM {
sum_ms += v;
} else {
count += v as u64;
}
}
seen.then_some((sum_ms, count))
}
/// Windowed mean bootstrap wait (seconds) between two cumulative scrapes.
/// None on the first scrape, when no bootstrap completed in the window, or
/// on a counter reset (engine restart — the caller rebases onto `cur`).
fn bootstrap_window_mean(prev: (f64, u64), cur: (f64, u64)) -> Option<f64> {
let dc = cur.1.checked_sub(prev.1)?;
let ds = cur.0 - prev.0;
if dc == 0 || ds < 0.0 {
return None;
}
Some(ds / dc as f64 / 1000.0)
}
/// Normalize a worker base URL (trailing slashes removed). pub(crate) so
/// shadow bookkeeping (pd_router) keys the prefix table identically.
pub(crate) fn normalize_url(url: &str) -> String {
@@ -1383,4 +1492,44 @@ mod tests {
assert!(parse_node_load(&json!({})).is_err());
assert!(parse_node_load(&json!({"loads": {}})).is_err());
}
/// Prometheus text parsing (T4): multi-rank sum/count aggregation,
/// bucket lines and unrelated metrics ignored, absent metric -> None.
#[test]
fn parse_bootstrap_histogram_aggregates_series() {
let text = "\
# HELP sglang:kv_transfer_bootstrap_ms KV transfer bootstrap latency
# TYPE sglang:kv_transfer_bootstrap_ms histogram
sglang:kv_transfer_bootstrap_ms_bucket{dp_rank=\"0\",le=\"10.0\"} 3
sglang:kv_transfer_bootstrap_ms_bucket{dp_rank=\"0\",le=\"+Inf\"} 5
sglang:kv_transfer_bootstrap_ms_sum{dp_rank=\"0\"} 250.5
sglang:kv_transfer_bootstrap_ms_count{dp_rank=\"0\"} 5
sglang:kv_transfer_bootstrap_ms_sum{dp_rank=\"1\"} 149.5
sglang:kv_transfer_bootstrap_ms_count{dp_rank=\"1\"} 3
sglang:token_usage{dp_rank=\"0\"} 0.5
";
let (sum, count) = parse_bootstrap_histogram(text).expect("metric present");
assert!((sum - 400.0).abs() < 1e-9, "sums aggregate over ranks");
assert_eq!(count, 8, "counts aggregate over ranks");
// Metric absent -> None (engine without the histogram, non-PD, ...).
assert!(parse_bootstrap_histogram("sglang:token_usage 0.5\n").is_none());
// A sum-only / count-only exposition still parses (partial data is
// better than none; the window mean just uses what is there).
assert_eq!(
parse_bootstrap_histogram("sglang:kv_transfer_bootstrap_ms_count 4\n"),
Some((0.0, 4))
);
}
#[test]
fn bootstrap_window_mean_diffs_cumulative_scrapes() {
// 400ms over 8 bootstraps in the window -> 50ms mean.
let mean = bootstrap_window_mean((1000.0, 10), (1400.0, 18)).unwrap();
assert!((mean - 0.05).abs() < 1e-9);
// No new samples in the window -> None.
assert_eq!(bootstrap_window_mean((1000.0, 10), (1000.0, 10)), None);
// Counter reset (engine restart) -> None; caller rebases.
assert_eq!(bootstrap_window_mean((1000.0, 10), (50.0, 1)), None);
}
}