gateway g9i: scrape P /metrics kv_transfer_bootstrap_ms as est_T_D ground truth (P1 shadow T4)
This commit is contained in:
@@ -309,6 +309,11 @@ pub struct G9iState {
|
||||
/// Input of the est_T_D shadow estimate (design 2026-10-14 §2.1);
|
||||
/// populated only while shadow mode is on.
|
||||
d_snapshots: RwLock<Vec<(String, NodeLoad)>>,
|
||||
/// Per-prefill-node cumulative (sum_ms, count) of the engine's
|
||||
/// `sglang:kv_transfer_bootstrap_ms` Prometheus histogram — the est_T_D
|
||||
/// ground truth (design §2.3-2). Windowed means come from diffing
|
||||
/// successive scrapes.
|
||||
bootstrap_stats: RwLock<HashMap<String, (f64, u64)>>,
|
||||
}
|
||||
|
||||
/// Freshness of a cached probe answer. Short on purpose: an eviction on the
|
||||
@@ -398,6 +403,7 @@ impl G9iState {
|
||||
shadow,
|
||||
shadow_queue,
|
||||
d_snapshots: RwLock::new(Vec::new()),
|
||||
bootstrap_stats: RwLock::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -477,6 +483,16 @@ impl G9iState {
|
||||
if let Ok(mut d) = self.d_snapshots.write() {
|
||||
*d = d_results;
|
||||
}
|
||||
// est_T_D P1 ground truth (design 2026-10-14 §2.3-2): scrape
|
||||
// each prefill node's /metrics for the per-request
|
||||
// kv_transfer_bootstrap_ms histogram and publish the windowed
|
||||
// mean next to the estimate. P1 only measures side by side —
|
||||
// the correction loop is P3.
|
||||
for worker in &workers {
|
||||
if matches!(worker.worker_type(), WorkerType::Prefill { .. }) {
|
||||
self.scrape_bootstrap_metric(client, worker.as_ref()).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Ok(mut snap) = self.snapshots.write() {
|
||||
@@ -488,6 +504,51 @@ impl G9iState {
|
||||
self.publish_ttft_percentiles();
|
||||
}
|
||||
|
||||
/// Scrape one prefill node's /metrics for `sglang:kv_transfer_bootstrap_ms`
|
||||
/// and publish the windowed mean (seconds) since the previous scrape.
|
||||
/// All failures are debug-logged and skipped — this is a shadow-side
|
||||
/// observability path, never worth a warn.
|
||||
async fn scrape_bootstrap_metric(&self, client: &Client, worker: &dyn Worker) {
|
||||
let url = normalize_url(&worker.url().to_string());
|
||||
let mut req = client
|
||||
.get(format!("{url}/metrics"))
|
||||
.timeout(Duration::from_secs(3));
|
||||
if let Some(key) = worker.api_key() {
|
||||
req = req.bearer_auth(key);
|
||||
}
|
||||
let text = match req.send().await {
|
||||
Ok(r) if r.status().is_success() => match r.text().await {
|
||||
Ok(t) => t,
|
||||
Err(e) => {
|
||||
debug!("G9i /metrics read failed for {}: {}", url, e);
|
||||
return;
|
||||
}
|
||||
},
|
||||
Ok(r) => {
|
||||
debug!("G9i /metrics fetch HTTP {} for {}", r.status(), url);
|
||||
return;
|
||||
}
|
||||
Err(e) => {
|
||||
debug!("G9i /metrics fetch failed for {}: {}", url, e);
|
||||
return;
|
||||
}
|
||||
};
|
||||
let Some(cur) = parse_bootstrap_histogram(&text) else {
|
||||
return;
|
||||
};
|
||||
let prev = match self.bootstrap_stats.write() {
|
||||
Ok(mut s) => s.insert(url.clone(), cur),
|
||||
Err(_) => None,
|
||||
};
|
||||
if let Some(mean_secs) = prev.and_then(|p| bootstrap_window_mean(p, cur)) {
|
||||
metrics::gauge!(
|
||||
super::g9i_shadow::METRIC_BOOTSTRAP_ACTUAL,
|
||||
"worker" => url,
|
||||
)
|
||||
.set(mean_secs);
|
||||
}
|
||||
}
|
||||
|
||||
fn snapshot(&self, url: &str) -> Option<NodeLoad> {
|
||||
self.snapshots
|
||||
.read()
|
||||
@@ -1239,6 +1300,54 @@ fn parse_node_load(json: &Value) -> Result<NodeLoad, String> {
|
||||
Ok(node)
|
||||
}
|
||||
|
||||
/// Aggregate `sglang:kv_transfer_bootstrap_ms` histogram totals (sum_ms,
|
||||
/// count) from a Prometheus text exposition, summing over every label
|
||||
/// variant (per-dp-rank series). None when the metric is absent entirely.
|
||||
fn parse_bootstrap_histogram(text: &str) -> Option<(f64, u64)> {
|
||||
const SUM: &str = "sglang:kv_transfer_bootstrap_ms_sum";
|
||||
const COUNT: &str = "sglang:kv_transfer_bootstrap_ms_count";
|
||||
let mut sum_ms = 0.0f64;
|
||||
let mut count = 0u64;
|
||||
let mut seen = false;
|
||||
for line in text.lines() {
|
||||
// Series name ends at the first '{' (label block) or whitespace.
|
||||
let name_end = line
|
||||
.find(|c: char| c == '{' || c == ' ' || c == '\t')
|
||||
.unwrap_or(line.len());
|
||||
let name = &line[..name_end];
|
||||
if name != SUM && name != COUNT {
|
||||
continue;
|
||||
}
|
||||
// The sample value is the last whitespace-separated token (label
|
||||
// values of this metric — dp_rank & friends — never contain spaces).
|
||||
let Some(val_idx) = line.rfind([' ', '\t']) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(v) = line[val_idx + 1..].parse::<f64>() else {
|
||||
continue;
|
||||
};
|
||||
seen = true;
|
||||
if name == SUM {
|
||||
sum_ms += v;
|
||||
} else {
|
||||
count += v as u64;
|
||||
}
|
||||
}
|
||||
seen.then_some((sum_ms, count))
|
||||
}
|
||||
|
||||
/// Windowed mean bootstrap wait (seconds) between two cumulative scrapes.
|
||||
/// None on the first scrape, when no bootstrap completed in the window, or
|
||||
/// on a counter reset (engine restart — the caller rebases onto `cur`).
|
||||
fn bootstrap_window_mean(prev: (f64, u64), cur: (f64, u64)) -> Option<f64> {
|
||||
let dc = cur.1.checked_sub(prev.1)?;
|
||||
let ds = cur.0 - prev.0;
|
||||
if dc == 0 || ds < 0.0 {
|
||||
return None;
|
||||
}
|
||||
Some(ds / dc as f64 / 1000.0)
|
||||
}
|
||||
|
||||
/// Normalize a worker base URL (trailing slashes removed). pub(crate) so
|
||||
/// shadow bookkeeping (pd_router) keys the prefix table identically.
|
||||
pub(crate) fn normalize_url(url: &str) -> String {
|
||||
@@ -1383,4 +1492,44 @@ mod tests {
|
||||
assert!(parse_node_load(&json!({})).is_err());
|
||||
assert!(parse_node_load(&json!({"loads": {}})).is_err());
|
||||
}
|
||||
|
||||
/// Prometheus text parsing (T4): multi-rank sum/count aggregation,
|
||||
/// bucket lines and unrelated metrics ignored, absent metric -> None.
|
||||
#[test]
|
||||
fn parse_bootstrap_histogram_aggregates_series() {
|
||||
let text = "\
|
||||
# HELP sglang:kv_transfer_bootstrap_ms KV transfer bootstrap latency
|
||||
# TYPE sglang:kv_transfer_bootstrap_ms histogram
|
||||
sglang:kv_transfer_bootstrap_ms_bucket{dp_rank=\"0\",le=\"10.0\"} 3
|
||||
sglang:kv_transfer_bootstrap_ms_bucket{dp_rank=\"0\",le=\"+Inf\"} 5
|
||||
sglang:kv_transfer_bootstrap_ms_sum{dp_rank=\"0\"} 250.5
|
||||
sglang:kv_transfer_bootstrap_ms_count{dp_rank=\"0\"} 5
|
||||
sglang:kv_transfer_bootstrap_ms_sum{dp_rank=\"1\"} 149.5
|
||||
sglang:kv_transfer_bootstrap_ms_count{dp_rank=\"1\"} 3
|
||||
sglang:token_usage{dp_rank=\"0\"} 0.5
|
||||
";
|
||||
let (sum, count) = parse_bootstrap_histogram(text).expect("metric present");
|
||||
assert!((sum - 400.0).abs() < 1e-9, "sums aggregate over ranks");
|
||||
assert_eq!(count, 8, "counts aggregate over ranks");
|
||||
|
||||
// Metric absent -> None (engine without the histogram, non-PD, ...).
|
||||
assert!(parse_bootstrap_histogram("sglang:token_usage 0.5\n").is_none());
|
||||
// A sum-only / count-only exposition still parses (partial data is
|
||||
// better than none; the window mean just uses what is there).
|
||||
assert_eq!(
|
||||
parse_bootstrap_histogram("sglang:kv_transfer_bootstrap_ms_count 4\n"),
|
||||
Some((0.0, 4))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bootstrap_window_mean_diffs_cumulative_scrapes() {
|
||||
// 400ms over 8 bootstraps in the window -> 50ms mean.
|
||||
let mean = bootstrap_window_mean((1000.0, 10), (1400.0, 18)).unwrap();
|
||||
assert!((mean - 0.05).abs() < 1e-9);
|
||||
// No new samples in the window -> None.
|
||||
assert_eq!(bootstrap_window_mean((1000.0, 10), (1000.0, 10)), None);
|
||||
// Counter reset (engine restart) -> None; caller rebases.
|
||||
assert_eq!(bootstrap_window_mean((1000.0, 10), (50.0, 1)), None);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user