From 047be72a8cb59db896d5a7a9a39ba151dde26790 Mon Sep 17 00:00:00 2001 From: Shangming Cai Date: Thu, 1 Oct 2026 14:15:42 +0800 Subject: [PATCH] [Router] Harden peer-bootstrap edges: refuse redirects and metric docs (#41999) Co-authored-by: Claude Fable 5 --- experimental/sgl-router/monitoring/README.md | 7 +++ .../sgl-router/src/server/routes/cache.rs | 13 ++++- .../src/state/kv_events/bootstrap/peers.rs | 8 ++- .../sgl-router/src/state/kv_events/index.rs | 6 ++ .../src/state/kv_events/index/tests.rs | 55 +++++++++++++++++++ 5 files changed, 86 insertions(+), 3 deletions(-) diff --git a/experimental/sgl-router/monitoring/README.md b/experimental/sgl-router/monitoring/README.md index a8d997c751..65d1a3c9f8 100644 --- a/experimental/sgl-router/monitoring/README.md +++ b/experimental/sgl-router/monitoring/README.md @@ -44,6 +44,13 @@ Families the router emits. The dashboard graphs all of them except the | `sgl_router_kv_tree_maintained` | Gauge | 1 when this router maintains its own KV tree, 0 under an external Indexer | | `sgl_router_kv_bootstrap_peers` | Gauge | Ready sibling router replicas peer bootstrap could pull a tree snapshot from. Emitted only with `--kv-peer-selector` | | `sgl_router_kv_bootstrap_peers_synced` | Gauge | 1 once peer discovery has reported at least once; a 0 that never becomes 1 means the peer watch is not delivering (check EndpointSlice RBAC). Emitted only with `--kv-peer-selector` | +| `sgl_router_kv_tree_nodes` | Gauge | Nodes in this replica's cache-aware tree. 0 after bootstrap settles means the replica is routing cache-blind. Emitted only with `--kv-peer-selector` | +| `sgl_router_kv_bootstrap_settled` | Gauge | 1 once initial peer bootstrap settled (every rank terminal, or the `--kv-bootstrap-timeout-ms` deadline). Settling says nothing about success — join with `sgl_router_kv_peer_snapshot_total` | +| `sgl_router_kv_bootstrap_seed_failed` | Gauge | 1 when the last sweep over a non-empty candidate set timed out: siblings were there and their tree could not be pulled. Emitted whether or not `--kv-bootstrap-seed-required` gates on it | +| `sgl_router_kv_bootstrap_state` | Gauge | Per-rank bootstrap state by `worker_url` / `dp_rank`: 0 pending (events held back, heading for overflow), 1 recovered, 2 failed (routing on live deltas alone) | +| `sgl_router_kv_peer_snapshot_total` | Counter | Peer snapshot fetches by `outcome` (`accepted` / `unreachable` / `cold_peer` / `rejected`). A fleet pinned at `unreachable` with warm siblings means the per-fetch bound is too small for the body | +| `sgl_router_kv_bootstrap_rank_total` | Counter | Final per-rank bootstrap verdicts by `outcome` (`warm`, `warm_unwitnessed`, `from_origin`, `gap`, `uncovered`, `abandoned`, `overflow`, `publisher_reset`, `tree_rejected`); one count per rank per incarnation | +| `sgl_router_kv_bootstrap_sweep_total` | Counter | Peer sweeps by `result` (`found` / `no_peers` / `fleet_cold` / `timed_out` / `ranks_resolved`) — what separates a healthy early settle on a cold fleet from burning the whole deadline | The legacy `sgl_router_overlap_blocks` metric was removed with the `cache_aware_zmq` policy and has no direct replacement. Remove queries, alerts, diff --git a/experimental/sgl-router/src/server/routes/cache.rs b/experimental/sgl-router/src/server/routes/cache.rs index fac43408bd..31929fdbc7 100644 --- a/experimental/sgl-router/src/server/routes/cache.rs +++ b/experimental/sgl-router/src/server/routes/cache.rs @@ -118,7 +118,18 @@ pub async fn kv_snapshot( if params.cursors_only { // Read live, so there is nothing for `max_age_ms` to bound; it is // ignored when both are sent. - return json_ok(index.peer_cursors_body()); + let body = index.peer_cursors_body(); + if body.is_empty() { + // Same shape as the full export's failure answer below: a 200 + // carrying an empty body would hand the caller JSON it cannot + // decode. Unreachable in practice (the body is a plain struct), + // kept so the two paths cannot drift apart. + return json_error( + StatusCode::SERVICE_UNAVAILABLE, + "cursor table could not be encoded", + ); + } + return json_ok(body); } let max_age = params .max_age_ms diff --git a/experimental/sgl-router/src/state/kv_events/bootstrap/peers.rs b/experimental/sgl-router/src/state/kv_events/bootstrap/peers.rs index e0c045fd0d..ca7848fe62 100644 --- a/experimental/sgl-router/src/state/kv_events/bootstrap/peers.rs +++ b/experimental/sgl-router/src/state/kv_events/bootstrap/peers.rs @@ -2,7 +2,7 @@ use parking_lot::Mutex; use rand::seq::SliceRandom; -use tracing::info; +use tracing::{debug, info}; /// Sibling replicas a snapshot may be pulled from. An empty set is conclusive /// only after a sync and only if siblings were never seen: before the first @@ -39,7 +39,11 @@ impl PeerRegistry { changed }; if let Some(peers) = changed { - info!(count = peers.len(), peers = ?peers, "kv-bootstrap: peer set updated"); + // Count at info; the full URL list only at debug — on a large + // fleet every rolling update re-lists every sibling, and one log + // line per change carrying every URL gets loud. + info!(count = peers.len(), "kv-bootstrap: peer set updated"); + debug!(peers = ?peers, "kv-bootstrap: peer set contents"); } } diff --git a/experimental/sgl-router/src/state/kv_events/index.rs b/experimental/sgl-router/src/state/kv_events/index.rs index d4627b574e..c04a90c0e4 100644 --- a/experimental/sgl-router/src/state/kv_events/index.rs +++ b/experimental/sgl-router/src/state/kv_events/index.rs @@ -384,6 +384,12 @@ impl KvEventIndex { .connect_timeout(SNAPSHOT_FETCH_CONNECT_TIMEOUT) .read_timeout(SNAPSHOT_FETCH_READ_TIMEOUT) .timeout(per_fetch) + // A sibling router never redirects this route, so a redirect is + // either a misconfigured peer or a hostile one steering the fetch + // — and its multi-gigabyte buffering budget — at an arbitrary + // in-cluster URL. Refuse to follow: the 3xx lands as + // `FetchAnswer::NoBody` and the peer is just not a source. + .redirect(reqwest::redirect::Policy::none()) .build() { Ok(client) => client, diff --git a/experimental/sgl-router/src/state/kv_events/index/tests.rs b/experimental/sgl-router/src/state/kv_events/index/tests.rs index 49c9a6aad5..c0d0615882 100644 --- a/experimental/sgl-router/src/state/kv_events/index/tests.rs +++ b/experimental/sgl-router/src/state/kv_events/index/tests.rs @@ -862,3 +862,58 @@ async fn remove_worker_clears_every_rank_before_returning() { assert_eq!(tree.node_count(), 0, "cleared chains must prune"); index.shutdown().await; } + +/// The snapshot client must not follow redirects: a sibling router never +/// redirects this route, so a 3xx is a misconfigured or hostile peer steering +/// the fetch — and its multi-gigabyte buffering budget — at an arbitrary +/// in-cluster URL. The 3xx must land as `FetchAnswer::NoBody` (peer is not a +/// source) with the redirect target never contacted. +/// +/// Asserted against the index's OWN client, not a test-local one: reqwest's +/// default policy follows up to ten redirects, so a copy built in the test +/// would prove nothing about the client the sweep and the probe actually use. +#[tokio::test] +async fn snapshot_client_refuses_to_follow_a_redirecting_peer() { + use crate::state::kv_events::bootstrap::{fetch_snapshot, FetchAnswer, SNAPSHOT_PATH}; + + let followed = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let poisoned = Arc::clone(&followed); + let app = axum::Router::new() + .route( + SNAPSHOT_PATH, + axum::routing::get(|| async { + ( + axum::http::StatusCode::FOUND, + [(axum::http::header::LOCATION, "/poison")], + ) + }), + ) + .route( + "/poison", + axum::routing::get(move || { + let poisoned = Arc::clone(&poisoned); + async move { + poisoned.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + "gotcha" + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + + let index = KvEventIndex::new(); + let got = fetch_snapshot(&index.snapshot_http, &base, None) + .await + .expect("a refused redirect is an answer, not a transport error"); + assert!( + matches!(got, FetchAnswer::NoBody(status) if status.is_redirection()), + "a redirecting peer must read as 'no snapshot here'", + ); + assert_eq!( + followed.load(std::sync::atomic::Ordering::SeqCst), + 0, + "the redirect target must never be contacted", + ); + index.shutdown().await; +}