[Router] Harden peer-bootstrap edges: refuse redirects and metric docs (#41999)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Shangming Cai
2026-10-01 14:15:42 +08:00
committed by GitHub
co-authored by Claude Fable 5
parent 99c9d65b32
commit 047be72a8c
5 changed files with 86 additions and 3 deletions
@@ -44,6 +44,13 @@ Families the router emits. The dashboard graphs all of them except the
| `sgl_router_kv_tree_maintained` | Gauge | 1 when this router maintains its own KV tree, 0 under an external Indexer |
| `sgl_router_kv_bootstrap_peers` | Gauge | Ready sibling router replicas peer bootstrap could pull a tree snapshot from. Emitted only with `--kv-peer-selector` |
| `sgl_router_kv_bootstrap_peers_synced` | Gauge | 1 once peer discovery has reported at least once; a 0 that never becomes 1 means the peer watch is not delivering (check EndpointSlice RBAC). Emitted only with `--kv-peer-selector` |
| `sgl_router_kv_tree_nodes` | Gauge | Nodes in this replica's cache-aware tree. 0 after bootstrap settles means the replica is routing cache-blind. Emitted only with `--kv-peer-selector` |
| `sgl_router_kv_bootstrap_settled` | Gauge | 1 once initial peer bootstrap settled (every rank terminal, or the `--kv-bootstrap-timeout-ms` deadline). Settling says nothing about success — join with `sgl_router_kv_peer_snapshot_total` |
| `sgl_router_kv_bootstrap_seed_failed` | Gauge | 1 when the last sweep over a non-empty candidate set timed out: siblings were there and their tree could not be pulled. Emitted whether or not `--kv-bootstrap-seed-required` gates on it |
| `sgl_router_kv_bootstrap_state` | Gauge | Per-rank bootstrap state by `worker_url` / `dp_rank`: 0 pending (events held back, heading for overflow), 1 recovered, 2 failed (routing on live deltas alone) |
| `sgl_router_kv_peer_snapshot_total` | Counter | Peer snapshot fetches by `outcome` (`accepted` / `unreachable` / `cold_peer` / `rejected`). A fleet pinned at `unreachable` with warm siblings means the per-fetch bound is too small for the body |
| `sgl_router_kv_bootstrap_rank_total` | Counter | Final per-rank bootstrap verdicts by `outcome` (`warm`, `warm_unwitnessed`, `from_origin`, `gap`, `uncovered`, `abandoned`, `overflow`, `publisher_reset`, `tree_rejected`); one count per rank per incarnation |
| `sgl_router_kv_bootstrap_sweep_total` | Counter | Peer sweeps by `result` (`found` / `no_peers` / `fleet_cold` / `timed_out` / `ranks_resolved`) — what separates a healthy early settle on a cold fleet from burning the whole deadline |
The legacy `sgl_router_overlap_blocks` metric was removed with the
`cache_aware_zmq` policy and has no direct replacement. Remove queries, alerts,
@@ -118,7 +118,18 @@ pub async fn kv_snapshot(
if params.cursors_only {
// Read live, so there is nothing for `max_age_ms` to bound; it is
// ignored when both are sent.
return json_ok(index.peer_cursors_body());
let body = index.peer_cursors_body();
if body.is_empty() {
// Same shape as the full export's failure answer below: a 200
// carrying an empty body would hand the caller JSON it cannot
// decode. Unreachable in practice (the body is a plain struct),
// kept so the two paths cannot drift apart.
return json_error(
StatusCode::SERVICE_UNAVAILABLE,
"cursor table could not be encoded",
);
}
return json_ok(body);
}
let max_age = params
.max_age_ms
@@ -2,7 +2,7 @@
use parking_lot::Mutex;
use rand::seq::SliceRandom;
use tracing::info;
use tracing::{debug, info};
/// Sibling replicas a snapshot may be pulled from. An empty set is conclusive
/// only after a sync and only if siblings were never seen: before the first
@@ -39,7 +39,11 @@ impl PeerRegistry {
changed
};
if let Some(peers) = changed {
info!(count = peers.len(), peers = ?peers, "kv-bootstrap: peer set updated");
// Count at info; the full URL list only at debug — on a large
// fleet every rolling update re-lists every sibling, and one log
// line per change carrying every URL gets loud.
info!(count = peers.len(), "kv-bootstrap: peer set updated");
debug!(peers = ?peers, "kv-bootstrap: peer set contents");
}
}
@@ -384,6 +384,12 @@ impl KvEventIndex {
.connect_timeout(SNAPSHOT_FETCH_CONNECT_TIMEOUT)
.read_timeout(SNAPSHOT_FETCH_READ_TIMEOUT)
.timeout(per_fetch)
// A sibling router never redirects this route, so a redirect is
// either a misconfigured peer or a hostile one steering the fetch
// — and its multi-gigabyte buffering budget — at an arbitrary
// in-cluster URL. Refuse to follow: the 3xx lands as
// `FetchAnswer::NoBody` and the peer is just not a source.
.redirect(reqwest::redirect::Policy::none())
.build()
{
Ok(client) => client,
@@ -862,3 +862,58 @@ async fn remove_worker_clears_every_rank_before_returning() {
assert_eq!(tree.node_count(), 0, "cleared chains must prune");
index.shutdown().await;
}
/// The snapshot client must not follow redirects: a sibling router never
/// redirects this route, so a 3xx is a misconfigured or hostile peer steering
/// the fetch — and its multi-gigabyte buffering budget — at an arbitrary
/// in-cluster URL. The 3xx must land as `FetchAnswer::NoBody` (peer is not a
/// source) with the redirect target never contacted.
///
/// Asserted against the index's OWN client, not a test-local one: reqwest's
/// default policy follows up to ten redirects, so a copy built in the test
/// would prove nothing about the client the sweep and the probe actually use.
#[tokio::test]
async fn snapshot_client_refuses_to_follow_a_redirecting_peer() {
use crate::state::kv_events::bootstrap::{fetch_snapshot, FetchAnswer, SNAPSHOT_PATH};
let followed = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let poisoned = Arc::clone(&followed);
let app = axum::Router::new()
.route(
SNAPSHOT_PATH,
axum::routing::get(|| async {
(
axum::http::StatusCode::FOUND,
[(axum::http::header::LOCATION, "/poison")],
)
}),
)
.route(
"/poison",
axum::routing::get(move || {
let poisoned = Arc::clone(&poisoned);
async move {
poisoned.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
"gotcha"
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let base = format!("http://{}", listener.local_addr().unwrap());
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let index = KvEventIndex::new();
let got = fetch_snapshot(&index.snapshot_http, &base, None)
.await
.expect("a refused redirect is an answer, not a transport error");
assert!(
matches!(got, FetchAnswer::NoBody(status) if status.is_redirection()),
"a redirecting peer must read as 'no snapshot here'",
);
assert_eq!(
followed.load(std::sync::atomic::Ordering::SeqCst),
0,
"the redirect target must never be contacted",
);
index.shutdown().await;
}