[Router] Harden peer-bootstrap edges: refuse redirects and metric docs (#41999)
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
99c9d65b32
commit
047be72a8c
@@ -44,6 +44,13 @@ Families the router emits. The dashboard graphs all of them except the
|
||||
| `sgl_router_kv_tree_maintained` | Gauge | 1 when this router maintains its own KV tree, 0 under an external Indexer |
|
||||
| `sgl_router_kv_bootstrap_peers` | Gauge | Ready sibling router replicas peer bootstrap could pull a tree snapshot from. Emitted only with `--kv-peer-selector` |
|
||||
| `sgl_router_kv_bootstrap_peers_synced` | Gauge | 1 once peer discovery has reported at least once; a 0 that never becomes 1 means the peer watch is not delivering (check EndpointSlice RBAC). Emitted only with `--kv-peer-selector` |
|
||||
| `sgl_router_kv_tree_nodes` | Gauge | Nodes in this replica's cache-aware tree. 0 after bootstrap settles means the replica is routing cache-blind. Emitted only with `--kv-peer-selector` |
|
||||
| `sgl_router_kv_bootstrap_settled` | Gauge | 1 once initial peer bootstrap settled (every rank terminal, or the `--kv-bootstrap-timeout-ms` deadline). Settling says nothing about success — join with `sgl_router_kv_peer_snapshot_total` |
|
||||
| `sgl_router_kv_bootstrap_seed_failed` | Gauge | 1 when the last sweep over a non-empty candidate set timed out: siblings were there and their tree could not be pulled. Emitted whether or not `--kv-bootstrap-seed-required` gates on it |
|
||||
| `sgl_router_kv_bootstrap_state` | Gauge | Per-rank bootstrap state by `worker_url` / `dp_rank`: 0 pending (events held back, heading for overflow), 1 recovered, 2 failed (routing on live deltas alone) |
|
||||
| `sgl_router_kv_peer_snapshot_total` | Counter | Peer snapshot fetches by `outcome` (`accepted` / `unreachable` / `cold_peer` / `rejected`). A fleet pinned at `unreachable` with warm siblings means the per-fetch bound is too small for the body |
|
||||
| `sgl_router_kv_bootstrap_rank_total` | Counter | Final per-rank bootstrap verdicts by `outcome` (`warm`, `warm_unwitnessed`, `from_origin`, `gap`, `uncovered`, `abandoned`, `overflow`, `publisher_reset`, `tree_rejected`); one count per rank per incarnation |
|
||||
| `sgl_router_kv_bootstrap_sweep_total` | Counter | Peer sweeps by `result` (`found` / `no_peers` / `fleet_cold` / `timed_out` / `ranks_resolved`) — what separates a healthy early settle on a cold fleet from burning the whole deadline |
|
||||
|
||||
The legacy `sgl_router_overlap_blocks` metric was removed with the
|
||||
`cache_aware_zmq` policy and has no direct replacement. Remove queries, alerts,
|
||||
|
||||
@@ -118,7 +118,18 @@ pub async fn kv_snapshot(
|
||||
if params.cursors_only {
|
||||
// Read live, so there is nothing for `max_age_ms` to bound; it is
|
||||
// ignored when both are sent.
|
||||
return json_ok(index.peer_cursors_body());
|
||||
let body = index.peer_cursors_body();
|
||||
if body.is_empty() {
|
||||
// Same shape as the full export's failure answer below: a 200
|
||||
// carrying an empty body would hand the caller JSON it cannot
|
||||
// decode. Unreachable in practice (the body is a plain struct),
|
||||
// kept so the two paths cannot drift apart.
|
||||
return json_error(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"cursor table could not be encoded",
|
||||
);
|
||||
}
|
||||
return json_ok(body);
|
||||
}
|
||||
let max_age = params
|
||||
.max_age_ms
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
use parking_lot::Mutex;
|
||||
use rand::seq::SliceRandom;
|
||||
use tracing::info;
|
||||
use tracing::{debug, info};
|
||||
|
||||
/// Sibling replicas a snapshot may be pulled from. An empty set is conclusive
|
||||
/// only after a sync and only if siblings were never seen: before the first
|
||||
@@ -39,7 +39,11 @@ impl PeerRegistry {
|
||||
changed
|
||||
};
|
||||
if let Some(peers) = changed {
|
||||
info!(count = peers.len(), peers = ?peers, "kv-bootstrap: peer set updated");
|
||||
// Count at info; the full URL list only at debug — on a large
|
||||
// fleet every rolling update re-lists every sibling, and one log
|
||||
// line per change carrying every URL gets loud.
|
||||
info!(count = peers.len(), "kv-bootstrap: peer set updated");
|
||||
debug!(peers = ?peers, "kv-bootstrap: peer set contents");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -384,6 +384,12 @@ impl KvEventIndex {
|
||||
.connect_timeout(SNAPSHOT_FETCH_CONNECT_TIMEOUT)
|
||||
.read_timeout(SNAPSHOT_FETCH_READ_TIMEOUT)
|
||||
.timeout(per_fetch)
|
||||
// A sibling router never redirects this route, so a redirect is
|
||||
// either a misconfigured peer or a hostile one steering the fetch
|
||||
// — and its multi-gigabyte buffering budget — at an arbitrary
|
||||
// in-cluster URL. Refuse to follow: the 3xx lands as
|
||||
// `FetchAnswer::NoBody` and the peer is just not a source.
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.build()
|
||||
{
|
||||
Ok(client) => client,
|
||||
|
||||
@@ -862,3 +862,58 @@ async fn remove_worker_clears_every_rank_before_returning() {
|
||||
assert_eq!(tree.node_count(), 0, "cleared chains must prune");
|
||||
index.shutdown().await;
|
||||
}
|
||||
|
||||
/// The snapshot client must not follow redirects: a sibling router never
|
||||
/// redirects this route, so a 3xx is a misconfigured or hostile peer steering
|
||||
/// the fetch — and its multi-gigabyte buffering budget — at an arbitrary
|
||||
/// in-cluster URL. The 3xx must land as `FetchAnswer::NoBody` (peer is not a
|
||||
/// source) with the redirect target never contacted.
|
||||
///
|
||||
/// Asserted against the index's OWN client, not a test-local one: reqwest's
|
||||
/// default policy follows up to ten redirects, so a copy built in the test
|
||||
/// would prove nothing about the client the sweep and the probe actually use.
|
||||
#[tokio::test]
|
||||
async fn snapshot_client_refuses_to_follow_a_redirecting_peer() {
|
||||
use crate::state::kv_events::bootstrap::{fetch_snapshot, FetchAnswer, SNAPSHOT_PATH};
|
||||
|
||||
let followed = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let poisoned = Arc::clone(&followed);
|
||||
let app = axum::Router::new()
|
||||
.route(
|
||||
SNAPSHOT_PATH,
|
||||
axum::routing::get(|| async {
|
||||
(
|
||||
axum::http::StatusCode::FOUND,
|
||||
[(axum::http::header::LOCATION, "/poison")],
|
||||
)
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/poison",
|
||||
axum::routing::get(move || {
|
||||
let poisoned = Arc::clone(&poisoned);
|
||||
async move {
|
||||
poisoned.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
"gotcha"
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let base = format!("http://{}", listener.local_addr().unwrap());
|
||||
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
|
||||
|
||||
let index = KvEventIndex::new();
|
||||
let got = fetch_snapshot(&index.snapshot_http, &base, None)
|
||||
.await
|
||||
.expect("a refused redirect is an answer, not a transport error");
|
||||
assert!(
|
||||
matches!(got, FetchAnswer::NoBody(status) if status.is_redirection()),
|
||||
"a redirecting peer must read as 'no snapshot here'",
|
||||
);
|
||||
assert_eq!(
|
||||
followed.load(std::sync::atomic::Ordering::SeqCst),
|
||||
0,
|
||||
"the redirect target must never be contacted",
|
||||
);
|
||||
index.shutdown().await;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user