[router] Raise chat body cap to 5 MiB for long contexts (#28742)

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

experimental/sgl-router/src/server/metrics.rs

@@ -45,13 +45,14 @@ use std::collections::HashMap;
 use std::sync::atomic::{AtomicI64, AtomicU64, Ordering};
 use std::sync::Arc;
 
-/// Histogram bucket upper bounds for `sgl_router_overlap_blocks`. Chosen to
-/// span 0 → ~1k blocks: blocks are 32–64 tokens each, and our `MAX_CHAT_BODY_BYTES`
-/// cap (1 MiB ≈ 250 k tokens) implies an upper bound around 4–8 k blocks for
-/// a maximum-length context. The `+Inf` bucket catches everything beyond
-/// 1000.
+/// Histogram bucket upper bounds for `sgl_router_overlap_blocks`. Blocks are
+/// 32–64 tokens each, and the `MAX_CHAT_BODY_BYTES` cap bounds context length —
+/// putting the practical ceiling for a maximum-length context in the low tens
+/// of thousands of blocks. The ladder spans 0 → ~8k blocks at the resolution
+/// worth charting; the `+Inf` bucket catches the longer-context tail beyond
+/// 8000.
 const OVERLAP_BLOCKS_BUCKETS: &[f64] = &[
-    0.0, 1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1000.0,
+    0.0, 1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1000.0, 2000.0, 4000.0, 8000.0,
 ];
 
 /// Histogram bucket upper bounds (seconds) for
@@ -1056,11 +1057,11 @@ mod tests {
     #[test]
     fn histogram_plus_inf_bucket_catches_overflow() {
         let reg = MetricsRegistry::new();
-        // 1001 is just above the last finite bucket (1000); it should land
+        // 8001 is just above the last finite bucket (8000); it should land
         // in +Inf only.
-        reg.observe_overlap_blocks("m", 1001);
+        reg.observe_overlap_blocks("m", 8001);
         let out = reg.render();
-        assert!(out.contains(r#"sgl_router_overlap_blocks_bucket{model_id="m",le="1000"} 0"#));
+        assert!(out.contains(r#"sgl_router_overlap_blocks_bucket{model_id="m",le="8000"} 0"#));
         assert!(out.contains(r#"sgl_router_overlap_blocks_bucket{model_id="m",le="+Inf"} 1"#));
     }
 }

experimental/sgl-router/src/server/routes/chat.rs

@@ -39,21 +39,21 @@ const X_SGL_DECODE_URL: HeaderName = HeaderName::from_static("x-sgl-decode-url")
 /// purpose.
 const CHARS_PER_TOKEN_ESTIMATE: usize = 4;
 
-/// Per-route body-size cap on `/v1/chat/completions`. 1 MiB is comfortable
-/// for normal chat traffic (a 200 k-token context tokenized as JSON is well
-/// under this) while preventing a hostile client from forcing the router to
+/// Per-route body-size cap on `/v1/chat/completions`. 5 MiB accommodates a
+/// long context — a ~1 M-token context tokenized as JSON fits under this —
+/// while preventing a hostile client from forcing the router to
 /// heap-allocate hundreds of MiB before forwarding. The cap is wired in
 /// `crate::server::app::build_router` as a route-level `DefaultBodyLimit`
 /// layer; axum's `Bytes` extractor enforces it and returns 413
 /// PAYLOAD_TOO_LARGE before this handler runs.
-pub const MAX_CHAT_BODY_BYTES: usize = 1 << 20;
+pub const MAX_CHAT_BODY_BYTES: usize = 5 << 20;
 
 /// Minimal probe over the request body — we only need the `stream` field
 /// and the `model` field to decide between buffered vs SSE forwarding and
 /// to select a worker. Deserializing into this struct (vs `serde_json::Value`)
 /// does two things:
 ///
-/// 1. Avoids the per-field heap allocation of `Value` for a 1 MiB body.
+/// 1. Avoids the per-field heap allocation of `Value` for a multi-MiB body.
 /// 2. Pins the contract: the body MUST be a JSON object. Degenerate
 ///    shapes (`null`, `[]`, `"hi"`) fail at this step rather than being
 ///    silently forwarded with `stream=false`.

experimental/sgl-router/tests/proxy/chat_routing.rs

@@ -10,6 +10,7 @@ use sgl_router::policies::factory::build_registry_with_defaults as build_policy_
 use sgl_router::proxy::Proxy;
 use sgl_router::server::app::build_router;
 use sgl_router::server::app_context::AppContext;
+use sgl_router::server::routes::chat::MAX_CHAT_BODY_BYTES;
 use sgl_router::tokenizer::TokenizerRegistry;
 use sgl_router::workers::{Worker, WorkerRegistry};
 
@@ -514,8 +515,9 @@ async fn oversized_request_body_returns_413() {
     let ctx = build_ctx_with_worker(&worker.url);
     let app = build_router(ctx);
 
-    // 2 MiB body — the configured limit is 1 MiB.
-    let big = vec![b'x'; 2 * 1024 * 1024];
+    // One byte over the configured cap, so the test tracks the cap
+    // (`MAX_CHAT_BODY_BYTES`) instead of a hardcoded size.
+    let big = vec![b'x'; MAX_CHAT_BODY_BYTES + 1];
     let req = Request::builder()
         .method("POST")
         .uri("/v1/chat/completions")