diff --git a/experimental/sgl-router/src/server/metrics.rs b/experimental/sgl-router/src/server/metrics.rs index 55e602688..cc9a0f587 100644 --- a/experimental/sgl-router/src/server/metrics.rs +++ b/experimental/sgl-router/src/server/metrics.rs @@ -45,13 +45,14 @@ use std::collections::HashMap; use std::sync::atomic::{AtomicI64, AtomicU64, Ordering}; use std::sync::Arc; -/// Histogram bucket upper bounds for `sgl_router_overlap_blocks`. Chosen to -/// span 0 → ~1k blocks: blocks are 32–64 tokens each, and our `MAX_CHAT_BODY_BYTES` -/// cap (1 MiB ≈ 250 k tokens) implies an upper bound around 4–8 k blocks for -/// a maximum-length context. The `+Inf` bucket catches everything beyond -/// 1000. +/// Histogram bucket upper bounds for `sgl_router_overlap_blocks`. Blocks are +/// 32–64 tokens each, and the `MAX_CHAT_BODY_BYTES` cap bounds context length — +/// putting the practical ceiling for a maximum-length context in the low tens +/// of thousands of blocks. The ladder spans 0 → ~8k blocks at the resolution +/// worth charting; the `+Inf` bucket catches the longer-context tail beyond +/// 8000. const OVERLAP_BLOCKS_BUCKETS: &[f64] = &[ - 0.0, 1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1000.0, + 0.0, 1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1000.0, 2000.0, 4000.0, 8000.0, ]; /// Histogram bucket upper bounds (seconds) for @@ -1056,11 +1057,11 @@ mod tests { #[test] fn histogram_plus_inf_bucket_catches_overflow() { let reg = MetricsRegistry::new(); - // 1001 is just above the last finite bucket (1000); it should land + // 8001 is just above the last finite bucket (8000); it should land // in +Inf only. - reg.observe_overlap_blocks("m", 1001); + reg.observe_overlap_blocks("m", 8001); let out = reg.render(); - assert!(out.contains(r#"sgl_router_overlap_blocks_bucket{model_id="m",le="1000"} 0"#)); + assert!(out.contains(r#"sgl_router_overlap_blocks_bucket{model_id="m",le="8000"} 0"#)); assert!(out.contains(r#"sgl_router_overlap_blocks_bucket{model_id="m",le="+Inf"} 1"#)); } } diff --git a/experimental/sgl-router/src/server/routes/chat.rs b/experimental/sgl-router/src/server/routes/chat.rs index d2ce1c1d2..bac3d799b 100644 --- a/experimental/sgl-router/src/server/routes/chat.rs +++ b/experimental/sgl-router/src/server/routes/chat.rs @@ -39,21 +39,21 @@ const X_SGL_DECODE_URL: HeaderName = HeaderName::from_static("x-sgl-decode-url") /// purpose. const CHARS_PER_TOKEN_ESTIMATE: usize = 4; -/// Per-route body-size cap on `/v1/chat/completions`. 1 MiB is comfortable -/// for normal chat traffic (a 200 k-token context tokenized as JSON is well -/// under this) while preventing a hostile client from forcing the router to +/// Per-route body-size cap on `/v1/chat/completions`. 5 MiB accommodates a +/// long context — a ~1 M-token context tokenized as JSON fits under this — +/// while preventing a hostile client from forcing the router to /// heap-allocate hundreds of MiB before forwarding. The cap is wired in /// `crate::server::app::build_router` as a route-level `DefaultBodyLimit` /// layer; axum's `Bytes` extractor enforces it and returns 413 /// PAYLOAD_TOO_LARGE before this handler runs. -pub const MAX_CHAT_BODY_BYTES: usize = 1 << 20; +pub const MAX_CHAT_BODY_BYTES: usize = 5 << 20; /// Minimal probe over the request body — we only need the `stream` field /// and the `model` field to decide between buffered vs SSE forwarding and /// to select a worker. Deserializing into this struct (vs `serde_json::Value`) /// does two things: /// -/// 1. Avoids the per-field heap allocation of `Value` for a 1 MiB body. +/// 1. Avoids the per-field heap allocation of `Value` for a multi-MiB body. /// 2. Pins the contract: the body MUST be a JSON object. Degenerate /// shapes (`null`, `[]`, `"hi"`) fail at this step rather than being /// silently forwarded with `stream=false`. diff --git a/experimental/sgl-router/tests/proxy/chat_routing.rs b/experimental/sgl-router/tests/proxy/chat_routing.rs index 72f4e80d1..b91404eb1 100644 --- a/experimental/sgl-router/tests/proxy/chat_routing.rs +++ b/experimental/sgl-router/tests/proxy/chat_routing.rs @@ -10,6 +10,7 @@ use sgl_router::policies::factory::build_registry_with_defaults as build_policy_ use sgl_router::proxy::Proxy; use sgl_router::server::app::build_router; use sgl_router::server::app_context::AppContext; +use sgl_router::server::routes::chat::MAX_CHAT_BODY_BYTES; use sgl_router::tokenizer::TokenizerRegistry; use sgl_router::workers::{Worker, WorkerRegistry}; @@ -514,8 +515,9 @@ async fn oversized_request_body_returns_413() { let ctx = build_ctx_with_worker(&worker.url); let app = build_router(ctx); - // 2 MiB body — the configured limit is 1 MiB. - let big = vec![b'x'; 2 * 1024 * 1024]; + // One byte over the configured cap, so the test tracks the cap + // (`MAX_CHAT_BODY_BYTES`) instead of a hardcoded size. + let big = vec![b'x'; MAX_CHAT_BODY_BYTES + 1]; let req = Request::builder() .method("POST") .uri("/v1/chat/completions")