[router] Raise chat body cap to 32 MB for multimodal payloads (#38735)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Kan Wu
2026-09-09 22:14:49 -07:00
committed by GitHub
co-authored by Claude Fable 5
parent 01ad27ecd9
commit 6f481ad0e3
@@ -107,14 +107,9 @@ fn prefill_policy_reason(
}
}
/// Per-route body-size cap on `/v1/chat/completions`. 5 MiB accommodates a
/// long context — a ~1 M-token context tokenized as JSON fits under this —
/// while preventing a hostile client from forcing the router to
/// heap-allocate hundreds of MiB before forwarding. The cap is wired in
/// `crate::server::app::build_router` as a route-level `DefaultBodyLimit`
/// layer; axum's `Bytes` extractor enforces it and returns 413
/// PAYLOAD_TOO_LARGE before this handler runs.
pub const MAX_CHAT_BODY_BYTES: usize = 5 << 20;
/// Maximum buffered chat-completions body (32MiB). Sized for base64 multimodal inputs;
/// enforced by the `DefaultBodyLimit`, and returns 413 PAYLOAD_TOO_LARGE.
pub const MAX_CHAT_BODY_BYTES: usize = 32 << 20;
/// Minimal probe over the request body — we only need the `stream` field
/// and the `model` field to decide between buffered vs SSE forwarding and