[Fix] rust-server: Drop the fixed 64-item multimodal cap in favor of --limit-mm-data-per-request (#45432)

Co-authored-by: shodoco <shodoco@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-20 20:25:22 -07:00
committed by GitHub
co-authored by shodoco
parent aba892fbdf
commit 6bd1c76735
7 changed files with 119 additions and 86 deletions
+5 -1
View File
@@ -164,7 +164,11 @@ def _build_server_args(
if ref.lora_name is not None
},
enable_strict_thinking=get_serving().enable_strict_thinking,
limit_mm_data_per_request=get_mm().limit_mm_data_per_request or {},
# A list of pairs keeps the configured order, which decides the error
# when several modalities are over their limit.
limit_mm_data_per_request=list(
(get_mm().limit_mm_data_per_request or {}).items()
),
allow_auto_truncate=get_serving().allow_auto_truncate,
enable_return_hidden_states=sa.enable_return_hidden_states,
# Not a `server_args` field: `TokenizerManager` derives it, and the
+4 -3
View File
@@ -108,9 +108,10 @@ Adding one = a `MmFamilyProcessor` impl in `src/<model>/mod.rs` plus a
`common::fetch` matches the Python `get_image_bytes` semantics
(`REQUEST_TIMEOUT` env, `HTTP(S)_PROXY` / `ALL_PROXY` / `NO_PROXY` including
IPv4-CIDR and `host:port` entries) with deliberate safety bounds: every media
source contributes to a shared 64-item / 1.25 GiB request budget, and each
remote I/O stream has an additional 64 MiB cap. `file://` URLs also work (the
Python helper passes the un-stripped URL to `open()`).
source contributes to a shared 1.25 GiB request budget, and each remote I/O
stream has an additional 64 MiB cap. As in Python, there is no fixed item-count
cap. `file://` URLs also work (the Python helper passes the un-stripped URL to
`open()`).
## Python API
+15 -16
View File
@@ -8,9 +8,11 @@
use crate::common::{self, fetch, par, token_layout};
use crate::pipeline::{DecodedMedia, MmFamilyProcessor, PositionOutput, ProcessedItem};
/// Per-request bounds. Every source consumes the aggregate budget;
/// [`fetch::MAX_FETCH_BYTES`] additionally caps each remote I/O stream.
pub const MAX_ITEMS_PER_REQUEST: usize = 64;
/// Per-request media byte bound. Every source consumes this aggregate budget;
/// [`fetch::MAX_FETCH_BYTES`] additionally caps each remote I/O stream. There
/// is deliberately no item-count bound: like the Python frontend, a request
/// may carry any number of items within the byte budget (per-modality counts
/// are the server's `--limit-mm-data-per-request`).
pub const MAX_REQUEST_BYTES: u64 = 1280 << 20;
/// One raw image source from the request.
@@ -82,11 +84,6 @@ pub fn process(family: &dyn MmFamilyProcessor, input: MmInput) -> Result<Output,
if input.images.is_empty() {
return Err("multimodal request without image sources".into());
}
if input.images.len() > MAX_ITEMS_PER_REQUEST {
return Err(format!(
"multimodal request exceeds {MAX_ITEMS_PER_REQUEST} media items"
));
}
// Stage 1 (fetch) is blocking I/O and runs inline, sequentially — never on
// the CPU pool, where a slow URL or file read would starve decode/resize
// for other requests. Contract: callers on a fixed worker pool (the
@@ -220,17 +217,19 @@ mod tests {
assert_eq!(out.input_ids.len(), 5 + 3 * 3); // each placeholder → 4 tokens
}
/// Only bytes are bounded per request: a request with more images than the
/// old 64-item cap processes in full, while the byte budget still rejects.
#[test]
fn per_request_caps_enforced() {
fn per_request_bound_is_bytes_not_items() {
let family = pipeline_from_spec(SPEC).unwrap();
let too_many = MmInput {
input_ids: vec![1],
images: (0..=MAX_ITEMS_PER_REQUEST)
.map(|_| ImageSource::Bytes(vec![]))
.collect(),
let n = 100;
let many = MmInput {
input_ids: vec![1; n],
images: (0..n).map(|_| ImageSource::Bytes(png(8, 8))).collect(),
};
let err = process(family.as_ref(), too_many).err().unwrap();
assert!(err.contains("media items"), "{err}");
let out = process(family.as_ref(), many).unwrap();
assert_eq!(out.items.len(), n);
assert_eq!(out.input_ids.len(), 4 * n);
let err = add_media_bytes(MAX_REQUEST_BYTES / 2, (MAX_REQUEST_BYTES / 2 + 1) as usize)
.unwrap_err();
@@ -3,7 +3,7 @@
//! (model info, model list, server info, flush cache, weights check, tokenize,
//! detokenize, health probe).
use std::collections::{BTreeMap, HashMap};
use std::collections::HashMap;
use std::sync::{
Arc, Mutex,
atomic::{AtomicBool, Ordering},
@@ -39,7 +39,7 @@ pub(crate) struct CoreConfig {
pub(crate) response_activity: ActivityCounter,
pub(crate) startup_ready: bool,
pub(crate) is_disaggregation: bool,
pub(crate) mm_limits: BTreeMap<String, usize>,
pub(crate) mm_limits: Vec<(String, usize)>,
pub(crate) metadata: CoreMetadata,
}
@@ -107,7 +107,7 @@ struct CoreInner {
response_activity: ActivityCounter,
startup_ready: AtomicBool,
is_disaggregation: bool,
mm_limits: BTreeMap<String, usize>,
mm_limits: Vec<(String, usize)>,
metadata: CoreMetadata,
generations: Generations,
}
@@ -1018,7 +1018,7 @@ mod tests {
Default::default(),
false,
false,
BTreeMap::new(),
Vec::new(),
)
}
@@ -1029,7 +1029,7 @@ mod tests {
response_activity: ActivityCounter,
startup_ready: bool,
is_disaggregation: bool,
mm_limits: BTreeMap<String, usize>,
mm_limits: Vec<(String, usize)>,
) -> CoreHandle {
CoreHandle::new(
intake_tx,
@@ -1072,7 +1072,7 @@ mod tests {
response_activity: Default::default(),
startup_ready: false,
is_disaggregation: false,
mm_limits: BTreeMap::new(),
mm_limits: Vec::new(),
metadata: CoreMetadata {
served_model_name: "served-model".into(),
max_context_length: 1_u64 << 32,
@@ -1324,7 +1324,7 @@ mod tests {
Default::default(),
false,
false,
BTreeMap::from([("image".to_owned(), 1)]),
vec![("image".to_owned(), 1)],
);
let request = GenerateRequest {
mm: Some(Box::new(MmData {
@@ -1358,7 +1358,7 @@ mod tests {
activity.clone(),
true,
false,
BTreeMap::new(),
Vec::new(),
);
let probe_handle = handle.clone();
let probe =
@@ -1392,7 +1392,7 @@ mod tests {
Default::default(),
true,
true,
BTreeMap::new(),
Vec::new(),
);
let probe_task =
@@ -1454,7 +1454,7 @@ mod tests {
Default::default(),
false,
true,
BTreeMap::new(),
Vec::new(),
);
let warm_up = tokio::spawn(async move { handle.warm_up(true).await });
@@ -9,12 +9,11 @@
//! [`crate::message::request::MmData::prefetched`], which
//! [`crate::multi_modality::payload::to_mm_input`] swaps back in.
use std::collections::BTreeMap;
use std::sync::{Arc, LazyLock};
use bytes::Bytes;
use sglang_mm::common::fetch::{ByteBudget, fetch_bytes_budgeted, fetch_local_file_budgeted};
use sglang_mm::driver::{MAX_ITEMS_PER_REQUEST, MAX_REQUEST_BYTES};
use sglang_mm::driver::MAX_REQUEST_BYTES;
use tokio::sync::Semaphore;
use crate::message::request::{GenerateRequest, MmData};
@@ -27,15 +26,20 @@ static PERMITS: LazyLock<Arc<Semaphore>> = LazyLock::new(|| Arc::new(Semaphore::
/// Fill [`MmData::prefetched`] for every request, all fetches across the batch
/// concurrent. Any failure rejects the call (a 400, as on the Python path).
///
/// The driver's budgets ([`MAX_ITEMS_PER_REQUEST`], [`MAX_REQUEST_BYTES`]) are
/// enforced *here* rather than in `sglang_mm::driver::process`, where 64 sources
/// of 64 MiB would already be resident. The driver keeps its own checks as the
/// backstop for callers without a prefetch layer.
/// `modality_limits` is `--limit-mm-data-per-request`, checked before a single
/// byte is fetched, in its configured order and with Python's message
/// (`TokenizerManager._validate_mm_limits`). Unset means no per-modality limit,
/// as in Python. Python has no fixed item cap, and neither does this.
///
/// [`MAX_REQUEST_BYTES`] is enforced while fetching, here rather than in
/// `sglang_mm::driver::process`, where every source would already be resident.
/// It bounds buffered media bytes regardless of the item count. The driver
/// keeps its own byte check as the backstop for callers without a prefetch
/// layer.
pub async fn prefetch_all(
requests: &mut [GenerateRequest],
modality_limits: &BTreeMap<String, usize>,
modality_limits: &[(String, usize)],
) -> Result<(), String> {
// The item budget rejects before a single byte is fetched.
let plan = |mm: &Option<Box<MmData>>| -> Result<Vec<String>, String> {
let Some(mm) = mm.as_deref() else {
return Ok(Vec::new());
@@ -45,21 +49,13 @@ pub async fn prefetch_all(
("video", &mm.video_data),
("audio", &mm.audio_data),
];
let items = modalities
.iter()
.map(|(_, items)| items.len())
.sum::<usize>();
if items > MAX_ITEMS_PER_REQUEST {
return Err(format!(
"multimodal request exceeds {MAX_ITEMS_PER_REQUEST} media items"
));
}
for (modality, items) in modalities {
for (modality, limit) in modality_limits {
let Some((name, items)) = modalities.iter().find(|(name, _)| name == modality) else {
continue;
};
let count = items.len();
if let Some(limit) = modality_limits.get(modality)
&& count > *limit
{
let display = modality[..1].to_uppercase() + &modality[1..];
if count > *limit {
let display = name[..1].to_uppercase() + &name[1..];
return Err(format!(
"{display} count {count} exceeds limit {limit} per request."
));
@@ -198,7 +194,7 @@ mod tests {
})),
..Default::default()
}];
prefetch_all(&mut requests, &BTreeMap::new()).await.unwrap();
prefetch_all(&mut requests, &[]).await.unwrap();
std::fs::remove_dir_all(base).ok();
let fetched = &requests[0].mm.as_ref().unwrap().prefetched;
assert_eq!(
@@ -223,7 +219,7 @@ mod tests {
]),
GenerateRequest::default(),
];
prefetch_all(&mut requests, &BTreeMap::new()).await.unwrap();
prefetch_all(&mut requests, &[]).await.unwrap();
std::fs::remove_file(&path).ok();
let fetched = &requests[0].mm.as_ref().unwrap().prefetched;
// The one-shot server answers in accept order, so contents may swap
@@ -237,49 +233,72 @@ mod tests {
#[tokio::test]
async fn failed_download_rejects() {
let mut requests = vec![mm_request(vec![src("http://127.0.0.1:1/nope.png")])];
let err = prefetch_all(&mut requests, &BTreeMap::new())
.await
.err()
.unwrap();
let err = prefetch_all(&mut requests, &[]).await.err().unwrap();
assert!(err.contains("media fetch"), "{err}");
}
/// The item budget rejects before any source is touched: all of these would
/// fail to fetch, so a fetch error would prove fetching started.
/// Without `--limit-mm-data-per-request`, a request with far more items
/// than the old fixed cap (64) is fetched in full, as Python serves it.
#[tokio::test]
async fn item_budget_rejects_before_fetching() {
let sources: Vec<MmItem> = (0..=MAX_ITEMS_PER_REQUEST)
.map(|i| src(format!("/definitely/not/here-{i}.png")))
.collect();
let mut requests = vec![mm_request(sources)];
let err = prefetch_all(&mut requests, &BTreeMap::new())
.await
.err()
.unwrap();
assert_eq!(
err,
format!("multimodal request exceeds {MAX_ITEMS_PER_REQUEST} media items")
);
assert!(requests[0].mm.as_ref().unwrap().prefetched.is_empty());
async fn items_beyond_the_old_cap_are_fetched_without_a_limit() {
let n = 200;
let path =
std::env::temp_dir().join(format!("sglang-prefetch-many-{}", std::process::id()));
std::fs::write(&path, b"img").unwrap();
let mut requests = vec![mm_request(
(0..n).map(|_| src(path.display().to_string())).collect(),
)];
let fetched = prefetch_all(&mut requests, &[]).await;
std::fs::remove_file(&path).ok();
fetched.unwrap();
assert_eq!(requests[0].mm.as_ref().unwrap().prefetched.len(), n);
}
/// `--limit-mm-data-per-request` matches Python `_validate_mm_limits`: a
/// count at the limit passes, the first over-limit modality in configured
/// order is reported with Python's message, and nothing is fetched.
#[tokio::test]
async fn per_modality_budget_rejects_before_fetching() {
let mut requests = vec![GenerateRequest {
async fn per_modality_limits_match_python_before_fetching() {
let request = || GenerateRequest {
mm: Some(Box::new(MmData {
image_data: vec![
src("/definitely/not/here-0.png"),
src("/definitely/not/here-1.png"),
],
video_data: vec![src("/definitely/not/here.mp4")],
video_data: vec![
src("/definitely/not/here-0.mp4"),
src("/definitely/not/here-1.mp4"),
],
..Default::default()
})),
..Default::default()
}];
let limits = BTreeMap::from([("image".to_owned(), 1), ("video".to_owned(), 1)]);
let err = prefetch_all(&mut requests, &limits).await.err().unwrap();
assert_eq!(err, "Image count 2 exceeds limit 1 per request.");
assert!(requests[0].mm.as_ref().unwrap().prefetched.is_empty());
};
let owned = |pairs: &[(&str, usize)]| -> Vec<(String, usize)> {
pairs.iter().map(|(m, n)| ((*m).to_owned(), *n)).collect()
};
for (limits, expected) in [
(
owned(&[("video", 1), ("image", 1)]),
"Video count 2 exceeds limit 1 per request.",
),
(
owned(&[("audio", 0), ("image", 1), ("video", 1)]),
"Image count 2 exceeds limit 1 per request.",
),
] {
let mut requests = vec![request()];
let err = prefetch_all(&mut requests, &limits).await.err().unwrap();
assert_eq!(err, expected);
assert!(requests[0].mm.as_ref().unwrap().prefetched.is_empty());
}
// At the limit the request passes the count check and reaches fetching.
let mut requests = vec![request()];
let err = prefetch_all(&mut requests, &owned(&[("image", 2)]))
.await
.err()
.unwrap();
assert!(err.contains("media fetch"), "{err}");
}
/// Sources legal alone but collectively over the limit are rejected while
+6 -4
View File
@@ -159,8 +159,10 @@ pub struct ServerArgs {
/// Native-generation launch policy normally applied by Python ingress.
pub lora_ids: BTreeMap<String, String>,
pub enable_strict_thinking: bool,
/// Per-modality media-count limits from `--limit-mm-data-per-request`.
pub limit_mm_data_per_request: BTreeMap<String, usize>,
/// Per-modality media-count limits from `--limit-mm-data-per-request`, in
/// configured order: Python checks them in that order and reports the first
/// one exceeded. Empty means no per-modality limit.
pub limit_mm_data_per_request: Vec<(String, usize)>,
/// Over-long inputs are truncated to fit the context instead of 400ing, and
/// `max_new_tokens` is clamped rather than rejected (Python
/// `TokenizerManager._validate_one_request`).
@@ -278,7 +280,7 @@ impl ServerArgs {
preferred_sampling_params: Option<PreferredSamplingParams>,
lora_ids: BTreeMap<String, String>,
enable_strict_thinking: bool,
limit_mm_data_per_request: BTreeMap<String, usize>,
limit_mm_data_per_request: Vec<(String, usize)>,
allow_auto_truncate: bool,
enable_return_hidden_states: bool,
num_reserved_tokens: u64,
@@ -376,7 +378,7 @@ impl Default for ServerArgs {
preferred_sampling_params: None,
lora_ids: BTreeMap::new(),
enable_strict_thinking: false,
limit_mm_data_per_request: BTreeMap::new(),
limit_mm_data_per_request: Vec::new(),
allow_auto_truncate: false,
enable_return_hidden_states: false,
num_reserved_tokens: 0,
@@ -52,6 +52,12 @@ def _scheduler_for_typed_config():
"default_priority_value",
5,
),
({}, "limit_mm_data_per_request", []),
(
{"limit_mm_data_per_request": {"video": 1, "image": 2}},
"limit_mm_data_per_request",
[("video", 1), ("image", 2)],
),
],
ids=[
"native-grpc",
@@ -59,6 +65,8 @@ def _scheduler_for_typed_config():
"deprecated-legacy-smg",
"priority-off",
"priority-on",
"mm-limits-unset",
"mm-limits-keep-order",
],
)
def test_typed_config_forwards_resolved_launch_fields(overrides, kwarg, expected):