From 6bd1c76735ca1e28b1347b54293c90d138cf03c3 Mon Sep 17 00:00:00 2001 From: metamergebot Date: Tue, 20 Oct 2026 20:25:22 -0700 Subject: [PATCH] [Fix] rust-server: Drop the fixed 64-item multimodal cap in favor of --limit-mm-data-per-request (#45432) Co-authored-by: shodoco --- python/sglang/srt/rust_server/config.py | 6 +- rust/sglang-mm/README.md | 7 +- rust/sglang-mm/src/driver.rs | 31 +++-- .../src/api_server/core/handle.rs | 20 +-- .../src/api_server/core/prefetch.rs | 123 ++++++++++-------- rust/sglang-server/src/message/config.rs | 10 +- .../entrypoints/test_rust_server_dp_ports.py | 8 ++ 7 files changed, 119 insertions(+), 86 deletions(-) diff --git a/python/sglang/srt/rust_server/config.py b/python/sglang/srt/rust_server/config.py index 6b9a97f1cd..c90788aeb8 100644 --- a/python/sglang/srt/rust_server/config.py +++ b/python/sglang/srt/rust_server/config.py @@ -164,7 +164,11 @@ def _build_server_args( if ref.lora_name is not None }, enable_strict_thinking=get_serving().enable_strict_thinking, - limit_mm_data_per_request=get_mm().limit_mm_data_per_request or {}, + # A list of pairs keeps the configured order, which decides the error + # when several modalities are over their limit. + limit_mm_data_per_request=list( + (get_mm().limit_mm_data_per_request or {}).items() + ), allow_auto_truncate=get_serving().allow_auto_truncate, enable_return_hidden_states=sa.enable_return_hidden_states, # Not a `server_args` field: `TokenizerManager` derives it, and the diff --git a/rust/sglang-mm/README.md b/rust/sglang-mm/README.md index 6cd8846274..b08d19e346 100644 --- a/rust/sglang-mm/README.md +++ b/rust/sglang-mm/README.md @@ -108,9 +108,10 @@ Adding one = a `MmFamilyProcessor` impl in `src//mod.rs` plus a `common::fetch` matches the Python `get_image_bytes` semantics (`REQUEST_TIMEOUT` env, `HTTP(S)_PROXY` / `ALL_PROXY` / `NO_PROXY` including IPv4-CIDR and `host:port` entries) with deliberate safety bounds: every media -source contributes to a shared 64-item / 1.25 GiB request budget, and each -remote I/O stream has an additional 64 MiB cap. `file://` URLs also work (the -Python helper passes the un-stripped URL to `open()`). +source contributes to a shared 1.25 GiB request budget, and each remote I/O +stream has an additional 64 MiB cap. As in Python, there is no fixed item-count +cap. `file://` URLs also work (the Python helper passes the un-stripped URL to +`open()`). ## Python API diff --git a/rust/sglang-mm/src/driver.rs b/rust/sglang-mm/src/driver.rs index 224bbc231a..e755491248 100644 --- a/rust/sglang-mm/src/driver.rs +++ b/rust/sglang-mm/src/driver.rs @@ -8,9 +8,11 @@ use crate::common::{self, fetch, par, token_layout}; use crate::pipeline::{DecodedMedia, MmFamilyProcessor, PositionOutput, ProcessedItem}; -/// Per-request bounds. Every source consumes the aggregate budget; -/// [`fetch::MAX_FETCH_BYTES`] additionally caps each remote I/O stream. -pub const MAX_ITEMS_PER_REQUEST: usize = 64; +/// Per-request media byte bound. Every source consumes this aggregate budget; +/// [`fetch::MAX_FETCH_BYTES`] additionally caps each remote I/O stream. There +/// is deliberately no item-count bound: like the Python frontend, a request +/// may carry any number of items within the byte budget (per-modality counts +/// are the server's `--limit-mm-data-per-request`). pub const MAX_REQUEST_BYTES: u64 = 1280 << 20; /// One raw image source from the request. @@ -82,11 +84,6 @@ pub fn process(family: &dyn MmFamilyProcessor, input: MmInput) -> Result MAX_ITEMS_PER_REQUEST { - return Err(format!( - "multimodal request exceeds {MAX_ITEMS_PER_REQUEST} media items" - )); - } // Stage 1 (fetch) is blocking I/O and runs inline, sequentially — never on // the CPU pool, where a slow URL or file read would starve decode/resize // for other requests. Contract: callers on a fixed worker pool (the @@ -220,17 +217,19 @@ mod tests { assert_eq!(out.input_ids.len(), 5 + 3 * 3); // each placeholder → 4 tokens } + /// Only bytes are bounded per request: a request with more images than the + /// old 64-item cap processes in full, while the byte budget still rejects. #[test] - fn per_request_caps_enforced() { + fn per_request_bound_is_bytes_not_items() { let family = pipeline_from_spec(SPEC).unwrap(); - let too_many = MmInput { - input_ids: vec![1], - images: (0..=MAX_ITEMS_PER_REQUEST) - .map(|_| ImageSource::Bytes(vec![])) - .collect(), + let n = 100; + let many = MmInput { + input_ids: vec![1; n], + images: (0..n).map(|_| ImageSource::Bytes(png(8, 8))).collect(), }; - let err = process(family.as_ref(), too_many).err().unwrap(); - assert!(err.contains("media items"), "{err}"); + let out = process(family.as_ref(), many).unwrap(); + assert_eq!(out.items.len(), n); + assert_eq!(out.input_ids.len(), 4 * n); let err = add_media_bytes(MAX_REQUEST_BYTES / 2, (MAX_REQUEST_BYTES / 2 + 1) as usize) .unwrap_err(); diff --git a/rust/sglang-server/src/api_server/core/handle.rs b/rust/sglang-server/src/api_server/core/handle.rs index 21e049cf08..b8cbad55ee 100644 --- a/rust/sglang-server/src/api_server/core/handle.rs +++ b/rust/sglang-server/src/api_server/core/handle.rs @@ -3,7 +3,7 @@ //! (model info, model list, server info, flush cache, weights check, tokenize, //! detokenize, health probe). -use std::collections::{BTreeMap, HashMap}; +use std::collections::HashMap; use std::sync::{ Arc, Mutex, atomic::{AtomicBool, Ordering}, @@ -39,7 +39,7 @@ pub(crate) struct CoreConfig { pub(crate) response_activity: ActivityCounter, pub(crate) startup_ready: bool, pub(crate) is_disaggregation: bool, - pub(crate) mm_limits: BTreeMap, + pub(crate) mm_limits: Vec<(String, usize)>, pub(crate) metadata: CoreMetadata, } @@ -107,7 +107,7 @@ struct CoreInner { response_activity: ActivityCounter, startup_ready: AtomicBool, is_disaggregation: bool, - mm_limits: BTreeMap, + mm_limits: Vec<(String, usize)>, metadata: CoreMetadata, generations: Generations, } @@ -1018,7 +1018,7 @@ mod tests { Default::default(), false, false, - BTreeMap::new(), + Vec::new(), ) } @@ -1029,7 +1029,7 @@ mod tests { response_activity: ActivityCounter, startup_ready: bool, is_disaggregation: bool, - mm_limits: BTreeMap, + mm_limits: Vec<(String, usize)>, ) -> CoreHandle { CoreHandle::new( intake_tx, @@ -1072,7 +1072,7 @@ mod tests { response_activity: Default::default(), startup_ready: false, is_disaggregation: false, - mm_limits: BTreeMap::new(), + mm_limits: Vec::new(), metadata: CoreMetadata { served_model_name: "served-model".into(), max_context_length: 1_u64 << 32, @@ -1324,7 +1324,7 @@ mod tests { Default::default(), false, false, - BTreeMap::from([("image".to_owned(), 1)]), + vec![("image".to_owned(), 1)], ); let request = GenerateRequest { mm: Some(Box::new(MmData { @@ -1358,7 +1358,7 @@ mod tests { activity.clone(), true, false, - BTreeMap::new(), + Vec::new(), ); let probe_handle = handle.clone(); let probe = @@ -1392,7 +1392,7 @@ mod tests { Default::default(), true, true, - BTreeMap::new(), + Vec::new(), ); let probe_task = @@ -1454,7 +1454,7 @@ mod tests { Default::default(), false, true, - BTreeMap::new(), + Vec::new(), ); let warm_up = tokio::spawn(async move { handle.warm_up(true).await }); diff --git a/rust/sglang-server/src/api_server/core/prefetch.rs b/rust/sglang-server/src/api_server/core/prefetch.rs index c67472d9c7..c3b34e1287 100644 --- a/rust/sglang-server/src/api_server/core/prefetch.rs +++ b/rust/sglang-server/src/api_server/core/prefetch.rs @@ -9,12 +9,11 @@ //! [`crate::message::request::MmData::prefetched`], which //! [`crate::multi_modality::payload::to_mm_input`] swaps back in. -use std::collections::BTreeMap; use std::sync::{Arc, LazyLock}; use bytes::Bytes; use sglang_mm::common::fetch::{ByteBudget, fetch_bytes_budgeted, fetch_local_file_budgeted}; -use sglang_mm::driver::{MAX_ITEMS_PER_REQUEST, MAX_REQUEST_BYTES}; +use sglang_mm::driver::MAX_REQUEST_BYTES; use tokio::sync::Semaphore; use crate::message::request::{GenerateRequest, MmData}; @@ -27,15 +26,20 @@ static PERMITS: LazyLock> = LazyLock::new(|| Arc::new(Semaphore:: /// Fill [`MmData::prefetched`] for every request, all fetches across the batch /// concurrent. Any failure rejects the call (a 400, as on the Python path). /// -/// The driver's budgets ([`MAX_ITEMS_PER_REQUEST`], [`MAX_REQUEST_BYTES`]) are -/// enforced *here* rather than in `sglang_mm::driver::process`, where 64 sources -/// of 64 MiB would already be resident. The driver keeps its own checks as the -/// backstop for callers without a prefetch layer. +/// `modality_limits` is `--limit-mm-data-per-request`, checked before a single +/// byte is fetched, in its configured order and with Python's message +/// (`TokenizerManager._validate_mm_limits`). Unset means no per-modality limit, +/// as in Python. Python has no fixed item cap, and neither does this. +/// +/// [`MAX_REQUEST_BYTES`] is enforced while fetching, here rather than in +/// `sglang_mm::driver::process`, where every source would already be resident. +/// It bounds buffered media bytes regardless of the item count. The driver +/// keeps its own byte check as the backstop for callers without a prefetch +/// layer. pub async fn prefetch_all( requests: &mut [GenerateRequest], - modality_limits: &BTreeMap, + modality_limits: &[(String, usize)], ) -> Result<(), String> { - // The item budget rejects before a single byte is fetched. let plan = |mm: &Option>| -> Result, String> { let Some(mm) = mm.as_deref() else { return Ok(Vec::new()); @@ -45,21 +49,13 @@ pub async fn prefetch_all( ("video", &mm.video_data), ("audio", &mm.audio_data), ]; - let items = modalities - .iter() - .map(|(_, items)| items.len()) - .sum::(); - if items > MAX_ITEMS_PER_REQUEST { - return Err(format!( - "multimodal request exceeds {MAX_ITEMS_PER_REQUEST} media items" - )); - } - for (modality, items) in modalities { + for (modality, limit) in modality_limits { + let Some((name, items)) = modalities.iter().find(|(name, _)| name == modality) else { + continue; + }; let count = items.len(); - if let Some(limit) = modality_limits.get(modality) - && count > *limit - { - let display = modality[..1].to_uppercase() + &modality[1..]; + if count > *limit { + let display = name[..1].to_uppercase() + &name[1..]; return Err(format!( "{display} count {count} exceeds limit {limit} per request." )); @@ -198,7 +194,7 @@ mod tests { })), ..Default::default() }]; - prefetch_all(&mut requests, &BTreeMap::new()).await.unwrap(); + prefetch_all(&mut requests, &[]).await.unwrap(); std::fs::remove_dir_all(base).ok(); let fetched = &requests[0].mm.as_ref().unwrap().prefetched; assert_eq!( @@ -223,7 +219,7 @@ mod tests { ]), GenerateRequest::default(), ]; - prefetch_all(&mut requests, &BTreeMap::new()).await.unwrap(); + prefetch_all(&mut requests, &[]).await.unwrap(); std::fs::remove_file(&path).ok(); let fetched = &requests[0].mm.as_ref().unwrap().prefetched; // The one-shot server answers in accept order, so contents may swap @@ -237,49 +233,72 @@ mod tests { #[tokio::test] async fn failed_download_rejects() { let mut requests = vec![mm_request(vec![src("http://127.0.0.1:1/nope.png")])]; - let err = prefetch_all(&mut requests, &BTreeMap::new()) - .await - .err() - .unwrap(); + let err = prefetch_all(&mut requests, &[]).await.err().unwrap(); assert!(err.contains("media fetch"), "{err}"); } - /// The item budget rejects before any source is touched: all of these would - /// fail to fetch, so a fetch error would prove fetching started. + /// Without `--limit-mm-data-per-request`, a request with far more items + /// than the old fixed cap (64) is fetched in full, as Python serves it. #[tokio::test] - async fn item_budget_rejects_before_fetching() { - let sources: Vec = (0..=MAX_ITEMS_PER_REQUEST) - .map(|i| src(format!("/definitely/not/here-{i}.png"))) - .collect(); - let mut requests = vec![mm_request(sources)]; - let err = prefetch_all(&mut requests, &BTreeMap::new()) - .await - .err() - .unwrap(); - assert_eq!( - err, - format!("multimodal request exceeds {MAX_ITEMS_PER_REQUEST} media items") - ); - assert!(requests[0].mm.as_ref().unwrap().prefetched.is_empty()); + async fn items_beyond_the_old_cap_are_fetched_without_a_limit() { + let n = 200; + let path = + std::env::temp_dir().join(format!("sglang-prefetch-many-{}", std::process::id())); + std::fs::write(&path, b"img").unwrap(); + let mut requests = vec![mm_request( + (0..n).map(|_| src(path.display().to_string())).collect(), + )]; + let fetched = prefetch_all(&mut requests, &[]).await; + std::fs::remove_file(&path).ok(); + fetched.unwrap(); + assert_eq!(requests[0].mm.as_ref().unwrap().prefetched.len(), n); } + /// `--limit-mm-data-per-request` matches Python `_validate_mm_limits`: a + /// count at the limit passes, the first over-limit modality in configured + /// order is reported with Python's message, and nothing is fetched. #[tokio::test] - async fn per_modality_budget_rejects_before_fetching() { - let mut requests = vec![GenerateRequest { + async fn per_modality_limits_match_python_before_fetching() { + let request = || GenerateRequest { mm: Some(Box::new(MmData { image_data: vec![ src("/definitely/not/here-0.png"), src("/definitely/not/here-1.png"), ], - video_data: vec![src("/definitely/not/here.mp4")], + video_data: vec![ + src("/definitely/not/here-0.mp4"), + src("/definitely/not/here-1.mp4"), + ], ..Default::default() })), ..Default::default() - }]; - let limits = BTreeMap::from([("image".to_owned(), 1), ("video".to_owned(), 1)]); - let err = prefetch_all(&mut requests, &limits).await.err().unwrap(); - assert_eq!(err, "Image count 2 exceeds limit 1 per request."); - assert!(requests[0].mm.as_ref().unwrap().prefetched.is_empty()); + }; + let owned = |pairs: &[(&str, usize)]| -> Vec<(String, usize)> { + pairs.iter().map(|(m, n)| ((*m).to_owned(), *n)).collect() + }; + for (limits, expected) in [ + ( + owned(&[("video", 1), ("image", 1)]), + "Video count 2 exceeds limit 1 per request.", + ), + ( + owned(&[("audio", 0), ("image", 1), ("video", 1)]), + "Image count 2 exceeds limit 1 per request.", + ), + ] { + let mut requests = vec![request()]; + let err = prefetch_all(&mut requests, &limits).await.err().unwrap(); + assert_eq!(err, expected); + assert!(requests[0].mm.as_ref().unwrap().prefetched.is_empty()); + } + + // At the limit the request passes the count check and reaches fetching. + let mut requests = vec![request()]; + let err = prefetch_all(&mut requests, &owned(&[("image", 2)])) + .await + .err() + .unwrap(); + assert!(err.contains("media fetch"), "{err}"); } /// Sources legal alone but collectively over the limit are rejected while diff --git a/rust/sglang-server/src/message/config.rs b/rust/sglang-server/src/message/config.rs index 3e5e336dae..cba144e18f 100644 --- a/rust/sglang-server/src/message/config.rs +++ b/rust/sglang-server/src/message/config.rs @@ -159,8 +159,10 @@ pub struct ServerArgs { /// Native-generation launch policy normally applied by Python ingress. pub lora_ids: BTreeMap, pub enable_strict_thinking: bool, - /// Per-modality media-count limits from `--limit-mm-data-per-request`. - pub limit_mm_data_per_request: BTreeMap, + /// Per-modality media-count limits from `--limit-mm-data-per-request`, in + /// configured order: Python checks them in that order and reports the first + /// one exceeded. Empty means no per-modality limit. + pub limit_mm_data_per_request: Vec<(String, usize)>, /// Over-long inputs are truncated to fit the context instead of 400ing, and /// `max_new_tokens` is clamped rather than rejected (Python /// `TokenizerManager._validate_one_request`). @@ -278,7 +280,7 @@ impl ServerArgs { preferred_sampling_params: Option, lora_ids: BTreeMap, enable_strict_thinking: bool, - limit_mm_data_per_request: BTreeMap, + limit_mm_data_per_request: Vec<(String, usize)>, allow_auto_truncate: bool, enable_return_hidden_states: bool, num_reserved_tokens: u64, @@ -376,7 +378,7 @@ impl Default for ServerArgs { preferred_sampling_params: None, lora_ids: BTreeMap::new(), enable_strict_thinking: false, - limit_mm_data_per_request: BTreeMap::new(), + limit_mm_data_per_request: Vec::new(), allow_auto_truncate: false, enable_return_hidden_states: false, num_reserved_tokens: 0, diff --git a/test/registered/unit/entrypoints/test_rust_server_dp_ports.py b/test/registered/unit/entrypoints/test_rust_server_dp_ports.py index 1fd2630c24..6e400c7e3c 100644 --- a/test/registered/unit/entrypoints/test_rust_server_dp_ports.py +++ b/test/registered/unit/entrypoints/test_rust_server_dp_ports.py @@ -52,6 +52,12 @@ def _scheduler_for_typed_config(): "default_priority_value", 5, ), + ({}, "limit_mm_data_per_request", []), + ( + {"limit_mm_data_per_request": {"video": 1, "image": 2}}, + "limit_mm_data_per_request", + [("video", 1), ("image", 2)], + ), ], ids=[ "native-grpc", @@ -59,6 +65,8 @@ def _scheduler_for_typed_config(): "deprecated-legacy-smg", "priority-off", "priority-on", + "mm-limits-unset", + "mm-limits-keep-order", ], ) def test_typed_config_forwards_resolved_launch_fields(overrides, kwarg, expected):