[Fix] rust-server: Drop the fixed 64-item multimodal cap in favor of --limit-mm-data-per-request (#45432)
Co-authored-by: shodoco <shodoco@users.noreply.github.com>
This commit is contained in:
@@ -164,7 +164,11 @@ def _build_server_args(
|
||||
if ref.lora_name is not None
|
||||
},
|
||||
enable_strict_thinking=get_serving().enable_strict_thinking,
|
||||
limit_mm_data_per_request=get_mm().limit_mm_data_per_request or {},
|
||||
# A list of pairs keeps the configured order, which decides the error
|
||||
# when several modalities are over their limit.
|
||||
limit_mm_data_per_request=list(
|
||||
(get_mm().limit_mm_data_per_request or {}).items()
|
||||
),
|
||||
allow_auto_truncate=get_serving().allow_auto_truncate,
|
||||
enable_return_hidden_states=sa.enable_return_hidden_states,
|
||||
# Not a `server_args` field: `TokenizerManager` derives it, and the
|
||||
|
||||
@@ -108,9 +108,10 @@ Adding one = a `MmFamilyProcessor` impl in `src/<model>/mod.rs` plus a
|
||||
`common::fetch` matches the Python `get_image_bytes` semantics
|
||||
(`REQUEST_TIMEOUT` env, `HTTP(S)_PROXY` / `ALL_PROXY` / `NO_PROXY` including
|
||||
IPv4-CIDR and `host:port` entries) with deliberate safety bounds: every media
|
||||
source contributes to a shared 64-item / 1.25 GiB request budget, and each
|
||||
remote I/O stream has an additional 64 MiB cap. `file://` URLs also work (the
|
||||
Python helper passes the un-stripped URL to `open()`).
|
||||
source contributes to a shared 1.25 GiB request budget, and each remote I/O
|
||||
stream has an additional 64 MiB cap. As in Python, there is no fixed item-count
|
||||
cap. `file://` URLs also work (the Python helper passes the un-stripped URL to
|
||||
`open()`).
|
||||
|
||||
## Python API
|
||||
|
||||
|
||||
@@ -8,9 +8,11 @@
|
||||
use crate::common::{self, fetch, par, token_layout};
|
||||
use crate::pipeline::{DecodedMedia, MmFamilyProcessor, PositionOutput, ProcessedItem};
|
||||
|
||||
/// Per-request bounds. Every source consumes the aggregate budget;
|
||||
/// [`fetch::MAX_FETCH_BYTES`] additionally caps each remote I/O stream.
|
||||
pub const MAX_ITEMS_PER_REQUEST: usize = 64;
|
||||
/// Per-request media byte bound. Every source consumes this aggregate budget;
|
||||
/// [`fetch::MAX_FETCH_BYTES`] additionally caps each remote I/O stream. There
|
||||
/// is deliberately no item-count bound: like the Python frontend, a request
|
||||
/// may carry any number of items within the byte budget (per-modality counts
|
||||
/// are the server's `--limit-mm-data-per-request`).
|
||||
pub const MAX_REQUEST_BYTES: u64 = 1280 << 20;
|
||||
|
||||
/// One raw image source from the request.
|
||||
@@ -82,11 +84,6 @@ pub fn process(family: &dyn MmFamilyProcessor, input: MmInput) -> Result<Output,
|
||||
if input.images.is_empty() {
|
||||
return Err("multimodal request without image sources".into());
|
||||
}
|
||||
if input.images.len() > MAX_ITEMS_PER_REQUEST {
|
||||
return Err(format!(
|
||||
"multimodal request exceeds {MAX_ITEMS_PER_REQUEST} media items"
|
||||
));
|
||||
}
|
||||
// Stage 1 (fetch) is blocking I/O and runs inline, sequentially — never on
|
||||
// the CPU pool, where a slow URL or file read would starve decode/resize
|
||||
// for other requests. Contract: callers on a fixed worker pool (the
|
||||
@@ -220,17 +217,19 @@ mod tests {
|
||||
assert_eq!(out.input_ids.len(), 5 + 3 * 3); // each placeholder → 4 tokens
|
||||
}
|
||||
|
||||
/// Only bytes are bounded per request: a request with more images than the
|
||||
/// old 64-item cap processes in full, while the byte budget still rejects.
|
||||
#[test]
|
||||
fn per_request_caps_enforced() {
|
||||
fn per_request_bound_is_bytes_not_items() {
|
||||
let family = pipeline_from_spec(SPEC).unwrap();
|
||||
let too_many = MmInput {
|
||||
input_ids: vec![1],
|
||||
images: (0..=MAX_ITEMS_PER_REQUEST)
|
||||
.map(|_| ImageSource::Bytes(vec![]))
|
||||
.collect(),
|
||||
let n = 100;
|
||||
let many = MmInput {
|
||||
input_ids: vec![1; n],
|
||||
images: (0..n).map(|_| ImageSource::Bytes(png(8, 8))).collect(),
|
||||
};
|
||||
let err = process(family.as_ref(), too_many).err().unwrap();
|
||||
assert!(err.contains("media items"), "{err}");
|
||||
let out = process(family.as_ref(), many).unwrap();
|
||||
assert_eq!(out.items.len(), n);
|
||||
assert_eq!(out.input_ids.len(), 4 * n);
|
||||
|
||||
let err = add_media_bytes(MAX_REQUEST_BYTES / 2, (MAX_REQUEST_BYTES / 2 + 1) as usize)
|
||||
.unwrap_err();
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
//! (model info, model list, server info, flush cache, weights check, tokenize,
|
||||
//! detokenize, health probe).
|
||||
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
@@ -39,7 +39,7 @@ pub(crate) struct CoreConfig {
|
||||
pub(crate) response_activity: ActivityCounter,
|
||||
pub(crate) startup_ready: bool,
|
||||
pub(crate) is_disaggregation: bool,
|
||||
pub(crate) mm_limits: BTreeMap<String, usize>,
|
||||
pub(crate) mm_limits: Vec<(String, usize)>,
|
||||
pub(crate) metadata: CoreMetadata,
|
||||
}
|
||||
|
||||
@@ -107,7 +107,7 @@ struct CoreInner {
|
||||
response_activity: ActivityCounter,
|
||||
startup_ready: AtomicBool,
|
||||
is_disaggregation: bool,
|
||||
mm_limits: BTreeMap<String, usize>,
|
||||
mm_limits: Vec<(String, usize)>,
|
||||
metadata: CoreMetadata,
|
||||
generations: Generations,
|
||||
}
|
||||
@@ -1018,7 +1018,7 @@ mod tests {
|
||||
Default::default(),
|
||||
false,
|
||||
false,
|
||||
BTreeMap::new(),
|
||||
Vec::new(),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1029,7 +1029,7 @@ mod tests {
|
||||
response_activity: ActivityCounter,
|
||||
startup_ready: bool,
|
||||
is_disaggregation: bool,
|
||||
mm_limits: BTreeMap<String, usize>,
|
||||
mm_limits: Vec<(String, usize)>,
|
||||
) -> CoreHandle {
|
||||
CoreHandle::new(
|
||||
intake_tx,
|
||||
@@ -1072,7 +1072,7 @@ mod tests {
|
||||
response_activity: Default::default(),
|
||||
startup_ready: false,
|
||||
is_disaggregation: false,
|
||||
mm_limits: BTreeMap::new(),
|
||||
mm_limits: Vec::new(),
|
||||
metadata: CoreMetadata {
|
||||
served_model_name: "served-model".into(),
|
||||
max_context_length: 1_u64 << 32,
|
||||
@@ -1324,7 +1324,7 @@ mod tests {
|
||||
Default::default(),
|
||||
false,
|
||||
false,
|
||||
BTreeMap::from([("image".to_owned(), 1)]),
|
||||
vec![("image".to_owned(), 1)],
|
||||
);
|
||||
let request = GenerateRequest {
|
||||
mm: Some(Box::new(MmData {
|
||||
@@ -1358,7 +1358,7 @@ mod tests {
|
||||
activity.clone(),
|
||||
true,
|
||||
false,
|
||||
BTreeMap::new(),
|
||||
Vec::new(),
|
||||
);
|
||||
let probe_handle = handle.clone();
|
||||
let probe =
|
||||
@@ -1392,7 +1392,7 @@ mod tests {
|
||||
Default::default(),
|
||||
true,
|
||||
true,
|
||||
BTreeMap::new(),
|
||||
Vec::new(),
|
||||
);
|
||||
|
||||
let probe_task =
|
||||
@@ -1454,7 +1454,7 @@ mod tests {
|
||||
Default::default(),
|
||||
false,
|
||||
true,
|
||||
BTreeMap::new(),
|
||||
Vec::new(),
|
||||
);
|
||||
let warm_up = tokio::spawn(async move { handle.warm_up(true).await });
|
||||
|
||||
|
||||
@@ -9,12 +9,11 @@
|
||||
//! [`crate::message::request::MmData::prefetched`], which
|
||||
//! [`crate::multi_modality::payload::to_mm_input`] swaps back in.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::{Arc, LazyLock};
|
||||
|
||||
use bytes::Bytes;
|
||||
use sglang_mm::common::fetch::{ByteBudget, fetch_bytes_budgeted, fetch_local_file_budgeted};
|
||||
use sglang_mm::driver::{MAX_ITEMS_PER_REQUEST, MAX_REQUEST_BYTES};
|
||||
use sglang_mm::driver::MAX_REQUEST_BYTES;
|
||||
use tokio::sync::Semaphore;
|
||||
|
||||
use crate::message::request::{GenerateRequest, MmData};
|
||||
@@ -27,15 +26,20 @@ static PERMITS: LazyLock<Arc<Semaphore>> = LazyLock::new(|| Arc::new(Semaphore::
|
||||
/// Fill [`MmData::prefetched`] for every request, all fetches across the batch
|
||||
/// concurrent. Any failure rejects the call (a 400, as on the Python path).
|
||||
///
|
||||
/// The driver's budgets ([`MAX_ITEMS_PER_REQUEST`], [`MAX_REQUEST_BYTES`]) are
|
||||
/// enforced *here* rather than in `sglang_mm::driver::process`, where 64 sources
|
||||
/// of 64 MiB would already be resident. The driver keeps its own checks as the
|
||||
/// backstop for callers without a prefetch layer.
|
||||
/// `modality_limits` is `--limit-mm-data-per-request`, checked before a single
|
||||
/// byte is fetched, in its configured order and with Python's message
|
||||
/// (`TokenizerManager._validate_mm_limits`). Unset means no per-modality limit,
|
||||
/// as in Python. Python has no fixed item cap, and neither does this.
|
||||
///
|
||||
/// [`MAX_REQUEST_BYTES`] is enforced while fetching, here rather than in
|
||||
/// `sglang_mm::driver::process`, where every source would already be resident.
|
||||
/// It bounds buffered media bytes regardless of the item count. The driver
|
||||
/// keeps its own byte check as the backstop for callers without a prefetch
|
||||
/// layer.
|
||||
pub async fn prefetch_all(
|
||||
requests: &mut [GenerateRequest],
|
||||
modality_limits: &BTreeMap<String, usize>,
|
||||
modality_limits: &[(String, usize)],
|
||||
) -> Result<(), String> {
|
||||
// The item budget rejects before a single byte is fetched.
|
||||
let plan = |mm: &Option<Box<MmData>>| -> Result<Vec<String>, String> {
|
||||
let Some(mm) = mm.as_deref() else {
|
||||
return Ok(Vec::new());
|
||||
@@ -45,21 +49,13 @@ pub async fn prefetch_all(
|
||||
("video", &mm.video_data),
|
||||
("audio", &mm.audio_data),
|
||||
];
|
||||
let items = modalities
|
||||
.iter()
|
||||
.map(|(_, items)| items.len())
|
||||
.sum::<usize>();
|
||||
if items > MAX_ITEMS_PER_REQUEST {
|
||||
return Err(format!(
|
||||
"multimodal request exceeds {MAX_ITEMS_PER_REQUEST} media items"
|
||||
));
|
||||
}
|
||||
for (modality, items) in modalities {
|
||||
for (modality, limit) in modality_limits {
|
||||
let Some((name, items)) = modalities.iter().find(|(name, _)| name == modality) else {
|
||||
continue;
|
||||
};
|
||||
let count = items.len();
|
||||
if let Some(limit) = modality_limits.get(modality)
|
||||
&& count > *limit
|
||||
{
|
||||
let display = modality[..1].to_uppercase() + &modality[1..];
|
||||
if count > *limit {
|
||||
let display = name[..1].to_uppercase() + &name[1..];
|
||||
return Err(format!(
|
||||
"{display} count {count} exceeds limit {limit} per request."
|
||||
));
|
||||
@@ -198,7 +194,7 @@ mod tests {
|
||||
})),
|
||||
..Default::default()
|
||||
}];
|
||||
prefetch_all(&mut requests, &BTreeMap::new()).await.unwrap();
|
||||
prefetch_all(&mut requests, &[]).await.unwrap();
|
||||
std::fs::remove_dir_all(base).ok();
|
||||
let fetched = &requests[0].mm.as_ref().unwrap().prefetched;
|
||||
assert_eq!(
|
||||
@@ -223,7 +219,7 @@ mod tests {
|
||||
]),
|
||||
GenerateRequest::default(),
|
||||
];
|
||||
prefetch_all(&mut requests, &BTreeMap::new()).await.unwrap();
|
||||
prefetch_all(&mut requests, &[]).await.unwrap();
|
||||
std::fs::remove_file(&path).ok();
|
||||
let fetched = &requests[0].mm.as_ref().unwrap().prefetched;
|
||||
// The one-shot server answers in accept order, so contents may swap
|
||||
@@ -237,49 +233,72 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn failed_download_rejects() {
|
||||
let mut requests = vec![mm_request(vec![src("http://127.0.0.1:1/nope.png")])];
|
||||
let err = prefetch_all(&mut requests, &BTreeMap::new())
|
||||
.await
|
||||
.err()
|
||||
.unwrap();
|
||||
let err = prefetch_all(&mut requests, &[]).await.err().unwrap();
|
||||
assert!(err.contains("media fetch"), "{err}");
|
||||
}
|
||||
|
||||
/// The item budget rejects before any source is touched: all of these would
|
||||
/// fail to fetch, so a fetch error would prove fetching started.
|
||||
/// Without `--limit-mm-data-per-request`, a request with far more items
|
||||
/// than the old fixed cap (64) is fetched in full, as Python serves it.
|
||||
#[tokio::test]
|
||||
async fn item_budget_rejects_before_fetching() {
|
||||
let sources: Vec<MmItem> = (0..=MAX_ITEMS_PER_REQUEST)
|
||||
.map(|i| src(format!("/definitely/not/here-{i}.png")))
|
||||
.collect();
|
||||
let mut requests = vec![mm_request(sources)];
|
||||
let err = prefetch_all(&mut requests, &BTreeMap::new())
|
||||
.await
|
||||
.err()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
err,
|
||||
format!("multimodal request exceeds {MAX_ITEMS_PER_REQUEST} media items")
|
||||
);
|
||||
assert!(requests[0].mm.as_ref().unwrap().prefetched.is_empty());
|
||||
async fn items_beyond_the_old_cap_are_fetched_without_a_limit() {
|
||||
let n = 200;
|
||||
let path =
|
||||
std::env::temp_dir().join(format!("sglang-prefetch-many-{}", std::process::id()));
|
||||
std::fs::write(&path, b"img").unwrap();
|
||||
let mut requests = vec![mm_request(
|
||||
(0..n).map(|_| src(path.display().to_string())).collect(),
|
||||
)];
|
||||
let fetched = prefetch_all(&mut requests, &[]).await;
|
||||
std::fs::remove_file(&path).ok();
|
||||
fetched.unwrap();
|
||||
assert_eq!(requests[0].mm.as_ref().unwrap().prefetched.len(), n);
|
||||
}
|
||||
|
||||
/// `--limit-mm-data-per-request` matches Python `_validate_mm_limits`: a
|
||||
/// count at the limit passes, the first over-limit modality in configured
|
||||
/// order is reported with Python's message, and nothing is fetched.
|
||||
#[tokio::test]
|
||||
async fn per_modality_budget_rejects_before_fetching() {
|
||||
let mut requests = vec![GenerateRequest {
|
||||
async fn per_modality_limits_match_python_before_fetching() {
|
||||
let request = || GenerateRequest {
|
||||
mm: Some(Box::new(MmData {
|
||||
image_data: vec![
|
||||
src("/definitely/not/here-0.png"),
|
||||
src("/definitely/not/here-1.png"),
|
||||
],
|
||||
video_data: vec![src("/definitely/not/here.mp4")],
|
||||
video_data: vec![
|
||||
src("/definitely/not/here-0.mp4"),
|
||||
src("/definitely/not/here-1.mp4"),
|
||||
],
|
||||
..Default::default()
|
||||
})),
|
||||
..Default::default()
|
||||
}];
|
||||
let limits = BTreeMap::from([("image".to_owned(), 1), ("video".to_owned(), 1)]);
|
||||
let err = prefetch_all(&mut requests, &limits).await.err().unwrap();
|
||||
assert_eq!(err, "Image count 2 exceeds limit 1 per request.");
|
||||
assert!(requests[0].mm.as_ref().unwrap().prefetched.is_empty());
|
||||
};
|
||||
let owned = |pairs: &[(&str, usize)]| -> Vec<(String, usize)> {
|
||||
pairs.iter().map(|(m, n)| ((*m).to_owned(), *n)).collect()
|
||||
};
|
||||
for (limits, expected) in [
|
||||
(
|
||||
owned(&[("video", 1), ("image", 1)]),
|
||||
"Video count 2 exceeds limit 1 per request.",
|
||||
),
|
||||
(
|
||||
owned(&[("audio", 0), ("image", 1), ("video", 1)]),
|
||||
"Image count 2 exceeds limit 1 per request.",
|
||||
),
|
||||
] {
|
||||
let mut requests = vec![request()];
|
||||
let err = prefetch_all(&mut requests, &limits).await.err().unwrap();
|
||||
assert_eq!(err, expected);
|
||||
assert!(requests[0].mm.as_ref().unwrap().prefetched.is_empty());
|
||||
}
|
||||
|
||||
// At the limit the request passes the count check and reaches fetching.
|
||||
let mut requests = vec![request()];
|
||||
let err = prefetch_all(&mut requests, &owned(&[("image", 2)]))
|
||||
.await
|
||||
.err()
|
||||
.unwrap();
|
||||
assert!(err.contains("media fetch"), "{err}");
|
||||
}
|
||||
|
||||
/// Sources legal alone but collectively over the limit are rejected while
|
||||
|
||||
@@ -159,8 +159,10 @@ pub struct ServerArgs {
|
||||
/// Native-generation launch policy normally applied by Python ingress.
|
||||
pub lora_ids: BTreeMap<String, String>,
|
||||
pub enable_strict_thinking: bool,
|
||||
/// Per-modality media-count limits from `--limit-mm-data-per-request`.
|
||||
pub limit_mm_data_per_request: BTreeMap<String, usize>,
|
||||
/// Per-modality media-count limits from `--limit-mm-data-per-request`, in
|
||||
/// configured order: Python checks them in that order and reports the first
|
||||
/// one exceeded. Empty means no per-modality limit.
|
||||
pub limit_mm_data_per_request: Vec<(String, usize)>,
|
||||
/// Over-long inputs are truncated to fit the context instead of 400ing, and
|
||||
/// `max_new_tokens` is clamped rather than rejected (Python
|
||||
/// `TokenizerManager._validate_one_request`).
|
||||
@@ -278,7 +280,7 @@ impl ServerArgs {
|
||||
preferred_sampling_params: Option<PreferredSamplingParams>,
|
||||
lora_ids: BTreeMap<String, String>,
|
||||
enable_strict_thinking: bool,
|
||||
limit_mm_data_per_request: BTreeMap<String, usize>,
|
||||
limit_mm_data_per_request: Vec<(String, usize)>,
|
||||
allow_auto_truncate: bool,
|
||||
enable_return_hidden_states: bool,
|
||||
num_reserved_tokens: u64,
|
||||
@@ -376,7 +378,7 @@ impl Default for ServerArgs {
|
||||
preferred_sampling_params: None,
|
||||
lora_ids: BTreeMap::new(),
|
||||
enable_strict_thinking: false,
|
||||
limit_mm_data_per_request: BTreeMap::new(),
|
||||
limit_mm_data_per_request: Vec::new(),
|
||||
allow_auto_truncate: false,
|
||||
enable_return_hidden_states: false,
|
||||
num_reserved_tokens: 0,
|
||||
|
||||
@@ -52,6 +52,12 @@ def _scheduler_for_typed_config():
|
||||
"default_priority_value",
|
||||
5,
|
||||
),
|
||||
({}, "limit_mm_data_per_request", []),
|
||||
(
|
||||
{"limit_mm_data_per_request": {"video": 1, "image": 2}},
|
||||
"limit_mm_data_per_request",
|
||||
[("video", 1), ("image", 2)],
|
||||
),
|
||||
],
|
||||
ids=[
|
||||
"native-grpc",
|
||||
@@ -59,6 +65,8 @@ def _scheduler_for_typed_config():
|
||||
"deprecated-legacy-smg",
|
||||
"priority-off",
|
||||
"priority-on",
|
||||
"mm-limits-unset",
|
||||
"mm-limits-keep-order",
|
||||
],
|
||||
)
|
||||
def test_typed_config_forwards_resolved_launch_fields(overrides, kwarg, expected):
|
||||
|
||||
Reference in New Issue
Block a user