diff --git a/rust/sglang-server/src/multi_modality/worker.rs b/rust/sglang-server/src/multi_modality/worker.rs index bb909fe725..494606c936 100644 --- a/rust/sglang-server/src/multi_modality/worker.rs +++ b/rust/sglang-server/src/multi_modality/worker.rs @@ -67,6 +67,11 @@ pub struct MmProcessOutput { /// owned. pub trait MmProcessor: Send + Sync { fn process(&self, input_ids: TokenIds, mm: MmData) -> Result; + + /// Finish item layout after caller hash overrides, before feature publication. + fn finalize(&self, _entry: &mut MmEncodedEntry) -> Result<(), String> { + Ok(()) + } } struct QwenMmProcessor { @@ -265,6 +270,8 @@ fn process( output.result.items.iter_mut().map(|item| &mut item.hash), &caller_hashes, ); + ctx.processor.finalize(&mut output.result)?; + output.result.validate(output.input_ids.len())?; let buffers = make_buffers(output.result, ctx.feature_shm)?; Ok((output.input_ids, buffers)) } @@ -711,6 +718,63 @@ mod tests { assert!(text.contains("clip_index"), "{text}"); } + #[test] + fn finalization_runs_after_caller_hashes_and_is_validated() { + struct FinalizingProcessor { + final_end: u32, + } + + impl MmProcessor for FinalizingProcessor { + fn process(&self, ids: TokenIds, mm: MmData) -> Result { + ExternalProcessor { + shape: vec![1], + offsets: vec![(0, 0)], + } + .process(ids, mm) + } + + fn finalize(&self, entry: &mut MmEncodedEntry) -> Result<(), String> { + let parent_hash = entry.items[0].hash; + assert_eq!(parent_hash, 42); + entry.items[0].hash = parent_hash + 1; + entry.items.push(MmEncodedItem { + modality: MmModality::Image, + feature: Tensor { + shape: vec![1], + data: TensorData::F32(vec![2.0]), + }, + hash: parent_hash + 2, + offsets: vec![(1, self.final_end)], + model_specific_data: BTreeMap::new(), + }); + Ok(()) + } + } + + for final_end in [1, 2] { + let ctx = MmContext::with_processor(Arc::new(FinalizingProcessor { final_end }), false); + let result = process( + &ctx, + vec![1, 2], + MmData { + mm_hashes: vec!["2a".to_owned()], + ..Default::default() + }, + ); + if final_end == 2 { + assert!(result.is_err(), "finalized spans must be validated"); + continue; + } + let (ids, buffers) = result.expect("valid finalized items"); + assert_eq!(ids, [1, 2]); + assert!(find(&buffers, "mm.feature.1").is_some()); + let meta: serde_json::Value = rmpv::ext::from_value(decoded_meta(&buffers)).unwrap(); + assert_eq!(meta["items"].as_array().unwrap().len(), 2); + assert_eq!(meta["items"][0]["hash"].as_u64(), Some(43)); + assert_eq!(meta["items"][1]["hash"].as_u64(), Some(44)); + } + } + /// Malformed results are rejected before any buffer exists, so nothing /// reaches the ring for them. #[test]