[Feat] rust-server: Let a multimodal processor finalize item layout after caller hashes (#45914)

Co-authored-by: metamergebot <metamergebot@users.noreply.github.com>
Co-authored-by: Lu Fang <30275821+houseroad@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-23 09:26:52 -07:00
committed by GitHub
co-authored by metamergebot Lu Fang
parent 30dc2f9430
commit 74c08f3dc5
@@ -67,6 +67,11 @@ pub struct MmProcessOutput {
/// owned.
pub trait MmProcessor: Send + Sync {
fn process(&self, input_ids: TokenIds, mm: MmData) -> Result<MmProcessOutput, String>;
/// Finish item layout after caller hash overrides, before feature publication.
fn finalize(&self, _entry: &mut MmEncodedEntry) -> Result<(), String> {
Ok(())
}
}
struct QwenMmProcessor {
@@ -265,6 +270,8 @@ fn process(
output.result.items.iter_mut().map(|item| &mut item.hash),
&caller_hashes,
);
ctx.processor.finalize(&mut output.result)?;
output.result.validate(output.input_ids.len())?;
let buffers = make_buffers(output.result, ctx.feature_shm)?;
Ok((output.input_ids, buffers))
}
@@ -711,6 +718,63 @@ mod tests {
assert!(text.contains("clip_index"), "{text}");
}
#[test]
fn finalization_runs_after_caller_hashes_and_is_validated() {
struct FinalizingProcessor {
final_end: u32,
}
impl MmProcessor for FinalizingProcessor {
fn process(&self, ids: TokenIds, mm: MmData) -> Result<MmProcessOutput, String> {
ExternalProcessor {
shape: vec![1],
offsets: vec![(0, 0)],
}
.process(ids, mm)
}
fn finalize(&self, entry: &mut MmEncodedEntry) -> Result<(), String> {
let parent_hash = entry.items[0].hash;
assert_eq!(parent_hash, 42);
entry.items[0].hash = parent_hash + 1;
entry.items.push(MmEncodedItem {
modality: MmModality::Image,
feature: Tensor {
shape: vec![1],
data: TensorData::F32(vec![2.0]),
},
hash: parent_hash + 2,
offsets: vec![(1, self.final_end)],
model_specific_data: BTreeMap::new(),
});
Ok(())
}
}
for final_end in [1, 2] {
let ctx = MmContext::with_processor(Arc::new(FinalizingProcessor { final_end }), false);
let result = process(
&ctx,
vec![1, 2],
MmData {
mm_hashes: vec!["2a".to_owned()],
..Default::default()
},
);
if final_end == 2 {
assert!(result.is_err(), "finalized spans must be validated");
continue;
}
let (ids, buffers) = result.expect("valid finalized items");
assert_eq!(ids, [1, 2]);
assert!(find(&buffers, "mm.feature.1").is_some());
let meta: serde_json::Value = rmpv::ext::from_value(decoded_meta(&buffers)).unwrap();
assert_eq!(meta["items"].as_array().unwrap().len(), 2);
assert_eq!(meta["items"][0]["hash"].as_u64(), Some(43));
assert_eq!(meta["items"][1]["hash"].as_u64(), Some(44));
}
}
/// Malformed results are rejected before any buffer exists, so nothing
/// reaches the ring for them.
#[test]