[Feat] runtime: Support tokenized multimodal generation (#42833)
This commit is contained in:
@@ -135,8 +135,10 @@ message GenerateRequest {
|
||||
optional bool require_reasoning = 15;
|
||||
optional uint32 max_thinking_tokens = 16;
|
||||
optional KVHintsEnvelope kv_hints = 17;
|
||||
// Also return the detokenized text, as HTTP /generate does; off by default.
|
||||
optional bool return_text = 18;
|
||||
repeated string image_data = 19;
|
||||
repeated string video_data = 20;
|
||||
repeated string audio_data = 21;
|
||||
}
|
||||
|
||||
// Versioned KV-hint envelope forwarded from a trusted orchestrator.
|
||||
|
||||
@@ -169,9 +169,14 @@ pub struct GenerateRequest {
|
||||
pub max_thinking_tokens: ::core::option::Option<u32>,
|
||||
#[prost(message, optional, tag = "17")]
|
||||
pub kv_hints: ::core::option::Option<KvHintsEnvelope>,
|
||||
/// Also return the detokenized text, as HTTP /generate does; off by default.
|
||||
#[prost(bool, optional, tag = "18")]
|
||||
pub return_text: ::core::option::Option<bool>,
|
||||
#[prost(string, repeated, tag = "19")]
|
||||
pub image_data: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
|
||||
#[prost(string, repeated, tag = "20")]
|
||||
pub video_data: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
|
||||
#[prost(string, repeated, tag = "21")]
|
||||
pub audio_data: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
|
||||
}
|
||||
/// Versioned KV-hint envelope forwarded from a trusted orchestrator.
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
|
||||
@@ -329,6 +329,15 @@ pub(crate) fn build_generate_dict(
|
||||
let mut d = HashMap::new();
|
||||
d.insert("rid".into(), serde_json::json!(rid));
|
||||
d.insert("input_ids".into(), serde_json::json!(req.input_ids));
|
||||
for (name, sources) in [
|
||||
("image_data", &req.image_data),
|
||||
("video_data", &req.video_data),
|
||||
("audio_data", &req.audio_data),
|
||||
] {
|
||||
if !sources.is_empty() {
|
||||
d.insert(name.into(), serde_json::json!(sources));
|
||||
}
|
||||
}
|
||||
d.insert(
|
||||
"sampling_params".into(),
|
||||
sampling_params_to_map(&req.sampling_params)?,
|
||||
@@ -647,6 +656,65 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tokenized_multimodal_request_preserves_generation_params() {
|
||||
let request = proto::GenerateRequest {
|
||||
input_ids: vec![2, 10, 20],
|
||||
image_data: vec![
|
||||
"https://example.com/image.png".to_string(),
|
||||
"data:image/png;base64,aW1hZ2U=".to_string(),
|
||||
],
|
||||
video_data: vec!["https://example.com/clip.mp4".to_string()],
|
||||
audio_data: vec!["data:audio/wav;base64,UklGRg==".to_string()],
|
||||
sampling_params: Some(proto::SamplingParams {
|
||||
temperature: Some(0.5),
|
||||
max_new_tokens: Some(64),
|
||||
..Default::default()
|
||||
}),
|
||||
disaggregated_params: Some(proto::DisaggregatedParams {
|
||||
bootstrap_host: "10.0.0.1".to_string(),
|
||||
bootstrap_port: 8998,
|
||||
bootstrap_room: i64::MAX,
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let mapped = build_generate_dict("request", &request).unwrap();
|
||||
|
||||
assert_eq!(mapped["input_ids"], serde_json::json!([2, 10, 20]));
|
||||
assert_eq!(
|
||||
mapped["image_data"],
|
||||
serde_json::json!([
|
||||
"https://example.com/image.png",
|
||||
"data:image/png;base64,aW1hZ2U="
|
||||
])
|
||||
);
|
||||
assert_eq!(
|
||||
mapped["video_data"],
|
||||
serde_json::json!(["https://example.com/clip.mp4"])
|
||||
);
|
||||
assert_eq!(
|
||||
mapped["audio_data"],
|
||||
serde_json::json!(["data:audio/wav;base64,UklGRg=="])
|
||||
);
|
||||
assert_eq!(
|
||||
mapped["sampling_params"],
|
||||
serde_json::json!({"temperature": 0.5, "max_new_tokens": 64})
|
||||
);
|
||||
assert_eq!(mapped["bootstrap_host"], serde_json::json!("10.0.0.1"));
|
||||
assert_eq!(mapped["bootstrap_port"], serde_json::json!(8998));
|
||||
assert_eq!(mapped["bootstrap_room"], serde_json::json!(i64::MAX));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tokenized_request_omits_empty_media_data() {
|
||||
let mapped = build_generate_dict("request", &proto::GenerateRequest::default()).unwrap();
|
||||
|
||||
for key in ["image_data", "video_data", "audio_data"] {
|
||||
assert!(!mapped.contains_key(key));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn guided_choice_maps_to_escaped_regex() {
|
||||
let request = proto::GenerateRequest {
|
||||
|
||||
Reference in New Issue
Block a user