From 3acfa2efcc51a0c473930e60614e7f7c39863fad Mon Sep 17 00:00:00 2001 From: Kaushik Chatterjee <31332999+kchatr@users.noreply.github.com> Date: Sat, 17 Oct 2026 20:35:51 -0700 Subject: [PATCH] [Feat] runtime: Support tokenized multimodal generation (#42833) --- proto/sglang/runtime/v1/sglang.proto | 4 +- .../src/generated/sglang.runtime.v1.rs | 7 +- rust/sglang-grpc/src/utils/request_utils.rs | 68 +++++++++++++++++++ 3 files changed, 77 insertions(+), 2 deletions(-) diff --git a/proto/sglang/runtime/v1/sglang.proto b/proto/sglang/runtime/v1/sglang.proto index 36744c069f..84cbdbae06 100644 --- a/proto/sglang/runtime/v1/sglang.proto +++ b/proto/sglang/runtime/v1/sglang.proto @@ -135,8 +135,10 @@ message GenerateRequest { optional bool require_reasoning = 15; optional uint32 max_thinking_tokens = 16; optional KVHintsEnvelope kv_hints = 17; - // Also return the detokenized text, as HTTP /generate does; off by default. optional bool return_text = 18; + repeated string image_data = 19; + repeated string video_data = 20; + repeated string audio_data = 21; } // Versioned KV-hint envelope forwarded from a trusted orchestrator. diff --git a/rust/sglang-api-types/src/generated/sglang.runtime.v1.rs b/rust/sglang-api-types/src/generated/sglang.runtime.v1.rs index e6314bdc19..b0b6cb0c2c 100644 --- a/rust/sglang-api-types/src/generated/sglang.runtime.v1.rs +++ b/rust/sglang-api-types/src/generated/sglang.runtime.v1.rs @@ -169,9 +169,14 @@ pub struct GenerateRequest { pub max_thinking_tokens: ::core::option::Option, #[prost(message, optional, tag = "17")] pub kv_hints: ::core::option::Option, - /// Also return the detokenized text, as HTTP /generate does; off by default. #[prost(bool, optional, tag = "18")] pub return_text: ::core::option::Option, + #[prost(string, repeated, tag = "19")] + pub image_data: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(string, repeated, tag = "20")] + pub video_data: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(string, repeated, tag = "21")] + pub audio_data: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, } /// Versioned KV-hint envelope forwarded from a trusted orchestrator. #[derive(Clone, PartialEq, ::prost::Message)] diff --git a/rust/sglang-grpc/src/utils/request_utils.rs b/rust/sglang-grpc/src/utils/request_utils.rs index 06f7e9e915..4337fb63ea 100644 --- a/rust/sglang-grpc/src/utils/request_utils.rs +++ b/rust/sglang-grpc/src/utils/request_utils.rs @@ -329,6 +329,15 @@ pub(crate) fn build_generate_dict( let mut d = HashMap::new(); d.insert("rid".into(), serde_json::json!(rid)); d.insert("input_ids".into(), serde_json::json!(req.input_ids)); + for (name, sources) in [ + ("image_data", &req.image_data), + ("video_data", &req.video_data), + ("audio_data", &req.audio_data), + ] { + if !sources.is_empty() { + d.insert(name.into(), serde_json::json!(sources)); + } + } d.insert( "sampling_params".into(), sampling_params_to_map(&req.sampling_params)?, @@ -647,6 +656,65 @@ mod tests { } } + #[test] + fn tokenized_multimodal_request_preserves_generation_params() { + let request = proto::GenerateRequest { + input_ids: vec![2, 10, 20], + image_data: vec![ + "https://example.com/image.png".to_string(), + "data:image/png;base64,aW1hZ2U=".to_string(), + ], + video_data: vec!["https://example.com/clip.mp4".to_string()], + audio_data: vec!["data:audio/wav;base64,UklGRg==".to_string()], + sampling_params: Some(proto::SamplingParams { + temperature: Some(0.5), + max_new_tokens: Some(64), + ..Default::default() + }), + disaggregated_params: Some(proto::DisaggregatedParams { + bootstrap_host: "10.0.0.1".to_string(), + bootstrap_port: 8998, + bootstrap_room: i64::MAX, + }), + ..Default::default() + }; + + let mapped = build_generate_dict("request", &request).unwrap(); + + assert_eq!(mapped["input_ids"], serde_json::json!([2, 10, 20])); + assert_eq!( + mapped["image_data"], + serde_json::json!([ + "https://example.com/image.png", + "data:image/png;base64,aW1hZ2U=" + ]) + ); + assert_eq!( + mapped["video_data"], + serde_json::json!(["https://example.com/clip.mp4"]) + ); + assert_eq!( + mapped["audio_data"], + serde_json::json!(["data:audio/wav;base64,UklGRg=="]) + ); + assert_eq!( + mapped["sampling_params"], + serde_json::json!({"temperature": 0.5, "max_new_tokens": 64}) + ); + assert_eq!(mapped["bootstrap_host"], serde_json::json!("10.0.0.1")); + assert_eq!(mapped["bootstrap_port"], serde_json::json!(8998)); + assert_eq!(mapped["bootstrap_room"], serde_json::json!(i64::MAX)); + } + + #[test] + fn tokenized_request_omits_empty_media_data() { + let mapped = build_generate_dict("request", &proto::GenerateRequest::default()).unwrap(); + + for key in ["image_data", "video_data", "audio_data"] { + assert!(!mapped.contains_key(key)); + } + } + #[test] fn guided_choice_maps_to_escaped_regex() { let request = proto::GenerateRequest {