VLM: enhance VL embedding model with video input support and revise warm-up strategy (#16635)

Co-authored-by: Mick <mickjagger19@icloud.com>

python/sglang/srt/entrypoints/http_server.py

@@ -1514,8 +1514,13 @@ def _execute_server_warmup(server_args: ServerArgs):
         # TODO Workaround the bug that embedding errors for list of size 1
         if server_args.dp_size == 1:
             json_data["input_ids"] = json_data["input_ids"][0]
-    elif is_vlm and server_args.disaggregation_mode == "null":
+    elif (
+        is_vlm
+        and server_args.disaggregation_mode == "null"
+        and model_info["is_generation"]
+    ):
         # TODO: ChatCompletionRequest does not have bootstrap info required by disaggregation mode, disable image-warmup for now
+        # Only use chat completions format for generation models, not embedding models
         json_data = {
             "model": _global_state.tokenizer_manager.served_model_name,
             "messages": [

python/sglang/srt/entrypoints/openai/protocol.py

@@ -780,6 +780,7 @@ class ChatCompletionStreamResponse(BaseModel):
 class MultimodalEmbeddingInput(BaseModel):
     text: Optional[str] = None
     image: Optional[str] = None
+    video: Optional[str] = None
 
 
 EmbeddingInput = Union[

python/sglang/srt/entrypoints/openai/serving_embedding.py

@@ -89,16 +89,18 @@ class OpenAIServingEmbedding(OpenAIServingBase):
                 # Handle multimodal embedding inputs
                 texts = []
                 images = []
+                videos = []
                 for item in prompt:
                     # Use padding for text if None - this could be improved
                     texts.append(item.text if item.text is not None else "padding")
                     images.append(item.image if item.image is not None else None)
+                    videos.append(item.video if item.video is not None else None)
 
                 generate_prompts = []
                 # Check if we have a chat template for multimodal embeddings
                 if self.template_manager.chat_template_name is not None:
                     convs = generate_embedding_convs(
-                        texts, images, self.template_manager.chat_template_name
+                        texts, images, videos, self.template_manager.chat_template_name
                     )
                     for conv in convs:
                         generate_prompts.append(conv.get_prompt())
@@ -109,11 +111,13 @@ class OpenAIServingEmbedding(OpenAIServingBase):
                     prompt_kwargs = {
                         "text": generate_prompts[0],
                         "image_data": images[0],
+                        "video_data": videos[0],
                     }
                 else:
                     prompt_kwargs = {
                         "text": generate_prompts,
                         "image_data": images,
+                        "video_data": videos,
                     }
             else:
                 # List of integers (token IDs) or empty list

python/sglang/srt/managers/tokenizer_manager.py

@@ -683,6 +683,8 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi
         if self.mm_processor and obj.contains_mm_input():
             if obj.image_data is not None and not isinstance(obj.image_data, list):
                 obj.image_data = [obj.image_data]
+            if obj.video_data is not None and not isinstance(obj.video_data, list):
+                obj.video_data = [obj.video_data]
             if obj.audio_data is not None and not isinstance(obj.audio_data, list):
                 obj.audio_data = [obj.audio_data]
             self._validate_mm_limits(obj)

python/sglang/srt/parser/conversation.py

@@ -510,11 +510,11 @@ def chat_template_exists(template_name: str) -> bool:
 
 
 def generate_embedding_convs(
-    texts: List[str], images: List[str], template_name: str
+    texts: List[str], images: List[str], videos: List[str], template_name: str
 ) -> List[Conversation]:
     conv_template = chat_templates[template_name].copy()
     convs = []
-    for text, image in zip(texts, images):
+    for text, image, video in zip(texts, images, videos):
         conv = Conversation(
             name=conv_template.name,
             system_template=conv_template.system_template,
@@ -544,6 +544,8 @@ def generate_embedding_convs(
                 else conv.image_token
             )
             real_content += image_token
+        if video is not None:
+            real_content += conv.video_token
         if text is not None:
             real_content += text
         conv.append_message(conv.roles[0], real_content)