Fix multimodal /v1/embeddings Jinja chat template handling (#20835)

Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>

python/sglang/srt/entrypoints/openai/serving_embedding.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 
+import jinja2
 from fastapi import Request
 from fastapi.responses import ORJSONResponse
 
@@ -17,6 +18,7 @@ from sglang.srt.entrypoints.openai.serving_base import OpenAIServingBase
 from sglang.srt.entrypoints.openai.utils import convert_embeds_to_tensors
 from sglang.srt.managers.io_struct import EmbeddingReqInput
 from sglang.srt.parser.conversation import generate_embedding_convs
+from sglang.srt.parser.jinja_template_utils import process_content_for_template_format
 
 if TYPE_CHECKING:
     from sglang.srt.managers.template_manager import TemplateManager
@@ -92,21 +94,31 @@ class OpenAIServingEmbedding(OpenAIServingBase):
                 images = []
                 videos = []
                 for item in prompt:
-                    # Use padding for text if None - this could be improved
-                    texts.append(item.text if item.text is not None else "padding")
+                    texts.append(item.text)
                     images.append(item.image if item.image is not None else None)
                     videos.append(item.video if item.video is not None else None)
 
+                # Precedence: a SGLang-registered conversation template wins
+                # over the tokenizer's own HF Jinja template when both exist.
                 generate_prompts = []
-                # Check if we have a chat template for multimodal embeddings
                 if self.template_manager.chat_template_name is not None:
                     convs = generate_embedding_convs(
                         texts, images, videos, self.template_manager.chat_template_name
                     )
                     for conv in convs:
                         generate_prompts.append(conv.get_prompt())
+                elif (
+                    self.tokenizer_manager.tokenizer is not None
+                    and getattr(self.tokenizer_manager.tokenizer, "chat_template", None)
+                    is not None
+                ):
+                    generate_prompts = self._apply_jinja_template_to_embedding_inputs(
+                        texts, images, videos
+                    )
                 else:
-                    generate_prompts = texts
+                    generate_prompts = [
+                        text if text is not None else "padding" for text in texts
+                    ]
 
                 if len(generate_prompts) == 1:
                     prompt_kwargs = {
@@ -163,6 +175,68 @@ class OpenAIServingEmbedding(OpenAIServingBase):
 
         return adapted_request, request
 
+    def _apply_jinja_template_to_embedding_inputs(
+        self,
+        texts: List[Optional[str]],
+        images: List[Optional[str]],
+        videos: List[Optional[str]],
+    ) -> List[str]:
+        """Render each multimodal embedding input through the tokenizer's Jinja chat template.
+
+        Image/video bytes are threaded to the engine separately via
+        ``EmbeddingReqInput.image_data``/``video_data``; this method only produces
+        the prompt string. ``text=None`` emits no text chunk (no ``"padding"``
+        literal). Jinja failures are re-raised as ``ValueError`` so the caller
+        returns HTTP 400 instead of 500.
+        """
+        prompts: List[str] = []
+        template_content_format = self.template_manager.jinja_template_content_format
+
+        for text, image, video in zip(texts, images, videos):
+            content_parts = []
+            if image is not None:
+                content_parts.append({"type": "image_url", "image_url": {"url": image}})
+            if video is not None:
+                content_parts.append({"type": "video_url", "video_url": {"url": video}})
+            if text is not None:
+                content_parts.append({"type": "text", "text": text})
+
+            msg_dict = {
+                "role": "user",
+                "content": content_parts if content_parts else "",
+            }
+            # Empty list args: this helper is only used to normalize the content
+            # shape (e.g. image_url -> image); real payloads ride on the outer
+            # images/videos lists, not EmbeddingReqInput fields derived here.
+            processed_msg = process_content_for_template_format(
+                msg_dict,
+                template_content_format,
+                image_data=[],
+                video_data=[],
+                audio_data=[],
+                modalities=[],
+            )
+            try:
+                prompt = self.tokenizer_manager.tokenizer.apply_chat_template(
+                    [processed_msg],
+                    tokenize=False,
+                    add_generation_prompt=True,
+                )
+            except jinja2.TemplateError as template_error:
+                location = getattr(template_error, "lineno", None)
+                name = getattr(template_error, "name", None)
+                suffix = ""
+                if name or location:
+                    suffix = f" (template={name or '<unknown>'}, line={location})"
+                raise ValueError(f"{template_error}{suffix}") from template_error
+            except (TypeError, KeyError, AttributeError) as template_error:
+                raise ValueError(
+                    f"Failed to render chat template for embedding input: {template_error}"
+                ) from template_error
+            prompts.append(prompt)
+
+        return prompts
+
     async def _handle_non_streaming_request(
         self,
         adapted_request: EmbeddingReqInput,