Fix multimodal /v1/embeddings Jinja chat template handling (#20835)

Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>

python/sglang/srt/entrypoints/openai/serving_embedding.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 
+import jinja2
 from fastapi import Request
 from fastapi.responses import ORJSONResponse
 
@@ -17,6 +18,7 @@ from sglang.srt.entrypoints.openai.serving_base import OpenAIServingBase
 from sglang.srt.entrypoints.openai.utils import convert_embeds_to_tensors
 from sglang.srt.managers.io_struct import EmbeddingReqInput
 from sglang.srt.parser.conversation import generate_embedding_convs
+from sglang.srt.parser.jinja_template_utils import process_content_for_template_format
 
 if TYPE_CHECKING:
     from sglang.srt.managers.template_manager import TemplateManager
@@ -92,21 +94,31 @@ class OpenAIServingEmbedding(OpenAIServingBase):
                 images = []
                 videos = []
                 for item in prompt:
-                    # Use padding for text if None - this could be improved
+                    texts.append(item.text)
-                    texts.append(item.text if item.text is not None else "padding")
                     images.append(item.image if item.image is not None else None)
                     videos.append(item.video if item.video is not None else None)
 
+                # Precedence: a SGLang-registered conversation template wins
+                # over the tokenizer's own HF Jinja template when both exist.
                 generate_prompts = []
-                # Check if we have a chat template for multimodal embeddings
                 if self.template_manager.chat_template_name is not None:
                     convs = generate_embedding_convs(
                         texts, images, videos, self.template_manager.chat_template_name
                     )
                     for conv in convs:
                         generate_prompts.append(conv.get_prompt())
+                elif (
+                    self.tokenizer_manager.tokenizer is not None
+                    and getattr(self.tokenizer_manager.tokenizer, "chat_template", None)
+                    is not None
+                ):
+                    generate_prompts = self._apply_jinja_template_to_embedding_inputs(
+                        texts, images, videos
+                    )
                 else:
-                    generate_prompts = texts
+                    generate_prompts = [
+                        text if text is not None else "padding" for text in texts
+                    ]
 
                 if len(generate_prompts) == 1:
                     prompt_kwargs = {
@@ -163,6 +175,68 @@ class OpenAIServingEmbedding(OpenAIServingBase):
 
         return adapted_request, request
 
+    def _apply_jinja_template_to_embedding_inputs(
+        self,
+        texts: List[Optional[str]],
+        images: List[Optional[str]],
+        videos: List[Optional[str]],
+    ) -> List[str]:
+        """Render each multimodal embedding input through the tokenizer's Jinja chat template.
+
+        Image/video bytes are threaded to the engine separately via
+        ``EmbeddingReqInput.image_data``/``video_data``; this method only produces
+        the prompt string. ``text=None`` emits no text chunk (no ``"padding"``
+        literal). Jinja failures are re-raised as ``ValueError`` so the caller
+        returns HTTP 400 instead of 500.
+        """
+        prompts: List[str] = []
+        template_content_format = self.template_manager.jinja_template_content_format
+
+        for text, image, video in zip(texts, images, videos):
+            content_parts = []
+            if image is not None:
+                content_parts.append({"type": "image_url", "image_url": {"url": image}})
+            if video is not None:
+                content_parts.append({"type": "video_url", "video_url": {"url": video}})
+            if text is not None:
+                content_parts.append({"type": "text", "text": text})
+
+            msg_dict = {
+                "role": "user",
+                "content": content_parts if content_parts else "",
+            }
+            # Empty list args: this helper is only used to normalize the content
+            # shape (e.g. image_url -> image); real payloads ride on the outer
+            # images/videos lists, not EmbeddingReqInput fields derived here.
+            processed_msg = process_content_for_template_format(
+                msg_dict,
+                template_content_format,
+                image_data=[],
+                video_data=[],
+                audio_data=[],
+                modalities=[],
+            )
+            try:
+                prompt = self.tokenizer_manager.tokenizer.apply_chat_template(
+                    [processed_msg],
+                    tokenize=False,
+                    add_generation_prompt=True,
+                )
+            except jinja2.TemplateError as template_error:
+                location = getattr(template_error, "lineno", None)
+                name = getattr(template_error, "name", None)
+                suffix = ""
+                if name or location:
+                    suffix = f" (template={name or '<unknown>'}, line={location})"
+                raise ValueError(f"{template_error}{suffix}") from template_error
+            except (TypeError, KeyError, AttributeError) as template_error:
+                raise ValueError(
+                    f"Failed to render chat template for embedding input: {template_error}"
+                ) from template_error
+            prompts.append(prompt)
+
+        return prompts
+
     async def _handle_non_streaming_request(
         self,
         adapted_request: EmbeddingReqInput,

test/registered/unit/entrypoints/openai/test_serving_embedding.py

@@ -11,6 +11,8 @@ import unittest
 import uuid
 from unittest.mock import MagicMock, Mock
 
+import jinja2
+
 
 # Stub out sgl_kernel (and all submodules) before any sglang import so
 # the test runs on CPU-only runners without the real CUDA library.
@@ -90,7 +92,7 @@ class _MockTokenizerManager:
 class _MockTemplateManager:
     def __init__(self):
         self.chat_template_name = None  # None for embeddings usually
-        self.jinja_template_content_format = None
+        self.jinja_template_content_format = "openai"
         self.completion_template_name = None
 
 
@@ -124,6 +126,22 @@ class ServingEmbeddingTestCase(unittest.TestCase):
             ],
             encoding_format="float",
         )
+        self.image_only_multimodal_req = EmbeddingRequest(
+            model="test-model",
+            input=[
+                MultimodalEmbeddingInput(text=None, image="base64_image_data"),
+            ],
+            encoding_format="float",
+        )
+        self.video_multimodal_req = EmbeddingRequest(
+            model="test-model",
+            input=[
+                MultimodalEmbeddingInput(
+                    text="Describe", image=None, video="base64_video_data"
+                ),
+            ],
+            encoding_format="float",
+        )
         self.token_ids_req = EmbeddingRequest(
             model="test-model",
             input=[1, 2, 3, 4, 5],
@@ -180,6 +198,146 @@ class ServingEmbeddingTestCase(unittest.TestCase):
         self.assertIsNone(adapted_request.image_data[1])
         # self.assertEqual(adapted_request.rid, "test-id")
 
+    def test_convert_multimodal_request_with_jinja_chat_template(self):
+        """Multimodal embeddings should apply explicit/HF Jinja chat templates."""
+        self.tokenizer_manager.tokenizer.chat_template = "mock-template"
+        self.tokenizer_manager.tokenizer.apply_chat_template = Mock(
+            side_effect=[
+                "<prompt>Hello<image></prompt>",
+                "<prompt>World</prompt>",
+            ]
+        )
+
+        adapted_request, _ = self.serving_embedding._convert_to_internal_request(
+            self.multimodal_req
+        )
+
+        self.assertEqual(
+            adapted_request.text,
+            ["<prompt>Hello<image></prompt>", "<prompt>World</prompt>"],
+        )
+        self.assertEqual(adapted_request.image_data[0], "base64_image_data")
+        self.assertIsNone(adapted_request.image_data[1])
+        self.assertEqual(
+            self.tokenizer_manager.tokenizer.apply_chat_template.call_count, 2
+        )
+        first_call = (
+            self.tokenizer_manager.tokenizer.apply_chat_template.call_args_list[0]
+        )
+        first_messages = first_call.args[0]
+        self.assertEqual(first_messages[0]["role"], "user")
+        self.assertEqual(first_messages[0]["content"][0]["type"], "image")
+        self.assertEqual(first_messages[0]["content"][1]["type"], "text")
+        self.assertEqual(first_messages[0]["content"][1]["text"], "Hello")
+        self.assertEqual(first_call.kwargs["tokenize"], False)
+        self.assertEqual(first_call.kwargs["add_generation_prompt"], True)
+
+        second_call = (
+            self.tokenizer_manager.tokenizer.apply_chat_template.call_args_list[1]
+        )
+        second_messages = second_call.args[0]
+        self.assertEqual(len(second_messages[0]["content"]), 1)
+        self.assertEqual(second_messages[0]["content"][0]["type"], "text")
+        self.assertEqual(second_messages[0]["content"][0]["text"], "World")
+
+    def test_convert_image_only_multimodal_request_with_jinja_chat_template(self):
+        """Image-only requests should not inject literal padding into Jinja prompts."""
+        self.tokenizer_manager.tokenizer.chat_template = "mock-template"
+        self.tokenizer_manager.tokenizer.apply_chat_template = Mock(
+            return_value="<prompt><image></prompt>"
+        )
+
+        adapted_request, _ = self.serving_embedding._convert_to_internal_request(
+            self.image_only_multimodal_req
+        )
+
+        self.assertEqual(adapted_request.text, "<prompt><image></prompt>")
+        first_call = self.tokenizer_manager.tokenizer.apply_chat_template.call_args
+        first_messages = first_call.args[0]
+        self.assertEqual(first_messages[0]["role"], "user")
+        self.assertEqual(len(first_messages[0]["content"]), 1)
+        self.assertEqual(first_messages[0]["content"][0]["type"], "image")
+
+    def test_convert_video_multimodal_request_with_jinja_chat_template(self):
+        """Video inputs should land in video_data and flow through the Jinja branch."""
+        self.tokenizer_manager.tokenizer.chat_template = "mock-template"
+        self.tokenizer_manager.tokenizer.apply_chat_template = Mock(
+            return_value="<prompt>Describe<video></prompt>"
+        )
+
+        adapted_request, _ = self.serving_embedding._convert_to_internal_request(
+            self.video_multimodal_req
+        )
+
+        self.assertEqual(adapted_request.text, "<prompt>Describe<video></prompt>")
+        self.assertEqual(adapted_request.video_data, "base64_video_data")
+        self.assertIsNone(adapted_request.image_data)
+        first_messages = (
+            self.tokenizer_manager.tokenizer.apply_chat_template.call_args.args[0]
+        )
+        content = first_messages[0]["content"]
+        self.assertEqual([c["type"] for c in content], ["video", "text"])
+
+    def test_multimodal_request_falls_back_when_no_chat_template(self):
+        """Without any chat template the raw-text fallback must run without raising."""
+        self.tokenizer_manager.tokenizer.chat_template = None
+
+        adapted_request, _ = self.serving_embedding._convert_to_internal_request(
+            self.image_only_multimodal_req
+        )
+
+        # text=None on an image-only input falls back to the "padding" literal.
+        self.assertEqual(adapted_request.text, "padding")
+        self.assertEqual(adapted_request.image_data, "base64_image_data")
+
+    def test_multimodal_request_with_no_tokenizer_uses_fallback(self):
+        """Missing tokenizer should not crash the Jinja branch check."""
+        self.tokenizer_manager.tokenizer = None
+
+        adapted_request, _ = self.serving_embedding._convert_to_internal_request(
+            self.multimodal_req
+        )
+
+        self.assertEqual(adapted_request.text, ["Hello", "World"])
+
+    def test_jinja_template_errors_are_raised_as_value_error(self):
+        """Template failures should be converted to ValueError for a 400 response."""
+        self.tokenizer_manager.tokenizer.chat_template = "mock-template"
+        self.tokenizer_manager.tokenizer.apply_chat_template = Mock(
+            side_effect=jinja2.TemplateError("bad template")
+        )
+
+        with self.assertRaisesRegex(ValueError, "bad template"):
+            self.serving_embedding._convert_to_internal_request(
+                self.image_only_multimodal_req
+            )
+
+    def test_jinja_template_syntax_error_includes_location(self):
+        """TemplateSyntaxError should surface template name and line number."""
+        err = jinja2.TemplateSyntaxError("unexpected end", lineno=7, name="mock.jinja")
+        self.tokenizer_manager.tokenizer.chat_template = "mock-template"
+        self.tokenizer_manager.tokenizer.apply_chat_template = Mock(side_effect=err)
+
+        with self.assertRaises(ValueError) as ctx:
+            self.serving_embedding._convert_to_internal_request(
+                self.image_only_multimodal_req
+            )
+        message = str(ctx.exception)
+        self.assertIn("mock.jinja", message)
+        self.assertIn("line=7", message)
+
+    def test_non_jinja_template_errors_are_raised_as_value_error(self):
+        """TypeError / KeyError from apply_chat_template should map to 400, not 500."""
+        self.tokenizer_manager.tokenizer.chat_template = "mock-template"
+        self.tokenizer_manager.tokenizer.apply_chat_template = Mock(
+            side_effect=KeyError("missing_field")
+        )
+
+        with self.assertRaisesRegex(ValueError, "missing_field"):
+            self.serving_embedding._convert_to_internal_request(
+                self.image_only_multimodal_req
+            )
+
 
 if __name__ == "__main__":
     unittest.main(verbosity=2)