diff --git a/python/sglang/srt/entrypoints/openai/serving_embedding.py b/python/sglang/srt/entrypoints/openai/serving_embedding.py index 5f2f3658c..10f66cfd3 100644 --- a/python/sglang/srt/entrypoints/openai/serving_embedding.py +++ b/python/sglang/srt/entrypoints/openai/serving_embedding.py @@ -2,6 +2,7 @@ from __future__ import annotations from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +import jinja2 from fastapi import Request from fastapi.responses import ORJSONResponse @@ -17,6 +18,7 @@ from sglang.srt.entrypoints.openai.serving_base import OpenAIServingBase from sglang.srt.entrypoints.openai.utils import convert_embeds_to_tensors from sglang.srt.managers.io_struct import EmbeddingReqInput from sglang.srt.parser.conversation import generate_embedding_convs +from sglang.srt.parser.jinja_template_utils import process_content_for_template_format if TYPE_CHECKING: from sglang.srt.managers.template_manager import TemplateManager @@ -92,21 +94,31 @@ class OpenAIServingEmbedding(OpenAIServingBase): images = [] videos = [] for item in prompt: - # Use padding for text if None - this could be improved - texts.append(item.text if item.text is not None else "padding") + texts.append(item.text) images.append(item.image if item.image is not None else None) videos.append(item.video if item.video is not None else None) + # Precedence: a SGLang-registered conversation template wins + # over the tokenizer's own HF Jinja template when both exist. generate_prompts = [] - # Check if we have a chat template for multimodal embeddings if self.template_manager.chat_template_name is not None: convs = generate_embedding_convs( texts, images, videos, self.template_manager.chat_template_name ) for conv in convs: generate_prompts.append(conv.get_prompt()) + elif ( + self.tokenizer_manager.tokenizer is not None + and getattr(self.tokenizer_manager.tokenizer, "chat_template", None) + is not None + ): + generate_prompts = self._apply_jinja_template_to_embedding_inputs( + texts, images, videos + ) else: - generate_prompts = texts + generate_prompts = [ + text if text is not None else "padding" for text in texts + ] if len(generate_prompts) == 1: prompt_kwargs = { @@ -163,6 +175,68 @@ class OpenAIServingEmbedding(OpenAIServingBase): return adapted_request, request + def _apply_jinja_template_to_embedding_inputs( + self, + texts: List[Optional[str]], + images: List[Optional[str]], + videos: List[Optional[str]], + ) -> List[str]: + """Render each multimodal embedding input through the tokenizer's Jinja chat template. + + Image/video bytes are threaded to the engine separately via + ``EmbeddingReqInput.image_data``/``video_data``; this method only produces + the prompt string. ``text=None`` emits no text chunk (no ``"padding"`` + literal). Jinja failures are re-raised as ``ValueError`` so the caller + returns HTTP 400 instead of 500. + """ + prompts: List[str] = [] + template_content_format = self.template_manager.jinja_template_content_format + + for text, image, video in zip(texts, images, videos): + content_parts = [] + if image is not None: + content_parts.append({"type": "image_url", "image_url": {"url": image}}) + if video is not None: + content_parts.append({"type": "video_url", "video_url": {"url": video}}) + if text is not None: + content_parts.append({"type": "text", "text": text}) + + msg_dict = { + "role": "user", + "content": content_parts if content_parts else "", + } + # Empty list args: this helper is only used to normalize the content + # shape (e.g. image_url -> image); real payloads ride on the outer + # images/videos lists, not EmbeddingReqInput fields derived here. + processed_msg = process_content_for_template_format( + msg_dict, + template_content_format, + image_data=[], + video_data=[], + audio_data=[], + modalities=[], + ) + try: + prompt = self.tokenizer_manager.tokenizer.apply_chat_template( + [processed_msg], + tokenize=False, + add_generation_prompt=True, + ) + except jinja2.TemplateError as template_error: + location = getattr(template_error, "lineno", None) + name = getattr(template_error, "name", None) + suffix = "" + if name or location: + suffix = f" (template={name or ''}, line={location})" + raise ValueError(f"{template_error}{suffix}") from template_error + except (TypeError, KeyError, AttributeError) as template_error: + raise ValueError( + f"Failed to render chat template for embedding input: {template_error}" + ) from template_error + prompts.append(prompt) + + return prompts + async def _handle_non_streaming_request( self, adapted_request: EmbeddingReqInput, diff --git a/test/registered/unit/entrypoints/openai/test_serving_embedding.py b/test/registered/unit/entrypoints/openai/test_serving_embedding.py index abbec63ec..9b8704e32 100644 --- a/test/registered/unit/entrypoints/openai/test_serving_embedding.py +++ b/test/registered/unit/entrypoints/openai/test_serving_embedding.py @@ -11,6 +11,8 @@ import unittest import uuid from unittest.mock import MagicMock, Mock +import jinja2 + # Stub out sgl_kernel (and all submodules) before any sglang import so # the test runs on CPU-only runners without the real CUDA library. @@ -90,7 +92,7 @@ class _MockTokenizerManager: class _MockTemplateManager: def __init__(self): self.chat_template_name = None # None for embeddings usually - self.jinja_template_content_format = None + self.jinja_template_content_format = "openai" self.completion_template_name = None @@ -124,6 +126,22 @@ class ServingEmbeddingTestCase(unittest.TestCase): ], encoding_format="float", ) + self.image_only_multimodal_req = EmbeddingRequest( + model="test-model", + input=[ + MultimodalEmbeddingInput(text=None, image="base64_image_data"), + ], + encoding_format="float", + ) + self.video_multimodal_req = EmbeddingRequest( + model="test-model", + input=[ + MultimodalEmbeddingInput( + text="Describe", image=None, video="base64_video_data" + ), + ], + encoding_format="float", + ) self.token_ids_req = EmbeddingRequest( model="test-model", input=[1, 2, 3, 4, 5], @@ -180,6 +198,146 @@ class ServingEmbeddingTestCase(unittest.TestCase): self.assertIsNone(adapted_request.image_data[1]) # self.assertEqual(adapted_request.rid, "test-id") + def test_convert_multimodal_request_with_jinja_chat_template(self): + """Multimodal embeddings should apply explicit/HF Jinja chat templates.""" + self.tokenizer_manager.tokenizer.chat_template = "mock-template" + self.tokenizer_manager.tokenizer.apply_chat_template = Mock( + side_effect=[ + "Hello", + "World", + ] + ) + + adapted_request, _ = self.serving_embedding._convert_to_internal_request( + self.multimodal_req + ) + + self.assertEqual( + adapted_request.text, + ["Hello", "World"], + ) + self.assertEqual(adapted_request.image_data[0], "base64_image_data") + self.assertIsNone(adapted_request.image_data[1]) + self.assertEqual( + self.tokenizer_manager.tokenizer.apply_chat_template.call_count, 2 + ) + first_call = ( + self.tokenizer_manager.tokenizer.apply_chat_template.call_args_list[0] + ) + first_messages = first_call.args[0] + self.assertEqual(first_messages[0]["role"], "user") + self.assertEqual(first_messages[0]["content"][0]["type"], "image") + self.assertEqual(first_messages[0]["content"][1]["type"], "text") + self.assertEqual(first_messages[0]["content"][1]["text"], "Hello") + self.assertEqual(first_call.kwargs["tokenize"], False) + self.assertEqual(first_call.kwargs["add_generation_prompt"], True) + + second_call = ( + self.tokenizer_manager.tokenizer.apply_chat_template.call_args_list[1] + ) + second_messages = second_call.args[0] + self.assertEqual(len(second_messages[0]["content"]), 1) + self.assertEqual(second_messages[0]["content"][0]["type"], "text") + self.assertEqual(second_messages[0]["content"][0]["text"], "World") + + def test_convert_image_only_multimodal_request_with_jinja_chat_template(self): + """Image-only requests should not inject literal padding into Jinja prompts.""" + self.tokenizer_manager.tokenizer.chat_template = "mock-template" + self.tokenizer_manager.tokenizer.apply_chat_template = Mock( + return_value="" + ) + + adapted_request, _ = self.serving_embedding._convert_to_internal_request( + self.image_only_multimodal_req + ) + + self.assertEqual(adapted_request.text, "") + first_call = self.tokenizer_manager.tokenizer.apply_chat_template.call_args + first_messages = first_call.args[0] + self.assertEqual(first_messages[0]["role"], "user") + self.assertEqual(len(first_messages[0]["content"]), 1) + self.assertEqual(first_messages[0]["content"][0]["type"], "image") + + def test_convert_video_multimodal_request_with_jinja_chat_template(self): + """Video inputs should land in video_data and flow through the Jinja branch.""" + self.tokenizer_manager.tokenizer.chat_template = "mock-template" + self.tokenizer_manager.tokenizer.apply_chat_template = Mock( + return_value="Describe" + ) + + adapted_request, _ = self.serving_embedding._convert_to_internal_request( + self.video_multimodal_req + ) + + self.assertEqual(adapted_request.text, "Describe") + self.assertEqual(adapted_request.video_data, "base64_video_data") + self.assertIsNone(adapted_request.image_data) + first_messages = ( + self.tokenizer_manager.tokenizer.apply_chat_template.call_args.args[0] + ) + content = first_messages[0]["content"] + self.assertEqual([c["type"] for c in content], ["video", "text"]) + + def test_multimodal_request_falls_back_when_no_chat_template(self): + """Without any chat template the raw-text fallback must run without raising.""" + self.tokenizer_manager.tokenizer.chat_template = None + + adapted_request, _ = self.serving_embedding._convert_to_internal_request( + self.image_only_multimodal_req + ) + + # text=None on an image-only input falls back to the "padding" literal. + self.assertEqual(adapted_request.text, "padding") + self.assertEqual(adapted_request.image_data, "base64_image_data") + + def test_multimodal_request_with_no_tokenizer_uses_fallback(self): + """Missing tokenizer should not crash the Jinja branch check.""" + self.tokenizer_manager.tokenizer = None + + adapted_request, _ = self.serving_embedding._convert_to_internal_request( + self.multimodal_req + ) + + self.assertEqual(adapted_request.text, ["Hello", "World"]) + + def test_jinja_template_errors_are_raised_as_value_error(self): + """Template failures should be converted to ValueError for a 400 response.""" + self.tokenizer_manager.tokenizer.chat_template = "mock-template" + self.tokenizer_manager.tokenizer.apply_chat_template = Mock( + side_effect=jinja2.TemplateError("bad template") + ) + + with self.assertRaisesRegex(ValueError, "bad template"): + self.serving_embedding._convert_to_internal_request( + self.image_only_multimodal_req + ) + + def test_jinja_template_syntax_error_includes_location(self): + """TemplateSyntaxError should surface template name and line number.""" + err = jinja2.TemplateSyntaxError("unexpected end", lineno=7, name="mock.jinja") + self.tokenizer_manager.tokenizer.chat_template = "mock-template" + self.tokenizer_manager.tokenizer.apply_chat_template = Mock(side_effect=err) + + with self.assertRaises(ValueError) as ctx: + self.serving_embedding._convert_to_internal_request( + self.image_only_multimodal_req + ) + message = str(ctx.exception) + self.assertIn("mock.jinja", message) + self.assertIn("line=7", message) + + def test_non_jinja_template_errors_are_raised_as_value_error(self): + """TypeError / KeyError from apply_chat_template should map to 400, not 500.""" + self.tokenizer_manager.tokenizer.chat_template = "mock-template" + self.tokenizer_manager.tokenizer.apply_chat_template = Mock( + side_effect=KeyError("missing_field") + ) + + with self.assertRaisesRegex(ValueError, "missing_field"): + self.serving_embedding._convert_to_internal_request( + self.image_only_multimodal_req + ) + if __name__ == "__main__": unittest.main(verbosity=2)