minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Fix multimodal /v1/embeddings Jinja chat template handling (#20835)

Browse Source 
Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>
This commit is contained in:
robellliu-dev
2026-04-28 13:05:45 -07:00
committed by GitHub
co-authored by Xinyuan Tong Xinyuan Tong
parent dc1eac4903
commit 914ef7c7f3
2 changed files with 237 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/entrypoints/openai/serving_embedding.py
+78 -4
View File
@@ -2,6 +2,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
import jinja2
from fastapi import Request
from fastapi.responses import ORJSONResponse
@@ -17,6 +18,7 @@ from sglang.srt.entrypoints.openai.serving_base import OpenAIServingBase
from sglang.srt.entrypoints.openai.utils import convert_embeds_to_tensors
from sglang.srt.managers.io_struct import EmbeddingReqInput
from sglang.srt.parser.conversation import generate_embedding_convs
from sglang.srt.parser.jinja_template_utils import process_content_for_template_format
if TYPE_CHECKING:
from sglang.srt.managers.template_manager import TemplateManager
@@ -92,21 +94,31 @@ class OpenAIServingEmbedding(OpenAIServingBase):
images = []
videos = []
for item in prompt:
# Use padding for text if None - this could be improved
texts.append(item.text if item.text is not None else "padding")
texts.append(item.text)
images.append(item.image if item.image is not None else None)
videos.append(item.video if item.video is not None else None)
# Precedence: a SGLang-registered conversation template wins
# over the tokenizer's own HF Jinja template when both exist.
generate_prompts = []
# Check if we have a chat template for multimodal embeddings
if self.template_manager.chat_template_name is not None:
convs = generate_embedding_convs(
texts, images, videos, self.template_manager.chat_template_name
)
for conv in convs:
generate_prompts.append(conv.get_prompt())
elif (
self.tokenizer_manager.tokenizer is not None
and getattr(self.tokenizer_manager.tokenizer, "chat_template", None)
is not None
):
generate_prompts = self._apply_jinja_template_to_embedding_inputs(
texts, images, videos
)
else:
generate_prompts = texts
generate_prompts = [
text if text is not None else "padding" for text in texts
]
if len(generate_prompts) == 1:
prompt_kwargs = {
@@ -163,6 +175,68 @@ class OpenAIServingEmbedding(OpenAIServingBase):
return adapted_request, request
def _apply_jinja_template_to_embedding_inputs(
self,
texts: List[Optional[str]],
images: List[Optional[str]],
videos: List[Optional[str]],
) -> List[str]:
"""Render each multimodal embedding input through the tokenizer's Jinja chat template.
Image/video bytes are threaded to the engine separately via
``EmbeddingReqInput.image_data``/``video_data``; this method only produces
the prompt string. ``text=None`` emits no text chunk (no ``"padding"``
literal). Jinja failures are re-raised as ``ValueError`` so the caller
returns HTTP 400 instead of 500.
"""
prompts: List[str] = []
template_content_format = self.template_manager.jinja_template_content_format
for text, image, video in zip(texts, images, videos):
content_parts = []
if image is not None:
content_parts.append({"type": "image_url", "image_url": {"url": image}})
if video is not None:
content_parts.append({"type": "video_url", "video_url": {"url": video}})
if text is not None:
content_parts.append({"type": "text", "text": text})
msg_dict = {
"role": "user",
"content": content_parts if content_parts else "",
}
# Empty list args: this helper is only used to normalize the content
# shape (e.g. image_url -> image); real payloads ride on the outer
# images/videos lists, not EmbeddingReqInput fields derived here.
processed_msg = process_content_for_template_format(
msg_dict,
template_content_format,
image_data=[],
video_data=[],
audio_data=[],
modalities=[],
)
try:
prompt = self.tokenizer_manager.tokenizer.apply_chat_template(
[processed_msg],
tokenize=False,
add_generation_prompt=True,
)
except jinja2.TemplateError as template_error:
location = getattr(template_error, "lineno", None)
name = getattr(template_error, "name", None)
suffix = ""
if name or location:
suffix = f" (template={name or '<unknown>'}, line={location})"
raise ValueError(f"{template_error}{suffix}") from template_error
except (TypeError, KeyError, AttributeError) as template_error:
raise ValueError(
f"Failed to render chat template for embedding input: {template_error}"
) from template_error
prompts.append(prompt)
return prompts
async def _handle_non_streaming_request(
self,
adapted_request: EmbeddingReqInput,
test/registered/unit/entrypoints/openai/test_serving_embedding.py
+159 -1
View File
@@ -11,6 +11,8 @@ import unittest
import uuid
from unittest.mock import MagicMock, Mock
import jinja2
# Stub out sgl_kernel (and all submodules) before any sglang import so
# the test runs on CPU-only runners without the real CUDA library.
@@ -90,7 +92,7 @@ class _MockTokenizerManager:
class _MockTemplateManager:
def __init__(self):
self.chat_template_name = None # None for embeddings usually
self.jinja_template_content_format = None
self.jinja_template_content_format = "openai"
self.completion_template_name = None
@@ -124,6 +126,22 @@ class ServingEmbeddingTestCase(unittest.TestCase):
],
encoding_format="float",
)
self.image_only_multimodal_req = EmbeddingRequest(
model="test-model",
input=[
MultimodalEmbeddingInput(text=None, image="base64_image_data"),
],
encoding_format="float",
)
self.video_multimodal_req = EmbeddingRequest(
model="test-model",
input=[
MultimodalEmbeddingInput(
text="Describe", image=None, video="base64_video_data"
),
],
encoding_format="float",
)
self.token_ids_req = EmbeddingRequest(
model="test-model",
input=[1, 2, 3, 4, 5],
@@ -180,6 +198,146 @@ class ServingEmbeddingTestCase(unittest.TestCase):
self.assertIsNone(adapted_request.image_data[1])
# self.assertEqual(adapted_request.rid, "test-id")
def test_convert_multimodal_request_with_jinja_chat_template(self):
"""Multimodal embeddings should apply explicit/HF Jinja chat templates."""
self.tokenizer_manager.tokenizer.chat_template = "mock-template"
self.tokenizer_manager.tokenizer.apply_chat_template = Mock(
side_effect=[
"<prompt>Hello<image></prompt>",
"<prompt>World</prompt>",
]
)
adapted_request, _ = self.serving_embedding._convert_to_internal_request(
self.multimodal_req
)
self.assertEqual(
adapted_request.text,
["<prompt>Hello<image></prompt>", "<prompt>World</prompt>"],
)
self.assertEqual(adapted_request.image_data[0], "base64_image_data")
self.assertIsNone(adapted_request.image_data[1])
self.assertEqual(
self.tokenizer_manager.tokenizer.apply_chat_template.call_count, 2
)
first_call = (
self.tokenizer_manager.tokenizer.apply_chat_template.call_args_list[0]
)
first_messages = first_call.args[0]
self.assertEqual(first_messages[0]["role"], "user")
self.assertEqual(first_messages[0]["content"][0]["type"], "image")
self.assertEqual(first_messages[0]["content"][1]["type"], "text")
self.assertEqual(first_messages[0]["content"][1]["text"], "Hello")
self.assertEqual(first_call.kwargs["tokenize"], False)
self.assertEqual(first_call.kwargs["add_generation_prompt"], True)
second_call = (
self.tokenizer_manager.tokenizer.apply_chat_template.call_args_list[1]
)
second_messages = second_call.args[0]
self.assertEqual(len(second_messages[0]["content"]), 1)
self.assertEqual(second_messages[0]["content"][0]["type"], "text")
self.assertEqual(second_messages[0]["content"][0]["text"], "World")
def test_convert_image_only_multimodal_request_with_jinja_chat_template(self):
"""Image-only requests should not inject literal padding into Jinja prompts."""
self.tokenizer_manager.tokenizer.chat_template = "mock-template"
self.tokenizer_manager.tokenizer.apply_chat_template = Mock(
return_value="<prompt><image></prompt>"
)
adapted_request, _ = self.serving_embedding._convert_to_internal_request(
self.image_only_multimodal_req
)
self.assertEqual(adapted_request.text, "<prompt><image></prompt>")
first_call = self.tokenizer_manager.tokenizer.apply_chat_template.call_args
first_messages = first_call.args[0]
self.assertEqual(first_messages[0]["role"], "user")
self.assertEqual(len(first_messages[0]["content"]), 1)
self.assertEqual(first_messages[0]["content"][0]["type"], "image")
def test_convert_video_multimodal_request_with_jinja_chat_template(self):
"""Video inputs should land in video_data and flow through the Jinja branch."""
self.tokenizer_manager.tokenizer.chat_template = "mock-template"
self.tokenizer_manager.tokenizer.apply_chat_template = Mock(
return_value="<prompt>Describe<video></prompt>"
)
adapted_request, _ = self.serving_embedding._convert_to_internal_request(
self.video_multimodal_req
)
self.assertEqual(adapted_request.text, "<prompt>Describe<video></prompt>")
self.assertEqual(adapted_request.video_data, "base64_video_data")
self.assertIsNone(adapted_request.image_data)
first_messages = (
self.tokenizer_manager.tokenizer.apply_chat_template.call_args.args[0]
)
content = first_messages[0]["content"]
self.assertEqual([c["type"] for c in content], ["video", "text"])
def test_multimodal_request_falls_back_when_no_chat_template(self):
"""Without any chat template the raw-text fallback must run without raising."""
self.tokenizer_manager.tokenizer.chat_template = None
adapted_request, _ = self.serving_embedding._convert_to_internal_request(
self.image_only_multimodal_req
)
# text=None on an image-only input falls back to the "padding" literal.
self.assertEqual(adapted_request.text, "padding")
self.assertEqual(adapted_request.image_data, "base64_image_data")
def test_multimodal_request_with_no_tokenizer_uses_fallback(self):
"""Missing tokenizer should not crash the Jinja branch check."""
self.tokenizer_manager.tokenizer = None
adapted_request, _ = self.serving_embedding._convert_to_internal_request(
self.multimodal_req
)
self.assertEqual(adapted_request.text, ["Hello", "World"])
def test_jinja_template_errors_are_raised_as_value_error(self):
"""Template failures should be converted to ValueError for a 400 response."""
self.tokenizer_manager.tokenizer.chat_template = "mock-template"
self.tokenizer_manager.tokenizer.apply_chat_template = Mock(
side_effect=jinja2.TemplateError("bad template")
)
with self.assertRaisesRegex(ValueError, "bad template"):
self.serving_embedding._convert_to_internal_request(
self.image_only_multimodal_req
)
def test_jinja_template_syntax_error_includes_location(self):
"""TemplateSyntaxError should surface template name and line number."""
err = jinja2.TemplateSyntaxError("unexpected end", lineno=7, name="mock.jinja")
self.tokenizer_manager.tokenizer.chat_template = "mock-template"
self.tokenizer_manager.tokenizer.apply_chat_template = Mock(side_effect=err)
with self.assertRaises(ValueError) as ctx:
self.serving_embedding._convert_to_internal_request(
self.image_only_multimodal_req
)
message = str(ctx.exception)
self.assertIn("mock.jinja", message)
self.assertIn("line=7", message)
def test_non_jinja_template_errors_are_raised_as_value_error(self):
"""TypeError / KeyError from apply_chat_template should map to 400, not 500."""
self.tokenizer_manager.tokenizer.chat_template = "mock-template"
self.tokenizer_manager.tokenizer.apply_chat_template = Mock(
side_effect=KeyError("missing_field")
)
with self.assertRaisesRegex(ValueError, "missing_field"):
self.serving_embedding._convert_to_internal_request(
self.image_only_multimodal_req
)
if __name__ == "__main__":
unittest.main(verbosity=2)