minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

[Fix] Alpha-channel images and tool-result media ordering (port of #36507) (#37320)

Browse Source 
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>
This commit is contained in:
Khoa Pham
2026-09-02 18:23:06 -07:00
committed by GitHub
co-authored by Claude Opus 5 Xinyuan Tong Xinyuan Tong
parent 6e41f1ad29
commit fbf909b460
14 changed files with 772 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/unit/entrypoints/openai/test_serving_chat.py
+182
View File
@@ -39,6 +39,9 @@ from sglang.srt.entrypoints.openai.serving_chat import (
from sglang.srt.environ import envs
from sglang.srt.function_call.kimik3_format import TOOLS_CLOSE, TOOLS_OPEN
from sglang.srt.managers.io_struct import GenerateReqInput
from sglang.srt.parser.jinja_template_utils import (
jinja_template_may_reorder_tool_results,
)
from sglang.srt.parser.template_detection import ReasoningToggleConfig
from sglang.srt.utils import get_or_create_event_loop
from sglang.test.ci.ci_register import register_cpu_ci
@@ -80,6 +83,24 @@ _DSV4_OFFICIAL_ENCODER = (
'DEFAULT_REASONING_EFFORT = "low"\n'
)
_TOOL_RESULT_REORDER_TEMPLATE = """
{%- for assistant in messages
if assistant.role == 'assistant' and assistant.tool_calls -%}
{%- for tool_call in assistant.tool_calls -%}
{%- for result in messages
if result.role == 'tool' and result.tool_call_id == tool_call.id -%}
{%- for part in result.content -%}
{%- if part.type == 'text' -%}
{{- part.text -}}
{%- else -%}
{{- '<' + part.type + '>' -}}
{%- endif -%}
{%- endfor -%}
{%- endfor -%}
{%- endfor -%}
{%- endfor -%}
"""
def _create_dsv4_checkpoint(test_case: unittest.TestCase, source: str) -> str:
model_dir = tempfile.TemporaryDirectory()
@@ -163,6 +184,7 @@ class _MockTemplateManager:
self.completion_template_name: Optional[str] = None
self.reasoning_config = None
self.force_reasoning = False
self.jinja_template_may_reorder_tool_results = False
class ServingChatTestCase(unittest.TestCase):
@@ -192,6 +214,166 @@ class ServingChatTestCase(unittest.TestCase):
self.fastapi_request = Mock(spec=Request)
self.fastapi_request.headers = {}
@staticmethod
def _render_tool_results_in_call_order(messages, **kwargs):
"""Block-level tool_call_id association, like the GLM chat templates."""
del kwargs
rendered = []
index = 0
while index < len(messages):
message = messages[index]
index += 1
tool_calls = message.get("tool_calls") or []
if message.get("role") != "assistant" or not tool_calls:
continue
run = []
while index < len(messages) and messages[index].get("role") == "tool":
run.append(messages[index])
index += 1
by_id = {result.get("tool_call_id"): result for result in run}
for tool_call in tool_calls:
result = by_id.get(tool_call.get("id"))
if result is None:
continue
for part in result.get("content") or []:
if part.get("type") == "text":
rendered.append(part.get("text", ""))
else:
rendered.append(f"<{part.get('type')}>")
return "".join(rendered)
@staticmethod
def _tool_round(call_ids, result_ids, part_type="image_url"):
assistant = {
"role": "assistant",
"content": "",
"tool_calls": [
{"id": call_id, "function": {"name": call_id, "arguments": {}}}
for call_id in call_ids
],
}
part_key = {"image_url": "image_url", "video_url": "video_url"}[part_type]
results = [
{
"role": "tool",
"tool_call_id": result_id,
"content": [{"type": part_type, part_key: {"url": result_id}}],
}
for result_id in result_ids
]
return [assistant] + results
def test_canonicalize_tool_message_order_sorts_media_runs(self):
messages = self._tool_round(
["call-a", "call-b"], ["call-b", "call-a"]
) + self._tool_round(["call-c", "call-d"], ["call-d", "call-c"], "video_url")
canonical = self.chat._canonicalize_tool_message_order(messages)
self.assertEqual(
[
message["tool_call_id"]
for message in canonical
if message.get("role") == "tool"
],
["call-a", "call-b", "call-c", "call-d"],
)
def test_canonicalize_tool_message_order_keeps_unassociable_runs(self):
cases = {
"unknown_id": (["call-a", "call-b"], ["call-b", "call-z"]),
"duplicate_result_id": (["call-a", "call-b"], ["call-b", "call-b"]),
"missing_call_id": ([None, "call-b"], ["call-b", None]),
}
for name, (call_ids, result_ids) in cases.items():
with self.subTest(name=name):
messages = self._tool_round(call_ids, result_ids)
canonical = self.chat._canonicalize_tool_message_order(messages)
self.assertEqual(canonical, messages)
def test_canonicalize_tool_message_order_keeps_text_only_runs(self):
messages = self._tool_round(["call-a", "call-b"], ["call-b", "call-a"])
for message in messages[1:]:
message["content"] = [{"type": "text", "text": "done"}]
canonical = self.chat._canonicalize_tool_message_order(messages)
self.assertEqual(canonical, messages)
def test_jinja_path_recovers_tool_result_images_only_when_template_needs_it(self):
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "openai"
self.template_manager.jinja_template_may_reorder_tool_results = (
jinja_template_may_reorder_tool_results(_TOOL_RESULT_REORDER_TEMPLATE)
)
self.assertTrue(self.template_manager.jinja_template_may_reorder_tool_results)
self.tm.tokenizer.apply_chat_template.side_effect = (
self._render_tool_results_in_call_order
)
request = ChatCompletionRequest(
model="x",
messages=[
{"role": "user", "content": "inspect"},
{
"role": "assistant",
"content": "",
"tool_calls": [
{
"id": "call-a",
"type": "function",
"function": {"name": "a", "arguments": {}},
},
{
"id": "call-b",
"type": "function",
"function": {"name": "b", "arguments": {}},
},
],
},
{
"role": "tool",
"tool_call_id": "call-b",
"content": [{"type": "image_url", "image_url": {"url": "image-b"}}],
},
{
"role": "tool",
"tool_call_id": "call-a",
"content": [{"type": "image_url", "image_url": {"url": "image-a"}}],
},
],
)
result = self.chat._apply_jinja_template(request, None, is_multimodal=True)
self.assertEqual(
[item.url for item in result.image_data], ["image-a", "image-b"]
)
self.assertEqual(self.tm.tokenizer.apply_chat_template.call_count, 1)
rendered_messages = self.tm.tokenizer.apply_chat_template.call_args[0][0]
self.assertEqual(
[m.get("tool_call_id") for m in rendered_messages if "tool_call_id" in m],
["call-a", "call-b"],
)
# An already-ordered request must produce the exact same render input.
ordered_request = ChatCompletionRequest(
model="x",
messages=request.messages[:2] + request.messages[2:][::-1],
)
self.tm.tokenizer.apply_chat_template.reset_mock()
self.chat._apply_jinja_template(ordered_request, None, is_multimodal=True)
self.assertEqual(
rendered_messages, self.tm.tokenizer.apply_chat_template.call_args[0][0]
)
self.template_manager.jinja_template_may_reorder_tool_results = False
self.tm.tokenizer.apply_chat_template.reset_mock()
result = self.chat._apply_jinja_template(request, None, is_multimodal=True)
self.assertEqual(
[item.url for item in result.image_data], ["image-b", "image-a"]
)
self.assertEqual(self.tm.tokenizer.apply_chat_template.call_count, 1)
def test_parsers_follow_the_control_plane_overlay(self):
"""Template detection records the parsers through `override`, so they
answer from the bags; `ServerArgs` keeps the launcher's seed."""
test/registered/unit/entrypoints/openai/test_serving_completions.py
+1
View File
@@ -58,6 +58,7 @@ class _MockTemplateManager:
self.completion_template_name: Optional[str] = (
None # Set to None to avoid template processing
)
self.jinja_template_may_reorder_tool_results = False
class ServingCompletionTestCase(unittest.TestCase):
test/registered/unit/entrypoints/openai/test_serving_embedding.py
+1
View File
@@ -97,6 +97,7 @@ class _MockTemplateManager:
self.chat_template_name = None # None for embeddings usually
self.jinja_template_content_format = "openai"
self.completion_template_name = None
self.jinja_template_may_reorder_tool_results = False
class ServingEmbeddingTestCase(unittest.TestCase):
test/registered/unit/entrypoints/openai/utils.py
+1
View File
@@ -79,6 +79,7 @@ class MockTemplateManager:
self.completion_template_name = None
self.reasoning_config = None
self.force_reasoning = False
self.jinja_template_may_reorder_tool_results = False
def make_serving(*, is_multimodal: bool = False) -> OpenAIServingResponses:
test/registered/unit/multimodal/test_processor_async_call_sites.py
+5
View File
@@ -27,6 +27,11 @@ _MULTIMODAL_ROOT = (
/ "srt"
/ "multimodal"
)
if not _MULTIMODAL_ROOT.is_dir():
raise RuntimeError(
f"multimodal processor tree not found at {_MULTIMODAL_ROOT}; "
"these tests must run from a full source checkout"
)
# The async helper and the sync body live side by side here by design.
_EXEMPT = {"base_processor.py"}
test/registered/unit/parser/test_jinja_template_utils.py
+56 -11
View File
@@ -4,8 +4,10 @@ import unittest
from sglang.srt.parser.jinja_template_utils import (
detect_jinja_template_content_format,
jinja_template_may_reorder_tool_results,
process_content_for_template_format,
)
from sglang.srt.utils import VideoData
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
@@ -16,6 +18,41 @@ register_cpu_ci(est_time=6, suite="base-c-test-cpu")
class TestTemplateContentFormatDetection(CustomTestCase):
"""Test template content format detection functionality."""
def test_detect_tool_result_id_association(self):
attribute_template = """
{% for message in messages %}
{{ message.tool_call_id }}
{% endfor %}
"""
item_template = "{{ messages[0]['tool_call_id'] }}"
get_template = "{{ messages[0].get('tool_call_id') }}"
select_template = (
"{{ messages | selectattr('tool_call_id', 'equalto', 'call-a') | list }}"
)
self.assertTrue(jinja_template_may_reorder_tool_results(attribute_template))
self.assertTrue(jinja_template_may_reorder_tool_results(item_template))
self.assertTrue(jinja_template_may_reorder_tool_results(get_template))
self.assertTrue(jinja_template_may_reorder_tool_results(select_template))
def test_sort_by_tool_call_id_value_is_not_association(self):
# sort/groupby order by the id string value, which message-order
# canonicalization cannot reproduce, so they must not activate it.
sort_template = "{{ messages | sort(attribute='tool_call_id') }}"
groupby_template = "{{ messages | groupby('tool_call_id') }}"
self.assertFalse(jinja_template_may_reorder_tool_results(sort_template))
self.assertFalse(jinja_template_may_reorder_tool_results(groupby_template))
def test_tool_call_id_text_does_not_enable_order_recovery(self):
self.assertFalse(
jinja_template_may_reorder_tool_results(
"{# tool_call_id is mentioned only in a comment #}{{ messages }}"
)
)
self.assertFalse(jinja_template_may_reorder_tool_results("{{{{ invalid"))
self.assertFalse(jinja_template_may_reorder_tool_results(None))
def test_detect_llama4_openai_format(self):
"""Test detection of llama4-style template (should be 'openai' format)."""
llama4_pattern = """
@@ -312,30 +349,38 @@ class TestTemplateContentFormatDetection(CustomTestCase):
self.assertEqual(video_data[0], "http://example.com/v.mp4")
self.assertEqual(result["content"][1], {"type": "video"})
def test_process_content_video_with_max_dynamic_patch(self):
"""Test video_url with max_dynamic_patch stores structured dict."""
def test_process_content_video_structured_fields_become_video_data(self):
"""video_url with mdp/fps-style fields lands in VideoData.preprocess_kwargs."""
msg_dict = {
"role": "user",
"content": [
{
"type": "video_url",
"video_url": {
"url": "http://example.com/v.mp4",
"url": "http://example.com/a.mp4",
"max_dynamic_patch": 4,
},
},
{
"type": "video_url",
"video_url": {
"url": "http://example.com/b.mp4",
"fps": 1.5,
"max_frames": 16,
},
},
],
}
image_data = []
video_data = []
audio_data = []
modalities = []
result = process_content_for_template_format(
msg_dict, "openai", image_data, video_data, audio_data, modalities
process_content_for_template_format(msg_dict, "openai", [], video_data, [], [])
self.assertEqual(
[(item.url, item.preprocess_kwargs) for item in video_data],
[
("http://example.com/a.mp4", {"max_dynamic_patch": 4}),
("http://example.com/b.mp4", {"fps": 1.5, "max_frames": 16}),
],
)
self.assertEqual(len(video_data), 1)
self.assertIsInstance(video_data[0], dict)
self.assertEqual(video_data[0]["max_dynamic_patch"], 4)
self.assertIsInstance(video_data[0], VideoData)
def test_process_content_v32_encoding(self):
"""Test v32 encoding mode flattens text and ignores structured content parts."""