[Fix] Alpha-channel images and tool-result media ordering (port of #36507) (#37320)

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>

test/registered/unit/parser/test_jinja_template_utils.py

@@ -4,8 +4,10 @@ import unittest
 
 from sglang.srt.parser.jinja_template_utils import (
     detect_jinja_template_content_format,
+    jinja_template_may_reorder_tool_results,
     process_content_for_template_format,
 )
+from sglang.srt.utils import VideoData
 from sglang.test.ci.ci_register import register_cpu_ci
 from sglang.test.test_utils import CustomTestCase
 
@@ -16,6 +18,41 @@ register_cpu_ci(est_time=6, suite="base-c-test-cpu")
 class TestTemplateContentFormatDetection(CustomTestCase):
     """Test template content format detection functionality."""
 
+    def test_detect_tool_result_id_association(self):
+        attribute_template = """
+        {% for message in messages %}
+            {{ message.tool_call_id }}
+        {% endfor %}
+        """
+        item_template = "{{ messages[0]['tool_call_id'] }}"
+        get_template = "{{ messages[0].get('tool_call_id') }}"
+        select_template = (
+            "{{ messages | selectattr('tool_call_id', 'equalto', 'call-a') | list }}"
+        )
+
+        self.assertTrue(jinja_template_may_reorder_tool_results(attribute_template))
+        self.assertTrue(jinja_template_may_reorder_tool_results(item_template))
+        self.assertTrue(jinja_template_may_reorder_tool_results(get_template))
+        self.assertTrue(jinja_template_may_reorder_tool_results(select_template))
+
+    def test_sort_by_tool_call_id_value_is_not_association(self):
+        # sort/groupby order by the id string value, which message-order
+        # canonicalization cannot reproduce, so they must not activate it.
+        sort_template = "{{ messages | sort(attribute='tool_call_id') }}"
+        groupby_template = "{{ messages | groupby('tool_call_id') }}"
+
+        self.assertFalse(jinja_template_may_reorder_tool_results(sort_template))
+        self.assertFalse(jinja_template_may_reorder_tool_results(groupby_template))
+
+    def test_tool_call_id_text_does_not_enable_order_recovery(self):
+        self.assertFalse(
+            jinja_template_may_reorder_tool_results(
+                "{# tool_call_id is mentioned only in a comment #}{{ messages }}"
+            )
+        )
+        self.assertFalse(jinja_template_may_reorder_tool_results("{{{{ invalid"))
+        self.assertFalse(jinja_template_may_reorder_tool_results(None))
+
     def test_detect_llama4_openai_format(self):
         """Test detection of llama4-style template (should be 'openai' format)."""
         llama4_pattern = """
@@ -312,30 +349,38 @@ class TestTemplateContentFormatDetection(CustomTestCase):
         self.assertEqual(video_data[0], "http://example.com/v.mp4")
         self.assertEqual(result["content"][1], {"type": "video"})
 
-    def test_process_content_video_with_max_dynamic_patch(self):
-        """Test video_url with max_dynamic_patch stores structured dict."""
+    def test_process_content_video_structured_fields_become_video_data(self):
+        """video_url with mdp/fps-style fields lands in VideoData.preprocess_kwargs."""
         msg_dict = {
             "role": "user",
             "content": [
                 {
                     "type": "video_url",
                     "video_url": {
-                        "url": "http://example.com/v.mp4",
+                        "url": "http://example.com/a.mp4",
                         "max_dynamic_patch": 4,
                     },
                 },
+                {
+                    "type": "video_url",
+                    "video_url": {
+                        "url": "http://example.com/b.mp4",
+                        "fps": 1.5,
+                        "max_frames": 16,
+                    },
+                },
             ],
         }
-        image_data = []
         video_data = []
-        audio_data = []
-        modalities = []
-        result = process_content_for_template_format(
-            msg_dict, "openai", image_data, video_data, audio_data, modalities
+        process_content_for_template_format(msg_dict, "openai", [], video_data, [], [])
+        self.assertEqual(
+            [(item.url, item.preprocess_kwargs) for item in video_data],
+            [
+                ("http://example.com/a.mp4", {"max_dynamic_patch": 4}),
+                ("http://example.com/b.mp4", {"fps": 1.5, "max_frames": 16}),
+            ],
         )
-        self.assertEqual(len(video_data), 1)
-        self.assertIsInstance(video_data[0], dict)
-        self.assertEqual(video_data[0]["max_dynamic_patch"], 4)
+        self.assertIsInstance(video_data[0], VideoData)
 
     def test_process_content_v32_encoding(self):
         """Test v32 encoding mode flattens text and ignores structured content parts."""