[Fix] Alpha-channel images and tool-result media ordering (port of #36507) (#37320)

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>

python/sglang/srt/entrypoints/openai/protocol.py

@@ -566,6 +566,10 @@ class ChatCompletionMessageContentVideoURL(BaseModel):
     url: str
     max_dynamic_patch: Optional[int] = None
     min_dynamic_patch: Optional[int] = None
+    fps: Optional[float] = None
+    max_frames: Optional[int] = None
+    max_tokens_per_frame: Optional[int] = None
+    max_image_tokens: Optional[int] = None
 
 
 class ChatCompletionMessageContentAudioURL(BaseModel):

python/sglang/srt/entrypoints/openai/serving_chat.py

@@ -89,7 +89,10 @@ from sglang.srt.function_call.utils import (
 )
 from sglang.srt.managers.io_struct import GenerateReqInput
 from sglang.srt.parser.conversation import generate_chat_conv
-from sglang.srt.parser.jinja_template_utils import process_content_for_template_format
+from sglang.srt.parser.jinja_template_utils import (
+    MEDIA_URL_PART_TYPES,
+    process_content_for_template_format,
+)
 from sglang.srt.parser.reasoning_parser import ReasoningParser
 from sglang.srt.utils.weight_versions import build_endpoint_weight_version_metadata
 
@@ -717,6 +720,66 @@ class OpenAIServingChat(OpenAIServingBase):
             prompt_tokens = max(0, prompt_tokens - self._KIMI_K3_GENERATION_STUB_TOKENS)
         return prompt_tokens
 
+    @staticmethod
+    def _sort_tool_message_run(
+        run: List[Dict[str, Any]], tool_calls: List[Dict[str, Any]]
+    ) -> List[Dict[str, Any]]:
+        """Order a tool-message run by tool_call position.
+
+        Templates that associate results by tool_call_id render the run in
+        tool_calls order; sorting the run upfront keeps extraction order and
+        placeholder order the same. Runs the template itself would refuse to
+        associate (missing/duplicate/unknown ids) are left untouched, as are
+        text-only runs, whose order text-only templates may rely on.
+        """
+        if len(run) < 2:
+            return run
+        call_ids = [tc.get("id") for tc in tool_calls]
+        if any(call_id is None for call_id in call_ids) or len(set(call_ids)) != len(
+            call_ids
+        ):
+            return run
+        result_ids = [message.get("tool_call_id") for message in run]
+        if any(result_id not in call_ids for result_id in result_ids) or len(
+            set(result_ids)
+        ) != len(result_ids):
+            return run
+        has_media = any(
+            isinstance(message.get("content"), list)
+            and any(
+                isinstance(part, dict) and part.get("type") in MEDIA_URL_PART_TYPES
+                for part in message["content"]
+            )
+            for message in run
+        )
+        if not has_media:
+            return run
+        position = {call_id: index for index, call_id in enumerate(call_ids)}
+        return sorted(run, key=lambda message: position[message["tool_call_id"]])
+
+    @classmethod
+    def _canonicalize_tool_message_order(
+        cls, messages: List[Dict[str, Any]]
+    ) -> List[Dict[str, Any]]:
+        canonical = []
+        index = 0
+        while index < len(messages):
+            message = messages[index]
+            canonical.append(message)
+            index += 1
+            tool_calls = message.get("tool_calls") or []
+            if message.get("role") != "assistant" or not tool_calls:
+                continue
+            run = []
+            while index < len(messages) and messages[index].get("role") in (
+                "tool",
+                "function",
+            ):
+                run.append(messages[index])
+                index += 1
+            canonical.extend(cls._sort_tool_message_run(run, tool_calls))
+        return canonical
+
     async def _generate_stream_content(
         self,
         content: Dict[str, Any],
@@ -1329,6 +1392,8 @@ class OpenAIServingChat(OpenAIServingBase):
                     prompt_ids, assistant_prefix
                 )
         else:
+            if self.template_manager.jinja_template_may_reorder_tool_results:
+                messages = self._canonicalize_tool_message_order(messages)
             for msg_dict in copy.deepcopy(messages):
                 if msg_dict.get("content") is None:
                     msg_dict["content"] = ""

python/sglang/srt/multimodal/processors/base_processor.py

@@ -57,6 +57,7 @@ from sglang.srt.utils import (
     load_image,
     load_video,
     logger,
+    smart_to_rgb,
 )
 
 _is_cpu = is_cpu()
@@ -210,6 +211,8 @@ def _tokenizer_of(processor):
 class BaseMultimodalProcessor(ABC):
     models = []
     gpu_image_decode = True  # Enable GPU decoding by default
+    smart_rgb_conversion = False
+    video_preprocessing_device = None
     prefer_tokenized_input = False
     precompute_hash_before_cpu_transfer = False
     # Set by processors that already build input_ids from the request's own
@@ -811,6 +814,10 @@ class BaseMultimodalProcessor(ABC):
             if processor_device is not None:
                 kwargs["device"] = processor_device
 
+        # Long-video preprocessing stays on CPU to avoid competing with scheduler GPU pools.
+        if videos and self.video_preprocessing_device is not None:
+            kwargs["device"] = self.video_preprocessing_device
+
         # Avoid double BOS when the chat template already wrote one.
         if self._tokenizer_auto_adds_specials and isinstance(input_text, str):
             bos = getattr(tokenizer, "bos_token", None)
@@ -895,8 +902,11 @@ class BaseMultimodalProcessor(ABC):
                 img, _ = load_image(data, cls.gpu_image_decode)
                 if isinstance(img, torch.Tensor):
                     return img  # JPEG already decoded on GPU by nvJPEG
-                if discard_alpha_channel and img.mode != "RGB":
-                    return img.convert("RGB")
+                if discard_alpha_channel:
+                    if cls.smart_rgb_conversion:
+                        return smart_to_rgb(img)
+                    if img.mode != "RGB":
+                        return img.convert("RGB")
                 return img
             elif modality == Modality.VIDEO:
                 return load_video(data, frame_count_limit)

python/sglang/srt/multimodal/processors/glm4v.py

@@ -1,5 +1,10 @@
+import asyncio
+import math
 from typing import List, Union
 
+import numpy as np
+import torch
+
 from sglang.srt.layers.rotary_embedding import MRotaryEmbedding
 from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput
 from sglang.srt.models.glm4v import Glm4vForConditionalGeneration
@@ -10,19 +15,248 @@ from sglang.srt.multimodal.processors.base_processor import (
 from sglang.srt.multimodal.processors.base_processor import (
     MultimodalSpecialTokens,
 )
+from sglang.srt.utils import GLM_MEDIA_CONFIG_KEYS
+from sglang.srt.utils.video_decoder import VideoDecoderWrapper
 
 try:
     from sglang.srt.models.glm_ocr import GlmOcrForConditionalGeneration
 except ImportError:
     GlmOcrForConditionalGeneration = None
 
+try:
+    from sglang.srt.models.glm5_next import Glm5NextForConditionalGeneration
+except ImportError:
+    Glm5NextForConditionalGeneration = None
+
+GLM_VIDEO_DEFAULT_FPS = 2.0
+GLM_VIDEO_DEFAULT_MAX_FRAMES = 2048
+GLM_VIDEO_PATCH_SIZE = 14
+GLM_VIDEO_MERGE_SIZE = 2
+GLM_VIDEO_PATCH_EXPAND_FACTOR = 4
+
+
+def _glm_video_metadata(total_num_frames, fps, duration, frames_indices):
+    return {
+        "total_num_frames": int(total_num_frames),
+        "fps": float(fps),
+        "duration": float(duration),
+        "video_backend": "sglang",
+        "frames_indices": list(frames_indices),
+    }
+
+
+def _glm_item_config(item):
+    config = dict(getattr(item, "preprocess_kwargs", None) or {})
+    if isinstance(item, dict):
+        config.update(item.get("preprocess_kwargs") or {})
+        config.update(
+            {
+                key: item[key]
+                for key in GLM_MEDIA_CONFIG_KEYS
+                if item.get(key) is not None
+            }
+        )
+    return {
+        key: config[key] for key in GLM_MEDIA_CONFIG_KEYS if config.get(key) is not None
+    }
+
+
+def split_glm_video_items(mm_data):
+    if mm_data is None:
+        return None, []
+    items = mm_data if isinstance(mm_data, (list, tuple)) else [mm_data]
+    urls, configs = [], []
+    for item in items:
+        if isinstance(item, dict) and "format" not in item and "url" in item:
+            urls.append(item["url"])
+        elif hasattr(item, "url") and hasattr(item, "preprocess_kwargs"):
+            urls.append(item.url)
+        else:
+            urls.append(item)
+        configs.append(_glm_item_config(item))
+    return urls, configs
+
+
+def glm_processor_video_config(processor):
+    if processor is None:
+        return {}
+    return {
+        key: value
+        for key in GLM_MEDIA_CONFIG_KEYS
+        if (value := getattr(processor, key, None)) is not None
+    }
+
+
+def _merge_glm_video_configs(default_config, item_configs):
+    defaults = dict(default_config or {})
+    return [{**defaults, **dict(config or {})} for config in item_configs]
+
+
+def _hf_sample_frame_indices(
+    video_processor, total_frames, fps, duration, video_config
+):
+    """Sample indices with the model's own HF processor so behavior tracks the pinned transformers."""
+    if video_config.get("max_frames") is not None:
+        return None
+    sample_frames = getattr(video_processor, "sample_frames", None)
+    if sample_frames is None:
+        return None
+    try:
+        from transformers.video_utils import VideoMetadata
+    except ImportError:
+        return None
+    metadata = VideoMetadata(
+        total_num_frames=int(total_frames),
+        fps=float(fps),
+        duration=float(duration),
+    )
+    indices = sample_frames(metadata, fps=video_config.get("fps"))
+    return [int(index) for index in indices]
+
+
+def glm_sample_frame_indices(
+    total_frames,
+    fps,
+    duration,
+    *,
+    target_fps=None,
+    max_frame_count=None,
+):
+    """Fallback sampler for processors without sample_frames or when max_frames is requested."""
+    if total_frames <= 0:
+        return []
+    target_fps = GLM_VIDEO_DEFAULT_FPS if target_fps is None else float(target_fps)
+    max_frame_count = (
+        GLM_VIDEO_DEFAULT_MAX_FRAMES
+        if max_frame_count is None
+        else int(max_frame_count)
+    )
+    if target_fps <= 0 or max_frame_count <= 0:
+        return []
+
+    max_frame_idx = total_frames - 1
+    if not duration:
+        duration = round(max_frame_idx / fps) + 1 if fps else 0
+    extract_t = min(int(duration * target_fps), int(max_frame_count))
+    extract_t = max(1, extract_t)
+
+    if fps:
+        duration_per_frame = 1 / fps
+        timestamps = [index * duration_per_frame for index in range(total_frames)]
+        max_second = int(duration)
+        indices = []
+        current_second = 0.0
+        interval = 1 / target_fps
+        for frame_index, timestamp in enumerate(timestamps):
+            if timestamp >= current_second:
+                current_second += interval
+                indices.append(frame_index)
+                if current_second >= max_second:
+                    break
+    else:
+        indices = []
+
+    if len(indices) < extract_t:
+        start = indices[0] if indices else 0
+        end = indices[-1] if indices else max(total_frames - 1, 0)
+        indices = np.linspace(start, end, extract_t, dtype=int).tolist()
+    elif len(indices) > extract_t:
+        indices = np.linspace(0, total_frames - 1, extract_t, dtype=int).tolist()
+
+    seen = set()
+    unique_indices = []
+    for index in indices:
+        index = int(index)
+        if index not in seen:
+            seen.add(index)
+            unique_indices.append(index)
+    if len(unique_indices) & 1:
+        unique_indices.append(unique_indices[-1])
+    return unique_indices
+
+
+def _resize_frames_to_max_tokens(frames, max_tokens_per_frame):
+    import torchvision.transforms.functional as TF
+
+    if not isinstance(frames, torch.Tensor):
+        frames = torch.from_numpy(np.asarray(frames))
+    nchw = frames.permute(0, 3, 1, 2)
+    _, _, height, width = nchw.shape
+    pixels_per_token = (GLM_VIDEO_PATCH_SIZE * GLM_VIDEO_MERGE_SIZE) ** 2
+    factor = GLM_VIDEO_PATCH_SIZE * GLM_VIDEO_MERGE_SIZE * GLM_VIDEO_PATCH_EXPAND_FACTOR
+    max_pixels = max(
+        int(max_tokens_per_frame) * pixels_per_token,
+        factor * factor,
+    )
+    resized_height = max(factor, round(height / factor) * factor)
+    resized_width = max(factor, round(width / factor) * factor)
+    if resized_height * resized_width > max_pixels:
+        scale = math.sqrt((height * width) / max_pixels)
+        resized_height = max(factor, math.floor(height / scale / factor) * factor)
+        resized_width = max(factor, math.floor(width / scale / factor) * factor)
+    if (resized_height, resized_width) != (height, width):
+        nchw = TF.resize(
+            nchw,
+            [resized_height, resized_width],
+            interpolation=TF.InterpolationMode.BICUBIC,
+            antialias=True,
+        )
+    return nchw.permute(0, 2, 3, 1).contiguous()
+
+
+def glm_decode_frames_at(vr, indices, video_config=None):
+    indices = list(indices)
+    if not indices:
+        return None
+    video_config = video_config or {}
+    if hasattr(vr, "get_frames_as_tensor"):
+        frames = vr.get_frames_as_tensor(indices)
+    else:
+        frames = vr.get_frames_at(indices)
+    max_tokens_per_frame = video_config.get("max_tokens_per_frame")
+    if max_tokens_per_frame is not None:
+        frames = _resize_frames_to_max_tokens(frames, max_tokens_per_frame)
+    return frames
+
+
+def glm_sample_and_decode_sync(vr, video_config=None, video_processor=None):
+    video_config = video_config or {}
+    fps = vr.avg_fps
+    if not fps or fps <= 0:
+        raise ValueError(f"Cannot determine video fps (avg_fps={fps!r})")
+    duration = len(vr) / fps
+    indices = _hf_sample_frame_indices(
+        video_processor, len(vr), fps, duration, video_config
+    )
+    if indices is None:
+        indices = glm_sample_frame_indices(
+            len(vr),
+            fps,
+            duration,
+            target_fps=video_config.get("fps"),
+            max_frame_count=video_config.get("max_frames"),
+        )
+    if not indices:
+        raise ValueError("Video frame sampling produced no frames")
+    frames = glm_decode_frames_at(vr, indices, video_config)
+    return frames, _glm_video_metadata(len(vr), fps, duration, indices)
+
+
+def _passthrough_video_metadata(video, video_config):
+    num_frames = video.shape[0] if hasattr(video, "shape") else len(video)
+    fps = float(video_config.get("fps") or GLM_VIDEO_DEFAULT_FPS)
+    return _glm_video_metadata(num_frames, fps, num_frames / fps, range(num_frames))
+
 
 class Glm4vImageProcessor(SGLangBaseProcessor):
+    smart_rgb_conversion = True
+    video_preprocessing_device = "cpu"
     models = [
         m
         for m in [
             Glm4vForConditionalGeneration,
             Glm4vMoeForConditionalGeneration,
+            Glm5NextForConditionalGeneration,
             GlmOcrForConditionalGeneration,
         ]
         if m is not None
@@ -46,6 +280,8 @@ class Glm4vImageProcessor(SGLangBaseProcessor):
         self.IMAGE_END_TOKEN_ID = hf_config.image_end_token_id
         self.VIDEO_START_TOKEN_ID = hf_config.video_start_token_id
         self.VIDEO_END_TOKEN_ID = hf_config.video_end_token_id
+        self.IM_START_TOKEN_ID = self.IMAGE_START_TOKEN_ID
+        self.IM_END_TOKEN_ID = self.IMAGE_END_TOKEN_ID
 
         # Vision config
         self.IMAGE_FACTOR = 28
@@ -90,17 +326,71 @@ class Glm4vImageProcessor(SGLangBaseProcessor):
         *args,
         **kwargs,
     ):
+        # Bare base64 video must use SGLang's decoder because HF treats it as a path-like string.
+        video_urls, video_configs = split_glm_video_items(request_obj.video_data)
+        video_processor = getattr(self._processor, "video_processor", None)
+        default_video_config = glm_processor_video_config(video_processor)
+        default_video_config.update(self.video_config)
+        video_configs = _merge_glm_video_configs(default_video_config, video_configs)
+
         base_output = await self.load_mm_data(
             prompt=input_text,
             image_data=image_data,
-            video_data=request_obj.video_data,
+            video_data=video_urls,
             multimodal_tokens=self.mm_tokens,
         )
 
-        if base_output.videos:
-            base_output.videos = request_obj.video_data
+        video_metadata = None
+        if base_output.videos and not isinstance(base_output.videos[0], dict):
+            loop = asyncio.get_running_loop()
+            decode_tasks = []
+            for index, video in enumerate(base_output.videos):
+                video_config = (
+                    video_configs[index] if index < len(video_configs) else {}
+                )
+                if isinstance(video, VideoDecoderWrapper):
+                    decode_tasks.append(
+                        loop.run_in_executor(
+                            self.io_executor,
+                            glm_sample_and_decode_sync,
+                            video,
+                            video_config,
+                            video_processor,
+                        )
+                    )
+                else:
+                    decode_tasks.append(
+                        asyncio.sleep(
+                            0,
+                            result=(
+                                video,
+                                _passthrough_video_metadata(video, video_config),
+                            ),
+                        )
+                    )
+
+            try:
+                videos_processed = await asyncio.gather(*decode_tasks)
+            finally:
+                for video in base_output.videos:
+                    close = getattr(video, "close", None)
+                    if callable(close):
+                        close()
+            base_output.videos, video_metadata = map(list, zip(*videos_processed))
+
+        combine_kwargs = {}
+        if video_metadata is not None:
+            # Skip HF resampling because these frames already carry their original indices.
+            combine_kwargs["video_metadata"] = video_metadata
+            combine_kwargs["do_sample_frames"] = False
+            combine_kwargs["processor_video_config"] = {
+                key: value
+                for key, value in self.video_config.items()
+                if key not in {"fps", "max_frames", "max_tokens_per_frame"}
+            }
+
         mm_items, input_ids, ret = await self.process_and_combine_mm_data_async(
-            base_output, self.mm_tokens
+            base_output, self.mm_tokens, **combine_kwargs
         )
 
         input_ids = input_ids.flatten()

python/sglang/srt/parser/conversation.py

@@ -35,7 +35,12 @@ from typing import Callable, Dict, List, Optional, Tuple, Union
 from typing_extensions import Literal
 
 from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest
-from sglang.srt.utils import ImageData, VideoData, read_system_prompt_from_file
+from sglang.srt.utils import (
+    GLM_MEDIA_CONFIG_KEYS,
+    ImageData,
+    VideoData,
+    read_system_prompt_from_file,
+)
 
 
 class SeparatorStyle(IntEnum):
@@ -681,7 +686,16 @@ def generate_chat_conv(
                         )
                     elif content.type == "video_url":
                         real_content += video_token
-                        conv.append_video(content.video_url.url)
+                        preprocess_kwargs = {
+                            key: value
+                            for key in GLM_MEDIA_CONFIG_KEYS
+                            if (value := getattr(content.video_url, key, None))
+                            is not None
+                        }
+                        conv.append_video(
+                            content.video_url.url,
+                            preprocess_kwargs=preprocess_kwargs or None,
+                        )
                     elif content.type == "audio_url":
                         real_content += audio_token
                         conv.append_audio(content.audio_url.url)

python/sglang/srt/parser/jinja_template_utils.py

@@ -9,10 +9,12 @@ import logging
 import jinja2
 import transformers.utils.chat_template_utils as hf_chat_utils
 
-from sglang.srt.utils import ImageData
+from sglang.srt.utils import GLM_MEDIA_CONFIG_KEYS, ImageData, VideoData
 
 logger = logging.getLogger(__name__)
 
+MEDIA_URL_PART_TYPES = ("image_url", "input_image", "video_url", "audio_url")
+
 # ============================================================================
 # JINJA TEMPLATE CONTENT FORMAT DETECTION
 # ============================================================================
@@ -120,6 +122,64 @@ def detect_jinja_template_content_format(chat_template: str) -> str:
         return "string"
 
 
+def jinja_template_may_reorder_tool_results(chat_template: str) -> bool:
+    """Detect templates that associate tool results with tool_calls by tool_call_id.
+
+    Such templates may emit media placeholders in tool_calls order rather than
+    request message order. Templates that sort/group by the tool_call_id string
+    value are intentionally excluded: their order cannot be reproduced from
+    message order alone.
+
+    This is an over-approximation: templates that merely print or validate
+    tool_call_id while rendering in message order (e.g. Mistral) also match.
+    That is safe because canonicalization keeps extraction and rendering
+    consistent for those templates too; it only reorders prompts the client
+    had already sent out of tool_calls order.
+    """
+    if not isinstance(chat_template, str):
+        return False
+
+    jinja_ast = _try_extract_ast(chat_template)
+    if jinja_ast is None:
+        return False
+
+    def is_tool_call_id(node: jinja2.nodes.Node) -> bool:
+        return isinstance(node, jinja2.nodes.Const) and node.value == "tool_call_id"
+
+    if any(
+        node.attr == "tool_call_id" for node in jinja_ast.find_all(jinja2.nodes.Getattr)
+    ):
+        return True
+
+    if any(
+        is_tool_call_id(node.arg) for node in jinja_ast.find_all(jinja2.nodes.Getitem)
+    ):
+        return True
+
+    for call in jinja_ast.find_all(jinja2.nodes.Call):
+        if (
+            isinstance(call.node, jinja2.nodes.Getattr)
+            and call.node.attr == "get"
+            and call.args
+            and is_tool_call_id(call.args[0])
+        ):
+            return True
+
+    attribute_filters = {"map", "rejectattr", "selectattr"}
+    for filter_node in jinja_ast.find_all(jinja2.nodes.Filter):
+        if filter_node.name not in attribute_filters:
+            continue
+        if filter_node.args and is_tool_call_id(filter_node.args[0]):
+            return True
+        if any(
+            keyword.key == "attribute" and is_tool_call_id(keyword.value)
+            for keyword in filter_node.kwargs
+        ):
+            return True
+
+    return False
+
+
 def process_content_for_template_format(
     msg_dict: dict,
     content_format: str,
@@ -179,15 +239,23 @@ def process_content_for_template_format(
                 elif chunk_type == "video_url":
                     video_obj = chunk.get("video_url") or {}
                     mdp = video_obj.get("max_dynamic_patch", None)
-                    if mdp is None:
+                    preprocess_kwargs = {
+                        key: video_obj[key]
+                        for key in GLM_MEDIA_CONFIG_KEYS
+                        if video_obj.get(key) is not None
+                    }
+                    if mdp is not None:
+                        preprocess_kwargs["max_dynamic_patch"] = mdp
+                    if not preprocess_kwargs:
                         video_data.append(chunk["video_url"]["url"])
                     else:
-                        # Keep structured info for backend, but template only sees {"type":"video"}
+                        # VideoData survives load_video on every processor; a
+                        # plain dict only the GLM consumer understands.
                         video_data.append(
-                            {
-                                "url": video_obj["url"],
-                                "max_dynamic_patch": mdp,
-                            }
+                            VideoData(
+                                url=video_obj["url"],
+                                preprocess_kwargs=preprocess_kwargs,
+                            )
                         )
                     if chunk.get("modalities"):
                         modalities.append(chunk.get("modalities"))
@@ -207,9 +275,7 @@ def process_content_for_template_format(
                             {"type": "text", "text": chunk["text"]}
                         )
                 elif chunk_type == "tool_reference":
-                    # GLM-specific extension: pass through so the chat template
-                    # can match tool_reference.name against tools[*].function.name
-                    # and render the referenced tool schemas inline.
+                    # Preserve this extension because GLM templates resolve referenced tool schemas by function name.
                     processed_content_parts.append(chunk)
 
         new_msg = {

python/sglang/srt/parser/template_manager.py

@@ -38,7 +38,10 @@ from sglang.srt.parser.conversation import (
     get_conv_template_by_model_path,
     register_conv_template,
 )
-from sglang.srt.parser.jinja_template_utils import detect_jinja_template_content_format
+from sglang.srt.parser.jinja_template_utils import (
+    detect_jinja_template_content_format,
+    jinja_template_may_reorder_tool_results,
+)
 from sglang.srt.parser.template_detection import (
     REASONING_PARSER_RULES,
     TOOL_CALL_PARSER_RULES,
@@ -68,6 +71,7 @@ class TemplateManager:
         self._reasoning_config: Optional[ReasoningToggleConfig] = None
         self._suggested_reasoning_parser: Optional[str] = None
         self._suggested_tool_call_parser: Optional[str] = None
+        self._jinja_template_may_reorder_tool_results: bool = False
 
     @property
     def chat_template_name(self) -> Optional[str]:
@@ -109,8 +113,15 @@ class TemplateManager:
         """Get the auto-detected tool-call parser name, or None."""
         return self._suggested_tool_call_parser
 
+    @property
+    def jinja_template_may_reorder_tool_results(self) -> bool:
+        return self._jinja_template_may_reorder_tool_results
+
     def _run_template_detection(self, template, tokenizer) -> None:
         """Run reasoning pattern and parser detection on a template."""
+        self._jinja_template_may_reorder_tool_results = (
+            jinja_template_may_reorder_tool_results(template)
+        )
         self._force_reasoning, self._reasoning_config = detect_reasoning_pattern(
             template
         )

python/sglang/srt/utils/common.py

@@ -1788,6 +1788,14 @@ class ImageData:
     content_hash: Optional[str] = None
 
 
+GLM_MEDIA_CONFIG_KEYS = (
+    "fps",
+    "max_frames",
+    "max_tokens_per_frame",
+    "max_image_tokens",
+)
+
+
 @dataclass
 class VideoData:
     url: str
@@ -1798,6 +1806,44 @@ image_extension_names = (".png", ".jpg", ".jpeg", ".webp", ".gif")
 GPUImageDecodeMode = Union[bool, Literal["nvjpeg_fancy"]]
 
 
+def smart_to_rgb(
+    image: Union[torch.Tensor, Image.Image],
+) -> Union[torch.Tensor, Image.Image]:
+    if not isinstance(image, Image.Image):
+        return image
+
+    if image.mode in ("RGBA", "LA") or "transparency" in image.info:
+        image = image.convert("RGBA")
+        width, height = image.size
+        edge_pixels = []
+
+        for x in range(0, width, max(1, width // 20)):
+            for y in (0, height - 1):
+                pixel = image.getpixel((x, y))
+                if pixel[3] > 128:
+                    edge_pixels.append(pixel[:3])
+
+        for y in range(0, height, max(1, height // 20)):
+            for x in (0, width - 1):
+                pixel = image.getpixel((x, y))
+                if pixel[3] > 128:
+                    edge_pixels.append(pixel[:3])
+
+        if edge_pixels:
+            avg_brightness = sum(sum(pixel) for pixel in edge_pixels) / (
+                len(edge_pixels) * 3
+            )
+            background_color = (32, 32, 32) if avg_brightness > 128 else (240, 240, 240)
+        else:
+            background_color = (255, 255, 255)
+
+        background = Image.new("RGB", image.size, background_color)
+        background.paste(image, mask=image.getchannel("A"))
+        return background
+
+    return image.convert("RGB")
+
+
 def is_jpeg_with_cuda(
     image_bytes: bytes = b"", gpu_image_decode: GPUImageDecodeMode = True
 ) -> bool:

test/registered/unit/entrypoints/openai/test_serving_chat.py

@@ -39,6 +39,9 @@ from sglang.srt.entrypoints.openai.serving_chat import (
 from sglang.srt.environ import envs
 from sglang.srt.function_call.kimik3_format import TOOLS_CLOSE, TOOLS_OPEN
 from sglang.srt.managers.io_struct import GenerateReqInput
+from sglang.srt.parser.jinja_template_utils import (
+    jinja_template_may_reorder_tool_results,
+)
 from sglang.srt.parser.template_detection import ReasoningToggleConfig
 from sglang.srt.utils import get_or_create_event_loop
 from sglang.test.ci.ci_register import register_cpu_ci
@@ -80,6 +83,24 @@ _DSV4_OFFICIAL_ENCODER = (
     'DEFAULT_REASONING_EFFORT = "low"\n'
 )
 
+_TOOL_RESULT_REORDER_TEMPLATE = """
+{%- for assistant in messages
+    if assistant.role == 'assistant' and assistant.tool_calls -%}
+    {%- for tool_call in assistant.tool_calls -%}
+        {%- for result in messages
+            if result.role == 'tool' and result.tool_call_id == tool_call.id -%}
+            {%- for part in result.content -%}
+                {%- if part.type == 'text' -%}
+                    {{- part.text -}}
+                {%- else -%}
+                    {{- '<' + part.type + '>' -}}
+                {%- endif -%}
+            {%- endfor -%}
+        {%- endfor -%}
+    {%- endfor -%}
+{%- endfor -%}
+"""
+
 
 def _create_dsv4_checkpoint(test_case: unittest.TestCase, source: str) -> str:
     model_dir = tempfile.TemporaryDirectory()
@@ -163,6 +184,7 @@ class _MockTemplateManager:
         self.completion_template_name: Optional[str] = None
         self.reasoning_config = None
         self.force_reasoning = False
+        self.jinja_template_may_reorder_tool_results = False
 
 
 class ServingChatTestCase(unittest.TestCase):
@@ -192,6 +214,166 @@ class ServingChatTestCase(unittest.TestCase):
         self.fastapi_request = Mock(spec=Request)
         self.fastapi_request.headers = {}
 
+    @staticmethod
+    def _render_tool_results_in_call_order(messages, **kwargs):
+        """Block-level tool_call_id association, like the GLM chat templates."""
+        del kwargs
+        rendered = []
+        index = 0
+        while index < len(messages):
+            message = messages[index]
+            index += 1
+            tool_calls = message.get("tool_calls") or []
+            if message.get("role") != "assistant" or not tool_calls:
+                continue
+            run = []
+            while index < len(messages) and messages[index].get("role") == "tool":
+                run.append(messages[index])
+                index += 1
+            by_id = {result.get("tool_call_id"): result for result in run}
+            for tool_call in tool_calls:
+                result = by_id.get(tool_call.get("id"))
+                if result is None:
+                    continue
+                for part in result.get("content") or []:
+                    if part.get("type") == "text":
+                        rendered.append(part.get("text", ""))
+                    else:
+                        rendered.append(f"<{part.get('type')}>")
+        return "".join(rendered)
+
+    @staticmethod
+    def _tool_round(call_ids, result_ids, part_type="image_url"):
+        assistant = {
+            "role": "assistant",
+            "content": "",
+            "tool_calls": [
+                {"id": call_id, "function": {"name": call_id, "arguments": {}}}
+                for call_id in call_ids
+            ],
+        }
+        part_key = {"image_url": "image_url", "video_url": "video_url"}[part_type]
+        results = [
+            {
+                "role": "tool",
+                "tool_call_id": result_id,
+                "content": [{"type": part_type, part_key: {"url": result_id}}],
+            }
+            for result_id in result_ids
+        ]
+        return [assistant] + results
+
+    def test_canonicalize_tool_message_order_sorts_media_runs(self):
+        messages = self._tool_round(
+            ["call-a", "call-b"], ["call-b", "call-a"]
+        ) + self._tool_round(["call-c", "call-d"], ["call-d", "call-c"], "video_url")
+
+        canonical = self.chat._canonicalize_tool_message_order(messages)
+
+        self.assertEqual(
+            [
+                message["tool_call_id"]
+                for message in canonical
+                if message.get("role") == "tool"
+            ],
+            ["call-a", "call-b", "call-c", "call-d"],
+        )
+
+    def test_canonicalize_tool_message_order_keeps_unassociable_runs(self):
+        cases = {
+            "unknown_id": (["call-a", "call-b"], ["call-b", "call-z"]),
+            "duplicate_result_id": (["call-a", "call-b"], ["call-b", "call-b"]),
+            "missing_call_id": ([None, "call-b"], ["call-b", None]),
+        }
+        for name, (call_ids, result_ids) in cases.items():
+            with self.subTest(name=name):
+                messages = self._tool_round(call_ids, result_ids)
+                canonical = self.chat._canonicalize_tool_message_order(messages)
+                self.assertEqual(canonical, messages)
+
+    def test_canonicalize_tool_message_order_keeps_text_only_runs(self):
+        messages = self._tool_round(["call-a", "call-b"], ["call-b", "call-a"])
+        for message in messages[1:]:
+            message["content"] = [{"type": "text", "text": "done"}]
+
+        canonical = self.chat._canonicalize_tool_message_order(messages)
+
+        self.assertEqual(canonical, messages)
+
+    def test_jinja_path_recovers_tool_result_images_only_when_template_needs_it(self):
+        self.template_manager.chat_template_name = None
+        self.template_manager.jinja_template_content_format = "openai"
+        self.template_manager.jinja_template_may_reorder_tool_results = (
+            jinja_template_may_reorder_tool_results(_TOOL_RESULT_REORDER_TEMPLATE)
+        )
+        self.assertTrue(self.template_manager.jinja_template_may_reorder_tool_results)
+        self.tm.tokenizer.apply_chat_template.side_effect = (
+            self._render_tool_results_in_call_order
+        )
+        request = ChatCompletionRequest(
+            model="x",
+            messages=[
+                {"role": "user", "content": "inspect"},
+                {
+                    "role": "assistant",
+                    "content": "",
+                    "tool_calls": [
+                        {
+                            "id": "call-a",
+                            "type": "function",
+                            "function": {"name": "a", "arguments": {}},
+                        },
+                        {
+                            "id": "call-b",
+                            "type": "function",
+                            "function": {"name": "b", "arguments": {}},
+                        },
+                    ],
+                },
+                {
+                    "role": "tool",
+                    "tool_call_id": "call-b",
+                    "content": [{"type": "image_url", "image_url": {"url": "image-b"}}],
+                },
+                {
+                    "role": "tool",
+                    "tool_call_id": "call-a",
+                    "content": [{"type": "image_url", "image_url": {"url": "image-a"}}],
+                },
+            ],
+        )
+
+        result = self.chat._apply_jinja_template(request, None, is_multimodal=True)
+
+        self.assertEqual(
+            [item.url for item in result.image_data], ["image-a", "image-b"]
+        )
+        self.assertEqual(self.tm.tokenizer.apply_chat_template.call_count, 1)
+        rendered_messages = self.tm.tokenizer.apply_chat_template.call_args[0][0]
+        self.assertEqual(
+            [m.get("tool_call_id") for m in rendered_messages if "tool_call_id" in m],
+            ["call-a", "call-b"],
+        )
+
+        # An already-ordered request must produce the exact same render input.
+        ordered_request = ChatCompletionRequest(
+            model="x",
+            messages=request.messages[:2] + request.messages[2:][::-1],
+        )
+        self.tm.tokenizer.apply_chat_template.reset_mock()
+        self.chat._apply_jinja_template(ordered_request, None, is_multimodal=True)
+        self.assertEqual(
+            rendered_messages, self.tm.tokenizer.apply_chat_template.call_args[0][0]
+        )
+
+        self.template_manager.jinja_template_may_reorder_tool_results = False
+        self.tm.tokenizer.apply_chat_template.reset_mock()
+        result = self.chat._apply_jinja_template(request, None, is_multimodal=True)
+        self.assertEqual(
+            [item.url for item in result.image_data], ["image-b", "image-a"]
+        )
+        self.assertEqual(self.tm.tokenizer.apply_chat_template.call_count, 1)
+
     def test_parsers_follow_the_control_plane_overlay(self):
         """Template detection records the parsers through `override`, so they
         answer from the bags; `ServerArgs` keeps the launcher's seed."""

test/registered/unit/entrypoints/openai/test_serving_completions.py

@@ -58,6 +58,7 @@ class _MockTemplateManager:
         self.completion_template_name: Optional[str] = (
             None  # Set to None to avoid template processing
         )
+        self.jinja_template_may_reorder_tool_results = False
 
 
 class ServingCompletionTestCase(unittest.TestCase):

test/registered/unit/entrypoints/openai/test_serving_embedding.py

@@ -97,6 +97,7 @@ class _MockTemplateManager:
         self.chat_template_name = None  # None for embeddings usually
         self.jinja_template_content_format = "openai"
         self.completion_template_name = None
+        self.jinja_template_may_reorder_tool_results = False
 
 
 class ServingEmbeddingTestCase(unittest.TestCase):

test/registered/unit/entrypoints/openai/utils.py

@@ -79,6 +79,7 @@ class MockTemplateManager:
         self.completion_template_name = None
         self.reasoning_config = None
         self.force_reasoning = False
+        self.jinja_template_may_reorder_tool_results = False
 
 
 def make_serving(*, is_multimodal: bool = False) -> OpenAIServingResponses:

test/registered/unit/multimodal/test_processor_async_call_sites.py

@@ -27,6 +27,11 @@ _MULTIMODAL_ROOT = (
     / "srt"
     / "multimodal"
 )
+if not _MULTIMODAL_ROOT.is_dir():
+    raise RuntimeError(
+        f"multimodal processor tree not found at {_MULTIMODAL_ROOT}; "
+        "these tests must run from a full source checkout"
+    )
 # The async helper and the sync body live side by side here by design.
 _EXEMPT = {"base_processor.py"}
 

test/registered/unit/parser/test_jinja_template_utils.py

@@ -4,8 +4,10 @@ import unittest
 
 from sglang.srt.parser.jinja_template_utils import (
     detect_jinja_template_content_format,
+    jinja_template_may_reorder_tool_results,
     process_content_for_template_format,
 )
+from sglang.srt.utils import VideoData
 from sglang.test.ci.ci_register import register_cpu_ci
 from sglang.test.test_utils import CustomTestCase
 
@@ -16,6 +18,41 @@ register_cpu_ci(est_time=6, suite="base-c-test-cpu")
 class TestTemplateContentFormatDetection(CustomTestCase):
     """Test template content format detection functionality."""
 
+    def test_detect_tool_result_id_association(self):
+        attribute_template = """
+        {% for message in messages %}
+            {{ message.tool_call_id }}
+        {% endfor %}
+        """
+        item_template = "{{ messages[0]['tool_call_id'] }}"
+        get_template = "{{ messages[0].get('tool_call_id') }}"
+        select_template = (
+            "{{ messages | selectattr('tool_call_id', 'equalto', 'call-a') | list }}"
+        )
+
+        self.assertTrue(jinja_template_may_reorder_tool_results(attribute_template))
+        self.assertTrue(jinja_template_may_reorder_tool_results(item_template))
+        self.assertTrue(jinja_template_may_reorder_tool_results(get_template))
+        self.assertTrue(jinja_template_may_reorder_tool_results(select_template))
+
+    def test_sort_by_tool_call_id_value_is_not_association(self):
+        # sort/groupby order by the id string value, which message-order
+        # canonicalization cannot reproduce, so they must not activate it.
+        sort_template = "{{ messages | sort(attribute='tool_call_id') }}"
+        groupby_template = "{{ messages | groupby('tool_call_id') }}"
+
+        self.assertFalse(jinja_template_may_reorder_tool_results(sort_template))
+        self.assertFalse(jinja_template_may_reorder_tool_results(groupby_template))
+
+    def test_tool_call_id_text_does_not_enable_order_recovery(self):
+        self.assertFalse(
+            jinja_template_may_reorder_tool_results(
+                "{# tool_call_id is mentioned only in a comment #}{{ messages }}"
+            )
+        )
+        self.assertFalse(jinja_template_may_reorder_tool_results("{{{{ invalid"))
+        self.assertFalse(jinja_template_may_reorder_tool_results(None))
+
     def test_detect_llama4_openai_format(self):
         """Test detection of llama4-style template (should be 'openai' format)."""
         llama4_pattern = """
@@ -312,30 +349,38 @@ class TestTemplateContentFormatDetection(CustomTestCase):
         self.assertEqual(video_data[0], "http://example.com/v.mp4")
         self.assertEqual(result["content"][1], {"type": "video"})
 
-    def test_process_content_video_with_max_dynamic_patch(self):
-        """Test video_url with max_dynamic_patch stores structured dict."""
+    def test_process_content_video_structured_fields_become_video_data(self):
+        """video_url with mdp/fps-style fields lands in VideoData.preprocess_kwargs."""
         msg_dict = {
             "role": "user",
             "content": [
                 {
                     "type": "video_url",
                     "video_url": {
-                        "url": "http://example.com/v.mp4",
+                        "url": "http://example.com/a.mp4",
                         "max_dynamic_patch": 4,
                     },
                 },
+                {
+                    "type": "video_url",
+                    "video_url": {
+                        "url": "http://example.com/b.mp4",
+                        "fps": 1.5,
+                        "max_frames": 16,
+                    },
+                },
             ],
         }
-        image_data = []
         video_data = []
-        audio_data = []
-        modalities = []
-        result = process_content_for_template_format(
-            msg_dict, "openai", image_data, video_data, audio_data, modalities
+        process_content_for_template_format(msg_dict, "openai", [], video_data, [], [])
+        self.assertEqual(
+            [(item.url, item.preprocess_kwargs) for item in video_data],
+            [
+                ("http://example.com/a.mp4", {"max_dynamic_patch": 4}),
+                ("http://example.com/b.mp4", {"fps": 1.5, "max_frames": 16}),
+            ],
         )
-        self.assertEqual(len(video_data), 1)
-        self.assertIsInstance(video_data[0], dict)
-        self.assertEqual(video_data[0]["max_dynamic_patch"], 4)
+        self.assertIsInstance(video_data[0], VideoData)
 
     def test_process_content_v32_encoding(self):
         """Test v32 encoding mode flattens text and ignores structured content parts."""