From fbf909b460ff2803fc41fc3dcd2de1db3854d47c Mon Sep 17 00:00:00 2001 From: Khoa Pham Date: Wed, 2 Sep 2026 18:23:06 -0700 Subject: [PATCH] [Fix] Alpha-channel images and tool-result media ordering (port of #36507) (#37320) Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Xinyuan Tong Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com> --- .../sglang/srt/entrypoints/openai/protocol.py | 4 + .../srt/entrypoints/openai/serving_chat.py | 67 +++- .../multimodal/processors/base_processor.py | 14 +- .../sglang/srt/multimodal/processors/glm4v.py | 298 +++++++++++++++++- python/sglang/srt/parser/conversation.py | 18 +- .../sglang/srt/parser/jinja_template_utils.py | 86 ++++- python/sglang/srt/parser/template_manager.py | 13 +- python/sglang/srt/utils/common.py | 46 +++ .../entrypoints/openai/test_serving_chat.py | 182 +++++++++++ .../openai/test_serving_completions.py | 1 + .../openai/test_serving_embedding.py | 1 + .../unit/entrypoints/openai/utils.py | 1 + .../test_processor_async_call_sites.py | 5 + .../unit/parser/test_jinja_template_utils.py | 67 +++- 14 files changed, 772 insertions(+), 31 deletions(-) diff --git a/python/sglang/srt/entrypoints/openai/protocol.py b/python/sglang/srt/entrypoints/openai/protocol.py index 109542667..82a384a09 100644 --- a/python/sglang/srt/entrypoints/openai/protocol.py +++ b/python/sglang/srt/entrypoints/openai/protocol.py @@ -566,6 +566,10 @@ class ChatCompletionMessageContentVideoURL(BaseModel): url: str max_dynamic_patch: Optional[int] = None min_dynamic_patch: Optional[int] = None + fps: Optional[float] = None + max_frames: Optional[int] = None + max_tokens_per_frame: Optional[int] = None + max_image_tokens: Optional[int] = None class ChatCompletionMessageContentAudioURL(BaseModel): diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index 17ecfa782..790f84a28 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -89,7 +89,10 @@ from sglang.srt.function_call.utils import ( ) from sglang.srt.managers.io_struct import GenerateReqInput from sglang.srt.parser.conversation import generate_chat_conv -from sglang.srt.parser.jinja_template_utils import process_content_for_template_format +from sglang.srt.parser.jinja_template_utils import ( + MEDIA_URL_PART_TYPES, + process_content_for_template_format, +) from sglang.srt.parser.reasoning_parser import ReasoningParser from sglang.srt.utils.weight_versions import build_endpoint_weight_version_metadata @@ -717,6 +720,66 @@ class OpenAIServingChat(OpenAIServingBase): prompt_tokens = max(0, prompt_tokens - self._KIMI_K3_GENERATION_STUB_TOKENS) return prompt_tokens + @staticmethod + def _sort_tool_message_run( + run: List[Dict[str, Any]], tool_calls: List[Dict[str, Any]] + ) -> List[Dict[str, Any]]: + """Order a tool-message run by tool_call position. + + Templates that associate results by tool_call_id render the run in + tool_calls order; sorting the run upfront keeps extraction order and + placeholder order the same. Runs the template itself would refuse to + associate (missing/duplicate/unknown ids) are left untouched, as are + text-only runs, whose order text-only templates may rely on. + """ + if len(run) < 2: + return run + call_ids = [tc.get("id") for tc in tool_calls] + if any(call_id is None for call_id in call_ids) or len(set(call_ids)) != len( + call_ids + ): + return run + result_ids = [message.get("tool_call_id") for message in run] + if any(result_id not in call_ids for result_id in result_ids) or len( + set(result_ids) + ) != len(result_ids): + return run + has_media = any( + isinstance(message.get("content"), list) + and any( + isinstance(part, dict) and part.get("type") in MEDIA_URL_PART_TYPES + for part in message["content"] + ) + for message in run + ) + if not has_media: + return run + position = {call_id: index for index, call_id in enumerate(call_ids)} + return sorted(run, key=lambda message: position[message["tool_call_id"]]) + + @classmethod + def _canonicalize_tool_message_order( + cls, messages: List[Dict[str, Any]] + ) -> List[Dict[str, Any]]: + canonical = [] + index = 0 + while index < len(messages): + message = messages[index] + canonical.append(message) + index += 1 + tool_calls = message.get("tool_calls") or [] + if message.get("role") != "assistant" or not tool_calls: + continue + run = [] + while index < len(messages) and messages[index].get("role") in ( + "tool", + "function", + ): + run.append(messages[index]) + index += 1 + canonical.extend(cls._sort_tool_message_run(run, tool_calls)) + return canonical + async def _generate_stream_content( self, content: Dict[str, Any], @@ -1329,6 +1392,8 @@ class OpenAIServingChat(OpenAIServingBase): prompt_ids, assistant_prefix ) else: + if self.template_manager.jinja_template_may_reorder_tool_results: + messages = self._canonicalize_tool_message_order(messages) for msg_dict in copy.deepcopy(messages): if msg_dict.get("content") is None: msg_dict["content"] = "" diff --git a/python/sglang/srt/multimodal/processors/base_processor.py b/python/sglang/srt/multimodal/processors/base_processor.py index 919bd1b3c..30440de8b 100644 --- a/python/sglang/srt/multimodal/processors/base_processor.py +++ b/python/sglang/srt/multimodal/processors/base_processor.py @@ -57,6 +57,7 @@ from sglang.srt.utils import ( load_image, load_video, logger, + smart_to_rgb, ) _is_cpu = is_cpu() @@ -210,6 +211,8 @@ def _tokenizer_of(processor): class BaseMultimodalProcessor(ABC): models = [] gpu_image_decode = True # Enable GPU decoding by default + smart_rgb_conversion = False + video_preprocessing_device = None prefer_tokenized_input = False precompute_hash_before_cpu_transfer = False # Set by processors that already build input_ids from the request's own @@ -811,6 +814,10 @@ class BaseMultimodalProcessor(ABC): if processor_device is not None: kwargs["device"] = processor_device + # Long-video preprocessing stays on CPU to avoid competing with scheduler GPU pools. + if videos and self.video_preprocessing_device is not None: + kwargs["device"] = self.video_preprocessing_device + # Avoid double BOS when the chat template already wrote one. if self._tokenizer_auto_adds_specials and isinstance(input_text, str): bos = getattr(tokenizer, "bos_token", None) @@ -895,8 +902,11 @@ class BaseMultimodalProcessor(ABC): img, _ = load_image(data, cls.gpu_image_decode) if isinstance(img, torch.Tensor): return img # JPEG already decoded on GPU by nvJPEG - if discard_alpha_channel and img.mode != "RGB": - return img.convert("RGB") + if discard_alpha_channel: + if cls.smart_rgb_conversion: + return smart_to_rgb(img) + if img.mode != "RGB": + return img.convert("RGB") return img elif modality == Modality.VIDEO: return load_video(data, frame_count_limit) diff --git a/python/sglang/srt/multimodal/processors/glm4v.py b/python/sglang/srt/multimodal/processors/glm4v.py index c4d363f47..7eecb6341 100644 --- a/python/sglang/srt/multimodal/processors/glm4v.py +++ b/python/sglang/srt/multimodal/processors/glm4v.py @@ -1,5 +1,10 @@ +import asyncio +import math from typing import List, Union +import numpy as np +import torch + from sglang.srt.layers.rotary_embedding import MRotaryEmbedding from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput from sglang.srt.models.glm4v import Glm4vForConditionalGeneration @@ -10,19 +15,248 @@ from sglang.srt.multimodal.processors.base_processor import ( from sglang.srt.multimodal.processors.base_processor import ( MultimodalSpecialTokens, ) +from sglang.srt.utils import GLM_MEDIA_CONFIG_KEYS +from sglang.srt.utils.video_decoder import VideoDecoderWrapper try: from sglang.srt.models.glm_ocr import GlmOcrForConditionalGeneration except ImportError: GlmOcrForConditionalGeneration = None +try: + from sglang.srt.models.glm5_next import Glm5NextForConditionalGeneration +except ImportError: + Glm5NextForConditionalGeneration = None + +GLM_VIDEO_DEFAULT_FPS = 2.0 +GLM_VIDEO_DEFAULT_MAX_FRAMES = 2048 +GLM_VIDEO_PATCH_SIZE = 14 +GLM_VIDEO_MERGE_SIZE = 2 +GLM_VIDEO_PATCH_EXPAND_FACTOR = 4 + + +def _glm_video_metadata(total_num_frames, fps, duration, frames_indices): + return { + "total_num_frames": int(total_num_frames), + "fps": float(fps), + "duration": float(duration), + "video_backend": "sglang", + "frames_indices": list(frames_indices), + } + + +def _glm_item_config(item): + config = dict(getattr(item, "preprocess_kwargs", None) or {}) + if isinstance(item, dict): + config.update(item.get("preprocess_kwargs") or {}) + config.update( + { + key: item[key] + for key in GLM_MEDIA_CONFIG_KEYS + if item.get(key) is not None + } + ) + return { + key: config[key] for key in GLM_MEDIA_CONFIG_KEYS if config.get(key) is not None + } + + +def split_glm_video_items(mm_data): + if mm_data is None: + return None, [] + items = mm_data if isinstance(mm_data, (list, tuple)) else [mm_data] + urls, configs = [], [] + for item in items: + if isinstance(item, dict) and "format" not in item and "url" in item: + urls.append(item["url"]) + elif hasattr(item, "url") and hasattr(item, "preprocess_kwargs"): + urls.append(item.url) + else: + urls.append(item) + configs.append(_glm_item_config(item)) + return urls, configs + + +def glm_processor_video_config(processor): + if processor is None: + return {} + return { + key: value + for key in GLM_MEDIA_CONFIG_KEYS + if (value := getattr(processor, key, None)) is not None + } + + +def _merge_glm_video_configs(default_config, item_configs): + defaults = dict(default_config or {}) + return [{**defaults, **dict(config or {})} for config in item_configs] + + +def _hf_sample_frame_indices( + video_processor, total_frames, fps, duration, video_config +): + """Sample indices with the model's own HF processor so behavior tracks the pinned transformers.""" + if video_config.get("max_frames") is not None: + return None + sample_frames = getattr(video_processor, "sample_frames", None) + if sample_frames is None: + return None + try: + from transformers.video_utils import VideoMetadata + except ImportError: + return None + metadata = VideoMetadata( + total_num_frames=int(total_frames), + fps=float(fps), + duration=float(duration), + ) + indices = sample_frames(metadata, fps=video_config.get("fps")) + return [int(index) for index in indices] + + +def glm_sample_frame_indices( + total_frames, + fps, + duration, + *, + target_fps=None, + max_frame_count=None, +): + """Fallback sampler for processors without sample_frames or when max_frames is requested.""" + if total_frames <= 0: + return [] + target_fps = GLM_VIDEO_DEFAULT_FPS if target_fps is None else float(target_fps) + max_frame_count = ( + GLM_VIDEO_DEFAULT_MAX_FRAMES + if max_frame_count is None + else int(max_frame_count) + ) + if target_fps <= 0 or max_frame_count <= 0: + return [] + + max_frame_idx = total_frames - 1 + if not duration: + duration = round(max_frame_idx / fps) + 1 if fps else 0 + extract_t = min(int(duration * target_fps), int(max_frame_count)) + extract_t = max(1, extract_t) + + if fps: + duration_per_frame = 1 / fps + timestamps = [index * duration_per_frame for index in range(total_frames)] + max_second = int(duration) + indices = [] + current_second = 0.0 + interval = 1 / target_fps + for frame_index, timestamp in enumerate(timestamps): + if timestamp >= current_second: + current_second += interval + indices.append(frame_index) + if current_second >= max_second: + break + else: + indices = [] + + if len(indices) < extract_t: + start = indices[0] if indices else 0 + end = indices[-1] if indices else max(total_frames - 1, 0) + indices = np.linspace(start, end, extract_t, dtype=int).tolist() + elif len(indices) > extract_t: + indices = np.linspace(0, total_frames - 1, extract_t, dtype=int).tolist() + + seen = set() + unique_indices = [] + for index in indices: + index = int(index) + if index not in seen: + seen.add(index) + unique_indices.append(index) + if len(unique_indices) & 1: + unique_indices.append(unique_indices[-1]) + return unique_indices + + +def _resize_frames_to_max_tokens(frames, max_tokens_per_frame): + import torchvision.transforms.functional as TF + + if not isinstance(frames, torch.Tensor): + frames = torch.from_numpy(np.asarray(frames)) + nchw = frames.permute(0, 3, 1, 2) + _, _, height, width = nchw.shape + pixels_per_token = (GLM_VIDEO_PATCH_SIZE * GLM_VIDEO_MERGE_SIZE) ** 2 + factor = GLM_VIDEO_PATCH_SIZE * GLM_VIDEO_MERGE_SIZE * GLM_VIDEO_PATCH_EXPAND_FACTOR + max_pixels = max( + int(max_tokens_per_frame) * pixels_per_token, + factor * factor, + ) + resized_height = max(factor, round(height / factor) * factor) + resized_width = max(factor, round(width / factor) * factor) + if resized_height * resized_width > max_pixels: + scale = math.sqrt((height * width) / max_pixels) + resized_height = max(factor, math.floor(height / scale / factor) * factor) + resized_width = max(factor, math.floor(width / scale / factor) * factor) + if (resized_height, resized_width) != (height, width): + nchw = TF.resize( + nchw, + [resized_height, resized_width], + interpolation=TF.InterpolationMode.BICUBIC, + antialias=True, + ) + return nchw.permute(0, 2, 3, 1).contiguous() + + +def glm_decode_frames_at(vr, indices, video_config=None): + indices = list(indices) + if not indices: + return None + video_config = video_config or {} + if hasattr(vr, "get_frames_as_tensor"): + frames = vr.get_frames_as_tensor(indices) + else: + frames = vr.get_frames_at(indices) + max_tokens_per_frame = video_config.get("max_tokens_per_frame") + if max_tokens_per_frame is not None: + frames = _resize_frames_to_max_tokens(frames, max_tokens_per_frame) + return frames + + +def glm_sample_and_decode_sync(vr, video_config=None, video_processor=None): + video_config = video_config or {} + fps = vr.avg_fps + if not fps or fps <= 0: + raise ValueError(f"Cannot determine video fps (avg_fps={fps!r})") + duration = len(vr) / fps + indices = _hf_sample_frame_indices( + video_processor, len(vr), fps, duration, video_config + ) + if indices is None: + indices = glm_sample_frame_indices( + len(vr), + fps, + duration, + target_fps=video_config.get("fps"), + max_frame_count=video_config.get("max_frames"), + ) + if not indices: + raise ValueError("Video frame sampling produced no frames") + frames = glm_decode_frames_at(vr, indices, video_config) + return frames, _glm_video_metadata(len(vr), fps, duration, indices) + + +def _passthrough_video_metadata(video, video_config): + num_frames = video.shape[0] if hasattr(video, "shape") else len(video) + fps = float(video_config.get("fps") or GLM_VIDEO_DEFAULT_FPS) + return _glm_video_metadata(num_frames, fps, num_frames / fps, range(num_frames)) + class Glm4vImageProcessor(SGLangBaseProcessor): + smart_rgb_conversion = True + video_preprocessing_device = "cpu" models = [ m for m in [ Glm4vForConditionalGeneration, Glm4vMoeForConditionalGeneration, + Glm5NextForConditionalGeneration, GlmOcrForConditionalGeneration, ] if m is not None @@ -46,6 +280,8 @@ class Glm4vImageProcessor(SGLangBaseProcessor): self.IMAGE_END_TOKEN_ID = hf_config.image_end_token_id self.VIDEO_START_TOKEN_ID = hf_config.video_start_token_id self.VIDEO_END_TOKEN_ID = hf_config.video_end_token_id + self.IM_START_TOKEN_ID = self.IMAGE_START_TOKEN_ID + self.IM_END_TOKEN_ID = self.IMAGE_END_TOKEN_ID # Vision config self.IMAGE_FACTOR = 28 @@ -90,17 +326,71 @@ class Glm4vImageProcessor(SGLangBaseProcessor): *args, **kwargs, ): + # Bare base64 video must use SGLang's decoder because HF treats it as a path-like string. + video_urls, video_configs = split_glm_video_items(request_obj.video_data) + video_processor = getattr(self._processor, "video_processor", None) + default_video_config = glm_processor_video_config(video_processor) + default_video_config.update(self.video_config) + video_configs = _merge_glm_video_configs(default_video_config, video_configs) + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, - video_data=request_obj.video_data, + video_data=video_urls, multimodal_tokens=self.mm_tokens, ) - if base_output.videos: - base_output.videos = request_obj.video_data + video_metadata = None + if base_output.videos and not isinstance(base_output.videos[0], dict): + loop = asyncio.get_running_loop() + decode_tasks = [] + for index, video in enumerate(base_output.videos): + video_config = ( + video_configs[index] if index < len(video_configs) else {} + ) + if isinstance(video, VideoDecoderWrapper): + decode_tasks.append( + loop.run_in_executor( + self.io_executor, + glm_sample_and_decode_sync, + video, + video_config, + video_processor, + ) + ) + else: + decode_tasks.append( + asyncio.sleep( + 0, + result=( + video, + _passthrough_video_metadata(video, video_config), + ), + ) + ) + + try: + videos_processed = await asyncio.gather(*decode_tasks) + finally: + for video in base_output.videos: + close = getattr(video, "close", None) + if callable(close): + close() + base_output.videos, video_metadata = map(list, zip(*videos_processed)) + + combine_kwargs = {} + if video_metadata is not None: + # Skip HF resampling because these frames already carry their original indices. + combine_kwargs["video_metadata"] = video_metadata + combine_kwargs["do_sample_frames"] = False + combine_kwargs["processor_video_config"] = { + key: value + for key, value in self.video_config.items() + if key not in {"fps", "max_frames", "max_tokens_per_frame"} + } + mm_items, input_ids, ret = await self.process_and_combine_mm_data_async( - base_output, self.mm_tokens + base_output, self.mm_tokens, **combine_kwargs ) input_ids = input_ids.flatten() diff --git a/python/sglang/srt/parser/conversation.py b/python/sglang/srt/parser/conversation.py index 920ac33e3..96d3a8fa9 100644 --- a/python/sglang/srt/parser/conversation.py +++ b/python/sglang/srt/parser/conversation.py @@ -35,7 +35,12 @@ from typing import Callable, Dict, List, Optional, Tuple, Union from typing_extensions import Literal from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest -from sglang.srt.utils import ImageData, VideoData, read_system_prompt_from_file +from sglang.srt.utils import ( + GLM_MEDIA_CONFIG_KEYS, + ImageData, + VideoData, + read_system_prompt_from_file, +) class SeparatorStyle(IntEnum): @@ -681,7 +686,16 @@ def generate_chat_conv( ) elif content.type == "video_url": real_content += video_token - conv.append_video(content.video_url.url) + preprocess_kwargs = { + key: value + for key in GLM_MEDIA_CONFIG_KEYS + if (value := getattr(content.video_url, key, None)) + is not None + } + conv.append_video( + content.video_url.url, + preprocess_kwargs=preprocess_kwargs or None, + ) elif content.type == "audio_url": real_content += audio_token conv.append_audio(content.audio_url.url) diff --git a/python/sglang/srt/parser/jinja_template_utils.py b/python/sglang/srt/parser/jinja_template_utils.py index f22222fd9..dce42ffd8 100644 --- a/python/sglang/srt/parser/jinja_template_utils.py +++ b/python/sglang/srt/parser/jinja_template_utils.py @@ -9,10 +9,12 @@ import logging import jinja2 import transformers.utils.chat_template_utils as hf_chat_utils -from sglang.srt.utils import ImageData +from sglang.srt.utils import GLM_MEDIA_CONFIG_KEYS, ImageData, VideoData logger = logging.getLogger(__name__) +MEDIA_URL_PART_TYPES = ("image_url", "input_image", "video_url", "audio_url") + # ============================================================================ # JINJA TEMPLATE CONTENT FORMAT DETECTION # ============================================================================ @@ -120,6 +122,64 @@ def detect_jinja_template_content_format(chat_template: str) -> str: return "string" +def jinja_template_may_reorder_tool_results(chat_template: str) -> bool: + """Detect templates that associate tool results with tool_calls by tool_call_id. + + Such templates may emit media placeholders in tool_calls order rather than + request message order. Templates that sort/group by the tool_call_id string + value are intentionally excluded: their order cannot be reproduced from + message order alone. + + This is an over-approximation: templates that merely print or validate + tool_call_id while rendering in message order (e.g. Mistral) also match. + That is safe because canonicalization keeps extraction and rendering + consistent for those templates too; it only reorders prompts the client + had already sent out of tool_calls order. + """ + if not isinstance(chat_template, str): + return False + + jinja_ast = _try_extract_ast(chat_template) + if jinja_ast is None: + return False + + def is_tool_call_id(node: jinja2.nodes.Node) -> bool: + return isinstance(node, jinja2.nodes.Const) and node.value == "tool_call_id" + + if any( + node.attr == "tool_call_id" for node in jinja_ast.find_all(jinja2.nodes.Getattr) + ): + return True + + if any( + is_tool_call_id(node.arg) for node in jinja_ast.find_all(jinja2.nodes.Getitem) + ): + return True + + for call in jinja_ast.find_all(jinja2.nodes.Call): + if ( + isinstance(call.node, jinja2.nodes.Getattr) + and call.node.attr == "get" + and call.args + and is_tool_call_id(call.args[0]) + ): + return True + + attribute_filters = {"map", "rejectattr", "selectattr"} + for filter_node in jinja_ast.find_all(jinja2.nodes.Filter): + if filter_node.name not in attribute_filters: + continue + if filter_node.args and is_tool_call_id(filter_node.args[0]): + return True + if any( + keyword.key == "attribute" and is_tool_call_id(keyword.value) + for keyword in filter_node.kwargs + ): + return True + + return False + + def process_content_for_template_format( msg_dict: dict, content_format: str, @@ -179,15 +239,23 @@ def process_content_for_template_format( elif chunk_type == "video_url": video_obj = chunk.get("video_url") or {} mdp = video_obj.get("max_dynamic_patch", None) - if mdp is None: + preprocess_kwargs = { + key: video_obj[key] + for key in GLM_MEDIA_CONFIG_KEYS + if video_obj.get(key) is not None + } + if mdp is not None: + preprocess_kwargs["max_dynamic_patch"] = mdp + if not preprocess_kwargs: video_data.append(chunk["video_url"]["url"]) else: - # Keep structured info for backend, but template only sees {"type":"video"} + # VideoData survives load_video on every processor; a + # plain dict only the GLM consumer understands. video_data.append( - { - "url": video_obj["url"], - "max_dynamic_patch": mdp, - } + VideoData( + url=video_obj["url"], + preprocess_kwargs=preprocess_kwargs, + ) ) if chunk.get("modalities"): modalities.append(chunk.get("modalities")) @@ -207,9 +275,7 @@ def process_content_for_template_format( {"type": "text", "text": chunk["text"]} ) elif chunk_type == "tool_reference": - # GLM-specific extension: pass through so the chat template - # can match tool_reference.name against tools[*].function.name - # and render the referenced tool schemas inline. + # Preserve this extension because GLM templates resolve referenced tool schemas by function name. processed_content_parts.append(chunk) new_msg = { diff --git a/python/sglang/srt/parser/template_manager.py b/python/sglang/srt/parser/template_manager.py index f8faabc7c..d69458f87 100644 --- a/python/sglang/srt/parser/template_manager.py +++ b/python/sglang/srt/parser/template_manager.py @@ -38,7 +38,10 @@ from sglang.srt.parser.conversation import ( get_conv_template_by_model_path, register_conv_template, ) -from sglang.srt.parser.jinja_template_utils import detect_jinja_template_content_format +from sglang.srt.parser.jinja_template_utils import ( + detect_jinja_template_content_format, + jinja_template_may_reorder_tool_results, +) from sglang.srt.parser.template_detection import ( REASONING_PARSER_RULES, TOOL_CALL_PARSER_RULES, @@ -68,6 +71,7 @@ class TemplateManager: self._reasoning_config: Optional[ReasoningToggleConfig] = None self._suggested_reasoning_parser: Optional[str] = None self._suggested_tool_call_parser: Optional[str] = None + self._jinja_template_may_reorder_tool_results: bool = False @property def chat_template_name(self) -> Optional[str]: @@ -109,8 +113,15 @@ class TemplateManager: """Get the auto-detected tool-call parser name, or None.""" return self._suggested_tool_call_parser + @property + def jinja_template_may_reorder_tool_results(self) -> bool: + return self._jinja_template_may_reorder_tool_results + def _run_template_detection(self, template, tokenizer) -> None: """Run reasoning pattern and parser detection on a template.""" + self._jinja_template_may_reorder_tool_results = ( + jinja_template_may_reorder_tool_results(template) + ) self._force_reasoning, self._reasoning_config = detect_reasoning_pattern( template ) diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index 35614abcb..87958d618 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -1788,6 +1788,14 @@ class ImageData: content_hash: Optional[str] = None +GLM_MEDIA_CONFIG_KEYS = ( + "fps", + "max_frames", + "max_tokens_per_frame", + "max_image_tokens", +) + + @dataclass class VideoData: url: str @@ -1798,6 +1806,44 @@ image_extension_names = (".png", ".jpg", ".jpeg", ".webp", ".gif") GPUImageDecodeMode = Union[bool, Literal["nvjpeg_fancy"]] +def smart_to_rgb( + image: Union[torch.Tensor, Image.Image], +) -> Union[torch.Tensor, Image.Image]: + if not isinstance(image, Image.Image): + return image + + if image.mode in ("RGBA", "LA") or "transparency" in image.info: + image = image.convert("RGBA") + width, height = image.size + edge_pixels = [] + + for x in range(0, width, max(1, width // 20)): + for y in (0, height - 1): + pixel = image.getpixel((x, y)) + if pixel[3] > 128: + edge_pixels.append(pixel[:3]) + + for y in range(0, height, max(1, height // 20)): + for x in (0, width - 1): + pixel = image.getpixel((x, y)) + if pixel[3] > 128: + edge_pixels.append(pixel[:3]) + + if edge_pixels: + avg_brightness = sum(sum(pixel) for pixel in edge_pixels) / ( + len(edge_pixels) * 3 + ) + background_color = (32, 32, 32) if avg_brightness > 128 else (240, 240, 240) + else: + background_color = (255, 255, 255) + + background = Image.new("RGB", image.size, background_color) + background.paste(image, mask=image.getchannel("A")) + return background + + return image.convert("RGB") + + def is_jpeg_with_cuda( image_bytes: bytes = b"", gpu_image_decode: GPUImageDecodeMode = True ) -> bool: diff --git a/test/registered/unit/entrypoints/openai/test_serving_chat.py b/test/registered/unit/entrypoints/openai/test_serving_chat.py index 2500f0c9c..8ea5bc466 100644 --- a/test/registered/unit/entrypoints/openai/test_serving_chat.py +++ b/test/registered/unit/entrypoints/openai/test_serving_chat.py @@ -39,6 +39,9 @@ from sglang.srt.entrypoints.openai.serving_chat import ( from sglang.srt.environ import envs from sglang.srt.function_call.kimik3_format import TOOLS_CLOSE, TOOLS_OPEN from sglang.srt.managers.io_struct import GenerateReqInput +from sglang.srt.parser.jinja_template_utils import ( + jinja_template_may_reorder_tool_results, +) from sglang.srt.parser.template_detection import ReasoningToggleConfig from sglang.srt.utils import get_or_create_event_loop from sglang.test.ci.ci_register import register_cpu_ci @@ -80,6 +83,24 @@ _DSV4_OFFICIAL_ENCODER = ( 'DEFAULT_REASONING_EFFORT = "low"\n' ) +_TOOL_RESULT_REORDER_TEMPLATE = """ +{%- for assistant in messages + if assistant.role == 'assistant' and assistant.tool_calls -%} + {%- for tool_call in assistant.tool_calls -%} + {%- for result in messages + if result.role == 'tool' and result.tool_call_id == tool_call.id -%} + {%- for part in result.content -%} + {%- if part.type == 'text' -%} + {{- part.text -}} + {%- else -%} + {{- '<' + part.type + '>' -}} + {%- endif -%} + {%- endfor -%} + {%- endfor -%} + {%- endfor -%} +{%- endfor -%} +""" + def _create_dsv4_checkpoint(test_case: unittest.TestCase, source: str) -> str: model_dir = tempfile.TemporaryDirectory() @@ -163,6 +184,7 @@ class _MockTemplateManager: self.completion_template_name: Optional[str] = None self.reasoning_config = None self.force_reasoning = False + self.jinja_template_may_reorder_tool_results = False class ServingChatTestCase(unittest.TestCase): @@ -192,6 +214,166 @@ class ServingChatTestCase(unittest.TestCase): self.fastapi_request = Mock(spec=Request) self.fastapi_request.headers = {} + @staticmethod + def _render_tool_results_in_call_order(messages, **kwargs): + """Block-level tool_call_id association, like the GLM chat templates.""" + del kwargs + rendered = [] + index = 0 + while index < len(messages): + message = messages[index] + index += 1 + tool_calls = message.get("tool_calls") or [] + if message.get("role") != "assistant" or not tool_calls: + continue + run = [] + while index < len(messages) and messages[index].get("role") == "tool": + run.append(messages[index]) + index += 1 + by_id = {result.get("tool_call_id"): result for result in run} + for tool_call in tool_calls: + result = by_id.get(tool_call.get("id")) + if result is None: + continue + for part in result.get("content") or []: + if part.get("type") == "text": + rendered.append(part.get("text", "")) + else: + rendered.append(f"<{part.get('type')}>") + return "".join(rendered) + + @staticmethod + def _tool_round(call_ids, result_ids, part_type="image_url"): + assistant = { + "role": "assistant", + "content": "", + "tool_calls": [ + {"id": call_id, "function": {"name": call_id, "arguments": {}}} + for call_id in call_ids + ], + } + part_key = {"image_url": "image_url", "video_url": "video_url"}[part_type] + results = [ + { + "role": "tool", + "tool_call_id": result_id, + "content": [{"type": part_type, part_key: {"url": result_id}}], + } + for result_id in result_ids + ] + return [assistant] + results + + def test_canonicalize_tool_message_order_sorts_media_runs(self): + messages = self._tool_round( + ["call-a", "call-b"], ["call-b", "call-a"] + ) + self._tool_round(["call-c", "call-d"], ["call-d", "call-c"], "video_url") + + canonical = self.chat._canonicalize_tool_message_order(messages) + + self.assertEqual( + [ + message["tool_call_id"] + for message in canonical + if message.get("role") == "tool" + ], + ["call-a", "call-b", "call-c", "call-d"], + ) + + def test_canonicalize_tool_message_order_keeps_unassociable_runs(self): + cases = { + "unknown_id": (["call-a", "call-b"], ["call-b", "call-z"]), + "duplicate_result_id": (["call-a", "call-b"], ["call-b", "call-b"]), + "missing_call_id": ([None, "call-b"], ["call-b", None]), + } + for name, (call_ids, result_ids) in cases.items(): + with self.subTest(name=name): + messages = self._tool_round(call_ids, result_ids) + canonical = self.chat._canonicalize_tool_message_order(messages) + self.assertEqual(canonical, messages) + + def test_canonicalize_tool_message_order_keeps_text_only_runs(self): + messages = self._tool_round(["call-a", "call-b"], ["call-b", "call-a"]) + for message in messages[1:]: + message["content"] = [{"type": "text", "text": "done"}] + + canonical = self.chat._canonicalize_tool_message_order(messages) + + self.assertEqual(canonical, messages) + + def test_jinja_path_recovers_tool_result_images_only_when_template_needs_it(self): + self.template_manager.chat_template_name = None + self.template_manager.jinja_template_content_format = "openai" + self.template_manager.jinja_template_may_reorder_tool_results = ( + jinja_template_may_reorder_tool_results(_TOOL_RESULT_REORDER_TEMPLATE) + ) + self.assertTrue(self.template_manager.jinja_template_may_reorder_tool_results) + self.tm.tokenizer.apply_chat_template.side_effect = ( + self._render_tool_results_in_call_order + ) + request = ChatCompletionRequest( + model="x", + messages=[ + {"role": "user", "content": "inspect"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call-a", + "type": "function", + "function": {"name": "a", "arguments": {}}, + }, + { + "id": "call-b", + "type": "function", + "function": {"name": "b", "arguments": {}}, + }, + ], + }, + { + "role": "tool", + "tool_call_id": "call-b", + "content": [{"type": "image_url", "image_url": {"url": "image-b"}}], + }, + { + "role": "tool", + "tool_call_id": "call-a", + "content": [{"type": "image_url", "image_url": {"url": "image-a"}}], + }, + ], + ) + + result = self.chat._apply_jinja_template(request, None, is_multimodal=True) + + self.assertEqual( + [item.url for item in result.image_data], ["image-a", "image-b"] + ) + self.assertEqual(self.tm.tokenizer.apply_chat_template.call_count, 1) + rendered_messages = self.tm.tokenizer.apply_chat_template.call_args[0][0] + self.assertEqual( + [m.get("tool_call_id") for m in rendered_messages if "tool_call_id" in m], + ["call-a", "call-b"], + ) + + # An already-ordered request must produce the exact same render input. + ordered_request = ChatCompletionRequest( + model="x", + messages=request.messages[:2] + request.messages[2:][::-1], + ) + self.tm.tokenizer.apply_chat_template.reset_mock() + self.chat._apply_jinja_template(ordered_request, None, is_multimodal=True) + self.assertEqual( + rendered_messages, self.tm.tokenizer.apply_chat_template.call_args[0][0] + ) + + self.template_manager.jinja_template_may_reorder_tool_results = False + self.tm.tokenizer.apply_chat_template.reset_mock() + result = self.chat._apply_jinja_template(request, None, is_multimodal=True) + self.assertEqual( + [item.url for item in result.image_data], ["image-b", "image-a"] + ) + self.assertEqual(self.tm.tokenizer.apply_chat_template.call_count, 1) + def test_parsers_follow_the_control_plane_overlay(self): """Template detection records the parsers through `override`, so they answer from the bags; `ServerArgs` keeps the launcher's seed.""" diff --git a/test/registered/unit/entrypoints/openai/test_serving_completions.py b/test/registered/unit/entrypoints/openai/test_serving_completions.py index 4b7c73fc5..d926de73b 100644 --- a/test/registered/unit/entrypoints/openai/test_serving_completions.py +++ b/test/registered/unit/entrypoints/openai/test_serving_completions.py @@ -58,6 +58,7 @@ class _MockTemplateManager: self.completion_template_name: Optional[str] = ( None # Set to None to avoid template processing ) + self.jinja_template_may_reorder_tool_results = False class ServingCompletionTestCase(unittest.TestCase): diff --git a/test/registered/unit/entrypoints/openai/test_serving_embedding.py b/test/registered/unit/entrypoints/openai/test_serving_embedding.py index e2d0f1085..40b7d292d 100644 --- a/test/registered/unit/entrypoints/openai/test_serving_embedding.py +++ b/test/registered/unit/entrypoints/openai/test_serving_embedding.py @@ -97,6 +97,7 @@ class _MockTemplateManager: self.chat_template_name = None # None for embeddings usually self.jinja_template_content_format = "openai" self.completion_template_name = None + self.jinja_template_may_reorder_tool_results = False class ServingEmbeddingTestCase(unittest.TestCase): diff --git a/test/registered/unit/entrypoints/openai/utils.py b/test/registered/unit/entrypoints/openai/utils.py index 9957bbd77..1d57cdd9b 100644 --- a/test/registered/unit/entrypoints/openai/utils.py +++ b/test/registered/unit/entrypoints/openai/utils.py @@ -79,6 +79,7 @@ class MockTemplateManager: self.completion_template_name = None self.reasoning_config = None self.force_reasoning = False + self.jinja_template_may_reorder_tool_results = False def make_serving(*, is_multimodal: bool = False) -> OpenAIServingResponses: diff --git a/test/registered/unit/multimodal/test_processor_async_call_sites.py b/test/registered/unit/multimodal/test_processor_async_call_sites.py index 31b84d81e..fe42b96e1 100644 --- a/test/registered/unit/multimodal/test_processor_async_call_sites.py +++ b/test/registered/unit/multimodal/test_processor_async_call_sites.py @@ -27,6 +27,11 @@ _MULTIMODAL_ROOT = ( / "srt" / "multimodal" ) +if not _MULTIMODAL_ROOT.is_dir(): + raise RuntimeError( + f"multimodal processor tree not found at {_MULTIMODAL_ROOT}; " + "these tests must run from a full source checkout" + ) # The async helper and the sync body live side by side here by design. _EXEMPT = {"base_processor.py"} diff --git a/test/registered/unit/parser/test_jinja_template_utils.py b/test/registered/unit/parser/test_jinja_template_utils.py index d23b88fc3..0e08ba885 100644 --- a/test/registered/unit/parser/test_jinja_template_utils.py +++ b/test/registered/unit/parser/test_jinja_template_utils.py @@ -4,8 +4,10 @@ import unittest from sglang.srt.parser.jinja_template_utils import ( detect_jinja_template_content_format, + jinja_template_may_reorder_tool_results, process_content_for_template_format, ) +from sglang.srt.utils import VideoData from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -16,6 +18,41 @@ register_cpu_ci(est_time=6, suite="base-c-test-cpu") class TestTemplateContentFormatDetection(CustomTestCase): """Test template content format detection functionality.""" + def test_detect_tool_result_id_association(self): + attribute_template = """ + {% for message in messages %} + {{ message.tool_call_id }} + {% endfor %} + """ + item_template = "{{ messages[0]['tool_call_id'] }}" + get_template = "{{ messages[0].get('tool_call_id') }}" + select_template = ( + "{{ messages | selectattr('tool_call_id', 'equalto', 'call-a') | list }}" + ) + + self.assertTrue(jinja_template_may_reorder_tool_results(attribute_template)) + self.assertTrue(jinja_template_may_reorder_tool_results(item_template)) + self.assertTrue(jinja_template_may_reorder_tool_results(get_template)) + self.assertTrue(jinja_template_may_reorder_tool_results(select_template)) + + def test_sort_by_tool_call_id_value_is_not_association(self): + # sort/groupby order by the id string value, which message-order + # canonicalization cannot reproduce, so they must not activate it. + sort_template = "{{ messages | sort(attribute='tool_call_id') }}" + groupby_template = "{{ messages | groupby('tool_call_id') }}" + + self.assertFalse(jinja_template_may_reorder_tool_results(sort_template)) + self.assertFalse(jinja_template_may_reorder_tool_results(groupby_template)) + + def test_tool_call_id_text_does_not_enable_order_recovery(self): + self.assertFalse( + jinja_template_may_reorder_tool_results( + "{# tool_call_id is mentioned only in a comment #}{{ messages }}" + ) + ) + self.assertFalse(jinja_template_may_reorder_tool_results("{{{{ invalid")) + self.assertFalse(jinja_template_may_reorder_tool_results(None)) + def test_detect_llama4_openai_format(self): """Test detection of llama4-style template (should be 'openai' format).""" llama4_pattern = """ @@ -312,30 +349,38 @@ class TestTemplateContentFormatDetection(CustomTestCase): self.assertEqual(video_data[0], "http://example.com/v.mp4") self.assertEqual(result["content"][1], {"type": "video"}) - def test_process_content_video_with_max_dynamic_patch(self): - """Test video_url with max_dynamic_patch stores structured dict.""" + def test_process_content_video_structured_fields_become_video_data(self): + """video_url with mdp/fps-style fields lands in VideoData.preprocess_kwargs.""" msg_dict = { "role": "user", "content": [ { "type": "video_url", "video_url": { - "url": "http://example.com/v.mp4", + "url": "http://example.com/a.mp4", "max_dynamic_patch": 4, }, }, + { + "type": "video_url", + "video_url": { + "url": "http://example.com/b.mp4", + "fps": 1.5, + "max_frames": 16, + }, + }, ], } - image_data = [] video_data = [] - audio_data = [] - modalities = [] - result = process_content_for_template_format( - msg_dict, "openai", image_data, video_data, audio_data, modalities + process_content_for_template_format(msg_dict, "openai", [], video_data, [], []) + self.assertEqual( + [(item.url, item.preprocess_kwargs) for item in video_data], + [ + ("http://example.com/a.mp4", {"max_dynamic_patch": 4}), + ("http://example.com/b.mp4", {"fps": 1.5, "max_frames": 16}), + ], ) - self.assertEqual(len(video_data), 1) - self.assertIsInstance(video_data[0], dict) - self.assertEqual(video_data[0]["max_dynamic_patch"], 4) + self.assertIsInstance(video_data[0], VideoData) def test_process_content_v32_encoding(self): """Test v32 encoding mode flattens text and ignores structured content parts."""