[VLM] Support Video for InternVL3_5 (#15942)

Co-authored-by: luoyuan.luo <luoyuan.luo@antgroup.com>

python/sglang/srt/models/internvl.py

@@ -539,6 +539,7 @@ class InternVLChatModel(nn.Module):
 
         self.external_mm_data_embedding_funcs = {
             Modality.IMAGE: self.get_image_feature,
+            Modality.VIDEO: self.get_video_feature,
         }
 
         self.model = self.language_model.model
@@ -594,6 +595,13 @@ class InternVLChatModel(nn.Module):
         image_features = self.extract_feature(pixel_values)
         return image_features
 
+    def get_video_feature(self, items: List[MultimodalDataItem]):
+        # items: each item corresponds to one video (recommended)
+        # item.feature shape: [num_frames, 3, 448, 448]  (or [num_tiles, 3, 448, 448])
+        pixel_values = torch.cat([item.feature for item in items], dim=0)
+        video_features = self.extract_feature(pixel_values)
+        return video_features
+
     @torch.no_grad()
     def forward(
         self,

python/sglang/srt/multimodal/processors/internvl.py

@@ -1,6 +1,8 @@
 # Adapted from https://huggingface.co/OpenGVLab/InternVL2-4B/blob/main/modeling_intern_vit.py
 
+import logging
 from functools import lru_cache
+from typing import List
 
 import numpy as np
 import torch
@@ -15,26 +17,44 @@ from sglang.srt.multimodal.processors.base_processor import (
     MultimodalSpecialTokens,
 )
 
+logger = logging.getLogger(__name__)
 
-class InternVLImageProcessor(BaseMultimodalProcessor):
+
+class InternVLProcessor(BaseMultimodalProcessor):
     models = [InternVLChatModel, InternS1ForConditionalGeneration]
 
     IMAGENET_MEAN = [0.485, 0.456, 0.406]
     IMAGENET_STD = [0.229, 0.224, 0.225]
 
+    DEFAULT_VIDEO_NUM_FRAMES = 32
+    VIDEO_MAX_NUM = 1
+    VIDEO_USE_THUMBNAIL = False
+
+    CONTEXT_FALLBACK = 40960
+    CONTEXT_RESERVED = 256
+
+    # OpenAI multimodal placeholder tokens
+    IMAGE_PLACEHOLDER_TOKEN = "<image>"
+    VIDEO_PLACEHOLDER_TOKEN = "<video>"
+
+    IMG_START = "<img>"
+    IMG_END = "</img>"
+    IMG_CONTEXT = "<IMG_CONTEXT>"
+
     @staticmethod
     @lru_cache(maxsize=1)
     def _get_normalize_tensors(device="cuda", dtype=torch.float32):
         mean = torch.tensor(
-            InternVLImageProcessor.IMAGENET_MEAN, device=device, dtype=dtype
+            InternVLProcessor.IMAGENET_MEAN, device=device, dtype=dtype
         ).view(-1, 1, 1)
         std = torch.tensor(
-            InternVLImageProcessor.IMAGENET_STD, device=device, dtype=dtype
+            InternVLProcessor.IMAGENET_STD, device=device, dtype=dtype
         ).view(-1, 1, 1)
         return mean, std
 
     def __init__(self, hf_config, server_args, _image_processor, *args, **kwargs):
         super().__init__(hf_config, server_args, _image_processor, *args, **kwargs)
+
         image_size = (
             getattr(hf_config, "force_image_size", None)
             or hf_config.vision_config.image_size
@@ -45,88 +65,65 @@ class InternVLImageProcessor(BaseMultimodalProcessor):
         if isinstance(patch_size, list):
             patch_size = patch_size[0]
 
-        self.IMG_CONTEXT_TOKEN = "<IMG_CONTEXT>"
-        self.IMG_START_TOKEN = "<img>"
-        self.IMG_END_TOKEN = "</img>"
-        self.num_image_token = int(
-            (image_size // patch_size) ** 2 * (hf_config.downsample_ratio**2)
-        )
         if hasattr(self._processor, "tokenizer"):
             tokenizer = self._processor.tokenizer
         else:
             tokenizer = self._processor
         self.tokenizer = tokenizer
 
-        self.img_start_token_id = tokenizer.convert_tokens_to_ids(self.IMG_START_TOKEN)
+        llm_arch = hf_config.llm_config.architectures[0]
-        self.img_end_token_id = tokenizer.convert_tokens_to_ids(self.IMG_END_TOKEN)
+        self.llm_arch = llm_arch
+        video_token_map = {
+            "Qwen2ForCausalLM": "<|video_pad|>",
+            "Qwen3ForCausalLM": "<|video_pad|>",
+            "Qwen3MoeForCausalLM": "<|video_pad|>",
+            "GptOssForCausalLM": "<|reserved_200000|>",
+        }
+        self.VIDEO_CONTEXT_TOKEN = video_token_map.get(llm_arch, None)
+        self.video_token_id = (
+            tokenizer.convert_tokens_to_ids(self.VIDEO_CONTEXT_TOKEN)
+            if self.VIDEO_CONTEXT_TOKEN
+            else None
+        )
+
+        self.num_image_token = int(
+            (image_size // patch_size) ** 2 * (hf_config.downsample_ratio**2)
+        )
+
+        self.img_start_token_id = tokenizer.convert_tokens_to_ids(self.IMG_START)
+        self.img_end_token_id = tokenizer.convert_tokens_to_ids(self.IMG_END)
+
+        # Placeholder token use <image>/<video>
+        # Offset token id use IMG_CONTEXT / VIDEO_CONTEXT
         self.mm_tokens = MultimodalSpecialTokens(
-            image_token="<IMG_CONTEXT>",
+            image_token=self.IMAGE_PLACEHOLDER_TOKEN,
-            image_token_id=tokenizer.convert_tokens_to_ids(self.IMG_CONTEXT_TOKEN),
+            image_token_id=tokenizer.convert_tokens_to_ids(self.IMG_CONTEXT),
+            video_token=self.VIDEO_PLACEHOLDER_TOKEN,
+            video_token_id=self.video_token_id,
         ).build(_image_processor)
 
-    @staticmethod
+        # Cache token id for IMG_CONTEXT (used by both branches)
-    def get_index(bound, fps, max_frame, first_idx=0, num_segments=32):
+        self.img_context_token_id = tokenizer.convert_tokens_to_ids(self.IMG_CONTEXT)
-        if bound:
+
-            start, end = bound[0], bound[1]
+        # InternLM2 legacy multimodal tokens: use <IMG_CONTEXT> as placeholder
-        else:
+        self.mm_tokens_internlm2 = MultimodalSpecialTokens(
-            start, end = -100000, 100000
+            image_token=self.IMG_CONTEXT,
-        start_idx = max(first_idx, round(start * fps))
+            image_token_id=self.img_context_token_id,
-        end_idx = min(round(end * fps), max_frame)
+        ).build(_image_processor)
-        seg_size = float(end_idx - start_idx) / num_segments
+
-        frame_indices = np.array(
+        self.max_context_len = (
-            [
+            getattr(server_args, "context_length", None)
-                int(start_idx + (seg_size / 2) + np.round(seg_size * idx))
+            or getattr(server_args, "max_context_len", None)
-                for idx in range(num_segments)
+            or getattr(hf_config, "max_position_embeddings", None)
-            ]
+            or getattr(
+                getattr(hf_config, "llm_config", None), "max_position_embeddings", None
             )
-        return frame_indices
+            or self.CONTEXT_FALLBACK
-
-    @staticmethod
-    def load_video(video_path, bound=None, input_size=448, max_num=1, num_segments=32):
-        try:
-            vr = VideoReader(video_path, ctx=gpu(0), num_threads=1)
-            use_gpu = True
-        except (RuntimeError, OSError) as e:
-            print(
-                f"[WARNING] Load video on gpu decoding failed: {e}. Falling back to CPU."
         )
-            vr = VideoReader(video_path, ctx=cpu(0), num_threads=1)
-            use_gpu = False
-
-        max_frame = len(vr) - 1
-        fps = float(vr.get_avg_fps())
-
-        pixel_values_list = []
-        num_patches_list = []
-        frame_indices = InternVLImageProcessor.get_index(
-            bound, fps, max_frame, first_idx=0, num_segments=num_segments
-        )
-
-        mean, std = InternVLImageProcessor._get_normalize_tensors(device="cuda")
-
-        for frame_index in frame_indices:
-            # Load frame
-            frame = vr[frame_index]
-            if use_gpu:
-                img = frame.cuda().permute(2, 0, 1).float() / 255.0
-            else:
-                img_np = frame.asnumpy()
-                img = torch.from_numpy(img_np).permute(2, 0, 1).cuda().float() / 255.0
-
-            img = (img - mean) / std
-
-            tiles = InternVLImageProcessor.dynamic_preprocess(
-                img, image_size=input_size, max_num=max_num, use_thumbnail=True
-            )
-
-            pixel_values_list.append(tiles)
-            num_patches_list.append(tiles.shape[0])
-
-        pixel_values = torch.cat(pixel_values_list, dim=0)
-        return pixel_values, num_patches_list
 
     @staticmethod
     def dynamic_preprocess(tensor, image_size=448, max_num=12, use_thumbnail=False):
+        # Tensor: (C,H,W) float on GPU
         C, H, W = tensor.shape
         aspect_ratio = W / H
 
@@ -187,69 +184,249 @@ class InternVLImageProcessor(BaseMultimodalProcessor):
 
         return torch.stack(tiles).to(torch.bfloat16)
 
+    @staticmethod
+    def _open_video_reader(path: str) -> VideoReader:
+        try:
+            return VideoReader(path, ctx=gpu(0), num_threads=1)
+        except (RuntimeError, OSError) as e:
+            logger.warning(
+                "[internvl] VideoReader gpu decode failed (%s), fallback CPU", e
+            )
+            return VideoReader(path, ctx=cpu(0), num_threads=1)
+
+    def _ensure_placeholders_before_assistant(
+        self, prompt: str, placeholder: str, want: int
+    ) -> str:
+        if want <= 0:
+            return prompt
+        have = (prompt or "").count(placeholder)
+        missing = want - have
+        if missing <= 0:
+            return prompt
+
+        insert = "\n" + "\n".join([placeholder] * missing) + "\n"
+
+        marker = "<|im_start|>assistant"
+        idx = (prompt or "").rfind(marker)
+        if idx != -1:
+            return (prompt or "")[:idx] + insert + (prompt or "")[idx:]
+        return (prompt or "") + insert
+
+    def _token_len(self, text: str) -> int:
+        try:
+            ids = self.tokenizer(text, return_tensors="pt")["input_ids"].flatten()
+            return int(ids.numel())
+        except Exception:
+            return 0
+
+    def _resolve_video_num_frames(
+        self, *, requested: int, num_videos: int, text_len: int, image_tile_cnt: int
+    ) -> int:
+        if num_videos <= 0:
+            return 0
+        if not self.VIDEO_CONTEXT_TOKEN or not self.video_token_id:
+            return 0
+        image_tokens = image_tile_cnt * self.num_image_token
+        budget = (
+            int(self.max_context_len)
+            - int(text_len)
+            - int(image_tokens)
+            - int(self.CONTEXT_RESERVED)
+        )
+        if budget <= 0:
+            return 1
+        max_total_frames = max(1, budget // self.num_image_token)
+        frames_per_video = max(1, max_total_frames // max(num_videos, 1))
+        return max(1, min(int(requested), int(frames_per_video)))
+
     async def process_mm_data_async(
         self, image_data, input_text, request_obj, **kwargs
     ):
-        base_output = self.load_mm_data(
+
-            prompt=input_text,
+        is_internlm2 = self.llm_arch == "InternLM2ForCausalLM"
+
+        if is_internlm2:
+            return await self.process_internlm2_mm_data_async(
                 image_data=image_data,
-            multimodal_tokens=self.mm_tokens,
+                input_text=input_text,
+                request_obj=request_obj,
+                **kwargs,
+            )
+        else:
+            # Default branch uses OpenAI-style placeholders
+            return await self.process_qwen_mm_data_async(
+                image_data=image_data,
+                input_text=input_text,
+                request_obj=request_obj,
+                **kwargs,
+            )
+
+    async def process_qwen_mm_data_async(
+        self, image_data, input_text, request_obj, **kwargs
+    ):
+        # Qwen/Qwen3 branch: OpenAI-style placeholders <image>/<video>
+        prompt = input_text or ""
+        video_data = getattr(request_obj, "video_data", None) or []
+
+        if image_data:
+            prompt = self._ensure_placeholders_before_assistant(
+                prompt, self.IMAGE_PLACEHOLDER_TOKEN, len(image_data)
+            )
+        if video_data:
+            prompt = self._ensure_placeholders_before_assistant(
+                prompt, self.VIDEO_PLACEHOLDER_TOKEN, len(video_data)
+            )
+
+        logger.info(
+            "[internvl][qwen] placeholders image=%d video=%d",
+            prompt.count(self.IMAGE_PLACEHOLDER_TOKEN),
+            prompt.count(self.VIDEO_PLACEHOLDER_TOKEN),
+        )
+
+        base_output = self.load_mm_data(
+            prompt=prompt,
+            image_data=image_data,
+            video_data=video_data,
+            multimodal_tokens=self.mm_tokens,  # expects <image>/<video>
             discard_alpha_channel=True,
         )
 
-        num_patches_list = []
+        logger.info(
-        pixel_values = []
+            "[internvl][qwen] loaded images=%d videos=%d",
+            len(base_output.images),
+            len(base_output.videos),
+        )
 
-        mean, std = InternVLImageProcessor._get_normalize_tensors(device="cuda")
+        mean, std = self._get_normalize_tensors(device="cuda")
 
-        # Process each input with allocated frames
+        # ----- Images -> tiles -----
-        for image_index, image in enumerate(base_output.images):
+        num_patches_list: List[int] = []
-            try:
+        pixel_values_list: List[torch.Tensor] = []
-                # TODO: video input
+
-                # Convert PIL to GPU tensor
+        for image in base_output.images:
             if isinstance(image, Image.Image):
                 img_np = np.array(image.convert("RGB"))
                 tensor = (
                     torch.from_numpy(img_np).permute(2, 0, 1).cuda().float() / 255.0
                 )
             else:
-                    tensor = image.cuda()  # assume already tensor
+                tensor = image.cuda()
 
             tensor = (tensor - mean) / std
             tiles = self.dynamic_preprocess(
                 tensor, image_size=448, max_num=12, use_thumbnail=True
             )
+            pixel_values_list.append(tiles)
+            num_patches_list.append(int(tiles.shape[0]))
 
-                pixel_values.append(tiles)
+        if image_data and not pixel_values_list:
-                num_patches_list.append(tiles.shape[0])
+            raise ValueError(
-
+                "[internvl][qwen] image_data provided but no images parsed from prompt placeholders"
-            except Exception as e:
-                print(f"[Error] Failed to process image {image_index}: {e}")
-                return None
-
-        # Concatenate all
-        pixel_values = torch.cat(pixel_values, dim=0)
-
-        original_placeholder = "<<<__IMG_CONTEXT_PLACEHOLDER__>>>"
-
-        input_text = base_output.input_text.replace(
-            self.IMG_CONTEXT_TOKEN, original_placeholder
             )
 
-        input_text_updated = input_text
+        image_tensor = (
+            torch.cat(pixel_values_list, dim=0) if pixel_values_list else None
+        )
+
+        # ----- Videos -> frame tiles (optional) -----
+        video_tensor = None
+        video_patch_lists = []
+        video_pixel_values = []
+
+        requested_frames = int(
+            kwargs.get("video_num_frames", self.DEFAULT_VIDEO_NUM_FRAMES)
+        )
+        num_frames = self._resolve_video_num_frames(
+            requested=requested_frames,
+            num_videos=len(base_output.videos),
+            text_len=self._token_len(base_output.input_text or prompt),
+            image_tile_cnt=int(sum(num_patches_list)) if num_patches_list else 0,
+        )
+
+        if base_output.videos and num_frames > 0 and self.video_token_id is not None:
+            for video in base_output.videos:
+                vr = (
+                    video
+                    if isinstance(video, VideoReader)
+                    else self._open_video_reader(str(video))
+                )
+                max_frame = len(vr) - 1
+                frame_indices = (
+                    [0]
+                    if num_frames == 1
+                    else np.linspace(0, max_frame, num=num_frames, dtype=int).tolist()
+                )
+
+                per_video_tiles = []
+                per_video_patch_cnt = []
+                for fi in frame_indices:
+                    frame = vr[int(fi)]
+                    img_np = (
+                        frame.asnumpy()
+                        if hasattr(frame, "asnumpy")
+                        else np.array(frame)
+                    )
+                    frame_t = (
+                        torch.from_numpy(img_np).permute(2, 0, 1).cuda().float() / 255.0
+                    )
+                    frame_t = (frame_t - mean) / std
+
+                    tiles = self.dynamic_preprocess(
+                        frame_t,
+                        image_size=448,
+                        max_num=self.VIDEO_MAX_NUM,
+                        use_thumbnail=self.VIDEO_USE_THUMBNAIL,
+                    )
+                    per_video_tiles.append(tiles)
+                    per_video_patch_cnt.append(int(tiles.shape[0]))
+
+                pv = torch.cat(per_video_tiles, dim=0)
+                video_pixel_values.append(pv)
+                video_patch_lists.append(per_video_patch_cnt)
+
+            video_tensor = (
+                torch.cat(video_pixel_values, dim=0) if video_pixel_values else None
+            )
+
+        # ----- Build prompt text with <img> + CONTEXT*n + </img> -----
+        img_ph = "<<<__IMG_PLACEHOLDER__>>>"
+        vid_ph = "<<<__VID_PLACEHOLDER__>>>"
+
+        input_text_mid = base_output.input_text or prompt
+        input_text_mid = input_text_mid.replace(self.IMAGE_PLACEHOLDER_TOKEN, img_ph)
+
+        if self.VIDEO_CONTEXT_TOKEN and self.video_token_id is not None:
+            input_text_mid = input_text_mid.replace(
+                self.VIDEO_PLACEHOLDER_TOKEN, vid_ph
+            )
+        else:
+            input_text_mid = input_text_mid.replace(self.VIDEO_PLACEHOLDER_TOKEN, "")
+
+        input_text_updated = input_text_mid
+
+        # Expand images
         for num_patches in num_patches_list:
             image_tokens = (
-                self.IMG_START_TOKEN
+                self.IMG_START
-                + self.IMG_CONTEXT_TOKEN * self.num_image_token * num_patches
+                + (self.IMG_CONTEXT * (self.num_image_token * int(num_patches)))
-                + self.IMG_END_TOKEN
+                + self.IMG_END
-            )
-            input_text_updated = input_text_updated.replace(
-                original_placeholder, image_tokens, 1
             )
+            input_text_updated = input_text_updated.replace(img_ph, image_tokens, 1)
 
-        input_text_updated = input_text_updated.replace(
+        # Expand videos (each frame is one <img>...</img>)
-            original_placeholder, self.IMG_CONTEXT_TOKEN
+        if video_patch_lists and self.VIDEO_CONTEXT_TOKEN:
+            for frame_patch_list in video_patch_lists:
+                frame_lines = []
+                for i, patch_cnt in enumerate(frame_patch_list):
+                    ctx_cnt = int(self.num_image_token) * int(patch_cnt)
+                    frame_tokens = (
+                        self.IMG_START
+                        + (self.VIDEO_CONTEXT_TOKEN * ctx_cnt)
+                        + self.IMG_END
                     )
+                    frame_lines.append(f"Frame {i+1}: {frame_tokens}")
+                video_tokens = "\n".join(frame_lines) + "\n"
+                input_text_updated = input_text_updated.replace(vid_ph, video_tokens, 1)
 
         # Tokenize
         input_ids_tensor = self.tokenizer(input_text_updated, return_tensors="pt")[
@@ -257,24 +434,147 @@ class InternVLImageProcessor(BaseMultimodalProcessor):
         ].flatten()
         input_ids = input_ids_tensor.tolist()
 
-        # Get image token offsets
+        # Offsets
+        image_offsets = []
+        if image_tensor is not None:
             image_offsets = self.get_mm_items_offset(
                 input_ids=input_ids_tensor.to("cuda"),
-            mm_token_id=self.mm_tokens.image_token_id,
+                mm_token_id=self.img_context_token_id,
             )
 
-        items = [
+        video_offsets = []
+        if video_tensor is not None and self.video_token_id is not None:
+            video_offsets = self.get_mm_items_offset(
+                input_ids=input_ids_tensor.to("cuda"),
+                mm_token_id=self.video_token_id,
+            )
+
+        items = []
+        if image_tensor is not None:
+            items.append(
                 MultimodalDataItem(
-                feature=pixel_values,
+                    feature=image_tensor, modality=Modality.IMAGE, offsets=image_offsets
-                modality=Modality.IMAGE,
+                )
-                offsets=image_offsets,
+            )
+        if video_tensor is not None:
+            items.append(
+                MultimodalDataItem(
+                    feature=video_tensor, modality=Modality.VIDEO, offsets=video_offsets
+                )
             )
-        ]
 
         return {
             "input_ids": input_ids,
             "mm_items": items,
             "im_start_id": self.img_start_token_id,
             "im_end_id": self.img_end_token_id,
-            "im_token_id": self.mm_tokens.image_token_id,
+            "im_token_id": self.img_context_token_id,
+            "video_token_id": self.video_token_id,
+        }
+
+    async def process_internlm2_mm_data_async(
+        self, image_data, input_text, request_obj, **kwargs
+    ):
+        # InternLM2 branch: legacy placeholder <IMG_CONTEXT> (stable for InternLM2 prompt behavior)
+        prompt = input_text or ""
+        video_data = getattr(request_obj, "video_data", None) or []
+        if video_data:
+            logger.warning(
+                "[internvl][internlm2] video input ignored for InternLM2 branch"
+            )
+
+        # Convert any OpenAI-style <image> into <IMG_CONTEXT>
+        prompt = prompt.replace(self.IMAGE_PLACEHOLDER_TOKEN, self.IMG_CONTEXT)
+
+        if image_data:
+            prompt = self._ensure_placeholders_before_assistant(
+                prompt, self.IMG_CONTEXT, len(image_data)
+            )
+
+        logger.info(
+            "[internvl][internlm2] placeholders img_context=%d",
+            prompt.count(self.IMG_CONTEXT),
+        )
+
+        base_output = self.load_mm_data(
+            prompt=prompt,
+            image_data=image_data,
+            multimodal_tokens=self.mm_tokens_internlm2,  # expects <IMG_CONTEXT>
+            discard_alpha_channel=True,
+        )
+
+        mean, std = self._get_normalize_tensors(device="cuda")
+
+        num_patches_list: List[int] = []
+        pixel_values_list: List[torch.Tensor] = []
+
+        for image in base_output.images:
+            if isinstance(image, Image.Image):
+                img_np = np.array(image.convert("RGB"))
+                tensor = (
+                    torch.from_numpy(img_np).permute(2, 0, 1).cuda().float() / 255.0
+                )
+            else:
+                tensor = image.cuda()
+
+            tensor = (tensor - mean) / std
+            tiles = self.dynamic_preprocess(
+                tensor, image_size=448, max_num=12, use_thumbnail=True
+            )
+            pixel_values_list.append(tiles)
+            num_patches_list.append(int(tiles.shape[0]))
+
+        if image_data and not pixel_values_list:
+            raise ValueError(
+                "[internvl][internlm2] image_data provided but no images parsed from prompt placeholders"
+            )
+
+        pixel_values = (
+            torch.cat(pixel_values_list, dim=0) if pixel_values_list else None
+        )
+
+        # Expand each <IMG_CONTEXT> into <img> + <IMG_CONTEXT>*N + </img>
+        ph = "<<<__IMG_CONTEXT_PLACEHOLDER__>>>"
+        input_text_base = (base_output.input_text or prompt).replace(
+            self.IMG_CONTEXT, ph
+        )
+
+        input_text_updated = input_text_base
+        for num_patches in num_patches_list:
+            image_tokens = (
+                self.IMG_START
+                + (self.IMG_CONTEXT * (self.num_image_token * int(num_patches)))
+                + self.IMG_END
+            )
+            input_text_updated = input_text_updated.replace(ph, image_tokens, 1)
+
+        # Tokenize
+        input_ids_tensor = self.tokenizer(input_text_updated, return_tensors="pt")[
+            "input_ids"
+        ].flatten()
+        input_ids = input_ids_tensor.tolist()
+
+        # Offsets
+        image_offsets = []
+        if pixel_values is not None:
+            image_offsets = self.get_mm_items_offset(
+                input_ids=input_ids_tensor.to("cuda"),
+                mm_token_id=self.img_context_token_id,
+            )
+
+        items = []
+        if pixel_values is not None:
+            items.append(
+                MultimodalDataItem(
+                    feature=pixel_values, modality=Modality.IMAGE, offsets=image_offsets
+                )
+            )
+
+        return {
+            "input_ids": input_ids,
+            "mm_items": items,
+            "im_start_id": self.img_start_token_id,
+            "im_end_id": self.img_end_token_id,
+            "im_token_id": self.img_context_token_id,
+            "video_token_id": self.video_token_id,
         }