[vlm][internVL] Support processor and embedding inputs for InternVL (#19127)

python/sglang/srt/models/internvl.py

@@ -616,6 +616,10 @@ class InternVLChatModel(nn.Module):
             image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`).
         """
         pixel_values = torch.cat([item.feature for item in items])
+        # If already precomputed embeddings (not raw pixel values), skip vision encoder.
+        # Normal pixel_values are 4D [N, C, H, W]; precomputed embeddings are 2D or 3D.
+        if pixel_values.dim() != 4:
+            return pixel_values
         image_features = self.extract_feature(pixel_values)
         return image_features
 
@@ -623,6 +627,9 @@ class InternVLChatModel(nn.Module):
         # items: each item corresponds to one video (recommended)
         # item.feature shape: [num_frames, 3, 448, 448]  (or [num_tiles, 3, 448, 448])
         pixel_values = torch.cat([item.feature for item in items], dim=0)
+        # If already precomputed embeddings, skip vision encoder.
+        if pixel_values.dim() != 4:
+            return pixel_values
         video_features = self.extract_feature(pixel_values)
         return video_features
 

python/sglang/srt/multimodal/processors/base_processor.py

@@ -182,6 +182,13 @@ class BaseMultimodalProcessor(ABC):
         self.server_args = server_args
         self.transport_mode = transport_mode
 
+        # Resolve tokenizer: some processors (e.g. InternVL) pass a tokenizer
+        # directly as _processor rather than a processor that wraps a tokenizer.
+        if hasattr(self._processor, "tokenizer"):
+            self._tokenizer = self._processor.tokenizer
+        else:
+            self._tokenizer = self._processor
+
         # FIXME: not accurate, model and image specific
         self.NUM_TOKEN_PER_FRAME = 330
 
@@ -252,7 +259,7 @@ class BaseMultimodalProcessor(ABC):
         Use prompt and img_grid_thw to build input_ids
         """
         if not isinstance(prompt, list):
-            prompt = self._processor.tokenizer.encode(prompt)
+            prompt = self._tokenizer.encode(prompt)
 
         img_token_id = self.IM_TOKEN_ID
         spatial_merge_size = self.spatial_merge_size
@@ -636,7 +643,7 @@ class BaseMultimodalProcessor(ABC):
         multimodal_tokens_pattern = multimodal_tokens.get_combined_regex()
         if isinstance(prompt, list) and return_text:
             assert len(prompt) and isinstance(prompt[0], int)
-            prompt = self._processor.tokenizer.decode(prompt)
+            prompt = self._tokenizer.decode(prompt)
         else:
             prompt = prompt
 
@@ -709,7 +716,7 @@ class BaseMultimodalProcessor(ABC):
         # Convert prompt into str
         if isinstance(prompt, list) and return_text:
             assert len(prompt) and isinstance(prompt[0], int)
-            prompt_str = self._processor.tokenizer.decode(prompt)
+            prompt_str = self._tokenizer.decode(prompt)
         else:
             assert isinstance(prompt, str)
             prompt_str = prompt
@@ -793,7 +800,7 @@ class BaseMultimodalProcessor(ABC):
         multimodal_tokens_pattern = multimodal_tokens.get_combined_regex()
         if isinstance(prompt, list) and return_text:
             assert len(prompt) and isinstance(prompt[0], int)
-            prompt = self._processor.tokenizer.decode(prompt)
+            prompt = self._tokenizer.decode(prompt)
         else:
             prompt = prompt
 
@@ -988,7 +995,7 @@ class BaseMultimodalProcessor(ABC):
         all_loaded_data = base_output.organize_results()
         # Handle text-only case
         if not all_loaded_data:
-            input_ids = self._processor.tokenizer(
+            input_ids = self._tokenizer(
                 base_output.input_text,
                 return_tensors="pt",
                 add_special_tokens=True,
@@ -1044,7 +1051,7 @@ class BaseMultimodalProcessor(ABC):
                 )
         # Fallback tokenization if no raw items were processed
         if input_ids is None:
-            input_ids = self._processor.tokenizer(
+            input_ids = self._tokenizer(
                 base_output.input_text,
                 return_tensors="pt",
                 add_special_tokens=True,

python/sglang/srt/multimodal/processors/internvl.py

@@ -9,11 +9,15 @@ import torch
 from decord import VideoReader, cpu, gpu
 from PIL import Image
 
-from sglang.srt.managers.schedule_batch import Modality, MultimodalDataItem
+from sglang.srt.managers.schedule_batch import (
+    Modality,
+    MultimodalDataItem,
+)
 from sglang.srt.models.interns1 import InternS1ForConditionalGeneration
 from sglang.srt.models.internvl import InternVLChatModel
 from sglang.srt.multimodal.processors.base_processor import (
     BaseMultimodalProcessor,
+    BaseMultiModalProcessorOutput,
     MultimodalSpecialTokens,
 )
 
@@ -255,9 +259,113 @@ class InternVLProcessor(BaseMultimodalProcessor):
         frames_per_video = max(1, max_total_frames // max(num_videos, 1))
         return max(1, min(int(requested), int(frames_per_video)))
 
+    @staticmethod
+    def _has_special_format(image_data, video_data):
+        """Check if any input items use processor_output or precomputed_embedding format."""
+        for data in list(image_data or []) + list(video_data or []):
+            if isinstance(data, dict) and data.get("format") in (
+                "processor_output",
+                "precomputed_embedding",
+            ):
+                return True
+        return False
+
+    async def _process_special_format(
+        self, image_data, video_data, input_text, request_obj, **kwargs
+    ):
+        """Handle processor_output and precomputed_embedding input formats.
+
+        Delegates to the base class process_and_combine_mm_data which has
+        built-in support for these formats.
+        """
+        # When user provides input_ids directly, input_text may be a list of ints
+        if isinstance(input_text, list):
+            user_input_ids = input_text
+            prompt = ""
+        else:
+            user_input_ids = None
+            prompt = input_text or ""
+
+        # When the prompt is empty (user provided input_ids directly),
+        # load_mm_data can't match multimodal tokens to data items.
+        # Build BaseMultiModalProcessorOutput directly from the dict items.
+        if not prompt and (image_data or video_data):
+            images = [d for d in (image_data or []) if isinstance(d, dict)]
+            videos = [d for d in (video_data or []) if isinstance(d, dict)]
+
+            # Raise if raw (non-dict) images/videos were silently filtered out.
+            # InternVL cannot process raw images without a text prompt because
+            # dynamic tiling and placeholder expansion require the prompt string.
+            raw_img_dropped = len(image_data or []) - len(images)
+            raw_vid_dropped = len(video_data or []) - len(videos)
+            if raw_img_dropped > 0 or raw_vid_dropped > 0:
+                raise ValueError(
+                    f"[internvl] Cannot process raw images/videos with pre-tokenized "
+                    f"input_ids. Provide multimodal data in 'processor_output' or "
+                    f"'precomputed_embedding' format, or use a text prompt instead. "
+                    f"(raw images dropped: {raw_img_dropped}, "
+                    f"raw videos dropped: {raw_vid_dropped})"
+                )
+
+            base_output = BaseMultiModalProcessorOutput(
+                input_text=prompt,
+                images=images,
+                videos=videos,
+            )
+        else:
+            base_output = self.load_mm_data(
+                prompt=prompt,
+                image_data=image_data,
+                video_data=video_data,
+                multimodal_tokens=self.mm_tokens,
+                discard_alpha_channel=True,
+            )
+
+        mm_items, input_ids_tensor, ret = self.process_and_combine_mm_data(
+            base_output, self.mm_tokens
+        )
+
+        # If user provided input_ids directly, use those and recompute offsets
+        if user_input_ids is not None:
+            input_ids_tensor = torch.tensor(user_input_ids, dtype=torch.long)
+            for mm_item in mm_items:
+                if (
+                    mm_item.modality == Modality.VIDEO
+                    and self.video_token_id is not None
+                ):
+                    mm_token_id = self.video_token_id
+                else:
+                    mm_token_id = self.img_context_token_id
+                mm_item.offsets = self.get_mm_items_offset(
+                    input_ids=input_ids_tensor,
+                    mm_token_id=mm_token_id,
+                )
+
+        return {
+            "input_ids": input_ids_tensor.flatten().tolist(),
+            "mm_items": mm_items,
+            "im_start_id": self.img_start_token_id,
+            "im_end_id": self.img_end_token_id,
+            "im_token_id": self.img_context_token_id,
+            "video_token_id": self.video_token_id,
+        }
+
     async def process_mm_data_async(
         self, image_data, input_text, request_obj, **kwargs
     ):
+        video_data = getattr(request_obj, "video_data", None) or []
+
+        # Handle processor_output and precomputed_embedding formats
+        if isinstance(input_text, list) or self._has_special_format(
+            image_data, video_data
+        ):
+            return await self._process_special_format(
+                image_data=image_data,
+                video_data=video_data,
+                input_text=input_text,
+                request_obj=request_obj,
+                **kwargs,
+            )
 
         is_internlm2 = self.llm_arch == "InternLM2ForCausalLM"
 

test/registered/vlm/test_vlm_input_format.py

@@ -349,5 +349,158 @@ class TestKimiVLImageUnderstandsImage(
 #         return dict(processor_output, format="processor_output")
 
 
+class TestInternVLUnderstandsImage(VLMInputTestBase, unittest.IsolatedAsyncioTestCase):
+    model_path = "OpenGVLab/InternVL2-2B"
+    chat_template = "internvl-2-5"
+
+    @classmethod
+    def setUpClass(cls):
+        assert cls.model_path is not None, "Set model_path in subclass"
+        assert cls.chat_template is not None, "Set chat_template in subclass"
+        cls.image_urls = [IMAGE_MAN_IRONING_URL, IMAGE_SGL_LOGO_URL]
+        cls.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        cls.main_image = []
+        for image_url in cls.image_urls:
+            response = requests.get(image_url)
+            cls.main_image.append(Image.open(BytesIO(response.content)))
+
+        # InternVL models (2, 3, 3.5) do not ship a standard HuggingFace
+        # Processor; AutoProcessor.from_pretrained returns a bare tokenizer.
+        # Use AutoTokenizer explicitly so the intent is clear.
+        from transformers import AutoTokenizer
+
+        cls.processor = AutoTokenizer.from_pretrained(
+            cls.model_path, trust_remote_code=True
+        )
+        cls._init_visual()
+
+    @classmethod
+    def _init_visual(cls):
+        model = AutoModel.from_pretrained(
+            cls.model_path, trust_remote_code=True, torch_dtype=torch.bfloat16
+        )
+        cls.vision_model = model.vision_model.eval().to(cls.device)
+        cls.mlp1 = model.mlp1.eval().to(cls.device)
+
+        config = model.config
+        cls.internvl_config = config
+        image_size = getattr(config, "force_image_size", None) or (
+            config.vision_config.image_size
+        )
+        patch_size = config.vision_config.patch_size
+        cls.num_image_token = int(
+            (image_size // patch_size) ** 2 * (config.downsample_ratio**2)
+        )
+        cls.internvl_image_size = image_size
+        cls.internvl_downsample_ratio = config.downsample_ratio
+        cls.internvl_ps_version = config.ps_version
+        cls.internvl_select_layer = config.select_layer
+
+        del model
+
+        def pixel_shuffle(x, scale_factor):
+            n, w, h, c = x.size()
+            x = x.view(n, w, int(h * scale_factor), int(c / scale_factor))
+            x = x.permute(0, 2, 1, 3).contiguous()
+            x = x.view(
+                n,
+                int(h * scale_factor),
+                int(w * scale_factor),
+                int(c / (scale_factor * scale_factor)),
+            )
+            if cls.internvl_ps_version != "v1":
+                x = x.permute(0, 2, 1, 3).contiguous()
+            return x
+
+        def visual_func(processor_output):
+            pixel_values = processor_output["pixel_values"].to(
+                cls.device, dtype=torch.bfloat16
+            )
+            if cls.internvl_select_layer == -1:
+                vit_embeds = cls.vision_model(
+                    pixel_values=pixel_values,
+                    output_hidden_states=False,
+                    return_dict=True,
+                ).last_hidden_state
+            else:
+                vit_embeds = cls.vision_model(
+                    pixel_values=pixel_values,
+                    output_hidden_states=True,
+                    return_dict=True,
+                ).hidden_states[cls.internvl_select_layer]
+            vit_embeds = vit_embeds[:, 1:, :]
+
+            h = w = int(vit_embeds.shape[1] ** 0.5)
+            vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], h, w, -1)
+            vit_embeds = pixel_shuffle(
+                vit_embeds, scale_factor=cls.internvl_downsample_ratio
+            )
+            vit_embeds = vit_embeds.reshape(
+                vit_embeds.shape[0], -1, vit_embeds.shape[-1]
+            )
+            vit_embeds = cls.mlp1(vit_embeds)
+            return vit_embeds
+
+        cls.visual = visual_func
+
+    def get_processor_output(self, req=None):
+        """Override to handle InternVL's custom preprocessing.
+
+        Uses shared ``image_to_pixel_values`` from ``internvl_utils`` for
+        image preprocessing (dynamic tiling + normalize) and expands
+        ``<IMG_CONTEXT>`` placeholders into ``<img>`` + context tokens +
+        ``</img>`` — mirroring the logic in
+        ``InternVLProcessor.process_internlm2_mm_data_async``.
+        """
+        from sglang.srt.multimodal.internvl_utils import image_to_pixel_values
+        from sglang.srt.multimodal.processors.internvl import InternVLProcessor
+
+        if req is None:
+            req = self.get_completion_request()
+        conv = generate_chat_conv(req, template_name=self.chat_template)
+        text = conv.get_prompt()
+
+        # Preprocess images using the shared utility (dynamic tiling +
+        # bicubic resize + ImageNet normalize), same pipeline as the engine.
+        all_pixel_values = []
+        num_patches_list = []
+        for img in self.main_image:
+            pv = image_to_pixel_values(
+                img,
+                input_size=self.internvl_image_size,
+                max_num_tiles=InternVLProcessor.IMAGE_MAX_NUM,
+                use_thumbnail=True,
+            )
+            all_pixel_values.append(pv)
+            num_patches_list.append(pv.shape[0])
+
+        pixel_values = torch.cat(all_pixel_values, dim=0).to(self.device)
+
+        # Expand each <IMG_CONTEXT> placeholder into <img> + <IMG_CONTEXT>*N + </img>.
+        # This mirrors InternVLProcessor.process_internlm2_mm_data_async.
+        ph = "<<<__IMG_PH__>>>"
+        expanded_text = text.replace(InternVLProcessor.IMG_CONTEXT, ph)
+        for num_patches in num_patches_list:
+            image_tokens = (
+                InternVLProcessor.IMG_START
+                + InternVLProcessor.IMG_CONTEXT * (self.num_image_token * num_patches)
+                + InternVLProcessor.IMG_END
+            )
+            expanded_text = expanded_text.replace(ph, image_tokens, 1)
+        # Remove any remaining placeholders (more placeholders than images)
+        expanded_text = expanded_text.replace(ph, "")
+
+        # Tokenize the expanded text
+        input_ids = self.processor(expanded_text, return_tensors="pt")["input_ids"]
+
+        return {
+            "input_ids": input_ids,
+            "pixel_values": pixel_values,
+        }, text
+
+    def _processor_output_image_data(self, processor_output):
+        return dict(processor_output, format="processor_output")
+
+
 if __name__ == "__main__":
     unittest.main()