[vlm][minicpm] support input formats of processor output and embedding (#19614)

python/sglang/srt/models/minicpmo.py

@@ -44,6 +44,7 @@ from sglang.srt.managers.mm_utils import (
 )
 from sglang.srt.managers.schedule_batch import (
     MultimodalDataItem,
+    MultimodalInputFormat,
     MultimodalInputs,
     flatten_nested_list,
 )
@@ -1803,6 +1804,10 @@ class MiniCPMO(MiniCPMBaseModel):
         return audio_embs
 
     def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
+        if items and items[0].format == MultimodalInputFormat.PRECOMPUTED_EMBEDDING:
+            result = torch.cat([item.feature for item in items])
+            return result.reshape(-1, result.shape[-1])
+
         # list of tensors
         pixel_values = flatten_nested_list([item.feature for item in items])
         tgt_sizes = torch.stack(

python/sglang/srt/models/minicpmv.py

@@ -51,7 +51,11 @@ from sglang.srt.managers.mm_utils import (
     MultiModalityDataPaddingPatternTokenPairs,
     general_mm_embed_routine,
 )
-from sglang.srt.managers.schedule_batch import MultimodalDataItem, MultimodalInputs
+from sglang.srt.managers.schedule_batch import (
+    MultimodalDataItem,
+    MultimodalInputFormat,
+    MultimodalInputs,
+)
 from sglang.srt.model_executor.forward_batch_info import ForwardBatch
 from sglang.srt.model_loader.utils import set_default_torch_dtype
 from sglang.srt.model_loader.weight_utils import default_weight_loader
@@ -939,6 +943,10 @@ class MiniCPMV2_6(MiniCPMBaseModel):
         return vision_embedding
 
     def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
+        if items and items[0].format == MultimodalInputFormat.PRECOMPUTED_EMBEDDING:
+            result = torch.cat([item.feature for item in items])
+            return result.reshape(-1, result.shape[-1])
+
         # list of tensors
         pixel_values = flatten_nested_list([item.feature for item in items])
         tgt_sizes = torch.stack(
@@ -1097,6 +1105,10 @@ class MiniCPMV4_0(MiniCPMBaseModel):
         return vision_embedding
 
     def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
+        if items and items[0].format == MultimodalInputFormat.PRECOMPUTED_EMBEDDING:
+            result = torch.cat([item.feature for item in items])
+            return result.reshape(-1, result.shape[-1])
+
         # list of tensors
         pixel_values = flatten_nested_list([item.feature for item in items])
         tgt_sizes = torch.stack(
@@ -1259,6 +1271,10 @@ class MiniCPMV4_5(MiniCPMBaseModel):
         return vision_embedding
 
     def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
+        if items and items[0].format == MultimodalInputFormat.PRECOMPUTED_EMBEDDING:
+            result = torch.cat([item.feature for item in items])
+            return result.reshape(-1, result.shape[-1])
+
         # list of tensors
         pixel_values = flatten_nested_list([item.feature for item in items])
         tgt_sizes = torch.stack(

python/sglang/srt/multimodal/processors/minicpm.py

@@ -2,11 +2,15 @@ from typing import List, Union
 
 import torch
 
-from sglang.srt.managers.schedule_batch import Modality, MultimodalDataItem
+from sglang.srt.managers.schedule_batch import (
+    Modality,
+    MultimodalDataItem,
+)
 from sglang.srt.models.minicpmo import MiniCPMO
 from sglang.srt.models.minicpmv import MiniCPMV
 from sglang.srt.multimodal.processors.base_processor import (
     BaseMultimodalProcessor,
+    BaseMultiModalProcessorOutput,
     MultimodalSpecialTokens,
 )
 
@@ -34,6 +38,137 @@ class MiniCPMMultimodalProcessor(BaseMultimodalProcessor):
             image_token_id=self.im_token_id,
         ).build(_processor)
 
+    @staticmethod
+    def _has_special_format(image_data, audio_data):
+        """Check if any input items use processor_output or precomputed_embedding format."""
+        for data in list(image_data or []) + list(audio_data or []):
+            if isinstance(data, dict) and data.get("format") in (
+                "processor_output",
+                "precomputed_embedding",
+            ):
+                return True
+        return False
+
+    async def _process_special_format(
+        self, image_data, audio_data, input_text, request_obj, **kwargs
+    ):
+        """Handle processor_output and precomputed_embedding input formats.
+
+        Delegates to the base class process_and_combine_mm_data which has
+        built-in support for these formats.
+        """
+        if isinstance(input_text, list):
+            user_input_ids = input_text
+            prompt = ""
+        else:
+            user_input_ids = None
+            prompt = input_text or ""
+
+        # Normalize dicts: the HF MiniCPM processor returns "tgt_sizes" (plural)
+        # but the base class ATTR_NAME_TO_MODALITY maps "tgt_size" (singular).
+        # Also flatten the nested batch dimension so the structure matches
+        # what the NORMAL path produces (flat list of per-patch tensors).
+        normalized_images = []
+        for d in image_data or []:
+            if isinstance(d, dict):
+                d = dict(d)
+                if "tgt_sizes" in d and "tgt_size" not in d:
+                    d["tgt_size"] = d.pop("tgt_sizes")
+                if d.get("format") == "processor_output":
+                    pixel_values = d.get("pixel_values")
+                    tgt_size = d.get("tgt_size")
+                    if pixel_values is not None and tgt_size is not None:
+                        pv_flat, ts_flat = [], []
+                        for pixel_b, tgt_b in zip(pixel_values, tgt_size):
+                            if isinstance(pixel_b, (list, tuple)):
+                                for pixel_n, tgt_n in zip(pixel_b, tgt_b):
+                                    pv_flat.append(pixel_n)
+                                    ts_flat.append(tgt_n)
+                            else:
+                                pv_flat.append(pixel_b)
+                                ts_flat.append(tgt_b)
+                        d["pixel_values"] = pv_flat
+                        d["tgt_size"] = ts_flat
+                normalized_images.append(d)
+            else:
+                normalized_images.append(d)
+
+        normalized_audios = list(audio_data or [])
+
+        if not prompt and (normalized_images or normalized_audios):
+            images = [d for d in normalized_images if isinstance(d, dict)]
+            audios = [d for d in normalized_audios if isinstance(d, dict)]
+
+            raw_img_dropped = len(normalized_images) - len(images)
+            raw_aud_dropped = len(normalized_audios) - len(audios)
+            if raw_img_dropped > 0 or raw_aud_dropped > 0:
+                raise ValueError(
+                    f"[minicpm] Cannot process raw media with pre-tokenized "
+                    f"input_ids. Provide multimodal data in 'processor_output' or "
+                    f"'precomputed_embedding' format, or use a text prompt instead. "
+                    f"(raw images dropped: {raw_img_dropped}, "
+                    f"raw audios dropped: {raw_aud_dropped})"
+                )
+
+            base_output = BaseMultiModalProcessorOutput(
+                input_text=prompt,
+                images=images,
+                audios=audios,
+            )
+        else:
+            base_output = self.load_mm_data(
+                prompt=prompt,
+                image_data=normalized_images,
+                audio_data=audio_data,
+                multimodal_tokens=self.mm_tokens,
+            )
+
+        if base_output is None:
+            return None
+
+        mm_items, input_ids_tensor, ret = self.process_and_combine_mm_data(
+            base_output, self.mm_tokens
+        )
+
+        if user_input_ids is not None:
+            input_ids_tensor = torch.tensor(user_input_ids, dtype=torch.long)
+            for mm_item in mm_items:
+                if mm_item.modality == Modality.IMAGE:
+                    image_offsets = self.get_mm_items_offset_by_pair(
+                        input_ids=input_ids_tensor,
+                        mm_start_id=self.im_start_id,
+                        mm_end_id=self.im_end_id,
+                    )
+                    slice_offsets = self.get_mm_items_offset_by_pair(
+                        input_ids=input_ids_tensor,
+                        mm_start_id=self.slice_start_id,
+                        mm_end_id=self.slice_end_id,
+                    )
+                    image_offsets.extend(slice_offsets)
+                    mm_item.offsets = sorted(image_offsets)
+                elif mm_item.modality == Modality.AUDIO:
+                    if (
+                        self.audio_start_id is not None
+                        and self.audio_end_id is not None
+                    ):
+                        mm_item.offsets = self.get_mm_items_offset_by_pair(
+                            input_ids=input_ids_tensor,
+                            mm_start_id=self.audio_start_id,
+                            mm_end_id=self.audio_end_id,
+                        )
+
+        return {
+            "mm_items": mm_items,
+            "input_ids": input_ids_tensor.flatten().tolist(),
+            "audio_start_id": self.audio_start_id,
+            "audio_end_id": self.audio_end_id,
+            "im_token_id": self.im_token_id,
+            "im_start_id": self.im_start_id,
+            "im_end_id": self.im_end_id,
+            "slice_start_id": self.slice_start_id,
+            "slice_end_id": self.slice_end_id,
+        }
+
     async def process_mm_data_async(
         self,
         image_data: List[Union[str, bytes]],
@@ -42,6 +177,17 @@ class MiniCPMMultimodalProcessor(BaseMultimodalProcessor):
         request_obj,
         **kwargs,
     ):
+        if isinstance(input_text, list) or self._has_special_format(
+            image_data, audio_data
+        ):
+            return await self._process_special_format(
+                image_data=image_data,
+                audio_data=audio_data,
+                input_text=input_text,
+                request_obj=request_obj,
+                **kwargs,
+            )
+
         base_output = self.load_mm_data(
             prompt=input_text,
             audio_data=audio_data,

test/registered/vlm/test_vlm_input_format.py

@@ -502,5 +502,87 @@ class TestInternVLUnderstandsImage(VLMInputTestBase, unittest.IsolatedAsyncioTes
         return dict(processor_output, format="processor_output")
 
 
+class TestMiniCPMVUnderstandsImage(VLMInputTestBase, unittest.IsolatedAsyncioTestCase):
+    model_path = "openbmb/MiniCPM-V-4"
+    chat_template = "minicpmv"
+
+    @classmethod
+    def setUpClass(cls):
+        assert cls.model_path is not None, "Set model_path in subclass"
+        assert cls.chat_template is not None, "Set chat_template in subclass"
+        cls.image_urls = [IMAGE_MAN_IRONING_URL, IMAGE_SGL_LOGO_URL]
+        cls.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        cls.main_image = []
+        for image_url in cls.image_urls:
+            response = requests.get(image_url)
+            cls.main_image.append(Image.open(BytesIO(response.content)))
+
+        cls.processor = AutoProcessor.from_pretrained(
+            cls.model_path, trust_remote_code=True
+        )
+        cls._init_visual()
+
+    @classmethod
+    def _init_visual(cls):
+        model = AutoModel.from_pretrained(
+            cls.model_path, trust_remote_code=True, torch_dtype=torch.bfloat16
+        )
+        cls.vpm_model = model.vpm.eval().to(cls.device)
+        cls.resampler_model = model.resampler.eval().to(cls.device)
+        del model
+
+        def visual_func(processor_output):
+            pixel_values = processor_output["pixel_values"]
+            tgt_sizes = processor_output["tgt_sizes"]
+
+            pixel_values_flat = []
+            tgt_sizes_flat = []
+            for pixel_b, tgt_b in zip(pixel_values, tgt_sizes):
+                if isinstance(pixel_b, (list, tuple)):
+                    for pixel_n, tgt_n in zip(pixel_b, tgt_b):
+                        pixel_values_flat.append(pixel_n)
+                        tgt_sizes_flat.append(tgt_n)
+                else:
+                    pixel_values_flat.append(pixel_b)
+                    tgt_sizes_flat.append(tgt_b)
+
+            tgt_sizes_tensor = torch.stack(tgt_sizes_flat, dim=0)
+            device = cls.vpm_model.embeddings.position_embedding.weight.device
+            dtype = cls.vpm_model.embeddings.position_embedding.weight.dtype
+
+            all_pixel_values_lst = [
+                i.flatten(end_dim=1).permute(1, 0) for i in pixel_values_flat
+            ]
+            max_patches = int(
+                (tgt_sizes_tensor[:, 0] * tgt_sizes_tensor[:, 1]).max().item()
+            )
+            all_pixel_values = torch.nn.utils.rnn.pad_sequence(
+                all_pixel_values_lst, batch_first=True, padding_value=0.0
+            )
+            B, L, _ = all_pixel_values.shape
+            all_pixel_values = all_pixel_values.permute(0, 2, 1).reshape(B, 3, -1, L)
+            patch_attn_mask = torch.zeros(
+                (B, 1, max_patches), dtype=torch.bool, device=device
+            )
+            tgt_sizes_dev = tgt_sizes_tensor.to(device)
+            mask_shapes = tgt_sizes_dev[:, 0] * tgt_sizes_dev[:, 1]
+            patch_attn_mask[:, 0, :] = torch.arange(
+                max_patches, device=device
+            ).unsqueeze(0) < mask_shapes.unsqueeze(1)
+
+            vision_output = cls.vpm_model(
+                all_pixel_values.type(dtype),
+                patch_attention_mask=patch_attn_mask,
+                tgt_sizes=tgt_sizes_tensor,
+            )
+            vision_embedding = vision_output.last_hidden_state
+            return cls.resampler_model(vision_embedding, tgt_sizes_tensor)
+
+        cls.visual = visual_func
+
+    def _processor_output_image_data(self, processor_output):
+        return dict(processor_output, format="processor_output")
+
+
 if __name__ == "__main__":
     unittest.main()