[VLM] accept precomputed multimodal metadata (#26101)

python/sglang/srt/multimodal/processors/base_processor.py

@@ -1080,28 +1080,53 @@ class BaseMultimodalProcessor(ABC):
         self, data_dict: dict, modality: Modality = None
     ) -> List[MultimodalDataItem]:
         """
-        Create mm_items from processor output. Initially creates one item per modality;
+        Create mm_items from processor output.
-        these are later split into per-image/video items by get_new_expanded_mm_items.
 
-        Note that the data_dict can be passed via offline engine api
+        Initially creates one item per modality; these are later split into per-image/video items by get_new_expanded_mm_items.
+
+        Note that the data_dict can be hf processor output, or passed via offline engine api
+
+        Args:
+            modality: if provided, force the data into a single MultimodalDataItem of that modality
         """
 
+        # universal getter for data_dict
+        get_data_value = (
+            data_dict.get
+            if hasattr(data_dict, "get")
+            else lambda name, default=None: getattr(data_dict, name, default)
+        )
+
+        # decide explicitly-set modality
+        explicit_modality = modality
+        modality_value = get_data_value("modality")
+        if explicit_modality is None and modality_value is not None:
+            explicit_modality = (
+                modality_value
+                if isinstance(modality_value, Modality)
+                else Modality.from_str(str(modality_value))
+            )
+
         items: dict[Modality, MultimodalDataItem] = {}
         for attr_name, value in data_dict.items():
-            if attr_name == "input_ids":
+            if attr_name in (
+                "input_ids",
+                "format",
+                "modality",
+                "hash",
+                "pad_value",
+                "offsets",
+            ):
+                # metadata fields need explicit handling, skip generic item.set
                 continue
 
             # Get modality for this attribute
-            current_modality = modality or self.ATTR_NAME_TO_MODALITY.get(attr_name)
+            current_modality = explicit_modality or self.ATTR_NAME_TO_MODALITY.get(
+                attr_name
+            )
 
             if attr_name == "precomputed_embeddings":
-                modality_str = data_dict.get("modality")
+                current_modality = current_modality or Modality.IMAGE
-                current_modality = Modality.IMAGE
-                if modality_str:
-                    try:
-                        current_modality = Modality.from_str(modality_str)
-                    except ValueError:
-                        pass
 
             if current_modality:
                 # Create item if needed
@@ -1115,6 +1140,30 @@ class BaseMultimodalProcessor(ABC):
 
                 items[current_modality].set(attr_name, value)
 
+        # deal with metadata fields when data_dict is preprocessed input: convert from tensor to expected python types
+        # the attribution of the metadata fields is only clear when number of MultimodalDataItem is 1
+        if len(items) == 1:
+            item = next(iter(items.values()))
+
+            # adjust offset
+            offsets = get_data_value("offsets")
+            if offsets is not None:
+                if isinstance(offsets, torch.Tensor):
+                    offsets = offsets.detach().cpu().tolist()
+                item.offsets = [(int(start), int(end)) for start, end in offsets]
+
+            # adjust hash_value
+            hash_value = get_data_value("hash")
+            if hash_value is not None:
+                if isinstance(hash_value, torch.Tensor):
+                    hash_value = hash_value.item()
+                item.hash = int(hash_value)
+                pad_value = get_data_value("pad_value")
+                if pad_value is not None:
+                    if isinstance(pad_value, torch.Tensor):
+                        pad_value = pad_value.item()
+                    item.pad_value = int(pad_value)
+
         return list(items.values())
 
     def _process_and_collect_mm_items(
@@ -1240,6 +1289,8 @@ class BaseMultimodalProcessor(ABC):
 
         # Add offsets to all items
         for mm_item in all_collected_items:
+            if mm_item.offsets is not None:
+                continue
             mm_token_id = mm_tokens.get_token_id_by_modality(mm_item.modality)
             if mm_token_id is None:
                 raise ValueError(f"No token id found for modality: {mm_item.modality}")

python/sglang/srt/multimodal/processors/qwen_vl.py

@@ -400,20 +400,13 @@ class QwenVLImageProcessor(SGLangBaseProcessor):
         )
         return mrope_positions.squeeze(1), mrope_position_delta
 
-    # TODO: consider moving it to SGLangBaseProcessor
     @staticmethod
     def _get_processor_output_value(ret, key):
-        """get value with key from returned value of processor"""
         if ret is None:
             return None
-        if hasattr(ret, "get"):
+        return ret.get(key) if hasattr(ret, "get") else getattr(ret, key, None)
-            value = ret.get(key)
-            if value is not None:
-                return value
-        return getattr(ret, key, None)
 
     def _get_precomputed_mrope_from_output(self, ret):
-        """get the precomputed mrope from processor output"""
         mrope_positions = self._get_processor_output_value(ret, "mrope_positions")
         mrope_position_delta = self._get_processor_output_value(
             ret, "mrope_position_delta"
@@ -430,9 +423,7 @@ class QwenVLImageProcessor(SGLangBaseProcessor):
             return None
 
         mrope_position_delta = torch.as_tensor(mrope_position_delta)
-        if mrope_position_delta.ndim == 0:
+        if mrope_position_delta.ndim <= 1:
-            mrope_position_delta = mrope_position_delta.reshape(1, 1)
-        elif mrope_position_delta.ndim == 1:
             mrope_position_delta = mrope_position_delta.reshape(-1, 1)
         return mrope_positions, mrope_position_delta
 
@@ -723,17 +714,11 @@ class QwenVLImageProcessor(SGLangBaseProcessor):
         else:
             padded_input_ids = list(padded_input_ids)
 
-        image_grid_thw = None
+        image_grid_thw = self._get_processor_output_value(ret, "image_grid_thw")
-        if hasattr(ret, "image_grid_thw"):
-            image_grid_thw = ret.image_grid_thw
-
         if image_grid_thw is None and image_data and isinstance(image_data[0], dict):
             image_grid_thw = image_data[0].get("image_grid_thw")
 
-        video_grid_thw = None
+        video_grid_thw = self._get_processor_output_value(ret, "video_grid_thw")
-        if hasattr(ret, "video_grid_thw"):
-            video_grid_thw = ret.video_grid_thw
-
         if video_grid_thw is None and request_obj.video_data:
             first_video = request_obj.video_data[0]
             if isinstance(first_video, dict):
@@ -777,8 +762,8 @@ class QwenVLImageProcessor(SGLangBaseProcessor):
             )
 
         mrope_positions, mrope_position_delta = mrope_result
+        if mrope_positions.ndim == 3:
             mrope_positions = mrope_positions.squeeze(1)
-
         get_rope_index_time = time.perf_counter()
         logger.debug(
             f"[QwenVLProcessor Perf] {rid=}, "