diff --git a/python/sglang/srt/multimodal/processors/base_processor.py b/python/sglang/srt/multimodal/processors/base_processor.py index 842cd903c..bd92d1e42 100644 --- a/python/sglang/srt/multimodal/processors/base_processor.py +++ b/python/sglang/srt/multimodal/processors/base_processor.py @@ -1080,28 +1080,53 @@ class BaseMultimodalProcessor(ABC): self, data_dict: dict, modality: Modality = None ) -> List[MultimodalDataItem]: """ - Create mm_items from processor output. Initially creates one item per modality; - these are later split into per-image/video items by get_new_expanded_mm_items. + Create mm_items from processor output. - Note that the data_dict can be passed via offline engine api + Initially creates one item per modality; these are later split into per-image/video items by get_new_expanded_mm_items. + + Note that the data_dict can be hf processor output, or passed via offline engine api + + Args: + modality: if provided, force the data into a single MultimodalDataItem of that modality """ + # universal getter for data_dict + get_data_value = ( + data_dict.get + if hasattr(data_dict, "get") + else lambda name, default=None: getattr(data_dict, name, default) + ) + + # decide explicitly-set modality + explicit_modality = modality + modality_value = get_data_value("modality") + if explicit_modality is None and modality_value is not None: + explicit_modality = ( + modality_value + if isinstance(modality_value, Modality) + else Modality.from_str(str(modality_value)) + ) + items: dict[Modality, MultimodalDataItem] = {} for attr_name, value in data_dict.items(): - if attr_name == "input_ids": + if attr_name in ( + "input_ids", + "format", + "modality", + "hash", + "pad_value", + "offsets", + ): + # metadata fields need explicit handling, skip generic item.set continue # Get modality for this attribute - current_modality = modality or self.ATTR_NAME_TO_MODALITY.get(attr_name) + current_modality = explicit_modality or self.ATTR_NAME_TO_MODALITY.get( + attr_name + ) if attr_name == "precomputed_embeddings": - modality_str = data_dict.get("modality") - current_modality = Modality.IMAGE - if modality_str: - try: - current_modality = Modality.from_str(modality_str) - except ValueError: - pass + current_modality = current_modality or Modality.IMAGE if current_modality: # Create item if needed @@ -1115,6 +1140,30 @@ class BaseMultimodalProcessor(ABC): items[current_modality].set(attr_name, value) + # deal with metadata fields when data_dict is preprocessed input: convert from tensor to expected python types + # the attribution of the metadata fields is only clear when number of MultimodalDataItem is 1 + if len(items) == 1: + item = next(iter(items.values())) + + # adjust offset + offsets = get_data_value("offsets") + if offsets is not None: + if isinstance(offsets, torch.Tensor): + offsets = offsets.detach().cpu().tolist() + item.offsets = [(int(start), int(end)) for start, end in offsets] + + # adjust hash_value + hash_value = get_data_value("hash") + if hash_value is not None: + if isinstance(hash_value, torch.Tensor): + hash_value = hash_value.item() + item.hash = int(hash_value) + pad_value = get_data_value("pad_value") + if pad_value is not None: + if isinstance(pad_value, torch.Tensor): + pad_value = pad_value.item() + item.pad_value = int(pad_value) + return list(items.values()) def _process_and_collect_mm_items( @@ -1240,6 +1289,8 @@ class BaseMultimodalProcessor(ABC): # Add offsets to all items for mm_item in all_collected_items: + if mm_item.offsets is not None: + continue mm_token_id = mm_tokens.get_token_id_by_modality(mm_item.modality) if mm_token_id is None: raise ValueError(f"No token id found for modality: {mm_item.modality}") diff --git a/python/sglang/srt/multimodal/processors/qwen_vl.py b/python/sglang/srt/multimodal/processors/qwen_vl.py index 962cf9b43..fa3cd7961 100644 --- a/python/sglang/srt/multimodal/processors/qwen_vl.py +++ b/python/sglang/srt/multimodal/processors/qwen_vl.py @@ -400,20 +400,13 @@ class QwenVLImageProcessor(SGLangBaseProcessor): ) return mrope_positions.squeeze(1), mrope_position_delta - # TODO: consider moving it to SGLangBaseProcessor @staticmethod def _get_processor_output_value(ret, key): - """get value with key from returned value of processor""" if ret is None: return None - if hasattr(ret, "get"): - value = ret.get(key) - if value is not None: - return value - return getattr(ret, key, None) + return ret.get(key) if hasattr(ret, "get") else getattr(ret, key, None) def _get_precomputed_mrope_from_output(self, ret): - """get the precomputed mrope from processor output""" mrope_positions = self._get_processor_output_value(ret, "mrope_positions") mrope_position_delta = self._get_processor_output_value( ret, "mrope_position_delta" @@ -430,9 +423,7 @@ class QwenVLImageProcessor(SGLangBaseProcessor): return None mrope_position_delta = torch.as_tensor(mrope_position_delta) - if mrope_position_delta.ndim == 0: - mrope_position_delta = mrope_position_delta.reshape(1, 1) - elif mrope_position_delta.ndim == 1: + if mrope_position_delta.ndim <= 1: mrope_position_delta = mrope_position_delta.reshape(-1, 1) return mrope_positions, mrope_position_delta @@ -723,17 +714,11 @@ class QwenVLImageProcessor(SGLangBaseProcessor): else: padded_input_ids = list(padded_input_ids) - image_grid_thw = None - if hasattr(ret, "image_grid_thw"): - image_grid_thw = ret.image_grid_thw - + image_grid_thw = self._get_processor_output_value(ret, "image_grid_thw") if image_grid_thw is None and image_data and isinstance(image_data[0], dict): image_grid_thw = image_data[0].get("image_grid_thw") - video_grid_thw = None - if hasattr(ret, "video_grid_thw"): - video_grid_thw = ret.video_grid_thw - + video_grid_thw = self._get_processor_output_value(ret, "video_grid_thw") if video_grid_thw is None and request_obj.video_data: first_video = request_obj.video_data[0] if isinstance(first_video, dict): @@ -777,8 +762,8 @@ class QwenVLImageProcessor(SGLangBaseProcessor): ) mrope_positions, mrope_position_delta = mrope_result - mrope_positions = mrope_positions.squeeze(1) - + if mrope_positions.ndim == 3: + mrope_positions = mrope_positions.squeeze(1) get_rope_index_time = time.perf_counter() logger.debug( f"[QwenVLProcessor Perf] {rid=}, "