minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[VLM] accept precomputed multimodal metadata (#26101)

Browse Source
This commit is contained in:
Mick
2026-05-24 15:43:21 +08:00
committed by GitHub
parent 826a4de062
commit 4c2b32bfbf
2 changed files with 69 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/multimodal/processors/base_processor.py
+63 -12
View File
@@ -1080,28 +1080,53 @@ class BaseMultimodalProcessor(ABC):
self, data_dict: dict, modality: Modality = None
) -> List[MultimodalDataItem]:
"""
Create mm_items from processor output. Initially creates one item per modality;
these are later split into per-image/video items by get_new_expanded_mm_items.
Create mm_items from processor output.
Note that the data_dict can be passed via offline engine api
Initially creates one item per modality; these are later split into per-image/video items by get_new_expanded_mm_items.
Note that the data_dict can be hf processor output, or passed via offline engine api
Args:
modality: if provided, force the data into a single MultimodalDataItem of that modality
"""
# universal getter for data_dict
get_data_value = (
data_dict.get
if hasattr(data_dict, "get")
else lambda name, default=None: getattr(data_dict, name, default)
)
# decide explicitly-set modality
explicit_modality = modality
modality_value = get_data_value("modality")
if explicit_modality is None and modality_value is not None:
explicit_modality = (
modality_value
if isinstance(modality_value, Modality)
else Modality.from_str(str(modality_value))
)
items: dict[Modality, MultimodalDataItem] = {}
for attr_name, value in data_dict.items():
if attr_name == "input_ids":
if attr_name in (
"input_ids",
"format",
"modality",
"hash",
"pad_value",
"offsets",
):
# metadata fields need explicit handling, skip generic item.set
continue
# Get modality for this attribute
current_modality = modality or self.ATTR_NAME_TO_MODALITY.get(attr_name)
current_modality = explicit_modality or self.ATTR_NAME_TO_MODALITY.get(
attr_name
)
if attr_name == "precomputed_embeddings":
modality_str = data_dict.get("modality")
current_modality = Modality.IMAGE
if modality_str:
try:
current_modality = Modality.from_str(modality_str)
except ValueError:
pass
current_modality = current_modality or Modality.IMAGE
if current_modality:
# Create item if needed
@@ -1115,6 +1140,30 @@ class BaseMultimodalProcessor(ABC):
items[current_modality].set(attr_name, value)
# deal with metadata fields when data_dict is preprocessed input: convert from tensor to expected python types
# the attribution of the metadata fields is only clear when number of MultimodalDataItem is 1
if len(items) == 1:
item = next(iter(items.values()))
# adjust offset
offsets = get_data_value("offsets")
if offsets is not None:
if isinstance(offsets, torch.Tensor):
offsets = offsets.detach().cpu().tolist()
item.offsets = [(int(start), int(end)) for start, end in offsets]
# adjust hash_value
hash_value = get_data_value("hash")
if hash_value is not None:
if isinstance(hash_value, torch.Tensor):
hash_value = hash_value.item()
item.hash = int(hash_value)
pad_value = get_data_value("pad_value")
if pad_value is not None:
if isinstance(pad_value, torch.Tensor):
pad_value = pad_value.item()
item.pad_value = int(pad_value)
return list(items.values())
def _process_and_collect_mm_items(
@@ -1240,6 +1289,8 @@ class BaseMultimodalProcessor(ABC):
# Add offsets to all items
for mm_item in all_collected_items:
if mm_item.offsets is not None:
continue
mm_token_id = mm_tokens.get_token_id_by_modality(mm_item.modality)
if mm_token_id is None:
raise ValueError(f"No token id found for modality: {mm_item.modality}")
python/sglang/srt/multimodal/processors/qwen_vl.py
+6 -21
View File
@@ -400,20 +400,13 @@ class QwenVLImageProcessor(SGLangBaseProcessor):
)
return mrope_positions.squeeze(1), mrope_position_delta
# TODO: consider moving it to SGLangBaseProcessor
@staticmethod
def _get_processor_output_value(ret, key):
"""get value with key from returned value of processor"""
if ret is None:
return None
if hasattr(ret, "get"):
value = ret.get(key)
if value is not None:
return value
return getattr(ret, key, None)
return ret.get(key) if hasattr(ret, "get") else getattr(ret, key, None)
def _get_precomputed_mrope_from_output(self, ret):
"""get the precomputed mrope from processor output"""
mrope_positions = self._get_processor_output_value(ret, "mrope_positions")
mrope_position_delta = self._get_processor_output_value(
ret, "mrope_position_delta"
@@ -430,9 +423,7 @@ class QwenVLImageProcessor(SGLangBaseProcessor):
return None
mrope_position_delta = torch.as_tensor(mrope_position_delta)
if mrope_position_delta.ndim == 0:
mrope_position_delta = mrope_position_delta.reshape(1, 1)
elif mrope_position_delta.ndim == 1:
if mrope_position_delta.ndim <= 1:
mrope_position_delta = mrope_position_delta.reshape(-1, 1)
return mrope_positions, mrope_position_delta
@@ -723,17 +714,11 @@ class QwenVLImageProcessor(SGLangBaseProcessor):
else:
padded_input_ids = list(padded_input_ids)
image_grid_thw = None
if hasattr(ret, "image_grid_thw"):
image_grid_thw = ret.image_grid_thw
image_grid_thw = self._get_processor_output_value(ret, "image_grid_thw")
if image_grid_thw is None and image_data and isinstance(image_data[0], dict):
image_grid_thw = image_data[0].get("image_grid_thw")
video_grid_thw = None
if hasattr(ret, "video_grid_thw"):
video_grid_thw = ret.video_grid_thw
video_grid_thw = self._get_processor_output_value(ret, "video_grid_thw")
if video_grid_thw is None and request_obj.video_data:
first_video = request_obj.video_data[0]
if isinstance(first_video, dict):
@@ -777,8 +762,8 @@ class QwenVLImageProcessor(SGLangBaseProcessor):
)
mrope_positions, mrope_position_delta = mrope_result
mrope_positions = mrope_positions.squeeze(1)
if mrope_positions.ndim == 3:
mrope_positions = mrope_positions.squeeze(1)
get_rope_index_time = time.perf_counter()
logger.debug(
f"[QwenVLProcessor Perf] {rid=}, "