[Fix] --mm-process-config crash when video config contains (#30260)
This commit is contained in:
@@ -29,7 +29,7 @@ def transform_patches_to_flatten(
|
||||
patch_size: int,
|
||||
merge_size: int,
|
||||
) -> torch.Tensor:
|
||||
patches = patches.view(
|
||||
patches = patches.reshape(
|
||||
batch_size * grid_t,
|
||||
temporal_patch_size * channel,
|
||||
grid_h // merge_size,
|
||||
|
||||
@@ -489,6 +489,7 @@ class BaseMultimodalProcessor(ABC):
|
||||
videos=None,
|
||||
audios=None,
|
||||
processor=None,
|
||||
processor_video_config: Optional[Dict[str, Any]] = None,
|
||||
**kwargs,
|
||||
) -> dict:
|
||||
"""
|
||||
@@ -502,8 +503,13 @@ class BaseMultimodalProcessor(ABC):
|
||||
kwargs.setdefault("images_kwargs", {}).update(self.image_config)
|
||||
if videos:
|
||||
kwargs["videos"] = videos
|
||||
if self.video_config:
|
||||
kwargs.setdefault("videos_kwargs", {}).update(self.video_config)
|
||||
video_config = (
|
||||
self.video_config
|
||||
if processor_video_config is None
|
||||
else processor_video_config
|
||||
)
|
||||
if video_config:
|
||||
kwargs.setdefault("videos_kwargs", {}).update(video_config)
|
||||
if audios:
|
||||
if processor.__class__.__name__ in {
|
||||
"Gemma3nProcessor",
|
||||
|
||||
@@ -59,6 +59,30 @@ FPS = 2.0
|
||||
FPS_MIN_FRAMES = 4
|
||||
FPS_MAX_FRAMES = 768
|
||||
|
||||
QWEN_VIDEO_PREPROCESS_CONFIG_KEYS = frozenset(
|
||||
{
|
||||
"fps",
|
||||
"nframes",
|
||||
"min_frames",
|
||||
"max_frames",
|
||||
"min_pixels",
|
||||
"max_pixels",
|
||||
"total_pixels",
|
||||
"resized_height",
|
||||
"resized_width",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _get_processor_video_config(video_config, video_metadata):
|
||||
if video_metadata and all(metadata is not None for metadata in video_metadata):
|
||||
return {
|
||||
key: value
|
||||
for key, value in video_config.items()
|
||||
if key not in QWEN_VIDEO_PREPROCESS_CONFIG_KEYS
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
_is_cpu_amx_available = cpu_has_amx_support()
|
||||
_is_cpu = is_cpu()
|
||||
@@ -719,6 +743,13 @@ class QwenVLImageProcessor(SGLangBaseProcessor):
|
||||
|
||||
preprocess_time = time.perf_counter()
|
||||
|
||||
processor_kwargs = {}
|
||||
processor_video_config = _get_processor_video_config(
|
||||
self.video_config, video_metadata
|
||||
)
|
||||
if processor_video_config is not None:
|
||||
processor_kwargs["processor_video_config"] = processor_video_config
|
||||
|
||||
# NOTE: for qwen3-vl, video_meta need to be passed in, since do_sample_frames is already done in preprocess_video
|
||||
if self.hf_config.model_type in (
|
||||
"qwen3_vl",
|
||||
@@ -727,16 +758,14 @@ class QwenVLImageProcessor(SGLangBaseProcessor):
|
||||
"qwen3_5_moe",
|
||||
"intern_s2_preview",
|
||||
):
|
||||
mm_items, input_ids, ret = await self.process_and_combine_mm_data_async(
|
||||
base_output,
|
||||
self.mm_tokens,
|
||||
processor_kwargs.update(
|
||||
video_metadata=video_metadata,
|
||||
do_sample_frames=False,
|
||||
)
|
||||
else:
|
||||
mm_items, input_ids, ret = await self.process_and_combine_mm_data_async(
|
||||
base_output, self.mm_tokens
|
||||
)
|
||||
|
||||
mm_items, input_ids, ret = await self.process_and_combine_mm_data_async(
|
||||
base_output, self.mm_tokens, **processor_kwargs
|
||||
)
|
||||
|
||||
audio_feature_lengths = None
|
||||
|
||||
|
||||
Reference in New Issue
Block a user