[Fix] --mm-process-config crash when video config contains (#30260)

This commit is contained in:
longxin9715
2026-07-28 09:24:40 +08:00
committed by GitHub
parent edc0e5489f
commit 356c11d5d9
4 changed files with 105 additions and 10 deletions
@@ -29,7 +29,7 @@ def transform_patches_to_flatten(
patch_size: int,
merge_size: int,
) -> torch.Tensor:
patches = patches.view(
patches = patches.reshape(
batch_size * grid_t,
temporal_patch_size * channel,
grid_h // merge_size,
@@ -489,6 +489,7 @@ class BaseMultimodalProcessor(ABC):
videos=None,
audios=None,
processor=None,
processor_video_config: Optional[Dict[str, Any]] = None,
**kwargs,
) -> dict:
"""
@@ -502,8 +503,13 @@ class BaseMultimodalProcessor(ABC):
kwargs.setdefault("images_kwargs", {}).update(self.image_config)
if videos:
kwargs["videos"] = videos
if self.video_config:
kwargs.setdefault("videos_kwargs", {}).update(self.video_config)
video_config = (
self.video_config
if processor_video_config is None
else processor_video_config
)
if video_config:
kwargs.setdefault("videos_kwargs", {}).update(video_config)
if audios:
if processor.__class__.__name__ in {
"Gemma3nProcessor",
@@ -59,6 +59,30 @@ FPS = 2.0
FPS_MIN_FRAMES = 4
FPS_MAX_FRAMES = 768
QWEN_VIDEO_PREPROCESS_CONFIG_KEYS = frozenset(
{
"fps",
"nframes",
"min_frames",
"max_frames",
"min_pixels",
"max_pixels",
"total_pixels",
"resized_height",
"resized_width",
}
)
def _get_processor_video_config(video_config, video_metadata):
if video_metadata and all(metadata is not None for metadata in video_metadata):
return {
key: value
for key, value in video_config.items()
if key not in QWEN_VIDEO_PREPROCESS_CONFIG_KEYS
}
return None
_is_cpu_amx_available = cpu_has_amx_support()
_is_cpu = is_cpu()
@@ -719,6 +743,13 @@ class QwenVLImageProcessor(SGLangBaseProcessor):
preprocess_time = time.perf_counter()
processor_kwargs = {}
processor_video_config = _get_processor_video_config(
self.video_config, video_metadata
)
if processor_video_config is not None:
processor_kwargs["processor_video_config"] = processor_video_config
# NOTE: for qwen3-vl, video_meta need to be passed in, since do_sample_frames is already done in preprocess_video
if self.hf_config.model_type in (
"qwen3_vl",
@@ -727,16 +758,14 @@ class QwenVLImageProcessor(SGLangBaseProcessor):
"qwen3_5_moe",
"intern_s2_preview",
):
mm_items, input_ids, ret = await self.process_and_combine_mm_data_async(
base_output,
self.mm_tokens,
processor_kwargs.update(
video_metadata=video_metadata,
do_sample_frames=False,
)
else:
mm_items, input_ids, ret = await self.process_and_combine_mm_data_async(
base_output, self.mm_tokens
)
mm_items, input_ids, ret = await self.process_and_combine_mm_data_async(
base_output, self.mm_tokens, **processor_kwargs
)
audio_feature_lengths = None