[VLM] route every multimodal processor through the worker pool's call site (#35342)

Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
2026-08-27 10:27:45 +08:00
parent 5adc2880f9
commit 07570202f0
34 changed files with 130 additions and 33 deletions
@@ -26,7 +26,7 @@ class ClipImageProcessor(BaseMultimodalProcessor):
            image_data=image_data,
        )

-        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+        mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
            base_output, self.mm_tokens
        )

@@ -57,7 +57,7 @@ class Cohere2VisionSGLangImageProcessor(SGLangBaseProcessor):
            discard_alpha_channel=True,
        )

-        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+        mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
            base_output, self.mm_tokens
        )
        return MultimodalProcessorOutput(
@@ -35,7 +35,7 @@ class DeepseekOCRProcessor(BaseMultimodalProcessor):
            image_data=image_data,
        )

-        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+        mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
            base_output, self.mm_tokens
        )

@@ -49,7 +49,7 @@ class DeepseekVL2ImageProcessor(BaseMultimodalProcessor):
            image_data=image_data,
            multimodal_tokens=self.mm_tokens,
        )
-        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+        mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
            base_output,
            self.mm_tokens,
            max_req_input_len=max_req_input_len,
@@ -75,7 +75,7 @@ class DotsVLMImageProcessor(BaseMultimodalProcessor):
            multimodal_tokens=self.mm_tokens,
        )

-        combined_mm_item, input_ids, _ = self.process_and_combine_mm_data(
+        combined_mm_item, input_ids, _ = await self.process_and_combine_mm_data_async(
            base_output, self.mm_tokens
        )
        if combined_mm_item is None:
@@ -404,7 +404,7 @@ class Ernie4_5_VLImageProcessor(SGLangBaseProcessor):
            ]
            base_output.videos, _ = map(list, zip(*videos_processed))

-        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+        mm_items, input_ids, ret = await self.process_and_combine_mm_data_async(
            base_output, self.mm_tokens
        )

@@ -44,7 +44,7 @@ class Gemma3SGLangImageProcessor(SGLangBaseProcessor):
            discard_alpha_channel=True,
        )

-        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+        mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
            base_output, self.mm_tokens
        )
        return MultimodalProcessorOutput(
@@ -59,7 +59,7 @@ class Gemma3nSGLangProcessor(SGLangBaseProcessor):
            multimodal_tokens=self.mm_tokens,
        )

-        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+        mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
            base_output, self.mm_tokens
        )

@@ -145,7 +145,7 @@ class Gemma4SGLangProcessor(SGLangBaseProcessor):
            multimodal_tokens=self.mm_tokens,
        )

-        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+        mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
            base_output, self.mm_tokens
        )

@@ -99,7 +99,7 @@ class Glm4vImageProcessor(SGLangBaseProcessor):

        if base_output.videos:
            base_output.videos = request_obj.video_data
-        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+        mm_items, input_ids, ret = await self.process_and_combine_mm_data_async(
            base_output, self.mm_tokens
        )

@@ -258,7 +258,7 @@ class GlmImageProcessor(SGLangBaseProcessor):
                multimodal_tokens=self.mm_tokens,
            )

-            mm_items, input_ids, ret = self.process_and_combine_mm_data(
+            mm_items, input_ids, ret = await self.process_and_combine_mm_data_async(
                base_output, self.mm_tokens
            )

@@ -75,7 +75,7 @@ class GlmAsrProcessor(BaseMultimodalProcessor):
        )
        if base_output is None:
            return None
-        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+        mm_items, input_ids, ret = await self.process_and_combine_mm_data_async(
            base_output, self.mm_tokens
        )
        return MultimodalProcessorOutput(
@@ -69,7 +69,7 @@ class InternS1_1ImageProcessor(QwenVLImageProcessor):

        preprocess_time = time.perf_counter()

-        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+        mm_items, input_ids, ret = await self.process_and_combine_mm_data_async(
            base_output,
            self.mm_tokens,
            video_metadata=video_metadata,
@@ -318,7 +318,7 @@ class InternVLProcessor(BaseMultimodalProcessor):
                discard_alpha_channel=True,
            )

-        mm_items, input_ids_tensor, ret = self.process_and_combine_mm_data(
+        mm_items, input_ids_tensor, ret = await self.process_and_combine_mm_data_async(
            base_output, self.mm_tokens
        )

@@ -32,7 +32,7 @@ class JanusProImageProcessor(BaseMultimodalProcessor):
            multimodal_tokens=self.mm_tokens,
        )

-        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+        mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
            base_out, self.mm_tokens, prompt=base_out.input_text
        )

@@ -46,7 +46,7 @@ class KimiVLImageProcessor(KimiGridMMDataMixin, SGLangBaseProcessor):
                f"expected {expected_image_count}, loaded {len(base_output.images)}"
            )

-        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+        mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
            base_output, self.mm_tokens
        )

@@ -74,7 +74,7 @@ class Lfm2VlImageProcessor(SGLangBaseProcessor):
            multimodal_tokens=self.mm_tokens,
        )

-        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+        mm_items, input_ids, ret = await self.process_and_combine_mm_data_async(
            base_output, self.mm_tokens
        )

@@ -45,7 +45,7 @@ class LocateAnythingImageProcessor(SGLangBaseProcessor):
            multimodal_tokens=self.mm_tokens,
        )

-        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+        mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
            base_output, self.mm_tokens
        )

@@ -112,7 +112,7 @@ class MiDashengLMMultimodalProcessor(BaseMultimodalProcessor):
            logger.info("base_output is None")
            return None

-        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+        mm_items, input_ids, ret = await self.process_and_combine_mm_data_async(
            base_output, self.mm_tokens
        )
        logger.info(f"mm_items count: {len(mm_items)}")
@@ -128,7 +128,7 @@ class MiniCPMMultimodalProcessor(BaseMultimodalProcessor):
        if base_output is None:
            return None

-        mm_items, input_ids_tensor, ret = self.process_and_combine_mm_data(
+        mm_items, input_ids_tensor, ret = await self.process_and_combine_mm_data_async(
            base_output, self.mm_tokens
        )

@@ -267,7 +267,7 @@ class MiniMaxM3VLProcessor(BaseMultimodalProcessor):
            ]
            base_output.videos, video_metadata = map(list, zip(*videos_processed))

-        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+        mm_items, input_ids, ret = await self.process_and_combine_mm_data_async(
            base_output=base_output,
            mm_tokens=self.mm_tokens,
            video_metadata=video_metadata,
@@ -27,7 +27,7 @@ class MllamaImageProcessor(BaseMultimodalProcessor):
            multimodal_tokens=self.mm_tokens,
        )

-        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+        mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
            base_out, self.mm_tokens
        )

@@ -37,7 +37,7 @@ class Mllama4ImageProcessor(BaseMultimodalProcessor):
        )

        # Process the prompt and images
-        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+        mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
            base_output, self.mm_tokens
        )

@@ -65,7 +65,7 @@ class NVILAMultimodalProcessor(BaseMultimodalProcessor):
        for i, video in enumerate(base_output.videos):  # type: ignore
            base_output.videos[i] = [x.asnumpy() for x in video]  # type: ignore

-        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+        mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
            base_output,
            self.mm_tokens,
            do_sample_frames=True,
@@ -89,7 +89,7 @@ class Phi4MMMultimodalProcessor(BaseMultimodalProcessor):
                (audio, self.AUDIO_SAMPLE_RATE) for audio in base_output.audios
            ]

-        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+        mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
            base_output, self.mm_tokens
        )

@@ -83,7 +83,7 @@ class PixtralProcessor(BaseMultimodalProcessor):
            image_data=image_data,
            return_text=True,
        )
-        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+        mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
            mm_data, self.mm_tokens
        )

@@ -32,7 +32,7 @@ class POINTSV15ChatProcessor(QwenVLImageProcessor):
            multimodal_tokens=self.mm_tokens,
        )

-        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+        mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
            base_output, self.mm_tokens
        )

@@ -79,7 +79,7 @@ class Qwen3ASRMultimodalProcessor(BaseMultimodalProcessor):
        if base_output is None:
            return None

-        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+        mm_items, input_ids, ret = await self.process_and_combine_mm_data_async(
            base_output, self.mm_tokens
        )

@@ -167,7 +167,7 @@ class Qwen2AudioMultimodalProcessor(BaseMultimodalProcessor):

        self._warn_if_audio_exceeds_window(base_output.audios)

-        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+        mm_items, input_ids, ret = await self.process_and_combine_mm_data_async(
            base_output, self.mm_tokens
        )

@@ -68,7 +68,7 @@ class Sarashina2VisionProcessor(BaseMultimodalProcessor):
            multimodal_tokens=self.mm_tokens,
        )

-        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+        mm_items, input_ids, ret = await self.process_and_combine_mm_data_async(
            base_output=base_output,
            mm_tokens=self.mm_tokens,
        )
@@ -563,7 +563,7 @@ class Step3VLImageProcessor(SGLangBaseProcessor):
            multimodal_tokens=self.mm_tokens,
        )

-        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+        mm_items, input_ids, ret = await self.process_and_combine_mm_data_async(
            base_output, self.mm_tokens
        )

@@ -199,7 +199,7 @@ class TransformersAutoMultimodalProcessor(BaseMultimodalProcessor):
            )
            if base_output is None:
                return None
-            mm_items, input_ids, _ = self.process_and_combine_mm_data(
+            mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
                base_output, self.mm_tokens
            )
            ret = MultimodalProcessorOutput(
@@ -100,7 +100,7 @@ class UnlimitedOCRProcessor(BaseMultimodalProcessor):
            multimodal_tokens=self.mm_tokens,
            image_data=image_data,
        )
-        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+        mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
            base_output, self.mm_tokens, **processor_kwargs
        )