[VLM] route every multimodal processor through the worker pool's call site (#35342)

Co-authored-by: Claude Opus 5 <noreply@anthropic.com>

python/sglang/srt/multimodal/processors/clip.py

@@ -26,7 +26,7 @@ class ClipImageProcessor(BaseMultimodalProcessor):
             image_data=image_data,
         )
 
-        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+        mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
             base_output, self.mm_tokens
         )
 

python/sglang/srt/multimodal/processors/cohere2_vision.py

@@ -57,7 +57,7 @@ class Cohere2VisionSGLangImageProcessor(SGLangBaseProcessor):
             discard_alpha_channel=True,
         )
 
-        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+        mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
             base_output, self.mm_tokens
         )
         return MultimodalProcessorOutput(

python/sglang/srt/multimodal/processors/deepseek_ocr.py

@@ -35,7 +35,7 @@ class DeepseekOCRProcessor(BaseMultimodalProcessor):
             image_data=image_data,
         )
 
-        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+        mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
             base_output, self.mm_tokens
         )
 

python/sglang/srt/multimodal/processors/deepseek_vl_v2.py

@@ -49,7 +49,7 @@ class DeepseekVL2ImageProcessor(BaseMultimodalProcessor):
             image_data=image_data,
             multimodal_tokens=self.mm_tokens,
         )
-        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+        mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
             base_output,
             self.mm_tokens,
             max_req_input_len=max_req_input_len,

python/sglang/srt/multimodal/processors/dots_vlm.py

@@ -75,7 +75,7 @@ class DotsVLMImageProcessor(BaseMultimodalProcessor):
             multimodal_tokens=self.mm_tokens,
         )
 
-        combined_mm_item, input_ids, _ = self.process_and_combine_mm_data(
+        combined_mm_item, input_ids, _ = await self.process_and_combine_mm_data_async(
             base_output, self.mm_tokens
         )
         if combined_mm_item is None:

python/sglang/srt/multimodal/processors/ernie45_vl.py

@@ -404,7 +404,7 @@ class Ernie4_5_VLImageProcessor(SGLangBaseProcessor):
             ]
             base_output.videos, _ = map(list, zip(*videos_processed))
 
-        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+        mm_items, input_ids, ret = await self.process_and_combine_mm_data_async(
             base_output, self.mm_tokens
         )
 

python/sglang/srt/multimodal/processors/gemma3.py

@@ -44,7 +44,7 @@ class Gemma3SGLangImageProcessor(SGLangBaseProcessor):
             discard_alpha_channel=True,
         )
 
-        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+        mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
             base_output, self.mm_tokens
         )
         return MultimodalProcessorOutput(

python/sglang/srt/multimodal/processors/gemma3n.py

@@ -59,7 +59,7 @@ class Gemma3nSGLangProcessor(SGLangBaseProcessor):
             multimodal_tokens=self.mm_tokens,
         )
 
-        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+        mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
             base_output, self.mm_tokens
         )
 

python/sglang/srt/multimodal/processors/gemma4.py

@@ -145,7 +145,7 @@ class Gemma4SGLangProcessor(SGLangBaseProcessor):
             multimodal_tokens=self.mm_tokens,
         )
 
-        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+        mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
             base_output, self.mm_tokens
         )
 

python/sglang/srt/multimodal/processors/glm4v.py

@@ -99,7 +99,7 @@ class Glm4vImageProcessor(SGLangBaseProcessor):
 
         if base_output.videos:
             base_output.videos = request_obj.video_data
-        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+        mm_items, input_ids, ret = await self.process_and_combine_mm_data_async(
             base_output, self.mm_tokens
         )
 

python/sglang/srt/multimodal/processors/glm_image.py

@@ -258,7 +258,7 @@ class GlmImageProcessor(SGLangBaseProcessor):
                 multimodal_tokens=self.mm_tokens,
             )
 
-            mm_items, input_ids, ret = self.process_and_combine_mm_data(
+            mm_items, input_ids, ret = await self.process_and_combine_mm_data_async(
                 base_output, self.mm_tokens
             )
 

python/sglang/srt/multimodal/processors/glmasr.py

@@ -75,7 +75,7 @@ class GlmAsrProcessor(BaseMultimodalProcessor):
         )
         if base_output is None:
             return None
-        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+        mm_items, input_ids, ret = await self.process_and_combine_mm_data_async(
             base_output, self.mm_tokens
         )
         return MultimodalProcessorOutput(

python/sglang/srt/multimodal/processors/interns1pro.py

@@ -69,7 +69,7 @@ class InternS1_1ImageProcessor(QwenVLImageProcessor):
 
         preprocess_time = time.perf_counter()
 
-        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+        mm_items, input_ids, ret = await self.process_and_combine_mm_data_async(
             base_output,
             self.mm_tokens,
             video_metadata=video_metadata,

python/sglang/srt/multimodal/processors/internvl.py

@@ -318,7 +318,7 @@ class InternVLProcessor(BaseMultimodalProcessor):
                 discard_alpha_channel=True,
             )
 
-        mm_items, input_ids_tensor, ret = self.process_and_combine_mm_data(
+        mm_items, input_ids_tensor, ret = await self.process_and_combine_mm_data_async(
             base_output, self.mm_tokens
         )
 

python/sglang/srt/multimodal/processors/janus_pro.py

@@ -32,7 +32,7 @@ class JanusProImageProcessor(BaseMultimodalProcessor):
             multimodal_tokens=self.mm_tokens,
         )
 
-        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+        mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
             base_out, self.mm_tokens, prompt=base_out.input_text
         )
 

python/sglang/srt/multimodal/processors/kimi_vl.py

@@ -46,7 +46,7 @@ class KimiVLImageProcessor(KimiGridMMDataMixin, SGLangBaseProcessor):
                 f"expected {expected_image_count}, loaded {len(base_output.images)}"
             )
 
-        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+        mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
             base_output, self.mm_tokens
         )
 

python/sglang/srt/multimodal/processors/lfm2_vl.py

@@ -74,7 +74,7 @@ class Lfm2VlImageProcessor(SGLangBaseProcessor):
             multimodal_tokens=self.mm_tokens,
         )
 
-        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+        mm_items, input_ids, ret = await self.process_and_combine_mm_data_async(
             base_output, self.mm_tokens
         )
 

python/sglang/srt/multimodal/processors/locate_anything.py

@@ -45,7 +45,7 @@ class LocateAnythingImageProcessor(SGLangBaseProcessor):
             multimodal_tokens=self.mm_tokens,
         )
 
-        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+        mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
             base_output, self.mm_tokens
         )
 

python/sglang/srt/multimodal/processors/midashenglm.py

@@ -112,7 +112,7 @@ class MiDashengLMMultimodalProcessor(BaseMultimodalProcessor):
             logger.info("base_output is None")
             return None
 
-        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+        mm_items, input_ids, ret = await self.process_and_combine_mm_data_async(
             base_output, self.mm_tokens
         )
         logger.info(f"mm_items count: {len(mm_items)}")

python/sglang/srt/multimodal/processors/minicpm.py

@@ -128,7 +128,7 @@ class MiniCPMMultimodalProcessor(BaseMultimodalProcessor):
         if base_output is None:
             return None
 
-        mm_items, input_ids_tensor, ret = self.process_and_combine_mm_data(
+        mm_items, input_ids_tensor, ret = await self.process_and_combine_mm_data_async(
             base_output, self.mm_tokens
         )
 

python/sglang/srt/multimodal/processors/minimax_m3_vl.py

@@ -267,7 +267,7 @@ class MiniMaxM3VLProcessor(BaseMultimodalProcessor):
             ]
             base_output.videos, video_metadata = map(list, zip(*videos_processed))
 
-        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+        mm_items, input_ids, ret = await self.process_and_combine_mm_data_async(
             base_output=base_output,
             mm_tokens=self.mm_tokens,
             video_metadata=video_metadata,

python/sglang/srt/multimodal/processors/mlama.py

@@ -27,7 +27,7 @@ class MllamaImageProcessor(BaseMultimodalProcessor):
             multimodal_tokens=self.mm_tokens,
         )
 
-        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+        mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
             base_out, self.mm_tokens
         )
 

python/sglang/srt/multimodal/processors/mllama4.py

@@ -37,7 +37,7 @@ class Mllama4ImageProcessor(BaseMultimodalProcessor):
         )
 
         # Process the prompt and images
-        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+        mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
             base_output, self.mm_tokens
         )
 

python/sglang/srt/multimodal/processors/nvila.py

@@ -65,7 +65,7 @@ class NVILAMultimodalProcessor(BaseMultimodalProcessor):
         for i, video in enumerate(base_output.videos):  # type: ignore
             base_output.videos[i] = [x.asnumpy() for x in video]  # type: ignore
 
-        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+        mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
             base_output,
             self.mm_tokens,
             do_sample_frames=True,

python/sglang/srt/multimodal/processors/phi4mm.py

@@ -89,7 +89,7 @@ class Phi4MMMultimodalProcessor(BaseMultimodalProcessor):
                 (audio, self.AUDIO_SAMPLE_RATE) for audio in base_output.audios
             ]
 
-        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+        mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
             base_output, self.mm_tokens
         )
 

python/sglang/srt/multimodal/processors/pixtral.py

@@ -83,7 +83,7 @@ class PixtralProcessor(BaseMultimodalProcessor):
             image_data=image_data,
             return_text=True,
         )
-        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+        mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
             mm_data, self.mm_tokens
         )
 

python/sglang/srt/multimodal/processors/points_v15_chat.py

@@ -32,7 +32,7 @@ class POINTSV15ChatProcessor(QwenVLImageProcessor):
             multimodal_tokens=self.mm_tokens,
         )
 
-        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+        mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
             base_output, self.mm_tokens
         )
 

python/sglang/srt/multimodal/processors/qwen3_asr.py

@@ -79,7 +79,7 @@ class Qwen3ASRMultimodalProcessor(BaseMultimodalProcessor):
         if base_output is None:
             return None
 
-        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+        mm_items, input_ids, ret = await self.process_and_combine_mm_data_async(
             base_output, self.mm_tokens
         )
 

python/sglang/srt/multimodal/processors/qwen_audio.py

@@ -167,7 +167,7 @@ class Qwen2AudioMultimodalProcessor(BaseMultimodalProcessor):
 
         self._warn_if_audio_exceeds_window(base_output.audios)
 
-        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+        mm_items, input_ids, ret = await self.process_and_combine_mm_data_async(
             base_output, self.mm_tokens
         )
 

python/sglang/srt/multimodal/processors/sarashina2_vision.py

@@ -68,7 +68,7 @@ class Sarashina2VisionProcessor(BaseMultimodalProcessor):
             multimodal_tokens=self.mm_tokens,
         )
 
-        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+        mm_items, input_ids, ret = await self.process_and_combine_mm_data_async(
             base_output=base_output,
             mm_tokens=self.mm_tokens,
         )

python/sglang/srt/multimodal/processors/step3_vl.py

@@ -563,7 +563,7 @@ class Step3VLImageProcessor(SGLangBaseProcessor):
             multimodal_tokens=self.mm_tokens,
         )
 
-        mm_items, input_ids, ret = self.process_and_combine_mm_data(
+        mm_items, input_ids, ret = await self.process_and_combine_mm_data_async(
             base_output, self.mm_tokens
         )
 

python/sglang/srt/multimodal/processors/transformers_auto.py

@@ -199,7 +199,7 @@ class TransformersAutoMultimodalProcessor(BaseMultimodalProcessor):
             )
             if base_output is None:
                 return None
-            mm_items, input_ids, _ = self.process_and_combine_mm_data(
+            mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
                 base_output, self.mm_tokens
             )
             ret = MultimodalProcessorOutput(

python/sglang/srt/multimodal/processors/unlimited_ocr.py

@@ -100,7 +100,7 @@ class UnlimitedOCRProcessor(BaseMultimodalProcessor):
             multimodal_tokens=self.mm_tokens,
             image_data=image_data,
         )
-        mm_items, input_ids, _ = self.process_and_combine_mm_data(
+        mm_items, input_ids, _ = await self.process_and_combine_mm_data_async(
             base_output, self.mm_tokens, **processor_kwargs
         )
 

test/registered/unit/multimodal/test_processor_async_call_sites.py

@@ -0,0 +1,97 @@
+"""Every processor must reach preprocessing through the executor-backed helper.
+
+`process_and_combine_mm_data` is the function the multimodal processor worker
+pool actually runs. A processor that calls it directly can never use those
+workers: it will build the thread pool and its processor clones on startup and
+then route every request past them. That failure is silent -- the model just
+serves at one-worker speed -- so pin the call site instead of the symptom.
+
+`process_and_combine_mm_data_async` delegates straight to the sync function when
+no executor exists, so using it costs nothing until a model opts into
+concurrency.
+"""
+
+import ast
+import pathlib
+
+import pytest
+
+from sglang.test.ci.ci_register import register_cpu_ci
+
+register_cpu_ci(est_time=3, suite="base-a-test-cpu")
+
+_MULTIMODAL_ROOT = (
+    pathlib.Path(__file__).resolve().parents[4]
+    / "python"
+    / "sglang"
+    / "srt"
+    / "multimodal"
+)
+# The async helper and the sync body live side by side here by design.
+_EXEMPT = {"base_processor.py"}
+
+
+def _enclosing_function(node, parents):
+    current = parents.get(id(node))
+    while current is not None:
+        if isinstance(current, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            return current
+        current = parents.get(id(current))
+    return None
+
+
+def _call_sites():
+    """Yield (path, lineno, attribute, enclosing_function) for every call."""
+    for path in sorted(_MULTIMODAL_ROOT.rglob("*.py")):
+        if path.name in _EXEMPT:
+            continue
+        tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
+        parents = {
+            id(child): parent
+            for parent in ast.walk(tree)
+            for child in ast.iter_child_nodes(parent)
+        }
+        for node in ast.walk(tree):
+            if not isinstance(node, ast.Call):
+                continue
+            func = node.func
+            if not (
+                isinstance(func, ast.Attribute)
+                and func.attr.startswith("process_and_combine_mm_data")
+            ):
+                continue
+            yield path, node.lineno, func.attr, _enclosing_function(node, parents)
+
+
+def test_no_processor_bypasses_the_worker_pool():
+    offenders = [
+        f"{path.relative_to(_MULTIMODAL_ROOT)}:{lineno}"
+        for path, lineno, attr, _ in _call_sites()
+        if not attr.endswith("_async")
+    ]
+    assert not offenders, (
+        "these call sites bypass the multimodal processor worker pool; use "
+        "`await self.process_and_combine_mm_data_async(...)`: " + ", ".join(offenders)
+    )
+
+
+def test_every_call_site_can_await():
+    """An `await` needs an async def around it, so the migration stays possible."""
+    offenders = [
+        f"{path.relative_to(_MULTIMODAL_ROOT)}:{lineno}"
+        for path, lineno, _, enclosing in _call_sites()
+        if not isinstance(enclosing, ast.AsyncFunctionDef)
+    ]
+    assert not offenders, (
+        "preprocessing is reached from a non-async function, so it cannot go "
+        "through the worker pool: " + ", ".join(offenders)
+    )
+
+
+def test_the_scan_actually_finds_call_sites():
+    """Guard against the scan silently matching nothing after a rename."""
+    assert len(list(_call_sites())) > 20
+
+
+if __name__ == "__main__":
+    raise SystemExit(pytest.main([__file__, "-v"]))