fix(mm): make multimodal data loading non-blocking to prevent health check stalls (#24751)

Signed-off-by: abinggo <107740309+abinggo@users.noreply.github.com>
Co-authored-by: Yang <ID+Y-aang@users.noreply.github.com>

python/sglang/srt/multimodal/processors/base_processor.py

@@ -1,3 +1,4 @@
+import asyncio
 import concurrent
 import concurrent.futures
 import dataclasses
@@ -729,7 +730,7 @@ class BaseMultimodalProcessor(ABC):
 
         return is_precomputed, images, videos, audios
 
-    def load_mm_data(
+    async def load_mm_data(
         self,
         prompt: str,
         multimodal_tokens: MultimodalSpecialTokens,
@@ -772,7 +773,7 @@ class BaseMultimodalProcessor(ABC):
             or cnt[Modality.AUDIO] != n_audio
             or getattr(self, "support_dynamic_frame_expansion", False)
         ):
-            return self.legacy_load_mm_data(
+            return await self.legacy_load_mm_data(
                 prompt=prompt,
                 multimodal_tokens=multimodal_tokens,
                 image_data=image_data,
@@ -784,7 +785,7 @@ class BaseMultimodalProcessor(ABC):
             )
         # For models other than MiniCPMO and MiniCPMV,
         # totally align multimodal_tokens, fast path
-        return self.fast_load_mm_data(
+        return await self.fast_load_mm_data(
             prompt=prompt,
             multimodal_tokens=multimodal_tokens,
             image_data=image_data,
@@ -795,7 +796,7 @@ class BaseMultimodalProcessor(ABC):
             audio_sample_rate=audio_sample_rate,
         )
 
-    def fast_load_mm_data(
+    async def fast_load_mm_data(
         self,
         prompt: str,
         multimodal_tokens: MultimodalSpecialTokens,
@@ -847,7 +848,7 @@ class BaseMultimodalProcessor(ABC):
 
         for modality, idx, future in futures:
             try:
-                result = future.result()
+                result = await asyncio.wrap_future(future)
             except Exception as e:
                 logger.exception(
                     "[load_mm_data(simple)] error loading %s data at index=%d",
@@ -879,7 +880,7 @@ class BaseMultimodalProcessor(ABC):
             input_text=prompt_str,
         )
 
-    def legacy_load_mm_data(
+    async def legacy_load_mm_data(
         self,
         prompt: str,
         multimodal_tokens: MultimodalSpecialTokens,
@@ -939,7 +940,7 @@ class BaseMultimodalProcessor(ABC):
             try:
                 if multimodal_tokens_pattern.match(text_part):
                     modality, raw_data, frame_limit = next(task_info_iter)
-                    result = next(futures_iter).result()
+                    result = await asyncio.wrap_future(next(futures_iter))
 
                     is_precomputed, new_imgs, new_vids, new_auds = (
                         self._process_loaded_mm_data(modality, raw_data, result)

python/sglang/srt/multimodal/processors/clip.py

@@ -20,7 +20,7 @@ class ClipImageProcessor(BaseMultimodalProcessor):
     async def process_mm_data_async(
         self, image_data: List[Union[str, bytes]], input_text, *args, **kwargs
     ):
-        base_output = self.load_mm_data(
+        base_output = await self.load_mm_data(
             prompt=input_text,
             multimodal_tokens=self.mm_tokens,
             image_data=image_data,

python/sglang/srt/multimodal/processors/deepseek_ocr.py

@@ -29,7 +29,7 @@ class DeepseekOCRProcessor(BaseMultimodalProcessor):
     async def process_mm_data_async(
         self, image_data: List[Union[str, bytes]], input_text, *args, **kwargs
     ):
-        base_output = self.load_mm_data(
+        base_output = await self.load_mm_data(
             prompt=input_text,
             multimodal_tokens=self.mm_tokens,
             image_data=image_data,

python/sglang/srt/multimodal/processors/deepseek_vl_v2.py

@@ -44,7 +44,7 @@ class DeepseekVL2ImageProcessor(BaseMultimodalProcessor):
         *args,
         **kwargs,
     ):
-        base_output = self.load_mm_data(
+        base_output = await self.load_mm_data(
             input_text,
             image_data=image_data,
             multimodal_tokens=self.mm_tokens,

python/sglang/srt/multimodal/processors/dots_vlm.py

@@ -61,7 +61,7 @@ class DotsVLMImageProcessor(BaseMultimodalProcessor):
         ):
             image_data = sum(image_data, [])
 
-        base_output = self.load_mm_data(
+        base_output = await self.load_mm_data(
             prompt=input_text,
             image_data=image_data,
             multimodal_tokens=self.mm_tokens,

python/sglang/srt/multimodal/processors/ernie45_vl.py

@@ -388,7 +388,7 @@ class Ernie4_5_VLImageProcessor(SGLangBaseProcessor):
         *args,
         **kwargs,
     ):
-        base_output = self.load_mm_data(
+        base_output = await self.load_mm_data(
             prompt=input_text,
             image_data=image_data,
             video_data=request_obj.video_data,

python/sglang/srt/multimodal/processors/gemma3.py

@@ -37,7 +37,7 @@ class Gemma3SGLangImageProcessor(SGLangBaseProcessor):
         *args,
         **kwargs,
     ):
-        base_output = self.load_mm_data(
+        base_output = await self.load_mm_data(
             prompt=input_text,
             image_data=image_data,
             multimodal_tokens=self.mm_tokens,

python/sglang/srt/multimodal/processors/gemma3n.py

@@ -52,7 +52,7 @@ class Gemma3nSGLangProcessor(SGLangBaseProcessor):
         **kwargs,
     ):
         """Process multimodal data including images and audio."""
-        base_output = self.load_mm_data(
+        base_output = await self.load_mm_data(
             prompt=input_text,
             image_data=image_data,
             audio_data=audio_data,

python/sglang/srt/multimodal/processors/gemma4.py

@@ -124,7 +124,7 @@ class Gemma4SGLangProcessor(SGLangBaseProcessor):
         **kwargs,
     ):
         """Process multimodal data including images, video, and audio."""
-        base_output = self.load_mm_data(
+        base_output = await self.load_mm_data(
             prompt=input_text,
             image_data=image_data,
             video_data=request_obj.video_data if request_obj else None,

python/sglang/srt/multimodal/processors/glm4v.py

@@ -90,7 +90,7 @@ class Glm4vImageProcessor(SGLangBaseProcessor):
         *args,
         **kwargs,
     ):
-        base_output = self.load_mm_data(
+        base_output = await self.load_mm_data(
             prompt=input_text,
             image_data=image_data,
             video_data=request_obj.video_data,

python/sglang/srt/multimodal/processors/glmasr.py

@@ -35,7 +35,7 @@ class GlmAsrProcessor(BaseMultimodalProcessor):
         input_text,
         **kwargs,
     ):
-        base_output = self.load_mm_data(
+        base_output = await self.load_mm_data(
             prompt=input_text,
             audio_data=audio_data,
             multimodal_tokens=self.mm_tokens,

python/sglang/srt/multimodal/processors/interns1pro.py

@@ -49,7 +49,7 @@ class InternS1_1ImageProcessor(QwenVLImageProcessor):
         **kwargs,
     ):
         entry_time = time.perf_counter()
-        base_output = self.load_mm_data(
+        base_output = await self.load_mm_data(
             prompt=input_text,
             image_data=image_data,
             video_data=request_obj.video_data,

python/sglang/srt/multimodal/processors/internvl.py

@@ -310,7 +310,7 @@ class InternVLProcessor(BaseMultimodalProcessor):
                 videos=videos,
             )
         else:
-            base_output = self.load_mm_data(
+            base_output = await self.load_mm_data(
                 prompt=prompt,
                 image_data=image_data,
                 video_data=video_data,
@@ -423,7 +423,7 @@ class InternVLProcessor(BaseMultimodalProcessor):
             prompt.count(self.VIDEO_PLACEHOLDER_TOKEN),
         )
 
-        base_output = self.load_mm_data(
+        base_output = await self.load_mm_data(
             prompt=prompt,
             image_data=image_data,
             video_data=video_data,
@@ -644,7 +644,7 @@ class InternVLProcessor(BaseMultimodalProcessor):
             prompt.count(self.IMG_CONTEXT),
         )
 
-        base_output = self.load_mm_data(
+        base_output = await self.load_mm_data(
             prompt=prompt,
             image_data=image_data,
             multimodal_tokens=self.mm_tokens_internlm2,  # expects <IMG_CONTEXT>

python/sglang/srt/multimodal/processors/janus_pro.py

@@ -26,7 +26,7 @@ class JanusProImageProcessor(BaseMultimodalProcessor):
         request_obj,
         **kwargs,
     ):
-        base_out = self.load_mm_data(
+        base_out = await self.load_mm_data(
             prompt=input_text,
             image_data=image_data,
             multimodal_tokens=self.mm_tokens,

python/sglang/srt/multimodal/processors/kimi_k25.py

@@ -373,7 +373,7 @@ class KimiK2_5VLImageProcessor(KimiGridMMDataMixin, SGLangBaseProcessor):
         *args,
         **kwargs,
     ):
-        base_output = self.load_mm_data(
+        base_output = await self.load_mm_data(
             prompt=input_text,
             image_data=image_data,
             multimodal_tokens=self.mm_tokens,

python/sglang/srt/multimodal/processors/kimi_vl.py

@@ -34,7 +34,7 @@ class KimiVLImageProcessor(KimiGridMMDataMixin, SGLangBaseProcessor):
         *args,
         **kwargs,
     ):
-        base_output = self.load_mm_data(
+        base_output = await self.load_mm_data(
             prompt=input_text,
             image_data=image_data,
             multimodal_tokens=self.mm_tokens,

python/sglang/srt/multimodal/processors/lfm2_vl.py

@@ -68,7 +68,7 @@ class Lfm2VlImageProcessor(SGLangBaseProcessor):
                 "im_token_id": self.IMAGE_TOKEN_ID,
             }
 
-        base_output = self.load_mm_data(
+        base_output = await self.load_mm_data(
             prompt=input_text,
             image_data=image_data,
             multimodal_tokens=self.mm_tokens,

python/sglang/srt/multimodal/processors/midashenglm.py

@@ -103,7 +103,7 @@ class MiDashengLMMultimodalProcessor(BaseMultimodalProcessor):
             input_text = f"{self.AUDIO_TOKEN}{input_text}"
             logger.info("Auto-prepended audio token")
 
-        base_output = self.load_mm_data(
+        base_output = await self.load_mm_data(
             prompt=input_text,
             audio_data=audio_data,
             multimodal_tokens=self.mm_tokens,

python/sglang/srt/multimodal/processors/mimo_v2.py

@@ -2107,7 +2107,7 @@ class MiMoV2Processor(BaseMultimodalProcessor):
             input_text = f"{self.mm_tokens.audio_token}{input_text}"
 
         video_data = getattr(request_obj, "video_data", [])
-        base_output = self.load_mm_data(
+        base_output = await self.load_mm_data(
             prompt=input_text,
             image_data=image_data,
             video_data=video_data,

python/sglang/srt/multimodal/processors/minicpm.py

@@ -118,7 +118,7 @@ class MiniCPMMultimodalProcessor(BaseMultimodalProcessor):
                 audios=audios,
             )
         else:
-            base_output = self.load_mm_data(
+            base_output = await self.load_mm_data(
                 prompt=prompt,
                 image_data=normalized_images,
                 audio_data=audio_data,
@@ -190,7 +190,7 @@ class MiniCPMMultimodalProcessor(BaseMultimodalProcessor):
                 **kwargs,
             )
 
-        base_output = self.load_mm_data(
+        base_output = await self.load_mm_data(
             prompt=input_text,
             audio_data=audio_data,
             image_data=image_data,

python/sglang/srt/multimodal/processors/minicpmv4_6.py

@@ -419,7 +419,7 @@ class MiniCPMV4_6MultimodalProcessor(BaseMultimodalProcessor):
         video_data = getattr(request_obj, "video_data", None) or kwargs.get(
             "video_data"
         )
-        base = self.load_mm_data(
+        base = await self.load_mm_data(
             prompt=input_text,
             audio_data=audio_data,
             image_data=image_data,

python/sglang/srt/multimodal/processors/mlama.py

@@ -21,7 +21,7 @@ class MllamaImageProcessor(BaseMultimodalProcessor):
     async def process_mm_data_async(
         self, image_data: List[Union[str, bytes]], input_text, *args, **kwargs
     ):
-        base_out = self.load_mm_data(
+        base_out = await self.load_mm_data(
             prompt=input_text,
             image_data=image_data,
             multimodal_tokens=self.mm_tokens,

python/sglang/srt/multimodal/processors/mllama4.py

@@ -30,7 +30,7 @@ class Mllama4ImageProcessor(BaseMultimodalProcessor):
         *args,
         **kwargs,
     ):
-        base_output = self.load_mm_data(
+        base_output = await self.load_mm_data(
             prompt=input_text,
             image_data=image_data,
             multimodal_tokens=self.mm_tokens,

python/sglang/srt/multimodal/processors/moss_vl.py

@@ -511,7 +511,7 @@ class MossVLImageProcessor(SGLangBaseProcessor):
         )
 
         try:
-            base_output = self.load_mm_data(
+            base_output = await self.load_mm_data(
                 prompt=input_text,
                 image_data=image_data,
                 multimodal_tokens=self.image_only_mm_tokens,

python/sglang/srt/multimodal/processors/nano_nemotron_vl.py

@@ -213,7 +213,7 @@ class NanoNemotronVLImageProcessor(BaseMultimodalProcessor):
     async def process_mm_data_async(
         self, image_data, audio_data, input_text, request_obj, **kwargs
     ):
-        base_output = self.load_mm_data(
+        base_output = await self.load_mm_data(
             prompt=input_text,
             image_data=image_data,
             video_data=request_obj.video_data,

python/sglang/srt/multimodal/processors/nvila.py

@@ -55,7 +55,7 @@ class NVILAMultimodalProcessor(BaseMultimodalProcessor):
         request_obj: GenerateReqInput,
         **kwargs,
     ) -> dict[str, Any] | None:
-        base_output = self.load_mm_data(
+        base_output = await self.load_mm_data(
             prompt=input_text,
             multimodal_tokens=self.mm_tokens,
             image_data=request_obj.image_data,  # type: ignore

python/sglang/srt/multimodal/processors/phi4mm.py

@@ -74,7 +74,7 @@ class Phi4MMMultimodalProcessor(BaseMultimodalProcessor):
         request_obj,
         **kwargs,
     ):
-        base_output = self.load_mm_data(
+        base_output = await self.load_mm_data(
             prompt=input_text,
             audio_data=audio_data,
             image_data=image_data,

python/sglang/srt/multimodal/processors/pixtral.py

@@ -71,7 +71,7 @@ class PixtralProcessor(BaseMultimodalProcessor):
         *args,
         **kwargs,
     ):
-        mm_data = self.load_mm_data(
+        mm_data = await self.load_mm_data(
             prompt=input_text,
             multimodal_tokens=self.mm_tokens,
             image_data=image_data,

python/sglang/srt/multimodal/processors/points_v15_chat.py

@@ -26,7 +26,7 @@ class POINTSV15ChatProcessor(QwenVLImageProcessor):
         *args,
         **kwargs,
     ):
-        base_output = self.load_mm_data(
+        base_output = await self.load_mm_data(
             prompt=input_text,
             image_data=image_data,
             multimodal_tokens=self.mm_tokens,

python/sglang/srt/multimodal/processors/qwen3_asr.py

@@ -71,7 +71,7 @@ class Qwen3ASRMultimodalProcessor(BaseMultimodalProcessor):
 
         prompt = self._build_transcription_prompt(input_text)
 
-        base_output = self.load_mm_data(
+        base_output = await self.load_mm_data(
             prompt=prompt,
             audio_data=audio_data,
             multimodal_tokens=self.mm_tokens,

python/sglang/srt/multimodal/processors/qwen_audio.py

@@ -87,7 +87,7 @@ class Qwen2AudioMultimodalProcessor(BaseMultimodalProcessor):
         input_text,
         **kwargs,
     ):
-        base_output = self.load_mm_data(
+        base_output = await self.load_mm_data(
             prompt=input_text,
             audio_data=audio_data,
             multimodal_tokens=self.mm_tokens,

python/sglang/srt/multimodal/processors/qwen_vl.py

@@ -505,7 +505,7 @@ class QwenVLImageProcessor(SGLangBaseProcessor):
         **kwargs,
     ):
         entry_time = time.perf_counter()
-        base_output = self.load_mm_data(
+        base_output = await self.load_mm_data(
             prompt=input_text,
             image_data=image_data,
             video_data=request_obj.video_data,

python/sglang/srt/multimodal/processors/sarashina2_vision.py

@@ -62,7 +62,7 @@ class Sarashina2VisionProcessor(BaseMultimodalProcessor):
         **kwargs,
     ):
         """Process image data for Sarashina2Vision model using standard SGLang pattern."""
-        base_output = self.load_mm_data(
+        base_output = await self.load_mm_data(
             prompt=input_text,
             image_data=image_data,
             multimodal_tokens=self.mm_tokens,

python/sglang/srt/multimodal/processors/step3_vl.py

@@ -504,7 +504,7 @@ class Step3VLImageProcessor(SGLangBaseProcessor):
         *args,
         **kwargs,
     ):
-        base_output = self.load_mm_data(
+        base_output = await self.load_mm_data(
             prompt=input_text,
             image_data=image_data,
             video_data=request_obj.video_data,

python/sglang/srt/multimodal/processors/voxtral.py

@@ -80,7 +80,7 @@ class VoxtralMultimodalProcessor(BaseMultimodalProcessor):
         # load_mm_data handles async loading, format detection, resampling.
         # process_and_combine_mm_data cannot be used: HF VoxtralProcessor.__call__
         # does not support audio (only apply_chat_template does).
-        base_output = self.load_mm_data(
+        base_output = await self.load_mm_data(
             prompt=prompt_with_placeholders,
             audio_data=audio_data,
             multimodal_tokens=self.mm_tokens,