diff --git a/python/sglang/srt/multimodal/processors/base_processor.py b/python/sglang/srt/multimodal/processors/base_processor.py index cf64b06fe..a647e9e0d 100644 --- a/python/sglang/srt/multimodal/processors/base_processor.py +++ b/python/sglang/srt/multimodal/processors/base_processor.py @@ -1,3 +1,4 @@ +import asyncio import concurrent import concurrent.futures import dataclasses @@ -729,7 +730,7 @@ class BaseMultimodalProcessor(ABC): return is_precomputed, images, videos, audios - def load_mm_data( + async def load_mm_data( self, prompt: str, multimodal_tokens: MultimodalSpecialTokens, @@ -772,7 +773,7 @@ class BaseMultimodalProcessor(ABC): or cnt[Modality.AUDIO] != n_audio or getattr(self, "support_dynamic_frame_expansion", False) ): - return self.legacy_load_mm_data( + return await self.legacy_load_mm_data( prompt=prompt, multimodal_tokens=multimodal_tokens, image_data=image_data, @@ -784,7 +785,7 @@ class BaseMultimodalProcessor(ABC): ) # For models other than MiniCPMO and MiniCPMV, # totally align multimodal_tokens, fast path - return self.fast_load_mm_data( + return await self.fast_load_mm_data( prompt=prompt, multimodal_tokens=multimodal_tokens, image_data=image_data, @@ -795,7 +796,7 @@ class BaseMultimodalProcessor(ABC): audio_sample_rate=audio_sample_rate, ) - def fast_load_mm_data( + async def fast_load_mm_data( self, prompt: str, multimodal_tokens: MultimodalSpecialTokens, @@ -847,7 +848,7 @@ class BaseMultimodalProcessor(ABC): for modality, idx, future in futures: try: - result = future.result() + result = await asyncio.wrap_future(future) except Exception as e: logger.exception( "[load_mm_data(simple)] error loading %s data at index=%d", @@ -879,7 +880,7 @@ class BaseMultimodalProcessor(ABC): input_text=prompt_str, ) - def legacy_load_mm_data( + async def legacy_load_mm_data( self, prompt: str, multimodal_tokens: MultimodalSpecialTokens, @@ -939,7 +940,7 @@ class BaseMultimodalProcessor(ABC): try: if multimodal_tokens_pattern.match(text_part): modality, raw_data, frame_limit = next(task_info_iter) - result = next(futures_iter).result() + result = await asyncio.wrap_future(next(futures_iter)) is_precomputed, new_imgs, new_vids, new_auds = ( self._process_loaded_mm_data(modality, raw_data, result) diff --git a/python/sglang/srt/multimodal/processors/clip.py b/python/sglang/srt/multimodal/processors/clip.py index 06f785b85..3265b6ab4 100644 --- a/python/sglang/srt/multimodal/processors/clip.py +++ b/python/sglang/srt/multimodal/processors/clip.py @@ -20,7 +20,7 @@ class ClipImageProcessor(BaseMultimodalProcessor): async def process_mm_data_async( self, image_data: List[Union[str, bytes]], input_text, *args, **kwargs ): - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, multimodal_tokens=self.mm_tokens, image_data=image_data, diff --git a/python/sglang/srt/multimodal/processors/deepseek_ocr.py b/python/sglang/srt/multimodal/processors/deepseek_ocr.py index becb0b2b3..bbf64cbd5 100644 --- a/python/sglang/srt/multimodal/processors/deepseek_ocr.py +++ b/python/sglang/srt/multimodal/processors/deepseek_ocr.py @@ -29,7 +29,7 @@ class DeepseekOCRProcessor(BaseMultimodalProcessor): async def process_mm_data_async( self, image_data: List[Union[str, bytes]], input_text, *args, **kwargs ): - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, multimodal_tokens=self.mm_tokens, image_data=image_data, diff --git a/python/sglang/srt/multimodal/processors/deepseek_vl_v2.py b/python/sglang/srt/multimodal/processors/deepseek_vl_v2.py index 3a08a3c61..56f325175 100644 --- a/python/sglang/srt/multimodal/processors/deepseek_vl_v2.py +++ b/python/sglang/srt/multimodal/processors/deepseek_vl_v2.py @@ -44,7 +44,7 @@ class DeepseekVL2ImageProcessor(BaseMultimodalProcessor): *args, **kwargs, ): - base_output = self.load_mm_data( + base_output = await self.load_mm_data( input_text, image_data=image_data, multimodal_tokens=self.mm_tokens, diff --git a/python/sglang/srt/multimodal/processors/dots_vlm.py b/python/sglang/srt/multimodal/processors/dots_vlm.py index c8e76562a..bc269cdae 100644 --- a/python/sglang/srt/multimodal/processors/dots_vlm.py +++ b/python/sglang/srt/multimodal/processors/dots_vlm.py @@ -61,7 +61,7 @@ class DotsVLMImageProcessor(BaseMultimodalProcessor): ): image_data = sum(image_data, []) - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, multimodal_tokens=self.mm_tokens, diff --git a/python/sglang/srt/multimodal/processors/ernie45_vl.py b/python/sglang/srt/multimodal/processors/ernie45_vl.py index 8bb3475be..1bd690555 100644 --- a/python/sglang/srt/multimodal/processors/ernie45_vl.py +++ b/python/sglang/srt/multimodal/processors/ernie45_vl.py @@ -388,7 +388,7 @@ class Ernie4_5_VLImageProcessor(SGLangBaseProcessor): *args, **kwargs, ): - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, video_data=request_obj.video_data, diff --git a/python/sglang/srt/multimodal/processors/gemma3.py b/python/sglang/srt/multimodal/processors/gemma3.py index c6b35e843..7390f14ea 100644 --- a/python/sglang/srt/multimodal/processors/gemma3.py +++ b/python/sglang/srt/multimodal/processors/gemma3.py @@ -37,7 +37,7 @@ class Gemma3SGLangImageProcessor(SGLangBaseProcessor): *args, **kwargs, ): - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, multimodal_tokens=self.mm_tokens, diff --git a/python/sglang/srt/multimodal/processors/gemma3n.py b/python/sglang/srt/multimodal/processors/gemma3n.py index 5cb6d7962..6c6c62064 100644 --- a/python/sglang/srt/multimodal/processors/gemma3n.py +++ b/python/sglang/srt/multimodal/processors/gemma3n.py @@ -52,7 +52,7 @@ class Gemma3nSGLangProcessor(SGLangBaseProcessor): **kwargs, ): """Process multimodal data including images and audio.""" - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, audio_data=audio_data, diff --git a/python/sglang/srt/multimodal/processors/gemma4.py b/python/sglang/srt/multimodal/processors/gemma4.py index 80bb37061..d8fd6bd0a 100644 --- a/python/sglang/srt/multimodal/processors/gemma4.py +++ b/python/sglang/srt/multimodal/processors/gemma4.py @@ -124,7 +124,7 @@ class Gemma4SGLangProcessor(SGLangBaseProcessor): **kwargs, ): """Process multimodal data including images, video, and audio.""" - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, video_data=request_obj.video_data if request_obj else None, diff --git a/python/sglang/srt/multimodal/processors/glm4v.py b/python/sglang/srt/multimodal/processors/glm4v.py index a44f14b6c..db684259d 100644 --- a/python/sglang/srt/multimodal/processors/glm4v.py +++ b/python/sglang/srt/multimodal/processors/glm4v.py @@ -90,7 +90,7 @@ class Glm4vImageProcessor(SGLangBaseProcessor): *args, **kwargs, ): - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, video_data=request_obj.video_data, diff --git a/python/sglang/srt/multimodal/processors/glmasr.py b/python/sglang/srt/multimodal/processors/glmasr.py index 1fcaf490a..e55656fe1 100644 --- a/python/sglang/srt/multimodal/processors/glmasr.py +++ b/python/sglang/srt/multimodal/processors/glmasr.py @@ -35,7 +35,7 @@ class GlmAsrProcessor(BaseMultimodalProcessor): input_text, **kwargs, ): - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, audio_data=audio_data, multimodal_tokens=self.mm_tokens, diff --git a/python/sglang/srt/multimodal/processors/interns1pro.py b/python/sglang/srt/multimodal/processors/interns1pro.py index 0f4a909ad..21c6ff16f 100644 --- a/python/sglang/srt/multimodal/processors/interns1pro.py +++ b/python/sglang/srt/multimodal/processors/interns1pro.py @@ -49,7 +49,7 @@ class InternS1_1ImageProcessor(QwenVLImageProcessor): **kwargs, ): entry_time = time.perf_counter() - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, video_data=request_obj.video_data, diff --git a/python/sglang/srt/multimodal/processors/internvl.py b/python/sglang/srt/multimodal/processors/internvl.py index 800f68110..899c4acce 100644 --- a/python/sglang/srt/multimodal/processors/internvl.py +++ b/python/sglang/srt/multimodal/processors/internvl.py @@ -310,7 +310,7 @@ class InternVLProcessor(BaseMultimodalProcessor): videos=videos, ) else: - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=prompt, image_data=image_data, video_data=video_data, @@ -423,7 +423,7 @@ class InternVLProcessor(BaseMultimodalProcessor): prompt.count(self.VIDEO_PLACEHOLDER_TOKEN), ) - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=prompt, image_data=image_data, video_data=video_data, @@ -644,7 +644,7 @@ class InternVLProcessor(BaseMultimodalProcessor): prompt.count(self.IMG_CONTEXT), ) - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=prompt, image_data=image_data, multimodal_tokens=self.mm_tokens_internlm2, # expects diff --git a/python/sglang/srt/multimodal/processors/janus_pro.py b/python/sglang/srt/multimodal/processors/janus_pro.py index f6711058d..4c8a755c2 100644 --- a/python/sglang/srt/multimodal/processors/janus_pro.py +++ b/python/sglang/srt/multimodal/processors/janus_pro.py @@ -26,7 +26,7 @@ class JanusProImageProcessor(BaseMultimodalProcessor): request_obj, **kwargs, ): - base_out = self.load_mm_data( + base_out = await self.load_mm_data( prompt=input_text, image_data=image_data, multimodal_tokens=self.mm_tokens, diff --git a/python/sglang/srt/multimodal/processors/kimi_k25.py b/python/sglang/srt/multimodal/processors/kimi_k25.py index 9838ca510..ff2af7b92 100644 --- a/python/sglang/srt/multimodal/processors/kimi_k25.py +++ b/python/sglang/srt/multimodal/processors/kimi_k25.py @@ -373,7 +373,7 @@ class KimiK2_5VLImageProcessor(KimiGridMMDataMixin, SGLangBaseProcessor): *args, **kwargs, ): - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, multimodal_tokens=self.mm_tokens, diff --git a/python/sglang/srt/multimodal/processors/kimi_vl.py b/python/sglang/srt/multimodal/processors/kimi_vl.py index 6c0e16a1c..7bcf2885c 100644 --- a/python/sglang/srt/multimodal/processors/kimi_vl.py +++ b/python/sglang/srt/multimodal/processors/kimi_vl.py @@ -34,7 +34,7 @@ class KimiVLImageProcessor(KimiGridMMDataMixin, SGLangBaseProcessor): *args, **kwargs, ): - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, multimodal_tokens=self.mm_tokens, diff --git a/python/sglang/srt/multimodal/processors/lfm2_vl.py b/python/sglang/srt/multimodal/processors/lfm2_vl.py index c80720651..28d98dc59 100644 --- a/python/sglang/srt/multimodal/processors/lfm2_vl.py +++ b/python/sglang/srt/multimodal/processors/lfm2_vl.py @@ -68,7 +68,7 @@ class Lfm2VlImageProcessor(SGLangBaseProcessor): "im_token_id": self.IMAGE_TOKEN_ID, } - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, multimodal_tokens=self.mm_tokens, diff --git a/python/sglang/srt/multimodal/processors/midashenglm.py b/python/sglang/srt/multimodal/processors/midashenglm.py index 526cdc979..985be22fd 100644 --- a/python/sglang/srt/multimodal/processors/midashenglm.py +++ b/python/sglang/srt/multimodal/processors/midashenglm.py @@ -103,7 +103,7 @@ class MiDashengLMMultimodalProcessor(BaseMultimodalProcessor): input_text = f"{self.AUDIO_TOKEN}{input_text}" logger.info("Auto-prepended audio token") - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, audio_data=audio_data, multimodal_tokens=self.mm_tokens, diff --git a/python/sglang/srt/multimodal/processors/mimo_v2.py b/python/sglang/srt/multimodal/processors/mimo_v2.py index 89c2619f3..3c1a02a93 100644 --- a/python/sglang/srt/multimodal/processors/mimo_v2.py +++ b/python/sglang/srt/multimodal/processors/mimo_v2.py @@ -2107,7 +2107,7 @@ class MiMoV2Processor(BaseMultimodalProcessor): input_text = f"{self.mm_tokens.audio_token}{input_text}" video_data = getattr(request_obj, "video_data", []) - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, video_data=video_data, diff --git a/python/sglang/srt/multimodal/processors/minicpm.py b/python/sglang/srt/multimodal/processors/minicpm.py index d4c407c13..9df74cbde 100644 --- a/python/sglang/srt/multimodal/processors/minicpm.py +++ b/python/sglang/srt/multimodal/processors/minicpm.py @@ -118,7 +118,7 @@ class MiniCPMMultimodalProcessor(BaseMultimodalProcessor): audios=audios, ) else: - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=prompt, image_data=normalized_images, audio_data=audio_data, @@ -190,7 +190,7 @@ class MiniCPMMultimodalProcessor(BaseMultimodalProcessor): **kwargs, ) - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, audio_data=audio_data, image_data=image_data, diff --git a/python/sglang/srt/multimodal/processors/minicpmv4_6.py b/python/sglang/srt/multimodal/processors/minicpmv4_6.py index 25529b9b8..c2a0f67b1 100644 --- a/python/sglang/srt/multimodal/processors/minicpmv4_6.py +++ b/python/sglang/srt/multimodal/processors/minicpmv4_6.py @@ -419,7 +419,7 @@ class MiniCPMV4_6MultimodalProcessor(BaseMultimodalProcessor): video_data = getattr(request_obj, "video_data", None) or kwargs.get( "video_data" ) - base = self.load_mm_data( + base = await self.load_mm_data( prompt=input_text, audio_data=audio_data, image_data=image_data, diff --git a/python/sglang/srt/multimodal/processors/mlama.py b/python/sglang/srt/multimodal/processors/mlama.py index 52129765c..a12c9e2c4 100644 --- a/python/sglang/srt/multimodal/processors/mlama.py +++ b/python/sglang/srt/multimodal/processors/mlama.py @@ -21,7 +21,7 @@ class MllamaImageProcessor(BaseMultimodalProcessor): async def process_mm_data_async( self, image_data: List[Union[str, bytes]], input_text, *args, **kwargs ): - base_out = self.load_mm_data( + base_out = await self.load_mm_data( prompt=input_text, image_data=image_data, multimodal_tokens=self.mm_tokens, diff --git a/python/sglang/srt/multimodal/processors/mllama4.py b/python/sglang/srt/multimodal/processors/mllama4.py index 3983df275..470e6de58 100644 --- a/python/sglang/srt/multimodal/processors/mllama4.py +++ b/python/sglang/srt/multimodal/processors/mllama4.py @@ -30,7 +30,7 @@ class Mllama4ImageProcessor(BaseMultimodalProcessor): *args, **kwargs, ): - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, multimodal_tokens=self.mm_tokens, diff --git a/python/sglang/srt/multimodal/processors/moss_vl.py b/python/sglang/srt/multimodal/processors/moss_vl.py index a4b77a739..409d9eb23 100644 --- a/python/sglang/srt/multimodal/processors/moss_vl.py +++ b/python/sglang/srt/multimodal/processors/moss_vl.py @@ -511,7 +511,7 @@ class MossVLImageProcessor(SGLangBaseProcessor): ) try: - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, multimodal_tokens=self.image_only_mm_tokens, diff --git a/python/sglang/srt/multimodal/processors/nano_nemotron_vl.py b/python/sglang/srt/multimodal/processors/nano_nemotron_vl.py index 04f9b5f3f..32bab7ae4 100644 --- a/python/sglang/srt/multimodal/processors/nano_nemotron_vl.py +++ b/python/sglang/srt/multimodal/processors/nano_nemotron_vl.py @@ -213,7 +213,7 @@ class NanoNemotronVLImageProcessor(BaseMultimodalProcessor): async def process_mm_data_async( self, image_data, audio_data, input_text, request_obj, **kwargs ): - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, video_data=request_obj.video_data, diff --git a/python/sglang/srt/multimodal/processors/nvila.py b/python/sglang/srt/multimodal/processors/nvila.py index 5fe64d10d..63a706218 100644 --- a/python/sglang/srt/multimodal/processors/nvila.py +++ b/python/sglang/srt/multimodal/processors/nvila.py @@ -55,7 +55,7 @@ class NVILAMultimodalProcessor(BaseMultimodalProcessor): request_obj: GenerateReqInput, **kwargs, ) -> dict[str, Any] | None: - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, multimodal_tokens=self.mm_tokens, image_data=request_obj.image_data, # type: ignore diff --git a/python/sglang/srt/multimodal/processors/phi4mm.py b/python/sglang/srt/multimodal/processors/phi4mm.py index 6ae194eac..0cce5b296 100644 --- a/python/sglang/srt/multimodal/processors/phi4mm.py +++ b/python/sglang/srt/multimodal/processors/phi4mm.py @@ -74,7 +74,7 @@ class Phi4MMMultimodalProcessor(BaseMultimodalProcessor): request_obj, **kwargs, ): - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, audio_data=audio_data, image_data=image_data, diff --git a/python/sglang/srt/multimodal/processors/pixtral.py b/python/sglang/srt/multimodal/processors/pixtral.py index 963bd6820..a9ee65e81 100644 --- a/python/sglang/srt/multimodal/processors/pixtral.py +++ b/python/sglang/srt/multimodal/processors/pixtral.py @@ -71,7 +71,7 @@ class PixtralProcessor(BaseMultimodalProcessor): *args, **kwargs, ): - mm_data = self.load_mm_data( + mm_data = await self.load_mm_data( prompt=input_text, multimodal_tokens=self.mm_tokens, image_data=image_data, diff --git a/python/sglang/srt/multimodal/processors/points_v15_chat.py b/python/sglang/srt/multimodal/processors/points_v15_chat.py index 7fac7e909..9bf7490fc 100644 --- a/python/sglang/srt/multimodal/processors/points_v15_chat.py +++ b/python/sglang/srt/multimodal/processors/points_v15_chat.py @@ -26,7 +26,7 @@ class POINTSV15ChatProcessor(QwenVLImageProcessor): *args, **kwargs, ): - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, multimodal_tokens=self.mm_tokens, diff --git a/python/sglang/srt/multimodal/processors/qwen3_asr.py b/python/sglang/srt/multimodal/processors/qwen3_asr.py index 31368077f..8b82334bc 100644 --- a/python/sglang/srt/multimodal/processors/qwen3_asr.py +++ b/python/sglang/srt/multimodal/processors/qwen3_asr.py @@ -71,7 +71,7 @@ class Qwen3ASRMultimodalProcessor(BaseMultimodalProcessor): prompt = self._build_transcription_prompt(input_text) - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=prompt, audio_data=audio_data, multimodal_tokens=self.mm_tokens, diff --git a/python/sglang/srt/multimodal/processors/qwen_audio.py b/python/sglang/srt/multimodal/processors/qwen_audio.py index 5ca7c957c..88f931903 100644 --- a/python/sglang/srt/multimodal/processors/qwen_audio.py +++ b/python/sglang/srt/multimodal/processors/qwen_audio.py @@ -87,7 +87,7 @@ class Qwen2AudioMultimodalProcessor(BaseMultimodalProcessor): input_text, **kwargs, ): - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, audio_data=audio_data, multimodal_tokens=self.mm_tokens, diff --git a/python/sglang/srt/multimodal/processors/qwen_vl.py b/python/sglang/srt/multimodal/processors/qwen_vl.py index fb9fd856b..4e86c25aa 100644 --- a/python/sglang/srt/multimodal/processors/qwen_vl.py +++ b/python/sglang/srt/multimodal/processors/qwen_vl.py @@ -505,7 +505,7 @@ class QwenVLImageProcessor(SGLangBaseProcessor): **kwargs, ): entry_time = time.perf_counter() - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, video_data=request_obj.video_data, diff --git a/python/sglang/srt/multimodal/processors/sarashina2_vision.py b/python/sglang/srt/multimodal/processors/sarashina2_vision.py index c56f969c6..761c067d8 100644 --- a/python/sglang/srt/multimodal/processors/sarashina2_vision.py +++ b/python/sglang/srt/multimodal/processors/sarashina2_vision.py @@ -62,7 +62,7 @@ class Sarashina2VisionProcessor(BaseMultimodalProcessor): **kwargs, ): """Process image data for Sarashina2Vision model using standard SGLang pattern.""" - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, multimodal_tokens=self.mm_tokens, diff --git a/python/sglang/srt/multimodal/processors/step3_vl.py b/python/sglang/srt/multimodal/processors/step3_vl.py index e31985192..20ae17ac4 100644 --- a/python/sglang/srt/multimodal/processors/step3_vl.py +++ b/python/sglang/srt/multimodal/processors/step3_vl.py @@ -504,7 +504,7 @@ class Step3VLImageProcessor(SGLangBaseProcessor): *args, **kwargs, ): - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, video_data=request_obj.video_data, diff --git a/python/sglang/srt/multimodal/processors/voxtral.py b/python/sglang/srt/multimodal/processors/voxtral.py index e6dc15321..7ed9b544c 100644 --- a/python/sglang/srt/multimodal/processors/voxtral.py +++ b/python/sglang/srt/multimodal/processors/voxtral.py @@ -80,7 +80,7 @@ class VoxtralMultimodalProcessor(BaseMultimodalProcessor): # load_mm_data handles async loading, format detection, resampling. # process_and_combine_mm_data cannot be used: HF VoxtralProcessor.__call__ # does not support audio (only apply_chat_template does). - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=prompt_with_placeholders, audio_data=audio_data, multimodal_tokens=self.mm_tokens,