From 679d2851ba28a1c5d85efa116faab9f0087bd4d8 Mon Sep 17 00:00:00 2001 From: Xiaoshuai Zhang Date: Tue, 6 Oct 2026 22:15:14 +0800 Subject: [PATCH] fix(mimo-vl): keep multimodal features when capturing aux hidden states and correct vision preprocessing (#40888) Co-authored-by: jet --- python/sglang/srt/models/mimo_v2.py | 21 +++++++++---------- python/sglang/srt/models/mimo_vl.py | 8 ++++++- .../srt/multimodal/processors/mimo_v2.py | 12 +++++++++-- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/python/sglang/srt/models/mimo_v2.py b/python/sglang/srt/models/mimo_v2.py index c79e645483..56338884cf 100644 --- a/python/sglang/srt/models/mimo_v2.py +++ b/python/sglang/srt/models/mimo_v2.py @@ -1326,16 +1326,11 @@ class MiMoV2ForCausalLM(nn.Module, AudioEncoderMixin): ) aux_hidden_states = None - if self.capture_aux_hidden_states: - hidden_states, hidden_states_before_norm, aux_hidden_states = self.model( - input_ids, - positions, - forward_batch, - input_embeds, - pp_proxy_tensors=pp_proxy_tensors, - ) - elif self._is_multimodal: - hidden_states, hidden_states_before_norm = general_mm_embed_routine( + # Multimodal embedding must run even when aux hidden states are captured + # (DFLASH/MTP). Taking the capture branch first skipped image/audio + # feature placement and left placeholder tokens as plain text embeddings. + if self._is_multimodal: + model_out = general_mm_embed_routine( input_ids=input_ids, forward_batch=forward_batch, language_model=self.model, @@ -1344,13 +1339,17 @@ class MiMoV2ForCausalLM(nn.Module, AudioEncoderMixin): pp_proxy_tensors=pp_proxy_tensors, ) else: - hidden_states, hidden_states_before_norm = self.model( + model_out = self.model( input_ids, positions, forward_batch, input_embeds, pp_proxy_tensors=pp_proxy_tensors, ) + if self.capture_aux_hidden_states: + hidden_states, hidden_states_before_norm, aux_hidden_states = model_out + else: + hidden_states, hidden_states_before_norm = model_out if self.pp_group.is_last_rank: return self.logits_processor( diff --git a/python/sglang/srt/models/mimo_vl.py b/python/sglang/srt/models/mimo_vl.py index 17ec05c119..944075c80b 100644 --- a/python/sglang/srt/models/mimo_vl.py +++ b/python/sglang/srt/models/mimo_vl.py @@ -318,6 +318,10 @@ class MiMoVisionTransformer(nn.Module): prefix=add_prefix("merger", prefix), use_data_parallel=self.use_data_parallel, ) + # MiMo-VL merger ln_q is LayerNorm (see modeling_mimo_v2.py + # MiMoVisionPatchMerger), not the Qwen2.5-VL RMSNorm. Checkpoint ships + # visual.merger.ln_q.weight only (bias omitted → zeros). + self.merger.ln_q = nn.LayerNorm(hidden_size, eps=1e-6, bias=False) self._post_init() def apply_index(self, tensor: torch.Tensor, index: torch.Tensor): @@ -328,7 +332,9 @@ class MiMoVisionTransformer(nn.Module): def _post_init(self): for name, param in self.named_parameters(): - if "bias" in name: + # Also zero sinks: they are torch.empty at ctor and some checkpoints + # omit individual visual.blocks.*.attn.sinks keys. + if "bias" in name or name.endswith("sinks"): param.data.zero_() def get_window_index_1d(self, grid_thw, col=True): diff --git a/python/sglang/srt/multimodal/processors/mimo_v2.py b/python/sglang/srt/multimodal/processors/mimo_v2.py index b6951b0acb..aca68a8886 100644 --- a/python/sglang/srt/multimodal/processors/mimo_v2.py +++ b/python/sglang/srt/multimodal/processors/mimo_v2.py @@ -214,8 +214,16 @@ class Content: ) -_QWEN2VL_PIXEL_MEAN = torch.Tensor([123.675, 116.28, 103.53]).view(-1, 1, 1) -_QWEN2VL_PIXEL_STD = torch.Tensor([58.395, 57.12, 57.375]).view(-1, 1, 1) +# OPENAI_CLIP stats, matching Qwen2VLImageProcessor (preprocessor_config.json +# image_mean/image_std with do_rescale 1/255). Kept in 0-255 space because +# standardize_batch sees un-rescaled PIL tensors. ImageNet values were wrong +# for MiMo-VL and scrambled colors. +_QWEN2VL_PIXEL_MEAN = ( + torch.Tensor([0.48145466, 0.4578275, 0.40821073]).view(-1, 1, 1) * 255.0 +) +_QWEN2VL_PIXEL_STD = ( + torch.Tensor([0.26862954, 0.26130258, 0.27577711]).view(-1, 1, 1) * 255.0 +) _mean_std_cache = {}