fix(mimo-vl): keep multimodal features when capturing aux hidden states and correct vision preprocessing (#40888)

Co-authored-by: jet <dev@jetd.one>
This commit is contained in:
Xiaoshuai Zhang
2026-10-06 22:15:14 +08:00
committed by GitHub
co-authored by jet
parent 16a23a672b
commit 679d2851ba
3 changed files with 27 additions and 14 deletions
+10 -11
View File
@@ -1326,16 +1326,11 @@ class MiMoV2ForCausalLM(nn.Module, AudioEncoderMixin):
)
aux_hidden_states = None
if self.capture_aux_hidden_states:
hidden_states, hidden_states_before_norm, aux_hidden_states = self.model(
input_ids,
positions,
forward_batch,
input_embeds,
pp_proxy_tensors=pp_proxy_tensors,
)
elif self._is_multimodal:
hidden_states, hidden_states_before_norm = general_mm_embed_routine(
# Multimodal embedding must run even when aux hidden states are captured
# (DFLASH/MTP). Taking the capture branch first skipped image/audio
# feature placement and left placeholder tokens as plain text embeddings.
if self._is_multimodal:
model_out = general_mm_embed_routine(
input_ids=input_ids,
forward_batch=forward_batch,
language_model=self.model,
@@ -1344,13 +1339,17 @@ class MiMoV2ForCausalLM(nn.Module, AudioEncoderMixin):
pp_proxy_tensors=pp_proxy_tensors,
)
else:
hidden_states, hidden_states_before_norm = self.model(
model_out = self.model(
input_ids,
positions,
forward_batch,
input_embeds,
pp_proxy_tensors=pp_proxy_tensors,
)
if self.capture_aux_hidden_states:
hidden_states, hidden_states_before_norm, aux_hidden_states = model_out
else:
hidden_states, hidden_states_before_norm = model_out
if self.pp_group.is_last_rank:
return self.logits_processor(
+7 -1
View File
@@ -318,6 +318,10 @@ class MiMoVisionTransformer(nn.Module):
prefix=add_prefix("merger", prefix),
use_data_parallel=self.use_data_parallel,
)
# MiMo-VL merger ln_q is LayerNorm (see modeling_mimo_v2.py
# MiMoVisionPatchMerger), not the Qwen2.5-VL RMSNorm. Checkpoint ships
# visual.merger.ln_q.weight only (bias omitted → zeros).
self.merger.ln_q = nn.LayerNorm(hidden_size, eps=1e-6, bias=False)
self._post_init()
def apply_index(self, tensor: torch.Tensor, index: torch.Tensor):
@@ -328,7 +332,9 @@ class MiMoVisionTransformer(nn.Module):
def _post_init(self):
for name, param in self.named_parameters():
if "bias" in name:
# Also zero sinks: they are torch.empty at ctor and some checkpoints
# omit individual visual.blocks.*.attn.sinks keys.
if "bias" in name or name.endswith("sinks"):
param.data.zero_()
def get_window_index_1d(self, grid_thw, col=True):
@@ -214,8 +214,16 @@ class Content:
)
_QWEN2VL_PIXEL_MEAN = torch.Tensor([123.675, 116.28, 103.53]).view(-1, 1, 1)
_QWEN2VL_PIXEL_STD = torch.Tensor([58.395, 57.12, 57.375]).view(-1, 1, 1)
# OPENAI_CLIP stats, matching Qwen2VLImageProcessor (preprocessor_config.json
# image_mean/image_std with do_rescale 1/255). Kept in 0-255 space because
# standardize_batch sees un-rescaled PIL tensors. ImageNet values were wrong
# for MiMo-VL and scrambled colors.
_QWEN2VL_PIXEL_MEAN = (
torch.Tensor([0.48145466, 0.4578275, 0.40821073]).view(-1, 1, 1) * 255.0
)
_QWEN2VL_PIXEL_STD = (
torch.Tensor([0.26862954, 0.26130258, 0.27577711]).view(-1, 1, 1) * 255.0
)
_mean_std_cache = {}