fix(mimo-vl): keep multimodal features when capturing aux hidden states and correct vision preprocessing (#40888)
Co-authored-by: jet <dev@jetd.one>
This commit is contained in:
@@ -1326,16 +1326,11 @@ class MiMoV2ForCausalLM(nn.Module, AudioEncoderMixin):
|
||||
)
|
||||
|
||||
aux_hidden_states = None
|
||||
if self.capture_aux_hidden_states:
|
||||
hidden_states, hidden_states_before_norm, aux_hidden_states = self.model(
|
||||
input_ids,
|
||||
positions,
|
||||
forward_batch,
|
||||
input_embeds,
|
||||
pp_proxy_tensors=pp_proxy_tensors,
|
||||
)
|
||||
elif self._is_multimodal:
|
||||
hidden_states, hidden_states_before_norm = general_mm_embed_routine(
|
||||
# Multimodal embedding must run even when aux hidden states are captured
|
||||
# (DFLASH/MTP). Taking the capture branch first skipped image/audio
|
||||
# feature placement and left placeholder tokens as plain text embeddings.
|
||||
if self._is_multimodal:
|
||||
model_out = general_mm_embed_routine(
|
||||
input_ids=input_ids,
|
||||
forward_batch=forward_batch,
|
||||
language_model=self.model,
|
||||
@@ -1344,13 +1339,17 @@ class MiMoV2ForCausalLM(nn.Module, AudioEncoderMixin):
|
||||
pp_proxy_tensors=pp_proxy_tensors,
|
||||
)
|
||||
else:
|
||||
hidden_states, hidden_states_before_norm = self.model(
|
||||
model_out = self.model(
|
||||
input_ids,
|
||||
positions,
|
||||
forward_batch,
|
||||
input_embeds,
|
||||
pp_proxy_tensors=pp_proxy_tensors,
|
||||
)
|
||||
if self.capture_aux_hidden_states:
|
||||
hidden_states, hidden_states_before_norm, aux_hidden_states = model_out
|
||||
else:
|
||||
hidden_states, hidden_states_before_norm = model_out
|
||||
|
||||
if self.pp_group.is_last_rank:
|
||||
return self.logits_processor(
|
||||
|
||||
@@ -318,6 +318,10 @@ class MiMoVisionTransformer(nn.Module):
|
||||
prefix=add_prefix("merger", prefix),
|
||||
use_data_parallel=self.use_data_parallel,
|
||||
)
|
||||
# MiMo-VL merger ln_q is LayerNorm (see modeling_mimo_v2.py
|
||||
# MiMoVisionPatchMerger), not the Qwen2.5-VL RMSNorm. Checkpoint ships
|
||||
# visual.merger.ln_q.weight only (bias omitted → zeros).
|
||||
self.merger.ln_q = nn.LayerNorm(hidden_size, eps=1e-6, bias=False)
|
||||
self._post_init()
|
||||
|
||||
def apply_index(self, tensor: torch.Tensor, index: torch.Tensor):
|
||||
@@ -328,7 +332,9 @@ class MiMoVisionTransformer(nn.Module):
|
||||
|
||||
def _post_init(self):
|
||||
for name, param in self.named_parameters():
|
||||
if "bias" in name:
|
||||
# Also zero sinks: they are torch.empty at ctor and some checkpoints
|
||||
# omit individual visual.blocks.*.attn.sinks keys.
|
||||
if "bias" in name or name.endswith("sinks"):
|
||||
param.data.zero_()
|
||||
|
||||
def get_window_index_1d(self, grid_thw, col=True):
|
||||
|
||||
@@ -214,8 +214,16 @@ class Content:
|
||||
)
|
||||
|
||||
|
||||
_QWEN2VL_PIXEL_MEAN = torch.Tensor([123.675, 116.28, 103.53]).view(-1, 1, 1)
|
||||
_QWEN2VL_PIXEL_STD = torch.Tensor([58.395, 57.12, 57.375]).view(-1, 1, 1)
|
||||
# OPENAI_CLIP stats, matching Qwen2VLImageProcessor (preprocessor_config.json
|
||||
# image_mean/image_std with do_rescale 1/255). Kept in 0-255 space because
|
||||
# standardize_batch sees un-rescaled PIL tensors. ImageNet values were wrong
|
||||
# for MiMo-VL and scrambled colors.
|
||||
_QWEN2VL_PIXEL_MEAN = (
|
||||
torch.Tensor([0.48145466, 0.4578275, 0.40821073]).view(-1, 1, 1) * 255.0
|
||||
)
|
||||
_QWEN2VL_PIXEL_STD = (
|
||||
torch.Tensor([0.26862954, 0.26130258, 0.27577711]).view(-1, 1, 1) * 255.0
|
||||
)
|
||||
_mean_std_cache = {}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user