[lora] Support LoRA for the NemotronH_Nano_VL_V2 model family (#39798)
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Po-Han Huang <pohanh@nvidia.com> Co-authored-by: Yanbin Jiang <jybsuper@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 5
Po-Han Huang
Yanbin Jiang
parent
ff3f8c348c
commit
2a8e9db3b0
@@ -76,8 +76,6 @@ class NemotronH_Nano_VL_V2_Config(PretrainedConfig):
|
||||
if vision_config is None:
|
||||
vision_config = kwargs.pop("raw_vision_config", None)
|
||||
|
||||
super().__init__(**kwargs)
|
||||
|
||||
# Handle both cases: when loading from JSON (llm_config is dict) and when called internally by transformers (llm_config; vision_config are None)
|
||||
if llm_config is not None:
|
||||
self.llm_config = NemotronHConfig(**llm_config)
|
||||
@@ -88,6 +86,8 @@ class NemotronH_Nano_VL_V2_Config(PretrainedConfig):
|
||||
self.llm_config = NemotronHConfig()
|
||||
self.raw_vision_config = {}
|
||||
|
||||
super().__init__(**kwargs)
|
||||
|
||||
# Audio (Parakeet) config: stored as a PretrainedConfig sub-object
|
||||
if sound_config is not None and isinstance(sound_config, dict):
|
||||
self.sound_config = PretrainedConfig.from_dict(sound_config)
|
||||
@@ -146,6 +146,9 @@ class NemotronH_Nano_VL_V2_Config(PretrainedConfig):
|
||||
"video_maintain_aspect_ratio", True
|
||||
)
|
||||
|
||||
def get_text_config(self, decoder=None, encoder=None):
|
||||
return self.llm_config
|
||||
|
||||
def create_radio_config(self):
|
||||
config = self.raw_vision_config
|
||||
model_name = config["args"]["model"]
|
||||
|
||||
@@ -1085,6 +1085,25 @@ class LoRAManager:
|
||||
self.lm_head_module = lora_module
|
||||
continue
|
||||
|
||||
# Routed banks live outside `model.layers.`, so the gate below drops
|
||||
# them before the FusedMoE branch can reject them.
|
||||
if (
|
||||
module_name.startswith("model.meta_mlp.")
|
||||
and isinstance(module, (FusedMoE, InklingBatchDenseMLP))
|
||||
and all(x in self.target_modules for x in ["gate_up_proj", "down_proj"])
|
||||
and get_layer_id(module_name) is None
|
||||
):
|
||||
raise ValueError(
|
||||
"LoRA on Intern-S2-Mobius model.meta_mlp routed banks "
|
||||
"is not supported by the baseline; remove routed-expert "
|
||||
"targets or use a future bank-aware LoRA implementation."
|
||||
)
|
||||
|
||||
if getattr(
|
||||
self.base_model, "should_apply_lora", None
|
||||
) and not self.base_model.should_apply_lora(module_name):
|
||||
continue
|
||||
|
||||
# Handle DeepSeek MLA fused projection: set the boundary
|
||||
# between q_a and kv_a output partitions so the LoRA layer
|
||||
# can apply separate B projections for each.
|
||||
@@ -1123,12 +1142,6 @@ class LoRAManager:
|
||||
):
|
||||
layer_id = get_layer_id(module_name)
|
||||
if layer_id is None:
|
||||
if module_name.startswith("model.meta_mlp."):
|
||||
raise ValueError(
|
||||
"LoRA on Intern-S2-Mobius model.meta_mlp routed banks "
|
||||
"is not supported by the baseline; remove routed-expert "
|
||||
"targets or use a future bank-aware LoRA implementation."
|
||||
)
|
||||
# FusedMoE submodules outside the decoder layer hierarchy
|
||||
# (e.g. nested helpers under non-".layers." prefixes) have
|
||||
# no resolvable layer id; skip them so we don't index
|
||||
|
||||
@@ -1099,12 +1099,8 @@ class Gemma4ForConditionalGeneration(PreTrainedModel):
|
||||
logger.log(level, "%s: %s", msg, names)
|
||||
return loaded_params
|
||||
|
||||
lora_pattern = re.compile(
|
||||
r"^language_model\.layers\.(\d+)\.(?:self_attn|mlp)\.(?:qkv_proj|o_proj|down_proj|gate_up_proj)"
|
||||
)
|
||||
|
||||
def should_apply_lora(self, module_name: str) -> bool:
|
||||
return bool(self.lora_pattern.match(module_name))
|
||||
return module_name.startswith("language_model.layers.")
|
||||
|
||||
def get_hidden_dim(self, module_name, layer_idx):
|
||||
# return input_dim, output_dim
|
||||
|
||||
@@ -2,7 +2,6 @@ import json as json_lib
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
from array import array
|
||||
from collections.abc import Iterable
|
||||
from typing import List, Optional, Set, Tuple
|
||||
@@ -447,11 +446,6 @@ class Llama4ForConditionalGeneration(nn.Module):
|
||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||
}
|
||||
|
||||
# Pattern to match language model layers only (skip vision_model and multi_modal_projector)
|
||||
lora_pattern = re.compile(
|
||||
r"^language_model\.model\.layers\.(\d+)\.(?:self_attn|mlp)\.(?:qkv_proj|o_proj|down_proj|gate_up_proj)"
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Llama4Config,
|
||||
@@ -585,7 +579,7 @@ class Llama4ForConditionalGeneration(nn.Module):
|
||||
|
||||
def should_apply_lora(self, module_name: str) -> bool:
|
||||
"""Skip vision model and multi_modal_projector for LoRA."""
|
||||
return bool(self.lora_pattern.match(module_name))
|
||||
return module_name.startswith("language_model.model.layers.")
|
||||
|
||||
def forward(
|
||||
self,
|
||||
|
||||
@@ -58,10 +58,21 @@ class NemotronH_Nano_VL_V2(EVS):
|
||||
},
|
||||
)
|
||||
|
||||
supported_lora_modules = NemotronHForCausalLM.supported_lora_modules
|
||||
|
||||
@staticmethod
|
||||
def create_evs_config(config: NemotronH_Nano_VL_V2_Config):
|
||||
return EVSConfig(video_pruning_rate=config.video_pruning_rate)
|
||||
|
||||
def should_apply_lora(self, module_name: str) -> bool:
|
||||
return module_name.startswith("language_model.")
|
||||
|
||||
def get_hidden_dim(self, module_name, layer_idx):
|
||||
return self.language_model.get_hidden_dim(module_name, layer_idx)
|
||||
|
||||
def get_stacked_multiply(self, module_name):
|
||||
return self.language_model.get_stacked_multiply(module_name)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: NemotronH_Nano_VL_V2_Config,
|
||||
|
||||
@@ -2114,6 +2114,9 @@ class Qwen4ExpForConditionalGeneration(Qwen3VLForConditionalGeneration):
|
||||
packed_modules_mapping = Qwen3_5ForCausalLM.packed_modules_mapping
|
||||
hf_to_sglang_mapper = None
|
||||
|
||||
def should_apply_lora(self, module_name: str) -> bool:
|
||||
return module_name.startswith("model.layers.")
|
||||
|
||||
@staticmethod
|
||||
def shared_experts_fusion_disable_reason(hf_config, quant_config):
|
||||
return Qwen4ExpVLModel.shared_experts_fusion_disable_reason(
|
||||
|
||||
Reference in New Issue
Block a user