[lora] Support LoRA for the NemotronH_Nano_VL_V2 model family (#39798)

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Po-Han Huang <pohanh@nvidia.com>
Co-authored-by: Yanbin Jiang <jybsuper@gmail.com>
This commit is contained in:
elvischenv
2026-10-13 11:29:48 -07:00
committed by GitHub
co-authored by Claude Opus 5 Po-Han Huang Yanbin Jiang
parent ff3f8c348c
commit 2a8e9db3b0
10 changed files with 286 additions and 112 deletions
@@ -76,8 +76,6 @@ class NemotronH_Nano_VL_V2_Config(PretrainedConfig):
if vision_config is None:
vision_config = kwargs.pop("raw_vision_config", None)
super().__init__(**kwargs)
# Handle both cases: when loading from JSON (llm_config is dict) and when called internally by transformers (llm_config; vision_config are None)
if llm_config is not None:
self.llm_config = NemotronHConfig(**llm_config)
@@ -88,6 +86,8 @@ class NemotronH_Nano_VL_V2_Config(PretrainedConfig):
self.llm_config = NemotronHConfig()
self.raw_vision_config = {}
super().__init__(**kwargs)
# Audio (Parakeet) config: stored as a PretrainedConfig sub-object
if sound_config is not None and isinstance(sound_config, dict):
self.sound_config = PretrainedConfig.from_dict(sound_config)
@@ -146,6 +146,9 @@ class NemotronH_Nano_VL_V2_Config(PretrainedConfig):
"video_maintain_aspect_ratio", True
)
def get_text_config(self, decoder=None, encoder=None):
return self.llm_config
def create_radio_config(self):
config = self.raw_vision_config
model_name = config["args"]["model"]
+19 -6
View File
@@ -1085,6 +1085,25 @@ class LoRAManager:
self.lm_head_module = lora_module
continue
# Routed banks live outside `model.layers.`, so the gate below drops
# them before the FusedMoE branch can reject them.
if (
module_name.startswith("model.meta_mlp.")
and isinstance(module, (FusedMoE, InklingBatchDenseMLP))
and all(x in self.target_modules for x in ["gate_up_proj", "down_proj"])
and get_layer_id(module_name) is None
):
raise ValueError(
"LoRA on Intern-S2-Mobius model.meta_mlp routed banks "
"is not supported by the baseline; remove routed-expert "
"targets or use a future bank-aware LoRA implementation."
)
if getattr(
self.base_model, "should_apply_lora", None
) and not self.base_model.should_apply_lora(module_name):
continue
# Handle DeepSeek MLA fused projection: set the boundary
# between q_a and kv_a output partitions so the LoRA layer
# can apply separate B projections for each.
@@ -1123,12 +1142,6 @@ class LoRAManager:
):
layer_id = get_layer_id(module_name)
if layer_id is None:
if module_name.startswith("model.meta_mlp."):
raise ValueError(
"LoRA on Intern-S2-Mobius model.meta_mlp routed banks "
"is not supported by the baseline; remove routed-expert "
"targets or use a future bank-aware LoRA implementation."
)
# FusedMoE submodules outside the decoder layer hierarchy
# (e.g. nested helpers under non-".layers." prefixes) have
# no resolvable layer id; skip them so we don't index
+1 -5
View File
@@ -1099,12 +1099,8 @@ class Gemma4ForConditionalGeneration(PreTrainedModel):
logger.log(level, "%s: %s", msg, names)
return loaded_params
lora_pattern = re.compile(
r"^language_model\.layers\.(\d+)\.(?:self_attn|mlp)\.(?:qkv_proj|o_proj|down_proj|gate_up_proj)"
)
def should_apply_lora(self, module_name: str) -> bool:
return bool(self.lora_pattern.match(module_name))
return module_name.startswith("language_model.layers.")
def get_hidden_dim(self, module_name, layer_idx):
# return input_dim, output_dim
+1 -7
View File
@@ -2,7 +2,6 @@ import json as json_lib
import logging
import math
import os
import re
from array import array
from collections.abc import Iterable
from typing import List, Optional, Set, Tuple
@@ -447,11 +446,6 @@ class Llama4ForConditionalGeneration(nn.Module):
"gate_up_proj": ["gate_proj", "up_proj"],
}
# Pattern to match language model layers only (skip vision_model and multi_modal_projector)
lora_pattern = re.compile(
r"^language_model\.model\.layers\.(\d+)\.(?:self_attn|mlp)\.(?:qkv_proj|o_proj|down_proj|gate_up_proj)"
)
def __init__(
self,
config: Llama4Config,
@@ -585,7 +579,7 @@ class Llama4ForConditionalGeneration(nn.Module):
def should_apply_lora(self, module_name: str) -> bool:
"""Skip vision model and multi_modal_projector for LoRA."""
return bool(self.lora_pattern.match(module_name))
return module_name.startswith("language_model.model.layers.")
def forward(
self,
@@ -58,10 +58,21 @@ class NemotronH_Nano_VL_V2(EVS):
},
)
supported_lora_modules = NemotronHForCausalLM.supported_lora_modules
@staticmethod
def create_evs_config(config: NemotronH_Nano_VL_V2_Config):
return EVSConfig(video_pruning_rate=config.video_pruning_rate)
def should_apply_lora(self, module_name: str) -> bool:
return module_name.startswith("language_model.")
def get_hidden_dim(self, module_name, layer_idx):
return self.language_model.get_hidden_dim(module_name, layer_idx)
def get_stacked_multiply(self, module_name):
return self.language_model.get_stacked_multiply(module_name)
def __init__(
self,
config: NemotronH_Nano_VL_V2_Config,
+3
View File
@@ -2114,6 +2114,9 @@ class Qwen4ExpForConditionalGeneration(Qwen3VLForConditionalGeneration):
packed_modules_mapping = Qwen3_5ForCausalLM.packed_modules_mapping
hf_to_sglang_mapper = None
def should_apply_lora(self, module_name: str) -> bool:
return module_name.startswith("model.layers.")
@staticmethod
def shared_experts_fusion_disable_reason(hf_config, quant_config):
return Qwen4ExpVLModel.shared_experts_fusion_disable_reason(