From 2a8e9db3b09f260ce7b381c989063a199303cbb5 Mon Sep 17 00:00:00 2001 From: elvischenv <219235043+elvischenv@users.noreply.github.com> Date: Wed, 14 Oct 2026 02:29:48 +0800 Subject: [PATCH] [lora] Support LoRA for the NemotronH_Nano_VL_V2 model family (#39798) Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Po-Han Huang Co-authored-by: Yanbin Jiang --- python/sglang/srt/configs/nano_nemotron_vl.py | 7 +- python/sglang/srt/lora/lora_manager.py | 25 ++- python/sglang/srt/models/gemma4_mm.py | 6 +- python/sglang/srt/models/mllama4.py | 8 +- python/sglang/srt/models/nano_nemotron_vl.py | 11 + python/sglang/srt/models/qwen4_exp.py | 3 + .../configs/test_nano_nemotron_vl_config.py | 12 +- .../lora/test_lora_manager_init_modules.py | 205 ++++++++++++++++++ .../lora/test_lora_manager_tied_lm_head.py | 91 -------- .../unit/models/test_nano_nemotron_vl.py | 30 +++ 10 files changed, 286 insertions(+), 112 deletions(-) create mode 100644 test/registered/unit/lora/test_lora_manager_init_modules.py delete mode 100644 test/registered/unit/lora/test_lora_manager_tied_lm_head.py diff --git a/python/sglang/srt/configs/nano_nemotron_vl.py b/python/sglang/srt/configs/nano_nemotron_vl.py index 2e37987831..5a9f5acae4 100644 --- a/python/sglang/srt/configs/nano_nemotron_vl.py +++ b/python/sglang/srt/configs/nano_nemotron_vl.py @@ -76,8 +76,6 @@ class NemotronH_Nano_VL_V2_Config(PretrainedConfig): if vision_config is None: vision_config = kwargs.pop("raw_vision_config", None) - super().__init__(**kwargs) - # Handle both cases: when loading from JSON (llm_config is dict) and when called internally by transformers (llm_config; vision_config are None) if llm_config is not None: self.llm_config = NemotronHConfig(**llm_config) @@ -88,6 +86,8 @@ class NemotronH_Nano_VL_V2_Config(PretrainedConfig): self.llm_config = NemotronHConfig() self.raw_vision_config = {} + super().__init__(**kwargs) + # Audio (Parakeet) config: stored as a PretrainedConfig sub-object if sound_config is not None and isinstance(sound_config, dict): self.sound_config = PretrainedConfig.from_dict(sound_config) @@ -146,6 +146,9 @@ class NemotronH_Nano_VL_V2_Config(PretrainedConfig): "video_maintain_aspect_ratio", True ) + def get_text_config(self, decoder=None, encoder=None): + return self.llm_config + def create_radio_config(self): config = self.raw_vision_config model_name = config["args"]["model"] diff --git a/python/sglang/srt/lora/lora_manager.py b/python/sglang/srt/lora/lora_manager.py index 6142089f95..9327307b39 100644 --- a/python/sglang/srt/lora/lora_manager.py +++ b/python/sglang/srt/lora/lora_manager.py @@ -1085,6 +1085,25 @@ class LoRAManager: self.lm_head_module = lora_module continue + # Routed banks live outside `model.layers.`, so the gate below drops + # them before the FusedMoE branch can reject them. + if ( + module_name.startswith("model.meta_mlp.") + and isinstance(module, (FusedMoE, InklingBatchDenseMLP)) + and all(x in self.target_modules for x in ["gate_up_proj", "down_proj"]) + and get_layer_id(module_name) is None + ): + raise ValueError( + "LoRA on Intern-S2-Mobius model.meta_mlp routed banks " + "is not supported by the baseline; remove routed-expert " + "targets or use a future bank-aware LoRA implementation." + ) + + if getattr( + self.base_model, "should_apply_lora", None + ) and not self.base_model.should_apply_lora(module_name): + continue + # Handle DeepSeek MLA fused projection: set the boundary # between q_a and kv_a output partitions so the LoRA layer # can apply separate B projections for each. @@ -1123,12 +1142,6 @@ class LoRAManager: ): layer_id = get_layer_id(module_name) if layer_id is None: - if module_name.startswith("model.meta_mlp."): - raise ValueError( - "LoRA on Intern-S2-Mobius model.meta_mlp routed banks " - "is not supported by the baseline; remove routed-expert " - "targets or use a future bank-aware LoRA implementation." - ) # FusedMoE submodules outside the decoder layer hierarchy # (e.g. nested helpers under non-".layers." prefixes) have # no resolvable layer id; skip them so we don't index diff --git a/python/sglang/srt/models/gemma4_mm.py b/python/sglang/srt/models/gemma4_mm.py index 86a4cdce72..41ae82ca1c 100644 --- a/python/sglang/srt/models/gemma4_mm.py +++ b/python/sglang/srt/models/gemma4_mm.py @@ -1099,12 +1099,8 @@ class Gemma4ForConditionalGeneration(PreTrainedModel): logger.log(level, "%s: %s", msg, names) return loaded_params - lora_pattern = re.compile( - r"^language_model\.layers\.(\d+)\.(?:self_attn|mlp)\.(?:qkv_proj|o_proj|down_proj|gate_up_proj)" - ) - def should_apply_lora(self, module_name: str) -> bool: - return bool(self.lora_pattern.match(module_name)) + return module_name.startswith("language_model.layers.") def get_hidden_dim(self, module_name, layer_idx): # return input_dim, output_dim diff --git a/python/sglang/srt/models/mllama4.py b/python/sglang/srt/models/mllama4.py index 6c3f2d57f3..84db91f121 100644 --- a/python/sglang/srt/models/mllama4.py +++ b/python/sglang/srt/models/mllama4.py @@ -2,7 +2,6 @@ import json as json_lib import logging import math import os -import re from array import array from collections.abc import Iterable from typing import List, Optional, Set, Tuple @@ -447,11 +446,6 @@ class Llama4ForConditionalGeneration(nn.Module): "gate_up_proj": ["gate_proj", "up_proj"], } - # Pattern to match language model layers only (skip vision_model and multi_modal_projector) - lora_pattern = re.compile( - r"^language_model\.model\.layers\.(\d+)\.(?:self_attn|mlp)\.(?:qkv_proj|o_proj|down_proj|gate_up_proj)" - ) - def __init__( self, config: Llama4Config, @@ -585,7 +579,7 @@ class Llama4ForConditionalGeneration(nn.Module): def should_apply_lora(self, module_name: str) -> bool: """Skip vision model and multi_modal_projector for LoRA.""" - return bool(self.lora_pattern.match(module_name)) + return module_name.startswith("language_model.model.layers.") def forward( self, diff --git a/python/sglang/srt/models/nano_nemotron_vl.py b/python/sglang/srt/models/nano_nemotron_vl.py index 25b8d2f88b..b8c87afe8c 100644 --- a/python/sglang/srt/models/nano_nemotron_vl.py +++ b/python/sglang/srt/models/nano_nemotron_vl.py @@ -58,10 +58,21 @@ class NemotronH_Nano_VL_V2(EVS): }, ) + supported_lora_modules = NemotronHForCausalLM.supported_lora_modules + @staticmethod def create_evs_config(config: NemotronH_Nano_VL_V2_Config): return EVSConfig(video_pruning_rate=config.video_pruning_rate) + def should_apply_lora(self, module_name: str) -> bool: + return module_name.startswith("language_model.") + + def get_hidden_dim(self, module_name, layer_idx): + return self.language_model.get_hidden_dim(module_name, layer_idx) + + def get_stacked_multiply(self, module_name): + return self.language_model.get_stacked_multiply(module_name) + def __init__( self, config: NemotronH_Nano_VL_V2_Config, diff --git a/python/sglang/srt/models/qwen4_exp.py b/python/sglang/srt/models/qwen4_exp.py index faebe4d497..7dbf76c514 100644 --- a/python/sglang/srt/models/qwen4_exp.py +++ b/python/sglang/srt/models/qwen4_exp.py @@ -2114,6 +2114,9 @@ class Qwen4ExpForConditionalGeneration(Qwen3VLForConditionalGeneration): packed_modules_mapping = Qwen3_5ForCausalLM.packed_modules_mapping hf_to_sglang_mapper = None + def should_apply_lora(self, module_name: str) -> bool: + return module_name.startswith("model.layers.") + @staticmethod def shared_experts_fusion_disable_reason(hf_config, quant_config): return Qwen4ExpVLModel.shared_experts_fusion_disable_reason( diff --git a/test/registered/unit/configs/test_nano_nemotron_vl_config.py b/test/registered/unit/configs/test_nano_nemotron_vl_config.py index a8612ccb28..2bac548aa5 100644 --- a/test/registered/unit/configs/test_nano_nemotron_vl_config.py +++ b/test/registered/unit/configs/test_nano_nemotron_vl_config.py @@ -8,7 +8,7 @@ from sglang.srt.configs.nano_nemotron_vl import ( from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase -register_cpu_ci(est_time=2, suite="base-a-test-cpu") +register_cpu_ci(est_time=3, suite="base-a-test-cpu") class TestNemotronHOmniConfig(CustomTestCase): @@ -46,6 +46,16 @@ class TestNemotronHOmniConfig(CustomTestCase): ["linear_attention", "moe", "full_attention"], ) + def test_text_config_resolves_to_the_language_model_config(self): + """get_text_config() returns llm_config, which LoRAManager sizes from.""" + config = NemotronH_Omni_Reasoning_V3_Config( + vision_config={"args": {"model": "radio"}}, + llm_config={}, + ) + + self.assertIs(config.get_text_config(), config.llm_config) + self.assertIs(config.get_text_config(encoder=True), config.llm_config) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/lora/test_lora_manager_init_modules.py b/test/registered/unit/lora/test_lora_manager_init_modules.py new file mode 100644 index 0000000000..115238c0cf --- /dev/null +++ b/test/registered/unit/lora/test_lora_manager_init_modules.py @@ -0,0 +1,205 @@ +# Copyright 2023-2026 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Module-scan regressions in LoRAManager.init_lora_modules.""" + +import sys +import types +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE +from sglang.srt.layers.vocab_parallel_embedding import VocabParallelEmbedding +from sglang.srt.lora import lora_manager as lora_manager_module +from sglang.srt.lora.lora_manager import LoRAManager +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=18, suite="base-a-test-cpu") + +# A VL model whose vision tower nests under `layers..` just like the decoder, +# so get_layer_id() resolves both to layer 0 and only should_apply_lora can tell +# them apart. +LANGUAGE_QKV = "language_model.model.layers.0.mixer.qkv_proj" +VISION_QKV = "vision_model.encoder.layers.0.attn.attn.qkv_proj" +LANGUAGE_EMBED_TOKENS = "language_model.model.embed_tokens" +# Intern-S2-Mobius routed banks: a FusedMoE outside the decoder hierarchy, so +# get_layer_id() returns None and should_apply_lora rejects the prefix. +META_MLP_BANK = "model.meta_mlp.0.experts" + + +class _TiedEmbedding(torch.nn.Module): + def __init__(self): + super().__init__() + self.org_vocab_size = 8 + self.embedding_dim = 4 + self.weight = torch.nn.Parameter(torch.randn(8, 4)) + + +class _ParallelLMHead(torch.nn.Module): + def __init__( + self, + num_embeddings, + embedding_dim, + params_dtype, + org_num_embeddings, + ): + super().__init__() + self.weight = torch.nn.Parameter( + torch.empty(num_embeddings, embedding_dim, dtype=params_dtype) + ) + + +class TestTiedLMHeadLoRA(CustomTestCase): + def test_tied_head_gets_independent_wrapper_with_shared_base_weight(self): + """An lm_head-only adapter must survive tied input/output embeddings.""" + model = torch.nn.Module() + tied_embedding = _TiedEmbedding() + model.embed_tokens = tied_embedding + model.lm_head = tied_embedding + + manager = LoRAManager.__new__(LoRAManager) + manager.base_model = model + manager.base_hf_config = SimpleNamespace(num_hidden_layers=0) + manager.target_modules = {"lm_head"} + wrapped_lm_head = object() + + inkling_module = types.ModuleType("sglang.srt.models.inkling_common.dense_mlp") + inkling_module.InklingBatchDenseMLP = type("InklingBatchDenseMLP", (), {}) + + with ( + patch.object(lora_manager_module, "ParallelLMHead", _ParallelLMHead), + patch.object( + manager, "set_lora_module", return_value=wrapped_lm_head + ) as set_lora_module, + patch.dict( + sys.modules, + {"sglang.srt.models.inkling_common.dense_mlp": inkling_module}, + ), + ): + manager.init_lora_modules() + + self.assertIsNot(model.lm_head, model.embed_tokens) + self.assertIs(model.lm_head.weight, model.embed_tokens.weight) + self.assertIs(manager.lm_head_module, wrapped_lm_head) + set_lora_module.assert_called_once_with("lm_head", model.lm_head) + + +class TestShouldApplyLoRAGate(CustomTestCase): + """init_lora_modules wraps only modules that should_apply_lora admits.""" + + @staticmethod + def _manager(modules, target_modules): + manager = LoRAManager.__new__(LoRAManager) + manager.base_model = SimpleNamespace( + named_modules=lambda: modules, + should_apply_lora=lambda name: name.startswith( + "language_model.model.layers." + ), + ) + manager.base_hf_config = SimpleNamespace(num_hidden_layers=1) + manager.target_modules = target_modules + return manager + + def test_vision_tower_module_sharing_a_target_name_is_skipped(self): + manager = self._manager( + modules=[ + (LANGUAGE_QKV, torch.nn.Module()), + (VISION_QKV, torch.nn.Module()), + ], + target_modules={"qkv_proj"}, + ) + + with patch.object(manager, "set_lora_module", side_effect=lambda name, _: name): + manager.init_lora_modules() + + self.assertEqual(list(manager.lora_modules[0]), [LANGUAGE_QKV]) + + def test_model_scope_keeps_decoder_projections(self): + """Decoder FFN, expert, and hybrid projections stay inside LoRA scope.""" + from sglang.srt.models.gemma4_mm import Gemma4ForConditionalGeneration + from sglang.srt.models.mllama4 import Llama4ForConditionalGeneration + from sglang.srt.models.qwen4_exp import Qwen4ExpForConditionalGeneration + + for model_class, prefix, projections, encoders in ( + ( + Llama4ForConditionalGeneration, + "language_model.model.layers.0.", + ( + "feed_forward.down_proj", + "feed_forward.experts", + "feed_forward.shared_expert.gate_up_proj", + ), + ("vision_model", "multi_modal_projector"), + ), + ( + Gemma4ForConditionalGeneration, + "language_model.layers.0.", + ("moe.experts",), + ("vision_tower", "audio_tower"), + ), + ( + Qwen4ExpForConditionalGeneration, + "model.layers.0.", + ( + "qkv_proj", + "o_proj", + "linear_attn.in_proj_qkvz", + "mlp.experts", + "mlp.shared_expert.down_proj", + ), + ("visual",), + ), + ): + with self.subTest(model=model_class.__name__): + model = object.__new__(model_class) + torch.nn.Module.__init__(model) + for projection in projections: + name = prefix + projection + self.assertTrue(model.should_apply_lora(name), name) + for encoder in encoders: + name = encoder + ".layers.0.qkv_proj" + self.assertFalse(model.should_apply_lora(name), name) + + def test_embed_tokens_outside_the_gated_prefix_is_still_wrapped(self): + """The gate stays below the embed_tokens / lm_head handling.""" + embed_tokens = VocabParallelEmbedding.__new__(VocabParallelEmbedding) + manager = self._manager( + modules=[(LANGUAGE_EMBED_TOKENS, embed_tokens)], + target_modules={"embed_tokens"}, + ) + + with patch.object(manager, "set_lora_module", side_effect=lambda name, _: name): + manager.init_lora_modules() + + self.assertEqual(manager.embed_tokens_module, LANGUAGE_EMBED_TOKENS) + + def test_routed_bank_outside_the_gated_prefix_still_raises(self): + """Routed-bank targets raise instead of being skipped by the gate.""" + bank = FusedMoE.__new__(FusedMoE) + manager = self._manager( + modules=[(META_MLP_BANK, bank)], + target_modules={"gate_up_proj", "down_proj"}, + ) + + with patch.object(manager, "set_lora_module", side_effect=lambda name, _: name): + with self.assertRaisesRegex(ValueError, "meta_mlp routed banks"): + manager.init_lora_modules() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/lora/test_lora_manager_tied_lm_head.py b/test/registered/unit/lora/test_lora_manager_tied_lm_head.py deleted file mode 100644 index 0290bdfb96..0000000000 --- a/test/registered/unit/lora/test_lora_manager_tied_lm_head.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright 2023-2026 SGLang Team -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Regression for #18634: LoRA wrapping of an object-shared lm_head.""" - -import sys -import types -import unittest -from types import SimpleNamespace -from unittest.mock import patch - -import torch - -from sglang.srt.lora import lora_manager as lora_manager_module -from sglang.srt.lora.lora_manager import LoRAManager -from sglang.test.ci.ci_register import register_cpu_ci -from sglang.test.test_utils import CustomTestCase - -register_cpu_ci(est_time=12, suite="base-a-test-cpu") - - -class _TiedEmbedding(torch.nn.Module): - def __init__(self): - super().__init__() - self.org_vocab_size = 8 - self.embedding_dim = 4 - self.weight = torch.nn.Parameter(torch.randn(8, 4)) - - -class _ParallelLMHead(torch.nn.Module): - def __init__( - self, - num_embeddings, - embedding_dim, - params_dtype, - org_num_embeddings, - ): - super().__init__() - self.weight = torch.nn.Parameter( - torch.empty(num_embeddings, embedding_dim, dtype=params_dtype) - ) - - -class TestTiedLMHeadLoRA(CustomTestCase): - def test_tied_head_gets_independent_wrapper_with_shared_base_weight(self): - """An lm_head-only adapter must survive tied input/output embeddings.""" - model = torch.nn.Module() - tied_embedding = _TiedEmbedding() - model.embed_tokens = tied_embedding - model.lm_head = tied_embedding - - manager = LoRAManager.__new__(LoRAManager) - manager.base_model = model - manager.base_hf_config = SimpleNamespace(num_hidden_layers=0) - manager.target_modules = {"lm_head"} - wrapped_lm_head = object() - - inkling_module = types.ModuleType("sglang.srt.models.inkling_common.dense_mlp") - inkling_module.InklingBatchDenseMLP = type("InklingBatchDenseMLP", (), {}) - - with ( - patch.object(lora_manager_module, "ParallelLMHead", _ParallelLMHead), - patch.object( - manager, "set_lora_module", return_value=wrapped_lm_head - ) as set_lora_module, - patch.dict( - sys.modules, - {"sglang.srt.models.inkling_common.dense_mlp": inkling_module}, - ), - ): - manager.init_lora_modules() - - self.assertIsNot(model.lm_head, model.embed_tokens) - self.assertIs(model.lm_head.weight, model.embed_tokens.weight) - self.assertIs(manager.lm_head_module, wrapped_lm_head) - set_lora_module.assert_called_once_with("lm_head", model.lm_head) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/unit/models/test_nano_nemotron_vl.py b/test/registered/unit/models/test_nano_nemotron_vl.py index 88cad16b75..63782c7736 100644 --- a/test/registered/unit/models/test_nano_nemotron_vl.py +++ b/test/registered/unit/models/test_nano_nemotron_vl.py @@ -138,6 +138,36 @@ class TestNemotronHOmniModel(CustomTestCase): self.assertIs(model_class, NemotronH_Omni_Reasoning_V3) self.assertEqual(architecture, "NemotronH_Omni_Reasoning_V3") + def test_lora_scope_excludes_the_vision_tower(self): + """RADIO qkv_proj under a layers.. path is outside the LoRA scope.""" + model = object.__new__(NemotronH_Omni_Reasoning_V3) + nn.Module.__init__(model) + + self.assertTrue( + model.should_apply_lora("language_model.model.layers.7.mixer.qkv_proj") + ) + self.assertFalse( + model.should_apply_lora( + "vision_model.radio_model.model.encoder.layers.7.attn.attn.qkv_proj" + ) + ) + + def test_lora_shape_hooks_reach_the_language_model(self): + """The LoRA shape hooks forward to the inner language model.""" + model = object.__new__(NemotronH_Omni_Reasoning_V3) + nn.Module.__init__(model) + model.language_model = SimpleNamespace( + get_hidden_dim=lambda module_name, layer_idx: (module_name, layer_idx), + get_stacked_multiply=lambda module_name: (module_name, "mult"), + ) + + self.assertEqual(model.get_hidden_dim("out_proj", 5), ("out_proj", 5)) + self.assertEqual( + model.get_stacked_multiply("gate_up_proj_moe"), + ("gate_up_proj_moe", "mult"), + ) + self.assertIn("in_proj", NemotronH_Omni_Reasoning_V3.supported_lora_modules) + def test_vision_final_layernorm_is_loaded_and_applied(self): model = object.__new__(NemotronH_Omni_Reasoning_V3) nn.Module.__init__(model)