diff --git a/python/sglang/multimodal_gen/runtime/layers/lora/linear.py b/python/sglang/multimodal_gen/runtime/layers/lora/linear.py index 3bb2afd756..61d32ddbcf 100644 --- a/python/sglang/multimodal_gen/runtime/layers/lora/linear.py +++ b/python/sglang/multimodal_gen/runtime/layers/lora/linear.py @@ -438,14 +438,7 @@ class BaseLayerWithLoRA(nn.Module): ) if clear_existing: - self.lora_weights_list.clear() - # Also clear backward compatibility attributes - self.lora_A = None - self.lora_B = None - self.lora_output_offset = None - self.has_lora_output_offset = False - self.lora_path = None - self.strength = 1.0 + self.clear_lora_weights() # Add to list for multi-LoRA support self.lora_weights_list.append( @@ -475,6 +468,16 @@ class BaseLayerWithLoRA(nn.Module): elif self.merged: self.unmerge_lora_weights() + def clear_lora_weights(self) -> None: + self.lora_weights_list.clear() + # Also clear backward compatibility attributes + self.lora_A = None + self.lora_B = None + self.lora_output_offset = None + self.has_lora_output_offset = False + self.lora_path = None + self.strength = 1.0 + def _active_lora_list(self) -> list[LoRAWeightEntry]: if self.lora_weights_list: return list(self.lora_weights_list) @@ -790,9 +793,6 @@ class BaseLayerWithLoRA(nn.Module): @torch.no_grad() def unmerge_lora_weights(self) -> None: - if self.disable_lora: - return - if not self.merged: raise ValueError( "LoRA weights not merged. Please merge them first before unmerging." diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/lora/pipeline.py b/python/sglang/multimodal_gen/runtime/pipelines_core/lora/pipeline.py index 9c053bcc0a..aafd8fbab0 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/lora/pipeline.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/lora/pipeline.py @@ -851,6 +851,11 @@ class LoRAPipeline(ComposedPipelineBase): for n in lora_nicknames ) if not has_any_lora: + # A previous adapter may still be merged here. + if clear_existing: + if layer.merged: + layer.unmerge_lora_weights() + layer.clear_lora_weights() layer.disable_lora = True if rank == 0: diff --git a/python/sglang/multimodal_gen/test/unit/test_lora_pipeline.py b/python/sglang/multimodal_gen/test/unit/test_lora_pipeline.py index 175a7930b6..f9858015be 100644 --- a/python/sglang/multimodal_gen/test/unit/test_lora_pipeline.py +++ b/python/sglang/multimodal_gen/test/unit/test_lora_pipeline.py @@ -402,3 +402,42 @@ def test_cached_merge_restores_untouched_base_view(): layer.merge_lora_weights(strength=0.5) layer.unmerge_lora_weights() torch.testing.assert_close(base.weight, original, rtol=0, atol=0) + + +def test_switching_adapters_unmerges_layers_the_new_adapter_skips(): + layer_x, layer_y = _make_layer(), _make_layer() + base_x = layer_x.base_layer.weight.detach().clone() + pipeline = _make_pipeline(layer_x) + pipeline.lora_layers = {"x": layer_x, "y": layer_y} + pipeline.lora_adapters["a"]["x.lora_A"] = torch.ones(1, 2) + pipeline.lora_adapters["a"]["x.lora_B"] = torch.ones(2, 1) + pipeline.lora_adapters["b"]["y.lora_A"] = torch.ones(1, 2) + pipeline.lora_adapters["b"]["y.lora_B"] = torch.ones(2, 1) + + with patch(_RANK_PATCH, return_value=0): + pipeline._apply_lora_to_layers( + pipeline.lora_layers, ["a"], ["/a"], 0, [1.0], clear_existing=True + ) + assert not torch.equal(layer_x.base_layer.weight, base_x) + pipeline._apply_lora_to_layers( + pipeline.lora_layers, ["b"], ["/b"], 0, [1.0], clear_existing=True + ) + + assert not layer_x.merged + assert layer_x.disable_lora + assert layer_x.lora_weights_list == [] + torch.testing.assert_close(layer_x.base_layer.weight, base_x) + x = torch.randn(3, 2) + torch.testing.assert_close(layer_x(x), x @ base_x.T) + + +def test_unmerge_restores_base_even_when_lora_disabled(): + layer = _make_layer() + base = layer.base_layer.weight.detach().clone() + layer.set_lora_weights(torch.ones(1, 2), torch.ones(2, 1), merge_weights=True) + layer.disable_lora = True + + layer.unmerge_lora_weights() + + assert not layer.merged + torch.testing.assert_close(layer.base_layer.weight, base)