[Fix] diffusion: Unmerge LoRA layers the next adapter does not cover (#44770)

This commit is contained in:
Avaya Aggarwal
2026-10-19 12:33:40 +08:00
committed by GitHub
parent 266550048a
commit 58ad0748a6
3 changed files with 55 additions and 11 deletions
@@ -438,14 +438,7 @@ class BaseLayerWithLoRA(nn.Module):
)
if clear_existing:
self.lora_weights_list.clear()
# Also clear backward compatibility attributes
self.lora_A = None
self.lora_B = None
self.lora_output_offset = None
self.has_lora_output_offset = False
self.lora_path = None
self.strength = 1.0
self.clear_lora_weights()
# Add to list for multi-LoRA support
self.lora_weights_list.append(
@@ -475,6 +468,16 @@ class BaseLayerWithLoRA(nn.Module):
elif self.merged:
self.unmerge_lora_weights()
def clear_lora_weights(self) -> None:
self.lora_weights_list.clear()
# Also clear backward compatibility attributes
self.lora_A = None
self.lora_B = None
self.lora_output_offset = None
self.has_lora_output_offset = False
self.lora_path = None
self.strength = 1.0
def _active_lora_list(self) -> list[LoRAWeightEntry]:
if self.lora_weights_list:
return list(self.lora_weights_list)
@@ -790,9 +793,6 @@ class BaseLayerWithLoRA(nn.Module):
@torch.no_grad()
def unmerge_lora_weights(self) -> None:
if self.disable_lora:
return
if not self.merged:
raise ValueError(
"LoRA weights not merged. Please merge them first before unmerging."
@@ -851,6 +851,11 @@ class LoRAPipeline(ComposedPipelineBase):
for n in lora_nicknames
)
if not has_any_lora:
# A previous adapter may still be merged here.
if clear_existing:
if layer.merged:
layer.unmerge_lora_weights()
layer.clear_lora_weights()
layer.disable_lora = True
if rank == 0:
@@ -402,3 +402,42 @@ def test_cached_merge_restores_untouched_base_view():
layer.merge_lora_weights(strength=0.5)
layer.unmerge_lora_weights()
torch.testing.assert_close(base.weight, original, rtol=0, atol=0)
def test_switching_adapters_unmerges_layers_the_new_adapter_skips():
layer_x, layer_y = _make_layer(), _make_layer()
base_x = layer_x.base_layer.weight.detach().clone()
pipeline = _make_pipeline(layer_x)
pipeline.lora_layers = {"x": layer_x, "y": layer_y}
pipeline.lora_adapters["a"]["x.lora_A"] = torch.ones(1, 2)
pipeline.lora_adapters["a"]["x.lora_B"] = torch.ones(2, 1)
pipeline.lora_adapters["b"]["y.lora_A"] = torch.ones(1, 2)
pipeline.lora_adapters["b"]["y.lora_B"] = torch.ones(2, 1)
with patch(_RANK_PATCH, return_value=0):
pipeline._apply_lora_to_layers(
pipeline.lora_layers, ["a"], ["/a"], 0, [1.0], clear_existing=True
)
assert not torch.equal(layer_x.base_layer.weight, base_x)
pipeline._apply_lora_to_layers(
pipeline.lora_layers, ["b"], ["/b"], 0, [1.0], clear_existing=True
)
assert not layer_x.merged
assert layer_x.disable_lora
assert layer_x.lora_weights_list == []
torch.testing.assert_close(layer_x.base_layer.weight, base_x)
x = torch.randn(3, 2)
torch.testing.assert_close(layer_x(x), x @ base_x.T)
def test_unmerge_restores_base_even_when_lora_disabled():
layer = _make_layer()
base = layer.base_layer.weight.detach().clone()
layer.set_lora_weights(torch.ones(1, 2), torch.ones(2, 1), merge_weights=True)
layer.disable_lora = True
layer.unmerge_lora_weights()
assert not layer.merged
torch.testing.assert_close(layer.base_layer.weight, base)