[Fix] diffusion: Unmerge LoRA layers the next adapter does not cover (#44770)
This commit is contained in:
@@ -438,14 +438,7 @@ class BaseLayerWithLoRA(nn.Module):
|
||||
)
|
||||
|
||||
if clear_existing:
|
||||
self.lora_weights_list.clear()
|
||||
# Also clear backward compatibility attributes
|
||||
self.lora_A = None
|
||||
self.lora_B = None
|
||||
self.lora_output_offset = None
|
||||
self.has_lora_output_offset = False
|
||||
self.lora_path = None
|
||||
self.strength = 1.0
|
||||
self.clear_lora_weights()
|
||||
|
||||
# Add to list for multi-LoRA support
|
||||
self.lora_weights_list.append(
|
||||
@@ -475,6 +468,16 @@ class BaseLayerWithLoRA(nn.Module):
|
||||
elif self.merged:
|
||||
self.unmerge_lora_weights()
|
||||
|
||||
def clear_lora_weights(self) -> None:
|
||||
self.lora_weights_list.clear()
|
||||
# Also clear backward compatibility attributes
|
||||
self.lora_A = None
|
||||
self.lora_B = None
|
||||
self.lora_output_offset = None
|
||||
self.has_lora_output_offset = False
|
||||
self.lora_path = None
|
||||
self.strength = 1.0
|
||||
|
||||
def _active_lora_list(self) -> list[LoRAWeightEntry]:
|
||||
if self.lora_weights_list:
|
||||
return list(self.lora_weights_list)
|
||||
@@ -790,9 +793,6 @@ class BaseLayerWithLoRA(nn.Module):
|
||||
|
||||
@torch.no_grad()
|
||||
def unmerge_lora_weights(self) -> None:
|
||||
if self.disable_lora:
|
||||
return
|
||||
|
||||
if not self.merged:
|
||||
raise ValueError(
|
||||
"LoRA weights not merged. Please merge them first before unmerging."
|
||||
|
||||
@@ -851,6 +851,11 @@ class LoRAPipeline(ComposedPipelineBase):
|
||||
for n in lora_nicknames
|
||||
)
|
||||
if not has_any_lora:
|
||||
# A previous adapter may still be merged here.
|
||||
if clear_existing:
|
||||
if layer.merged:
|
||||
layer.unmerge_lora_weights()
|
||||
layer.clear_lora_weights()
|
||||
layer.disable_lora = True
|
||||
|
||||
if rank == 0:
|
||||
|
||||
@@ -402,3 +402,42 @@ def test_cached_merge_restores_untouched_base_view():
|
||||
layer.merge_lora_weights(strength=0.5)
|
||||
layer.unmerge_lora_weights()
|
||||
torch.testing.assert_close(base.weight, original, rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_switching_adapters_unmerges_layers_the_new_adapter_skips():
|
||||
layer_x, layer_y = _make_layer(), _make_layer()
|
||||
base_x = layer_x.base_layer.weight.detach().clone()
|
||||
pipeline = _make_pipeline(layer_x)
|
||||
pipeline.lora_layers = {"x": layer_x, "y": layer_y}
|
||||
pipeline.lora_adapters["a"]["x.lora_A"] = torch.ones(1, 2)
|
||||
pipeline.lora_adapters["a"]["x.lora_B"] = torch.ones(2, 1)
|
||||
pipeline.lora_adapters["b"]["y.lora_A"] = torch.ones(1, 2)
|
||||
pipeline.lora_adapters["b"]["y.lora_B"] = torch.ones(2, 1)
|
||||
|
||||
with patch(_RANK_PATCH, return_value=0):
|
||||
pipeline._apply_lora_to_layers(
|
||||
pipeline.lora_layers, ["a"], ["/a"], 0, [1.0], clear_existing=True
|
||||
)
|
||||
assert not torch.equal(layer_x.base_layer.weight, base_x)
|
||||
pipeline._apply_lora_to_layers(
|
||||
pipeline.lora_layers, ["b"], ["/b"], 0, [1.0], clear_existing=True
|
||||
)
|
||||
|
||||
assert not layer_x.merged
|
||||
assert layer_x.disable_lora
|
||||
assert layer_x.lora_weights_list == []
|
||||
torch.testing.assert_close(layer_x.base_layer.weight, base_x)
|
||||
x = torch.randn(3, 2)
|
||||
torch.testing.assert_close(layer_x(x), x @ base_x.T)
|
||||
|
||||
|
||||
def test_unmerge_restores_base_even_when_lora_disabled():
|
||||
layer = _make_layer()
|
||||
base = layer.base_layer.weight.detach().clone()
|
||||
layer.set_lora_weights(torch.ones(1, 2), torch.ones(2, 1), merge_weights=True)
|
||||
layer.disable_lora = True
|
||||
|
||||
layer.unmerge_lora_weights()
|
||||
|
||||
assert not layer.merged
|
||||
torch.testing.assert_close(layer.base_layer.weight, base)
|
||||
|
||||
Reference in New Issue
Block a user