[diffusion] fix: avoid redundant LoRA scaling while preserving exact numerics (#43028)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
@@ -189,6 +189,20 @@ class BaseLayerWithLoRA(nn.Module):
|
||||
def bias(self):
|
||||
return getattr(self.base_layer, "bias", None)
|
||||
|
||||
def _scale_lora_delta(
|
||||
self, delta: torch.Tensor, runtime_lora_scale: float
|
||||
) -> torch.Tensor:
|
||||
if self.lora_alpha != self.lora_rank:
|
||||
delta = delta * (self.lora_alpha / self.lora_rank) # type: ignore
|
||||
if _request_allows_lossless():
|
||||
scale = self.strength * runtime_lora_scale
|
||||
return delta if scale == 1.0 else delta * scale
|
||||
if self.strength != 1.0:
|
||||
delta = delta * self.strength
|
||||
if runtime_lora_scale != 1.0:
|
||||
delta = delta * runtime_lora_scale
|
||||
return delta
|
||||
|
||||
@staticmethod
|
||||
def _runtime_lora_scale() -> float:
|
||||
context = get_forward_context_or_none()
|
||||
@@ -253,13 +267,7 @@ class BaseLayerWithLoRA(nn.Module):
|
||||
return _add_lora_delta_in_gemm(out, x_lora, lora_A, lora_B, scale)
|
||||
|
||||
delta = _compute_lora_delta(x_lora, lora_A, lora_B)
|
||||
if self.lora_alpha != self.lora_rank:
|
||||
delta = delta * (self.lora_alpha / self.lora_rank) # type: ignore
|
||||
# x * 1.0 is exact, so skipping a unit scale keeps the reference bits
|
||||
if self.strength != 1.0:
|
||||
delta = delta * self.strength
|
||||
if runtime_lora_scale != 1.0:
|
||||
delta = delta * runtime_lora_scale
|
||||
delta = self._scale_lora_delta(delta, runtime_lora_scale)
|
||||
return out + delta.to(dtype=out.dtype)
|
||||
|
||||
def slice_lora_a_weights(self, A: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
@@ -73,18 +73,27 @@ def _reference_chain(layer, out, x, runtime_scale):
|
||||
return out + delta.to(dtype=out.dtype)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device", ["cpu", "cuda"])
|
||||
@pytest.mark.parametrize("request_batch", [_batch("exact"), None])
|
||||
@pytest.mark.parametrize("stacked", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"alpha, strength, runtime_scale",
|
||||
[(RANK, 1.0, 1.0), (2 * RANK, 1.0, 1.0), (RANK, 0.7, 1.0), (8, 0.7, 0.5)],
|
||||
[
|
||||
(RANK, 1.0, 1.0),
|
||||
(2 * RANK, 1.0, 1.0),
|
||||
(RANK, 0.7, 1.0),
|
||||
(8, 0.7, 0.5),
|
||||
(8, 0.7, 0.6),
|
||||
],
|
||||
)
|
||||
def test_exact_tier_keeps_the_reference_chain_bit_for_bit(
|
||||
stacked, alpha, strength, runtime_scale, request_batch
|
||||
stacked, alpha, strength, runtime_scale, request_batch, device
|
||||
):
|
||||
if device == "cuda" and not torch.cuda.is_available():
|
||||
pytest.skip("CUDA unavailable")
|
||||
a, b = _adapter(stacked, torch.bfloat16)
|
||||
layer = _layer(a, b, alpha, strength)
|
||||
x = _activations(TOKENS, IN_DIM, seed=2).to(torch.bfloat16)
|
||||
layer = _layer(a, b, alpha, strength).to(device)
|
||||
x = _activations(TOKENS, IN_DIM, seed=2).to(device=device, dtype=torch.bfloat16)
|
||||
with torch.inference_mode(), set_forward_context(0, None, request_batch):
|
||||
out = layer.base_layer(x)
|
||||
actual = layer._add_lora_delta(out.clone(), x, runtime_scale)
|
||||
|
||||
@@ -3,6 +3,7 @@ from types import SimpleNamespace
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.utils._python_dispatch import TorchDispatchMode
|
||||
|
||||
from sglang.multimodal_gen.runtime.layers.linear import ReplicatedLinear
|
||||
from sglang.multimodal_gen.runtime.layers.lora.linear import (
|
||||
@@ -102,3 +103,52 @@ def test_lora_merge_unmerge_handles_inference_base_weight():
|
||||
assert not layer.merged
|
||||
assert not layer.base_layer.weight.is_inference()
|
||||
assert torch.allclose(layer.base_layer.weight, base_weight)
|
||||
|
||||
|
||||
class _AtenMulCounter(TorchDispatchMode):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.count = 0
|
||||
|
||||
def __torch_dispatch__(self, func, types, args=(), kwargs=None):
|
||||
if func.overloadpacket is torch.ops.aten.mul:
|
||||
self.count += 1
|
||||
return func(*args, **(kwargs or {}))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("quality", ["exact", "lossless"])
|
||||
@pytest.mark.parametrize("kind", ["linear", "replicated"])
|
||||
@pytest.mark.parametrize("strength", [1.0, 0.5])
|
||||
@pytest.mark.parametrize("runtime_scale", [None, 1.0, 0.5])
|
||||
def test_unit_request_lora_scale_avoids_redundant_multiplies(
|
||||
kind, strength, runtime_scale, quality
|
||||
):
|
||||
"""Unit scales are skipped; exact retains the nonunit scale order."""
|
||||
base = (
|
||||
ReplicatedLinear(4, 3, bias=False)
|
||||
if kind == "replicated"
|
||||
else nn.Linear(4, 3, bias=False)
|
||||
)
|
||||
layer = wrap_with_lora_layer(base, lora_rank=2, lora_alpha=2)
|
||||
layer.set_lora_weights(
|
||||
torch.ones(2, 4), torch.ones(3, 2), strength=strength, merge_weights=False
|
||||
)
|
||||
x = torch.ones(2, 4)
|
||||
counter = _AtenMulCounter()
|
||||
if runtime_scale is None:
|
||||
with counter:
|
||||
layer(x)
|
||||
else:
|
||||
batch = SimpleNamespace(
|
||||
sampling_params=SimpleNamespace(quality=quality),
|
||||
runtime_lora_scale=runtime_scale,
|
||||
)
|
||||
with set_forward_context(0, None, forward_batch=batch), counter:
|
||||
layer(x)
|
||||
scale = 1.0 if runtime_scale is None else runtime_scale
|
||||
expected_multiplies = (
|
||||
int(strength * scale != 1.0)
|
||||
if runtime_scale is not None and quality == "lossless"
|
||||
else int(strength != 1.0) + int(scale != 1.0)
|
||||
)
|
||||
assert counter.count == expected_multiplies
|
||||
|
||||
Reference in New Issue
Block a user