[diffusion] fix: avoid redundant LoRA scaling while preserving exact numerics (#43028)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
WenhaoZhang
2026-10-13 12:27:04 +08:00
committed by GitHub
co-authored by Mick Qian
parent cda267ab0f
commit feba04fa49
3 changed files with 78 additions and 11 deletions
@@ -189,6 +189,20 @@ class BaseLayerWithLoRA(nn.Module):
def bias(self):
return getattr(self.base_layer, "bias", None)
def _scale_lora_delta(
self, delta: torch.Tensor, runtime_lora_scale: float
) -> torch.Tensor:
if self.lora_alpha != self.lora_rank:
delta = delta * (self.lora_alpha / self.lora_rank) # type: ignore
if _request_allows_lossless():
scale = self.strength * runtime_lora_scale
return delta if scale == 1.0 else delta * scale
if self.strength != 1.0:
delta = delta * self.strength
if runtime_lora_scale != 1.0:
delta = delta * runtime_lora_scale
return delta
@staticmethod
def _runtime_lora_scale() -> float:
context = get_forward_context_or_none()
@@ -253,13 +267,7 @@ class BaseLayerWithLoRA(nn.Module):
return _add_lora_delta_in_gemm(out, x_lora, lora_A, lora_B, scale)
delta = _compute_lora_delta(x_lora, lora_A, lora_B)
if self.lora_alpha != self.lora_rank:
delta = delta * (self.lora_alpha / self.lora_rank) # type: ignore
# x * 1.0 is exact, so skipping a unit scale keeps the reference bits
if self.strength != 1.0:
delta = delta * self.strength
if runtime_lora_scale != 1.0:
delta = delta * runtime_lora_scale
delta = self._scale_lora_delta(delta, runtime_lora_scale)
return out + delta.to(dtype=out.dtype)
def slice_lora_a_weights(self, A: torch.Tensor) -> torch.Tensor:
@@ -73,18 +73,27 @@ def _reference_chain(layer, out, x, runtime_scale):
return out + delta.to(dtype=out.dtype)
@pytest.mark.parametrize("device", ["cpu", "cuda"])
@pytest.mark.parametrize("request_batch", [_batch("exact"), None])
@pytest.mark.parametrize("stacked", [False, True])
@pytest.mark.parametrize(
"alpha, strength, runtime_scale",
[(RANK, 1.0, 1.0), (2 * RANK, 1.0, 1.0), (RANK, 0.7, 1.0), (8, 0.7, 0.5)],
[
(RANK, 1.0, 1.0),
(2 * RANK, 1.0, 1.0),
(RANK, 0.7, 1.0),
(8, 0.7, 0.5),
(8, 0.7, 0.6),
],
)
def test_exact_tier_keeps_the_reference_chain_bit_for_bit(
stacked, alpha, strength, runtime_scale, request_batch
stacked, alpha, strength, runtime_scale, request_batch, device
):
if device == "cuda" and not torch.cuda.is_available():
pytest.skip("CUDA unavailable")
a, b = _adapter(stacked, torch.bfloat16)
layer = _layer(a, b, alpha, strength)
x = _activations(TOKENS, IN_DIM, seed=2).to(torch.bfloat16)
layer = _layer(a, b, alpha, strength).to(device)
x = _activations(TOKENS, IN_DIM, seed=2).to(device=device, dtype=torch.bfloat16)
with torch.inference_mode(), set_forward_context(0, None, request_batch):
out = layer.base_layer(x)
actual = layer._add_lora_delta(out.clone(), x, runtime_scale)
@@ -3,6 +3,7 @@ from types import SimpleNamespace
import pytest
import torch
from torch import nn
from torch.utils._python_dispatch import TorchDispatchMode
from sglang.multimodal_gen.runtime.layers.linear import ReplicatedLinear
from sglang.multimodal_gen.runtime.layers.lora.linear import (
@@ -102,3 +103,52 @@ def test_lora_merge_unmerge_handles_inference_base_weight():
assert not layer.merged
assert not layer.base_layer.weight.is_inference()
assert torch.allclose(layer.base_layer.weight, base_weight)
class _AtenMulCounter(TorchDispatchMode):
def __init__(self):
super().__init__()
self.count = 0
def __torch_dispatch__(self, func, types, args=(), kwargs=None):
if func.overloadpacket is torch.ops.aten.mul:
self.count += 1
return func(*args, **(kwargs or {}))
@pytest.mark.parametrize("quality", ["exact", "lossless"])
@pytest.mark.parametrize("kind", ["linear", "replicated"])
@pytest.mark.parametrize("strength", [1.0, 0.5])
@pytest.mark.parametrize("runtime_scale", [None, 1.0, 0.5])
def test_unit_request_lora_scale_avoids_redundant_multiplies(
kind, strength, runtime_scale, quality
):
"""Unit scales are skipped; exact retains the nonunit scale order."""
base = (
ReplicatedLinear(4, 3, bias=False)
if kind == "replicated"
else nn.Linear(4, 3, bias=False)
)
layer = wrap_with_lora_layer(base, lora_rank=2, lora_alpha=2)
layer.set_lora_weights(
torch.ones(2, 4), torch.ones(3, 2), strength=strength, merge_weights=False
)
x = torch.ones(2, 4)
counter = _AtenMulCounter()
if runtime_scale is None:
with counter:
layer(x)
else:
batch = SimpleNamespace(
sampling_params=SimpleNamespace(quality=quality),
runtime_lora_scale=runtime_scale,
)
with set_forward_context(0, None, forward_batch=batch), counter:
layer(x)
scale = 1.0 if runtime_scale is None else runtime_scale
expected_multiplies = (
int(strength * scale != 1.0)
if runtime_scale is not None and quality == "lossless"
else int(strength != 1.0) + int(scale != 1.0)
)
assert counter.count == expected_multiplies