diff --git a/python/sglang/multimodal_gen/runtime/layers/lora/linear.py b/python/sglang/multimodal_gen/runtime/layers/lora/linear.py index 78454b1a1b..2db42bd231 100644 --- a/python/sglang/multimodal_gen/runtime/layers/lora/linear.py +++ b/python/sglang/multimodal_gen/runtime/layers/lora/linear.py @@ -189,6 +189,20 @@ class BaseLayerWithLoRA(nn.Module): def bias(self): return getattr(self.base_layer, "bias", None) + def _scale_lora_delta( + self, delta: torch.Tensor, runtime_lora_scale: float + ) -> torch.Tensor: + if self.lora_alpha != self.lora_rank: + delta = delta * (self.lora_alpha / self.lora_rank) # type: ignore + if _request_allows_lossless(): + scale = self.strength * runtime_lora_scale + return delta if scale == 1.0 else delta * scale + if self.strength != 1.0: + delta = delta * self.strength + if runtime_lora_scale != 1.0: + delta = delta * runtime_lora_scale + return delta + @staticmethod def _runtime_lora_scale() -> float: context = get_forward_context_or_none() @@ -253,13 +267,7 @@ class BaseLayerWithLoRA(nn.Module): return _add_lora_delta_in_gemm(out, x_lora, lora_A, lora_B, scale) delta = _compute_lora_delta(x_lora, lora_A, lora_B) - if self.lora_alpha != self.lora_rank: - delta = delta * (self.lora_alpha / self.lora_rank) # type: ignore - # x * 1.0 is exact, so skipping a unit scale keeps the reference bits - if self.strength != 1.0: - delta = delta * self.strength - if runtime_lora_scale != 1.0: - delta = delta * runtime_lora_scale + delta = self._scale_lora_delta(delta, runtime_lora_scale) return out + delta.to(dtype=out.dtype) def slice_lora_a_weights(self, A: torch.Tensor) -> torch.Tensor: diff --git a/python/sglang/multimodal_gen/test/unit/test_lora_dynamic_delta_in_gemm.py b/python/sglang/multimodal_gen/test/unit/test_lora_dynamic_delta_in_gemm.py index a0b08795a9..35afa09483 100644 --- a/python/sglang/multimodal_gen/test/unit/test_lora_dynamic_delta_in_gemm.py +++ b/python/sglang/multimodal_gen/test/unit/test_lora_dynamic_delta_in_gemm.py @@ -73,18 +73,27 @@ def _reference_chain(layer, out, x, runtime_scale): return out + delta.to(dtype=out.dtype) +@pytest.mark.parametrize("device", ["cpu", "cuda"]) @pytest.mark.parametrize("request_batch", [_batch("exact"), None]) @pytest.mark.parametrize("stacked", [False, True]) @pytest.mark.parametrize( "alpha, strength, runtime_scale", - [(RANK, 1.0, 1.0), (2 * RANK, 1.0, 1.0), (RANK, 0.7, 1.0), (8, 0.7, 0.5)], + [ + (RANK, 1.0, 1.0), + (2 * RANK, 1.0, 1.0), + (RANK, 0.7, 1.0), + (8, 0.7, 0.5), + (8, 0.7, 0.6), + ], ) def test_exact_tier_keeps_the_reference_chain_bit_for_bit( - stacked, alpha, strength, runtime_scale, request_batch + stacked, alpha, strength, runtime_scale, request_batch, device ): + if device == "cuda" and not torch.cuda.is_available(): + pytest.skip("CUDA unavailable") a, b = _adapter(stacked, torch.bfloat16) - layer = _layer(a, b, alpha, strength) - x = _activations(TOKENS, IN_DIM, seed=2).to(torch.bfloat16) + layer = _layer(a, b, alpha, strength).to(device) + x = _activations(TOKENS, IN_DIM, seed=2).to(device=device, dtype=torch.bfloat16) with torch.inference_mode(), set_forward_context(0, None, request_batch): out = layer.base_layer(x) actual = layer._add_lora_delta(out.clone(), x, runtime_scale) diff --git a/python/sglang/multimodal_gen/test/unit/test_lora_inference_mode.py b/python/sglang/multimodal_gen/test/unit/test_lora_inference_mode.py index dfd26eba84..db66f29d1e 100644 --- a/python/sglang/multimodal_gen/test/unit/test_lora_inference_mode.py +++ b/python/sglang/multimodal_gen/test/unit/test_lora_inference_mode.py @@ -3,6 +3,7 @@ from types import SimpleNamespace import pytest import torch from torch import nn +from torch.utils._python_dispatch import TorchDispatchMode from sglang.multimodal_gen.runtime.layers.linear import ReplicatedLinear from sglang.multimodal_gen.runtime.layers.lora.linear import ( @@ -102,3 +103,52 @@ def test_lora_merge_unmerge_handles_inference_base_weight(): assert not layer.merged assert not layer.base_layer.weight.is_inference() assert torch.allclose(layer.base_layer.weight, base_weight) + + +class _AtenMulCounter(TorchDispatchMode): + def __init__(self): + super().__init__() + self.count = 0 + + def __torch_dispatch__(self, func, types, args=(), kwargs=None): + if func.overloadpacket is torch.ops.aten.mul: + self.count += 1 + return func(*args, **(kwargs or {})) + + +@pytest.mark.parametrize("quality", ["exact", "lossless"]) +@pytest.mark.parametrize("kind", ["linear", "replicated"]) +@pytest.mark.parametrize("strength", [1.0, 0.5]) +@pytest.mark.parametrize("runtime_scale", [None, 1.0, 0.5]) +def test_unit_request_lora_scale_avoids_redundant_multiplies( + kind, strength, runtime_scale, quality +): + """Unit scales are skipped; exact retains the nonunit scale order.""" + base = ( + ReplicatedLinear(4, 3, bias=False) + if kind == "replicated" + else nn.Linear(4, 3, bias=False) + ) + layer = wrap_with_lora_layer(base, lora_rank=2, lora_alpha=2) + layer.set_lora_weights( + torch.ones(2, 4), torch.ones(3, 2), strength=strength, merge_weights=False + ) + x = torch.ones(2, 4) + counter = _AtenMulCounter() + if runtime_scale is None: + with counter: + layer(x) + else: + batch = SimpleNamespace( + sampling_params=SimpleNamespace(quality=quality), + runtime_lora_scale=runtime_scale, + ) + with set_forward_context(0, None, forward_batch=batch), counter: + layer(x) + scale = 1.0 if runtime_scale is None else runtime_scale + expected_multiplies = ( + int(strength * scale != 1.0) + if runtime_scale is not None and quality == "lossless" + else int(strength != 1.0) + int(scale != 1.0) + ) + assert counter.count == expected_multiplies