From 1be30131184a797ff433d07ba4a44441bb06d32f Mon Sep 17 00:00:00 2001 From: Jzz1943 Date: Tue, 6 Oct 2026 16:15:21 +0800 Subject: [PATCH] [diffusion] feat: support Ideogram TurboTime LoRA (#30487) --- .../multimodal_gen/configs/sample/ideogram.py | 27 ++++++ .../runtime/cache/cache_dit_integration.py | 10 ++- .../runtime/layers/lora/linear.py | 83 +++++++++++++++++-- .../runtime/models/dits/base.py | 1 + .../runtime/pipelines_core/schedule_batch.py | 1 + .../stages/model_specific_stages/ideogram.py | 35 +++++++- .../progressive_resolution/denoising.py | 7 +- .../stages/progressive_resolution/ideogram.py | 30 ++++++- .../test/unit/test_cache_dit_per_request.py | 31 +++++++ .../test/unit/test_ideogram4.py | 26 ++++++ .../test/unit/test_lora_inference_mode.py | 53 ++++++++++++ 11 files changed, 287 insertions(+), 17 deletions(-) diff --git a/python/sglang/multimodal_gen/configs/sample/ideogram.py b/python/sglang/multimodal_gen/configs/sample/ideogram.py index 561a3b845d..981e9521bf 100644 --- a/python/sglang/multimodal_gen/configs/sample/ideogram.py +++ b/python/sglang/multimodal_gen/configs/sample/ideogram.py @@ -26,6 +26,33 @@ IDEOGRAM4_PRESETS: dict[str, dict[str, object]] = { "mu": 0.5, "std": 1.75, }, + "V4_TURBOTIME_LORA_2": { + "num_steps": 2, + "guidance_schedule": (1.0,) * 2, + "mu": 0.5, + "std": 1.75, + "skip_unconditional": True, + "lora_scale": 1.0, + "requires_lora": True, + }, + "V4_TURBOTIME_LORA_4": { + "num_steps": 4, + "guidance_schedule": (1.0,) * 4, + "mu": 0.5, + "std": 1.75, + "skip_unconditional": True, + "lora_scale": 1.0, + "requires_lora": True, + }, + "V4_TURBOTIME_LORA_8": { + "num_steps": 8, + "guidance_schedule": (1.0,) * 8, + "mu": 0.5, + "std": 1.75, + "skip_unconditional": True, + "lora_scale": 1.0, + "requires_lora": True, + }, "V4_FAST_20": { "num_steps": 20, "guidance_schedule": (1.0,) * 20, diff --git a/python/sglang/multimodal_gen/runtime/cache/cache_dit_integration.py b/python/sglang/multimodal_gen/runtime/cache/cache_dit_integration.py index bd10bc51f1..edc19f80c3 100644 --- a/python/sglang/multimodal_gen/runtime/cache/cache_dit_integration.py +++ b/python/sglang/multimodal_gen/runtime/cache/cache_dit_integration.py @@ -865,19 +865,23 @@ def refresh_context_on_transformer( num_inference_steps: int, scm_preset: str | None = None, verbose: bool = False, + steps_computation_mask: Optional[List[int]] = None, + steps_computation_policy: str | None = None, ) -> None: """Refresh cache-dit context for transformer.""" - steps_computation_mask = None - if scm_preset is not None: + if steps_computation_mask is None and scm_preset is not None: steps_computation_mask = cache_dit.steps_mask( mask_policy=scm_preset, total_steps=num_inference_steps ) + policy = ( + steps_computation_policy if steps_computation_policy is not None else scm_preset + ) cache_dit.refresh_context( transformer, cache_config=DBCacheConfig().reset( num_inference_steps=num_inference_steps, steps_computation_mask=steps_computation_mask, - steps_computation_policy=scm_preset, + steps_computation_policy=policy, ), verbose=verbose, ) diff --git a/python/sglang/multimodal_gen/runtime/layers/lora/linear.py b/python/sglang/multimodal_gen/runtime/layers/lora/linear.py index 13346c80e5..3dda849ef5 100644 --- a/python/sglang/multimodal_gen/runtime/layers/lora/linear.py +++ b/python/sglang/multimodal_gen/runtime/layers/lora/linear.py @@ -29,9 +29,15 @@ from sglang.multimodal_gen.runtime.layers.linear import ( RowParallelLinear, UnquantizedLinearMethod, ) +from sglang.multimodal_gen.runtime.layers.quantization.weight_only_fp8 import ( + WeightOnlyFP8Linear, +) from sglang.multimodal_gen.runtime.layers.vocab_parallel_embedding import ( VocabParallelEmbedding, ) +from sglang.multimodal_gen.runtime.managers.forward_context import ( + get_forward_context_or_none, +) from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import ( write_dense_weight, ) @@ -120,6 +126,16 @@ class BaseLayerWithLoRA(nn.Module): def bias(self): return getattr(self.base_layer, "bias", None) + @staticmethod + def _runtime_lora_scale() -> float: + context = get_forward_context_or_none() + if context is None: + return 1.0 + forward_batch = context.forward_batch + if forward_batch is None: + return 1.0 + return float(forward_batch.runtime_lora_scale) + @property def can_merge_base_weight(self) -> bool: """Whether a LoRA delta may safely replace the stored base weight.""" @@ -148,6 +164,10 @@ class BaseLayerWithLoRA(nn.Module): lora_A = self.lora_A.to_local() # TODO: Support multiple LoRA adapters when use not merged mode + runtime_lora_scale = self._runtime_lora_scale() + if runtime_lora_scale == 0.0: + return self.base_layer(x) + lora_dtype = lora_A.dtype x_lora = x.to(dtype=lora_dtype) lora_A_sliced = self.slice_lora_a_weights( @@ -161,7 +181,7 @@ class BaseLayerWithLoRA(nn.Module): delta = delta * ( self.lora_alpha / self.lora_rank # type: ignore ) # type: ignore - delta = delta * self.strength + delta = delta * self.strength * runtime_lora_scale out, output_bias = self.base_layer(x) out = out + delta.to(dtype=out.dtype) return self._add_lora_output_offset(out), output_bias @@ -208,6 +228,8 @@ class BaseLayerWithLoRA(nn.Module): offset = self._active_lora_output_offset() if offset is None: return output + if not self.merged: + offset = offset * self._runtime_lora_scale() return output + offset.to(device=output.device, dtype=output.dtype) @staticmethod @@ -718,6 +740,10 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA): if self.disable_lora or (self.merged and not self.has_lora_output_offset): return self.base_layer(input_) + runtime_lora_scale = self._runtime_lora_scale() + if runtime_lora_scale == 0.0: + return self.base_layer(input_) + lora_A = self.lora_A lora_B = self.lora_B if isinstance(self.lora_B, DTensor): @@ -744,7 +770,7 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA): delta_parallel = delta_parallel * ( self.lora_alpha / self.lora_rank # type: ignore ) # type: ignore - delta_parallel = delta_parallel * self.strength + delta_parallel = delta_parallel * self.strength * runtime_lora_scale output_parallel = output_parallel + delta_parallel.to( dtype=output_parallel.dtype ) @@ -852,6 +878,10 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA): if self.disable_lora or (self.merged and not self.has_lora_output_offset): return self.base_layer(input_) + runtime_lora_scale = self._runtime_lora_scale() + if runtime_lora_scale == 0.0: + return self.base_layer(input_) + lora_A = self.lora_A lora_B = self.lora_B if isinstance(self.lora_B, DTensor): @@ -885,7 +915,7 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA): delta_parallel = delta_parallel * ( self.lora_alpha / self.lora_rank # type: ignore ) # type: ignore - delta_parallel = delta_parallel * self.strength + delta_parallel = delta_parallel * self.strength * runtime_lora_scale output_parallel = output_parallel + delta_parallel.to( dtype=output_parallel.dtype ) @@ -944,6 +974,11 @@ class LinearWithLoRA(BaseLayerWithLoRA): lora_A = self.lora_A.to_local() # TODO: Support multiple LoRA adapters when use not merged mode + runtime_lora_scale = self._runtime_lora_scale() + if runtime_lora_scale == 0.0: + # nn.Linear.forward() returns a single tensor + return self.base_layer(x) + if not self.merged and not self.disable_lora: lora_dtype = lora_A.dtype x_lora = x.to(dtype=lora_dtype) @@ -958,7 +993,7 @@ class LinearWithLoRA(BaseLayerWithLoRA): delta = delta * ( self.lora_alpha / self.lora_rank # type: ignore ) # type: ignore - delta = delta * self.strength + delta = delta * self.strength * runtime_lora_scale # nn.Linear.forward() returns a single tensor, not a tuple out = self.base_layer(x) out = out + delta.to(dtype=out.dtype) @@ -976,6 +1011,39 @@ def _use_owned_base_snapshot( return snapshot_base or device_type not in ("cpu", "meta") +class WeightOnlyFP8LinearWithLoRA(LinearWithLoRA): + """ + Dynamic-only LoRA wrapper for storage-only FP8 linear layers. + + Merging LoRA into FP8 storage weights requires dequantizing, applying the + delta, and requantizing weight_scale consistently. Keep the first FP8 path + explicit and only support dynamic LoRA. + """ + + @property + def can_merge_base_weight(self) -> bool: + return False + + def set_lora_weights( + self, + A: torch.Tensor, + B: torch.Tensor, + lora_path: str | None = None, + strength: float = 1.0, + clear_existing: bool = False, + merge_weights: bool = True, + output_offset: torch.Tensor | None = None, + ) -> None: + if merge_weights: + raise ValueError( + "Weight-only FP8 LoRA only supports dynamic mode; " + "please use --lora-merge-mode dynamic." + ) + super().set_lora_weights( + A, B, lora_path, strength, clear_existing, False, output_offset + ) + + def wrap_with_lora_layer( layer: nn.Module, lora_rank: int | None = None, @@ -985,11 +1053,12 @@ def wrap_with_lora_layer( """ transform the given layer to its corresponding LoRA layer """ - supported_layer_types: dict[ - type[LinearBase] | type[nn.Linear], type[BaseLayerWithLoRA] - ] = { + supported_layer_types: dict[type[nn.Module], type[BaseLayerWithLoRA]] = { # the order matters # VocabParallelEmbedding: VocabParallelEmbeddingWithLoRA, + # Weight-only FP8 LoRA is currently dynamic-only and intended for + # single-GPU deployments. + WeightOnlyFP8Linear: WeightOnlyFP8LinearWithLoRA, QKVParallelLinear: QKVParallelLinearWithLoRA, MergedColumnParallelLinear: MergedColumnParallelLinearWithLoRA, ColumnParallelLinear: ColumnParallelLinearWithLoRA, diff --git a/python/sglang/multimodal_gen/runtime/models/dits/base.py b/python/sglang/multimodal_gen/runtime/models/dits/base.py index 60108e3663..898a7dee8a 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/base.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/base.py @@ -34,6 +34,7 @@ class BaseDiT(nn.Module, ABC): _fsdp_forward_methods: tuple[str, ...] = () param_names_mapping: dict reverse_param_names_mapping: dict + lora_param_names_mapping: dict = {} hidden_size: int num_attention_heads: int num_channels_latents: int diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/schedule_batch.py b/python/sglang/multimodal_gen/runtime/pipelines_core/schedule_batch.py index 52f8a2d26f..454a4d39be 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/schedule_batch.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/schedule_batch.py @@ -82,6 +82,7 @@ class Req: """ sampling_params: SamplingParams | None = None + runtime_lora_scale: float = 1.0 generator: torch.Generator | list[torch.Generator] | None = None diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ideogram.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ideogram.py index c854946563..0785ae27fc 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ideogram.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ideogram.py @@ -350,6 +350,37 @@ class Ideogram4DenoisingStage(DenoisingStage): preset_cfg = IDEOGRAM4_PRESETS[preset] num_steps = int(preset_cfg["num_steps"]) device = get_local_torch_device() + + skip_unconditional = bool(preset_cfg.get("skip_unconditional", False)) + lora_scale = float(preset_cfg.get("lora_scale", 0.0)) + requires_lora = bool(preset_cfg.get("requires_lora", False)) + + if skip_unconditional and any( + float(guidance) != 1.0 for guidance in preset_cfg["guidance_schedule"] + ): + raise ValueError("skip_unconditional requires guidance_schedule to be 1.0") + + if ( + lora_scale == 0.0 + and getattr(server_args, "lora_path", None) is not None + and getattr(server_args, "lora_merge_mode", "auto") != "dynamic" + ): + raise ValueError( + f"Ideogram 4 preset {preset} disables LoRA, but request-local LoRA " + "switching requires --lora-merge-mode dynamic when --lora-path is set." + ) + + pipeline = self.pipeline() if self.pipeline else None + if requires_lora and ( + pipeline is None or not pipeline.is_lora_effective("transformer") + ): + raise ValueError( + f"Ideogram 4 preset {preset} requires an active LoRA on transformer. " + "Please start the server with --lora-path and --lora-merge-mode dynamic." + ) + + batch.runtime_lora_scale = lora_scale + schedule = get_schedule_for_resolution( (batch.height, batch.width), known_mean=float(preset_cfg["mu"]), @@ -405,6 +436,7 @@ class Ideogram4DenoisingStage(DenoisingStage): "ideogram4_schedule_deltas": schedule_deltas, "ideogram4_guidance_schedule": guidance_schedule, "ideogram4_text_z_padding": text_z_padding, + "ideogram4_skip_unconditional": skip_unconditional, "ideogram4_attn_mask": attn_mask, "ideogram4_attn_mask_meta": build_varlen_mask_meta(attn_mask), "ideogram4_neg_position_ids": neg_position_ids, @@ -432,6 +464,7 @@ class Ideogram4DenoisingStage(DenoisingStage): schedule_values = ctx.extra["ideogram4_schedule_values"] schedule_deltas = ctx.extra["ideogram4_schedule_deltas"] guidance_schedule = ctx.extra["ideogram4_guidance_schedule"] + skip_unconditional = ctx.extra.get("ideogram4_skip_unconditional", False) i = step.t_int t_val = schedule_values[i + 1] @@ -461,7 +494,7 @@ class Ideogram4DenoisingStage(DenoisingStage): pos_v = pos_out[:, max_text_tokens : max_text_tokens + num_image_tokens] neg_v = None - if self.unconditional_transformer is not None: + if not skip_unconditional and self.unconditional_transformer is not None: self._manage_unconditional_transformer_use_site(batch) with set_forward_context( current_timestep=i, diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/progressive_resolution/denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/progressive_resolution/denoising.py index 44aee43d70..e9ef10f212 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/progressive_resolution/denoising.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/progressive_resolution/denoising.py @@ -303,7 +303,10 @@ class ProgressiveDenoisingStage(DenoisingStage): return None if (preset is None or preset == "none") else preset def _refresh_cache_dit_context( - self, n_remaining: int, scm_preset: str | None + self, + n_remaining: int, + scm_preset: str | None, + ctx: DenoisingContext | None = None, ) -> None: """Refresh cache-dit activations and step counter at a stage transition. @@ -615,7 +618,7 @@ class ProgressiveDenoisingStage(DenoisingStage): if self._cache_dit_enabled: n_remaining = n_steps - stage_end self._refresh_cache_dit_context( - n_remaining, self._effective_scm_preset() + n_remaining, self._effective_scm_preset(), ctx ) logger.info( "cache-dit context refreshed at stage transition " diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/progressive_resolution/ideogram.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/progressive_resolution/ideogram.py index aede267c4d..1af0af7005 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/progressive_resolution/ideogram.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/progressive_resolution/ideogram.py @@ -22,6 +22,7 @@ from diffusers.utils.torch_utils import randn_tensor from sglang.multimodal_gen.configs.sample.ideogram import IDEOGRAM4_PRESETS from sglang.multimodal_gen.runtime.cache.cache_dit_integration import ( refresh_context_on_dual_transformer, + refresh_context_on_transformer, ) from sglang.multimodal_gen.runtime.distributed import get_local_torch_device from sglang.multimodal_gen.runtime.layers.attention import build_varlen_mask_meta @@ -452,14 +453,35 @@ class Ideogram4ProgressiveDenoisingStage( # ------------------------------------------------------------------ def _refresh_cache_dit_context( - self, n_remaining: int, scm_preset: str | None + self, + n_remaining: int, + scm_preset: str | None, + ctx: DenoisingContext | None = None, ) -> None: - """Refresh both conditional and unconditional transformers.""" + """Refresh active Cache-DiT transformer contexts after a stage transition.""" # Recompute the full SCM config here so custom compute/cache bins stay # active after a progressive-resolution stage transition. - _, scm_policy, steps_computation_mask, steps_computation_mask_2 = ( - self._cache_dit_scm_masks(n_remaining, n_remaining) + skip_unconditional = bool( + self.unconditional_transformer is None + or ( + ctx is not None and ctx.extra.get("ideogram4_skip_unconditional", False) + ) ) + + _, scm_policy, steps_computation_mask, steps_computation_mask_2 = ( + self._cache_dit_scm_masks( + n_remaining, None if skip_unconditional else n_remaining + ) + ) + if skip_unconditional: + refresh_context_on_transformer( + self.transformer, + n_remaining, + steps_computation_mask=steps_computation_mask, + steps_computation_policy=scm_policy, + ) + return + refresh_context_on_dual_transformer( self.transformer, self.unconditional_transformer, diff --git a/python/sglang/multimodal_gen/test/unit/test_cache_dit_per_request.py b/python/sglang/multimodal_gen/test/unit/test_cache_dit_per_request.py index d2d985799b..7eb9c68d98 100644 --- a/python/sglang/multimodal_gen/test/unit/test_cache_dit_per_request.py +++ b/python/sglang/multimodal_gen/test/unit/test_cache_dit_per_request.py @@ -6,6 +6,9 @@ import unittest from types import SimpleNamespace from unittest.mock import patch +import torch + +from sglang.multimodal_gen.runtime.cache import cache_dit_integration from sglang.multimodal_gen.runtime.cache.cache_dit_integration import ( cache_dit_overrides_key, resolve_cache_dit_request_overrides, @@ -16,6 +19,34 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages import ( from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import ( DenoisingStage, ) +from sglang.multimodal_gen.runtime.pipelines_core.stages.progressive_resolution.ideogram import ( + Ideogram4ProgressiveDenoisingStage, +) + + +class TestIdeogramActiveCacheRefresh(unittest.TestCase): + def test_request_override_and_active_transformers_survive_transition(self): + for skip, has_unconditional in ((False, True), (True, True), (False, False)): + with self.subTest(skip=skip, has_unconditional=has_unconditional): + stage = object.__new__(Ideogram4ProgressiveDenoisingStage) + stage.transformer = torch.nn.Linear(2, 2) + stage.unconditional_transformer = ( + torch.nn.Linear(2, 2) if has_unconditional else None + ) + stage._cache_dit_request_overrides = {"scm_preset": "none"} + ctx = SimpleNamespace(extra={"ideogram4_skip_unconditional": skip}) + with patch.object( + cache_dit_integration.cache_dit, "refresh_context" + ) as refresh: + stage._refresh_cache_dit_context( + 4, stage._effective_scm_preset(), ctx + ) + targets = [call.args[0] for call in refresh.call_args_list] + expected = [stage.transformer] + if has_unconditional and not skip: + expected.append(stage.unconditional_transformer) + self.assertEqual(targets, expected) + self.assertIsNone(stage._effective_scm_preset()) class TestResolveCacheDitRequestOverrides(unittest.TestCase): diff --git a/python/sglang/multimodal_gen/test/unit/test_ideogram4.py b/python/sglang/multimodal_gen/test/unit/test_ideogram4.py index cbb3e989fc..4a5de7fd81 100644 --- a/python/sglang/multimodal_gen/test/unit/test_ideogram4.py +++ b/python/sglang/multimodal_gen/test/unit/test_ideogram4.py @@ -472,6 +472,18 @@ class TestIdeogram4(unittest.TestCase): with self.assertRaisesRegex(ValueError, "Unknown Ideogram 4 preset"): Ideogram4SamplingParams(preset="V4_FAST") + def test_turbotime_and_distilled_presets_coexist(self): + for steps in (2, 4, 8): + preset = f"V4_TURBOTIME_LORA_{steps}" + with self.subTest(preset=preset): + params = Ideogram4SamplingParams(preset=preset) + self.assertEqual(params.num_inference_steps, steps) + self.assertEqual(params.guidance_scale, 1.0) + self.assertTrue(IDEOGRAM4_PRESETS[preset]["skip_unconditional"]) + self.assertTrue(IDEOGRAM4_PRESETS[preset]["requires_lora"]) + self.assertEqual(Ideogram4FastSamplingParams().preset, "V4_FAST_20") + self.assertEqual(Ideogram4InstantSamplingParams().preset, "V4_INSTANT_8") + def test_ideogram_distilled_sampling_defaults(self): fast = Ideogram4FastSamplingParams() instant = Ideogram4InstantSamplingParams() @@ -1580,6 +1592,20 @@ class TestIdeogram4(unittest.TestCase): with patch.object(stage, "_run_ideogram_transformer", side_effect=fake_run): stage._run_denoising_step(ctx, step, batch, args) + for skip_unconditional, unconditional in ( + (True, unconditional_transformer), + (False, None), + ): + with self.subTest(skip_unconditional=skip_unconditional): + ctx.latents.zero_() + ctx.extra["ideogram4_skip_unconditional"] = skip_unconditional + stage.unconditional_transformer = unconditional + with patch.object( + stage, "_run_ideogram_transformer", side_effect=fake_run + ) as run: + stage._run_denoising_step(ctx, step, batch, args) + self.assertEqual(run.call_count, 1) + self.assertIs(run.call_args.args[0], transformer) finally: set_global_server_args(prev_args) diff --git a/python/sglang/multimodal_gen/test/unit/test_lora_inference_mode.py b/python/sglang/multimodal_gen/test/unit/test_lora_inference_mode.py index ae1b7db422..dfd26eba84 100644 --- a/python/sglang/multimodal_gen/test/unit/test_lora_inference_mode.py +++ b/python/sglang/multimodal_gen/test/unit/test_lora_inference_mode.py @@ -1,10 +1,63 @@ +from types import SimpleNamespace + +import pytest import torch from torch import nn +from sglang.multimodal_gen.runtime.layers.linear import ReplicatedLinear from sglang.multimodal_gen.runtime.layers.lora.linear import ( LinearWithLoRA, _compute_lora_delta, + wrap_with_lora_layer, ) +from sglang.multimodal_gen.runtime.layers.quantization.weight_only_fp8 import ( + WeightOnlyFP8Linear, +) +from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context + + +@pytest.mark.parametrize("kind", ["linear", "replicated", "fp8"]) +@pytest.mark.parametrize("device", ["cpu", "cuda"]) +def test_request_lora_scale_preserves_delta_offset_and_base(kind, device): + if device == "cuda" and not torch.cuda.is_available(): + pytest.skip("CUDA unavailable") + if kind == "replicated": + base = ReplicatedLinear(4, 3, bias=False) + elif kind == "fp8": + base = WeightOnlyFP8Linear(4, 3, bias=False, enable_fused_w8a8=False) + else: + base = nn.Linear(4, 3, bias=False) + base = base.to(device) + with torch.no_grad(): + base.weight.copy_(torch.ones(3, 4, device=device)) + if kind == "fp8": + base.weight_scale.fill_(1.0) + layer = wrap_with_lora_layer(base, lora_rank=2, lora_alpha=2) + layer.set_lora_weights( + torch.ones(2, 4, device=device), + torch.ones(3, 2, device=device), + strength=0.5, + merge_weights=False, + output_offset=torch.full((3,), 2.0, device=device), + ) + x = torch.ones(2, 4, device=device) + base_weight = base.weight.detach().clone() + batch = SimpleNamespace(runtime_lora_scale=0.0) + for scale in (0.0, 1.0, 0.5, 0.0): + batch.runtime_lora_scale = scale + with set_forward_context(0, None, forward_batch=batch): + actual = layer(x) + if kind == "replicated": + actual, output_bias = actual + assert output_bias is None + torch.testing.assert_close( + actual.float(), torch.full((2, 3), 4.0 + 5.0 * scale, device=device) + ) + torch.testing.assert_close(base.weight.float(), base_weight.float()) + if kind == "fp8": + assert not layer.can_merge_base_weight + with pytest.raises(ValueError, match="only supports dynamic"): + layer.set_lora_weights(torch.ones(2, 4), torch.ones(3, 2)) def test_stacked_lora_delta_preserves_projection_order():