[diffusion] feat: support Ideogram TurboTime LoRA (#30487)
This commit is contained in:
@@ -26,6 +26,33 @@ IDEOGRAM4_PRESETS: dict[str, dict[str, object]] = {
|
||||
"mu": 0.5,
|
||||
"std": 1.75,
|
||||
},
|
||||
"V4_TURBOTIME_LORA_2": {
|
||||
"num_steps": 2,
|
||||
"guidance_schedule": (1.0,) * 2,
|
||||
"mu": 0.5,
|
||||
"std": 1.75,
|
||||
"skip_unconditional": True,
|
||||
"lora_scale": 1.0,
|
||||
"requires_lora": True,
|
||||
},
|
||||
"V4_TURBOTIME_LORA_4": {
|
||||
"num_steps": 4,
|
||||
"guidance_schedule": (1.0,) * 4,
|
||||
"mu": 0.5,
|
||||
"std": 1.75,
|
||||
"skip_unconditional": True,
|
||||
"lora_scale": 1.0,
|
||||
"requires_lora": True,
|
||||
},
|
||||
"V4_TURBOTIME_LORA_8": {
|
||||
"num_steps": 8,
|
||||
"guidance_schedule": (1.0,) * 8,
|
||||
"mu": 0.5,
|
||||
"std": 1.75,
|
||||
"skip_unconditional": True,
|
||||
"lora_scale": 1.0,
|
||||
"requires_lora": True,
|
||||
},
|
||||
"V4_FAST_20": {
|
||||
"num_steps": 20,
|
||||
"guidance_schedule": (1.0,) * 20,
|
||||
|
||||
@@ -865,19 +865,23 @@ def refresh_context_on_transformer(
|
||||
num_inference_steps: int,
|
||||
scm_preset: str | None = None,
|
||||
verbose: bool = False,
|
||||
steps_computation_mask: Optional[List[int]] = None,
|
||||
steps_computation_policy: str | None = None,
|
||||
) -> None:
|
||||
"""Refresh cache-dit context for transformer."""
|
||||
steps_computation_mask = None
|
||||
if scm_preset is not None:
|
||||
if steps_computation_mask is None and scm_preset is not None:
|
||||
steps_computation_mask = cache_dit.steps_mask(
|
||||
mask_policy=scm_preset, total_steps=num_inference_steps
|
||||
)
|
||||
policy = (
|
||||
steps_computation_policy if steps_computation_policy is not None else scm_preset
|
||||
)
|
||||
cache_dit.refresh_context(
|
||||
transformer,
|
||||
cache_config=DBCacheConfig().reset(
|
||||
num_inference_steps=num_inference_steps,
|
||||
steps_computation_mask=steps_computation_mask,
|
||||
steps_computation_policy=scm_preset,
|
||||
steps_computation_policy=policy,
|
||||
),
|
||||
verbose=verbose,
|
||||
)
|
||||
|
||||
@@ -29,9 +29,15 @@ from sglang.multimodal_gen.runtime.layers.linear import (
|
||||
RowParallelLinear,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.weight_only_fp8 import (
|
||||
WeightOnlyFP8Linear,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import (
|
||||
get_forward_context_or_none,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
|
||||
write_dense_weight,
|
||||
)
|
||||
@@ -120,6 +126,16 @@ class BaseLayerWithLoRA(nn.Module):
|
||||
def bias(self):
|
||||
return getattr(self.base_layer, "bias", None)
|
||||
|
||||
@staticmethod
|
||||
def _runtime_lora_scale() -> float:
|
||||
context = get_forward_context_or_none()
|
||||
if context is None:
|
||||
return 1.0
|
||||
forward_batch = context.forward_batch
|
||||
if forward_batch is None:
|
||||
return 1.0
|
||||
return float(forward_batch.runtime_lora_scale)
|
||||
|
||||
@property
|
||||
def can_merge_base_weight(self) -> bool:
|
||||
"""Whether a LoRA delta may safely replace the stored base weight."""
|
||||
@@ -148,6 +164,10 @@ class BaseLayerWithLoRA(nn.Module):
|
||||
lora_A = self.lora_A.to_local()
|
||||
|
||||
# TODO: Support multiple LoRA adapters when use not merged mode
|
||||
runtime_lora_scale = self._runtime_lora_scale()
|
||||
if runtime_lora_scale == 0.0:
|
||||
return self.base_layer(x)
|
||||
|
||||
lora_dtype = lora_A.dtype
|
||||
x_lora = x.to(dtype=lora_dtype)
|
||||
lora_A_sliced = self.slice_lora_a_weights(
|
||||
@@ -161,7 +181,7 @@ class BaseLayerWithLoRA(nn.Module):
|
||||
delta = delta * (
|
||||
self.lora_alpha / self.lora_rank # type: ignore
|
||||
) # type: ignore
|
||||
delta = delta * self.strength
|
||||
delta = delta * self.strength * runtime_lora_scale
|
||||
out, output_bias = self.base_layer(x)
|
||||
out = out + delta.to(dtype=out.dtype)
|
||||
return self._add_lora_output_offset(out), output_bias
|
||||
@@ -208,6 +228,8 @@ class BaseLayerWithLoRA(nn.Module):
|
||||
offset = self._active_lora_output_offset()
|
||||
if offset is None:
|
||||
return output
|
||||
if not self.merged:
|
||||
offset = offset * self._runtime_lora_scale()
|
||||
return output + offset.to(device=output.device, dtype=output.dtype)
|
||||
|
||||
@staticmethod
|
||||
@@ -718,6 +740,10 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
|
||||
if self.disable_lora or (self.merged and not self.has_lora_output_offset):
|
||||
return self.base_layer(input_)
|
||||
|
||||
runtime_lora_scale = self._runtime_lora_scale()
|
||||
if runtime_lora_scale == 0.0:
|
||||
return self.base_layer(input_)
|
||||
|
||||
lora_A = self.lora_A
|
||||
lora_B = self.lora_B
|
||||
if isinstance(self.lora_B, DTensor):
|
||||
@@ -744,7 +770,7 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
|
||||
delta_parallel = delta_parallel * (
|
||||
self.lora_alpha / self.lora_rank # type: ignore
|
||||
) # type: ignore
|
||||
delta_parallel = delta_parallel * self.strength
|
||||
delta_parallel = delta_parallel * self.strength * runtime_lora_scale
|
||||
output_parallel = output_parallel + delta_parallel.to(
|
||||
dtype=output_parallel.dtype
|
||||
)
|
||||
@@ -852,6 +878,10 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
|
||||
if self.disable_lora or (self.merged and not self.has_lora_output_offset):
|
||||
return self.base_layer(input_)
|
||||
|
||||
runtime_lora_scale = self._runtime_lora_scale()
|
||||
if runtime_lora_scale == 0.0:
|
||||
return self.base_layer(input_)
|
||||
|
||||
lora_A = self.lora_A
|
||||
lora_B = self.lora_B
|
||||
if isinstance(self.lora_B, DTensor):
|
||||
@@ -885,7 +915,7 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
|
||||
delta_parallel = delta_parallel * (
|
||||
self.lora_alpha / self.lora_rank # type: ignore
|
||||
) # type: ignore
|
||||
delta_parallel = delta_parallel * self.strength
|
||||
delta_parallel = delta_parallel * self.strength * runtime_lora_scale
|
||||
output_parallel = output_parallel + delta_parallel.to(
|
||||
dtype=output_parallel.dtype
|
||||
)
|
||||
@@ -944,6 +974,11 @@ class LinearWithLoRA(BaseLayerWithLoRA):
|
||||
lora_A = self.lora_A.to_local()
|
||||
|
||||
# TODO: Support multiple LoRA adapters when use not merged mode
|
||||
runtime_lora_scale = self._runtime_lora_scale()
|
||||
if runtime_lora_scale == 0.0:
|
||||
# nn.Linear.forward() returns a single tensor
|
||||
return self.base_layer(x)
|
||||
|
||||
if not self.merged and not self.disable_lora:
|
||||
lora_dtype = lora_A.dtype
|
||||
x_lora = x.to(dtype=lora_dtype)
|
||||
@@ -958,7 +993,7 @@ class LinearWithLoRA(BaseLayerWithLoRA):
|
||||
delta = delta * (
|
||||
self.lora_alpha / self.lora_rank # type: ignore
|
||||
) # type: ignore
|
||||
delta = delta * self.strength
|
||||
delta = delta * self.strength * runtime_lora_scale
|
||||
# nn.Linear.forward() returns a single tensor, not a tuple
|
||||
out = self.base_layer(x)
|
||||
out = out + delta.to(dtype=out.dtype)
|
||||
@@ -976,6 +1011,39 @@ def _use_owned_base_snapshot(
|
||||
return snapshot_base or device_type not in ("cpu", "meta")
|
||||
|
||||
|
||||
class WeightOnlyFP8LinearWithLoRA(LinearWithLoRA):
|
||||
"""
|
||||
Dynamic-only LoRA wrapper for storage-only FP8 linear layers.
|
||||
|
||||
Merging LoRA into FP8 storage weights requires dequantizing, applying the
|
||||
delta, and requantizing weight_scale consistently. Keep the first FP8 path
|
||||
explicit and only support dynamic LoRA.
|
||||
"""
|
||||
|
||||
@property
|
||||
def can_merge_base_weight(self) -> bool:
|
||||
return False
|
||||
|
||||
def set_lora_weights(
|
||||
self,
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
lora_path: str | None = None,
|
||||
strength: float = 1.0,
|
||||
clear_existing: bool = False,
|
||||
merge_weights: bool = True,
|
||||
output_offset: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
if merge_weights:
|
||||
raise ValueError(
|
||||
"Weight-only FP8 LoRA only supports dynamic mode; "
|
||||
"please use --lora-merge-mode dynamic."
|
||||
)
|
||||
super().set_lora_weights(
|
||||
A, B, lora_path, strength, clear_existing, False, output_offset
|
||||
)
|
||||
|
||||
|
||||
def wrap_with_lora_layer(
|
||||
layer: nn.Module,
|
||||
lora_rank: int | None = None,
|
||||
@@ -985,11 +1053,12 @@ def wrap_with_lora_layer(
|
||||
"""
|
||||
transform the given layer to its corresponding LoRA layer
|
||||
"""
|
||||
supported_layer_types: dict[
|
||||
type[LinearBase] | type[nn.Linear], type[BaseLayerWithLoRA]
|
||||
] = {
|
||||
supported_layer_types: dict[type[nn.Module], type[BaseLayerWithLoRA]] = {
|
||||
# the order matters
|
||||
# VocabParallelEmbedding: VocabParallelEmbeddingWithLoRA,
|
||||
# Weight-only FP8 LoRA is currently dynamic-only and intended for
|
||||
# single-GPU deployments.
|
||||
WeightOnlyFP8Linear: WeightOnlyFP8LinearWithLoRA,
|
||||
QKVParallelLinear: QKVParallelLinearWithLoRA,
|
||||
MergedColumnParallelLinear: MergedColumnParallelLinearWithLoRA,
|
||||
ColumnParallelLinear: ColumnParallelLinearWithLoRA,
|
||||
|
||||
@@ -34,6 +34,7 @@ class BaseDiT(nn.Module, ABC):
|
||||
_fsdp_forward_methods: tuple[str, ...] = ()
|
||||
param_names_mapping: dict
|
||||
reverse_param_names_mapping: dict
|
||||
lora_param_names_mapping: dict = {}
|
||||
hidden_size: int
|
||||
num_attention_heads: int
|
||||
num_channels_latents: int
|
||||
|
||||
@@ -82,6 +82,7 @@ class Req:
|
||||
"""
|
||||
|
||||
sampling_params: SamplingParams | None = None
|
||||
runtime_lora_scale: float = 1.0
|
||||
|
||||
generator: torch.Generator | list[torch.Generator] | None = None
|
||||
|
||||
|
||||
+34
-1
@@ -350,6 +350,37 @@ class Ideogram4DenoisingStage(DenoisingStage):
|
||||
preset_cfg = IDEOGRAM4_PRESETS[preset]
|
||||
num_steps = int(preset_cfg["num_steps"])
|
||||
device = get_local_torch_device()
|
||||
|
||||
skip_unconditional = bool(preset_cfg.get("skip_unconditional", False))
|
||||
lora_scale = float(preset_cfg.get("lora_scale", 0.0))
|
||||
requires_lora = bool(preset_cfg.get("requires_lora", False))
|
||||
|
||||
if skip_unconditional and any(
|
||||
float(guidance) != 1.0 for guidance in preset_cfg["guidance_schedule"]
|
||||
):
|
||||
raise ValueError("skip_unconditional requires guidance_schedule to be 1.0")
|
||||
|
||||
if (
|
||||
lora_scale == 0.0
|
||||
and getattr(server_args, "lora_path", None) is not None
|
||||
and getattr(server_args, "lora_merge_mode", "auto") != "dynamic"
|
||||
):
|
||||
raise ValueError(
|
||||
f"Ideogram 4 preset {preset} disables LoRA, but request-local LoRA "
|
||||
"switching requires --lora-merge-mode dynamic when --lora-path is set."
|
||||
)
|
||||
|
||||
pipeline = self.pipeline() if self.pipeline else None
|
||||
if requires_lora and (
|
||||
pipeline is None or not pipeline.is_lora_effective("transformer")
|
||||
):
|
||||
raise ValueError(
|
||||
f"Ideogram 4 preset {preset} requires an active LoRA on transformer. "
|
||||
"Please start the server with --lora-path and --lora-merge-mode dynamic."
|
||||
)
|
||||
|
||||
batch.runtime_lora_scale = lora_scale
|
||||
|
||||
schedule = get_schedule_for_resolution(
|
||||
(batch.height, batch.width),
|
||||
known_mean=float(preset_cfg["mu"]),
|
||||
@@ -405,6 +436,7 @@ class Ideogram4DenoisingStage(DenoisingStage):
|
||||
"ideogram4_schedule_deltas": schedule_deltas,
|
||||
"ideogram4_guidance_schedule": guidance_schedule,
|
||||
"ideogram4_text_z_padding": text_z_padding,
|
||||
"ideogram4_skip_unconditional": skip_unconditional,
|
||||
"ideogram4_attn_mask": attn_mask,
|
||||
"ideogram4_attn_mask_meta": build_varlen_mask_meta(attn_mask),
|
||||
"ideogram4_neg_position_ids": neg_position_ids,
|
||||
@@ -432,6 +464,7 @@ class Ideogram4DenoisingStage(DenoisingStage):
|
||||
schedule_values = ctx.extra["ideogram4_schedule_values"]
|
||||
schedule_deltas = ctx.extra["ideogram4_schedule_deltas"]
|
||||
guidance_schedule = ctx.extra["ideogram4_guidance_schedule"]
|
||||
skip_unconditional = ctx.extra.get("ideogram4_skip_unconditional", False)
|
||||
i = step.t_int
|
||||
|
||||
t_val = schedule_values[i + 1]
|
||||
@@ -461,7 +494,7 @@ class Ideogram4DenoisingStage(DenoisingStage):
|
||||
pos_v = pos_out[:, max_text_tokens : max_text_tokens + num_image_tokens]
|
||||
|
||||
neg_v = None
|
||||
if self.unconditional_transformer is not None:
|
||||
if not skip_unconditional and self.unconditional_transformer is not None:
|
||||
self._manage_unconditional_transformer_use_site(batch)
|
||||
with set_forward_context(
|
||||
current_timestep=i,
|
||||
|
||||
+5
-2
@@ -303,7 +303,10 @@ class ProgressiveDenoisingStage(DenoisingStage):
|
||||
return None if (preset is None or preset == "none") else preset
|
||||
|
||||
def _refresh_cache_dit_context(
|
||||
self, n_remaining: int, scm_preset: str | None
|
||||
self,
|
||||
n_remaining: int,
|
||||
scm_preset: str | None,
|
||||
ctx: DenoisingContext | None = None,
|
||||
) -> None:
|
||||
"""Refresh cache-dit activations and step counter at a stage transition.
|
||||
|
||||
@@ -615,7 +618,7 @@ class ProgressiveDenoisingStage(DenoisingStage):
|
||||
if self._cache_dit_enabled:
|
||||
n_remaining = n_steps - stage_end
|
||||
self._refresh_cache_dit_context(
|
||||
n_remaining, self._effective_scm_preset()
|
||||
n_remaining, self._effective_scm_preset(), ctx
|
||||
)
|
||||
logger.info(
|
||||
"cache-dit context refreshed at stage transition "
|
||||
|
||||
+26
-4
@@ -22,6 +22,7 @@ from diffusers.utils.torch_utils import randn_tensor
|
||||
from sglang.multimodal_gen.configs.sample.ideogram import IDEOGRAM4_PRESETS
|
||||
from sglang.multimodal_gen.runtime.cache.cache_dit_integration import (
|
||||
refresh_context_on_dual_transformer,
|
||||
refresh_context_on_transformer,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
|
||||
from sglang.multimodal_gen.runtime.layers.attention import build_varlen_mask_meta
|
||||
@@ -452,14 +453,35 @@ class Ideogram4ProgressiveDenoisingStage(
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _refresh_cache_dit_context(
|
||||
self, n_remaining: int, scm_preset: str | None
|
||||
self,
|
||||
n_remaining: int,
|
||||
scm_preset: str | None,
|
||||
ctx: DenoisingContext | None = None,
|
||||
) -> None:
|
||||
"""Refresh both conditional and unconditional transformers."""
|
||||
"""Refresh active Cache-DiT transformer contexts after a stage transition."""
|
||||
# Recompute the full SCM config here so custom compute/cache bins stay
|
||||
# active after a progressive-resolution stage transition.
|
||||
_, scm_policy, steps_computation_mask, steps_computation_mask_2 = (
|
||||
self._cache_dit_scm_masks(n_remaining, n_remaining)
|
||||
skip_unconditional = bool(
|
||||
self.unconditional_transformer is None
|
||||
or (
|
||||
ctx is not None and ctx.extra.get("ideogram4_skip_unconditional", False)
|
||||
)
|
||||
)
|
||||
|
||||
_, scm_policy, steps_computation_mask, steps_computation_mask_2 = (
|
||||
self._cache_dit_scm_masks(
|
||||
n_remaining, None if skip_unconditional else n_remaining
|
||||
)
|
||||
)
|
||||
if skip_unconditional:
|
||||
refresh_context_on_transformer(
|
||||
self.transformer,
|
||||
n_remaining,
|
||||
steps_computation_mask=steps_computation_mask,
|
||||
steps_computation_policy=scm_policy,
|
||||
)
|
||||
return
|
||||
|
||||
refresh_context_on_dual_transformer(
|
||||
self.transformer,
|
||||
self.unconditional_transformer,
|
||||
|
||||
@@ -6,6 +6,9 @@ import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.cache import cache_dit_integration
|
||||
from sglang.multimodal_gen.runtime.cache.cache_dit_integration import (
|
||||
cache_dit_overrides_key,
|
||||
resolve_cache_dit_request_overrides,
|
||||
@@ -16,6 +19,34 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages import (
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import (
|
||||
DenoisingStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.progressive_resolution.ideogram import (
|
||||
Ideogram4ProgressiveDenoisingStage,
|
||||
)
|
||||
|
||||
|
||||
class TestIdeogramActiveCacheRefresh(unittest.TestCase):
|
||||
def test_request_override_and_active_transformers_survive_transition(self):
|
||||
for skip, has_unconditional in ((False, True), (True, True), (False, False)):
|
||||
with self.subTest(skip=skip, has_unconditional=has_unconditional):
|
||||
stage = object.__new__(Ideogram4ProgressiveDenoisingStage)
|
||||
stage.transformer = torch.nn.Linear(2, 2)
|
||||
stage.unconditional_transformer = (
|
||||
torch.nn.Linear(2, 2) if has_unconditional else None
|
||||
)
|
||||
stage._cache_dit_request_overrides = {"scm_preset": "none"}
|
||||
ctx = SimpleNamespace(extra={"ideogram4_skip_unconditional": skip})
|
||||
with patch.object(
|
||||
cache_dit_integration.cache_dit, "refresh_context"
|
||||
) as refresh:
|
||||
stage._refresh_cache_dit_context(
|
||||
4, stage._effective_scm_preset(), ctx
|
||||
)
|
||||
targets = [call.args[0] for call in refresh.call_args_list]
|
||||
expected = [stage.transformer]
|
||||
if has_unconditional and not skip:
|
||||
expected.append(stage.unconditional_transformer)
|
||||
self.assertEqual(targets, expected)
|
||||
self.assertIsNone(stage._effective_scm_preset())
|
||||
|
||||
|
||||
class TestResolveCacheDitRequestOverrides(unittest.TestCase):
|
||||
|
||||
@@ -472,6 +472,18 @@ class TestIdeogram4(unittest.TestCase):
|
||||
with self.assertRaisesRegex(ValueError, "Unknown Ideogram 4 preset"):
|
||||
Ideogram4SamplingParams(preset="V4_FAST")
|
||||
|
||||
def test_turbotime_and_distilled_presets_coexist(self):
|
||||
for steps in (2, 4, 8):
|
||||
preset = f"V4_TURBOTIME_LORA_{steps}"
|
||||
with self.subTest(preset=preset):
|
||||
params = Ideogram4SamplingParams(preset=preset)
|
||||
self.assertEqual(params.num_inference_steps, steps)
|
||||
self.assertEqual(params.guidance_scale, 1.0)
|
||||
self.assertTrue(IDEOGRAM4_PRESETS[preset]["skip_unconditional"])
|
||||
self.assertTrue(IDEOGRAM4_PRESETS[preset]["requires_lora"])
|
||||
self.assertEqual(Ideogram4FastSamplingParams().preset, "V4_FAST_20")
|
||||
self.assertEqual(Ideogram4InstantSamplingParams().preset, "V4_INSTANT_8")
|
||||
|
||||
def test_ideogram_distilled_sampling_defaults(self):
|
||||
fast = Ideogram4FastSamplingParams()
|
||||
instant = Ideogram4InstantSamplingParams()
|
||||
@@ -1580,6 +1592,20 @@ class TestIdeogram4(unittest.TestCase):
|
||||
|
||||
with patch.object(stage, "_run_ideogram_transformer", side_effect=fake_run):
|
||||
stage._run_denoising_step(ctx, step, batch, args)
|
||||
for skip_unconditional, unconditional in (
|
||||
(True, unconditional_transformer),
|
||||
(False, None),
|
||||
):
|
||||
with self.subTest(skip_unconditional=skip_unconditional):
|
||||
ctx.latents.zero_()
|
||||
ctx.extra["ideogram4_skip_unconditional"] = skip_unconditional
|
||||
stage.unconditional_transformer = unconditional
|
||||
with patch.object(
|
||||
stage, "_run_ideogram_transformer", side_effect=fake_run
|
||||
) as run:
|
||||
stage._run_denoising_step(ctx, step, batch, args)
|
||||
self.assertEqual(run.call_count, 1)
|
||||
self.assertIs(run.call_args.args[0], transformer)
|
||||
finally:
|
||||
set_global_server_args(prev_args)
|
||||
|
||||
|
||||
@@ -1,10 +1,63 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.multimodal_gen.runtime.layers.linear import ReplicatedLinear
|
||||
from sglang.multimodal_gen.runtime.layers.lora.linear import (
|
||||
LinearWithLoRA,
|
||||
_compute_lora_delta,
|
||||
wrap_with_lora_layer,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.weight_only_fp8 import (
|
||||
WeightOnlyFP8Linear,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["linear", "replicated", "fp8"])
|
||||
@pytest.mark.parametrize("device", ["cpu", "cuda"])
|
||||
def test_request_lora_scale_preserves_delta_offset_and_base(kind, device):
|
||||
if device == "cuda" and not torch.cuda.is_available():
|
||||
pytest.skip("CUDA unavailable")
|
||||
if kind == "replicated":
|
||||
base = ReplicatedLinear(4, 3, bias=False)
|
||||
elif kind == "fp8":
|
||||
base = WeightOnlyFP8Linear(4, 3, bias=False, enable_fused_w8a8=False)
|
||||
else:
|
||||
base = nn.Linear(4, 3, bias=False)
|
||||
base = base.to(device)
|
||||
with torch.no_grad():
|
||||
base.weight.copy_(torch.ones(3, 4, device=device))
|
||||
if kind == "fp8":
|
||||
base.weight_scale.fill_(1.0)
|
||||
layer = wrap_with_lora_layer(base, lora_rank=2, lora_alpha=2)
|
||||
layer.set_lora_weights(
|
||||
torch.ones(2, 4, device=device),
|
||||
torch.ones(3, 2, device=device),
|
||||
strength=0.5,
|
||||
merge_weights=False,
|
||||
output_offset=torch.full((3,), 2.0, device=device),
|
||||
)
|
||||
x = torch.ones(2, 4, device=device)
|
||||
base_weight = base.weight.detach().clone()
|
||||
batch = SimpleNamespace(runtime_lora_scale=0.0)
|
||||
for scale in (0.0, 1.0, 0.5, 0.0):
|
||||
batch.runtime_lora_scale = scale
|
||||
with set_forward_context(0, None, forward_batch=batch):
|
||||
actual = layer(x)
|
||||
if kind == "replicated":
|
||||
actual, output_bias = actual
|
||||
assert output_bias is None
|
||||
torch.testing.assert_close(
|
||||
actual.float(), torch.full((2, 3), 4.0 + 5.0 * scale, device=device)
|
||||
)
|
||||
torch.testing.assert_close(base.weight.float(), base_weight.float())
|
||||
if kind == "fp8":
|
||||
assert not layer.can_merge_base_weight
|
||||
with pytest.raises(ValueError, match="only supports dynamic"):
|
||||
layer.set_lora_weights(torch.ones(2, 4), torch.ones(3, 2))
|
||||
|
||||
|
||||
def test_stacked_lora_delta_preserves_projection_order():
|
||||
|
||||
Reference in New Issue
Block a user