[diffusion] feat: support Ideogram TurboTime LoRA (#30487)

This commit is contained in:
Jzz1943
2026-10-06 16:15:21 +08:00
committed by GitHub
parent 01345197af
commit 1be3013118
11 changed files with 287 additions and 17 deletions
@@ -26,6 +26,33 @@ IDEOGRAM4_PRESETS: dict[str, dict[str, object]] = {
"mu": 0.5,
"std": 1.75,
},
"V4_TURBOTIME_LORA_2": {
"num_steps": 2,
"guidance_schedule": (1.0,) * 2,
"mu": 0.5,
"std": 1.75,
"skip_unconditional": True,
"lora_scale": 1.0,
"requires_lora": True,
},
"V4_TURBOTIME_LORA_4": {
"num_steps": 4,
"guidance_schedule": (1.0,) * 4,
"mu": 0.5,
"std": 1.75,
"skip_unconditional": True,
"lora_scale": 1.0,
"requires_lora": True,
},
"V4_TURBOTIME_LORA_8": {
"num_steps": 8,
"guidance_schedule": (1.0,) * 8,
"mu": 0.5,
"std": 1.75,
"skip_unconditional": True,
"lora_scale": 1.0,
"requires_lora": True,
},
"V4_FAST_20": {
"num_steps": 20,
"guidance_schedule": (1.0,) * 20,
@@ -865,19 +865,23 @@ def refresh_context_on_transformer(
num_inference_steps: int,
scm_preset: str | None = None,
verbose: bool = False,
steps_computation_mask: Optional[List[int]] = None,
steps_computation_policy: str | None = None,
) -> None:
"""Refresh cache-dit context for transformer."""
steps_computation_mask = None
if scm_preset is not None:
if steps_computation_mask is None and scm_preset is not None:
steps_computation_mask = cache_dit.steps_mask(
mask_policy=scm_preset, total_steps=num_inference_steps
)
policy = (
steps_computation_policy if steps_computation_policy is not None else scm_preset
)
cache_dit.refresh_context(
transformer,
cache_config=DBCacheConfig().reset(
num_inference_steps=num_inference_steps,
steps_computation_mask=steps_computation_mask,
steps_computation_policy=scm_preset,
steps_computation_policy=policy,
),
verbose=verbose,
)
@@ -29,9 +29,15 @@ from sglang.multimodal_gen.runtime.layers.linear import (
RowParallelLinear,
UnquantizedLinearMethod,
)
from sglang.multimodal_gen.runtime.layers.quantization.weight_only_fp8 import (
WeightOnlyFP8Linear,
)
from sglang.multimodal_gen.runtime.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from sglang.multimodal_gen.runtime.managers.forward_context import (
get_forward_context_or_none,
)
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
write_dense_weight,
)
@@ -120,6 +126,16 @@ class BaseLayerWithLoRA(nn.Module):
def bias(self):
return getattr(self.base_layer, "bias", None)
@staticmethod
def _runtime_lora_scale() -> float:
context = get_forward_context_or_none()
if context is None:
return 1.0
forward_batch = context.forward_batch
if forward_batch is None:
return 1.0
return float(forward_batch.runtime_lora_scale)
@property
def can_merge_base_weight(self) -> bool:
"""Whether a LoRA delta may safely replace the stored base weight."""
@@ -148,6 +164,10 @@ class BaseLayerWithLoRA(nn.Module):
lora_A = self.lora_A.to_local()
# TODO: Support multiple LoRA adapters when use not merged mode
runtime_lora_scale = self._runtime_lora_scale()
if runtime_lora_scale == 0.0:
return self.base_layer(x)
lora_dtype = lora_A.dtype
x_lora = x.to(dtype=lora_dtype)
lora_A_sliced = self.slice_lora_a_weights(
@@ -161,7 +181,7 @@ class BaseLayerWithLoRA(nn.Module):
delta = delta * (
self.lora_alpha / self.lora_rank # type: ignore
) # type: ignore
delta = delta * self.strength
delta = delta * self.strength * runtime_lora_scale
out, output_bias = self.base_layer(x)
out = out + delta.to(dtype=out.dtype)
return self._add_lora_output_offset(out), output_bias
@@ -208,6 +228,8 @@ class BaseLayerWithLoRA(nn.Module):
offset = self._active_lora_output_offset()
if offset is None:
return output
if not self.merged:
offset = offset * self._runtime_lora_scale()
return output + offset.to(device=output.device, dtype=output.dtype)
@staticmethod
@@ -718,6 +740,10 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
if self.disable_lora or (self.merged and not self.has_lora_output_offset):
return self.base_layer(input_)
runtime_lora_scale = self._runtime_lora_scale()
if runtime_lora_scale == 0.0:
return self.base_layer(input_)
lora_A = self.lora_A
lora_B = self.lora_B
if isinstance(self.lora_B, DTensor):
@@ -744,7 +770,7 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
delta_parallel = delta_parallel * (
self.lora_alpha / self.lora_rank # type: ignore
) # type: ignore
delta_parallel = delta_parallel * self.strength
delta_parallel = delta_parallel * self.strength * runtime_lora_scale
output_parallel = output_parallel + delta_parallel.to(
dtype=output_parallel.dtype
)
@@ -852,6 +878,10 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
if self.disable_lora or (self.merged and not self.has_lora_output_offset):
return self.base_layer(input_)
runtime_lora_scale = self._runtime_lora_scale()
if runtime_lora_scale == 0.0:
return self.base_layer(input_)
lora_A = self.lora_A
lora_B = self.lora_B
if isinstance(self.lora_B, DTensor):
@@ -885,7 +915,7 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
delta_parallel = delta_parallel * (
self.lora_alpha / self.lora_rank # type: ignore
) # type: ignore
delta_parallel = delta_parallel * self.strength
delta_parallel = delta_parallel * self.strength * runtime_lora_scale
output_parallel = output_parallel + delta_parallel.to(
dtype=output_parallel.dtype
)
@@ -944,6 +974,11 @@ class LinearWithLoRA(BaseLayerWithLoRA):
lora_A = self.lora_A.to_local()
# TODO: Support multiple LoRA adapters when use not merged mode
runtime_lora_scale = self._runtime_lora_scale()
if runtime_lora_scale == 0.0:
# nn.Linear.forward() returns a single tensor
return self.base_layer(x)
if not self.merged and not self.disable_lora:
lora_dtype = lora_A.dtype
x_lora = x.to(dtype=lora_dtype)
@@ -958,7 +993,7 @@ class LinearWithLoRA(BaseLayerWithLoRA):
delta = delta * (
self.lora_alpha / self.lora_rank # type: ignore
) # type: ignore
delta = delta * self.strength
delta = delta * self.strength * runtime_lora_scale
# nn.Linear.forward() returns a single tensor, not a tuple
out = self.base_layer(x)
out = out + delta.to(dtype=out.dtype)
@@ -976,6 +1011,39 @@ def _use_owned_base_snapshot(
return snapshot_base or device_type not in ("cpu", "meta")
class WeightOnlyFP8LinearWithLoRA(LinearWithLoRA):
"""
Dynamic-only LoRA wrapper for storage-only FP8 linear layers.
Merging LoRA into FP8 storage weights requires dequantizing, applying the
delta, and requantizing weight_scale consistently. Keep the first FP8 path
explicit and only support dynamic LoRA.
"""
@property
def can_merge_base_weight(self) -> bool:
return False
def set_lora_weights(
self,
A: torch.Tensor,
B: torch.Tensor,
lora_path: str | None = None,
strength: float = 1.0,
clear_existing: bool = False,
merge_weights: bool = True,
output_offset: torch.Tensor | None = None,
) -> None:
if merge_weights:
raise ValueError(
"Weight-only FP8 LoRA only supports dynamic mode; "
"please use --lora-merge-mode dynamic."
)
super().set_lora_weights(
A, B, lora_path, strength, clear_existing, False, output_offset
)
def wrap_with_lora_layer(
layer: nn.Module,
lora_rank: int | None = None,
@@ -985,11 +1053,12 @@ def wrap_with_lora_layer(
"""
transform the given layer to its corresponding LoRA layer
"""
supported_layer_types: dict[
type[LinearBase] | type[nn.Linear], type[BaseLayerWithLoRA]
] = {
supported_layer_types: dict[type[nn.Module], type[BaseLayerWithLoRA]] = {
# the order matters
# VocabParallelEmbedding: VocabParallelEmbeddingWithLoRA,
# Weight-only FP8 LoRA is currently dynamic-only and intended for
# single-GPU deployments.
WeightOnlyFP8Linear: WeightOnlyFP8LinearWithLoRA,
QKVParallelLinear: QKVParallelLinearWithLoRA,
MergedColumnParallelLinear: MergedColumnParallelLinearWithLoRA,
ColumnParallelLinear: ColumnParallelLinearWithLoRA,
@@ -34,6 +34,7 @@ class BaseDiT(nn.Module, ABC):
_fsdp_forward_methods: tuple[str, ...] = ()
param_names_mapping: dict
reverse_param_names_mapping: dict
lora_param_names_mapping: dict = {}
hidden_size: int
num_attention_heads: int
num_channels_latents: int
@@ -82,6 +82,7 @@ class Req:
"""
sampling_params: SamplingParams | None = None
runtime_lora_scale: float = 1.0
generator: torch.Generator | list[torch.Generator] | None = None
@@ -350,6 +350,37 @@ class Ideogram4DenoisingStage(DenoisingStage):
preset_cfg = IDEOGRAM4_PRESETS[preset]
num_steps = int(preset_cfg["num_steps"])
device = get_local_torch_device()
skip_unconditional = bool(preset_cfg.get("skip_unconditional", False))
lora_scale = float(preset_cfg.get("lora_scale", 0.0))
requires_lora = bool(preset_cfg.get("requires_lora", False))
if skip_unconditional and any(
float(guidance) != 1.0 for guidance in preset_cfg["guidance_schedule"]
):
raise ValueError("skip_unconditional requires guidance_schedule to be 1.0")
if (
lora_scale == 0.0
and getattr(server_args, "lora_path", None) is not None
and getattr(server_args, "lora_merge_mode", "auto") != "dynamic"
):
raise ValueError(
f"Ideogram 4 preset {preset} disables LoRA, but request-local LoRA "
"switching requires --lora-merge-mode dynamic when --lora-path is set."
)
pipeline = self.pipeline() if self.pipeline else None
if requires_lora and (
pipeline is None or not pipeline.is_lora_effective("transformer")
):
raise ValueError(
f"Ideogram 4 preset {preset} requires an active LoRA on transformer. "
"Please start the server with --lora-path and --lora-merge-mode dynamic."
)
batch.runtime_lora_scale = lora_scale
schedule = get_schedule_for_resolution(
(batch.height, batch.width),
known_mean=float(preset_cfg["mu"]),
@@ -405,6 +436,7 @@ class Ideogram4DenoisingStage(DenoisingStage):
"ideogram4_schedule_deltas": schedule_deltas,
"ideogram4_guidance_schedule": guidance_schedule,
"ideogram4_text_z_padding": text_z_padding,
"ideogram4_skip_unconditional": skip_unconditional,
"ideogram4_attn_mask": attn_mask,
"ideogram4_attn_mask_meta": build_varlen_mask_meta(attn_mask),
"ideogram4_neg_position_ids": neg_position_ids,
@@ -432,6 +464,7 @@ class Ideogram4DenoisingStage(DenoisingStage):
schedule_values = ctx.extra["ideogram4_schedule_values"]
schedule_deltas = ctx.extra["ideogram4_schedule_deltas"]
guidance_schedule = ctx.extra["ideogram4_guidance_schedule"]
skip_unconditional = ctx.extra.get("ideogram4_skip_unconditional", False)
i = step.t_int
t_val = schedule_values[i + 1]
@@ -461,7 +494,7 @@ class Ideogram4DenoisingStage(DenoisingStage):
pos_v = pos_out[:, max_text_tokens : max_text_tokens + num_image_tokens]
neg_v = None
if self.unconditional_transformer is not None:
if not skip_unconditional and self.unconditional_transformer is not None:
self._manage_unconditional_transformer_use_site(batch)
with set_forward_context(
current_timestep=i,
@@ -303,7 +303,10 @@ class ProgressiveDenoisingStage(DenoisingStage):
return None if (preset is None or preset == "none") else preset
def _refresh_cache_dit_context(
self, n_remaining: int, scm_preset: str | None
self,
n_remaining: int,
scm_preset: str | None,
ctx: DenoisingContext | None = None,
) -> None:
"""Refresh cache-dit activations and step counter at a stage transition.
@@ -615,7 +618,7 @@ class ProgressiveDenoisingStage(DenoisingStage):
if self._cache_dit_enabled:
n_remaining = n_steps - stage_end
self._refresh_cache_dit_context(
n_remaining, self._effective_scm_preset()
n_remaining, self._effective_scm_preset(), ctx
)
logger.info(
"cache-dit context refreshed at stage transition "
@@ -22,6 +22,7 @@ from diffusers.utils.torch_utils import randn_tensor
from sglang.multimodal_gen.configs.sample.ideogram import IDEOGRAM4_PRESETS
from sglang.multimodal_gen.runtime.cache.cache_dit_integration import (
refresh_context_on_dual_transformer,
refresh_context_on_transformer,
)
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
from sglang.multimodal_gen.runtime.layers.attention import build_varlen_mask_meta
@@ -452,14 +453,35 @@ class Ideogram4ProgressiveDenoisingStage(
# ------------------------------------------------------------------
def _refresh_cache_dit_context(
self, n_remaining: int, scm_preset: str | None
self,
n_remaining: int,
scm_preset: str | None,
ctx: DenoisingContext | None = None,
) -> None:
"""Refresh both conditional and unconditional transformers."""
"""Refresh active Cache-DiT transformer contexts after a stage transition."""
# Recompute the full SCM config here so custom compute/cache bins stay
# active after a progressive-resolution stage transition.
_, scm_policy, steps_computation_mask, steps_computation_mask_2 = (
self._cache_dit_scm_masks(n_remaining, n_remaining)
skip_unconditional = bool(
self.unconditional_transformer is None
or (
ctx is not None and ctx.extra.get("ideogram4_skip_unconditional", False)
)
)
_, scm_policy, steps_computation_mask, steps_computation_mask_2 = (
self._cache_dit_scm_masks(
n_remaining, None if skip_unconditional else n_remaining
)
)
if skip_unconditional:
refresh_context_on_transformer(
self.transformer,
n_remaining,
steps_computation_mask=steps_computation_mask,
steps_computation_policy=scm_policy,
)
return
refresh_context_on_dual_transformer(
self.transformer,
self.unconditional_transformer,
@@ -6,6 +6,9 @@ import unittest
from types import SimpleNamespace
from unittest.mock import patch
import torch
from sglang.multimodal_gen.runtime.cache import cache_dit_integration
from sglang.multimodal_gen.runtime.cache.cache_dit_integration import (
cache_dit_overrides_key,
resolve_cache_dit_request_overrides,
@@ -16,6 +19,34 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages import (
from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import (
DenoisingStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.progressive_resolution.ideogram import (
Ideogram4ProgressiveDenoisingStage,
)
class TestIdeogramActiveCacheRefresh(unittest.TestCase):
def test_request_override_and_active_transformers_survive_transition(self):
for skip, has_unconditional in ((False, True), (True, True), (False, False)):
with self.subTest(skip=skip, has_unconditional=has_unconditional):
stage = object.__new__(Ideogram4ProgressiveDenoisingStage)
stage.transformer = torch.nn.Linear(2, 2)
stage.unconditional_transformer = (
torch.nn.Linear(2, 2) if has_unconditional else None
)
stage._cache_dit_request_overrides = {"scm_preset": "none"}
ctx = SimpleNamespace(extra={"ideogram4_skip_unconditional": skip})
with patch.object(
cache_dit_integration.cache_dit, "refresh_context"
) as refresh:
stage._refresh_cache_dit_context(
4, stage._effective_scm_preset(), ctx
)
targets = [call.args[0] for call in refresh.call_args_list]
expected = [stage.transformer]
if has_unconditional and not skip:
expected.append(stage.unconditional_transformer)
self.assertEqual(targets, expected)
self.assertIsNone(stage._effective_scm_preset())
class TestResolveCacheDitRequestOverrides(unittest.TestCase):
@@ -472,6 +472,18 @@ class TestIdeogram4(unittest.TestCase):
with self.assertRaisesRegex(ValueError, "Unknown Ideogram 4 preset"):
Ideogram4SamplingParams(preset="V4_FAST")
def test_turbotime_and_distilled_presets_coexist(self):
for steps in (2, 4, 8):
preset = f"V4_TURBOTIME_LORA_{steps}"
with self.subTest(preset=preset):
params = Ideogram4SamplingParams(preset=preset)
self.assertEqual(params.num_inference_steps, steps)
self.assertEqual(params.guidance_scale, 1.0)
self.assertTrue(IDEOGRAM4_PRESETS[preset]["skip_unconditional"])
self.assertTrue(IDEOGRAM4_PRESETS[preset]["requires_lora"])
self.assertEqual(Ideogram4FastSamplingParams().preset, "V4_FAST_20")
self.assertEqual(Ideogram4InstantSamplingParams().preset, "V4_INSTANT_8")
def test_ideogram_distilled_sampling_defaults(self):
fast = Ideogram4FastSamplingParams()
instant = Ideogram4InstantSamplingParams()
@@ -1580,6 +1592,20 @@ class TestIdeogram4(unittest.TestCase):
with patch.object(stage, "_run_ideogram_transformer", side_effect=fake_run):
stage._run_denoising_step(ctx, step, batch, args)
for skip_unconditional, unconditional in (
(True, unconditional_transformer),
(False, None),
):
with self.subTest(skip_unconditional=skip_unconditional):
ctx.latents.zero_()
ctx.extra["ideogram4_skip_unconditional"] = skip_unconditional
stage.unconditional_transformer = unconditional
with patch.object(
stage, "_run_ideogram_transformer", side_effect=fake_run
) as run:
stage._run_denoising_step(ctx, step, batch, args)
self.assertEqual(run.call_count, 1)
self.assertIs(run.call_args.args[0], transformer)
finally:
set_global_server_args(prev_args)
@@ -1,10 +1,63 @@
from types import SimpleNamespace
import pytest
import torch
from torch import nn
from sglang.multimodal_gen.runtime.layers.linear import ReplicatedLinear
from sglang.multimodal_gen.runtime.layers.lora.linear import (
LinearWithLoRA,
_compute_lora_delta,
wrap_with_lora_layer,
)
from sglang.multimodal_gen.runtime.layers.quantization.weight_only_fp8 import (
WeightOnlyFP8Linear,
)
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
@pytest.mark.parametrize("kind", ["linear", "replicated", "fp8"])
@pytest.mark.parametrize("device", ["cpu", "cuda"])
def test_request_lora_scale_preserves_delta_offset_and_base(kind, device):
if device == "cuda" and not torch.cuda.is_available():
pytest.skip("CUDA unavailable")
if kind == "replicated":
base = ReplicatedLinear(4, 3, bias=False)
elif kind == "fp8":
base = WeightOnlyFP8Linear(4, 3, bias=False, enable_fused_w8a8=False)
else:
base = nn.Linear(4, 3, bias=False)
base = base.to(device)
with torch.no_grad():
base.weight.copy_(torch.ones(3, 4, device=device))
if kind == "fp8":
base.weight_scale.fill_(1.0)
layer = wrap_with_lora_layer(base, lora_rank=2, lora_alpha=2)
layer.set_lora_weights(
torch.ones(2, 4, device=device),
torch.ones(3, 2, device=device),
strength=0.5,
merge_weights=False,
output_offset=torch.full((3,), 2.0, device=device),
)
x = torch.ones(2, 4, device=device)
base_weight = base.weight.detach().clone()
batch = SimpleNamespace(runtime_lora_scale=0.0)
for scale in (0.0, 1.0, 0.5, 0.0):
batch.runtime_lora_scale = scale
with set_forward_context(0, None, forward_batch=batch):
actual = layer(x)
if kind == "replicated":
actual, output_bias = actual
assert output_bias is None
torch.testing.assert_close(
actual.float(), torch.full((2, 3), 4.0 + 5.0 * scale, device=device)
)
torch.testing.assert_close(base.weight.float(), base_weight.float())
if kind == "fp8":
assert not layer.can_merge_base_weight
with pytest.raises(ValueError, match="only supports dynamic"):
layer.set_lora_weights(torch.ones(2, 4), torch.ones(3, 2))
def test_stacked_lora_delta_preserves_projection_order():