[Fix] diffusion: Keep dynamic LoRA deltas on fused linear-GELU sites (#45943)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-10-23 21:19:53 +08:00
committed by GitHub
co-authored by Mick Qian Claude Opus 5.5
parent 0e34962884
commit 02b983603d
6 changed files with 424 additions and 7 deletions
@@ -9,13 +9,16 @@ tanh-approximate GELU (max abs diff ~5e-6 vs ``F.gelu(approximate="tanh")`` in
fp32), so for half-precision inference the fused path differs from the
reference only at bf16/fp16 rounding-order level -- close, but not bit-exact.
Because it is not bit-exact, the fused path is **mounted only for
``quality="high"`` requests** (see ``SamplingParams.quality``): model code marks
its GELU up-projection sites with ``FUSED_LINEAR_GELU.mark`` (default: off,
reference path, bit-exact), and the denoising stage mounts / unmounts
``FUSED_LINEAR_GELU`` at batch boundaries. Mounting is all-or-nothing per transformer: if any marked site
fails the static guards (quantized weights, missing bias, non-half dtype, ...)
no site on that transformer is fused.
Because it is not bit-exact, the fused path is a ``"lossless"``-tier fast path
(see ``SamplingParams.quality``): model code marks its GELU up-projection sites
with ``FUSED_LINEAR_GELU.mark`` (unmounted: bit-exact reference path), and the
denoising stage mounts it at batch boundaries for ``quality="lossless"`` (the
default) and ``"high"`` requests and unmounts it for ``"exact"``. Mounting is
all-or-nothing per transformer: if any marked site fails the static guards
(quantized weights, missing bias, non-half dtype, ...) no site on that
transformer is fused. A mounted site still runs its linear's forward while a
LoRA adapter on that linear adds output: the epilogue multiplies only the base
weight.
The fused GEMM is exposed as a registered custom op (``register_custom_op``)
exactly like the other diffusion kernels (e.g. qknorm_rope), so it stays a
@@ -91,7 +94,13 @@ def _static_reject_reason(linear: Any) -> str | None:
exactly what the reference forward multiplies. The weight's *device* is
deliberately not checked here -- under CPU offload the weights live on CPU
between requests -- the runtime guard checks the input device per call.
A LoRA wrapper is judged by the layer it wraps, whose weight the epilogue
multiplies; whether its adapter adds output is a per-call check.
"""
from sglang.multimodal_gen.runtime.layers.lora.linear import BaseLayerWithLoRA
if isinstance(linear, BaseLayerWithLoRA):
linear = linear.base_layer
if not _HAS_ADDMM_ACTIVATION:
return "torch._addmm_activation unavailable"
if not _is_unquantized(linear):
@@ -134,8 +143,13 @@ def _input_independent_ok(linear: Any) -> bool:
def can_use_linear_gelu(linear: Any, x: torch.Tensor) -> bool:
"""Whether ``gelu(linear(x))`` can use the fused cublasLt epilogue now."""
from sglang.multimodal_gen.runtime.layers.lora.linear import BaseLayerWithLoRA
if not (x.is_cuda and x.dtype in (torch.bfloat16, torch.float16)):
return False
# the epilogue reads the base weight only; an active adapter needs the forward
if isinstance(linear, BaseLayerWithLoRA) and linear.lora_active:
return False
if getattr(linear, "weight", None) is None or x.dtype != linear.weight.dtype:
return False
if torch.compiler.is_compiling():
@@ -645,6 +645,44 @@ if not current_platform.is_hip():
T2V_sampling_params,
),
)
# Dynamic LoRA at the default quality: the lossless-tier fusions (the fused
# linear-GELU epilogue among them) must keep the adapter's delta. GT-free: the
# output must match quality="exact" and differ from the adapter off. Skipped on
# AMD: the match/change thresholds are calibrated on NVIDIA only.
if not current_platform.is_hip():
ONE_GPU_CASES.append(
DiffusionTestCase(
"wan2_1_t2v_1_3b_dynamic_lora_default_quality_1gpu",
DiffusionServerArgs(
model_path=DEFAULT_WAN_2_1_T2V_1_3B_MODEL_NAME_FOR_TEST,
lora_path="Cseti/Wan-LoRA-Arcane-Jinx-v1",
extras=[
"--lora-merge-mode",
"dynamic",
# layerwise offload leaves placeholder weights, which keep
# the fused linear-GELU path unmounted
"--dit-layerwise-offload",
"false",
],
),
DiffusionSamplingParams(
prompt="csetiarcane Nfj1nx with blue hair, a woman walking in a cyberpunk city at night",
output_size="480x272",
num_frames=9,
# CRF 0, so the frame comparisons measure the model, not the encoder
extras={
"num_inference_steps": 8,
"seed": 42,
"output_compression": 100,
},
),
run_perf_check=False,
perf_warmup_requests=1,
run_consistency_check=False,
reference_quality="exact",
run_lora_effect_check=True,
),
)
# Skip all ModelOpt tests on AMD: FP8 requires torch._scaled_mm (HIPBLAS_STATUS_NOT_SUPPORTED
# on ROCm), NVFP4 requires flashinfer or sgl_kernel FP4 kernels (CUDA-only).
# Run FP8 cases on the regular H100 1-GPU CI shard and keep only B200-only
@@ -2293,6 +2293,39 @@
"runtime_peak_allocated_mb": 9999.0,
"estimated_full_test_time_s": 90
},
"wan2_1_t2v_1_3b_dynamic_lora_default_quality_1gpu": {
"stages_ms": {
"InputValidationStage": 0.04,
"TextEncodingStage": 2.36,
"LatentPreparationStage": 0.11,
"TimestepPreparationStage": 0.48,
"DenoisingStage": 687.26,
"DecodingStage": 88.7,
"per_frame_generation": null
},
"denoise_step_ms": {
"0": 107.57,
"1": 99.77,
"2": 78.34,
"3": 78.47,
"4": 82.14,
"5": 79.72,
"6": 80.1,
"7": 80.0
},
"expected_e2e_ms": 782.31,
"expected_load_ms": 31976.56,
"expected_avg_denoise_ms": 85.76,
"expected_median_denoise_ms": 80.05,
"load_peak_vram_mb": 4148.0,
"runtime_peak_vram_mb": 13494.0,
"warmup_peak_vram_mb": 13968.0,
"load_peak_allocated_mb": 3976.25,
"runtime_peak_allocated_mb": 5521.42,
"load_peak_host_anon_mb": 10335.67,
"runtime_peak_host_anon_mb": 10335.67,
"estimated_full_test_time_s": 51.9
},
"wan2_1_i2v_14b_lora_2gpu": {
"expected_load_ms": 66554.82,
"stages_ms": {
@@ -13,6 +13,7 @@ import os
import queue
import threading
import time
from dataclasses import replace
from pathlib import Path
from typing import Any, Callable
@@ -42,6 +43,9 @@ from sglang.multimodal_gen.test.server.test_server_utils import (
)
from sglang.multimodal_gen.test.server.testcase_configs import (
BASELINE_CONFIG,
OUTPUT_CHANGE_MIN_MEAN_ABS_DIFF,
OUTPUT_MATCH_MAX_MEAN_ABS_DIFF,
OUTPUT_MATCH_MIN_PSNR,
DiffusionTestCase,
PerformanceSummary,
ScenarioConfig,
@@ -56,6 +60,8 @@ from sglang.multimodal_gen.test.test_utils import (
audio_gt_exists,
compare_audio_with_gt,
compare_with_gt,
compute_mean_abs_diff,
compute_psnr,
encode_audio_gt_wav,
extract_audio_pcm_from_video_bytes,
extract_key_frames_from_video,
@@ -116,6 +122,22 @@ def _print_case_log_separator(case_id: str, state: str) -> None:
)
def _key_frame_distances(
case_id: str, label: str, frames: list[np.ndarray], reference: list[np.ndarray]
) -> tuple[list[float], list[float]]:
"""Per-key-frame mean_abs_diff and PSNR of ``frames`` against ``reference``."""
pairs = list(zip(frames, reference, strict=True))
mean_abs_diffs = [compute_mean_abs_diff(frame, ref) for frame, ref in pairs]
psnrs = [compute_psnr(frame, ref) for frame, ref in pairs]
print(
f"[Output Reference] {case_id}: {label}: "
f"mean_abs_diff=[{', '.join(f'{value:.3f}' for value in mean_abs_diffs)}] "
f"psnr=[{', '.join(f'{value:.2f}' for value in psnrs)}]",
flush=True,
)
return mean_abs_diffs, psnrs
@pytest.fixture
def diffusion_server(case: DiffusionTestCase) -> ServerContext:
"""Start a diffusion server for a single case and tear it down afterwards."""
@@ -1463,6 +1485,85 @@ Pinned revision used by this check: {SGL_TEST_FILES_CI_DATA_REVISION}
logger.info("[Multi-LoRA] All multi-LoRA tests passed for %s", case.id)
def _key_frames(self, case: DiffusionTestCase, content: bytes) -> list[np.ndarray]:
if case.server_args.modality == "video":
return extract_key_frames_from_video(content)
return [image_bytes_to_numpy(content)]
def _generate_key_frames(
self, ctx: ServerContext, case: DiffusionTestCase, **extras: Any
) -> list[np.ndarray]:
"""Key frames of the case's request with ``extras`` overriding its own."""
generate_fn = get_generate_fn(
model_path=case.server_args.model_path,
modality=case.server_args.modality,
sampling_params=replace(
case.sampling_params,
extras={**case.sampling_params.extras, **extras},
),
)
_, content = self._run_generation_with_server_watchdog(
ctx, case.id, generate_fn, self._client(ctx)
)
return self._key_frames(case, content)
def _test_output_references(
self, ctx: ServerContext, case: DiffusionTestCase, content: bytes
) -> None:
"""Compare the case's output with other requests to the same server.
Same prompt and seed, so a reference differs from the case's request
only in its quality level, or in the LoRA adapter being off.
"""
compared = [("output", self._key_frames(case, content))]
failures = []
if case.reference_quality is not None:
label = f"quality={case.reference_quality}"
reference = self._generate_key_frames(
ctx, case, quality=case.reference_quality
)
mean_abs_diffs, psnrs = _key_frame_distances(
case.id, f"output vs {label}", compared[0][1], reference
)
if (
max(mean_abs_diffs) > OUTPUT_MATCH_MAX_MEAN_ABS_DIFF
or min(psnrs) < OUTPUT_MATCH_MIN_PSNR
):
failures.append(
f"output does not match {label}: "
f"max mean_abs_diff={max(mean_abs_diffs):.3f} "
f"(<= {OUTPUT_MATCH_MAX_MEAN_ABS_DIFF}), "
f"min psnr={min(psnrs):.2f} (>= {OUTPUT_MATCH_MIN_PSNR})"
)
compared.append((label, reference))
if case.run_lora_effect_check:
base_url = f"http://localhost:{ctx.port}/v1"
# unmerge turns the adapter off in every merge mode; merge restores it
resp = requests.post(
f"{base_url}/unmerge_lora_weights", timeout=_CONTROL_API_TIMEOUT_SECS
)
assert resp.status_code == 200, f"unmerge_lora_weights failed: {resp.text}"
try:
without_lora = self._generate_key_frames(ctx, case)
finally:
resp = requests.post(
f"{base_url}/merge_lora_weights",
timeout=_CONTROL_API_TIMEOUT_SECS,
)
assert resp.status_code == 200, f"merge_lora_weights failed: {resp.text}"
for label, frames in compared:
mean_abs_diffs, _ = _key_frame_distances(
case.id, f"{label} vs LoRA off", frames, without_lora
)
if min(mean_abs_diffs) < OUTPUT_CHANGE_MIN_MEAN_ABS_DIFF:
failures.append(
f"{label} barely changes with the LoRA off: "
f"min mean_abs_diff={min(mean_abs_diffs):.3f} "
f"(>= {OUTPUT_CHANGE_MIN_MEAN_ABS_DIFF})"
)
assert not failures, "\n".join(failures)
def _test_v1_models_endpoint(
self, ctx: ServerContext, case: DiffusionTestCase
) -> None:
@@ -1774,6 +1875,13 @@ Pinned revision used by this check: {SGL_TEST_FILES_CI_DATA_REVISION}
lambda: self._validate_audio_consistency(case, content),
)
# before the LoRA checks below, which change the server's adapter state
if case.reference_quality is not None or case.run_lora_effect_check:
run_case_check(
"output references",
lambda: self._test_output_references(diffusion_server, case, content),
)
if case.run_lora_basic_api_check:
run_case_check(
"LoRA basic API",
@@ -351,6 +351,11 @@ class DiffusionTestCase:
run_multi_lora_api_check: bool = False
# Cached MiniMax goldens require its implicit-quality cache mode.
pin_consistency_quality: bool = True
# GT-free checks against other requests to the same server, same seed: the
# output must match the request at this quality level ...
reference_quality: str | None = None
# ... and must change once the server's LoRA adapter is deactivated
run_lora_effect_check: bool = False
def __post_init__(self) -> None:
if self.perf_repeat_requests < 1:
@@ -441,6 +446,17 @@ class DiffusionTestCase:
f"{self.id}: run_multi_lora_api_check requires lora_path and second_lora_path"
)
if self.run_lora_effect_check and not (has_startup_lora or has_dynamic_lora):
raise ValueError(
f"{self.id}: run_lora_effect_check requires lora_path or dynamic_lora_path"
)
# per-key-frame bounds in 0-255 levels: lossless-tier drift is ~1 level, a
# full-strength LoRA moves tens; the 4x gap keeps a dropped delta out of "match"
OUTPUT_MATCH_MAX_MEAN_ABS_DIFF = 3.0
OUTPUT_MATCH_MIN_PSNR = 30.0
OUTPUT_CHANGE_MIN_MEAN_ABS_DIFF = 12.0
_REALTIME_MODEL_COMMON_EXTRAS = {
"seed": 42,
@@ -0,0 +1,208 @@
"""Mounting a quality fusion must not drop an active LoRA adapter's output.
A fused path that multiplies a linear's weight itself sees only the base weight
of a LoRA wrapper. Every fusion in ``_QUALITY_FUSION_HANDLERS`` is either run
below on a site whose linears carry a dynamic adapter, or listed with the
reason its sites cannot drop one, so a new fusion has to pick a side.
"""
import pytest
import torch
import torch.nn as nn
from sglang.kernels.ops.diffusion import (
FLUX2_NVFP4_SWIGLU_QUANT,
FUSED_GATE_RMSNORM,
FUSED_LINEAR_GELU,
FUSED_LN_MODULATE,
HELIOS_GATED_RESIDUAL,
HUNYUAN_QKNORM,
LINGBOT_VIDEO_GATED_RESIDUAL,
LINGBOT_VIDEO_RMSNORM,
LTX2_QKNORM_SPLIT_ROPE,
LTX2_RMS_NORM_MODULATE,
MINIMAX_H3_NORM_MODULATE,
NVFP4_BIAS_GELU,
QWEN_IMAGE_ADDED_QKV,
SANA_VIDEO_LINEAR_ATTENTION,
can_use_linear_gelu,
)
from sglang.kernels.ops.diffusion.sites.fused_linear_gelu_site import (
can_use_linear_gelu_static,
)
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
maybe_init_distributed_environment_and_model_parallel,
model_parallel_is_initialized,
)
from sglang.multimodal_gen.runtime.layers.linear import (
ColumnParallelLinear,
MergedColumnParallelLinear,
)
from sglang.multimodal_gen.runtime.layers.lora.linear import (
BaseLayerWithLoRA,
wrap_with_lora_layer,
)
from sglang.multimodal_gen.runtime.models.dits.qwen_image import (
QwenImageCrossAttention,
)
from sglang.multimodal_gen.runtime.models.dits.wanvideo import _WanGELUMLP
from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import (
_QUALITY_FUSION_HANDLERS,
)
from sglang.multimodal_gen.test.single_test_file.component_accuracy.utils import (
ensure_distributed_env_defaults,
)
DIM = 64
LORA_RANK = 4
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
requires_cuda = pytest.mark.skipif(
not torch.cuda.is_available(), reason="the fused path runs on CUDA only"
)
# fusion -> why mounting it cannot drop an adapter's output
_ADAPTER_SAFE_FUSIONS = {
FLUX2_NVFP4_SWIGLU_QUANT: "fuses only a linear_in that NVFP4 loading marked "
"ready; a LoRA wrapper carries no such mark",
NVFP4_BIAS_GELU: "rejects an fc_in without the ModelOpt FP4 method, "
"which a LoRA wrapper does not expose",
FUSED_LN_MODULATE: "norm and modulation, no linear",
LTX2_QKNORM_SPLIT_ROPE: "runs on the projections' outputs",
LTX2_RMS_NORM_MODULATE: "norm and modulation, no linear",
FUSED_GATE_RMSNORM: "norm, gate and residual, no linear",
HUNYUAN_QKNORM: "runs on the projections' outputs",
LINGBOT_VIDEO_RMSNORM: "norm, no linear",
LINGBOT_VIDEO_GATED_RESIDUAL: "residual update, no linear",
HELIOS_GATED_RESIDUAL: "residual update, no linear",
MINIMAX_H3_NORM_MODULATE: "norm and modulation, no linear",
SANA_VIDEO_LINEAR_ATTENTION: "runs on the projections' outputs",
}
def _ensure_single_process_parallel_runtime() -> None:
if model_parallel_is_initialized():
return
ensure_distributed_env_defaults()
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
def _wan_gelu_mlp():
site = _WanGELUMLP(DIM, 4 * DIM, prefix="", quant_config=None)
return site, ("fc_in",), lambda site, x: site(x)
def _qwen_image_added_qkv():
# only the state _get_added_qkv_projections reads
site = object.__new__(QwenImageCrossAttention)
nn.Module.__init__(site)
site.use_fused_added_qkv = True
site._unquantized_added_qkv_is_packed = True
site.to_added_qkv = MergedColumnParallelLinear(DIM, [DIM] * 3, bias=True)
QWEN_IMAGE_ADDED_QKV.mark(site)
return (
site,
("to_added_qkv",),
lambda site, x: torch.cat(site._get_added_qkv_projections(x), dim=-1),
)
# a site builder returns (site, names of its LoRA-wrappable linears, call)
_LORA_SITES = [
pytest.param(
FUSED_LINEAR_GELU, _wan_gelu_mlp, id="linear_gelu", marks=requires_cuda
),
pytest.param(QWEN_IMAGE_ADDED_QKV, _qwen_image_added_qkv, id="qwen_added_qkv"),
]
def _attach_dynamic_lora(site: nn.Module, name: str) -> BaseLayerWithLoRA:
linear = site.get_submodule(name)
out_features, in_features = linear.weight.shape
wrapper = wrap_with_lora_layer(linear, lora_rank=LORA_RANK, lora_alpha=LORA_RANK)
wrapper.set_lora_weights(
torch.randn(LORA_RANK, in_features, device=DEVICE, dtype=torch.bfloat16),
torch.randn(out_features, LORA_RANK, device=DEVICE, dtype=torch.bfloat16) * 0.1,
merge_weights=False,
)
site.register_module(name, wrapper)
return wrapper
def _site_with_dynamic_lora(make_site):
_ensure_single_process_parallel_runtime()
torch.manual_seed(0)
site, linear_names, run = make_site()
site = site.to(device=DEVICE, dtype=torch.bfloat16)
for parameter in site.parameters():
parameter.normal_(mean=0.0, std=0.02)
wrappers = [_attach_dynamic_lora(site, name) for name in linear_names]
x = torch.randn(2, 129, DIM, device=DEVICE, dtype=torch.bfloat16)
return site, wrappers, run, x
def test_every_quality_fusion_has_a_lora_site_or_a_reason():
with_site = {param.values[0] for param in _LORA_SITES}
assert with_site.isdisjoint(_ADAPTER_SAFE_FUSIONS)
unclassified = {fusion for _, fusion in _QUALITY_FUSION_HANDLERS} - (
with_site | set(_ADAPTER_SAFE_FUSIONS)
)
assert not unclassified, (
"add a LoRA site for these fusions, or say why they cannot drop an "
f"adapter's output: {sorted(fusion.name for fusion in unclassified)}"
)
@pytest.mark.parametrize("fusion, make_site", _LORA_SITES)
@torch.no_grad()
def test_mounted_fusion_keeps_the_dynamic_lora_delta(fusion, make_site):
site, wrappers, run, x = _site_with_dynamic_lora(make_site)
reference = run(site, x)
for wrapper in wrappers:
wrapper.disable_lora = True
base_only = run(site, x)
for wrapper in wrappers:
wrapper.disable_lora = False
assert not torch.allclose(reference, base_only, rtol=1e-2, atol=1e-2)
fusion.mount(site)
try:
assert torch.equal(run(site, x), reference)
finally:
fusion.unmount(site)
@requires_cuda
@torch.no_grad()
def test_linear_gelu_guard_follows_the_adapter_state():
site, (wrapper,), _, x = _site_with_dynamic_lora(_wan_gelu_mlp)
fusable = torch.cuda.get_device_capability() < (12, 0)
assert FUSED_LINEAR_GELU.mount(site)
try:
assert not can_use_linear_gelu(wrapper, x)
wrapper.disable_lora = True
assert can_use_linear_gelu(wrapper, x) == fusable
wrapper.disable_lora = False
# merged: the base weight the epilogue reads already holds the update
wrapper.merge_lora_weights()
assert can_use_linear_gelu(wrapper, x) == fusable
finally:
FUSED_LINEAR_GELU.unmount(site)
def test_linear_gelu_static_guard_reads_the_wrapped_layer():
_ensure_single_process_parallel_runtime()
linear = ColumnParallelLinear(
DIM, 4 * DIM, bias=True, gather_output=False, params_dtype=torch.bfloat16
)
wrapper = wrap_with_lora_layer(linear, lora_rank=LORA_RANK, lora_alpha=LORA_RANK)
assert can_use_linear_gelu_static(wrapper)
# the per-shard epilogue would skip the cross-rank gather
linear.gather_output, linear.tp_size = True, 2
assert not can_use_linear_gelu_static(wrapper)
linear.gather_output, linear.tp_size = False, 1
# the epilogue adds the bias the layer hands back to its caller
linear.skip_bias_add = True
assert not can_use_linear_gelu_static(wrapper)