[Fix] diffusion: Keep dynamic LoRA deltas on fused linear-GELU sites (#45943)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Mick Qian
Claude Opus 5.5
parent
0e34962884
commit
02b983603d
@@ -9,13 +9,16 @@ tanh-approximate GELU (max abs diff ~5e-6 vs ``F.gelu(approximate="tanh")`` in
|
||||
fp32), so for half-precision inference the fused path differs from the
|
||||
reference only at bf16/fp16 rounding-order level -- close, but not bit-exact.
|
||||
|
||||
Because it is not bit-exact, the fused path is **mounted only for
|
||||
``quality="high"`` requests** (see ``SamplingParams.quality``): model code marks
|
||||
its GELU up-projection sites with ``FUSED_LINEAR_GELU.mark`` (default: off,
|
||||
reference path, bit-exact), and the denoising stage mounts / unmounts
|
||||
``FUSED_LINEAR_GELU`` at batch boundaries. Mounting is all-or-nothing per transformer: if any marked site
|
||||
fails the static guards (quantized weights, missing bias, non-half dtype, ...)
|
||||
no site on that transformer is fused.
|
||||
Because it is not bit-exact, the fused path is a ``"lossless"``-tier fast path
|
||||
(see ``SamplingParams.quality``): model code marks its GELU up-projection sites
|
||||
with ``FUSED_LINEAR_GELU.mark`` (unmounted: bit-exact reference path), and the
|
||||
denoising stage mounts it at batch boundaries for ``quality="lossless"`` (the
|
||||
default) and ``"high"`` requests and unmounts it for ``"exact"``. Mounting is
|
||||
all-or-nothing per transformer: if any marked site fails the static guards
|
||||
(quantized weights, missing bias, non-half dtype, ...) no site on that
|
||||
transformer is fused. A mounted site still runs its linear's forward while a
|
||||
LoRA adapter on that linear adds output: the epilogue multiplies only the base
|
||||
weight.
|
||||
|
||||
The fused GEMM is exposed as a registered custom op (``register_custom_op``)
|
||||
exactly like the other diffusion kernels (e.g. qknorm_rope), so it stays a
|
||||
@@ -91,7 +94,13 @@ def _static_reject_reason(linear: Any) -> str | None:
|
||||
exactly what the reference forward multiplies. The weight's *device* is
|
||||
deliberately not checked here -- under CPU offload the weights live on CPU
|
||||
between requests -- the runtime guard checks the input device per call.
|
||||
A LoRA wrapper is judged by the layer it wraps, whose weight the epilogue
|
||||
multiplies; whether its adapter adds output is a per-call check.
|
||||
"""
|
||||
from sglang.multimodal_gen.runtime.layers.lora.linear import BaseLayerWithLoRA
|
||||
|
||||
if isinstance(linear, BaseLayerWithLoRA):
|
||||
linear = linear.base_layer
|
||||
if not _HAS_ADDMM_ACTIVATION:
|
||||
return "torch._addmm_activation unavailable"
|
||||
if not _is_unquantized(linear):
|
||||
@@ -134,8 +143,13 @@ def _input_independent_ok(linear: Any) -> bool:
|
||||
|
||||
def can_use_linear_gelu(linear: Any, x: torch.Tensor) -> bool:
|
||||
"""Whether ``gelu(linear(x))`` can use the fused cublasLt epilogue now."""
|
||||
from sglang.multimodal_gen.runtime.layers.lora.linear import BaseLayerWithLoRA
|
||||
|
||||
if not (x.is_cuda and x.dtype in (torch.bfloat16, torch.float16)):
|
||||
return False
|
||||
# the epilogue reads the base weight only; an active adapter needs the forward
|
||||
if isinstance(linear, BaseLayerWithLoRA) and linear.lora_active:
|
||||
return False
|
||||
if getattr(linear, "weight", None) is None or x.dtype != linear.weight.dtype:
|
||||
return False
|
||||
if torch.compiler.is_compiling():
|
||||
|
||||
@@ -645,6 +645,44 @@ if not current_platform.is_hip():
|
||||
T2V_sampling_params,
|
||||
),
|
||||
)
|
||||
# Dynamic LoRA at the default quality: the lossless-tier fusions (the fused
|
||||
# linear-GELU epilogue among them) must keep the adapter's delta. GT-free: the
|
||||
# output must match quality="exact" and differ from the adapter off. Skipped on
|
||||
# AMD: the match/change thresholds are calibrated on NVIDIA only.
|
||||
if not current_platform.is_hip():
|
||||
ONE_GPU_CASES.append(
|
||||
DiffusionTestCase(
|
||||
"wan2_1_t2v_1_3b_dynamic_lora_default_quality_1gpu",
|
||||
DiffusionServerArgs(
|
||||
model_path=DEFAULT_WAN_2_1_T2V_1_3B_MODEL_NAME_FOR_TEST,
|
||||
lora_path="Cseti/Wan-LoRA-Arcane-Jinx-v1",
|
||||
extras=[
|
||||
"--lora-merge-mode",
|
||||
"dynamic",
|
||||
# layerwise offload leaves placeholder weights, which keep
|
||||
# the fused linear-GELU path unmounted
|
||||
"--dit-layerwise-offload",
|
||||
"false",
|
||||
],
|
||||
),
|
||||
DiffusionSamplingParams(
|
||||
prompt="csetiarcane Nfj1nx with blue hair, a woman walking in a cyberpunk city at night",
|
||||
output_size="480x272",
|
||||
num_frames=9,
|
||||
# CRF 0, so the frame comparisons measure the model, not the encoder
|
||||
extras={
|
||||
"num_inference_steps": 8,
|
||||
"seed": 42,
|
||||
"output_compression": 100,
|
||||
},
|
||||
),
|
||||
run_perf_check=False,
|
||||
perf_warmup_requests=1,
|
||||
run_consistency_check=False,
|
||||
reference_quality="exact",
|
||||
run_lora_effect_check=True,
|
||||
),
|
||||
)
|
||||
# Skip all ModelOpt tests on AMD: FP8 requires torch._scaled_mm (HIPBLAS_STATUS_NOT_SUPPORTED
|
||||
# on ROCm), NVFP4 requires flashinfer or sgl_kernel FP4 kernels (CUDA-only).
|
||||
# Run FP8 cases on the regular H100 1-GPU CI shard and keep only B200-only
|
||||
|
||||
@@ -2293,6 +2293,39 @@
|
||||
"runtime_peak_allocated_mb": 9999.0,
|
||||
"estimated_full_test_time_s": 90
|
||||
},
|
||||
"wan2_1_t2v_1_3b_dynamic_lora_default_quality_1gpu": {
|
||||
"stages_ms": {
|
||||
"InputValidationStage": 0.04,
|
||||
"TextEncodingStage": 2.36,
|
||||
"LatentPreparationStage": 0.11,
|
||||
"TimestepPreparationStage": 0.48,
|
||||
"DenoisingStage": 687.26,
|
||||
"DecodingStage": 88.7,
|
||||
"per_frame_generation": null
|
||||
},
|
||||
"denoise_step_ms": {
|
||||
"0": 107.57,
|
||||
"1": 99.77,
|
||||
"2": 78.34,
|
||||
"3": 78.47,
|
||||
"4": 82.14,
|
||||
"5": 79.72,
|
||||
"6": 80.1,
|
||||
"7": 80.0
|
||||
},
|
||||
"expected_e2e_ms": 782.31,
|
||||
"expected_load_ms": 31976.56,
|
||||
"expected_avg_denoise_ms": 85.76,
|
||||
"expected_median_denoise_ms": 80.05,
|
||||
"load_peak_vram_mb": 4148.0,
|
||||
"runtime_peak_vram_mb": 13494.0,
|
||||
"warmup_peak_vram_mb": 13968.0,
|
||||
"load_peak_allocated_mb": 3976.25,
|
||||
"runtime_peak_allocated_mb": 5521.42,
|
||||
"load_peak_host_anon_mb": 10335.67,
|
||||
"runtime_peak_host_anon_mb": 10335.67,
|
||||
"estimated_full_test_time_s": 51.9
|
||||
},
|
||||
"wan2_1_i2v_14b_lora_2gpu": {
|
||||
"expected_load_ms": 66554.82,
|
||||
"stages_ms": {
|
||||
|
||||
@@ -13,6 +13,7 @@ import os
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
@@ -42,6 +43,9 @@ from sglang.multimodal_gen.test.server.test_server_utils import (
|
||||
)
|
||||
from sglang.multimodal_gen.test.server.testcase_configs import (
|
||||
BASELINE_CONFIG,
|
||||
OUTPUT_CHANGE_MIN_MEAN_ABS_DIFF,
|
||||
OUTPUT_MATCH_MAX_MEAN_ABS_DIFF,
|
||||
OUTPUT_MATCH_MIN_PSNR,
|
||||
DiffusionTestCase,
|
||||
PerformanceSummary,
|
||||
ScenarioConfig,
|
||||
@@ -56,6 +60,8 @@ from sglang.multimodal_gen.test.test_utils import (
|
||||
audio_gt_exists,
|
||||
compare_audio_with_gt,
|
||||
compare_with_gt,
|
||||
compute_mean_abs_diff,
|
||||
compute_psnr,
|
||||
encode_audio_gt_wav,
|
||||
extract_audio_pcm_from_video_bytes,
|
||||
extract_key_frames_from_video,
|
||||
@@ -116,6 +122,22 @@ def _print_case_log_separator(case_id: str, state: str) -> None:
|
||||
)
|
||||
|
||||
|
||||
def _key_frame_distances(
|
||||
case_id: str, label: str, frames: list[np.ndarray], reference: list[np.ndarray]
|
||||
) -> tuple[list[float], list[float]]:
|
||||
"""Per-key-frame mean_abs_diff and PSNR of ``frames`` against ``reference``."""
|
||||
pairs = list(zip(frames, reference, strict=True))
|
||||
mean_abs_diffs = [compute_mean_abs_diff(frame, ref) for frame, ref in pairs]
|
||||
psnrs = [compute_psnr(frame, ref) for frame, ref in pairs]
|
||||
print(
|
||||
f"[Output Reference] {case_id}: {label}: "
|
||||
f"mean_abs_diff=[{', '.join(f'{value:.3f}' for value in mean_abs_diffs)}] "
|
||||
f"psnr=[{', '.join(f'{value:.2f}' for value in psnrs)}]",
|
||||
flush=True,
|
||||
)
|
||||
return mean_abs_diffs, psnrs
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def diffusion_server(case: DiffusionTestCase) -> ServerContext:
|
||||
"""Start a diffusion server for a single case and tear it down afterwards."""
|
||||
@@ -1463,6 +1485,85 @@ Pinned revision used by this check: {SGL_TEST_FILES_CI_DATA_REVISION}
|
||||
|
||||
logger.info("[Multi-LoRA] All multi-LoRA tests passed for %s", case.id)
|
||||
|
||||
def _key_frames(self, case: DiffusionTestCase, content: bytes) -> list[np.ndarray]:
|
||||
if case.server_args.modality == "video":
|
||||
return extract_key_frames_from_video(content)
|
||||
return [image_bytes_to_numpy(content)]
|
||||
|
||||
def _generate_key_frames(
|
||||
self, ctx: ServerContext, case: DiffusionTestCase, **extras: Any
|
||||
) -> list[np.ndarray]:
|
||||
"""Key frames of the case's request with ``extras`` overriding its own."""
|
||||
generate_fn = get_generate_fn(
|
||||
model_path=case.server_args.model_path,
|
||||
modality=case.server_args.modality,
|
||||
sampling_params=replace(
|
||||
case.sampling_params,
|
||||
extras={**case.sampling_params.extras, **extras},
|
||||
),
|
||||
)
|
||||
_, content = self._run_generation_with_server_watchdog(
|
||||
ctx, case.id, generate_fn, self._client(ctx)
|
||||
)
|
||||
return self._key_frames(case, content)
|
||||
|
||||
def _test_output_references(
|
||||
self, ctx: ServerContext, case: DiffusionTestCase, content: bytes
|
||||
) -> None:
|
||||
"""Compare the case's output with other requests to the same server.
|
||||
|
||||
Same prompt and seed, so a reference differs from the case's request
|
||||
only in its quality level, or in the LoRA adapter being off.
|
||||
"""
|
||||
compared = [("output", self._key_frames(case, content))]
|
||||
failures = []
|
||||
if case.reference_quality is not None:
|
||||
label = f"quality={case.reference_quality}"
|
||||
reference = self._generate_key_frames(
|
||||
ctx, case, quality=case.reference_quality
|
||||
)
|
||||
mean_abs_diffs, psnrs = _key_frame_distances(
|
||||
case.id, f"output vs {label}", compared[0][1], reference
|
||||
)
|
||||
if (
|
||||
max(mean_abs_diffs) > OUTPUT_MATCH_MAX_MEAN_ABS_DIFF
|
||||
or min(psnrs) < OUTPUT_MATCH_MIN_PSNR
|
||||
):
|
||||
failures.append(
|
||||
f"output does not match {label}: "
|
||||
f"max mean_abs_diff={max(mean_abs_diffs):.3f} "
|
||||
f"(<= {OUTPUT_MATCH_MAX_MEAN_ABS_DIFF}), "
|
||||
f"min psnr={min(psnrs):.2f} (>= {OUTPUT_MATCH_MIN_PSNR})"
|
||||
)
|
||||
compared.append((label, reference))
|
||||
|
||||
if case.run_lora_effect_check:
|
||||
base_url = f"http://localhost:{ctx.port}/v1"
|
||||
# unmerge turns the adapter off in every merge mode; merge restores it
|
||||
resp = requests.post(
|
||||
f"{base_url}/unmerge_lora_weights", timeout=_CONTROL_API_TIMEOUT_SECS
|
||||
)
|
||||
assert resp.status_code == 200, f"unmerge_lora_weights failed: {resp.text}"
|
||||
try:
|
||||
without_lora = self._generate_key_frames(ctx, case)
|
||||
finally:
|
||||
resp = requests.post(
|
||||
f"{base_url}/merge_lora_weights",
|
||||
timeout=_CONTROL_API_TIMEOUT_SECS,
|
||||
)
|
||||
assert resp.status_code == 200, f"merge_lora_weights failed: {resp.text}"
|
||||
for label, frames in compared:
|
||||
mean_abs_diffs, _ = _key_frame_distances(
|
||||
case.id, f"{label} vs LoRA off", frames, without_lora
|
||||
)
|
||||
if min(mean_abs_diffs) < OUTPUT_CHANGE_MIN_MEAN_ABS_DIFF:
|
||||
failures.append(
|
||||
f"{label} barely changes with the LoRA off: "
|
||||
f"min mean_abs_diff={min(mean_abs_diffs):.3f} "
|
||||
f"(>= {OUTPUT_CHANGE_MIN_MEAN_ABS_DIFF})"
|
||||
)
|
||||
assert not failures, "\n".join(failures)
|
||||
|
||||
def _test_v1_models_endpoint(
|
||||
self, ctx: ServerContext, case: DiffusionTestCase
|
||||
) -> None:
|
||||
@@ -1774,6 +1875,13 @@ Pinned revision used by this check: {SGL_TEST_FILES_CI_DATA_REVISION}
|
||||
lambda: self._validate_audio_consistency(case, content),
|
||||
)
|
||||
|
||||
# before the LoRA checks below, which change the server's adapter state
|
||||
if case.reference_quality is not None or case.run_lora_effect_check:
|
||||
run_case_check(
|
||||
"output references",
|
||||
lambda: self._test_output_references(diffusion_server, case, content),
|
||||
)
|
||||
|
||||
if case.run_lora_basic_api_check:
|
||||
run_case_check(
|
||||
"LoRA basic API",
|
||||
|
||||
@@ -351,6 +351,11 @@ class DiffusionTestCase:
|
||||
run_multi_lora_api_check: bool = False
|
||||
# Cached MiniMax goldens require its implicit-quality cache mode.
|
||||
pin_consistency_quality: bool = True
|
||||
# GT-free checks against other requests to the same server, same seed: the
|
||||
# output must match the request at this quality level ...
|
||||
reference_quality: str | None = None
|
||||
# ... and must change once the server's LoRA adapter is deactivated
|
||||
run_lora_effect_check: bool = False
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.perf_repeat_requests < 1:
|
||||
@@ -441,6 +446,17 @@ class DiffusionTestCase:
|
||||
f"{self.id}: run_multi_lora_api_check requires lora_path and second_lora_path"
|
||||
)
|
||||
|
||||
if self.run_lora_effect_check and not (has_startup_lora or has_dynamic_lora):
|
||||
raise ValueError(
|
||||
f"{self.id}: run_lora_effect_check requires lora_path or dynamic_lora_path"
|
||||
)
|
||||
|
||||
|
||||
# per-key-frame bounds in 0-255 levels: lossless-tier drift is ~1 level, a
|
||||
# full-strength LoRA moves tens; the 4x gap keeps a dropped delta out of "match"
|
||||
OUTPUT_MATCH_MAX_MEAN_ABS_DIFF = 3.0
|
||||
OUTPUT_MATCH_MIN_PSNR = 30.0
|
||||
OUTPUT_CHANGE_MIN_MEAN_ABS_DIFF = 12.0
|
||||
|
||||
_REALTIME_MODEL_COMMON_EXTRAS = {
|
||||
"seed": 42,
|
||||
|
||||
@@ -0,0 +1,208 @@
|
||||
"""Mounting a quality fusion must not drop an active LoRA adapter's output.
|
||||
|
||||
A fused path that multiplies a linear's weight itself sees only the base weight
|
||||
of a LoRA wrapper. Every fusion in ``_QUALITY_FUSION_HANDLERS`` is either run
|
||||
below on a site whose linears carry a dynamic adapter, or listed with the
|
||||
reason its sites cannot drop one, so a new fusion has to pick a side.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from sglang.kernels.ops.diffusion import (
|
||||
FLUX2_NVFP4_SWIGLU_QUANT,
|
||||
FUSED_GATE_RMSNORM,
|
||||
FUSED_LINEAR_GELU,
|
||||
FUSED_LN_MODULATE,
|
||||
HELIOS_GATED_RESIDUAL,
|
||||
HUNYUAN_QKNORM,
|
||||
LINGBOT_VIDEO_GATED_RESIDUAL,
|
||||
LINGBOT_VIDEO_RMSNORM,
|
||||
LTX2_QKNORM_SPLIT_ROPE,
|
||||
LTX2_RMS_NORM_MODULATE,
|
||||
MINIMAX_H3_NORM_MODULATE,
|
||||
NVFP4_BIAS_GELU,
|
||||
QWEN_IMAGE_ADDED_QKV,
|
||||
SANA_VIDEO_LINEAR_ATTENTION,
|
||||
can_use_linear_gelu,
|
||||
)
|
||||
from sglang.kernels.ops.diffusion.sites.fused_linear_gelu_site import (
|
||||
can_use_linear_gelu_static,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
maybe_init_distributed_environment_and_model_parallel,
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
MergedColumnParallelLinear,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.lora.linear import (
|
||||
BaseLayerWithLoRA,
|
||||
wrap_with_lora_layer,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.qwen_image import (
|
||||
QwenImageCrossAttention,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.wanvideo import _WanGELUMLP
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import (
|
||||
_QUALITY_FUSION_HANDLERS,
|
||||
)
|
||||
from sglang.multimodal_gen.test.single_test_file.component_accuracy.utils import (
|
||||
ensure_distributed_env_defaults,
|
||||
)
|
||||
|
||||
DIM = 64
|
||||
LORA_RANK = 4
|
||||
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
|
||||
requires_cuda = pytest.mark.skipif(
|
||||
not torch.cuda.is_available(), reason="the fused path runs on CUDA only"
|
||||
)
|
||||
|
||||
# fusion -> why mounting it cannot drop an adapter's output
|
||||
_ADAPTER_SAFE_FUSIONS = {
|
||||
FLUX2_NVFP4_SWIGLU_QUANT: "fuses only a linear_in that NVFP4 loading marked "
|
||||
"ready; a LoRA wrapper carries no such mark",
|
||||
NVFP4_BIAS_GELU: "rejects an fc_in without the ModelOpt FP4 method, "
|
||||
"which a LoRA wrapper does not expose",
|
||||
FUSED_LN_MODULATE: "norm and modulation, no linear",
|
||||
LTX2_QKNORM_SPLIT_ROPE: "runs on the projections' outputs",
|
||||
LTX2_RMS_NORM_MODULATE: "norm and modulation, no linear",
|
||||
FUSED_GATE_RMSNORM: "norm, gate and residual, no linear",
|
||||
HUNYUAN_QKNORM: "runs on the projections' outputs",
|
||||
LINGBOT_VIDEO_RMSNORM: "norm, no linear",
|
||||
LINGBOT_VIDEO_GATED_RESIDUAL: "residual update, no linear",
|
||||
HELIOS_GATED_RESIDUAL: "residual update, no linear",
|
||||
MINIMAX_H3_NORM_MODULATE: "norm and modulation, no linear",
|
||||
SANA_VIDEO_LINEAR_ATTENTION: "runs on the projections' outputs",
|
||||
}
|
||||
|
||||
|
||||
def _ensure_single_process_parallel_runtime() -> None:
|
||||
if model_parallel_is_initialized():
|
||||
return
|
||||
ensure_distributed_env_defaults()
|
||||
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
|
||||
|
||||
|
||||
def _wan_gelu_mlp():
|
||||
site = _WanGELUMLP(DIM, 4 * DIM, prefix="", quant_config=None)
|
||||
return site, ("fc_in",), lambda site, x: site(x)
|
||||
|
||||
|
||||
def _qwen_image_added_qkv():
|
||||
# only the state _get_added_qkv_projections reads
|
||||
site = object.__new__(QwenImageCrossAttention)
|
||||
nn.Module.__init__(site)
|
||||
site.use_fused_added_qkv = True
|
||||
site._unquantized_added_qkv_is_packed = True
|
||||
site.to_added_qkv = MergedColumnParallelLinear(DIM, [DIM] * 3, bias=True)
|
||||
QWEN_IMAGE_ADDED_QKV.mark(site)
|
||||
return (
|
||||
site,
|
||||
("to_added_qkv",),
|
||||
lambda site, x: torch.cat(site._get_added_qkv_projections(x), dim=-1),
|
||||
)
|
||||
|
||||
|
||||
# a site builder returns (site, names of its LoRA-wrappable linears, call)
|
||||
_LORA_SITES = [
|
||||
pytest.param(
|
||||
FUSED_LINEAR_GELU, _wan_gelu_mlp, id="linear_gelu", marks=requires_cuda
|
||||
),
|
||||
pytest.param(QWEN_IMAGE_ADDED_QKV, _qwen_image_added_qkv, id="qwen_added_qkv"),
|
||||
]
|
||||
|
||||
|
||||
def _attach_dynamic_lora(site: nn.Module, name: str) -> BaseLayerWithLoRA:
|
||||
linear = site.get_submodule(name)
|
||||
out_features, in_features = linear.weight.shape
|
||||
wrapper = wrap_with_lora_layer(linear, lora_rank=LORA_RANK, lora_alpha=LORA_RANK)
|
||||
wrapper.set_lora_weights(
|
||||
torch.randn(LORA_RANK, in_features, device=DEVICE, dtype=torch.bfloat16),
|
||||
torch.randn(out_features, LORA_RANK, device=DEVICE, dtype=torch.bfloat16) * 0.1,
|
||||
merge_weights=False,
|
||||
)
|
||||
site.register_module(name, wrapper)
|
||||
return wrapper
|
||||
|
||||
|
||||
def _site_with_dynamic_lora(make_site):
|
||||
_ensure_single_process_parallel_runtime()
|
||||
torch.manual_seed(0)
|
||||
site, linear_names, run = make_site()
|
||||
site = site.to(device=DEVICE, dtype=torch.bfloat16)
|
||||
for parameter in site.parameters():
|
||||
parameter.normal_(mean=0.0, std=0.02)
|
||||
wrappers = [_attach_dynamic_lora(site, name) for name in linear_names]
|
||||
x = torch.randn(2, 129, DIM, device=DEVICE, dtype=torch.bfloat16)
|
||||
return site, wrappers, run, x
|
||||
|
||||
|
||||
def test_every_quality_fusion_has_a_lora_site_or_a_reason():
|
||||
with_site = {param.values[0] for param in _LORA_SITES}
|
||||
assert with_site.isdisjoint(_ADAPTER_SAFE_FUSIONS)
|
||||
unclassified = {fusion for _, fusion in _QUALITY_FUSION_HANDLERS} - (
|
||||
with_site | set(_ADAPTER_SAFE_FUSIONS)
|
||||
)
|
||||
assert not unclassified, (
|
||||
"add a LoRA site for these fusions, or say why they cannot drop an "
|
||||
f"adapter's output: {sorted(fusion.name for fusion in unclassified)}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fusion, make_site", _LORA_SITES)
|
||||
@torch.no_grad()
|
||||
def test_mounted_fusion_keeps_the_dynamic_lora_delta(fusion, make_site):
|
||||
site, wrappers, run, x = _site_with_dynamic_lora(make_site)
|
||||
reference = run(site, x)
|
||||
for wrapper in wrappers:
|
||||
wrapper.disable_lora = True
|
||||
base_only = run(site, x)
|
||||
for wrapper in wrappers:
|
||||
wrapper.disable_lora = False
|
||||
assert not torch.allclose(reference, base_only, rtol=1e-2, atol=1e-2)
|
||||
|
||||
fusion.mount(site)
|
||||
try:
|
||||
assert torch.equal(run(site, x), reference)
|
||||
finally:
|
||||
fusion.unmount(site)
|
||||
|
||||
|
||||
@requires_cuda
|
||||
@torch.no_grad()
|
||||
def test_linear_gelu_guard_follows_the_adapter_state():
|
||||
site, (wrapper,), _, x = _site_with_dynamic_lora(_wan_gelu_mlp)
|
||||
fusable = torch.cuda.get_device_capability() < (12, 0)
|
||||
assert FUSED_LINEAR_GELU.mount(site)
|
||||
try:
|
||||
assert not can_use_linear_gelu(wrapper, x)
|
||||
wrapper.disable_lora = True
|
||||
assert can_use_linear_gelu(wrapper, x) == fusable
|
||||
wrapper.disable_lora = False
|
||||
# merged: the base weight the epilogue reads already holds the update
|
||||
wrapper.merge_lora_weights()
|
||||
assert can_use_linear_gelu(wrapper, x) == fusable
|
||||
finally:
|
||||
FUSED_LINEAR_GELU.unmount(site)
|
||||
|
||||
|
||||
def test_linear_gelu_static_guard_reads_the_wrapped_layer():
|
||||
_ensure_single_process_parallel_runtime()
|
||||
linear = ColumnParallelLinear(
|
||||
DIM, 4 * DIM, bias=True, gather_output=False, params_dtype=torch.bfloat16
|
||||
)
|
||||
wrapper = wrap_with_lora_layer(linear, lora_rank=LORA_RANK, lora_alpha=LORA_RANK)
|
||||
assert can_use_linear_gelu_static(wrapper)
|
||||
|
||||
# the per-shard epilogue would skip the cross-rank gather
|
||||
linear.gather_output, linear.tp_size = True, 2
|
||||
assert not can_use_linear_gelu_static(wrapper)
|
||||
linear.gather_output, linear.tp_size = False, 1
|
||||
|
||||
# the epilogue adds the bias the layer hands back to its caller
|
||||
linear.skip_bias_add = True
|
||||
assert not can_use_linear_gelu_static(wrapper)
|
||||
Reference in New Issue
Block a user