diff --git a/python/sglang/kernels/ops/diffusion/sites/fused_linear_gelu_site.py b/python/sglang/kernels/ops/diffusion/sites/fused_linear_gelu_site.py index 1d8e84ca88..b914da3123 100644 --- a/python/sglang/kernels/ops/diffusion/sites/fused_linear_gelu_site.py +++ b/python/sglang/kernels/ops/diffusion/sites/fused_linear_gelu_site.py @@ -9,13 +9,16 @@ tanh-approximate GELU (max abs diff ~5e-6 vs ``F.gelu(approximate="tanh")`` in fp32), so for half-precision inference the fused path differs from the reference only at bf16/fp16 rounding-order level -- close, but not bit-exact. -Because it is not bit-exact, the fused path is **mounted only for -``quality="high"`` requests** (see ``SamplingParams.quality``): model code marks -its GELU up-projection sites with ``FUSED_LINEAR_GELU.mark`` (default: off, -reference path, bit-exact), and the denoising stage mounts / unmounts -``FUSED_LINEAR_GELU`` at batch boundaries. Mounting is all-or-nothing per transformer: if any marked site -fails the static guards (quantized weights, missing bias, non-half dtype, ...) -no site on that transformer is fused. +Because it is not bit-exact, the fused path is a ``"lossless"``-tier fast path +(see ``SamplingParams.quality``): model code marks its GELU up-projection sites +with ``FUSED_LINEAR_GELU.mark`` (unmounted: bit-exact reference path), and the +denoising stage mounts it at batch boundaries for ``quality="lossless"`` (the +default) and ``"high"`` requests and unmounts it for ``"exact"``. Mounting is +all-or-nothing per transformer: if any marked site fails the static guards +(quantized weights, missing bias, non-half dtype, ...) no site on that +transformer is fused. A mounted site still runs its linear's forward while a +LoRA adapter on that linear adds output: the epilogue multiplies only the base +weight. The fused GEMM is exposed as a registered custom op (``register_custom_op``) exactly like the other diffusion kernels (e.g. qknorm_rope), so it stays a @@ -91,7 +94,13 @@ def _static_reject_reason(linear: Any) -> str | None: exactly what the reference forward multiplies. The weight's *device* is deliberately not checked here -- under CPU offload the weights live on CPU between requests -- the runtime guard checks the input device per call. + A LoRA wrapper is judged by the layer it wraps, whose weight the epilogue + multiplies; whether its adapter adds output is a per-call check. """ + from sglang.multimodal_gen.runtime.layers.lora.linear import BaseLayerWithLoRA + + if isinstance(linear, BaseLayerWithLoRA): + linear = linear.base_layer if not _HAS_ADDMM_ACTIVATION: return "torch._addmm_activation unavailable" if not _is_unquantized(linear): @@ -134,8 +143,13 @@ def _input_independent_ok(linear: Any) -> bool: def can_use_linear_gelu(linear: Any, x: torch.Tensor) -> bool: """Whether ``gelu(linear(x))`` can use the fused cublasLt epilogue now.""" + from sglang.multimodal_gen.runtime.layers.lora.linear import BaseLayerWithLoRA + if not (x.is_cuda and x.dtype in (torch.bfloat16, torch.float16)): return False + # the epilogue reads the base weight only; an active adapter needs the forward + if isinstance(linear, BaseLayerWithLoRA) and linear.lora_active: + return False if getattr(linear, "weight", None) is None or x.dtype != linear.weight.dtype: return False if torch.compiler.is_compiling(): diff --git a/python/sglang/multimodal_gen/test/server/gpu_cases.py b/python/sglang/multimodal_gen/test/server/gpu_cases.py index 0d4c162d1e..97de0ec4cd 100644 --- a/python/sglang/multimodal_gen/test/server/gpu_cases.py +++ b/python/sglang/multimodal_gen/test/server/gpu_cases.py @@ -645,6 +645,44 @@ if not current_platform.is_hip(): T2V_sampling_params, ), ) +# Dynamic LoRA at the default quality: the lossless-tier fusions (the fused +# linear-GELU epilogue among them) must keep the adapter's delta. GT-free: the +# output must match quality="exact" and differ from the adapter off. Skipped on +# AMD: the match/change thresholds are calibrated on NVIDIA only. +if not current_platform.is_hip(): + ONE_GPU_CASES.append( + DiffusionTestCase( + "wan2_1_t2v_1_3b_dynamic_lora_default_quality_1gpu", + DiffusionServerArgs( + model_path=DEFAULT_WAN_2_1_T2V_1_3B_MODEL_NAME_FOR_TEST, + lora_path="Cseti/Wan-LoRA-Arcane-Jinx-v1", + extras=[ + "--lora-merge-mode", + "dynamic", + # layerwise offload leaves placeholder weights, which keep + # the fused linear-GELU path unmounted + "--dit-layerwise-offload", + "false", + ], + ), + DiffusionSamplingParams( + prompt="csetiarcane Nfj1nx with blue hair, a woman walking in a cyberpunk city at night", + output_size="480x272", + num_frames=9, + # CRF 0, so the frame comparisons measure the model, not the encoder + extras={ + "num_inference_steps": 8, + "seed": 42, + "output_compression": 100, + }, + ), + run_perf_check=False, + perf_warmup_requests=1, + run_consistency_check=False, + reference_quality="exact", + run_lora_effect_check=True, + ), + ) # Skip all ModelOpt tests on AMD: FP8 requires torch._scaled_mm (HIPBLAS_STATUS_NOT_SUPPORTED # on ROCm), NVFP4 requires flashinfer or sgl_kernel FP4 kernels (CUDA-only). # Run FP8 cases on the regular H100 1-GPU CI shard and keep only B200-only diff --git a/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json b/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json index 3827fca72c..72208d860e 100644 --- a/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json +++ b/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json @@ -2293,6 +2293,39 @@ "runtime_peak_allocated_mb": 9999.0, "estimated_full_test_time_s": 90 }, + "wan2_1_t2v_1_3b_dynamic_lora_default_quality_1gpu": { + "stages_ms": { + "InputValidationStage": 0.04, + "TextEncodingStage": 2.36, + "LatentPreparationStage": 0.11, + "TimestepPreparationStage": 0.48, + "DenoisingStage": 687.26, + "DecodingStage": 88.7, + "per_frame_generation": null + }, + "denoise_step_ms": { + "0": 107.57, + "1": 99.77, + "2": 78.34, + "3": 78.47, + "4": 82.14, + "5": 79.72, + "6": 80.1, + "7": 80.0 + }, + "expected_e2e_ms": 782.31, + "expected_load_ms": 31976.56, + "expected_avg_denoise_ms": 85.76, + "expected_median_denoise_ms": 80.05, + "load_peak_vram_mb": 4148.0, + "runtime_peak_vram_mb": 13494.0, + "warmup_peak_vram_mb": 13968.0, + "load_peak_allocated_mb": 3976.25, + "runtime_peak_allocated_mb": 5521.42, + "load_peak_host_anon_mb": 10335.67, + "runtime_peak_host_anon_mb": 10335.67, + "estimated_full_test_time_s": 51.9 + }, "wan2_1_i2v_14b_lora_2gpu": { "expected_load_ms": 66554.82, "stages_ms": { diff --git a/python/sglang/multimodal_gen/test/server/test_server_common.py b/python/sglang/multimodal_gen/test/server/test_server_common.py index d9423b99fc..9d38ec7e17 100644 --- a/python/sglang/multimodal_gen/test/server/test_server_common.py +++ b/python/sglang/multimodal_gen/test/server/test_server_common.py @@ -13,6 +13,7 @@ import os import queue import threading import time +from dataclasses import replace from pathlib import Path from typing import Any, Callable @@ -42,6 +43,9 @@ from sglang.multimodal_gen.test.server.test_server_utils import ( ) from sglang.multimodal_gen.test.server.testcase_configs import ( BASELINE_CONFIG, + OUTPUT_CHANGE_MIN_MEAN_ABS_DIFF, + OUTPUT_MATCH_MAX_MEAN_ABS_DIFF, + OUTPUT_MATCH_MIN_PSNR, DiffusionTestCase, PerformanceSummary, ScenarioConfig, @@ -56,6 +60,8 @@ from sglang.multimodal_gen.test.test_utils import ( audio_gt_exists, compare_audio_with_gt, compare_with_gt, + compute_mean_abs_diff, + compute_psnr, encode_audio_gt_wav, extract_audio_pcm_from_video_bytes, extract_key_frames_from_video, @@ -116,6 +122,22 @@ def _print_case_log_separator(case_id: str, state: str) -> None: ) +def _key_frame_distances( + case_id: str, label: str, frames: list[np.ndarray], reference: list[np.ndarray] +) -> tuple[list[float], list[float]]: + """Per-key-frame mean_abs_diff and PSNR of ``frames`` against ``reference``.""" + pairs = list(zip(frames, reference, strict=True)) + mean_abs_diffs = [compute_mean_abs_diff(frame, ref) for frame, ref in pairs] + psnrs = [compute_psnr(frame, ref) for frame, ref in pairs] + print( + f"[Output Reference] {case_id}: {label}: " + f"mean_abs_diff=[{', '.join(f'{value:.3f}' for value in mean_abs_diffs)}] " + f"psnr=[{', '.join(f'{value:.2f}' for value in psnrs)}]", + flush=True, + ) + return mean_abs_diffs, psnrs + + @pytest.fixture def diffusion_server(case: DiffusionTestCase) -> ServerContext: """Start a diffusion server for a single case and tear it down afterwards.""" @@ -1463,6 +1485,85 @@ Pinned revision used by this check: {SGL_TEST_FILES_CI_DATA_REVISION} logger.info("[Multi-LoRA] All multi-LoRA tests passed for %s", case.id) + def _key_frames(self, case: DiffusionTestCase, content: bytes) -> list[np.ndarray]: + if case.server_args.modality == "video": + return extract_key_frames_from_video(content) + return [image_bytes_to_numpy(content)] + + def _generate_key_frames( + self, ctx: ServerContext, case: DiffusionTestCase, **extras: Any + ) -> list[np.ndarray]: + """Key frames of the case's request with ``extras`` overriding its own.""" + generate_fn = get_generate_fn( + model_path=case.server_args.model_path, + modality=case.server_args.modality, + sampling_params=replace( + case.sampling_params, + extras={**case.sampling_params.extras, **extras}, + ), + ) + _, content = self._run_generation_with_server_watchdog( + ctx, case.id, generate_fn, self._client(ctx) + ) + return self._key_frames(case, content) + + def _test_output_references( + self, ctx: ServerContext, case: DiffusionTestCase, content: bytes + ) -> None: + """Compare the case's output with other requests to the same server. + + Same prompt and seed, so a reference differs from the case's request + only in its quality level, or in the LoRA adapter being off. + """ + compared = [("output", self._key_frames(case, content))] + failures = [] + if case.reference_quality is not None: + label = f"quality={case.reference_quality}" + reference = self._generate_key_frames( + ctx, case, quality=case.reference_quality + ) + mean_abs_diffs, psnrs = _key_frame_distances( + case.id, f"output vs {label}", compared[0][1], reference + ) + if ( + max(mean_abs_diffs) > OUTPUT_MATCH_MAX_MEAN_ABS_DIFF + or min(psnrs) < OUTPUT_MATCH_MIN_PSNR + ): + failures.append( + f"output does not match {label}: " + f"max mean_abs_diff={max(mean_abs_diffs):.3f} " + f"(<= {OUTPUT_MATCH_MAX_MEAN_ABS_DIFF}), " + f"min psnr={min(psnrs):.2f} (>= {OUTPUT_MATCH_MIN_PSNR})" + ) + compared.append((label, reference)) + + if case.run_lora_effect_check: + base_url = f"http://localhost:{ctx.port}/v1" + # unmerge turns the adapter off in every merge mode; merge restores it + resp = requests.post( + f"{base_url}/unmerge_lora_weights", timeout=_CONTROL_API_TIMEOUT_SECS + ) + assert resp.status_code == 200, f"unmerge_lora_weights failed: {resp.text}" + try: + without_lora = self._generate_key_frames(ctx, case) + finally: + resp = requests.post( + f"{base_url}/merge_lora_weights", + timeout=_CONTROL_API_TIMEOUT_SECS, + ) + assert resp.status_code == 200, f"merge_lora_weights failed: {resp.text}" + for label, frames in compared: + mean_abs_diffs, _ = _key_frame_distances( + case.id, f"{label} vs LoRA off", frames, without_lora + ) + if min(mean_abs_diffs) < OUTPUT_CHANGE_MIN_MEAN_ABS_DIFF: + failures.append( + f"{label} barely changes with the LoRA off: " + f"min mean_abs_diff={min(mean_abs_diffs):.3f} " + f"(>= {OUTPUT_CHANGE_MIN_MEAN_ABS_DIFF})" + ) + assert not failures, "\n".join(failures) + def _test_v1_models_endpoint( self, ctx: ServerContext, case: DiffusionTestCase ) -> None: @@ -1774,6 +1875,13 @@ Pinned revision used by this check: {SGL_TEST_FILES_CI_DATA_REVISION} lambda: self._validate_audio_consistency(case, content), ) + # before the LoRA checks below, which change the server's adapter state + if case.reference_quality is not None or case.run_lora_effect_check: + run_case_check( + "output references", + lambda: self._test_output_references(diffusion_server, case, content), + ) + if case.run_lora_basic_api_check: run_case_check( "LoRA basic API", diff --git a/python/sglang/multimodal_gen/test/server/testcase_configs.py b/python/sglang/multimodal_gen/test/server/testcase_configs.py index 894de807ed..ed90d6330e 100644 --- a/python/sglang/multimodal_gen/test/server/testcase_configs.py +++ b/python/sglang/multimodal_gen/test/server/testcase_configs.py @@ -351,6 +351,11 @@ class DiffusionTestCase: run_multi_lora_api_check: bool = False # Cached MiniMax goldens require its implicit-quality cache mode. pin_consistency_quality: bool = True + # GT-free checks against other requests to the same server, same seed: the + # output must match the request at this quality level ... + reference_quality: str | None = None + # ... and must change once the server's LoRA adapter is deactivated + run_lora_effect_check: bool = False def __post_init__(self) -> None: if self.perf_repeat_requests < 1: @@ -441,6 +446,17 @@ class DiffusionTestCase: f"{self.id}: run_multi_lora_api_check requires lora_path and second_lora_path" ) + if self.run_lora_effect_check and not (has_startup_lora or has_dynamic_lora): + raise ValueError( + f"{self.id}: run_lora_effect_check requires lora_path or dynamic_lora_path" + ) + + +# per-key-frame bounds in 0-255 levels: lossless-tier drift is ~1 level, a +# full-strength LoRA moves tens; the 4x gap keeps a dropped delta out of "match" +OUTPUT_MATCH_MAX_MEAN_ABS_DIFF = 3.0 +OUTPUT_MATCH_MIN_PSNR = 30.0 +OUTPUT_CHANGE_MIN_MEAN_ABS_DIFF = 12.0 _REALTIME_MODEL_COMMON_EXTRAS = { "seed": 42, diff --git a/python/sglang/multimodal_gen/test/unit/test_quality_fusion_lora.py b/python/sglang/multimodal_gen/test/unit/test_quality_fusion_lora.py new file mode 100644 index 0000000000..d369bdef2d --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_quality_fusion_lora.py @@ -0,0 +1,208 @@ +"""Mounting a quality fusion must not drop an active LoRA adapter's output. + +A fused path that multiplies a linear's weight itself sees only the base weight +of a LoRA wrapper. Every fusion in ``_QUALITY_FUSION_HANDLERS`` is either run +below on a site whose linears carry a dynamic adapter, or listed with the +reason its sites cannot drop one, so a new fusion has to pick a side. +""" + +import pytest +import torch +import torch.nn as nn + +from sglang.kernels.ops.diffusion import ( + FLUX2_NVFP4_SWIGLU_QUANT, + FUSED_GATE_RMSNORM, + FUSED_LINEAR_GELU, + FUSED_LN_MODULATE, + HELIOS_GATED_RESIDUAL, + HUNYUAN_QKNORM, + LINGBOT_VIDEO_GATED_RESIDUAL, + LINGBOT_VIDEO_RMSNORM, + LTX2_QKNORM_SPLIT_ROPE, + LTX2_RMS_NORM_MODULATE, + MINIMAX_H3_NORM_MODULATE, + NVFP4_BIAS_GELU, + QWEN_IMAGE_ADDED_QKV, + SANA_VIDEO_LINEAR_ATTENTION, + can_use_linear_gelu, +) +from sglang.kernels.ops.diffusion.sites.fused_linear_gelu_site import ( + can_use_linear_gelu_static, +) +from sglang.multimodal_gen.runtime.distributed.parallel_state import ( + maybe_init_distributed_environment_and_model_parallel, + model_parallel_is_initialized, +) +from sglang.multimodal_gen.runtime.layers.linear import ( + ColumnParallelLinear, + MergedColumnParallelLinear, +) +from sglang.multimodal_gen.runtime.layers.lora.linear import ( + BaseLayerWithLoRA, + wrap_with_lora_layer, +) +from sglang.multimodal_gen.runtime.models.dits.qwen_image import ( + QwenImageCrossAttention, +) +from sglang.multimodal_gen.runtime.models.dits.wanvideo import _WanGELUMLP +from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import ( + _QUALITY_FUSION_HANDLERS, +) +from sglang.multimodal_gen.test.single_test_file.component_accuracy.utils import ( + ensure_distributed_env_defaults, +) + +DIM = 64 +LORA_RANK = 4 +DEVICE = "cuda" if torch.cuda.is_available() else "cpu" + +requires_cuda = pytest.mark.skipif( + not torch.cuda.is_available(), reason="the fused path runs on CUDA only" +) + +# fusion -> why mounting it cannot drop an adapter's output +_ADAPTER_SAFE_FUSIONS = { + FLUX2_NVFP4_SWIGLU_QUANT: "fuses only a linear_in that NVFP4 loading marked " + "ready; a LoRA wrapper carries no such mark", + NVFP4_BIAS_GELU: "rejects an fc_in without the ModelOpt FP4 method, " + "which a LoRA wrapper does not expose", + FUSED_LN_MODULATE: "norm and modulation, no linear", + LTX2_QKNORM_SPLIT_ROPE: "runs on the projections' outputs", + LTX2_RMS_NORM_MODULATE: "norm and modulation, no linear", + FUSED_GATE_RMSNORM: "norm, gate and residual, no linear", + HUNYUAN_QKNORM: "runs on the projections' outputs", + LINGBOT_VIDEO_RMSNORM: "norm, no linear", + LINGBOT_VIDEO_GATED_RESIDUAL: "residual update, no linear", + HELIOS_GATED_RESIDUAL: "residual update, no linear", + MINIMAX_H3_NORM_MODULATE: "norm and modulation, no linear", + SANA_VIDEO_LINEAR_ATTENTION: "runs on the projections' outputs", +} + + +def _ensure_single_process_parallel_runtime() -> None: + if model_parallel_is_initialized(): + return + ensure_distributed_env_defaults() + maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1) + + +def _wan_gelu_mlp(): + site = _WanGELUMLP(DIM, 4 * DIM, prefix="", quant_config=None) + return site, ("fc_in",), lambda site, x: site(x) + + +def _qwen_image_added_qkv(): + # only the state _get_added_qkv_projections reads + site = object.__new__(QwenImageCrossAttention) + nn.Module.__init__(site) + site.use_fused_added_qkv = True + site._unquantized_added_qkv_is_packed = True + site.to_added_qkv = MergedColumnParallelLinear(DIM, [DIM] * 3, bias=True) + QWEN_IMAGE_ADDED_QKV.mark(site) + return ( + site, + ("to_added_qkv",), + lambda site, x: torch.cat(site._get_added_qkv_projections(x), dim=-1), + ) + + +# a site builder returns (site, names of its LoRA-wrappable linears, call) +_LORA_SITES = [ + pytest.param( + FUSED_LINEAR_GELU, _wan_gelu_mlp, id="linear_gelu", marks=requires_cuda + ), + pytest.param(QWEN_IMAGE_ADDED_QKV, _qwen_image_added_qkv, id="qwen_added_qkv"), +] + + +def _attach_dynamic_lora(site: nn.Module, name: str) -> BaseLayerWithLoRA: + linear = site.get_submodule(name) + out_features, in_features = linear.weight.shape + wrapper = wrap_with_lora_layer(linear, lora_rank=LORA_RANK, lora_alpha=LORA_RANK) + wrapper.set_lora_weights( + torch.randn(LORA_RANK, in_features, device=DEVICE, dtype=torch.bfloat16), + torch.randn(out_features, LORA_RANK, device=DEVICE, dtype=torch.bfloat16) * 0.1, + merge_weights=False, + ) + site.register_module(name, wrapper) + return wrapper + + +def _site_with_dynamic_lora(make_site): + _ensure_single_process_parallel_runtime() + torch.manual_seed(0) + site, linear_names, run = make_site() + site = site.to(device=DEVICE, dtype=torch.bfloat16) + for parameter in site.parameters(): + parameter.normal_(mean=0.0, std=0.02) + wrappers = [_attach_dynamic_lora(site, name) for name in linear_names] + x = torch.randn(2, 129, DIM, device=DEVICE, dtype=torch.bfloat16) + return site, wrappers, run, x + + +def test_every_quality_fusion_has_a_lora_site_or_a_reason(): + with_site = {param.values[0] for param in _LORA_SITES} + assert with_site.isdisjoint(_ADAPTER_SAFE_FUSIONS) + unclassified = {fusion for _, fusion in _QUALITY_FUSION_HANDLERS} - ( + with_site | set(_ADAPTER_SAFE_FUSIONS) + ) + assert not unclassified, ( + "add a LoRA site for these fusions, or say why they cannot drop an " + f"adapter's output: {sorted(fusion.name for fusion in unclassified)}" + ) + + +@pytest.mark.parametrize("fusion, make_site", _LORA_SITES) +@torch.no_grad() +def test_mounted_fusion_keeps_the_dynamic_lora_delta(fusion, make_site): + site, wrappers, run, x = _site_with_dynamic_lora(make_site) + reference = run(site, x) + for wrapper in wrappers: + wrapper.disable_lora = True + base_only = run(site, x) + for wrapper in wrappers: + wrapper.disable_lora = False + assert not torch.allclose(reference, base_only, rtol=1e-2, atol=1e-2) + + fusion.mount(site) + try: + assert torch.equal(run(site, x), reference) + finally: + fusion.unmount(site) + + +@requires_cuda +@torch.no_grad() +def test_linear_gelu_guard_follows_the_adapter_state(): + site, (wrapper,), _, x = _site_with_dynamic_lora(_wan_gelu_mlp) + fusable = torch.cuda.get_device_capability() < (12, 0) + assert FUSED_LINEAR_GELU.mount(site) + try: + assert not can_use_linear_gelu(wrapper, x) + wrapper.disable_lora = True + assert can_use_linear_gelu(wrapper, x) == fusable + wrapper.disable_lora = False + # merged: the base weight the epilogue reads already holds the update + wrapper.merge_lora_weights() + assert can_use_linear_gelu(wrapper, x) == fusable + finally: + FUSED_LINEAR_GELU.unmount(site) + + +def test_linear_gelu_static_guard_reads_the_wrapped_layer(): + _ensure_single_process_parallel_runtime() + linear = ColumnParallelLinear( + DIM, 4 * DIM, bias=True, gather_output=False, params_dtype=torch.bfloat16 + ) + wrapper = wrap_with_lora_layer(linear, lora_rank=LORA_RANK, lora_alpha=LORA_RANK) + assert can_use_linear_gelu_static(wrapper) + + # the per-shard epilogue would skip the cross-rank gather + linear.gather_output, linear.tp_size = True, 2 + assert not can_use_linear_gelu_static(wrapper) + linear.gather_output, linear.tp_size = False, 1 + + # the epilogue adds the bias the layer hands back to its caller + linear.skip_bias_add = True + assert not can_use_linear_gelu_static(wrapper)