From 65386cb95a925863e7be461abdd40b045dbabc17 Mon Sep 17 00:00:00 2001 From: Mick Date: Sun, 11 Oct 2026 08:50:53 +0800 Subject: [PATCH] [diffusion] perf: add the dynamic LoRA delta inside the second GEMM (#43501) Co-authored-by: Mick Qian Co-authored-by: Claude Opus 5.5 --- docs/docs/sglang-diffusion/fused_kernels.mdx | 1 + .../runtime/layers/lora/linear.py | 212 ++++++++++-------- .../unit/test_lora_dynamic_delta_in_gemm.py | 181 +++++++++++++++ 3 files changed, 298 insertions(+), 96 deletions(-) create mode 100644 python/sglang/multimodal_gen/test/unit/test_lora_dynamic_delta_in_gemm.py diff --git a/docs/docs/sglang-diffusion/fused_kernels.mdx b/docs/docs/sglang-diffusion/fused_kernels.mdx index 206bb39a91..fff6c90e35 100644 --- a/docs/docs/sglang-diffusion/fused_kernels.mdx +++ b/docs/docs/sglang-diffusion/fused_kernels.mdx @@ -99,6 +99,7 @@ These fusion families mount under both `quality="lossless"` and | LingBot Video fused RMSNorm | Replaces the handwritten cast, square, mean, rsqrt, and multiply chain with existing Triton RMSNorm kernels | | LingBot Video per-token gated residual + RMSNorm modulate | Folds `residual + gate * update` (per-token `[B, S, 1]` gate) and the `rmsnorm(x) * (1 + scale) + shift` adaLN chain (strided `[B, S, 6D]` chunk views) into single kernels | | SANA-Video BF16-input linear attention | Keeps the first linear-attention GEMM's inputs in BF16 with FP32 accumulation/output; the second GEMM remains FP32 | +| Dynamic LoRA delta | An unmerged adapter's scaled update accumulated onto the base output by its second GEMM (`addmm`, one rounding), and a row-parallel layer's bias added in the base GEMM's epilogue, instead of materializing the `[tokens, out]` delta and scaling and adding it in separate passes | | FLUX-family VAE fast paths | Channels-last decode, GroupNorm(+SiLU), upsample, and attention replacements for FLUX.2 and AutoencoderKL-based FLUX.1, Z-Image, and SD3 pipelines | | Wan VAE RMSNorm + SiLU | Replaces the channel-first RMSNorm/SiLU chain while keeping the decode in `channels_last_3d` | | MiniMax-H3 VAE decoder fast path | Per-head fp32 RMSNorm, rounded to the activation dtype, and NeoX RoPE in one launch over the strided Q/K views of the ViT decoder's QKV buffer; cuDNN SDPA for the block's attention | diff --git a/python/sglang/multimodal_gen/runtime/layers/lora/linear.py b/python/sglang/multimodal_gen/runtime/layers/lora/linear.py index b7d0fa34a0..78454b1a1b 100644 --- a/python/sglang/multimodal_gen/runtime/layers/lora/linear.py +++ b/python/sglang/multimodal_gen/runtime/layers/lora/linear.py @@ -13,6 +13,7 @@ from torch.distributed._composable.fsdp import ( ) from torch.distributed.tensor import DTensor +from sglang.multimodal_gen.configs.sample.sampling_params import quality_allows from sglang.multimodal_gen.runtime.distributed import ( get_local_torch_device, get_tp_rank, @@ -76,6 +77,71 @@ def _compute_lora_delta( ) +def _request_allows_lossless() -> bool: + # only a request that allows it: many stages set the forward context + # without forward_batch, and an exact request must still get exact bytes + context = get_forward_context_or_none() + forward_batch = None if context is None else context.forward_batch + sampling_params = getattr(forward_batch, "sampling_params", None) + quality = getattr(sampling_params, "quality", None) + return quality is not None and quality_allows(quality, "lossless") + + +def _can_add_lora_delta_in_gemm( + out: torch.Tensor, lora_A: torch.Tensor, lora_B: torch.Tensor +) -> bool: + if not isinstance(lora_B, torch.Tensor) or lora_A.dim() != lora_B.dim(): + return False + if lora_A.dim() == 3 and lora_A.shape[0] != lora_B.shape[0]: + return False + return lora_A.dim() in (2, 3) and out.is_contiguous() and _request_allows_lossless() + + +def _add_lora_delta_in_gemm( + out: torch.Tensor, + x: torch.Tensor, + lora_A: torch.Tensor, + lora_B: torch.Tensor, + scale: float, +) -> torch.Tensor: + """``out + scale * delta``, accumulated onto ``out`` by the second GEMM. + + Lossless tier: the reference path rounds the delta after the GEMM, after + each scale and after the add, moving the full ``[tokens, out]`` delta + through memory for each; here it is never materialized. The scale goes on + the rank-wide intermediate as a tensor op, not as the GEMM's ``alpha``: a + compiled forward reads the request scale as a symbolic float, which a + kwarg would freeze at its first value. + """ + out_2d = out.view(-1, out.shape[-1]) + x_2d = x.reshape(-1, x.shape[-1]) + if lora_A.dim() == 2: + hidden = x_2d @ lora_A.t() + else: + hidden = x_2d @ lora_A.reshape(-1, lora_A.shape[-1]).t() + if scale != 1.0: + hidden = hidden * scale + if lora_A.dim() == 2: + groups = [(out_2d, hidden, lora_B)] + else: + # stacked [groups, rank, in] / [groups, out, rank], one GEMM per group + rank, width = lora_A.shape[1], lora_B.shape[1] + groups = [ + ( + out_2d[:, g * width : (g + 1) * width], + hidden[:, g * rank : (g + 1) * rank], + lora_B[g], + ) + for g in range(lora_B.shape[0]) + ] + for target, rows, b in groups: + if target.dtype == rows.dtype: + target.addmm_(rows, b.t()) + else: + target.add_(rows @ b.t()) + return out + + class BaseLayerWithLoRA(nn.Module): def __init__( self, @@ -154,34 +220,47 @@ class BaseLayerWithLoRA(nn.Module): @torch.compile() def _forward_with_delta(self, x: torch.Tensor) -> torch.Tensor: - lora_A = self.lora_A - lora_B = self.lora_B - if isinstance(self.lora_B, DTensor): - lora_B = self.lora_B.to_local() - lora_A = self.lora_A.to_local() - # TODO: Support multiple LoRA adapters when use not merged mode runtime_lora_scale = self._runtime_lora_scale() if runtime_lora_scale == 0.0: return self.base_layer(x) - lora_dtype = lora_A.dtype - x_lora = x.to(dtype=lora_dtype) - lora_A_sliced = self.slice_lora_a_weights( + out, output_bias = self.base_layer(x) + out = self._add_lora_delta(out, x, runtime_lora_scale) + return self._add_lora_output_offset(out), output_bias + + def _add_lora_delta( + self, out: torch.Tensor, x: torch.Tensor, runtime_lora_scale: float + ) -> torch.Tensor: + """``out`` plus the active adapter's update for input ``x``.""" + lora_A = self.lora_A + lora_B = self.lora_B + if isinstance(lora_B, DTensor): + lora_B = lora_B.to_local() + lora_A = lora_A.to_local() + + x_lora = x.to(dtype=lora_A.dtype) + lora_A = self.slice_lora_a_weights( lora_A.to(device=x.device, non_blocking=True) ) - lora_B_sliced = self.slice_lora_b_weights( + lora_B = self.slice_lora_b_weights( lora_B.to(device=x.device, non_blocking=True) ) - delta = _compute_lora_delta(x_lora, lora_A_sliced, lora_B_sliced) + if _can_add_lora_delta_in_gemm(out, lora_A, lora_B): + scale = self.strength * runtime_lora_scale + if self.lora_alpha != self.lora_rank: + scale *= self.lora_alpha / self.lora_rank # type: ignore + return _add_lora_delta_in_gemm(out, x_lora, lora_A, lora_B, scale) + + delta = _compute_lora_delta(x_lora, lora_A, lora_B) if self.lora_alpha != self.lora_rank: - delta = delta * ( - self.lora_alpha / self.lora_rank # type: ignore - ) # type: ignore - delta = delta * self.strength * runtime_lora_scale - out, output_bias = self.base_layer(x) - out = out + delta.to(dtype=out.dtype) - return self._add_lora_output_offset(out), output_bias + delta = delta * (self.lora_alpha / self.lora_rank) # type: ignore + # x * 1.0 is exact, so skipping a unit scale keeps the reference bits + if self.strength != 1.0: + delta = delta * self.strength + if runtime_lora_scale != 1.0: + delta = delta * runtime_lora_scale + return out + delta.to(dtype=out.dtype) def slice_lora_a_weights(self, A: torch.Tensor) -> torch.Tensor: return A @@ -688,35 +767,13 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA): if runtime_lora_scale == 0.0: return self.base_layer(input_) - lora_A = self.lora_A - lora_B = self.lora_B - if isinstance(self.lora_B, DTensor): - lora_B = self.lora_B.to_local() - lora_A = self.lora_A.to_local() - bias = self.base_layer.bias if not self.base_layer.skip_bias_add else None output_parallel = self.base_layer.quant_method.apply( self.base_layer, input_, bias ) if not self.merged and not self.disable_lora: - lora_dtype = lora_A.dtype - input_lora = input_.to(dtype=lora_dtype) - lora_A_sliced = self.slice_lora_a_weights( - lora_A.to(device=input_.device, non_blocking=True) - ) - lora_B_sliced = self.slice_lora_b_weights( - lora_B.to(device=input_.device, non_blocking=True) - ) - delta_parallel = _compute_lora_delta( - input_lora, lora_A_sliced, lora_B_sliced - ) - if self.lora_alpha != self.lora_rank: - delta_parallel = delta_parallel * ( - self.lora_alpha / self.lora_rank # type: ignore - ) # type: ignore - delta_parallel = delta_parallel * self.strength * runtime_lora_scale - output_parallel = output_parallel + delta_parallel.to( - dtype=output_parallel.dtype + output_parallel = self._add_lora_delta( + output_parallel, input_, runtime_lora_scale ) output_parallel = self._add_lora_output_offset(output_parallel) if self.base_layer.gather_output: @@ -790,12 +847,6 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA): if runtime_lora_scale == 0.0: return self.base_layer(input_) - lora_A = self.lora_A - lora_B = self.lora_B - if isinstance(self.lora_B, DTensor): - lora_B = self.lora_B.to_local() - lora_A = self.lora_A.to_local() - if self.base_layer.input_is_parallel: input_parallel = input_ else: @@ -804,28 +855,22 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA): input_, num_partitions=self.base_layer.tp_size ) input_parallel = splitted_input[tp_rank].contiguous() + bias = self.base_layer.bias + # Lossless tier: the bias goes into the GEMM epilogue on rank 0, as in + # RowParallelLinear, instead of a separate pass after the reduce. + bias_in_gemm = ( + bias is not None + and not self.base_layer.skip_bias_add + and _request_allows_lossless() + ) output_parallel = self.base_layer.quant_method.apply( - self.base_layer, input_parallel + self.base_layer, + input_parallel, + bias=bias if bias_in_gemm and self.base_layer.tp_rank == 0 else None, ) if not self.merged and not self.disable_lora: - lora_dtype = lora_A.dtype - input_parallel_lora = input_parallel.to(dtype=lora_dtype) - lora_A_sliced = self.slice_lora_a_weights( - lora_A.to(device=input_parallel.device, non_blocking=True) - ) - lora_B_sliced = self.slice_lora_b_weights( - lora_B.to(device=input_parallel.device, non_blocking=True) - ) - delta_parallel = _compute_lora_delta( - input_parallel_lora, lora_A_sliced, lora_B_sliced - ) - if self.lora_alpha != self.lora_rank: - delta_parallel = delta_parallel * ( - self.lora_alpha / self.lora_rank # type: ignore - ) # type: ignore - delta_parallel = delta_parallel * self.strength * runtime_lora_scale - output_parallel = output_parallel + delta_parallel.to( - dtype=output_parallel.dtype + output_parallel = self._add_lora_delta( + output_parallel, input_parallel, runtime_lora_scale ) if self.base_layer.reduce_results and self.base_layer.tp_size > 1: @@ -835,9 +880,7 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA): if not self.base_layer.skip_bias_add: output = ( - output_ + self.base_layer.bias - if self.base_layer.bias is not None - else output_ + output_ + bias if bias is not None and not bias_in_gemm else output_ ) output_bias = None else: @@ -863,39 +906,16 @@ class LinearWithLoRA(BaseLayerWithLoRA): @torch.compile() def forward(self, x: torch.Tensor) -> torch.Tensor: - lora_A = self.lora_A - lora_B = self.lora_B - if isinstance(self.lora_B, DTensor): - lora_B = self.lora_B.to_local() - lora_A = self.lora_A.to_local() - # TODO: Support multiple LoRA adapters when use not merged mode runtime_lora_scale = self._runtime_lora_scale() if runtime_lora_scale == 0.0: # nn.Linear.forward() returns a single tensor return self.base_layer(x) + # nn.Linear.forward() returns a single tensor, not a tuple + out = self.base_layer(x) if not self.merged and not self.disable_lora: - lora_dtype = lora_A.dtype - x_lora = x.to(dtype=lora_dtype) - lora_A_sliced = self.slice_lora_a_weights( - lora_A.to(device=x.device, non_blocking=True) - ) - lora_B_sliced = self.slice_lora_b_weights( - lora_B.to(device=x.device, non_blocking=True) - ) - delta = _compute_lora_delta(x_lora, lora_A_sliced, lora_B_sliced) - if self.lora_alpha != self.lora_rank: - delta = delta * ( - self.lora_alpha / self.lora_rank # type: ignore - ) # type: ignore - delta = delta * self.strength * runtime_lora_scale - # nn.Linear.forward() returns a single tensor, not a tuple - out = self.base_layer(x) - out = out + delta.to(dtype=out.dtype) - else: - # nn.Linear.forward() returns a single tensor - out = self.base_layer(x) + out = self._add_lora_delta(out, x, runtime_lora_scale) return self._add_lora_output_offset(out) diff --git a/python/sglang/multimodal_gen/test/unit/test_lora_dynamic_delta_in_gemm.py b/python/sglang/multimodal_gen/test/unit/test_lora_dynamic_delta_in_gemm.py new file mode 100644 index 0000000000..a0b08795a9 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_lora_dynamic_delta_in_gemm.py @@ -0,0 +1,181 @@ +"""The dynamic LoRA delta: bit-exact at quality "exact", inside the GEMM above it. + +At "exact" the delta keeps the reference chain (GEMM, alpha scale, strength, +request scale, add) and only drops multiplications by 1.0, which are exact. At +"lossless" the second GEMM accumulates the scaled delta onto the base output and +rounds once, and a row-parallel bias moves into the base GEMM's epilogue. A +forward whose stage does not put its request in the forward context runs the +exact chain. +""" + +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +import torch +import torch.nn.functional as F +from torch import nn + +from sglang.multimodal_gen.runtime.layers.lora.linear import ( + LinearWithLoRA, + RowParallelLinearWithLoRA, + _compute_lora_delta, + _request_allows_lossless, +) +from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context +from sglang.multimodal_gen.test.quality_tier_admission import ( + assert_error_no_worse_than_reference, +) + +_TP_RANK_PATCH = "sglang.multimodal_gen.runtime.layers.lora.linear.get_tp_rank" +TOKENS, IN_DIM, OUT_DIM, RANK = 256, 192, 96, 16 + + +def _batch(quality: str, runtime_scale: float = 1.0): + return SimpleNamespace( + sampling_params=SimpleNamespace(quality=quality), + runtime_lora_scale=runtime_scale, + ) + + +def _activations(rows: int, cols: int, seed: int) -> torch.Tensor: + g = torch.Generator().manual_seed(seed) + x = torch.randn(rows, cols, generator=g) + x[:, ::17] *= 8.0 # the outlier channels a DiT block carries + return x + + +def _adapter(stacked: bool, dtype: torch.dtype, seed: int = 1): + g = torch.Generator().manual_seed(seed) + if stacked: + a = torch.randn(3, RANK, IN_DIM, generator=g) * 0.05 + b = torch.randn(3, OUT_DIM // 3, RANK, generator=g) * 0.05 + else: + a = torch.randn(RANK, IN_DIM, generator=g) * 0.05 + b = torch.randn(OUT_DIM, RANK, generator=g) * 0.05 + return a.to(dtype), b.to(dtype) + + +def _layer(a, b, alpha: int, strength: float) -> LinearWithLoRA: + base = nn.Linear(IN_DIM, OUT_DIM, bias=True, dtype=torch.bfloat16) + layer = LinearWithLoRA(base, lora_rank=RANK, lora_alpha=alpha) + layer.set_lora_weights(a, b, strength=strength, merge_weights=False) + return layer + + +def _reference_chain(layer, out, x, runtime_scale): + """The dynamic delta as main computed it.""" + a, b = layer.lora_A, layer.lora_B + delta = _compute_lora_delta(x.to(a.dtype), a, b) + if layer.lora_alpha != layer.lora_rank: + delta = delta * (layer.lora_alpha / layer.lora_rank) + delta = delta * layer.strength * runtime_scale + return out + delta.to(dtype=out.dtype) + + +@pytest.mark.parametrize("request_batch", [_batch("exact"), None]) +@pytest.mark.parametrize("stacked", [False, True]) +@pytest.mark.parametrize( + "alpha, strength, runtime_scale", + [(RANK, 1.0, 1.0), (2 * RANK, 1.0, 1.0), (RANK, 0.7, 1.0), (8, 0.7, 0.5)], +) +def test_exact_tier_keeps_the_reference_chain_bit_for_bit( + stacked, alpha, strength, runtime_scale, request_batch +): + a, b = _adapter(stacked, torch.bfloat16) + layer = _layer(a, b, alpha, strength) + x = _activations(TOKENS, IN_DIM, seed=2).to(torch.bfloat16) + with torch.inference_mode(), set_forward_context(0, None, request_batch): + out = layer.base_layer(x) + actual = layer._add_lora_delta(out.clone(), x, runtime_scale) + expected = _reference_chain(layer, out, x, runtime_scale) + assert torch.equal(actual, expected) + + +@pytest.mark.parametrize( + "stacked, adapter_dtype", + [(False, torch.bfloat16), (True, torch.bfloat16), (False, torch.float32)], +) +def test_lossless_tier_is_admitted_against_the_fp64_math(stacked, adapter_dtype): + a, b = _adapter(stacked, adapter_dtype) + layer = _layer(a, b, alpha=2 * RANK, strength=0.8) + x = _activations(TOKENS, IN_DIM, seed=3).to(torch.bfloat16) + with torch.inference_mode(), set_forward_context(0, None, _batch("lossless")): + out = layer.base_layer(x) + reference = out.double() + _compute_lora_delta( + x.double(), a.double(), b.double() + ) * (2.0 * 0.8) + baseline = _reference_chain(layer, out, x, 1.0) + candidate = layer._add_lora_delta(out.clone(), x, 1.0) + assert_error_no_worse_than_reference( + reference_fp64=reference, + baseline=baseline, + candidate=candidate, + label=f"LoRA delta in the GEMM ({'stacked' if stacked else '2D'}, {adapter_dtype})", + ) + + +def test_only_a_request_that_allows_it_takes_the_lossless_path(): + # LTX-2's denoising, among others, sets the context without forward_batch + assert not _request_allows_lossless() # no forward context + with set_forward_context(0, None): + assert not _request_allows_lossless() + with set_forward_context(0, None, SimpleNamespace(runtime_lora_scale=1.0)): + assert not _request_allows_lossless() + with set_forward_context(0, None, _batch("exact")): + assert not _request_allows_lossless() + for quality in ("lossless", "high"): + with set_forward_context(0, None, _batch(quality)): + assert _request_allows_lossless() + + +class _RowLinear(nn.Module): + """The RowParallelLinear surface the LoRA wrapper reads, at TP 1.""" + + def __init__(self): + super().__init__() + g = torch.Generator().manual_seed(4) + self.weight = nn.Parameter( + (torch.randn(OUT_DIM, IN_DIM, generator=g) * 0.05).to(torch.bfloat16) + ) + self.bias = nn.Parameter(torch.randn(OUT_DIM, generator=g).to(torch.bfloat16)) + self.input_is_parallel = True + self.reduce_results = True + self.skip_bias_add = False + self.tp_size = 1 + self.tp_rank = 0 + self.input_size_per_partition = IN_DIM + self.quant_method = SimpleNamespace( + apply=lambda layer, x, bias=None: F.linear(x, layer.weight, bias) + ) + + def forward(self, x): + return F.linear(x, self.weight, self.bias), None + + +def test_row_parallel_bias_moves_into_the_gemm_only_above_exact(): + a, b = _adapter(False, torch.bfloat16) + layer = RowParallelLinearWithLoRA(_RowLinear(), lora_rank=RANK, lora_alpha=RANK) + layer.set_lora_weights(a, b, merge_weights=False) + base = layer.base_layer + x = _activations(TOKENS, IN_DIM, seed=5).to(torch.bfloat16) + with patch(_TP_RANK_PATCH, return_value=0), torch.inference_mode(): + with set_forward_context(0, None, _batch("exact")): + exact, _ = layer(x) + with set_forward_context(0, None, _batch("lossless")): + lossless, _ = layer(x) + reference_chain = ( + _reference_chain(layer, F.linear(x, base.weight), x, 1.0) + base.bias + ) + reference = ( + x.double() @ base.weight.double().t() + + base.bias.double() + + _compute_lora_delta(x.double(), a.double(), b.double()) + ) + assert torch.equal(exact, reference_chain) + assert_error_no_worse_than_reference( + reference_fp64=reference, + baseline=exact, + candidate=lossless, + label="row-parallel LoRA with the bias in the GEMM", + )