[diffusion] perf: add the dynamic LoRA delta inside the second GEMM (#43501)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-10-11 08:50:53 +08:00
committed by GitHub
co-authored by Mick Qian Claude Opus 5.5
parent e328c7ecbd
commit 65386cb95a
3 changed files with 298 additions and 96 deletions
@@ -99,6 +99,7 @@ These fusion families mount under both `quality="lossless"` and
| LingBot Video fused RMSNorm | Replaces the handwritten cast, square, mean, rsqrt, and multiply chain with existing Triton RMSNorm kernels |
| LingBot Video per-token gated residual + RMSNorm modulate | Folds `residual + gate * update` (per-token `[B, S, 1]` gate) and the `rmsnorm(x) * (1 + scale) + shift` adaLN chain (strided `[B, S, 6D]` chunk views) into single kernels |
| SANA-Video BF16-input linear attention | Keeps the first linear-attention GEMM's inputs in BF16 with FP32 accumulation/output; the second GEMM remains FP32 |
| Dynamic LoRA delta | An unmerged adapter's scaled update accumulated onto the base output by its second GEMM (`addmm`, one rounding), and a row-parallel layer's bias added in the base GEMM's epilogue, instead of materializing the `[tokens, out]` delta and scaling and adding it in separate passes |
| FLUX-family VAE fast paths | Channels-last decode, GroupNorm(+SiLU), upsample, and attention replacements for FLUX.2 and AutoencoderKL-based FLUX.1, Z-Image, and SD3 pipelines |
| Wan VAE RMSNorm + SiLU | Replaces the channel-first RMSNorm/SiLU chain while keeping the decode in `channels_last_3d` |
| MiniMax-H3 VAE decoder fast path | Per-head fp32 RMSNorm, rounded to the activation dtype, and NeoX RoPE in one launch over the strided Q/K views of the ViT decoder's QKV buffer; cuDNN SDPA for the block's attention |
@@ -13,6 +13,7 @@ from torch.distributed._composable.fsdp import (
)
from torch.distributed.tensor import DTensor
from sglang.multimodal_gen.configs.sample.sampling_params import quality_allows
from sglang.multimodal_gen.runtime.distributed import (
get_local_torch_device,
get_tp_rank,
@@ -76,6 +77,71 @@ def _compute_lora_delta(
)
def _request_allows_lossless() -> bool:
# only a request that allows it: many stages set the forward context
# without forward_batch, and an exact request must still get exact bytes
context = get_forward_context_or_none()
forward_batch = None if context is None else context.forward_batch
sampling_params = getattr(forward_batch, "sampling_params", None)
quality = getattr(sampling_params, "quality", None)
return quality is not None and quality_allows(quality, "lossless")
def _can_add_lora_delta_in_gemm(
out: torch.Tensor, lora_A: torch.Tensor, lora_B: torch.Tensor
) -> bool:
if not isinstance(lora_B, torch.Tensor) or lora_A.dim() != lora_B.dim():
return False
if lora_A.dim() == 3 and lora_A.shape[0] != lora_B.shape[0]:
return False
return lora_A.dim() in (2, 3) and out.is_contiguous() and _request_allows_lossless()
def _add_lora_delta_in_gemm(
out: torch.Tensor,
x: torch.Tensor,
lora_A: torch.Tensor,
lora_B: torch.Tensor,
scale: float,
) -> torch.Tensor:
"""``out + scale * delta``, accumulated onto ``out`` by the second GEMM.
Lossless tier: the reference path rounds the delta after the GEMM, after
each scale and after the add, moving the full ``[tokens, out]`` delta
through memory for each; here it is never materialized. The scale goes on
the rank-wide intermediate as a tensor op, not as the GEMM's ``alpha``: a
compiled forward reads the request scale as a symbolic float, which a
kwarg would freeze at its first value.
"""
out_2d = out.view(-1, out.shape[-1])
x_2d = x.reshape(-1, x.shape[-1])
if lora_A.dim() == 2:
hidden = x_2d @ lora_A.t()
else:
hidden = x_2d @ lora_A.reshape(-1, lora_A.shape[-1]).t()
if scale != 1.0:
hidden = hidden * scale
if lora_A.dim() == 2:
groups = [(out_2d, hidden, lora_B)]
else:
# stacked [groups, rank, in] / [groups, out, rank], one GEMM per group
rank, width = lora_A.shape[1], lora_B.shape[1]
groups = [
(
out_2d[:, g * width : (g + 1) * width],
hidden[:, g * rank : (g + 1) * rank],
lora_B[g],
)
for g in range(lora_B.shape[0])
]
for target, rows, b in groups:
if target.dtype == rows.dtype:
target.addmm_(rows, b.t())
else:
target.add_(rows @ b.t())
return out
class BaseLayerWithLoRA(nn.Module):
def __init__(
self,
@@ -154,34 +220,47 @@ class BaseLayerWithLoRA(nn.Module):
@torch.compile()
def _forward_with_delta(self, x: torch.Tensor) -> torch.Tensor:
lora_A = self.lora_A
lora_B = self.lora_B
if isinstance(self.lora_B, DTensor):
lora_B = self.lora_B.to_local()
lora_A = self.lora_A.to_local()
# TODO: Support multiple LoRA adapters when use not merged mode
runtime_lora_scale = self._runtime_lora_scale()
if runtime_lora_scale == 0.0:
return self.base_layer(x)
lora_dtype = lora_A.dtype
x_lora = x.to(dtype=lora_dtype)
lora_A_sliced = self.slice_lora_a_weights(
out, output_bias = self.base_layer(x)
out = self._add_lora_delta(out, x, runtime_lora_scale)
return self._add_lora_output_offset(out), output_bias
def _add_lora_delta(
self, out: torch.Tensor, x: torch.Tensor, runtime_lora_scale: float
) -> torch.Tensor:
"""``out`` plus the active adapter's update for input ``x``."""
lora_A = self.lora_A
lora_B = self.lora_B
if isinstance(lora_B, DTensor):
lora_B = lora_B.to_local()
lora_A = lora_A.to_local()
x_lora = x.to(dtype=lora_A.dtype)
lora_A = self.slice_lora_a_weights(
lora_A.to(device=x.device, non_blocking=True)
)
lora_B_sliced = self.slice_lora_b_weights(
lora_B = self.slice_lora_b_weights(
lora_B.to(device=x.device, non_blocking=True)
)
delta = _compute_lora_delta(x_lora, lora_A_sliced, lora_B_sliced)
if _can_add_lora_delta_in_gemm(out, lora_A, lora_B):
scale = self.strength * runtime_lora_scale
if self.lora_alpha != self.lora_rank:
scale *= self.lora_alpha / self.lora_rank # type: ignore
return _add_lora_delta_in_gemm(out, x_lora, lora_A, lora_B, scale)
delta = _compute_lora_delta(x_lora, lora_A, lora_B)
if self.lora_alpha != self.lora_rank:
delta = delta * (
self.lora_alpha / self.lora_rank # type: ignore
) # type: ignore
delta = delta * self.strength * runtime_lora_scale
out, output_bias = self.base_layer(x)
out = out + delta.to(dtype=out.dtype)
return self._add_lora_output_offset(out), output_bias
delta = delta * (self.lora_alpha / self.lora_rank) # type: ignore
# x * 1.0 is exact, so skipping a unit scale keeps the reference bits
if self.strength != 1.0:
delta = delta * self.strength
if runtime_lora_scale != 1.0:
delta = delta * runtime_lora_scale
return out + delta.to(dtype=out.dtype)
def slice_lora_a_weights(self, A: torch.Tensor) -> torch.Tensor:
return A
@@ -688,35 +767,13 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
if runtime_lora_scale == 0.0:
return self.base_layer(input_)
lora_A = self.lora_A
lora_B = self.lora_B
if isinstance(self.lora_B, DTensor):
lora_B = self.lora_B.to_local()
lora_A = self.lora_A.to_local()
bias = self.base_layer.bias if not self.base_layer.skip_bias_add else None
output_parallel = self.base_layer.quant_method.apply(
self.base_layer, input_, bias
)
if not self.merged and not self.disable_lora:
lora_dtype = lora_A.dtype
input_lora = input_.to(dtype=lora_dtype)
lora_A_sliced = self.slice_lora_a_weights(
lora_A.to(device=input_.device, non_blocking=True)
)
lora_B_sliced = self.slice_lora_b_weights(
lora_B.to(device=input_.device, non_blocking=True)
)
delta_parallel = _compute_lora_delta(
input_lora, lora_A_sliced, lora_B_sliced
)
if self.lora_alpha != self.lora_rank:
delta_parallel = delta_parallel * (
self.lora_alpha / self.lora_rank # type: ignore
) # type: ignore
delta_parallel = delta_parallel * self.strength * runtime_lora_scale
output_parallel = output_parallel + delta_parallel.to(
dtype=output_parallel.dtype
output_parallel = self._add_lora_delta(
output_parallel, input_, runtime_lora_scale
)
output_parallel = self._add_lora_output_offset(output_parallel)
if self.base_layer.gather_output:
@@ -790,12 +847,6 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
if runtime_lora_scale == 0.0:
return self.base_layer(input_)
lora_A = self.lora_A
lora_B = self.lora_B
if isinstance(self.lora_B, DTensor):
lora_B = self.lora_B.to_local()
lora_A = self.lora_A.to_local()
if self.base_layer.input_is_parallel:
input_parallel = input_
else:
@@ -804,28 +855,22 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
input_, num_partitions=self.base_layer.tp_size
)
input_parallel = splitted_input[tp_rank].contiguous()
bias = self.base_layer.bias
# Lossless tier: the bias goes into the GEMM epilogue on rank 0, as in
# RowParallelLinear, instead of a separate pass after the reduce.
bias_in_gemm = (
bias is not None
and not self.base_layer.skip_bias_add
and _request_allows_lossless()
)
output_parallel = self.base_layer.quant_method.apply(
self.base_layer, input_parallel
self.base_layer,
input_parallel,
bias=bias if bias_in_gemm and self.base_layer.tp_rank == 0 else None,
)
if not self.merged and not self.disable_lora:
lora_dtype = lora_A.dtype
input_parallel_lora = input_parallel.to(dtype=lora_dtype)
lora_A_sliced = self.slice_lora_a_weights(
lora_A.to(device=input_parallel.device, non_blocking=True)
)
lora_B_sliced = self.slice_lora_b_weights(
lora_B.to(device=input_parallel.device, non_blocking=True)
)
delta_parallel = _compute_lora_delta(
input_parallel_lora, lora_A_sliced, lora_B_sliced
)
if self.lora_alpha != self.lora_rank:
delta_parallel = delta_parallel * (
self.lora_alpha / self.lora_rank # type: ignore
) # type: ignore
delta_parallel = delta_parallel * self.strength * runtime_lora_scale
output_parallel = output_parallel + delta_parallel.to(
dtype=output_parallel.dtype
output_parallel = self._add_lora_delta(
output_parallel, input_parallel, runtime_lora_scale
)
if self.base_layer.reduce_results and self.base_layer.tp_size > 1:
@@ -835,9 +880,7 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
if not self.base_layer.skip_bias_add:
output = (
output_ + self.base_layer.bias
if self.base_layer.bias is not None
else output_
output_ + bias if bias is not None and not bias_in_gemm else output_
)
output_bias = None
else:
@@ -863,39 +906,16 @@ class LinearWithLoRA(BaseLayerWithLoRA):
@torch.compile()
def forward(self, x: torch.Tensor) -> torch.Tensor:
lora_A = self.lora_A
lora_B = self.lora_B
if isinstance(self.lora_B, DTensor):
lora_B = self.lora_B.to_local()
lora_A = self.lora_A.to_local()
# TODO: Support multiple LoRA adapters when use not merged mode
runtime_lora_scale = self._runtime_lora_scale()
if runtime_lora_scale == 0.0:
# nn.Linear.forward() returns a single tensor
return self.base_layer(x)
# nn.Linear.forward() returns a single tensor, not a tuple
out = self.base_layer(x)
if not self.merged and not self.disable_lora:
lora_dtype = lora_A.dtype
x_lora = x.to(dtype=lora_dtype)
lora_A_sliced = self.slice_lora_a_weights(
lora_A.to(device=x.device, non_blocking=True)
)
lora_B_sliced = self.slice_lora_b_weights(
lora_B.to(device=x.device, non_blocking=True)
)
delta = _compute_lora_delta(x_lora, lora_A_sliced, lora_B_sliced)
if self.lora_alpha != self.lora_rank:
delta = delta * (
self.lora_alpha / self.lora_rank # type: ignore
) # type: ignore
delta = delta * self.strength * runtime_lora_scale
# nn.Linear.forward() returns a single tensor, not a tuple
out = self.base_layer(x)
out = out + delta.to(dtype=out.dtype)
else:
# nn.Linear.forward() returns a single tensor
out = self.base_layer(x)
out = self._add_lora_delta(out, x, runtime_lora_scale)
return self._add_lora_output_offset(out)
@@ -0,0 +1,181 @@
"""The dynamic LoRA delta: bit-exact at quality "exact", inside the GEMM above it.
At "exact" the delta keeps the reference chain (GEMM, alpha scale, strength,
request scale, add) and only drops multiplications by 1.0, which are exact. At
"lossless" the second GEMM accumulates the scaled delta onto the base output and
rounds once, and a row-parallel bias moves into the base GEMM's epilogue. A
forward whose stage does not put its request in the forward context runs the
exact chain.
"""
from types import SimpleNamespace
from unittest.mock import patch
import pytest
import torch
import torch.nn.functional as F
from torch import nn
from sglang.multimodal_gen.runtime.layers.lora.linear import (
LinearWithLoRA,
RowParallelLinearWithLoRA,
_compute_lora_delta,
_request_allows_lossless,
)
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
from sglang.multimodal_gen.test.quality_tier_admission import (
assert_error_no_worse_than_reference,
)
_TP_RANK_PATCH = "sglang.multimodal_gen.runtime.layers.lora.linear.get_tp_rank"
TOKENS, IN_DIM, OUT_DIM, RANK = 256, 192, 96, 16
def _batch(quality: str, runtime_scale: float = 1.0):
return SimpleNamespace(
sampling_params=SimpleNamespace(quality=quality),
runtime_lora_scale=runtime_scale,
)
def _activations(rows: int, cols: int, seed: int) -> torch.Tensor:
g = torch.Generator().manual_seed(seed)
x = torch.randn(rows, cols, generator=g)
x[:, ::17] *= 8.0 # the outlier channels a DiT block carries
return x
def _adapter(stacked: bool, dtype: torch.dtype, seed: int = 1):
g = torch.Generator().manual_seed(seed)
if stacked:
a = torch.randn(3, RANK, IN_DIM, generator=g) * 0.05
b = torch.randn(3, OUT_DIM // 3, RANK, generator=g) * 0.05
else:
a = torch.randn(RANK, IN_DIM, generator=g) * 0.05
b = torch.randn(OUT_DIM, RANK, generator=g) * 0.05
return a.to(dtype), b.to(dtype)
def _layer(a, b, alpha: int, strength: float) -> LinearWithLoRA:
base = nn.Linear(IN_DIM, OUT_DIM, bias=True, dtype=torch.bfloat16)
layer = LinearWithLoRA(base, lora_rank=RANK, lora_alpha=alpha)
layer.set_lora_weights(a, b, strength=strength, merge_weights=False)
return layer
def _reference_chain(layer, out, x, runtime_scale):
"""The dynamic delta as main computed it."""
a, b = layer.lora_A, layer.lora_B
delta = _compute_lora_delta(x.to(a.dtype), a, b)
if layer.lora_alpha != layer.lora_rank:
delta = delta * (layer.lora_alpha / layer.lora_rank)
delta = delta * layer.strength * runtime_scale
return out + delta.to(dtype=out.dtype)
@pytest.mark.parametrize("request_batch", [_batch("exact"), None])
@pytest.mark.parametrize("stacked", [False, True])
@pytest.mark.parametrize(
"alpha, strength, runtime_scale",
[(RANK, 1.0, 1.0), (2 * RANK, 1.0, 1.0), (RANK, 0.7, 1.0), (8, 0.7, 0.5)],
)
def test_exact_tier_keeps_the_reference_chain_bit_for_bit(
stacked, alpha, strength, runtime_scale, request_batch
):
a, b = _adapter(stacked, torch.bfloat16)
layer = _layer(a, b, alpha, strength)
x = _activations(TOKENS, IN_DIM, seed=2).to(torch.bfloat16)
with torch.inference_mode(), set_forward_context(0, None, request_batch):
out = layer.base_layer(x)
actual = layer._add_lora_delta(out.clone(), x, runtime_scale)
expected = _reference_chain(layer, out, x, runtime_scale)
assert torch.equal(actual, expected)
@pytest.mark.parametrize(
"stacked, adapter_dtype",
[(False, torch.bfloat16), (True, torch.bfloat16), (False, torch.float32)],
)
def test_lossless_tier_is_admitted_against_the_fp64_math(stacked, adapter_dtype):
a, b = _adapter(stacked, adapter_dtype)
layer = _layer(a, b, alpha=2 * RANK, strength=0.8)
x = _activations(TOKENS, IN_DIM, seed=3).to(torch.bfloat16)
with torch.inference_mode(), set_forward_context(0, None, _batch("lossless")):
out = layer.base_layer(x)
reference = out.double() + _compute_lora_delta(
x.double(), a.double(), b.double()
) * (2.0 * 0.8)
baseline = _reference_chain(layer, out, x, 1.0)
candidate = layer._add_lora_delta(out.clone(), x, 1.0)
assert_error_no_worse_than_reference(
reference_fp64=reference,
baseline=baseline,
candidate=candidate,
label=f"LoRA delta in the GEMM ({'stacked' if stacked else '2D'}, {adapter_dtype})",
)
def test_only_a_request_that_allows_it_takes_the_lossless_path():
# LTX-2's denoising, among others, sets the context without forward_batch
assert not _request_allows_lossless() # no forward context
with set_forward_context(0, None):
assert not _request_allows_lossless()
with set_forward_context(0, None, SimpleNamespace(runtime_lora_scale=1.0)):
assert not _request_allows_lossless()
with set_forward_context(0, None, _batch("exact")):
assert not _request_allows_lossless()
for quality in ("lossless", "high"):
with set_forward_context(0, None, _batch(quality)):
assert _request_allows_lossless()
class _RowLinear(nn.Module):
"""The RowParallelLinear surface the LoRA wrapper reads, at TP 1."""
def __init__(self):
super().__init__()
g = torch.Generator().manual_seed(4)
self.weight = nn.Parameter(
(torch.randn(OUT_DIM, IN_DIM, generator=g) * 0.05).to(torch.bfloat16)
)
self.bias = nn.Parameter(torch.randn(OUT_DIM, generator=g).to(torch.bfloat16))
self.input_is_parallel = True
self.reduce_results = True
self.skip_bias_add = False
self.tp_size = 1
self.tp_rank = 0
self.input_size_per_partition = IN_DIM
self.quant_method = SimpleNamespace(
apply=lambda layer, x, bias=None: F.linear(x, layer.weight, bias)
)
def forward(self, x):
return F.linear(x, self.weight, self.bias), None
def test_row_parallel_bias_moves_into_the_gemm_only_above_exact():
a, b = _adapter(False, torch.bfloat16)
layer = RowParallelLinearWithLoRA(_RowLinear(), lora_rank=RANK, lora_alpha=RANK)
layer.set_lora_weights(a, b, merge_weights=False)
base = layer.base_layer
x = _activations(TOKENS, IN_DIM, seed=5).to(torch.bfloat16)
with patch(_TP_RANK_PATCH, return_value=0), torch.inference_mode():
with set_forward_context(0, None, _batch("exact")):
exact, _ = layer(x)
with set_forward_context(0, None, _batch("lossless")):
lossless, _ = layer(x)
reference_chain = (
_reference_chain(layer, F.linear(x, base.weight), x, 1.0) + base.bias
)
reference = (
x.double() @ base.weight.double().t()
+ base.bias.double()
+ _compute_lora_delta(x.double(), a.double(), b.double())
)
assert torch.equal(exact, reference_chain)
assert_error_no_worse_than_reference(
reference_fp64=reference,
baseline=exact,
candidate=lossless,
label="row-parallel LoRA with the bias in the GEMM",
)