From dcf05724f72ea2bda1f08e7fbf6e17c8af1797e0 Mon Sep 17 00:00:00 2001 From: Sun <2114377220@qq.com> Date: Thu, 22 Oct 2026 21:18:39 +0800 Subject: [PATCH] [Perf] diffusion: accumulate chunked LoRA merges with addmm (#37620) Co-authored-by: Mick --- .../runtime/layers/lora/linear.py | 5 +- .../unit/manual/bench_lora_merge_addmm.py | 240 ++++++++++++++++++ .../test/unit/test_lora_inference_mode.py | 60 +++++ 3 files changed, 303 insertions(+), 2 deletions(-) create mode 100644 python/sglang/multimodal_gen/test/unit/manual/bench_lora_merge_addmm.py diff --git a/python/sglang/multimodal_gen/runtime/layers/lora/linear.py b/python/sglang/multimodal_gen/runtime/layers/lora/linear.py index 61d32ddbcf..fbd513379d 100644 --- a/python/sglang/multimodal_gen/runtime/layers/lora/linear.py +++ b/python/sglang/multimodal_gen/runtime/layers/lora/linear.py @@ -585,8 +585,9 @@ class BaseLayerWithLoRA(nn.Module): ) for start in range(0, lora_B_2d.shape[0], chunk_rows): end = min(start + chunk_rows, lora_B_2d.shape[0]) - chunk_delta = lora_B_2d[start:end] @ lora_A_sliced - data_2d[start:end].add_(chunk_delta, alpha=scale) + data_2d[start:end].addmm_( + lora_B_2d[start:end], lora_A_sliced, alpha=scale + ) def _should_merge_in_fp32( self, diff --git a/python/sglang/multimodal_gen/test/unit/manual/bench_lora_merge_addmm.py b/python/sglang/multimodal_gen/test/unit/manual/bench_lora_merge_addmm.py new file mode 100644 index 0000000000..8ae678d178 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/manual/bench_lora_merge_addmm.py @@ -0,0 +1,240 @@ +# SPDX-License-Identifier: Apache-2.0 + +"""Paired CUDA benchmark for chunked diffusion LoRA weight merging. + +Example: + python bench_lora_merge_addmm.py --output /tmp/results.csv +""" + +import argparse +import csv +import math +import statistics +from collections import defaultdict +from collections.abc import Callable +from pathlib import Path + +import torch +from torch.profiler import ProfilerActivity, profile + +CHUNK_BYTES = 32 * 1024 * 1024 +CASES = [ + (320, 320, 4, 1.0), + (320, 320, 8, 0.25), + (1280, 1280, 8, 1.0), + (1280, 1280, 16, -0.5), + (3072, 3072, 16, 0.25), + (3072, 3072, 32, 1.0), + (4096, 4096, 32, -0.5), + (4096, 4096, 64, 1.0), + (12288, 4096, 4, 0.25), + (12288, 4096, 64, 1.0), +] +DTYPES = [torch.float16, torch.bfloat16, torch.float32] + + +def baseline(dst, lora_b, lora_a, scale, chunk_rows): + for start in range(0, dst.shape[0], chunk_rows): + end = min(start + chunk_rows, dst.shape[0]) + delta = lora_b[start:end] @ lora_a + dst[start:end].add_(delta, alpha=scale) + + +def candidate(dst, lora_b, lora_a, scale, chunk_rows): + for start in range(0, dst.shape[0], chunk_rows): + end = min(start + chunk_rows, dst.shape[0]) + dst[start:end].addmm_(lora_b[start:end], lora_a, alpha=scale) + + +def time_variant( + fn: Callable, + initial: torch.Tensor, + lora_b: torch.Tensor, + lora_a: torch.Tensor, + scale: float, + chunk_rows: int, + repetitions: int, +) -> float: + dst = initial.clone() + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + start.record() + for _ in range(repetitions): + fn(dst, lora_b, lora_a, scale, chunk_rows) + end.record() + end.synchronize() + return start.elapsed_time(end) * 1000.0 / repetitions + + +def peak_extra( + fn: Callable, + initial: torch.Tensor, + lora_b: torch.Tensor, + lora_a: torch.Tensor, + scale: float, + chunk_rows: int, +) -> int: + dst = initial.clone() + torch.cuda.synchronize() + torch.cuda.reset_peak_memory_stats() + before = torch.cuda.memory_allocated() + fn(dst, lora_b, lora_a, scale, chunk_rows) + torch.cuda.synchronize() + return torch.cuda.max_memory_allocated() - before + + +def validate(initial, lora_b, lora_a, scale, chunk_rows) -> float: + expected = initial.clone() + actual = initial.clone() + baseline(expected, lora_b, lora_a, scale, chunk_rows) + candidate(actual, lora_b, lora_a, scale, chunk_rows) + atol = 1e-5 if initial.dtype is torch.float32 else 1e-2 + torch.testing.assert_close(actual, expected, atol=atol, rtol=atol) + return (actual.float() - expected.float()).abs().max().item() + + +def profile_variant(name, fn, initial, lora_b, lora_a, scale, chunk_rows): + dst = initial.clone() + with profile(activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA]) as prof: + fn(dst, lora_b, lora_a, scale, chunk_rows) + torch.cuda.synchronize() + return ( + name + + "\n" + + prof.key_averages().table(sort_by="self_cuda_time_total", row_limit=30) + ) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--profile-output", type=Path) + parser.add_argument("--warmups", type=int, default=50) + parser.add_argument("--repetitions", type=int, default=500) + args = parser.parse_args() + + if not torch.cuda.is_available(): + raise RuntimeError("CUDA is required") + torch.manual_seed(0) + rows_out = [] + representative = None + free_bytes, _ = torch.cuda.mem_get_info() + + for dtype in DTYPES: + for rows, columns, rank, scale in CASES: + element_size = torch.empty((), dtype=dtype).element_size() + estimate = ( + element_size * (2 * rows * columns + rows * rank + rank * columns) + + CHUNK_BYTES + ) + if estimate > int(0.8 * free_bytes): + print(f"skip dtype={dtype} shape=({rows},{columns}) rank={rank}") + continue + chunk_rows = max(1, CHUNK_BYTES // (columns * element_size)) + initial = torch.randn(rows, columns, device="cuda", dtype=dtype) * 1e-3 + lora_b = torch.randn(rows, rank, device="cuda", dtype=dtype) * 1e-3 + lora_a = torch.randn(rank, columns, device="cuda", dtype=dtype) * 1e-3 + max_abs = validate(initial, lora_b, lora_a, scale, chunk_rows) + warm = initial.clone() + for _ in range(args.warmups): + baseline(warm, lora_b, lora_a, scale, chunk_rows) + candidate(warm, lora_b, lora_a, scale, chunk_rows) + torch.cuda.synchronize() + + peaks = { + "baseline": peak_extra( + baseline, initial, lora_b, lora_a, scale, chunk_rows + ), + "candidate": peak_extra( + candidate, initial, lora_b, lora_a, scale, chunk_rows + ), + } + variants = {"baseline": baseline, "candidate": candidate} + for round_idx in range(7): + order = ( + ["baseline", "candidate"] + if round_idx % 2 == 0 + else ["candidate", "baseline"] + ) + for variant in order: + latency = time_variant( + variants[variant], + initial, + lora_b, + lora_a, + scale, + chunk_rows, + args.repetitions, + ) + rows_out.append( + { + "dtype": str(dtype).removeprefix("torch."), + "rows": rows, + "columns": columns, + "rank": rank, + "scale": scale, + "variant": variant, + "round": round_idx, + "latency_us": f"{latency:.6f}", + "peak_extra_bytes": peaks[variant], + "max_abs": f"{max_abs:.9g}", + } + ) + if dtype is torch.bfloat16 and (rows, columns, rank) == (4096, 4096, 32): + representative = (initial, lora_b, lora_a, scale, chunk_rows) + + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open("w", newline="") as file: + writer = csv.DictWriter(file, fieldnames=list(rows_out[0])) + writer.writeheader() + writer.writerows(rows_out) + + groups = defaultdict(lambda: defaultdict(list)) + for row in rows_out: + key = ( + row["dtype"], + row["rows"], + row["columns"], + row["rank"], + row["scale"], + ) + groups[key][row["variant"]].append(float(row["latency_us"])) + speedups = [] + print("dtype shape rank scale baseline_us candidate_us speedup max_abs") + for key in sorted(groups): + baseline_us = statistics.median(groups[key]["baseline"]) + candidate_us = statistics.median(groups[key]["candidate"]) + speedup = baseline_us / candidate_us + speedups.append(speedup) + dtype, rows, columns, rank, scale = key + max_abs = next( + row["max_abs"] + for row in rows_out + if (row["dtype"], row["rows"], row["columns"], row["rank"], row["scale"]) + == key + ) + print( + f"{dtype} {rows}x{columns} {rank} {scale:g} " + f"{baseline_us:.3f} {candidate_us:.3f} {speedup:.3f}x {max_abs}" + ) + geomean = math.exp(sum(math.log(value) for value in speedups) / len(speedups)) + print(f"geomean_speedup={geomean:.3f}x minimum_speedup={min(speedups):.3f}x") + + if args.profile_output is not None: + if representative is None: + raise RuntimeError("representative profiler case was skipped") + initial, lora_b, lora_a, scale, chunk_rows = representative + tables = [ + profile_variant( + "baseline", baseline, initial, lora_b, lora_a, scale, chunk_rows + ), + profile_variant( + "candidate", candidate, initial, lora_b, lora_a, scale, chunk_rows + ), + ] + args.profile_output.parent.mkdir(parents=True, exist_ok=True) + args.profile_output.write_text("\n\n".join(tables)) + + +if __name__ == "__main__": + main() diff --git a/python/sglang/multimodal_gen/test/unit/test_lora_inference_mode.py b/python/sglang/multimodal_gen/test/unit/test_lora_inference_mode.py index db66f29d1e..f4b5bf5999 100644 --- a/python/sglang/multimodal_gen/test/unit/test_lora_inference_mode.py +++ b/python/sglang/multimodal_gen/test/unit/test_lora_inference_mode.py @@ -3,8 +3,10 @@ from types import SimpleNamespace import pytest import torch from torch import nn +from torch.overrides import TorchFunctionMode from torch.utils._python_dispatch import TorchDispatchMode +import sglang.multimodal_gen.runtime.layers.lora.linear as lora_linear from sglang.multimodal_gen.runtime.layers.linear import ReplicatedLinear from sglang.multimodal_gen.runtime.layers.lora.linear import ( LinearWithLoRA, @@ -61,6 +63,17 @@ def test_request_lora_scale_preserves_delta_offset_and_base(kind, device): layer.set_lora_weights(torch.ones(2, 4), torch.ones(3, 2)) +class _AddmmInplaceRecorder(TorchFunctionMode): + def __init__(self): + super().__init__() + self.count = 0 + + def __torch_function__(self, func, types, args=(), kwargs=None): + if func is torch.Tensor.addmm_: + self.count += 1 + return func(*args, **(kwargs or {})) + + def test_stacked_lora_delta_preserves_projection_order(): x = torch.tensor([[2.0, 3.0]]) lora_a = torch.tensor([[[1.0, 0.0]], [[0.0, 1.0]]]) @@ -105,6 +118,53 @@ def test_lora_merge_unmerge_handles_inference_base_weight(): assert torch.allclose(layer.base_layer.weight, base_weight) +@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("noncontiguous", [False, True]) +def test_chunked_2d_merge_accumulates_directly_with_addmm( + monkeypatch, dtype, noncontiguous +): + torch.manual_seed(2) + out_features, in_features, rank = 5, 7, 3 + base = nn.Linear(in_features, out_features, bias=False) + layer = LinearWithLoRA(base, lora_rank=rank, lora_alpha=rank) + if noncontiguous: + data = torch.randn(in_features, out_features, dtype=dtype).T + lora_a = torch.randn(in_features, rank, dtype=dtype).T + lora_b = torch.randn(rank, out_features, dtype=dtype).T + else: + data = torch.randn(out_features, in_features, dtype=dtype) + lora_a = torch.randn(rank, in_features, dtype=dtype) + lora_b = torch.randn(out_features, rank, dtype=dtype) + lora_a_2 = torch.randn_like(lora_a) + lora_b_2 = torch.randn_like(lora_b) + entries = [ + (lora_a, lora_b, None, -0.375, rank, rank, None), + (lora_a_2, lora_b_2, None, 0.25, rank, 2 * rank, None), + ] + expected = data.clone(memory_format=torch.preserve_format) + for entry in entries: + a, b, _, strength, lora_rank, alpha, _ = entry + scale = strength * alpha / lora_rank + for start in range(0, out_features, 2): + expected[start : start + 2].add_(b[start : start + 2] @ a, alpha=scale) + storage_ptr = data.data_ptr() + monkeypatch.setattr( + lora_linear, + "LORA_MERGE_CHUNK_BYTES", + 2 * in_features * data.element_size(), + ) + recorder = _AddmmInplaceRecorder() + + with recorder: + layer._merge_lora_into_data(data, entries) + + assert recorder.count == 6 + assert data.data_ptr() == storage_ptr + atol = {torch.float32: 1e-5, torch.float16: 5e-3, torch.bfloat16: 3e-2}[dtype] + rtol = atol + torch.testing.assert_close(data, expected, atol=atol, rtol=rtol) + + class _AtenMulCounter(TorchDispatchMode): def __init__(self): super().__init__()