[Perf] diffusion: accumulate chunked LoRA merges with addmm (#37620)
Co-authored-by: Mick <mickjagger19@icloud.com>
This commit is contained in:
@@ -585,8 +585,9 @@ class BaseLayerWithLoRA(nn.Module):
|
||||
)
|
||||
for start in range(0, lora_B_2d.shape[0], chunk_rows):
|
||||
end = min(start + chunk_rows, lora_B_2d.shape[0])
|
||||
chunk_delta = lora_B_2d[start:end] @ lora_A_sliced
|
||||
data_2d[start:end].add_(chunk_delta, alpha=scale)
|
||||
data_2d[start:end].addmm_(
|
||||
lora_B_2d[start:end], lora_A_sliced, alpha=scale
|
||||
)
|
||||
|
||||
def _should_merge_in_fp32(
|
||||
self,
|
||||
|
||||
@@ -0,0 +1,240 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
"""Paired CUDA benchmark for chunked diffusion LoRA weight merging.
|
||||
|
||||
Example:
|
||||
python bench_lora_merge_addmm.py --output /tmp/results.csv
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import math
|
||||
import statistics
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
from torch.profiler import ProfilerActivity, profile
|
||||
|
||||
CHUNK_BYTES = 32 * 1024 * 1024
|
||||
CASES = [
|
||||
(320, 320, 4, 1.0),
|
||||
(320, 320, 8, 0.25),
|
||||
(1280, 1280, 8, 1.0),
|
||||
(1280, 1280, 16, -0.5),
|
||||
(3072, 3072, 16, 0.25),
|
||||
(3072, 3072, 32, 1.0),
|
||||
(4096, 4096, 32, -0.5),
|
||||
(4096, 4096, 64, 1.0),
|
||||
(12288, 4096, 4, 0.25),
|
||||
(12288, 4096, 64, 1.0),
|
||||
]
|
||||
DTYPES = [torch.float16, torch.bfloat16, torch.float32]
|
||||
|
||||
|
||||
def baseline(dst, lora_b, lora_a, scale, chunk_rows):
|
||||
for start in range(0, dst.shape[0], chunk_rows):
|
||||
end = min(start + chunk_rows, dst.shape[0])
|
||||
delta = lora_b[start:end] @ lora_a
|
||||
dst[start:end].add_(delta, alpha=scale)
|
||||
|
||||
|
||||
def candidate(dst, lora_b, lora_a, scale, chunk_rows):
|
||||
for start in range(0, dst.shape[0], chunk_rows):
|
||||
end = min(start + chunk_rows, dst.shape[0])
|
||||
dst[start:end].addmm_(lora_b[start:end], lora_a, alpha=scale)
|
||||
|
||||
|
||||
def time_variant(
|
||||
fn: Callable,
|
||||
initial: torch.Tensor,
|
||||
lora_b: torch.Tensor,
|
||||
lora_a: torch.Tensor,
|
||||
scale: float,
|
||||
chunk_rows: int,
|
||||
repetitions: int,
|
||||
) -> float:
|
||||
dst = initial.clone()
|
||||
start = torch.cuda.Event(enable_timing=True)
|
||||
end = torch.cuda.Event(enable_timing=True)
|
||||
start.record()
|
||||
for _ in range(repetitions):
|
||||
fn(dst, lora_b, lora_a, scale, chunk_rows)
|
||||
end.record()
|
||||
end.synchronize()
|
||||
return start.elapsed_time(end) * 1000.0 / repetitions
|
||||
|
||||
|
||||
def peak_extra(
|
||||
fn: Callable,
|
||||
initial: torch.Tensor,
|
||||
lora_b: torch.Tensor,
|
||||
lora_a: torch.Tensor,
|
||||
scale: float,
|
||||
chunk_rows: int,
|
||||
) -> int:
|
||||
dst = initial.clone()
|
||||
torch.cuda.synchronize()
|
||||
torch.cuda.reset_peak_memory_stats()
|
||||
before = torch.cuda.memory_allocated()
|
||||
fn(dst, lora_b, lora_a, scale, chunk_rows)
|
||||
torch.cuda.synchronize()
|
||||
return torch.cuda.max_memory_allocated() - before
|
||||
|
||||
|
||||
def validate(initial, lora_b, lora_a, scale, chunk_rows) -> float:
|
||||
expected = initial.clone()
|
||||
actual = initial.clone()
|
||||
baseline(expected, lora_b, lora_a, scale, chunk_rows)
|
||||
candidate(actual, lora_b, lora_a, scale, chunk_rows)
|
||||
atol = 1e-5 if initial.dtype is torch.float32 else 1e-2
|
||||
torch.testing.assert_close(actual, expected, atol=atol, rtol=atol)
|
||||
return (actual.float() - expected.float()).abs().max().item()
|
||||
|
||||
|
||||
def profile_variant(name, fn, initial, lora_b, lora_a, scale, chunk_rows):
|
||||
dst = initial.clone()
|
||||
with profile(activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA]) as prof:
|
||||
fn(dst, lora_b, lora_a, scale, chunk_rows)
|
||||
torch.cuda.synchronize()
|
||||
return (
|
||||
name
|
||||
+ "\n"
|
||||
+ prof.key_averages().table(sort_by="self_cuda_time_total", row_limit=30)
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--profile-output", type=Path)
|
||||
parser.add_argument("--warmups", type=int, default=50)
|
||||
parser.add_argument("--repetitions", type=int, default=500)
|
||||
args = parser.parse_args()
|
||||
|
||||
if not torch.cuda.is_available():
|
||||
raise RuntimeError("CUDA is required")
|
||||
torch.manual_seed(0)
|
||||
rows_out = []
|
||||
representative = None
|
||||
free_bytes, _ = torch.cuda.mem_get_info()
|
||||
|
||||
for dtype in DTYPES:
|
||||
for rows, columns, rank, scale in CASES:
|
||||
element_size = torch.empty((), dtype=dtype).element_size()
|
||||
estimate = (
|
||||
element_size * (2 * rows * columns + rows * rank + rank * columns)
|
||||
+ CHUNK_BYTES
|
||||
)
|
||||
if estimate > int(0.8 * free_bytes):
|
||||
print(f"skip dtype={dtype} shape=({rows},{columns}) rank={rank}")
|
||||
continue
|
||||
chunk_rows = max(1, CHUNK_BYTES // (columns * element_size))
|
||||
initial = torch.randn(rows, columns, device="cuda", dtype=dtype) * 1e-3
|
||||
lora_b = torch.randn(rows, rank, device="cuda", dtype=dtype) * 1e-3
|
||||
lora_a = torch.randn(rank, columns, device="cuda", dtype=dtype) * 1e-3
|
||||
max_abs = validate(initial, lora_b, lora_a, scale, chunk_rows)
|
||||
warm = initial.clone()
|
||||
for _ in range(args.warmups):
|
||||
baseline(warm, lora_b, lora_a, scale, chunk_rows)
|
||||
candidate(warm, lora_b, lora_a, scale, chunk_rows)
|
||||
torch.cuda.synchronize()
|
||||
|
||||
peaks = {
|
||||
"baseline": peak_extra(
|
||||
baseline, initial, lora_b, lora_a, scale, chunk_rows
|
||||
),
|
||||
"candidate": peak_extra(
|
||||
candidate, initial, lora_b, lora_a, scale, chunk_rows
|
||||
),
|
||||
}
|
||||
variants = {"baseline": baseline, "candidate": candidate}
|
||||
for round_idx in range(7):
|
||||
order = (
|
||||
["baseline", "candidate"]
|
||||
if round_idx % 2 == 0
|
||||
else ["candidate", "baseline"]
|
||||
)
|
||||
for variant in order:
|
||||
latency = time_variant(
|
||||
variants[variant],
|
||||
initial,
|
||||
lora_b,
|
||||
lora_a,
|
||||
scale,
|
||||
chunk_rows,
|
||||
args.repetitions,
|
||||
)
|
||||
rows_out.append(
|
||||
{
|
||||
"dtype": str(dtype).removeprefix("torch."),
|
||||
"rows": rows,
|
||||
"columns": columns,
|
||||
"rank": rank,
|
||||
"scale": scale,
|
||||
"variant": variant,
|
||||
"round": round_idx,
|
||||
"latency_us": f"{latency:.6f}",
|
||||
"peak_extra_bytes": peaks[variant],
|
||||
"max_abs": f"{max_abs:.9g}",
|
||||
}
|
||||
)
|
||||
if dtype is torch.bfloat16 and (rows, columns, rank) == (4096, 4096, 32):
|
||||
representative = (initial, lora_b, lora_a, scale, chunk_rows)
|
||||
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
with args.output.open("w", newline="") as file:
|
||||
writer = csv.DictWriter(file, fieldnames=list(rows_out[0]))
|
||||
writer.writeheader()
|
||||
writer.writerows(rows_out)
|
||||
|
||||
groups = defaultdict(lambda: defaultdict(list))
|
||||
for row in rows_out:
|
||||
key = (
|
||||
row["dtype"],
|
||||
row["rows"],
|
||||
row["columns"],
|
||||
row["rank"],
|
||||
row["scale"],
|
||||
)
|
||||
groups[key][row["variant"]].append(float(row["latency_us"]))
|
||||
speedups = []
|
||||
print("dtype shape rank scale baseline_us candidate_us speedup max_abs")
|
||||
for key in sorted(groups):
|
||||
baseline_us = statistics.median(groups[key]["baseline"])
|
||||
candidate_us = statistics.median(groups[key]["candidate"])
|
||||
speedup = baseline_us / candidate_us
|
||||
speedups.append(speedup)
|
||||
dtype, rows, columns, rank, scale = key
|
||||
max_abs = next(
|
||||
row["max_abs"]
|
||||
for row in rows_out
|
||||
if (row["dtype"], row["rows"], row["columns"], row["rank"], row["scale"])
|
||||
== key
|
||||
)
|
||||
print(
|
||||
f"{dtype} {rows}x{columns} {rank} {scale:g} "
|
||||
f"{baseline_us:.3f} {candidate_us:.3f} {speedup:.3f}x {max_abs}"
|
||||
)
|
||||
geomean = math.exp(sum(math.log(value) for value in speedups) / len(speedups))
|
||||
print(f"geomean_speedup={geomean:.3f}x minimum_speedup={min(speedups):.3f}x")
|
||||
|
||||
if args.profile_output is not None:
|
||||
if representative is None:
|
||||
raise RuntimeError("representative profiler case was skipped")
|
||||
initial, lora_b, lora_a, scale, chunk_rows = representative
|
||||
tables = [
|
||||
profile_variant(
|
||||
"baseline", baseline, initial, lora_b, lora_a, scale, chunk_rows
|
||||
),
|
||||
profile_variant(
|
||||
"candidate", candidate, initial, lora_b, lora_a, scale, chunk_rows
|
||||
),
|
||||
]
|
||||
args.profile_output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.profile_output.write_text("\n\n".join(tables))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -3,8 +3,10 @@ from types import SimpleNamespace
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.overrides import TorchFunctionMode
|
||||
from torch.utils._python_dispatch import TorchDispatchMode
|
||||
|
||||
import sglang.multimodal_gen.runtime.layers.lora.linear as lora_linear
|
||||
from sglang.multimodal_gen.runtime.layers.linear import ReplicatedLinear
|
||||
from sglang.multimodal_gen.runtime.layers.lora.linear import (
|
||||
LinearWithLoRA,
|
||||
@@ -61,6 +63,17 @@ def test_request_lora_scale_preserves_delta_offset_and_base(kind, device):
|
||||
layer.set_lora_weights(torch.ones(2, 4), torch.ones(3, 2))
|
||||
|
||||
|
||||
class _AddmmInplaceRecorder(TorchFunctionMode):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.count = 0
|
||||
|
||||
def __torch_function__(self, func, types, args=(), kwargs=None):
|
||||
if func is torch.Tensor.addmm_:
|
||||
self.count += 1
|
||||
return func(*args, **(kwargs or {}))
|
||||
|
||||
|
||||
def test_stacked_lora_delta_preserves_projection_order():
|
||||
x = torch.tensor([[2.0, 3.0]])
|
||||
lora_a = torch.tensor([[[1.0, 0.0]], [[0.0, 1.0]]])
|
||||
@@ -105,6 +118,53 @@ def test_lora_merge_unmerge_handles_inference_base_weight():
|
||||
assert torch.allclose(layer.base_layer.weight, base_weight)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("noncontiguous", [False, True])
|
||||
def test_chunked_2d_merge_accumulates_directly_with_addmm(
|
||||
monkeypatch, dtype, noncontiguous
|
||||
):
|
||||
torch.manual_seed(2)
|
||||
out_features, in_features, rank = 5, 7, 3
|
||||
base = nn.Linear(in_features, out_features, bias=False)
|
||||
layer = LinearWithLoRA(base, lora_rank=rank, lora_alpha=rank)
|
||||
if noncontiguous:
|
||||
data = torch.randn(in_features, out_features, dtype=dtype).T
|
||||
lora_a = torch.randn(in_features, rank, dtype=dtype).T
|
||||
lora_b = torch.randn(rank, out_features, dtype=dtype).T
|
||||
else:
|
||||
data = torch.randn(out_features, in_features, dtype=dtype)
|
||||
lora_a = torch.randn(rank, in_features, dtype=dtype)
|
||||
lora_b = torch.randn(out_features, rank, dtype=dtype)
|
||||
lora_a_2 = torch.randn_like(lora_a)
|
||||
lora_b_2 = torch.randn_like(lora_b)
|
||||
entries = [
|
||||
(lora_a, lora_b, None, -0.375, rank, rank, None),
|
||||
(lora_a_2, lora_b_2, None, 0.25, rank, 2 * rank, None),
|
||||
]
|
||||
expected = data.clone(memory_format=torch.preserve_format)
|
||||
for entry in entries:
|
||||
a, b, _, strength, lora_rank, alpha, _ = entry
|
||||
scale = strength * alpha / lora_rank
|
||||
for start in range(0, out_features, 2):
|
||||
expected[start : start + 2].add_(b[start : start + 2] @ a, alpha=scale)
|
||||
storage_ptr = data.data_ptr()
|
||||
monkeypatch.setattr(
|
||||
lora_linear,
|
||||
"LORA_MERGE_CHUNK_BYTES",
|
||||
2 * in_features * data.element_size(),
|
||||
)
|
||||
recorder = _AddmmInplaceRecorder()
|
||||
|
||||
with recorder:
|
||||
layer._merge_lora_into_data(data, entries)
|
||||
|
||||
assert recorder.count == 6
|
||||
assert data.data_ptr() == storage_ptr
|
||||
atol = {torch.float32: 1e-5, torch.float16: 5e-3, torch.bfloat16: 3e-2}[dtype]
|
||||
rtol = atol
|
||||
torch.testing.assert_close(data, expected, atol=atol, rtol=rtol)
|
||||
|
||||
|
||||
class _AtenMulCounter(TorchDispatchMode):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
Reference in New Issue
Block a user