106 lines
4.0 KiB
Python
106 lines
4.0 KiB
Python
"""Compare CSGMV early-return expand, compact expand, and split-K on one GPU.
|
|
|
|
PYTHONPATH=python python benchmark/kernels/lora_csgmv/bench_csgmv_split_k.py
|
|
Timings cover shrink + expand under CUDA graphs; routing is excluded.
|
|
The split-K case uses production dispatch, including tuned-config fallbacks.
|
|
"""
|
|
|
|
import argparse
|
|
import itertools
|
|
import json
|
|
import math
|
|
|
|
import torch
|
|
import triton
|
|
|
|
from sglang.kernels.ops.lora.dense.chunked_sgmv_expand import (
|
|
chunked_sgmv_lora_expand_forward,
|
|
)
|
|
from sglang.kernels.ops.lora.dense.chunked_sgmv_shrink import (
|
|
chunked_sgmv_lora_shrink_forward,
|
|
)
|
|
from sglang.srt.environ import envs
|
|
from sglang.srt.lora.utils import LoRABatchInfo
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--tokens", type=int, nargs="+", default=[32, 128, 8192])
|
|
parser.add_argument("--adapters", type=int, default=8)
|
|
args = parser.parse_args()
|
|
torch.manual_seed(0)
|
|
envs.SGLANG_ENABLE_DETERMINISTIC_INFERENCE.set(False)
|
|
print(
|
|
json.dumps(
|
|
{"gpu": torch.cuda.get_device_name(), "dtype": "bfloat16", "rank": 32}
|
|
)
|
|
)
|
|
for total in args.tokens:
|
|
ids = torch.arange(total) % args.adapters
|
|
permutation = torch.argsort(ids, stable=True).int().cuda()
|
|
counts = torch.bincount(ids, minlength=args.adapters).tolist()
|
|
lengths = [min(16, n - j) for n in counts for j in range(0, n, 16)]
|
|
adapter_ids = [a for a, n in enumerate(counts) for _ in range(0, n, 16)]
|
|
info = LoRABatchInfo(
|
|
use_cuda_graph=False,
|
|
bs=total,
|
|
num_segments=len(lengths),
|
|
max_len=16,
|
|
seg_lens=None,
|
|
permutation=permutation,
|
|
seg_indptr=torch.tensor(
|
|
[0, *itertools.accumulate(lengths)], device="cuda", dtype=torch.int32
|
|
),
|
|
weight_indices=torch.tensor(adapter_ids, device="cuda", dtype=torch.int32),
|
|
lora_ranks=torch.full(
|
|
(args.adapters,), 32, device="cuda", dtype=torch.int32
|
|
),
|
|
scalings=torch.ones(args.adapters, device="cuda"),
|
|
)
|
|
for name, k, offsets in [
|
|
("qkv", 4096, [0, 8192, 9216, 10240]),
|
|
("gate_up", 4096, [0, 12288, 24576]),
|
|
("down", 12288, [0, 4096]),
|
|
]:
|
|
slices, n = len(offsets) - 1, offsets[-1]
|
|
x = torch.randn(total, k, device="cuda", dtype=torch.bfloat16)
|
|
a = torch.randn(
|
|
args.adapters, slices * 32, k, device="cuda", dtype=torch.bfloat16
|
|
) / math.sqrt(k)
|
|
b = torch.randn(
|
|
args.adapters, n, 32, device="cuda", dtype=torch.bfloat16
|
|
) / math.sqrt(32)
|
|
base = torch.zeros(total, n, device="cuda", dtype=torch.bfloat16)
|
|
cpu_offsets = torch.tensor(offsets, dtype=torch.int32)
|
|
gpu_offsets = cpu_offsets.cuda()
|
|
maxwidth = max(hi - lo for lo, hi in zip(offsets, offsets[1:]))
|
|
result = dict(tokens=total, adapters=args.adapters, projection=name)
|
|
for label, enable_split_k, compact in [
|
|
("early_return", False, False),
|
|
("compact", False, True),
|
|
("compact_split_k", True, True),
|
|
]:
|
|
envs.SGLANG_ENABLE_LORA_CSGMV_SPLIT_K.set(enable_split_k)
|
|
|
|
def run():
|
|
h = chunked_sgmv_lora_shrink_forward(x, a, info, slices)
|
|
chunked_sgmv_lora_expand_forward(
|
|
h,
|
|
b,
|
|
info,
|
|
gpu_offsets,
|
|
maxwidth,
|
|
base,
|
|
cpu_offsets if compact else None,
|
|
)
|
|
|
|
result[label + "_us"] = (
|
|
triton.testing.do_bench_cudagraph(run, rep=100) * 1000
|
|
)
|
|
result["speedup"] = result["early_return_us"] / result["compact_split_k_us"]
|
|
print(json.dumps(result), flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|