From c0584cf9afefdb4a3004a57af8b1aec501d44a4a Mon Sep 17 00:00:00 2001 From: Kailash Ranganathan Date: Tue, 13 Oct 2026 10:16:14 -0700 Subject: [PATCH] [perf] [multi-lora] remove empty CTAs from fused LoRA expand and add split-K to shrink (#41951) Co-authored-by: Yanbin Jiang --- .../kernels/lora_csgmv/bench_csgmv_split_k.py | 105 +++++++++ .../kernels/lora_csgmv/tune_lora_csgmv.py | 6 + .../ops/lora/dense/chunked_sgmv_expand.py | 53 ++++- .../ops/lora/dense/chunked_sgmv_shrink.py | 199 ++++++++++++++++++ python/sglang/srt/environ.py | 2 + .../srt/lora/backend/chunked_backend.py | 6 +- .../ops/lora/test_chunked_sgmv_cuda_graph.py | 187 +++++++++++++++- 7 files changed, 547 insertions(+), 11 deletions(-) create mode 100644 benchmark/kernels/lora_csgmv/bench_csgmv_split_k.py diff --git a/benchmark/kernels/lora_csgmv/bench_csgmv_split_k.py b/benchmark/kernels/lora_csgmv/bench_csgmv_split_k.py new file mode 100644 index 0000000000..8cfda9c564 --- /dev/null +++ b/benchmark/kernels/lora_csgmv/bench_csgmv_split_k.py @@ -0,0 +1,105 @@ +"""Compare CSGMV early-return expand, compact expand, and split-K on one GPU. + +PYTHONPATH=python python benchmark/kernels/lora_csgmv/bench_csgmv_split_k.py +Timings cover shrink + expand under CUDA graphs; routing is excluded. +The split-K case uses production dispatch, including tuned-config fallbacks. +""" + +import argparse +import itertools +import json +import math + +import torch +import triton + +from sglang.kernels.ops.lora.dense.chunked_sgmv_expand import ( + chunked_sgmv_lora_expand_forward, +) +from sglang.kernels.ops.lora.dense.chunked_sgmv_shrink import ( + chunked_sgmv_lora_shrink_forward, +) +from sglang.srt.environ import envs +from sglang.srt.lora.utils import LoRABatchInfo + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--tokens", type=int, nargs="+", default=[32, 128, 8192]) + parser.add_argument("--adapters", type=int, default=8) + args = parser.parse_args() + torch.manual_seed(0) + envs.SGLANG_ENABLE_DETERMINISTIC_INFERENCE.set(False) + print( + json.dumps( + {"gpu": torch.cuda.get_device_name(), "dtype": "bfloat16", "rank": 32} + ) + ) + for total in args.tokens: + ids = torch.arange(total) % args.adapters + permutation = torch.argsort(ids, stable=True).int().cuda() + counts = torch.bincount(ids, minlength=args.adapters).tolist() + lengths = [min(16, n - j) for n in counts for j in range(0, n, 16)] + adapter_ids = [a for a, n in enumerate(counts) for _ in range(0, n, 16)] + info = LoRABatchInfo( + use_cuda_graph=False, + bs=total, + num_segments=len(lengths), + max_len=16, + seg_lens=None, + permutation=permutation, + seg_indptr=torch.tensor( + [0, *itertools.accumulate(lengths)], device="cuda", dtype=torch.int32 + ), + weight_indices=torch.tensor(adapter_ids, device="cuda", dtype=torch.int32), + lora_ranks=torch.full( + (args.adapters,), 32, device="cuda", dtype=torch.int32 + ), + scalings=torch.ones(args.adapters, device="cuda"), + ) + for name, k, offsets in [ + ("qkv", 4096, [0, 8192, 9216, 10240]), + ("gate_up", 4096, [0, 12288, 24576]), + ("down", 12288, [0, 4096]), + ]: + slices, n = len(offsets) - 1, offsets[-1] + x = torch.randn(total, k, device="cuda", dtype=torch.bfloat16) + a = torch.randn( + args.adapters, slices * 32, k, device="cuda", dtype=torch.bfloat16 + ) / math.sqrt(k) + b = torch.randn( + args.adapters, n, 32, device="cuda", dtype=torch.bfloat16 + ) / math.sqrt(32) + base = torch.zeros(total, n, device="cuda", dtype=torch.bfloat16) + cpu_offsets = torch.tensor(offsets, dtype=torch.int32) + gpu_offsets = cpu_offsets.cuda() + maxwidth = max(hi - lo for lo, hi in zip(offsets, offsets[1:])) + result = dict(tokens=total, adapters=args.adapters, projection=name) + for label, enable_split_k, compact in [ + ("early_return", False, False), + ("compact", False, True), + ("compact_split_k", True, True), + ]: + envs.SGLANG_ENABLE_LORA_CSGMV_SPLIT_K.set(enable_split_k) + + def run(): + h = chunked_sgmv_lora_shrink_forward(x, a, info, slices) + chunked_sgmv_lora_expand_forward( + h, + b, + info, + gpu_offsets, + maxwidth, + base, + cpu_offsets if compact else None, + ) + + result[label + "_us"] = ( + triton.testing.do_bench_cudagraph(run, rep=100) * 1000 + ) + result["speedup"] = result["early_return_us"] / result["compact_split_k_us"] + print(json.dumps(result), flush=True) + + +if __name__ == "__main__": + main() diff --git a/benchmark/kernels/lora_csgmv/tune_lora_csgmv.py b/benchmark/kernels/lora_csgmv/tune_lora_csgmv.py index 1bd4f08e61..5c7ae695c8 100755 --- a/benchmark/kernels/lora_csgmv/tune_lora_csgmv.py +++ b/benchmark/kernels/lora_csgmv/tune_lora_csgmv.py @@ -290,6 +290,8 @@ def benchmark_expand_config( x=x, weights=weights, output=output, + output_stride_0=output.stride(0), + output_stride_1=output.stride(1), seg_indptr=batch_info.seg_indptr, weight_indices=batch_info.weight_indices, lora_ranks=batch_info.lora_ranks, @@ -302,6 +304,7 @@ def benchmark_expand_config( MAX_RANK=max_rank, BLOCK_M=batch_info.max_len, BLOCK_N=config["BLOCK_N"], + SLICE_OFFSETS=(), BLOCK_K=config["BLOCK_K"], **extra_kwargs, ) @@ -315,6 +318,8 @@ def benchmark_expand_config( x=x, weights=weights, output=output, + output_stride_0=output.stride(0), + output_stride_1=output.stride(1), seg_indptr=batch_info.seg_indptr, weight_indices=batch_info.weight_indices, lora_ranks=batch_info.lora_ranks, @@ -327,6 +332,7 @@ def benchmark_expand_config( MAX_RANK=max_rank, BLOCK_M=batch_info.max_len, BLOCK_N=config["BLOCK_N"], + SLICE_OFFSETS=(), BLOCK_K=config["BLOCK_K"], **extra_kwargs, ) diff --git a/python/sglang/kernels/ops/lora/dense/chunked_sgmv_expand.py b/python/sglang/kernels/ops/lora/dense/chunked_sgmv_expand.py index 8dd7430ec5..b091659cb0 100644 --- a/python/sglang/kernels/ops/lora/dense/chunked_sgmv_expand.py +++ b/python/sglang/kernels/ops/lora/dense/chunked_sgmv_expand.py @@ -10,7 +10,16 @@ from sglang.srt.utils import cached_triton_kernel @cached_triton_kernel( - lambda _, kwargs: (kwargs["NUM_SLICES"], kwargs["BLOCK_M"], kwargs["OUTPUT_DIM"]) + lambda _, kwargs: ( + kwargs["NUM_SLICES"], + kwargs["BLOCK_M"], + kwargs["OUTPUT_DIM"], + kwargs["MAX_RANK"], + kwargs["BLOCK_N"], + kwargs["BLOCK_K"], + kwargs["SLICE_OFFSETS"], + kwargs["x"].dtype, + ) ) @triton.jit(do_not_specialize=["num_segs", "output_stride_0", "output_stride_1"]) def _chunked_lora_expand_kernel( @@ -39,6 +48,7 @@ def _chunked_lora_expand_kernel( BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr, + SLICE_OFFSETS: tl.constexpr, ): """ Computes a chunked SGMV for LoRA expand operations. @@ -63,7 +73,7 @@ def _chunked_lora_expand_kernel( w_stride_1: tl.constexpr = MAX_RANK w_stride_2: tl.constexpr = 1 - pid_s = tl.program_id(axis=2) + pid_s = tl.program_id(axis=1) if len(SLICE_OFFSETS) > 0 else tl.program_id(axis=2) if pid_s >= num_segs: return @@ -82,9 +92,30 @@ def _chunked_lora_expand_kernel( if cur_rank == 0: return - slice_id = tl.program_id(axis=1) - slice_start = tl.load(slice_offsets + slice_id) - slice_end = tl.load(slice_offsets + slice_id + 1) + if len(SLICE_OFFSETS) > 0: + # Flatten the ragged slice grid. SLICE_OFFSETS is model metadata, known on CPU. + flat_tile = tl.program_id(0) + slice_id = tl.full((), 0, tl.int32) + slice_start = tl.full((), 0, tl.int32) + slice_end = tl.full((), 0, tl.int32) + pid_n = tl.full((), 0, tl.int32) + tile_start = tl.full((), 0, tl.int32) + for s in tl.static_range(NUM_SLICES): + tiles = (SLICE_OFFSETS[s + 1] - SLICE_OFFSETS[s] + BLOCK_N - 1) // BLOCK_N + belongs = (flat_tile >= tile_start) & (flat_tile < tile_start + tiles) + slice_id = tl.where(belongs, s, slice_id) + slice_start = tl.where(belongs, SLICE_OFFSETS[s], slice_start) + slice_end = tl.where(belongs, SLICE_OFFSETS[s + 1], slice_end) + pid_n = tl.where(belongs, flat_tile - tile_start, pid_n) + tile_start += tiles + + else: + slice_id = tl.program_id(axis=1) + slice_start = tl.load(slice_offsets + slice_id) + slice_end = tl.load(slice_offsets + slice_id + 1) + pid_n = tl.program_id(axis=0) + if pid_n * BLOCK_N >= slice_end - slice_start: + return scaling = tl.load(scalings + w_index) # Adjust K (rank) according to the specific LoRA adapter @@ -99,7 +130,6 @@ def _chunked_lora_expand_kernel( # Create pointers for the first block of x and weights[batch_id][n_start: n_end][:] # The pointers will be advanced as we move in the K direction # and accumulate - pid_n = tl.program_id(axis=0) n_offset = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N + slice_start k_offset = tl.arange(0, BLOCK_K) @@ -153,8 +183,8 @@ def chunked_sgmv_lora_expand_forward( slice_offsets: torch.Tensor, max_slice_size: int, base_output: Optional[torch.Tensor], + slice_offsets_cpu: Optional[torch.Tensor] = None, ) -> torch.Tensor: - # x: (s, slice_num * r) # weights: (num_lora, output_dim, r) # slice_offsets: boundaries for different slices in the output dimension @@ -192,12 +222,20 @@ def chunked_sgmv_lora_expand_forward( else num_segments ) + # Layer metadata is already on the CPU; never copy device offsets here. + offsets = tuple(slice_offsets_cpu.tolist()) if slice_offsets_cpu is not None else () grid = ( triton.cdiv(max_slice_size, BLOCK_N), num_slices, # number of slices in the input/output segment_grid, ) + if offsets: + grid = ( + sum(triton.cdiv(hi - lo, BLOCK_N) for lo, hi in zip(offsets, offsets[1:])), + segment_grid, + ) + if base_output is None: output = torch.zeros((M, OUTPUT_DIM), device=x.device, dtype=x.dtype) else: @@ -232,6 +270,7 @@ def chunked_sgmv_lora_expand_forward( BLOCK_M=BLOCK_M, BLOCK_N=BLOCK_N, BLOCK_K=BLOCK_K, + SLICE_OFFSETS=offsets, **extra_kwargs, ) diff --git a/python/sglang/kernels/ops/lora/dense/chunked_sgmv_shrink.py b/python/sglang/kernels/ops/lora/dense/chunked_sgmv_shrink.py index 8502cfc8b0..61e1976e96 100644 --- a/python/sglang/kernels/ops/lora/dense/chunked_sgmv_shrink.py +++ b/python/sglang/kernels/ops/lora/dense/chunked_sgmv_shrink.py @@ -3,6 +3,7 @@ import triton import triton.language as tl from sglang.kernels.ops.lora.dense.lora_tuning_config import get_lora_shrink_config +from sglang.srt.environ import envs from sglang.srt.lora.utils import LoRABatchInfo from sglang.srt.utils import cached_triton_kernel @@ -122,12 +123,158 @@ def _chunked_lora_shrink_kernel( tl.store(output_ptr, partial_sum, mask=output_mask) +@triton.jit(do_not_specialize=["num_segs", "TOTAL_ROWS"]) +def _chunked_lora_shrink_split_k_kernel( + # Pointers to matrices + x, + weights, + output, + # Information on sequence lengths,ranks and weight id + seg_indptr, + weight_indices, + lora_ranks, + permutation, + num_segs, + # Meta parameters + N: tl.constexpr, # num_slices * r + K: tl.constexpr, # input_dim + NUM_SLICES: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + SPLIT_K: tl.constexpr, + TOTAL_ROWS, +): + """Write FP32 partials with shape [SPLIT_K, TOTAL_ROWS, N].""" + x_stride_1: tl.constexpr = 1 + x_stride_0: tl.constexpr = K + + w_stride_0: tl.constexpr = N * K + w_stride_1: tl.constexpr = K + w_stride_2: tl.constexpr = 1 + + output_stride_0: tl.constexpr = N + output_stride_1: tl.constexpr = 1 + + pid_s = tl.program_id(1) + if pid_s >= num_segs: + return + + pid_n = tl.program_id(0) + + seg_start = tl.load(seg_indptr + pid_s) + seg_end = tl.load(seg_indptr + pid_s + 1) + if seg_start == seg_end: + return + + # Current block computes sequence with batch_id, + # which starts from row seg_start of x with length seg_len + w_index = tl.load(weight_indices + pid_s) + rank = tl.load(lora_ranks + w_index) + + # If rank is 0, this kernel becomes a no-op as the output is always trivially correct. + if rank == 0: + return + + # Adjust N dim according to the specific LoRA adapter + cur_n = tl.minimum(N, rank * NUM_SLICES) + + # Map logical sequence index to physical index + s_offset_logical = tl.arange(0, BLOCK_M) + seg_start + s_offset_physical = tl.load( + permutation + s_offset_logical, mask=s_offset_logical < seg_end, other=0 + ) + + n_offset = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N + split_id = tl.program_id(2) + steps: tl.constexpr = tl.cdiv(K, BLOCK_K * SPLIT_K) + k_offset = tl.arange(0, BLOCK_K) + split_id * steps * BLOCK_K + x_ptrs = x + ( + s_offset_physical[:, None] * x_stride_0 + k_offset[None, :] * x_stride_1 + ) + w_ptrs = (weights + w_index * w_stride_0) + ( + k_offset[:, None] * w_stride_2 + n_offset[None, :] * w_stride_1 + ) + + # Iterate to compute the block in output matrix + partial_sum = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32) + for k in range(0, steps): + x_tile = tl.load( + x_ptrs, + mask=(s_offset_logical[:, None] < seg_end) + & (k_offset[None, :] < K - k * BLOCK_K), + other=0.0, + ) + w_tile = tl.load( + w_ptrs, + mask=(k_offset[:, None] < K - k * BLOCK_K) & (n_offset[None, :] < cur_n), + other=0.0, + ) + partial_sum += tl.dot(x_tile, w_tile) + + x_ptrs += BLOCK_K * x_stride_1 + w_ptrs += BLOCK_K * w_stride_2 + + # Store result to output matrix + output_ptr = ( + output + + split_id * TOTAL_ROWS * N + + ( + s_offset_physical[:, None] * output_stride_0 + + n_offset[None, :] * output_stride_1 + ) + ) + output_mask = (s_offset_logical[:, None] < seg_end) & (n_offset[None, :] < cur_n) + tl.store(output_ptr, partial_sum, mask=output_mask) + + +@triton.jit(do_not_specialize=["TOTAL_ROWS"]) +def _reduce_shrink_split_k( + partials, + output, + seg_indptr, + weight_indices, + lora_ranks, + permutation, + N: tl.constexpr, + TOTAL_ROWS, + NUM_SLICES: tl.constexpr, + SPLIT_K: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, +): + segment = tl.program_id(1) + start = tl.load(seg_indptr + segment) + end = tl.load(seg_indptr + segment + 1) + if start == end: + return + adapter = tl.load(weight_indices + segment) + rank = tl.load(lora_ranks + adapter) + if rank == 0: + return + logical_rows = start + tl.arange(0, BLOCK_M) + rows = tl.load(permutation + logical_rows, mask=logical_rows < end, other=0) + columns = tl.program_id(0) * BLOCK_N + tl.arange(0, BLOCK_N) + mask = (logical_rows[:, None] < end) & ( + columns[None, :] < tl.minimum(N, rank * NUM_SLICES) + ) + offsets = rows[:, None] * N + columns[None, :] + acc = tl.full((BLOCK_M, BLOCK_N), 0, tl.float32) + for split in tl.static_range(SPLIT_K): + acc += tl.load(partials + split * TOTAL_ROWS * N + offsets, mask=mask, other=0) + tl.store(output + offsets, acc, mask=mask) + + def chunked_sgmv_lora_shrink_forward( x: torch.Tensor, weights: torch.Tensor, batch_info: LoRABatchInfo, num_slices: int, + split_k: int | None = None, ) -> torch.Tensor: + if split_k is not None and split_k < 1: + raise ValueError("split_k must be positive") + # x: (s, input_dim) # weights: (num_lora, num_slices * r, input_dim) # output: (s, num_slices * r) @@ -173,7 +320,59 @@ def chunked_sgmv_lora_shrink_forward( if "num_stages" in config: extra_kwargs["num_stages"] = config["num_stages"] + # Opt-in policy measured on H200 with BF16 rank-32 adapters. Keep other + # shapes and deterministic inference on the original accumulation order. + if ( + split_k is None + and envs.SGLANG_ENABLE_LORA_CSGMV_SPLIT_K.get() + and MAX_RANK == 32 + and K in (4096, 12288) + and x.dtype == torch.bfloat16 + and (BLOCK_M, BLOCK_N, BLOCK_K) == (16, 16, 256) + ): + split_k = 16 if K == 12288 else 8 + if split_k is None or envs.SGLANG_ENABLE_DETERMINISTIC_INFERENCE.get(): + split_k = 1 + output = torch.empty((S, N), device=x.device, dtype=x.dtype) + if split_k > 1: + partials = torch.empty((split_k, S, N), device=x.device, dtype=torch.float32) + _chunked_lora_shrink_split_k_kernel[(*grid, split_k)]( + x=x, + weights=weights, + output=partials, + seg_indptr=batch_info.seg_indptr, + weight_indices=batch_info.weight_indices, + lora_ranks=batch_info.lora_ranks, + permutation=batch_info.permutation, + num_segs=segment_grid, + N=N, + K=K, + NUM_SLICES=num_slices, + BLOCK_M=BLOCK_M, + BLOCK_N=BLOCK_N, + BLOCK_K=BLOCK_K, + SPLIT_K=split_k, + TOTAL_ROWS=S, + **extra_kwargs, + ) + # Reduce only live rows/rank columns. Unused adapter slots and graph + # padding have no initialized partials and must not be read. + _reduce_shrink_split_k[grid]( + partials, + output, + batch_info.seg_indptr, + batch_info.weight_indices, + batch_info.lora_ranks, + batch_info.permutation, + N=N, + TOTAL_ROWS=S, + NUM_SLICES=num_slices, + SPLIT_K=split_k, + BLOCK_M=BLOCK_M, + BLOCK_N=BLOCK_N, + ) + return output _chunked_lora_shrink_kernel[grid]( x=x, weights=weights, diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index a6a38c32b1..77b16ef36f 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1121,6 +1121,8 @@ class Envs: # Master switch for the experimental TRT-LLM LoRA fast path; when OFF (default) every # fine-grained opt switch reads False, keeping non-experimental paths byte-identical. SGLANG_EXPERIMENTAL_LORA_OPTI = EnvBool(False) + # Small-batch CSGMV split-K changes rounding. Disabled in deterministic mode. + SGLANG_ENABLE_LORA_CSGMV_SPLIT_K = EnvBool(False) # SGLang needs to know FlashInfer NVFP4 4over6 config to compute the global scale factor. FLASHINFER_NVFP4_4OVER6 = EnvBool(False) FLASHINFER_NVFP4_4OVER6_E4M3_USE_256 = EnvBool(False) diff --git a/python/sglang/srt/lora/backend/chunked_backend.py b/python/sglang/srt/lora/backend/chunked_backend.py index 64533908ef..c89f6f7d8b 100644 --- a/python/sglang/srt/lora/backend/chunked_backend.py +++ b/python/sglang/srt/lora/backend/chunked_backend.py @@ -108,6 +108,7 @@ class ChunkedSgmvLoRABackend(BaseLoRABackend): weights=weights, batch_info=batch_info, slice_offsets=output_offset, + slice_offsets_cpu=kwargs.get("output_offset_cpu"), max_slice_size=max_slice_size, base_output=base_output, ) @@ -124,7 +125,6 @@ class ChunkedSgmvLoRABackend(BaseLoRABackend): *args, **kwargs, ) -> torch.Tensor: - # x: (s, input_dim) # qkv_lora_a: (num_lora, n_slices * r, input_dim) # qkv_lora_b: (num_lora, total_output_dim, r) @@ -141,6 +141,7 @@ class ChunkedSgmvLoRABackend(BaseLoRABackend): weights=qkv_lora_b, batch_info=self.batch_info, slice_offsets=output_offset, + slice_offsets_cpu=kwargs.get("output_offset_cpu"), max_slice_size=max_qkv_out_dim, base_output=base_output, ) @@ -156,7 +157,6 @@ class ChunkedSgmvLoRABackend(BaseLoRABackend): *args, **kwargs, ) -> torch.Tensor: - # x: (s, input_dim) # gate_up_lora_a: (num_lora, 2 * r, input_dim) # gate_up_lora_b: (num_lora, 2 * output_dim, r) @@ -175,6 +175,7 @@ class ChunkedSgmvLoRABackend(BaseLoRABackend): weights=gate_up_lora_b, batch_info=self.batch_info, slice_offsets=output_offset, + slice_offsets_cpu=kwargs.get("output_offset_cpu"), max_slice_size=output_dim, base_output=base_output, ) @@ -393,7 +394,6 @@ class ChunkedSgmvLoRABackend(BaseLoRABackend): weight_indices: list[int], batch_info: LoRABatchInfo, ) -> Tuple[Optional[LoRABatchInfo], Optional[List[LoRABatchInfo]]]: - # Precompute lm_head_batch_info for pruned lm_head LoRA pruned_lens = get_lm_head_pruned_lens(forward_batch) lm_head_batch_info = None diff --git a/test/registered/kernels/ops/lora/test_chunked_sgmv_cuda_graph.py b/test/registered/kernels/ops/lora/test_chunked_sgmv_cuda_graph.py index 7c069fa679..f189cfab5f 100644 --- a/test/registered/kernels/ops/lora/test_chunked_sgmv_cuda_graph.py +++ b/test/registered/kernels/ops/lora/test_chunked_sgmv_cuda_graph.py @@ -12,11 +12,15 @@ # limitations under the License. # ============================================================================== -"""CUDA-graph regressions for #28371: dynamic chunked-SGMV LoRA segments.""" +"""Numerical and CUDA-graph regressions for chunked-SGMV LoRA.""" + +import math +from unittest.mock import patch import pytest import torch +import sglang.kernels.ops.lora.dense.chunked_sgmv_shrink as shrink_module from sglang.kernels.ops.lora.dense.chunked_sgmv_expand import ( _chunked_lora_expand_kernel, chunked_sgmv_lora_expand_forward, @@ -31,6 +35,7 @@ from sglang.kernels.ops.lora.dense.kv_b_lora_absorbed import ( step_b_q_fwd, step_b_v_fwd, ) +from sglang.srt.environ import envs from sglang.srt.lora.backend.chunked_backend import ChunkedSgmvLoRABackend from sglang.srt.lora.utils import LoRABatchInfo from sglang.srt.model_executor.forward_batch_info import ForwardMode @@ -293,5 +298,185 @@ def test_absorbed_kv_b_replay_uses_all_current_segments(): torch.testing.assert_close(captured_v, expected_v, rtol=RTOL, atol=ATOL) +def _set_mixed_segment_state(batch_info, *, active): + _set_segment_state(batch_info, active=active) + if active: + batch_info.lora_ranks.copy_( + torch.tensor([0, 8, 16, 32, 0], dtype=torch.int32, device="cuda") + ) + batch_info.permutation.copy_( + torch.tensor([0, 4, 1, 5, 2, 6, 3, 7], dtype=torch.int32, device="cuda") + ) + batch_info.scalings.copy_(torch.tensor([1.0, -0.7, 1.0, 1.5, 0.3], device="cuda")) + + +def test_split_k_preserves_fp32_partials_and_reduction_tail(): + """Split boundaries and rank tails must match FP32 matmul before the final cast.""" + torch.manual_seed(5) + batch_info = _make_batch_info() + _set_mixed_segment_state(batch_info, active=True) + k = 12345 + inputs = torch.randn(BS, k, device="cuda", dtype=torch.bfloat16) + weights = torch.randn( + NUM_LORAS, 96, k, device="cuda", dtype=torch.bfloat16 + ) / math.sqrt(k) + # Cancellation loses the small terms if partials are rounded before reduction. + starts = [0, 1792, 3584, 5376, 7168, 8960] + small_terms = [start + 1 for start in starts] + [10753] + weights[:, 0].zero_() + inputs[:, starts + small_terms] = 1 + weights[:, 0, starts] = torch.tensor( + [256, -256, 256, -256, 256, -256], device="cuda", dtype=torch.bfloat16 + ) + weights[:, 0, small_terms] = 0.125 + split_kernel = shrink_module._chunked_lora_shrink_split_k_kernel + with ( + envs.SGLANG_ENABLE_LORA_CSGMV_SPLIT_K.override(False), + envs.SGLANG_ENABLE_DETERMINISTIC_INFERENCE.override(False), + patch.object( + shrink_module, + "get_lora_shrink_config", + return_value={"BLOCK_N": 16, "BLOCK_K": 256}, + ), + patch.object(split_kernel, "run", wraps=split_kernel.run) as launches, + ): + actual = chunked_sgmv_lora_shrink_forward( + inputs, weights, batch_info, 3, split_k=8 + ) + assert launches.call_count == 1 + for adapter, rank in enumerate((8, 16, 32), start=1): + rows = batch_info.permutation[2 * (adapter - 1) : 2 * adapter].long() + expected = ( + inputs[rows].float() @ weights[adapter, : 3 * rank].float().T + ).bfloat16() + torch.testing.assert_close( + actual[rows, : 3 * rank], expected, rtol=0.025, atol=0.04 + ) + assert torch.equal(expected[:, 0], torch.full_like(expected[:, 0], 0.875)) + assert torch.equal(actual[rows, 0], expected[:, 0]) + + +def test_compact_expand_preserves_ragged_slices_and_output_stride(): + """Empty and partial slices must preserve their neighbors and wider output padding.""" + torch.manual_seed(6) + batch_info = _make_batch_info() + _set_mixed_segment_state(batch_info, active=True) + offsets = (0, 65, 65, 194) + offsets_cpu = torch.tensor(offsets, dtype=torch.int32) + offsets_gpu = offsets_cpu.cuda() + inputs = torch.randn(BS, 96, device="cuda", dtype=torch.bfloat16) + weights = torch.randn( + NUM_LORAS, 194, 32, device="cuda", dtype=torch.bfloat16 + ) / math.sqrt(32) + initial = torch.randn(BS, 211, device="cuda", dtype=torch.bfloat16) + rectangular, compact = initial.clone(), initial.clone() + chunked_sgmv_lora_expand_forward( + inputs, weights, batch_info, offsets_gpu, 129, rectangular[:, :194] + ) + chunked_sgmv_lora_expand_forward( + inputs, weights, batch_info, offsets_gpu, 129, compact[:, :194], offsets_cpu + ) + assert torch.equal(compact, rectangular) + expected = initial.clone() + for adapter, rank in enumerate((8, 16, 32), start=1): + rows = batch_info.permutation[2 * (adapter - 1) : 2 * adapter].long() + for s, (lo, hi) in enumerate(zip(offsets, offsets[1:])): + delta = ( + inputs[rows, s * rank : (s + 1) * rank].float() + @ weights[adapter, lo:hi, :rank].float().T + ) + expected[rows, lo:hi] += (delta * batch_info.scalings[adapter]).bfloat16() + torch.testing.assert_close(compact, expected, rtol=0.03, atol=0.08) + assert torch.equal(compact[:, 194:], initial[:, 194:]) + inactive_rows = batch_info.permutation[6:].long() + assert torch.equal(compact[inactive_rows], initial[inactive_rows]) + + +def test_split_k_compact_replay_uses_current_metadata(): + """Captured split-K and compact expand must follow inactive and active adapter updates.""" + torch.manual_seed(7) + batch_info = _make_batch_info() + _set_mixed_segment_state(batch_info, active=False) + inputs = torch.randn(BS, 4096, device="cuda", dtype=torch.bfloat16) + a = torch.randn(NUM_LORAS, 96, 4096, device="cuda", dtype=torch.bfloat16) / 64 + b = torch.randn( + NUM_LORAS, 194, 32, device="cuda", dtype=torch.bfloat16 + ) / math.sqrt(32) + offsets_cpu = torch.tensor([0, 65, 65, 194], dtype=torch.int32) + offsets_gpu = offsets_cpu.cuda() + base = torch.randn(BS, 211, device="cuda", dtype=torch.bfloat16) + + def run(): + h = chunked_sgmv_lora_shrink_forward(inputs, a, batch_info, 3, split_k=8) + output = base.clone() + chunked_sgmv_lora_expand_forward( + h, b, batch_info, offsets_gpu, 129, output[:, :194], offsets_cpu + ) + return output + + split_kernel = shrink_module._chunked_lora_shrink_split_k_kernel + with ( + envs.SGLANG_ENABLE_LORA_CSGMV_SPLIT_K.override(False), + envs.SGLANG_ENABLE_DETERMINISTIC_INFERENCE.override(False), + patch.object(split_kernel, "run", wraps=split_kernel.run) as launches, + ): + graph, captured = _capture(run) + assert launches.call_count == 4 + for active in (True, False, True): + _set_mixed_segment_state(batch_info, active=active) + expected = run() + graph.replay() + torch.cuda.synchronize() + torch.testing.assert_close(captured, expected, rtol=0, atol=0) + + +def test_deterministic_mode_keeps_original_shrink_path(): + """Deterministic mode must suppress both automatic and explicit split-K launches.""" + batch_info = _make_batch_info() + _set_mixed_segment_state(batch_info, active=True) + inputs = torch.randn(BS, 4096, device="cuda", dtype=torch.bfloat16) + weights = torch.randn(NUM_LORAS, 96, 4096, device="cuda", dtype=torch.bfloat16) / 64 + split_kernel = shrink_module._chunked_lora_shrink_split_k_kernel + # Installed tuning tables may disable the policy; pin the eligibility inputs. + with ( + patch.object( + shrink_module, + "get_lora_shrink_config", + return_value={"BLOCK_N": 16, "BLOCK_K": 256}, + ), + envs.SGLANG_ENABLE_DETERMINISTIC_INFERENCE.override(False), + ): + with envs.SGLANG_ENABLE_LORA_CSGMV_SPLIT_K.override(False): + expected = chunked_sgmv_lora_shrink_forward(inputs, weights, batch_info, 3) + with ( + envs.SGLANG_ENABLE_LORA_CSGMV_SPLIT_K.override(True), + patch.object(split_kernel, "run", wraps=split_kernel.run) as launches, + ): + chunked_sgmv_lora_shrink_forward(inputs, weights, batch_info, 3) + assert launches.call_count == 1 + assert launches.call_args.kwargs["SPLIT_K"] == 8 + chunked_sgmv_lora_shrink_forward(inputs, weights, batch_info, 3, split_k=4) + assert launches.call_count == 2 + assert launches.call_args.kwargs["SPLIT_K"] == 4 + chunked_sgmv_lora_shrink_forward(inputs, weights, batch_info, 3, split_k=1) + assert launches.call_count == 2 + for deterministic in (False, True): + with ( + envs.SGLANG_ENABLE_DETERMINISTIC_INFERENCE.override(deterministic), + pytest.raises(ValueError, match="split_k must be positive"), + ): + chunked_sgmv_lora_shrink_forward( + inputs, weights, batch_info, 3, split_k=0 + ) + with envs.SGLANG_ENABLE_DETERMINISTIC_INFERENCE.override(True): + actual = chunked_sgmv_lora_shrink_forward( + inputs, weights, batch_info, 3, split_k=8 + ) + assert launches.call_count == 2 + for adapter, rank in enumerate((8, 16, 32), start=1): + rows = batch_info.permutation[2 * (adapter - 1) : 2 * adapter].long() + assert torch.equal(actual[rows, : 3 * rank], expected[rows, : 3 * rank]) + + if __name__ == "__main__": raise SystemExit(pytest.main([__file__, "-v"]))