[qwen 3.8 next] Fuse small CUDA graph input buffer copies (#41166)
This commit is contained in:
@@ -0,0 +1,136 @@
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
|
||||
@triton.jit(
|
||||
do_not_specialize=["sizes", "cols", "source_strides", "dest_strides"],
|
||||
do_not_specialize_on_alignment=["sizes", "cols", "source_strides", "dest_strides"],
|
||||
)
|
||||
def _small_copy_kernel(
|
||||
sources,
|
||||
destinations,
|
||||
sizes,
|
||||
cols,
|
||||
source_strides,
|
||||
dest_strides,
|
||||
BIT_WIDTHS: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
field = tl.program_id(0)
|
||||
offsets = (tl.program_id(1) * BLOCK + tl.arange(0, BLOCK)).to(tl.int64)
|
||||
for i in tl.static_range(len(BIT_WIDTHS)):
|
||||
if field == i:
|
||||
src_offset = (
|
||||
offsets // cols[i] * source_strides[i][0]
|
||||
+ offsets % cols[i] * source_strides[i][1]
|
||||
)
|
||||
dst_offset = (
|
||||
offsets // cols[i] * dest_strides[i][0]
|
||||
+ offsets % cols[i] * dest_strides[i][1]
|
||||
)
|
||||
src = sources[i]
|
||||
dst = destinations[i]
|
||||
if BIT_WIDTHS[i] == 8:
|
||||
src = src.to(tl.pointer_type(tl.uint8))
|
||||
dst = dst.to(tl.pointer_type(tl.uint8))
|
||||
elif BIT_WIDTHS[i] == 16:
|
||||
src = src.to(tl.pointer_type(tl.uint16))
|
||||
dst = dst.to(tl.pointer_type(tl.uint16))
|
||||
elif BIT_WIDTHS[i] == 32:
|
||||
src = src.to(tl.pointer_type(tl.uint32))
|
||||
dst = dst.to(tl.pointer_type(tl.uint32))
|
||||
elif BIT_WIDTHS[i] == 64:
|
||||
src = src.to(tl.pointer_type(tl.uint64))
|
||||
dst = dst.to(tl.pointer_type(tl.uint64))
|
||||
values = tl.load(src + src_offset, offsets < sizes[i], other=0)
|
||||
tl.store(dst + dst_offset, values, offsets < sizes[i])
|
||||
|
||||
|
||||
def try_small_copy(dsts, srcs):
|
||||
if len(dsts) < 2 or len(dsts) != len(srcs):
|
||||
return False
|
||||
supported = (
|
||||
torch.bool,
|
||||
torch.uint8,
|
||||
torch.int8,
|
||||
torch.int16,
|
||||
torch.int32,
|
||||
torch.int64,
|
||||
torch.float16,
|
||||
torch.bfloat16,
|
||||
torch.float32,
|
||||
torch.float64,
|
||||
)
|
||||
device = dsts[0].device
|
||||
if device.type != "cuda":
|
||||
return False
|
||||
sizes, cols, src_strides, dst_strides, bit_widths = [], [], [], [], []
|
||||
src_ranges, dst_ranges = [], []
|
||||
for dst, src in zip(dsts, srcs):
|
||||
if (
|
||||
dst.dtype not in supported
|
||||
or src.dtype not in supported
|
||||
or dst.device != device
|
||||
or src.device != device
|
||||
or dst.shape != src.shape
|
||||
or dst.ndim not in (1, 2)
|
||||
or dst.numel() > 8192
|
||||
or dst.requires_grad
|
||||
or src.requires_grad
|
||||
or src.is_neg()
|
||||
or dst.is_neg()
|
||||
):
|
||||
return False
|
||||
same_dtype = dst.dtype == src.dtype
|
||||
if not same_dtype and not (
|
||||
dst.dtype in (torch.int32, torch.int64)
|
||||
and src.dtype in (torch.int32, torch.int64)
|
||||
):
|
||||
return False
|
||||
n = dst.numel()
|
||||
c = max(dst.shape[-1], 1)
|
||||
ds = dst.stride() if dst.ndim == 2 else (0, dst.stride(0))
|
||||
ss = src.stride() if src.ndim == 2 else (0, src.stride(0))
|
||||
if ds[1] <= 0 or (dst.ndim == 2 and ds[0] < c * ds[1]):
|
||||
return False
|
||||
sizes.append(n)
|
||||
cols.append(c)
|
||||
src_strides.append(ss)
|
||||
dst_strides.append(ds)
|
||||
bit_widths.append(dst.element_size() * 8 if same_dtype else 0)
|
||||
for tensor, ranges in ((src, src_ranges), (dst, dst_ranges)):
|
||||
first = tensor.data_ptr()
|
||||
extent = (
|
||||
(sum((d - 1) * s for d, s in zip(tensor.shape, tensor.stride())) + 1)
|
||||
* tensor.element_size()
|
||||
if n
|
||||
else 0
|
||||
)
|
||||
ranges.append((first, first + extent))
|
||||
for i, (first, last) in enumerate(dst_ranges):
|
||||
for j, (other_first, other_last) in enumerate(src_ranges):
|
||||
if first < other_last and other_first < last:
|
||||
if (
|
||||
i != j
|
||||
or dsts[i].data_ptr() != srcs[i].data_ptr()
|
||||
or dsts[i].stride() != srcs[i].stride()
|
||||
or dsts[i].dtype != srcs[i].dtype
|
||||
):
|
||||
return False
|
||||
for other_first, other_last in dst_ranges[:i]:
|
||||
if first < other_last and other_first < last:
|
||||
return False
|
||||
largest = max(sizes)
|
||||
if largest:
|
||||
_small_copy_kernel[(len(dsts), triton.cdiv(largest, 256))](
|
||||
tuple(srcs),
|
||||
tuple(dsts),
|
||||
tuple(sizes),
|
||||
tuple(cols),
|
||||
tuple(src_strides),
|
||||
tuple(dst_strides),
|
||||
tuple(bit_widths),
|
||||
BLOCK=256,
|
||||
)
|
||||
return True
|
||||
@@ -57,6 +57,12 @@ def _grouped_foreach_copy_(dsts: List[torch.Tensor], srcs: List[torch.Tensor]) -
|
||||
for dst, src in zip(group_dsts, group_srcs):
|
||||
dst.copy_(src)
|
||||
|
||||
if dsts and dsts[0].is_cuda:
|
||||
from sglang.kernels.ops.memory.small_copy import try_small_copy
|
||||
|
||||
if try_small_copy(dsts, srcs):
|
||||
return
|
||||
|
||||
groups: Dict[Tuple[torch.dtype, torch.dtype], Tuple[List, List]] = {}
|
||||
for dst, src in zip(dsts, srcs):
|
||||
key = (dst.dtype, src.dtype)
|
||||
|
||||
@@ -47,6 +47,12 @@ def _grouped_foreach_copy_(dsts: List[torch.Tensor], srcs: List[torch.Tensor]) -
|
||||
for dst, src in zip(dsts, srcs):
|
||||
dst.copy_(src)
|
||||
|
||||
if dsts and dsts[0].is_cuda:
|
||||
from sglang.kernels.ops.memory.small_copy import try_small_copy
|
||||
|
||||
if try_small_copy(dsts, srcs):
|
||||
return
|
||||
|
||||
groups: Dict[Tuple[torch.dtype, torch.dtype], Tuple[List, List]] = {}
|
||||
for dst, src in zip(dsts, srcs):
|
||||
key = (dst.dtype, src.dtype)
|
||||
|
||||
Reference in New Issue
Block a user