diff --git a/python/sglang/kernels/ops/attention/dsv4/request_window_copy.py b/python/sglang/kernels/ops/attention/dsv4/request_window_copy.py new file mode 100644 index 0000000000..9388e9980d --- /dev/null +++ b/python/sglang/kernels/ops/attention/dsv4/request_window_copy.py @@ -0,0 +1,65 @@ +"""Copy packed request-window KV rows without materializing byte index grids.""" + +import triton +import triton.language as tl + + +@triton.jit +def _copy_packed_rows( + src, + dst, + src_loc, + dst_loc, + SRC_STRIDE: tl.constexpr, + DST_STRIDE: tl.constexpr, + SRC_LOC_STRIDE: tl.constexpr, + DST_LOC_STRIDE: tl.constexpr, + PAGE: tl.constexpr, + DATA: tl.constexpr, + SCALE: tl.constexpr, + BLOCK: tl.constexpr, +): + row = tl.program_id(0) + source = tl.load(src_loc + row * SRC_LOC_STRIDE).to(tl.int64) + target = tl.load(dst_loc + row * DST_LOC_STRIDE).to(tl.int64) + col = tl.arange(0, BLOCK) + # Each page contains all data rows, followed by all scale rows. Its + # physical stride may also contain alignment padding, which is untouched. + source_offset = tl.where( + col < DATA, + (source % PAGE) * DATA + col, + PAGE * DATA + (source % PAGE) * SCALE + col - DATA, + ) + target_offset = tl.where( + col < DATA, + (target % PAGE) * DATA + col, + PAGE * DATA + (target % PAGE) * SCALE + col - DATA, + ) + value = tl.load( + src + source // PAGE * SRC_STRIDE + source_offset, + mask=col < DATA + SCALE, + other=0, + ) + tl.store( + dst + target // PAGE * DST_STRIDE + target_offset, + value, + mask=col < DATA + SCALE, + ) + + +def copy_packed_rows(src, dst, src_loc, dst_loc, *, page_size, layout): + _copy_packed_rows[(src_loc.numel(),)]( + src, + dst, + src_loc, + dst_loc, + src.stride(0), + dst.stride(0), + src_loc.stride(0), + dst_loc.stride(0), + page_size, + layout.data_bytes, + layout.scale_bytes, + triton.next_power_of_2(layout.bytes_per_token), + num_warps=4, + ) diff --git a/python/sglang/srt/mem_cache/dsv41_request_window.py b/python/sglang/srt/mem_cache/dsv41_request_window.py index f336d6ae79..786a7242ef 100644 --- a/python/sglang/srt/mem_cache/dsv41_request_window.py +++ b/python/sglang/srt/mem_cache/dsv41_request_window.py @@ -4,6 +4,7 @@ import msgspec import torch from sglang.kernels.ops.attention.dsv4.kv_layout import KVLayout +from sglang.kernels.ops.attention.dsv4.request_window_copy import copy_packed_rows from sglang.srt.model_executor.runner_utils.capture_mode import get_is_capture_mode @@ -126,20 +127,7 @@ def copy_packed_tokens(src, dst, src_loc, dst_loc, *, page_size, layout=KVLayout """Move tokens between paged buffers of ``layout``: a data row and a scale row.""" if not src_loc.numel(): return - src_loc, dst_loc = src_loc.long(), dst_loc.long() - for width, base in ( - (layout.data_bytes, 0), - (layout.scale_bytes, page_size * layout.data_bytes), - ): - cols = torch.arange(width, device=src.device) - values = src[ - src_loc[:, None] // page_size, - base + (src_loc[:, None] % page_size) * width + cols, - ] - dst[ - dst_loc[:, None] // page_size, - base + (dst_loc[:, None] % page_size) * width + cols, - ] = values + copy_packed_rows(src, dst, src_loc, dst_loc, page_size=page_size, layout=layout) def _capturing() -> bool: diff --git a/test/registered/kernels/ops/attention/test_dsv41_request_window_copy.py b/test/registered/kernels/ops/attention/test_dsv41_request_window_copy.py new file mode 100644 index 0000000000..4cbda0d61a --- /dev/null +++ b/test/registered/kernels/ops/attention/test_dsv41_request_window_copy.py @@ -0,0 +1,93 @@ +"""Request-window copies preserve data, scales, padding and graph addresses.""" + +import unittest + +import torch + +from sglang.kernels.ops.attention.dsv4.kv_layout import KVLayout +from sglang.srt.mem_cache.dsv41_request_window import copy_packed_tokens +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=20, stage="base-b-kernel-unit", runner_config="1-gpu-large") + + +def reference(src, dst, source, target, page, layout): + # Independent scalar row slicing also checks every untouched padding byte. + result = dst.clone() + for s, d in zip(source.tolist(), target.tolist()): + for width, base in ( + (layout.data_bytes, 0), + (layout.scale_bytes, page * layout.data_bytes), + ): + result[ + d // page, base + d % page * width : base + (d % page + 1) * width + ] = src[s // page, base + s % page * width : base + (s % page + 1) * width] + return result + + +@unittest.skipUnless(torch.cuda.is_available(), "CUDA required") +class TestRequestWindowCopy(CustomTestCase): + def test_layouts_pages_and_strides(self): + for layout in KVLayout: + for page in (16, 64, 256): + for count in (0, 1, 17, 129): + with self.subTest(layout=layout, page=page, count=count): + width = layout.page_bytes(page) + src = torch.randint( + 0, 256, (12, width + 64), dtype=torch.uint8 + )[:, :width] + dst = torch.full((12, width + 128), 193, dtype=torch.uint8)[ + :, :width + ] + source = (torch.arange(count) * 7 + page - 1) % (12 * page) + target = torch.randperm(12 * page)[:count] + expected = reference(src, dst, source, target, page, layout) + sg = torch.empty( + (12, width + 64), dtype=torch.uint8, device="cuda" + )[:, :width] + dg = torch.empty( + (12, width + 128), dtype=torch.uint8, device="cuda" + )[:, :width] + sg.copy_(src) + dg.copy_(dst) + # Non-contiguous location arrays, mixed integer widths. + sl = torch.zeros(count * 2, dtype=torch.int32, device="cuda")[ + ::2 + ] + dl = torch.zeros(count * 2, dtype=torch.int64, device="cuda")[ + ::2 + ] + sl.copy_(source) + dl.copy_(target) + copy_packed_tokens( + sg, dg, sl, dl, page_size=page, layout=layout + ) + torch.testing.assert_close(dg.cpu(), expected, rtol=0, atol=0) + + def test_graph_replay_refreshes_locations(self): + page, layout = 64, KVLayout.V41 + src = torch.randint( + 0, 256, (8, layout.page_bytes(page)), dtype=torch.uint8, device="cuda" + ) + dst = torch.full_like(src, 193) + source = torch.arange(128, dtype=torch.int64, device="cuda") + target = source.clone() + copy_packed_tokens(src, dst, source, target, page_size=page, layout=layout) + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + copy_packed_tokens(src, dst, source, target, page_size=page, layout=layout) + for offset in (63, 127, 256): + source.copy_(torch.arange(128, device="cuda") + offset) + target.copy_(torch.arange(128, device="cuda") + 128) + dst.fill_(193) + expected = reference( + src.cpu(), dst.cpu(), source.cpu(), target.cpu(), page, layout + ) + graph.replay() + torch.testing.assert_close(dst.cpu(), expected, rtol=0, atol=0) + + +if __name__ == "__main__": + unittest.main()