[DSv4.1] Fuse request-window KV copies (#44555)

This commit is contained in:
Yuwei An
2026-10-15 14:11:51 -07:00
committed by GitHub
parent 877be2974a
commit 91f6ef2322
3 changed files with 160 additions and 14 deletions
@@ -0,0 +1,65 @@
"""Copy packed request-window KV rows without materializing byte index grids."""
import triton
import triton.language as tl
@triton.jit
def _copy_packed_rows(
src,
dst,
src_loc,
dst_loc,
SRC_STRIDE: tl.constexpr,
DST_STRIDE: tl.constexpr,
SRC_LOC_STRIDE: tl.constexpr,
DST_LOC_STRIDE: tl.constexpr,
PAGE: tl.constexpr,
DATA: tl.constexpr,
SCALE: tl.constexpr,
BLOCK: tl.constexpr,
):
row = tl.program_id(0)
source = tl.load(src_loc + row * SRC_LOC_STRIDE).to(tl.int64)
target = tl.load(dst_loc + row * DST_LOC_STRIDE).to(tl.int64)
col = tl.arange(0, BLOCK)
# Each page contains all data rows, followed by all scale rows. Its
# physical stride may also contain alignment padding, which is untouched.
source_offset = tl.where(
col < DATA,
(source % PAGE) * DATA + col,
PAGE * DATA + (source % PAGE) * SCALE + col - DATA,
)
target_offset = tl.where(
col < DATA,
(target % PAGE) * DATA + col,
PAGE * DATA + (target % PAGE) * SCALE + col - DATA,
)
value = tl.load(
src + source // PAGE * SRC_STRIDE + source_offset,
mask=col < DATA + SCALE,
other=0,
)
tl.store(
dst + target // PAGE * DST_STRIDE + target_offset,
value,
mask=col < DATA + SCALE,
)
def copy_packed_rows(src, dst, src_loc, dst_loc, *, page_size, layout):
_copy_packed_rows[(src_loc.numel(),)](
src,
dst,
src_loc,
dst_loc,
src.stride(0),
dst.stride(0),
src_loc.stride(0),
dst_loc.stride(0),
page_size,
layout.data_bytes,
layout.scale_bytes,
triton.next_power_of_2(layout.bytes_per_token),
num_warps=4,
)
@@ -4,6 +4,7 @@ import msgspec
import torch
from sglang.kernels.ops.attention.dsv4.kv_layout import KVLayout
from sglang.kernels.ops.attention.dsv4.request_window_copy import copy_packed_rows
from sglang.srt.model_executor.runner_utils.capture_mode import get_is_capture_mode
@@ -126,20 +127,7 @@ def copy_packed_tokens(src, dst, src_loc, dst_loc, *, page_size, layout=KVLayout
"""Move tokens between paged buffers of ``layout``: a data row and a scale row."""
if not src_loc.numel():
return
src_loc, dst_loc = src_loc.long(), dst_loc.long()
for width, base in (
(layout.data_bytes, 0),
(layout.scale_bytes, page_size * layout.data_bytes),
):
cols = torch.arange(width, device=src.device)
values = src[
src_loc[:, None] // page_size,
base + (src_loc[:, None] % page_size) * width + cols,
]
dst[
dst_loc[:, None] // page_size,
base + (dst_loc[:, None] % page_size) * width + cols,
] = values
copy_packed_rows(src, dst, src_loc, dst_loc, page_size=page_size, layout=layout)
def _capturing() -> bool:
@@ -0,0 +1,93 @@
"""Request-window copies preserve data, scales, padding and graph addresses."""
import unittest
import torch
from sglang.kernels.ops.attention.dsv4.kv_layout import KVLayout
from sglang.srt.mem_cache.dsv41_request_window import copy_packed_tokens
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=20, stage="base-b-kernel-unit", runner_config="1-gpu-large")
def reference(src, dst, source, target, page, layout):
# Independent scalar row slicing also checks every untouched padding byte.
result = dst.clone()
for s, d in zip(source.tolist(), target.tolist()):
for width, base in (
(layout.data_bytes, 0),
(layout.scale_bytes, page * layout.data_bytes),
):
result[
d // page, base + d % page * width : base + (d % page + 1) * width
] = src[s // page, base + s % page * width : base + (s % page + 1) * width]
return result
@unittest.skipUnless(torch.cuda.is_available(), "CUDA required")
class TestRequestWindowCopy(CustomTestCase):
def test_layouts_pages_and_strides(self):
for layout in KVLayout:
for page in (16, 64, 256):
for count in (0, 1, 17, 129):
with self.subTest(layout=layout, page=page, count=count):
width = layout.page_bytes(page)
src = torch.randint(
0, 256, (12, width + 64), dtype=torch.uint8
)[:, :width]
dst = torch.full((12, width + 128), 193, dtype=torch.uint8)[
:, :width
]
source = (torch.arange(count) * 7 + page - 1) % (12 * page)
target = torch.randperm(12 * page)[:count]
expected = reference(src, dst, source, target, page, layout)
sg = torch.empty(
(12, width + 64), dtype=torch.uint8, device="cuda"
)[:, :width]
dg = torch.empty(
(12, width + 128), dtype=torch.uint8, device="cuda"
)[:, :width]
sg.copy_(src)
dg.copy_(dst)
# Non-contiguous location arrays, mixed integer widths.
sl = torch.zeros(count * 2, dtype=torch.int32, device="cuda")[
::2
]
dl = torch.zeros(count * 2, dtype=torch.int64, device="cuda")[
::2
]
sl.copy_(source)
dl.copy_(target)
copy_packed_tokens(
sg, dg, sl, dl, page_size=page, layout=layout
)
torch.testing.assert_close(dg.cpu(), expected, rtol=0, atol=0)
def test_graph_replay_refreshes_locations(self):
page, layout = 64, KVLayout.V41
src = torch.randint(
0, 256, (8, layout.page_bytes(page)), dtype=torch.uint8, device="cuda"
)
dst = torch.full_like(src, 193)
source = torch.arange(128, dtype=torch.int64, device="cuda")
target = source.clone()
copy_packed_tokens(src, dst, source, target, page_size=page, layout=layout)
torch.cuda.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
copy_packed_tokens(src, dst, source, target, page_size=page, layout=layout)
for offset in (63, 127, 256):
source.copy_(torch.arange(128, device="cuda") + offset)
target.copy_(torch.arange(128, device="cuda") + 128)
dst.fill_(193)
expected = reference(
src.cpu(), dst.cpu(), source.cpu(), target.cpu(), page, layout
)
graph.replay()
torch.testing.assert_close(dst.cpu(), expected, rtol=0, atol=0)
if __name__ == "__main__":
unittest.main()