[DSv4.1] Fuse request-window KV copies (#44555)

This commit is contained in:
Yuwei An
2026-10-15 14:11:51 -07:00
committed by GitHub
parent 877be2974a
commit 91f6ef2322
3 changed files with 160 additions and 14 deletions
@@ -0,0 +1,65 @@
"""Copy packed request-window KV rows without materializing byte index grids."""
import triton
import triton.language as tl
@triton.jit
def _copy_packed_rows(
src,
dst,
src_loc,
dst_loc,
SRC_STRIDE: tl.constexpr,
DST_STRIDE: tl.constexpr,
SRC_LOC_STRIDE: tl.constexpr,
DST_LOC_STRIDE: tl.constexpr,
PAGE: tl.constexpr,
DATA: tl.constexpr,
SCALE: tl.constexpr,
BLOCK: tl.constexpr,
):
row = tl.program_id(0)
source = tl.load(src_loc + row * SRC_LOC_STRIDE).to(tl.int64)
target = tl.load(dst_loc + row * DST_LOC_STRIDE).to(tl.int64)
col = tl.arange(0, BLOCK)
# Each page contains all data rows, followed by all scale rows. Its
# physical stride may also contain alignment padding, which is untouched.
source_offset = tl.where(
col < DATA,
(source % PAGE) * DATA + col,
PAGE * DATA + (source % PAGE) * SCALE + col - DATA,
)
target_offset = tl.where(
col < DATA,
(target % PAGE) * DATA + col,
PAGE * DATA + (target % PAGE) * SCALE + col - DATA,
)
value = tl.load(
src + source // PAGE * SRC_STRIDE + source_offset,
mask=col < DATA + SCALE,
other=0,
)
tl.store(
dst + target // PAGE * DST_STRIDE + target_offset,
value,
mask=col < DATA + SCALE,
)
def copy_packed_rows(src, dst, src_loc, dst_loc, *, page_size, layout):
_copy_packed_rows[(src_loc.numel(),)](
src,
dst,
src_loc,
dst_loc,
src.stride(0),
dst.stride(0),
src_loc.stride(0),
dst_loc.stride(0),
page_size,
layout.data_bytes,
layout.scale_bytes,
triton.next_power_of_2(layout.bytes_per_token),
num_warps=4,
)
@@ -4,6 +4,7 @@ import msgspec
import torch
from sglang.kernels.ops.attention.dsv4.kv_layout import KVLayout
from sglang.kernels.ops.attention.dsv4.request_window_copy import copy_packed_rows
from sglang.srt.model_executor.runner_utils.capture_mode import get_is_capture_mode
@@ -126,20 +127,7 @@ def copy_packed_tokens(src, dst, src_loc, dst_loc, *, page_size, layout=KVLayout
"""Move tokens between paged buffers of ``layout``: a data row and a scale row."""
if not src_loc.numel():
return
src_loc, dst_loc = src_loc.long(), dst_loc.long()
for width, base in (
(layout.data_bytes, 0),
(layout.scale_bytes, page_size * layout.data_bytes),
):
cols = torch.arange(width, device=src.device)
values = src[
src_loc[:, None] // page_size,
base + (src_loc[:, None] % page_size) * width + cols,
]
dst[
dst_loc[:, None] // page_size,
base + (dst_loc[:, None] % page_size) * width + cols,
] = values
copy_packed_rows(src, dst, src_loc, dst_loc, page_size=page_size, layout=layout)
def _capturing() -> bool: