[HiCache] Add the page-unified KV load-back JIT kernel (#39726)
Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: huangtingwei <141888744+huangtingwei9988@users.noreply.github.com> Co-authored-by: Zhangheng <hzh0425@apache.org>
This commit is contained in:
co-authored by
Claude Opus 5
huangtingwei
Zhangheng
parent
d10a09ecf3
commit
e581520c67
@@ -218,6 +218,21 @@ struct HicacheKernelParams {
|
||||
uint32_t num_layers = 0; // only used in all_layer transfer
|
||||
};
|
||||
|
||||
/// \brief Operands of \ref hicache_transfer_per_layer_page_unified.
|
||||
struct HicachePageUnifiedKernelParams {
|
||||
void* __restrict__ k_cache_dst;
|
||||
void* __restrict__ v_cache_dst;
|
||||
const void* __restrict__ src;
|
||||
const void* __restrict__ src_indices;
|
||||
const void* __restrict__ dst_indices;
|
||||
uint64_t total_vecs;
|
||||
int64_t num_items;
|
||||
int64_t num_groups;
|
||||
int64_t num_layers;
|
||||
int64_t page_size;
|
||||
int64_t layer_id;
|
||||
};
|
||||
|
||||
template <
|
||||
typename T,
|
||||
int64_t kElementSize,
|
||||
@@ -313,6 +328,67 @@ SGL_HICACHE_KERNEL void hicache_transfer_all_layer(const __grid_constant__ Hicac
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief Read one layer out of page-unified host pages into a per-layer device buffer.
|
||||
*
|
||||
* The source page block is (head_group, layer, 2, page_size, head_in_group, dim)
|
||||
* with K=0 and V=1 -- the exact byte order the page-unified write-back leaves
|
||||
* behind, so a page round-trips unchanged. MLA drops the head-group, K/V and
|
||||
* head axes, leaving (layer, page_size, dim).
|
||||
*
|
||||
* The destination is the device pool's own per-layer rows, (token, head, dim),
|
||||
* where head group \c g owns the contiguous run [g * kElementSize, (g+1) * kElementSize).
|
||||
* That run is exactly one source cell, which is why the permutation costs
|
||||
* nothing: it only reorders addresses the copy already computes per group.
|
||||
*
|
||||
* \c kElementSize is one head group's token row here, not the whole row the
|
||||
* other kernels in this file copy: one source cell is what a thread block
|
||||
* walks, and the device row is \c num_groups of them side by side.
|
||||
*
|
||||
* Each thread moves 16 bytes. The linear index runs dim-fastest, then item, so
|
||||
* lanes of a warp read consecutive host bytes for tokens of one (head group,
|
||||
* layer, K/V) cell -- the reads cross PCIe, so their coalescing is what sets
|
||||
* the achieved bandwidth.
|
||||
*/
|
||||
template <typename T, int64_t kElementSize, uint32_t kBlockSize, bool kIsMLA = false>
|
||||
SGL_HICACHE_KERNEL void
|
||||
hicache_transfer_per_layer_page_unified(const __grid_constant__ HicachePageUnifiedKernelParams params) {
|
||||
using namespace device;
|
||||
static_assert(kElementSize > 0 && kElementSize % 16 == 0);
|
||||
constexpr int64_t kGroupVecs = kElementSize / 16;
|
||||
constexpr int64_t kComponents = kIsMLA ? 1 : 2;
|
||||
const int64_t num_groups = kIsMLA ? 1 : params.num_groups;
|
||||
|
||||
const auto tid = static_cast<uint64_t>(blockIdx.x) * blockDim.x + threadIdx.x;
|
||||
const auto stride = static_cast<uint64_t>(gridDim.x) * blockDim.x;
|
||||
for (uint64_t i = tid; i < params.total_vecs; i += stride) {
|
||||
const auto vec = static_cast<int64_t>(i % kGroupVecs);
|
||||
auto remaining = static_cast<int64_t>(i / kGroupVecs);
|
||||
const auto item = remaining % params.num_items;
|
||||
remaining /= params.num_items;
|
||||
const auto kv = remaining % kComponents;
|
||||
const auto group = remaining / kComponents;
|
||||
|
||||
const auto src_token = static_cast<int64_t>(static_cast<const T*>(params.src_indices)[item]);
|
||||
const auto dst_token = static_cast<int64_t>(static_cast<const T*>(params.dst_indices)[item]);
|
||||
const auto page = src_token / params.page_size;
|
||||
const auto token_in_page = src_token % params.page_size;
|
||||
|
||||
const auto src_offset =
|
||||
(((((page * num_groups + group) * params.num_layers + params.layer_id) * kComponents + kv) * params.page_size +
|
||||
token_in_page) *
|
||||
kElementSize) +
|
||||
vec * 16;
|
||||
const auto dst_offset = (dst_token * num_groups + group) * kElementSize + vec * 16;
|
||||
const auto dst_base = kv == 0 ? params.k_cache_dst : params.v_cache_dst;
|
||||
|
||||
const auto src = pointer::offset(static_cast<const void*>(params.src), src_offset);
|
||||
const auto dst = pointer::offset(static_cast<void*>(dst_base), dst_offset);
|
||||
const auto value = details::load_nc(reinterpret_cast<const uint4*>(src));
|
||||
details::store_nc(reinterpret_cast<uint4*>(dst), value);
|
||||
}
|
||||
}
|
||||
|
||||
template <int64_t kElementSize, uint32_t kUnroll, uint32_t kBlockQuota, uint32_t kBlockSize>
|
||||
struct HiCacheKernel {
|
||||
template <typename T>
|
||||
@@ -325,6 +401,9 @@ struct HiCacheKernel {
|
||||
template <typename T>
|
||||
static constexpr auto kernel_all_mla =
|
||||
hicache_transfer_all_layer<T, kElementSize, kUnroll, kBlockQuota, kBlockSize, true>;
|
||||
template <typename T, bool kIsMLA>
|
||||
static constexpr auto kernel_one_page_unified =
|
||||
hicache_transfer_per_layer_page_unified<T, kElementSize, kBlockSize, kIsMLA>;
|
||||
|
||||
static void run_one(
|
||||
const tvm::ffi::TensorView k_cache_dst,
|
||||
@@ -565,6 +644,101 @@ struct HiCacheKernel {
|
||||
const auto kernel = use_int32 ? kernel_all_mla<int32_t> : kernel_all_mla<int64_t>;
|
||||
LaunchKernel(num_blocks, kBlockSize, device)(kernel, params);
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief Load-back half of the page-unified HiCache layout: host pages -> device pool.
|
||||
*
|
||||
* The mirror of the page-unified write-back. Where write-back stages a whole
|
||||
* page on the device and hands the bulk transfer to the copy engine, load-back
|
||||
* is driven per layer by the SMs reading the pinned host mapping directly: the
|
||||
* transfer engine walks layers and unblocks the forward pass one layer at a
|
||||
* time, so staging a whole page per layer would move \c num_layers times the
|
||||
* bytes the layer actually needs.
|
||||
*
|
||||
* \tparam kIsMLA Whether the pool has a single latent component and no head axis
|
||||
* \param k_cache_dst One layer of the device K pool, flattened to (token, head * dim)
|
||||
* \param v_cache_dst One layer of the device V pool; ignored when \c kIsMLA
|
||||
* \param src Host pool flattened to (page, page elements), page-unified order
|
||||
* \param src_indices Host-pool TOKEN indices, on the destination device
|
||||
* \param dst_indices Device-pool token indices, same length and order
|
||||
* \param layer_id Layer to read, indexing the host pool's layer axis
|
||||
* \param num_layers Layer extent of the host pool's layer axis
|
||||
* \param num_groups Head groups the page block is cut into; 1 when \c kIsMLA
|
||||
* \param page_size Tokens per page
|
||||
*/
|
||||
template <bool kIsMLA>
|
||||
static void run_one_page_unified(
|
||||
const tvm::ffi::TensorView k_cache_dst,
|
||||
const tvm::ffi::TensorView v_cache_dst,
|
||||
const tvm::ffi::TensorView src,
|
||||
const tvm::ffi::TensorView src_indices,
|
||||
const tvm::ffi::TensorView dst_indices,
|
||||
const int64_t layer_id,
|
||||
const int64_t num_layers,
|
||||
const int64_t num_groups,
|
||||
const int64_t page_size) {
|
||||
using namespace host;
|
||||
|
||||
auto B = SymbolicSize{"page elements"};
|
||||
auto C = SymbolicSize{"layer elements per token"};
|
||||
auto N = SymbolicSize{"indices length"};
|
||||
auto cache_dtype = SymbolicDType{};
|
||||
auto indices_dtype = SymbolicDType{};
|
||||
auto device_ = SymbolicDevice{};
|
||||
|
||||
// The device pool binds the launch device: the source is host memory read
|
||||
// over its pinned mapping, so it cannot name a stream.
|
||||
TensorMatcher({-1, C}).with_dtype(cache_dtype).with_device<kDLGPU>(device_).verify(k_cache_dst);
|
||||
if constexpr (!kIsMLA) {
|
||||
TensorMatcher({-1, C}).with_dtype(cache_dtype).with_device<kDLGPU>(device_).verify(v_cache_dst);
|
||||
}
|
||||
TensorMatcher({-1, B}).with_dtype(cache_dtype).with_device<kDLCPU, kDLGPUHost, kDLGPU>().verify(src);
|
||||
TensorMatcher({N})
|
||||
.with_dtype<int32_t, int64_t>(indices_dtype)
|
||||
.with_device<kDLGPU>(device_)
|
||||
.verify(src_indices)
|
||||
.verify(dst_indices);
|
||||
|
||||
constexpr int64_t kComponents = kIsMLA ? 1 : 2;
|
||||
RuntimeCheck(num_layers > 0 && num_groups > 0 && page_size > 0, "Page-unified load-back: invalid dimensions");
|
||||
RuntimeCheck(layer_id >= 0 && layer_id < num_layers, "Page-unified load-back: layer id out of range");
|
||||
RuntimeCheck(!kIsMLA || num_groups == 1, "Page-unified MLA load-back: expected one latent group");
|
||||
const auto dtype_size = dtype_bytes(cache_dtype.unwrap());
|
||||
RuntimeCheck(
|
||||
B.unwrap() * dtype_size == num_groups * num_layers * kComponents * page_size * kElementSize,
|
||||
"Page-unified load-back: page byte size mismatch");
|
||||
RuntimeCheck(
|
||||
C.unwrap() * dtype_size == num_groups * kElementSize, "Page-unified load-back: device row byte size mismatch");
|
||||
RuntimeCheck(
|
||||
reinterpret_cast<uintptr_t>(src.data_ptr()) % 16 == 0 &&
|
||||
reinterpret_cast<uintptr_t>(k_cache_dst.data_ptr()) % 16 == 0 &&
|
||||
(kIsMLA || reinterpret_cast<uintptr_t>(v_cache_dst.data_ptr()) % 16 == 0),
|
||||
"Page-unified load-back: buffers must be 16-byte aligned");
|
||||
if (N.unwrap() == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const auto params = HicachePageUnifiedKernelParams{
|
||||
.k_cache_dst = k_cache_dst.data_ptr(),
|
||||
.v_cache_dst = kIsMLA ? nullptr : v_cache_dst.data_ptr(),
|
||||
.src = runtime::get_device_accessible_ptr(src),
|
||||
.src_indices = src_indices.data_ptr(),
|
||||
.dst_indices = dst_indices.data_ptr(),
|
||||
.total_vecs = static_cast<uint64_t>(N.unwrap()) * num_groups * kComponents * (kElementSize / 16),
|
||||
.num_items = N.unwrap(),
|
||||
.num_groups = num_groups,
|
||||
.num_layers = num_layers,
|
||||
.page_size = page_size,
|
||||
.layer_id = layer_id,
|
||||
};
|
||||
// A quota-capped persistent grid, as in the other entry points: the load
|
||||
// overlaps the forward pass, so more blocks buy bandwidth by taking SMs
|
||||
// from it.
|
||||
const auto num_blocks = std::min(div_ceil(params.total_vecs, uint64_t{kBlockSize}), uint64_t{kBlockQuota});
|
||||
const auto use_int32 = indices_dtype.unwrap().bits == 32;
|
||||
const auto kernel = use_int32 ? kernel_one_page_unified<int32_t, kIsMLA> : kernel_one_page_unified<int64_t, kIsMLA>;
|
||||
LaunchKernel(static_cast<uint32_t>(num_blocks), kBlockSize, device_.unwrap())(kernel, params);
|
||||
}
|
||||
};
|
||||
|
||||
#undef SGL_HICACHE_KERNEL
|
||||
|
||||
@@ -494,3 +494,186 @@ def transfer_hicache_all_layer_mla_staged_lf_pf(
|
||||
ptr_src,
|
||||
page_size,
|
||||
)
|
||||
|
||||
|
||||
# A separate module from `_jit_hicache_module`: this one instantiates
|
||||
# HiCacheKernel with `kElementSize` set to one head group's token row rather than
|
||||
# the whole row, which is often not the 128-byte multiple `run_one` / `run_all`
|
||||
# need. Only the exported entry points are instantiated, so those never compile
|
||||
# here.
|
||||
@cache_once
|
||||
def _jit_hicache_page_unified_module(*, element_size: int, block_quota: int) -> Module:
|
||||
args = make_cpp_args(
|
||||
element_size,
|
||||
1, # kUnroll: unread by this entry point, which always moves 16B per thread
|
||||
block_quota,
|
||||
1024, # num_threads, can be tuned for performance
|
||||
)
|
||||
return load_jit(
|
||||
"hicache_page_unified",
|
||||
*args,
|
||||
cuda_files=[
|
||||
"kvcacheio/hicache.cuh",
|
||||
],
|
||||
cuda_wrappers=[
|
||||
(
|
||||
"launch_one_page_unified",
|
||||
f"&HiCacheKernel<{args}>::run_one_page_unified<false>",
|
||||
),
|
||||
(
|
||||
"launch_one_mla_page_unified",
|
||||
f"&HiCacheKernel<{args}>::run_one_page_unified<true>",
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def can_use_page_unified_load_back_jit_kernel(
|
||||
*,
|
||||
group_bytes: int,
|
||||
block_quota: int | None = None, # can be tuned for less interference
|
||||
) -> bool:
|
||||
logger = logging.getLogger(__name__)
|
||||
if group_bytes <= 0 or group_bytes % 16 != 0:
|
||||
logger.warning(f"Unsupported {group_bytes = } for page-unified JIT load-back")
|
||||
return False
|
||||
try:
|
||||
_jit_hicache_page_unified_module(
|
||||
element_size=group_bytes,
|
||||
block_quota=block_quota or DEFAULT_BLOCK_QUOTA,
|
||||
)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load page-unified JIT load-back kernel: {e}")
|
||||
return False
|
||||
|
||||
|
||||
@debug_kernel_api
|
||||
def transfer_hicache_one_layer_page_unified_lf(
|
||||
k_cache_dst: torch.Tensor,
|
||||
v_cache_dst: torch.Tensor,
|
||||
src: torch.Tensor,
|
||||
src_indices: torch.Tensor,
|
||||
dst_indices: torch.Tensor,
|
||||
layer_id: int,
|
||||
*,
|
||||
block_quota: int | None = None, # can be tuned for less interference
|
||||
) -> None:
|
||||
"""Load one layer of page_unified host pages into the device KV pool.
|
||||
|
||||
``src`` is a contiguous host tensor with layout
|
||||
(page, head_group, layer, 2, page_size, head_in_group, dim); K=0, V=1 --
|
||||
the order the page-unified write-back produces. It must be pinned or
|
||||
CUDA-registered, because the kernel reads it in place over its device
|
||||
mapping; no staging buffer is allocated.
|
||||
|
||||
``k_cache_dst`` / ``v_cache_dst`` are one layer of the device pool, shaped
|
||||
(token, head, dim) with the pool's full head count. ``src_indices`` holds
|
||||
host-pool TOKEN indices (the page is ``index // page_size``) and
|
||||
``dst_indices`` the device-pool token indices they land on, in matching
|
||||
order; both are CUDA int32/int64 vectors on the destination device.
|
||||
One head group's token row must be a positive multiple of 16 bytes.
|
||||
|
||||
Runs on the current CUDA stream. Keep ``src`` alive and unwritten until
|
||||
that stream completes.
|
||||
"""
|
||||
if src.ndim != 7:
|
||||
raise ValueError(
|
||||
"Expected (page, head_group, layer, 2, page_size, head_in_group, dim)"
|
||||
)
|
||||
if src.shape[3] != 2 or any(d <= 0 for d in src.shape[1:]):
|
||||
raise ValueError(
|
||||
"Page dimensions must be positive and the K/V dimension must be 2"
|
||||
)
|
||||
_transfer_hicache_one_layer_page_unified(
|
||||
k_cache_dst,
|
||||
v_cache_dst,
|
||||
src,
|
||||
src_indices,
|
||||
dst_indices,
|
||||
layer_id=layer_id,
|
||||
group_bytes=src.shape[5] * src.shape[6] * src.element_size(),
|
||||
num_layers=src.shape[2],
|
||||
num_groups=src.shape[1],
|
||||
page_size=src.shape[4],
|
||||
is_mla=False,
|
||||
block_quota=block_quota,
|
||||
)
|
||||
|
||||
|
||||
@debug_kernel_api
|
||||
def transfer_hicache_one_layer_mla_page_unified_lf(
|
||||
cache_dst: torch.Tensor,
|
||||
src: torch.Tensor,
|
||||
src_indices: torch.Tensor,
|
||||
dst_indices: torch.Tensor,
|
||||
layer_id: int,
|
||||
*,
|
||||
block_quota: int | None = None,
|
||||
) -> None:
|
||||
"""Load one layer of compressed MLA pages in (page, layer, page_size, dim) order.
|
||||
|
||||
MLA has one latent cache per layer, with no head-group or separate K/V
|
||||
axes; ``dim`` covers all stored latent and positional components.
|
||||
``cache_dst`` is one layer of the device pool, shaped (token, 1, dim).
|
||||
Host memory, index and stream requirements match the MHA entry point.
|
||||
"""
|
||||
if src.ndim != 4:
|
||||
raise ValueError("Expected MLA (page, layer, page_size, dim)")
|
||||
if any(d <= 0 for d in src.shape[1:]):
|
||||
raise ValueError("MLA page dimensions must be positive")
|
||||
_transfer_hicache_one_layer_page_unified(
|
||||
cache_dst,
|
||||
cache_dst,
|
||||
src,
|
||||
src_indices,
|
||||
dst_indices,
|
||||
layer_id=layer_id,
|
||||
group_bytes=src.shape[3] * src.element_size(),
|
||||
num_layers=src.shape[1],
|
||||
num_groups=1,
|
||||
page_size=src.shape[2],
|
||||
is_mla=True,
|
||||
block_quota=block_quota,
|
||||
)
|
||||
|
||||
|
||||
def _transfer_hicache_one_layer_page_unified(
|
||||
k_cache_dst: torch.Tensor,
|
||||
v_cache_dst: torch.Tensor,
|
||||
src: torch.Tensor,
|
||||
src_indices: torch.Tensor,
|
||||
dst_indices: torch.Tensor,
|
||||
*,
|
||||
layer_id: int,
|
||||
group_bytes: int,
|
||||
num_layers: int,
|
||||
num_groups: int,
|
||||
page_size: int,
|
||||
is_mla: bool,
|
||||
block_quota: int | None,
|
||||
) -> None:
|
||||
if not src.is_contiguous():
|
||||
raise ValueError("The host pool must be contiguous")
|
||||
if src.device.type != "cpu" and not src.is_cuda:
|
||||
raise ValueError("Expected a CPU or CUDA-registered host pool")
|
||||
if group_bytes % 16:
|
||||
raise ValueError("Each copied token row must be 16-byte aligned")
|
||||
module = _jit_hicache_page_unified_module(
|
||||
element_size=group_bytes,
|
||||
block_quota=block_quota or DEFAULT_BLOCK_QUOTA,
|
||||
)
|
||||
launch = (
|
||||
module.launch_one_mla_page_unified if is_mla else module.launch_one_page_unified
|
||||
)
|
||||
launch(
|
||||
k_cache_dst.view(k_cache_dst.shape[0], -1),
|
||||
v_cache_dst.view(v_cache_dst.shape[0], -1),
|
||||
src.view(src.shape[0], -1),
|
||||
src_indices,
|
||||
dst_indices,
|
||||
layer_id,
|
||||
num_layers,
|
||||
num_groups,
|
||||
page_size,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user