[HiCache] Add the page-unified KV load-back JIT kernel (#39726)

Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: huangtingwei <141888744+huangtingwei9988@users.noreply.github.com>
Co-authored-by: Zhangheng <hzh0425@apache.org>
This commit is contained in:
Shunkangz
2026-09-28 01:45:02 +08:00
committed by GitHub
co-authored by Claude Opus 5 huangtingwei Zhangheng
parent d10a09ecf3
commit e581520c67
3 changed files with 646 additions and 2 deletions
@@ -218,6 +218,21 @@ struct HicacheKernelParams {
uint32_t num_layers = 0; // only used in all_layer transfer
};
/// \brief Operands of \ref hicache_transfer_per_layer_page_unified.
struct HicachePageUnifiedKernelParams {
void* __restrict__ k_cache_dst;
void* __restrict__ v_cache_dst;
const void* __restrict__ src;
const void* __restrict__ src_indices;
const void* __restrict__ dst_indices;
uint64_t total_vecs;
int64_t num_items;
int64_t num_groups;
int64_t num_layers;
int64_t page_size;
int64_t layer_id;
};
template <
typename T,
int64_t kElementSize,
@@ -313,6 +328,67 @@ SGL_HICACHE_KERNEL void hicache_transfer_all_layer(const __grid_constant__ Hicac
}
}
/**
* \brief Read one layer out of page-unified host pages into a per-layer device buffer.
*
* The source page block is (head_group, layer, 2, page_size, head_in_group, dim)
* with K=0 and V=1 -- the exact byte order the page-unified write-back leaves
* behind, so a page round-trips unchanged. MLA drops the head-group, K/V and
* head axes, leaving (layer, page_size, dim).
*
* The destination is the device pool's own per-layer rows, (token, head, dim),
* where head group \c g owns the contiguous run [g * kElementSize, (g+1) * kElementSize).
* That run is exactly one source cell, which is why the permutation costs
* nothing: it only reorders addresses the copy already computes per group.
*
* \c kElementSize is one head group's token row here, not the whole row the
* other kernels in this file copy: one source cell is what a thread block
* walks, and the device row is \c num_groups of them side by side.
*
* Each thread moves 16 bytes. The linear index runs dim-fastest, then item, so
* lanes of a warp read consecutive host bytes for tokens of one (head group,
* layer, K/V) cell -- the reads cross PCIe, so their coalescing is what sets
* the achieved bandwidth.
*/
template <typename T, int64_t kElementSize, uint32_t kBlockSize, bool kIsMLA = false>
SGL_HICACHE_KERNEL void
hicache_transfer_per_layer_page_unified(const __grid_constant__ HicachePageUnifiedKernelParams params) {
using namespace device;
static_assert(kElementSize > 0 && kElementSize % 16 == 0);
constexpr int64_t kGroupVecs = kElementSize / 16;
constexpr int64_t kComponents = kIsMLA ? 1 : 2;
const int64_t num_groups = kIsMLA ? 1 : params.num_groups;
const auto tid = static_cast<uint64_t>(blockIdx.x) * blockDim.x + threadIdx.x;
const auto stride = static_cast<uint64_t>(gridDim.x) * blockDim.x;
for (uint64_t i = tid; i < params.total_vecs; i += stride) {
const auto vec = static_cast<int64_t>(i % kGroupVecs);
auto remaining = static_cast<int64_t>(i / kGroupVecs);
const auto item = remaining % params.num_items;
remaining /= params.num_items;
const auto kv = remaining % kComponents;
const auto group = remaining / kComponents;
const auto src_token = static_cast<int64_t>(static_cast<const T*>(params.src_indices)[item]);
const auto dst_token = static_cast<int64_t>(static_cast<const T*>(params.dst_indices)[item]);
const auto page = src_token / params.page_size;
const auto token_in_page = src_token % params.page_size;
const auto src_offset =
(((((page * num_groups + group) * params.num_layers + params.layer_id) * kComponents + kv) * params.page_size +
token_in_page) *
kElementSize) +
vec * 16;
const auto dst_offset = (dst_token * num_groups + group) * kElementSize + vec * 16;
const auto dst_base = kv == 0 ? params.k_cache_dst : params.v_cache_dst;
const auto src = pointer::offset(static_cast<const void*>(params.src), src_offset);
const auto dst = pointer::offset(static_cast<void*>(dst_base), dst_offset);
const auto value = details::load_nc(reinterpret_cast<const uint4*>(src));
details::store_nc(reinterpret_cast<uint4*>(dst), value);
}
}
template <int64_t kElementSize, uint32_t kUnroll, uint32_t kBlockQuota, uint32_t kBlockSize>
struct HiCacheKernel {
template <typename T>
@@ -325,6 +401,9 @@ struct HiCacheKernel {
template <typename T>
static constexpr auto kernel_all_mla =
hicache_transfer_all_layer<T, kElementSize, kUnroll, kBlockQuota, kBlockSize, true>;
template <typename T, bool kIsMLA>
static constexpr auto kernel_one_page_unified =
hicache_transfer_per_layer_page_unified<T, kElementSize, kBlockSize, kIsMLA>;
static void run_one(
const tvm::ffi::TensorView k_cache_dst,
@@ -565,6 +644,101 @@ struct HiCacheKernel {
const auto kernel = use_int32 ? kernel_all_mla<int32_t> : kernel_all_mla<int64_t>;
LaunchKernel(num_blocks, kBlockSize, device)(kernel, params);
}
/**
* \brief Load-back half of the page-unified HiCache layout: host pages -> device pool.
*
* The mirror of the page-unified write-back. Where write-back stages a whole
* page on the device and hands the bulk transfer to the copy engine, load-back
* is driven per layer by the SMs reading the pinned host mapping directly: the
* transfer engine walks layers and unblocks the forward pass one layer at a
* time, so staging a whole page per layer would move \c num_layers times the
* bytes the layer actually needs.
*
* \tparam kIsMLA Whether the pool has a single latent component and no head axis
* \param k_cache_dst One layer of the device K pool, flattened to (token, head * dim)
* \param v_cache_dst One layer of the device V pool; ignored when \c kIsMLA
* \param src Host pool flattened to (page, page elements), page-unified order
* \param src_indices Host-pool TOKEN indices, on the destination device
* \param dst_indices Device-pool token indices, same length and order
* \param layer_id Layer to read, indexing the host pool's layer axis
* \param num_layers Layer extent of the host pool's layer axis
* \param num_groups Head groups the page block is cut into; 1 when \c kIsMLA
* \param page_size Tokens per page
*/
template <bool kIsMLA>
static void run_one_page_unified(
const tvm::ffi::TensorView k_cache_dst,
const tvm::ffi::TensorView v_cache_dst,
const tvm::ffi::TensorView src,
const tvm::ffi::TensorView src_indices,
const tvm::ffi::TensorView dst_indices,
const int64_t layer_id,
const int64_t num_layers,
const int64_t num_groups,
const int64_t page_size) {
using namespace host;
auto B = SymbolicSize{"page elements"};
auto C = SymbolicSize{"layer elements per token"};
auto N = SymbolicSize{"indices length"};
auto cache_dtype = SymbolicDType{};
auto indices_dtype = SymbolicDType{};
auto device_ = SymbolicDevice{};
// The device pool binds the launch device: the source is host memory read
// over its pinned mapping, so it cannot name a stream.
TensorMatcher({-1, C}).with_dtype(cache_dtype).with_device<kDLGPU>(device_).verify(k_cache_dst);
if constexpr (!kIsMLA) {
TensorMatcher({-1, C}).with_dtype(cache_dtype).with_device<kDLGPU>(device_).verify(v_cache_dst);
}
TensorMatcher({-1, B}).with_dtype(cache_dtype).with_device<kDLCPU, kDLGPUHost, kDLGPU>().verify(src);
TensorMatcher({N})
.with_dtype<int32_t, int64_t>(indices_dtype)
.with_device<kDLGPU>(device_)
.verify(src_indices)
.verify(dst_indices);
constexpr int64_t kComponents = kIsMLA ? 1 : 2;
RuntimeCheck(num_layers > 0 && num_groups > 0 && page_size > 0, "Page-unified load-back: invalid dimensions");
RuntimeCheck(layer_id >= 0 && layer_id < num_layers, "Page-unified load-back: layer id out of range");
RuntimeCheck(!kIsMLA || num_groups == 1, "Page-unified MLA load-back: expected one latent group");
const auto dtype_size = dtype_bytes(cache_dtype.unwrap());
RuntimeCheck(
B.unwrap() * dtype_size == num_groups * num_layers * kComponents * page_size * kElementSize,
"Page-unified load-back: page byte size mismatch");
RuntimeCheck(
C.unwrap() * dtype_size == num_groups * kElementSize, "Page-unified load-back: device row byte size mismatch");
RuntimeCheck(
reinterpret_cast<uintptr_t>(src.data_ptr()) % 16 == 0 &&
reinterpret_cast<uintptr_t>(k_cache_dst.data_ptr()) % 16 == 0 &&
(kIsMLA || reinterpret_cast<uintptr_t>(v_cache_dst.data_ptr()) % 16 == 0),
"Page-unified load-back: buffers must be 16-byte aligned");
if (N.unwrap() == 0) {
return;
}
const auto params = HicachePageUnifiedKernelParams{
.k_cache_dst = k_cache_dst.data_ptr(),
.v_cache_dst = kIsMLA ? nullptr : v_cache_dst.data_ptr(),
.src = runtime::get_device_accessible_ptr(src),
.src_indices = src_indices.data_ptr(),
.dst_indices = dst_indices.data_ptr(),
.total_vecs = static_cast<uint64_t>(N.unwrap()) * num_groups * kComponents * (kElementSize / 16),
.num_items = N.unwrap(),
.num_groups = num_groups,
.num_layers = num_layers,
.page_size = page_size,
.layer_id = layer_id,
};
// A quota-capped persistent grid, as in the other entry points: the load
// overlaps the forward pass, so more blocks buy bandwidth by taking SMs
// from it.
const auto num_blocks = std::min(div_ceil(params.total_vecs, uint64_t{kBlockSize}), uint64_t{kBlockQuota});
const auto use_int32 = indices_dtype.unwrap().bits == 32;
const auto kernel = use_int32 ? kernel_one_page_unified<int32_t, kIsMLA> : kernel_one_page_unified<int64_t, kIsMLA>;
LaunchKernel(static_cast<uint32_t>(num_blocks), kBlockSize, device_.unwrap())(kernel, params);
}
};
#undef SGL_HICACHE_KERNEL
@@ -494,3 +494,186 @@ def transfer_hicache_all_layer_mla_staged_lf_pf(
ptr_src,
page_size,
)
# A separate module from `_jit_hicache_module`: this one instantiates
# HiCacheKernel with `kElementSize` set to one head group's token row rather than
# the whole row, which is often not the 128-byte multiple `run_one` / `run_all`
# need. Only the exported entry points are instantiated, so those never compile
# here.
@cache_once
def _jit_hicache_page_unified_module(*, element_size: int, block_quota: int) -> Module:
args = make_cpp_args(
element_size,
1, # kUnroll: unread by this entry point, which always moves 16B per thread
block_quota,
1024, # num_threads, can be tuned for performance
)
return load_jit(
"hicache_page_unified",
*args,
cuda_files=[
"kvcacheio/hicache.cuh",
],
cuda_wrappers=[
(
"launch_one_page_unified",
f"&HiCacheKernel<{args}>::run_one_page_unified<false>",
),
(
"launch_one_mla_page_unified",
f"&HiCacheKernel<{args}>::run_one_page_unified<true>",
),
],
)
def can_use_page_unified_load_back_jit_kernel(
*,
group_bytes: int,
block_quota: int | None = None, # can be tuned for less interference
) -> bool:
logger = logging.getLogger(__name__)
if group_bytes <= 0 or group_bytes % 16 != 0:
logger.warning(f"Unsupported {group_bytes = } for page-unified JIT load-back")
return False
try:
_jit_hicache_page_unified_module(
element_size=group_bytes,
block_quota=block_quota or DEFAULT_BLOCK_QUOTA,
)
return True
except Exception as e:
logger.warning(f"Failed to load page-unified JIT load-back kernel: {e}")
return False
@debug_kernel_api
def transfer_hicache_one_layer_page_unified_lf(
k_cache_dst: torch.Tensor,
v_cache_dst: torch.Tensor,
src: torch.Tensor,
src_indices: torch.Tensor,
dst_indices: torch.Tensor,
layer_id: int,
*,
block_quota: int | None = None, # can be tuned for less interference
) -> None:
"""Load one layer of page_unified host pages into the device KV pool.
``src`` is a contiguous host tensor with layout
(page, head_group, layer, 2, page_size, head_in_group, dim); K=0, V=1 --
the order the page-unified write-back produces. It must be pinned or
CUDA-registered, because the kernel reads it in place over its device
mapping; no staging buffer is allocated.
``k_cache_dst`` / ``v_cache_dst`` are one layer of the device pool, shaped
(token, head, dim) with the pool's full head count. ``src_indices`` holds
host-pool TOKEN indices (the page is ``index // page_size``) and
``dst_indices`` the device-pool token indices they land on, in matching
order; both are CUDA int32/int64 vectors on the destination device.
One head group's token row must be a positive multiple of 16 bytes.
Runs on the current CUDA stream. Keep ``src`` alive and unwritten until
that stream completes.
"""
if src.ndim != 7:
raise ValueError(
"Expected (page, head_group, layer, 2, page_size, head_in_group, dim)"
)
if src.shape[3] != 2 or any(d <= 0 for d in src.shape[1:]):
raise ValueError(
"Page dimensions must be positive and the K/V dimension must be 2"
)
_transfer_hicache_one_layer_page_unified(
k_cache_dst,
v_cache_dst,
src,
src_indices,
dst_indices,
layer_id=layer_id,
group_bytes=src.shape[5] * src.shape[6] * src.element_size(),
num_layers=src.shape[2],
num_groups=src.shape[1],
page_size=src.shape[4],
is_mla=False,
block_quota=block_quota,
)
@debug_kernel_api
def transfer_hicache_one_layer_mla_page_unified_lf(
cache_dst: torch.Tensor,
src: torch.Tensor,
src_indices: torch.Tensor,
dst_indices: torch.Tensor,
layer_id: int,
*,
block_quota: int | None = None,
) -> None:
"""Load one layer of compressed MLA pages in (page, layer, page_size, dim) order.
MLA has one latent cache per layer, with no head-group or separate K/V
axes; ``dim`` covers all stored latent and positional components.
``cache_dst`` is one layer of the device pool, shaped (token, 1, dim).
Host memory, index and stream requirements match the MHA entry point.
"""
if src.ndim != 4:
raise ValueError("Expected MLA (page, layer, page_size, dim)")
if any(d <= 0 for d in src.shape[1:]):
raise ValueError("MLA page dimensions must be positive")
_transfer_hicache_one_layer_page_unified(
cache_dst,
cache_dst,
src,
src_indices,
dst_indices,
layer_id=layer_id,
group_bytes=src.shape[3] * src.element_size(),
num_layers=src.shape[1],
num_groups=1,
page_size=src.shape[2],
is_mla=True,
block_quota=block_quota,
)
def _transfer_hicache_one_layer_page_unified(
k_cache_dst: torch.Tensor,
v_cache_dst: torch.Tensor,
src: torch.Tensor,
src_indices: torch.Tensor,
dst_indices: torch.Tensor,
*,
layer_id: int,
group_bytes: int,
num_layers: int,
num_groups: int,
page_size: int,
is_mla: bool,
block_quota: int | None,
) -> None:
if not src.is_contiguous():
raise ValueError("The host pool must be contiguous")
if src.device.type != "cpu" and not src.is_cuda:
raise ValueError("Expected a CPU or CUDA-registered host pool")
if group_bytes % 16:
raise ValueError("Each copied token row must be 16-byte aligned")
module = _jit_hicache_page_unified_module(
element_size=group_bytes,
block_quota=block_quota or DEFAULT_BLOCK_QUOTA,
)
launch = (
module.launch_one_mla_page_unified if is_mla else module.launch_one_page_unified
)
launch(
k_cache_dst.view(k_cache_dst.shape[0], -1),
v_cache_dst.view(v_cache_dst.shape[0], -1),
src.view(src.shape[0], -1),
src_indices,
dst_indices,
layer_id,
num_layers,
num_groups,
page_size,
)