[HiCache] Add staged write-back for the page-unified KV cache layout (#39606)

This commit is contained in:
huangtingwei
2026-10-10 22:49:45 +08:00
committed by GitHub
parent 99d6f4e158
commit 406540ab5f
4 changed files with 482 additions and 13 deletions
@@ -87,4 +87,47 @@ inline void launch_hicache_relayout_kernel(
LaunchKernel(static_cast<uint32_t>(grid), kRelayoutBlockSize, device)(kernel, params);
}
struct HicachePageUnifiedRelayoutParams {
void* staging;
const void* k_ptr_src;
const void* v_ptr_src;
const void* src_pages;
uint64_t total_vecs;
int64_t num_groups;
int64_t num_layers;
int64_t page_size;
};
// Stage in destination order: (page, group, layer, kv, token, head, dim).
// MLA omits the group, K/V and head axes: (page, layer, token, dim).
// Each thread moves 16 bytes; adjacent threads write adjacent staging vectors.
// Source layers contain contiguous (token, group * head_in_group, dim) rows.
template <typename Index, int64_t kGroupBytes, bool kIsMLA>
__global__ void hicache_page_unified_relayout_kernel(const __grid_constant__ HicachePageUnifiedRelayoutParams p) {
static_assert(kGroupBytes > 0 && kGroupBytes % 16 == 0);
constexpr int64_t kGroupVecs = kGroupBytes / 16;
constexpr int64_t kComponents = kIsMLA ? 1 : 2;
const int64_t num_groups = kIsMLA ? 1 : p.num_groups;
const auto tid = static_cast<uint64_t>(blockIdx.x) * blockDim.x + threadIdx.x;
const auto stride = static_cast<uint64_t>(gridDim.x) * blockDim.x;
for (uint64_t i = tid; i < p.total_vecs; i += stride) {
auto remaining = i / kGroupVecs;
const auto vec = i % kGroupVecs;
const auto token = remaining % p.page_size;
remaining /= p.page_size;
const auto kv = remaining % kComponents;
remaining /= kComponents;
const auto layer = remaining % p.num_layers;
remaining /= p.num_layers;
const auto group = remaining % num_groups;
const auto page = remaining / num_groups;
const auto src_page = static_cast<const Index*>(p.src_pages)[page];
const auto src_token = static_cast<int64_t>(src_page) * p.page_size + token;
const auto ptrs = static_cast<const void* const*>(kv == 0 ? p.k_ptr_src : p.v_ptr_src);
const auto src = device::pointer::offset(ptrs[layer], (src_token * num_groups + group) * kGroupBytes + vec * 16);
const auto value = device::details::load_nc(static_cast<const uint4*>(src));
device::details::store_nc(static_cast<uint4*>(p.staging) + i, value);
}
}
} // namespace sglang
@@ -325,4 +325,72 @@ struct HiCacheStagedWriteBackKernel {
}
};
template <int64_t kGroupBytes, bool kIsMLA = false>
struct HiCachePageUnifiedWriteBackKernel {
static void
run(const tvm::ffi::TensorView dst,
const tvm::ffi::TensorView staging,
const tvm::ffi::TensorView k_ptr_src,
const tvm::ffi::TensorView v_ptr_src,
const tvm::ffi::TensorView src_pages,
const tvm::ffi::TensorView dst_pages,
const int64_t num_groups,
const int64_t page_size) {
using namespace host;
auto P = SymbolicSize{"page count"};
auto L = SymbolicSize{"layer count"};
auto B = SymbolicSize{"page elements"};
auto dtype = SymbolicDType{};
auto index_dtype = SymbolicDType{};
auto device = SymbolicDevice{};
TensorMatcher({L}).with_dtype<uint64_t>().with_device<kDLGPU>(device).verify(k_ptr_src);
if constexpr (!kIsMLA) {
TensorMatcher({L}).with_dtype<uint64_t>().with_device<kDLGPU>(device).verify(v_ptr_src);
}
TensorMatcher({P}).with_dtype<int32_t, int64_t>(index_dtype).with_device<kDLGPU>(device).verify(src_pages);
TensorMatcher({P}).with_dtype<int64_t>().with_device<kDLCPU, kDLGPUHost>().verify(dst_pages);
TensorMatcher({P, B}).with_dtype(dtype).with_device<kDLGPU>(device).verify(staging);
TensorMatcher({-1, B}).with_dtype(dtype).with_device<kDLCPU, kDLGPUHost>().verify(dst);
RuntimeCheck(num_groups > 0 && page_size > 0 && L.unwrap() > 0, "Page-unified write-back: invalid dimensions");
RuntimeCheck(!kIsMLA || num_groups == 1, "Page-unified MLA write-back: expected one latent group");
constexpr int64_t kComponents = kIsMLA ? 1 : 2;
RuntimeCheck(
B.unwrap() * dtype_bytes(dtype.unwrap()) == num_groups * L.unwrap() * kComponents * page_size * kGroupBytes,
"Page-unified write-back: page byte size mismatch");
RuntimeCheck(reinterpret_cast<uintptr_t>(staging.data_ptr()) % 16 == 0, "Staging must be 16-byte aligned");
const auto dst_ids = static_cast<const int64_t*>(dst_pages.data_ptr());
for (int64_t i = 0; i < P.unwrap(); ++i) {
RuntimeCheck(
dst_ids[i] >= 0 && dst_ids[i] < dst.size(0), "Page-unified write-back: destination page out of range");
}
if (P.unwrap() == 0) {
return;
}
const auto params = HicachePageUnifiedRelayoutParams{
.staging = staging.data_ptr(),
.k_ptr_src = k_ptr_src.data_ptr(),
.v_ptr_src = kIsMLA ? nullptr : v_ptr_src.data_ptr(),
.src_pages = src_pages.data_ptr(),
.total_vecs = static_cast<uint64_t>(P.unwrap()) * B.unwrap() * dtype_bytes(dtype.unwrap()) / 16,
.num_groups = num_groups,
.num_layers = L.unwrap(),
.page_size = page_size,
};
// Grid-stride traversal keeps large transfers within CUDA's grid limits.
constexpr uint32_t kThreads = 256;
const auto blocks =
static_cast<uint32_t>(std::min<uint64_t>(div_ceil(params.total_vecs, uint64_t{kThreads}), 65535));
const auto kernel = index_dtype.unwrap().bits == 32
? hicache_page_unified_relayout_kernel<int32_t, kGroupBytes, kIsMLA>
: hicache_page_unified_relayout_kernel<int64_t, kGroupBytes, kIsMLA>;
const auto dev = device.unwrap();
LaunchKernel(blocks, kThreads, dev)(kernel, params);
const auto stream = LaunchKernel::resolve_device(dev);
// The flattened tensors have one row per page (a single latent cache for MLA).
if (!try_copy_page_first_pages_batch({staging}, {dst}, dst_ids, P.unwrap(), 1, dev.device_id, stream)) {
copy_page_first_pages_fallback({staging}, {dst}, dst_ids, P.unwrap(), 1, stream);
}
}
};
} // namespace sglang
@@ -677,3 +677,151 @@ def _transfer_hicache_one_layer_page_unified(
num_groups,
page_size,
)
@cache_once
def _jit_page_unified_write_back(group_bytes: int, *, is_mla: bool = False) -> Module:
args = make_cpp_args(group_bytes, is_mla)
return load_jit(
"hicache_page_unified_write_back",
*args,
cuda_files=["kvcacheio/staged_write_back.cuh"],
cuda_wrappers=[("launch", f"&HiCachePageUnifiedWriteBackKernel<{args}>::run")],
)
@debug_kernel_api
def transfer_hicache_all_layer_staged_lf_page_unified(
k_ptr_src: torch.Tensor,
v_ptr_src: torch.Tensor,
src_pages: torch.Tensor,
dst_pages: torch.Tensor,
staging: torch.Tensor,
dst: torch.Tensor,
) -> None:
"""Write full GPU KV pages to pinned host memory in page_unified layout.
``staging`` and ``dst`` are contiguous tensors with layout
(page, head_group, layer, 2, page_size, head_in_group, dim); K=0, V=1.
Only their page capacities may differ. Each source pointer addresses a
contiguous (token, head_group * head_in_group, dim) GPU tensor of the same
dtype as ``dst``, with a 16-byte-aligned base. Pointer tables are contiguous
CUDA uint64 tensors, one pointer per layer, on the staging device.
``src_pages`` contains CUDA int32/int64 physical PAGE IDs, not token offsets.
``dst_pages`` contains CPU int64 physical page IDs, in matching order.
Source IDs must be in bounds; destination IDs must be unique. The caller
owns the source allocations, which cannot be validated via pointer tables.
One group's token data must be a positive multiple of 16 bytes.
Reuses caller-provided staging in chunks on the current CUDA stream.
Keep all inputs alive and staging exclusive until that stream completes;
wait for completion before reading the host output. No GPU buffer is allocated.
"""
if dst.ndim != 7 or staging.ndim != 7:
raise ValueError(
"Expected (page, head_group, layer, 2, page_size, head_in_group, dim)"
)
if dst.shape[3] != 2 or any(d <= 0 for d in dst.shape[1:]):
raise ValueError(
"Page dimensions must be positive and the K/V dimension must be 2"
)
_transfer_hicache_all_layer_staged_page_unified(
k_ptr_src,
v_ptr_src,
src_pages,
dst_pages,
staging,
dst,
group_bytes=dst.shape[5] * dst.shape[6] * dst.element_size(),
num_groups=dst.shape[1],
page_size=dst.shape[4],
is_mla=False,
)
@debug_kernel_api
def transfer_hicache_all_layer_mla_staged_lf_page_unified(
ptr_src: torch.Tensor,
src_pages: torch.Tensor,
dst_pages: torch.Tensor,
staging: torch.Tensor,
dst: torch.Tensor,
) -> None:
"""Write compressed MLA KV pages in (page, layer, page_size, dim) order.
MLA has one latent cache per layer, with no head-group or separate K/V
axes. Source pointers address contiguous (token, dim) tensors; ``dim``
includes all stored latent and positional components. No dtype conversion
is performed. Each token row must be a positive multiple of 16 bytes.
Pointer tables, page IDs, pinned destination memory, staging reuse and
stream lifetime requirements match the MHA page_unified entry point.
``src_pages`` and ``dst_pages`` contain physical page IDs, not token offsets.
"""
if dst.ndim != 4 or staging.ndim != 4:
raise ValueError("Expected MLA (page, layer, page_size, dim)")
if any(d <= 0 for d in dst.shape[1:]):
raise ValueError("MLA page dimensions must be positive")
_transfer_hicache_all_layer_staged_page_unified(
ptr_src,
ptr_src,
src_pages,
dst_pages,
staging,
dst,
group_bytes=dst.shape[3] * dst.element_size(),
num_groups=1,
page_size=dst.shape[2],
is_mla=True,
)
def _transfer_hicache_all_layer_staged_page_unified(
k_ptr_src: torch.Tensor,
v_ptr_src: torch.Tensor,
src_pages: torch.Tensor,
dst_pages: torch.Tensor,
staging: torch.Tensor,
dst: torch.Tensor,
*,
group_bytes: int,
num_groups: int,
page_size: int,
is_mla: bool,
) -> None:
if staging.shape[1:] != dst.shape[1:] or staging.dtype != dst.dtype:
raise ValueError(
"Staging and destination must have matching page shapes and dtype"
)
if not staging.is_contiguous() or not dst.is_contiguous():
raise ValueError("Staging and destination must be contiguous")
if not staging.is_cuda or dst.device.type != "cpu" or not dst.is_pinned():
raise ValueError("Expected CUDA staging and a pinned CPU destination")
if staging.shape[0] == 0:
raise ValueError("Staging must hold at least one page")
if (
src_pages.ndim != 1
or dst_pages.ndim != 1
or src_pages.numel() != dst_pages.numel()
):
raise ValueError("Source and destination page IDs must be equal-length vectors")
if group_bytes % 16:
raise ValueError("Each copied token row must be 16-byte aligned")
module = _jit_page_unified_write_back(group_bytes, is_mla=is_mla)
capacity = staging.shape[0]
page_elements = staging[0].numel()
staging_flat = staging.view(capacity, page_elements)
dst_flat = dst.view(dst.shape[0], page_elements)
for begin in range(0, src_pages.numel(), capacity):
end = min(begin + capacity, src_pages.numel())
module.launch(
dst_flat,
staging_flat[: end - begin],
k_ptr_src,
v_ptr_src,
src_pages[begin:end],
dst_pages[begin:end],
num_groups,
page_size,
)