[HiCache] Add staged write-back for the page-unified KV cache layout (#39606)
This commit is contained in:
@@ -87,4 +87,47 @@ inline void launch_hicache_relayout_kernel(
|
||||
LaunchKernel(static_cast<uint32_t>(grid), kRelayoutBlockSize, device)(kernel, params);
|
||||
}
|
||||
|
||||
struct HicachePageUnifiedRelayoutParams {
|
||||
void* staging;
|
||||
const void* k_ptr_src;
|
||||
const void* v_ptr_src;
|
||||
const void* src_pages;
|
||||
uint64_t total_vecs;
|
||||
int64_t num_groups;
|
||||
int64_t num_layers;
|
||||
int64_t page_size;
|
||||
};
|
||||
|
||||
// Stage in destination order: (page, group, layer, kv, token, head, dim).
|
||||
// MLA omits the group, K/V and head axes: (page, layer, token, dim).
|
||||
// Each thread moves 16 bytes; adjacent threads write adjacent staging vectors.
|
||||
// Source layers contain contiguous (token, group * head_in_group, dim) rows.
|
||||
template <typename Index, int64_t kGroupBytes, bool kIsMLA>
|
||||
__global__ void hicache_page_unified_relayout_kernel(const __grid_constant__ HicachePageUnifiedRelayoutParams p) {
|
||||
static_assert(kGroupBytes > 0 && kGroupBytes % 16 == 0);
|
||||
constexpr int64_t kGroupVecs = kGroupBytes / 16;
|
||||
constexpr int64_t kComponents = kIsMLA ? 1 : 2;
|
||||
const int64_t num_groups = kIsMLA ? 1 : p.num_groups;
|
||||
const auto tid = static_cast<uint64_t>(blockIdx.x) * blockDim.x + threadIdx.x;
|
||||
const auto stride = static_cast<uint64_t>(gridDim.x) * blockDim.x;
|
||||
for (uint64_t i = tid; i < p.total_vecs; i += stride) {
|
||||
auto remaining = i / kGroupVecs;
|
||||
const auto vec = i % kGroupVecs;
|
||||
const auto token = remaining % p.page_size;
|
||||
remaining /= p.page_size;
|
||||
const auto kv = remaining % kComponents;
|
||||
remaining /= kComponents;
|
||||
const auto layer = remaining % p.num_layers;
|
||||
remaining /= p.num_layers;
|
||||
const auto group = remaining % num_groups;
|
||||
const auto page = remaining / num_groups;
|
||||
const auto src_page = static_cast<const Index*>(p.src_pages)[page];
|
||||
const auto src_token = static_cast<int64_t>(src_page) * p.page_size + token;
|
||||
const auto ptrs = static_cast<const void* const*>(kv == 0 ? p.k_ptr_src : p.v_ptr_src);
|
||||
const auto src = device::pointer::offset(ptrs[layer], (src_token * num_groups + group) * kGroupBytes + vec * 16);
|
||||
const auto value = device::details::load_nc(static_cast<const uint4*>(src));
|
||||
device::details::store_nc(static_cast<uint4*>(p.staging) + i, value);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace sglang
|
||||
|
||||
@@ -325,4 +325,72 @@ struct HiCacheStagedWriteBackKernel {
|
||||
}
|
||||
};
|
||||
|
||||
template <int64_t kGroupBytes, bool kIsMLA = false>
|
||||
struct HiCachePageUnifiedWriteBackKernel {
|
||||
static void
|
||||
run(const tvm::ffi::TensorView dst,
|
||||
const tvm::ffi::TensorView staging,
|
||||
const tvm::ffi::TensorView k_ptr_src,
|
||||
const tvm::ffi::TensorView v_ptr_src,
|
||||
const tvm::ffi::TensorView src_pages,
|
||||
const tvm::ffi::TensorView dst_pages,
|
||||
const int64_t num_groups,
|
||||
const int64_t page_size) {
|
||||
using namespace host;
|
||||
auto P = SymbolicSize{"page count"};
|
||||
auto L = SymbolicSize{"layer count"};
|
||||
auto B = SymbolicSize{"page elements"};
|
||||
auto dtype = SymbolicDType{};
|
||||
auto index_dtype = SymbolicDType{};
|
||||
auto device = SymbolicDevice{};
|
||||
TensorMatcher({L}).with_dtype<uint64_t>().with_device<kDLGPU>(device).verify(k_ptr_src);
|
||||
if constexpr (!kIsMLA) {
|
||||
TensorMatcher({L}).with_dtype<uint64_t>().with_device<kDLGPU>(device).verify(v_ptr_src);
|
||||
}
|
||||
TensorMatcher({P}).with_dtype<int32_t, int64_t>(index_dtype).with_device<kDLGPU>(device).verify(src_pages);
|
||||
TensorMatcher({P}).with_dtype<int64_t>().with_device<kDLCPU, kDLGPUHost>().verify(dst_pages);
|
||||
TensorMatcher({P, B}).with_dtype(dtype).with_device<kDLGPU>(device).verify(staging);
|
||||
TensorMatcher({-1, B}).with_dtype(dtype).with_device<kDLCPU, kDLGPUHost>().verify(dst);
|
||||
RuntimeCheck(num_groups > 0 && page_size > 0 && L.unwrap() > 0, "Page-unified write-back: invalid dimensions");
|
||||
RuntimeCheck(!kIsMLA || num_groups == 1, "Page-unified MLA write-back: expected one latent group");
|
||||
constexpr int64_t kComponents = kIsMLA ? 1 : 2;
|
||||
RuntimeCheck(
|
||||
B.unwrap() * dtype_bytes(dtype.unwrap()) == num_groups * L.unwrap() * kComponents * page_size * kGroupBytes,
|
||||
"Page-unified write-back: page byte size mismatch");
|
||||
RuntimeCheck(reinterpret_cast<uintptr_t>(staging.data_ptr()) % 16 == 0, "Staging must be 16-byte aligned");
|
||||
const auto dst_ids = static_cast<const int64_t*>(dst_pages.data_ptr());
|
||||
for (int64_t i = 0; i < P.unwrap(); ++i) {
|
||||
RuntimeCheck(
|
||||
dst_ids[i] >= 0 && dst_ids[i] < dst.size(0), "Page-unified write-back: destination page out of range");
|
||||
}
|
||||
if (P.unwrap() == 0) {
|
||||
return;
|
||||
}
|
||||
const auto params = HicachePageUnifiedRelayoutParams{
|
||||
.staging = staging.data_ptr(),
|
||||
.k_ptr_src = k_ptr_src.data_ptr(),
|
||||
.v_ptr_src = kIsMLA ? nullptr : v_ptr_src.data_ptr(),
|
||||
.src_pages = src_pages.data_ptr(),
|
||||
.total_vecs = static_cast<uint64_t>(P.unwrap()) * B.unwrap() * dtype_bytes(dtype.unwrap()) / 16,
|
||||
.num_groups = num_groups,
|
||||
.num_layers = L.unwrap(),
|
||||
.page_size = page_size,
|
||||
};
|
||||
// Grid-stride traversal keeps large transfers within CUDA's grid limits.
|
||||
constexpr uint32_t kThreads = 256;
|
||||
const auto blocks =
|
||||
static_cast<uint32_t>(std::min<uint64_t>(div_ceil(params.total_vecs, uint64_t{kThreads}), 65535));
|
||||
const auto kernel = index_dtype.unwrap().bits == 32
|
||||
? hicache_page_unified_relayout_kernel<int32_t, kGroupBytes, kIsMLA>
|
||||
: hicache_page_unified_relayout_kernel<int64_t, kGroupBytes, kIsMLA>;
|
||||
const auto dev = device.unwrap();
|
||||
LaunchKernel(blocks, kThreads, dev)(kernel, params);
|
||||
const auto stream = LaunchKernel::resolve_device(dev);
|
||||
// The flattened tensors have one row per page (a single latent cache for MLA).
|
||||
if (!try_copy_page_first_pages_batch({staging}, {dst}, dst_ids, P.unwrap(), 1, dev.device_id, stream)) {
|
||||
copy_page_first_pages_fallback({staging}, {dst}, dst_ids, P.unwrap(), 1, stream);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace sglang
|
||||
|
||||
@@ -677,3 +677,151 @@ def _transfer_hicache_one_layer_page_unified(
|
||||
num_groups,
|
||||
page_size,
|
||||
)
|
||||
|
||||
|
||||
@cache_once
|
||||
def _jit_page_unified_write_back(group_bytes: int, *, is_mla: bool = False) -> Module:
|
||||
args = make_cpp_args(group_bytes, is_mla)
|
||||
return load_jit(
|
||||
"hicache_page_unified_write_back",
|
||||
*args,
|
||||
cuda_files=["kvcacheio/staged_write_back.cuh"],
|
||||
cuda_wrappers=[("launch", f"&HiCachePageUnifiedWriteBackKernel<{args}>::run")],
|
||||
)
|
||||
|
||||
|
||||
@debug_kernel_api
|
||||
def transfer_hicache_all_layer_staged_lf_page_unified(
|
||||
k_ptr_src: torch.Tensor,
|
||||
v_ptr_src: torch.Tensor,
|
||||
src_pages: torch.Tensor,
|
||||
dst_pages: torch.Tensor,
|
||||
staging: torch.Tensor,
|
||||
dst: torch.Tensor,
|
||||
) -> None:
|
||||
"""Write full GPU KV pages to pinned host memory in page_unified layout.
|
||||
|
||||
``staging`` and ``dst`` are contiguous tensors with layout
|
||||
(page, head_group, layer, 2, page_size, head_in_group, dim); K=0, V=1.
|
||||
Only their page capacities may differ. Each source pointer addresses a
|
||||
contiguous (token, head_group * head_in_group, dim) GPU tensor of the same
|
||||
dtype as ``dst``, with a 16-byte-aligned base. Pointer tables are contiguous
|
||||
CUDA uint64 tensors, one pointer per layer, on the staging device.
|
||||
|
||||
``src_pages`` contains CUDA int32/int64 physical PAGE IDs, not token offsets.
|
||||
``dst_pages`` contains CPU int64 physical page IDs, in matching order.
|
||||
Source IDs must be in bounds; destination IDs must be unique. The caller
|
||||
owns the source allocations, which cannot be validated via pointer tables.
|
||||
One group's token data must be a positive multiple of 16 bytes.
|
||||
|
||||
Reuses caller-provided staging in chunks on the current CUDA stream.
|
||||
Keep all inputs alive and staging exclusive until that stream completes;
|
||||
wait for completion before reading the host output. No GPU buffer is allocated.
|
||||
"""
|
||||
if dst.ndim != 7 or staging.ndim != 7:
|
||||
raise ValueError(
|
||||
"Expected (page, head_group, layer, 2, page_size, head_in_group, dim)"
|
||||
)
|
||||
if dst.shape[3] != 2 or any(d <= 0 for d in dst.shape[1:]):
|
||||
raise ValueError(
|
||||
"Page dimensions must be positive and the K/V dimension must be 2"
|
||||
)
|
||||
_transfer_hicache_all_layer_staged_page_unified(
|
||||
k_ptr_src,
|
||||
v_ptr_src,
|
||||
src_pages,
|
||||
dst_pages,
|
||||
staging,
|
||||
dst,
|
||||
group_bytes=dst.shape[5] * dst.shape[6] * dst.element_size(),
|
||||
num_groups=dst.shape[1],
|
||||
page_size=dst.shape[4],
|
||||
is_mla=False,
|
||||
)
|
||||
|
||||
|
||||
@debug_kernel_api
|
||||
def transfer_hicache_all_layer_mla_staged_lf_page_unified(
|
||||
ptr_src: torch.Tensor,
|
||||
src_pages: torch.Tensor,
|
||||
dst_pages: torch.Tensor,
|
||||
staging: torch.Tensor,
|
||||
dst: torch.Tensor,
|
||||
) -> None:
|
||||
"""Write compressed MLA KV pages in (page, layer, page_size, dim) order.
|
||||
|
||||
MLA has one latent cache per layer, with no head-group or separate K/V
|
||||
axes. Source pointers address contiguous (token, dim) tensors; ``dim``
|
||||
includes all stored latent and positional components. No dtype conversion
|
||||
is performed. Each token row must be a positive multiple of 16 bytes.
|
||||
|
||||
Pointer tables, page IDs, pinned destination memory, staging reuse and
|
||||
stream lifetime requirements match the MHA page_unified entry point.
|
||||
``src_pages`` and ``dst_pages`` contain physical page IDs, not token offsets.
|
||||
"""
|
||||
if dst.ndim != 4 or staging.ndim != 4:
|
||||
raise ValueError("Expected MLA (page, layer, page_size, dim)")
|
||||
if any(d <= 0 for d in dst.shape[1:]):
|
||||
raise ValueError("MLA page dimensions must be positive")
|
||||
_transfer_hicache_all_layer_staged_page_unified(
|
||||
ptr_src,
|
||||
ptr_src,
|
||||
src_pages,
|
||||
dst_pages,
|
||||
staging,
|
||||
dst,
|
||||
group_bytes=dst.shape[3] * dst.element_size(),
|
||||
num_groups=1,
|
||||
page_size=dst.shape[2],
|
||||
is_mla=True,
|
||||
)
|
||||
|
||||
|
||||
def _transfer_hicache_all_layer_staged_page_unified(
|
||||
k_ptr_src: torch.Tensor,
|
||||
v_ptr_src: torch.Tensor,
|
||||
src_pages: torch.Tensor,
|
||||
dst_pages: torch.Tensor,
|
||||
staging: torch.Tensor,
|
||||
dst: torch.Tensor,
|
||||
*,
|
||||
group_bytes: int,
|
||||
num_groups: int,
|
||||
page_size: int,
|
||||
is_mla: bool,
|
||||
) -> None:
|
||||
if staging.shape[1:] != dst.shape[1:] or staging.dtype != dst.dtype:
|
||||
raise ValueError(
|
||||
"Staging and destination must have matching page shapes and dtype"
|
||||
)
|
||||
if not staging.is_contiguous() or not dst.is_contiguous():
|
||||
raise ValueError("Staging and destination must be contiguous")
|
||||
if not staging.is_cuda or dst.device.type != "cpu" or not dst.is_pinned():
|
||||
raise ValueError("Expected CUDA staging and a pinned CPU destination")
|
||||
if staging.shape[0] == 0:
|
||||
raise ValueError("Staging must hold at least one page")
|
||||
if (
|
||||
src_pages.ndim != 1
|
||||
or dst_pages.ndim != 1
|
||||
or src_pages.numel() != dst_pages.numel()
|
||||
):
|
||||
raise ValueError("Source and destination page IDs must be equal-length vectors")
|
||||
if group_bytes % 16:
|
||||
raise ValueError("Each copied token row must be 16-byte aligned")
|
||||
module = _jit_page_unified_write_back(group_bytes, is_mla=is_mla)
|
||||
capacity = staging.shape[0]
|
||||
page_elements = staging[0].numel()
|
||||
staging_flat = staging.view(capacity, page_elements)
|
||||
dst_flat = dst.view(dst.shape[0], page_elements)
|
||||
for begin in range(0, src_pages.numel(), capacity):
|
||||
end = min(begin + capacity, src_pages.numel())
|
||||
module.launch(
|
||||
dst_flat,
|
||||
staging_flat[: end - begin],
|
||||
k_ptr_src,
|
||||
v_ptr_src,
|
||||
src_pages[begin:end],
|
||||
dst_pages[begin:end],
|
||||
num_groups,
|
||||
page_size,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user