diff --git a/python/sglang/kernels/jit/csrc/kvcacheio/hicache.cuh b/python/sglang/kernels/jit/csrc/kvcacheio/hicache.cuh index aa0038bcaf..df46e1e72d 100644 --- a/python/sglang/kernels/jit/csrc/kvcacheio/hicache.cuh +++ b/python/sglang/kernels/jit/csrc/kvcacheio/hicache.cuh @@ -218,6 +218,21 @@ struct HicacheKernelParams { uint32_t num_layers = 0; // only used in all_layer transfer }; +/// \brief Operands of \ref hicache_transfer_per_layer_page_unified. +struct HicachePageUnifiedKernelParams { + void* __restrict__ k_cache_dst; + void* __restrict__ v_cache_dst; + const void* __restrict__ src; + const void* __restrict__ src_indices; + const void* __restrict__ dst_indices; + uint64_t total_vecs; + int64_t num_items; + int64_t num_groups; + int64_t num_layers; + int64_t page_size; + int64_t layer_id; +}; + template < typename T, int64_t kElementSize, @@ -313,6 +328,67 @@ SGL_HICACHE_KERNEL void hicache_transfer_all_layer(const __grid_constant__ Hicac } } +/** + * \brief Read one layer out of page-unified host pages into a per-layer device buffer. + * + * The source page block is (head_group, layer, 2, page_size, head_in_group, dim) + * with K=0 and V=1 -- the exact byte order the page-unified write-back leaves + * behind, so a page round-trips unchanged. MLA drops the head-group, K/V and + * head axes, leaving (layer, page_size, dim). + * + * The destination is the device pool's own per-layer rows, (token, head, dim), + * where head group \c g owns the contiguous run [g * kElementSize, (g+1) * kElementSize). + * That run is exactly one source cell, which is why the permutation costs + * nothing: it only reorders addresses the copy already computes per group. + * + * \c kElementSize is one head group's token row here, not the whole row the + * other kernels in this file copy: one source cell is what a thread block + * walks, and the device row is \c num_groups of them side by side. + * + * Each thread moves 16 bytes. The linear index runs dim-fastest, then item, so + * lanes of a warp read consecutive host bytes for tokens of one (head group, + * layer, K/V) cell -- the reads cross PCIe, so their coalescing is what sets + * the achieved bandwidth. + */ +template +SGL_HICACHE_KERNEL void +hicache_transfer_per_layer_page_unified(const __grid_constant__ HicachePageUnifiedKernelParams params) { + using namespace device; + static_assert(kElementSize > 0 && kElementSize % 16 == 0); + constexpr int64_t kGroupVecs = kElementSize / 16; + constexpr int64_t kComponents = kIsMLA ? 1 : 2; + const int64_t num_groups = kIsMLA ? 1 : params.num_groups; + + const auto tid = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + const auto stride = static_cast(gridDim.x) * blockDim.x; + for (uint64_t i = tid; i < params.total_vecs; i += stride) { + const auto vec = static_cast(i % kGroupVecs); + auto remaining = static_cast(i / kGroupVecs); + const auto item = remaining % params.num_items; + remaining /= params.num_items; + const auto kv = remaining % kComponents; + const auto group = remaining / kComponents; + + const auto src_token = static_cast(static_cast(params.src_indices)[item]); + const auto dst_token = static_cast(static_cast(params.dst_indices)[item]); + const auto page = src_token / params.page_size; + const auto token_in_page = src_token % params.page_size; + + const auto src_offset = + (((((page * num_groups + group) * params.num_layers + params.layer_id) * kComponents + kv) * params.page_size + + token_in_page) * + kElementSize) + + vec * 16; + const auto dst_offset = (dst_token * num_groups + group) * kElementSize + vec * 16; + const auto dst_base = kv == 0 ? params.k_cache_dst : params.v_cache_dst; + + const auto src = pointer::offset(static_cast(params.src), src_offset); + const auto dst = pointer::offset(static_cast(dst_base), dst_offset); + const auto value = details::load_nc(reinterpret_cast(src)); + details::store_nc(reinterpret_cast(dst), value); + } +} + template struct HiCacheKernel { template @@ -325,6 +401,9 @@ struct HiCacheKernel { template static constexpr auto kernel_all_mla = hicache_transfer_all_layer; + template + static constexpr auto kernel_one_page_unified = + hicache_transfer_per_layer_page_unified; static void run_one( const tvm::ffi::TensorView k_cache_dst, @@ -565,6 +644,101 @@ struct HiCacheKernel { const auto kernel = use_int32 ? kernel_all_mla : kernel_all_mla; LaunchKernel(num_blocks, kBlockSize, device)(kernel, params); } + + /** + * \brief Load-back half of the page-unified HiCache layout: host pages -> device pool. + * + * The mirror of the page-unified write-back. Where write-back stages a whole + * page on the device and hands the bulk transfer to the copy engine, load-back + * is driven per layer by the SMs reading the pinned host mapping directly: the + * transfer engine walks layers and unblocks the forward pass one layer at a + * time, so staging a whole page per layer would move \c num_layers times the + * bytes the layer actually needs. + * + * \tparam kIsMLA Whether the pool has a single latent component and no head axis + * \param k_cache_dst One layer of the device K pool, flattened to (token, head * dim) + * \param v_cache_dst One layer of the device V pool; ignored when \c kIsMLA + * \param src Host pool flattened to (page, page elements), page-unified order + * \param src_indices Host-pool TOKEN indices, on the destination device + * \param dst_indices Device-pool token indices, same length and order + * \param layer_id Layer to read, indexing the host pool's layer axis + * \param num_layers Layer extent of the host pool's layer axis + * \param num_groups Head groups the page block is cut into; 1 when \c kIsMLA + * \param page_size Tokens per page + */ + template + static void run_one_page_unified( + const tvm::ffi::TensorView k_cache_dst, + const tvm::ffi::TensorView v_cache_dst, + const tvm::ffi::TensorView src, + const tvm::ffi::TensorView src_indices, + const tvm::ffi::TensorView dst_indices, + const int64_t layer_id, + const int64_t num_layers, + const int64_t num_groups, + const int64_t page_size) { + using namespace host; + + auto B = SymbolicSize{"page elements"}; + auto C = SymbolicSize{"layer elements per token"}; + auto N = SymbolicSize{"indices length"}; + auto cache_dtype = SymbolicDType{}; + auto indices_dtype = SymbolicDType{}; + auto device_ = SymbolicDevice{}; + + // The device pool binds the launch device: the source is host memory read + // over its pinned mapping, so it cannot name a stream. + TensorMatcher({-1, C}).with_dtype(cache_dtype).with_device(device_).verify(k_cache_dst); + if constexpr (!kIsMLA) { + TensorMatcher({-1, C}).with_dtype(cache_dtype).with_device(device_).verify(v_cache_dst); + } + TensorMatcher({-1, B}).with_dtype(cache_dtype).with_device().verify(src); + TensorMatcher({N}) + .with_dtype(indices_dtype) + .with_device(device_) + .verify(src_indices) + .verify(dst_indices); + + constexpr int64_t kComponents = kIsMLA ? 1 : 2; + RuntimeCheck(num_layers > 0 && num_groups > 0 && page_size > 0, "Page-unified load-back: invalid dimensions"); + RuntimeCheck(layer_id >= 0 && layer_id < num_layers, "Page-unified load-back: layer id out of range"); + RuntimeCheck(!kIsMLA || num_groups == 1, "Page-unified MLA load-back: expected one latent group"); + const auto dtype_size = dtype_bytes(cache_dtype.unwrap()); + RuntimeCheck( + B.unwrap() * dtype_size == num_groups * num_layers * kComponents * page_size * kElementSize, + "Page-unified load-back: page byte size mismatch"); + RuntimeCheck( + C.unwrap() * dtype_size == num_groups * kElementSize, "Page-unified load-back: device row byte size mismatch"); + RuntimeCheck( + reinterpret_cast(src.data_ptr()) % 16 == 0 && + reinterpret_cast(k_cache_dst.data_ptr()) % 16 == 0 && + (kIsMLA || reinterpret_cast(v_cache_dst.data_ptr()) % 16 == 0), + "Page-unified load-back: buffers must be 16-byte aligned"); + if (N.unwrap() == 0) { + return; + } + + const auto params = HicachePageUnifiedKernelParams{ + .k_cache_dst = k_cache_dst.data_ptr(), + .v_cache_dst = kIsMLA ? nullptr : v_cache_dst.data_ptr(), + .src = runtime::get_device_accessible_ptr(src), + .src_indices = src_indices.data_ptr(), + .dst_indices = dst_indices.data_ptr(), + .total_vecs = static_cast(N.unwrap()) * num_groups * kComponents * (kElementSize / 16), + .num_items = N.unwrap(), + .num_groups = num_groups, + .num_layers = num_layers, + .page_size = page_size, + .layer_id = layer_id, + }; + // A quota-capped persistent grid, as in the other entry points: the load + // overlaps the forward pass, so more blocks buy bandwidth by taking SMs + // from it. + const auto num_blocks = std::min(div_ceil(params.total_vecs, uint64_t{kBlockSize}), uint64_t{kBlockQuota}); + const auto use_int32 = indices_dtype.unwrap().bits == 32; + const auto kernel = use_int32 ? kernel_one_page_unified : kernel_one_page_unified; + LaunchKernel(static_cast(num_blocks), kBlockSize, device_.unwrap())(kernel, params); + } }; #undef SGL_HICACHE_KERNEL diff --git a/python/sglang/kernels/ops/kvcache/hicache.py b/python/sglang/kernels/ops/kvcache/hicache.py index 8b3ff3803d..895f9e6c78 100644 --- a/python/sglang/kernels/ops/kvcache/hicache.py +++ b/python/sglang/kernels/ops/kvcache/hicache.py @@ -494,3 +494,186 @@ def transfer_hicache_all_layer_mla_staged_lf_pf( ptr_src, page_size, ) + + +# A separate module from `_jit_hicache_module`: this one instantiates +# HiCacheKernel with `kElementSize` set to one head group's token row rather than +# the whole row, which is often not the 128-byte multiple `run_one` / `run_all` +# need. Only the exported entry points are instantiated, so those never compile +# here. +@cache_once +def _jit_hicache_page_unified_module(*, element_size: int, block_quota: int) -> Module: + args = make_cpp_args( + element_size, + 1, # kUnroll: unread by this entry point, which always moves 16B per thread + block_quota, + 1024, # num_threads, can be tuned for performance + ) + return load_jit( + "hicache_page_unified", + *args, + cuda_files=[ + "kvcacheio/hicache.cuh", + ], + cuda_wrappers=[ + ( + "launch_one_page_unified", + f"&HiCacheKernel<{args}>::run_one_page_unified", + ), + ( + "launch_one_mla_page_unified", + f"&HiCacheKernel<{args}>::run_one_page_unified", + ), + ], + ) + + +def can_use_page_unified_load_back_jit_kernel( + *, + group_bytes: int, + block_quota: int | None = None, # can be tuned for less interference +) -> bool: + logger = logging.getLogger(__name__) + if group_bytes <= 0 or group_bytes % 16 != 0: + logger.warning(f"Unsupported {group_bytes = } for page-unified JIT load-back") + return False + try: + _jit_hicache_page_unified_module( + element_size=group_bytes, + block_quota=block_quota or DEFAULT_BLOCK_QUOTA, + ) + return True + except Exception as e: + logger.warning(f"Failed to load page-unified JIT load-back kernel: {e}") + return False + + +@debug_kernel_api +def transfer_hicache_one_layer_page_unified_lf( + k_cache_dst: torch.Tensor, + v_cache_dst: torch.Tensor, + src: torch.Tensor, + src_indices: torch.Tensor, + dst_indices: torch.Tensor, + layer_id: int, + *, + block_quota: int | None = None, # can be tuned for less interference +) -> None: + """Load one layer of page_unified host pages into the device KV pool. + + ``src`` is a contiguous host tensor with layout + (page, head_group, layer, 2, page_size, head_in_group, dim); K=0, V=1 -- + the order the page-unified write-back produces. It must be pinned or + CUDA-registered, because the kernel reads it in place over its device + mapping; no staging buffer is allocated. + + ``k_cache_dst`` / ``v_cache_dst`` are one layer of the device pool, shaped + (token, head, dim) with the pool's full head count. ``src_indices`` holds + host-pool TOKEN indices (the page is ``index // page_size``) and + ``dst_indices`` the device-pool token indices they land on, in matching + order; both are CUDA int32/int64 vectors on the destination device. + One head group's token row must be a positive multiple of 16 bytes. + + Runs on the current CUDA stream. Keep ``src`` alive and unwritten until + that stream completes. + """ + if src.ndim != 7: + raise ValueError( + "Expected (page, head_group, layer, 2, page_size, head_in_group, dim)" + ) + if src.shape[3] != 2 or any(d <= 0 for d in src.shape[1:]): + raise ValueError( + "Page dimensions must be positive and the K/V dimension must be 2" + ) + _transfer_hicache_one_layer_page_unified( + k_cache_dst, + v_cache_dst, + src, + src_indices, + dst_indices, + layer_id=layer_id, + group_bytes=src.shape[5] * src.shape[6] * src.element_size(), + num_layers=src.shape[2], + num_groups=src.shape[1], + page_size=src.shape[4], + is_mla=False, + block_quota=block_quota, + ) + + +@debug_kernel_api +def transfer_hicache_one_layer_mla_page_unified_lf( + cache_dst: torch.Tensor, + src: torch.Tensor, + src_indices: torch.Tensor, + dst_indices: torch.Tensor, + layer_id: int, + *, + block_quota: int | None = None, +) -> None: + """Load one layer of compressed MLA pages in (page, layer, page_size, dim) order. + + MLA has one latent cache per layer, with no head-group or separate K/V + axes; ``dim`` covers all stored latent and positional components. + ``cache_dst`` is one layer of the device pool, shaped (token, 1, dim). + Host memory, index and stream requirements match the MHA entry point. + """ + if src.ndim != 4: + raise ValueError("Expected MLA (page, layer, page_size, dim)") + if any(d <= 0 for d in src.shape[1:]): + raise ValueError("MLA page dimensions must be positive") + _transfer_hicache_one_layer_page_unified( + cache_dst, + cache_dst, + src, + src_indices, + dst_indices, + layer_id=layer_id, + group_bytes=src.shape[3] * src.element_size(), + num_layers=src.shape[1], + num_groups=1, + page_size=src.shape[2], + is_mla=True, + block_quota=block_quota, + ) + + +def _transfer_hicache_one_layer_page_unified( + k_cache_dst: torch.Tensor, + v_cache_dst: torch.Tensor, + src: torch.Tensor, + src_indices: torch.Tensor, + dst_indices: torch.Tensor, + *, + layer_id: int, + group_bytes: int, + num_layers: int, + num_groups: int, + page_size: int, + is_mla: bool, + block_quota: int | None, +) -> None: + if not src.is_contiguous(): + raise ValueError("The host pool must be contiguous") + if src.device.type != "cpu" and not src.is_cuda: + raise ValueError("Expected a CPU or CUDA-registered host pool") + if group_bytes % 16: + raise ValueError("Each copied token row must be 16-byte aligned") + module = _jit_hicache_page_unified_module( + element_size=group_bytes, + block_quota=block_quota or DEFAULT_BLOCK_QUOTA, + ) + launch = ( + module.launch_one_mla_page_unified if is_mla else module.launch_one_page_unified + ) + launch( + k_cache_dst.view(k_cache_dst.shape[0], -1), + v_cache_dst.view(v_cache_dst.shape[0], -1), + src.view(src.shape[0], -1), + src_indices, + dst_indices, + layer_id, + num_layers, + num_groups, + page_size, + ) diff --git a/test/registered/kernels/ops/kvcache/test_hicache.py b/test/registered/kernels/ops/kvcache/test_hicache.py index 4f2fe71c4c..83c61eee76 100644 --- a/test/registered/kernels/ops/kvcache/test_hicache.py +++ b/test/registered/kernels/ops/kvcache/test_hicache.py @@ -3,7 +3,12 @@ import sys import pytest import torch -from sglang.kernels.ops.kvcache.hicache import can_use_write_back_jit_kernel +from sglang.kernels.ops.kvcache.hicache import ( + can_use_page_unified_load_back_jit_kernel, + can_use_write_back_jit_kernel, + transfer_hicache_one_layer_mla_page_unified_lf, + transfer_hicache_one_layer_page_unified_lf, +) from sglang.srt.mem_cache.l2_transfer import L2Transfer, L2TransferEngine from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool, MLATokenToKVPool from sglang.srt.mem_cache.pool_host.common import ( @@ -19,7 +24,7 @@ from sglang.srt.server_args import ServerArgs from sglang.srt.utils import is_cuda, is_hip, is_npu, is_xpu from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -register_cuda_ci(est_time=12, stage="base-b", runner_config="1-gpu-large") +register_cuda_ci(est_time=37, stage="base-b", runner_config="1-gpu-large") register_amd_ci(est_time=12, stage="jit-kernel-unit", runner_config="amd") pytestmark = pytest.mark.skipif( @@ -579,5 +584,287 @@ def test_hicache_page_first_staged_write_back_mla_staged_only_alignment() -> Non _run_page_first_staged_write_back_mla("page_first", 72, 65) +# --------------------------------------------------------------------------- +# page_unified load-back (host -> device), HiCacheKernel::run_one_page_unified +# +# Unlike the cases above, these drive the op directly rather than through a host +# pool: the layout has no pool wrapper yet. The reference is built by plain +# PyTorch indexing of the host tensor rather than by re-deriving the kernel's +# offsets, and the payload is seeded noise rather than a counter -- a counter is +# periodic in the layer stride, so a layer mix-up would compare equal. +# --------------------------------------------------------------------------- + +PAGE_UNIFIED_NUM_PAGES = 8 +PAGE_UNIFIED_SENTINEL = 255 +# Repeats a source page and is unordered; reading the same host token into two +# device slots is a real prefix-sharing case. +PAGE_UNIFIED_SRC_PATTERN = [5, 0, 3, 5, 1, 2, 4, 3] + + +def _page_unified_host_pages(shape, seed): + """Pinned host pages of exactly representable noise.""" + generator = torch.Generator().manual_seed(seed) + return torch.randint(0, 128, shape, generator=generator, dtype=torch.int32) + + +def _page_unified_index_pairs(token_count, host_tokens, device_tokens): + """Host token ids (repeats allowed, unordered) paired with distinct device ids. + + Device ids must be distinct: two items writing the same device slot would + make the expected result depend on which block happens to land last. + """ + src_ids = [i % host_tokens for i in PAGE_UNIFIED_SRC_PATTERN[:token_count]] + dst_ids = list(reversed(range(token_count))) + assert token_count <= device_tokens + return src_ids, dst_ids + + +@pytest.mark.parametrize( + "groups,layers,page_size,heads_per_group,dim", + [(1, 1, 1, 1, 16), (3, 5, 3, 2, 24), (8, 3, 64, 1, 128)], +) +@pytest.mark.parametrize("dtype", [torch.uint8, torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("index_dtype", [torch.int32, torch.int64]) +@pytest.mark.parametrize("token_count", [0, 1, 8]) +def test_page_unified_load_back( + groups, layers, page_size, heads_per_group, dim, dtype, index_dtype, token_count +): + heads = groups * heads_per_group + host_tokens = PAGE_UNIFIED_NUM_PAGES * page_size + device_tokens = host_tokens + page_size + src = ( + _page_unified_host_pages( + ( + PAGE_UNIFIED_NUM_PAGES, + groups, + layers, + 2, + page_size, + heads_per_group, + dim, + ), + 1234, + ) + .to(dtype) + .pin_memory() + ) + src_ids, dst_ids = _page_unified_index_pairs( + token_count, host_tokens, device_tokens + ) + + src_pages = torch.tensor(src_ids, dtype=index_dtype, device=DEVICE) + dst_pages = torch.tensor(dst_ids, dtype=index_dtype, device=DEVICE) + k_dst = torch.full( + (device_tokens, heads, dim), PAGE_UNIFIED_SENTINEL, dtype=dtype, device=DEVICE + ) + v_dst = torch.full_like(k_dst, PAGE_UNIFIED_SENTINEL) + + # Every layer is loaded separately: the layer stride is the axis a wrong + # permutation is most likely to get wrong, and only sweeping it catches that. + for layer_id in range(layers): + k_dst.fill_(PAGE_UNIFIED_SENTINEL) + v_dst.fill_(PAGE_UNIFIED_SENTINEL) + expected_k = torch.full_like(k_dst.cpu(), PAGE_UNIFIED_SENTINEL) + expected_v = torch.full_like(expected_k, PAGE_UNIFIED_SENTINEL) + for src_token, dst_token in zip(src_ids, dst_ids): + page, token = divmod(src_token, page_size) + for group in range(groups): + head_slice = slice( + group * heads_per_group, (group + 1) * heads_per_group + ) + expected_k[dst_token, head_slice] = src[page, group, layer_id, 0, token] + expected_v[dst_token, head_slice] = src[page, group, layer_id, 1, token] + + transfer_hicache_one_layer_page_unified_lf( + k_dst, v_dst, src, src_pages, dst_pages, layer_id + ) + torch.cuda.synchronize() + # Compare every element, including rows that should retain the sentinel. + assert torch.equal(k_dst.cpu(), expected_k) + assert torch.equal(v_dst.cpu(), expected_v) + + +@pytest.mark.parametrize( + "layers,page_size,dim", [(1, 1, 16), (5, 3, 512), (3, 64, 576)] +) +@pytest.mark.parametrize("dtype", [torch.uint8, torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("index_dtype", [torch.int32, torch.int64]) +@pytest.mark.parametrize("token_count", [0, 1, 8]) +def test_page_unified_mla_load_back( + layers, page_size, dim, dtype, index_dtype, token_count +): + host_tokens = PAGE_UNIFIED_NUM_PAGES * page_size + device_tokens = host_tokens + page_size + src = ( + _page_unified_host_pages((PAGE_UNIFIED_NUM_PAGES, layers, page_size, dim), 5678) + .to(dtype) + .pin_memory() + ) + src_ids, dst_ids = _page_unified_index_pairs( + token_count, host_tokens, device_tokens + ) + + src_pages = torch.tensor(src_ids, dtype=index_dtype, device=DEVICE) + dst_pages = torch.tensor(dst_ids, dtype=index_dtype, device=DEVICE) + dst = torch.full( + (device_tokens, 1, dim), PAGE_UNIFIED_SENTINEL, dtype=dtype, device=DEVICE + ) + + for layer_id in range(layers): + dst.fill_(PAGE_UNIFIED_SENTINEL) + expected = torch.full_like(dst.cpu(), PAGE_UNIFIED_SENTINEL) + for src_token, dst_token in zip(src_ids, dst_ids): + page, token = divmod(src_token, page_size) + expected[dst_token, 0] = src[page, layer_id, token] + + transfer_hicache_one_layer_mla_page_unified_lf( + dst, src, src_pages, dst_pages, layer_id + ) + torch.cuda.synchronize() + assert torch.equal(dst.cpu(), expected) + + +def test_page_unified_load_back_non_default_stream(): + """All work must land on the caller's stream, including the host reads.""" + groups, layers, page_size, heads_per_group, dim = 2, 3, 8, 2, 64 + heads = groups * heads_per_group + src = ( + _page_unified_host_pages( + ( + PAGE_UNIFIED_NUM_PAGES, + groups, + layers, + 2, + page_size, + heads_per_group, + dim, + ), + 99, + ) + .to(torch.bfloat16) + .pin_memory() + ) + src_ids = list(range(0, PAGE_UNIFIED_NUM_PAGES * page_size, 3)) + dst_ids = list(range(len(src_ids))) + layer_id = layers - 1 + + expected_k = torch.full( + (len(dst_ids), heads, dim), PAGE_UNIFIED_SENTINEL, dtype=torch.bfloat16 + ) + expected_v = torch.full_like(expected_k, PAGE_UNIFIED_SENTINEL) + for src_token, dst_token in zip(src_ids, dst_ids): + page, token = divmod(src_token, page_size) + for group in range(groups): + head_slice = slice(group * heads_per_group, (group + 1) * heads_per_group) + expected_k[dst_token, head_slice] = src[page, group, layer_id, 0, token] + expected_v[dst_token, head_slice] = src[page, group, layer_id, 1, token] + + stream = torch.cuda.Stream() + with torch.cuda.stream(stream): + src_pages = torch.tensor(src_ids, dtype=torch.int64, device=DEVICE) + dst_pages = torch.tensor(dst_ids, dtype=torch.int64, device=DEVICE) + k_dst = torch.full( + (len(dst_ids), heads, dim), + PAGE_UNIFIED_SENTINEL, + dtype=torch.bfloat16, + device=DEVICE, + ) + v_dst = torch.full_like(k_dst, PAGE_UNIFIED_SENTINEL) + transfer_hicache_one_layer_page_unified_lf( + k_dst, v_dst, src, src_pages, dst_pages, layer_id + ) + stream.synchronize() + assert torch.equal(k_dst.cpu(), expected_k) + assert torch.equal(v_dst.cpu(), expected_v) + + +# The C++ launcher also checks the page and device-row byte sizes against the +# compiled kElementSize, for callers that reach the module directly. Those are +# unreachable here by construction: the entry points derive every geometry +# argument from the tensor shapes they are handed. +@pytest.mark.parametrize( + "invalid,match", + [ + ("rank", "Expected \\(page"), + ("kv_axis", "K/V dimension"), + ("group_alignment", "16-byte aligned"), + ("noncontiguous", "must be contiguous"), + ("layer_id", "layer id out of range"), + ("device_heads", "device row byte size mismatch"), + ("index_length", "indices length"), + ], +) +def test_page_unified_load_back_invalid_input(invalid, match): + groups, layers, page_size, heads_per_group, dim = 2, 3, 4, 1, 16 + shape = [PAGE_UNIFIED_NUM_PAGES, groups, layers, 2, page_size, heads_per_group, dim] + if invalid == "kv_axis": + shape[3] = 3 + elif invalid == "group_alignment": + shape[6] = 7 + elif invalid == "rank": + shape.pop() + src = torch.zeros(shape, dtype=torch.float16).pin_memory() + if invalid == "noncontiguous": + src = src.transpose(1, 2).contiguous().transpose(1, 2) + + heads = groups * heads_per_group + if invalid == "device_heads": + heads += 1 + k_dst = torch.zeros((8, heads, dim), dtype=torch.float16, device=DEVICE) + v_dst = torch.zeros_like(k_dst) + src_pages = torch.zeros(2, dtype=torch.int64, device=DEVICE) + dst_pages = torch.zeros(2, dtype=torch.int64, device=DEVICE) + if invalid == "index_length": + dst_pages = dst_pages[:1] + layer_id = layers if invalid == "layer_id" else 0 + + # TVM FFI uses its own exception type for C++ RuntimeCheck failures. + with pytest.raises(Exception, match=match): + transfer_hicache_one_layer_page_unified_lf( + k_dst, v_dst, src, src_pages, dst_pages, layer_id + ) + + +@pytest.mark.parametrize( + "invalid,match", + [ + ("rank", "Expected MLA"), + ("dimension", "must be positive"), + ("alignment", "16-byte aligned"), + ("layer_id", "layer id out of range"), + ], +) +def test_page_unified_mla_load_back_invalid_input(invalid, match): + layers, page_size, dim = 3, 4, 16 + if invalid == "dimension": + dim = 0 + elif invalid == "alignment": + dim = 7 + src = torch.zeros( + (PAGE_UNIFIED_NUM_PAGES, layers, page_size, dim), dtype=torch.float16 + ) + if dim > 0: + src = src.pin_memory() + if invalid == "rank": + src = src.unsqueeze(1) + dst = torch.zeros((8, 1, max(dim, 1)), dtype=torch.float16, device=DEVICE) + src_pages = torch.zeros(2, dtype=torch.int64, device=DEVICE) + dst_pages = torch.zeros(2, dtype=torch.int64, device=DEVICE) + layer_id = layers if invalid == "layer_id" else 0 + + with pytest.raises(Exception, match=match): + transfer_hicache_one_layer_mla_page_unified_lf( + dst, src, src_pages, dst_pages, layer_id + ) + + +@pytest.mark.parametrize("group_bytes,expected", [(0, False), (8, False), (256, True)]) +def test_can_use_page_unified_load_back_jit_kernel(group_bytes, expected): + """The probe owns the alignment precondition and compiles the specialisation.""" + assert ( + can_use_page_unified_load_back_jit_kernel(group_bytes=group_bytes) is expected + ) + + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v", "-s"]))