diff --git a/python/sglang/kernels/jit/csrc/kvcacheio/relayout.cuh b/python/sglang/kernels/jit/csrc/kvcacheio/relayout.cuh index d17a09d8d3..42695df8c2 100644 --- a/python/sglang/kernels/jit/csrc/kvcacheio/relayout.cuh +++ b/python/sglang/kernels/jit/csrc/kvcacheio/relayout.cuh @@ -87,4 +87,47 @@ inline void launch_hicache_relayout_kernel( LaunchKernel(static_cast(grid), kRelayoutBlockSize, device)(kernel, params); } +struct HicachePageUnifiedRelayoutParams { + void* staging; + const void* k_ptr_src; + const void* v_ptr_src; + const void* src_pages; + uint64_t total_vecs; + int64_t num_groups; + int64_t num_layers; + int64_t page_size; +}; + +// Stage in destination order: (page, group, layer, kv, token, head, dim). +// MLA omits the group, K/V and head axes: (page, layer, token, dim). +// Each thread moves 16 bytes; adjacent threads write adjacent staging vectors. +// Source layers contain contiguous (token, group * head_in_group, dim) rows. +template +__global__ void hicache_page_unified_relayout_kernel(const __grid_constant__ HicachePageUnifiedRelayoutParams p) { + static_assert(kGroupBytes > 0 && kGroupBytes % 16 == 0); + constexpr int64_t kGroupVecs = kGroupBytes / 16; + constexpr int64_t kComponents = kIsMLA ? 1 : 2; + const int64_t num_groups = kIsMLA ? 1 : p.num_groups; + const auto tid = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + const auto stride = static_cast(gridDim.x) * blockDim.x; + for (uint64_t i = tid; i < p.total_vecs; i += stride) { + auto remaining = i / kGroupVecs; + const auto vec = i % kGroupVecs; + const auto token = remaining % p.page_size; + remaining /= p.page_size; + const auto kv = remaining % kComponents; + remaining /= kComponents; + const auto layer = remaining % p.num_layers; + remaining /= p.num_layers; + const auto group = remaining % num_groups; + const auto page = remaining / num_groups; + const auto src_page = static_cast(p.src_pages)[page]; + const auto src_token = static_cast(src_page) * p.page_size + token; + const auto ptrs = static_cast(kv == 0 ? p.k_ptr_src : p.v_ptr_src); + const auto src = device::pointer::offset(ptrs[layer], (src_token * num_groups + group) * kGroupBytes + vec * 16); + const auto value = device::details::load_nc(static_cast(src)); + device::details::store_nc(static_cast(p.staging) + i, value); + } +} + } // namespace sglang diff --git a/python/sglang/kernels/jit/csrc/kvcacheio/staged_write_back.cuh b/python/sglang/kernels/jit/csrc/kvcacheio/staged_write_back.cuh index 93322643a4..6cb8b16da7 100644 --- a/python/sglang/kernels/jit/csrc/kvcacheio/staged_write_back.cuh +++ b/python/sglang/kernels/jit/csrc/kvcacheio/staged_write_back.cuh @@ -325,4 +325,72 @@ struct HiCacheStagedWriteBackKernel { } }; +template +struct HiCachePageUnifiedWriteBackKernel { + static void + run(const tvm::ffi::TensorView dst, + const tvm::ffi::TensorView staging, + const tvm::ffi::TensorView k_ptr_src, + const tvm::ffi::TensorView v_ptr_src, + const tvm::ffi::TensorView src_pages, + const tvm::ffi::TensorView dst_pages, + const int64_t num_groups, + const int64_t page_size) { + using namespace host; + auto P = SymbolicSize{"page count"}; + auto L = SymbolicSize{"layer count"}; + auto B = SymbolicSize{"page elements"}; + auto dtype = SymbolicDType{}; + auto index_dtype = SymbolicDType{}; + auto device = SymbolicDevice{}; + TensorMatcher({L}).with_dtype().with_device(device).verify(k_ptr_src); + if constexpr (!kIsMLA) { + TensorMatcher({L}).with_dtype().with_device(device).verify(v_ptr_src); + } + TensorMatcher({P}).with_dtype(index_dtype).with_device(device).verify(src_pages); + TensorMatcher({P}).with_dtype().with_device().verify(dst_pages); + TensorMatcher({P, B}).with_dtype(dtype).with_device(device).verify(staging); + TensorMatcher({-1, B}).with_dtype(dtype).with_device().verify(dst); + RuntimeCheck(num_groups > 0 && page_size > 0 && L.unwrap() > 0, "Page-unified write-back: invalid dimensions"); + RuntimeCheck(!kIsMLA || num_groups == 1, "Page-unified MLA write-back: expected one latent group"); + constexpr int64_t kComponents = kIsMLA ? 1 : 2; + RuntimeCheck( + B.unwrap() * dtype_bytes(dtype.unwrap()) == num_groups * L.unwrap() * kComponents * page_size * kGroupBytes, + "Page-unified write-back: page byte size mismatch"); + RuntimeCheck(reinterpret_cast(staging.data_ptr()) % 16 == 0, "Staging must be 16-byte aligned"); + const auto dst_ids = static_cast(dst_pages.data_ptr()); + for (int64_t i = 0; i < P.unwrap(); ++i) { + RuntimeCheck( + dst_ids[i] >= 0 && dst_ids[i] < dst.size(0), "Page-unified write-back: destination page out of range"); + } + if (P.unwrap() == 0) { + return; + } + const auto params = HicachePageUnifiedRelayoutParams{ + .staging = staging.data_ptr(), + .k_ptr_src = k_ptr_src.data_ptr(), + .v_ptr_src = kIsMLA ? nullptr : v_ptr_src.data_ptr(), + .src_pages = src_pages.data_ptr(), + .total_vecs = static_cast(P.unwrap()) * B.unwrap() * dtype_bytes(dtype.unwrap()) / 16, + .num_groups = num_groups, + .num_layers = L.unwrap(), + .page_size = page_size, + }; + // Grid-stride traversal keeps large transfers within CUDA's grid limits. + constexpr uint32_t kThreads = 256; + const auto blocks = + static_cast(std::min(div_ceil(params.total_vecs, uint64_t{kThreads}), 65535)); + const auto kernel = index_dtype.unwrap().bits == 32 + ? hicache_page_unified_relayout_kernel + : hicache_page_unified_relayout_kernel; + const auto dev = device.unwrap(); + LaunchKernel(blocks, kThreads, dev)(kernel, params); + const auto stream = LaunchKernel::resolve_device(dev); + // The flattened tensors have one row per page (a single latent cache for MLA). + if (!try_copy_page_first_pages_batch({staging}, {dst}, dst_ids, P.unwrap(), 1, dev.device_id, stream)) { + copy_page_first_pages_fallback({staging}, {dst}, dst_ids, P.unwrap(), 1, stream); + } + } +}; + } // namespace sglang diff --git a/python/sglang/kernels/ops/kvcache/hicache.py b/python/sglang/kernels/ops/kvcache/hicache.py index 895f9e6c78..ff2f47482e 100644 --- a/python/sglang/kernels/ops/kvcache/hicache.py +++ b/python/sglang/kernels/ops/kvcache/hicache.py @@ -677,3 +677,151 @@ def _transfer_hicache_one_layer_page_unified( num_groups, page_size, ) + + +@cache_once +def _jit_page_unified_write_back(group_bytes: int, *, is_mla: bool = False) -> Module: + args = make_cpp_args(group_bytes, is_mla) + return load_jit( + "hicache_page_unified_write_back", + *args, + cuda_files=["kvcacheio/staged_write_back.cuh"], + cuda_wrappers=[("launch", f"&HiCachePageUnifiedWriteBackKernel<{args}>::run")], + ) + + +@debug_kernel_api +def transfer_hicache_all_layer_staged_lf_page_unified( + k_ptr_src: torch.Tensor, + v_ptr_src: torch.Tensor, + src_pages: torch.Tensor, + dst_pages: torch.Tensor, + staging: torch.Tensor, + dst: torch.Tensor, +) -> None: + """Write full GPU KV pages to pinned host memory in page_unified layout. + + ``staging`` and ``dst`` are contiguous tensors with layout + (page, head_group, layer, 2, page_size, head_in_group, dim); K=0, V=1. + Only their page capacities may differ. Each source pointer addresses a + contiguous (token, head_group * head_in_group, dim) GPU tensor of the same + dtype as ``dst``, with a 16-byte-aligned base. Pointer tables are contiguous + CUDA uint64 tensors, one pointer per layer, on the staging device. + + ``src_pages`` contains CUDA int32/int64 physical PAGE IDs, not token offsets. + ``dst_pages`` contains CPU int64 physical page IDs, in matching order. + Source IDs must be in bounds; destination IDs must be unique. The caller + owns the source allocations, which cannot be validated via pointer tables. + One group's token data must be a positive multiple of 16 bytes. + + Reuses caller-provided staging in chunks on the current CUDA stream. + Keep all inputs alive and staging exclusive until that stream completes; + wait for completion before reading the host output. No GPU buffer is allocated. + """ + if dst.ndim != 7 or staging.ndim != 7: + raise ValueError( + "Expected (page, head_group, layer, 2, page_size, head_in_group, dim)" + ) + if dst.shape[3] != 2 or any(d <= 0 for d in dst.shape[1:]): + raise ValueError( + "Page dimensions must be positive and the K/V dimension must be 2" + ) + _transfer_hicache_all_layer_staged_page_unified( + k_ptr_src, + v_ptr_src, + src_pages, + dst_pages, + staging, + dst, + group_bytes=dst.shape[5] * dst.shape[6] * dst.element_size(), + num_groups=dst.shape[1], + page_size=dst.shape[4], + is_mla=False, + ) + + +@debug_kernel_api +def transfer_hicache_all_layer_mla_staged_lf_page_unified( + ptr_src: torch.Tensor, + src_pages: torch.Tensor, + dst_pages: torch.Tensor, + staging: torch.Tensor, + dst: torch.Tensor, +) -> None: + """Write compressed MLA KV pages in (page, layer, page_size, dim) order. + + MLA has one latent cache per layer, with no head-group or separate K/V + axes. Source pointers address contiguous (token, dim) tensors; ``dim`` + includes all stored latent and positional components. No dtype conversion + is performed. Each token row must be a positive multiple of 16 bytes. + + Pointer tables, page IDs, pinned destination memory, staging reuse and + stream lifetime requirements match the MHA page_unified entry point. + ``src_pages`` and ``dst_pages`` contain physical page IDs, not token offsets. + """ + if dst.ndim != 4 or staging.ndim != 4: + raise ValueError("Expected MLA (page, layer, page_size, dim)") + if any(d <= 0 for d in dst.shape[1:]): + raise ValueError("MLA page dimensions must be positive") + _transfer_hicache_all_layer_staged_page_unified( + ptr_src, + ptr_src, + src_pages, + dst_pages, + staging, + dst, + group_bytes=dst.shape[3] * dst.element_size(), + num_groups=1, + page_size=dst.shape[2], + is_mla=True, + ) + + +def _transfer_hicache_all_layer_staged_page_unified( + k_ptr_src: torch.Tensor, + v_ptr_src: torch.Tensor, + src_pages: torch.Tensor, + dst_pages: torch.Tensor, + staging: torch.Tensor, + dst: torch.Tensor, + *, + group_bytes: int, + num_groups: int, + page_size: int, + is_mla: bool, +) -> None: + if staging.shape[1:] != dst.shape[1:] or staging.dtype != dst.dtype: + raise ValueError( + "Staging and destination must have matching page shapes and dtype" + ) + if not staging.is_contiguous() or not dst.is_contiguous(): + raise ValueError("Staging and destination must be contiguous") + if not staging.is_cuda or dst.device.type != "cpu" or not dst.is_pinned(): + raise ValueError("Expected CUDA staging and a pinned CPU destination") + if staging.shape[0] == 0: + raise ValueError("Staging must hold at least one page") + if ( + src_pages.ndim != 1 + or dst_pages.ndim != 1 + or src_pages.numel() != dst_pages.numel() + ): + raise ValueError("Source and destination page IDs must be equal-length vectors") + if group_bytes % 16: + raise ValueError("Each copied token row must be 16-byte aligned") + module = _jit_page_unified_write_back(group_bytes, is_mla=is_mla) + capacity = staging.shape[0] + page_elements = staging[0].numel() + staging_flat = staging.view(capacity, page_elements) + dst_flat = dst.view(dst.shape[0], page_elements) + for begin in range(0, src_pages.numel(), capacity): + end = min(begin + capacity, src_pages.numel()) + module.launch( + dst_flat, + staging_flat[: end - begin], + k_ptr_src, + v_ptr_src, + src_pages[begin:end], + dst_pages[begin:end], + num_groups, + page_size, + ) diff --git a/test/registered/kernels/ops/kvcache/test_hicache_page_first_write_back.py b/test/registered/kernels/ops/kvcache/test_hicache_page_first_write_back.py index 73499e35c5..0d61e20674 100644 --- a/test/registered/kernels/ops/kvcache/test_hicache_page_first_write_back.py +++ b/test/registered/kernels/ops/kvcache/test_hicache_page_first_write_back.py @@ -13,20 +13,11 @@ import sys import pytest import torch -from sglang.kernels.ops.kvcache.hicache import can_use_write_back_jit_kernel -from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool, MLATokenToKVPool -from sglang.srt.mem_cache.pool_host import common as pool_host_common -from sglang.srt.mem_cache.pool_host.common import ( - ALLOC_MEMORY_FUNCS, - HostTensorAllocator, - _cuda_host_unregister, - _resolve_device_accessible_ptr_fn, - alloc_with_host_register, - alloc_with_pin_memory, - make_kernel_ptr_table, +from sglang.kernels.ops.kvcache.hicache import ( + can_use_write_back_jit_kernel, + transfer_hicache_all_layer_mla_staged_lf_page_unified, + transfer_hicache_all_layer_staged_lf_page_unified, ) -from sglang.srt.mem_cache.pool_host.mha import MHATokenToKVPoolHost -from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost from sglang.srt.utils import is_cuda, is_hip, is_npu, is_xpu from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci @@ -69,6 +60,11 @@ def _token_indices_for_pages( def _pinned_host_pool(host_pool_cls, **kwargs): + from sglang.srt.mem_cache.pool_host.common import ( + ALLOC_MEMORY_FUNCS, + alloc_with_pin_memory, + ) + original_alloc = ALLOC_MEMORY_FUNCS[DEVICE] ALLOC_MEMORY_FUNCS[DEVICE] = alloc_with_pin_memory try: @@ -114,6 +110,9 @@ def _assert_pages_equal(host_ref, device_ref, host_pages, device_pages) -> None: def _run_mha(element_dim: int, page_count: int) -> None: + from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool + from sglang.srt.mem_cache.pool_host.mha import MHATokenToKVPoolHost + pool_size = PAGE_SIZE * (page_count + 8) device_pool = MHATokenToKVPool( size=pool_size, @@ -197,6 +196,9 @@ def _run_mha(element_dim: int, page_count: int) -> None: def _run_mla(element_dim: int, page_count: int) -> None: + from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool + from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost + pool_size = PAGE_SIZE * (page_count + 8) device_pool = MLATokenToKVPool( size=pool_size, @@ -273,6 +275,197 @@ def test_page_first_staged_write_back_mla(element_dim: int, page_count: int) -> _run_mla(element_dim, page_count) +@pytest.mark.parametrize( + "groups,layers,page_size,heads_per_group,dim", + [(1, 1, 1, 1, 16), (3, 5, 3, 2, 24), (8, 3, 64, 1, 128)], +) +@pytest.mark.parametrize("dtype", [torch.uint8, torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("index_dtype", [torch.int32, torch.int64]) +@pytest.mark.parametrize("page_count", [0, 1, 2, 5]) +def test_page_unified_staged_write_back( + groups, layers, page_size, heads_per_group, dim, dtype, index_dtype, page_count +): + # Capacity 2 covers an exact chunk, repeated reuse, and a short final chunk. + # Small and >=128 KiB pages exercise the fallback/batch-copy size branches + # (the batch API additionally requires a supported CUDA runtime/driver). + shape = (groups, layers, 2, page_size, heads_per_group, dim) + generator = torch.Generator().manual_seed(1234) + source_cpu = [ + [ + torch.randint( + 0, + 128, + (6 * page_size, groups * heads_per_group, dim), + generator=generator, + dtype=torch.int32, + ).to(dtype) + for _ in range(layers) + ] + for _ in range(2) + ] + src_ids = [4, 0, 3, 4, 1][:page_count] # Includes a repeated source page. + dst_ids = [6, 2, 0, 5, 3][:page_count] + dst = torch.full((8, *shape), 255, dtype=dtype, pin_memory=True) + expected = dst.clone() + for src_page, dst_page in zip(src_ids, dst_ids): + for group in range(groups): + for layer in range(layers): + for kv in range(2): + expected[dst_page, group, layer, kv].copy_( + source_cpu[kv][layer][ + src_page * page_size : (src_page + 1) * page_size, + group * heads_per_group : (group + 1) * heads_per_group, + ] + ) + + # All GPU work uses a non-default stream; only synchronize after the call. + # This also checks ordering when the same staging allocation is reused. + stream = torch.cuda.Stream() + with torch.cuda.stream(stream): + source_gpu = [[t.to(DEVICE) for t in component] for component in source_cpu] + ptrs = [ + torch.tensor( + [t.data_ptr() for t in component], dtype=torch.uint64, device=DEVICE + ) + for component in source_gpu + ] + src_pages = torch.tensor(src_ids, dtype=index_dtype, device=DEVICE) + dst_pages = torch.tensor(dst_ids, dtype=torch.int64) + staging = torch.empty((2, *shape), dtype=dtype, device=DEVICE) + transfer_hicache_all_layer_staged_lf_page_unified( + *ptrs, src_pages, dst_pages, staging, dst + ) + stream.synchronize() + # Compare every element, including pages that should retain the sentinel. + assert torch.equal(dst, expected) + + +@pytest.mark.parametrize( + "invalid,match", + [ + ("kv_axis", "K/V dimension"), + ("group_alignment", "16-byte aligned"), + ("staging_capacity", "at least one page"), + ("page_shape", "matching page shapes"), + ("noncontiguous", "must be contiguous"), + ("page_count", "equal-length vectors"), + ("dst_page", "destination page out of range"), + ("layer_count", "page byte size mismatch"), + ], +) +def test_page_unified_write_back_invalid_input(invalid, match): + shape = (2, 3, 2, 4, 1, 16) + if invalid == "kv_axis": + shape = (2, 3, 3, 4, 1, 16) + elif invalid == "group_alignment": + shape = (2, 3, 2, 4, 1, 7) + staging = torch.empty((1, *shape), dtype=torch.float16, device=DEVICE) + dst = torch.empty((2, *shape), dtype=torch.float16, pin_memory=True) + # Validation must reject the inputs before any pointer is dereferenced. + ptrs = torch.zeros(3, dtype=torch.uint64, device=DEVICE) + src_pages = torch.zeros(1, dtype=torch.int64, device=DEVICE) + dst_pages = torch.zeros(1, dtype=torch.int64) + if invalid == "staging_capacity": + staging = staging[:0] + elif invalid == "page_shape": + staging = staging[:, :1] + elif invalid == "noncontiguous": + staging = staging.transpose(1, 2).contiguous().transpose(1, 2) + elif invalid == "page_count": + dst_pages = dst_pages[:0] + elif invalid == "dst_page": + dst_pages.fill_(dst.shape[0]) + elif invalid == "layer_count": + ptrs = ptrs[:2] + # TVM FFI uses its own exception type for C++ RuntimeCheck failures. + with pytest.raises(Exception, match=match): + transfer_hicache_all_layer_staged_lf_page_unified( + ptrs, ptrs, src_pages, dst_pages, staging, dst + ) + + +@pytest.mark.parametrize( + "layers,page_size,dim", [(1, 1, 16), (5, 3, 512), (3, 64, 576)] +) +@pytest.mark.parametrize("dtype", [torch.uint8, torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("index_dtype", [torch.int32, torch.int64]) +@pytest.mark.parametrize("page_count", [0, 1, 2, 5]) +def test_page_unified_mla_staged_write_back( + layers, page_size, dim, dtype, index_dtype, page_count +): + generator = torch.Generator().manual_seed(5678) + source_cpu = [ + torch.randint(0, 128, (6 * page_size, dim), generator=generator).to(dtype) + for _ in range(layers) + ] + src_ids = [4, 0, 3, 4, 1][:page_count] + dst_ids = [6, 2, 0, 5, 3][:page_count] + shape = (layers, page_size, dim) + dst = torch.full((8, *shape), 255, dtype=dtype, pin_memory=True) + expected = dst.clone() + for src_page, dst_page in zip(src_ids, dst_ids): + for layer in range(layers): + expected[dst_page, layer].copy_( + source_cpu[layer][src_page * page_size : (src_page + 1) * page_size] + ) + stream = torch.cuda.Stream() + with torch.cuda.stream(stream): + source_gpu = [t.to(DEVICE) for t in source_cpu] + ptrs = torch.tensor( + [t.data_ptr() for t in source_gpu], dtype=torch.uint64, device=DEVICE + ) + src_pages = torch.tensor(src_ids, dtype=index_dtype, device=DEVICE) + dst_pages = torch.tensor(dst_ids, dtype=torch.int64) + staging = torch.empty((2, *shape), dtype=dtype, device=DEVICE) + transfer_hicache_all_layer_mla_staged_lf_page_unified( + ptrs, src_pages, dst_pages, staging, dst + ) + stream.synchronize() + assert torch.equal(dst, expected) + + +@pytest.mark.parametrize( + "invalid,match", + [ + ("rank", "Expected MLA"), + ("dimension", "must be positive"), + ("alignment", "16-byte aligned"), + ("staging_capacity", "at least one page"), + ("page_shape", "matching page shapes"), + ("noncontiguous", "must be contiguous"), + ("dst_page", "destination page out of range"), + ("layer_count", "page byte size mismatch"), + ], +) +def test_page_unified_mla_write_back_invalid_input(invalid, match): + shape = (3, 4, 16) + if invalid == "dimension": + shape = (3, 4, 0) + elif invalid == "alignment": + shape = (3, 4, 7) + staging = torch.empty((1, *shape), dtype=torch.float16, device=DEVICE) + dst = torch.empty((2, *shape), dtype=torch.float16, pin_memory=True) + ptrs = torch.zeros(3, dtype=torch.uint64, device=DEVICE) + src_pages = torch.zeros(1, dtype=torch.int64, device=DEVICE) + dst_pages = torch.zeros(1, dtype=torch.int64) + if invalid == "rank": + staging = staging.unsqueeze(1) + elif invalid == "staging_capacity": + staging = staging[:0] + elif invalid == "page_shape": + staging = staging[:, :1] + elif invalid == "noncontiguous": + staging = staging.transpose(1, 2).contiguous().transpose(1, 2) + elif invalid == "dst_page": + dst_pages.fill_(-1) + elif invalid == "layer_count": + ptrs = ptrs[:2] + with pytest.raises(Exception, match=match): + transfer_hicache_all_layer_mla_staged_lf_page_unified( + ptrs, src_pages, dst_pages, staging, dst + ) + + @pytest.mark.skipif( is_hip(), reason="ROCm maps registered host memory at a distinct device address.", @@ -282,6 +475,15 @@ def test_registered_mmap_kernel_ptr_table_fallback_matches_device_alias( ) -> None: """CUDA maps registered host memory at the host address itself; ``make_kernel_ptr_table``'s raw-host-address fallback depends on it.""" + from sglang.srt.mem_cache.pool_host import common as pool_host_common + from sglang.srt.mem_cache.pool_host.common import ( + HostTensorAllocator, + _cuda_host_unregister, + _resolve_device_accessible_ptr_fn, + alloc_with_host_register, + make_kernel_ptr_table, + ) + if _resolve_device_accessible_ptr_fn() is None: pytest.skip( "installed sglang-kernel has no get_device_accessible_ptr; " @@ -307,6 +509,10 @@ def test_registered_mmap_kernel_ptr_table_fallback_matches_device_alias( def test_registered_mmap_pointer_domains_and_all_layer_transfer() -> None: + from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool + from sglang.srt.mem_cache.pool_host.common import _resolve_device_accessible_ptr_fn + from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost + get_device_accessible_ptr = _resolve_device_accessible_ptr_fn() if get_device_accessible_ptr is None: pytest.skip( @@ -366,6 +572,10 @@ def test_registered_mmap_pointer_domains_and_all_layer_transfer() -> None: def test_registered_mmap_page_first_kernel_operands_and_graph( pool_kind: str, ) -> None: + from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool, MLATokenToKVPool + from sglang.srt.mem_cache.pool_host.mha import MHATokenToKVPoolHost + from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost + if pool_kind == "mha": device_pool = MHATokenToKVPool( size=PAGE_SIZE * 4,