From b6ba1483848dda193c18f9cf9d0cb52f0de60d34 Mon Sep 17 00:00:00 2001 From: Ke Bao Date: Mon, 12 Oct 2026 15:40:06 +0800 Subject: [PATCH] Capture KV host transfer buffers (#43711) --- python/sglang/srt/mem_cache/pool_host/mha.py | 195 ++++++++---------- python/sglang/srt/mem_cache/pool_host/mla.py | 122 +++++------ .../pool_host/test_kvcacheio_asymmetric.py | 19 +- .../test_asymmetric_mha_pool_host_unit.py | 18 +- .../mem_cache/test_hicache_dcp_host_pool.py | 87 +++++++- .../mem_cache/test_hicache_host_register.py | 3 + .../test_hicache_seaweedfs_storage.py | 10 +- ...test_hicache_staged_write_back_dispatch.py | 11 +- .../unit/mem_cache/test_mem_pool_host.py | 84 ++++++++ .../test_mla_host_dedup_primitives.py | 4 + 10 files changed, 366 insertions(+), 187 deletions(-) diff --git a/python/sglang/srt/mem_cache/pool_host/mha.py b/python/sglang/srt/mem_cache/pool_host/mha.py index 331eb32398..51ab281630 100644 --- a/python/sglang/srt/mem_cache/pool_host/mha.py +++ b/python/sglang/srt/mem_cache/pool_host/mha.py @@ -123,9 +123,11 @@ def prepare_mha_write_back_staging( class MHATokenToKVPoolHost(HostKVCache): - device_pool: MHATokenToKVPool | None = None + device_pool: MHATokenToKVPool mtp_draft_device_pools: tuple[MHATokenToKVPool, ...] = () + _transfer_device: torch.device + def __init__( self, device_pool: MHATokenToKVPool, @@ -142,8 +144,20 @@ class MHATokenToKVPoolHost(HostKVCache): ): self.mtp_draft_device_pools = tuple(mtp_draft_device_pools) self.target_layer_num = device_pool.layer_num + # Transfer buffers are captured below. The pool remains available for + # NPU transfer APIs and device layer ownership. self.device_pool = device_pool + self._transfer_device = torch.device(device_pool.device) self.dtype = device_pool.store_dtype + self.head_dim = device_pool.head_dim + self.v_head_dim = device_pool.v_head_dim + # Equal-width pools may store multiple attention rows per allocator token. + self.head_num = ( + device_pool.row_dim // self.head_dim + if self.head_dim == self.v_head_dim + else device_pool.head_num + ) + self.layer_num = self.target_layer_num + len(self.mtp_draft_device_pools) # Scale-bearing subclasses need page coverage when computing token bytes. self.page_size = page_size device_capacity = getattr(device_pool, "host_capacity_tokens", None) @@ -164,14 +178,14 @@ class MHATokenToKVPoolHost(HostKVCache): allocator_type=allocator_type, pool_label=pool_label, ) + pools = (self.device_pool, *self.mtp_draft_device_pools) + self.device_k_buffers = tuple(b for pool in pools for b in pool.k_buffer) + self.device_v_buffers = tuple(b for pool in pools for b in pool.v_buffer) + self._initialize_transfer_views() + + def _initialize_transfer_views(self) -> None: self.element_dim = self.head_num * self.head_dim - self._init_device_row_stride( - buf - for pool in (self.device_pool, *self.mtp_draft_device_pools) - if pool is not None - for side in ("k_buffer", "v_buffer") - for buf in getattr(pool, side, None) or () - ) + self._init_device_row_stride((*self.device_k_buffers, *self.device_v_buffers)) # The JIT HiCache kernels also build with hipcc (ROCm): the PTX-only # helpers in hicache.cuh are guarded by USE_ROCM and the staged # write-back kernel has a ROCm path, so enable them on HIP too. This @@ -193,44 +207,36 @@ class MHATokenToKVPoolHost(HostKVCache): self.v_data_refs = [self.v_buffer[i] for i in range(self.layer_num)] self.k_data_ptrs = make_kernel_ptr_table( self.k_data_refs, - self.device_pool.device, + self._transfer_device, host_memory_registered=self.pin_memory, ) self.v_data_ptrs = make_kernel_ptr_table( self.v_data_refs, - self.device_pool.device, + self._transfer_device, host_memory_registered=self.pin_memory, ) - if self.mtp_draft_device_pools and not _is_npu: - device_pools = (self.device_pool, *self.mtp_draft_device_pools) - if not _is_npu: - self.packed_device_k_data_ptrs = torch.cat( - [pool.k_data_ptrs for pool in device_pools] - ) - self.packed_device_v_data_ptrs = torch.cat( - [pool.v_data_ptrs for pool in device_pools] - ) - else: - self.packed_device_k_data_ptrs = None - self.packed_device_v_data_ptrs = None - self.packed_device_k_buffers = [ - buffer for pool in device_pools for buffer in pool.k_buffer - ] - self.packed_device_v_buffers = [ - buffer for pool in device_pools for buffer in pool.v_buffer - ] - self.packed_device_kv_buffers = ( - self.packed_device_k_buffers + self.packed_device_v_buffers + if not _is_npu: + pools = (self.device_pool, *self.mtp_draft_device_pools) + self.device_k_data_ptrs = ( + torch.cat([pool.k_data_ptrs for pool in pools]) + if self.mtp_draft_device_pools + else self.device_pool.k_data_ptrs + ) + self.device_v_data_ptrs = ( + torch.cat([pool.v_data_ptrs for pool in pools]) + if self.mtp_draft_device_pools + else self.device_pool.v_data_ptrs ) self.host_kv_data_refs = self.k_data_refs + self.v_data_refs self._init_write_back_staging_buffers() def get_size_per_token(self): - # One allocator token may hold multiple attention rows. - self.head_num = self.device_pool.row_dim // self.device_pool.head_dim - self.head_dim = self.device_pool.head_dim - self.layer_num = self.target_layer_num + len(self.mtp_draft_device_pools) - return self.head_dim * self.head_num * self.layer_num * self.dtype.itemsize * 2 + return ( + (self.head_dim + self.v_head_dim) + * self.head_num + * self.layer_num + * self.dtype.itemsize + ) def get_hybrid_pool_buffer(self): # Expose the K/V host tensors required for zero-copy I/O registration. @@ -267,7 +273,7 @@ class MHATokenToKVPoolHost(HostKVCache): self.token_stride_size = self.head_num * self.head_dim * self.dtype.itemsize self.layout_dim = self.token_stride_size * self.layer_num - alloc_func = ALLOC_MEMORY_FUNCS[self.device_pool.device] + alloc_func = ALLOC_MEMORY_FUNCS[self._transfer_device.type] buffer = alloc_func( dims, dtype=self.dtype, @@ -330,22 +336,22 @@ class MHATokenToKVPoolHost(HostKVCache): *, is_draft: bool = False, ): - if self.device_pool is not None: - if not is_draft and not self._is_device_layer_owned(device_pool, layer_id): - return - # MTP draft layers do not participate in CP layer sharding. - host_layer_id = layer_id if is_draft else self._host_layer_index(layer_id) - device_layer_id = 0 if is_draft else layer_id - else: - host_layer_id = device_layer_id = layer_id + if not is_draft and not self._is_device_layer_owned(device_pool, layer_id): + return + # MTP draft layers do not participate in CP layer sharding. + host_layer_id = layer_id if is_draft else self._host_layer_index(layer_id) + + if io_backend != "kernel_ascend": + device_k = self.device_k_buffers[layer_id] + device_v = self.device_v_buffers[layer_id] if io_backend == "kernel": if self.layout == "layer_first": if self.can_use_jit: jit_transfer_hicache_one_layer( page_size=self.page_size, - k_cache_dst=device_pool.k_buffer[device_layer_id], - v_cache_dst=device_pool.v_buffer[device_layer_id], + k_cache_dst=device_k, + v_cache_dst=device_v, k_cache_src=self.k_buffer[host_layer_id], v_cache_src=self.v_buffer[host_layer_id], indices_dst=device_indices, @@ -356,9 +362,9 @@ class MHATokenToKVPoolHost(HostKVCache): self._require_packed_device_rows("transfer_kv_per_layer") transfer_kv_per_layer( src_k=self.k_buffer[host_layer_id], - dst_k=device_pool.k_buffer[device_layer_id], + dst_k=device_k, src_v=self.v_buffer[host_layer_id], - dst_v=device_pool.v_buffer[device_layer_id], + dst_v=device_v, src_indices=host_indices, dst_indices=device_indices, item_size=self.token_stride_size, @@ -370,8 +376,8 @@ class MHATokenToKVPoolHost(HostKVCache): # The kernel handles different src/dst strides automatically. jit_transfer_hicache_one_layer( page_size=self.page_size, - k_cache_dst=device_pool.k_buffer[device_layer_id], - v_cache_dst=device_pool.v_buffer[device_layer_id], + k_cache_dst=device_k, + v_cache_dst=device_v, k_cache_src=self.k_data_refs[host_layer_id], v_cache_src=self.v_data_refs[host_layer_id], indices_dst=device_indices, @@ -382,9 +388,9 @@ class MHATokenToKVPoolHost(HostKVCache): self._require_packed_device_rows("transfer_kv_per_layer_pf_lf") transfer_kv_per_layer_pf_lf( src_k=self.k_buffer, - dst_k=device_pool.k_buffer[device_layer_id], + dst_k=device_k, src_v=self.v_buffer, - dst_v=device_pool.v_buffer[device_layer_id], + dst_v=device_v, src_indices=host_indices, dst_indices=device_indices, layer_id=host_layer_id, @@ -395,9 +401,9 @@ class MHATokenToKVPoolHost(HostKVCache): self._require_packed_device_rows("transfer_kv_per_layer_ph_lf") transfer_kv_per_layer_ph_lf( src_k=self.k_buffer, - dst_k=device_pool.k_buffer[device_layer_id], + dst_k=device_k, src_v=self.v_buffer, - dst_v=device_pool.v_buffer[device_layer_id], + dst_v=device_v, src_indices=host_indices, dst_indices=device_indices, layer_id=host_layer_id, @@ -416,8 +422,8 @@ class MHATokenToKVPoolHost(HostKVCache): self.v_buffer[host_layer_id], ], dst_layers=[ - device_pool.k_buffer[device_layer_id], - device_pool.v_buffer[device_layer_id], + device_k, + device_v, ], src_indices=host_indices, dst_indices=device_indices, @@ -427,8 +433,8 @@ class MHATokenToKVPoolHost(HostKVCache): transfer_kv_per_layer_direct_pf_lf( src_ptrs=[self.k_buffer, self.v_buffer], dst_ptrs=[ - device_pool.k_buffer[device_layer_id], - device_pool.v_buffer[device_layer_id], + device_k, + device_v, ], src_indices=host_indices, dst_indices=device_indices, @@ -462,21 +468,6 @@ class MHATokenToKVPoolHost(HostKVCache): else: raise ValueError(f"Unsupported IO backend: {io_backend}") - def _resolve_device_transfer_buffers(self, device_pool): - if self.mtp_draft_device_pools: - return ( - self.packed_device_k_data_ptrs, - self.packed_device_v_data_ptrs, - self.packed_device_k_buffers, - self.packed_device_v_buffers, - ) - return ( - device_pool.k_data_ptrs, - device_pool.v_data_ptrs, - device_pool.k_buffer, - device_pool.v_buffer, - ) - def _npu_transfer_buffers(self, target_device_pool): layer_start = 0 for pool in (target_device_pool, *self.mtp_draft_device_pools): @@ -498,13 +489,9 @@ class MHATokenToKVPoolHost(HostKVCache): # not build the CUDA-style k_data_ptrs/v_data_ptrs arrays. device_kv_buffers = None else: - ( - device_k_data_ptrs, - device_v_data_ptrs, - device_k_buffers, - device_v_buffers, - ) = self._resolve_device_transfer_buffers(device_pool) - device_kv_buffers = device_k_buffers + device_v_buffers + device_k_data_ptrs = self.device_k_data_ptrs + device_v_data_ptrs = self.device_v_data_ptrs + device_kv_buffers = self.device_k_buffers + self.device_v_buffers if io_backend == "kernel": if self.layout == "layer_first": if self.can_use_jit: @@ -1239,18 +1226,6 @@ class AsymmetricMHATokenToKVPoolHost(MHATokenToKVPoolHost): kernels derive copy sizes from each call's first tensor. """ - def get_size_per_token(self): - self.head_num = self.device_pool.head_num - self.head_dim = self.device_pool.head_dim - self.layer_num = self.target_layer_num + len(self.mtp_draft_device_pools) - self.v_head_dim = self.device_pool.v_head_dim - return ( - (self.head_dim + self.v_head_dim) - * self.head_num - * self.layer_num - * self.dtype.itemsize - ) - def get_ksize_per_token(self): return self.head_dim * self.head_num * self.layer_num * self.dtype.itemsize @@ -1284,7 +1259,7 @@ class AsymmetricMHATokenToKVPoolHost(MHATokenToKVPoolHost): # shared stride is a bug. Such callers will fail loudly with # AttributeError rather than silently use the K stride for V copies. - alloc_func = ALLOC_MEMORY_FUNCS[self.device_pool.device] + alloc_func = ALLOC_MEMORY_FUNCS[self._transfer_device.type] k_buffer = alloc_func( k_dims, dtype=self.dtype, @@ -1332,14 +1307,14 @@ class AsymmetricMHATokenToKVPoolHost(MHATokenToKVPoolHost): *, is_draft: bool = False, ): - if self.device_pool is not None: - if not is_draft and not self._is_device_layer_owned(device_pool, layer_id): - return - # MTP draft layers do not participate in CP layer sharding. - host_layer_id = layer_id if is_draft else self._host_layer_index(layer_id) - device_layer_id = 0 if is_draft else layer_id - else: - host_layer_id = device_layer_id = layer_id + if not is_draft and not self._is_device_layer_owned(device_pool, layer_id): + return + # MTP draft layers do not participate in CP layer sharding. + host_layer_id = layer_id if is_draft else self._host_layer_index(layer_id) + + if io_backend != "kernel_ascend": + device_k = self.device_k_buffers[layer_id] + device_v = self.device_v_buffers[layer_id] if io_backend == "kernel": if self.layout != "page_first": @@ -1349,7 +1324,7 @@ class AsymmetricMHATokenToKVPoolHost(MHATokenToKVPoolHost): ) transfer_kv_per_layer_mla_pf_lf( src=self.k_buffer, - dst=device_pool.k_buffer[device_layer_id], + dst=device_k, src_indices=host_indices, dst_indices=device_indices, layer_id=host_layer_id, @@ -1358,7 +1333,7 @@ class AsymmetricMHATokenToKVPoolHost(MHATokenToKVPoolHost): ) transfer_kv_per_layer_mla_pf_lf( src=self.v_buffer, - dst=device_pool.v_buffer[device_layer_id], + dst=device_v, src_indices=host_indices, dst_indices=device_indices, layer_id=host_layer_id, @@ -1374,7 +1349,7 @@ class AsymmetricMHATokenToKVPoolHost(MHATokenToKVPoolHost): ) transfer_kv_per_layer_direct_pf_lf( src_ptrs=[self.k_buffer], - dst_ptrs=[device_pool.k_buffer[device_layer_id]], + dst_ptrs=[device_k], src_indices=host_indices, dst_indices=device_indices, layer_id=host_layer_id, @@ -1382,7 +1357,7 @@ class AsymmetricMHATokenToKVPoolHost(MHATokenToKVPoolHost): ) transfer_kv_per_layer_direct_pf_lf( src_ptrs=[self.v_buffer], - dst_ptrs=[device_pool.v_buffer[device_layer_id]], + dst_ptrs=[device_v], src_indices=host_indices, dst_indices=device_indices, layer_id=host_layer_id, @@ -1397,12 +1372,10 @@ class AsymmetricMHATokenToKVPoolHost(MHATokenToKVPoolHost): def backup_from_device_all_layer( self, device_pool, host_indices, device_indices, io_backend ): - ( - device_k_data_ptrs, - device_v_data_ptrs, - device_k_buffers, - device_v_buffers, - ) = self._resolve_device_transfer_buffers(device_pool) + device_k_data_ptrs = self.device_k_data_ptrs + device_v_data_ptrs = self.device_v_data_ptrs + device_k_buffers = self.device_k_buffers + device_v_buffers = self.device_v_buffers if io_backend == "kernel": if self.layout != "page_first": raise ValueError( diff --git a/python/sglang/srt/mem_cache/pool_host/mla.py b/python/sglang/srt/mem_cache/pool_host/mla.py index 4688fd7a7b..1b12b559dc 100644 --- a/python/sglang/srt/mem_cache/pool_host/mla.py +++ b/python/sglang/srt/mem_cache/pool_host/mla.py @@ -65,6 +65,8 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): device_pool: MLATokenToKVPool mtp_draft_device_pools: tuple[MLATokenToKVPool, ...] = () + _transfer_device: torch.device + def __init__( self, device_pool: MLATokenToKVPool, @@ -87,9 +89,24 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): self.mtp_draft_device_pools = tuple(mtp_draft_device_pools) self._is_dummy = is_dummy + # Transfer buffers are captured below. The pool remains available for + # NPU transfer APIs and device layer ownership. self.device_pool = device_pool + if not is_dummy: + self._transfer_device = torch.device(device_pool.device) self.dtype = device_pool.store_dtype self.layout = layout + self.kv_lora_rank = device_pool.kv_lora_rank + self.qk_rope_head_dim = device_pool.qk_rope_head_dim + # L3 excludes the dead V component when the device stores packed FP8 KV. + self.dsa_kv_cache_store_fp8 = getattr( + device_pool, "dsa_kv_cache_store_fp8", False + ) + self.target_layer_num = self._effective_host_layer_num() + self.layer_num = self.target_layer_num + len(self.mtp_draft_device_pools) + self.kv_cache_dim = override_kv_cache_dim or ( + self.kv_lora_rank + self.qk_rope_head_dim + ) device_capacity = getattr(device_pool, "host_capacity_tokens", None) if is_dummy: device_capacity = device_capacity or device_pool.size @@ -131,6 +148,19 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): ) return + # Ascend transfers split K, rope and index tensors through its own API. + self.device_kv_buffers = ( + () + if _is_npu + else tuple( + b + for pool in (device_pool, *self.mtp_draft_device_pools) + for b in pool.kv_buffer + ) + ) + self._initialize_transfer_views() + + def _initialize_transfer_views(self) -> None: # The JIT HiCache kernels also build with hipcc (ROCm): the PTX-only # helpers in hicache.cuh are guarded by USE_ROCM and the staged # write-back kernel has a ROCm path, so enable them on HIP too. This @@ -150,22 +180,17 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): self.data_refs = [self.kv_buffer[i] for i in range(self.layer_num)] self.data_ptrs = make_kernel_ptr_table( self.data_refs, - self.device_pool.device, + self._transfer_device, host_memory_registered=self.pin_memory, ) - if self.mtp_draft_device_pools: - device_pools = (self.device_pool, *self.mtp_draft_device_pools) - self.packed_device_data_ptrs = torch.cat( - [pool.data_ptrs for pool in device_pools] + if not _is_npu: + pools = (self.device_pool, *self.mtp_draft_device_pools) + self.device_data_ptrs = ( + torch.cat([pool.data_ptrs for pool in pools]) + if self.mtp_draft_device_pools + else self.device_pool.data_ptrs ) - self.packed_device_kv_buffers = [ - buffer for pool in device_pools for buffer in pool.kv_buffer - ] - self._init_device_row_stride( - buf - for pool in (self.device_pool, *self.mtp_draft_device_pools) - for buf in getattr(pool, "kv_buffer", None) or () - ) + self._init_device_row_stride(self.device_kv_buffers) self._init_write_back_staging_buffers() def get_contiguous_buf_infos(self): @@ -184,19 +209,6 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): return data_ptrs, data_lens, item_lens def get_size_per_token(self): - self.kv_lora_rank = self.device_pool.kv_lora_rank - self.qk_rope_head_dim = self.device_pool.qk_rope_head_dim - # FP8 DSA packs K/V into the single device k_buffer (device v_buffer is - # empty and never transferred). Exposed for the L3 store to skip the - # dead v component when generating per-page keys/pointers. - self.dsa_kv_cache_store_fp8 = getattr( - self.device_pool, "dsa_kv_cache_store_fp8", False - ) - self.target_layer_num = self._effective_host_layer_num() - self.layer_num = self.target_layer_num + len(self.mtp_draft_device_pools) - self.kv_cache_dim = self.override_kv_cache_dim or ( - self.kv_lora_rank + self.qk_rope_head_dim - ) size_per_token = self.kv_cache_dim * self.dtype.itemsize * self.layer_num if ( self.layout == "page_first_kv_split" @@ -258,7 +270,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): if num_indexer_layers is None: num_indexer_layers = self.layer_num indexer_dims = (self.page_num, num_indexer_layers, self.page_size, 1) - alloc_func = ALLOC_MEMORY_FUNCS[self.device_pool.device] + alloc_func = ALLOC_MEMORY_FUNCS[self._transfer_device.type] if getattr(self.device_pool, "dsa_kv_cache_store_fp8", False): # FP8 DSA packs latent+RoPE+scale into the device k_buffer; # mirror the packed width so the 2D memcpy row width matches. @@ -335,7 +347,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): self.token_stride_size = self.kv_cache_dim * self.dtype.itemsize self.layout_dim = self.token_stride_size * self.layer_num - alloc_func = ALLOC_MEMORY_FUNCS[self.device_pool.device] + alloc_func = ALLOC_MEMORY_FUNCS[self._transfer_device.type] buffer = alloc_func( dims, dtype=self.dtype, @@ -383,7 +395,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): self.kv_cache_dim, ), dtype=self.dtype, - device=self.device_pool.device, + device=self._transfer_device, ) def _indexer_slot_range_for_layer(self, device_pool, device_layer_id): @@ -624,14 +636,15 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): ) # MTP draft layers do not participate in CP layer sharding. host_layer_id = layer_id if is_draft else self._host_layer_index(layer_id) - device_layer_id = 0 if is_draft else layer_id + if io_backend != "kernel_ascend": + device_buffer = self.device_kv_buffers[layer_id] if io_backend == "kernel": if self.layout == "layer_first": if self.can_use_jit: jit_transfer_hicache_one_layer_mla( page_size=self.page_size, - cache_dst=device_pool.kv_buffer[device_layer_id], + cache_dst=device_buffer, cache_src=self.kv_buffer[host_layer_id], indices_dst=device_indices, indices_src=host_indices, @@ -641,7 +654,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): self._require_packed_device_rows("transfer_kv_per_layer_mla") transfer_kv_per_layer_mla( src=self.kv_buffer[host_layer_id], - dst=device_pool.kv_buffer[device_layer_id], + dst=device_buffer, src_indices=host_indices, dst_indices=device_indices, item_size=self.token_stride_size, @@ -650,7 +663,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): if self.can_use_jit: jit_transfer_hicache_one_layer_mla( page_size=self.page_size, - cache_dst=device_pool.kv_buffer[device_layer_id], + cache_dst=device_buffer, cache_src=self.data_refs[host_layer_id], indices_dst=device_indices, indices_src=host_indices, @@ -660,7 +673,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): self._require_packed_device_rows("transfer_kv_per_layer_mla_pf_lf") transfer_kv_per_layer_mla_pf_lf( src=self.kv_buffer, - dst=device_pool.kv_buffer[device_layer_id], + dst=device_buffer, src_indices=host_indices, dst_indices=device_indices, layer_id=host_layer_id, @@ -673,7 +686,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): if self.layout == "layer_first": transfer_kv_direct( src_layers=[self.kv_buffer[host_layer_id]], - dst_layers=[device_pool.kv_buffer[device_layer_id]], + dst_layers=[device_buffer], src_indices=host_indices, dst_indices=device_indices, page_size=self.page_size, @@ -681,7 +694,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): elif self.layout == "page_first_direct": transfer_kv_per_layer_direct_pf_lf( src_ptrs=[self.kv_buffer], - dst_ptrs=[device_pool.kv_buffer[device_layer_id]], + dst_ptrs=[device_buffer], src_indices=host_indices, dst_indices=device_indices, layer_id=host_layer_id, @@ -690,6 +703,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): else: raise ValueError(f"Unsupported layout: {self.layout}") elif io_backend == "kernel_ascend": + device_layer_id = 0 if is_draft else layer_id if self.layout == "page_first_kv_split": if _is_npu and ascendc_io_enabled(): # The per-layer complete(i) event recorded by the caller lets @@ -740,7 +754,6 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): def _backup_from_device_per_layer( self, - device_pool, host_indices, device_indices, layer_id, @@ -751,10 +764,14 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): assert not getattr(self, "_is_dummy", False), ( "backup on a dummy (non-src MLA) host pool" ) + if io_backend not in ("kernel", "direct"): + raise ValueError( + f"Layer-sharded HiCache backup does not support IO backend: {io_backend}" + ) # Indices arrive already translated by backup_from_device_all_layer. # MTP draft layers do not participate in CP layer sharding. host_layer_id = layer_id if is_draft else self._host_layer_index(layer_id) - device_layer_id = 0 if is_draft else layer_id + device_buffer = self.device_kv_buffers[layer_id] if io_backend == "kernel": if self.layout == "layer_first": @@ -762,7 +779,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): jit_transfer_hicache_one_layer_mla( page_size=self.page_size, cache_dst=self.kv_buffer[host_layer_id], - cache_src=device_pool.kv_buffer[device_layer_id], + cache_src=device_buffer, indices_dst=host_indices, indices_src=device_indices, element_dim=self.kv_cache_dim, @@ -770,7 +787,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): else: self._require_packed_device_rows("transfer_kv_per_layer_mla") transfer_kv_per_layer_mla( - src=device_pool.kv_buffer[device_layer_id], + src=device_buffer, dst=self.kv_buffer[host_layer_id], src_indices=device_indices, dst_indices=host_indices, @@ -781,7 +798,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): jit_transfer_hicache_one_layer_mla( page_size=self.page_size, cache_dst=self.data_refs[host_layer_id], - cache_src=device_pool.kv_buffer[device_layer_id], + cache_src=device_buffer, indices_dst=host_indices, indices_src=device_indices, element_dim=self.kv_cache_dim, @@ -798,7 +815,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): elif io_backend == "direct": if self.layout == "layer_first": transfer_kv_direct( - src_layers=[device_pool.kv_buffer[device_layer_id]], + src_layers=[device_buffer], dst_layers=[self.kv_buffer[host_layer_id]], src_indices=device_indices, dst_indices=host_indices, @@ -809,15 +826,6 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): "Layer-sharded direct HiCache backup only supports " f"layer_first layout, got {self.layout}" ) - else: - raise ValueError( - f"Layer-sharded HiCache backup does not support IO backend: {io_backend}" - ) - - def _resolve_device_transfer_buffers(self, device_pool): - if self.mtp_draft_device_pools: - return self.packed_device_data_ptrs, self.packed_device_kv_buffers - return device_pool.data_ptrs, device_pool.kv_buffer def backup_from_device_all_layer( self, device_pool, host_indices, device_indices, io_backend @@ -834,13 +842,10 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): if self._is_device_layer_sharded(device_pool): for layer_id in self._owned_device_layer_ids(device_pool): self._backup_from_device_per_layer( - device_pool, host_indices, device_indices, layer_id, io_backend + host_indices, device_indices, layer_id, io_backend ) - for draft_layer_id, draft_device_pool in enumerate( - self.mtp_draft_device_pools - ): + for draft_layer_id in range(len(self.mtp_draft_device_pools)): self._backup_from_device_per_layer( - draft_device_pool, host_indices, device_indices, self.device_pool.layer_num + draft_layer_id, @@ -854,9 +859,8 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): # not build the CUDA-style data_ptrs array. device_data_ptrs, device_kv_buffers = None, None else: - device_data_ptrs, device_kv_buffers = self._resolve_device_transfer_buffers( - device_pool - ) + device_data_ptrs = self.device_data_ptrs + device_kv_buffers = self.device_kv_buffers if io_backend == "kernel": if self.layout == "layer_first": diff --git a/test/registered/unit/mem_cache/pool_host/test_kvcacheio_asymmetric.py b/test/registered/unit/mem_cache/pool_host/test_kvcacheio_asymmetric.py index fa137f0264..8261840d32 100644 --- a/test/registered/unit/mem_cache/pool_host/test_kvcacheio_asymmetric.py +++ b/test/registered/unit/mem_cache/pool_host/test_kvcacheio_asymmetric.py @@ -71,7 +71,7 @@ def make_host_pool(dtype, layout="page_first"): return host -def make_device_pool(dtype): +def make_device_pool(dtype, host): k_buffer = [ torch.empty(TOTAL_ITEMS, HEAD_NUM, K_HEAD_DIM, dtype=dtype, device=DEVICE) for _ in range(NUM_LAYERS) @@ -84,7 +84,17 @@ def make_device_pool(dtype): fill_with_offset(k_buffer[layer_id], layer_id * 1000) fill_with_offset(v_buffer[layer_id], layer_id * 1000 + 100) - return SimpleNamespace( + host.device_k_buffers = tuple(k_buffer) + host.device_v_buffers = tuple(v_buffer) + host.device_k_data_ptrs = torch.tensor( + [x.data_ptr() for x in k_buffer], dtype=torch.uint64, device=DEVICE + ) + host.device_v_data_ptrs = torch.tensor( + [x.data_ptr() for x in v_buffer], dtype=torch.uint64, device=DEVICE + ) + host.device_pool = SimpleNamespace( + layer_num=host.layer_num, + layer_shard_enabled=False, k_buffer=k_buffer, v_buffer=v_buffer, k_data_ptrs=torch.tensor( @@ -94,6 +104,7 @@ def make_device_pool(dtype): [x.data_ptr() for x in v_buffer], dtype=torch.uint64, device=DEVICE ), ) + return host.device_pool def _host_k_tokens(host, indices, layer_id): @@ -139,7 +150,7 @@ def test_asymmetric_mha_kernel_page_first_roundtrip(dtype): # Covers D2H backup + H2D load through AsymmetricMHATokenToKVPoolHost using # MiMoV2's real K/V head dims and the real MLA single-buffer kernels. host = make_host_pool(dtype) - device_pool = make_device_pool(dtype) + device_pool = make_device_pool(dtype, host) device_pages = torch.tensor([1, 2, 3], dtype=torch.int64) host_pages = torch.tensor([0, 1, 2], dtype=torch.int64) @@ -175,7 +186,7 @@ def test_asymmetric_mha_direct_page_first_direct_roundtrip(dtype): # page_first_direct/direct. K and V are copied through separate direct calls # because their per-token strides differ. host = make_host_pool(dtype, layout="page_first_direct") - device_pool = make_device_pool(dtype) + device_pool = make_device_pool(dtype, host) direct_stream = torch.cuda.Stream() device_pages = torch.tensor([1, 2, 3], dtype=torch.int64) diff --git a/test/registered/unit/mem_cache/test_asymmetric_mha_pool_host_unit.py b/test/registered/unit/mem_cache/test_asymmetric_mha_pool_host_unit.py index bd6eed718f..5af75cdb9f 100644 --- a/test/registered/unit/mem_cache/test_asymmetric_mha_pool_host_unit.py +++ b/test/registered/unit/mem_cache/test_asymmetric_mha_pool_host_unit.py @@ -20,6 +20,7 @@ register_cpu_ci(est_time=11, suite="base-a-test-cpu") def _make_host(layout: str) -> AsymmetricMHATokenToKVPoolHost: host = AsymmetricMHATokenToKVPoolHost.__new__(AsymmetricMHATokenToKVPoolHost) + host._transfer_device = torch.device("cpu") host.layout = layout host.page_size = 2 host.layer_num = 3 @@ -63,12 +64,23 @@ def _make_device_pool(host: AsymmetricMHATokenToKVPoolHost) -> SimpleNamespace: v_buffer = [ torch.empty(size, host.head_num, host.v_head_dim) for _ in range(host.layer_num) ] - return SimpleNamespace( + host.device_k_buffers = tuple(k_buffer) + host.device_v_buffers = tuple(v_buffer) + host.device_k_data_ptrs = torch.tensor( + [x.data_ptr() for x in k_buffer], dtype=torch.uint64 + ) + host.device_v_data_ptrs = torch.tensor( + [x.data_ptr() for x in v_buffer], dtype=torch.uint64 + ) + host.device_pool = SimpleNamespace( + layer_num=host.layer_num, + layer_shard_enabled=False, k_buffer=k_buffer, v_buffer=v_buffer, - k_data_ptrs=torch.tensor([x.data_ptr() for x in k_buffer], dtype=torch.uint64), - v_data_ptrs=torch.tensor([x.data_ptr() for x in v_buffer], dtype=torch.uint64), + k_data_ptrs=host.device_k_data_ptrs, + v_data_ptrs=host.device_v_data_ptrs, ) + return host.device_pool class TestAsymmetricMHATokenToKVPoolHost(CustomTestCase): diff --git a/test/registered/unit/mem_cache/test_hicache_dcp_host_pool.py b/test/registered/unit/mem_cache/test_hicache_dcp_host_pool.py index af0d8db9e7..be6f86e2b5 100644 --- a/test/registered/unit/mem_cache/test_hicache_dcp_host_pool.py +++ b/test/registered/unit/mem_cache/test_hicache_dcp_host_pool.py @@ -29,7 +29,12 @@ WIDENED_PAGE = PHYSICAL_PAGE * DCP_SIZE def _fake_mla_device_pool(size: int = 1024) -> SimpleNamespace: + buffers = [ + torch.empty(size + PHYSICAL_PAGE, 1, 12, dtype=torch.float16) for _ in range(2) + ] return SimpleNamespace( + kv_buffer=buffers, + data_ptrs=torch.tensor([b.data_ptr() for b in buffers], dtype=torch.uint64), size=size, # Match KVCache's default: this static pool's size already counts tokens. host_capacity_tokens=None, @@ -160,10 +165,7 @@ class TestHostPoolSizingUnderDcp(CustomTestCase): class TestTransferEntryPointsTranslate(CustomTestCase): def _run_backup(self, pool, host_indices, device_indices): - device_pool = SimpleNamespace( - data_ptrs=torch.zeros(2, dtype=torch.uint64), - kv_buffer=[torch.zeros(1)] * 2, - ) + device_pool = pool.device_pool # create=True: mla.py imports the kernel only under `if _is_cuda or # _is_hip`, so the name is absent on the CPU runner this test targets. with mock.patch( @@ -194,5 +196,82 @@ class TestTransferEntryPointsTranslate(CustomTestCase): pool.get_data_page(0) +class TestLayerShardedHostTransfers(CustomTestCase): + def test_layer_sharded_ascend_reports_unsupported_backend(self): + """Unsupported Ascend backup must report the backend, not a buffer error.""" + pool = _fake_mla_device_pool(size=128) + pool.layer_shard_enabled = True + pool.layer_shard_size = 2 + pool._owned_local_layer_range = lambda: (0, 1) + host = MLATokenToKVPoolHost( + pool, 2, 0, PHYSICAL_PAGE, "layer_first", pin_memory=False + ) + self.addCleanup(host.destroy) + # Ascend has no CUDA-style per-layer KV buffers. + host.device_kv_buffers = () + indices = torch.arange(PHYSICAL_PAGE) + with self.assertRaisesRegex( + ValueError, + "Layer-sharded HiCache backup does not support IO backend: kernel_ascend", + ): + host.backup_from_device_all_layer( + pool, indices, indices, io_backend="kernel_ascend" + ) + + def test_owned_device_layers_keep_their_original_positions(self): + # Rank 1 owns device layers 2/3, which occupy host layers 0/1. + pool = _fake_mla_device_pool(size=128) + pool.layer_num = 4 + pool.end_layer = 3 + pool.layer_shard_enabled = True + pool.layer_shard_size = 2 + pool._owned_local_layer_range = lambda: (2, 4) + owned = [ + torch.full((192, 1, 12), value, dtype=torch.float16) for value in (2, 3) + ] + pool.kv_buffer = [ + torch.empty(0, 1, 12, dtype=torch.float16) for _ in range(2) + ] + owned + pool.data_ptrs = torch.tensor( + [b.data_ptr() for b in pool.kv_buffer], dtype=torch.uint64 + ) + host = MLATokenToKVPoolHost( + pool, + 2, + 0, + PHYSICAL_PAGE, + "layer_first", + pin_memory=False, + ) + self.addCleanup(host.destroy) + self.assertEqual(host.layer_num, 2) + source = torch.arange(64, 128) + destination = torch.arange(64) + host_indices = torch.arange(128, 192) + + def copy_pages(*, src_layers, dst_layers, src_indices, dst_indices, page_size): + for src, dst in zip(src_layers, dst_layers, strict=True): + dst[dst_indices] = src[src_indices] + + with mock.patch( + "sglang.srt.mem_cache.pool_host.mla.transfer_kv_direct", + side_effect=copy_pages, + create=True, + ) as transfer: + host.backup_from_device_all_layer(pool, host_indices, source, "direct") + for buffer in owned: + buffer.zero_() + for layer in range(4): + host.load_to_device_per_layer( + pool, host_indices, destination, layer, "direct" + ) + self.assertEqual(transfer.call_count, 4) + for buffer, value in zip(owned, (2, 3), strict=True): + torch.testing.assert_close( + buffer[destination], torch.full((64, 1, 12), value, dtype=torch.float16) + ) + self.assertEqual(torch.count_nonzero(buffer[64:]).item(), 0) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/mem_cache/test_hicache_host_register.py b/test/registered/unit/mem_cache/test_hicache_host_register.py index 7712664be1..0e67a5af3c 100644 --- a/test/registered/unit/mem_cache/test_hicache_host_register.py +++ b/test/registered/unit/mem_cache/test_hicache_host_register.py @@ -110,6 +110,7 @@ class TestHiCacheHostRegister(unittest.TestCase): pool.kv_cache_dim = 5 pool.dtype = torch.float16 pool.device_pool = SimpleNamespace(device="cuda") + pool._transfer_device = torch.device("cuda") pool.device = "cpu" pool.pin_memory = True pool.allocator = object() @@ -133,6 +134,7 @@ class TestHiCacheHostRegister(unittest.TestCase): pool.head_dim = 4 pool.dtype = torch.float16 pool.device_pool = SimpleNamespace(device="cuda") + pool._transfer_device = torch.device("cuda") pool.device = "cpu" pool.pin_memory = True pool.allocator = object() @@ -290,6 +292,7 @@ class TestHiCacheHostRegister(unittest.TestCase): pool.v_head_dim = 7 pool.dtype = torch.float16 pool.device_pool = SimpleNamespace(device="cuda") + pool._transfer_device = torch.device("cuda") pool.device = "cpu" pool.pin_memory = True pool.allocator = object() diff --git a/test/registered/unit/mem_cache/test_hicache_seaweedfs_storage.py b/test/registered/unit/mem_cache/test_hicache_seaweedfs_storage.py index 9195cf5151..5a5f1ac95f 100755 --- a/test/registered/unit/mem_cache/test_hicache_seaweedfs_storage.py +++ b/test/registered/unit/mem_cache/test_hicache_seaweedfs_storage.py @@ -27,6 +27,7 @@ from sglang.srt.mem_cache.hicache_storage import ( PoolName, PoolTransfer, ) +from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool from sglang.srt.mem_cache.pool_host.mha import MHATokenToKVPoolHost from sglang.srt.mem_cache.storage import backend_factory from sglang.srt.mem_cache.storage.seaweedfs import seaweedfs_store @@ -93,19 +94,18 @@ class InMemoryS3: def _host_pool(layout: str) -> MHATokenToKVPoolHost: - device_pool = SimpleNamespace( - store_dtype=torch.bfloat16, + device_pool = MHATokenToKVPool( + dtype=torch.bfloat16, size=64, + page_size=PAGE_SIZE, layer_num=4, start_layer=0, end_layer=4, head_num=2, head_dim=64, v_head_dim=64, - row_dim=128, device="cpu", - layer_shard_enabled=False, - hicache_write_back_staging=None, + enable_memory_saver=False, ) return MHATokenToKVPoolHost( device_pool=device_pool, diff --git a/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py b/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py index 4689cf4b90..89d3d8930b 100644 --- a/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py +++ b/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py @@ -518,6 +518,11 @@ class TestHiCacheStagedWriteBackDispatch(CustomTestCase): ) host = MHATokenToKVPoolHost.__new__(MHATokenToKVPoolHost) + host.device_pool = device_pool + host.device_k_buffers = tuple(k_layers) + host.device_v_buffers = tuple(v_layers) + host.device_k_data_ptrs = device_pool.k_data_ptrs + host.device_v_data_ptrs = device_pool.v_data_ptrs host.layout = "page_first" host.page_size = 1 host.layer_num = layer_num @@ -603,12 +608,14 @@ class TestHiCacheStagedWriteBackDispatch(CustomTestCase): device_k = torch.empty(2, 3, 2, 1, 1) device_v = torch.empty_like(device_k) - device_pool = SimpleNamespace( + device_pool = _device_pool_stub( + layer_num=2, # FIA exposes lists here; these must not be sent to the operator. k_buffer=[device_k[layer].reshape(-1, 1, 1, 1) for layer in range(2)], v_buffer=[device_v[layer].reshape(-1, 1, 1, 1) for layer in range(2)], get_hicache_transfer_buffers=mock.Mock(return_value=(device_k, device_v)), ) + host.device_pool = device_pool host_indices = _indices(0, 2) device_indices = _indices(2, 4) directions = SimpleNamespace(H2D="H2D", D2H="D2H") @@ -668,6 +675,8 @@ class TestHiCacheStagedWriteBackDispatch(CustomTestCase): host = MLATokenToKVPoolHost.__new__(MLATokenToKVPoolHost) host.device_pool = device_pool + host.device_kv_buffers = tuple(device_layers) + host.device_data_ptrs = device_pool.data_ptrs host.layout = "page_first" host.page_size = 1 host.layer_num = layer_num diff --git a/test/registered/unit/mem_cache/test_mem_pool_host.py b/test/registered/unit/mem_cache/test_mem_pool_host.py index 8dae448426..1c8f3702a1 100644 --- a/test/registered/unit/mem_cache/test_mem_pool_host.py +++ b/test/registered/unit/mem_cache/test_mem_pool_host.py @@ -1,5 +1,6 @@ """Unit tests for host-pool allocation and free-list bookkeeping.""" +import contextlib import threading import unittest import unittest.mock @@ -13,6 +14,7 @@ from sglang.srt.mem_cache.memory_pool import ( MambaPool, MHATokenToKOnlyPool, MHATokenToKVPool, + MLATokenToKVPool, ) from sglang.srt.mem_cache.memory_pool_host import ( DeepSeekV4PagedHostPool, @@ -61,6 +63,88 @@ class TestHostKVCache(CustomTestCase): allocator_type="default", ) + def test_packed_transfers_use_captured_target_and_draft_buffers(self): + """Target and draft rows retain their bound destinations during transfer.""" + + def copy_rows(*, src_layers, dst_layers, src_indices, dst_indices, page_size): + for src, dst in zip(src_layers, dst_layers, strict=True): + dst[dst_indices] = src[src_indices] + + for mla in (False, True): + with self.subTest(mla=mla): + pool_type = MLATokenToKVPool if mla else MHATokenToKVPool + host_type = MLATokenToKVPoolHost if mla else MHATokenToKVPoolHost + geometry = ( + dict(kv_lora_rank=2, qk_rope_head_dim=2) + if mla + else dict(head_num=1, head_dim=4) + ) + pools = tuple( + pool_type( + size=4, + page_size=2, + dtype=torch.float16, + layer_num=layers, + device="cpu", + enable_memory_saver=False, + **geometry, + ) + for layers in (2, 1, 1) + ) + host = host_type( + pools[0], + host_to_device_ratio=2, + host_size=0, + page_size=2, + layout="layer_first", + pin_memory=False, + mtp_draft_device_pools=pools[1:], + ) + self.addCleanup(host.destroy) + names = ("kv_buffer",) if mla else ("k_buffer", "v_buffer") + buffers = [ + buffer + for name in names + for pool in pools + for buffer in getattr(pool, name) + ] + for value, buffer in enumerate(buffers, 1): + buffer.fill_(value) + source = torch.tensor([2, 3]) + saved = [buffer[source].clone() for buffer in buffers] + destination = torch.tensor([4, 5]) + host_indices = torch.tensor([0, 1]) + module = "mla" if mla else "mha" + with unittest.mock.patch( + f"sglang.srt.mem_cache.pool_host.{module}.transfer_kv_direct", + side_effect=copy_rows, + create=True, + ): + with contextlib.ExitStack() as patches: + for pool in pools: + for name in names: + patches.enter_context( + unittest.mock.patch.object(pool, name, None) + ) + host.backup_from_device_all_layer( + pools[0], host_indices, source, "direct" + ) + for buffer in buffers: + buffer.zero_() + for layer in range(4): + pool = pools[0] if layer < 2 else pools[layer - 1] + host.load_to_device_per_layer( + pool, + host_indices, + destination, + layer, + "direct", + is_draft=layer >= 2, + ) + for buffer, expected in zip(buffers, saved, strict=True): + torch.testing.assert_close(buffer[destination], expected) + self.assertEqual(torch.count_nonzero(buffer[:4]).item(), 0) + def test_index_k_host_pool_joins_a_host_pool_group(self): """Grouping a main KV pool with a K-only index pool raised AttributeError.""" index_pool = MHATokenToKOnlyPool( diff --git a/test/registered/unit/mem_cache/test_mla_host_dedup_primitives.py b/test/registered/unit/mem_cache/test_mla_host_dedup_primitives.py index 588d4066d6..bd11033d93 100644 --- a/test/registered/unit/mem_cache/test_mla_host_dedup_primitives.py +++ b/test/registered/unit/mem_cache/test_mla_host_dedup_primitives.py @@ -120,6 +120,10 @@ class TestMLAHostDedupPrimitives(unittest.TestCase): device="cpu", kv_buffer=[torch.zeros(10, 1, 6) for _ in range(2)], ) + pool.data_ptrs = torch.tensor( + [buffer.data_ptr() for buffer in pool.kv_buffer], + dtype=torch.uint64, + ) kwargs = dict( host_to_device_ratio=2, host_size=host_size,