Capture KV host transfer buffers (#43711)

This commit is contained in:
Ke Bao
2026-10-12 15:40:06 +08:00
committed by GitHub
parent 06887827c7
commit b6ba148384
10 changed files with 366 additions and 187 deletions
+84 -111
View File
@@ -123,9 +123,11 @@ def prepare_mha_write_back_staging(
class MHATokenToKVPoolHost(HostKVCache):
device_pool: MHATokenToKVPool | None = None
device_pool: MHATokenToKVPool
mtp_draft_device_pools: tuple[MHATokenToKVPool, ...] = ()
_transfer_device: torch.device
def __init__(
self,
device_pool: MHATokenToKVPool,
@@ -142,8 +144,20 @@ class MHATokenToKVPoolHost(HostKVCache):
):
self.mtp_draft_device_pools = tuple(mtp_draft_device_pools)
self.target_layer_num = device_pool.layer_num
# Transfer buffers are captured below. The pool remains available for
# NPU transfer APIs and device layer ownership.
self.device_pool = device_pool
self._transfer_device = torch.device(device_pool.device)
self.dtype = device_pool.store_dtype
self.head_dim = device_pool.head_dim
self.v_head_dim = device_pool.v_head_dim
# Equal-width pools may store multiple attention rows per allocator token.
self.head_num = (
device_pool.row_dim // self.head_dim
if self.head_dim == self.v_head_dim
else device_pool.head_num
)
self.layer_num = self.target_layer_num + len(self.mtp_draft_device_pools)
# Scale-bearing subclasses need page coverage when computing token bytes.
self.page_size = page_size
device_capacity = getattr(device_pool, "host_capacity_tokens", None)
@@ -164,14 +178,14 @@ class MHATokenToKVPoolHost(HostKVCache):
allocator_type=allocator_type,
pool_label=pool_label,
)
pools = (self.device_pool, *self.mtp_draft_device_pools)
self.device_k_buffers = tuple(b for pool in pools for b in pool.k_buffer)
self.device_v_buffers = tuple(b for pool in pools for b in pool.v_buffer)
self._initialize_transfer_views()
def _initialize_transfer_views(self) -> None:
self.element_dim = self.head_num * self.head_dim
self._init_device_row_stride(
buf
for pool in (self.device_pool, *self.mtp_draft_device_pools)
if pool is not None
for side in ("k_buffer", "v_buffer")
for buf in getattr(pool, side, None) or ()
)
self._init_device_row_stride((*self.device_k_buffers, *self.device_v_buffers))
# The JIT HiCache kernels also build with hipcc (ROCm): the PTX-only
# helpers in hicache.cuh are guarded by USE_ROCM and the staged
# write-back kernel has a ROCm path, so enable them on HIP too. This
@@ -193,44 +207,36 @@ class MHATokenToKVPoolHost(HostKVCache):
self.v_data_refs = [self.v_buffer[i] for i in range(self.layer_num)]
self.k_data_ptrs = make_kernel_ptr_table(
self.k_data_refs,
self.device_pool.device,
self._transfer_device,
host_memory_registered=self.pin_memory,
)
self.v_data_ptrs = make_kernel_ptr_table(
self.v_data_refs,
self.device_pool.device,
self._transfer_device,
host_memory_registered=self.pin_memory,
)
if self.mtp_draft_device_pools and not _is_npu:
device_pools = (self.device_pool, *self.mtp_draft_device_pools)
if not _is_npu:
self.packed_device_k_data_ptrs = torch.cat(
[pool.k_data_ptrs for pool in device_pools]
)
self.packed_device_v_data_ptrs = torch.cat(
[pool.v_data_ptrs for pool in device_pools]
)
else:
self.packed_device_k_data_ptrs = None
self.packed_device_v_data_ptrs = None
self.packed_device_k_buffers = [
buffer for pool in device_pools for buffer in pool.k_buffer
]
self.packed_device_v_buffers = [
buffer for pool in device_pools for buffer in pool.v_buffer
]
self.packed_device_kv_buffers = (
self.packed_device_k_buffers + self.packed_device_v_buffers
if not _is_npu:
pools = (self.device_pool, *self.mtp_draft_device_pools)
self.device_k_data_ptrs = (
torch.cat([pool.k_data_ptrs for pool in pools])
if self.mtp_draft_device_pools
else self.device_pool.k_data_ptrs
)
self.device_v_data_ptrs = (
torch.cat([pool.v_data_ptrs for pool in pools])
if self.mtp_draft_device_pools
else self.device_pool.v_data_ptrs
)
self.host_kv_data_refs = self.k_data_refs + self.v_data_refs
self._init_write_back_staging_buffers()
def get_size_per_token(self):
# One allocator token may hold multiple attention rows.
self.head_num = self.device_pool.row_dim // self.device_pool.head_dim
self.head_dim = self.device_pool.head_dim
self.layer_num = self.target_layer_num + len(self.mtp_draft_device_pools)
return self.head_dim * self.head_num * self.layer_num * self.dtype.itemsize * 2
return (
(self.head_dim + self.v_head_dim)
* self.head_num
* self.layer_num
* self.dtype.itemsize
)
def get_hybrid_pool_buffer(self):
# Expose the K/V host tensors required for zero-copy I/O registration.
@@ -267,7 +273,7 @@ class MHATokenToKVPoolHost(HostKVCache):
self.token_stride_size = self.head_num * self.head_dim * self.dtype.itemsize
self.layout_dim = self.token_stride_size * self.layer_num
alloc_func = ALLOC_MEMORY_FUNCS[self.device_pool.device]
alloc_func = ALLOC_MEMORY_FUNCS[self._transfer_device.type]
buffer = alloc_func(
dims,
dtype=self.dtype,
@@ -330,22 +336,22 @@ class MHATokenToKVPoolHost(HostKVCache):
*,
is_draft: bool = False,
):
if self.device_pool is not None:
if not is_draft and not self._is_device_layer_owned(device_pool, layer_id):
return
# MTP draft layers do not participate in CP layer sharding.
host_layer_id = layer_id if is_draft else self._host_layer_index(layer_id)
device_layer_id = 0 if is_draft else layer_id
else:
host_layer_id = device_layer_id = layer_id
if not is_draft and not self._is_device_layer_owned(device_pool, layer_id):
return
# MTP draft layers do not participate in CP layer sharding.
host_layer_id = layer_id if is_draft else self._host_layer_index(layer_id)
if io_backend != "kernel_ascend":
device_k = self.device_k_buffers[layer_id]
device_v = self.device_v_buffers[layer_id]
if io_backend == "kernel":
if self.layout == "layer_first":
if self.can_use_jit:
jit_transfer_hicache_one_layer(
page_size=self.page_size,
k_cache_dst=device_pool.k_buffer[device_layer_id],
v_cache_dst=device_pool.v_buffer[device_layer_id],
k_cache_dst=device_k,
v_cache_dst=device_v,
k_cache_src=self.k_buffer[host_layer_id],
v_cache_src=self.v_buffer[host_layer_id],
indices_dst=device_indices,
@@ -356,9 +362,9 @@ class MHATokenToKVPoolHost(HostKVCache):
self._require_packed_device_rows("transfer_kv_per_layer")
transfer_kv_per_layer(
src_k=self.k_buffer[host_layer_id],
dst_k=device_pool.k_buffer[device_layer_id],
dst_k=device_k,
src_v=self.v_buffer[host_layer_id],
dst_v=device_pool.v_buffer[device_layer_id],
dst_v=device_v,
src_indices=host_indices,
dst_indices=device_indices,
item_size=self.token_stride_size,
@@ -370,8 +376,8 @@ class MHATokenToKVPoolHost(HostKVCache):
# The kernel handles different src/dst strides automatically.
jit_transfer_hicache_one_layer(
page_size=self.page_size,
k_cache_dst=device_pool.k_buffer[device_layer_id],
v_cache_dst=device_pool.v_buffer[device_layer_id],
k_cache_dst=device_k,
v_cache_dst=device_v,
k_cache_src=self.k_data_refs[host_layer_id],
v_cache_src=self.v_data_refs[host_layer_id],
indices_dst=device_indices,
@@ -382,9 +388,9 @@ class MHATokenToKVPoolHost(HostKVCache):
self._require_packed_device_rows("transfer_kv_per_layer_pf_lf")
transfer_kv_per_layer_pf_lf(
src_k=self.k_buffer,
dst_k=device_pool.k_buffer[device_layer_id],
dst_k=device_k,
src_v=self.v_buffer,
dst_v=device_pool.v_buffer[device_layer_id],
dst_v=device_v,
src_indices=host_indices,
dst_indices=device_indices,
layer_id=host_layer_id,
@@ -395,9 +401,9 @@ class MHATokenToKVPoolHost(HostKVCache):
self._require_packed_device_rows("transfer_kv_per_layer_ph_lf")
transfer_kv_per_layer_ph_lf(
src_k=self.k_buffer,
dst_k=device_pool.k_buffer[device_layer_id],
dst_k=device_k,
src_v=self.v_buffer,
dst_v=device_pool.v_buffer[device_layer_id],
dst_v=device_v,
src_indices=host_indices,
dst_indices=device_indices,
layer_id=host_layer_id,
@@ -416,8 +422,8 @@ class MHATokenToKVPoolHost(HostKVCache):
self.v_buffer[host_layer_id],
],
dst_layers=[
device_pool.k_buffer[device_layer_id],
device_pool.v_buffer[device_layer_id],
device_k,
device_v,
],
src_indices=host_indices,
dst_indices=device_indices,
@@ -427,8 +433,8 @@ class MHATokenToKVPoolHost(HostKVCache):
transfer_kv_per_layer_direct_pf_lf(
src_ptrs=[self.k_buffer, self.v_buffer],
dst_ptrs=[
device_pool.k_buffer[device_layer_id],
device_pool.v_buffer[device_layer_id],
device_k,
device_v,
],
src_indices=host_indices,
dst_indices=device_indices,
@@ -462,21 +468,6 @@ class MHATokenToKVPoolHost(HostKVCache):
else:
raise ValueError(f"Unsupported IO backend: {io_backend}")
def _resolve_device_transfer_buffers(self, device_pool):
if self.mtp_draft_device_pools:
return (
self.packed_device_k_data_ptrs,
self.packed_device_v_data_ptrs,
self.packed_device_k_buffers,
self.packed_device_v_buffers,
)
return (
device_pool.k_data_ptrs,
device_pool.v_data_ptrs,
device_pool.k_buffer,
device_pool.v_buffer,
)
def _npu_transfer_buffers(self, target_device_pool):
layer_start = 0
for pool in (target_device_pool, *self.mtp_draft_device_pools):
@@ -498,13 +489,9 @@ class MHATokenToKVPoolHost(HostKVCache):
# not build the CUDA-style k_data_ptrs/v_data_ptrs arrays.
device_kv_buffers = None
else:
(
device_k_data_ptrs,
device_v_data_ptrs,
device_k_buffers,
device_v_buffers,
) = self._resolve_device_transfer_buffers(device_pool)
device_kv_buffers = device_k_buffers + device_v_buffers
device_k_data_ptrs = self.device_k_data_ptrs
device_v_data_ptrs = self.device_v_data_ptrs
device_kv_buffers = self.device_k_buffers + self.device_v_buffers
if io_backend == "kernel":
if self.layout == "layer_first":
if self.can_use_jit:
@@ -1239,18 +1226,6 @@ class AsymmetricMHATokenToKVPoolHost(MHATokenToKVPoolHost):
kernels derive copy sizes from each call's first tensor.
"""
def get_size_per_token(self):
self.head_num = self.device_pool.head_num
self.head_dim = self.device_pool.head_dim
self.layer_num = self.target_layer_num + len(self.mtp_draft_device_pools)
self.v_head_dim = self.device_pool.v_head_dim
return (
(self.head_dim + self.v_head_dim)
* self.head_num
* self.layer_num
* self.dtype.itemsize
)
def get_ksize_per_token(self):
return self.head_dim * self.head_num * self.layer_num * self.dtype.itemsize
@@ -1284,7 +1259,7 @@ class AsymmetricMHATokenToKVPoolHost(MHATokenToKVPoolHost):
# shared stride is a bug. Such callers will fail loudly with
# AttributeError rather than silently use the K stride for V copies.
alloc_func = ALLOC_MEMORY_FUNCS[self.device_pool.device]
alloc_func = ALLOC_MEMORY_FUNCS[self._transfer_device.type]
k_buffer = alloc_func(
k_dims,
dtype=self.dtype,
@@ -1332,14 +1307,14 @@ class AsymmetricMHATokenToKVPoolHost(MHATokenToKVPoolHost):
*,
is_draft: bool = False,
):
if self.device_pool is not None:
if not is_draft and not self._is_device_layer_owned(device_pool, layer_id):
return
# MTP draft layers do not participate in CP layer sharding.
host_layer_id = layer_id if is_draft else self._host_layer_index(layer_id)
device_layer_id = 0 if is_draft else layer_id
else:
host_layer_id = device_layer_id = layer_id
if not is_draft and not self._is_device_layer_owned(device_pool, layer_id):
return
# MTP draft layers do not participate in CP layer sharding.
host_layer_id = layer_id if is_draft else self._host_layer_index(layer_id)
if io_backend != "kernel_ascend":
device_k = self.device_k_buffers[layer_id]
device_v = self.device_v_buffers[layer_id]
if io_backend == "kernel":
if self.layout != "page_first":
@@ -1349,7 +1324,7 @@ class AsymmetricMHATokenToKVPoolHost(MHATokenToKVPoolHost):
)
transfer_kv_per_layer_mla_pf_lf(
src=self.k_buffer,
dst=device_pool.k_buffer[device_layer_id],
dst=device_k,
src_indices=host_indices,
dst_indices=device_indices,
layer_id=host_layer_id,
@@ -1358,7 +1333,7 @@ class AsymmetricMHATokenToKVPoolHost(MHATokenToKVPoolHost):
)
transfer_kv_per_layer_mla_pf_lf(
src=self.v_buffer,
dst=device_pool.v_buffer[device_layer_id],
dst=device_v,
src_indices=host_indices,
dst_indices=device_indices,
layer_id=host_layer_id,
@@ -1374,7 +1349,7 @@ class AsymmetricMHATokenToKVPoolHost(MHATokenToKVPoolHost):
)
transfer_kv_per_layer_direct_pf_lf(
src_ptrs=[self.k_buffer],
dst_ptrs=[device_pool.k_buffer[device_layer_id]],
dst_ptrs=[device_k],
src_indices=host_indices,
dst_indices=device_indices,
layer_id=host_layer_id,
@@ -1382,7 +1357,7 @@ class AsymmetricMHATokenToKVPoolHost(MHATokenToKVPoolHost):
)
transfer_kv_per_layer_direct_pf_lf(
src_ptrs=[self.v_buffer],
dst_ptrs=[device_pool.v_buffer[device_layer_id]],
dst_ptrs=[device_v],
src_indices=host_indices,
dst_indices=device_indices,
layer_id=host_layer_id,
@@ -1397,12 +1372,10 @@ class AsymmetricMHATokenToKVPoolHost(MHATokenToKVPoolHost):
def backup_from_device_all_layer(
self, device_pool, host_indices, device_indices, io_backend
):
(
device_k_data_ptrs,
device_v_data_ptrs,
device_k_buffers,
device_v_buffers,
) = self._resolve_device_transfer_buffers(device_pool)
device_k_data_ptrs = self.device_k_data_ptrs
device_v_data_ptrs = self.device_v_data_ptrs
device_k_buffers = self.device_k_buffers
device_v_buffers = self.device_v_buffers
if io_backend == "kernel":
if self.layout != "page_first":
raise ValueError(
+63 -59
View File
@@ -65,6 +65,8 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
device_pool: MLATokenToKVPool
mtp_draft_device_pools: tuple[MLATokenToKVPool, ...] = ()
_transfer_device: torch.device
def __init__(
self,
device_pool: MLATokenToKVPool,
@@ -87,9 +89,24 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
self.mtp_draft_device_pools = tuple(mtp_draft_device_pools)
self._is_dummy = is_dummy
# Transfer buffers are captured below. The pool remains available for
# NPU transfer APIs and device layer ownership.
self.device_pool = device_pool
if not is_dummy:
self._transfer_device = torch.device(device_pool.device)
self.dtype = device_pool.store_dtype
self.layout = layout
self.kv_lora_rank = device_pool.kv_lora_rank
self.qk_rope_head_dim = device_pool.qk_rope_head_dim
# L3 excludes the dead V component when the device stores packed FP8 KV.
self.dsa_kv_cache_store_fp8 = getattr(
device_pool, "dsa_kv_cache_store_fp8", False
)
self.target_layer_num = self._effective_host_layer_num()
self.layer_num = self.target_layer_num + len(self.mtp_draft_device_pools)
self.kv_cache_dim = override_kv_cache_dim or (
self.kv_lora_rank + self.qk_rope_head_dim
)
device_capacity = getattr(device_pool, "host_capacity_tokens", None)
if is_dummy:
device_capacity = device_capacity or device_pool.size
@@ -131,6 +148,19 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
)
return
# Ascend transfers split K, rope and index tensors through its own API.
self.device_kv_buffers = (
()
if _is_npu
else tuple(
b
for pool in (device_pool, *self.mtp_draft_device_pools)
for b in pool.kv_buffer
)
)
self._initialize_transfer_views()
def _initialize_transfer_views(self) -> None:
# The JIT HiCache kernels also build with hipcc (ROCm): the PTX-only
# helpers in hicache.cuh are guarded by USE_ROCM and the staged
# write-back kernel has a ROCm path, so enable them on HIP too. This
@@ -150,22 +180,17 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
self.data_refs = [self.kv_buffer[i] for i in range(self.layer_num)]
self.data_ptrs = make_kernel_ptr_table(
self.data_refs,
self.device_pool.device,
self._transfer_device,
host_memory_registered=self.pin_memory,
)
if self.mtp_draft_device_pools:
device_pools = (self.device_pool, *self.mtp_draft_device_pools)
self.packed_device_data_ptrs = torch.cat(
[pool.data_ptrs for pool in device_pools]
if not _is_npu:
pools = (self.device_pool, *self.mtp_draft_device_pools)
self.device_data_ptrs = (
torch.cat([pool.data_ptrs for pool in pools])
if self.mtp_draft_device_pools
else self.device_pool.data_ptrs
)
self.packed_device_kv_buffers = [
buffer for pool in device_pools for buffer in pool.kv_buffer
]
self._init_device_row_stride(
buf
for pool in (self.device_pool, *self.mtp_draft_device_pools)
for buf in getattr(pool, "kv_buffer", None) or ()
)
self._init_device_row_stride(self.device_kv_buffers)
self._init_write_back_staging_buffers()
def get_contiguous_buf_infos(self):
@@ -184,19 +209,6 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
return data_ptrs, data_lens, item_lens
def get_size_per_token(self):
self.kv_lora_rank = self.device_pool.kv_lora_rank
self.qk_rope_head_dim = self.device_pool.qk_rope_head_dim
# FP8 DSA packs K/V into the single device k_buffer (device v_buffer is
# empty and never transferred). Exposed for the L3 store to skip the
# dead v component when generating per-page keys/pointers.
self.dsa_kv_cache_store_fp8 = getattr(
self.device_pool, "dsa_kv_cache_store_fp8", False
)
self.target_layer_num = self._effective_host_layer_num()
self.layer_num = self.target_layer_num + len(self.mtp_draft_device_pools)
self.kv_cache_dim = self.override_kv_cache_dim or (
self.kv_lora_rank + self.qk_rope_head_dim
)
size_per_token = self.kv_cache_dim * self.dtype.itemsize * self.layer_num
if (
self.layout == "page_first_kv_split"
@@ -258,7 +270,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
if num_indexer_layers is None:
num_indexer_layers = self.layer_num
indexer_dims = (self.page_num, num_indexer_layers, self.page_size, 1)
alloc_func = ALLOC_MEMORY_FUNCS[self.device_pool.device]
alloc_func = ALLOC_MEMORY_FUNCS[self._transfer_device.type]
if getattr(self.device_pool, "dsa_kv_cache_store_fp8", False):
# FP8 DSA packs latent+RoPE+scale into the device k_buffer;
# mirror the packed width so the 2D memcpy row width matches.
@@ -335,7 +347,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
self.token_stride_size = self.kv_cache_dim * self.dtype.itemsize
self.layout_dim = self.token_stride_size * self.layer_num
alloc_func = ALLOC_MEMORY_FUNCS[self.device_pool.device]
alloc_func = ALLOC_MEMORY_FUNCS[self._transfer_device.type]
buffer = alloc_func(
dims,
dtype=self.dtype,
@@ -383,7 +395,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
self.kv_cache_dim,
),
dtype=self.dtype,
device=self.device_pool.device,
device=self._transfer_device,
)
def _indexer_slot_range_for_layer(self, device_pool, device_layer_id):
@@ -624,14 +636,15 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
)
# MTP draft layers do not participate in CP layer sharding.
host_layer_id = layer_id if is_draft else self._host_layer_index(layer_id)
device_layer_id = 0 if is_draft else layer_id
if io_backend != "kernel_ascend":
device_buffer = self.device_kv_buffers[layer_id]
if io_backend == "kernel":
if self.layout == "layer_first":
if self.can_use_jit:
jit_transfer_hicache_one_layer_mla(
page_size=self.page_size,
cache_dst=device_pool.kv_buffer[device_layer_id],
cache_dst=device_buffer,
cache_src=self.kv_buffer[host_layer_id],
indices_dst=device_indices,
indices_src=host_indices,
@@ -641,7 +654,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
self._require_packed_device_rows("transfer_kv_per_layer_mla")
transfer_kv_per_layer_mla(
src=self.kv_buffer[host_layer_id],
dst=device_pool.kv_buffer[device_layer_id],
dst=device_buffer,
src_indices=host_indices,
dst_indices=device_indices,
item_size=self.token_stride_size,
@@ -650,7 +663,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
if self.can_use_jit:
jit_transfer_hicache_one_layer_mla(
page_size=self.page_size,
cache_dst=device_pool.kv_buffer[device_layer_id],
cache_dst=device_buffer,
cache_src=self.data_refs[host_layer_id],
indices_dst=device_indices,
indices_src=host_indices,
@@ -660,7 +673,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
self._require_packed_device_rows("transfer_kv_per_layer_mla_pf_lf")
transfer_kv_per_layer_mla_pf_lf(
src=self.kv_buffer,
dst=device_pool.kv_buffer[device_layer_id],
dst=device_buffer,
src_indices=host_indices,
dst_indices=device_indices,
layer_id=host_layer_id,
@@ -673,7 +686,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
if self.layout == "layer_first":
transfer_kv_direct(
src_layers=[self.kv_buffer[host_layer_id]],
dst_layers=[device_pool.kv_buffer[device_layer_id]],
dst_layers=[device_buffer],
src_indices=host_indices,
dst_indices=device_indices,
page_size=self.page_size,
@@ -681,7 +694,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
elif self.layout == "page_first_direct":
transfer_kv_per_layer_direct_pf_lf(
src_ptrs=[self.kv_buffer],
dst_ptrs=[device_pool.kv_buffer[device_layer_id]],
dst_ptrs=[device_buffer],
src_indices=host_indices,
dst_indices=device_indices,
layer_id=host_layer_id,
@@ -690,6 +703,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
else:
raise ValueError(f"Unsupported layout: {self.layout}")
elif io_backend == "kernel_ascend":
device_layer_id = 0 if is_draft else layer_id
if self.layout == "page_first_kv_split":
if _is_npu and ascendc_io_enabled():
# The per-layer complete(i) event recorded by the caller lets
@@ -740,7 +754,6 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
def _backup_from_device_per_layer(
self,
device_pool,
host_indices,
device_indices,
layer_id,
@@ -751,10 +764,14 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
assert not getattr(self, "_is_dummy", False), (
"backup on a dummy (non-src MLA) host pool"
)
if io_backend not in ("kernel", "direct"):
raise ValueError(
f"Layer-sharded HiCache backup does not support IO backend: {io_backend}"
)
# Indices arrive already translated by backup_from_device_all_layer.
# MTP draft layers do not participate in CP layer sharding.
host_layer_id = layer_id if is_draft else self._host_layer_index(layer_id)
device_layer_id = 0 if is_draft else layer_id
device_buffer = self.device_kv_buffers[layer_id]
if io_backend == "kernel":
if self.layout == "layer_first":
@@ -762,7 +779,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
jit_transfer_hicache_one_layer_mla(
page_size=self.page_size,
cache_dst=self.kv_buffer[host_layer_id],
cache_src=device_pool.kv_buffer[device_layer_id],
cache_src=device_buffer,
indices_dst=host_indices,
indices_src=device_indices,
element_dim=self.kv_cache_dim,
@@ -770,7 +787,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
else:
self._require_packed_device_rows("transfer_kv_per_layer_mla")
transfer_kv_per_layer_mla(
src=device_pool.kv_buffer[device_layer_id],
src=device_buffer,
dst=self.kv_buffer[host_layer_id],
src_indices=device_indices,
dst_indices=host_indices,
@@ -781,7 +798,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
jit_transfer_hicache_one_layer_mla(
page_size=self.page_size,
cache_dst=self.data_refs[host_layer_id],
cache_src=device_pool.kv_buffer[device_layer_id],
cache_src=device_buffer,
indices_dst=host_indices,
indices_src=device_indices,
element_dim=self.kv_cache_dim,
@@ -798,7 +815,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
elif io_backend == "direct":
if self.layout == "layer_first":
transfer_kv_direct(
src_layers=[device_pool.kv_buffer[device_layer_id]],
src_layers=[device_buffer],
dst_layers=[self.kv_buffer[host_layer_id]],
src_indices=device_indices,
dst_indices=host_indices,
@@ -809,15 +826,6 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
"Layer-sharded direct HiCache backup only supports "
f"layer_first layout, got {self.layout}"
)
else:
raise ValueError(
f"Layer-sharded HiCache backup does not support IO backend: {io_backend}"
)
def _resolve_device_transfer_buffers(self, device_pool):
if self.mtp_draft_device_pools:
return self.packed_device_data_ptrs, self.packed_device_kv_buffers
return device_pool.data_ptrs, device_pool.kv_buffer
def backup_from_device_all_layer(
self, device_pool, host_indices, device_indices, io_backend
@@ -834,13 +842,10 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
if self._is_device_layer_sharded(device_pool):
for layer_id in self._owned_device_layer_ids(device_pool):
self._backup_from_device_per_layer(
device_pool, host_indices, device_indices, layer_id, io_backend
host_indices, device_indices, layer_id, io_backend
)
for draft_layer_id, draft_device_pool in enumerate(
self.mtp_draft_device_pools
):
for draft_layer_id in range(len(self.mtp_draft_device_pools)):
self._backup_from_device_per_layer(
draft_device_pool,
host_indices,
device_indices,
self.device_pool.layer_num + draft_layer_id,
@@ -854,9 +859,8 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
# not build the CUDA-style data_ptrs array.
device_data_ptrs, device_kv_buffers = None, None
else:
device_data_ptrs, device_kv_buffers = self._resolve_device_transfer_buffers(
device_pool
)
device_data_ptrs = self.device_data_ptrs
device_kv_buffers = self.device_kv_buffers
if io_backend == "kernel":
if self.layout == "layer_first":