[Refactor] memory-pool: Build KV host pools from buffer info (#40917)

This commit is contained in:
Ke Bao
2026-10-18 23:21:16 +08:00
committed by GitHub
parent bb1d4cddd9
commit 4ec19ca27f
35 changed files with 1796 additions and 464 deletions
@@ -190,6 +190,33 @@ class NPUMHATokenToKVPool(MHATokenToKVPool):
"""Return contiguous all-layer KV tensors for NPU HiCache IO."""
return self._hicache_k_buffer, self._hicache_v_buffer
def _mha_buffer_info(self):
from sglang.srt.mem_cache.device_pool_info import (
KVBufferLayout,
KVBuffers,
MHABufferInfo,
)
return MHABufferInfo(
buffers=KVBuffers(
k=tuple(self.k_buffer),
v=tuple(self.v_buffer),
k_data_ptrs=None,
v_data_ptrs=None,
layout=KVBufferLayout.PAGED_NHD,
),
page_size=self.page_size,
head_num=self.row_dim // self.head_dim
if self.head_dim == self.v_head_dim
else self.head_num,
head_dim=self.head_dim,
v_head_dim=self.v_head_dim,
dtype=self.store_dtype,
# self.device is the bare backend name; tensors carry the rank index.
device=self._hicache_k_buffer.device,
transfer_buffers=self.get_hicache_transfer_buffers,
)
def _init_kv_copy_and_warmup(self):
# implementation relies on self.data_strides / self.data_ptrs, which the
# NPU paged buffer layout never builds.
@@ -610,6 +637,9 @@ class NPUMLATokenToKVPool(MLATokenToKVPool):
end_layer=end_layer,
)
self.model_layer_ids = tuple(
range(self.start_layer, self.start_layer + self.layer_num)
)
self.kv_lora_rank = kv_lora_rank
self.qk_rope_head_dim = qk_rope_head_dim
self.index_head_dim = index_head_dim
@@ -731,6 +761,45 @@ class NPUMLATokenToKVPool(MLATokenToKVPool):
self._finalize_allocation_log(size)
def _mla_buffer_info(self):
from sglang.srt.mem_cache.device_pool_info import MLABufferInfo
return MLABufferInfo(
buffers=self._mla_native_buffers(),
native_buffers=self._mla_native_buffers,
data_ptrs=None,
layer_indices=tuple(range(self.layer_num)),
device_layer_num=self.layer_num,
page_size=self.page_size,
kv_lora_rank=self.kv_lora_rank,
qk_rope_head_dim=self.qk_rope_head_dim,
kv_cache_dim=self.kv_cache_dim + self.kr_cache_dim,
dtype=self.store_dtype,
# self.device is the bare backend name; tensors carry the rank index.
device=next(
buffer.device
for buffer in (self.k_buffer, self.index_k_buffer)
if buffer is not None
),
packed_fp8=self.dsa_kv_cache_store_fp8,
)
def _mla_native_buffers(self):
from sglang.srt.mem_cache.device_pool_info import MLASplitBuffers
return MLASplitBuffers(
k=self.k_buffer,
v=self.v_buffer,
index_k=self.index_k_buffer,
index_scales=self.index_k_scale_buffer,
index_layer_indices=tuple(
i - self.start_layer for i in self.indexer_layer_ids
),
k_width=self.kv_cache_dim,
v_width=self.kr_cache_dim,
index_width=self.index_head_dim or 0,
)
def _copy_indices_for_buffer(self, indices, uses_global_slots):
if uses_global_slots or self.dcp_size <= 1:
return indices
@@ -601,7 +601,14 @@ class HiSparseCoordinator:
"tensor-parallel size."
)
self.mem_pool_host = HiSparseMHATokenToKVPoolHost(
device_pool=self.mem_pool_device,
buffer_info=self.mem_pool_device.get_device_pool_infos()[
0
].buffer_info,
device_capacity_tokens=self.mem_pool_device.size
if self.mem_pool_device.host_capacity_tokens is None
else self.mem_pool_device.host_capacity_tokens,
start_layer=self.mem_pool_device.start_layer,
end_layer=self.mem_pool_device.end_layer,
host_to_device_ratio=host_to_device_ratio,
page_size=self.mem_pool_device.page_size,
)
@@ -609,7 +616,15 @@ class HiSparseCoordinator:
else:
self.mem_pool_device: HiSparseDSATokenToKVPool = kvcache
self.mem_pool_host = MLATokenToKVPoolHost(
device_pool=self.mem_pool_device,
buffer_info=self.mem_pool_device.get_device_pool_infos()[
0
].buffer_info,
device_capacity_tokens=self.mem_pool_device.size
if self.mem_pool_device.host_capacity_tokens is None
else self.mem_pool_device.host_capacity_tokens,
start_layer=self.mem_pool_device.start_layer,
end_layer=self.mem_pool_device.end_layer,
target_layer_num=self.mem_pool_device.layer_num,
host_to_device_ratio=host_to_device_ratio,
host_size=0,
page_size=self.mem_pool_device.page_size,
+158 -1
View File
@@ -6,6 +6,9 @@ layer routing and transfers. This first consumer describes Mamba checkpoints.
from __future__ import annotations
from collections.abc import Callable
from enum import Enum
import msgspec
import torch
@@ -57,12 +60,166 @@ class MambaStateBufferInfo(msgspec.Struct, frozen=True, kw_only=True):
)
class KVBufferLayout(Enum):
NHD = "nhd"
PAGED_NHD = "paged_nhd"
HND = "hnd"
VECTORIZED_5D = "vectorized_5d"
class KVBuffers(msgspec.Struct, frozen=True, kw_only=True):
"""Per-layer K/V views and their existing device pointer allocations.
NHD views may be strided. Page/head/vector axes stay in the native layout;
this description neither flattens buffers nor enables new transfer backends.
"""
k: tuple[torch.Tensor, ...]
v: tuple[torch.Tensor, ...]
# Some packed formats have no pointer tables; providers must not allocate them.
k_data_ptrs: torch.Tensor | None
v_data_ptrs: torch.Tensor | None
layout: KVBufferLayout
class KVScaleLayout(Enum):
TOKEN = "token"
MXFP8_INTERLEAVED = "mxfp8_interleaved"
class KVScaleBuffers(msgspec.Struct, frozen=True, kw_only=True):
k: tuple[torch.Tensor, ...]
v: tuple[torch.Tensor, ...]
layout: KVScaleLayout
class MHABufferInfo(msgspec.Struct, frozen=True, kw_only=True):
"""Physical MHA input for host construction; no capacity or allocation policy.
Geometry is explicit for empty stages. head_num counts physical attention
rows per allocator token, which can exceed the model's attention head count.
Existing pointer tables and tensor strides must be preserved by consumers.
"""
buffers: KVBuffers
page_size: int
head_num: int
head_dim: int
v_head_dim: int
dtype: torch.dtype
device: torch.device
scales: KVScaleBuffers | None = None
# Resolve current contiguous backing at transfer time for relocatable pools.
transfer_buffers: Callable[[], tuple[torch.Tensor, torch.Tensor]] | None = None
def validate(self, *, layer_ids: tuple[int, ...] | None = None) -> None:
layers = len(self.buffers.k)
if len(self.buffers.v) != layers or (
layer_ids is not None and len(layer_ids) != layers
):
raise ValueError(
f"MHA layers: k={layers}, v={len(self.buffers.v)}, layer_ids={layer_ids}"
)
if min(self.page_size, self.head_num, self.head_dim, self.v_head_dim) <= 0:
raise ValueError(
f"Invalid MHA geometry: page_size={self.page_size}, "
f"head_num={self.head_num}, head_dim={self.head_dim}, "
f"v_head_dim={self.v_head_dim}"
)
for name in ("k", "v"):
tensors = self.buffers.k if name == "k" else self.buffers.v
for layer, tensor in enumerate(tensors):
if tensor.dtype != self.dtype or tensor.device != self.device:
raise ValueError(
f"MHA {name}[{layer}]: dtype={tensor.dtype}, device={tensor.device}; "
f"expected dtype={self.dtype}, device={self.device}"
)
if self.scales is not None and (
len(self.scales.k) != layers or len(self.scales.v) != layers
):
raise ValueError(
f"MHA scales: k={len(self.scales.k)}, v={len(self.scales.v)}, "
f"expected layers={layers}"
)
class MLASplitBuffers(msgspec.Struct, frozen=True, kw_only=True):
"""Native NPU allocations; index slots have their own pool-local layer order."""
k: torch.Tensor | None
v: torch.Tensor | None
index_k: torch.Tensor | None
index_scales: torch.Tensor | None
index_layer_indices: tuple[int, ...]
k_width: int
v_width: int
index_width: int
class MLABufferInfo(msgspec.Struct, frozen=True, kw_only=True):
"""Persistent MLA rows or native split components, including empty stages.
layer_indices address the pool's controller slots, not model layer IDs or
reserved host rows. Existing pointer allocations are views, never rebuilt.
"""
buffers: tuple[torch.Tensor, ...] | MLASplitBuffers
data_ptrs: torch.Tensor | None
layer_indices: tuple[int, ...]
device_layer_num: int
page_size: int
kv_lora_rank: int
qk_rope_head_dim: int
kv_cache_dim: int
dtype: torch.dtype
device: torch.device
packed_fp8: bool = False
scales: tuple[torch.Tensor, ...] | None = None
native_buffers: Callable[[], MLASplitBuffers] | None = None
def validate(self, *, layer_ids: tuple[int, ...] | None = None) -> None:
if layer_ids is not None and len(layer_ids) != len(self.layer_indices):
raise ValueError(
f"MLA model layers={layer_ids} differ from buffer slots={self.layer_indices}"
)
if len(set(self.layer_indices)) != len(self.layer_indices) or any(
i < 0 or i >= self.device_layer_num for i in self.layer_indices
):
raise ValueError(
f"MLA slots={self.layer_indices} outside device_layer_num={self.device_layer_num}"
)
if isinstance(self.buffers, tuple):
if len(self.buffers) != len(self.layer_indices):
raise ValueError(
f"MLA buffers={len(self.buffers)}, slots={self.layer_indices}"
)
for i, tensor in zip(self.layer_indices, self.buffers):
if (
tensor.ndim != 3
or tensor.shape[1] != 1
or tensor.dtype != self.dtype
or tensor.device != self.device
):
raise ValueError(
f"MLA slot={i}: shape={tuple(tensor.shape)}, dtype={tensor.dtype}, "
f"device={tensor.device}; expected [token, 1, width], "
f"dtype={self.dtype}, device={self.device}"
)
elif self.buffers.index_k is not None and self.buffers.index_k.shape[0] != len(
self.buffers.index_layer_indices
):
raise ValueError(
f"MLA index shape={tuple(self.buffers.index_k.shape)}, "
f"index layers={self.buffers.index_layer_indices}"
)
class DevicePoolInfo(msgspec.Struct, frozen=True, kw_only=True):
pool_name: PoolName
indices_from_pool: PoolName
# Model layer IDs, including the PP stage offset, in buffer-axis order.
layer_ids: tuple[int, ...]
buffer_info: MambaStateBufferInfo
buffer_info: MambaStateBufferInfo | MHABufferInfo | MLABufferInfo
def __post_init__(self) -> None:
# A PP stage may own no checkpoint layers. Buffer validation decides
@@ -37,6 +37,7 @@ from sglang.srt.mem_cache.pool_host.qsa import QSAIndexerPoolHost
from sglang.srt.mem_cache.pool_host.unified import UnifiedPageEnvelopeHostPool
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
from sglang.srt.runtime_context import get_memory, get_parallel, get_serving
from sglang.srt.utils import is_npu
if TYPE_CHECKING:
import torch
@@ -134,6 +135,27 @@ def _resolve_deepseek_v4_layer_mappings(
)
def _mha_host_inputs(pool, drafts=()):
info = pool.get_device_pool_infos()[0].buffer_info
return (
get_mha_host_pool_cls(info),
info,
dict(
device_capacity_tokens=(
pool.size
if pool.host_capacity_tokens is None
else pool.host_capacity_tokens
),
start_layer=pool.start_layer,
end_layer=pool.end_layer,
draft_buffer_infos=tuple(
draft.get_device_pool_infos()[0].buffer_info for draft in drafts
),
write_back_staging=pool.hicache_write_back_staging,
),
)
def build_kv_host_pool(
*,
kv_pool: Any,
@@ -146,21 +168,46 @@ def build_kv_host_pool(
):
if use_mla is None:
use_mla = is_mla_pool(kv_pool)
kv_host_pool_cls = (
MLATokenToKVPoolHost if use_mla else get_mha_host_pool_cls(kv_pool)
)
if use_mla:
kv_host_pool_cls = MLATokenToKVPoolHost
host_input = kv_pool.get_device_pool_infos()[0].buffer_info
mha_kwargs = {}
else:
kv_host_pool_cls, host_input, mha_kwargs = _mha_host_inputs(
kv_pool, mtp_draft_device_pools
)
if use_mla and override_kv_cache_dim is None:
# An fp8 DSA store is wider than kv_lora_rank + qk_rope_head_dim. The
# device pool already resolved the row width.
override_kv_cache_dim = kv_pool.kv_cache_dim
check_packed_kv_rows(
kv_pool=kv_pool, drafts=mtp_draft_device_pools, use_mla=use_mla
)
if use_mla or is_npu():
check_packed_kv_rows(
kv_pool=kv_pool, drafts=mtp_draft_device_pools, use_mla=use_mla
)
kwargs = {}
if override_kv_cache_dim is not None:
kwargs["override_kv_cache_dim"] = override_kv_cache_dim
if mtp_draft_device_pools:
kwargs["mtp_draft_device_pools"] = mtp_draft_device_pools
if use_mla:
kwargs.update(
device_capacity_tokens=kv_pool.size
if kv_pool.host_capacity_tokens is None
else kv_pool.host_capacity_tokens,
start_layer=kv_pool.start_layer,
end_layer=kv_pool.end_layer,
draft_buffer_infos=tuple(
pool.get_device_pool_infos()[0].buffer_info
for pool in mtp_draft_device_pools
),
layer_shard_enabled=kv_pool.layer_shard_enabled,
target_layer_num=(
(kv_pool.layer_num + kv_pool.layer_shard_size - 1)
// kv_pool.layer_shard_size
if kv_pool.layer_shard_enabled
else kv_pool.layer_num
),
)
else:
kwargs.update(mha_kwargs)
parallel = get_parallel()
if parallel.dcp_enabled:
assert use_mla, (
@@ -170,7 +217,7 @@ def build_kv_host_pool(
kwargs["dcp_size"] = parallel.attn_dcp_size
kwargs["dcp_rank"] = parallel.attn_dcp_rank
return kv_host_pool_cls(
kv_pool,
host_input,
get_memory().hicache_ratio,
get_memory().hicache_size if host_size is None else host_size,
page_size,
@@ -421,14 +468,15 @@ def build_hybrid_swa_group(
# from full attention. Use a separate host pool with KV indices instead of
# packing their bytes into either target pool.
first, *rest = mtp_swa_full_device_pools
draft_host_pool = get_mha_host_pool_cls(first)(
first,
host_cls, host_input, host_kwargs = _mha_host_inputs(first, tuple(rest))
draft_host_pool = host_cls(
host_input,
host_to_device_ratio=kv_host_pool.size / first.size,
host_size=0,
page_size=page_size,
layout=get_memory().hicache_mem_layout,
allocator_type=_get_allocator_type(),
mtp_draft_device_pools=rest,
**host_kwargs,
pool_label="draft_swa",
)
group.add_entry(
@@ -1533,9 +1581,16 @@ def _build_mha_mla_host_pool(
pool_label=pool_label,
)
if isinstance(pool, MHATokenToKVPool):
return get_mha_host_pool_cls(pool)(pool, **kwargs)
host_cls, host_input, host_kwargs = _mha_host_inputs(pool)
return host_cls(host_input, **host_kwargs, **kwargs)
return MLATokenToKVPoolHost(
pool,
pool.get_device_pool_infos()[0].buffer_info,
device_capacity_tokens=pool.size
if pool.host_capacity_tokens is None
else pool.host_capacity_tokens,
start_layer=pool.start_layer,
end_layer=pool.end_layer,
target_layer_num=pool.layer_num,
override_kv_cache_dim=pool.kv_cache_dim,
**kwargs,
)
@@ -58,7 +58,7 @@ from sglang.srt.runtime_context import (
get_schedule,
)
from sglang.srt.speculative.base_spec_worker import HiCacheDraftMode
from sglang.srt.utils import ceil_align, is_hip
from sglang.srt.utils import ceil_align, is_hip, is_npu
if TYPE_CHECKING:
from sglang.srt.configs.model_config import ModelConfig
@@ -72,6 +72,8 @@ def prepare_hicache_staging(
*, tp_worker: BaseTpWorker, draft_plan: Optional[HiCacheDraftPlan] = None
) -> None:
"""Materialize MHA transfer buffers before the final KV budget is measured."""
if is_npu():
return
memory = get_memory()
page_size = get_schedule().page_size
if memory.hicache_mem_layout != "page_first" or not (
@@ -93,11 +95,13 @@ def prepare_hicache_staging(
if memory.hicache_size <= 0 and not sidecar:
page_capacity = int(pool.size * memory.hicache_ratio) // page_size + 1
staging = prepare_mha_write_back_staging(
pool,
pool.get_device_pool_infos()[0].buffer_info,
layer_num=pool.layer_num + len(packed_drafts),
page_size=page_size,
page_capacity=page_capacity,
staging=pool.hicache_write_back_staging,
)
pool.hicache_write_back_staging = staging
if staging is not None:
logger.info(
"HiCache staging prepared before KV sizing: %.1f MiB, %d layers",
+139 -1
View File
@@ -100,7 +100,7 @@ from sglang.srt.utils.torch_memory_saver_adapter import TorchMemorySaverAdapter
if TYPE_CHECKING:
from sglang.srt.managers.cache_controller import LayerDoneCounter
from sglang.srt.managers.schedule_batch import Req
from sglang.srt.mem_cache.device_pool_info import DevicePoolInfo
from sglang.srt.mem_cache.device_pool_info import DevicePoolInfo, MHABufferInfo
logger = logging.getLogger(__name__)
@@ -2075,6 +2075,14 @@ class KVCache(abc.ABC):
maybe_init_custom_mem_pool(device=self.device)
)
def get_device_pool_infos(self) -> tuple[DevicePoolInfo, ...]:
"""Persistent buffers for migrated host consumers; see device_pool_info.py.
Unmigrated formats retain their existing construction entry points.
Consumers must not use exceptions or empty descriptions to pick a path.
"""
raise NotImplementedError(f"{type(self).__name__} has no device pool provider")
def host_pool_decls(self):
"""Host pools HiCache keeps for the buffers this device pool (and its
sub-pools) owns: the KV pool itself, plus dependent pools such as sparse
@@ -2225,6 +2233,11 @@ class MHATokenToKVPool(KVCache):
):
self.k_buffer = None
self.v_buffer = None
if not _is_npu:
self.k_data_ptrs = None
self.v_data_ptrs = None
self.k_scale_buffer = None
self.v_scale_buffer = None
if post_capture_active:
# Reserved upper bound only (unbacked VA): page-align UP so
# (size + page_size) % page_size == 0 holds for paged layouts.
@@ -2324,6 +2337,64 @@ class MHATokenToKVPool(KVCache):
# for store_cache JIT kernel
self.row_dim = self.head_num * self.head_dim
self.v_row_dim = self.head_num * self.v_head_dim
self.model_layer_ids = tuple(
range(self.start_layer, self.start_layer + self.layer_num)
)
def get_device_pool_infos(self) -> tuple[DevicePoolInfo, ...]:
"""Describe persistent KV buffers for host assembly without allocating."""
from sglang.srt.mem_cache.device_pool_info import DevicePoolInfo
from sglang.srt.mem_cache.hicache_storage import PoolName
return (
DevicePoolInfo(
pool_name=PoolName.KV,
indices_from_pool=PoolName.KV,
layer_ids=self.model_layer_ids,
buffer_info=self._mha_buffer_info(),
),
)
def _mha_buffers(self):
from sglang.srt.mem_cache.device_pool_info import KVBufferLayout, KVBuffers
return KVBuffers(
k=tuple(self.k_buffer),
v=tuple(self.v_buffer),
k_data_ptrs=self.k_data_ptrs,
v_data_ptrs=self.v_data_ptrs,
layout=KVBufferLayout.NHD
if self.kv_cache_layout == "page_major"
else KVBufferLayout(self.kv_cache_layout),
)
def _mha_scale_buffers(self):
from sglang.srt.mem_cache.device_pool_info import KVScaleBuffers, KVScaleLayout
if self.k_scale_buffer is None:
return None
return KVScaleBuffers(
k=tuple(self.k_scale_buffer),
v=tuple(self.v_scale_buffer),
layout=KVScaleLayout.TOKEN,
)
def _mha_buffer_info(self) -> MHABufferInfo:
from sglang.srt.mem_cache.device_pool_info import MHABufferInfo
buffers = self._mha_buffers()
return MHABufferInfo(
buffers=buffers,
page_size=self.page_size,
head_num=self.row_dim // self.head_dim
if self.head_dim == self.v_head_dim
else self.head_num,
head_dim=self.head_dim,
v_head_dim=self.v_head_dim,
dtype=self.store_dtype,
device=buffers.k[0].device if len(buffers.k) else torch.device(self.device),
scales=self._mha_scale_buffers(),
)
def _init_kv_copy_and_warmup(self):
# Zero-layer pool (e.g. all-SWA model's full sub-pool) has no buffers.
@@ -3789,6 +3860,17 @@ class MHATokenToKVPoolMXFP8(MHATokenToKVPool):
MXFP8_SCALE_BLOCK_SIZE = 32
def _mha_scale_buffers(self):
from sglang.srt.mem_cache.device_pool_info import KVScaleBuffers, KVScaleLayout
return KVScaleBuffers(
k=tuple(self.k_scale_buffer),
v=tuple(self.v_scale_buffer),
layout=KVScaleLayout.MXFP8_INTERLEAVED
if self.mxfp8_sf_interleaved
else KVScaleLayout.TOKEN,
)
def _create_buffers(self):
with self.memory_saver_adapter.region(GPU_MEMORY_TYPE_KV_CACHE):
with (
@@ -4272,6 +4354,9 @@ class HybridLinearKVPool(KVCache):
qk_rope_head_dim=qk_rope_head_dim,
enable_memory_saver=enable_memory_saver,
)
if isinstance(self.full_kv_pool, MHATokenToKVPool):
# Only the hybrid root knows the model IDs of its compact child.
self.full_kv_pool.model_layer_ids = tuple(full_attention_layer_ids)
self.full_attention_layer_id_mapping = {
id: i for i, id in enumerate(full_attention_layer_ids)
}
@@ -4716,6 +4801,11 @@ class MLATokenToKVPool(KVCache):
else (kv_lora_rank + qk_rope_head_dim)
)
self.model_layer_ids = tuple(
range(self.start_layer, self.start_layer + self.layer_num)
)
self.kv_scale_buffer = None
self._create_buffers()
self.data_ptrs = torch.tensor(
@@ -4727,6 +4817,47 @@ class MLATokenToKVPool(KVCache):
# DSA will allocate indexer KV cache later and then log the total size
self._finalize_allocation_log(size)
def get_device_pool_infos(self) -> tuple[DevicePoolInfo, ...]:
from sglang.srt.mem_cache.device_pool_info import DevicePoolInfo
from sglang.srt.mem_cache.hicache_storage import PoolName
info = self._mla_buffer_info()
return (
DevicePoolInfo(
pool_name=PoolName.KV,
indices_from_pool=PoolName.KV,
layer_ids=tuple(self.model_layer_ids[i] for i in info.layer_indices),
buffer_info=info,
),
)
def _mla_buffer_info(self):
from sglang.srt.mem_cache.device_pool_info import MLABufferInfo
start, end = (
self._owned_local_layer_range()
if self.layer_shard_enabled
else (0, self.layer_num)
)
return MLABufferInfo(
buffers=tuple(self.kv_buffer[start:end]),
data_ptrs=self.data_ptrs
if (start, end) == (0, self.layer_num)
else self.data_ptrs[start:end],
layer_indices=tuple(range(start, end)),
device_layer_num=self.layer_num,
page_size=self.page_size,
kv_lora_rank=self.kv_lora_rank,
qk_rope_head_dim=self.qk_rope_head_dim,
kv_cache_dim=self.kv_cache_dim,
dtype=self.store_dtype,
device=self.data_ptrs.device,
packed_fp8=self.dsa_kv_cache_store_fp8,
scales=None
if self.kv_scale_buffer is None
else tuple(self.kv_scale_buffer[start:end]),
)
def _create_buffers(self):
with self.memory_saver_adapter.region(GPU_MEMORY_TYPE_KV_CACHE):
with (
@@ -5858,6 +5989,13 @@ class MiniMaxSparseKVPool(KVCache):
else None
)
if enable_hisparse:
self.main_pool.model_layer_ids = tuple(local_sparse_layer_ids)
if self.dense_pool is not None:
self.dense_pool.model_layer_ids = tuple(local_dense_layer_ids)
if self.index_kv_pool is not None:
self.index_kv_pool.model_layer_ids = tuple(local_kv_sparse_layer_ids)
self.mem_usage = self.main_pool.mem_usage
if self.dense_pool is not None:
self.mem_usage += self.dense_pool.mem_usage
+116 -104
View File
@@ -28,10 +28,12 @@ from sglang.kernels.ops.kvcache.hicache import (
from sglang.kernels.ops.kvcache.hicache import (
transfer_hicache_one_layer_mla as jit_transfer_hicache_one_layer_mla,
)
from sglang.srt.mem_cache.device_pool_info import (
KVScaleLayout,
MHABufferInfo,
)
from sglang.srt.mem_cache.memory_pool import (
MHATokenToKOnlyPool,
MHATokenToKVPool,
MHATokenToKVPoolMXFP8,
)
from sglang.srt.mem_cache.pool_host.base import (
_WRITE_BACK_STAGING_PAGE_CHUNK,
@@ -76,46 +78,37 @@ logger = logging.getLogger(__name__)
def prepare_mha_write_back_staging(
device_pool: MHATokenToKVPool,
buffer_info: MHABufferInfo,
*,
layer_num: int,
page_size: int,
page_capacity: int = _WRITE_BACK_STAGING_PAGE_CHUNK,
retain: bool = True,
staging: tuple[torch.Tensor, torch.Tensor] | None = None,
) -> tuple[torch.Tensor, torch.Tensor] | None:
if not (_is_cuda or _is_hip) or layer_num == 0:
return None
head_num = device_pool.head_num
if device_pool.head_dim == device_pool.v_head_dim:
head_num = device_pool.row_dim // device_pool.head_dim
head_num = buffer_info.head_num
if not all(
can_use_write_back_jit_kernel(
element_size=head_num * dim * device_pool.store_dtype.itemsize
element_size=head_num * dim * buffer_info.dtype.itemsize
)
for dim in (device_pool.head_dim, device_pool.v_head_dim)
for dim in (buffer_info.head_dim, buffer_info.v_head_dim)
):
return None
token_capacity = min(page_capacity, _WRITE_BACK_STAGING_PAGE_CHUNK) * page_size
shapes = tuple(
(token_capacity, layer_num, head_num, dim)
for dim in (device_pool.head_dim, device_pool.v_head_dim)
for dim in (buffer_info.head_dim, buffer_info.v_head_dim)
)
staging = device_pool.hicache_write_back_staging
if staging is None:
staging = (
torch.empty(
shapes[0], dtype=device_pool.store_dtype, device=device_pool.device
),
torch.empty(
shapes[1], dtype=device_pool.store_dtype, device=device_pool.device
),
torch.empty(shapes[0], dtype=buffer_info.dtype, device=buffer_info.device),
torch.empty(shapes[1], dtype=buffer_info.dtype, device=buffer_info.device),
)
if retain:
device_pool.hicache_write_back_staging = staging
else:
for buffer, shape in zip(staging, shapes):
if (
buffer.dtype != device_pool.store_dtype
buffer.dtype != buffer_info.dtype
or tuple(buffer.shape[1:]) != shape[1:]
or buffer.shape[0] < shape[0]
):
@@ -124,14 +117,9 @@ def prepare_mha_write_back_staging(
class MHATokenToKVPoolHost(HostKVCache):
device_pool: MHATokenToKVPool
mtp_draft_device_pools: tuple[MHATokenToKVPool, ...] = ()
_transfer_device: torch.device
def __init__(
self,
device_pool: MHATokenToKVPool,
buffer_info: MHABufferInfo,
host_to_device_ratio: float,
host_size: int,
page_size: int,
@@ -140,41 +128,70 @@ class MHATokenToKVPoolHost(HostKVCache):
device: str = "cpu",
allocator_type: str = "default",
*,
mtp_draft_device_pools: Sequence[MHATokenToKVPool] = (),
device_capacity_tokens: int,
start_layer: int,
end_layer: int,
draft_buffer_infos: Sequence[MHABufferInfo] = (),
write_back_staging: tuple[torch.Tensor, torch.Tensor] | None = None,
pool_label: str = "kv",
head_group_num: int = 1,
):
self.mtp_draft_device_pools = tuple(mtp_draft_device_pools)
self.target_layer_num = device_pool.layer_num
# page_unified cuts the kv-head axis into this many groups, and the cut
# is part of the byte order, so it has to be known before the buffer is
# allocated in init_kv_buffer() below.
infos = (buffer_info, *draft_buffer_infos)
for info in infos:
if (
info.scales is not None
and info.scales.layout != KVScaleLayout.MXFP8_INTERLEAVED
):
raise ValueError(
f"{pool_label}: MHA host transfers do not support "
f"KV scale layout {info.scales.layout.value!r}."
)
info.validate()
if info.transfer_buffers is None and (
info.buffers.k_data_ptrs is None or info.buffers.v_data_ptrs is None
):
raise ValueError(
f"{pool_label}: MHA host requires existing K/V pointer tables"
)
for depth, draft in enumerate(draft_buffer_infos):
if len(draft.buffers.k) != 1:
raise ValueError(
f"{pool_label}: packed draft[{depth}] has {len(draft.buffers.k)} layers; expected 1"
)
target_geometry = (
buffer_info.head_num,
buffer_info.head_dim,
buffer_info.v_head_dim,
buffer_info.dtype,
buffer_info.device,
)
draft_geometry = (
draft.head_num,
draft.head_dim,
draft.v_head_dim,
draft.dtype,
draft.device,
)
if draft_geometry != target_geometry:
raise ValueError(
f"{pool_label}: packed draft[{depth}] (heads, K width, V width, dtype, device)={draft_geometry}, expected {target_geometry}"
)
self.target_layer_num = len(buffer_info.buffers.k)
self._transfer_device = buffer_info.device
self.dtype = buffer_info.dtype
self.head_dim = buffer_info.head_dim
self.v_head_dim = buffer_info.v_head_dim
self.head_num = buffer_info.head_num
self.layer_num = self.target_layer_num + len(draft_buffer_infos)
self._buffer_infos = infos
self.head_group_num = head_group_num
# Transfer buffers are captured below. The pool remains available for
# NPU transfer APIs and device layer ownership.
self.device_pool = device_pool
self._transfer_device = torch.device(device_pool.device)
self.dtype = device_pool.store_dtype
self.head_dim = device_pool.head_dim
self.v_head_dim = device_pool.v_head_dim
# Equal-width pools may store multiple attention rows per allocator token.
self.head_num = (
device_pool.row_dim // self.head_dim
if self.head_dim == self.v_head_dim
else device_pool.head_num
)
self.layer_num = self.target_layer_num + len(self.mtp_draft_device_pools)
# Scale-bearing subclasses need page coverage when computing token bytes.
self.page_size = page_size
device_capacity = getattr(device_pool, "host_capacity_tokens", None)
super().__init__(
dtype=self.dtype,
device_capacity_tokens=(
device_pool.size if device_capacity is None else device_capacity
),
device_capacity_tokens=device_capacity_tokens,
size_per_token=self.get_size_per_token(),
start_layer=device_pool.start_layer,
end_layer=device_pool.end_layer,
start_layer=start_layer,
end_layer=end_layer,
host_to_device_ratio=host_to_device_ratio,
host_size=host_size,
page_size=page_size,
@@ -184,12 +201,22 @@ class MHATokenToKVPoolHost(HostKVCache):
allocator_type=allocator_type,
pool_label=pool_label,
)
pools = (self.device_pool, *self.mtp_draft_device_pools)
self.device_k_buffers = tuple(b for pool in pools for b in pool.k_buffer)
self.device_v_buffers = tuple(b for pool in pools for b in pool.v_buffer)
self._initialize_transfer_views()
self.device_k_buffers = tuple(b for info in infos for b in info.buffers.k)
self.device_v_buffers = tuple(b for info in infos for b in info.buffers.v)
if buffer_info.transfer_buffers is None:
self.device_k_data_ptrs = (
torch.cat([info.buffers.k_data_ptrs for info in infos])
if draft_buffer_infos
else buffer_info.buffers.k_data_ptrs
)
self.device_v_data_ptrs = (
torch.cat([info.buffers.v_data_ptrs for info in infos])
if draft_buffer_infos
else buffer_info.buffers.v_data_ptrs
)
self._initialize_transfer_views(buffer_info, write_back_staging)
def _initialize_transfer_views(self) -> None:
def _initialize_transfer_views(self, buffer_info, write_back_staging) -> None:
self.element_dim = self.head_num * self.head_dim
self._init_device_row_stride((*self.device_k_buffers, *self.device_v_buffers))
# The JIT HiCache kernels also build with hipcc (ROCm): the PTX-only
@@ -235,20 +262,8 @@ class MHATokenToKVPoolHost(HostKVCache):
if self.v_data_refs
else None
)
if not _is_npu:
pools = (self.device_pool, *self.mtp_draft_device_pools)
self.device_k_data_ptrs = (
torch.cat([pool.k_data_ptrs for pool in pools])
if self.mtp_draft_device_pools
else self.device_pool.k_data_ptrs
)
self.device_v_data_ptrs = (
torch.cat([pool.v_data_ptrs for pool in pools])
if self.mtp_draft_device_pools
else self.device_pool.v_data_ptrs
)
self.host_kv_data_refs = self.k_data_refs + self.v_data_refs
self._init_write_back_staging_buffers()
self._init_write_back_staging_buffers(buffer_info, write_back_staging)
def get_size_per_token(self):
return (
@@ -322,7 +337,7 @@ class MHATokenToKVPoolHost(HostKVCache):
)
return buffer
def _init_write_back_staging_buffers(self):
def _init_write_back_staging_buffers(self, buffer_info, write_back_staging):
self.staging_page_capacity = 0
self.staging_token_capacity = 0
self.staging_k_buffer = None
@@ -337,11 +352,11 @@ class MHATokenToKVPoolHost(HostKVCache):
return
page_capacity = min(self.page_num, _WRITE_BACK_STAGING_PAGE_CHUNK)
staging = prepare_mha_write_back_staging(
self.device_pool,
buffer_info,
layer_num=self.layer_num,
page_size=self.page_size,
page_capacity=page_capacity,
retain=False,
staging=write_back_staging,
)
if staging is None:
return
@@ -370,10 +385,9 @@ class MHATokenToKVPoolHost(HostKVCache):
*,
is_draft: bool = False,
):
if not is_draft and not self._is_device_layer_owned(device_pool, layer_id):
if not is_draft and not 0 <= layer_id < self.target_layer_num:
return
# MTP draft layers do not participate in CP layer sharding.
host_layer_id = layer_id if is_draft else self._host_layer_index(layer_id)
host_layer_id = layer_id
if io_backend != "kernel_ascend":
device_k = self.device_k_buffers[layer_id]
@@ -487,7 +501,7 @@ class MHATokenToKVPoolHost(HostKVCache):
device_v,
host_k,
host_v,
) in self._npu_transfer_buffers(device_pool):
) in self._npu_transfer_buffers():
transfer_kv_dim_exchange(
device_indices=device_indices,
host_indices=host_indices,
@@ -503,10 +517,10 @@ class MHATokenToKVPoolHost(HostKVCache):
else:
raise ValueError(f"Unsupported IO backend: {io_backend}")
def _npu_transfer_buffers(self, target_device_pool):
def _npu_transfer_buffers(self):
layer_start = 0
for pool in (target_device_pool, *self.mtp_draft_device_pools):
device_k, device_v = pool.get_hicache_transfer_buffers()
for info in self._buffer_infos:
device_k, device_v = info.transfer_buffers()
layer_end = layer_start + device_k.shape[0]
yield (
device_k,
@@ -634,9 +648,7 @@ class MHATokenToKVPoolHost(HostKVCache):
raise ValueError(f"Unsupported layout: {self.layout}")
elif io_backend == "kernel_ascend":
if self.layout == "page_first_direct":
for device_k, device_v, host_k, host_v in self._npu_transfer_buffers(
device_pool
):
for device_k, device_v, host_k, host_v in self._npu_transfer_buffers():
transfer_kv_dim_exchange(
device_indices=device_indices,
host_indices=host_indices,
@@ -1185,16 +1197,23 @@ class HiSparseMHATokenToKVPoolHost(HiSparseHostPoolMixin, MHATokenToKVPoolHost):
def __init__(
self,
device_pool: MHATokenToKVPool,
buffer_info: MHABufferInfo,
host_to_device_ratio: float,
page_size: int,
*,
device_capacity_tokens: int,
start_layer: int,
end_layer: int,
):
super().__init__(
device_pool=device_pool,
buffer_info=buffer_info,
host_to_device_ratio=host_to_device_ratio,
host_size=0,
page_size=page_size,
layout="layer_first",
device_capacity_tokens=device_capacity_tokens,
start_layer=start_layer,
end_layer=end_layer,
)
def get_contiguous_buf_infos(self):
@@ -1343,10 +1362,9 @@ class AsymmetricMHATokenToKVPoolHost(MHATokenToKVPoolHost):
*,
is_draft: bool = False,
):
if not is_draft and not self._is_device_layer_owned(device_pool, layer_id):
if not is_draft and not 0 <= layer_id < self.target_layer_num:
return
# MTP draft layers do not participate in CP layer sharding.
host_layer_id = layer_id if is_draft else self._host_layer_index(layer_id)
host_layer_id = layer_id
if io_backend != "kernel_ascend":
device_k = self.device_k_buffers[layer_id]
@@ -1574,25 +1592,19 @@ class AsymmetricMHATokenToKVPoolHost(MHATokenToKVPoolHost):
)
def get_mha_host_pool_cls(device_pool: MHATokenToKVPool) -> type:
"""Pick the right MHA host-pool class based on the device pool's K/V dims.
Returns ``MHATokenToKVPoolMXFP8Host`` for the block-scaled MXFP8 pool (its
UE8M0 scales must travel with the payload),
``AsymmetricMHATokenToKVPoolHost`` when ``head_dim != v_head_dim``
(e.g. MiMo-V2), else the default ``MHATokenToKVPoolHost``.
"""
if isinstance(device_pool, MHATokenToKVPoolMXFP8):
if device_pool.head_dim != device_pool.v_head_dim:
def get_mha_host_pool_cls(buffer_info: MHABufferInfo) -> type:
is_mxfp8 = (
buffer_info.scales is not None
and buffer_info.scales.layout == KVScaleLayout.MXFP8_INTERLEAVED
)
if is_mxfp8:
if buffer_info.head_dim != buffer_info.v_head_dim:
raise NotImplementedError(
"MXFP8 HiCache does not support asymmetric K/V head dimensions yet."
)
from sglang.srt.mem_cache.pool_host.mha_mxfp8 import (
MHATokenToKVPoolMXFP8Host,
)
from sglang.srt.mem_cache.pool_host.mha_mxfp8 import MHATokenToKVPoolMXFP8Host
return MHATokenToKVPoolMXFP8Host
if device_pool.head_dim != device_pool.v_head_dim:
if buffer_info.head_dim != buffer_info.v_head_dim:
return AsymmetricMHATokenToKVPoolHost
return MHATokenToKVPoolHost
@@ -23,7 +23,7 @@ from sglang.kernels.ops.kvcache.hicache import (
from sglang.kernels.ops.kvcache.hicache import (
transfer_hicache_one_layer_mla as jit_transfer_hicache_one_layer_mla,
)
from sglang.srt.mem_cache.memory_pool import MHATokenToKVPoolMXFP8
from sglang.srt.mem_cache.device_pool_info import KVScaleLayout, MHABufferInfo
from sglang.srt.mem_cache.pool_host.common import (
ALLOC_MEMORY_FUNCS,
_cuda_host_unregister,
@@ -36,11 +36,9 @@ from sglang.srt.mem_cache.pool_host.mha import (
class MHATokenToKVPoolMXFP8Host(MHATokenToKVPoolHost):
device_pool: MHATokenToKVPoolMXFP8
def __init__(
self,
device_pool: MHATokenToKVPoolMXFP8,
buffer_info: MHABufferInfo,
host_to_device_ratio: float,
host_size: int,
page_size: int,
@@ -49,29 +47,38 @@ class MHATokenToKVPoolMXFP8Host(MHATokenToKVPoolHost):
device: str = "cpu",
allocator_type: str = "default",
*,
mtp_draft_device_pools: Sequence = (),
device_capacity_tokens: int,
start_layer: int,
end_layer: int,
draft_buffer_infos: Sequence[MHABufferInfo] = (),
write_back_staging: tuple[torch.Tensor, torch.Tensor] | None = None,
pool_label: str = "kv",
):
if layout != "page_first":
raise NotImplementedError(
f"MXFP8 KV host pool supports only the page_first layout, got {layout!r}."
)
if mtp_draft_device_pools:
if draft_buffer_infos:
raise NotImplementedError(
"MXFP8 KV host pool does not pack MTP draft KV layers."
)
if not device_pool.mxfp8_sf_interleaved:
interleaved = (
buffer_info.scales is not None
and buffer_info.scales.layout == KVScaleLayout.MXFP8_INTERLEAVED
)
if not interleaved:
raise NotImplementedError(
"MXFP8 KV host pool requires the interleaved (page_size=128) scale layout."
)
# Bytes of UE8M0 scales one page holds per layer; needed by
# get_size_per_token before the base constructor sizes the pool.
self.k_sf_page_bytes = device_pool.k_scale_buffer[0][0].numel()
self.v_sf_page_bytes = device_pool.v_scale_buffer[0][0].numel()
k_scales = buffer_info.scales.k
v_scales = buffer_info.scales.v
# HostKVCache calls get_size_per_token during base initialization.
self.k_sf_page_bytes = k_scales[0][0].numel()
self.v_sf_page_bytes = v_scales[0][0].numel()
self.k_scale_host: torch.Tensor | None = None
self.v_scale_host: torch.Tensor | None = None
super().__init__(
device_pool,
buffer_info,
host_to_device_ratio,
host_size,
page_size,
@@ -79,13 +86,17 @@ class MHATokenToKVPoolMXFP8Host(MHATokenToKVPoolHost):
pin_memory,
device,
allocator_type,
device_capacity_tokens=device_capacity_tokens,
start_layer=start_layer,
end_layer=end_layer,
write_back_staging=write_back_staging,
pool_label=pool_label,
)
if self.page_size != device_pool.page_size:
if self.page_size != buffer_info.page_size:
raise ValueError(
"MXFP8 KV host pool moves scales per page, so the host page size "
f"({self.page_size}) must equal the device page size "
f"({device_pool.page_size})."
f"({buffer_info.page_size})."
)
if not self.can_use_write_back_jit or not all(
can_use_write_back_jit_kernel(element_size=size)
@@ -95,15 +106,15 @@ class MHATokenToKVPoolMXFP8Host(MHATokenToKVPoolHost):
"MXFP8 KV host pool needs the staged JIT write-back kernel "
"(io_backend='kernel', page_first layout, CUDA or HIP)."
)
self._init_scale_buffers()
self._init_scale_buffers(k_scales, v_scales)
def get_size_per_token(self):
payload = super().get_size_per_token()
scales = (self.k_sf_page_bytes + self.v_sf_page_bytes) // self.page_size
return payload + scales * self.layer_num
def _init_scale_buffers(self):
alloc_func = ALLOC_MEMORY_FUNCS[self.device_pool.device]
def _init_scale_buffers(self, k_scales, v_scales):
alloc_func = ALLOC_MEMORY_FUNCS[self._transfer_device.type]
# Host: page-first like the payload, one contiguous scale block per
# (page, layer) so a page's layers are a single memcpy span.
self.k_scale_host = alloc_func(
@@ -126,32 +137,30 @@ class MHATokenToKVPoolMXFP8Host(MHATokenToKVPoolHost):
# Device: each layer's interleaved scale tensor as [page, bytes] rows.
self.k_scale_device_rows = [
buf.view(torch.uint8).reshape(buf.shape[0], -1)
for buf in self.device_pool.k_scale_buffer
buf.view(torch.uint8).reshape(buf.shape[0], -1) for buf in k_scales
]
self.v_scale_device_rows = [
buf.view(torch.uint8).reshape(buf.shape[0], -1)
for buf in self.device_pool.v_scale_buffer
buf.view(torch.uint8).reshape(buf.shape[0], -1) for buf in v_scales
]
self.k_scale_device_ptrs = torch.tensor(
[rows.data_ptr() for rows in self.k_scale_device_rows],
dtype=torch.uint64,
device=self.device_pool.device,
device=self._transfer_device,
)
self.v_scale_device_ptrs = torch.tensor(
[rows.data_ptr() for rows in self.v_scale_device_rows],
dtype=torch.uint64,
device=self.device_pool.device,
device=self._transfer_device,
)
self.k_scale_staging = torch.empty(
(self.staging_page_capacity, self.layer_num, self.k_sf_page_bytes),
dtype=torch.uint8,
device=self.device_pool.device,
device=self._transfer_device,
)
self.v_scale_staging = torch.empty(
(self.staging_page_capacity, self.layer_num, self.v_sf_page_bytes),
dtype=torch.uint8,
device=self.device_pool.device,
device=self._transfer_device,
)
def _page_ids(self, indices: torch.Tensor) -> torch.Tensor:
@@ -209,9 +218,9 @@ class MHATokenToKVPoolMXFP8Host(MHATokenToKVPoolHost):
raise NotImplementedError(
f"MXFP8 KV host pool supports only io_backend='kernel', got {io_backend!r}."
)
if not self._is_device_layer_owned(device_pool, layer_id):
if not 0 <= layer_id < self.target_layer_num:
return
host_layer_id = self._host_layer_index(layer_id)
host_layer_id = layer_id
device_pages = self._page_ids(device_indices)
host_pages = self._page_ids(host_indices)
for dst_rows, src_rows, sf_page_bytes in (
+182 -154
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import logging
from typing import Optional, Sequence
import msgspec
import torch
from sglang.kernels.ops.kvcache.hicache import (
@@ -19,7 +20,7 @@ from sglang.kernels.ops.kvcache.hicache import (
transfer_hicache_one_layer_mla as jit_transfer_hicache_one_layer_mla,
)
from sglang.srt.layers.dcp.layout import maybe_dcp_kernel_indices
from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool
from sglang.srt.mem_cache.device_pool_info import MLABufferInfo, MLASplitBuffers
from sglang.srt.mem_cache.pool_host.base import (
_WRITE_BACK_STAGING_PAGE_CHUNK,
HostKVCache,
@@ -62,15 +63,28 @@ if _is_npu:
logger = logging.getLogger(__name__)
class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
device_pool: MLATokenToKVPool
mtp_draft_device_pools: tuple[MLATokenToKVPool, ...] = ()
class MLAHostAllocation(msgspec.Struct, frozen=True, kw_only=True):
"""Allocator-only geometry; it carries no device buffers or transfer promise."""
_transfer_device: torch.device
dtype: torch.dtype
kv_lora_rank: int
qk_rope_head_dim: int
kv_cache_dim: int
size_per_token: int
draft_layer_num: int = 0
packed_fp8: bool = False
class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
"""Host mirror built by the assembler or HiSparse from captured MLA buffers.
Capacity and reserved CP rows remain caller policy. Transfer methods retain
the shared HostKVCache signature but use the captured tensors and slot map.
"""
def __init__(
self,
device_pool: MLATokenToKVPool,
buffer_info: MLABufferInfo | MLAHostAllocation,
host_to_device_ratio: float,
host_size: int,
page_size: int,
@@ -79,46 +93,117 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
device: str = "cpu",
allocator_type: str = "default",
override_kv_cache_dim: Optional[int] = None,
mtp_draft_device_pools: Sequence[MLATokenToKVPool] = (),
draft_buffer_infos: Sequence[MLABufferInfo] = (),
dcp_size: int = 1,
dcp_rank: int = 0,
*,
device_capacity_tokens: int,
start_layer: int,
end_layer: int,
target_layer_num: int,
layer_shard_enabled: bool = False,
pool_label: str = "kv",
is_dummy: bool = False,
):
self._is_dummy = isinstance(buffer_info, MLAHostAllocation)
self.override_kv_cache_dim = override_kv_cache_dim
self.mtp_draft_device_pools = tuple(mtp_draft_device_pools)
self._is_dummy = is_dummy
# Transfer buffers are captured below. The pool remains available for
# NPU transfer APIs and device layer ownership.
self.device_pool = device_pool
if not is_dummy:
self._transfer_device = torch.device(device_pool.device)
self.dtype = device_pool.store_dtype
self.dtype = buffer_info.dtype
self.layout = layout
self.kv_lora_rank = device_pool.kv_lora_rank
self.qk_rope_head_dim = device_pool.qk_rope_head_dim
# L3 excludes the dead V component when the device stores packed FP8 KV.
self.dsa_kv_cache_store_fp8 = getattr(
device_pool, "dsa_kv_cache_store_fp8", False
)
self.target_layer_num = self._effective_host_layer_num()
self.layer_num = self.target_layer_num + len(self.mtp_draft_device_pools)
self.kv_lora_rank = buffer_info.kv_lora_rank
self.qk_rope_head_dim = buffer_info.qk_rope_head_dim
# Existing builders explicitly resolve packed row width before construction.
self.kv_cache_dim = override_kv_cache_dim or (
self.kv_lora_rank + self.qk_rope_head_dim
buffer_info.kv_cache_dim
if self._is_dummy
else self.kv_lora_rank + self.qk_rope_head_dim
)
device_capacity = getattr(device_pool, "host_capacity_tokens", None)
if is_dummy:
device_capacity = device_capacity or device_pool.size
self.dsa_kv_cache_store_fp8 = buffer_info.packed_fp8
self.target_layer_num = target_layer_num
self.layer_num = target_layer_num + (
buffer_info.draft_layer_num if self._is_dummy else len(draft_buffer_infos)
)
self._layer_shard_enabled = layer_shard_enabled
self._native_buffers = None
if self._is_dummy:
if draft_buffer_infos:
raise ValueError(
"Allocator-only MLA does not accept physical draft buffers"
)
self._size_per_token = buffer_info.size_per_token
else:
infos = (buffer_info, *draft_buffer_infos)
for index, info in enumerate(infos):
info.validate()
if info.scales is not None:
role = "target" if index == 0 else f"draft[{index - 1}]"
raise ValueError(
f"MLA host cache does not support {role} FP4 KV: "
"packed payload and separate block scales cannot be restored"
)
if index and (
(info.kv_cache_dim, info.dtype)
!= (buffer_info.kv_cache_dim, buffer_info.dtype)
or type(info.buffers) is not type(buffer_info.buffers)
):
raise ValueError(
f"packed draft KV row {(info.kv_cache_dim, info.dtype)} differs "
f"from target KV row {(buffer_info.kv_cache_dim, buffer_info.dtype)}"
)
if index and len(info.layer_indices) != 1:
raise ValueError(
f"MLA packed draft[{index - 1}] must contain one layer, got {info.layer_indices}"
)
self._transfer_device = buffer_info.device
self._device_layer_num = buffer_info.device_layer_num
self._target_layer_indices = buffer_info.layer_indices
self._draft_layer_num = len(draft_buffer_infos)
slots = (
*buffer_info.layer_indices,
*range(
self._device_layer_num,
self._device_layer_num + self._draft_layer_num,
),
)
self._buffer_index_by_layer = {layer: i for i, layer in enumerate(slots)}
self._host_layer_by_controller = dict(
zip(buffer_info.layer_indices, range(len(buffer_info.layer_indices)))
)
self._host_layer_by_controller.update(
{
self._device_layer_num + i: target_layer_num + i
for i in range(self._draft_layer_num)
}
)
self._size_per_token = (
self.kv_cache_dim * self.dtype.itemsize * self.layer_num
)
if isinstance(buffer_info.buffers, MLASplitBuffers):
self._native_buffers = buffer_info.buffers
self._native_infos = infos
native = self._native_buffers
index_layers = len(native.index_layer_indices)
if layout == "page_first_kv_split" and native.index_k is not None:
self._size_per_token += (
native.index_width * self.dtype.itemsize * index_layers
)
if native.index_scales is not None:
self._size_per_token += 4 * index_layers
self.device_kv_buffers = ()
self.device_data_ptrs = None
else:
self.device_kv_buffers = tuple(
tensor for info in infos for tensor in info.buffers
)
self.device_data_ptrs = (
torch.cat([info.data_ptrs for info in infos])
if draft_buffer_infos
else buffer_info.data_ptrs
)
super().__init__(
dtype=self.dtype,
device_capacity_tokens=(
device_pool.size if device_capacity is None else device_capacity
),
device_capacity_tokens=device_capacity_tokens,
size_per_token=self.get_size_per_token(),
start_layer=device_pool.start_layer,
end_layer=device_pool.end_layer,
start_layer=start_layer,
end_layer=end_layer,
host_to_device_ratio=host_to_device_ratio,
host_size=host_size,
page_size=page_size,
@@ -129,9 +214,9 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
dcp_size=dcp_size,
dcp_rank=dcp_rank,
pool_label=pool_label,
is_dummy=is_dummy,
is_dummy=self._is_dummy,
)
if is_dummy:
if self._is_dummy:
self.token_stride_size = self.kv_cache_dim * self.dtype.itemsize
self.layout_dim = self.token_stride_size * self.layer_num
self.can_use_jit = False
@@ -149,16 +234,6 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
)
return
# Ascend transfers split K, rope and index tensors through its own API.
self.device_kv_buffers = (
()
if _is_npu
else tuple(
b
for pool in (device_pool, *self.mtp_draft_device_pools)
for b in pool.kv_buffer
)
)
self._initialize_transfer_views()
def _initialize_transfer_views(self) -> None:
@@ -184,14 +259,8 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
self._transfer_device,
host_memory_registered=self.pin_memory,
)
if not _is_npu:
pools = (self.device_pool, *self.mtp_draft_device_pools)
self.device_data_ptrs = (
torch.cat([pool.data_ptrs for pool in pools])
if self.mtp_draft_device_pools
else self.device_pool.data_ptrs
)
self._init_device_row_stride(self.device_kv_buffers)
if self._native_buffers is None:
self._init_device_row_stride(self.device_kv_buffers)
self._init_write_back_staging_buffers()
def get_contiguous_buf_infos(self):
@@ -210,27 +279,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
return data_ptrs, data_lens, item_lens
def get_size_per_token(self):
size_per_token = self.kv_cache_dim * self.dtype.itemsize * self.layer_num
if (
self.layout == "page_first_kv_split"
and self.device_pool.index_head_dim is not None
):
# Indexer buffers only exist for physical Indexer layers, which can
# be a subset of all layers (e.g. GLM 5.2: 21 of 78). Mirror the
# layer count used by init_kv_buffer so host capacity sizing stays
# consistent with the actually-allocated buffers.
num_indexer_layers = getattr(self.device_pool, "num_indexer_layers", None)
if num_indexer_layers is None:
num_indexer_layers = self.layer_num
size_per_token += (
self.device_pool.index_head_dim
* self.dtype.itemsize
* num_indexer_layers
)
if getattr(self.device_pool, "index_k_scale_buffer", None) is not None:
# FP32 quantization scale per token per indexer layer.
size_per_token += 4 * num_indexer_layers
return size_per_token
return self._size_per_token
def get_ksize_per_token(self):
return self.get_size_per_token()
@@ -282,15 +331,14 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
# pool packs them as (num_indexer_layers, page, ...); mirror that
# layer count here so transfer_kv_dim_exchange's layer check
# (device dim0 == host dim1) holds.
num_indexer_layers = getattr(self.device_pool, "num_indexer_layers", None)
if num_indexer_layers is None:
num_indexer_layers = self.layer_num
native = self._native_buffers
num_indexer_layers = len(native.index_layer_indices)
indexer_dims = (self.page_num, num_indexer_layers, self.page_size, 1)
alloc_func = ALLOC_MEMORY_FUNCS[self._transfer_device.type]
if getattr(self.device_pool, "dsa_kv_cache_store_fp8", False):
if self.dsa_kv_cache_store_fp8:
# FP8 DSA packs latent+RoPE+scale into the device k_buffer;
# mirror the packed width so the 2D memcpy row width matches.
k_width = self.device_pool.kv_cache_dim
k_width = native.k_width
else:
k_width = self.kv_lora_rank
if _is_npu and memfabric_host_memory_enabled():
@@ -305,15 +353,15 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
* (k_width + self.qk_rope_head_dim)
* self.dtype.itemsize
)
if self.device_pool.index_head_dim is not None:
if native.index_k is not None:
total_bytes += (
self.page_num
* self.page_size
* num_indexer_layers
* self.device_pool.index_head_dim
* native.index_width
* self.dtype.itemsize
)
if getattr(self.device_pool, "index_k_scale_buffer", None) is not None:
if native.index_scales is not None:
# FP32 scale mirror
total_bytes += (
self.page_num * self.page_size * num_indexer_layers * 4
@@ -335,9 +383,9 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
allocator=self.allocator,
)
self.index_k_buffer = None
if self.device_pool.index_head_dim is not None:
if native.index_k is not None:
self.index_k_buffer = alloc_func(
(*indexer_dims, self.device_pool.index_head_dim),
(*indexer_dims, native.index_width),
dtype=self.dtype,
device=self.device,
pin_memory=self.pin_memory,
@@ -347,7 +395,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
# (see NPUMLATokenToKVPool.index_k_scale_buffer). Only present when
# the device pool carries one (FP8 DSA + npu_quant_lightning_indexer).
self.index_k_scale_buffer = None
if getattr(self.device_pool, "index_k_scale_buffer", None) is not None:
if native.index_scales is not None:
self.index_k_scale_buffer = alloc_func(
(*indexer_dims, 1),
dtype=torch.float32,
@@ -386,6 +434,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
# The staged kernel reads whole device pages, so it needs packed rows.
if (
self.layout != "page_first"
or self._layer_shard_enabled
or not self.device_rows_packed
or not current_platform.capabilities.hicache_device_kernels
):
@@ -414,27 +463,17 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
device=self._transfer_device,
)
def _indexer_slot_range_for_layer(self, device_pool, device_layer_id):
"""Map a device layer onto the indexer slot space.
Returns ``(slot, 1)`` when the layer is an indexer layer, ``(0, 0)``
to skip the indexer components, or ``(0, -1)`` (all indexer layers)
when the mapping is unavailable. ``indexer_layer_ids`` holds
absolute (PP-global) layer ids while ``device_layer_id`` is
pool-local; convert before matching.
"""
indexer_layer_ids = getattr(device_pool, "indexer_layer_ids", None)
if not indexer_layer_ids or self.index_k_buffer is None:
def _indexer_slot_range_for_layer(self, buffers, device_layer_id):
if not buffers.index_layer_indices or self.index_k_buffer is None:
return 0, -1
start_layer = getattr(device_pool, "start_layer", 0)
for slot, layer in enumerate(indexer_layer_ids):
if layer - start_layer == device_layer_id:
for slot, layer in enumerate(buffers.index_layer_indices):
if layer == device_layer_id:
return slot, 1
return 0, 0
def _transfer_ascendc_sparse_copy(
self,
device_pool,
buffers,
host_indices,
device_indices,
direction: TransferDirection,
@@ -459,7 +498,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
The kernel de-references host pool pointers directly, so the host
pool must be Memfabric-mapped.
"""
device = device_pool.k_buffer.device
device = buffers.k.device
# The kernel reads the token indices directly from device memory.
# Upload without a stream sync: a plain .to(device) from pageable
# memory synchronizes the stream and would serialize the pipeline.
@@ -474,7 +513,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
device_indices.record_stream(stream)
comps = self._build_ascendc_component_meta(
device_pool,
buffers,
layer_start,
layer_num,
index_k_layer_start,
@@ -528,7 +567,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
def _build_ascendc_component_meta(
self,
device_pool,
buffers,
layer_start: int,
layer_num: int,
index_k_layer_start: int,
@@ -536,12 +575,12 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
) -> list:
"""Assemble the per-component metadata list for kv_exchange_copy."""
k_lo = layer_start
k_hi = device_pool.k_buffer.shape[0] if layer_num < 0 else k_lo + layer_num
k_hi = buffers.k.shape[0] if layer_num < 0 else k_lo + layer_num
# Both pools must share the layer index space (same limitation as the
# legacy memcpy2d exchange op); catches e.g. MTP draft pools, whose
# host rows live past the main pool's layers.
host_layer_num = self.k_buffer.shape[1]
device_layer_num = device_pool.k_buffer.shape[0]
device_layer_num = buffers.k.shape[0]
if k_hi > host_layer_num or k_hi > device_layer_num:
raise RuntimeError(
f"AscendC kv_exchange layer range [{k_lo}, {k_hi}) exceeds the "
@@ -550,13 +589,13 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
)
comp_meta = self._ascendc_comp_meta
comps = [comp_meta(device_pool.k_buffer, self.k_buffer, k_lo, k_hi)]
comps = [comp_meta(buffers.k, self.k_buffer, k_lo, k_hi)]
# FP8 DSA packs V into the device k_buffer; the device v_buffer is
# empty and must be skipped.
if device_pool.v_buffer.numel() > 0 and self.v_buffer.numel() > 0:
comps.append(comp_meta(device_pool.v_buffer, self.v_buffer, k_lo, k_hi))
if buffers.v.numel() > 0 and self.v_buffer.numel() > 0:
comps.append(comp_meta(buffers.v, self.v_buffer, k_lo, k_hi))
device_index_k = getattr(device_pool, "index_k_buffer", None)
device_index_k = buffers.index_k
if self.index_k_buffer is not None and device_index_k is not None:
if index_k_layer_num < 0:
ik_lo, ik_hi = 0, self.index_k_buffer.shape[1]
@@ -567,7 +606,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
comps.append(
comp_meta(device_index_k, self.index_k_buffer, ik_lo, ik_hi)
)
device_scale = getattr(device_pool, "index_k_scale_buffer", None)
device_scale = buffers.index_scales
if self.index_k_scale_buffer is not None and device_scale is not None:
comps.append(
comp_meta(
@@ -639,21 +678,22 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
*,
is_draft: bool = False,
):
if not is_draft and not self._is_device_layer_owned(device_pool, layer_id):
assert not self._is_dummy, "load on a dummy (non-src MLA) host pool"
if not is_draft and layer_id not in self._target_layer_indices:
return
assert not getattr(self, "_is_dummy", False), (
"load on a dummy (non-src MLA) host pool"
)
host_indices = maybe_dcp_kernel_indices(
host_indices, self.dcp_size, self.dcp_rank
)
device_indices = maybe_dcp_kernel_indices(
device_indices, self.dcp_size, self.dcp_rank
)
# MTP draft layers do not participate in CP layer sharding.
host_layer_id = layer_id if is_draft else self._host_layer_index(layer_id)
# Draft transfer IDs follow all target device layers, while host rows
# follow only this rank's target layers.
host_layer_id = self._host_layer_by_controller[layer_id]
if io_backend != "kernel_ascend":
device_buffer = self.device_kv_buffers[layer_id]
device_buffer = self.device_kv_buffers[
self._buffer_index_by_layer[layer_id]
]
if io_backend == "kernel":
if self.layout == "layer_first":
@@ -720,16 +760,18 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
else:
raise ValueError(f"Unsupported layout: {self.layout}")
elif io_backend == "kernel_ascend":
info_index = layer_id - self._device_layer_num + 1 if is_draft else 0
native = self._native_infos[info_index].native_buffers()
device_layer_id = 0 if is_draft else layer_id
if self.layout == "page_first_kv_split":
if _is_npu and ascendc_io_enabled():
# The per-layer complete(i) event recorded by the caller lets
# later layers' DMA overlap the current layer's compute.
ik_start, ik_num = self._indexer_slot_range_for_layer(
device_pool, device_layer_id
native, device_layer_id
)
self._transfer_ascendc_sparse_copy(
device_pool,
native,
host_indices,
device_indices,
TransferDirection.H2D,
@@ -747,19 +789,13 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
transfer_kv_dim_exchange(
device_indices=device_indices,
host_indices=host_indices,
device_k=getattr(
device_pool, "k_buffer_tensor", device_pool.k_buffer
),
device_k=native.k,
host_k=self.k_buffer,
device_v=getattr(
device_pool, "v_buffer_tensor", device_pool.v_buffer
),
device_v=native.v,
host_v=self.v_buffer,
device_index_k=device_pool.index_k_buffer,
device_index_k=native.index_k,
host_index_k=self.index_k_buffer,
device_index_k_scale=getattr(
device_pool, "index_k_scale_buffer", None
),
device_index_k_scale=native.index_scales,
host_index_k_scale=self.index_k_scale_buffer,
page_size=self.page_size,
direction=TransferDirection.H2D,
@@ -778,17 +814,16 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
*,
is_draft: bool = False,
):
assert not getattr(self, "_is_dummy", False), (
"backup on a dummy (non-src MLA) host pool"
)
assert not self._is_dummy, "backup on a dummy (non-src MLA) host pool"
if io_backend not in ("kernel", "direct"):
raise ValueError(
f"Layer-sharded HiCache backup does not support IO backend: {io_backend}"
)
# Indices arrive already translated by backup_from_device_all_layer.
# MTP draft layers do not participate in CP layer sharding.
host_layer_id = layer_id if is_draft else self._host_layer_index(layer_id)
device_buffer = self.device_kv_buffers[layer_id]
# Draft transfer IDs follow all target device layers, while host rows
# follow only this rank's target layers.
host_layer_id = self._host_layer_by_controller[layer_id]
device_buffer = self.device_kv_buffers[self._buffer_index_by_layer[layer_id]]
if io_backend == "kernel":
if self.layout == "layer_first":
@@ -847,25 +882,23 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
def backup_from_device_all_layer(
self, device_pool, host_indices, device_indices, io_backend
):
assert not getattr(self, "_is_dummy", False), (
"backup on a dummy (non-src MLA) host pool"
)
assert not self._is_dummy, "backup on a dummy (non-src MLA) host pool"
host_indices = maybe_dcp_kernel_indices(
host_indices, self.dcp_size, self.dcp_rank
)
device_indices = maybe_dcp_kernel_indices(
device_indices, self.dcp_size, self.dcp_rank
)
if self._is_device_layer_sharded(device_pool):
for layer_id in self._owned_device_layer_ids(device_pool):
if self._layer_shard_enabled:
for layer_id in self._target_layer_indices:
self._backup_from_device_per_layer(
host_indices, device_indices, layer_id, io_backend
)
for draft_layer_id in range(len(self.mtp_draft_device_pools)):
for draft_layer_id in range(self._draft_layer_num):
self._backup_from_device_per_layer(
host_indices,
device_indices,
self.device_pool.layer_num + draft_layer_id,
self._device_layer_num + draft_layer_id,
io_backend,
is_draft=True,
)
@@ -959,10 +992,11 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
else:
raise ValueError(f"Unsupported layout: {self.layout}")
elif io_backend == "kernel_ascend":
native = self._native_infos[0].native_buffers()
if self.layout == "page_first_kv_split":
if _is_npu and ascendc_io_enabled():
self._transfer_ascendc_sparse_copy(
device_pool,
native,
host_indices,
device_indices,
TransferDirection.D2H,
@@ -971,19 +1005,13 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
transfer_kv_dim_exchange(
device_indices=device_indices,
host_indices=host_indices,
device_k=getattr(
device_pool, "k_buffer_tensor", device_pool.k_buffer
),
device_k=native.k,
host_k=self.k_buffer,
device_v=getattr(
device_pool, "v_buffer_tensor", device_pool.v_buffer
),
device_v=native.v,
host_v=self.v_buffer,
device_index_k=device_pool.index_k_buffer,
device_index_k=native.index_k,
host_index_k=self.index_k_buffer,
device_index_k_scale=getattr(
device_pool, "index_k_scale_buffer", None
),
device_index_k_scale=native.index_scales,
host_index_k_scale=self.index_k_scale_buffer,
page_size=self.page_size,
direction=TransferDirection.D2H,
@@ -31,16 +31,17 @@ class AibrixKVCacheStorage(HiCacheStorage):
else:
self.is_mla_backend = False
self.local_rank = 0
kv_cache = mem_pool.device_pool
self.page_size = mem_pool.page_size
self.kv_cache_dtype = kv_cache.dtype
self.layer_num = kv_cache.layer_num
storage_info = mem_pool
self.kv_cache_dtype = storage_info.dtype
self.layer_num = mem_pool.target_layer_num
self.kv_head_ids = [
self.local_rank * kv_cache.head_num + i for i in range(kv_cache.head_num)
self.local_rank * storage_info.head_num + i
for i in range(storage_info.head_num)
]
if not self.is_mla_backend:
self.layer_ids = range(
kv_cache.start_layer, kv_cache.end_layer
storage_info.start_layer, storage_info.end_layer
) # for pipeline parallel
self.block_spec = KVCacheBlockSpec(
@@ -50,7 +51,7 @@ class AibrixKVCacheStorage(HiCacheStorage):
tensor_spec=KVCacheTensorSpec(
heads=self.kv_head_ids,
layers=self.layer_ids,
head_size=kv_cache.head_dim,
head_size=storage_info.head_dim,
),
)
logger.info(self.block_spec)
@@ -51,7 +51,19 @@ class AIBrixKVCacheStorageTest:
0,
layer_num,
)
mem_pool = MHATokenToKVPoolHost(kv_cache, 2, 0, page_size, "layer_first")
mem_pool = MHATokenToKVPoolHost(
kv_cache.get_device_pool_infos()[0].buffer_info,
2,
0,
page_size,
"layer_first",
device_capacity_tokens=kv_cache.size
if kv_cache.host_capacity_tokens is None
else kv_cache.host_capacity_tokens,
start_layer=kv_cache.start_layer,
end_layer=kv_cache.end_layer,
write_back_staging=kv_cache.hicache_write_back_staging,
)
query_length = batch_size * 2
partial = batch_size
self.aibrix_kvcache = AibrixKVCacheStorage(config, mem_pool)
@@ -489,7 +489,7 @@ def _validate_storage_config(
"TensorCast FULL v1 supports only page_first and page_first_direct, "
f"got layout={mem_pool_host.layout!r}"
)
if mem_pool_host.mtp_draft_device_pools:
if mem_pool_host.layer_num != mem_pool_host.target_layer_num:
raise NotImplementedError(
"TensorCast FULL v1 does not support packed MTP draft pools"
)
@@ -247,7 +247,7 @@ def _make_pool(
pool.layer_num = layer_num
pool.head_num = head_num
pool.dtype = dtype
pool.mtp_draft_device_pools = ()
pool.target_layer_num = layer_num
pool.dcp_size = 1
pool.dcp_rank = 0
pool.allocator = allocator or HostTensorAllocator()
@@ -332,7 +332,7 @@ def test_fragment_registration_dispatch_is_exact_and_specific() -> None:
[
({"layout": "layer_first"}, {}, "supports only"),
({"layout": "page_head"}, {}, "supports only"),
({"mtp_draft_device_pools": (object(),)}, {}, "MTP"),
({"target_layer_num": 0}, {}, "MTP"),
({"dcp_size": 2}, {}, "DCP"),
({}, {"attn_cp_size": 2}, "context parallel"),
({}, {"tp_lcm_size": 2}, "TP-LCM"),
@@ -105,6 +105,12 @@ class SWAKVPool(BaseSWAKVPool):
layer_num=self.swa_layer_nums,
**swa_kv_pool_kwargs,
)
for pool, layer_ids in (
(self.full_kv_pool, full_attention_layer_ids),
(self.swa_kv_pool, swa_attention_layer_ids),
):
if isinstance(pool, MHATokenToKVPool):
pool.model_layer_ids = tuple(layer_ids)
self.dsa_kv_cache_store_fp8 = False
self.kv_cache_dim = None
self.index_head_dim = None
@@ -1909,6 +1909,9 @@ class UnifiedSWAKVPool(SWAKVPool):
end_layer=end_layer,
)
self.full_kv_pool.model_layer_ids = tuple(full_attention_layer_ids)
self.swa_kv_pool.model_layer_ids = tuple(swa_attention_layer_ids)
# disagg/nvlink disabled; keep attrs present to avoid AttributeError.
self.enable_custom_mem_pool = False
self.custom_mem_pool = None