[Refactor] memory-pool: Build KV host pools from buffer info (#40917)
This commit is contained in:
@@ -190,6 +190,33 @@ class NPUMHATokenToKVPool(MHATokenToKVPool):
|
||||
"""Return contiguous all-layer KV tensors for NPU HiCache IO."""
|
||||
return self._hicache_k_buffer, self._hicache_v_buffer
|
||||
|
||||
def _mha_buffer_info(self):
|
||||
from sglang.srt.mem_cache.device_pool_info import (
|
||||
KVBufferLayout,
|
||||
KVBuffers,
|
||||
MHABufferInfo,
|
||||
)
|
||||
|
||||
return MHABufferInfo(
|
||||
buffers=KVBuffers(
|
||||
k=tuple(self.k_buffer),
|
||||
v=tuple(self.v_buffer),
|
||||
k_data_ptrs=None,
|
||||
v_data_ptrs=None,
|
||||
layout=KVBufferLayout.PAGED_NHD,
|
||||
),
|
||||
page_size=self.page_size,
|
||||
head_num=self.row_dim // self.head_dim
|
||||
if self.head_dim == self.v_head_dim
|
||||
else self.head_num,
|
||||
head_dim=self.head_dim,
|
||||
v_head_dim=self.v_head_dim,
|
||||
dtype=self.store_dtype,
|
||||
# self.device is the bare backend name; tensors carry the rank index.
|
||||
device=self._hicache_k_buffer.device,
|
||||
transfer_buffers=self.get_hicache_transfer_buffers,
|
||||
)
|
||||
|
||||
def _init_kv_copy_and_warmup(self):
|
||||
# implementation relies on self.data_strides / self.data_ptrs, which the
|
||||
# NPU paged buffer layout never builds.
|
||||
@@ -610,6 +637,9 @@ class NPUMLATokenToKVPool(MLATokenToKVPool):
|
||||
end_layer=end_layer,
|
||||
)
|
||||
|
||||
self.model_layer_ids = tuple(
|
||||
range(self.start_layer, self.start_layer + self.layer_num)
|
||||
)
|
||||
self.kv_lora_rank = kv_lora_rank
|
||||
self.qk_rope_head_dim = qk_rope_head_dim
|
||||
self.index_head_dim = index_head_dim
|
||||
@@ -731,6 +761,45 @@ class NPUMLATokenToKVPool(MLATokenToKVPool):
|
||||
|
||||
self._finalize_allocation_log(size)
|
||||
|
||||
def _mla_buffer_info(self):
|
||||
from sglang.srt.mem_cache.device_pool_info import MLABufferInfo
|
||||
|
||||
return MLABufferInfo(
|
||||
buffers=self._mla_native_buffers(),
|
||||
native_buffers=self._mla_native_buffers,
|
||||
data_ptrs=None,
|
||||
layer_indices=tuple(range(self.layer_num)),
|
||||
device_layer_num=self.layer_num,
|
||||
page_size=self.page_size,
|
||||
kv_lora_rank=self.kv_lora_rank,
|
||||
qk_rope_head_dim=self.qk_rope_head_dim,
|
||||
kv_cache_dim=self.kv_cache_dim + self.kr_cache_dim,
|
||||
dtype=self.store_dtype,
|
||||
# self.device is the bare backend name; tensors carry the rank index.
|
||||
device=next(
|
||||
buffer.device
|
||||
for buffer in (self.k_buffer, self.index_k_buffer)
|
||||
if buffer is not None
|
||||
),
|
||||
packed_fp8=self.dsa_kv_cache_store_fp8,
|
||||
)
|
||||
|
||||
def _mla_native_buffers(self):
|
||||
from sglang.srt.mem_cache.device_pool_info import MLASplitBuffers
|
||||
|
||||
return MLASplitBuffers(
|
||||
k=self.k_buffer,
|
||||
v=self.v_buffer,
|
||||
index_k=self.index_k_buffer,
|
||||
index_scales=self.index_k_scale_buffer,
|
||||
index_layer_indices=tuple(
|
||||
i - self.start_layer for i in self.indexer_layer_ids
|
||||
),
|
||||
k_width=self.kv_cache_dim,
|
||||
v_width=self.kr_cache_dim,
|
||||
index_width=self.index_head_dim or 0,
|
||||
)
|
||||
|
||||
def _copy_indices_for_buffer(self, indices, uses_global_slots):
|
||||
if uses_global_slots or self.dcp_size <= 1:
|
||||
return indices
|
||||
|
||||
@@ -601,7 +601,14 @@ class HiSparseCoordinator:
|
||||
"tensor-parallel size."
|
||||
)
|
||||
self.mem_pool_host = HiSparseMHATokenToKVPoolHost(
|
||||
device_pool=self.mem_pool_device,
|
||||
buffer_info=self.mem_pool_device.get_device_pool_infos()[
|
||||
0
|
||||
].buffer_info,
|
||||
device_capacity_tokens=self.mem_pool_device.size
|
||||
if self.mem_pool_device.host_capacity_tokens is None
|
||||
else self.mem_pool_device.host_capacity_tokens,
|
||||
start_layer=self.mem_pool_device.start_layer,
|
||||
end_layer=self.mem_pool_device.end_layer,
|
||||
host_to_device_ratio=host_to_device_ratio,
|
||||
page_size=self.mem_pool_device.page_size,
|
||||
)
|
||||
@@ -609,7 +616,15 @@ class HiSparseCoordinator:
|
||||
else:
|
||||
self.mem_pool_device: HiSparseDSATokenToKVPool = kvcache
|
||||
self.mem_pool_host = MLATokenToKVPoolHost(
|
||||
device_pool=self.mem_pool_device,
|
||||
buffer_info=self.mem_pool_device.get_device_pool_infos()[
|
||||
0
|
||||
].buffer_info,
|
||||
device_capacity_tokens=self.mem_pool_device.size
|
||||
if self.mem_pool_device.host_capacity_tokens is None
|
||||
else self.mem_pool_device.host_capacity_tokens,
|
||||
start_layer=self.mem_pool_device.start_layer,
|
||||
end_layer=self.mem_pool_device.end_layer,
|
||||
target_layer_num=self.mem_pool_device.layer_num,
|
||||
host_to_device_ratio=host_to_device_ratio,
|
||||
host_size=0,
|
||||
page_size=self.mem_pool_device.page_size,
|
||||
|
||||
@@ -6,6 +6,9 @@ layer routing and transfers. This first consumer describes Mamba checkpoints.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from enum import Enum
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
@@ -57,12 +60,166 @@ class MambaStateBufferInfo(msgspec.Struct, frozen=True, kw_only=True):
|
||||
)
|
||||
|
||||
|
||||
class KVBufferLayout(Enum):
|
||||
NHD = "nhd"
|
||||
PAGED_NHD = "paged_nhd"
|
||||
HND = "hnd"
|
||||
VECTORIZED_5D = "vectorized_5d"
|
||||
|
||||
|
||||
class KVBuffers(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Per-layer K/V views and their existing device pointer allocations.
|
||||
|
||||
NHD views may be strided. Page/head/vector axes stay in the native layout;
|
||||
this description neither flattens buffers nor enables new transfer backends.
|
||||
"""
|
||||
|
||||
k: tuple[torch.Tensor, ...]
|
||||
v: tuple[torch.Tensor, ...]
|
||||
# Some packed formats have no pointer tables; providers must not allocate them.
|
||||
k_data_ptrs: torch.Tensor | None
|
||||
v_data_ptrs: torch.Tensor | None
|
||||
layout: KVBufferLayout
|
||||
|
||||
|
||||
class KVScaleLayout(Enum):
|
||||
TOKEN = "token"
|
||||
MXFP8_INTERLEAVED = "mxfp8_interleaved"
|
||||
|
||||
|
||||
class KVScaleBuffers(msgspec.Struct, frozen=True, kw_only=True):
|
||||
k: tuple[torch.Tensor, ...]
|
||||
v: tuple[torch.Tensor, ...]
|
||||
layout: KVScaleLayout
|
||||
|
||||
|
||||
class MHABufferInfo(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Physical MHA input for host construction; no capacity or allocation policy.
|
||||
|
||||
Geometry is explicit for empty stages. head_num counts physical attention
|
||||
rows per allocator token, which can exceed the model's attention head count.
|
||||
Existing pointer tables and tensor strides must be preserved by consumers.
|
||||
"""
|
||||
|
||||
buffers: KVBuffers
|
||||
page_size: int
|
||||
head_num: int
|
||||
head_dim: int
|
||||
v_head_dim: int
|
||||
dtype: torch.dtype
|
||||
device: torch.device
|
||||
scales: KVScaleBuffers | None = None
|
||||
# Resolve current contiguous backing at transfer time for relocatable pools.
|
||||
transfer_buffers: Callable[[], tuple[torch.Tensor, torch.Tensor]] | None = None
|
||||
|
||||
def validate(self, *, layer_ids: tuple[int, ...] | None = None) -> None:
|
||||
layers = len(self.buffers.k)
|
||||
if len(self.buffers.v) != layers or (
|
||||
layer_ids is not None and len(layer_ids) != layers
|
||||
):
|
||||
raise ValueError(
|
||||
f"MHA layers: k={layers}, v={len(self.buffers.v)}, layer_ids={layer_ids}"
|
||||
)
|
||||
if min(self.page_size, self.head_num, self.head_dim, self.v_head_dim) <= 0:
|
||||
raise ValueError(
|
||||
f"Invalid MHA geometry: page_size={self.page_size}, "
|
||||
f"head_num={self.head_num}, head_dim={self.head_dim}, "
|
||||
f"v_head_dim={self.v_head_dim}"
|
||||
)
|
||||
for name in ("k", "v"):
|
||||
tensors = self.buffers.k if name == "k" else self.buffers.v
|
||||
for layer, tensor in enumerate(tensors):
|
||||
if tensor.dtype != self.dtype or tensor.device != self.device:
|
||||
raise ValueError(
|
||||
f"MHA {name}[{layer}]: dtype={tensor.dtype}, device={tensor.device}; "
|
||||
f"expected dtype={self.dtype}, device={self.device}"
|
||||
)
|
||||
if self.scales is not None and (
|
||||
len(self.scales.k) != layers or len(self.scales.v) != layers
|
||||
):
|
||||
raise ValueError(
|
||||
f"MHA scales: k={len(self.scales.k)}, v={len(self.scales.v)}, "
|
||||
f"expected layers={layers}"
|
||||
)
|
||||
|
||||
|
||||
class MLASplitBuffers(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Native NPU allocations; index slots have their own pool-local layer order."""
|
||||
|
||||
k: torch.Tensor | None
|
||||
v: torch.Tensor | None
|
||||
index_k: torch.Tensor | None
|
||||
index_scales: torch.Tensor | None
|
||||
index_layer_indices: tuple[int, ...]
|
||||
k_width: int
|
||||
v_width: int
|
||||
index_width: int
|
||||
|
||||
|
||||
class MLABufferInfo(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Persistent MLA rows or native split components, including empty stages.
|
||||
|
||||
layer_indices address the pool's controller slots, not model layer IDs or
|
||||
reserved host rows. Existing pointer allocations are views, never rebuilt.
|
||||
"""
|
||||
|
||||
buffers: tuple[torch.Tensor, ...] | MLASplitBuffers
|
||||
data_ptrs: torch.Tensor | None
|
||||
layer_indices: tuple[int, ...]
|
||||
device_layer_num: int
|
||||
page_size: int
|
||||
kv_lora_rank: int
|
||||
qk_rope_head_dim: int
|
||||
kv_cache_dim: int
|
||||
dtype: torch.dtype
|
||||
device: torch.device
|
||||
packed_fp8: bool = False
|
||||
scales: tuple[torch.Tensor, ...] | None = None
|
||||
native_buffers: Callable[[], MLASplitBuffers] | None = None
|
||||
|
||||
def validate(self, *, layer_ids: tuple[int, ...] | None = None) -> None:
|
||||
if layer_ids is not None and len(layer_ids) != len(self.layer_indices):
|
||||
raise ValueError(
|
||||
f"MLA model layers={layer_ids} differ from buffer slots={self.layer_indices}"
|
||||
)
|
||||
if len(set(self.layer_indices)) != len(self.layer_indices) or any(
|
||||
i < 0 or i >= self.device_layer_num for i in self.layer_indices
|
||||
):
|
||||
raise ValueError(
|
||||
f"MLA slots={self.layer_indices} outside device_layer_num={self.device_layer_num}"
|
||||
)
|
||||
if isinstance(self.buffers, tuple):
|
||||
if len(self.buffers) != len(self.layer_indices):
|
||||
raise ValueError(
|
||||
f"MLA buffers={len(self.buffers)}, slots={self.layer_indices}"
|
||||
)
|
||||
for i, tensor in zip(self.layer_indices, self.buffers):
|
||||
if (
|
||||
tensor.ndim != 3
|
||||
or tensor.shape[1] != 1
|
||||
or tensor.dtype != self.dtype
|
||||
or tensor.device != self.device
|
||||
):
|
||||
raise ValueError(
|
||||
f"MLA slot={i}: shape={tuple(tensor.shape)}, dtype={tensor.dtype}, "
|
||||
f"device={tensor.device}; expected [token, 1, width], "
|
||||
f"dtype={self.dtype}, device={self.device}"
|
||||
)
|
||||
elif self.buffers.index_k is not None and self.buffers.index_k.shape[0] != len(
|
||||
self.buffers.index_layer_indices
|
||||
):
|
||||
raise ValueError(
|
||||
f"MLA index shape={tuple(self.buffers.index_k.shape)}, "
|
||||
f"index layers={self.buffers.index_layer_indices}"
|
||||
)
|
||||
|
||||
|
||||
class DevicePoolInfo(msgspec.Struct, frozen=True, kw_only=True):
|
||||
pool_name: PoolName
|
||||
indices_from_pool: PoolName
|
||||
# Model layer IDs, including the PP stage offset, in buffer-axis order.
|
||||
layer_ids: tuple[int, ...]
|
||||
buffer_info: MambaStateBufferInfo
|
||||
buffer_info: MambaStateBufferInfo | MHABufferInfo | MLABufferInfo
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
# A PP stage may own no checkpoint layers. Buffer validation decides
|
||||
|
||||
@@ -37,6 +37,7 @@ from sglang.srt.mem_cache.pool_host.qsa import QSAIndexerPoolHost
|
||||
from sglang.srt.mem_cache.pool_host.unified import UnifiedPageEnvelopeHostPool
|
||||
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
|
||||
from sglang.srt.runtime_context import get_memory, get_parallel, get_serving
|
||||
from sglang.srt.utils import is_npu
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
@@ -134,6 +135,27 @@ def _resolve_deepseek_v4_layer_mappings(
|
||||
)
|
||||
|
||||
|
||||
def _mha_host_inputs(pool, drafts=()):
|
||||
info = pool.get_device_pool_infos()[0].buffer_info
|
||||
return (
|
||||
get_mha_host_pool_cls(info),
|
||||
info,
|
||||
dict(
|
||||
device_capacity_tokens=(
|
||||
pool.size
|
||||
if pool.host_capacity_tokens is None
|
||||
else pool.host_capacity_tokens
|
||||
),
|
||||
start_layer=pool.start_layer,
|
||||
end_layer=pool.end_layer,
|
||||
draft_buffer_infos=tuple(
|
||||
draft.get_device_pool_infos()[0].buffer_info for draft in drafts
|
||||
),
|
||||
write_back_staging=pool.hicache_write_back_staging,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def build_kv_host_pool(
|
||||
*,
|
||||
kv_pool: Any,
|
||||
@@ -146,21 +168,46 @@ def build_kv_host_pool(
|
||||
):
|
||||
if use_mla is None:
|
||||
use_mla = is_mla_pool(kv_pool)
|
||||
kv_host_pool_cls = (
|
||||
MLATokenToKVPoolHost if use_mla else get_mha_host_pool_cls(kv_pool)
|
||||
)
|
||||
if use_mla:
|
||||
kv_host_pool_cls = MLATokenToKVPoolHost
|
||||
host_input = kv_pool.get_device_pool_infos()[0].buffer_info
|
||||
mha_kwargs = {}
|
||||
else:
|
||||
kv_host_pool_cls, host_input, mha_kwargs = _mha_host_inputs(
|
||||
kv_pool, mtp_draft_device_pools
|
||||
)
|
||||
if use_mla and override_kv_cache_dim is None:
|
||||
# An fp8 DSA store is wider than kv_lora_rank + qk_rope_head_dim. The
|
||||
# device pool already resolved the row width.
|
||||
override_kv_cache_dim = kv_pool.kv_cache_dim
|
||||
check_packed_kv_rows(
|
||||
kv_pool=kv_pool, drafts=mtp_draft_device_pools, use_mla=use_mla
|
||||
)
|
||||
if use_mla or is_npu():
|
||||
check_packed_kv_rows(
|
||||
kv_pool=kv_pool, drafts=mtp_draft_device_pools, use_mla=use_mla
|
||||
)
|
||||
kwargs = {}
|
||||
if override_kv_cache_dim is not None:
|
||||
kwargs["override_kv_cache_dim"] = override_kv_cache_dim
|
||||
if mtp_draft_device_pools:
|
||||
kwargs["mtp_draft_device_pools"] = mtp_draft_device_pools
|
||||
if use_mla:
|
||||
kwargs.update(
|
||||
device_capacity_tokens=kv_pool.size
|
||||
if kv_pool.host_capacity_tokens is None
|
||||
else kv_pool.host_capacity_tokens,
|
||||
start_layer=kv_pool.start_layer,
|
||||
end_layer=kv_pool.end_layer,
|
||||
draft_buffer_infos=tuple(
|
||||
pool.get_device_pool_infos()[0].buffer_info
|
||||
for pool in mtp_draft_device_pools
|
||||
),
|
||||
layer_shard_enabled=kv_pool.layer_shard_enabled,
|
||||
target_layer_num=(
|
||||
(kv_pool.layer_num + kv_pool.layer_shard_size - 1)
|
||||
// kv_pool.layer_shard_size
|
||||
if kv_pool.layer_shard_enabled
|
||||
else kv_pool.layer_num
|
||||
),
|
||||
)
|
||||
else:
|
||||
kwargs.update(mha_kwargs)
|
||||
parallel = get_parallel()
|
||||
if parallel.dcp_enabled:
|
||||
assert use_mla, (
|
||||
@@ -170,7 +217,7 @@ def build_kv_host_pool(
|
||||
kwargs["dcp_size"] = parallel.attn_dcp_size
|
||||
kwargs["dcp_rank"] = parallel.attn_dcp_rank
|
||||
return kv_host_pool_cls(
|
||||
kv_pool,
|
||||
host_input,
|
||||
get_memory().hicache_ratio,
|
||||
get_memory().hicache_size if host_size is None else host_size,
|
||||
page_size,
|
||||
@@ -421,14 +468,15 @@ def build_hybrid_swa_group(
|
||||
# from full attention. Use a separate host pool with KV indices instead of
|
||||
# packing their bytes into either target pool.
|
||||
first, *rest = mtp_swa_full_device_pools
|
||||
draft_host_pool = get_mha_host_pool_cls(first)(
|
||||
first,
|
||||
host_cls, host_input, host_kwargs = _mha_host_inputs(first, tuple(rest))
|
||||
draft_host_pool = host_cls(
|
||||
host_input,
|
||||
host_to_device_ratio=kv_host_pool.size / first.size,
|
||||
host_size=0,
|
||||
page_size=page_size,
|
||||
layout=get_memory().hicache_mem_layout,
|
||||
allocator_type=_get_allocator_type(),
|
||||
mtp_draft_device_pools=rest,
|
||||
**host_kwargs,
|
||||
pool_label="draft_swa",
|
||||
)
|
||||
group.add_entry(
|
||||
@@ -1533,9 +1581,16 @@ def _build_mha_mla_host_pool(
|
||||
pool_label=pool_label,
|
||||
)
|
||||
if isinstance(pool, MHATokenToKVPool):
|
||||
return get_mha_host_pool_cls(pool)(pool, **kwargs)
|
||||
host_cls, host_input, host_kwargs = _mha_host_inputs(pool)
|
||||
return host_cls(host_input, **host_kwargs, **kwargs)
|
||||
return MLATokenToKVPoolHost(
|
||||
pool,
|
||||
pool.get_device_pool_infos()[0].buffer_info,
|
||||
device_capacity_tokens=pool.size
|
||||
if pool.host_capacity_tokens is None
|
||||
else pool.host_capacity_tokens,
|
||||
start_layer=pool.start_layer,
|
||||
end_layer=pool.end_layer,
|
||||
target_layer_num=pool.layer_num,
|
||||
override_kv_cache_dim=pool.kv_cache_dim,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@@ -58,7 +58,7 @@ from sglang.srt.runtime_context import (
|
||||
get_schedule,
|
||||
)
|
||||
from sglang.srt.speculative.base_spec_worker import HiCacheDraftMode
|
||||
from sglang.srt.utils import ceil_align, is_hip
|
||||
from sglang.srt.utils import ceil_align, is_hip, is_npu
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.configs.model_config import ModelConfig
|
||||
@@ -72,6 +72,8 @@ def prepare_hicache_staging(
|
||||
*, tp_worker: BaseTpWorker, draft_plan: Optional[HiCacheDraftPlan] = None
|
||||
) -> None:
|
||||
"""Materialize MHA transfer buffers before the final KV budget is measured."""
|
||||
if is_npu():
|
||||
return
|
||||
memory = get_memory()
|
||||
page_size = get_schedule().page_size
|
||||
if memory.hicache_mem_layout != "page_first" or not (
|
||||
@@ -93,11 +95,13 @@ def prepare_hicache_staging(
|
||||
if memory.hicache_size <= 0 and not sidecar:
|
||||
page_capacity = int(pool.size * memory.hicache_ratio) // page_size + 1
|
||||
staging = prepare_mha_write_back_staging(
|
||||
pool,
|
||||
pool.get_device_pool_infos()[0].buffer_info,
|
||||
layer_num=pool.layer_num + len(packed_drafts),
|
||||
page_size=page_size,
|
||||
page_capacity=page_capacity,
|
||||
staging=pool.hicache_write_back_staging,
|
||||
)
|
||||
pool.hicache_write_back_staging = staging
|
||||
if staging is not None:
|
||||
logger.info(
|
||||
"HiCache staging prepared before KV sizing: %.1f MiB, %d layers",
|
||||
|
||||
@@ -100,7 +100,7 @@ from sglang.srt.utils.torch_memory_saver_adapter import TorchMemorySaverAdapter
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.cache_controller import LayerDoneCounter
|
||||
from sglang.srt.managers.schedule_batch import Req
|
||||
from sglang.srt.mem_cache.device_pool_info import DevicePoolInfo
|
||||
from sglang.srt.mem_cache.device_pool_info import DevicePoolInfo, MHABufferInfo
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -2075,6 +2075,14 @@ class KVCache(abc.ABC):
|
||||
maybe_init_custom_mem_pool(device=self.device)
|
||||
)
|
||||
|
||||
def get_device_pool_infos(self) -> tuple[DevicePoolInfo, ...]:
|
||||
"""Persistent buffers for migrated host consumers; see device_pool_info.py.
|
||||
|
||||
Unmigrated formats retain their existing construction entry points.
|
||||
Consumers must not use exceptions or empty descriptions to pick a path.
|
||||
"""
|
||||
raise NotImplementedError(f"{type(self).__name__} has no device pool provider")
|
||||
|
||||
def host_pool_decls(self):
|
||||
"""Host pools HiCache keeps for the buffers this device pool (and its
|
||||
sub-pools) owns: the KV pool itself, plus dependent pools such as sparse
|
||||
@@ -2225,6 +2233,11 @@ class MHATokenToKVPool(KVCache):
|
||||
):
|
||||
self.k_buffer = None
|
||||
self.v_buffer = None
|
||||
if not _is_npu:
|
||||
self.k_data_ptrs = None
|
||||
self.v_data_ptrs = None
|
||||
self.k_scale_buffer = None
|
||||
self.v_scale_buffer = None
|
||||
if post_capture_active:
|
||||
# Reserved upper bound only (unbacked VA): page-align UP so
|
||||
# (size + page_size) % page_size == 0 holds for paged layouts.
|
||||
@@ -2324,6 +2337,64 @@ class MHATokenToKVPool(KVCache):
|
||||
# for store_cache JIT kernel
|
||||
self.row_dim = self.head_num * self.head_dim
|
||||
self.v_row_dim = self.head_num * self.v_head_dim
|
||||
self.model_layer_ids = tuple(
|
||||
range(self.start_layer, self.start_layer + self.layer_num)
|
||||
)
|
||||
|
||||
def get_device_pool_infos(self) -> tuple[DevicePoolInfo, ...]:
|
||||
"""Describe persistent KV buffers for host assembly without allocating."""
|
||||
from sglang.srt.mem_cache.device_pool_info import DevicePoolInfo
|
||||
from sglang.srt.mem_cache.hicache_storage import PoolName
|
||||
|
||||
return (
|
||||
DevicePoolInfo(
|
||||
pool_name=PoolName.KV,
|
||||
indices_from_pool=PoolName.KV,
|
||||
layer_ids=self.model_layer_ids,
|
||||
buffer_info=self._mha_buffer_info(),
|
||||
),
|
||||
)
|
||||
|
||||
def _mha_buffers(self):
|
||||
from sglang.srt.mem_cache.device_pool_info import KVBufferLayout, KVBuffers
|
||||
|
||||
return KVBuffers(
|
||||
k=tuple(self.k_buffer),
|
||||
v=tuple(self.v_buffer),
|
||||
k_data_ptrs=self.k_data_ptrs,
|
||||
v_data_ptrs=self.v_data_ptrs,
|
||||
layout=KVBufferLayout.NHD
|
||||
if self.kv_cache_layout == "page_major"
|
||||
else KVBufferLayout(self.kv_cache_layout),
|
||||
)
|
||||
|
||||
def _mha_scale_buffers(self):
|
||||
from sglang.srt.mem_cache.device_pool_info import KVScaleBuffers, KVScaleLayout
|
||||
|
||||
if self.k_scale_buffer is None:
|
||||
return None
|
||||
return KVScaleBuffers(
|
||||
k=tuple(self.k_scale_buffer),
|
||||
v=tuple(self.v_scale_buffer),
|
||||
layout=KVScaleLayout.TOKEN,
|
||||
)
|
||||
|
||||
def _mha_buffer_info(self) -> MHABufferInfo:
|
||||
from sglang.srt.mem_cache.device_pool_info import MHABufferInfo
|
||||
|
||||
buffers = self._mha_buffers()
|
||||
return MHABufferInfo(
|
||||
buffers=buffers,
|
||||
page_size=self.page_size,
|
||||
head_num=self.row_dim // self.head_dim
|
||||
if self.head_dim == self.v_head_dim
|
||||
else self.head_num,
|
||||
head_dim=self.head_dim,
|
||||
v_head_dim=self.v_head_dim,
|
||||
dtype=self.store_dtype,
|
||||
device=buffers.k[0].device if len(buffers.k) else torch.device(self.device),
|
||||
scales=self._mha_scale_buffers(),
|
||||
)
|
||||
|
||||
def _init_kv_copy_and_warmup(self):
|
||||
# Zero-layer pool (e.g. all-SWA model's full sub-pool) has no buffers.
|
||||
@@ -3789,6 +3860,17 @@ class MHATokenToKVPoolMXFP8(MHATokenToKVPool):
|
||||
|
||||
MXFP8_SCALE_BLOCK_SIZE = 32
|
||||
|
||||
def _mha_scale_buffers(self):
|
||||
from sglang.srt.mem_cache.device_pool_info import KVScaleBuffers, KVScaleLayout
|
||||
|
||||
return KVScaleBuffers(
|
||||
k=tuple(self.k_scale_buffer),
|
||||
v=tuple(self.v_scale_buffer),
|
||||
layout=KVScaleLayout.MXFP8_INTERLEAVED
|
||||
if self.mxfp8_sf_interleaved
|
||||
else KVScaleLayout.TOKEN,
|
||||
)
|
||||
|
||||
def _create_buffers(self):
|
||||
with self.memory_saver_adapter.region(GPU_MEMORY_TYPE_KV_CACHE):
|
||||
with (
|
||||
@@ -4272,6 +4354,9 @@ class HybridLinearKVPool(KVCache):
|
||||
qk_rope_head_dim=qk_rope_head_dim,
|
||||
enable_memory_saver=enable_memory_saver,
|
||||
)
|
||||
if isinstance(self.full_kv_pool, MHATokenToKVPool):
|
||||
# Only the hybrid root knows the model IDs of its compact child.
|
||||
self.full_kv_pool.model_layer_ids = tuple(full_attention_layer_ids)
|
||||
self.full_attention_layer_id_mapping = {
|
||||
id: i for i, id in enumerate(full_attention_layer_ids)
|
||||
}
|
||||
@@ -4716,6 +4801,11 @@ class MLATokenToKVPool(KVCache):
|
||||
else (kv_lora_rank + qk_rope_head_dim)
|
||||
)
|
||||
|
||||
self.model_layer_ids = tuple(
|
||||
range(self.start_layer, self.start_layer + self.layer_num)
|
||||
)
|
||||
self.kv_scale_buffer = None
|
||||
|
||||
self._create_buffers()
|
||||
|
||||
self.data_ptrs = torch.tensor(
|
||||
@@ -4727,6 +4817,47 @@ class MLATokenToKVPool(KVCache):
|
||||
# DSA will allocate indexer KV cache later and then log the total size
|
||||
self._finalize_allocation_log(size)
|
||||
|
||||
def get_device_pool_infos(self) -> tuple[DevicePoolInfo, ...]:
|
||||
from sglang.srt.mem_cache.device_pool_info import DevicePoolInfo
|
||||
from sglang.srt.mem_cache.hicache_storage import PoolName
|
||||
|
||||
info = self._mla_buffer_info()
|
||||
return (
|
||||
DevicePoolInfo(
|
||||
pool_name=PoolName.KV,
|
||||
indices_from_pool=PoolName.KV,
|
||||
layer_ids=tuple(self.model_layer_ids[i] for i in info.layer_indices),
|
||||
buffer_info=info,
|
||||
),
|
||||
)
|
||||
|
||||
def _mla_buffer_info(self):
|
||||
from sglang.srt.mem_cache.device_pool_info import MLABufferInfo
|
||||
|
||||
start, end = (
|
||||
self._owned_local_layer_range()
|
||||
if self.layer_shard_enabled
|
||||
else (0, self.layer_num)
|
||||
)
|
||||
return MLABufferInfo(
|
||||
buffers=tuple(self.kv_buffer[start:end]),
|
||||
data_ptrs=self.data_ptrs
|
||||
if (start, end) == (0, self.layer_num)
|
||||
else self.data_ptrs[start:end],
|
||||
layer_indices=tuple(range(start, end)),
|
||||
device_layer_num=self.layer_num,
|
||||
page_size=self.page_size,
|
||||
kv_lora_rank=self.kv_lora_rank,
|
||||
qk_rope_head_dim=self.qk_rope_head_dim,
|
||||
kv_cache_dim=self.kv_cache_dim,
|
||||
dtype=self.store_dtype,
|
||||
device=self.data_ptrs.device,
|
||||
packed_fp8=self.dsa_kv_cache_store_fp8,
|
||||
scales=None
|
||||
if self.kv_scale_buffer is None
|
||||
else tuple(self.kv_scale_buffer[start:end]),
|
||||
)
|
||||
|
||||
def _create_buffers(self):
|
||||
with self.memory_saver_adapter.region(GPU_MEMORY_TYPE_KV_CACHE):
|
||||
with (
|
||||
@@ -5858,6 +5989,13 @@ class MiniMaxSparseKVPool(KVCache):
|
||||
else None
|
||||
)
|
||||
|
||||
if enable_hisparse:
|
||||
self.main_pool.model_layer_ids = tuple(local_sparse_layer_ids)
|
||||
if self.dense_pool is not None:
|
||||
self.dense_pool.model_layer_ids = tuple(local_dense_layer_ids)
|
||||
if self.index_kv_pool is not None:
|
||||
self.index_kv_pool.model_layer_ids = tuple(local_kv_sparse_layer_ids)
|
||||
|
||||
self.mem_usage = self.main_pool.mem_usage
|
||||
if self.dense_pool is not None:
|
||||
self.mem_usage += self.dense_pool.mem_usage
|
||||
|
||||
@@ -28,10 +28,12 @@ from sglang.kernels.ops.kvcache.hicache import (
|
||||
from sglang.kernels.ops.kvcache.hicache import (
|
||||
transfer_hicache_one_layer_mla as jit_transfer_hicache_one_layer_mla,
|
||||
)
|
||||
from sglang.srt.mem_cache.device_pool_info import (
|
||||
KVScaleLayout,
|
||||
MHABufferInfo,
|
||||
)
|
||||
from sglang.srt.mem_cache.memory_pool import (
|
||||
MHATokenToKOnlyPool,
|
||||
MHATokenToKVPool,
|
||||
MHATokenToKVPoolMXFP8,
|
||||
)
|
||||
from sglang.srt.mem_cache.pool_host.base import (
|
||||
_WRITE_BACK_STAGING_PAGE_CHUNK,
|
||||
@@ -76,46 +78,37 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def prepare_mha_write_back_staging(
|
||||
device_pool: MHATokenToKVPool,
|
||||
buffer_info: MHABufferInfo,
|
||||
*,
|
||||
layer_num: int,
|
||||
page_size: int,
|
||||
page_capacity: int = _WRITE_BACK_STAGING_PAGE_CHUNK,
|
||||
retain: bool = True,
|
||||
staging: tuple[torch.Tensor, torch.Tensor] | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor] | None:
|
||||
if not (_is_cuda or _is_hip) or layer_num == 0:
|
||||
return None
|
||||
head_num = device_pool.head_num
|
||||
if device_pool.head_dim == device_pool.v_head_dim:
|
||||
head_num = device_pool.row_dim // device_pool.head_dim
|
||||
head_num = buffer_info.head_num
|
||||
if not all(
|
||||
can_use_write_back_jit_kernel(
|
||||
element_size=head_num * dim * device_pool.store_dtype.itemsize
|
||||
element_size=head_num * dim * buffer_info.dtype.itemsize
|
||||
)
|
||||
for dim in (device_pool.head_dim, device_pool.v_head_dim)
|
||||
for dim in (buffer_info.head_dim, buffer_info.v_head_dim)
|
||||
):
|
||||
return None
|
||||
token_capacity = min(page_capacity, _WRITE_BACK_STAGING_PAGE_CHUNK) * page_size
|
||||
shapes = tuple(
|
||||
(token_capacity, layer_num, head_num, dim)
|
||||
for dim in (device_pool.head_dim, device_pool.v_head_dim)
|
||||
for dim in (buffer_info.head_dim, buffer_info.v_head_dim)
|
||||
)
|
||||
staging = device_pool.hicache_write_back_staging
|
||||
if staging is None:
|
||||
staging = (
|
||||
torch.empty(
|
||||
shapes[0], dtype=device_pool.store_dtype, device=device_pool.device
|
||||
),
|
||||
torch.empty(
|
||||
shapes[1], dtype=device_pool.store_dtype, device=device_pool.device
|
||||
),
|
||||
torch.empty(shapes[0], dtype=buffer_info.dtype, device=buffer_info.device),
|
||||
torch.empty(shapes[1], dtype=buffer_info.dtype, device=buffer_info.device),
|
||||
)
|
||||
if retain:
|
||||
device_pool.hicache_write_back_staging = staging
|
||||
else:
|
||||
for buffer, shape in zip(staging, shapes):
|
||||
if (
|
||||
buffer.dtype != device_pool.store_dtype
|
||||
buffer.dtype != buffer_info.dtype
|
||||
or tuple(buffer.shape[1:]) != shape[1:]
|
||||
or buffer.shape[0] < shape[0]
|
||||
):
|
||||
@@ -124,14 +117,9 @@ def prepare_mha_write_back_staging(
|
||||
|
||||
|
||||
class MHATokenToKVPoolHost(HostKVCache):
|
||||
device_pool: MHATokenToKVPool
|
||||
mtp_draft_device_pools: tuple[MHATokenToKVPool, ...] = ()
|
||||
|
||||
_transfer_device: torch.device
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
device_pool: MHATokenToKVPool,
|
||||
buffer_info: MHABufferInfo,
|
||||
host_to_device_ratio: float,
|
||||
host_size: int,
|
||||
page_size: int,
|
||||
@@ -140,41 +128,70 @@ class MHATokenToKVPoolHost(HostKVCache):
|
||||
device: str = "cpu",
|
||||
allocator_type: str = "default",
|
||||
*,
|
||||
mtp_draft_device_pools: Sequence[MHATokenToKVPool] = (),
|
||||
device_capacity_tokens: int,
|
||||
start_layer: int,
|
||||
end_layer: int,
|
||||
draft_buffer_infos: Sequence[MHABufferInfo] = (),
|
||||
write_back_staging: tuple[torch.Tensor, torch.Tensor] | None = None,
|
||||
pool_label: str = "kv",
|
||||
head_group_num: int = 1,
|
||||
):
|
||||
self.mtp_draft_device_pools = tuple(mtp_draft_device_pools)
|
||||
self.target_layer_num = device_pool.layer_num
|
||||
# page_unified cuts the kv-head axis into this many groups, and the cut
|
||||
# is part of the byte order, so it has to be known before the buffer is
|
||||
# allocated in init_kv_buffer() below.
|
||||
infos = (buffer_info, *draft_buffer_infos)
|
||||
for info in infos:
|
||||
if (
|
||||
info.scales is not None
|
||||
and info.scales.layout != KVScaleLayout.MXFP8_INTERLEAVED
|
||||
):
|
||||
raise ValueError(
|
||||
f"{pool_label}: MHA host transfers do not support "
|
||||
f"KV scale layout {info.scales.layout.value!r}."
|
||||
)
|
||||
info.validate()
|
||||
if info.transfer_buffers is None and (
|
||||
info.buffers.k_data_ptrs is None or info.buffers.v_data_ptrs is None
|
||||
):
|
||||
raise ValueError(
|
||||
f"{pool_label}: MHA host requires existing K/V pointer tables"
|
||||
)
|
||||
for depth, draft in enumerate(draft_buffer_infos):
|
||||
if len(draft.buffers.k) != 1:
|
||||
raise ValueError(
|
||||
f"{pool_label}: packed draft[{depth}] has {len(draft.buffers.k)} layers; expected 1"
|
||||
)
|
||||
target_geometry = (
|
||||
buffer_info.head_num,
|
||||
buffer_info.head_dim,
|
||||
buffer_info.v_head_dim,
|
||||
buffer_info.dtype,
|
||||
buffer_info.device,
|
||||
)
|
||||
draft_geometry = (
|
||||
draft.head_num,
|
||||
draft.head_dim,
|
||||
draft.v_head_dim,
|
||||
draft.dtype,
|
||||
draft.device,
|
||||
)
|
||||
if draft_geometry != target_geometry:
|
||||
raise ValueError(
|
||||
f"{pool_label}: packed draft[{depth}] (heads, K width, V width, dtype, device)={draft_geometry}, expected {target_geometry}"
|
||||
)
|
||||
self.target_layer_num = len(buffer_info.buffers.k)
|
||||
self._transfer_device = buffer_info.device
|
||||
self.dtype = buffer_info.dtype
|
||||
self.head_dim = buffer_info.head_dim
|
||||
self.v_head_dim = buffer_info.v_head_dim
|
||||
self.head_num = buffer_info.head_num
|
||||
self.layer_num = self.target_layer_num + len(draft_buffer_infos)
|
||||
self._buffer_infos = infos
|
||||
self.head_group_num = head_group_num
|
||||
# Transfer buffers are captured below. The pool remains available for
|
||||
# NPU transfer APIs and device layer ownership.
|
||||
self.device_pool = device_pool
|
||||
self._transfer_device = torch.device(device_pool.device)
|
||||
self.dtype = device_pool.store_dtype
|
||||
self.head_dim = device_pool.head_dim
|
||||
self.v_head_dim = device_pool.v_head_dim
|
||||
# Equal-width pools may store multiple attention rows per allocator token.
|
||||
self.head_num = (
|
||||
device_pool.row_dim // self.head_dim
|
||||
if self.head_dim == self.v_head_dim
|
||||
else device_pool.head_num
|
||||
)
|
||||
self.layer_num = self.target_layer_num + len(self.mtp_draft_device_pools)
|
||||
# Scale-bearing subclasses need page coverage when computing token bytes.
|
||||
self.page_size = page_size
|
||||
device_capacity = getattr(device_pool, "host_capacity_tokens", None)
|
||||
super().__init__(
|
||||
dtype=self.dtype,
|
||||
device_capacity_tokens=(
|
||||
device_pool.size if device_capacity is None else device_capacity
|
||||
),
|
||||
device_capacity_tokens=device_capacity_tokens,
|
||||
size_per_token=self.get_size_per_token(),
|
||||
start_layer=device_pool.start_layer,
|
||||
end_layer=device_pool.end_layer,
|
||||
start_layer=start_layer,
|
||||
end_layer=end_layer,
|
||||
host_to_device_ratio=host_to_device_ratio,
|
||||
host_size=host_size,
|
||||
page_size=page_size,
|
||||
@@ -184,12 +201,22 @@ class MHATokenToKVPoolHost(HostKVCache):
|
||||
allocator_type=allocator_type,
|
||||
pool_label=pool_label,
|
||||
)
|
||||
pools = (self.device_pool, *self.mtp_draft_device_pools)
|
||||
self.device_k_buffers = tuple(b for pool in pools for b in pool.k_buffer)
|
||||
self.device_v_buffers = tuple(b for pool in pools for b in pool.v_buffer)
|
||||
self._initialize_transfer_views()
|
||||
self.device_k_buffers = tuple(b for info in infos for b in info.buffers.k)
|
||||
self.device_v_buffers = tuple(b for info in infos for b in info.buffers.v)
|
||||
if buffer_info.transfer_buffers is None:
|
||||
self.device_k_data_ptrs = (
|
||||
torch.cat([info.buffers.k_data_ptrs for info in infos])
|
||||
if draft_buffer_infos
|
||||
else buffer_info.buffers.k_data_ptrs
|
||||
)
|
||||
self.device_v_data_ptrs = (
|
||||
torch.cat([info.buffers.v_data_ptrs for info in infos])
|
||||
if draft_buffer_infos
|
||||
else buffer_info.buffers.v_data_ptrs
|
||||
)
|
||||
self._initialize_transfer_views(buffer_info, write_back_staging)
|
||||
|
||||
def _initialize_transfer_views(self) -> None:
|
||||
def _initialize_transfer_views(self, buffer_info, write_back_staging) -> None:
|
||||
self.element_dim = self.head_num * self.head_dim
|
||||
self._init_device_row_stride((*self.device_k_buffers, *self.device_v_buffers))
|
||||
# The JIT HiCache kernels also build with hipcc (ROCm): the PTX-only
|
||||
@@ -235,20 +262,8 @@ class MHATokenToKVPoolHost(HostKVCache):
|
||||
if self.v_data_refs
|
||||
else None
|
||||
)
|
||||
if not _is_npu:
|
||||
pools = (self.device_pool, *self.mtp_draft_device_pools)
|
||||
self.device_k_data_ptrs = (
|
||||
torch.cat([pool.k_data_ptrs for pool in pools])
|
||||
if self.mtp_draft_device_pools
|
||||
else self.device_pool.k_data_ptrs
|
||||
)
|
||||
self.device_v_data_ptrs = (
|
||||
torch.cat([pool.v_data_ptrs for pool in pools])
|
||||
if self.mtp_draft_device_pools
|
||||
else self.device_pool.v_data_ptrs
|
||||
)
|
||||
self.host_kv_data_refs = self.k_data_refs + self.v_data_refs
|
||||
self._init_write_back_staging_buffers()
|
||||
self._init_write_back_staging_buffers(buffer_info, write_back_staging)
|
||||
|
||||
def get_size_per_token(self):
|
||||
return (
|
||||
@@ -322,7 +337,7 @@ class MHATokenToKVPoolHost(HostKVCache):
|
||||
)
|
||||
return buffer
|
||||
|
||||
def _init_write_back_staging_buffers(self):
|
||||
def _init_write_back_staging_buffers(self, buffer_info, write_back_staging):
|
||||
self.staging_page_capacity = 0
|
||||
self.staging_token_capacity = 0
|
||||
self.staging_k_buffer = None
|
||||
@@ -337,11 +352,11 @@ class MHATokenToKVPoolHost(HostKVCache):
|
||||
return
|
||||
page_capacity = min(self.page_num, _WRITE_BACK_STAGING_PAGE_CHUNK)
|
||||
staging = prepare_mha_write_back_staging(
|
||||
self.device_pool,
|
||||
buffer_info,
|
||||
layer_num=self.layer_num,
|
||||
page_size=self.page_size,
|
||||
page_capacity=page_capacity,
|
||||
retain=False,
|
||||
staging=write_back_staging,
|
||||
)
|
||||
if staging is None:
|
||||
return
|
||||
@@ -370,10 +385,9 @@ class MHATokenToKVPoolHost(HostKVCache):
|
||||
*,
|
||||
is_draft: bool = False,
|
||||
):
|
||||
if not is_draft and not self._is_device_layer_owned(device_pool, layer_id):
|
||||
if not is_draft and not 0 <= layer_id < self.target_layer_num:
|
||||
return
|
||||
# MTP draft layers do not participate in CP layer sharding.
|
||||
host_layer_id = layer_id if is_draft else self._host_layer_index(layer_id)
|
||||
host_layer_id = layer_id
|
||||
|
||||
if io_backend != "kernel_ascend":
|
||||
device_k = self.device_k_buffers[layer_id]
|
||||
@@ -487,7 +501,7 @@ class MHATokenToKVPoolHost(HostKVCache):
|
||||
device_v,
|
||||
host_k,
|
||||
host_v,
|
||||
) in self._npu_transfer_buffers(device_pool):
|
||||
) in self._npu_transfer_buffers():
|
||||
transfer_kv_dim_exchange(
|
||||
device_indices=device_indices,
|
||||
host_indices=host_indices,
|
||||
@@ -503,10 +517,10 @@ class MHATokenToKVPoolHost(HostKVCache):
|
||||
else:
|
||||
raise ValueError(f"Unsupported IO backend: {io_backend}")
|
||||
|
||||
def _npu_transfer_buffers(self, target_device_pool):
|
||||
def _npu_transfer_buffers(self):
|
||||
layer_start = 0
|
||||
for pool in (target_device_pool, *self.mtp_draft_device_pools):
|
||||
device_k, device_v = pool.get_hicache_transfer_buffers()
|
||||
for info in self._buffer_infos:
|
||||
device_k, device_v = info.transfer_buffers()
|
||||
layer_end = layer_start + device_k.shape[0]
|
||||
yield (
|
||||
device_k,
|
||||
@@ -634,9 +648,7 @@ class MHATokenToKVPoolHost(HostKVCache):
|
||||
raise ValueError(f"Unsupported layout: {self.layout}")
|
||||
elif io_backend == "kernel_ascend":
|
||||
if self.layout == "page_first_direct":
|
||||
for device_k, device_v, host_k, host_v in self._npu_transfer_buffers(
|
||||
device_pool
|
||||
):
|
||||
for device_k, device_v, host_k, host_v in self._npu_transfer_buffers():
|
||||
transfer_kv_dim_exchange(
|
||||
device_indices=device_indices,
|
||||
host_indices=host_indices,
|
||||
@@ -1185,16 +1197,23 @@ class HiSparseMHATokenToKVPoolHost(HiSparseHostPoolMixin, MHATokenToKVPoolHost):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
device_pool: MHATokenToKVPool,
|
||||
buffer_info: MHABufferInfo,
|
||||
host_to_device_ratio: float,
|
||||
page_size: int,
|
||||
*,
|
||||
device_capacity_tokens: int,
|
||||
start_layer: int,
|
||||
end_layer: int,
|
||||
):
|
||||
super().__init__(
|
||||
device_pool=device_pool,
|
||||
buffer_info=buffer_info,
|
||||
host_to_device_ratio=host_to_device_ratio,
|
||||
host_size=0,
|
||||
page_size=page_size,
|
||||
layout="layer_first",
|
||||
device_capacity_tokens=device_capacity_tokens,
|
||||
start_layer=start_layer,
|
||||
end_layer=end_layer,
|
||||
)
|
||||
|
||||
def get_contiguous_buf_infos(self):
|
||||
@@ -1343,10 +1362,9 @@ class AsymmetricMHATokenToKVPoolHost(MHATokenToKVPoolHost):
|
||||
*,
|
||||
is_draft: bool = False,
|
||||
):
|
||||
if not is_draft and not self._is_device_layer_owned(device_pool, layer_id):
|
||||
if not is_draft and not 0 <= layer_id < self.target_layer_num:
|
||||
return
|
||||
# MTP draft layers do not participate in CP layer sharding.
|
||||
host_layer_id = layer_id if is_draft else self._host_layer_index(layer_id)
|
||||
host_layer_id = layer_id
|
||||
|
||||
if io_backend != "kernel_ascend":
|
||||
device_k = self.device_k_buffers[layer_id]
|
||||
@@ -1574,25 +1592,19 @@ class AsymmetricMHATokenToKVPoolHost(MHATokenToKVPoolHost):
|
||||
)
|
||||
|
||||
|
||||
def get_mha_host_pool_cls(device_pool: MHATokenToKVPool) -> type:
|
||||
"""Pick the right MHA host-pool class based on the device pool's K/V dims.
|
||||
|
||||
Returns ``MHATokenToKVPoolMXFP8Host`` for the block-scaled MXFP8 pool (its
|
||||
UE8M0 scales must travel with the payload),
|
||||
``AsymmetricMHATokenToKVPoolHost`` when ``head_dim != v_head_dim``
|
||||
(e.g. MiMo-V2), else the default ``MHATokenToKVPoolHost``.
|
||||
"""
|
||||
if isinstance(device_pool, MHATokenToKVPoolMXFP8):
|
||||
if device_pool.head_dim != device_pool.v_head_dim:
|
||||
def get_mha_host_pool_cls(buffer_info: MHABufferInfo) -> type:
|
||||
is_mxfp8 = (
|
||||
buffer_info.scales is not None
|
||||
and buffer_info.scales.layout == KVScaleLayout.MXFP8_INTERLEAVED
|
||||
)
|
||||
if is_mxfp8:
|
||||
if buffer_info.head_dim != buffer_info.v_head_dim:
|
||||
raise NotImplementedError(
|
||||
"MXFP8 HiCache does not support asymmetric K/V head dimensions yet."
|
||||
)
|
||||
|
||||
from sglang.srt.mem_cache.pool_host.mha_mxfp8 import (
|
||||
MHATokenToKVPoolMXFP8Host,
|
||||
)
|
||||
from sglang.srt.mem_cache.pool_host.mha_mxfp8 import MHATokenToKVPoolMXFP8Host
|
||||
|
||||
return MHATokenToKVPoolMXFP8Host
|
||||
if device_pool.head_dim != device_pool.v_head_dim:
|
||||
if buffer_info.head_dim != buffer_info.v_head_dim:
|
||||
return AsymmetricMHATokenToKVPoolHost
|
||||
return MHATokenToKVPoolHost
|
||||
|
||||
@@ -23,7 +23,7 @@ from sglang.kernels.ops.kvcache.hicache import (
|
||||
from sglang.kernels.ops.kvcache.hicache import (
|
||||
transfer_hicache_one_layer_mla as jit_transfer_hicache_one_layer_mla,
|
||||
)
|
||||
from sglang.srt.mem_cache.memory_pool import MHATokenToKVPoolMXFP8
|
||||
from sglang.srt.mem_cache.device_pool_info import KVScaleLayout, MHABufferInfo
|
||||
from sglang.srt.mem_cache.pool_host.common import (
|
||||
ALLOC_MEMORY_FUNCS,
|
||||
_cuda_host_unregister,
|
||||
@@ -36,11 +36,9 @@ from sglang.srt.mem_cache.pool_host.mha import (
|
||||
|
||||
|
||||
class MHATokenToKVPoolMXFP8Host(MHATokenToKVPoolHost):
|
||||
device_pool: MHATokenToKVPoolMXFP8
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
device_pool: MHATokenToKVPoolMXFP8,
|
||||
buffer_info: MHABufferInfo,
|
||||
host_to_device_ratio: float,
|
||||
host_size: int,
|
||||
page_size: int,
|
||||
@@ -49,29 +47,38 @@ class MHATokenToKVPoolMXFP8Host(MHATokenToKVPoolHost):
|
||||
device: str = "cpu",
|
||||
allocator_type: str = "default",
|
||||
*,
|
||||
mtp_draft_device_pools: Sequence = (),
|
||||
device_capacity_tokens: int,
|
||||
start_layer: int,
|
||||
end_layer: int,
|
||||
draft_buffer_infos: Sequence[MHABufferInfo] = (),
|
||||
write_back_staging: tuple[torch.Tensor, torch.Tensor] | None = None,
|
||||
pool_label: str = "kv",
|
||||
):
|
||||
if layout != "page_first":
|
||||
raise NotImplementedError(
|
||||
f"MXFP8 KV host pool supports only the page_first layout, got {layout!r}."
|
||||
)
|
||||
if mtp_draft_device_pools:
|
||||
if draft_buffer_infos:
|
||||
raise NotImplementedError(
|
||||
"MXFP8 KV host pool does not pack MTP draft KV layers."
|
||||
)
|
||||
if not device_pool.mxfp8_sf_interleaved:
|
||||
interleaved = (
|
||||
buffer_info.scales is not None
|
||||
and buffer_info.scales.layout == KVScaleLayout.MXFP8_INTERLEAVED
|
||||
)
|
||||
if not interleaved:
|
||||
raise NotImplementedError(
|
||||
"MXFP8 KV host pool requires the interleaved (page_size=128) scale layout."
|
||||
)
|
||||
# Bytes of UE8M0 scales one page holds per layer; needed by
|
||||
# get_size_per_token before the base constructor sizes the pool.
|
||||
self.k_sf_page_bytes = device_pool.k_scale_buffer[0][0].numel()
|
||||
self.v_sf_page_bytes = device_pool.v_scale_buffer[0][0].numel()
|
||||
k_scales = buffer_info.scales.k
|
||||
v_scales = buffer_info.scales.v
|
||||
# HostKVCache calls get_size_per_token during base initialization.
|
||||
self.k_sf_page_bytes = k_scales[0][0].numel()
|
||||
self.v_sf_page_bytes = v_scales[0][0].numel()
|
||||
self.k_scale_host: torch.Tensor | None = None
|
||||
self.v_scale_host: torch.Tensor | None = None
|
||||
super().__init__(
|
||||
device_pool,
|
||||
buffer_info,
|
||||
host_to_device_ratio,
|
||||
host_size,
|
||||
page_size,
|
||||
@@ -79,13 +86,17 @@ class MHATokenToKVPoolMXFP8Host(MHATokenToKVPoolHost):
|
||||
pin_memory,
|
||||
device,
|
||||
allocator_type,
|
||||
device_capacity_tokens=device_capacity_tokens,
|
||||
start_layer=start_layer,
|
||||
end_layer=end_layer,
|
||||
write_back_staging=write_back_staging,
|
||||
pool_label=pool_label,
|
||||
)
|
||||
if self.page_size != device_pool.page_size:
|
||||
if self.page_size != buffer_info.page_size:
|
||||
raise ValueError(
|
||||
"MXFP8 KV host pool moves scales per page, so the host page size "
|
||||
f"({self.page_size}) must equal the device page size "
|
||||
f"({device_pool.page_size})."
|
||||
f"({buffer_info.page_size})."
|
||||
)
|
||||
if not self.can_use_write_back_jit or not all(
|
||||
can_use_write_back_jit_kernel(element_size=size)
|
||||
@@ -95,15 +106,15 @@ class MHATokenToKVPoolMXFP8Host(MHATokenToKVPoolHost):
|
||||
"MXFP8 KV host pool needs the staged JIT write-back kernel "
|
||||
"(io_backend='kernel', page_first layout, CUDA or HIP)."
|
||||
)
|
||||
self._init_scale_buffers()
|
||||
self._init_scale_buffers(k_scales, v_scales)
|
||||
|
||||
def get_size_per_token(self):
|
||||
payload = super().get_size_per_token()
|
||||
scales = (self.k_sf_page_bytes + self.v_sf_page_bytes) // self.page_size
|
||||
return payload + scales * self.layer_num
|
||||
|
||||
def _init_scale_buffers(self):
|
||||
alloc_func = ALLOC_MEMORY_FUNCS[self.device_pool.device]
|
||||
def _init_scale_buffers(self, k_scales, v_scales):
|
||||
alloc_func = ALLOC_MEMORY_FUNCS[self._transfer_device.type]
|
||||
# Host: page-first like the payload, one contiguous scale block per
|
||||
# (page, layer) so a page's layers are a single memcpy span.
|
||||
self.k_scale_host = alloc_func(
|
||||
@@ -126,32 +137,30 @@ class MHATokenToKVPoolMXFP8Host(MHATokenToKVPoolHost):
|
||||
|
||||
# Device: each layer's interleaved scale tensor as [page, bytes] rows.
|
||||
self.k_scale_device_rows = [
|
||||
buf.view(torch.uint8).reshape(buf.shape[0], -1)
|
||||
for buf in self.device_pool.k_scale_buffer
|
||||
buf.view(torch.uint8).reshape(buf.shape[0], -1) for buf in k_scales
|
||||
]
|
||||
self.v_scale_device_rows = [
|
||||
buf.view(torch.uint8).reshape(buf.shape[0], -1)
|
||||
for buf in self.device_pool.v_scale_buffer
|
||||
buf.view(torch.uint8).reshape(buf.shape[0], -1) for buf in v_scales
|
||||
]
|
||||
self.k_scale_device_ptrs = torch.tensor(
|
||||
[rows.data_ptr() for rows in self.k_scale_device_rows],
|
||||
dtype=torch.uint64,
|
||||
device=self.device_pool.device,
|
||||
device=self._transfer_device,
|
||||
)
|
||||
self.v_scale_device_ptrs = torch.tensor(
|
||||
[rows.data_ptr() for rows in self.v_scale_device_rows],
|
||||
dtype=torch.uint64,
|
||||
device=self.device_pool.device,
|
||||
device=self._transfer_device,
|
||||
)
|
||||
self.k_scale_staging = torch.empty(
|
||||
(self.staging_page_capacity, self.layer_num, self.k_sf_page_bytes),
|
||||
dtype=torch.uint8,
|
||||
device=self.device_pool.device,
|
||||
device=self._transfer_device,
|
||||
)
|
||||
self.v_scale_staging = torch.empty(
|
||||
(self.staging_page_capacity, self.layer_num, self.v_sf_page_bytes),
|
||||
dtype=torch.uint8,
|
||||
device=self.device_pool.device,
|
||||
device=self._transfer_device,
|
||||
)
|
||||
|
||||
def _page_ids(self, indices: torch.Tensor) -> torch.Tensor:
|
||||
@@ -209,9 +218,9 @@ class MHATokenToKVPoolMXFP8Host(MHATokenToKVPoolHost):
|
||||
raise NotImplementedError(
|
||||
f"MXFP8 KV host pool supports only io_backend='kernel', got {io_backend!r}."
|
||||
)
|
||||
if not self._is_device_layer_owned(device_pool, layer_id):
|
||||
if not 0 <= layer_id < self.target_layer_num:
|
||||
return
|
||||
host_layer_id = self._host_layer_index(layer_id)
|
||||
host_layer_id = layer_id
|
||||
device_pages = self._page_ids(device_indices)
|
||||
host_pages = self._page_ids(host_indices)
|
||||
for dst_rows, src_rows, sf_page_bytes in (
|
||||
|
||||
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
import logging
|
||||
from typing import Optional, Sequence
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.kvcache.hicache import (
|
||||
@@ -19,7 +20,7 @@ from sglang.kernels.ops.kvcache.hicache import (
|
||||
transfer_hicache_one_layer_mla as jit_transfer_hicache_one_layer_mla,
|
||||
)
|
||||
from sglang.srt.layers.dcp.layout import maybe_dcp_kernel_indices
|
||||
from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool
|
||||
from sglang.srt.mem_cache.device_pool_info import MLABufferInfo, MLASplitBuffers
|
||||
from sglang.srt.mem_cache.pool_host.base import (
|
||||
_WRITE_BACK_STAGING_PAGE_CHUNK,
|
||||
HostKVCache,
|
||||
@@ -62,15 +63,28 @@ if _is_npu:
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
device_pool: MLATokenToKVPool
|
||||
mtp_draft_device_pools: tuple[MLATokenToKVPool, ...] = ()
|
||||
class MLAHostAllocation(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Allocator-only geometry; it carries no device buffers or transfer promise."""
|
||||
|
||||
_transfer_device: torch.device
|
||||
dtype: torch.dtype
|
||||
kv_lora_rank: int
|
||||
qk_rope_head_dim: int
|
||||
kv_cache_dim: int
|
||||
size_per_token: int
|
||||
draft_layer_num: int = 0
|
||||
packed_fp8: bool = False
|
||||
|
||||
|
||||
class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
"""Host mirror built by the assembler or HiSparse from captured MLA buffers.
|
||||
|
||||
Capacity and reserved CP rows remain caller policy. Transfer methods retain
|
||||
the shared HostKVCache signature but use the captured tensors and slot map.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
device_pool: MLATokenToKVPool,
|
||||
buffer_info: MLABufferInfo | MLAHostAllocation,
|
||||
host_to_device_ratio: float,
|
||||
host_size: int,
|
||||
page_size: int,
|
||||
@@ -79,46 +93,117 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
device: str = "cpu",
|
||||
allocator_type: str = "default",
|
||||
override_kv_cache_dim: Optional[int] = None,
|
||||
mtp_draft_device_pools: Sequence[MLATokenToKVPool] = (),
|
||||
draft_buffer_infos: Sequence[MLABufferInfo] = (),
|
||||
dcp_size: int = 1,
|
||||
dcp_rank: int = 0,
|
||||
*,
|
||||
device_capacity_tokens: int,
|
||||
start_layer: int,
|
||||
end_layer: int,
|
||||
target_layer_num: int,
|
||||
layer_shard_enabled: bool = False,
|
||||
pool_label: str = "kv",
|
||||
is_dummy: bool = False,
|
||||
):
|
||||
self._is_dummy = isinstance(buffer_info, MLAHostAllocation)
|
||||
self.override_kv_cache_dim = override_kv_cache_dim
|
||||
self.mtp_draft_device_pools = tuple(mtp_draft_device_pools)
|
||||
self._is_dummy = is_dummy
|
||||
|
||||
# Transfer buffers are captured below. The pool remains available for
|
||||
# NPU transfer APIs and device layer ownership.
|
||||
self.device_pool = device_pool
|
||||
if not is_dummy:
|
||||
self._transfer_device = torch.device(device_pool.device)
|
||||
self.dtype = device_pool.store_dtype
|
||||
self.dtype = buffer_info.dtype
|
||||
self.layout = layout
|
||||
self.kv_lora_rank = device_pool.kv_lora_rank
|
||||
self.qk_rope_head_dim = device_pool.qk_rope_head_dim
|
||||
# L3 excludes the dead V component when the device stores packed FP8 KV.
|
||||
self.dsa_kv_cache_store_fp8 = getattr(
|
||||
device_pool, "dsa_kv_cache_store_fp8", False
|
||||
)
|
||||
self.target_layer_num = self._effective_host_layer_num()
|
||||
self.layer_num = self.target_layer_num + len(self.mtp_draft_device_pools)
|
||||
self.kv_lora_rank = buffer_info.kv_lora_rank
|
||||
self.qk_rope_head_dim = buffer_info.qk_rope_head_dim
|
||||
# Existing builders explicitly resolve packed row width before construction.
|
||||
self.kv_cache_dim = override_kv_cache_dim or (
|
||||
self.kv_lora_rank + self.qk_rope_head_dim
|
||||
buffer_info.kv_cache_dim
|
||||
if self._is_dummy
|
||||
else self.kv_lora_rank + self.qk_rope_head_dim
|
||||
)
|
||||
device_capacity = getattr(device_pool, "host_capacity_tokens", None)
|
||||
if is_dummy:
|
||||
device_capacity = device_capacity or device_pool.size
|
||||
self.dsa_kv_cache_store_fp8 = buffer_info.packed_fp8
|
||||
self.target_layer_num = target_layer_num
|
||||
self.layer_num = target_layer_num + (
|
||||
buffer_info.draft_layer_num if self._is_dummy else len(draft_buffer_infos)
|
||||
)
|
||||
self._layer_shard_enabled = layer_shard_enabled
|
||||
self._native_buffers = None
|
||||
if self._is_dummy:
|
||||
if draft_buffer_infos:
|
||||
raise ValueError(
|
||||
"Allocator-only MLA does not accept physical draft buffers"
|
||||
)
|
||||
self._size_per_token = buffer_info.size_per_token
|
||||
else:
|
||||
infos = (buffer_info, *draft_buffer_infos)
|
||||
for index, info in enumerate(infos):
|
||||
info.validate()
|
||||
if info.scales is not None:
|
||||
role = "target" if index == 0 else f"draft[{index - 1}]"
|
||||
raise ValueError(
|
||||
f"MLA host cache does not support {role} FP4 KV: "
|
||||
"packed payload and separate block scales cannot be restored"
|
||||
)
|
||||
if index and (
|
||||
(info.kv_cache_dim, info.dtype)
|
||||
!= (buffer_info.kv_cache_dim, buffer_info.dtype)
|
||||
or type(info.buffers) is not type(buffer_info.buffers)
|
||||
):
|
||||
raise ValueError(
|
||||
f"packed draft KV row {(info.kv_cache_dim, info.dtype)} differs "
|
||||
f"from target KV row {(buffer_info.kv_cache_dim, buffer_info.dtype)}"
|
||||
)
|
||||
if index and len(info.layer_indices) != 1:
|
||||
raise ValueError(
|
||||
f"MLA packed draft[{index - 1}] must contain one layer, got {info.layer_indices}"
|
||||
)
|
||||
self._transfer_device = buffer_info.device
|
||||
self._device_layer_num = buffer_info.device_layer_num
|
||||
self._target_layer_indices = buffer_info.layer_indices
|
||||
self._draft_layer_num = len(draft_buffer_infos)
|
||||
slots = (
|
||||
*buffer_info.layer_indices,
|
||||
*range(
|
||||
self._device_layer_num,
|
||||
self._device_layer_num + self._draft_layer_num,
|
||||
),
|
||||
)
|
||||
self._buffer_index_by_layer = {layer: i for i, layer in enumerate(slots)}
|
||||
self._host_layer_by_controller = dict(
|
||||
zip(buffer_info.layer_indices, range(len(buffer_info.layer_indices)))
|
||||
)
|
||||
self._host_layer_by_controller.update(
|
||||
{
|
||||
self._device_layer_num + i: target_layer_num + i
|
||||
for i in range(self._draft_layer_num)
|
||||
}
|
||||
)
|
||||
self._size_per_token = (
|
||||
self.kv_cache_dim * self.dtype.itemsize * self.layer_num
|
||||
)
|
||||
if isinstance(buffer_info.buffers, MLASplitBuffers):
|
||||
self._native_buffers = buffer_info.buffers
|
||||
self._native_infos = infos
|
||||
native = self._native_buffers
|
||||
index_layers = len(native.index_layer_indices)
|
||||
if layout == "page_first_kv_split" and native.index_k is not None:
|
||||
self._size_per_token += (
|
||||
native.index_width * self.dtype.itemsize * index_layers
|
||||
)
|
||||
if native.index_scales is not None:
|
||||
self._size_per_token += 4 * index_layers
|
||||
self.device_kv_buffers = ()
|
||||
self.device_data_ptrs = None
|
||||
else:
|
||||
self.device_kv_buffers = tuple(
|
||||
tensor for info in infos for tensor in info.buffers
|
||||
)
|
||||
self.device_data_ptrs = (
|
||||
torch.cat([info.data_ptrs for info in infos])
|
||||
if draft_buffer_infos
|
||||
else buffer_info.data_ptrs
|
||||
)
|
||||
super().__init__(
|
||||
dtype=self.dtype,
|
||||
device_capacity_tokens=(
|
||||
device_pool.size if device_capacity is None else device_capacity
|
||||
),
|
||||
device_capacity_tokens=device_capacity_tokens,
|
||||
size_per_token=self.get_size_per_token(),
|
||||
start_layer=device_pool.start_layer,
|
||||
end_layer=device_pool.end_layer,
|
||||
start_layer=start_layer,
|
||||
end_layer=end_layer,
|
||||
host_to_device_ratio=host_to_device_ratio,
|
||||
host_size=host_size,
|
||||
page_size=page_size,
|
||||
@@ -129,9 +214,9 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
dcp_size=dcp_size,
|
||||
dcp_rank=dcp_rank,
|
||||
pool_label=pool_label,
|
||||
is_dummy=is_dummy,
|
||||
is_dummy=self._is_dummy,
|
||||
)
|
||||
if is_dummy:
|
||||
if self._is_dummy:
|
||||
self.token_stride_size = self.kv_cache_dim * self.dtype.itemsize
|
||||
self.layout_dim = self.token_stride_size * self.layer_num
|
||||
self.can_use_jit = False
|
||||
@@ -149,16 +234,6 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
)
|
||||
return
|
||||
|
||||
# Ascend transfers split K, rope and index tensors through its own API.
|
||||
self.device_kv_buffers = (
|
||||
()
|
||||
if _is_npu
|
||||
else tuple(
|
||||
b
|
||||
for pool in (device_pool, *self.mtp_draft_device_pools)
|
||||
for b in pool.kv_buffer
|
||||
)
|
||||
)
|
||||
self._initialize_transfer_views()
|
||||
|
||||
def _initialize_transfer_views(self) -> None:
|
||||
@@ -184,14 +259,8 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
self._transfer_device,
|
||||
host_memory_registered=self.pin_memory,
|
||||
)
|
||||
if not _is_npu:
|
||||
pools = (self.device_pool, *self.mtp_draft_device_pools)
|
||||
self.device_data_ptrs = (
|
||||
torch.cat([pool.data_ptrs for pool in pools])
|
||||
if self.mtp_draft_device_pools
|
||||
else self.device_pool.data_ptrs
|
||||
)
|
||||
self._init_device_row_stride(self.device_kv_buffers)
|
||||
if self._native_buffers is None:
|
||||
self._init_device_row_stride(self.device_kv_buffers)
|
||||
self._init_write_back_staging_buffers()
|
||||
|
||||
def get_contiguous_buf_infos(self):
|
||||
@@ -210,27 +279,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
return data_ptrs, data_lens, item_lens
|
||||
|
||||
def get_size_per_token(self):
|
||||
size_per_token = self.kv_cache_dim * self.dtype.itemsize * self.layer_num
|
||||
if (
|
||||
self.layout == "page_first_kv_split"
|
||||
and self.device_pool.index_head_dim is not None
|
||||
):
|
||||
# Indexer buffers only exist for physical Indexer layers, which can
|
||||
# be a subset of all layers (e.g. GLM 5.2: 21 of 78). Mirror the
|
||||
# layer count used by init_kv_buffer so host capacity sizing stays
|
||||
# consistent with the actually-allocated buffers.
|
||||
num_indexer_layers = getattr(self.device_pool, "num_indexer_layers", None)
|
||||
if num_indexer_layers is None:
|
||||
num_indexer_layers = self.layer_num
|
||||
size_per_token += (
|
||||
self.device_pool.index_head_dim
|
||||
* self.dtype.itemsize
|
||||
* num_indexer_layers
|
||||
)
|
||||
if getattr(self.device_pool, "index_k_scale_buffer", None) is not None:
|
||||
# FP32 quantization scale per token per indexer layer.
|
||||
size_per_token += 4 * num_indexer_layers
|
||||
return size_per_token
|
||||
return self._size_per_token
|
||||
|
||||
def get_ksize_per_token(self):
|
||||
return self.get_size_per_token()
|
||||
@@ -282,15 +331,14 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
# pool packs them as (num_indexer_layers, page, ...); mirror that
|
||||
# layer count here so transfer_kv_dim_exchange's layer check
|
||||
# (device dim0 == host dim1) holds.
|
||||
num_indexer_layers = getattr(self.device_pool, "num_indexer_layers", None)
|
||||
if num_indexer_layers is None:
|
||||
num_indexer_layers = self.layer_num
|
||||
native = self._native_buffers
|
||||
num_indexer_layers = len(native.index_layer_indices)
|
||||
indexer_dims = (self.page_num, num_indexer_layers, self.page_size, 1)
|
||||
alloc_func = ALLOC_MEMORY_FUNCS[self._transfer_device.type]
|
||||
if getattr(self.device_pool, "dsa_kv_cache_store_fp8", False):
|
||||
if self.dsa_kv_cache_store_fp8:
|
||||
# FP8 DSA packs latent+RoPE+scale into the device k_buffer;
|
||||
# mirror the packed width so the 2D memcpy row width matches.
|
||||
k_width = self.device_pool.kv_cache_dim
|
||||
k_width = native.k_width
|
||||
else:
|
||||
k_width = self.kv_lora_rank
|
||||
if _is_npu and memfabric_host_memory_enabled():
|
||||
@@ -305,15 +353,15 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
* (k_width + self.qk_rope_head_dim)
|
||||
* self.dtype.itemsize
|
||||
)
|
||||
if self.device_pool.index_head_dim is not None:
|
||||
if native.index_k is not None:
|
||||
total_bytes += (
|
||||
self.page_num
|
||||
* self.page_size
|
||||
* num_indexer_layers
|
||||
* self.device_pool.index_head_dim
|
||||
* native.index_width
|
||||
* self.dtype.itemsize
|
||||
)
|
||||
if getattr(self.device_pool, "index_k_scale_buffer", None) is not None:
|
||||
if native.index_scales is not None:
|
||||
# FP32 scale mirror
|
||||
total_bytes += (
|
||||
self.page_num * self.page_size * num_indexer_layers * 4
|
||||
@@ -335,9 +383,9 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
allocator=self.allocator,
|
||||
)
|
||||
self.index_k_buffer = None
|
||||
if self.device_pool.index_head_dim is not None:
|
||||
if native.index_k is not None:
|
||||
self.index_k_buffer = alloc_func(
|
||||
(*indexer_dims, self.device_pool.index_head_dim),
|
||||
(*indexer_dims, native.index_width),
|
||||
dtype=self.dtype,
|
||||
device=self.device,
|
||||
pin_memory=self.pin_memory,
|
||||
@@ -347,7 +395,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
# (see NPUMLATokenToKVPool.index_k_scale_buffer). Only present when
|
||||
# the device pool carries one (FP8 DSA + npu_quant_lightning_indexer).
|
||||
self.index_k_scale_buffer = None
|
||||
if getattr(self.device_pool, "index_k_scale_buffer", None) is not None:
|
||||
if native.index_scales is not None:
|
||||
self.index_k_scale_buffer = alloc_func(
|
||||
(*indexer_dims, 1),
|
||||
dtype=torch.float32,
|
||||
@@ -386,6 +434,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
# The staged kernel reads whole device pages, so it needs packed rows.
|
||||
if (
|
||||
self.layout != "page_first"
|
||||
or self._layer_shard_enabled
|
||||
or not self.device_rows_packed
|
||||
or not current_platform.capabilities.hicache_device_kernels
|
||||
):
|
||||
@@ -414,27 +463,17 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
device=self._transfer_device,
|
||||
)
|
||||
|
||||
def _indexer_slot_range_for_layer(self, device_pool, device_layer_id):
|
||||
"""Map a device layer onto the indexer slot space.
|
||||
|
||||
Returns ``(slot, 1)`` when the layer is an indexer layer, ``(0, 0)``
|
||||
to skip the indexer components, or ``(0, -1)`` (all indexer layers)
|
||||
when the mapping is unavailable. ``indexer_layer_ids`` holds
|
||||
absolute (PP-global) layer ids while ``device_layer_id`` is
|
||||
pool-local; convert before matching.
|
||||
"""
|
||||
indexer_layer_ids = getattr(device_pool, "indexer_layer_ids", None)
|
||||
if not indexer_layer_ids or self.index_k_buffer is None:
|
||||
def _indexer_slot_range_for_layer(self, buffers, device_layer_id):
|
||||
if not buffers.index_layer_indices or self.index_k_buffer is None:
|
||||
return 0, -1
|
||||
start_layer = getattr(device_pool, "start_layer", 0)
|
||||
for slot, layer in enumerate(indexer_layer_ids):
|
||||
if layer - start_layer == device_layer_id:
|
||||
for slot, layer in enumerate(buffers.index_layer_indices):
|
||||
if layer == device_layer_id:
|
||||
return slot, 1
|
||||
return 0, 0
|
||||
|
||||
def _transfer_ascendc_sparse_copy(
|
||||
self,
|
||||
device_pool,
|
||||
buffers,
|
||||
host_indices,
|
||||
device_indices,
|
||||
direction: TransferDirection,
|
||||
@@ -459,7 +498,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
The kernel de-references host pool pointers directly, so the host
|
||||
pool must be Memfabric-mapped.
|
||||
"""
|
||||
device = device_pool.k_buffer.device
|
||||
device = buffers.k.device
|
||||
# The kernel reads the token indices directly from device memory.
|
||||
# Upload without a stream sync: a plain .to(device) from pageable
|
||||
# memory synchronizes the stream and would serialize the pipeline.
|
||||
@@ -474,7 +513,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
device_indices.record_stream(stream)
|
||||
|
||||
comps = self._build_ascendc_component_meta(
|
||||
device_pool,
|
||||
buffers,
|
||||
layer_start,
|
||||
layer_num,
|
||||
index_k_layer_start,
|
||||
@@ -528,7 +567,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
|
||||
def _build_ascendc_component_meta(
|
||||
self,
|
||||
device_pool,
|
||||
buffers,
|
||||
layer_start: int,
|
||||
layer_num: int,
|
||||
index_k_layer_start: int,
|
||||
@@ -536,12 +575,12 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
) -> list:
|
||||
"""Assemble the per-component metadata list for kv_exchange_copy."""
|
||||
k_lo = layer_start
|
||||
k_hi = device_pool.k_buffer.shape[0] if layer_num < 0 else k_lo + layer_num
|
||||
k_hi = buffers.k.shape[0] if layer_num < 0 else k_lo + layer_num
|
||||
# Both pools must share the layer index space (same limitation as the
|
||||
# legacy memcpy2d exchange op); catches e.g. MTP draft pools, whose
|
||||
# host rows live past the main pool's layers.
|
||||
host_layer_num = self.k_buffer.shape[1]
|
||||
device_layer_num = device_pool.k_buffer.shape[0]
|
||||
device_layer_num = buffers.k.shape[0]
|
||||
if k_hi > host_layer_num or k_hi > device_layer_num:
|
||||
raise RuntimeError(
|
||||
f"AscendC kv_exchange layer range [{k_lo}, {k_hi}) exceeds the "
|
||||
@@ -550,13 +589,13 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
)
|
||||
|
||||
comp_meta = self._ascendc_comp_meta
|
||||
comps = [comp_meta(device_pool.k_buffer, self.k_buffer, k_lo, k_hi)]
|
||||
comps = [comp_meta(buffers.k, self.k_buffer, k_lo, k_hi)]
|
||||
# FP8 DSA packs V into the device k_buffer; the device v_buffer is
|
||||
# empty and must be skipped.
|
||||
if device_pool.v_buffer.numel() > 0 and self.v_buffer.numel() > 0:
|
||||
comps.append(comp_meta(device_pool.v_buffer, self.v_buffer, k_lo, k_hi))
|
||||
if buffers.v.numel() > 0 and self.v_buffer.numel() > 0:
|
||||
comps.append(comp_meta(buffers.v, self.v_buffer, k_lo, k_hi))
|
||||
|
||||
device_index_k = getattr(device_pool, "index_k_buffer", None)
|
||||
device_index_k = buffers.index_k
|
||||
if self.index_k_buffer is not None and device_index_k is not None:
|
||||
if index_k_layer_num < 0:
|
||||
ik_lo, ik_hi = 0, self.index_k_buffer.shape[1]
|
||||
@@ -567,7 +606,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
comps.append(
|
||||
comp_meta(device_index_k, self.index_k_buffer, ik_lo, ik_hi)
|
||||
)
|
||||
device_scale = getattr(device_pool, "index_k_scale_buffer", None)
|
||||
device_scale = buffers.index_scales
|
||||
if self.index_k_scale_buffer is not None and device_scale is not None:
|
||||
comps.append(
|
||||
comp_meta(
|
||||
@@ -639,21 +678,22 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
*,
|
||||
is_draft: bool = False,
|
||||
):
|
||||
if not is_draft and not self._is_device_layer_owned(device_pool, layer_id):
|
||||
assert not self._is_dummy, "load on a dummy (non-src MLA) host pool"
|
||||
if not is_draft and layer_id not in self._target_layer_indices:
|
||||
return
|
||||
assert not getattr(self, "_is_dummy", False), (
|
||||
"load on a dummy (non-src MLA) host pool"
|
||||
)
|
||||
host_indices = maybe_dcp_kernel_indices(
|
||||
host_indices, self.dcp_size, self.dcp_rank
|
||||
)
|
||||
device_indices = maybe_dcp_kernel_indices(
|
||||
device_indices, self.dcp_size, self.dcp_rank
|
||||
)
|
||||
# MTP draft layers do not participate in CP layer sharding.
|
||||
host_layer_id = layer_id if is_draft else self._host_layer_index(layer_id)
|
||||
# Draft transfer IDs follow all target device layers, while host rows
|
||||
# follow only this rank's target layers.
|
||||
host_layer_id = self._host_layer_by_controller[layer_id]
|
||||
if io_backend != "kernel_ascend":
|
||||
device_buffer = self.device_kv_buffers[layer_id]
|
||||
device_buffer = self.device_kv_buffers[
|
||||
self._buffer_index_by_layer[layer_id]
|
||||
]
|
||||
|
||||
if io_backend == "kernel":
|
||||
if self.layout == "layer_first":
|
||||
@@ -720,16 +760,18 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
else:
|
||||
raise ValueError(f"Unsupported layout: {self.layout}")
|
||||
elif io_backend == "kernel_ascend":
|
||||
info_index = layer_id - self._device_layer_num + 1 if is_draft else 0
|
||||
native = self._native_infos[info_index].native_buffers()
|
||||
device_layer_id = 0 if is_draft else layer_id
|
||||
if self.layout == "page_first_kv_split":
|
||||
if _is_npu and ascendc_io_enabled():
|
||||
# The per-layer complete(i) event recorded by the caller lets
|
||||
# later layers' DMA overlap the current layer's compute.
|
||||
ik_start, ik_num = self._indexer_slot_range_for_layer(
|
||||
device_pool, device_layer_id
|
||||
native, device_layer_id
|
||||
)
|
||||
self._transfer_ascendc_sparse_copy(
|
||||
device_pool,
|
||||
native,
|
||||
host_indices,
|
||||
device_indices,
|
||||
TransferDirection.H2D,
|
||||
@@ -747,19 +789,13 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
transfer_kv_dim_exchange(
|
||||
device_indices=device_indices,
|
||||
host_indices=host_indices,
|
||||
device_k=getattr(
|
||||
device_pool, "k_buffer_tensor", device_pool.k_buffer
|
||||
),
|
||||
device_k=native.k,
|
||||
host_k=self.k_buffer,
|
||||
device_v=getattr(
|
||||
device_pool, "v_buffer_tensor", device_pool.v_buffer
|
||||
),
|
||||
device_v=native.v,
|
||||
host_v=self.v_buffer,
|
||||
device_index_k=device_pool.index_k_buffer,
|
||||
device_index_k=native.index_k,
|
||||
host_index_k=self.index_k_buffer,
|
||||
device_index_k_scale=getattr(
|
||||
device_pool, "index_k_scale_buffer", None
|
||||
),
|
||||
device_index_k_scale=native.index_scales,
|
||||
host_index_k_scale=self.index_k_scale_buffer,
|
||||
page_size=self.page_size,
|
||||
direction=TransferDirection.H2D,
|
||||
@@ -778,17 +814,16 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
*,
|
||||
is_draft: bool = False,
|
||||
):
|
||||
assert not getattr(self, "_is_dummy", False), (
|
||||
"backup on a dummy (non-src MLA) host pool"
|
||||
)
|
||||
assert not self._is_dummy, "backup on a dummy (non-src MLA) host pool"
|
||||
if io_backend not in ("kernel", "direct"):
|
||||
raise ValueError(
|
||||
f"Layer-sharded HiCache backup does not support IO backend: {io_backend}"
|
||||
)
|
||||
# Indices arrive already translated by backup_from_device_all_layer.
|
||||
# MTP draft layers do not participate in CP layer sharding.
|
||||
host_layer_id = layer_id if is_draft else self._host_layer_index(layer_id)
|
||||
device_buffer = self.device_kv_buffers[layer_id]
|
||||
# Draft transfer IDs follow all target device layers, while host rows
|
||||
# follow only this rank's target layers.
|
||||
host_layer_id = self._host_layer_by_controller[layer_id]
|
||||
device_buffer = self.device_kv_buffers[self._buffer_index_by_layer[layer_id]]
|
||||
|
||||
if io_backend == "kernel":
|
||||
if self.layout == "layer_first":
|
||||
@@ -847,25 +882,23 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
def backup_from_device_all_layer(
|
||||
self, device_pool, host_indices, device_indices, io_backend
|
||||
):
|
||||
assert not getattr(self, "_is_dummy", False), (
|
||||
"backup on a dummy (non-src MLA) host pool"
|
||||
)
|
||||
assert not self._is_dummy, "backup on a dummy (non-src MLA) host pool"
|
||||
host_indices = maybe_dcp_kernel_indices(
|
||||
host_indices, self.dcp_size, self.dcp_rank
|
||||
)
|
||||
device_indices = maybe_dcp_kernel_indices(
|
||||
device_indices, self.dcp_size, self.dcp_rank
|
||||
)
|
||||
if self._is_device_layer_sharded(device_pool):
|
||||
for layer_id in self._owned_device_layer_ids(device_pool):
|
||||
if self._layer_shard_enabled:
|
||||
for layer_id in self._target_layer_indices:
|
||||
self._backup_from_device_per_layer(
|
||||
host_indices, device_indices, layer_id, io_backend
|
||||
)
|
||||
for draft_layer_id in range(len(self.mtp_draft_device_pools)):
|
||||
for draft_layer_id in range(self._draft_layer_num):
|
||||
self._backup_from_device_per_layer(
|
||||
host_indices,
|
||||
device_indices,
|
||||
self.device_pool.layer_num + draft_layer_id,
|
||||
self._device_layer_num + draft_layer_id,
|
||||
io_backend,
|
||||
is_draft=True,
|
||||
)
|
||||
@@ -959,10 +992,11 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
else:
|
||||
raise ValueError(f"Unsupported layout: {self.layout}")
|
||||
elif io_backend == "kernel_ascend":
|
||||
native = self._native_infos[0].native_buffers()
|
||||
if self.layout == "page_first_kv_split":
|
||||
if _is_npu and ascendc_io_enabled():
|
||||
self._transfer_ascendc_sparse_copy(
|
||||
device_pool,
|
||||
native,
|
||||
host_indices,
|
||||
device_indices,
|
||||
TransferDirection.D2H,
|
||||
@@ -971,19 +1005,13 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
transfer_kv_dim_exchange(
|
||||
device_indices=device_indices,
|
||||
host_indices=host_indices,
|
||||
device_k=getattr(
|
||||
device_pool, "k_buffer_tensor", device_pool.k_buffer
|
||||
),
|
||||
device_k=native.k,
|
||||
host_k=self.k_buffer,
|
||||
device_v=getattr(
|
||||
device_pool, "v_buffer_tensor", device_pool.v_buffer
|
||||
),
|
||||
device_v=native.v,
|
||||
host_v=self.v_buffer,
|
||||
device_index_k=device_pool.index_k_buffer,
|
||||
device_index_k=native.index_k,
|
||||
host_index_k=self.index_k_buffer,
|
||||
device_index_k_scale=getattr(
|
||||
device_pool, "index_k_scale_buffer", None
|
||||
),
|
||||
device_index_k_scale=native.index_scales,
|
||||
host_index_k_scale=self.index_k_scale_buffer,
|
||||
page_size=self.page_size,
|
||||
direction=TransferDirection.D2H,
|
||||
|
||||
@@ -31,16 +31,17 @@ class AibrixKVCacheStorage(HiCacheStorage):
|
||||
else:
|
||||
self.is_mla_backend = False
|
||||
self.local_rank = 0
|
||||
kv_cache = mem_pool.device_pool
|
||||
self.page_size = mem_pool.page_size
|
||||
self.kv_cache_dtype = kv_cache.dtype
|
||||
self.layer_num = kv_cache.layer_num
|
||||
storage_info = mem_pool
|
||||
self.kv_cache_dtype = storage_info.dtype
|
||||
self.layer_num = mem_pool.target_layer_num
|
||||
self.kv_head_ids = [
|
||||
self.local_rank * kv_cache.head_num + i for i in range(kv_cache.head_num)
|
||||
self.local_rank * storage_info.head_num + i
|
||||
for i in range(storage_info.head_num)
|
||||
]
|
||||
if not self.is_mla_backend:
|
||||
self.layer_ids = range(
|
||||
kv_cache.start_layer, kv_cache.end_layer
|
||||
storage_info.start_layer, storage_info.end_layer
|
||||
) # for pipeline parallel
|
||||
|
||||
self.block_spec = KVCacheBlockSpec(
|
||||
@@ -50,7 +51,7 @@ class AibrixKVCacheStorage(HiCacheStorage):
|
||||
tensor_spec=KVCacheTensorSpec(
|
||||
heads=self.kv_head_ids,
|
||||
layers=self.layer_ids,
|
||||
head_size=kv_cache.head_dim,
|
||||
head_size=storage_info.head_dim,
|
||||
),
|
||||
)
|
||||
logger.info(self.block_spec)
|
||||
|
||||
@@ -51,7 +51,19 @@ class AIBrixKVCacheStorageTest:
|
||||
0,
|
||||
layer_num,
|
||||
)
|
||||
mem_pool = MHATokenToKVPoolHost(kv_cache, 2, 0, page_size, "layer_first")
|
||||
mem_pool = MHATokenToKVPoolHost(
|
||||
kv_cache.get_device_pool_infos()[0].buffer_info,
|
||||
2,
|
||||
0,
|
||||
page_size,
|
||||
"layer_first",
|
||||
device_capacity_tokens=kv_cache.size
|
||||
if kv_cache.host_capacity_tokens is None
|
||||
else kv_cache.host_capacity_tokens,
|
||||
start_layer=kv_cache.start_layer,
|
||||
end_layer=kv_cache.end_layer,
|
||||
write_back_staging=kv_cache.hicache_write_back_staging,
|
||||
)
|
||||
query_length = batch_size * 2
|
||||
partial = batch_size
|
||||
self.aibrix_kvcache = AibrixKVCacheStorage(config, mem_pool)
|
||||
|
||||
@@ -489,7 +489,7 @@ def _validate_storage_config(
|
||||
"TensorCast FULL v1 supports only page_first and page_first_direct, "
|
||||
f"got layout={mem_pool_host.layout!r}"
|
||||
)
|
||||
if mem_pool_host.mtp_draft_device_pools:
|
||||
if mem_pool_host.layer_num != mem_pool_host.target_layer_num:
|
||||
raise NotImplementedError(
|
||||
"TensorCast FULL v1 does not support packed MTP draft pools"
|
||||
)
|
||||
|
||||
@@ -247,7 +247,7 @@ def _make_pool(
|
||||
pool.layer_num = layer_num
|
||||
pool.head_num = head_num
|
||||
pool.dtype = dtype
|
||||
pool.mtp_draft_device_pools = ()
|
||||
pool.target_layer_num = layer_num
|
||||
pool.dcp_size = 1
|
||||
pool.dcp_rank = 0
|
||||
pool.allocator = allocator or HostTensorAllocator()
|
||||
@@ -332,7 +332,7 @@ def test_fragment_registration_dispatch_is_exact_and_specific() -> None:
|
||||
[
|
||||
({"layout": "layer_first"}, {}, "supports only"),
|
||||
({"layout": "page_head"}, {}, "supports only"),
|
||||
({"mtp_draft_device_pools": (object(),)}, {}, "MTP"),
|
||||
({"target_layer_num": 0}, {}, "MTP"),
|
||||
({"dcp_size": 2}, {}, "DCP"),
|
||||
({}, {"attn_cp_size": 2}, "context parallel"),
|
||||
({}, {"tp_lcm_size": 2}, "TP-LCM"),
|
||||
|
||||
@@ -105,6 +105,12 @@ class SWAKVPool(BaseSWAKVPool):
|
||||
layer_num=self.swa_layer_nums,
|
||||
**swa_kv_pool_kwargs,
|
||||
)
|
||||
for pool, layer_ids in (
|
||||
(self.full_kv_pool, full_attention_layer_ids),
|
||||
(self.swa_kv_pool, swa_attention_layer_ids),
|
||||
):
|
||||
if isinstance(pool, MHATokenToKVPool):
|
||||
pool.model_layer_ids = tuple(layer_ids)
|
||||
self.dsa_kv_cache_store_fp8 = False
|
||||
self.kv_cache_dim = None
|
||||
self.index_head_dim = None
|
||||
|
||||
@@ -1909,6 +1909,9 @@ class UnifiedSWAKVPool(SWAKVPool):
|
||||
end_layer=end_layer,
|
||||
)
|
||||
|
||||
self.full_kv_pool.model_layer_ids = tuple(full_attention_layer_ids)
|
||||
self.swa_kv_pool.model_layer_ids = tuple(swa_attention_layer_ids)
|
||||
|
||||
# disagg/nvlink disabled; keep attrs present to avoid AttributeError.
|
||||
self.enable_custom_mem_pool = False
|
||||
self.custom_mem_pool = None
|
||||
|
||||
Reference in New Issue
Block a user