diff --git a/python/sglang/srt/hardware_backend/npu/memory_pool_npu.py b/python/sglang/srt/hardware_backend/npu/memory_pool_npu.py index 013e9db066..904e241c3c 100644 --- a/python/sglang/srt/hardware_backend/npu/memory_pool_npu.py +++ b/python/sglang/srt/hardware_backend/npu/memory_pool_npu.py @@ -190,6 +190,33 @@ class NPUMHATokenToKVPool(MHATokenToKVPool): """Return contiguous all-layer KV tensors for NPU HiCache IO.""" return self._hicache_k_buffer, self._hicache_v_buffer + def _mha_buffer_info(self): + from sglang.srt.mem_cache.device_pool_info import ( + KVBufferLayout, + KVBuffers, + MHABufferInfo, + ) + + return MHABufferInfo( + buffers=KVBuffers( + k=tuple(self.k_buffer), + v=tuple(self.v_buffer), + k_data_ptrs=None, + v_data_ptrs=None, + layout=KVBufferLayout.PAGED_NHD, + ), + page_size=self.page_size, + head_num=self.row_dim // self.head_dim + if self.head_dim == self.v_head_dim + else self.head_num, + head_dim=self.head_dim, + v_head_dim=self.v_head_dim, + dtype=self.store_dtype, + # self.device is the bare backend name; tensors carry the rank index. + device=self._hicache_k_buffer.device, + transfer_buffers=self.get_hicache_transfer_buffers, + ) + def _init_kv_copy_and_warmup(self): # implementation relies on self.data_strides / self.data_ptrs, which the # NPU paged buffer layout never builds. @@ -610,6 +637,9 @@ class NPUMLATokenToKVPool(MLATokenToKVPool): end_layer=end_layer, ) + self.model_layer_ids = tuple( + range(self.start_layer, self.start_layer + self.layer_num) + ) self.kv_lora_rank = kv_lora_rank self.qk_rope_head_dim = qk_rope_head_dim self.index_head_dim = index_head_dim @@ -731,6 +761,45 @@ class NPUMLATokenToKVPool(MLATokenToKVPool): self._finalize_allocation_log(size) + def _mla_buffer_info(self): + from sglang.srt.mem_cache.device_pool_info import MLABufferInfo + + return MLABufferInfo( + buffers=self._mla_native_buffers(), + native_buffers=self._mla_native_buffers, + data_ptrs=None, + layer_indices=tuple(range(self.layer_num)), + device_layer_num=self.layer_num, + page_size=self.page_size, + kv_lora_rank=self.kv_lora_rank, + qk_rope_head_dim=self.qk_rope_head_dim, + kv_cache_dim=self.kv_cache_dim + self.kr_cache_dim, + dtype=self.store_dtype, + # self.device is the bare backend name; tensors carry the rank index. + device=next( + buffer.device + for buffer in (self.k_buffer, self.index_k_buffer) + if buffer is not None + ), + packed_fp8=self.dsa_kv_cache_store_fp8, + ) + + def _mla_native_buffers(self): + from sglang.srt.mem_cache.device_pool_info import MLASplitBuffers + + return MLASplitBuffers( + k=self.k_buffer, + v=self.v_buffer, + index_k=self.index_k_buffer, + index_scales=self.index_k_scale_buffer, + index_layer_indices=tuple( + i - self.start_layer for i in self.indexer_layer_ids + ), + k_width=self.kv_cache_dim, + v_width=self.kr_cache_dim, + index_width=self.index_head_dim or 0, + ) + def _copy_indices_for_buffer(self, indices, uses_global_slots): if uses_global_slots or self.dcp_size <= 1: return indices diff --git a/python/sglang/srt/managers/hisparse_coordinator.py b/python/sglang/srt/managers/hisparse_coordinator.py index ecf25fb72e..e11507185c 100644 --- a/python/sglang/srt/managers/hisparse_coordinator.py +++ b/python/sglang/srt/managers/hisparse_coordinator.py @@ -601,7 +601,14 @@ class HiSparseCoordinator: "tensor-parallel size." ) self.mem_pool_host = HiSparseMHATokenToKVPoolHost( - device_pool=self.mem_pool_device, + buffer_info=self.mem_pool_device.get_device_pool_infos()[ + 0 + ].buffer_info, + device_capacity_tokens=self.mem_pool_device.size + if self.mem_pool_device.host_capacity_tokens is None + else self.mem_pool_device.host_capacity_tokens, + start_layer=self.mem_pool_device.start_layer, + end_layer=self.mem_pool_device.end_layer, host_to_device_ratio=host_to_device_ratio, page_size=self.mem_pool_device.page_size, ) @@ -609,7 +616,15 @@ class HiSparseCoordinator: else: self.mem_pool_device: HiSparseDSATokenToKVPool = kvcache self.mem_pool_host = MLATokenToKVPoolHost( - device_pool=self.mem_pool_device, + buffer_info=self.mem_pool_device.get_device_pool_infos()[ + 0 + ].buffer_info, + device_capacity_tokens=self.mem_pool_device.size + if self.mem_pool_device.host_capacity_tokens is None + else self.mem_pool_device.host_capacity_tokens, + start_layer=self.mem_pool_device.start_layer, + end_layer=self.mem_pool_device.end_layer, + target_layer_num=self.mem_pool_device.layer_num, host_to_device_ratio=host_to_device_ratio, host_size=0, page_size=self.mem_pool_device.page_size, diff --git a/python/sglang/srt/mem_cache/device_pool_info.py b/python/sglang/srt/mem_cache/device_pool_info.py index 6872c71f19..ef3e378405 100644 --- a/python/sglang/srt/mem_cache/device_pool_info.py +++ b/python/sglang/srt/mem_cache/device_pool_info.py @@ -6,6 +6,9 @@ layer routing and transfers. This first consumer describes Mamba checkpoints. from __future__ import annotations +from collections.abc import Callable +from enum import Enum + import msgspec import torch @@ -57,12 +60,166 @@ class MambaStateBufferInfo(msgspec.Struct, frozen=True, kw_only=True): ) +class KVBufferLayout(Enum): + NHD = "nhd" + PAGED_NHD = "paged_nhd" + HND = "hnd" + VECTORIZED_5D = "vectorized_5d" + + +class KVBuffers(msgspec.Struct, frozen=True, kw_only=True): + """Per-layer K/V views and their existing device pointer allocations. + + NHD views may be strided. Page/head/vector axes stay in the native layout; + this description neither flattens buffers nor enables new transfer backends. + """ + + k: tuple[torch.Tensor, ...] + v: tuple[torch.Tensor, ...] + # Some packed formats have no pointer tables; providers must not allocate them. + k_data_ptrs: torch.Tensor | None + v_data_ptrs: torch.Tensor | None + layout: KVBufferLayout + + +class KVScaleLayout(Enum): + TOKEN = "token" + MXFP8_INTERLEAVED = "mxfp8_interleaved" + + +class KVScaleBuffers(msgspec.Struct, frozen=True, kw_only=True): + k: tuple[torch.Tensor, ...] + v: tuple[torch.Tensor, ...] + layout: KVScaleLayout + + +class MHABufferInfo(msgspec.Struct, frozen=True, kw_only=True): + """Physical MHA input for host construction; no capacity or allocation policy. + + Geometry is explicit for empty stages. head_num counts physical attention + rows per allocator token, which can exceed the model's attention head count. + Existing pointer tables and tensor strides must be preserved by consumers. + """ + + buffers: KVBuffers + page_size: int + head_num: int + head_dim: int + v_head_dim: int + dtype: torch.dtype + device: torch.device + scales: KVScaleBuffers | None = None + # Resolve current contiguous backing at transfer time for relocatable pools. + transfer_buffers: Callable[[], tuple[torch.Tensor, torch.Tensor]] | None = None + + def validate(self, *, layer_ids: tuple[int, ...] | None = None) -> None: + layers = len(self.buffers.k) + if len(self.buffers.v) != layers or ( + layer_ids is not None and len(layer_ids) != layers + ): + raise ValueError( + f"MHA layers: k={layers}, v={len(self.buffers.v)}, layer_ids={layer_ids}" + ) + if min(self.page_size, self.head_num, self.head_dim, self.v_head_dim) <= 0: + raise ValueError( + f"Invalid MHA geometry: page_size={self.page_size}, " + f"head_num={self.head_num}, head_dim={self.head_dim}, " + f"v_head_dim={self.v_head_dim}" + ) + for name in ("k", "v"): + tensors = self.buffers.k if name == "k" else self.buffers.v + for layer, tensor in enumerate(tensors): + if tensor.dtype != self.dtype or tensor.device != self.device: + raise ValueError( + f"MHA {name}[{layer}]: dtype={tensor.dtype}, device={tensor.device}; " + f"expected dtype={self.dtype}, device={self.device}" + ) + if self.scales is not None and ( + len(self.scales.k) != layers or len(self.scales.v) != layers + ): + raise ValueError( + f"MHA scales: k={len(self.scales.k)}, v={len(self.scales.v)}, " + f"expected layers={layers}" + ) + + +class MLASplitBuffers(msgspec.Struct, frozen=True, kw_only=True): + """Native NPU allocations; index slots have their own pool-local layer order.""" + + k: torch.Tensor | None + v: torch.Tensor | None + index_k: torch.Tensor | None + index_scales: torch.Tensor | None + index_layer_indices: tuple[int, ...] + k_width: int + v_width: int + index_width: int + + +class MLABufferInfo(msgspec.Struct, frozen=True, kw_only=True): + """Persistent MLA rows or native split components, including empty stages. + + layer_indices address the pool's controller slots, not model layer IDs or + reserved host rows. Existing pointer allocations are views, never rebuilt. + """ + + buffers: tuple[torch.Tensor, ...] | MLASplitBuffers + data_ptrs: torch.Tensor | None + layer_indices: tuple[int, ...] + device_layer_num: int + page_size: int + kv_lora_rank: int + qk_rope_head_dim: int + kv_cache_dim: int + dtype: torch.dtype + device: torch.device + packed_fp8: bool = False + scales: tuple[torch.Tensor, ...] | None = None + native_buffers: Callable[[], MLASplitBuffers] | None = None + + def validate(self, *, layer_ids: tuple[int, ...] | None = None) -> None: + if layer_ids is not None and len(layer_ids) != len(self.layer_indices): + raise ValueError( + f"MLA model layers={layer_ids} differ from buffer slots={self.layer_indices}" + ) + if len(set(self.layer_indices)) != len(self.layer_indices) or any( + i < 0 or i >= self.device_layer_num for i in self.layer_indices + ): + raise ValueError( + f"MLA slots={self.layer_indices} outside device_layer_num={self.device_layer_num}" + ) + if isinstance(self.buffers, tuple): + if len(self.buffers) != len(self.layer_indices): + raise ValueError( + f"MLA buffers={len(self.buffers)}, slots={self.layer_indices}" + ) + for i, tensor in zip(self.layer_indices, self.buffers): + if ( + tensor.ndim != 3 + or tensor.shape[1] != 1 + or tensor.dtype != self.dtype + or tensor.device != self.device + ): + raise ValueError( + f"MLA slot={i}: shape={tuple(tensor.shape)}, dtype={tensor.dtype}, " + f"device={tensor.device}; expected [token, 1, width], " + f"dtype={self.dtype}, device={self.device}" + ) + elif self.buffers.index_k is not None and self.buffers.index_k.shape[0] != len( + self.buffers.index_layer_indices + ): + raise ValueError( + f"MLA index shape={tuple(self.buffers.index_k.shape)}, " + f"index layers={self.buffers.index_layer_indices}" + ) + + class DevicePoolInfo(msgspec.Struct, frozen=True, kw_only=True): pool_name: PoolName indices_from_pool: PoolName # Model layer IDs, including the PP stage offset, in buffer-axis order. layer_ids: tuple[int, ...] - buffer_info: MambaStateBufferInfo + buffer_info: MambaStateBufferInfo | MHABufferInfo | MLABufferInfo def __post_init__(self) -> None: # A PP stage may own no checkpoint layers. Buffer validation decides diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py index 9fbece9a17..d5526623d0 100644 --- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py +++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py @@ -37,6 +37,7 @@ from sglang.srt.mem_cache.pool_host.qsa import QSAIndexerPoolHost from sglang.srt.mem_cache.pool_host.unified import UnifiedPageEnvelopeHostPool from sglang.srt.mem_cache.unified_cache.component_type import ComponentType from sglang.srt.runtime_context import get_memory, get_parallel, get_serving +from sglang.srt.utils import is_npu if TYPE_CHECKING: import torch @@ -134,6 +135,27 @@ def _resolve_deepseek_v4_layer_mappings( ) +def _mha_host_inputs(pool, drafts=()): + info = pool.get_device_pool_infos()[0].buffer_info + return ( + get_mha_host_pool_cls(info), + info, + dict( + device_capacity_tokens=( + pool.size + if pool.host_capacity_tokens is None + else pool.host_capacity_tokens + ), + start_layer=pool.start_layer, + end_layer=pool.end_layer, + draft_buffer_infos=tuple( + draft.get_device_pool_infos()[0].buffer_info for draft in drafts + ), + write_back_staging=pool.hicache_write_back_staging, + ), + ) + + def build_kv_host_pool( *, kv_pool: Any, @@ -146,21 +168,46 @@ def build_kv_host_pool( ): if use_mla is None: use_mla = is_mla_pool(kv_pool) - kv_host_pool_cls = ( - MLATokenToKVPoolHost if use_mla else get_mha_host_pool_cls(kv_pool) - ) + if use_mla: + kv_host_pool_cls = MLATokenToKVPoolHost + host_input = kv_pool.get_device_pool_infos()[0].buffer_info + mha_kwargs = {} + else: + kv_host_pool_cls, host_input, mha_kwargs = _mha_host_inputs( + kv_pool, mtp_draft_device_pools + ) if use_mla and override_kv_cache_dim is None: # An fp8 DSA store is wider than kv_lora_rank + qk_rope_head_dim. The # device pool already resolved the row width. override_kv_cache_dim = kv_pool.kv_cache_dim - check_packed_kv_rows( - kv_pool=kv_pool, drafts=mtp_draft_device_pools, use_mla=use_mla - ) + if use_mla or is_npu(): + check_packed_kv_rows( + kv_pool=kv_pool, drafts=mtp_draft_device_pools, use_mla=use_mla + ) kwargs = {} if override_kv_cache_dim is not None: kwargs["override_kv_cache_dim"] = override_kv_cache_dim - if mtp_draft_device_pools: - kwargs["mtp_draft_device_pools"] = mtp_draft_device_pools + if use_mla: + kwargs.update( + device_capacity_tokens=kv_pool.size + if kv_pool.host_capacity_tokens is None + else kv_pool.host_capacity_tokens, + start_layer=kv_pool.start_layer, + end_layer=kv_pool.end_layer, + draft_buffer_infos=tuple( + pool.get_device_pool_infos()[0].buffer_info + for pool in mtp_draft_device_pools + ), + layer_shard_enabled=kv_pool.layer_shard_enabled, + target_layer_num=( + (kv_pool.layer_num + kv_pool.layer_shard_size - 1) + // kv_pool.layer_shard_size + if kv_pool.layer_shard_enabled + else kv_pool.layer_num + ), + ) + else: + kwargs.update(mha_kwargs) parallel = get_parallel() if parallel.dcp_enabled: assert use_mla, ( @@ -170,7 +217,7 @@ def build_kv_host_pool( kwargs["dcp_size"] = parallel.attn_dcp_size kwargs["dcp_rank"] = parallel.attn_dcp_rank return kv_host_pool_cls( - kv_pool, + host_input, get_memory().hicache_ratio, get_memory().hicache_size if host_size is None else host_size, page_size, @@ -421,14 +468,15 @@ def build_hybrid_swa_group( # from full attention. Use a separate host pool with KV indices instead of # packing their bytes into either target pool. first, *rest = mtp_swa_full_device_pools - draft_host_pool = get_mha_host_pool_cls(first)( - first, + host_cls, host_input, host_kwargs = _mha_host_inputs(first, tuple(rest)) + draft_host_pool = host_cls( + host_input, host_to_device_ratio=kv_host_pool.size / first.size, host_size=0, page_size=page_size, layout=get_memory().hicache_mem_layout, allocator_type=_get_allocator_type(), - mtp_draft_device_pools=rest, + **host_kwargs, pool_label="draft_swa", ) group.add_entry( @@ -1533,9 +1581,16 @@ def _build_mha_mla_host_pool( pool_label=pool_label, ) if isinstance(pool, MHATokenToKVPool): - return get_mha_host_pool_cls(pool)(pool, **kwargs) + host_cls, host_input, host_kwargs = _mha_host_inputs(pool) + return host_cls(host_input, **host_kwargs, **kwargs) return MLATokenToKVPoolHost( - pool, + pool.get_device_pool_infos()[0].buffer_info, + device_capacity_tokens=pool.size + if pool.host_capacity_tokens is None + else pool.host_capacity_tokens, + start_layer=pool.start_layer, + end_layer=pool.end_layer, + target_layer_num=pool.layer_num, override_kv_cache_dim=pool.kv_cache_dim, **kwargs, ) diff --git a/python/sglang/srt/mem_cache/kv_cache_builder.py b/python/sglang/srt/mem_cache/kv_cache_builder.py index 709be5b229..e796f14a5e 100644 --- a/python/sglang/srt/mem_cache/kv_cache_builder.py +++ b/python/sglang/srt/mem_cache/kv_cache_builder.py @@ -58,7 +58,7 @@ from sglang.srt.runtime_context import ( get_schedule, ) from sglang.srt.speculative.base_spec_worker import HiCacheDraftMode -from sglang.srt.utils import ceil_align, is_hip +from sglang.srt.utils import ceil_align, is_hip, is_npu if TYPE_CHECKING: from sglang.srt.configs.model_config import ModelConfig @@ -72,6 +72,8 @@ def prepare_hicache_staging( *, tp_worker: BaseTpWorker, draft_plan: Optional[HiCacheDraftPlan] = None ) -> None: """Materialize MHA transfer buffers before the final KV budget is measured.""" + if is_npu(): + return memory = get_memory() page_size = get_schedule().page_size if memory.hicache_mem_layout != "page_first" or not ( @@ -93,11 +95,13 @@ def prepare_hicache_staging( if memory.hicache_size <= 0 and not sidecar: page_capacity = int(pool.size * memory.hicache_ratio) // page_size + 1 staging = prepare_mha_write_back_staging( - pool, + pool.get_device_pool_infos()[0].buffer_info, layer_num=pool.layer_num + len(packed_drafts), page_size=page_size, page_capacity=page_capacity, + staging=pool.hicache_write_back_staging, ) + pool.hicache_write_back_staging = staging if staging is not None: logger.info( "HiCache staging prepared before KV sizing: %.1f MiB, %d layers", diff --git a/python/sglang/srt/mem_cache/memory_pool.py b/python/sglang/srt/mem_cache/memory_pool.py index 9568abec7c..5f519dd6ba 100644 --- a/python/sglang/srt/mem_cache/memory_pool.py +++ b/python/sglang/srt/mem_cache/memory_pool.py @@ -100,7 +100,7 @@ from sglang.srt.utils.torch_memory_saver_adapter import TorchMemorySaverAdapter if TYPE_CHECKING: from sglang.srt.managers.cache_controller import LayerDoneCounter from sglang.srt.managers.schedule_batch import Req - from sglang.srt.mem_cache.device_pool_info import DevicePoolInfo + from sglang.srt.mem_cache.device_pool_info import DevicePoolInfo, MHABufferInfo logger = logging.getLogger(__name__) @@ -2075,6 +2075,14 @@ class KVCache(abc.ABC): maybe_init_custom_mem_pool(device=self.device) ) + def get_device_pool_infos(self) -> tuple[DevicePoolInfo, ...]: + """Persistent buffers for migrated host consumers; see device_pool_info.py. + + Unmigrated formats retain their existing construction entry points. + Consumers must not use exceptions or empty descriptions to pick a path. + """ + raise NotImplementedError(f"{type(self).__name__} has no device pool provider") + def host_pool_decls(self): """Host pools HiCache keeps for the buffers this device pool (and its sub-pools) owns: the KV pool itself, plus dependent pools such as sparse @@ -2225,6 +2233,11 @@ class MHATokenToKVPool(KVCache): ): self.k_buffer = None self.v_buffer = None + if not _is_npu: + self.k_data_ptrs = None + self.v_data_ptrs = None + self.k_scale_buffer = None + self.v_scale_buffer = None if post_capture_active: # Reserved upper bound only (unbacked VA): page-align UP so # (size + page_size) % page_size == 0 holds for paged layouts. @@ -2324,6 +2337,64 @@ class MHATokenToKVPool(KVCache): # for store_cache JIT kernel self.row_dim = self.head_num * self.head_dim self.v_row_dim = self.head_num * self.v_head_dim + self.model_layer_ids = tuple( + range(self.start_layer, self.start_layer + self.layer_num) + ) + + def get_device_pool_infos(self) -> tuple[DevicePoolInfo, ...]: + """Describe persistent KV buffers for host assembly without allocating.""" + from sglang.srt.mem_cache.device_pool_info import DevicePoolInfo + from sglang.srt.mem_cache.hicache_storage import PoolName + + return ( + DevicePoolInfo( + pool_name=PoolName.KV, + indices_from_pool=PoolName.KV, + layer_ids=self.model_layer_ids, + buffer_info=self._mha_buffer_info(), + ), + ) + + def _mha_buffers(self): + from sglang.srt.mem_cache.device_pool_info import KVBufferLayout, KVBuffers + + return KVBuffers( + k=tuple(self.k_buffer), + v=tuple(self.v_buffer), + k_data_ptrs=self.k_data_ptrs, + v_data_ptrs=self.v_data_ptrs, + layout=KVBufferLayout.NHD + if self.kv_cache_layout == "page_major" + else KVBufferLayout(self.kv_cache_layout), + ) + + def _mha_scale_buffers(self): + from sglang.srt.mem_cache.device_pool_info import KVScaleBuffers, KVScaleLayout + + if self.k_scale_buffer is None: + return None + return KVScaleBuffers( + k=tuple(self.k_scale_buffer), + v=tuple(self.v_scale_buffer), + layout=KVScaleLayout.TOKEN, + ) + + def _mha_buffer_info(self) -> MHABufferInfo: + from sglang.srt.mem_cache.device_pool_info import MHABufferInfo + + buffers = self._mha_buffers() + return MHABufferInfo( + buffers=buffers, + page_size=self.page_size, + head_num=self.row_dim // self.head_dim + if self.head_dim == self.v_head_dim + else self.head_num, + head_dim=self.head_dim, + v_head_dim=self.v_head_dim, + dtype=self.store_dtype, + device=buffers.k[0].device if len(buffers.k) else torch.device(self.device), + scales=self._mha_scale_buffers(), + ) def _init_kv_copy_and_warmup(self): # Zero-layer pool (e.g. all-SWA model's full sub-pool) has no buffers. @@ -3789,6 +3860,17 @@ class MHATokenToKVPoolMXFP8(MHATokenToKVPool): MXFP8_SCALE_BLOCK_SIZE = 32 + def _mha_scale_buffers(self): + from sglang.srt.mem_cache.device_pool_info import KVScaleBuffers, KVScaleLayout + + return KVScaleBuffers( + k=tuple(self.k_scale_buffer), + v=tuple(self.v_scale_buffer), + layout=KVScaleLayout.MXFP8_INTERLEAVED + if self.mxfp8_sf_interleaved + else KVScaleLayout.TOKEN, + ) + def _create_buffers(self): with self.memory_saver_adapter.region(GPU_MEMORY_TYPE_KV_CACHE): with ( @@ -4272,6 +4354,9 @@ class HybridLinearKVPool(KVCache): qk_rope_head_dim=qk_rope_head_dim, enable_memory_saver=enable_memory_saver, ) + if isinstance(self.full_kv_pool, MHATokenToKVPool): + # Only the hybrid root knows the model IDs of its compact child. + self.full_kv_pool.model_layer_ids = tuple(full_attention_layer_ids) self.full_attention_layer_id_mapping = { id: i for i, id in enumerate(full_attention_layer_ids) } @@ -4716,6 +4801,11 @@ class MLATokenToKVPool(KVCache): else (kv_lora_rank + qk_rope_head_dim) ) + self.model_layer_ids = tuple( + range(self.start_layer, self.start_layer + self.layer_num) + ) + self.kv_scale_buffer = None + self._create_buffers() self.data_ptrs = torch.tensor( @@ -4727,6 +4817,47 @@ class MLATokenToKVPool(KVCache): # DSA will allocate indexer KV cache later and then log the total size self._finalize_allocation_log(size) + def get_device_pool_infos(self) -> tuple[DevicePoolInfo, ...]: + from sglang.srt.mem_cache.device_pool_info import DevicePoolInfo + from sglang.srt.mem_cache.hicache_storage import PoolName + + info = self._mla_buffer_info() + return ( + DevicePoolInfo( + pool_name=PoolName.KV, + indices_from_pool=PoolName.KV, + layer_ids=tuple(self.model_layer_ids[i] for i in info.layer_indices), + buffer_info=info, + ), + ) + + def _mla_buffer_info(self): + from sglang.srt.mem_cache.device_pool_info import MLABufferInfo + + start, end = ( + self._owned_local_layer_range() + if self.layer_shard_enabled + else (0, self.layer_num) + ) + return MLABufferInfo( + buffers=tuple(self.kv_buffer[start:end]), + data_ptrs=self.data_ptrs + if (start, end) == (0, self.layer_num) + else self.data_ptrs[start:end], + layer_indices=tuple(range(start, end)), + device_layer_num=self.layer_num, + page_size=self.page_size, + kv_lora_rank=self.kv_lora_rank, + qk_rope_head_dim=self.qk_rope_head_dim, + kv_cache_dim=self.kv_cache_dim, + dtype=self.store_dtype, + device=self.data_ptrs.device, + packed_fp8=self.dsa_kv_cache_store_fp8, + scales=None + if self.kv_scale_buffer is None + else tuple(self.kv_scale_buffer[start:end]), + ) + def _create_buffers(self): with self.memory_saver_adapter.region(GPU_MEMORY_TYPE_KV_CACHE): with ( @@ -5858,6 +5989,13 @@ class MiniMaxSparseKVPool(KVCache): else None ) + if enable_hisparse: + self.main_pool.model_layer_ids = tuple(local_sparse_layer_ids) + if self.dense_pool is not None: + self.dense_pool.model_layer_ids = tuple(local_dense_layer_ids) + if self.index_kv_pool is not None: + self.index_kv_pool.model_layer_ids = tuple(local_kv_sparse_layer_ids) + self.mem_usage = self.main_pool.mem_usage if self.dense_pool is not None: self.mem_usage += self.dense_pool.mem_usage diff --git a/python/sglang/srt/mem_cache/pool_host/mha.py b/python/sglang/srt/mem_cache/pool_host/mha.py index f4a0e40d9a..2af000962d 100644 --- a/python/sglang/srt/mem_cache/pool_host/mha.py +++ b/python/sglang/srt/mem_cache/pool_host/mha.py @@ -28,10 +28,12 @@ from sglang.kernels.ops.kvcache.hicache import ( from sglang.kernels.ops.kvcache.hicache import ( transfer_hicache_one_layer_mla as jit_transfer_hicache_one_layer_mla, ) +from sglang.srt.mem_cache.device_pool_info import ( + KVScaleLayout, + MHABufferInfo, +) from sglang.srt.mem_cache.memory_pool import ( MHATokenToKOnlyPool, - MHATokenToKVPool, - MHATokenToKVPoolMXFP8, ) from sglang.srt.mem_cache.pool_host.base import ( _WRITE_BACK_STAGING_PAGE_CHUNK, @@ -76,46 +78,37 @@ logger = logging.getLogger(__name__) def prepare_mha_write_back_staging( - device_pool: MHATokenToKVPool, + buffer_info: MHABufferInfo, *, layer_num: int, page_size: int, page_capacity: int = _WRITE_BACK_STAGING_PAGE_CHUNK, - retain: bool = True, + staging: tuple[torch.Tensor, torch.Tensor] | None = None, ) -> tuple[torch.Tensor, torch.Tensor] | None: if not (_is_cuda or _is_hip) or layer_num == 0: return None - head_num = device_pool.head_num - if device_pool.head_dim == device_pool.v_head_dim: - head_num = device_pool.row_dim // device_pool.head_dim + head_num = buffer_info.head_num if not all( can_use_write_back_jit_kernel( - element_size=head_num * dim * device_pool.store_dtype.itemsize + element_size=head_num * dim * buffer_info.dtype.itemsize ) - for dim in (device_pool.head_dim, device_pool.v_head_dim) + for dim in (buffer_info.head_dim, buffer_info.v_head_dim) ): return None token_capacity = min(page_capacity, _WRITE_BACK_STAGING_PAGE_CHUNK) * page_size shapes = tuple( (token_capacity, layer_num, head_num, dim) - for dim in (device_pool.head_dim, device_pool.v_head_dim) + for dim in (buffer_info.head_dim, buffer_info.v_head_dim) ) - staging = device_pool.hicache_write_back_staging if staging is None: staging = ( - torch.empty( - shapes[0], dtype=device_pool.store_dtype, device=device_pool.device - ), - torch.empty( - shapes[1], dtype=device_pool.store_dtype, device=device_pool.device - ), + torch.empty(shapes[0], dtype=buffer_info.dtype, device=buffer_info.device), + torch.empty(shapes[1], dtype=buffer_info.dtype, device=buffer_info.device), ) - if retain: - device_pool.hicache_write_back_staging = staging else: for buffer, shape in zip(staging, shapes): if ( - buffer.dtype != device_pool.store_dtype + buffer.dtype != buffer_info.dtype or tuple(buffer.shape[1:]) != shape[1:] or buffer.shape[0] < shape[0] ): @@ -124,14 +117,9 @@ def prepare_mha_write_back_staging( class MHATokenToKVPoolHost(HostKVCache): - device_pool: MHATokenToKVPool - mtp_draft_device_pools: tuple[MHATokenToKVPool, ...] = () - - _transfer_device: torch.device - def __init__( self, - device_pool: MHATokenToKVPool, + buffer_info: MHABufferInfo, host_to_device_ratio: float, host_size: int, page_size: int, @@ -140,41 +128,70 @@ class MHATokenToKVPoolHost(HostKVCache): device: str = "cpu", allocator_type: str = "default", *, - mtp_draft_device_pools: Sequence[MHATokenToKVPool] = (), + device_capacity_tokens: int, + start_layer: int, + end_layer: int, + draft_buffer_infos: Sequence[MHABufferInfo] = (), + write_back_staging: tuple[torch.Tensor, torch.Tensor] | None = None, pool_label: str = "kv", head_group_num: int = 1, ): - self.mtp_draft_device_pools = tuple(mtp_draft_device_pools) - self.target_layer_num = device_pool.layer_num - # page_unified cuts the kv-head axis into this many groups, and the cut - # is part of the byte order, so it has to be known before the buffer is - # allocated in init_kv_buffer() below. + infos = (buffer_info, *draft_buffer_infos) + for info in infos: + if ( + info.scales is not None + and info.scales.layout != KVScaleLayout.MXFP8_INTERLEAVED + ): + raise ValueError( + f"{pool_label}: MHA host transfers do not support " + f"KV scale layout {info.scales.layout.value!r}." + ) + info.validate() + if info.transfer_buffers is None and ( + info.buffers.k_data_ptrs is None or info.buffers.v_data_ptrs is None + ): + raise ValueError( + f"{pool_label}: MHA host requires existing K/V pointer tables" + ) + for depth, draft in enumerate(draft_buffer_infos): + if len(draft.buffers.k) != 1: + raise ValueError( + f"{pool_label}: packed draft[{depth}] has {len(draft.buffers.k)} layers; expected 1" + ) + target_geometry = ( + buffer_info.head_num, + buffer_info.head_dim, + buffer_info.v_head_dim, + buffer_info.dtype, + buffer_info.device, + ) + draft_geometry = ( + draft.head_num, + draft.head_dim, + draft.v_head_dim, + draft.dtype, + draft.device, + ) + if draft_geometry != target_geometry: + raise ValueError( + f"{pool_label}: packed draft[{depth}] (heads, K width, V width, dtype, device)={draft_geometry}, expected {target_geometry}" + ) + self.target_layer_num = len(buffer_info.buffers.k) + self._transfer_device = buffer_info.device + self.dtype = buffer_info.dtype + self.head_dim = buffer_info.head_dim + self.v_head_dim = buffer_info.v_head_dim + self.head_num = buffer_info.head_num + self.layer_num = self.target_layer_num + len(draft_buffer_infos) + self._buffer_infos = infos self.head_group_num = head_group_num - # Transfer buffers are captured below. The pool remains available for - # NPU transfer APIs and device layer ownership. - self.device_pool = device_pool - self._transfer_device = torch.device(device_pool.device) - self.dtype = device_pool.store_dtype - self.head_dim = device_pool.head_dim - self.v_head_dim = device_pool.v_head_dim - # Equal-width pools may store multiple attention rows per allocator token. - self.head_num = ( - device_pool.row_dim // self.head_dim - if self.head_dim == self.v_head_dim - else device_pool.head_num - ) - self.layer_num = self.target_layer_num + len(self.mtp_draft_device_pools) - # Scale-bearing subclasses need page coverage when computing token bytes. self.page_size = page_size - device_capacity = getattr(device_pool, "host_capacity_tokens", None) super().__init__( dtype=self.dtype, - device_capacity_tokens=( - device_pool.size if device_capacity is None else device_capacity - ), + device_capacity_tokens=device_capacity_tokens, size_per_token=self.get_size_per_token(), - start_layer=device_pool.start_layer, - end_layer=device_pool.end_layer, + start_layer=start_layer, + end_layer=end_layer, host_to_device_ratio=host_to_device_ratio, host_size=host_size, page_size=page_size, @@ -184,12 +201,22 @@ class MHATokenToKVPoolHost(HostKVCache): allocator_type=allocator_type, pool_label=pool_label, ) - pools = (self.device_pool, *self.mtp_draft_device_pools) - self.device_k_buffers = tuple(b for pool in pools for b in pool.k_buffer) - self.device_v_buffers = tuple(b for pool in pools for b in pool.v_buffer) - self._initialize_transfer_views() + self.device_k_buffers = tuple(b for info in infos for b in info.buffers.k) + self.device_v_buffers = tuple(b for info in infos for b in info.buffers.v) + if buffer_info.transfer_buffers is None: + self.device_k_data_ptrs = ( + torch.cat([info.buffers.k_data_ptrs for info in infos]) + if draft_buffer_infos + else buffer_info.buffers.k_data_ptrs + ) + self.device_v_data_ptrs = ( + torch.cat([info.buffers.v_data_ptrs for info in infos]) + if draft_buffer_infos + else buffer_info.buffers.v_data_ptrs + ) + self._initialize_transfer_views(buffer_info, write_back_staging) - def _initialize_transfer_views(self) -> None: + def _initialize_transfer_views(self, buffer_info, write_back_staging) -> None: self.element_dim = self.head_num * self.head_dim self._init_device_row_stride((*self.device_k_buffers, *self.device_v_buffers)) # The JIT HiCache kernels also build with hipcc (ROCm): the PTX-only @@ -235,20 +262,8 @@ class MHATokenToKVPoolHost(HostKVCache): if self.v_data_refs else None ) - if not _is_npu: - pools = (self.device_pool, *self.mtp_draft_device_pools) - self.device_k_data_ptrs = ( - torch.cat([pool.k_data_ptrs for pool in pools]) - if self.mtp_draft_device_pools - else self.device_pool.k_data_ptrs - ) - self.device_v_data_ptrs = ( - torch.cat([pool.v_data_ptrs for pool in pools]) - if self.mtp_draft_device_pools - else self.device_pool.v_data_ptrs - ) self.host_kv_data_refs = self.k_data_refs + self.v_data_refs - self._init_write_back_staging_buffers() + self._init_write_back_staging_buffers(buffer_info, write_back_staging) def get_size_per_token(self): return ( @@ -322,7 +337,7 @@ class MHATokenToKVPoolHost(HostKVCache): ) return buffer - def _init_write_back_staging_buffers(self): + def _init_write_back_staging_buffers(self, buffer_info, write_back_staging): self.staging_page_capacity = 0 self.staging_token_capacity = 0 self.staging_k_buffer = None @@ -337,11 +352,11 @@ class MHATokenToKVPoolHost(HostKVCache): return page_capacity = min(self.page_num, _WRITE_BACK_STAGING_PAGE_CHUNK) staging = prepare_mha_write_back_staging( - self.device_pool, + buffer_info, layer_num=self.layer_num, page_size=self.page_size, page_capacity=page_capacity, - retain=False, + staging=write_back_staging, ) if staging is None: return @@ -370,10 +385,9 @@ class MHATokenToKVPoolHost(HostKVCache): *, is_draft: bool = False, ): - if not is_draft and not self._is_device_layer_owned(device_pool, layer_id): + if not is_draft and not 0 <= layer_id < self.target_layer_num: return - # MTP draft layers do not participate in CP layer sharding. - host_layer_id = layer_id if is_draft else self._host_layer_index(layer_id) + host_layer_id = layer_id if io_backend != "kernel_ascend": device_k = self.device_k_buffers[layer_id] @@ -487,7 +501,7 @@ class MHATokenToKVPoolHost(HostKVCache): device_v, host_k, host_v, - ) in self._npu_transfer_buffers(device_pool): + ) in self._npu_transfer_buffers(): transfer_kv_dim_exchange( device_indices=device_indices, host_indices=host_indices, @@ -503,10 +517,10 @@ class MHATokenToKVPoolHost(HostKVCache): else: raise ValueError(f"Unsupported IO backend: {io_backend}") - def _npu_transfer_buffers(self, target_device_pool): + def _npu_transfer_buffers(self): layer_start = 0 - for pool in (target_device_pool, *self.mtp_draft_device_pools): - device_k, device_v = pool.get_hicache_transfer_buffers() + for info in self._buffer_infos: + device_k, device_v = info.transfer_buffers() layer_end = layer_start + device_k.shape[0] yield ( device_k, @@ -634,9 +648,7 @@ class MHATokenToKVPoolHost(HostKVCache): raise ValueError(f"Unsupported layout: {self.layout}") elif io_backend == "kernel_ascend": if self.layout == "page_first_direct": - for device_k, device_v, host_k, host_v in self._npu_transfer_buffers( - device_pool - ): + for device_k, device_v, host_k, host_v in self._npu_transfer_buffers(): transfer_kv_dim_exchange( device_indices=device_indices, host_indices=host_indices, @@ -1185,16 +1197,23 @@ class HiSparseMHATokenToKVPoolHost(HiSparseHostPoolMixin, MHATokenToKVPoolHost): def __init__( self, - device_pool: MHATokenToKVPool, + buffer_info: MHABufferInfo, host_to_device_ratio: float, page_size: int, + *, + device_capacity_tokens: int, + start_layer: int, + end_layer: int, ): super().__init__( - device_pool=device_pool, + buffer_info=buffer_info, host_to_device_ratio=host_to_device_ratio, host_size=0, page_size=page_size, layout="layer_first", + device_capacity_tokens=device_capacity_tokens, + start_layer=start_layer, + end_layer=end_layer, ) def get_contiguous_buf_infos(self): @@ -1343,10 +1362,9 @@ class AsymmetricMHATokenToKVPoolHost(MHATokenToKVPoolHost): *, is_draft: bool = False, ): - if not is_draft and not self._is_device_layer_owned(device_pool, layer_id): + if not is_draft and not 0 <= layer_id < self.target_layer_num: return - # MTP draft layers do not participate in CP layer sharding. - host_layer_id = layer_id if is_draft else self._host_layer_index(layer_id) + host_layer_id = layer_id if io_backend != "kernel_ascend": device_k = self.device_k_buffers[layer_id] @@ -1574,25 +1592,19 @@ class AsymmetricMHATokenToKVPoolHost(MHATokenToKVPoolHost): ) -def get_mha_host_pool_cls(device_pool: MHATokenToKVPool) -> type: - """Pick the right MHA host-pool class based on the device pool's K/V dims. - - Returns ``MHATokenToKVPoolMXFP8Host`` for the block-scaled MXFP8 pool (its - UE8M0 scales must travel with the payload), - ``AsymmetricMHATokenToKVPoolHost`` when ``head_dim != v_head_dim`` - (e.g. MiMo-V2), else the default ``MHATokenToKVPoolHost``. - """ - if isinstance(device_pool, MHATokenToKVPoolMXFP8): - if device_pool.head_dim != device_pool.v_head_dim: +def get_mha_host_pool_cls(buffer_info: MHABufferInfo) -> type: + is_mxfp8 = ( + buffer_info.scales is not None + and buffer_info.scales.layout == KVScaleLayout.MXFP8_INTERLEAVED + ) + if is_mxfp8: + if buffer_info.head_dim != buffer_info.v_head_dim: raise NotImplementedError( "MXFP8 HiCache does not support asymmetric K/V head dimensions yet." ) - - from sglang.srt.mem_cache.pool_host.mha_mxfp8 import ( - MHATokenToKVPoolMXFP8Host, - ) + from sglang.srt.mem_cache.pool_host.mha_mxfp8 import MHATokenToKVPoolMXFP8Host return MHATokenToKVPoolMXFP8Host - if device_pool.head_dim != device_pool.v_head_dim: + if buffer_info.head_dim != buffer_info.v_head_dim: return AsymmetricMHATokenToKVPoolHost return MHATokenToKVPoolHost diff --git a/python/sglang/srt/mem_cache/pool_host/mha_mxfp8.py b/python/sglang/srt/mem_cache/pool_host/mha_mxfp8.py index 1c4c0933ce..804a5ea246 100644 --- a/python/sglang/srt/mem_cache/pool_host/mha_mxfp8.py +++ b/python/sglang/srt/mem_cache/pool_host/mha_mxfp8.py @@ -23,7 +23,7 @@ from sglang.kernels.ops.kvcache.hicache import ( from sglang.kernels.ops.kvcache.hicache import ( transfer_hicache_one_layer_mla as jit_transfer_hicache_one_layer_mla, ) -from sglang.srt.mem_cache.memory_pool import MHATokenToKVPoolMXFP8 +from sglang.srt.mem_cache.device_pool_info import KVScaleLayout, MHABufferInfo from sglang.srt.mem_cache.pool_host.common import ( ALLOC_MEMORY_FUNCS, _cuda_host_unregister, @@ -36,11 +36,9 @@ from sglang.srt.mem_cache.pool_host.mha import ( class MHATokenToKVPoolMXFP8Host(MHATokenToKVPoolHost): - device_pool: MHATokenToKVPoolMXFP8 - def __init__( self, - device_pool: MHATokenToKVPoolMXFP8, + buffer_info: MHABufferInfo, host_to_device_ratio: float, host_size: int, page_size: int, @@ -49,29 +47,38 @@ class MHATokenToKVPoolMXFP8Host(MHATokenToKVPoolHost): device: str = "cpu", allocator_type: str = "default", *, - mtp_draft_device_pools: Sequence = (), + device_capacity_tokens: int, + start_layer: int, + end_layer: int, + draft_buffer_infos: Sequence[MHABufferInfo] = (), + write_back_staging: tuple[torch.Tensor, torch.Tensor] | None = None, pool_label: str = "kv", ): if layout != "page_first": raise NotImplementedError( f"MXFP8 KV host pool supports only the page_first layout, got {layout!r}." ) - if mtp_draft_device_pools: + if draft_buffer_infos: raise NotImplementedError( "MXFP8 KV host pool does not pack MTP draft KV layers." ) - if not device_pool.mxfp8_sf_interleaved: + interleaved = ( + buffer_info.scales is not None + and buffer_info.scales.layout == KVScaleLayout.MXFP8_INTERLEAVED + ) + if not interleaved: raise NotImplementedError( "MXFP8 KV host pool requires the interleaved (page_size=128) scale layout." ) - # Bytes of UE8M0 scales one page holds per layer; needed by - # get_size_per_token before the base constructor sizes the pool. - self.k_sf_page_bytes = device_pool.k_scale_buffer[0][0].numel() - self.v_sf_page_bytes = device_pool.v_scale_buffer[0][0].numel() + k_scales = buffer_info.scales.k + v_scales = buffer_info.scales.v + # HostKVCache calls get_size_per_token during base initialization. + self.k_sf_page_bytes = k_scales[0][0].numel() + self.v_sf_page_bytes = v_scales[0][0].numel() self.k_scale_host: torch.Tensor | None = None self.v_scale_host: torch.Tensor | None = None super().__init__( - device_pool, + buffer_info, host_to_device_ratio, host_size, page_size, @@ -79,13 +86,17 @@ class MHATokenToKVPoolMXFP8Host(MHATokenToKVPoolHost): pin_memory, device, allocator_type, + device_capacity_tokens=device_capacity_tokens, + start_layer=start_layer, + end_layer=end_layer, + write_back_staging=write_back_staging, pool_label=pool_label, ) - if self.page_size != device_pool.page_size: + if self.page_size != buffer_info.page_size: raise ValueError( "MXFP8 KV host pool moves scales per page, so the host page size " f"({self.page_size}) must equal the device page size " - f"({device_pool.page_size})." + f"({buffer_info.page_size})." ) if not self.can_use_write_back_jit or not all( can_use_write_back_jit_kernel(element_size=size) @@ -95,15 +106,15 @@ class MHATokenToKVPoolMXFP8Host(MHATokenToKVPoolHost): "MXFP8 KV host pool needs the staged JIT write-back kernel " "(io_backend='kernel', page_first layout, CUDA or HIP)." ) - self._init_scale_buffers() + self._init_scale_buffers(k_scales, v_scales) def get_size_per_token(self): payload = super().get_size_per_token() scales = (self.k_sf_page_bytes + self.v_sf_page_bytes) // self.page_size return payload + scales * self.layer_num - def _init_scale_buffers(self): - alloc_func = ALLOC_MEMORY_FUNCS[self.device_pool.device] + def _init_scale_buffers(self, k_scales, v_scales): + alloc_func = ALLOC_MEMORY_FUNCS[self._transfer_device.type] # Host: page-first like the payload, one contiguous scale block per # (page, layer) so a page's layers are a single memcpy span. self.k_scale_host = alloc_func( @@ -126,32 +137,30 @@ class MHATokenToKVPoolMXFP8Host(MHATokenToKVPoolHost): # Device: each layer's interleaved scale tensor as [page, bytes] rows. self.k_scale_device_rows = [ - buf.view(torch.uint8).reshape(buf.shape[0], -1) - for buf in self.device_pool.k_scale_buffer + buf.view(torch.uint8).reshape(buf.shape[0], -1) for buf in k_scales ] self.v_scale_device_rows = [ - buf.view(torch.uint8).reshape(buf.shape[0], -1) - for buf in self.device_pool.v_scale_buffer + buf.view(torch.uint8).reshape(buf.shape[0], -1) for buf in v_scales ] self.k_scale_device_ptrs = torch.tensor( [rows.data_ptr() for rows in self.k_scale_device_rows], dtype=torch.uint64, - device=self.device_pool.device, + device=self._transfer_device, ) self.v_scale_device_ptrs = torch.tensor( [rows.data_ptr() for rows in self.v_scale_device_rows], dtype=torch.uint64, - device=self.device_pool.device, + device=self._transfer_device, ) self.k_scale_staging = torch.empty( (self.staging_page_capacity, self.layer_num, self.k_sf_page_bytes), dtype=torch.uint8, - device=self.device_pool.device, + device=self._transfer_device, ) self.v_scale_staging = torch.empty( (self.staging_page_capacity, self.layer_num, self.v_sf_page_bytes), dtype=torch.uint8, - device=self.device_pool.device, + device=self._transfer_device, ) def _page_ids(self, indices: torch.Tensor) -> torch.Tensor: @@ -209,9 +218,9 @@ class MHATokenToKVPoolMXFP8Host(MHATokenToKVPoolHost): raise NotImplementedError( f"MXFP8 KV host pool supports only io_backend='kernel', got {io_backend!r}." ) - if not self._is_device_layer_owned(device_pool, layer_id): + if not 0 <= layer_id < self.target_layer_num: return - host_layer_id = self._host_layer_index(layer_id) + host_layer_id = layer_id device_pages = self._page_ids(device_indices) host_pages = self._page_ids(host_indices) for dst_rows, src_rows, sf_page_bytes in ( diff --git a/python/sglang/srt/mem_cache/pool_host/mla.py b/python/sglang/srt/mem_cache/pool_host/mla.py index f2ecfe9f38..9480cddabd 100644 --- a/python/sglang/srt/mem_cache/pool_host/mla.py +++ b/python/sglang/srt/mem_cache/pool_host/mla.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging from typing import Optional, Sequence +import msgspec import torch from sglang.kernels.ops.kvcache.hicache import ( @@ -19,7 +20,7 @@ from sglang.kernels.ops.kvcache.hicache import ( transfer_hicache_one_layer_mla as jit_transfer_hicache_one_layer_mla, ) from sglang.srt.layers.dcp.layout import maybe_dcp_kernel_indices -from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool +from sglang.srt.mem_cache.device_pool_info import MLABufferInfo, MLASplitBuffers from sglang.srt.mem_cache.pool_host.base import ( _WRITE_BACK_STAGING_PAGE_CHUNK, HostKVCache, @@ -62,15 +63,28 @@ if _is_npu: logger = logging.getLogger(__name__) -class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): - device_pool: MLATokenToKVPool - mtp_draft_device_pools: tuple[MLATokenToKVPool, ...] = () +class MLAHostAllocation(msgspec.Struct, frozen=True, kw_only=True): + """Allocator-only geometry; it carries no device buffers or transfer promise.""" - _transfer_device: torch.device + dtype: torch.dtype + kv_lora_rank: int + qk_rope_head_dim: int + kv_cache_dim: int + size_per_token: int + draft_layer_num: int = 0 + packed_fp8: bool = False + + +class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): + """Host mirror built by the assembler or HiSparse from captured MLA buffers. + + Capacity and reserved CP rows remain caller policy. Transfer methods retain + the shared HostKVCache signature but use the captured tensors and slot map. + """ def __init__( self, - device_pool: MLATokenToKVPool, + buffer_info: MLABufferInfo | MLAHostAllocation, host_to_device_ratio: float, host_size: int, page_size: int, @@ -79,46 +93,117 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): device: str = "cpu", allocator_type: str = "default", override_kv_cache_dim: Optional[int] = None, - mtp_draft_device_pools: Sequence[MLATokenToKVPool] = (), + draft_buffer_infos: Sequence[MLABufferInfo] = (), dcp_size: int = 1, dcp_rank: int = 0, *, + device_capacity_tokens: int, + start_layer: int, + end_layer: int, + target_layer_num: int, + layer_shard_enabled: bool = False, pool_label: str = "kv", - is_dummy: bool = False, ): + self._is_dummy = isinstance(buffer_info, MLAHostAllocation) self.override_kv_cache_dim = override_kv_cache_dim - self.mtp_draft_device_pools = tuple(mtp_draft_device_pools) - self._is_dummy = is_dummy - - # Transfer buffers are captured below. The pool remains available for - # NPU transfer APIs and device layer ownership. - self.device_pool = device_pool - if not is_dummy: - self._transfer_device = torch.device(device_pool.device) - self.dtype = device_pool.store_dtype + self.dtype = buffer_info.dtype self.layout = layout - self.kv_lora_rank = device_pool.kv_lora_rank - self.qk_rope_head_dim = device_pool.qk_rope_head_dim - # L3 excludes the dead V component when the device stores packed FP8 KV. - self.dsa_kv_cache_store_fp8 = getattr( - device_pool, "dsa_kv_cache_store_fp8", False - ) - self.target_layer_num = self._effective_host_layer_num() - self.layer_num = self.target_layer_num + len(self.mtp_draft_device_pools) + self.kv_lora_rank = buffer_info.kv_lora_rank + self.qk_rope_head_dim = buffer_info.qk_rope_head_dim + # Existing builders explicitly resolve packed row width before construction. self.kv_cache_dim = override_kv_cache_dim or ( - self.kv_lora_rank + self.qk_rope_head_dim + buffer_info.kv_cache_dim + if self._is_dummy + else self.kv_lora_rank + self.qk_rope_head_dim ) - device_capacity = getattr(device_pool, "host_capacity_tokens", None) - if is_dummy: - device_capacity = device_capacity or device_pool.size + self.dsa_kv_cache_store_fp8 = buffer_info.packed_fp8 + self.target_layer_num = target_layer_num + self.layer_num = target_layer_num + ( + buffer_info.draft_layer_num if self._is_dummy else len(draft_buffer_infos) + ) + self._layer_shard_enabled = layer_shard_enabled + self._native_buffers = None + if self._is_dummy: + if draft_buffer_infos: + raise ValueError( + "Allocator-only MLA does not accept physical draft buffers" + ) + self._size_per_token = buffer_info.size_per_token + else: + infos = (buffer_info, *draft_buffer_infos) + for index, info in enumerate(infos): + info.validate() + if info.scales is not None: + role = "target" if index == 0 else f"draft[{index - 1}]" + raise ValueError( + f"MLA host cache does not support {role} FP4 KV: " + "packed payload and separate block scales cannot be restored" + ) + if index and ( + (info.kv_cache_dim, info.dtype) + != (buffer_info.kv_cache_dim, buffer_info.dtype) + or type(info.buffers) is not type(buffer_info.buffers) + ): + raise ValueError( + f"packed draft KV row {(info.kv_cache_dim, info.dtype)} differs " + f"from target KV row {(buffer_info.kv_cache_dim, buffer_info.dtype)}" + ) + if index and len(info.layer_indices) != 1: + raise ValueError( + f"MLA packed draft[{index - 1}] must contain one layer, got {info.layer_indices}" + ) + self._transfer_device = buffer_info.device + self._device_layer_num = buffer_info.device_layer_num + self._target_layer_indices = buffer_info.layer_indices + self._draft_layer_num = len(draft_buffer_infos) + slots = ( + *buffer_info.layer_indices, + *range( + self._device_layer_num, + self._device_layer_num + self._draft_layer_num, + ), + ) + self._buffer_index_by_layer = {layer: i for i, layer in enumerate(slots)} + self._host_layer_by_controller = dict( + zip(buffer_info.layer_indices, range(len(buffer_info.layer_indices))) + ) + self._host_layer_by_controller.update( + { + self._device_layer_num + i: target_layer_num + i + for i in range(self._draft_layer_num) + } + ) + self._size_per_token = ( + self.kv_cache_dim * self.dtype.itemsize * self.layer_num + ) + if isinstance(buffer_info.buffers, MLASplitBuffers): + self._native_buffers = buffer_info.buffers + self._native_infos = infos + native = self._native_buffers + index_layers = len(native.index_layer_indices) + if layout == "page_first_kv_split" and native.index_k is not None: + self._size_per_token += ( + native.index_width * self.dtype.itemsize * index_layers + ) + if native.index_scales is not None: + self._size_per_token += 4 * index_layers + self.device_kv_buffers = () + self.device_data_ptrs = None + else: + self.device_kv_buffers = tuple( + tensor for info in infos for tensor in info.buffers + ) + self.device_data_ptrs = ( + torch.cat([info.data_ptrs for info in infos]) + if draft_buffer_infos + else buffer_info.data_ptrs + ) super().__init__( dtype=self.dtype, - device_capacity_tokens=( - device_pool.size if device_capacity is None else device_capacity - ), + device_capacity_tokens=device_capacity_tokens, size_per_token=self.get_size_per_token(), - start_layer=device_pool.start_layer, - end_layer=device_pool.end_layer, + start_layer=start_layer, + end_layer=end_layer, host_to_device_ratio=host_to_device_ratio, host_size=host_size, page_size=page_size, @@ -129,9 +214,9 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): dcp_size=dcp_size, dcp_rank=dcp_rank, pool_label=pool_label, - is_dummy=is_dummy, + is_dummy=self._is_dummy, ) - if is_dummy: + if self._is_dummy: self.token_stride_size = self.kv_cache_dim * self.dtype.itemsize self.layout_dim = self.token_stride_size * self.layer_num self.can_use_jit = False @@ -149,16 +234,6 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): ) return - # Ascend transfers split K, rope and index tensors through its own API. - self.device_kv_buffers = ( - () - if _is_npu - else tuple( - b - for pool in (device_pool, *self.mtp_draft_device_pools) - for b in pool.kv_buffer - ) - ) self._initialize_transfer_views() def _initialize_transfer_views(self) -> None: @@ -184,14 +259,8 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): self._transfer_device, host_memory_registered=self.pin_memory, ) - if not _is_npu: - pools = (self.device_pool, *self.mtp_draft_device_pools) - self.device_data_ptrs = ( - torch.cat([pool.data_ptrs for pool in pools]) - if self.mtp_draft_device_pools - else self.device_pool.data_ptrs - ) - self._init_device_row_stride(self.device_kv_buffers) + if self._native_buffers is None: + self._init_device_row_stride(self.device_kv_buffers) self._init_write_back_staging_buffers() def get_contiguous_buf_infos(self): @@ -210,27 +279,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): return data_ptrs, data_lens, item_lens def get_size_per_token(self): - size_per_token = self.kv_cache_dim * self.dtype.itemsize * self.layer_num - if ( - self.layout == "page_first_kv_split" - and self.device_pool.index_head_dim is not None - ): - # Indexer buffers only exist for physical Indexer layers, which can - # be a subset of all layers (e.g. GLM 5.2: 21 of 78). Mirror the - # layer count used by init_kv_buffer so host capacity sizing stays - # consistent with the actually-allocated buffers. - num_indexer_layers = getattr(self.device_pool, "num_indexer_layers", None) - if num_indexer_layers is None: - num_indexer_layers = self.layer_num - size_per_token += ( - self.device_pool.index_head_dim - * self.dtype.itemsize - * num_indexer_layers - ) - if getattr(self.device_pool, "index_k_scale_buffer", None) is not None: - # FP32 quantization scale per token per indexer layer. - size_per_token += 4 * num_indexer_layers - return size_per_token + return self._size_per_token def get_ksize_per_token(self): return self.get_size_per_token() @@ -282,15 +331,14 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): # pool packs them as (num_indexer_layers, page, ...); mirror that # layer count here so transfer_kv_dim_exchange's layer check # (device dim0 == host dim1) holds. - num_indexer_layers = getattr(self.device_pool, "num_indexer_layers", None) - if num_indexer_layers is None: - num_indexer_layers = self.layer_num + native = self._native_buffers + num_indexer_layers = len(native.index_layer_indices) indexer_dims = (self.page_num, num_indexer_layers, self.page_size, 1) alloc_func = ALLOC_MEMORY_FUNCS[self._transfer_device.type] - if getattr(self.device_pool, "dsa_kv_cache_store_fp8", False): + if self.dsa_kv_cache_store_fp8: # FP8 DSA packs latent+RoPE+scale into the device k_buffer; # mirror the packed width so the 2D memcpy row width matches. - k_width = self.device_pool.kv_cache_dim + k_width = native.k_width else: k_width = self.kv_lora_rank if _is_npu and memfabric_host_memory_enabled(): @@ -305,15 +353,15 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): * (k_width + self.qk_rope_head_dim) * self.dtype.itemsize ) - if self.device_pool.index_head_dim is not None: + if native.index_k is not None: total_bytes += ( self.page_num * self.page_size * num_indexer_layers - * self.device_pool.index_head_dim + * native.index_width * self.dtype.itemsize ) - if getattr(self.device_pool, "index_k_scale_buffer", None) is not None: + if native.index_scales is not None: # FP32 scale mirror total_bytes += ( self.page_num * self.page_size * num_indexer_layers * 4 @@ -335,9 +383,9 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): allocator=self.allocator, ) self.index_k_buffer = None - if self.device_pool.index_head_dim is not None: + if native.index_k is not None: self.index_k_buffer = alloc_func( - (*indexer_dims, self.device_pool.index_head_dim), + (*indexer_dims, native.index_width), dtype=self.dtype, device=self.device, pin_memory=self.pin_memory, @@ -347,7 +395,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): # (see NPUMLATokenToKVPool.index_k_scale_buffer). Only present when # the device pool carries one (FP8 DSA + npu_quant_lightning_indexer). self.index_k_scale_buffer = None - if getattr(self.device_pool, "index_k_scale_buffer", None) is not None: + if native.index_scales is not None: self.index_k_scale_buffer = alloc_func( (*indexer_dims, 1), dtype=torch.float32, @@ -386,6 +434,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): # The staged kernel reads whole device pages, so it needs packed rows. if ( self.layout != "page_first" + or self._layer_shard_enabled or not self.device_rows_packed or not current_platform.capabilities.hicache_device_kernels ): @@ -414,27 +463,17 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): device=self._transfer_device, ) - def _indexer_slot_range_for_layer(self, device_pool, device_layer_id): - """Map a device layer onto the indexer slot space. - - Returns ``(slot, 1)`` when the layer is an indexer layer, ``(0, 0)`` - to skip the indexer components, or ``(0, -1)`` (all indexer layers) - when the mapping is unavailable. ``indexer_layer_ids`` holds - absolute (PP-global) layer ids while ``device_layer_id`` is - pool-local; convert before matching. - """ - indexer_layer_ids = getattr(device_pool, "indexer_layer_ids", None) - if not indexer_layer_ids or self.index_k_buffer is None: + def _indexer_slot_range_for_layer(self, buffers, device_layer_id): + if not buffers.index_layer_indices or self.index_k_buffer is None: return 0, -1 - start_layer = getattr(device_pool, "start_layer", 0) - for slot, layer in enumerate(indexer_layer_ids): - if layer - start_layer == device_layer_id: + for slot, layer in enumerate(buffers.index_layer_indices): + if layer == device_layer_id: return slot, 1 return 0, 0 def _transfer_ascendc_sparse_copy( self, - device_pool, + buffers, host_indices, device_indices, direction: TransferDirection, @@ -459,7 +498,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): The kernel de-references host pool pointers directly, so the host pool must be Memfabric-mapped. """ - device = device_pool.k_buffer.device + device = buffers.k.device # The kernel reads the token indices directly from device memory. # Upload without a stream sync: a plain .to(device) from pageable # memory synchronizes the stream and would serialize the pipeline. @@ -474,7 +513,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): device_indices.record_stream(stream) comps = self._build_ascendc_component_meta( - device_pool, + buffers, layer_start, layer_num, index_k_layer_start, @@ -528,7 +567,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): def _build_ascendc_component_meta( self, - device_pool, + buffers, layer_start: int, layer_num: int, index_k_layer_start: int, @@ -536,12 +575,12 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): ) -> list: """Assemble the per-component metadata list for kv_exchange_copy.""" k_lo = layer_start - k_hi = device_pool.k_buffer.shape[0] if layer_num < 0 else k_lo + layer_num + k_hi = buffers.k.shape[0] if layer_num < 0 else k_lo + layer_num # Both pools must share the layer index space (same limitation as the # legacy memcpy2d exchange op); catches e.g. MTP draft pools, whose # host rows live past the main pool's layers. host_layer_num = self.k_buffer.shape[1] - device_layer_num = device_pool.k_buffer.shape[0] + device_layer_num = buffers.k.shape[0] if k_hi > host_layer_num or k_hi > device_layer_num: raise RuntimeError( f"AscendC kv_exchange layer range [{k_lo}, {k_hi}) exceeds the " @@ -550,13 +589,13 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): ) comp_meta = self._ascendc_comp_meta - comps = [comp_meta(device_pool.k_buffer, self.k_buffer, k_lo, k_hi)] + comps = [comp_meta(buffers.k, self.k_buffer, k_lo, k_hi)] # FP8 DSA packs V into the device k_buffer; the device v_buffer is # empty and must be skipped. - if device_pool.v_buffer.numel() > 0 and self.v_buffer.numel() > 0: - comps.append(comp_meta(device_pool.v_buffer, self.v_buffer, k_lo, k_hi)) + if buffers.v.numel() > 0 and self.v_buffer.numel() > 0: + comps.append(comp_meta(buffers.v, self.v_buffer, k_lo, k_hi)) - device_index_k = getattr(device_pool, "index_k_buffer", None) + device_index_k = buffers.index_k if self.index_k_buffer is not None and device_index_k is not None: if index_k_layer_num < 0: ik_lo, ik_hi = 0, self.index_k_buffer.shape[1] @@ -567,7 +606,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): comps.append( comp_meta(device_index_k, self.index_k_buffer, ik_lo, ik_hi) ) - device_scale = getattr(device_pool, "index_k_scale_buffer", None) + device_scale = buffers.index_scales if self.index_k_scale_buffer is not None and device_scale is not None: comps.append( comp_meta( @@ -639,21 +678,22 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): *, is_draft: bool = False, ): - if not is_draft and not self._is_device_layer_owned(device_pool, layer_id): + assert not self._is_dummy, "load on a dummy (non-src MLA) host pool" + if not is_draft and layer_id not in self._target_layer_indices: return - assert not getattr(self, "_is_dummy", False), ( - "load on a dummy (non-src MLA) host pool" - ) host_indices = maybe_dcp_kernel_indices( host_indices, self.dcp_size, self.dcp_rank ) device_indices = maybe_dcp_kernel_indices( device_indices, self.dcp_size, self.dcp_rank ) - # MTP draft layers do not participate in CP layer sharding. - host_layer_id = layer_id if is_draft else self._host_layer_index(layer_id) + # Draft transfer IDs follow all target device layers, while host rows + # follow only this rank's target layers. + host_layer_id = self._host_layer_by_controller[layer_id] if io_backend != "kernel_ascend": - device_buffer = self.device_kv_buffers[layer_id] + device_buffer = self.device_kv_buffers[ + self._buffer_index_by_layer[layer_id] + ] if io_backend == "kernel": if self.layout == "layer_first": @@ -720,16 +760,18 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): else: raise ValueError(f"Unsupported layout: {self.layout}") elif io_backend == "kernel_ascend": + info_index = layer_id - self._device_layer_num + 1 if is_draft else 0 + native = self._native_infos[info_index].native_buffers() device_layer_id = 0 if is_draft else layer_id if self.layout == "page_first_kv_split": if _is_npu and ascendc_io_enabled(): # The per-layer complete(i) event recorded by the caller lets # later layers' DMA overlap the current layer's compute. ik_start, ik_num = self._indexer_slot_range_for_layer( - device_pool, device_layer_id + native, device_layer_id ) self._transfer_ascendc_sparse_copy( - device_pool, + native, host_indices, device_indices, TransferDirection.H2D, @@ -747,19 +789,13 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): transfer_kv_dim_exchange( device_indices=device_indices, host_indices=host_indices, - device_k=getattr( - device_pool, "k_buffer_tensor", device_pool.k_buffer - ), + device_k=native.k, host_k=self.k_buffer, - device_v=getattr( - device_pool, "v_buffer_tensor", device_pool.v_buffer - ), + device_v=native.v, host_v=self.v_buffer, - device_index_k=device_pool.index_k_buffer, + device_index_k=native.index_k, host_index_k=self.index_k_buffer, - device_index_k_scale=getattr( - device_pool, "index_k_scale_buffer", None - ), + device_index_k_scale=native.index_scales, host_index_k_scale=self.index_k_scale_buffer, page_size=self.page_size, direction=TransferDirection.H2D, @@ -778,17 +814,16 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): *, is_draft: bool = False, ): - assert not getattr(self, "_is_dummy", False), ( - "backup on a dummy (non-src MLA) host pool" - ) + assert not self._is_dummy, "backup on a dummy (non-src MLA) host pool" if io_backend not in ("kernel", "direct"): raise ValueError( f"Layer-sharded HiCache backup does not support IO backend: {io_backend}" ) # Indices arrive already translated by backup_from_device_all_layer. - # MTP draft layers do not participate in CP layer sharding. - host_layer_id = layer_id if is_draft else self._host_layer_index(layer_id) - device_buffer = self.device_kv_buffers[layer_id] + # Draft transfer IDs follow all target device layers, while host rows + # follow only this rank's target layers. + host_layer_id = self._host_layer_by_controller[layer_id] + device_buffer = self.device_kv_buffers[self._buffer_index_by_layer[layer_id]] if io_backend == "kernel": if self.layout == "layer_first": @@ -847,25 +882,23 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): def backup_from_device_all_layer( self, device_pool, host_indices, device_indices, io_backend ): - assert not getattr(self, "_is_dummy", False), ( - "backup on a dummy (non-src MLA) host pool" - ) + assert not self._is_dummy, "backup on a dummy (non-src MLA) host pool" host_indices = maybe_dcp_kernel_indices( host_indices, self.dcp_size, self.dcp_rank ) device_indices = maybe_dcp_kernel_indices( device_indices, self.dcp_size, self.dcp_rank ) - if self._is_device_layer_sharded(device_pool): - for layer_id in self._owned_device_layer_ids(device_pool): + if self._layer_shard_enabled: + for layer_id in self._target_layer_indices: self._backup_from_device_per_layer( host_indices, device_indices, layer_id, io_backend ) - for draft_layer_id in range(len(self.mtp_draft_device_pools)): + for draft_layer_id in range(self._draft_layer_num): self._backup_from_device_per_layer( host_indices, device_indices, - self.device_pool.layer_num + draft_layer_id, + self._device_layer_num + draft_layer_id, io_backend, is_draft=True, ) @@ -959,10 +992,11 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): else: raise ValueError(f"Unsupported layout: {self.layout}") elif io_backend == "kernel_ascend": + native = self._native_infos[0].native_buffers() if self.layout == "page_first_kv_split": if _is_npu and ascendc_io_enabled(): self._transfer_ascendc_sparse_copy( - device_pool, + native, host_indices, device_indices, TransferDirection.D2H, @@ -971,19 +1005,13 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): transfer_kv_dim_exchange( device_indices=device_indices, host_indices=host_indices, - device_k=getattr( - device_pool, "k_buffer_tensor", device_pool.k_buffer - ), + device_k=native.k, host_k=self.k_buffer, - device_v=getattr( - device_pool, "v_buffer_tensor", device_pool.v_buffer - ), + device_v=native.v, host_v=self.v_buffer, - device_index_k=device_pool.index_k_buffer, + device_index_k=native.index_k, host_index_k=self.index_k_buffer, - device_index_k_scale=getattr( - device_pool, "index_k_scale_buffer", None - ), + device_index_k_scale=native.index_scales, host_index_k_scale=self.index_k_scale_buffer, page_size=self.page_size, direction=TransferDirection.D2H, diff --git a/python/sglang/srt/mem_cache/storage/aibrix_kvcache/aibrix_kvcache_storage.py b/python/sglang/srt/mem_cache/storage/aibrix_kvcache/aibrix_kvcache_storage.py index d4197a10a9..6d0333aacf 100644 --- a/python/sglang/srt/mem_cache/storage/aibrix_kvcache/aibrix_kvcache_storage.py +++ b/python/sglang/srt/mem_cache/storage/aibrix_kvcache/aibrix_kvcache_storage.py @@ -31,16 +31,17 @@ class AibrixKVCacheStorage(HiCacheStorage): else: self.is_mla_backend = False self.local_rank = 0 - kv_cache = mem_pool.device_pool self.page_size = mem_pool.page_size - self.kv_cache_dtype = kv_cache.dtype - self.layer_num = kv_cache.layer_num + storage_info = mem_pool + self.kv_cache_dtype = storage_info.dtype + self.layer_num = mem_pool.target_layer_num self.kv_head_ids = [ - self.local_rank * kv_cache.head_num + i for i in range(kv_cache.head_num) + self.local_rank * storage_info.head_num + i + for i in range(storage_info.head_num) ] if not self.is_mla_backend: self.layer_ids = range( - kv_cache.start_layer, kv_cache.end_layer + storage_info.start_layer, storage_info.end_layer ) # for pipeline parallel self.block_spec = KVCacheBlockSpec( @@ -50,7 +51,7 @@ class AibrixKVCacheStorage(HiCacheStorage): tensor_spec=KVCacheTensorSpec( heads=self.kv_head_ids, layers=self.layer_ids, - head_size=kv_cache.head_dim, + head_size=storage_info.head_dim, ), ) logger.info(self.block_spec) diff --git a/python/sglang/srt/mem_cache/storage/aibrix_kvcache/unit_test.py b/python/sglang/srt/mem_cache/storage/aibrix_kvcache/unit_test.py index 17ea4fd560..d02b42031e 100644 --- a/python/sglang/srt/mem_cache/storage/aibrix_kvcache/unit_test.py +++ b/python/sglang/srt/mem_cache/storage/aibrix_kvcache/unit_test.py @@ -51,7 +51,19 @@ class AIBrixKVCacheStorageTest: 0, layer_num, ) - mem_pool = MHATokenToKVPoolHost(kv_cache, 2, 0, page_size, "layer_first") + mem_pool = MHATokenToKVPoolHost( + kv_cache.get_device_pool_infos()[0].buffer_info, + 2, + 0, + page_size, + "layer_first", + device_capacity_tokens=kv_cache.size + if kv_cache.host_capacity_tokens is None + else kv_cache.host_capacity_tokens, + start_layer=kv_cache.start_layer, + end_layer=kv_cache.end_layer, + write_back_staging=kv_cache.hicache_write_back_staging, + ) query_length = batch_size * 2 partial = batch_size self.aibrix_kvcache = AibrixKVCacheStorage(config, mem_pool) diff --git a/python/sglang/srt/mem_cache/storage/tensorcast_store/tensorcast_store.py b/python/sglang/srt/mem_cache/storage/tensorcast_store/tensorcast_store.py index 3e3d3bd3b8..92169d55b0 100644 --- a/python/sglang/srt/mem_cache/storage/tensorcast_store/tensorcast_store.py +++ b/python/sglang/srt/mem_cache/storage/tensorcast_store/tensorcast_store.py @@ -489,7 +489,7 @@ def _validate_storage_config( "TensorCast FULL v1 supports only page_first and page_first_direct, " f"got layout={mem_pool_host.layout!r}" ) - if mem_pool_host.mtp_draft_device_pools: + if mem_pool_host.layer_num != mem_pool_host.target_layer_num: raise NotImplementedError( "TensorCast FULL v1 does not support packed MTP draft pools" ) diff --git a/python/sglang/srt/mem_cache/storage/tensorcast_store/test_tensorcast_store.py b/python/sglang/srt/mem_cache/storage/tensorcast_store/test_tensorcast_store.py index f16a82d5e7..5ebd327807 100644 --- a/python/sglang/srt/mem_cache/storage/tensorcast_store/test_tensorcast_store.py +++ b/python/sglang/srt/mem_cache/storage/tensorcast_store/test_tensorcast_store.py @@ -247,7 +247,7 @@ def _make_pool( pool.layer_num = layer_num pool.head_num = head_num pool.dtype = dtype - pool.mtp_draft_device_pools = () + pool.target_layer_num = layer_num pool.dcp_size = 1 pool.dcp_rank = 0 pool.allocator = allocator or HostTensorAllocator() @@ -332,7 +332,7 @@ def test_fragment_registration_dispatch_is_exact_and_specific() -> None: [ ({"layout": "layer_first"}, {}, "supports only"), ({"layout": "page_head"}, {}, "supports only"), - ({"mtp_draft_device_pools": (object(),)}, {}, "MTP"), + ({"target_layer_num": 0}, {}, "MTP"), ({"dcp_size": 2}, {}, "DCP"), ({}, {"attn_cp_size": 2}, "context parallel"), ({}, {"tp_lcm_size": 2}, "TP-LCM"), diff --git a/python/sglang/srt/mem_cache/swa_memory_pool.py b/python/sglang/srt/mem_cache/swa_memory_pool.py index 20ae99f2cf..b7da1ee5f5 100644 --- a/python/sglang/srt/mem_cache/swa_memory_pool.py +++ b/python/sglang/srt/mem_cache/swa_memory_pool.py @@ -105,6 +105,12 @@ class SWAKVPool(BaseSWAKVPool): layer_num=self.swa_layer_nums, **swa_kv_pool_kwargs, ) + for pool, layer_ids in ( + (self.full_kv_pool, full_attention_layer_ids), + (self.swa_kv_pool, swa_attention_layer_ids), + ): + if isinstance(pool, MHATokenToKVPool): + pool.model_layer_ids = tuple(layer_ids) self.dsa_kv_cache_store_fp8 = False self.kv_cache_dim = None self.index_head_dim = None diff --git a/python/sglang/srt/mem_cache/unified_memory_pool.py b/python/sglang/srt/mem_cache/unified_memory_pool.py index e7766b292a..f5820ff544 100644 --- a/python/sglang/srt/mem_cache/unified_memory_pool.py +++ b/python/sglang/srt/mem_cache/unified_memory_pool.py @@ -1909,6 +1909,9 @@ class UnifiedSWAKVPool(SWAKVPool): end_layer=end_layer, ) + self.full_kv_pool.model_layer_ids = tuple(full_attention_layer_ids) + self.swa_kv_pool.model_layer_ids = tuple(swa_attention_layer_ids) + # disagg/nvlink disabled; keep attrs present to avoid AttributeError. self.enable_custom_mem_pool = False self.custom_mem_pool = None diff --git a/test/registered/hicache/test_dsa_host_release.py b/test/registered/hicache/test_dsa_host_release.py index 1ed2eef62e..a32294580e 100644 --- a/test/registered/hicache/test_dsa_host_release.py +++ b/test/registered/hicache/test_dsa_host_release.py @@ -39,7 +39,11 @@ class TestDSAHostRelease(CustomTestCase): index_head_dim=128, ) anchor = MLATokenToKVPoolHost( - pool, + pool.get_device_pool_infos()[0].buffer_info, + device_capacity_tokens=pool.size, + start_layer=pool.start_layer, + end_layer=pool.end_layer, + target_layer_num=pool.layer_num, host_to_device_ratio=2, host_size=0, page_size=64, diff --git a/test/registered/hicache/test_k_only_host_release.py b/test/registered/hicache/test_k_only_host_release.py index 33242e2d0f..d39ead5e2b 100644 --- a/test/registered/hicache/test_k_only_host_release.py +++ b/test/registered/hicache/test_k_only_host_release.py @@ -32,7 +32,18 @@ class TestKOnlyHostRelease(CustomTestCase): main_pool = MHATokenToKVPool(**common) index_pool = MHATokenToKOnlyPool(**common) anchor = MHATokenToKVPoolHost( - main_pool, 2, 0, 64, "layer_first", pin_memory=False + main_pool.get_device_pool_infos()[0].buffer_info, + 2, + 0, + 64, + "layer_first", + pin_memory=False, + device_capacity_tokens=main_pool.size + if main_pool.host_capacity_tokens is None + else main_pool.host_capacity_tokens, + start_layer=main_pool.start_layer, + end_layer=main_pool.end_layer, + write_back_staging=main_pool.hicache_write_back_staging, ) self.addCleanup(anchor.destroy) host = MHATokenToKOnlyPoolHost(index_pool, anchor, "layer_first") diff --git a/test/registered/kernels/ops/attention/qsa/test_qsa_hicache.py b/test/registered/kernels/ops/attention/qsa/test_qsa_hicache.py index 4dd83a5fcb..4be7383181 100644 --- a/test/registered/kernels/ops/attention/qsa/test_qsa_hicache.py +++ b/test/registered/kernels/ops/attention/qsa/test_qsa_hicache.py @@ -50,7 +50,19 @@ def pools(request): layout, backend = request.param publish(ServerArgs(model_path="dummy"), role="test") device = make_device_pool([7, 11], start_layer=4) - anchor = MHATokenToKVPoolHost(device.full_kv_pool, 2, 0, PAGE, layout) + anchor = MHATokenToKVPoolHost( + device.full_kv_pool.get_device_pool_infos()[0].buffer_info, + 2, + 0, + PAGE, + layout, + device_capacity_tokens=device.full_kv_pool.size + if device.full_kv_pool.host_capacity_tokens is None + else device.full_kv_pool.host_capacity_tokens, + start_layer=device.full_kv_pool.start_layer, + end_layer=device.full_kv_pool.end_layer, + write_back_staging=device.full_kv_pool.hicache_write_back_staging, + ) host = QSAIndexerPoolHost(device, anchor) try: yield device, anchor, host, backend diff --git a/test/registered/kernels/ops/kvcache/test_hicache.py b/test/registered/kernels/ops/kvcache/test_hicache.py index 4fe68280ac..2735a2bde4 100644 --- a/test/registered/kernels/ops/kvcache/test_hicache.py +++ b/test/registered/kernels/ops/kvcache/test_hicache.py @@ -141,6 +141,17 @@ def _token_indices_for_pages( def _pinned_host_pool(host_pool_cls, **kwargs): + if "device_pool" in kwargs: + pool = kwargs.pop("device_pool") + kwargs.update( + buffer_info=pool.get_device_pool_infos()[0].buffer_info, + device_capacity_tokens=pool.size + if pool.host_capacity_tokens is None + else pool.host_capacity_tokens, + start_layer=pool.start_layer, + end_layer=pool.end_layer, + target_layer_num=pool.layer_num, + ) original_alloc = ALLOC_MEMORY_FUNCS[DEVICE] ALLOC_MEMORY_FUNCS[DEVICE] = alloc_with_pin_memory try: @@ -182,7 +193,13 @@ def _run_transfer_roundtrip_mha(layout: str, element_dim: int) -> None: ) host_pool = _pinned_host_pool( MHATokenToKVPoolHost, - device_pool=device_pool, + buffer_info=device_pool.get_device_pool_infos()[0].buffer_info, + device_capacity_tokens=device_pool.size + if device_pool.host_capacity_tokens is None + else device_pool.host_capacity_tokens, + start_layer=device_pool.start_layer, + end_layer=device_pool.end_layer, + write_back_staging=device_pool.hicache_write_back_staging, layout=layout, ) assert host_pool.can_use_jit, ( @@ -359,7 +376,13 @@ def _run_page_first_staged_write_back_mha( ) host_pool = _pinned_host_pool( MHATokenToKVPoolHost, - device_pool=device_pool, + buffer_info=device_pool.get_device_pool_infos()[0].buffer_info, + device_capacity_tokens=device_pool.size + if device_pool.host_capacity_tokens is None + else device_pool.host_capacity_tokens, + start_layer=device_pool.start_layer, + end_layer=device_pool.end_layer, + write_back_staging=device_pool.hicache_write_back_staging, layout=layout, ) assert can_use_write_back_jit_kernel( diff --git a/test/registered/kernels/ops/kvcache/test_hicache_page_first_write_back.py b/test/registered/kernels/ops/kvcache/test_hicache_page_first_write_back.py index 0d61e20674..85268e56d3 100644 --- a/test/registered/kernels/ops/kvcache/test_hicache_page_first_write_back.py +++ b/test/registered/kernels/ops/kvcache/test_hicache_page_first_write_back.py @@ -60,6 +60,17 @@ def _token_indices_for_pages( def _pinned_host_pool(host_pool_cls, **kwargs): + if "device_pool" in kwargs: + pool = kwargs.pop("device_pool") + kwargs.update( + buffer_info=pool.get_device_pool_infos()[0].buffer_info, + device_capacity_tokens=pool.size + if pool.host_capacity_tokens is None + else pool.host_capacity_tokens, + start_layer=pool.start_layer, + end_layer=pool.end_layer, + target_layer_num=pool.layer_num, + ) from sglang.srt.mem_cache.pool_host.common import ( ALLOC_MEMORY_FUNCS, alloc_with_pin_memory, @@ -81,6 +92,17 @@ def _pinned_host_pool(host_pool_cls, **kwargs): def _registered_host_pool(host_pool_cls, **kwargs): + if "device_pool" in kwargs: + pool = kwargs.pop("device_pool") + kwargs.update( + buffer_info=pool.get_device_pool_infos()[0].buffer_info, + device_capacity_tokens=pool.size + if pool.host_capacity_tokens is None + else pool.host_capacity_tokens, + start_layer=pool.start_layer, + end_layer=pool.end_layer, + target_layer_num=pool.layer_num, + ) return host_pool_cls( host_to_device_ratio=2.0, host_size=0, @@ -125,7 +147,15 @@ def _run_mha(element_dim: int, page_count: int) -> None: enable_memory_saver=False, ) host_pool = _pinned_host_pool( - MHATokenToKVPoolHost, device_pool=device_pool, layout="page_first" + MHATokenToKVPoolHost, + buffer_info=device_pool.get_device_pool_infos()[0].buffer_info, + device_capacity_tokens=device_pool.size + if device_pool.host_capacity_tokens is None + else device_pool.host_capacity_tokens, + start_layer=device_pool.start_layer, + end_layer=device_pool.end_layer, + write_back_staging=device_pool.hicache_write_back_staging, + layout="page_first", ) assert can_use_write_back_jit_kernel( element_size=element_dim * host_pool.dtype.itemsize, @@ -589,7 +619,13 @@ def test_registered_mmap_page_first_kernel_operands_and_graph( ) host_pool = _registered_host_pool( MHATokenToKVPoolHost, - device_pool=device_pool, + buffer_info=device_pool.get_device_pool_infos()[0].buffer_info, + device_capacity_tokens=device_pool.size + if device_pool.host_capacity_tokens is None + else device_pool.host_capacity_tokens, + start_layer=device_pool.start_layer, + end_layer=device_pool.end_layer, + write_back_staging=device_pool.hicache_write_back_staging, layout="page_first", ) host_refs = [host_pool.k_data_refs[0], host_pool.v_data_refs[0]] diff --git a/test/registered/kernels/ops/kvcache/test_hicache_strided_device_rows.py b/test/registered/kernels/ops/kvcache/test_hicache_strided_device_rows.py index 15bede77d6..1ab62e56ff 100644 --- a/test/registered/kernels/ops/kvcache/test_hicache_strided_device_rows.py +++ b/test/registered/kernels/ops/kvcache/test_hicache_strided_device_rows.py @@ -17,6 +17,7 @@ import torch from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool from sglang.srt.mem_cache.pool_host.mha import MHATokenToKVPoolHost from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase register_cuda_ci(est_time=15, stage="base-b-kernel-unit", runner_config="1-gpu-large") @@ -50,13 +51,13 @@ def _strided_device_pool(): @unittest.skipUnless(torch.cuda.is_available(), "requires CUDA") -class TestHiCacheStridedDeviceRows(unittest.TestCase): +class TestHiCacheStridedDeviceRows(CustomTestCase): def test_backup_and_load_keep_every_slot(self): for layout in ("layer_first", "page_first"): with self.subTest(layout=layout): device = _strided_device_pool() host = MHATokenToKVPoolHost( - device_pool=device, + buffer_info=device.get_device_pool_infos()[0].buffer_info, host_to_device_ratio=2.0, host_size=0, page_size=1, @@ -64,6 +65,12 @@ class TestHiCacheStridedDeviceRows(unittest.TestCase): pin_memory=True, device="cpu", allocator_type="default", + device_capacity_tokens=device.size + if device.host_capacity_tokens is None + else device.host_capacity_tokens, + start_layer=device.start_layer, + end_layer=device.end_layer, + write_back_staging=device.hicache_write_back_staging, ) self.addCleanup(host.destroy) if not host.can_use_jit: diff --git a/test/registered/kernels/ops/kvcache/test_mxfp8_host_allocation.py b/test/registered/kernels/ops/kvcache/test_mxfp8_host_allocation.py index 4b99821dbb..b8e61b454f 100644 --- a/test/registered/kernels/ops/kvcache/test_mxfp8_host_allocation.py +++ b/test/registered/kernels/ops/kvcache/test_mxfp8_host_allocation.py @@ -23,12 +23,18 @@ class TestMXFP8HostAllocation(CustomTestCase): enable_memory_saver=False, ) host = MHATokenToKVPoolMXFP8Host( - pool, + pool.get_device_pool_infos()[0].buffer_info, host_to_device_ratio=2, host_size=0, page_size=pool.page_size, layout="page_first", pin_memory=False, + device_capacity_tokens=pool.size + if pool.host_capacity_tokens is None + else pool.host_capacity_tokens, + start_layer=pool.start_layer, + end_layer=pool.end_layer, + write_back_staging=pool.hicache_write_back_staging, ) self.addCleanup(host.destroy) expected_token_bytes = ( @@ -45,6 +51,119 @@ class TestMXFP8HostAllocation(CustomTestCase): host.size * expected_token_bytes, ) + def test_host_reuses_prebudget_staging(self): + from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool + from sglang.srt.mem_cache.pool_host.mha import ( + MHATokenToKVPoolHost, + prepare_mha_write_back_staging, + ) + + pool = MHATokenToKVPool( + size=256, + page_size=16, + dtype=torch.bfloat16, + head_num=1, + head_dim=128, + layer_num=2, + device="cuda", + enable_memory_saver=False, + ) + info = pool.get_device_pool_infos()[0].buffer_info + reserved = prepare_mha_write_back_staging( + info, + layer_num=2, + page_size=16, + page_capacity=64, + ) + self.assertIs( + prepare_mha_write_back_staging( + info, + layer_num=2, + page_size=16, + page_capacity=33, + staging=reserved, + ), + reserved, + ) + host = MHATokenToKVPoolHost( + info, + 2, + 0, + 16, + "page_first", + pin_memory=False, + device_capacity_tokens=256, + start_layer=0, + end_layer=2, + write_back_staging=reserved, + ) + self.addCleanup(host.destroy) + self.assertTrue(host.can_use_write_back_jit) + for actual, original in zip( + (host.staging_k_buffer, host.staging_v_buffer), reserved, strict=True + ): + self.assertEqual(actual.data_ptr(), original.data_ptr()) + self.assertEqual( + actual.untyped_storage().data_ptr(), + original.untyped_storage().data_ptr(), + ) + self.assertEqual(actual.shape[0], host.staging_token_capacity) + self.assertIsNone(pool.hicache_write_back_staging) + + def test_payload_and_scales_restore_without_device_owner(self): + """Captured scale buffers must restore without the removed owner helpers.""" + pool = MHATokenToKVPoolMXFP8( + size=256, + page_size=128, + dtype=torch.float8_e4m3fn, + head_num=1, + head_dim=128, + layer_num=2, + device="cuda", + enable_memory_saver=False, + ) + info = pool.get_device_pool_infos()[0].buffer_info + host = MHATokenToKVPoolMXFP8Host( + info, + 2, + 0, + 128, + "page_first", + device_capacity_tokens=256, + start_layer=0, + end_layer=2, + ) + self.addCleanup(host.destroy) + del pool + payload = (*info.buffers.k, *info.buffers.v) + scales = (*info.scales.k, *info.scales.v) + for value, buffer in enumerate((*payload, *scales), 1): + buffer.view(torch.uint8).fill_(value) + expected = [b[128:256].view(torch.uint8).clone() for b in payload] + expected += [b[1:2].view(torch.uint8).clone() for b in scales] + device_indices = torch.arange(128, 256, device="cuda") + host_indices = torch.arange(128) + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + host.backup_from_device_all_layer( + object(), host_indices, device_indices, "kernel" + ) + stream.synchronize() + for buffer in (*payload, *scales): + buffer.view(torch.uint8).zero_() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + for layer in range(2): + host.load_to_device_per_layer( + object(), host_indices.cuda(), device_indices, layer, "kernel" + ) + stream.synchronize() + actual = [b[128:256].view(torch.uint8) for b in payload] + actual += [b[1:2].view(torch.uint8) for b in scales] + for before, after in zip(expected, actual, strict=True): + self.assertTrue(torch.equal(before, after)) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/mem_cache/pool_host/test_kvcacheio_asymmetric.py b/test/registered/unit/mem_cache/pool_host/test_kvcacheio_asymmetric.py index 8261840d32..622ef97f95 100644 --- a/test/registered/unit/mem_cache/pool_host/test_kvcacheio_asymmetric.py +++ b/test/registered/unit/mem_cache/pool_host/test_kvcacheio_asymmetric.py @@ -51,6 +51,7 @@ def make_host_pool(dtype, layout="page_first"): host.page_size = PAGE_SIZE host.page_num = TOTAL_ITEMS // PAGE_SIZE host.layer_num = NUM_LAYERS + host.target_layer_num = NUM_LAYERS host.head_num = HEAD_NUM host.head_dim = K_HEAD_DIM host.v_head_dim = V_HEAD_DIM diff --git a/test/registered/unit/mem_cache/test_asymmetric_mha_pool_host_unit.py b/test/registered/unit/mem_cache/test_asymmetric_mha_pool_host_unit.py index 5af75cdb9f..b1b0cc3018 100644 --- a/test/registered/unit/mem_cache/test_asymmetric_mha_pool_host_unit.py +++ b/test/registered/unit/mem_cache/test_asymmetric_mha_pool_host_unit.py @@ -24,6 +24,7 @@ def _make_host(layout: str) -> AsymmetricMHATokenToKVPoolHost: host.layout = layout host.page_size = 2 host.layer_num = 3 + host.target_layer_num = 3 host.head_num = 2 host.head_dim = 4 host.v_head_dim = 6 @@ -85,8 +86,27 @@ def _make_device_pool(host: AsymmetricMHATokenToKVPoolHost) -> SimpleNamespace: class TestAsymmetricMHATokenToKVPoolHost(CustomTestCase): def test_factory_selects_asymmetric_pool_for_mismatched_kv_dims(self): - symmetric_pool = SimpleNamespace(head_dim=4, v_head_dim=4) - asymmetric_pool = SimpleNamespace(head_dim=4, v_head_dim=6) + from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool + + def info(v_head_dim): + return ( + MHATokenToKVPool( + size=8, + page_size=2, + dtype=torch.float16, + head_num=1, + head_dim=4, + v_head_dim=v_head_dim, + layer_num=1, + device="cpu", + enable_memory_saver=False, + ) + .get_device_pool_infos()[0] + .buffer_info + ) + + symmetric_pool = info(4) + asymmetric_pool = info(6) self.assertIs(get_mha_host_pool_cls(symmetric_pool), MHATokenToKVPoolHost) self.assertIs( @@ -110,7 +130,7 @@ class TestAsymmetricMHATokenToKVPoolHost(CustomTestCase): enable_memory_saver=False, ) host = AsymmetricMHATokenToKVPoolHost( - device_pool=device, + buffer_info=device.get_device_pool_infos()[0].buffer_info, host_to_device_ratio=2.0, host_size=0, page_size=2, @@ -118,6 +138,12 @@ class TestAsymmetricMHATokenToKVPoolHost(CustomTestCase): pin_memory=False, device="cpu", allocator_type="default", + device_capacity_tokens=device.size + if device.host_capacity_tokens is None + else device.host_capacity_tokens, + start_layer=device.start_layer, + end_layer=device.end_layer, + write_back_staging=device.hicache_write_back_staging, ) self.assertTrue(host.device_rows_packed) @@ -125,13 +151,12 @@ class TestAsymmetricMHATokenToKVPoolHost(CustomTestCase): host = _make_host("page_first") host.page_num = 4 host.v_head_dim = 8 - host.device_pool = SimpleNamespace( - device="cuda", + buffer_info = SimpleNamespace( + device=torch.device("cuda"), head_num=host.head_num, head_dim=host.head_dim, v_head_dim=host.v_head_dim, - store_dtype=host.dtype, - hicache_write_back_staging=None, + dtype=host.dtype, ) cpu_empty = torch.empty @@ -156,7 +181,7 @@ class TestAsymmetricMHATokenToKVPoolHost(CustomTestCase): side_effect=_cpu_empty, ), ): - host._init_write_back_staging_buffers() + host._init_write_back_staging_buffers(buffer_info, None) self.assertTrue(host.can_use_write_back_jit) self.assertEqual(host.staging_page_capacity, 4) diff --git a/test/registered/unit/mem_cache/test_dsa_pool_host_unit.py b/test/registered/unit/mem_cache/test_dsa_pool_host_unit.py index 15e70342e4..d87ef5244b 100644 --- a/test/registered/unit/mem_cache/test_dsa_pool_host_unit.py +++ b/test/registered/unit/mem_cache/test_dsa_pool_host_unit.py @@ -77,7 +77,7 @@ class TestDSAHiCacheTransfer(unittest.TestCase): ALLOC_MEMORY_FUNCS["cuda"] = alloc_with_pin_memory try: mla_host = MLATokenToKVPoolHost( - device_pool=device_pool, + buffer_info=device_pool.get_device_pool_infos()[0].buffer_info, host_to_device_ratio=2.0, host_size=0, page_size=page_size, @@ -86,6 +86,20 @@ class TestDSAHiCacheTransfer(unittest.TestCase): device="cpu", allocator_type="default", override_kv_cache_dim=device_pool.kv_cache_dim, + device_capacity_tokens=( + device_pool.size + if device_pool.host_capacity_tokens is None + else device_pool.host_capacity_tokens + ), + start_layer=device_pool.start_layer, + end_layer=device_pool.end_layer, + target_layer_num=( + (device_pool.layer_num + device_pool.layer_shard_size - 1) + // device_pool.layer_shard_size + if device_pool.layer_shard_enabled + else device_pool.layer_num + ), + layer_shard_enabled=device_pool.layer_shard_enabled, ) indexer_host = DSAIndexerPoolHost( decl=make_dsa_indexer_pool_decl(device_pool), diff --git a/test/registered/unit/mem_cache/test_hicache_auto_size.py b/test/registered/unit/mem_cache/test_hicache_auto_size.py index a29029b729..97031cd78f 100644 --- a/test/registered/unit/mem_cache/test_hicache_auto_size.py +++ b/test/registered/unit/mem_cache/test_hicache_auto_size.py @@ -121,12 +121,36 @@ class TestHiCacheAutoSize(CustomTestCase): ratio = get_memory().hicache_ratio self.assertLess(ratio, 2.0) host = MHATokenToKVPoolHost( - pool, ratio, 0, 2, "layer_first", pin_memory=False, device="cpu" + pool.get_device_pool_infos()[0].buffer_info, + ratio, + 0, + 2, + "layer_first", + pin_memory=False, + device="cpu", + device_capacity_tokens=pool.size + if pool.host_capacity_tokens is None + else pool.host_capacity_tokens, + start_layer=pool.start_layer, + end_layer=pool.end_layer, + write_back_staging=pool.hicache_write_back_staging, ) self.assertLessEqual(host.size * host.size_per_token, 0.8 * rank_budget) with self.assertRaisesRegex(ValueError, "Not enough host memory"): MHATokenToKVPoolHost( - pool, ratio, 0, 2, "layer_first", pin_memory=False, device="cpu" + pool.get_device_pool_infos()[0].buffer_info, + ratio, + 0, + 2, + "layer_first", + pin_memory=False, + device="cpu", + device_capacity_tokens=pool.size + if pool.host_capacity_tokens is None + else pool.host_capacity_tokens, + start_layer=pool.start_layer, + end_layer=pool.end_layer, + write_back_staging=pool.hicache_write_back_staging, ) self.assertIsNone(base._host_memory_budget.get()) diff --git a/test/registered/unit/mem_cache/test_hicache_dcp_host_pool.py b/test/registered/unit/mem_cache/test_hicache_dcp_host_pool.py index be6f86e2b5..a6080206ff 100644 --- a/test/registered/unit/mem_cache/test_hicache_dcp_host_pool.py +++ b/test/registered/unit/mem_cache/test_hicache_dcp_host_pool.py @@ -11,12 +11,12 @@ kernels. """ import unittest -from types import SimpleNamespace from unittest import mock import torch from sglang.srt.layers.dcp.layout import maybe_dcp_kernel_indices +from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -28,31 +28,23 @@ PHYSICAL_PAGE = 64 WIDENED_PAGE = PHYSICAL_PAGE * DCP_SIZE -def _fake_mla_device_pool(size: int = 1024) -> SimpleNamespace: - buffers = [ - torch.empty(size + PHYSICAL_PAGE, 1, 12, dtype=torch.float16) for _ in range(2) - ] - return SimpleNamespace( - kv_buffer=buffers, - data_ptrs=torch.tensor([b.data_ptr() for b in buffers], dtype=torch.uint64), +def _fake_mla_device_pool(size: int = 1024) -> MLATokenToKVPool: + return MLATokenToKVPool( size=size, - # Match KVCache's default: this static pool's size already counts tokens. - host_capacity_tokens=None, - store_dtype=torch.float16, + page_size=PHYSICAL_PAGE, + dtype=torch.float16, kv_lora_rank=8, qk_rope_head_dim=4, layer_num=2, - start_layer=0, - end_layer=1, device="cpu", - layers_to_capture=None, - layer_shard_enabled=False, + enable_memory_saver=False, ) def _make_host_pool(dcp_rank: int, device_size: int = 1024) -> MLATokenToKVPoolHost: + device_pool = _fake_mla_device_pool(device_size) return MLATokenToKVPoolHost( - _fake_mla_device_pool(device_size), + device_pool.get_device_pool_infos()[0].buffer_info, host_to_device_ratio=2.0, host_size=0, page_size=WIDENED_PAGE, @@ -61,6 +53,20 @@ def _make_host_pool(dcp_rank: int, device_size: int = 1024) -> MLATokenToKVPoolH device="cpu", dcp_size=DCP_SIZE, dcp_rank=dcp_rank, + device_capacity_tokens=( + device_pool.size + if device_pool.host_capacity_tokens is None + else device_pool.host_capacity_tokens + ), + start_layer=device_pool.start_layer, + end_layer=device_pool.end_layer, + target_layer_num=( + (device_pool.layer_num + device_pool.layer_shard_size - 1) + // device_pool.layer_shard_size + if device_pool.layer_shard_enabled + else device_pool.layer_num + ), + layer_shard_enabled=device_pool.layer_shard_enabled, ) @@ -149,14 +155,29 @@ class TestHostPoolSizingUnderDcp(CustomTestCase): pool.alloc(PHYSICAL_PAGE) # not a multiple of the widened page def test_non_dcp_pool_unchanged(self): + device_pool = _fake_mla_device_pool() pool = MLATokenToKVPoolHost( - _fake_mla_device_pool(), + device_pool.get_device_pool_infos()[0].buffer_info, host_to_device_ratio=2.0, host_size=0, page_size=PHYSICAL_PAGE, layout="layer_first", pin_memory=False, device="cpu", + device_capacity_tokens=( + device_pool.size + if device_pool.host_capacity_tokens is None + else device_pool.host_capacity_tokens + ), + start_layer=device_pool.start_layer, + end_layer=device_pool.end_layer, + target_layer_num=( + (device_pool.layer_num + device_pool.layer_shard_size - 1) + // device_pool.layer_shard_size + if device_pool.layer_shard_enabled + else device_pool.layer_num + ), + layer_shard_enabled=device_pool.layer_shard_enabled, ) self.assertEqual(pool.page_size, PHYSICAL_PAGE) self.assertEqual(pool.logical_size, pool.size) @@ -165,7 +186,7 @@ class TestHostPoolSizingUnderDcp(CustomTestCase): class TestTransferEntryPointsTranslate(CustomTestCase): def _run_backup(self, pool, host_indices, device_indices): - device_pool = pool.device_pool + device_pool = None # create=True: mla.py imports the kernel only under `if _is_cuda or # _is_hip`, so the name is absent on the CPU runner this test targets. with mock.patch( @@ -204,7 +225,25 @@ class TestLayerShardedHostTransfers(CustomTestCase): pool.layer_shard_size = 2 pool._owned_local_layer_range = lambda: (0, 1) host = MLATokenToKVPoolHost( - pool, 2, 0, PHYSICAL_PAGE, "layer_first", pin_memory=False + pool.get_device_pool_infos()[0].buffer_info, + 2, + 0, + PHYSICAL_PAGE, + "layer_first", + pin_memory=False, + device_capacity_tokens=( + pool.size + if pool.host_capacity_tokens is None + else pool.host_capacity_tokens + ), + start_layer=pool.start_layer, + end_layer=pool.end_layer, + target_layer_num=( + (pool.layer_num + pool.layer_shard_size - 1) // pool.layer_shard_size + if pool.layer_shard_enabled + else pool.layer_num + ), + layer_shard_enabled=pool.layer_shard_enabled, ) self.addCleanup(host.destroy) # Ascend has no CUDA-style per-layer KV buffers. @@ -222,7 +261,8 @@ class TestLayerShardedHostTransfers(CustomTestCase): # Rank 1 owns device layers 2/3, which occupy host layers 0/1. pool = _fake_mla_device_pool(size=128) pool.layer_num = 4 - pool.end_layer = 3 + pool.end_layer = 4 + pool.model_layer_ids = tuple(range(4)) pool.layer_shard_enabled = True pool.layer_shard_size = 2 pool._owned_local_layer_range = lambda: (2, 4) @@ -236,12 +276,25 @@ class TestLayerShardedHostTransfers(CustomTestCase): [b.data_ptr() for b in pool.kv_buffer], dtype=torch.uint64 ) host = MLATokenToKVPoolHost( - pool, + pool.get_device_pool_infos()[0].buffer_info, 2, 0, PHYSICAL_PAGE, "layer_first", pin_memory=False, + device_capacity_tokens=( + pool.size + if pool.host_capacity_tokens is None + else pool.host_capacity_tokens + ), + start_layer=pool.start_layer, + end_layer=pool.end_layer, + target_layer_num=( + (pool.layer_num + pool.layer_shard_size - 1) // pool.layer_shard_size + if pool.layer_shard_enabled + else pool.layer_num + ), + layer_shard_enabled=pool.layer_shard_enabled, ) self.addCleanup(host.destroy) self.assertEqual(host.layer_num, 2) diff --git a/test/registered/unit/mem_cache/test_hicache_seaweedfs_storage.py b/test/registered/unit/mem_cache/test_hicache_seaweedfs_storage.py index 6e32a46b07..f0c4949eaf 100755 --- a/test/registered/unit/mem_cache/test_hicache_seaweedfs_storage.py +++ b/test/registered/unit/mem_cache/test_hicache_seaweedfs_storage.py @@ -108,13 +108,19 @@ def _host_pool(layout: str) -> MHATokenToKVPoolHost: enable_memory_saver=False, ) return MHATokenToKVPoolHost( - device_pool=device_pool, + buffer_info=device_pool.get_device_pool_infos()[0].buffer_info, host_to_device_ratio=2.0, host_size=0, page_size=PAGE_SIZE, layout=layout, pin_memory=False, device="cpu", + device_capacity_tokens=device_pool.size + if device_pool.host_capacity_tokens is None + else device_pool.host_capacity_tokens, + start_layer=device_pool.start_layer, + end_layer=device_pool.end_layer, + write_back_staging=device_pool.hicache_write_back_staging, ) diff --git a/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py b/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py index ab9b810b71..a8cc98cbc0 100644 --- a/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py +++ b/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py @@ -519,7 +519,7 @@ class TestHiCacheStagedWriteBackDispatch(CustomTestCase): ) host = MHATokenToKVPoolHost.__new__(MHATokenToKVPoolHost) - host.device_pool = device_pool + host.target_layer_num = layer_num host.device_k_buffers = tuple(k_layers) host.device_v_buffers = tuple(v_layers) host.device_k_data_ptrs = device_pool.k_data_ptrs @@ -602,57 +602,81 @@ class TestHiCacheStagedWriteBackDispatch(CustomTestCase): ) def test_npu_mha_transfer_uses_contiguous_hicache_backing(self): - host = MHATokenToKVPoolHost.__new__(MHATokenToKVPoolHost) - host.layout = "page_first_direct" - host.page_size = 2 - host.kv_buffer = torch.empty(2, 2, 2, 2, 1, 1) + """FIA views and packed drafts must restore without querying their owners.""" + from sglang.srt.hardware_backend.npu.memory_pool_npu import NPUMHATokenToKVPool - device_k = torch.empty(2, 3, 2, 1, 1) - device_v = torch.empty_like(device_k) - device_pool = _device_pool_stub( - layer_num=2, - # FIA exposes lists here; these must not be sent to the operator. - k_buffer=[device_k[layer].reshape(-1, 1, 1, 1) for layer in range(2)], - v_buffer=[device_v[layer].reshape(-1, 1, 1, 1) for layer in range(2)], - get_hicache_transfer_buffers=mock.Mock(return_value=(device_k, device_v)), + with mock.patch.dict("os.environ", {"ASCEND_USE_FIA": "True"}): + pools = tuple( + NPUMHATokenToKVPool( + size=32, + page_size=16, + dtype=torch.float16, + head_num=2, + head_dim=4, + layer_num=layers, + device="cpu", + enable_memory_saver=False, + enable_alt_stream=False, + ) + for layers in (2, 1) + ) + infos = tuple(pool.get_device_pool_infos()[0].buffer_info for pool in pools) + for pool in pools: + pool.get_hicache_transfer_buffers = mock.Mock( + side_effect=AssertionError("runtime owner query") + ) + host = MHATokenToKVPoolHost( + infos[0], + 2, + 0, + 16, + "page_first_direct", + pin_memory=False, + device_capacity_tokens=32, + start_layer=0, + end_layer=2, + draft_buffer_infos=infos[1:], ) - host.device_pool = device_pool - host_indices = _indices(0, 2) - device_indices = _indices(2, 4) - directions = SimpleNamespace(H2D="H2D", D2H="D2H") + self.addCleanup(host.destroy) + expected = [] + for i, info in enumerate(infos): + for tensor in info.transfer_buffers(): + tensor.copy_( + torch.arange(tensor.numel()).reshape(tensor.shape) + i * 100 + ) + expected.append(tensor[:, 1].clone()) + + def transfer(*, device_k, device_v, host_k, host_v, direction, **kwargs): + for device_buffer, host_buffer in ((device_k, host_k), (device_v, host_v)): + if direction == "D2H": + host_buffer[0].copy_(device_buffer[:, 1]) + else: + device_buffer[:, 1].copy_(host_buffer[0]) with ( mock.patch( f"{MHA_POOL_HOST_MODULE}.TransferDirection", - directions, + SimpleNamespace(H2D="H2D", D2H="D2H"), create=True, ), mock.patch( f"{MHA_POOL_HOST_MODULE}.transfer_kv_dim_exchange", + side_effect=transfer, create=True, - ) as transfer, + ), ): host.backup_from_device_all_layer( - device_pool, - host_indices, - device_indices, - io_backend="kernel_ascend", + object(), _indices(0, 16), _indices(16, 32), "kernel_ascend" ) + for info in infos: + for buffer in info.transfer_buffers(): + buffer.zero_() host.load_to_device_per_layer( - device_pool, - host_indices, - device_indices, - layer_id=0, - io_backend="kernel_ascend", + object(), _indices(0, 16), _indices(16, 32), 0, "kernel_ascend" ) - - self.assertEqual(device_pool.get_hicache_transfer_buffers.call_count, 2) - self.assertEqual(transfer.call_count, 2) - for call in transfer.call_args_list: - self.assertIs(call.kwargs["device_k"], device_k) - self.assertIs(call.kwargs["device_v"], device_v) - self.assertEqual(transfer.call_args_list[0].kwargs["direction"], "D2H") - self.assertEqual(transfer.call_args_list[1].kwargs["direction"], "H2D") + actual = [tensor[:, 1] for info in infos for tensor in info.transfer_buffers()] + for before, after in zip(expected, actual, strict=True): + self.assertTrue(torch.equal(before, after)) def test_mla_backup_then_load_roundtrip_uses_staged(self): layer_num = 2 @@ -675,6 +699,11 @@ class TestHiCacheStagedWriteBackDispatch(CustomTestCase): ) host = MLATokenToKVPoolHost.__new__(MLATokenToKVPoolHost) + host._is_dummy = False + host._layer_shard_enabled = False + host._target_layer_indices = tuple(range(layer_num)) + host._host_layer_by_controller = {i: i for i in range(layer_num)} + host._buffer_index_by_layer = {i: i for i in range(layer_num)} host.device_pool = device_pool host.device_kv_buffers = tuple(device_layers) host.device_data_ptrs = device_pool.data_ptrs diff --git a/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py b/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py index e2b0b24008..7340dc6327 100644 --- a/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py +++ b/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py @@ -643,21 +643,44 @@ class TestDeclaredStackStructure(CustomTestCase): packed drafts, mirror geometry, controller arguments, sidecars.""" def _run(self, *, pool, drafts, full_layer_mapping, page_size=64): - from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost + from sglang.srt.mem_cache.pool_host.mla import ( + MLAHostAllocation, + MLATokenToKVPoolHost, + ) real_indexer_host = pool_host_dsa.DSAIndexerPoolHost def dummy_kv_host(**kwargs): + pool = kwargs["kv_pool"] + drafts = kwargs["mtp_draft_device_pools"] + width = kwargs["kv_pool"].kv_cache_dim or ( + pool.kv_lora_rank + pool.qk_rope_head_dim + ) + target_layer_num = ( + (pool.layer_num + pool.layer_shard_size - 1) // pool.layer_shard_size + if pool.layer_shard_enabled + else pool.layer_num + ) return MLATokenToKVPoolHost( - kwargs["kv_pool"], + MLAHostAllocation( + dtype=pool.store_dtype, + kv_lora_rank=pool.kv_lora_rank, + qk_rope_head_dim=pool.qk_rope_head_dim, + kv_cache_dim=width, + size_per_token=width + * pool.store_dtype.itemsize + * (target_layer_num + len(drafts)), + draft_layer_num=len(drafts), + ), + device_capacity_tokens=pool.size, + start_layer=pool.start_layer, + end_layer=pool.end_layer, + target_layer_num=target_layer_num, host_to_device_ratio=2, host_size=0, page_size=kwargs["page_size"], layout="page_first", pin_memory=False, - is_dummy=True, - override_kv_cache_dim=kwargs["kv_pool"].kv_cache_dim, - mtp_draft_device_pools=kwargs["mtp_draft_device_pools"], ) def dummy_indexer_host( @@ -893,10 +916,30 @@ class TestKvHostPoolRow(CustomTestCase): def _build(self, pool, drafts=()): seen = {} + for device_pool in (pool, *drafts): + device_pool.model_layer_ids = tuple(range(device_pool.layer_num)) + device_pool.kv_buffer = [ + torch.empty( + ( + device_pool.size + device_pool.page_size, + 1, + device_pool.kv_cache_dim, + ), + dtype=device_pool.store_dtype, + ) + for _ in range(device_pool.layer_num) + ] + device_pool.data_ptrs = torch.tensor( + [buffer.data_ptr() for buffer in device_pool.kv_buffer], + dtype=torch.uint64, + ) + device_pool.kv_scale_buffer = None + device_pool.dsa_kv_cache_store_fp8 = False + device_pool.host_capacity_tokens = None - def fake_host(kv_pool, ratio, size, page_size, layout, **kwargs): + def fake_host(buffer_info, ratio, size, page_size, layout, **kwargs): seen.update(kwargs) - return SimpleNamespace(layer_num=kv_pool.layer_num) + return SimpleNamespace(layer_num=kwargs["target_layer_num"]) with ( patch.object(hybrid_pool_assembler, "MLATokenToKVPoolHost", fake_host), @@ -943,7 +986,10 @@ class TestHybridMambaDeclaredIndexer(CustomTestCase): DSA layers that own index buffers.""" def test_stack_declares_indexer_and_skips_empty_layers(self): - from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost + from sglang.srt.mem_cache.pool_host.mla import ( + MLAHostAllocation, + MLATokenToKVPoolHost, + ) kv_pool = _dsa_pool_stub(layer_num=3) kv_pool.skip_topk_layers = [False, True, False] # device layer 1: no buffer @@ -976,16 +1022,29 @@ class TestHybridMambaDeclaredIndexer(CustomTestCase): real_indexer_host = pool_host_dsa.DSAIndexerPoolHost def dummy_kv_host(**kwargs): + pool = kwargs["kv_pool"] + drafts = kwargs["mtp_draft_device_pools"] + width = None or (pool.kv_lora_rank + pool.qk_rope_head_dim) return MLATokenToKVPoolHost( - kwargs["kv_pool"], + MLAHostAllocation( + dtype=pool.store_dtype, + kv_lora_rank=pool.kv_lora_rank, + qk_rope_head_dim=pool.qk_rope_head_dim, + kv_cache_dim=width, + size_per_token=width + * pool.store_dtype.itemsize + * (pool.layer_num + len(drafts)), + draft_layer_num=len(drafts), + ), + device_capacity_tokens=pool.size, + start_layer=pool.start_layer, + end_layer=pool.end_layer, + target_layer_num=pool.layer_num, host_to_device_ratio=2, host_size=0, page_size=kwargs["page_size"], layout="page_first", pin_memory=False, - is_dummy=True, - override_kv_cache_dim=None, - mtp_draft_device_pools=kwargs["mtp_draft_device_pools"], ) def dummy_indexer_host( @@ -1523,5 +1582,99 @@ class TestUnifiedPageEnvelopeSelection(CustomTestCase): ) +class TestMtpMhaPhysicalConstruction(CustomTestCase): + def test_full_token_draft_swa_preserves_capacity_and_physical_rows(self): + """MTP sidecars keep their capacity override and both draft allocations.""" + from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool + from sglang.srt.mem_cache.pool_host.mha import MHATokenToKVPoolHost + + original_init = MHATokenToKVPoolHost.__init__ + allocated = [] + + def unpinned(host, *args, **kwargs): + kwargs["pin_memory"] = False + original_init(host, *args, **kwargs) + allocated.append(host) + + def pool(size, layers, start, heads): + return MHATokenToKVPool( + size=size, + page_size=4, + dtype=torch.float16, + head_num=heads, + head_dim=8, + layer_num=layers, + device="cpu", + enable_memory_saver=False, + start_layer=start, + end_layer=start + layers, + ) + + for layout in ("layer_first", "page_first"): + with self.subTest(layout=layout): + full = pool(32, 2, 20, 2) + swa = pool(16, 1, 20, 1) + first, second = pool(16, 1, 23, 1), pool(16, 1, 24, 1) + first.host_capacity_tokens = 32 + try: + with ( + patch.object(MHATokenToKVPoolHost, "__init__", unpinned), + patch.object( + hybrid_pool_assembler, + "get_memory", + return_value=SimpleNamespace( + hicache_ratio=1.0, + hicache_size=0, + hicache_mem_layout=layout, + ), + ), + patch.object( + hybrid_pool_assembler, + "get_parallel", + return_value=SimpleNamespace(dcp_enabled=False), + ), + patch.object( + hybrid_pool_assembler, + "_get_allocator_type", + return_value="default", + ), + ): + group = build_hybrid_swa_group( + page_size=4, + full_kv_pool=full, + swa_kv_pool=swa, + full_layer_mapping={0: 0, 2: 1}, + swa_layer_mapping={1: 0}, + use_mla=False, + mtp_swa_full_device_pools=(first, second), + transfer_layer_id_max=3, + ) + host = group.get_pool(PoolName.DRAFT_SWA) + self.assertEqual(group.get_pool(PoolName.KV).size, 36) + self.assertEqual(host.device_capacity_tokens, 32) + self.assertEqual(host.size, 76) + self.assertEqual((host.start_layer, host.end_layer), (23, 24)) + self.assertEqual((host.target_layer_num, host.layer_num), (1, 2)) + self.assertIs(host.device_k_buffers[0], first.k_buffer[0]) + self.assertIs(host.device_k_buffers[1], second.k_buffer[0]) + self.assertIs(host.device_v_buffers[0], first.v_buffer[0]) + self.assertIs(host.device_v_buffers[1], second.v_buffer[0]) + self.assertEqual( + [ + group.get_entry(PoolName.DRAFT_SWA).layer_mapper(i) + for i in (0, 3) + ], + [0, 1], + ) + self.assertEqual( + group.get_entry(PoolName.DRAFT_SWA).packed_draft_device_pools, + (second,), + ) + finally: + for host in allocated: + host.destroy() + allocated.clear() + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/mem_cache/test_mem_pool_host.py b/test/registered/unit/mem_cache/test_mem_pool_host.py index 8117c960fc..713bc3dae3 100644 --- a/test/registered/unit/mem_cache/test_mem_pool_host.py +++ b/test/registered/unit/mem_cache/test_mem_pool_host.py @@ -14,6 +14,7 @@ from sglang.srt.mem_cache.memory_pool import ( MambaPool, MHATokenToKOnlyPool, MHATokenToKVPool, + MHATokenToKVPoolFP4, MLATokenToKVPool, ) from sglang.srt.mem_cache.memory_pool_host import ( @@ -29,8 +30,9 @@ from sglang.srt.mem_cache.pool_host.mamba import MambaPoolHost from sglang.srt.mem_cache.pool_host.mha import ( MHATokenToKOnlyPoolHost, MHATokenToKVPoolHost, + get_mha_host_pool_cls, ) -from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost +from sglang.srt.mem_cache.pool_host.mla import MLAHostAllocation, MLATokenToKVPoolHost from sglang.srt.runtime_context import get_context from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -53,7 +55,7 @@ class TestHostKVCache(CustomTestCase): enable_memory_saver=False, ) self.host_pool = MHATokenToKVPoolHost( - device_pool=self.device_pool, + buffer_info=self.device_pool.get_device_pool_infos()[0].buffer_info, host_to_device_ratio=2.0, host_size=0, page_size=self.page_size, @@ -61,8 +63,76 @@ class TestHostKVCache(CustomTestCase): pin_memory=False, device="cpu", allocator_type="default", + device_capacity_tokens=self.device_pool.size + if self.device_pool.host_capacity_tokens is None + else self.device_pool.host_capacity_tokens, + start_layer=self.device_pool.start_layer, + end_layer=self.device_pool.end_layer, + write_back_staging=self.device_pool.hicache_write_back_staging, ) + def test_info_host_does_not_retain_owner_or_rebuild_pointer_tables(self): + """Host construction preserves physical allocations without retaining pools.""" + import gc + import weakref + + pool = MHATokenToKVPool( + size=16, + page_size=2, + dtype=torch.float16, + head_num=1, + head_dim=4, + layer_num=2, + start_layer=7, + end_layer=9, + device="cpu", + enable_memory_saver=False, + ) + info = pool.get_device_pool_infos()[0] + self.assertEqual(info.layer_ids, (7, 8)) + pointer_table = pool.k_data_ptrs + owner = weakref.ref(pool) + del pool + gc.collect() + self.assertIsNone(owner()) + host = MHATokenToKVPoolHost( + info.buffer_info, + 2, + 0, + 2, + "layer_first", + pin_memory=False, + device_capacity_tokens=8, + start_layer=7, + end_layer=9, + ) + self.addCleanup(host.destroy) + self.assertIs(host.device_k_data_ptrs, pointer_table) + self.assertEqual(host.size, 18) + self.assertIs(host.device_k_buffers[0], info.buffer_info.buffers.k[0]) + self.assertNotIn("device_pool", vars(host)) + self.assertNotIn("mtp_draft_device_pools", vars(host)) + + def test_mha_model_layer_ids_follow_hybrid_and_empty_stage(self): + """Hybrid compact buffer positions must not become model layer IDs.""" + from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool + + for layers in ([7, 12], []): + root = HybridLinearKVPool( + size=8, + page_size=2, + dtype=torch.float16, + head_num=1, + head_dim=4, + full_attention_layer_ids=layers, + device="cpu", + mamba_pool=None, + ) + info = root.full_kv_pool.get_device_pool_infos()[0] + self.assertEqual(info.layer_ids, tuple(layers)) + info.buffer_info.validate(layer_ids=info.layer_ids) + self.assertEqual(len(info.buffer_info.buffers.k), len(layers)) + def test_direct_page_first_direct_rejects_strided_rows(self): device = self.device_pool device.k_buffer = [ @@ -75,7 +145,7 @@ class TestHostKVCache(CustomTestCase): ] device._init_data_ptrs_and_strides() host = MHATokenToKVPoolHost( - device_pool=device, + buffer_info=device.get_device_pool_infos()[0].buffer_info, host_to_device_ratio=2.0, host_size=0, page_size=self.page_size, @@ -83,6 +153,9 @@ class TestHostKVCache(CustomTestCase): pin_memory=False, device="cpu", allocator_type="default", + device_capacity_tokens=device.size, + start_layer=device.start_layer, + end_layer=device.end_layer, ) self.addCleanup(host.destroy) src = torch.tensor([0, 1]) @@ -120,14 +193,25 @@ class TestHostKVCache(CustomTestCase): ) for layers in (2, 1, 1) ) + host_inputs = dict( + device_capacity_tokens=pools[0].size, + start_layer=pools[0].start_layer, + end_layer=pools[0].end_layer, + draft_buffer_infos=tuple( + pool.get_device_pool_infos()[0].buffer_info + for pool in pools[1:] + ), + ) + if mla: + host_inputs["target_layer_num"] = pools[0].layer_num host = host_type( - pools[0], + pools[0].get_device_pool_infos()[0].buffer_info, host_to_device_ratio=2, host_size=0, page_size=2, layout="layer_first", pin_memory=False, - mtp_draft_device_pools=pools[1:], + **host_inputs, ) self.addCleanup(host.destroy) names = ("kv_buffer",) if mla else ("k_buffer", "v_buffer") @@ -208,6 +292,115 @@ class TestHostKVCache(CustomTestCase): ) self.assertFalse(group.can_use_write_back_jit) + def test_reject_packed_scaled_kv_before_host_allocation(self): + """Packed target or draft KV previously lost scales and used wrong row bytes.""" + from sglang.srt.layers.quantization.fp4_kv_cache_quant_method import ( + FP4MXBlock16KVCacheMethod, + NVFP4KVCacheMethod, + ) + + for pool_cls, method in ( + (MHATokenToKVPool, FP4MXBlock16KVCacheMethod()), + ( + MHATokenToKVPool, + NVFP4KVCacheMethod( + num_layers=1, device="cpu", native_scale_layout=False + ), + ), + (MHATokenToKVPoolFP4, None), + ): + pool = pool_cls( + size=32, + page_size=16, + dtype=torch.bfloat16, + head_num=2, + head_dim=64, + layer_num=1, + device="cpu", + enable_memory_saver=False, + enable_alt_stream=False, + quant_method=method, + ) + for target, drafts in ((pool, ()), (self.device_pool, (pool,))): + with self.subTest( + pool=type(pool).__name__, method=method, draft=bool(drafts) + ): + with unittest.mock.patch.object( + base.HostKVCache, + "__init__", + side_effect=AssertionError( + "unsupported format reached allocation" + ), + ): + with self.assertRaisesRegex( + ValueError, "KV scale layout 'token'" + ): + MHATokenToKVPoolHost( + buffer_info=target.get_device_pool_infos()[ + 0 + ].buffer_info, + host_to_device_ratio=2, + host_size=0, + page_size=16, + layout="layer_first", + pin_memory=False, + draft_buffer_infos=tuple( + pool.get_device_pool_infos()[0].buffer_info + for pool in drafts + ), + device_capacity_tokens=target.size + if target.host_capacity_tokens is None + else target.host_capacity_tokens, + start_layer=target.start_layer, + end_layer=target.end_layer, + write_back_staging=target.hicache_write_back_staging, + ) + + def test_unscaled_fp8_and_asymmetric_host_capacity(self): + """The FP4 guard must not reject byte-backed FP8 or asymmetric rows.""" + from sglang.srt.layers.quantization.fp4_kv_cache_quant_method import ( + CPUFP8KVCacheMethod, + ) + + for dtype, method, v_head_dim in ( + (torch.float8_e4m3fn, None, 64), + (torch.bfloat16, CPUFP8KVCacheMethod(), 64), + (torch.bfloat16, None, 32), + ): + with self.subTest(dtype=dtype, method=method, v_head_dim=v_head_dim): + pool = MHATokenToKVPool( + size=32, + page_size=16, + dtype=dtype, + head_num=2, + head_dim=64, + v_head_dim=v_head_dim, + layer_num=1, + device="cpu", + enable_memory_saver=False, + enable_alt_stream=False, + quant_method=method, + ) + host = get_mha_host_pool_cls( + pool.get_device_pool_infos()[0].buffer_info + )( + buffer_info=pool.get_device_pool_infos()[0].buffer_info, + device_capacity_tokens=pool.size, + start_layer=pool.start_layer, + end_layer=pool.end_layer, + host_to_device_ratio=2, + host_size=0, + page_size=16, + layout="page_first", + pin_memory=False, + ) + self.addCleanup(host.destroy) + expected_bytes = sum( + buf.numel() * buf.element_size() + for buf in (*pool.k_buffer, *pool.v_buffer) + ) // (pool.size + pool.page_size) + self.assertEqual(host.get_size_per_token(), expected_bytes) + def test_multiple_attention_rows_per_token(self): for rows_per_token in (1, 3): device_pool = MHATokenToKVPool( @@ -230,12 +423,18 @@ class TestHostKVCache(CustomTestCase): ): with self.subTest(rows_per_token=rows_per_token, layout=layout): host_pool = MHATokenToKVPoolHost( - device_pool=device_pool, + buffer_info=device_pool.get_device_pool_infos()[0].buffer_info, host_to_device_ratio=2.0, host_size=0, page_size=self.page_size, layout=layout, pin_memory=False, + device_capacity_tokens=device_pool.size + if device_pool.host_capacity_tokens is None + else device_pool.host_capacity_tokens, + start_layer=device_pool.start_layer, + end_layer=device_pool.end_layer, + write_back_staging=device_pool.hicache_write_back_staging, ) device_row = device_pool.k_buffer[0][0] row_bytes = device_row.numel() * device_row.element_size() @@ -292,7 +491,7 @@ class TestHostKVCache(CustomTestCase): def test_shm_allocator(self): shm_host_pool = MHATokenToKVPoolHost( - device_pool=self.device_pool, + buffer_info=self.device_pool.get_device_pool_infos()[0].buffer_info, host_to_device_ratio=2.0, host_size=0, page_size=self.page_size, @@ -300,6 +499,12 @@ class TestHostKVCache(CustomTestCase): pin_memory=False, device="cpu", allocator_type="shm", + device_capacity_tokens=self.device_pool.size + if self.device_pool.host_capacity_tokens is None + else self.device_pool.host_capacity_tokens, + start_layer=self.device_pool.start_layer, + end_layer=self.device_pool.end_layer, + write_back_staging=self.device_pool.hicache_write_back_staging, ) self.assertIsNotNone(shm_host_pool.fd) self.assertGreaterEqual(shm_host_pool.fd, 0) @@ -648,13 +853,24 @@ class TestDSAIndexerPoolDecl(CustomTestCase): decl = make_dsa_indexer_pool_decl(stub) storage_info = decl.storage_info anchor = MLATokenToKVPoolHost( - stub, + MLAHostAllocation( + dtype=stub.store_dtype, + kv_lora_rank=stub.kv_lora_rank, + qk_rope_head_dim=stub.qk_rope_head_dim, + kv_cache_dim=stub.kv_lora_rank + stub.qk_rope_head_dim, + size_per_token=(stub.kv_lora_rank + stub.qk_rope_head_dim) + * stub.store_dtype.itemsize + * stub.layer_num, + ), + device_capacity_tokens=stub.size, + start_layer=stub.start_layer, + end_layer=stub.end_layer, + target_layer_num=stub.layer_num, host_to_device_ratio=2, host_size=0, page_size=64, layout="page_first", pin_memory=False, - is_dummy=True, ) mirror = DSAIndexerPoolHost( decl=decl, @@ -684,13 +900,24 @@ class TestDSAIndexerPoolDecl(CustomTestCase): decl = make_dsa_indexer_pool_decl(stub) self.assertEqual(decl.owned_device_layers, (0, 3)) anchor = MLATokenToKVPoolHost( - stub, + MLAHostAllocation( + dtype=stub.store_dtype, + kv_lora_rank=stub.kv_lora_rank, + qk_rope_head_dim=stub.qk_rope_head_dim, + kv_cache_dim=stub.kv_lora_rank + stub.qk_rope_head_dim, + size_per_token=(stub.kv_lora_rank + stub.qk_rope_head_dim) + * stub.store_dtype.itemsize + * stub.layer_num, + ), + device_capacity_tokens=stub.size, + start_layer=stub.start_layer, + end_layer=stub.end_layer, + target_layer_num=stub.layer_num, host_to_device_ratio=2, host_size=0, page_size=64, layout="page_first", pin_memory=False, - is_dummy=True, ) mirror = DSAIndexerPoolHost( decl=decl, diff --git a/test/registered/unit/mem_cache/test_minimax_sparse_pool_host_unit.py b/test/registered/unit/mem_cache/test_minimax_sparse_pool_host_unit.py index a6cee776b7..4e36b557ce 100644 --- a/test/registered/unit/mem_cache/test_minimax_sparse_pool_host_unit.py +++ b/test/registered/unit/mem_cache/test_minimax_sparse_pool_host_unit.py @@ -90,7 +90,7 @@ class TestMiniMaxSparseHiCacheIntegration(unittest.TestCase): with self.subTest(layout=layout): pool = _make_cpu_minimax_sparse_pool(start_layer=0) kv_host = MHATokenToKVPoolHost( - device_pool=pool.main_pool, + buffer_info=pool.main_pool.get_device_pool_infos()[0].buffer_info, host_to_device_ratio=2.0, host_size=0, page_size=pool.page_size, @@ -98,6 +98,12 @@ class TestMiniMaxSparseHiCacheIntegration(unittest.TestCase): pin_memory=False, device="cpu", allocator_type="default", + device_capacity_tokens=pool.main_pool.size + if pool.main_pool.host_capacity_tokens is None + else pool.main_pool.host_capacity_tokens, + start_layer=pool.main_pool.start_layer, + end_layer=pool.main_pool.end_layer, + write_back_staging=pool.main_pool.hicache_write_back_staging, ) index_host = MHATokenToKOnlyPoolHost( pool.index_k_pool, @@ -255,7 +261,9 @@ class TestMiniMaxSparseHiCacheTransfer(unittest.TestCase): ALLOC_MEMORY_FUNCS["cuda"] = alloc_with_pin_memory try: kv_host = MHATokenToKVPoolHost( - device_pool=device_pool.main_pool, + buffer_info=device_pool.main_pool.get_device_pool_infos()[ + 0 + ].buffer_info, host_to_device_ratio=2.0, host_size=0, page_size=page_size, @@ -263,6 +271,12 @@ class TestMiniMaxSparseHiCacheTransfer(unittest.TestCase): pin_memory=pin_memory, device="cpu", allocator_type="default", + device_capacity_tokens=device_pool.main_pool.size + if device_pool.main_pool.host_capacity_tokens is None + else device_pool.main_pool.host_capacity_tokens, + start_layer=device_pool.main_pool.start_layer, + end_layer=device_pool.main_pool.end_layer, + write_back_staging=device_pool.main_pool.hicache_write_back_staging, ) index_host = MHATokenToKOnlyPoolHost( device_pool.index_k_pool, diff --git a/test/registered/unit/mem_cache/test_minimax_sparse_pool_pd_unit.py b/test/registered/unit/mem_cache/test_minimax_sparse_pool_pd_unit.py index 96921199d6..99a248e9d6 100644 --- a/test/registered/unit/mem_cache/test_minimax_sparse_pool_pd_unit.py +++ b/test/registered/unit/mem_cache/test_minimax_sparse_pool_pd_unit.py @@ -5,10 +5,7 @@ import torch from sglang.srt.disaggregation.utils import get_kv_transfer_buf_infos from sglang.srt.mem_cache.memory_pool import MiniMaxSparseKVPool -from sglang.srt.mem_cache.pool_host.mha import ( - HiSparseMHATokenToKVPoolHost, - MHATokenToKVPoolHost, -) +from sglang.srt.mem_cache.pool_host.mha import HiSparseMHATokenToKVPoolHost from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -64,30 +61,37 @@ class TestMiniMaxSparsePoolPD(CustomTestCase): self.assertEqual(item_lens[i], buf[0].nbytes * pool.page_size) def test_hisparse_host_registration(self): - """PD startup must register every sparse host K/V buffer with page strides.""" - pool = _make_k_only_pool(enable_hisparse=True) - host = HiSparseMHATokenToKVPoolHost.__new__(HiSparseMHATokenToKVPoolHost) - with patch( - "sglang.srt.mem_cache.pool_host.base.host_memory_budget_bytes", - return_value=1 << 30, + """PD registers host capacity and sparse PP layers, not resident device KV.""" + pool = _make_k_only_pool(start_layer=4, enable_hisparse=True) + with ( + patch( + "sglang.srt.mem_cache.pool_host.base.host_memory_budget_bytes", + return_value=1 << 30, + ), + patch("sglang.srt.mem_cache.pool_host.common._cuda_host_register"), + patch("sglang.srt.mem_cache.pool_host.base._cuda_host_unregister"), ): - MHATokenToKVPoolHost.__init__( - host, - device_pool=pool.main_pool, + host = HiSparseMHATokenToKVPoolHost( + buffer_info=pool.main_pool.get_device_pool_infos()[0].buffer_info, + device_capacity_tokens=16, + start_layer=pool.main_pool.start_layer, + end_layer=pool.main_pool.end_layer, host_to_device_ratio=2, - host_size=0, page_size=pool.page_size, - layout="layer_first", - pin_memory=False, ) - ptrs, lens, item_lens = get_kv_transfer_buf_infos(host) - buffers = list(host.k_buffer.unbind()) + list(host.v_buffer.unbind()) - self.assertEqual(len(buffers), 8) - self.assertEqual(ptrs, [buffer.data_ptr() for buffer in buffers]) - self.assertEqual(lens, [buffer.nbytes for buffer in buffers]) - self.assertEqual( - item_lens, [buffer[0].nbytes * pool.page_size for buffer in buffers] - ) + try: + self.assertEqual((host.start_layer, host.end_layer), (7, 11)) + self.assertEqual(host.size, 36) + ptrs, lens, item_lens = get_kv_transfer_buf_infos(host) + buffers = list(host.k_buffer.unbind()) + list(host.v_buffer.unbind()) + self.assertEqual(len(buffers), 8) + self.assertEqual(ptrs, [buffer.data_ptr() for buffer in buffers]) + self.assertEqual(lens, [36 * 2 * 8 * 4] * 8) + self.assertEqual(item_lens, [4 * 2 * 8 * 4] * 8) + device_ptrs = pool.main_pool.get_contiguous_buf_infos()[0] + self.assertTrue(set(ptrs).isdisjoint(device_ptrs)) + finally: + host.destroy() def test_pd_registration_separates_dense_and_sparse_layers(self): """Both PD peers must keep dense device KV out of the sparse transfer list.""" diff --git a/test/registered/unit/mem_cache/test_mla_host_dedup_primitives.py b/test/registered/unit/mem_cache/test_mla_host_dedup_primitives.py index bd11033d93..0007586f9c 100644 --- a/test/registered/unit/mem_cache/test_mla_host_dedup_primitives.py +++ b/test/registered/unit/mem_cache/test_mla_host_dedup_primitives.py @@ -5,6 +5,7 @@ from unittest import mock import torch from sglang.srt.environ import envs +from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool from sglang.srt.mem_cache.mla_host_dedup import ( MLAHostDedupBroadcaster, MLAHostDedupContext, @@ -14,7 +15,7 @@ from sglang.srt.mem_cache.pool_host.dsa import ( DSAIndexerPoolHost, make_dsa_indexer_pool_decl, ) -from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost +from sglang.srt.mem_cache.pool_host.mla import MLAHostAllocation, MLATokenToKVPoolHost from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci(est_time=1, suite="base-a-test-cpu") @@ -55,13 +56,28 @@ class TestMLAHostDedupPrimitives(unittest.TestCase): end_layer=2, ) mla_host = MLATokenToKVPoolHost( - mla_device_pool, + MLAHostAllocation( + dtype=mla_device_pool.store_dtype, + kv_lora_rank=mla_device_pool.kv_lora_rank, + qk_rope_head_dim=mla_device_pool.qk_rope_head_dim, + kv_cache_dim=( + mla_device_pool.kv_lora_rank + mla_device_pool.qk_rope_head_dim + ), + size_per_token=( + mla_device_pool.kv_lora_rank + mla_device_pool.qk_rope_head_dim + ) + * mla_device_pool.store_dtype.itemsize + * (mla_device_pool.layer_num), + ), host_to_device_ratio=2, host_size=0, page_size=2, layout="page_first", pin_memory=False, - is_dummy=True, + device_capacity_tokens=mla_device_pool.size, + start_layer=mla_device_pool.start_layer, + end_layer=mla_device_pool.end_layer, + target_layer_num=mla_device_pool.layer_num, ) self.assertTrue(mla_host._is_dummy) @@ -108,22 +124,19 @@ class TestMLAHostDedupPrimitives(unittest.TestCase): for dcp_size in (1, 2): for host_size in (0, 0.000002): with self.subTest(dcp_size=dcp_size, host_size=host_size): - pool = _device_pool_stub( + pool = MLATokenToKVPool( + size=8, + page_size=2, layer_num=2, - store_dtype=torch.float16, + dtype=torch.float16, kv_lora_rank=4, qk_rope_head_dim=2, - size=8, - host_capacity_tokens=16, start_layer=7, end_layer=9, device="cpu", - kv_buffer=[torch.zeros(10, 1, 6) for _ in range(2)], - ) - pool.data_ptrs = torch.tensor( - [buffer.data_ptr() for buffer in pool.kv_buffer], - dtype=torch.uint64, + enable_memory_saver=False, ) + pool.host_capacity_tokens = 16 kwargs = dict( host_to_device_ratio=2, host_size=host_size, @@ -132,8 +145,40 @@ class TestMLAHostDedupPrimitives(unittest.TestCase): pin_memory=False, dcp_size=dcp_size, ) - host = MLATokenToKVPoolHost(pool, **kwargs) - dummy = MLATokenToKVPoolHost(pool, **kwargs, is_dummy=True) + host = MLATokenToKVPoolHost( + pool.get_device_pool_infos()[0].buffer_info, + device_capacity_tokens=( + pool.size + if pool.host_capacity_tokens is None + else pool.host_capacity_tokens + ), + start_layer=pool.start_layer, + end_layer=pool.end_layer, + target_layer_num=( + (pool.layer_num + pool.layer_shard_size - 1) + // pool.layer_shard_size + if pool.layer_shard_enabled + else pool.layer_num + ), + layer_shard_enabled=pool.layer_shard_enabled, + **kwargs, + ) + dummy = MLATokenToKVPoolHost( + MLAHostAllocation( + dtype=pool.store_dtype, + kv_lora_rank=pool.kv_lora_rank, + qk_rope_head_dim=pool.qk_rope_head_dim, + kv_cache_dim=(pool.kv_lora_rank + pool.qk_rope_head_dim), + size_per_token=(pool.kv_lora_rank + pool.qk_rope_head_dim) + * pool.store_dtype.itemsize + * (pool.layer_num), + ), + device_capacity_tokens=pool.host_capacity_tokens, + start_layer=pool.start_layer, + end_layer=pool.end_layer, + target_layer_num=pool.layer_num, + **kwargs, + ) self.addCleanup(host.destroy) self.addCleanup(dummy.destroy) self.assertEqual( diff --git a/test/registered/unit/mem_cache/test_mxfp8_mha_pool_host_unit.py b/test/registered/unit/mem_cache/test_mxfp8_mha_pool_host_unit.py index f216c4f565..cbd03db24b 100644 --- a/test/registered/unit/mem_cache/test_mxfp8_mha_pool_host_unit.py +++ b/test/registered/unit/mem_cache/test_mxfp8_mha_pool_host_unit.py @@ -2,7 +2,6 @@ through L2 alongside the fp8 payload.""" import unittest -from types import SimpleNamespace from unittest import mock import torch @@ -56,7 +55,7 @@ def _make_host(k_rows, v_rows): host.page_num = 4 host.k_sf_page_bytes = SF_PAGE_BYTES host.v_sf_page_bytes = SF_PAGE_BYTES - host.device_pool = SimpleNamespace(layer_shard_enabled=False, layer_num=LAYER_NUM) + host.target_layer_num = LAYER_NUM host.k_scale_host = torch.zeros( host.page_num, LAYER_NUM, SF_PAGE_BYTES, dtype=torch.uint8 ) @@ -80,9 +79,26 @@ def _make_host(k_rows, v_rows): class TestMXFP8MHATokenToKVPoolHost(CustomTestCase): def test_factory_selects_mxfp8_host_pool(self): - mxfp8_pool = MHATokenToKVPoolMXFP8.__new__(MHATokenToKVPoolMXFP8) - mxfp8_pool.head_dim = mxfp8_pool.v_head_dim = 128 - plain_pool = SimpleNamespace(head_dim=4, v_head_dim=4) + from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool + + def info(pool_type): + return ( + pool_type( + size=128, + page_size=128, + dtype=torch.float16, + head_num=1, + head_dim=32, + layer_num=1, + device="cpu", + enable_memory_saver=False, + ) + .get_device_pool_infos()[0] + .buffer_info + ) + + mxfp8_pool = info(MHATokenToKVPoolMXFP8) + plain_pool = info(MHATokenToKVPool) self.assertIs(get_mha_host_pool_cls(mxfp8_pool), MHATokenToKVPoolMXFP8Host) self.assertIs(get_mha_host_pool_cls(plain_pool), MHATokenToKVPoolHost) @@ -153,7 +169,7 @@ class TestMXFP8MHATokenToKVPoolHost(CustomTestCase): ) as one_layer, ): host.backup_from_device_all_layer( - host.device_pool, host_indices, device_indices, io_backend="kernel" + object(), host_indices, device_indices, io_backend="kernel" ) self.assertEqual(staged.call_count, 2) # Device page 2 landed in host page 1, device page 3 in host page 0. @@ -168,7 +184,7 @@ class TestMXFP8MHATokenToKVPoolHost(CustomTestCase): rows.zero_() for layer_id in range(LAYER_NUM): host.load_to_device_per_layer( - host.device_pool, + object(), host_indices, device_indices, layer_id,