diff --git a/python/sglang/srt/configs/inkling.py b/python/sglang/srt/configs/inkling.py index f4f8f115ef..b3fa0ec4d6 100644 --- a/python/sglang/srt/configs/inkling.py +++ b/python/sglang/srt/configs/inkling.py @@ -328,6 +328,9 @@ class InklingMMConfig(PretrainedConfig): if isinstance(text_config, InklingModelConfig) else InklingModelConfig(**(text_config or {})) ) + if isinstance(mtp_config, dict) and "n_layers" in mtp_config: + # HiCache reads the draft depth through the text config. + self.text_config.num_nextn_predict_layers = mtp_config["n_layers"] if isinstance(mtp_config, dict) and mtp_config.get("local_layer_ids"): # Banded MTP head: the checkpoint declares its sliding-window draft # depths on mtp_config. Canonicalize onto text_config so every diff --git a/python/sglang/srt/configs/model_config.py b/python/sglang/srt/configs/model_config.py index 9ef405b856..96dc392d19 100644 --- a/python/sglang/srt/configs/model_config.py +++ b/python/sglang/srt/configs/model_config.py @@ -2648,21 +2648,15 @@ def get_hybrid_layer_ids( i for i, x in enumerate(layer_types) if x == "full_attention" ] elif "InklingForConditionalGenerationMTP" in model_architectures: - # One block per MTP depth; a banded head marks its sliding-window depths - # in mtp_local_layer_ids. The per-depth pool routing in the KV-cache - # mixin is authoritative; this keeps model_config's swa/full lists - # self-consistent for other consumers. - mtp_local_layer_ids = hf_text_config.mtp_local_layer_ids - if mtp_local_layer_ids: - num_depths = hf_text_config.num_nextn_predict_layers - local_set = set(mtp_local_layer_ids) - swa_attention_layer_ids = sorted(local_set) - full_attention_layer_ids = [ - i for i in range(num_depths) if i not in local_set - ] - else: - swa_attention_layer_ids = [] - full_attention_layer_ids = [0] + # Runner-local layer setup selects one depth from these shared lists, + # including when every draft depth uses full attention. + local_set = set(hf_text_config.mtp_local_layer_ids) + swa_attention_layer_ids = sorted(local_set) + full_attention_layer_ids = [ + i + for i in range(hf_text_config.num_nextn_predict_layers) + if i not in local_set + ] elif "InklingForConditionalGeneration" in model_architectures: local_layer_ids = hf_text_config.local_layer_ids local_layer_id_set = set(local_layer_ids) diff --git a/python/sglang/srt/mem_cache/base_swa_memory_pool.py b/python/sglang/srt/mem_cache/base_swa_memory_pool.py index 2b44bf942e..3d381ab250 100644 --- a/python/sglang/srt/mem_cache/base_swa_memory_pool.py +++ b/python/sglang/srt/mem_cache/base_swa_memory_pool.py @@ -18,6 +18,10 @@ class BaseSWAKVPool(KVCache): # Set when SWA KV is a per-request ring of this many tokens (addressed by # req_pool_idx) rather than a paged token pool; SWA is then not budgeted per token. swa_req_ring_size: Optional[int] = None + # Independent draft SWA buffers can consume full-token IDs directly instead + # of the target SWA allocator's physical IDs. HiCache must preserve that + # index ownership even though both buffers implement sliding attention. + swa_uses_full_indices: bool = False @abc.abstractmethod def register_mapping(self, full_to_swa_index_mapping: torch.Tensor) -> None: diff --git a/python/sglang/srt/mem_cache/cache_init_params.py b/python/sglang/srt/mem_cache/cache_init_params.py index 7722516d2a..9794b59cc2 100644 --- a/python/sglang/srt/mem_cache/cache_init_params.py +++ b/python/sglang/srt/mem_cache/cache_init_params.py @@ -62,3 +62,4 @@ class CacheInitParams: tree_core_backend: Optional[str] = dataclasses.field(default=None, kw_only=True) mtp_draft_device_pools: tuple[object, ...] = () + mtp_draft_mamba_pools: tuple[object, ...] = () diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py index 7f899abce8..b2d571fb72 100644 --- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py +++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py @@ -897,10 +897,12 @@ class HybridCacheController(BaseHiCacheController): if draft_host_layer is None: continue + # Finish every draft before the final target-layer event, + # including chains deeper than the target model. def draft_layer_mapper( layer_id: int, *, - expected_layer_id: int = depth, + expected_layer_id: int = min(depth, self.transfer_layer_id_max - 1), host_layer_id: int = draft_host_layer, ) -> Optional[int]: if layer_id == expected_layer_id: diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py index 4aadb2a1a0..d8ff9d9daf 100644 --- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py +++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py @@ -288,13 +288,13 @@ def _uses_unified_page_envelope_host( full_kv_pool: Any, swa_kv_pool: Any, use_mla: bool, - mtp_swa_device_pools: tuple[Any, ...] = (), + mtp_device_pools: tuple[Any, ...] = (), ) -> bool: from sglang.srt.mem_cache.unified_memory_pool import UnifiedMHATokenToKVPool return ( not use_mla - and not mtp_swa_device_pools + and not mtp_device_pools and all( isinstance(pool, UnifiedMHATokenToKVPool) for pool in (full_kv_pool, swa_kv_pool) @@ -320,14 +320,29 @@ def build_hybrid_swa_group( host_swa_evict_fn: Optional[Callable[[int], Any]] = None, device_swa_evict_fn: Optional[Callable[[int], Any]] = None, swa_attn_allocator: Any = None, + mtp_full_device_pools: tuple[Any, ...] = (), mtp_swa_device_pools: tuple[Any, ...] = (), + mtp_swa_full_device_pools: tuple[Any, ...] = (), + transfer_layer_id_max: Optional[int] = None, ) -> HostPoolGroup: """Anchor (full) + SWA host pool group for a hybrid-SWA device pool.""" - transfer_layer_id_max = ( - max(full_layer_mapping.keys() | swa_layer_mapping.keys()) + 1 - ) + if transfer_layer_id_max is None: + transfer_layer_id_max = ( + max(full_layer_mapping.keys() | swa_layer_mapping.keys()) + 1 + ) + if mtp_swa_full_device_pools and kv_host_size: + # The sidecar has the anchor's token capacity. Reserve its bytes from + # the full-KV budget rather than exceeding an explicit --hicache-size. + anchor_pools = (full_kv_pool, *mtp_full_device_pools) + shares = _split_hicache_size( + kv_host_size, (*anchor_pools, *mtp_swa_full_device_pools) + ) + kv_host_size = sum(shares[: len(anchor_pools)]) if _uses_unified_page_envelope_host( - full_kv_pool, swa_kv_pool, use_mla, mtp_swa_device_pools + full_kv_pool, + swa_kv_pool, + use_mla, + mtp_full_device_pools + mtp_swa_device_pools + mtp_swa_full_device_pools, ): memory = get_memory() kv_host_pool, swa_host_pool = ( @@ -349,6 +364,7 @@ def build_hybrid_swa_group( page_size=page_size, use_mla=use_mla, host_size=kv_host_size, + mtp_draft_device_pools=mtp_full_device_pools, pool_label="full", ) swa_host_pool = build_kv_host_pool( @@ -359,6 +375,13 @@ def build_hybrid_swa_group( mtp_draft_device_pools=mtp_swa_device_pools, pool_label="swa", ) + if mtp_full_device_pools: + full_layer_mapping = with_packed_draft_layer_mapping( + full_layer_mapping, + transfer_layer_start=transfer_layer_id_max, + target_device_layer_num=full_kv_pool.layer_num, + draft_layer_num=len(mtp_full_device_pools), + ) if mtp_swa_device_pools: swa_layer_mapping = with_packed_draft_layer_mapping( swa_layer_mapping, @@ -366,16 +389,18 @@ def build_hybrid_swa_group( target_device_layer_num=swa_kv_pool.layer_num, draft_layer_num=len(mtp_swa_device_pools), ) - return HostPoolGroup( + group = HostPoolGroup( [ build_pool_entry( name=PoolName.KV, host_pool=kv_host_pool, device_pool=full_kv_pool, layer_mapping=full_layer_mapping, - transfer_layer_id_max=transfer_layer_id_max, + transfer_layer_id_max=transfer_layer_id_max + + len(mtp_full_device_pools), is_anchor=True, host_evict_fn=host_full_evict_fn if has_shared_arena else None, + packed_draft_device_pools=mtp_full_device_pools, ), build_pool_entry( name=PoolName.SWA, @@ -390,6 +415,53 @@ def build_hybrid_swa_group( ), ] ) + if mtp_swa_full_device_pools: + # SWA drafts with full-token addressing may have different head geometry + # from full attention. Use a separate host pool with KV indices instead of + # packing their bytes into either target pool. + first, *rest = mtp_swa_full_device_pools + draft_host_pool = get_mha_host_pool_cls(first)( + first, + host_to_device_ratio=kv_host_pool.size / first.size, + host_size=0, + page_size=page_size, + layout=get_memory().hicache_mem_layout, + allocator_type=_get_allocator_type(), + mtp_draft_device_pools=rest, + pool_label="draft_swa", + ) + group.add_entry( + build_pool_entry( + name=PoolName.DRAFT_SWA, + host_pool=draft_host_pool, + device_pool=first, + layer_mapping=with_packed_draft_layer_mapping( + {0: 0}, + transfer_layer_start=transfer_layer_id_max, + target_device_layer_num=first.layer_num, + draft_layer_num=len(rest), + ), + transfer_layer_id_max=transfer_layer_id_max + len(rest), + packed_draft_device_pools=tuple(rest), + ) + ) + return group + + +def _mtp_swa_device_pools( + params: CacheInitParams, *, full_indices: bool +) -> tuple[Any, ...]: + return tuple( + pool.swa_kv_pool + for pool in params.mtp_draft_device_pools + if pool.swa_kv_pool.layer_num and pool.swa_uses_full_indices == full_indices + ) + + +def _mtp_swa_sidecar_specs(group: HostPoolGroup) -> list[SidecarPoolSpec]: + if PoolName.DRAFT_SWA not in group.entry_map: + return [] + return [SidecarPoolSpec(PoolName.DRAFT_SWA, indices_from_pool=PoolName.KV)] def build_kv_only_stack( @@ -458,15 +530,18 @@ def build_hybrid_swa_stack( transfer_layer_id_max = ( max(full_layer_mapping.keys() | swa_layer_mapping.keys()) + 1 ) - # MTP draft pools follow the target SWA layout; select their SWA storage. - mtp_swa_device_pools = tuple( - pool.swa_kv_pool for pool in params.mtp_draft_device_pools + # Each draft depth may use full or sliding-window attention. + mtp_full_device_pools = tuple( + pool.full_kv_pool + for pool in params.mtp_draft_device_pools + if pool.full_kv_pool.layer_num ) + mtp_swa_device_pools = _mtp_swa_device_pools(params, full_indices=False) kv_host_size = swa_host_size = None memory = get_memory() if memory.hicache_size > 0 and not _uses_unified_page_envelope_host( - full_kv_pool, swa_kv_pool, use_mla, mtp_swa_device_pools + full_kv_pool, swa_kv_pool, use_mla, params.mtp_draft_device_pools ): kv_host_size, swa_host_size = _split_hicache_size( memory.hicache_size, (full_kv_pool, swa_kv_pool) @@ -486,7 +561,9 @@ def build_hybrid_swa_stack( device_swa_evict_fn=device_swa_evict_fn, # For SWA hybrid, device allocation goes through the inner allocator. swa_attn_allocator=params.token_to_kv_pool_allocator.swa_attn_allocator, + mtp_full_device_pools=mtp_full_device_pools, mtp_swa_device_pools=mtp_swa_device_pools, + mtp_swa_full_device_pools=_mtp_swa_device_pools(params, full_indices=True), ) cache_controller = HybridCacheController( params.token_to_kv_pool_allocator, @@ -1104,9 +1181,16 @@ def build_deepseek_v4_hicache_stack( return host_pool_group, cache_controller -def build_mamba_host_pool(mamba_pool, *, host_size: float, layout: str): +def build_mamba_host_pool( + mamba_pool, *, host_size: float, layout: str, mtp_draft_device_pools=() +): (pool_info,) = mamba_pool.get_device_pool_infos() pool_info.buffer_info.validate(layer_ids=pool_info.layer_ids) + draft_buffer_infos = [] + for draft_pool in mtp_draft_device_pools: + (draft_info,) = draft_pool.get_device_pool_infos() + draft_info.buffer_info.validate(layer_ids=draft_info.layer_ids) + draft_buffer_infos.append(draft_info.buffer_info) capacity = mamba_pool.host_capacity_tokens if capacity is None: capacity = mamba_pool.size @@ -1115,6 +1199,7 @@ def build_mamba_host_pool(mamba_pool, *, host_size: float, layout: str): get_memory().hicache_ratio, host_size, device_capacity=capacity, + mtp_draft_buffer_infos=draft_buffer_infos, allocator_type=_get_allocator_type(), layout=layout, ) @@ -1187,6 +1272,13 @@ def build_hybrid_mamba_stack( mamba_pool, host_size=mamba_host_size, layout=mamba_layout, + mtp_draft_device_pools=params.mtp_draft_mamba_pools, + ) + mamba_layer_mapping = with_packed_draft_layer_mapping( + mamba_layer_mapping, + transfer_layer_start=transfer_layer_id_max, + target_device_layer_num=mamba_pool.num_mamba_layers, + draft_layer_num=len(params.mtp_draft_mamba_pools), ) entries = _build_pool_entries(config=config, root_host_pool=kv_host_pool) + [ build_pool_entry( @@ -1194,12 +1286,14 @@ def build_hybrid_mamba_stack( host_pool=mamba_host_pool, device_pool=mamba_pool, layer_mapping=mamba_layer_mapping, - transfer_layer_id_max=transfer_layer_id_max, + transfer_layer_id_max=transfer_layer_id_max + + len(params.mtp_draft_mamba_pools), host_evict_fn=host_mamba_evict_fn, device_evict_fn=device_mamba_evict_fn, device_alloc_fn=mamba_allocator.alloc, device_free_fn=mamba_allocator.free, - ) + packed_draft_device_pools=params.mtp_draft_mamba_pools, + ), ] if qsa_pool is not None: entries.append( @@ -1281,57 +1375,54 @@ def build_hybrid_mamba_swa_stack( kv_host_size, swa_host_size, mamba_host_size = _split_hicache_size( get_memory().hicache_size, (full_kv_pool, swa_kv_pool, mamba_pool) ) - kv_host_pool = build_kv_host_pool( - kv_pool=full_kv_pool, + host_pool_group = build_hybrid_swa_group( page_size=page_size, + full_kv_pool=full_kv_pool, + swa_kv_pool=swa_kv_pool, + full_layer_mapping=full_layer_mapping, + swa_layer_mapping=swa_layer_mapping, use_mla=False, - host_size=kv_host_size, - pool_label="full", - ) - swa_host_pool = build_kv_host_pool( - kv_pool=swa_kv_pool, - page_size=page_size, - use_mla=False, - host_size=swa_host_size, - pool_label="swa", + kv_host_size=kv_host_size, + swa_host_size=swa_host_size, + host_swa_evict_fn=host_swa_evict_fn, + device_swa_evict_fn=device_swa_evict_fn, + swa_attn_allocator=swa_attn_allocator, + mtp_full_device_pools=tuple( + pool.full_kv_pool + for pool in params.mtp_draft_device_pools + if pool.full_kv_pool.layer_num + ), + mtp_swa_device_pools=_mtp_swa_device_pools(params, full_indices=False), + mtp_swa_full_device_pools=_mtp_swa_device_pools(params, full_indices=True), + transfer_layer_id_max=transfer_layer_id_max, ) mamba_host_pool = build_mamba_host_pool( mamba_pool, host_size=mamba_host_size, layout=get_memory().hicache_mem_layout, + mtp_draft_device_pools=params.mtp_draft_mamba_pools, ) - entries = [ - build_pool_entry( - name=PoolName.KV, - host_pool=kv_host_pool, - device_pool=full_kv_pool, - layer_mapping=full_layer_mapping, - transfer_layer_id_max=transfer_layer_id_max, - is_anchor=True, - ), - build_pool_entry( - name=PoolName.SWA, - host_pool=swa_host_pool, - device_pool=swa_kv_pool, - layer_mapping=swa_layer_mapping, - transfer_layer_id_max=transfer_layer_id_max, - host_evict_fn=host_swa_evict_fn, - device_evict_fn=device_swa_evict_fn, - **_swa_allocation_callbacks(swa_attn_allocator), - ), + mamba_layer_mapping = with_packed_draft_layer_mapping( + mamba_layer_mapping, + transfer_layer_start=transfer_layer_id_max, + target_device_layer_num=mamba_pool.num_mamba_layers, + draft_layer_num=len(params.mtp_draft_mamba_pools), + ) + host_pool_group.add_entry( build_pool_entry( name=PoolName.MAMBA, host_pool=mamba_host_pool, device_pool=mamba_pool, layer_mapping=mamba_layer_mapping, - transfer_layer_id_max=transfer_layer_id_max, + transfer_layer_id_max=transfer_layer_id_max + + len(params.mtp_draft_mamba_pools), host_evict_fn=host_mamba_evict_fn, device_evict_fn=device_mamba_evict_fn, device_alloc_fn=mamba_allocator.alloc, device_free_fn=mamba_allocator.free, + packed_draft_device_pools=params.mtp_draft_mamba_pools, ), - ] - host_pool_group = HostPoolGroup(entries) + ) cache_controller = HybridCacheController( params.token_to_kv_pool_allocator, host_pool_group, @@ -1916,6 +2007,7 @@ class _SwaStrategy(StackStrategy): ComponentType.SWA: host_pool_group.get_pool(PoolName.SWA), }, pools_desc="Full + SWA", + sidecars=_mtp_swa_sidecar_specs(host_pool_group), ) @@ -1985,6 +2077,7 @@ class _MambaSwaStrategy(StackStrategy): }, register_req_to_token_counter=True, pools_desc="KV + SWA + MAMBA", + sidecars=_mtp_swa_sidecar_specs(host_pool_group), ) diff --git a/python/sglang/srt/mem_cache/kv_cache_builder.py b/python/sglang/srt/mem_cache/kv_cache_builder.py index 4158412db9..488fa72667 100644 --- a/python/sglang/srt/mem_cache/kv_cache_builder.py +++ b/python/sglang/srt/mem_cache/kv_cache_builder.py @@ -361,6 +361,7 @@ def build_kv_cache( chunked_prefill_size=effective_chunked_prefill_size, sliding_window_size=sliding_window_size, mtp_draft_device_pools=mtp_draft_device_pools, + mtp_draft_mamba_pools=tp_worker.model_runner.mtp_draft_mamba_pools, ) tree_context = TreeCacheBuildContext( diff --git a/python/sglang/srt/mem_cache/kv_cache_configurator.py b/python/sglang/srt/mem_cache/kv_cache_configurator.py index 54f1a9e437..47b6fb2887 100644 --- a/python/sglang/srt/mem_cache/kv_cache_configurator.py +++ b/python/sglang/srt/mem_cache/kv_cache_configurator.py @@ -2641,6 +2641,7 @@ class KVCacheConfigurator: ) identity_mapping[-1] = -1 token_to_kv_pool.register_mapping(identity_mapping) + token_to_kv_pool.swa_uses_full_indices = True elif not uses_unified_virtual_ids: token_to_kv_pool.register_mapping( swa_allocator.full_to_swa_index_mapping diff --git a/python/sglang/srt/mem_cache/pool_host/mamba.py b/python/sglang/srt/mem_cache/pool_host/mamba.py index b08bfe8250..ea36ee7779 100644 --- a/python/sglang/srt/mem_cache/pool_host/mamba.py +++ b/python/sglang/srt/mem_cache/pool_host/mamba.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging import os import threading -from typing import Optional +from typing import Optional, Sequence import numpy as np import torch @@ -72,6 +72,8 @@ def _npu_hicache_mamba_io_mode() -> str: class MambaPoolHost(HostKVCache): + mtp_draft_buffer_infos: tuple[MambaStateBufferInfo, ...] = () + # Class-level defaults so a model without side state, and the test stubs that # skip __init__, still read an empty pair rather than raising. slot_state_device_tensors: tuple = () @@ -88,11 +90,13 @@ class MambaPoolHost(HostKVCache): device: str = "cpu", allocator_type: str = "default", layout: str = "layer_first", + mtp_draft_buffer_infos: Sequence[MambaStateBufferInfo] = (), ): buffer_info.validate() self.buffer_info = buffer_info self.device_pool = None self._buffer_device = buffer_info.temporal.device + self.mtp_draft_buffer_infos = tuple(mtp_draft_buffer_infos) self.page_size = 1 assert layout in [ @@ -104,7 +108,31 @@ class MambaPoolHost(HostKVCache): self.pin_memory = pin_memory self.device = device self.allocator = get_allocator_from_storage(allocator_type) - self.num_mamba_layers = buffer_info.temporal.shape[0] + self.num_mamba_layers = buffer_info.temporal.shape[0] + len( + self.mtp_draft_buffer_infos + ) + + buffer_infos = (buffer_info, *self.mtp_draft_buffer_infos) + for draft_info in self.mtp_draft_buffer_infos: + draft_info.validate() + assert draft_info.temporal.shape[0] == 1 + assert len(draft_info.conv) == len(buffer_info.conv) + for target, draft in zip( + (*buffer_info.conv, buffer_info.temporal), + (*draft_info.conv, draft_info.temporal), + strict=True, + ): + assert target.shape[1:] == draft.shape[1:] + assert target.dtype == draft.dtype + assert target.device == draft.device + + self.temporal_device_layers = [ + layer for info in buffer_infos for layer in info.temporal.unbind(0) + ] + self.conv_device_layers = [ + [layer for info in buffer_infos for layer in info.conv[i].unbind(0)] + for i in range(len(buffer_info.conv)) + ] self.conv_state_shapes = [ conv_state.shape[2:] for conv_state in self.buffer_info.conv @@ -119,7 +147,9 @@ class MambaPoolHost(HostKVCache): self.dtype = self.conv_dtype # Registered side states share the checkpoint slot with conv/temporal, # so a host round trip that skips them leaves the previous occupant's rows. - self.slot_state_device_tensors = list(buffer_info.extra_checkpoint_buffers) + self.slot_state_device_tensors = [ + buf for info in buffer_infos for buf in info.extra_checkpoint_buffers + ] self.slot_state_buffers = [] self.size_per_token = self.get_size_per_token() @@ -158,20 +188,17 @@ class MambaPoolHost(HostKVCache): ) self.temporal_device_ptrs = torch.tensor( - [ - self.buffer_info.temporal[i].data_ptr() - for i in range(self.num_mamba_layers) - ], + [layer.data_ptr() for layer in self.temporal_device_layers], dtype=torch.uint64, device=self._buffer_device, ) self.conv_device_ptrs = [ torch.tensor( - [conv_state[i].data_ptr() for i in range(self.num_mamba_layers)], + [layer.data_ptr() for layer in layers], dtype=torch.uint64, device=self._buffer_device, ) - for conv_state in self.buffer_info.conv + for layers in self.conv_device_layers ] self.kv_buffer = self.init_kv_buffer() @@ -543,7 +570,7 @@ class MambaPoolHost(HostKVCache): @staticmethod def _copy_tensor_all_layers_lf_pf( - src_layers: torch.Tensor, + src_layers: torch.Tensor | Sequence[torch.Tensor], dst: torch.Tensor, src_indices: torch.Tensor, dst_indices: torch.Tensor, @@ -601,7 +628,17 @@ class MambaPoolHost(HostKVCache): page_size=1, ) elif io_backend == "kernel_ascend": - if _npu_hicache_mamba_io_mode() == "async": + if not isinstance(src_layers, torch.Tensor): + # Packed drafts live in separate allocations; the NPU bulk + # kernels require a single contiguous layer-first tensor. + host_indices = dst_indices.to(dtype=torch.int64, device=dst.device) + for layer_id, src in enumerate(src_layers): + device_indices = src_indices.to( + dtype=torch.int64, device=src.device + ) + values = src.index_select(0, device_indices).to(device=dst.device) + dst[:, layer_id, 0].index_copy_(0, host_indices, values) + elif _npu_hicache_mamba_io_mode() == "async": transfer_state_all_layer_direct_lf_pf( device_states=[src_layers], host_states=[dst], @@ -646,13 +683,28 @@ class MambaPoolHost(HostKVCache): *, is_draft: bool = False, ): + temporal_layers = ( + self.temporal_device_layers + if self.mtp_draft_buffer_infos + else self.buffer_info.temporal + ) + conv_layers = ( + self.conv_device_layers + if self.mtp_draft_buffer_infos + else self.buffer_info.conv + ) + # Side state is not per-layer, so it rides on the first layer's call -- # before that layer's completion event, which is what every reader of a # restored slot waits on. if layer_id == 0: self._transfer_slot_states(host_indices, device_indices, to_device=True) if self.layout in ["page_first", "page_first_direct"]: - if io_backend == "kernel_ascend" and transfer_mamba_state is not None: + if ( + io_backend == "kernel_ascend" + and transfer_mamba_state is not None + and not self.mtp_draft_buffer_infos + ): # NPU: transfer all layers at once via dedicated kernel. # layer_id == 0 covers every layer, so later calls must skip. if layer_id == 0: @@ -679,7 +731,7 @@ class MambaPoolHost(HostKVCache): if self.temporal_state_elem_size > 0: self._copy_tensor_pf_lf( src=self.temporal_buffer, - dst=self.buffer_info.temporal[layer_id], + dst=temporal_layers[layer_id], src_indices=host_indices, dst_indices=device_indices, layer_id=layer_id, @@ -689,7 +741,7 @@ class MambaPoolHost(HostKVCache): for conv_idx in range(len(self.conv_state_shapes)): self._copy_tensor_pf_lf( src=self.conv_buffer[conv_idx], - dst=self.buffer_info.conv[conv_idx][layer_id], + dst=conv_layers[conv_idx][layer_id], src_indices=host_indices, dst_indices=device_indices, layer_id=layer_id, @@ -699,7 +751,7 @@ class MambaPoolHost(HostKVCache): else: self._copy_tensor( self.temporal_buffer[layer_id], - self.buffer_info.temporal[layer_id], + temporal_layers[layer_id], host_indices, device_indices, io_backend, @@ -707,7 +759,7 @@ class MambaPoolHost(HostKVCache): for conv_idx in range(len(self.conv_state_shapes)): self._copy_tensor( self.conv_buffer[conv_idx][layer_id], - self.buffer_info.conv[conv_idx][layer_id], + conv_layers[conv_idx][layer_id], host_indices, device_indices, io_backend, @@ -716,6 +768,16 @@ class MambaPoolHost(HostKVCache): def backup_from_device_all_layer( self, device_pool, host_indices, device_indices, io_backend="kernel" ): + temporal_layers = ( + self.temporal_device_layers + if self.mtp_draft_buffer_infos + else self.buffer_info.temporal + ) + conv_layers = ( + self.conv_device_layers + if self.mtp_draft_buffer_infos + else self.buffer_info.conv + ) if self.layout in ["page_first", "page_first_direct"]: if io_backend == "kernel" and host_indices.device != device_indices.device: # The mamba JIT kernel wants both index tensors on the device; @@ -725,7 +787,7 @@ class MambaPoolHost(HostKVCache): # no ssm state on conv-only models: a 0-size batched memcpy errors if self.temporal_state_elem_size > 0: self._copy_tensor_all_layers_lf_pf( - src_layers=self.buffer_info.temporal, + src_layers=temporal_layers, dst=self.temporal_buffer, src_indices=device_indices, dst_indices=host_indices, @@ -737,7 +799,7 @@ class MambaPoolHost(HostKVCache): ) for conv_idx in range(len(self.conv_state_shapes)): self._copy_tensor_all_layers_lf_pf( - src_layers=self.buffer_info.conv[conv_idx], + src_layers=conv_layers[conv_idx], dst=self.conv_buffer[conv_idx], src_indices=device_indices, dst_indices=host_indices, @@ -750,7 +812,7 @@ class MambaPoolHost(HostKVCache): else: for layer_id in range(self.num_mamba_layers): self._copy_tensor( - self.buffer_info.temporal[layer_id], + temporal_layers[layer_id], self.temporal_buffer[layer_id], device_indices, host_indices, @@ -758,7 +820,7 @@ class MambaPoolHost(HostKVCache): ) for conv_idx in range(len(self.conv_state_shapes)): self._copy_tensor( - self.buffer_info.conv[conv_idx][layer_id], + conv_layers[conv_idx][layer_id], self.conv_buffer[conv_idx][layer_id], device_indices, host_indices, diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index 5aa907ecbc..166724eeae 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -383,6 +383,7 @@ class ModelRunner: self.req_to_token_pool = req_to_token_pool self.token_to_kv_pool_allocator = token_to_kv_pool_allocator self.mtp_draft_device_pools = () + self.mtp_draft_mamba_pools = () self.is_hybrid_swa = model_config.is_hybrid_swa self.is_hybrid_swa_compress = model_config.is_hybrid_swa_compress self.use_mla_backend = self.model_config.attention_arch == AttentionArch.MLA diff --git a/python/sglang/srt/speculative/base_spec_worker.py b/python/sglang/srt/speculative/base_spec_worker.py index ae7910ea67..891956ca89 100644 --- a/python/sglang/srt/speculative/base_spec_worker.py +++ b/python/sglang/srt/speculative/base_spec_worker.py @@ -260,6 +260,7 @@ class BaseSpecWorker(ABC): def _build_hicache_draft_plan(self) -> HiCacheDraftPlan: target_model_runner = self.target_worker.model_runner target_model_runner.mtp_draft_device_pools = () + target_model_runner.mtp_draft_mamba_pools = () spec_algorithm = target_model_runner.spec_algorithm if not ( get_memory().enable_hierarchical_cache @@ -272,16 +273,29 @@ class BaseSpecWorker(ABC): if not draft_runners: return HiCacheDraftPlan() draft_pools = tuple(runner.token_to_kv_pool for runner in draft_runners) - if ( - "InklingForConditionalGenerationMTP" - in draft_runners[0].model_config.hf_config.architectures - ): - raise NotImplementedError( - "HiCache does not support Inkling MTP draft state yet." - ) - if _can_pack_hicache_mtp(spec_algorithm, draft_runners): target_model_runner.mtp_draft_device_pools = draft_pools + target_mamba_pool = getattr( + target_model_runner.req_to_token_pool, "mamba_pool", None + ) + draft_mamba_pools = tuple( + getattr(runner.req_to_token_pool, "mamba_pool", None) + for runner in draft_runners + ) + # Shared state is already backed up with the target. Separate + # draft state follows the target's Mamba slot indices. + if any( + pool is not None and pool is not target_mamba_pool + for pool in draft_mamba_pools + ): + assert target_mamba_pool is not None, ( + "Packed draft state requires target Mamba slot indices" + ) + assert all( + pool is not None and pool is not target_mamba_pool + for pool in draft_mamba_pools + ), "Packed MTP requires separate state for every draft layer" + target_model_runner.mtp_draft_mamba_pools = draft_mamba_pools return HiCacheDraftPlan( mode=HiCacheDraftMode.PACKED, device_pools=draft_pools, diff --git a/test/registered/kernels/ops/mamba/test_mtp_hicache.py b/test/registered/kernels/ops/mamba/test_mtp_hicache.py new file mode 100644 index 0000000000..7866719a7b --- /dev/null +++ b/test/registered/kernels/ops/mamba/test_mtp_hicache.py @@ -0,0 +1,317 @@ +import sys +from types import SimpleNamespace +from typing import cast + +import pytest +import torch + +from sglang.srt.configs.inkling import ( + InklingConvCacheParams, + InklingConvStateShape, + InklingStateDType, +) +from sglang.srt.managers.cache_controller import CacheOperation +from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator +from sglang.srt.mem_cache.cache_init_params import CacheInitParams +from sglang.srt.mem_cache.hicache_storage import PoolName, PoolTransfer +from sglang.srt.mem_cache.hybrid_cache import hybrid_pool_assembler as assembler +from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import ( + HybridCacheController, +) +from sglang.srt.mem_cache.l2_transfer import L2TransferEngine +from sglang.srt.mem_cache.memory_pool import ( + MambaPool, + MHATokenToKVPool, + ReqToTokenPool, +) +from sglang.srt.mem_cache.pool_host import HostPoolGroup +from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool +from sglang.srt.runtime_context import get_context +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large") + +pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") + + +def _mamba_pool(layer_ids: list[int], with_temporal: bool) -> MambaPool: + return MambaPool( + size=16, + spec_state_size=4, + cache_params=InklingConvCacheParams( + shape=InklingConvStateShape( + conv=[(3, 256), (3, 256), (3, 128), (3, 128), (3, 256), (3, 256)], + temporal=(2, 8, 16) if with_temporal else (0, 0, 0), + ), + layers=layer_ids, + dtype=InklingStateDType(conv=torch.bfloat16, temporal=torch.bfloat16), + ), + mamba_layer_ids=layer_ids, + device="cuda", + ) + + +@pytest.mark.parametrize( + ("io_backend", "layout"), + [("kernel", "page_first"), ("direct", "page_first_direct")], +) +@pytest.mark.parametrize( + ("local_drafts", "draft_full_indices"), + [([0, 1, 2], True), ([0, 2], True), ([], True), ([0, 1, 2], False)], +) +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) +@pytest.mark.parametrize("target_layers", [2, 4]) +@pytest.mark.parametrize("with_temporal", [False, True]) +def test_hicache_restores_all_mtp_kv_and_conv_state( + monkeypatch: pytest.MonkeyPatch, + layout: str, + io_backend: str, + local_drafts: list[int], + draft_full_indices: bool, + dtype: torch.dtype, + target_layers: int, + with_temporal: bool, +) -> None: + # Cover both ratio sizing and a fixed budget, including draft sidecars. + host_size_gb = 0.01 if target_layers == 4 else 0 + get_context().set_server_args( + ServerArgs( + model_path="test", + hicache_ratio=1.0, + hicache_size=host_size_gb, + hicache_mem_layout=layout, + hicache_io_backend=io_backend, + ) + ) + + def kv_pool( + full: list[int], swa: list[int], full_indices: bool = False + ) -> SWAKVPool: + pool = SWAKVPool( + size=128, + size_swa=128 if full_indices else 64, + page_size=16, + dtype=dtype, + head_num=2, + head_dim=128, + full_attention_layer_ids=full, + swa_attention_layer_ids=swa, + device="cuda", + token_to_kv_pool_class=MHATokenToKVPool, + swa_kv_pool_kwargs={ + "head_num": 1, + "head_dim": 128, + "page_size": 16, + "device": "cuda", + "enable_memory_saver": False, + }, + ) + pool.swa_uses_full_indices = full_indices + return pool + + full_layers = list(range(1, target_layers, 2)) + swa_layers = list(range(0, target_layers, 2)) + target_kv = kv_pool(full_layers, swa_layers) + drafts = tuple( + kv_pool( + [] if i in local_drafts else [i], + [i] if i in local_drafts else [], + draft_full_indices, + ) + for i in range(3) + ) + target_mamba = _mamba_pool(list(range(target_layers)), with_temporal) + draft_mamba = tuple(_mamba_pool([i], with_temporal) for i in range(3)) + allocator = SimpleNamespace(alloc=lambda n: None, free=lambda ids: None) + params = CacheInitParams( + disable=False, + req_to_token_pool=cast( + ReqToTokenPool, + SimpleNamespace(mamba_pool=target_mamba, mamba_allocator=allocator), + ), + token_to_kv_pool_allocator=cast( + BaseTokenToKVPoolAllocator, SimpleNamespace(swa_attn_allocator=allocator) + ), + page_size=16, + mtp_draft_device_pools=drafts, + mtp_draft_mamba_pools=draft_mamba, + ) + + # Exercise the real assembly and transfer engine without storage threads. + def controller_without_threads( + _token_allocator: object, + host_group: HostPoolGroup, + *_args: object, + transfer_layer_id_max: int, + **_kwargs: object, + ) -> HybridCacheController: + controller = object.__new__(HybridCacheController) + monkeypatch.setattr(controller, "mem_pool_host", host_group, raising=False) + controller.transfer_layer_id_max = transfer_layer_id_max + controller.io_backend = io_backend + controller.device = "cuda" + return controller + + monkeypatch.setattr(assembler, "HybridCacheController", controller_without_threads) + group, controller = assembler.build_hybrid_mamba_swa_stack( + params=params, + full_kv_pool=target_kv.full_kv_pool, + swa_kv_pool=target_kv.swa_kv_pool, + mamba_pool=target_mamba, + full_layer_mapping={layer: i for i, layer in enumerate(full_layers)}, + swa_layer_mapping={layer: i for i, layer in enumerate(swa_layers)}, + mamba_layer_mapping={i: i for i in range(target_layers)}, + page_size=16, + tp_group=None, + load_cache_event=None, + storage_backend=None, + ) + try: + if host_size_gb: + host_bytes = sum( + entry.host_pool.size * entry.host_pool.size_per_token + for entry in group.entries + ) + # Each pool rounds up by a page. A derived sidecar also inherits + # the anchor's rounding before rounding its own capacity. + page_padding = sum( + entry.host_pool.page_size + * entry.host_pool.size_per_token + * (2 if entry.name == PoolName.DRAFT_SWA else 1) + for entry in group.entries + ) + assert host_bytes <= host_size_gb * 1e9 + page_padding + if PoolName.DRAFT_SWA in group.entry_map: + assert ( + group.get_entry(PoolName.DRAFT_SWA).host_pool.size + >= group.anchor_entry.host_pool.size + ) + full_src, full_dst = ( + torch.arange(16, 32, device="cuda"), + torch.arange(64, 80, device="cuda"), + ) + swa_src, swa_dst = ( + torch.arange(32, 48, device="cuda"), + torch.arange(16, device="cuda"), + ) + state_src, state_dst = ( + torch.tensor([3, 5], device="cuda"), + torch.tensor([8, 10], device="cuda"), + ) + active_state = torch.tensor([12, 13], device="cuda") + # Token-major views let the same check cover every KV layer and all six + # convolution buffers and any temporal state. + buffers = [] + for pool in (target_kv, *drafts): + # Inkling drafts use identity mapping, while the target's SWA + # allocation has different source AND destination token IDs. + draft_swa_src, draft_swa_dst = ( + (full_src, full_dst) + if pool.swa_uses_full_indices + else (swa_src, swa_dst) + ) + for subpool, src, dst in ( + (pool.full_kv_pool, full_src, full_dst), + (pool.swa_kv_pool, draft_swa_src, draft_swa_dst), + ): + assert isinstance(subpool, MHATokenToKVPool) + assert subpool.k_buffer is not None and subpool.v_buffer is not None + buffers.extend( + (buf, src, dst) for buf in (*subpool.k_buffer, *subpool.v_buffer) + ) + for pool in (target_mamba, *draft_mamba): + buffers.extend( + (buf.transpose(0, 1), state_src, active_state) + for buf in (*pool.mamba_cache.conv, pool.mamba_cache.temporal) + if buf.numel() + ) + expected = [] + for i, (buf, src, _) in enumerate(buffers): + raw = buf.view(torch.uint8) + raw.copy_( + (torch.arange(raw.numel(), device="cuda").reshape(raw.shape) + i) + .remainder(251) + .to(torch.uint8) + ) + expected.append(raw[src].clone()) + + full_host, swa_host, state_host = ( + torch.arange(16), + torch.arange(16, 32), + torch.tensor([1, 2]), + ) + + def transfers( + full_indices: torch.Tensor, + swa_indices: torch.Tensor, + state_indices: torch.Tensor, + ) -> list[PoolTransfer]: + result = [ + PoolTransfer( + PoolName.SWA, host_indices=swa_host, device_indices=swa_indices + ), + PoolTransfer( + PoolName.MAMBA, + host_indices=state_host, + device_indices=state_indices, + ), + ] + specs = assembler._mtp_swa_sidecar_specs(group) + assert bool(specs) == (draft_full_indices and bool(local_drafts)) + for spec in specs: + assert spec.indices_from_pool == PoolName.KV + result.append( + PoolTransfer( + spec.pool_name, indices_from_pool=spec.indices_from_pool + ) + ) + return group.resolve_host_transfers( + result, + primary_device_indices=full_indices, + primary_host_indices=full_host, + ) + + engine = L2TransferEngine(io_backend) + backup = CacheOperation( + full_host, + full_src, + 1, + pool_transfers=transfers(full_src, swa_src, state_src), + ) + engine.submit_device_to_host( + controller._l2_transfers(*controller._move_write_operation(backup)) + ).finish_event.synchronize() + for buf, _, _ in buffers: + buf.zero_() + restore = CacheOperation( + full_host, + full_dst, + 1, + pool_transfers=transfers(full_dst, swa_dst, state_dst), + ) + layer_done = [ + torch.cuda.Event() for _ in range(controller.transfer_layer_id_max) + ] + engine.submit_host_to_device( + controller._l2_load_transfers(*controller._move_op_indices(restore)), + transfer_layer_id_max=controller.transfer_layer_id_max, + on_layer_done=lambda layer: layer_done[layer].record(), + ) + # Prefix reuse copies all target and draft convolution state on the + # forward stream after waiting for the final target-layer event. + layer_done[-1].wait() + for pool in (target_mamba, *draft_mamba): + pool.copy_from(state_dst, active_state) + torch.cuda.synchronize() + + for (buf, _, dst), reference in zip(buffers, expected, strict=True): + assert torch.equal(buf.view(torch.uint8)[dst], reference) + finally: + for entry in group.entries: + entry.host_pool.destroy() + + +if __name__ == "__main__": + args = [arg for arg in sys.argv[1:] if arg != "-f"] + raise SystemExit(pytest.main([__file__, "-v", "-x", *args])) diff --git a/test/registered/unit/mem_cache/test_hicache_mtp_state.py b/test/registered/unit/mem_cache/test_hicache_mtp_state.py new file mode 100644 index 0000000000..fb3e0937b9 --- /dev/null +++ b/test/registered/unit/mem_cache/test_hicache_mtp_state.py @@ -0,0 +1,192 @@ +"""Packed MTP discovery and host-state transfer regressions without model weights.""" + +import sys +from types import SimpleNamespace + +import pytest +import torch + +from sglang.srt.configs import model_config as model_config_module +from sglang.srt.configs.inkling import InklingMMConfig +from sglang.srt.configs.model_config import ModelConfig +from sglang.srt.mem_cache.device_pool_info import MambaStateBufferInfo +from sglang.srt.mem_cache.pool_host.mamba import MambaPoolHost +from sglang.srt.model_executor.model_runner_components.layer_setup import ( + resolve_layer_indices, +) +from sglang.srt.speculative import base_spec_worker as spec +from sglang.srt.speculative.spec_info import SpeculativeAlgorithm +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +def _worker(monkeypatch, draft_config, target_pool, draft_pools): + monkeypatch.setattr( + spec, "get_memory", lambda: SimpleNamespace(enable_hierarchical_cache=True) + ) + target = SimpleNamespace( + spec_algorithm=SpeculativeAlgorithm.EAGLE, + req_to_token_pool=SimpleNamespace(mamba_pool=target_pool), + ) + runners = tuple( + SimpleNamespace( + model_config=draft_config, + token_to_kv_pool=object(), + req_to_token_pool=SimpleNamespace(mamba_pool=pool), + ) + for pool in draft_pools + ) + worker = SimpleNamespace( + target_worker=SimpleNamespace(model_runner=target), + _draft_model_runners=lambda: runners, + ) + return worker, target, runners + + +@pytest.mark.parametrize("local_drafts", [[0, 1, 2], [0, 2], []]) +def test_inkling_checkpoint_packs_every_draft_pool(monkeypatch, tmp_path, local_drafts): + # Checkpoints store the MTP depth outside the text config read by ModelConfig. + config = InklingMMConfig( + architectures=["InklingForConditionalGeneration"], + num_nextn_predict_layers=3, + mtp_config={"n_layers": 3, "local_layer_ids": local_drafts}, + text_config={ + "hidden_size": 128, + "num_hidden_layers": 4, + "num_attention_heads": 2, + "num_key_value_heads": 1, + "vocab_size": 128, + "local_layer_ids": [0, 2], + "max_position_embeddings": 2048, + }, + ) + monkeypatch.setattr(model_config_module, "get_config", lambda *a, **kw: config) + monkeypatch.setattr( + model_config_module, "get_generation_config", lambda *a, **kw: None + ) + draft_config = ModelConfig( + str(tmp_path), is_draft_model=True, is_multi_layer_eagle=True, dtype="bfloat16" + ) + assert draft_config.num_nextn_predict_layers == 3 + # Each runner owns its draft depth while the checkpoint config is shared. + for depth in range(3): + layer_info = resolve_layer_indices( + model=SimpleNamespace(mtp_layer_id_is_depth=True), + model_config=draft_config, + is_draft_worker=True, + draft_model_idx=depth, + ) + assert layer_info.num_effective_layers == 1 + assert layer_info.is_hybrid_swa_mtp_draft + assert layer_info.swa_attention_layer_ids == ( + [depth] if depth in local_drafts else [] + ) + assert layer_info.full_attention_layer_ids == ( + [] if depth in local_drafts else [depth] + ) + assert draft_config.swa_attention_layer_ids == local_drafts + assert draft_config.full_attention_layer_ids == [ + i for i in range(3) if i not in local_drafts + ] + assert draft_config.hf_config.architectures == [ + "InklingForConditionalGenerationMTP" + ] + worker, target, runners = _worker( + monkeypatch, draft_config, object(), (object(), object(), object()) + ) + + plan = spec.BaseSpecWorker._build_hicache_draft_plan(worker) + + assert plan.mode == spec.HiCacheDraftMode.PACKED + assert plan.device_pools == tuple(r.token_to_kv_pool for r in runners) + assert target.mtp_draft_device_pools == plan.device_pools + assert target.mtp_draft_mamba_pools == tuple( + r.req_to_token_pool.mamba_pool for r in runners + ) + + +@pytest.mark.parametrize("has_state", [False, True]) +def test_shared_target_state_is_not_packed_twice(monkeypatch, has_state): + pool = object() if has_state else None + worker, target, _ = _worker( + monkeypatch, SimpleNamespace(num_nextn_predict_layers=3), pool, (pool,) * 3 + ) + target.mtp_draft_mamba_pools = (object(),) + plan = spec.BaseSpecWorker._build_hicache_draft_plan(worker) + assert plan.mode == spec.HiCacheDraftMode.PACKED + assert target.mtp_draft_mamba_pools == () + + +@pytest.mark.parametrize("missing_target", [False, True]) +def test_separate_state_requires_compatible_slot_ownership(monkeypatch, missing_target): + target_pool = None if missing_target else object() + pools = (object(),) * 3 if missing_target else (object(), target_pool, object()) + worker, _, _ = _worker( + monkeypatch, SimpleNamespace(num_nextn_predict_layers=3), target_pool, pools + ) + with pytest.raises(AssertionError, match="target Mamba slot|separate state"): + spec.BaseSpecWorker._build_hicache_draft_plan(worker) + + +@pytest.mark.parametrize("temporal_width", [0, 4]) +def test_ascend_packed_state_roundtrip_uses_buffer_descriptors( + monkeypatch, temporal_width +): + monkeypatch.setenv("SGLANG_NPU_HICACHE_MAMBA_IO", "sync") + infos = tuple( + MambaStateBufferInfo( + temporal=torch.arange(layers * 8 * temporal_width).reshape( + layers, 8, temporal_width + ) + + 100 * i, + conv=(torch.arange(layers * 8 * 4).reshape(layers, 8, 4) + 100 * i,), + extra_checkpoint_buffers=(torch.arange(16).reshape(8, 2) + 100 * i,), + ) + for i, layers in enumerate((2, 1, 1, 1)) + ) + host = MambaPoolHost( + infos[0], + host_to_device_ratio=2, + host_size=0, + device_capacity=7, + pin_memory=False, + layout="page_first", + mtp_draft_buffer_infos=infos[1:], + ) + try: + assert host.num_mamba_layers == 5 + assert host.temporal_device_ptrs.tolist() == [ + layer.data_ptr() for info in infos for layer in info.temporal.unbind(0) + ] + src_ids = torch.tensor([2, 5]) + host_ids = torch.tensor([1, 2]) + dst_ids = torch.tensor([3, 6]) + tensors = [ + layer + for info in infos + for tensor in (*info.conv, info.temporal) + for layer in tensor.unbind(0) + ] + [buf for info in infos for buf in info.extra_checkpoint_buffers] + expected = [tensor[src_ids].clone() for tensor in tensors] + host.backup_from_device_all_layer(None, host_ids, src_ids, "kernel_ascend") + for tensor in tensors: + tensor.zero_() + for layer_id in range(host.num_mamba_layers): + host.load_to_device_per_layer( + None, + host_ids, + dst_ids, + layer_id, + "kernel_ascend", + is_draft=layer_id >= 2, + ) + for tensor, reference in zip(tensors, expected, strict=True): + assert torch.equal(tensor[dst_ids], reference) + finally: + host.destroy() + + +if __name__ == "__main__": + args = [arg for arg in sys.argv[1:] if arg != "-f"] + raise SystemExit(pytest.main([__file__, "-v", "-x", *args])) diff --git a/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py b/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py index 264dc62580..94622c624e 100644 --- a/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py +++ b/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py @@ -748,13 +748,14 @@ class TestHybridMambaDeclaredIndexer(CustomTestCase): kv_pool = _dsa_pool_stub(layer_num=3) kv_pool.skip_topk_layers = [False, True, False] # device layer 1: no buffer - mamba_pool = SimpleNamespace(layer_num=2, size=8) + mamba_pool = SimpleNamespace(layer_num=2, num_mamba_layers=2, size=8) # transfer layers 0,2,4 are DSA (device 0,1,2), 1,3 are Mamba full_mapping = {0: 0, 2: 1, 4: 2} mamba_mapping = {1: 0, 3: 1} params = SimpleNamespace( page_size=64, mtp_draft_device_pools=(), + mtp_draft_mamba_pools=(), token_to_kv_pool_allocator=None, tp_cache_group=None, attn_cp_cache_group=None, @@ -1122,6 +1123,42 @@ def _build_unified_host_pair(bundle): class TestUnifiedPageEnvelopeHostPool(CustomTestCase): + def test_packed_drafts_are_not_silently_dropped_by_shared_arena(self): + bundle = _build_unified_swa_pool() + pool = bundle.token_to_kv_pool + for draft_arg in ( + "mtp_full_device_pools", + "mtp_swa_device_pools", + "mtp_swa_full_device_pools", + ): + with ( + self.subTest(draft_arg=draft_arg), + patch( + _ASSEMBLER + "get_memory", + return_value=SimpleNamespace(hicache_host_memory_mode="auto"), + ), + patch( + _ASSEMBLER + "build_kv_host_pool", + side_effect=NotImplementedError("packed pool reached host factory"), + ), + patch.object( + UnifiedPageEnvelopeHostPool, "build_hybrid_swa_pool_pair" + ) as build_pair, + ): + with self.assertRaisesRegex( + NotImplementedError, "reached host factory" + ): + build_hybrid_swa_group( + page_size=4, + full_kv_pool=pool.full_kv_pool, + swa_kv_pool=pool.swa_kv_pool, + full_layer_mapping={0: 0, 1: 1, 2: 2}, + swa_layer_mapping={3: 0}, + use_mla=False, + **{draft_arg: (object(),)}, + ) + build_pair.assert_not_called() + def test_sidecar_read_error_is_a_cache_miss_with_unified_memory(self): from sglang.srt.runtime_context import publish, reset_context from sglang.srt.server_args import ServerArgs diff --git a/test/registered/unit/mem_cache/test_linker_pool_assembler.py b/test/registered/unit/mem_cache/test_linker_pool_assembler.py index dbf2c712f4..9a0c6b6378 100644 --- a/test/registered/unit/mem_cache/test_linker_pool_assembler.py +++ b/test/registered/unit/mem_cache/test_linker_pool_assembler.py @@ -479,12 +479,15 @@ class TestHybridDevicePoolAssembler(CustomTestCase): draft = SimpleNamespace( token_to_kv_pool=object(), + req_to_token_pool=None, model_config=SimpleNamespace( num_nextn_predict_layers=0, hf_config=SimpleNamespace(architectures=["LlamaForCausalLM"]), ), ) - target = SimpleNamespace(spec_algorithm=SpeculativeAlgorithm.EAGLE) + target = SimpleNamespace( + spec_algorithm=SpeculativeAlgorithm.EAGLE, req_to_token_pool=None + ) worker = SimpleNamespace( target_worker=SimpleNamespace(model_runner=target), _draft_model_runners=lambda: (draft,), diff --git a/test/registered/unit/mem_cache/test_unified_hicache_regressions.py b/test/registered/unit/mem_cache/test_unified_hicache_regressions.py index 245a03a450..113c5d6346 100644 --- a/test/registered/unit/mem_cache/test_unified_hicache_regressions.py +++ b/test/registered/unit/mem_cache/test_unified_hicache_regressions.py @@ -173,6 +173,8 @@ class TestTriPoolAssembly(unittest.TestCase): ) composite = SimpleNamespace(swa_attn_allocator=swa_allocator) params = SimpleNamespace( + mtp_draft_device_pools=(), + mtp_draft_mamba_pools=(), token_to_kv_pool_allocator=composite, req_to_token_pool=SimpleNamespace( mamba_allocator=SimpleNamespace(alloc=Mock(), free=Mock()) @@ -193,7 +195,7 @@ class TestTriPoolAssembly(unittest.TestCase): params=params, full_kv_pool=object(), swa_kv_pool=object(), - mamba_pool=object(), + mamba_pool=SimpleNamespace(num_mamba_layers=1), full_layer_mapping={0: 0}, swa_layer_mapping={1: 0}, mamba_layer_mapping={2: 0},