[HiCache] Preserve all MTP KV and recurrent state (#40223)
Co-authored-by: alphabetc1 <2508695655@qq.com> Co-authored-by: Shuwen Wang <47200617+alphabetc1@users.noreply.github.com>
This commit is contained in:
co-authored by
alphabetc1
Shuwen Wang
parent
f5f5003323
commit
5ca0695aa6
@@ -328,6 +328,9 @@ class InklingMMConfig(PretrainedConfig):
|
||||
if isinstance(text_config, InklingModelConfig)
|
||||
else InklingModelConfig(**(text_config or {}))
|
||||
)
|
||||
if isinstance(mtp_config, dict) and "n_layers" in mtp_config:
|
||||
# HiCache reads the draft depth through the text config.
|
||||
self.text_config.num_nextn_predict_layers = mtp_config["n_layers"]
|
||||
if isinstance(mtp_config, dict) and mtp_config.get("local_layer_ids"):
|
||||
# Banded MTP head: the checkpoint declares its sliding-window draft
|
||||
# depths on mtp_config. Canonicalize onto text_config so every
|
||||
|
||||
@@ -2648,21 +2648,15 @@ def get_hybrid_layer_ids(
|
||||
i for i, x in enumerate(layer_types) if x == "full_attention"
|
||||
]
|
||||
elif "InklingForConditionalGenerationMTP" in model_architectures:
|
||||
# One block per MTP depth; a banded head marks its sliding-window depths
|
||||
# in mtp_local_layer_ids. The per-depth pool routing in the KV-cache
|
||||
# mixin is authoritative; this keeps model_config's swa/full lists
|
||||
# self-consistent for other consumers.
|
||||
mtp_local_layer_ids = hf_text_config.mtp_local_layer_ids
|
||||
if mtp_local_layer_ids:
|
||||
num_depths = hf_text_config.num_nextn_predict_layers
|
||||
local_set = set(mtp_local_layer_ids)
|
||||
swa_attention_layer_ids = sorted(local_set)
|
||||
full_attention_layer_ids = [
|
||||
i for i in range(num_depths) if i not in local_set
|
||||
]
|
||||
else:
|
||||
swa_attention_layer_ids = []
|
||||
full_attention_layer_ids = [0]
|
||||
# Runner-local layer setup selects one depth from these shared lists,
|
||||
# including when every draft depth uses full attention.
|
||||
local_set = set(hf_text_config.mtp_local_layer_ids)
|
||||
swa_attention_layer_ids = sorted(local_set)
|
||||
full_attention_layer_ids = [
|
||||
i
|
||||
for i in range(hf_text_config.num_nextn_predict_layers)
|
||||
if i not in local_set
|
||||
]
|
||||
elif "InklingForConditionalGeneration" in model_architectures:
|
||||
local_layer_ids = hf_text_config.local_layer_ids
|
||||
local_layer_id_set = set(local_layer_ids)
|
||||
|
||||
@@ -18,6 +18,10 @@ class BaseSWAKVPool(KVCache):
|
||||
# Set when SWA KV is a per-request ring of this many tokens (addressed by
|
||||
# req_pool_idx) rather than a paged token pool; SWA is then not budgeted per token.
|
||||
swa_req_ring_size: Optional[int] = None
|
||||
# Independent draft SWA buffers can consume full-token IDs directly instead
|
||||
# of the target SWA allocator's physical IDs. HiCache must preserve that
|
||||
# index ownership even though both buffers implement sliding attention.
|
||||
swa_uses_full_indices: bool = False
|
||||
|
||||
@abc.abstractmethod
|
||||
def register_mapping(self, full_to_swa_index_mapping: torch.Tensor) -> None:
|
||||
|
||||
@@ -62,3 +62,4 @@ class CacheInitParams:
|
||||
tree_core_backend: Optional[str] = dataclasses.field(default=None, kw_only=True)
|
||||
|
||||
mtp_draft_device_pools: tuple[object, ...] = ()
|
||||
mtp_draft_mamba_pools: tuple[object, ...] = ()
|
||||
|
||||
@@ -897,10 +897,12 @@ class HybridCacheController(BaseHiCacheController):
|
||||
if draft_host_layer is None:
|
||||
continue
|
||||
|
||||
# Finish every draft before the final target-layer event,
|
||||
# including chains deeper than the target model.
|
||||
def draft_layer_mapper(
|
||||
layer_id: int,
|
||||
*,
|
||||
expected_layer_id: int = depth,
|
||||
expected_layer_id: int = min(depth, self.transfer_layer_id_max - 1),
|
||||
host_layer_id: int = draft_host_layer,
|
||||
) -> Optional[int]:
|
||||
if layer_id == expected_layer_id:
|
||||
|
||||
@@ -288,13 +288,13 @@ def _uses_unified_page_envelope_host(
|
||||
full_kv_pool: Any,
|
||||
swa_kv_pool: Any,
|
||||
use_mla: bool,
|
||||
mtp_swa_device_pools: tuple[Any, ...] = (),
|
||||
mtp_device_pools: tuple[Any, ...] = (),
|
||||
) -> bool:
|
||||
from sglang.srt.mem_cache.unified_memory_pool import UnifiedMHATokenToKVPool
|
||||
|
||||
return (
|
||||
not use_mla
|
||||
and not mtp_swa_device_pools
|
||||
and not mtp_device_pools
|
||||
and all(
|
||||
isinstance(pool, UnifiedMHATokenToKVPool)
|
||||
for pool in (full_kv_pool, swa_kv_pool)
|
||||
@@ -320,14 +320,29 @@ def build_hybrid_swa_group(
|
||||
host_swa_evict_fn: Optional[Callable[[int], Any]] = None,
|
||||
device_swa_evict_fn: Optional[Callable[[int], Any]] = None,
|
||||
swa_attn_allocator: Any = None,
|
||||
mtp_full_device_pools: tuple[Any, ...] = (),
|
||||
mtp_swa_device_pools: tuple[Any, ...] = (),
|
||||
mtp_swa_full_device_pools: tuple[Any, ...] = (),
|
||||
transfer_layer_id_max: Optional[int] = None,
|
||||
) -> HostPoolGroup:
|
||||
"""Anchor (full) + SWA host pool group for a hybrid-SWA device pool."""
|
||||
transfer_layer_id_max = (
|
||||
max(full_layer_mapping.keys() | swa_layer_mapping.keys()) + 1
|
||||
)
|
||||
if transfer_layer_id_max is None:
|
||||
transfer_layer_id_max = (
|
||||
max(full_layer_mapping.keys() | swa_layer_mapping.keys()) + 1
|
||||
)
|
||||
if mtp_swa_full_device_pools and kv_host_size:
|
||||
# The sidecar has the anchor's token capacity. Reserve its bytes from
|
||||
# the full-KV budget rather than exceeding an explicit --hicache-size.
|
||||
anchor_pools = (full_kv_pool, *mtp_full_device_pools)
|
||||
shares = _split_hicache_size(
|
||||
kv_host_size, (*anchor_pools, *mtp_swa_full_device_pools)
|
||||
)
|
||||
kv_host_size = sum(shares[: len(anchor_pools)])
|
||||
if _uses_unified_page_envelope_host(
|
||||
full_kv_pool, swa_kv_pool, use_mla, mtp_swa_device_pools
|
||||
full_kv_pool,
|
||||
swa_kv_pool,
|
||||
use_mla,
|
||||
mtp_full_device_pools + mtp_swa_device_pools + mtp_swa_full_device_pools,
|
||||
):
|
||||
memory = get_memory()
|
||||
kv_host_pool, swa_host_pool = (
|
||||
@@ -349,6 +364,7 @@ def build_hybrid_swa_group(
|
||||
page_size=page_size,
|
||||
use_mla=use_mla,
|
||||
host_size=kv_host_size,
|
||||
mtp_draft_device_pools=mtp_full_device_pools,
|
||||
pool_label="full",
|
||||
)
|
||||
swa_host_pool = build_kv_host_pool(
|
||||
@@ -359,6 +375,13 @@ def build_hybrid_swa_group(
|
||||
mtp_draft_device_pools=mtp_swa_device_pools,
|
||||
pool_label="swa",
|
||||
)
|
||||
if mtp_full_device_pools:
|
||||
full_layer_mapping = with_packed_draft_layer_mapping(
|
||||
full_layer_mapping,
|
||||
transfer_layer_start=transfer_layer_id_max,
|
||||
target_device_layer_num=full_kv_pool.layer_num,
|
||||
draft_layer_num=len(mtp_full_device_pools),
|
||||
)
|
||||
if mtp_swa_device_pools:
|
||||
swa_layer_mapping = with_packed_draft_layer_mapping(
|
||||
swa_layer_mapping,
|
||||
@@ -366,16 +389,18 @@ def build_hybrid_swa_group(
|
||||
target_device_layer_num=swa_kv_pool.layer_num,
|
||||
draft_layer_num=len(mtp_swa_device_pools),
|
||||
)
|
||||
return HostPoolGroup(
|
||||
group = HostPoolGroup(
|
||||
[
|
||||
build_pool_entry(
|
||||
name=PoolName.KV,
|
||||
host_pool=kv_host_pool,
|
||||
device_pool=full_kv_pool,
|
||||
layer_mapping=full_layer_mapping,
|
||||
transfer_layer_id_max=transfer_layer_id_max,
|
||||
transfer_layer_id_max=transfer_layer_id_max
|
||||
+ len(mtp_full_device_pools),
|
||||
is_anchor=True,
|
||||
host_evict_fn=host_full_evict_fn if has_shared_arena else None,
|
||||
packed_draft_device_pools=mtp_full_device_pools,
|
||||
),
|
||||
build_pool_entry(
|
||||
name=PoolName.SWA,
|
||||
@@ -390,6 +415,53 @@ def build_hybrid_swa_group(
|
||||
),
|
||||
]
|
||||
)
|
||||
if mtp_swa_full_device_pools:
|
||||
# SWA drafts with full-token addressing may have different head geometry
|
||||
# from full attention. Use a separate host pool with KV indices instead of
|
||||
# packing their bytes into either target pool.
|
||||
first, *rest = mtp_swa_full_device_pools
|
||||
draft_host_pool = get_mha_host_pool_cls(first)(
|
||||
first,
|
||||
host_to_device_ratio=kv_host_pool.size / first.size,
|
||||
host_size=0,
|
||||
page_size=page_size,
|
||||
layout=get_memory().hicache_mem_layout,
|
||||
allocator_type=_get_allocator_type(),
|
||||
mtp_draft_device_pools=rest,
|
||||
pool_label="draft_swa",
|
||||
)
|
||||
group.add_entry(
|
||||
build_pool_entry(
|
||||
name=PoolName.DRAFT_SWA,
|
||||
host_pool=draft_host_pool,
|
||||
device_pool=first,
|
||||
layer_mapping=with_packed_draft_layer_mapping(
|
||||
{0: 0},
|
||||
transfer_layer_start=transfer_layer_id_max,
|
||||
target_device_layer_num=first.layer_num,
|
||||
draft_layer_num=len(rest),
|
||||
),
|
||||
transfer_layer_id_max=transfer_layer_id_max + len(rest),
|
||||
packed_draft_device_pools=tuple(rest),
|
||||
)
|
||||
)
|
||||
return group
|
||||
|
||||
|
||||
def _mtp_swa_device_pools(
|
||||
params: CacheInitParams, *, full_indices: bool
|
||||
) -> tuple[Any, ...]:
|
||||
return tuple(
|
||||
pool.swa_kv_pool
|
||||
for pool in params.mtp_draft_device_pools
|
||||
if pool.swa_kv_pool.layer_num and pool.swa_uses_full_indices == full_indices
|
||||
)
|
||||
|
||||
|
||||
def _mtp_swa_sidecar_specs(group: HostPoolGroup) -> list[SidecarPoolSpec]:
|
||||
if PoolName.DRAFT_SWA not in group.entry_map:
|
||||
return []
|
||||
return [SidecarPoolSpec(PoolName.DRAFT_SWA, indices_from_pool=PoolName.KV)]
|
||||
|
||||
|
||||
def build_kv_only_stack(
|
||||
@@ -458,15 +530,18 @@ def build_hybrid_swa_stack(
|
||||
transfer_layer_id_max = (
|
||||
max(full_layer_mapping.keys() | swa_layer_mapping.keys()) + 1
|
||||
)
|
||||
# MTP draft pools follow the target SWA layout; select their SWA storage.
|
||||
mtp_swa_device_pools = tuple(
|
||||
pool.swa_kv_pool for pool in params.mtp_draft_device_pools
|
||||
# Each draft depth may use full or sliding-window attention.
|
||||
mtp_full_device_pools = tuple(
|
||||
pool.full_kv_pool
|
||||
for pool in params.mtp_draft_device_pools
|
||||
if pool.full_kv_pool.layer_num
|
||||
)
|
||||
mtp_swa_device_pools = _mtp_swa_device_pools(params, full_indices=False)
|
||||
|
||||
kv_host_size = swa_host_size = None
|
||||
memory = get_memory()
|
||||
if memory.hicache_size > 0 and not _uses_unified_page_envelope_host(
|
||||
full_kv_pool, swa_kv_pool, use_mla, mtp_swa_device_pools
|
||||
full_kv_pool, swa_kv_pool, use_mla, params.mtp_draft_device_pools
|
||||
):
|
||||
kv_host_size, swa_host_size = _split_hicache_size(
|
||||
memory.hicache_size, (full_kv_pool, swa_kv_pool)
|
||||
@@ -486,7 +561,9 @@ def build_hybrid_swa_stack(
|
||||
device_swa_evict_fn=device_swa_evict_fn,
|
||||
# For SWA hybrid, device allocation goes through the inner allocator.
|
||||
swa_attn_allocator=params.token_to_kv_pool_allocator.swa_attn_allocator,
|
||||
mtp_full_device_pools=mtp_full_device_pools,
|
||||
mtp_swa_device_pools=mtp_swa_device_pools,
|
||||
mtp_swa_full_device_pools=_mtp_swa_device_pools(params, full_indices=True),
|
||||
)
|
||||
cache_controller = HybridCacheController(
|
||||
params.token_to_kv_pool_allocator,
|
||||
@@ -1104,9 +1181,16 @@ def build_deepseek_v4_hicache_stack(
|
||||
return host_pool_group, cache_controller
|
||||
|
||||
|
||||
def build_mamba_host_pool(mamba_pool, *, host_size: float, layout: str):
|
||||
def build_mamba_host_pool(
|
||||
mamba_pool, *, host_size: float, layout: str, mtp_draft_device_pools=()
|
||||
):
|
||||
(pool_info,) = mamba_pool.get_device_pool_infos()
|
||||
pool_info.buffer_info.validate(layer_ids=pool_info.layer_ids)
|
||||
draft_buffer_infos = []
|
||||
for draft_pool in mtp_draft_device_pools:
|
||||
(draft_info,) = draft_pool.get_device_pool_infos()
|
||||
draft_info.buffer_info.validate(layer_ids=draft_info.layer_ids)
|
||||
draft_buffer_infos.append(draft_info.buffer_info)
|
||||
capacity = mamba_pool.host_capacity_tokens
|
||||
if capacity is None:
|
||||
capacity = mamba_pool.size
|
||||
@@ -1115,6 +1199,7 @@ def build_mamba_host_pool(mamba_pool, *, host_size: float, layout: str):
|
||||
get_memory().hicache_ratio,
|
||||
host_size,
|
||||
device_capacity=capacity,
|
||||
mtp_draft_buffer_infos=draft_buffer_infos,
|
||||
allocator_type=_get_allocator_type(),
|
||||
layout=layout,
|
||||
)
|
||||
@@ -1187,6 +1272,13 @@ def build_hybrid_mamba_stack(
|
||||
mamba_pool,
|
||||
host_size=mamba_host_size,
|
||||
layout=mamba_layout,
|
||||
mtp_draft_device_pools=params.mtp_draft_mamba_pools,
|
||||
)
|
||||
mamba_layer_mapping = with_packed_draft_layer_mapping(
|
||||
mamba_layer_mapping,
|
||||
transfer_layer_start=transfer_layer_id_max,
|
||||
target_device_layer_num=mamba_pool.num_mamba_layers,
|
||||
draft_layer_num=len(params.mtp_draft_mamba_pools),
|
||||
)
|
||||
entries = _build_pool_entries(config=config, root_host_pool=kv_host_pool) + [
|
||||
build_pool_entry(
|
||||
@@ -1194,12 +1286,14 @@ def build_hybrid_mamba_stack(
|
||||
host_pool=mamba_host_pool,
|
||||
device_pool=mamba_pool,
|
||||
layer_mapping=mamba_layer_mapping,
|
||||
transfer_layer_id_max=transfer_layer_id_max,
|
||||
transfer_layer_id_max=transfer_layer_id_max
|
||||
+ len(params.mtp_draft_mamba_pools),
|
||||
host_evict_fn=host_mamba_evict_fn,
|
||||
device_evict_fn=device_mamba_evict_fn,
|
||||
device_alloc_fn=mamba_allocator.alloc,
|
||||
device_free_fn=mamba_allocator.free,
|
||||
)
|
||||
packed_draft_device_pools=params.mtp_draft_mamba_pools,
|
||||
),
|
||||
]
|
||||
if qsa_pool is not None:
|
||||
entries.append(
|
||||
@@ -1281,57 +1375,54 @@ def build_hybrid_mamba_swa_stack(
|
||||
kv_host_size, swa_host_size, mamba_host_size = _split_hicache_size(
|
||||
get_memory().hicache_size, (full_kv_pool, swa_kv_pool, mamba_pool)
|
||||
)
|
||||
kv_host_pool = build_kv_host_pool(
|
||||
kv_pool=full_kv_pool,
|
||||
host_pool_group = build_hybrid_swa_group(
|
||||
page_size=page_size,
|
||||
full_kv_pool=full_kv_pool,
|
||||
swa_kv_pool=swa_kv_pool,
|
||||
full_layer_mapping=full_layer_mapping,
|
||||
swa_layer_mapping=swa_layer_mapping,
|
||||
use_mla=False,
|
||||
host_size=kv_host_size,
|
||||
pool_label="full",
|
||||
)
|
||||
swa_host_pool = build_kv_host_pool(
|
||||
kv_pool=swa_kv_pool,
|
||||
page_size=page_size,
|
||||
use_mla=False,
|
||||
host_size=swa_host_size,
|
||||
pool_label="swa",
|
||||
kv_host_size=kv_host_size,
|
||||
swa_host_size=swa_host_size,
|
||||
host_swa_evict_fn=host_swa_evict_fn,
|
||||
device_swa_evict_fn=device_swa_evict_fn,
|
||||
swa_attn_allocator=swa_attn_allocator,
|
||||
mtp_full_device_pools=tuple(
|
||||
pool.full_kv_pool
|
||||
for pool in params.mtp_draft_device_pools
|
||||
if pool.full_kv_pool.layer_num
|
||||
),
|
||||
mtp_swa_device_pools=_mtp_swa_device_pools(params, full_indices=False),
|
||||
mtp_swa_full_device_pools=_mtp_swa_device_pools(params, full_indices=True),
|
||||
transfer_layer_id_max=transfer_layer_id_max,
|
||||
)
|
||||
mamba_host_pool = build_mamba_host_pool(
|
||||
mamba_pool,
|
||||
host_size=mamba_host_size,
|
||||
layout=get_memory().hicache_mem_layout,
|
||||
mtp_draft_device_pools=params.mtp_draft_mamba_pools,
|
||||
)
|
||||
entries = [
|
||||
build_pool_entry(
|
||||
name=PoolName.KV,
|
||||
host_pool=kv_host_pool,
|
||||
device_pool=full_kv_pool,
|
||||
layer_mapping=full_layer_mapping,
|
||||
transfer_layer_id_max=transfer_layer_id_max,
|
||||
is_anchor=True,
|
||||
),
|
||||
build_pool_entry(
|
||||
name=PoolName.SWA,
|
||||
host_pool=swa_host_pool,
|
||||
device_pool=swa_kv_pool,
|
||||
layer_mapping=swa_layer_mapping,
|
||||
transfer_layer_id_max=transfer_layer_id_max,
|
||||
host_evict_fn=host_swa_evict_fn,
|
||||
device_evict_fn=device_swa_evict_fn,
|
||||
**_swa_allocation_callbacks(swa_attn_allocator),
|
||||
),
|
||||
mamba_layer_mapping = with_packed_draft_layer_mapping(
|
||||
mamba_layer_mapping,
|
||||
transfer_layer_start=transfer_layer_id_max,
|
||||
target_device_layer_num=mamba_pool.num_mamba_layers,
|
||||
draft_layer_num=len(params.mtp_draft_mamba_pools),
|
||||
)
|
||||
host_pool_group.add_entry(
|
||||
build_pool_entry(
|
||||
name=PoolName.MAMBA,
|
||||
host_pool=mamba_host_pool,
|
||||
device_pool=mamba_pool,
|
||||
layer_mapping=mamba_layer_mapping,
|
||||
transfer_layer_id_max=transfer_layer_id_max,
|
||||
transfer_layer_id_max=transfer_layer_id_max
|
||||
+ len(params.mtp_draft_mamba_pools),
|
||||
host_evict_fn=host_mamba_evict_fn,
|
||||
device_evict_fn=device_mamba_evict_fn,
|
||||
device_alloc_fn=mamba_allocator.alloc,
|
||||
device_free_fn=mamba_allocator.free,
|
||||
packed_draft_device_pools=params.mtp_draft_mamba_pools,
|
||||
),
|
||||
]
|
||||
host_pool_group = HostPoolGroup(entries)
|
||||
)
|
||||
cache_controller = HybridCacheController(
|
||||
params.token_to_kv_pool_allocator,
|
||||
host_pool_group,
|
||||
@@ -1916,6 +2007,7 @@ class _SwaStrategy(StackStrategy):
|
||||
ComponentType.SWA: host_pool_group.get_pool(PoolName.SWA),
|
||||
},
|
||||
pools_desc="Full + SWA",
|
||||
sidecars=_mtp_swa_sidecar_specs(host_pool_group),
|
||||
)
|
||||
|
||||
|
||||
@@ -1985,6 +2077,7 @@ class _MambaSwaStrategy(StackStrategy):
|
||||
},
|
||||
register_req_to_token_counter=True,
|
||||
pools_desc="KV + SWA + MAMBA",
|
||||
sidecars=_mtp_swa_sidecar_specs(host_pool_group),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -361,6 +361,7 @@ def build_kv_cache(
|
||||
chunked_prefill_size=effective_chunked_prefill_size,
|
||||
sliding_window_size=sliding_window_size,
|
||||
mtp_draft_device_pools=mtp_draft_device_pools,
|
||||
mtp_draft_mamba_pools=tp_worker.model_runner.mtp_draft_mamba_pools,
|
||||
)
|
||||
|
||||
tree_context = TreeCacheBuildContext(
|
||||
|
||||
@@ -2641,6 +2641,7 @@ class KVCacheConfigurator:
|
||||
)
|
||||
identity_mapping[-1] = -1
|
||||
token_to_kv_pool.register_mapping(identity_mapping)
|
||||
token_to_kv_pool.swa_uses_full_indices = True
|
||||
elif not uses_unified_virtual_ids:
|
||||
token_to_kv_pool.register_mapping(
|
||||
swa_allocator.full_to_swa_index_mapping
|
||||
|
||||
@@ -3,7 +3,7 @@ from __future__ import annotations
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from typing import Optional
|
||||
from typing import Optional, Sequence
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -72,6 +72,8 @@ def _npu_hicache_mamba_io_mode() -> str:
|
||||
|
||||
|
||||
class MambaPoolHost(HostKVCache):
|
||||
mtp_draft_buffer_infos: tuple[MambaStateBufferInfo, ...] = ()
|
||||
|
||||
# Class-level defaults so a model without side state, and the test stubs that
|
||||
# skip __init__, still read an empty pair rather than raising.
|
||||
slot_state_device_tensors: tuple = ()
|
||||
@@ -88,11 +90,13 @@ class MambaPoolHost(HostKVCache):
|
||||
device: str = "cpu",
|
||||
allocator_type: str = "default",
|
||||
layout: str = "layer_first",
|
||||
mtp_draft_buffer_infos: Sequence[MambaStateBufferInfo] = (),
|
||||
):
|
||||
buffer_info.validate()
|
||||
self.buffer_info = buffer_info
|
||||
self.device_pool = None
|
||||
self._buffer_device = buffer_info.temporal.device
|
||||
self.mtp_draft_buffer_infos = tuple(mtp_draft_buffer_infos)
|
||||
self.page_size = 1
|
||||
|
||||
assert layout in [
|
||||
@@ -104,7 +108,31 @@ class MambaPoolHost(HostKVCache):
|
||||
self.pin_memory = pin_memory
|
||||
self.device = device
|
||||
self.allocator = get_allocator_from_storage(allocator_type)
|
||||
self.num_mamba_layers = buffer_info.temporal.shape[0]
|
||||
self.num_mamba_layers = buffer_info.temporal.shape[0] + len(
|
||||
self.mtp_draft_buffer_infos
|
||||
)
|
||||
|
||||
buffer_infos = (buffer_info, *self.mtp_draft_buffer_infos)
|
||||
for draft_info in self.mtp_draft_buffer_infos:
|
||||
draft_info.validate()
|
||||
assert draft_info.temporal.shape[0] == 1
|
||||
assert len(draft_info.conv) == len(buffer_info.conv)
|
||||
for target, draft in zip(
|
||||
(*buffer_info.conv, buffer_info.temporal),
|
||||
(*draft_info.conv, draft_info.temporal),
|
||||
strict=True,
|
||||
):
|
||||
assert target.shape[1:] == draft.shape[1:]
|
||||
assert target.dtype == draft.dtype
|
||||
assert target.device == draft.device
|
||||
|
||||
self.temporal_device_layers = [
|
||||
layer for info in buffer_infos for layer in info.temporal.unbind(0)
|
||||
]
|
||||
self.conv_device_layers = [
|
||||
[layer for info in buffer_infos for layer in info.conv[i].unbind(0)]
|
||||
for i in range(len(buffer_info.conv))
|
||||
]
|
||||
|
||||
self.conv_state_shapes = [
|
||||
conv_state.shape[2:] for conv_state in self.buffer_info.conv
|
||||
@@ -119,7 +147,9 @@ class MambaPoolHost(HostKVCache):
|
||||
self.dtype = self.conv_dtype
|
||||
# Registered side states share the checkpoint slot with conv/temporal,
|
||||
# so a host round trip that skips them leaves the previous occupant's rows.
|
||||
self.slot_state_device_tensors = list(buffer_info.extra_checkpoint_buffers)
|
||||
self.slot_state_device_tensors = [
|
||||
buf for info in buffer_infos for buf in info.extra_checkpoint_buffers
|
||||
]
|
||||
self.slot_state_buffers = []
|
||||
self.size_per_token = self.get_size_per_token()
|
||||
|
||||
@@ -158,20 +188,17 @@ class MambaPoolHost(HostKVCache):
|
||||
)
|
||||
|
||||
self.temporal_device_ptrs = torch.tensor(
|
||||
[
|
||||
self.buffer_info.temporal[i].data_ptr()
|
||||
for i in range(self.num_mamba_layers)
|
||||
],
|
||||
[layer.data_ptr() for layer in self.temporal_device_layers],
|
||||
dtype=torch.uint64,
|
||||
device=self._buffer_device,
|
||||
)
|
||||
self.conv_device_ptrs = [
|
||||
torch.tensor(
|
||||
[conv_state[i].data_ptr() for i in range(self.num_mamba_layers)],
|
||||
[layer.data_ptr() for layer in layers],
|
||||
dtype=torch.uint64,
|
||||
device=self._buffer_device,
|
||||
)
|
||||
for conv_state in self.buffer_info.conv
|
||||
for layers in self.conv_device_layers
|
||||
]
|
||||
|
||||
self.kv_buffer = self.init_kv_buffer()
|
||||
@@ -543,7 +570,7 @@ class MambaPoolHost(HostKVCache):
|
||||
|
||||
@staticmethod
|
||||
def _copy_tensor_all_layers_lf_pf(
|
||||
src_layers: torch.Tensor,
|
||||
src_layers: torch.Tensor | Sequence[torch.Tensor],
|
||||
dst: torch.Tensor,
|
||||
src_indices: torch.Tensor,
|
||||
dst_indices: torch.Tensor,
|
||||
@@ -601,7 +628,17 @@ class MambaPoolHost(HostKVCache):
|
||||
page_size=1,
|
||||
)
|
||||
elif io_backend == "kernel_ascend":
|
||||
if _npu_hicache_mamba_io_mode() == "async":
|
||||
if not isinstance(src_layers, torch.Tensor):
|
||||
# Packed drafts live in separate allocations; the NPU bulk
|
||||
# kernels require a single contiguous layer-first tensor.
|
||||
host_indices = dst_indices.to(dtype=torch.int64, device=dst.device)
|
||||
for layer_id, src in enumerate(src_layers):
|
||||
device_indices = src_indices.to(
|
||||
dtype=torch.int64, device=src.device
|
||||
)
|
||||
values = src.index_select(0, device_indices).to(device=dst.device)
|
||||
dst[:, layer_id, 0].index_copy_(0, host_indices, values)
|
||||
elif _npu_hicache_mamba_io_mode() == "async":
|
||||
transfer_state_all_layer_direct_lf_pf(
|
||||
device_states=[src_layers],
|
||||
host_states=[dst],
|
||||
@@ -646,13 +683,28 @@ class MambaPoolHost(HostKVCache):
|
||||
*,
|
||||
is_draft: bool = False,
|
||||
):
|
||||
temporal_layers = (
|
||||
self.temporal_device_layers
|
||||
if self.mtp_draft_buffer_infos
|
||||
else self.buffer_info.temporal
|
||||
)
|
||||
conv_layers = (
|
||||
self.conv_device_layers
|
||||
if self.mtp_draft_buffer_infos
|
||||
else self.buffer_info.conv
|
||||
)
|
||||
|
||||
# Side state is not per-layer, so it rides on the first layer's call --
|
||||
# before that layer's completion event, which is what every reader of a
|
||||
# restored slot waits on.
|
||||
if layer_id == 0:
|
||||
self._transfer_slot_states(host_indices, device_indices, to_device=True)
|
||||
if self.layout in ["page_first", "page_first_direct"]:
|
||||
if io_backend == "kernel_ascend" and transfer_mamba_state is not None:
|
||||
if (
|
||||
io_backend == "kernel_ascend"
|
||||
and transfer_mamba_state is not None
|
||||
and not self.mtp_draft_buffer_infos
|
||||
):
|
||||
# NPU: transfer all layers at once via dedicated kernel.
|
||||
# layer_id == 0 covers every layer, so later calls must skip.
|
||||
if layer_id == 0:
|
||||
@@ -679,7 +731,7 @@ class MambaPoolHost(HostKVCache):
|
||||
if self.temporal_state_elem_size > 0:
|
||||
self._copy_tensor_pf_lf(
|
||||
src=self.temporal_buffer,
|
||||
dst=self.buffer_info.temporal[layer_id],
|
||||
dst=temporal_layers[layer_id],
|
||||
src_indices=host_indices,
|
||||
dst_indices=device_indices,
|
||||
layer_id=layer_id,
|
||||
@@ -689,7 +741,7 @@ class MambaPoolHost(HostKVCache):
|
||||
for conv_idx in range(len(self.conv_state_shapes)):
|
||||
self._copy_tensor_pf_lf(
|
||||
src=self.conv_buffer[conv_idx],
|
||||
dst=self.buffer_info.conv[conv_idx][layer_id],
|
||||
dst=conv_layers[conv_idx][layer_id],
|
||||
src_indices=host_indices,
|
||||
dst_indices=device_indices,
|
||||
layer_id=layer_id,
|
||||
@@ -699,7 +751,7 @@ class MambaPoolHost(HostKVCache):
|
||||
else:
|
||||
self._copy_tensor(
|
||||
self.temporal_buffer[layer_id],
|
||||
self.buffer_info.temporal[layer_id],
|
||||
temporal_layers[layer_id],
|
||||
host_indices,
|
||||
device_indices,
|
||||
io_backend,
|
||||
@@ -707,7 +759,7 @@ class MambaPoolHost(HostKVCache):
|
||||
for conv_idx in range(len(self.conv_state_shapes)):
|
||||
self._copy_tensor(
|
||||
self.conv_buffer[conv_idx][layer_id],
|
||||
self.buffer_info.conv[conv_idx][layer_id],
|
||||
conv_layers[conv_idx][layer_id],
|
||||
host_indices,
|
||||
device_indices,
|
||||
io_backend,
|
||||
@@ -716,6 +768,16 @@ class MambaPoolHost(HostKVCache):
|
||||
def backup_from_device_all_layer(
|
||||
self, device_pool, host_indices, device_indices, io_backend="kernel"
|
||||
):
|
||||
temporal_layers = (
|
||||
self.temporal_device_layers
|
||||
if self.mtp_draft_buffer_infos
|
||||
else self.buffer_info.temporal
|
||||
)
|
||||
conv_layers = (
|
||||
self.conv_device_layers
|
||||
if self.mtp_draft_buffer_infos
|
||||
else self.buffer_info.conv
|
||||
)
|
||||
if self.layout in ["page_first", "page_first_direct"]:
|
||||
if io_backend == "kernel" and host_indices.device != device_indices.device:
|
||||
# The mamba JIT kernel wants both index tensors on the device;
|
||||
@@ -725,7 +787,7 @@ class MambaPoolHost(HostKVCache):
|
||||
# no ssm state on conv-only models: a 0-size batched memcpy errors
|
||||
if self.temporal_state_elem_size > 0:
|
||||
self._copy_tensor_all_layers_lf_pf(
|
||||
src_layers=self.buffer_info.temporal,
|
||||
src_layers=temporal_layers,
|
||||
dst=self.temporal_buffer,
|
||||
src_indices=device_indices,
|
||||
dst_indices=host_indices,
|
||||
@@ -737,7 +799,7 @@ class MambaPoolHost(HostKVCache):
|
||||
)
|
||||
for conv_idx in range(len(self.conv_state_shapes)):
|
||||
self._copy_tensor_all_layers_lf_pf(
|
||||
src_layers=self.buffer_info.conv[conv_idx],
|
||||
src_layers=conv_layers[conv_idx],
|
||||
dst=self.conv_buffer[conv_idx],
|
||||
src_indices=device_indices,
|
||||
dst_indices=host_indices,
|
||||
@@ -750,7 +812,7 @@ class MambaPoolHost(HostKVCache):
|
||||
else:
|
||||
for layer_id in range(self.num_mamba_layers):
|
||||
self._copy_tensor(
|
||||
self.buffer_info.temporal[layer_id],
|
||||
temporal_layers[layer_id],
|
||||
self.temporal_buffer[layer_id],
|
||||
device_indices,
|
||||
host_indices,
|
||||
@@ -758,7 +820,7 @@ class MambaPoolHost(HostKVCache):
|
||||
)
|
||||
for conv_idx in range(len(self.conv_state_shapes)):
|
||||
self._copy_tensor(
|
||||
self.buffer_info.conv[conv_idx][layer_id],
|
||||
conv_layers[conv_idx][layer_id],
|
||||
self.conv_buffer[conv_idx][layer_id],
|
||||
device_indices,
|
||||
host_indices,
|
||||
|
||||
@@ -383,6 +383,7 @@ class ModelRunner:
|
||||
self.req_to_token_pool = req_to_token_pool
|
||||
self.token_to_kv_pool_allocator = token_to_kv_pool_allocator
|
||||
self.mtp_draft_device_pools = ()
|
||||
self.mtp_draft_mamba_pools = ()
|
||||
self.is_hybrid_swa = model_config.is_hybrid_swa
|
||||
self.is_hybrid_swa_compress = model_config.is_hybrid_swa_compress
|
||||
self.use_mla_backend = self.model_config.attention_arch == AttentionArch.MLA
|
||||
|
||||
@@ -260,6 +260,7 @@ class BaseSpecWorker(ABC):
|
||||
def _build_hicache_draft_plan(self) -> HiCacheDraftPlan:
|
||||
target_model_runner = self.target_worker.model_runner
|
||||
target_model_runner.mtp_draft_device_pools = ()
|
||||
target_model_runner.mtp_draft_mamba_pools = ()
|
||||
spec_algorithm = target_model_runner.spec_algorithm
|
||||
if not (
|
||||
get_memory().enable_hierarchical_cache
|
||||
@@ -272,16 +273,29 @@ class BaseSpecWorker(ABC):
|
||||
if not draft_runners:
|
||||
return HiCacheDraftPlan()
|
||||
draft_pools = tuple(runner.token_to_kv_pool for runner in draft_runners)
|
||||
if (
|
||||
"InklingForConditionalGenerationMTP"
|
||||
in draft_runners[0].model_config.hf_config.architectures
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"HiCache does not support Inkling MTP draft state yet."
|
||||
)
|
||||
|
||||
if _can_pack_hicache_mtp(spec_algorithm, draft_runners):
|
||||
target_model_runner.mtp_draft_device_pools = draft_pools
|
||||
target_mamba_pool = getattr(
|
||||
target_model_runner.req_to_token_pool, "mamba_pool", None
|
||||
)
|
||||
draft_mamba_pools = tuple(
|
||||
getattr(runner.req_to_token_pool, "mamba_pool", None)
|
||||
for runner in draft_runners
|
||||
)
|
||||
# Shared state is already backed up with the target. Separate
|
||||
# draft state follows the target's Mamba slot indices.
|
||||
if any(
|
||||
pool is not None and pool is not target_mamba_pool
|
||||
for pool in draft_mamba_pools
|
||||
):
|
||||
assert target_mamba_pool is not None, (
|
||||
"Packed draft state requires target Mamba slot indices"
|
||||
)
|
||||
assert all(
|
||||
pool is not None and pool is not target_mamba_pool
|
||||
for pool in draft_mamba_pools
|
||||
), "Packed MTP requires separate state for every draft layer"
|
||||
target_model_runner.mtp_draft_mamba_pools = draft_mamba_pools
|
||||
return HiCacheDraftPlan(
|
||||
mode=HiCacheDraftMode.PACKED,
|
||||
device_pools=draft_pools,
|
||||
|
||||
Reference in New Issue
Block a user