[HiCache] Preserve all MTP KV and recurrent state (#40223)

Co-authored-by: alphabetc1 <2508695655@qq.com>
Co-authored-by: Shuwen Wang <47200617+alphabetc1@users.noreply.github.com>
This commit is contained in:
paulzhang-tm
2026-10-12 15:37:59 +08:00
committed by GitHub
co-authored by alphabetc1 Shuwen Wang
parent f5f5003323
commit 5ca0695aa6
16 changed files with 822 additions and 95 deletions
+3
View File
@@ -328,6 +328,9 @@ class InklingMMConfig(PretrainedConfig):
if isinstance(text_config, InklingModelConfig)
else InklingModelConfig(**(text_config or {}))
)
if isinstance(mtp_config, dict) and "n_layers" in mtp_config:
# HiCache reads the draft depth through the text config.
self.text_config.num_nextn_predict_layers = mtp_config["n_layers"]
if isinstance(mtp_config, dict) and mtp_config.get("local_layer_ids"):
# Banded MTP head: the checkpoint declares its sliding-window draft
# depths on mtp_config. Canonicalize onto text_config so every
+9 -15
View File
@@ -2648,21 +2648,15 @@ def get_hybrid_layer_ids(
i for i, x in enumerate(layer_types) if x == "full_attention"
]
elif "InklingForConditionalGenerationMTP" in model_architectures:
# One block per MTP depth; a banded head marks its sliding-window depths
# in mtp_local_layer_ids. The per-depth pool routing in the KV-cache
# mixin is authoritative; this keeps model_config's swa/full lists
# self-consistent for other consumers.
mtp_local_layer_ids = hf_text_config.mtp_local_layer_ids
if mtp_local_layer_ids:
num_depths = hf_text_config.num_nextn_predict_layers
local_set = set(mtp_local_layer_ids)
swa_attention_layer_ids = sorted(local_set)
full_attention_layer_ids = [
i for i in range(num_depths) if i not in local_set
]
else:
swa_attention_layer_ids = []
full_attention_layer_ids = [0]
# Runner-local layer setup selects one depth from these shared lists,
# including when every draft depth uses full attention.
local_set = set(hf_text_config.mtp_local_layer_ids)
swa_attention_layer_ids = sorted(local_set)
full_attention_layer_ids = [
i
for i in range(hf_text_config.num_nextn_predict_layers)
if i not in local_set
]
elif "InklingForConditionalGeneration" in model_architectures:
local_layer_ids = hf_text_config.local_layer_ids
local_layer_id_set = set(local_layer_ids)
@@ -18,6 +18,10 @@ class BaseSWAKVPool(KVCache):
# Set when SWA KV is a per-request ring of this many tokens (addressed by
# req_pool_idx) rather than a paged token pool; SWA is then not budgeted per token.
swa_req_ring_size: Optional[int] = None
# Independent draft SWA buffers can consume full-token IDs directly instead
# of the target SWA allocator's physical IDs. HiCache must preserve that
# index ownership even though both buffers implement sliding attention.
swa_uses_full_indices: bool = False
@abc.abstractmethod
def register_mapping(self, full_to_swa_index_mapping: torch.Tensor) -> None:
@@ -62,3 +62,4 @@ class CacheInitParams:
tree_core_backend: Optional[str] = dataclasses.field(default=None, kw_only=True)
mtp_draft_device_pools: tuple[object, ...] = ()
mtp_draft_mamba_pools: tuple[object, ...] = ()
@@ -897,10 +897,12 @@ class HybridCacheController(BaseHiCacheController):
if draft_host_layer is None:
continue
# Finish every draft before the final target-layer event,
# including chains deeper than the target model.
def draft_layer_mapper(
layer_id: int,
*,
expected_layer_id: int = depth,
expected_layer_id: int = min(depth, self.transfer_layer_id_max - 1),
host_layer_id: int = draft_host_layer,
) -> Optional[int]:
if layer_id == expected_layer_id:
@@ -288,13 +288,13 @@ def _uses_unified_page_envelope_host(
full_kv_pool: Any,
swa_kv_pool: Any,
use_mla: bool,
mtp_swa_device_pools: tuple[Any, ...] = (),
mtp_device_pools: tuple[Any, ...] = (),
) -> bool:
from sglang.srt.mem_cache.unified_memory_pool import UnifiedMHATokenToKVPool
return (
not use_mla
and not mtp_swa_device_pools
and not mtp_device_pools
and all(
isinstance(pool, UnifiedMHATokenToKVPool)
for pool in (full_kv_pool, swa_kv_pool)
@@ -320,14 +320,29 @@ def build_hybrid_swa_group(
host_swa_evict_fn: Optional[Callable[[int], Any]] = None,
device_swa_evict_fn: Optional[Callable[[int], Any]] = None,
swa_attn_allocator: Any = None,
mtp_full_device_pools: tuple[Any, ...] = (),
mtp_swa_device_pools: tuple[Any, ...] = (),
mtp_swa_full_device_pools: tuple[Any, ...] = (),
transfer_layer_id_max: Optional[int] = None,
) -> HostPoolGroup:
"""Anchor (full) + SWA host pool group for a hybrid-SWA device pool."""
transfer_layer_id_max = (
max(full_layer_mapping.keys() | swa_layer_mapping.keys()) + 1
)
if transfer_layer_id_max is None:
transfer_layer_id_max = (
max(full_layer_mapping.keys() | swa_layer_mapping.keys()) + 1
)
if mtp_swa_full_device_pools and kv_host_size:
# The sidecar has the anchor's token capacity. Reserve its bytes from
# the full-KV budget rather than exceeding an explicit --hicache-size.
anchor_pools = (full_kv_pool, *mtp_full_device_pools)
shares = _split_hicache_size(
kv_host_size, (*anchor_pools, *mtp_swa_full_device_pools)
)
kv_host_size = sum(shares[: len(anchor_pools)])
if _uses_unified_page_envelope_host(
full_kv_pool, swa_kv_pool, use_mla, mtp_swa_device_pools
full_kv_pool,
swa_kv_pool,
use_mla,
mtp_full_device_pools + mtp_swa_device_pools + mtp_swa_full_device_pools,
):
memory = get_memory()
kv_host_pool, swa_host_pool = (
@@ -349,6 +364,7 @@ def build_hybrid_swa_group(
page_size=page_size,
use_mla=use_mla,
host_size=kv_host_size,
mtp_draft_device_pools=mtp_full_device_pools,
pool_label="full",
)
swa_host_pool = build_kv_host_pool(
@@ -359,6 +375,13 @@ def build_hybrid_swa_group(
mtp_draft_device_pools=mtp_swa_device_pools,
pool_label="swa",
)
if mtp_full_device_pools:
full_layer_mapping = with_packed_draft_layer_mapping(
full_layer_mapping,
transfer_layer_start=transfer_layer_id_max,
target_device_layer_num=full_kv_pool.layer_num,
draft_layer_num=len(mtp_full_device_pools),
)
if mtp_swa_device_pools:
swa_layer_mapping = with_packed_draft_layer_mapping(
swa_layer_mapping,
@@ -366,16 +389,18 @@ def build_hybrid_swa_group(
target_device_layer_num=swa_kv_pool.layer_num,
draft_layer_num=len(mtp_swa_device_pools),
)
return HostPoolGroup(
group = HostPoolGroup(
[
build_pool_entry(
name=PoolName.KV,
host_pool=kv_host_pool,
device_pool=full_kv_pool,
layer_mapping=full_layer_mapping,
transfer_layer_id_max=transfer_layer_id_max,
transfer_layer_id_max=transfer_layer_id_max
+ len(mtp_full_device_pools),
is_anchor=True,
host_evict_fn=host_full_evict_fn if has_shared_arena else None,
packed_draft_device_pools=mtp_full_device_pools,
),
build_pool_entry(
name=PoolName.SWA,
@@ -390,6 +415,53 @@ def build_hybrid_swa_group(
),
]
)
if mtp_swa_full_device_pools:
# SWA drafts with full-token addressing may have different head geometry
# from full attention. Use a separate host pool with KV indices instead of
# packing their bytes into either target pool.
first, *rest = mtp_swa_full_device_pools
draft_host_pool = get_mha_host_pool_cls(first)(
first,
host_to_device_ratio=kv_host_pool.size / first.size,
host_size=0,
page_size=page_size,
layout=get_memory().hicache_mem_layout,
allocator_type=_get_allocator_type(),
mtp_draft_device_pools=rest,
pool_label="draft_swa",
)
group.add_entry(
build_pool_entry(
name=PoolName.DRAFT_SWA,
host_pool=draft_host_pool,
device_pool=first,
layer_mapping=with_packed_draft_layer_mapping(
{0: 0},
transfer_layer_start=transfer_layer_id_max,
target_device_layer_num=first.layer_num,
draft_layer_num=len(rest),
),
transfer_layer_id_max=transfer_layer_id_max + len(rest),
packed_draft_device_pools=tuple(rest),
)
)
return group
def _mtp_swa_device_pools(
params: CacheInitParams, *, full_indices: bool
) -> tuple[Any, ...]:
return tuple(
pool.swa_kv_pool
for pool in params.mtp_draft_device_pools
if pool.swa_kv_pool.layer_num and pool.swa_uses_full_indices == full_indices
)
def _mtp_swa_sidecar_specs(group: HostPoolGroup) -> list[SidecarPoolSpec]:
if PoolName.DRAFT_SWA not in group.entry_map:
return []
return [SidecarPoolSpec(PoolName.DRAFT_SWA, indices_from_pool=PoolName.KV)]
def build_kv_only_stack(
@@ -458,15 +530,18 @@ def build_hybrid_swa_stack(
transfer_layer_id_max = (
max(full_layer_mapping.keys() | swa_layer_mapping.keys()) + 1
)
# MTP draft pools follow the target SWA layout; select their SWA storage.
mtp_swa_device_pools = tuple(
pool.swa_kv_pool for pool in params.mtp_draft_device_pools
# Each draft depth may use full or sliding-window attention.
mtp_full_device_pools = tuple(
pool.full_kv_pool
for pool in params.mtp_draft_device_pools
if pool.full_kv_pool.layer_num
)
mtp_swa_device_pools = _mtp_swa_device_pools(params, full_indices=False)
kv_host_size = swa_host_size = None
memory = get_memory()
if memory.hicache_size > 0 and not _uses_unified_page_envelope_host(
full_kv_pool, swa_kv_pool, use_mla, mtp_swa_device_pools
full_kv_pool, swa_kv_pool, use_mla, params.mtp_draft_device_pools
):
kv_host_size, swa_host_size = _split_hicache_size(
memory.hicache_size, (full_kv_pool, swa_kv_pool)
@@ -486,7 +561,9 @@ def build_hybrid_swa_stack(
device_swa_evict_fn=device_swa_evict_fn,
# For SWA hybrid, device allocation goes through the inner allocator.
swa_attn_allocator=params.token_to_kv_pool_allocator.swa_attn_allocator,
mtp_full_device_pools=mtp_full_device_pools,
mtp_swa_device_pools=mtp_swa_device_pools,
mtp_swa_full_device_pools=_mtp_swa_device_pools(params, full_indices=True),
)
cache_controller = HybridCacheController(
params.token_to_kv_pool_allocator,
@@ -1104,9 +1181,16 @@ def build_deepseek_v4_hicache_stack(
return host_pool_group, cache_controller
def build_mamba_host_pool(mamba_pool, *, host_size: float, layout: str):
def build_mamba_host_pool(
mamba_pool, *, host_size: float, layout: str, mtp_draft_device_pools=()
):
(pool_info,) = mamba_pool.get_device_pool_infos()
pool_info.buffer_info.validate(layer_ids=pool_info.layer_ids)
draft_buffer_infos = []
for draft_pool in mtp_draft_device_pools:
(draft_info,) = draft_pool.get_device_pool_infos()
draft_info.buffer_info.validate(layer_ids=draft_info.layer_ids)
draft_buffer_infos.append(draft_info.buffer_info)
capacity = mamba_pool.host_capacity_tokens
if capacity is None:
capacity = mamba_pool.size
@@ -1115,6 +1199,7 @@ def build_mamba_host_pool(mamba_pool, *, host_size: float, layout: str):
get_memory().hicache_ratio,
host_size,
device_capacity=capacity,
mtp_draft_buffer_infos=draft_buffer_infos,
allocator_type=_get_allocator_type(),
layout=layout,
)
@@ -1187,6 +1272,13 @@ def build_hybrid_mamba_stack(
mamba_pool,
host_size=mamba_host_size,
layout=mamba_layout,
mtp_draft_device_pools=params.mtp_draft_mamba_pools,
)
mamba_layer_mapping = with_packed_draft_layer_mapping(
mamba_layer_mapping,
transfer_layer_start=transfer_layer_id_max,
target_device_layer_num=mamba_pool.num_mamba_layers,
draft_layer_num=len(params.mtp_draft_mamba_pools),
)
entries = _build_pool_entries(config=config, root_host_pool=kv_host_pool) + [
build_pool_entry(
@@ -1194,12 +1286,14 @@ def build_hybrid_mamba_stack(
host_pool=mamba_host_pool,
device_pool=mamba_pool,
layer_mapping=mamba_layer_mapping,
transfer_layer_id_max=transfer_layer_id_max,
transfer_layer_id_max=transfer_layer_id_max
+ len(params.mtp_draft_mamba_pools),
host_evict_fn=host_mamba_evict_fn,
device_evict_fn=device_mamba_evict_fn,
device_alloc_fn=mamba_allocator.alloc,
device_free_fn=mamba_allocator.free,
)
packed_draft_device_pools=params.mtp_draft_mamba_pools,
),
]
if qsa_pool is not None:
entries.append(
@@ -1281,57 +1375,54 @@ def build_hybrid_mamba_swa_stack(
kv_host_size, swa_host_size, mamba_host_size = _split_hicache_size(
get_memory().hicache_size, (full_kv_pool, swa_kv_pool, mamba_pool)
)
kv_host_pool = build_kv_host_pool(
kv_pool=full_kv_pool,
host_pool_group = build_hybrid_swa_group(
page_size=page_size,
full_kv_pool=full_kv_pool,
swa_kv_pool=swa_kv_pool,
full_layer_mapping=full_layer_mapping,
swa_layer_mapping=swa_layer_mapping,
use_mla=False,
host_size=kv_host_size,
pool_label="full",
)
swa_host_pool = build_kv_host_pool(
kv_pool=swa_kv_pool,
page_size=page_size,
use_mla=False,
host_size=swa_host_size,
pool_label="swa",
kv_host_size=kv_host_size,
swa_host_size=swa_host_size,
host_swa_evict_fn=host_swa_evict_fn,
device_swa_evict_fn=device_swa_evict_fn,
swa_attn_allocator=swa_attn_allocator,
mtp_full_device_pools=tuple(
pool.full_kv_pool
for pool in params.mtp_draft_device_pools
if pool.full_kv_pool.layer_num
),
mtp_swa_device_pools=_mtp_swa_device_pools(params, full_indices=False),
mtp_swa_full_device_pools=_mtp_swa_device_pools(params, full_indices=True),
transfer_layer_id_max=transfer_layer_id_max,
)
mamba_host_pool = build_mamba_host_pool(
mamba_pool,
host_size=mamba_host_size,
layout=get_memory().hicache_mem_layout,
mtp_draft_device_pools=params.mtp_draft_mamba_pools,
)
entries = [
build_pool_entry(
name=PoolName.KV,
host_pool=kv_host_pool,
device_pool=full_kv_pool,
layer_mapping=full_layer_mapping,
transfer_layer_id_max=transfer_layer_id_max,
is_anchor=True,
),
build_pool_entry(
name=PoolName.SWA,
host_pool=swa_host_pool,
device_pool=swa_kv_pool,
layer_mapping=swa_layer_mapping,
transfer_layer_id_max=transfer_layer_id_max,
host_evict_fn=host_swa_evict_fn,
device_evict_fn=device_swa_evict_fn,
**_swa_allocation_callbacks(swa_attn_allocator),
),
mamba_layer_mapping = with_packed_draft_layer_mapping(
mamba_layer_mapping,
transfer_layer_start=transfer_layer_id_max,
target_device_layer_num=mamba_pool.num_mamba_layers,
draft_layer_num=len(params.mtp_draft_mamba_pools),
)
host_pool_group.add_entry(
build_pool_entry(
name=PoolName.MAMBA,
host_pool=mamba_host_pool,
device_pool=mamba_pool,
layer_mapping=mamba_layer_mapping,
transfer_layer_id_max=transfer_layer_id_max,
transfer_layer_id_max=transfer_layer_id_max
+ len(params.mtp_draft_mamba_pools),
host_evict_fn=host_mamba_evict_fn,
device_evict_fn=device_mamba_evict_fn,
device_alloc_fn=mamba_allocator.alloc,
device_free_fn=mamba_allocator.free,
packed_draft_device_pools=params.mtp_draft_mamba_pools,
),
]
host_pool_group = HostPoolGroup(entries)
)
cache_controller = HybridCacheController(
params.token_to_kv_pool_allocator,
host_pool_group,
@@ -1916,6 +2007,7 @@ class _SwaStrategy(StackStrategy):
ComponentType.SWA: host_pool_group.get_pool(PoolName.SWA),
},
pools_desc="Full + SWA",
sidecars=_mtp_swa_sidecar_specs(host_pool_group),
)
@@ -1985,6 +2077,7 @@ class _MambaSwaStrategy(StackStrategy):
},
register_req_to_token_counter=True,
pools_desc="KV + SWA + MAMBA",
sidecars=_mtp_swa_sidecar_specs(host_pool_group),
)
@@ -361,6 +361,7 @@ def build_kv_cache(
chunked_prefill_size=effective_chunked_prefill_size,
sliding_window_size=sliding_window_size,
mtp_draft_device_pools=mtp_draft_device_pools,
mtp_draft_mamba_pools=tp_worker.model_runner.mtp_draft_mamba_pools,
)
tree_context = TreeCacheBuildContext(
@@ -2641,6 +2641,7 @@ class KVCacheConfigurator:
)
identity_mapping[-1] = -1
token_to_kv_pool.register_mapping(identity_mapping)
token_to_kv_pool.swa_uses_full_indices = True
elif not uses_unified_virtual_ids:
token_to_kv_pool.register_mapping(
swa_allocator.full_to_swa_index_mapping
+82 -20
View File
@@ -3,7 +3,7 @@ from __future__ import annotations
import logging
import os
import threading
from typing import Optional
from typing import Optional, Sequence
import numpy as np
import torch
@@ -72,6 +72,8 @@ def _npu_hicache_mamba_io_mode() -> str:
class MambaPoolHost(HostKVCache):
mtp_draft_buffer_infos: tuple[MambaStateBufferInfo, ...] = ()
# Class-level defaults so a model without side state, and the test stubs that
# skip __init__, still read an empty pair rather than raising.
slot_state_device_tensors: tuple = ()
@@ -88,11 +90,13 @@ class MambaPoolHost(HostKVCache):
device: str = "cpu",
allocator_type: str = "default",
layout: str = "layer_first",
mtp_draft_buffer_infos: Sequence[MambaStateBufferInfo] = (),
):
buffer_info.validate()
self.buffer_info = buffer_info
self.device_pool = None
self._buffer_device = buffer_info.temporal.device
self.mtp_draft_buffer_infos = tuple(mtp_draft_buffer_infos)
self.page_size = 1
assert layout in [
@@ -104,7 +108,31 @@ class MambaPoolHost(HostKVCache):
self.pin_memory = pin_memory
self.device = device
self.allocator = get_allocator_from_storage(allocator_type)
self.num_mamba_layers = buffer_info.temporal.shape[0]
self.num_mamba_layers = buffer_info.temporal.shape[0] + len(
self.mtp_draft_buffer_infos
)
buffer_infos = (buffer_info, *self.mtp_draft_buffer_infos)
for draft_info in self.mtp_draft_buffer_infos:
draft_info.validate()
assert draft_info.temporal.shape[0] == 1
assert len(draft_info.conv) == len(buffer_info.conv)
for target, draft in zip(
(*buffer_info.conv, buffer_info.temporal),
(*draft_info.conv, draft_info.temporal),
strict=True,
):
assert target.shape[1:] == draft.shape[1:]
assert target.dtype == draft.dtype
assert target.device == draft.device
self.temporal_device_layers = [
layer for info in buffer_infos for layer in info.temporal.unbind(0)
]
self.conv_device_layers = [
[layer for info in buffer_infos for layer in info.conv[i].unbind(0)]
for i in range(len(buffer_info.conv))
]
self.conv_state_shapes = [
conv_state.shape[2:] for conv_state in self.buffer_info.conv
@@ -119,7 +147,9 @@ class MambaPoolHost(HostKVCache):
self.dtype = self.conv_dtype
# Registered side states share the checkpoint slot with conv/temporal,
# so a host round trip that skips them leaves the previous occupant's rows.
self.slot_state_device_tensors = list(buffer_info.extra_checkpoint_buffers)
self.slot_state_device_tensors = [
buf for info in buffer_infos for buf in info.extra_checkpoint_buffers
]
self.slot_state_buffers = []
self.size_per_token = self.get_size_per_token()
@@ -158,20 +188,17 @@ class MambaPoolHost(HostKVCache):
)
self.temporal_device_ptrs = torch.tensor(
[
self.buffer_info.temporal[i].data_ptr()
for i in range(self.num_mamba_layers)
],
[layer.data_ptr() for layer in self.temporal_device_layers],
dtype=torch.uint64,
device=self._buffer_device,
)
self.conv_device_ptrs = [
torch.tensor(
[conv_state[i].data_ptr() for i in range(self.num_mamba_layers)],
[layer.data_ptr() for layer in layers],
dtype=torch.uint64,
device=self._buffer_device,
)
for conv_state in self.buffer_info.conv
for layers in self.conv_device_layers
]
self.kv_buffer = self.init_kv_buffer()
@@ -543,7 +570,7 @@ class MambaPoolHost(HostKVCache):
@staticmethod
def _copy_tensor_all_layers_lf_pf(
src_layers: torch.Tensor,
src_layers: torch.Tensor | Sequence[torch.Tensor],
dst: torch.Tensor,
src_indices: torch.Tensor,
dst_indices: torch.Tensor,
@@ -601,7 +628,17 @@ class MambaPoolHost(HostKVCache):
page_size=1,
)
elif io_backend == "kernel_ascend":
if _npu_hicache_mamba_io_mode() == "async":
if not isinstance(src_layers, torch.Tensor):
# Packed drafts live in separate allocations; the NPU bulk
# kernels require a single contiguous layer-first tensor.
host_indices = dst_indices.to(dtype=torch.int64, device=dst.device)
for layer_id, src in enumerate(src_layers):
device_indices = src_indices.to(
dtype=torch.int64, device=src.device
)
values = src.index_select(0, device_indices).to(device=dst.device)
dst[:, layer_id, 0].index_copy_(0, host_indices, values)
elif _npu_hicache_mamba_io_mode() == "async":
transfer_state_all_layer_direct_lf_pf(
device_states=[src_layers],
host_states=[dst],
@@ -646,13 +683,28 @@ class MambaPoolHost(HostKVCache):
*,
is_draft: bool = False,
):
temporal_layers = (
self.temporal_device_layers
if self.mtp_draft_buffer_infos
else self.buffer_info.temporal
)
conv_layers = (
self.conv_device_layers
if self.mtp_draft_buffer_infos
else self.buffer_info.conv
)
# Side state is not per-layer, so it rides on the first layer's call --
# before that layer's completion event, which is what every reader of a
# restored slot waits on.
if layer_id == 0:
self._transfer_slot_states(host_indices, device_indices, to_device=True)
if self.layout in ["page_first", "page_first_direct"]:
if io_backend == "kernel_ascend" and transfer_mamba_state is not None:
if (
io_backend == "kernel_ascend"
and transfer_mamba_state is not None
and not self.mtp_draft_buffer_infos
):
# NPU: transfer all layers at once via dedicated kernel.
# layer_id == 0 covers every layer, so later calls must skip.
if layer_id == 0:
@@ -679,7 +731,7 @@ class MambaPoolHost(HostKVCache):
if self.temporal_state_elem_size > 0:
self._copy_tensor_pf_lf(
src=self.temporal_buffer,
dst=self.buffer_info.temporal[layer_id],
dst=temporal_layers[layer_id],
src_indices=host_indices,
dst_indices=device_indices,
layer_id=layer_id,
@@ -689,7 +741,7 @@ class MambaPoolHost(HostKVCache):
for conv_idx in range(len(self.conv_state_shapes)):
self._copy_tensor_pf_lf(
src=self.conv_buffer[conv_idx],
dst=self.buffer_info.conv[conv_idx][layer_id],
dst=conv_layers[conv_idx][layer_id],
src_indices=host_indices,
dst_indices=device_indices,
layer_id=layer_id,
@@ -699,7 +751,7 @@ class MambaPoolHost(HostKVCache):
else:
self._copy_tensor(
self.temporal_buffer[layer_id],
self.buffer_info.temporal[layer_id],
temporal_layers[layer_id],
host_indices,
device_indices,
io_backend,
@@ -707,7 +759,7 @@ class MambaPoolHost(HostKVCache):
for conv_idx in range(len(self.conv_state_shapes)):
self._copy_tensor(
self.conv_buffer[conv_idx][layer_id],
self.buffer_info.conv[conv_idx][layer_id],
conv_layers[conv_idx][layer_id],
host_indices,
device_indices,
io_backend,
@@ -716,6 +768,16 @@ class MambaPoolHost(HostKVCache):
def backup_from_device_all_layer(
self, device_pool, host_indices, device_indices, io_backend="kernel"
):
temporal_layers = (
self.temporal_device_layers
if self.mtp_draft_buffer_infos
else self.buffer_info.temporal
)
conv_layers = (
self.conv_device_layers
if self.mtp_draft_buffer_infos
else self.buffer_info.conv
)
if self.layout in ["page_first", "page_first_direct"]:
if io_backend == "kernel" and host_indices.device != device_indices.device:
# The mamba JIT kernel wants both index tensors on the device;
@@ -725,7 +787,7 @@ class MambaPoolHost(HostKVCache):
# no ssm state on conv-only models: a 0-size batched memcpy errors
if self.temporal_state_elem_size > 0:
self._copy_tensor_all_layers_lf_pf(
src_layers=self.buffer_info.temporal,
src_layers=temporal_layers,
dst=self.temporal_buffer,
src_indices=device_indices,
dst_indices=host_indices,
@@ -737,7 +799,7 @@ class MambaPoolHost(HostKVCache):
)
for conv_idx in range(len(self.conv_state_shapes)):
self._copy_tensor_all_layers_lf_pf(
src_layers=self.buffer_info.conv[conv_idx],
src_layers=conv_layers[conv_idx],
dst=self.conv_buffer[conv_idx],
src_indices=device_indices,
dst_indices=host_indices,
@@ -750,7 +812,7 @@ class MambaPoolHost(HostKVCache):
else:
for layer_id in range(self.num_mamba_layers):
self._copy_tensor(
self.buffer_info.temporal[layer_id],
temporal_layers[layer_id],
self.temporal_buffer[layer_id],
device_indices,
host_indices,
@@ -758,7 +820,7 @@ class MambaPoolHost(HostKVCache):
)
for conv_idx in range(len(self.conv_state_shapes)):
self._copy_tensor(
self.buffer_info.conv[conv_idx][layer_id],
conv_layers[conv_idx][layer_id],
self.conv_buffer[conv_idx][layer_id],
device_indices,
host_indices,
@@ -383,6 +383,7 @@ class ModelRunner:
self.req_to_token_pool = req_to_token_pool
self.token_to_kv_pool_allocator = token_to_kv_pool_allocator
self.mtp_draft_device_pools = ()
self.mtp_draft_mamba_pools = ()
self.is_hybrid_swa = model_config.is_hybrid_swa
self.is_hybrid_swa_compress = model_config.is_hybrid_swa_compress
self.use_mla_backend = self.model_config.attention_arch == AttentionArch.MLA
@@ -260,6 +260,7 @@ class BaseSpecWorker(ABC):
def _build_hicache_draft_plan(self) -> HiCacheDraftPlan:
target_model_runner = self.target_worker.model_runner
target_model_runner.mtp_draft_device_pools = ()
target_model_runner.mtp_draft_mamba_pools = ()
spec_algorithm = target_model_runner.spec_algorithm
if not (
get_memory().enable_hierarchical_cache
@@ -272,16 +273,29 @@ class BaseSpecWorker(ABC):
if not draft_runners:
return HiCacheDraftPlan()
draft_pools = tuple(runner.token_to_kv_pool for runner in draft_runners)
if (
"InklingForConditionalGenerationMTP"
in draft_runners[0].model_config.hf_config.architectures
):
raise NotImplementedError(
"HiCache does not support Inkling MTP draft state yet."
)
if _can_pack_hicache_mtp(spec_algorithm, draft_runners):
target_model_runner.mtp_draft_device_pools = draft_pools
target_mamba_pool = getattr(
target_model_runner.req_to_token_pool, "mamba_pool", None
)
draft_mamba_pools = tuple(
getattr(runner.req_to_token_pool, "mamba_pool", None)
for runner in draft_runners
)
# Shared state is already backed up with the target. Separate
# draft state follows the target's Mamba slot indices.
if any(
pool is not None and pool is not target_mamba_pool
for pool in draft_mamba_pools
):
assert target_mamba_pool is not None, (
"Packed draft state requires target Mamba slot indices"
)
assert all(
pool is not None and pool is not target_mamba_pool
for pool in draft_mamba_pools
), "Packed MTP requires separate state for every draft layer"
target_model_runner.mtp_draft_mamba_pools = draft_mamba_pools
return HiCacheDraftPlan(
mode=HiCacheDraftMode.PACKED,
device_pools=draft_pools,