Declare hybrid Mamba indexer host pool (#40915)
This commit is contained in:
@@ -1131,7 +1131,7 @@ def build_deepseek_v4_hicache_stack(
|
||||
def build_hybrid_mamba_stack(
|
||||
*,
|
||||
params: CacheInitParams,
|
||||
kv_pool: Any,
|
||||
decls: tuple[HostPoolDecl, ...],
|
||||
mamba_pool: Any,
|
||||
full_layer_mapping: dict[int, int],
|
||||
mamba_layer_mapping: dict[int, int],
|
||||
@@ -1145,18 +1145,23 @@ def build_hybrid_mamba_stack(
|
||||
model_name: Optional[str] = None,
|
||||
storage_backend_extra_config: Optional[dict] = None,
|
||||
enable_storage_metrics: bool = False,
|
||||
) -> tuple[HostPoolGroup, HybridCacheController]:
|
||||
) -> tuple[HostPoolGroup, HybridCacheController, HostPoolGroupConfig]:
|
||||
"""KV plus every pool the hybrid pool declares (e.g. a sparse indexer),
|
||||
then the Mamba state pool, which keeps its own host path."""
|
||||
transfer_layer_id_max = (
|
||||
max(full_layer_mapping.keys() | mamba_layer_mapping.keys()) + 1
|
||||
)
|
||||
mamba_allocator = params.req_to_token_pool.mamba_allocator
|
||||
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool
|
||||
|
||||
qsa_draft_pools = params.mtp_draft_device_pools if qsa_pool is not None else ()
|
||||
mtp_draft_device_pools = tuple(
|
||||
pool.full_kv_pool if isinstance(pool, HybridLinearKVPool) else pool
|
||||
for pool in params.mtp_draft_device_pools
|
||||
config = prepare_host_pool_config(
|
||||
decls=decls,
|
||||
full_layer_mapping=full_layer_mapping,
|
||||
transfer_layer_id_max=transfer_layer_id_max,
|
||||
transfer_page_size=params.page_size,
|
||||
packed_draft_device_pools=params.mtp_draft_device_pools,
|
||||
)
|
||||
root = config.pools[0]
|
||||
kv_pool = root.decl.device_pool
|
||||
kv_host_size, mamba_host_size = None, 0
|
||||
if get_memory().hicache_size > 0:
|
||||
kv_host_size, mamba_host_size = _split_hicache_size(
|
||||
@@ -1175,15 +1180,8 @@ def build_hybrid_mamba_stack(
|
||||
page_size=params.page_size,
|
||||
use_mla=use_mla,
|
||||
host_size=kv_host_size,
|
||||
mtp_draft_device_pools=mtp_draft_device_pools,
|
||||
mtp_draft_device_pools=root.packed_draft_device_pools,
|
||||
)
|
||||
if mtp_draft_device_pools:
|
||||
full_layer_mapping = with_packed_draft_layer_mapping(
|
||||
full_layer_mapping,
|
||||
transfer_layer_start=transfer_layer_id_max,
|
||||
target_device_layer_num=kv_pool.layer_num,
|
||||
draft_layer_num=len(mtp_draft_device_pools),
|
||||
)
|
||||
# MambaPoolHost only supports page_first_direct; the global layout may be
|
||||
# page_first_kv_split (e.g. MLA + KDA hybrid on NPU). The Mamba/KDA state
|
||||
# pool has no separate K/V buffers, so kv_split does not apply; override
|
||||
@@ -1200,16 +1198,7 @@ def build_hybrid_mamba_stack(
|
||||
allocator_type=_get_allocator_type(),
|
||||
layout=mamba_layout,
|
||||
)
|
||||
entries = [
|
||||
build_pool_entry(
|
||||
name=PoolName.KV,
|
||||
host_pool=kv_host_pool,
|
||||
device_pool=kv_pool,
|
||||
layer_mapping=full_layer_mapping,
|
||||
transfer_layer_id_max=transfer_layer_id_max + len(mtp_draft_device_pools),
|
||||
is_anchor=True,
|
||||
packed_draft_device_pools=mtp_draft_device_pools,
|
||||
),
|
||||
entries = _build_pool_entries(config=config, root_host_pool=kv_host_pool) + [
|
||||
build_pool_entry(
|
||||
name=PoolName.MAMBA,
|
||||
host_pool=mamba_host_pool,
|
||||
@@ -1220,7 +1209,7 @@ def build_hybrid_mamba_stack(
|
||||
device_evict_fn=device_mamba_evict_fn,
|
||||
device_alloc_fn=mamba_allocator.alloc,
|
||||
device_free_fn=mamba_allocator.free,
|
||||
),
|
||||
)
|
||||
]
|
||||
if qsa_pool is not None:
|
||||
entries.append(
|
||||
@@ -1233,8 +1222,9 @@ def build_hybrid_mamba_stack(
|
||||
mtp_draft_device_pools=qsa_draft_pools,
|
||||
),
|
||||
device_pool=qsa_pool,
|
||||
layer_mapping=full_layer_mapping,
|
||||
transfer_layer_id_max=transfer_layer_id_max + len(qsa_draft_pools),
|
||||
# The KV config carries the packed draft tail layers.
|
||||
layer_mapping=root.layer_mapping,
|
||||
transfer_layer_id_max=root.transfer_layer_id_max,
|
||||
packed_draft_device_pools=qsa_draft_pools,
|
||||
)
|
||||
)
|
||||
@@ -1258,7 +1248,7 @@ def build_hybrid_mamba_stack(
|
||||
enable_storage_metrics=enable_storage_metrics,
|
||||
host_memory_mode=get_memory().hicache_host_memory_mode,
|
||||
)
|
||||
return host_pool_group, cache_controller
|
||||
return host_pool_group, cache_controller, config
|
||||
|
||||
|
||||
def build_hybrid_mamba_swa_stack(
|
||||
@@ -1390,13 +1380,20 @@ def build_host_pool_group(
|
||||
) -> HostPoolGroup:
|
||||
"""Allocate host pools and transfer entries from prepared configs."""
|
||||
root = config.pools[0]
|
||||
host_pools = {
|
||||
root.decl.pool_name: build_kv_host_pool(
|
||||
kv_pool=root.decl.device_pool,
|
||||
page_size=config.transfer_page_size,
|
||||
mtp_draft_device_pools=root.packed_draft_device_pools,
|
||||
)
|
||||
}
|
||||
root_host_pool = build_kv_host_pool(
|
||||
kv_pool=root.decl.device_pool,
|
||||
page_size=config.transfer_page_size,
|
||||
mtp_draft_device_pools=root.packed_draft_device_pools,
|
||||
)
|
||||
return HostPoolGroup(
|
||||
_build_pool_entries(config=config, root_host_pool=root_host_pool)
|
||||
)
|
||||
|
||||
|
||||
def _build_pool_entries(
|
||||
*, config: HostPoolGroupConfig, root_host_pool: Any
|
||||
) -> list[PoolEntry]:
|
||||
host_pools = {config.pools[0].decl.pool_name: root_host_pool}
|
||||
entries = []
|
||||
for pool_config in config.pools:
|
||||
decl = pool_config.decl
|
||||
@@ -1418,7 +1415,7 @@ def build_host_pool_group(
|
||||
packed_draft_device_pools=pool_config.packed_draft_device_pools,
|
||||
)
|
||||
)
|
||||
return HostPoolGroup(entries)
|
||||
return entries
|
||||
|
||||
|
||||
def _build_mha_mla_host_pool(
|
||||
@@ -1832,9 +1829,9 @@ class _MambaStrategy(StackStrategy):
|
||||
mamba_layer_mapping = _stage_local_layer_mapping(
|
||||
params.req_to_token_pool.mamba_map, kvcache.start_layer
|
||||
)
|
||||
host_pool_group, cache_controller = build_hybrid_mamba_stack(
|
||||
host_pool_group, cache_controller, config = build_hybrid_mamba_stack(
|
||||
params=params,
|
||||
kv_pool=kvcache.full_kv_pool,
|
||||
decls=kvcache.host_pool_decls(),
|
||||
mamba_pool=params.req_to_token_pool.mamba_pool,
|
||||
full_layer_mapping=full_layer_mapping,
|
||||
mamba_layer_mapping=mamba_layer_mapping,
|
||||
@@ -1856,13 +1853,21 @@ class _MambaStrategy(StackStrategy):
|
||||
ComponentType.FULL: host_pool_group.get_pool(PoolName.KV),
|
||||
ComponentType.MAMBA: host_pool_group.get_pool(PoolName.MAMBA),
|
||||
},
|
||||
sidecars=(
|
||||
sidecars=[
|
||||
c.decl.sidecar_spec() for c in config.pools if not c.decl.is_primary
|
||||
]
|
||||
+ (
|
||||
[SidecarPoolSpec(PoolName.INDEXER, PoolName.KV)]
|
||||
if qsa_pool is not None
|
||||
else []
|
||||
),
|
||||
pool_declarations=tuple(c.decl for c in config.pools),
|
||||
register_req_to_token_counter=True,
|
||||
pools_desc="KV + MAMBA + INDEXER" if qsa_pool is not None else "KV + MAMBA",
|
||||
pools_desc=" + ".join(
|
||||
[c.decl.pool_name.value.upper() for c in config.pools]
|
||||
+ ["MAMBA"]
|
||||
+ (["INDEXER"] if qsa_pool is not None else [])
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -2223,7 +2228,7 @@ def _select_strategy(kvcache: Any, components: set[ComponentType]) -> StackStrat
|
||||
|
||||
# Strategies that assemble from host_pool_decls(), so every declared pool has an
|
||||
# entry by construction. A miss here is a bug, not an unsupported combination.
|
||||
_DECLARATION_VERIFIED_STRATEGIES: tuple[type, ...] = (_DsaStrategy,)
|
||||
_DECLARATION_VERIFIED_STRATEGIES: tuple[type, ...] = (_DsaStrategy, _MambaStrategy)
|
||||
|
||||
|
||||
def _check_declared_pools_present(
|
||||
|
||||
Reference in New Issue
Block a user