[Feat] unified_cache: Support FULL + MAMBA MLA models (Kimi-K3) in the direct external linker (#44977)

Co-authored-by: Yutong Wu <yutong.wu@amd.com>
Co-authored-by: Zhangheng <hzh0425@apache.org>
This commit is contained in:
Shuwen Wang
2026-10-21 10:47:17 +08:00
committed by GitHub
co-authored by Yutong Wu Zhangheng
parent f1ba723c64
commit eb91a41f38
7 changed files with 324 additions and 49 deletions
@@ -72,6 +72,8 @@ class CacheInitParams:
tree_core_backend: Optional[str] = dataclasses.field(default=None, kw_only=True)
mtp_draft_device_pools: tuple[object, ...] = ()
# Draft KV pools that share the target KV slots but are stored apart from it.
sidecar_draft_device_pools: tuple[object, ...] = ()
mtp_draft_mamba_pools: tuple[object, ...] = ()
@property
@@ -1960,6 +1960,13 @@ class _MambaStrategy(StackStrategy):
ComponentType.MAMBA,
}
def build_direct_linker_pool_group(self, *, kvcache, params, page_size):
from sglang.srt.mem_cache.hybrid_cache.linker_pool_assembler import (
_build_mamba_device_pool_group,
)
return _build_mamba_device_pool_group(kvcache, params, page_size)
def build(
self,
*,
@@ -14,7 +14,7 @@ from sglang.srt.mem_cache.hicache_storage import (
PoolTransfer,
)
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
from sglang.srt.runtime_context import get_memory
from sglang.srt.runtime_context import get_memory, get_parallel
class DevicePoolEntry:
@@ -460,7 +460,11 @@ def _build_dsa_device_pool_group(
def _mha_device_pool_entry(
name: PoolName, pool: Any, layer_mapping: dict[int, int], page_size: int
name: PoolName,
pool: Any,
layer_mapping: dict[int, int],
page_size: int,
indices_from_pool: PoolName | None = None,
) -> DevicePoolEntry:
# TODO(alphabetc1): support mxfp8.
if pool.k_scale_buffer is not None or pool.native_k_scale_buffer is not None:
@@ -470,7 +474,7 @@ def _mha_device_pool_entry(
num_layers = len(pool.k_buffer)
return DevicePoolEntry(
name=name,
indices_from_pool=name,
indices_from_pool=indices_from_pool or name,
device_pool=pool,
components=[[*pool.k_buffer, *pool.v_buffer]],
layer_mapping={
@@ -488,6 +492,23 @@ def _mamba_device_pool_entry(
_stage_local_layer_mapping,
)
# TODO(alphabetc1): support unified memory for hybrid Mamba models.
if get_memory().enable_unified_memory:
raise ValueError(
"The direct external linker does not support --enable-unified-memory "
"on hybrid Mamba models yet."
)
# TODO(alphabetc1): support int8 Mamba checkpoints and Mamba side states.
if (
req_to_token_pool.mamba_ckpt_pool is not None
or req_to_token_pool.short_conv_pool.enabled
or req_to_token_pool.ngram_pool.enabled
):
raise ValueError(
"The direct external linker does not support int8 Mamba checkpoints "
"or Mamba side states yet."
)
mamba_pool = req_to_token_pool.mamba_pool
mamba_cache = mamba_pool.mamba_cache
layers = range(mamba_pool.num_mamba_layers)
@@ -517,24 +538,6 @@ def _build_mamba_swa_device_pool_group(
_swa_layer_mappings,
)
# TODO(alphabetc1): support unified memory for hybrid Mamba models.
if get_memory().enable_unified_memory:
raise ValueError(
"The direct external linker does not support --enable-unified-memory "
"on hybrid Mamba models yet."
)
# TODO(alphabetc1): support int8 Mamba checkpoints and Mamba side states.
req_to_token_pool = params.req_to_token_pool
if (
req_to_token_pool.mamba_ckpt_pool is not None
or req_to_token_pool.short_conv_pool.enabled
or req_to_token_pool.ngram_pool.enabled
):
raise ValueError(
"The direct external linker does not support int8 Mamba checkpoints "
"or Mamba side states yet."
)
full_mapping, swa_mapping = _swa_layer_mappings(kvcache)
entries = [
_mha_device_pool_entry(
@@ -549,6 +552,63 @@ def _build_mamba_swa_device_pool_group(
return DevicePoolGroup(entries, num_layers + 1, page_size)
def _build_mamba_device_pool_group(
kvcache: Any, params: Any, page_size: int
) -> DevicePoolGroup:
from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import (
_stage_local_layer_mapping,
)
from sglang.srt.mem_cache.memory_pool import (
HybridLinearKVPool,
MLATokenToKVPool,
)
# TODO(alphabetc1): support other full-attention pools, DCP and packed draft KV.
full_kv_pool = kvcache.full_kv_pool
if (
type(kvcache) is not HybridLinearKVPool
or type(full_kv_pool) is not MLATokenToKVPool
or get_parallel().attn_dcp_size > 1
or params.mtp_draft_device_pools
):
raise ValueError(
"The direct external linker supports FULL + MAMBA models only with a "
"plain MLA pool, without DCP or packed draft KV."
)
entries = [
DevicePoolEntry(
name=PoolName.KV,
indices_from_pool=PoolName.KV,
device_pool=full_kv_pool,
components=[full_kv_pool.kv_buffer],
layer_mapping=_stage_local_layer_mapping(
kvcache.full_attention_layer_id_mapping, kvcache.start_layer
),
page_size=page_size,
rows_are_pages=False,
),
_mamba_device_pool_entry(params.req_to_token_pool, kvcache.start_layer),
]
for draft_pool in params.sidecar_draft_device_pools:
if draft_pool.page_size != page_size:
raise ValueError("Draft KV page size must match the tree page size.")
# Draft layer i rides transfer layer i; the draft reads it only after
# the batch's loads finish.
entries.append(
_mha_device_pool_entry(
PoolName.DRAFT,
draft_pool,
{layer: layer for layer in range(draft_pool.layer_num)},
page_size,
indices_from_pool=PoolName.KV,
)
)
num_layers = max(layer for entry in entries for layer in entry.layer_mapping)
# Not rank_replicated: the Mamba state is TP-sharded.
return DevicePoolGroup(entries, num_layers + 1, page_size)
def resolve_hybrid_device_pool_group(
*,
kvcache: Any,
@@ -561,8 +621,15 @@ def resolve_hybrid_device_pool_group(
_select_strategy,
)
return _select_strategy(kvcache, components).build_direct_linker_pool_group(
group = _select_strategy(kvcache, components).build_direct_linker_pool_group(
kvcache=kvcache,
params=params,
page_size=page_size,
)
if getattr(params, "sidecar_draft_device_pools", ()) and (
PoolName.DRAFT not in group.entry_map
):
raise ValueError(
"The direct external linker does not support this model's draft KV."
)
return group
@@ -409,6 +409,12 @@ def build_kv_cache(
draft_swa_window=draft_swa_window,
mtp_draft_device_pools=mtp_draft_device_pools,
mtp_draft_mamba_pools=tp_worker.model_runner.mtp_draft_mamba_pools,
sidecar_draft_device_pools=(
hicache_draft_plan.device_pools
if hicache_draft_plan is not None
and hicache_draft_plan.mode == HiCacheDraftMode.SIDECAR
else ()
),
)
if params.swa_retain_window != sliding_window_size:
logger.info(
@@ -307,9 +307,9 @@ class BaseSpecWorker(ABC):
device_pools=draft_pools,
)
if get_memory().enable_unified_cache_external_linker:
if get_memory().enable_unified_cache_external_linker and len(draft_pools) > 1:
raise NotImplementedError(
"The external linker only supports packed draft KV caches."
"The external linker supports one non-packed draft KV pool."
)
return HiCacheDraftPlan(