[Feat] unified_cache: Support FULL + MAMBA MLA models (Kimi-K3) in the direct external linker (#44977)
Co-authored-by: Yutong Wu <yutong.wu@amd.com> Co-authored-by: Zhangheng <hzh0425@apache.org>
This commit is contained in:
co-authored by
Yutong Wu
Zhangheng
parent
f1ba723c64
commit
eb91a41f38
@@ -72,6 +72,8 @@ class CacheInitParams:
|
||||
tree_core_backend: Optional[str] = dataclasses.field(default=None, kw_only=True)
|
||||
|
||||
mtp_draft_device_pools: tuple[object, ...] = ()
|
||||
# Draft KV pools that share the target KV slots but are stored apart from it.
|
||||
sidecar_draft_device_pools: tuple[object, ...] = ()
|
||||
mtp_draft_mamba_pools: tuple[object, ...] = ()
|
||||
|
||||
@property
|
||||
|
||||
@@ -1960,6 +1960,13 @@ class _MambaStrategy(StackStrategy):
|
||||
ComponentType.MAMBA,
|
||||
}
|
||||
|
||||
def build_direct_linker_pool_group(self, *, kvcache, params, page_size):
|
||||
from sglang.srt.mem_cache.hybrid_cache.linker_pool_assembler import (
|
||||
_build_mamba_device_pool_group,
|
||||
)
|
||||
|
||||
return _build_mamba_device_pool_group(kvcache, params, page_size)
|
||||
|
||||
def build(
|
||||
self,
|
||||
*,
|
||||
|
||||
@@ -14,7 +14,7 @@ from sglang.srt.mem_cache.hicache_storage import (
|
||||
PoolTransfer,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
|
||||
from sglang.srt.runtime_context import get_memory
|
||||
from sglang.srt.runtime_context import get_memory, get_parallel
|
||||
|
||||
|
||||
class DevicePoolEntry:
|
||||
@@ -460,7 +460,11 @@ def _build_dsa_device_pool_group(
|
||||
|
||||
|
||||
def _mha_device_pool_entry(
|
||||
name: PoolName, pool: Any, layer_mapping: dict[int, int], page_size: int
|
||||
name: PoolName,
|
||||
pool: Any,
|
||||
layer_mapping: dict[int, int],
|
||||
page_size: int,
|
||||
indices_from_pool: PoolName | None = None,
|
||||
) -> DevicePoolEntry:
|
||||
# TODO(alphabetc1): support mxfp8.
|
||||
if pool.k_scale_buffer is not None or pool.native_k_scale_buffer is not None:
|
||||
@@ -470,7 +474,7 @@ def _mha_device_pool_entry(
|
||||
num_layers = len(pool.k_buffer)
|
||||
return DevicePoolEntry(
|
||||
name=name,
|
||||
indices_from_pool=name,
|
||||
indices_from_pool=indices_from_pool or name,
|
||||
device_pool=pool,
|
||||
components=[[*pool.k_buffer, *pool.v_buffer]],
|
||||
layer_mapping={
|
||||
@@ -488,6 +492,23 @@ def _mamba_device_pool_entry(
|
||||
_stage_local_layer_mapping,
|
||||
)
|
||||
|
||||
# TODO(alphabetc1): support unified memory for hybrid Mamba models.
|
||||
if get_memory().enable_unified_memory:
|
||||
raise ValueError(
|
||||
"The direct external linker does not support --enable-unified-memory "
|
||||
"on hybrid Mamba models yet."
|
||||
)
|
||||
# TODO(alphabetc1): support int8 Mamba checkpoints and Mamba side states.
|
||||
if (
|
||||
req_to_token_pool.mamba_ckpt_pool is not None
|
||||
or req_to_token_pool.short_conv_pool.enabled
|
||||
or req_to_token_pool.ngram_pool.enabled
|
||||
):
|
||||
raise ValueError(
|
||||
"The direct external linker does not support int8 Mamba checkpoints "
|
||||
"or Mamba side states yet."
|
||||
)
|
||||
|
||||
mamba_pool = req_to_token_pool.mamba_pool
|
||||
mamba_cache = mamba_pool.mamba_cache
|
||||
layers = range(mamba_pool.num_mamba_layers)
|
||||
@@ -517,24 +538,6 @@ def _build_mamba_swa_device_pool_group(
|
||||
_swa_layer_mappings,
|
||||
)
|
||||
|
||||
# TODO(alphabetc1): support unified memory for hybrid Mamba models.
|
||||
if get_memory().enable_unified_memory:
|
||||
raise ValueError(
|
||||
"The direct external linker does not support --enable-unified-memory "
|
||||
"on hybrid Mamba models yet."
|
||||
)
|
||||
# TODO(alphabetc1): support int8 Mamba checkpoints and Mamba side states.
|
||||
req_to_token_pool = params.req_to_token_pool
|
||||
if (
|
||||
req_to_token_pool.mamba_ckpt_pool is not None
|
||||
or req_to_token_pool.short_conv_pool.enabled
|
||||
or req_to_token_pool.ngram_pool.enabled
|
||||
):
|
||||
raise ValueError(
|
||||
"The direct external linker does not support int8 Mamba checkpoints "
|
||||
"or Mamba side states yet."
|
||||
)
|
||||
|
||||
full_mapping, swa_mapping = _swa_layer_mappings(kvcache)
|
||||
entries = [
|
||||
_mha_device_pool_entry(
|
||||
@@ -549,6 +552,63 @@ def _build_mamba_swa_device_pool_group(
|
||||
return DevicePoolGroup(entries, num_layers + 1, page_size)
|
||||
|
||||
|
||||
def _build_mamba_device_pool_group(
|
||||
kvcache: Any, params: Any, page_size: int
|
||||
) -> DevicePoolGroup:
|
||||
from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import (
|
||||
_stage_local_layer_mapping,
|
||||
)
|
||||
from sglang.srt.mem_cache.memory_pool import (
|
||||
HybridLinearKVPool,
|
||||
MLATokenToKVPool,
|
||||
)
|
||||
|
||||
# TODO(alphabetc1): support other full-attention pools, DCP and packed draft KV.
|
||||
full_kv_pool = kvcache.full_kv_pool
|
||||
if (
|
||||
type(kvcache) is not HybridLinearKVPool
|
||||
or type(full_kv_pool) is not MLATokenToKVPool
|
||||
or get_parallel().attn_dcp_size > 1
|
||||
or params.mtp_draft_device_pools
|
||||
):
|
||||
raise ValueError(
|
||||
"The direct external linker supports FULL + MAMBA models only with a "
|
||||
"plain MLA pool, without DCP or packed draft KV."
|
||||
)
|
||||
|
||||
entries = [
|
||||
DevicePoolEntry(
|
||||
name=PoolName.KV,
|
||||
indices_from_pool=PoolName.KV,
|
||||
device_pool=full_kv_pool,
|
||||
components=[full_kv_pool.kv_buffer],
|
||||
layer_mapping=_stage_local_layer_mapping(
|
||||
kvcache.full_attention_layer_id_mapping, kvcache.start_layer
|
||||
),
|
||||
page_size=page_size,
|
||||
rows_are_pages=False,
|
||||
),
|
||||
_mamba_device_pool_entry(params.req_to_token_pool, kvcache.start_layer),
|
||||
]
|
||||
for draft_pool in params.sidecar_draft_device_pools:
|
||||
if draft_pool.page_size != page_size:
|
||||
raise ValueError("Draft KV page size must match the tree page size.")
|
||||
# Draft layer i rides transfer layer i; the draft reads it only after
|
||||
# the batch's loads finish.
|
||||
entries.append(
|
||||
_mha_device_pool_entry(
|
||||
PoolName.DRAFT,
|
||||
draft_pool,
|
||||
{layer: layer for layer in range(draft_pool.layer_num)},
|
||||
page_size,
|
||||
indices_from_pool=PoolName.KV,
|
||||
)
|
||||
)
|
||||
num_layers = max(layer for entry in entries for layer in entry.layer_mapping)
|
||||
# Not rank_replicated: the Mamba state is TP-sharded.
|
||||
return DevicePoolGroup(entries, num_layers + 1, page_size)
|
||||
|
||||
|
||||
def resolve_hybrid_device_pool_group(
|
||||
*,
|
||||
kvcache: Any,
|
||||
@@ -561,8 +621,15 @@ def resolve_hybrid_device_pool_group(
|
||||
_select_strategy,
|
||||
)
|
||||
|
||||
return _select_strategy(kvcache, components).build_direct_linker_pool_group(
|
||||
group = _select_strategy(kvcache, components).build_direct_linker_pool_group(
|
||||
kvcache=kvcache,
|
||||
params=params,
|
||||
page_size=page_size,
|
||||
)
|
||||
if getattr(params, "sidecar_draft_device_pools", ()) and (
|
||||
PoolName.DRAFT not in group.entry_map
|
||||
):
|
||||
raise ValueError(
|
||||
"The direct external linker does not support this model's draft KV."
|
||||
)
|
||||
return group
|
||||
|
||||
@@ -409,6 +409,12 @@ def build_kv_cache(
|
||||
draft_swa_window=draft_swa_window,
|
||||
mtp_draft_device_pools=mtp_draft_device_pools,
|
||||
mtp_draft_mamba_pools=tp_worker.model_runner.mtp_draft_mamba_pools,
|
||||
sidecar_draft_device_pools=(
|
||||
hicache_draft_plan.device_pools
|
||||
if hicache_draft_plan is not None
|
||||
and hicache_draft_plan.mode == HiCacheDraftMode.SIDECAR
|
||||
else ()
|
||||
),
|
||||
)
|
||||
if params.swa_retain_window != sliding_window_size:
|
||||
logger.info(
|
||||
|
||||
@@ -307,9 +307,9 @@ class BaseSpecWorker(ABC):
|
||||
device_pools=draft_pools,
|
||||
)
|
||||
|
||||
if get_memory().enable_unified_cache_external_linker:
|
||||
if get_memory().enable_unified_cache_external_linker and len(draft_pools) > 1:
|
||||
raise NotImplementedError(
|
||||
"The external linker only supports packed draft KV caches."
|
||||
"The external linker supports one non-packed draft KV pool."
|
||||
)
|
||||
|
||||
return HiCacheDraftPlan(
|
||||
|
||||
Reference in New Issue
Block a user