[Feat] unified_cache: Support hybrid SWA + Mamba models in the Mooncake direct linker (#41343)
Co-authored-by: Zhangheng <hzh0425@apache.org> Co-authored-by: Teng Ma <sima.mt@alibaba-inc.com>
This commit is contained in:
co-authored by
Zhangheng
Teng Ma
parent
3b34d9f8b6
commit
05c1539098
@@ -2023,6 +2023,13 @@ class _MambaSwaStrategy(StackStrategy):
|
||||
== {ComponentType.FULL, ComponentType.SWA, ComponentType.MAMBA}
|
||||
)
|
||||
|
||||
def build_direct_linker_pool_group(self, *, kvcache, params, page_size):
|
||||
from sglang.srt.mem_cache.hybrid_cache.linker_pool_assembler import (
|
||||
_build_mamba_swa_device_pool_group,
|
||||
)
|
||||
|
||||
return _build_mamba_swa_device_pool_group(kvcache, params, page_size)
|
||||
|
||||
def build(
|
||||
self,
|
||||
*,
|
||||
|
||||
@@ -14,6 +14,7 @@ from sglang.srt.mem_cache.hicache_storage import (
|
||||
PoolTransfer,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
|
||||
from sglang.srt.runtime_context import get_memory
|
||||
|
||||
|
||||
class DevicePoolEntry:
|
||||
@@ -442,6 +443,96 @@ def _build_dsa_device_pool_group(
|
||||
return DevicePoolGroup(entries, num_layers, page_size, rank_replicated=True)
|
||||
|
||||
|
||||
def _mha_device_pool_entry(
|
||||
name: PoolName, pool: Any, layer_mapping: dict[int, int], page_size: int
|
||||
) -> DevicePoolEntry:
|
||||
# TODO(alphabetc1): support mxfp8.
|
||||
if pool.k_scale_buffer is not None or pool.native_k_scale_buffer is not None:
|
||||
raise ValueError(
|
||||
"The direct external linker does not support scaled KV caches yet."
|
||||
)
|
||||
num_layers = len(pool.k_buffer)
|
||||
return DevicePoolEntry(
|
||||
name=name,
|
||||
indices_from_pool=name,
|
||||
device_pool=pool,
|
||||
components=[[*pool.k_buffer, *pool.v_buffer]],
|
||||
layer_mapping={
|
||||
layer: (index, num_layers + index) for layer, index in layer_mapping.items()
|
||||
},
|
||||
page_size=page_size,
|
||||
rows_are_pages=False,
|
||||
)
|
||||
|
||||
|
||||
def _mamba_device_pool_entry(
|
||||
req_to_token_pool: Any, start_layer: int
|
||||
) -> DevicePoolEntry:
|
||||
from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import (
|
||||
_stage_local_layer_mapping,
|
||||
)
|
||||
|
||||
mamba_pool = req_to_token_pool.mamba_pool
|
||||
mamba_cache = mamba_pool.mamba_cache
|
||||
layers = range(mamba_pool.num_mamba_layers)
|
||||
# One row per state slot; conv-only models carry an empty temporal state.
|
||||
components = [
|
||||
[state[layer] for layer in layers]
|
||||
for state in (mamba_cache.temporal, *mamba_cache.conv)
|
||||
if state.numel() > 0
|
||||
]
|
||||
return DevicePoolEntry(
|
||||
name=PoolName.MAMBA,
|
||||
indices_from_pool=PoolName.MAMBA,
|
||||
device_pool=mamba_pool,
|
||||
components=components,
|
||||
layer_mapping=_stage_local_layer_mapping(
|
||||
req_to_token_pool.mamba_map, start_layer
|
||||
),
|
||||
page_size=1,
|
||||
rows_are_pages=True,
|
||||
)
|
||||
|
||||
|
||||
def _build_mamba_swa_device_pool_group(
|
||||
kvcache: Any, params: Any, page_size: int
|
||||
) -> DevicePoolGroup:
|
||||
from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import (
|
||||
_swa_layer_mappings,
|
||||
)
|
||||
|
||||
# TODO(alphabetc1): support unified memory for hybrid Mamba models.
|
||||
if get_memory().enable_unified_memory:
|
||||
raise ValueError(
|
||||
"The direct external linker does not support --enable-unified-memory "
|
||||
"on hybrid Mamba models yet."
|
||||
)
|
||||
# TODO(alphabetc1): support int8 Mamba checkpoints and Mamba side states.
|
||||
req_to_token_pool = params.req_to_token_pool
|
||||
if (
|
||||
req_to_token_pool.mamba_ckpt_pool is not None
|
||||
or req_to_token_pool.short_conv_pool.enabled
|
||||
or req_to_token_pool.ngram_pool.enabled
|
||||
):
|
||||
raise ValueError(
|
||||
"The direct external linker does not support int8 Mamba checkpoints "
|
||||
"or Mamba side states yet."
|
||||
)
|
||||
|
||||
full_mapping, swa_mapping = _swa_layer_mappings(kvcache)
|
||||
entries = [
|
||||
_mha_device_pool_entry(
|
||||
PoolName.KV, kvcache.full_kv_pool, full_mapping, page_size
|
||||
),
|
||||
_mha_device_pool_entry(
|
||||
PoolName.SWA, kvcache.swa_kv_pool, swa_mapping, page_size
|
||||
),
|
||||
_mamba_device_pool_entry(params.req_to_token_pool, kvcache.start_layer),
|
||||
]
|
||||
num_layers = max(layer for entry in entries for layer in entry.layer_mapping)
|
||||
return DevicePoolGroup(entries, num_layers + 1, page_size)
|
||||
|
||||
|
||||
def resolve_hybrid_device_pool_group(
|
||||
*,
|
||||
kvcache: Any,
|
||||
|
||||
@@ -21,6 +21,7 @@ from sglang.srt.mem_cache.hicache_storage import (
|
||||
PoolTransfer,
|
||||
PoolTransferResult,
|
||||
)
|
||||
from sglang.srt.mem_cache.hybrid_cache.linker_pool_assembler import DevicePoolEntry
|
||||
from sglang.srt.mem_cache.pool_host import HostKVCache, HostTensorAllocator
|
||||
from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost
|
||||
from sglang.srt.observability.metrics_collector import StorageMetrics
|
||||
@@ -772,6 +773,12 @@ class MooncakeStore(HiCacheStorage, MooncakeBaseStore):
|
||||
if host_pool is None:
|
||||
raise ValueError(f"Unregistered Mooncake hybrid pool: {transfer.name}")
|
||||
|
||||
if isinstance(host_pool, DevicePoolEntry):
|
||||
# A direct-linker entry packs every buffer of a page into one object.
|
||||
object_name = "k" if transfer.name == PoolName.KV else transfer.name
|
||||
suffix = f"_{self.mla_suffix}_{object_name}"
|
||||
return [f"{page_key}{suffix}" for page_key in page_keys], 1
|
||||
|
||||
# Suffix order must match get_page_buffer_meta() for one page, because
|
||||
# Mooncake zips object keys with registered buffer pointers.
|
||||
pool_name = transfer.name
|
||||
|
||||
@@ -31,6 +31,7 @@ from sglang.srt.mem_cache.unified_cache.components.base import (
|
||||
CacheTransferPhase,
|
||||
ComponentType,
|
||||
EvictLayer,
|
||||
ExternalLinkerLoadPhase,
|
||||
InternalStateBackup,
|
||||
LinkerTransferPhase,
|
||||
LRURefreshPhase,
|
||||
@@ -692,9 +693,62 @@ class MambaComponent(TreeComponent):
|
||||
node: Optional[UnifiedTreeNode],
|
||||
keys: Optional[Sequence[str]],
|
||||
) -> Optional[PoolTransfer]:
|
||||
raise AssertionError(
|
||||
"MambaComponent does not support external linker mode, will support soon"
|
||||
if phase == LinkerTransferPhase.OFFLOAD:
|
||||
if node is None or not node.hash_value:
|
||||
return None
|
||||
value = node.component_data[self.component_type].value
|
||||
if value is None:
|
||||
return None
|
||||
# One state per node, valid only at the node's end boundary.
|
||||
return PoolTransfer(
|
||||
name=PoolName.MAMBA,
|
||||
device_indices=value,
|
||||
keys=[node.hash_value[-1]],
|
||||
hit_policy=PoolHitPolicy.TRAILING_PAGES,
|
||||
)
|
||||
|
||||
if not keys:
|
||||
return None
|
||||
|
||||
transfer = PoolTransfer(
|
||||
name=PoolName.MAMBA,
|
||||
keys=[keys[-1]],
|
||||
hit_policy=PoolHitPolicy.TRAILING_PAGES,
|
||||
)
|
||||
if phase == LinkerTransferPhase.LOAD:
|
||||
transfer.device_indices = self._alloc_mamba_slot()
|
||||
return transfer
|
||||
|
||||
def update_external_linker_load(
|
||||
self,
|
||||
phase: ExternalLinkerLoadPhase,
|
||||
req: Req,
|
||||
full_transfer: PoolTransfer,
|
||||
transfer: PoolTransfer,
|
||||
prefix_len: int,
|
||||
) -> Optional[PoolTransfer]:
|
||||
pool = self.cache.req_to_token_pool
|
||||
if phase == ExternalLinkerLoadPhase.ABORT:
|
||||
pool.mamba_allocator.free(transfer.device_indices)
|
||||
return None
|
||||
|
||||
assert phase == ExternalLinkerLoadPhase.PREPARE
|
||||
if req.kv.holds_mamba:
|
||||
pool.mamba_allocator.free(req.kv.mamba_pool_idx.unsqueeze(0))
|
||||
# The layer-gated load writes the request's private state directly.
|
||||
req.kv.mamba_pool_idx = transfer.device_indices[0]
|
||||
self._adopt_loaded_state(req)
|
||||
return transfer
|
||||
|
||||
def _adopt_loaded_state(self, req: Req) -> None:
|
||||
# A loaded state supersedes the replay cursor and the deferred CoW/clear.
|
||||
pool = self.cache.req_to_token_pool
|
||||
write_pos = pool.mamba_pool.replayssm_write_pos
|
||||
if write_pos is not None:
|
||||
slot = req.kv.mamba_pool_idx.unsqueeze(0)
|
||||
write_pos[pool.translate_mamba_indices(slot)] = 0
|
||||
req.kv.mamba_cow_src_index = None
|
||||
req.kv.mamba_needs_clear = False
|
||||
|
||||
# ---- HiCache Hooks ----
|
||||
|
||||
@@ -1035,14 +1089,7 @@ class _MambaBufferLoadBackContext(BufferLoadBackContext):
|
||||
if not success:
|
||||
self.component.finalize_load_back(req, self.prep, success=False)
|
||||
return
|
||||
# The H2D supersedes the replay cursor and the deferred CoW/clear.
|
||||
req_to_token_pool = self.component.cache.req_to_token_pool
|
||||
write_pos = req_to_token_pool.mamba_pool.replayssm_write_pos
|
||||
if write_pos is not None:
|
||||
slot = req.kv.mamba_pool_idx.unsqueeze(0)
|
||||
write_pos[req_to_token_pool.translate_mamba_indices(slot)] = 0
|
||||
req.kv.mamba_cow_src_index = None
|
||||
req.kv.mamba_needs_clear = False
|
||||
self.component._adopt_loaded_state(req)
|
||||
|
||||
def get_insert_fields(self) -> dict[str, torch.Tensor]:
|
||||
return {"mamba_value": self.node_copy.device_indices}
|
||||
|
||||
@@ -59,6 +59,7 @@ _EXTERNAL_LINKER_SUPPORTED_COMPONENTS = frozenset(
|
||||
{
|
||||
ComponentType.FULL,
|
||||
ComponentType.SWA,
|
||||
ComponentType.MAMBA,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -220,7 +221,7 @@ class UnifiedCacheLinkerWrapper:
|
||||
component.name for component in sorted(unsupported, key=int)
|
||||
)
|
||||
raise ValueError(
|
||||
"External cache linker supports only Full and SWA tree "
|
||||
"External cache linker supports only Full, SWA and Mamba tree "
|
||||
f"components; unsupported: {names}"
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user