[Feat] unified_cache: Support hybrid SWA + Mamba models in the Mooncake direct linker (#41343)

Co-authored-by: Zhangheng <hzh0425@apache.org>
Co-authored-by: Teng Ma <sima.mt@alibaba-inc.com>
This commit is contained in:
Shuwen Wang
2026-10-17 13:24:08 -07:00
committed by GitHub
co-authored by Zhangheng Teng Ma
parent 3b34d9f8b6
commit 05c1539098
12 changed files with 445 additions and 46 deletions
@@ -2023,6 +2023,13 @@ class _MambaSwaStrategy(StackStrategy):
== {ComponentType.FULL, ComponentType.SWA, ComponentType.MAMBA}
)
def build_direct_linker_pool_group(self, *, kvcache, params, page_size):
from sglang.srt.mem_cache.hybrid_cache.linker_pool_assembler import (
_build_mamba_swa_device_pool_group,
)
return _build_mamba_swa_device_pool_group(kvcache, params, page_size)
def build(
self,
*,
@@ -14,6 +14,7 @@ from sglang.srt.mem_cache.hicache_storage import (
PoolTransfer,
)
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
from sglang.srt.runtime_context import get_memory
class DevicePoolEntry:
@@ -442,6 +443,96 @@ def _build_dsa_device_pool_group(
return DevicePoolGroup(entries, num_layers, page_size, rank_replicated=True)
def _mha_device_pool_entry(
name: PoolName, pool: Any, layer_mapping: dict[int, int], page_size: int
) -> DevicePoolEntry:
# TODO(alphabetc1): support mxfp8.
if pool.k_scale_buffer is not None or pool.native_k_scale_buffer is not None:
raise ValueError(
"The direct external linker does not support scaled KV caches yet."
)
num_layers = len(pool.k_buffer)
return DevicePoolEntry(
name=name,
indices_from_pool=name,
device_pool=pool,
components=[[*pool.k_buffer, *pool.v_buffer]],
layer_mapping={
layer: (index, num_layers + index) for layer, index in layer_mapping.items()
},
page_size=page_size,
rows_are_pages=False,
)
def _mamba_device_pool_entry(
req_to_token_pool: Any, start_layer: int
) -> DevicePoolEntry:
from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import (
_stage_local_layer_mapping,
)
mamba_pool = req_to_token_pool.mamba_pool
mamba_cache = mamba_pool.mamba_cache
layers = range(mamba_pool.num_mamba_layers)
# One row per state slot; conv-only models carry an empty temporal state.
components = [
[state[layer] for layer in layers]
for state in (mamba_cache.temporal, *mamba_cache.conv)
if state.numel() > 0
]
return DevicePoolEntry(
name=PoolName.MAMBA,
indices_from_pool=PoolName.MAMBA,
device_pool=mamba_pool,
components=components,
layer_mapping=_stage_local_layer_mapping(
req_to_token_pool.mamba_map, start_layer
),
page_size=1,
rows_are_pages=True,
)
def _build_mamba_swa_device_pool_group(
kvcache: Any, params: Any, page_size: int
) -> DevicePoolGroup:
from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import (
_swa_layer_mappings,
)
# TODO(alphabetc1): support unified memory for hybrid Mamba models.
if get_memory().enable_unified_memory:
raise ValueError(
"The direct external linker does not support --enable-unified-memory "
"on hybrid Mamba models yet."
)
# TODO(alphabetc1): support int8 Mamba checkpoints and Mamba side states.
req_to_token_pool = params.req_to_token_pool
if (
req_to_token_pool.mamba_ckpt_pool is not None
or req_to_token_pool.short_conv_pool.enabled
or req_to_token_pool.ngram_pool.enabled
):
raise ValueError(
"The direct external linker does not support int8 Mamba checkpoints "
"or Mamba side states yet."
)
full_mapping, swa_mapping = _swa_layer_mappings(kvcache)
entries = [
_mha_device_pool_entry(
PoolName.KV, kvcache.full_kv_pool, full_mapping, page_size
),
_mha_device_pool_entry(
PoolName.SWA, kvcache.swa_kv_pool, swa_mapping, page_size
),
_mamba_device_pool_entry(params.req_to_token_pool, kvcache.start_layer),
]
num_layers = max(layer for entry in entries for layer in entry.layer_mapping)
return DevicePoolGroup(entries, num_layers + 1, page_size)
def resolve_hybrid_device_pool_group(
*,
kvcache: Any,
@@ -21,6 +21,7 @@ from sglang.srt.mem_cache.hicache_storage import (
PoolTransfer,
PoolTransferResult,
)
from sglang.srt.mem_cache.hybrid_cache.linker_pool_assembler import DevicePoolEntry
from sglang.srt.mem_cache.pool_host import HostKVCache, HostTensorAllocator
from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost
from sglang.srt.observability.metrics_collector import StorageMetrics
@@ -772,6 +773,12 @@ class MooncakeStore(HiCacheStorage, MooncakeBaseStore):
if host_pool is None:
raise ValueError(f"Unregistered Mooncake hybrid pool: {transfer.name}")
if isinstance(host_pool, DevicePoolEntry):
# A direct-linker entry packs every buffer of a page into one object.
object_name = "k" if transfer.name == PoolName.KV else transfer.name
suffix = f"_{self.mla_suffix}_{object_name}"
return [f"{page_key}{suffix}" for page_key in page_keys], 1
# Suffix order must match get_page_buffer_meta() for one page, because
# Mooncake zips object keys with registered buffer pointers.
pool_name = transfer.name
@@ -31,6 +31,7 @@ from sglang.srt.mem_cache.unified_cache.components.base import (
CacheTransferPhase,
ComponentType,
EvictLayer,
ExternalLinkerLoadPhase,
InternalStateBackup,
LinkerTransferPhase,
LRURefreshPhase,
@@ -692,9 +693,62 @@ class MambaComponent(TreeComponent):
node: Optional[UnifiedTreeNode],
keys: Optional[Sequence[str]],
) -> Optional[PoolTransfer]:
raise AssertionError(
"MambaComponent does not support external linker mode, will support soon"
if phase == LinkerTransferPhase.OFFLOAD:
if node is None or not node.hash_value:
return None
value = node.component_data[self.component_type].value
if value is None:
return None
# One state per node, valid only at the node's end boundary.
return PoolTransfer(
name=PoolName.MAMBA,
device_indices=value,
keys=[node.hash_value[-1]],
hit_policy=PoolHitPolicy.TRAILING_PAGES,
)
if not keys:
return None
transfer = PoolTransfer(
name=PoolName.MAMBA,
keys=[keys[-1]],
hit_policy=PoolHitPolicy.TRAILING_PAGES,
)
if phase == LinkerTransferPhase.LOAD:
transfer.device_indices = self._alloc_mamba_slot()
return transfer
def update_external_linker_load(
self,
phase: ExternalLinkerLoadPhase,
req: Req,
full_transfer: PoolTransfer,
transfer: PoolTransfer,
prefix_len: int,
) -> Optional[PoolTransfer]:
pool = self.cache.req_to_token_pool
if phase == ExternalLinkerLoadPhase.ABORT:
pool.mamba_allocator.free(transfer.device_indices)
return None
assert phase == ExternalLinkerLoadPhase.PREPARE
if req.kv.holds_mamba:
pool.mamba_allocator.free(req.kv.mamba_pool_idx.unsqueeze(0))
# The layer-gated load writes the request's private state directly.
req.kv.mamba_pool_idx = transfer.device_indices[0]
self._adopt_loaded_state(req)
return transfer
def _adopt_loaded_state(self, req: Req) -> None:
# A loaded state supersedes the replay cursor and the deferred CoW/clear.
pool = self.cache.req_to_token_pool
write_pos = pool.mamba_pool.replayssm_write_pos
if write_pos is not None:
slot = req.kv.mamba_pool_idx.unsqueeze(0)
write_pos[pool.translate_mamba_indices(slot)] = 0
req.kv.mamba_cow_src_index = None
req.kv.mamba_needs_clear = False
# ---- HiCache Hooks ----
@@ -1035,14 +1089,7 @@ class _MambaBufferLoadBackContext(BufferLoadBackContext):
if not success:
self.component.finalize_load_back(req, self.prep, success=False)
return
# The H2D supersedes the replay cursor and the deferred CoW/clear.
req_to_token_pool = self.component.cache.req_to_token_pool
write_pos = req_to_token_pool.mamba_pool.replayssm_write_pos
if write_pos is not None:
slot = req.kv.mamba_pool_idx.unsqueeze(0)
write_pos[req_to_token_pool.translate_mamba_indices(slot)] = 0
req.kv.mamba_cow_src_index = None
req.kv.mamba_needs_clear = False
self.component._adopt_loaded_state(req)
def get_insert_fields(self) -> dict[str, torch.Tensor]:
return {"mamba_value": self.node_copy.device_indices}
@@ -59,6 +59,7 @@ _EXTERNAL_LINKER_SUPPORTED_COMPONENTS = frozenset(
{
ComponentType.FULL,
ComponentType.SWA,
ComponentType.MAMBA,
}
)
@@ -220,7 +221,7 @@ class UnifiedCacheLinkerWrapper:
component.name for component in sorted(unsupported, key=int)
)
raise ValueError(
"External cache linker supports only Full and SWA tree "
"External cache linker supports only Full, SWA and Mamba tree "
f"components; unsupported: {names}"
)