[Feat] unified_cache: Support FULL + MAMBA MLA models (Kimi-K3) in the direct external linker (#44977)

Co-authored-by: Yutong Wu <yutong.wu@amd.com>
Co-authored-by: Zhangheng <hzh0425@apache.org>
This commit is contained in:
Shuwen Wang
2026-10-21 10:47:17 +08:00
committed by GitHub
co-authored by Yutong Wu Zhangheng
parent f1ba723c64
commit eb91a41f38
7 changed files with 324 additions and 49 deletions
@@ -72,6 +72,8 @@ class CacheInitParams:
tree_core_backend: Optional[str] = dataclasses.field(default=None, kw_only=True)
mtp_draft_device_pools: tuple[object, ...] = ()
# Draft KV pools that share the target KV slots but are stored apart from it.
sidecar_draft_device_pools: tuple[object, ...] = ()
mtp_draft_mamba_pools: tuple[object, ...] = ()
@property
@@ -1960,6 +1960,13 @@ class _MambaStrategy(StackStrategy):
ComponentType.MAMBA,
}
def build_direct_linker_pool_group(self, *, kvcache, params, page_size):
from sglang.srt.mem_cache.hybrid_cache.linker_pool_assembler import (
_build_mamba_device_pool_group,
)
return _build_mamba_device_pool_group(kvcache, params, page_size)
def build(
self,
*,
@@ -14,7 +14,7 @@ from sglang.srt.mem_cache.hicache_storage import (
PoolTransfer,
)
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
from sglang.srt.runtime_context import get_memory
from sglang.srt.runtime_context import get_memory, get_parallel
class DevicePoolEntry:
@@ -460,7 +460,11 @@ def _build_dsa_device_pool_group(
def _mha_device_pool_entry(
name: PoolName, pool: Any, layer_mapping: dict[int, int], page_size: int
name: PoolName,
pool: Any,
layer_mapping: dict[int, int],
page_size: int,
indices_from_pool: PoolName | None = None,
) -> DevicePoolEntry:
# TODO(alphabetc1): support mxfp8.
if pool.k_scale_buffer is not None or pool.native_k_scale_buffer is not None:
@@ -470,7 +474,7 @@ def _mha_device_pool_entry(
num_layers = len(pool.k_buffer)
return DevicePoolEntry(
name=name,
indices_from_pool=name,
indices_from_pool=indices_from_pool or name,
device_pool=pool,
components=[[*pool.k_buffer, *pool.v_buffer]],
layer_mapping={
@@ -488,6 +492,23 @@ def _mamba_device_pool_entry(
_stage_local_layer_mapping,
)
# TODO(alphabetc1): support unified memory for hybrid Mamba models.
if get_memory().enable_unified_memory:
raise ValueError(
"The direct external linker does not support --enable-unified-memory "
"on hybrid Mamba models yet."
)
# TODO(alphabetc1): support int8 Mamba checkpoints and Mamba side states.
if (
req_to_token_pool.mamba_ckpt_pool is not None
or req_to_token_pool.short_conv_pool.enabled
or req_to_token_pool.ngram_pool.enabled
):
raise ValueError(
"The direct external linker does not support int8 Mamba checkpoints "
"or Mamba side states yet."
)
mamba_pool = req_to_token_pool.mamba_pool
mamba_cache = mamba_pool.mamba_cache
layers = range(mamba_pool.num_mamba_layers)
@@ -517,24 +538,6 @@ def _build_mamba_swa_device_pool_group(
_swa_layer_mappings,
)
# TODO(alphabetc1): support unified memory for hybrid Mamba models.
if get_memory().enable_unified_memory:
raise ValueError(
"The direct external linker does not support --enable-unified-memory "
"on hybrid Mamba models yet."
)
# TODO(alphabetc1): support int8 Mamba checkpoints and Mamba side states.
req_to_token_pool = params.req_to_token_pool
if (
req_to_token_pool.mamba_ckpt_pool is not None
or req_to_token_pool.short_conv_pool.enabled
or req_to_token_pool.ngram_pool.enabled
):
raise ValueError(
"The direct external linker does not support int8 Mamba checkpoints "
"or Mamba side states yet."
)
full_mapping, swa_mapping = _swa_layer_mappings(kvcache)
entries = [
_mha_device_pool_entry(
@@ -549,6 +552,63 @@ def _build_mamba_swa_device_pool_group(
return DevicePoolGroup(entries, num_layers + 1, page_size)
def _build_mamba_device_pool_group(
kvcache: Any, params: Any, page_size: int
) -> DevicePoolGroup:
from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import (
_stage_local_layer_mapping,
)
from sglang.srt.mem_cache.memory_pool import (
HybridLinearKVPool,
MLATokenToKVPool,
)
# TODO(alphabetc1): support other full-attention pools, DCP and packed draft KV.
full_kv_pool = kvcache.full_kv_pool
if (
type(kvcache) is not HybridLinearKVPool
or type(full_kv_pool) is not MLATokenToKVPool
or get_parallel().attn_dcp_size > 1
or params.mtp_draft_device_pools
):
raise ValueError(
"The direct external linker supports FULL + MAMBA models only with a "
"plain MLA pool, without DCP or packed draft KV."
)
entries = [
DevicePoolEntry(
name=PoolName.KV,
indices_from_pool=PoolName.KV,
device_pool=full_kv_pool,
components=[full_kv_pool.kv_buffer],
layer_mapping=_stage_local_layer_mapping(
kvcache.full_attention_layer_id_mapping, kvcache.start_layer
),
page_size=page_size,
rows_are_pages=False,
),
_mamba_device_pool_entry(params.req_to_token_pool, kvcache.start_layer),
]
for draft_pool in params.sidecar_draft_device_pools:
if draft_pool.page_size != page_size:
raise ValueError("Draft KV page size must match the tree page size.")
# Draft layer i rides transfer layer i; the draft reads it only after
# the batch's loads finish.
entries.append(
_mha_device_pool_entry(
PoolName.DRAFT,
draft_pool,
{layer: layer for layer in range(draft_pool.layer_num)},
page_size,
indices_from_pool=PoolName.KV,
)
)
num_layers = max(layer for entry in entries for layer in entry.layer_mapping)
# Not rank_replicated: the Mamba state is TP-sharded.
return DevicePoolGroup(entries, num_layers + 1, page_size)
def resolve_hybrid_device_pool_group(
*,
kvcache: Any,
@@ -561,8 +621,15 @@ def resolve_hybrid_device_pool_group(
_select_strategy,
)
return _select_strategy(kvcache, components).build_direct_linker_pool_group(
group = _select_strategy(kvcache, components).build_direct_linker_pool_group(
kvcache=kvcache,
params=params,
page_size=page_size,
)
if getattr(params, "sidecar_draft_device_pools", ()) and (
PoolName.DRAFT not in group.entry_map
):
raise ValueError(
"The direct external linker does not support this model's draft KV."
)
return group
@@ -409,6 +409,12 @@ def build_kv_cache(
draft_swa_window=draft_swa_window,
mtp_draft_device_pools=mtp_draft_device_pools,
mtp_draft_mamba_pools=tp_worker.model_runner.mtp_draft_mamba_pools,
sidecar_draft_device_pools=(
hicache_draft_plan.device_pools
if hicache_draft_plan is not None
and hicache_draft_plan.mode == HiCacheDraftMode.SIDECAR
else ()
),
)
if params.swa_retain_window != sliding_window_size:
logger.info(
@@ -307,9 +307,9 @@ class BaseSpecWorker(ABC):
device_pools=draft_pools,
)
if get_memory().enable_unified_cache_external_linker:
if get_memory().enable_unified_cache_external_linker and len(draft_pools) > 1:
raise NotImplementedError(
"The external linker only supports packed draft KV caches."
"The external linker supports one non-packed draft KV pool."
)
return HiCacheDraftPlan(
@@ -0,0 +1,115 @@
"""Mooncake direct linker on Kimi Linear (MLA + KDA, as Kimi-K3): a host hit decodes
bit-for-bit like a device hit."""
import unittest
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.unified_radix_cache_kit import _random_suffixes
from sglang.test.kl_test_utils import (
_extract_output_logprobs,
_flush_cache,
_generate,
get_input_ids,
)
from sglang.test.mooncake_utils import MooncakeTestServices
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
CustomTestCase,
find_available_port,
popen_launch_server,
terminate_and_kill_process_tree,
unified_radix_tree_server_env,
)
register_cuda_ci(
est_time=180,
stage="extra-b",
# Direct GPU transfers require an RDMA-capable runner, even for TP=1.
runner_config="4-gpu-h100",
)
_MODEL_PATH = "yujiepan/kimi-linear-tiny-random"
PAGE_SIZE = 64
TRACK_INTERVAL = 128
# A checkpoint boundary; requests end inside the next page so both hits stop here.
PREFIX_LEN = 1024
class TestKimiLinearUnifiedCacheLinker(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = _MODEL_PATH
cls.base_url = f"http://127.0.0.1:{find_available_port(30000)}"
cls.mooncake = MooncakeTestServices()
cls.addClassCleanup(cls.mooncake.stop)
cls.mooncake.start()
other_args = [
"--trust-remote-code",
"--attention-backend",
"triton",
"--linear-attn-backend",
"triton",
"--mamba-backend",
"triton",
"--page-size",
str(PAGE_SIZE),
"--mamba-radix-cache-strategy",
"extra_buffer",
"--mamba-track-interval",
str(TRACK_INTERVAL),
"--enable-deterministic-inference",
"--mem-fraction-static",
"0.6",
"--max-running-requests",
"1",
"--enable-cache-report",
"--enable-unified-cache-external-linker",
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_args,
env=unified_radix_tree_server_env("python", **cls.mooncake.server_env()),
)
cls.addClassCleanup(
terminate_and_kill_process_tree, cls.process, wait_timeout=60
)
cls.input_ids = get_input_ids(
tokenizer_path=cls.model, num_samples=4, trust_remote_code=True
)
def test_host_hit_matches_device_hit(self):
prompts = [list(ids[: PREFIX_LEN + 1]) for ids in self.input_ids]
for ids in prompts:
_generate(self.base_url, [ids], 1)
requests = [
prompt + suffix
for prompt, suffix in zip(
prompts, _random_suffixes(len(prompts), PAGE_SIZE - 1, seed=100)
)
]
run = lambda: [
_generate(self.base_url, [ids], 32, return_logprob=True)[0]
for ids in requests
]
device_results = run()
_flush_cache(self.base_url)
host_results = run()
for device, host in zip(device_results, host_results):
for result, host_tokens in ((device, 0), (host, PREFIX_LEN)):
meta = result["meta_info"]
details = meta.get("cached_tokens_details") or {}
self.assertEqual(meta["cached_tokens"], PREFIX_LEN)
self.assertEqual(int(details.get("host", 0)), host_tokens)
self.assertEqual(host["output_ids"], device["output_ids"])
self.assertEqual(
_extract_output_logprobs(host), _extract_output_logprobs(device)
)
if __name__ == "__main__":
unittest.main()
@@ -645,6 +645,15 @@ class TestHybridDevicePoolAssembler(CustomTestCase):
params=SimpleNamespace(mtp_draft_device_pools=draft_pools),
components={ComponentType.FULL},
)
with self.assertRaisesRegex(ValueError, "draft KV"):
resolve_hybrid_device_pool_group(
kvcache=kvcache,
page_size=2,
params=SimpleNamespace(
mtp_draft_device_pools=(), sidecar_draft_device_pools=(object(),)
),
components={ComponentType.FULL},
)
self.assertEqual(group.num_layers, 2)
self.assertTrue(group.rank_replicated)
@@ -672,8 +681,7 @@ class TestHybridDevicePoolAssembler(CustomTestCase):
self.assertEqual(sizes, [[11, 23]])
self.assertEqual(offsets, [[7, 35]])
def test_linker_requires_packed_draft(self):
"""Do not accept draft state that the linker would omit from storage."""
def test_draft_plan_packs_only_nextn_mtp(self):
from sglang.srt.speculative import base_spec_worker as spec
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
@@ -710,39 +718,109 @@ class TestHybridDevicePoolAssembler(CustomTestCase):
),
),
):
if linker_enabled and not nextn_layers:
with self.assertRaisesRegex(
NotImplementedError, "only supports packed"
):
spec.BaseSpecWorker._build_hicache_draft_plan(worker)
self.assertEqual(target.mtp_draft_device_pools, ())
else:
plan = spec.BaseSpecWorker._build_hicache_draft_plan(worker)
self.assertEqual(
plan.mode,
spec.HiCacheDraftMode.PACKED
if nextn_layers
else spec.HiCacheDraftMode.SIDECAR,
)
self.assertEqual(plan.device_pools, (draft.token_to_kv_pool,))
self.assertEqual(
target.mtp_draft_device_pools,
plan.device_pools if nextn_layers else (),
)
plan = spec.BaseSpecWorker._build_hicache_draft_plan(worker)
self.assertEqual(
plan.mode,
spec.HiCacheDraftMode.PACKED
if nextn_layers
else spec.HiCacheDraftMode.SIDECAR,
)
self.assertEqual(plan.device_pools, (draft.token_to_kv_pool,))
self.assertEqual(
target.mtp_draft_device_pools,
plan.device_pools if nextn_layers else (),
)
def test_unsupported_strategy_fails_with_context(self):
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool
def test_mla_mamba_maps_kv_pages_and_state_slots(self):
from sglang.srt.mem_cache.hybrid_cache import linker_pool_assembler
from sglang.srt.mem_cache.memory_pool import (
HybridLinearKVPool,
MHATokenToKVPool,
MLATokenToKVPool,
)
kvcache = HybridLinearKVPool.__new__(HybridLinearKVPool)
kvcache.start_layer = 0
kvcache.full_attention_layer_id_mapping = {1: 0, 3: 1}
kvcache.full_kv_pool = MLATokenToKVPool.__new__(MLATokenToKVPool)
kvcache.full_kv_pool.kv_buffer = [
torch.zeros((8, 1, 5), dtype=torch.uint8) for _ in range(2)
]
mamba_cache = SimpleNamespace(
temporal=torch.zeros((2, 4, 3), dtype=torch.uint8),
conv=[torch.zeros((2, 4, 2), dtype=torch.uint8)],
)
draft = MHATokenToKVPool.__new__(MHATokenToKVPool)
draft.page_size, draft.layer_num = 2, 1
draft.k_scale_buffer = draft.native_k_scale_buffer = None
draft.k_buffer = [torch.zeros((8, 3), dtype=torch.uint8)]
draft.v_buffer = [torch.zeros((8, 3), dtype=torch.uint8)]
params = SimpleNamespace(
mtp_draft_device_pools=(),
sidecar_draft_device_pools=(draft,),
req_to_token_pool=SimpleNamespace(
mamba_pool=SimpleNamespace(mamba_cache=mamba_cache, num_mamba_layers=2),
mamba_map={0: 0, 2: 1},
mamba_ckpt_pool=None,
short_conv_pool=SimpleNamespace(enabled=False),
ngram_pool=SimpleNamespace(enabled=False),
),
)
def resolve(dcp_size=1):
with (
patch.object(
linker_pool_assembler,
"get_memory",
return_value=SimpleNamespace(enable_unified_memory=False),
),
patch.object(
linker_pool_assembler,
"get_parallel",
return_value=SimpleNamespace(attn_dcp_size=dcp_size),
),
):
return resolve_hybrid_device_pool_group(
kvcache=kvcache,
page_size=2,
params=params,
components={ComponentType.FULL, ComponentType.MAMBA},
)
group = resolve()
self.assertEqual(group.num_layers, 4)
self.assertFalse(group.rank_replicated)
kv, mamba = group.entry_map[PoolName.KV], group.entry_map[PoolName.MAMBA]
self.assertEqual(kv.layer_mapping, {1: 0, 3: 1})
self.assertEqual(mamba.layer_mapping, {0: 0, 2: 1})
self.assertEqual(kv.prepare_locations(torch.tensor([2, 3])), [2])
self.assertEqual(mamba.prepare_locations(torch.tensor([3])), [3])
self.assertEqual(group.sources[PoolName.DRAFT], PoolName.KV)
self.assertEqual(group.entry_map[PoolName.DRAFT].layer_mapping, {0: (0, 1)})
with self.assertRaisesRegex(ValueError, "plain MLA"):
resolve(dcp_size=2)
params.mtp_draft_device_pools = (object(),)
with self.assertRaisesRegex(ValueError, "plain MLA"):
resolve()
params.mtp_draft_device_pools = ()
kvcache.full_kv_pool = MHATokenToKVPool.__new__(MHATokenToKVPool)
with self.assertRaisesRegex(ValueError, "plain MLA"):
resolve()
def test_unsupported_strategy_fails_with_context(self):
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
kvcache = SWAKVPool.__new__(SWAKVPool)
with self.assertRaisesRegex(
ValueError,
"does not support the direct external linker: _MambaStrategy",
"does not support the direct external linker: _SwaStrategy",
):
resolve_hybrid_device_pool_group(
kvcache=kvcache,
page_size=2,
params=SimpleNamespace(),
components={ComponentType.FULL, ComponentType.MAMBA},
components={ComponentType.FULL, ComponentType.SWA},
)