From eb91a41f380bad9eb320fbe6bd52e6f15061e314 Mon Sep 17 00:00:00 2001 From: Shuwen Wang <47200617+alphabetc1@users.noreply.github.com> Date: Wed, 21 Oct 2026 10:47:17 +0800 Subject: [PATCH] [Feat] unified_cache: Support FULL + MAMBA MLA models (Kimi-K3) in the direct external linker (#44977) Co-authored-by: Yutong Wu Co-authored-by: Zhangheng --- .../sglang/srt/mem_cache/cache_init_params.py | 2 + .../hybrid_cache/hybrid_pool_assembler.py | 7 + .../hybrid_cache/linker_pool_assembler.py | 111 ++++++++++++--- .../sglang/srt/mem_cache/kv_cache_builder.py | 6 + .../srt/speculative/base_spec_worker.py | 4 +- .../test_unified_cache_linker_kimi_linear.py | 115 ++++++++++++++++ .../mem_cache/test_linker_pool_assembler.py | 128 ++++++++++++++---- 7 files changed, 324 insertions(+), 49 deletions(-) create mode 100644 test/registered/radix_cache/unified_radix_tree/linker/test_unified_cache_linker_kimi_linear.py diff --git a/python/sglang/srt/mem_cache/cache_init_params.py b/python/sglang/srt/mem_cache/cache_init_params.py index ae4f1790a3..15125c238f 100644 --- a/python/sglang/srt/mem_cache/cache_init_params.py +++ b/python/sglang/srt/mem_cache/cache_init_params.py @@ -72,6 +72,8 @@ class CacheInitParams: tree_core_backend: Optional[str] = dataclasses.field(default=None, kw_only=True) mtp_draft_device_pools: tuple[object, ...] = () + # Draft KV pools that share the target KV slots but are stored apart from it. + sidecar_draft_device_pools: tuple[object, ...] = () mtp_draft_mamba_pools: tuple[object, ...] = () @property diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py index b810d65173..60be6a350d 100644 --- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py +++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py @@ -1960,6 +1960,13 @@ class _MambaStrategy(StackStrategy): ComponentType.MAMBA, } + def build_direct_linker_pool_group(self, *, kvcache, params, page_size): + from sglang.srt.mem_cache.hybrid_cache.linker_pool_assembler import ( + _build_mamba_device_pool_group, + ) + + return _build_mamba_device_pool_group(kvcache, params, page_size) + def build( self, *, diff --git a/python/sglang/srt/mem_cache/hybrid_cache/linker_pool_assembler.py b/python/sglang/srt/mem_cache/hybrid_cache/linker_pool_assembler.py index 334f0df9d0..a351d67cbf 100644 --- a/python/sglang/srt/mem_cache/hybrid_cache/linker_pool_assembler.py +++ b/python/sglang/srt/mem_cache/hybrid_cache/linker_pool_assembler.py @@ -14,7 +14,7 @@ from sglang.srt.mem_cache.hicache_storage import ( PoolTransfer, ) from sglang.srt.mem_cache.unified_cache.component_type import ComponentType -from sglang.srt.runtime_context import get_memory +from sglang.srt.runtime_context import get_memory, get_parallel class DevicePoolEntry: @@ -460,7 +460,11 @@ def _build_dsa_device_pool_group( def _mha_device_pool_entry( - name: PoolName, pool: Any, layer_mapping: dict[int, int], page_size: int + name: PoolName, + pool: Any, + layer_mapping: dict[int, int], + page_size: int, + indices_from_pool: PoolName | None = None, ) -> DevicePoolEntry: # TODO(alphabetc1): support mxfp8. if pool.k_scale_buffer is not None or pool.native_k_scale_buffer is not None: @@ -470,7 +474,7 @@ def _mha_device_pool_entry( num_layers = len(pool.k_buffer) return DevicePoolEntry( name=name, - indices_from_pool=name, + indices_from_pool=indices_from_pool or name, device_pool=pool, components=[[*pool.k_buffer, *pool.v_buffer]], layer_mapping={ @@ -488,6 +492,23 @@ def _mamba_device_pool_entry( _stage_local_layer_mapping, ) + # TODO(alphabetc1): support unified memory for hybrid Mamba models. + if get_memory().enable_unified_memory: + raise ValueError( + "The direct external linker does not support --enable-unified-memory " + "on hybrid Mamba models yet." + ) + # TODO(alphabetc1): support int8 Mamba checkpoints and Mamba side states. + if ( + req_to_token_pool.mamba_ckpt_pool is not None + or req_to_token_pool.short_conv_pool.enabled + or req_to_token_pool.ngram_pool.enabled + ): + raise ValueError( + "The direct external linker does not support int8 Mamba checkpoints " + "or Mamba side states yet." + ) + mamba_pool = req_to_token_pool.mamba_pool mamba_cache = mamba_pool.mamba_cache layers = range(mamba_pool.num_mamba_layers) @@ -517,24 +538,6 @@ def _build_mamba_swa_device_pool_group( _swa_layer_mappings, ) - # TODO(alphabetc1): support unified memory for hybrid Mamba models. - if get_memory().enable_unified_memory: - raise ValueError( - "The direct external linker does not support --enable-unified-memory " - "on hybrid Mamba models yet." - ) - # TODO(alphabetc1): support int8 Mamba checkpoints and Mamba side states. - req_to_token_pool = params.req_to_token_pool - if ( - req_to_token_pool.mamba_ckpt_pool is not None - or req_to_token_pool.short_conv_pool.enabled - or req_to_token_pool.ngram_pool.enabled - ): - raise ValueError( - "The direct external linker does not support int8 Mamba checkpoints " - "or Mamba side states yet." - ) - full_mapping, swa_mapping = _swa_layer_mappings(kvcache) entries = [ _mha_device_pool_entry( @@ -549,6 +552,63 @@ def _build_mamba_swa_device_pool_group( return DevicePoolGroup(entries, num_layers + 1, page_size) +def _build_mamba_device_pool_group( + kvcache: Any, params: Any, page_size: int +) -> DevicePoolGroup: + from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import ( + _stage_local_layer_mapping, + ) + from sglang.srt.mem_cache.memory_pool import ( + HybridLinearKVPool, + MLATokenToKVPool, + ) + + # TODO(alphabetc1): support other full-attention pools, DCP and packed draft KV. + full_kv_pool = kvcache.full_kv_pool + if ( + type(kvcache) is not HybridLinearKVPool + or type(full_kv_pool) is not MLATokenToKVPool + or get_parallel().attn_dcp_size > 1 + or params.mtp_draft_device_pools + ): + raise ValueError( + "The direct external linker supports FULL + MAMBA models only with a " + "plain MLA pool, without DCP or packed draft KV." + ) + + entries = [ + DevicePoolEntry( + name=PoolName.KV, + indices_from_pool=PoolName.KV, + device_pool=full_kv_pool, + components=[full_kv_pool.kv_buffer], + layer_mapping=_stage_local_layer_mapping( + kvcache.full_attention_layer_id_mapping, kvcache.start_layer + ), + page_size=page_size, + rows_are_pages=False, + ), + _mamba_device_pool_entry(params.req_to_token_pool, kvcache.start_layer), + ] + for draft_pool in params.sidecar_draft_device_pools: + if draft_pool.page_size != page_size: + raise ValueError("Draft KV page size must match the tree page size.") + # Draft layer i rides transfer layer i; the draft reads it only after + # the batch's loads finish. + entries.append( + _mha_device_pool_entry( + PoolName.DRAFT, + draft_pool, + {layer: layer for layer in range(draft_pool.layer_num)}, + page_size, + indices_from_pool=PoolName.KV, + ) + ) + num_layers = max(layer for entry in entries for layer in entry.layer_mapping) + # Not rank_replicated: the Mamba state is TP-sharded. + return DevicePoolGroup(entries, num_layers + 1, page_size) + + def resolve_hybrid_device_pool_group( *, kvcache: Any, @@ -561,8 +621,15 @@ def resolve_hybrid_device_pool_group( _select_strategy, ) - return _select_strategy(kvcache, components).build_direct_linker_pool_group( + group = _select_strategy(kvcache, components).build_direct_linker_pool_group( kvcache=kvcache, params=params, page_size=page_size, ) + if getattr(params, "sidecar_draft_device_pools", ()) and ( + PoolName.DRAFT not in group.entry_map + ): + raise ValueError( + "The direct external linker does not support this model's draft KV." + ) + return group diff --git a/python/sglang/srt/mem_cache/kv_cache_builder.py b/python/sglang/srt/mem_cache/kv_cache_builder.py index e796f14a5e..2b0b6ba6ce 100644 --- a/python/sglang/srt/mem_cache/kv_cache_builder.py +++ b/python/sglang/srt/mem_cache/kv_cache_builder.py @@ -409,6 +409,12 @@ def build_kv_cache( draft_swa_window=draft_swa_window, mtp_draft_device_pools=mtp_draft_device_pools, mtp_draft_mamba_pools=tp_worker.model_runner.mtp_draft_mamba_pools, + sidecar_draft_device_pools=( + hicache_draft_plan.device_pools + if hicache_draft_plan is not None + and hicache_draft_plan.mode == HiCacheDraftMode.SIDECAR + else () + ), ) if params.swa_retain_window != sliding_window_size: logger.info( diff --git a/python/sglang/srt/speculative/base_spec_worker.py b/python/sglang/srt/speculative/base_spec_worker.py index c2f4f2a2b4..e21fcff772 100644 --- a/python/sglang/srt/speculative/base_spec_worker.py +++ b/python/sglang/srt/speculative/base_spec_worker.py @@ -307,9 +307,9 @@ class BaseSpecWorker(ABC): device_pools=draft_pools, ) - if get_memory().enable_unified_cache_external_linker: + if get_memory().enable_unified_cache_external_linker and len(draft_pools) > 1: raise NotImplementedError( - "The external linker only supports packed draft KV caches." + "The external linker supports one non-packed draft KV pool." ) return HiCacheDraftPlan( diff --git a/test/registered/radix_cache/unified_radix_tree/linker/test_unified_cache_linker_kimi_linear.py b/test/registered/radix_cache/unified_radix_tree/linker/test_unified_cache_linker_kimi_linear.py new file mode 100644 index 0000000000..f1747598fc --- /dev/null +++ b/test/registered/radix_cache/unified_radix_tree/linker/test_unified_cache_linker_kimi_linear.py @@ -0,0 +1,115 @@ +"""Mooncake direct linker on Kimi Linear (MLA + KDA, as Kimi-K3): a host hit decodes +bit-for-bit like a device hit.""" + +import unittest + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kits.unified_radix_cache_kit import _random_suffixes +from sglang.test.kl_test_utils import ( + _extract_output_logprobs, + _flush_cache, + _generate, + get_input_ids, +) +from sglang.test.mooncake_utils import MooncakeTestServices +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + CustomTestCase, + find_available_port, + popen_launch_server, + terminate_and_kill_process_tree, + unified_radix_tree_server_env, +) + +register_cuda_ci( + est_time=180, + stage="extra-b", + # Direct GPU transfers require an RDMA-capable runner, even for TP=1. + runner_config="4-gpu-h100", +) + +_MODEL_PATH = "yujiepan/kimi-linear-tiny-random" + +PAGE_SIZE = 64 +TRACK_INTERVAL = 128 +# A checkpoint boundary; requests end inside the next page so both hits stop here. +PREFIX_LEN = 1024 + + +class TestKimiLinearUnifiedCacheLinker(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = _MODEL_PATH + cls.base_url = f"http://127.0.0.1:{find_available_port(30000)}" + cls.mooncake = MooncakeTestServices() + cls.addClassCleanup(cls.mooncake.stop) + cls.mooncake.start() + other_args = [ + "--trust-remote-code", + "--attention-backend", + "triton", + "--linear-attn-backend", + "triton", + "--mamba-backend", + "triton", + "--page-size", + str(PAGE_SIZE), + "--mamba-radix-cache-strategy", + "extra_buffer", + "--mamba-track-interval", + str(TRACK_INTERVAL), + "--enable-deterministic-inference", + "--mem-fraction-static", + "0.6", + "--max-running-requests", + "1", + "--enable-cache-report", + "--enable-unified-cache-external-linker", + ] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + env=unified_radix_tree_server_env("python", **cls.mooncake.server_env()), + ) + cls.addClassCleanup( + terminate_and_kill_process_tree, cls.process, wait_timeout=60 + ) + cls.input_ids = get_input_ids( + tokenizer_path=cls.model, num_samples=4, trust_remote_code=True + ) + + def test_host_hit_matches_device_hit(self): + prompts = [list(ids[: PREFIX_LEN + 1]) for ids in self.input_ids] + for ids in prompts: + _generate(self.base_url, [ids], 1) + requests = [ + prompt + suffix + for prompt, suffix in zip( + prompts, _random_suffixes(len(prompts), PAGE_SIZE - 1, seed=100) + ) + ] + + run = lambda: [ + _generate(self.base_url, [ids], 32, return_logprob=True)[0] + for ids in requests + ] + device_results = run() + _flush_cache(self.base_url) + host_results = run() + + for device, host in zip(device_results, host_results): + for result, host_tokens in ((device, 0), (host, PREFIX_LEN)): + meta = result["meta_info"] + details = meta.get("cached_tokens_details") or {} + self.assertEqual(meta["cached_tokens"], PREFIX_LEN) + self.assertEqual(int(details.get("host", 0)), host_tokens) + self.assertEqual(host["output_ids"], device["output_ids"]) + self.assertEqual( + _extract_output_logprobs(host), _extract_output_logprobs(device) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/mem_cache/test_linker_pool_assembler.py b/test/registered/unit/mem_cache/test_linker_pool_assembler.py index 09a2bedd6b..617be70161 100644 --- a/test/registered/unit/mem_cache/test_linker_pool_assembler.py +++ b/test/registered/unit/mem_cache/test_linker_pool_assembler.py @@ -645,6 +645,15 @@ class TestHybridDevicePoolAssembler(CustomTestCase): params=SimpleNamespace(mtp_draft_device_pools=draft_pools), components={ComponentType.FULL}, ) + with self.assertRaisesRegex(ValueError, "draft KV"): + resolve_hybrid_device_pool_group( + kvcache=kvcache, + page_size=2, + params=SimpleNamespace( + mtp_draft_device_pools=(), sidecar_draft_device_pools=(object(),) + ), + components={ComponentType.FULL}, + ) self.assertEqual(group.num_layers, 2) self.assertTrue(group.rank_replicated) @@ -672,8 +681,7 @@ class TestHybridDevicePoolAssembler(CustomTestCase): self.assertEqual(sizes, [[11, 23]]) self.assertEqual(offsets, [[7, 35]]) - def test_linker_requires_packed_draft(self): - """Do not accept draft state that the linker would omit from storage.""" + def test_draft_plan_packs_only_nextn_mtp(self): from sglang.srt.speculative import base_spec_worker as spec from sglang.srt.speculative.spec_info import SpeculativeAlgorithm @@ -710,39 +718,109 @@ class TestHybridDevicePoolAssembler(CustomTestCase): ), ), ): - if linker_enabled and not nextn_layers: - with self.assertRaisesRegex( - NotImplementedError, "only supports packed" - ): - spec.BaseSpecWorker._build_hicache_draft_plan(worker) - self.assertEqual(target.mtp_draft_device_pools, ()) - else: - plan = spec.BaseSpecWorker._build_hicache_draft_plan(worker) - self.assertEqual( - plan.mode, - spec.HiCacheDraftMode.PACKED - if nextn_layers - else spec.HiCacheDraftMode.SIDECAR, - ) - self.assertEqual(plan.device_pools, (draft.token_to_kv_pool,)) - self.assertEqual( - target.mtp_draft_device_pools, - plan.device_pools if nextn_layers else (), - ) + plan = spec.BaseSpecWorker._build_hicache_draft_plan(worker) + self.assertEqual( + plan.mode, + spec.HiCacheDraftMode.PACKED + if nextn_layers + else spec.HiCacheDraftMode.SIDECAR, + ) + self.assertEqual(plan.device_pools, (draft.token_to_kv_pool,)) + self.assertEqual( + target.mtp_draft_device_pools, + plan.device_pools if nextn_layers else (), + ) - def test_unsupported_strategy_fails_with_context(self): - from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool + def test_mla_mamba_maps_kv_pages_and_state_slots(self): + from sglang.srt.mem_cache.hybrid_cache import linker_pool_assembler + from sglang.srt.mem_cache.memory_pool import ( + HybridLinearKVPool, + MHATokenToKVPool, + MLATokenToKVPool, + ) kvcache = HybridLinearKVPool.__new__(HybridLinearKVPool) + kvcache.start_layer = 0 + kvcache.full_attention_layer_id_mapping = {1: 0, 3: 1} + kvcache.full_kv_pool = MLATokenToKVPool.__new__(MLATokenToKVPool) + kvcache.full_kv_pool.kv_buffer = [ + torch.zeros((8, 1, 5), dtype=torch.uint8) for _ in range(2) + ] + mamba_cache = SimpleNamespace( + temporal=torch.zeros((2, 4, 3), dtype=torch.uint8), + conv=[torch.zeros((2, 4, 2), dtype=torch.uint8)], + ) + draft = MHATokenToKVPool.__new__(MHATokenToKVPool) + draft.page_size, draft.layer_num = 2, 1 + draft.k_scale_buffer = draft.native_k_scale_buffer = None + draft.k_buffer = [torch.zeros((8, 3), dtype=torch.uint8)] + draft.v_buffer = [torch.zeros((8, 3), dtype=torch.uint8)] + params = SimpleNamespace( + mtp_draft_device_pools=(), + sidecar_draft_device_pools=(draft,), + req_to_token_pool=SimpleNamespace( + mamba_pool=SimpleNamespace(mamba_cache=mamba_cache, num_mamba_layers=2), + mamba_map={0: 0, 2: 1}, + mamba_ckpt_pool=None, + short_conv_pool=SimpleNamespace(enabled=False), + ngram_pool=SimpleNamespace(enabled=False), + ), + ) + + def resolve(dcp_size=1): + with ( + patch.object( + linker_pool_assembler, + "get_memory", + return_value=SimpleNamespace(enable_unified_memory=False), + ), + patch.object( + linker_pool_assembler, + "get_parallel", + return_value=SimpleNamespace(attn_dcp_size=dcp_size), + ), + ): + return resolve_hybrid_device_pool_group( + kvcache=kvcache, + page_size=2, + params=params, + components={ComponentType.FULL, ComponentType.MAMBA}, + ) + + group = resolve() + self.assertEqual(group.num_layers, 4) + self.assertFalse(group.rank_replicated) + kv, mamba = group.entry_map[PoolName.KV], group.entry_map[PoolName.MAMBA] + self.assertEqual(kv.layer_mapping, {1: 0, 3: 1}) + self.assertEqual(mamba.layer_mapping, {0: 0, 2: 1}) + self.assertEqual(kv.prepare_locations(torch.tensor([2, 3])), [2]) + self.assertEqual(mamba.prepare_locations(torch.tensor([3])), [3]) + self.assertEqual(group.sources[PoolName.DRAFT], PoolName.KV) + self.assertEqual(group.entry_map[PoolName.DRAFT].layer_mapping, {0: (0, 1)}) + + with self.assertRaisesRegex(ValueError, "plain MLA"): + resolve(dcp_size=2) + params.mtp_draft_device_pools = (object(),) + with self.assertRaisesRegex(ValueError, "plain MLA"): + resolve() + params.mtp_draft_device_pools = () + kvcache.full_kv_pool = MHATokenToKVPool.__new__(MHATokenToKVPool) + with self.assertRaisesRegex(ValueError, "plain MLA"): + resolve() + + def test_unsupported_strategy_fails_with_context(self): + from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool + + kvcache = SWAKVPool.__new__(SWAKVPool) with self.assertRaisesRegex( ValueError, - "does not support the direct external linker: _MambaStrategy", + "does not support the direct external linker: _SwaStrategy", ): resolve_hybrid_device_pool_group( kvcache=kvcache, page_size=2, params=SimpleNamespace(), - components={ComponentType.FULL, ComponentType.MAMBA}, + components={ComponentType.FULL, ComponentType.SWA}, )