From 05c15390984d80a8bb4bbc415a7653e174c27c0b Mon Sep 17 00:00:00 2001 From: Shuwen Wang <47200617+alphabetc1@users.noreply.github.com> Date: Sun, 18 Oct 2026 04:24:08 +0800 Subject: [PATCH] [Feat] unified_cache: Support hybrid SWA + Mamba models in the Mooncake direct linker (#41343) Co-authored-by: Zhangheng Co-authored-by: Teng Ma --- .../hybrid_cache/hybrid_pool_assembler.py | 7 + .../hybrid_cache/linker_pool_assembler.py | 91 +++++++++++ .../storage/mooncake_store/mooncake_store.py | 7 + .../unified_cache/components/mamba.py | 67 ++++++-- .../unified_cache/unified_cache_linker.py | 3 +- .../sglang-radix-tree/src/components/mamba.rs | 18 +++ rust/sglang-radix-tree/src/node.rs | 3 - .../src/tests/unified_tree_core.rs | 18 --- .../src/unified_tree_core.rs | 7 - .../test_unified_cache_linker_kl_inkling.py | 119 +++++++++++++++ .../test_rust_tree_core_integration.py | 7 +- .../mem_cache/test_unified_cache_linker.py | 144 +++++++++++++++++- 12 files changed, 445 insertions(+), 46 deletions(-) create mode 100644 test/registered/radix_cache/unified_radix_tree/linker/test_unified_cache_linker_kl_inkling.py diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py index d8ff9d9daf..a715eb9686 100644 --- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py +++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py @@ -2023,6 +2023,13 @@ class _MambaSwaStrategy(StackStrategy): == {ComponentType.FULL, ComponentType.SWA, ComponentType.MAMBA} ) + def build_direct_linker_pool_group(self, *, kvcache, params, page_size): + from sglang.srt.mem_cache.hybrid_cache.linker_pool_assembler import ( + _build_mamba_swa_device_pool_group, + ) + + return _build_mamba_swa_device_pool_group(kvcache, params, page_size) + def build( self, *, diff --git a/python/sglang/srt/mem_cache/hybrid_cache/linker_pool_assembler.py b/python/sglang/srt/mem_cache/hybrid_cache/linker_pool_assembler.py index 0a8cba5e5f..9d663790ab 100644 --- a/python/sglang/srt/mem_cache/hybrid_cache/linker_pool_assembler.py +++ b/python/sglang/srt/mem_cache/hybrid_cache/linker_pool_assembler.py @@ -14,6 +14,7 @@ from sglang.srt.mem_cache.hicache_storage import ( PoolTransfer, ) from sglang.srt.mem_cache.unified_cache.component_type import ComponentType +from sglang.srt.runtime_context import get_memory class DevicePoolEntry: @@ -442,6 +443,96 @@ def _build_dsa_device_pool_group( return DevicePoolGroup(entries, num_layers, page_size, rank_replicated=True) +def _mha_device_pool_entry( + name: PoolName, pool: Any, layer_mapping: dict[int, int], page_size: int +) -> DevicePoolEntry: + # TODO(alphabetc1): support mxfp8. + if pool.k_scale_buffer is not None or pool.native_k_scale_buffer is not None: + raise ValueError( + "The direct external linker does not support scaled KV caches yet." + ) + num_layers = len(pool.k_buffer) + return DevicePoolEntry( + name=name, + indices_from_pool=name, + device_pool=pool, + components=[[*pool.k_buffer, *pool.v_buffer]], + layer_mapping={ + layer: (index, num_layers + index) for layer, index in layer_mapping.items() + }, + page_size=page_size, + rows_are_pages=False, + ) + + +def _mamba_device_pool_entry( + req_to_token_pool: Any, start_layer: int +) -> DevicePoolEntry: + from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import ( + _stage_local_layer_mapping, + ) + + mamba_pool = req_to_token_pool.mamba_pool + mamba_cache = mamba_pool.mamba_cache + layers = range(mamba_pool.num_mamba_layers) + # One row per state slot; conv-only models carry an empty temporal state. + components = [ + [state[layer] for layer in layers] + for state in (mamba_cache.temporal, *mamba_cache.conv) + if state.numel() > 0 + ] + return DevicePoolEntry( + name=PoolName.MAMBA, + indices_from_pool=PoolName.MAMBA, + device_pool=mamba_pool, + components=components, + layer_mapping=_stage_local_layer_mapping( + req_to_token_pool.mamba_map, start_layer + ), + page_size=1, + rows_are_pages=True, + ) + + +def _build_mamba_swa_device_pool_group( + kvcache: Any, params: Any, page_size: int +) -> DevicePoolGroup: + from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import ( + _swa_layer_mappings, + ) + + # TODO(alphabetc1): support unified memory for hybrid Mamba models. + if get_memory().enable_unified_memory: + raise ValueError( + "The direct external linker does not support --enable-unified-memory " + "on hybrid Mamba models yet." + ) + # TODO(alphabetc1): support int8 Mamba checkpoints and Mamba side states. + req_to_token_pool = params.req_to_token_pool + if ( + req_to_token_pool.mamba_ckpt_pool is not None + or req_to_token_pool.short_conv_pool.enabled + or req_to_token_pool.ngram_pool.enabled + ): + raise ValueError( + "The direct external linker does not support int8 Mamba checkpoints " + "or Mamba side states yet." + ) + + full_mapping, swa_mapping = _swa_layer_mappings(kvcache) + entries = [ + _mha_device_pool_entry( + PoolName.KV, kvcache.full_kv_pool, full_mapping, page_size + ), + _mha_device_pool_entry( + PoolName.SWA, kvcache.swa_kv_pool, swa_mapping, page_size + ), + _mamba_device_pool_entry(params.req_to_token_pool, kvcache.start_layer), + ] + num_layers = max(layer for entry in entries for layer in entry.layer_mapping) + return DevicePoolGroup(entries, num_layers + 1, page_size) + + def resolve_hybrid_device_pool_group( *, kvcache: Any, diff --git a/python/sglang/srt/mem_cache/storage/mooncake_store/mooncake_store.py b/python/sglang/srt/mem_cache/storage/mooncake_store/mooncake_store.py index 917c1e721a..b7b0115b9c 100644 --- a/python/sglang/srt/mem_cache/storage/mooncake_store/mooncake_store.py +++ b/python/sglang/srt/mem_cache/storage/mooncake_store/mooncake_store.py @@ -21,6 +21,7 @@ from sglang.srt.mem_cache.hicache_storage import ( PoolTransfer, PoolTransferResult, ) +from sglang.srt.mem_cache.hybrid_cache.linker_pool_assembler import DevicePoolEntry from sglang.srt.mem_cache.pool_host import HostKVCache, HostTensorAllocator from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost from sglang.srt.observability.metrics_collector import StorageMetrics @@ -772,6 +773,12 @@ class MooncakeStore(HiCacheStorage, MooncakeBaseStore): if host_pool is None: raise ValueError(f"Unregistered Mooncake hybrid pool: {transfer.name}") + if isinstance(host_pool, DevicePoolEntry): + # A direct-linker entry packs every buffer of a page into one object. + object_name = "k" if transfer.name == PoolName.KV else transfer.name + suffix = f"_{self.mla_suffix}_{object_name}" + return [f"{page_key}{suffix}" for page_key in page_keys], 1 + # Suffix order must match get_page_buffer_meta() for one page, because # Mooncake zips object keys with registered buffer pointers. pool_name = transfer.name diff --git a/python/sglang/srt/mem_cache/unified_cache/components/mamba.py b/python/sglang/srt/mem_cache/unified_cache/components/mamba.py index 325cb92dec..04358a6cbb 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/mamba.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/mamba.py @@ -31,6 +31,7 @@ from sglang.srt.mem_cache.unified_cache.components.base import ( CacheTransferPhase, ComponentType, EvictLayer, + ExternalLinkerLoadPhase, InternalStateBackup, LinkerTransferPhase, LRURefreshPhase, @@ -692,9 +693,62 @@ class MambaComponent(TreeComponent): node: Optional[UnifiedTreeNode], keys: Optional[Sequence[str]], ) -> Optional[PoolTransfer]: - raise AssertionError( - "MambaComponent does not support external linker mode, will support soon" + if phase == LinkerTransferPhase.OFFLOAD: + if node is None or not node.hash_value: + return None + value = node.component_data[self.component_type].value + if value is None: + return None + # One state per node, valid only at the node's end boundary. + return PoolTransfer( + name=PoolName.MAMBA, + device_indices=value, + keys=[node.hash_value[-1]], + hit_policy=PoolHitPolicy.TRAILING_PAGES, + ) + + if not keys: + return None + + transfer = PoolTransfer( + name=PoolName.MAMBA, + keys=[keys[-1]], + hit_policy=PoolHitPolicy.TRAILING_PAGES, ) + if phase == LinkerTransferPhase.LOAD: + transfer.device_indices = self._alloc_mamba_slot() + return transfer + + def update_external_linker_load( + self, + phase: ExternalLinkerLoadPhase, + req: Req, + full_transfer: PoolTransfer, + transfer: PoolTransfer, + prefix_len: int, + ) -> Optional[PoolTransfer]: + pool = self.cache.req_to_token_pool + if phase == ExternalLinkerLoadPhase.ABORT: + pool.mamba_allocator.free(transfer.device_indices) + return None + + assert phase == ExternalLinkerLoadPhase.PREPARE + if req.kv.holds_mamba: + pool.mamba_allocator.free(req.kv.mamba_pool_idx.unsqueeze(0)) + # The layer-gated load writes the request's private state directly. + req.kv.mamba_pool_idx = transfer.device_indices[0] + self._adopt_loaded_state(req) + return transfer + + def _adopt_loaded_state(self, req: Req) -> None: + # A loaded state supersedes the replay cursor and the deferred CoW/clear. + pool = self.cache.req_to_token_pool + write_pos = pool.mamba_pool.replayssm_write_pos + if write_pos is not None: + slot = req.kv.mamba_pool_idx.unsqueeze(0) + write_pos[pool.translate_mamba_indices(slot)] = 0 + req.kv.mamba_cow_src_index = None + req.kv.mamba_needs_clear = False # ---- HiCache Hooks ---- @@ -1035,14 +1089,7 @@ class _MambaBufferLoadBackContext(BufferLoadBackContext): if not success: self.component.finalize_load_back(req, self.prep, success=False) return - # The H2D supersedes the replay cursor and the deferred CoW/clear. - req_to_token_pool = self.component.cache.req_to_token_pool - write_pos = req_to_token_pool.mamba_pool.replayssm_write_pos - if write_pos is not None: - slot = req.kv.mamba_pool_idx.unsqueeze(0) - write_pos[req_to_token_pool.translate_mamba_indices(slot)] = 0 - req.kv.mamba_cow_src_index = None - req.kv.mamba_needs_clear = False + self.component._adopt_loaded_state(req) def get_insert_fields(self) -> dict[str, torch.Tensor]: return {"mamba_value": self.node_copy.device_indices} diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_cache_linker.py b/python/sglang/srt/mem_cache/unified_cache/unified_cache_linker.py index cbfc622cab..3d31507e5f 100644 --- a/python/sglang/srt/mem_cache/unified_cache/unified_cache_linker.py +++ b/python/sglang/srt/mem_cache/unified_cache/unified_cache_linker.py @@ -59,6 +59,7 @@ _EXTERNAL_LINKER_SUPPORTED_COMPONENTS = frozenset( { ComponentType.FULL, ComponentType.SWA, + ComponentType.MAMBA, } ) @@ -220,7 +221,7 @@ class UnifiedCacheLinkerWrapper: component.name for component in sorted(unsupported, key=int) ) raise ValueError( - "External cache linker supports only Full and SWA tree " + "External cache linker supports only Full, SWA and Mamba tree " f"components; unsupported: {names}" ) diff --git a/rust/sglang-radix-tree/src/components/mamba.rs b/rust/sglang-radix-tree/src/components/mamba.rs index 9c04131e83..142a2d2474 100644 --- a/rust/sglang-radix-tree/src/components/mamba.rs +++ b/rust/sglang-radix-tree/src/components/mamba.rs @@ -597,6 +597,24 @@ impl TreeComponent for MambaComponent { }) } + fn build_external_linker_offload_transfer( + &self, + tree_core: &UnifiedTreeCore, + node_id: NodeIdx_, + ) -> Option { + let node = tree_core.arena.node(node_id); + let hash_value = node.hash_value.as_ref().filter(|h| !h.is_empty())?; + let device_indices = node.try_device_value(MAMBA)?; + // One state per node, valid only at the node's end boundary. + Some(PoolTransfer { + name: PoolName::Mamba, + device_indices: Some(device_indices.shallow_clone()), + keys: Some(vec![hash_value[hash_value.len() - 1].clone()]), + hit_policy: PoolHitPolicy::TrailingPages, + ..Default::default() + }) + } + /// Post-transfer mamba bookkeeping for the given phase. fn commit_hicache_transfer( &self, diff --git a/rust/sglang-radix-tree/src/node.rs b/rust/sglang-radix-tree/src/node.rs index 60ddc6a9f4..7c6fecd631 100644 --- a/rust/sglang-radix-tree/src/node.rs +++ b/rust/sglang-radix-tree/src/node.rs @@ -794,9 +794,6 @@ pub enum TreeCoreRuntimeError { #[cfg(any(test, feature = "inspection"))] #[error("{0}")] InspectionAssertion(String), - /// Direct external-cache linking does not support this tree component. - #[error("external cache linker does not support component {component_type:?}")] - ExternalCacheLinkerUnsupportedComponent { component_type: ComponentType }, /// The existing device anchor must be on the restored endpoint's root path. #[error("node {until_node_id} is not an ancestor of node {from_node_id}")] ExternalCachePathNotAncestor { diff --git a/rust/sglang-radix-tree/src/tests/unified_tree_core.rs b/rust/sglang-radix-tree/src/tests/unified_tree_core.rs index 1f0489cd36..600809dd37 100644 --- a/rust/sglang-radix-tree/src/tests/unified_tree_core.rs +++ b/rust/sglang-radix-tree/src/tests/unified_tree_core.rs @@ -2728,24 +2728,6 @@ fn external_linker_swa_offload_uses_complete_trailing_pages() { ); } -#[test] -fn external_linker_rejects_mamba_trees() { - let params = CacheInitParams { - mamba_cache_chunk_size: Some(1), - ..Default::default() - }; - let mut tc: UnifiedTreeCore> = UnifiedTreeCore::new(params, vec![FULL, MAMBA]); - let error = tc.set_enable_external_cache_linker(true).unwrap_err(); - assert!(matches!( - &error, - TreeCoreRuntimeError::ExternalCacheLinkerUnsupportedComponent { - component_type - } if *component_type == MAMBA - )); - assert!(error.to_string().contains("Mamba")); - assert!(!tc.enable_external_cache_linker); -} - #[test] fn external_linker_state_follows_load_offload_and_split_lifecycle() { let mut tc = core(); diff --git a/rust/sglang-radix-tree/src/unified_tree_core.rs b/rust/sglang-radix-tree/src/unified_tree_core.rs index 6f6fd1ea4c..e6d0777962 100644 --- a/rust/sglang-radix-tree/src/unified_tree_core.rs +++ b/rust/sglang-radix-tree/src/unified_tree_core.rs @@ -3257,13 +3257,6 @@ impl UnifiedTreeCore { &mut self, value: bool, ) -> Result<(), TreeCoreRuntimeError> { - if value && self.components_by_type[MAMBA.idx()].is_some() { - return Err( - TreeCoreRuntimeError::ExternalCacheLinkerUnsupportedComponent { - component_type: MAMBA, - }, - ); - } self.enable_external_cache_linker = value; Ok(()) } diff --git a/test/registered/radix_cache/unified_radix_tree/linker/test_unified_cache_linker_kl_inkling.py b/test/registered/radix_cache/unified_radix_tree/linker/test_unified_cache_linker_kl_inkling.py new file mode 100644 index 0000000000..e54c706a06 --- /dev/null +++ b/test/registered/radix_cache/unified_radix_tree/linker/test_unified_cache_linker_kl_inkling.py @@ -0,0 +1,119 @@ +"""Bit-exact KL for the Mooncake direct linker on Inkling (FULL + SWA + Mamba): +prefill and decode score identically, so any nonzero KL is a wrong restored state.""" + +import os +import unittest + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kits.unified_radix_cache_kit import _random_suffixes +from sglang.test.kl_multiturn_utils import ( + _extract_output_logprobs, + _flush_cache, + _generate, + _replay_and_compare_kl, +) +from sglang.test.kl_test_utils import get_input_ids +from sglang.test.mooncake_utils import MooncakeTestServices +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + CustomTestCase, + find_available_port, + popen_launch_server, + terminate_and_kill_process_tree, + unified_radix_tree_server_env, +) + +register_cuda_ci( + est_time=300, + stage="extra-a", + runner_config="1-gpu-large", + disabled="Re-enable after #44220 merges: Full prefill graph must wait for layer-wise loads; until 2026-10-29", +) + +_MODEL_PATH = os.environ.get("INKLING_TEST_MODEL_PATH", "thinkingmachines/Inkling") +_MODEL_REVISION = os.environ.get("INKLING_TEST_MODEL_REVISION", "test") + +# Measured 0; the floor only absorbs the strict `<`. +KL_DIV_THRESHOLD = 1e-9 +PAGE_SIZE = 128 +TRACK_INTERVAL = 128 + + +class TestInklingUnifiedCacheLinkerKL(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = _MODEL_PATH + cls.base_url = f"http://127.0.0.1:{find_available_port(30000)}" + cls.mooncake = MooncakeTestServices() + cls.addClassCleanup(cls.mooncake.stop) + cls.mooncake.start() + other_args = [ + "--trust-remote-code", + "--revision", + _MODEL_REVISION, + "--attention-backend", + "fa4", + "--page-size", + str(PAGE_SIZE), + "--mamba-radix-cache-strategy", + "extra_buffer", + "--mamba-track-interval", + str(TRACK_INTERVAL), + "--enable-deterministic-inference", + "--mem-fraction-static", + "0.6", + "--max-running-requests", + "1", + "--enable-cache-report", + "--enable-unified-cache-external-linker", + ] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + env=unified_radix_tree_server_env("python", **cls.mooncake.server_env()), + ) + cls.addClassCleanup( + terminate_and_kill_process_tree, cls.process, wait_timeout=60 + ) + cls.input_ids = get_input_ids( + tokenizer_path=cls.model, num_samples=2, trust_remote_code=True + ) + + def test_load_back_after_flush(self): + """Every second-turn prefix comes back from Mooncake alone.""" + first_turn = [list(ids[:1024]) for ids in self.input_ids] + histories = [ + ids + _generate(self.base_url, [ids], 256)[0]["output_ids"] + for ids in first_turn + ] + + _flush_cache(self.base_url) + sequences, logprobs = [], [] + for history, suffix in zip( + histories, _random_suffixes(len(histories), 256, seed=100) + ): + ids = history + suffix + (result,) = _generate(self.base_url, [ids], 512, return_logprob=True) + expected = (len(history) - 1) // TRACK_INTERVAL * TRACK_INTERVAL + cached_tokens = result["meta_info"]["cached_tokens"] + details = result["meta_info"].get("cached_tokens_details") or {} + self.assertGreaterEqual(cached_tokens, expected) + self.assertEqual(int(details.get("host", 0)), cached_tokens) + sequences.append(ids + result["output_ids"]) + logprobs.append(_extract_output_logprobs(result)) + + _replay_and_compare_kl( + self.base_url, + self.model, + KL_DIV_THRESHOLD, + sequences, + logprobs, + label="linker_load_back_after_flush", + sampling_temperature=0, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/mem_cache/test_rust_tree_core_integration.py b/test/registered/unit/mem_cache/test_rust_tree_core_integration.py index a31d5bbe60..a5c03c9c19 100644 --- a/test/registered/unit/mem_cache/test_rust_tree_core_integration.py +++ b/test/registered/unit/mem_cache/test_rust_tree_core_integration.py @@ -1214,7 +1214,7 @@ def test_configuration_reads_the_locked_rust_state(): assert swa_core.has_swa_host_pool is True -def test_external_cache_linker_enablement_and_component_guard(): +def test_external_cache_linker_enablement(): core = _tree_core() assert core.enable_external_cache_linker is False core.enable_external_cache_linker = True @@ -1222,11 +1222,6 @@ def test_external_cache_linker_enablement_and_component_guard(): core.enable_external_cache_linker = False assert core.enable_external_cache_linker is False - mamba_core = _mamba_tree_core() - with pytest.raises(AssertionError, match="(?i)mamba"): - mamba_core.enable_external_cache_linker = True - assert mamba_core.enable_external_cache_linker is False - def test_sanity_check_passes_after_the_full_flow(): core = _tree_core() diff --git a/test/registered/unit/mem_cache/test_unified_cache_linker.py b/test/registered/unit/mem_cache/test_unified_cache_linker.py index fc6703713e..cf696d1bca 100644 --- a/test/registered/unit/mem_cache/test_unified_cache_linker.py +++ b/test/registered/unit/mem_cache/test_unified_cache_linker.py @@ -183,7 +183,7 @@ def test_cache_linker_attachment_is_backend_independent(): assert cache.linker.layer_done_counter is linker.layer_done_counter -@pytest.mark.parametrize("component_type", [ComponentType.MAMBA, ComponentType.C128]) +@pytest.mark.parametrize("component_type", [ComponentType.C128]) def test_cache_linker_rejects_unsupported_tree_components(component_type): cache = _cache_for_wrapper(tree_components=(ComponentType.FULL, component_type)) @@ -663,6 +663,148 @@ class TestUnifiedCacheLinkerPythonBackend(_TreeCoreBackendTestMixin, _InsertWalk self.assertEqual(unpublished.device_prefix_len, 3) consumer.sanity_check() + def _mamba_linker_consumer(self, tokens): + self.cfg = CacheConfig( + page_size=1, + components=(ComponentType.FULL, ComponentType.SWA, ComponentType.MAMBA), + sliding_window_size=2, + kv_size=64, + max_context_len=64, + ) + stored_keys = defaultdict(set) + + producer, producer_allocator, producer_req_pool = build_fixture(self.cfg) + producer_linker = _InMemoryUnifiedCacheLinker(stored_keys) + producer.init_cache_linker(producer_linker) + inserted = self._insert(producer, producer_allocator, producer_req_pool, tokens) + # Ancestors of the chain carry no state; only the checkpoint node does. + *ancestors, offload = producer_linker.offload_calls + for transfers in ancestors: + self.assertEqual( + {transfer.name for transfer in transfers}, {PoolName.KV, PoolName.SWA} + ) + offloads = {transfer.name: transfer for transfer in offload} + self.assertEqual(set(offloads), {PoolName.KV, PoolName.SWA, PoolName.MAMBA}) + mamba_offload = offloads[PoolName.MAMBA] + self.assertEqual(mamba_offload.keys, offloads[PoolName.KV].keys[-1:]) + self.assertEqual(mamba_offload.hit_policy, PoolHitPolicy.TRAILING_PAGES) + self.assertTrue( + torch.equal( + mamba_offload.device_indices, + _device_value(producer, inserted.last_device_node, ComponentType.MAMBA), + ) + ) + for _ in producer_linker.offload_calls: + producer_linker.complete_next_offload(True) + producer.check_hicache_events() + + consumer, _, consumer_req_pool = build_fixture(self.cfg) + consumer_linker = _InMemoryUnifiedCacheLinker(stored_keys) + consumer.init_cache_linker(consumer_linker) + + return consumer, consumer_req_pool, consumer_linker + + def test_mamba_state_round_trips_only_at_its_checkpoint(self): + tokens = list(range(1, 7)) + consumer, consumer_req_pool, consumer_linker = self._mamba_linker_consumer( + tokens + ) + + # Every KV and SWA page is stored, but only the full prefix has a state. + short = consumer.match_prefix( + MatchPrefixParams( + key=RadixKey(array("q", tokens[:5])), + req=self._make_req(consumer_req_pool), + ) + ) + self.assertEqual(short.host_hit_length, 0) + + req = self._make_req(consumer_req_pool) + match = consumer.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", tokens)), req=req) + ) + self.assertEqual(match.host_hit_length, len(tokens)) + self.assertEqual(match.mamba_host_hit_length, 1) + self._apply_match_to_req(req, match) + loaded, _ = consumer.init_load_back( + InitLoadBackParams( + best_match_node=match.best_match_node, + host_hit_length=match.host_hit_length, + req=req, + ) + ) + self.assertEqual(loaded, len(tokens)) + loads = { + transfer.name: transfer + for transfer in consumer_linker.queued_loads[req.rid] + } + self.assertEqual(set(loads), {PoolName.KV, PoolName.SWA, PoolName.MAMBA}) + self.assertEqual(loads[PoolName.MAMBA].keys, loads[PoolName.KV].keys[-1:]) + loaded_state = loads[PoolName.MAMBA].device_indices + self.assertEqual(int(req.kv.mamba_pool_idx), int(loaded_state)) + self.assertFalse(req.kv.mamba_needs_clear) + + loaded_indices = consumer.linker.private_load_indices[req.rid] + self.assertGreaterEqual(consumer.ready_to_load_host_cache(), 0) + self._bind_loaded_request( + consumer, consumer_req_pool, req, tokens, loaded_indices + ) + self.assertEqual(consumer.finish_external_linker_loads([req]), []) + consumer.checkpoint(req, up_to=req.extend_end) + consumer.unlock(req.lock) + req.lock = None + final_match = consumer.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", tokens))) + ) + self.assertEqual(final_match.device_prefix_len, len(tokens)) + for _ in consumer_linker.offload_calls: + consumer_linker.complete_next_offload(True) + consumer.check_hicache_events() + consumer.sanity_check() + + def test_mamba_load_supersedes_a_shorter_device_cow(self): + tokens = list(range(1, 7)) + consumer, consumer_req_pool, consumer_linker = self._mamba_linker_consumer( + tokens + ) + self._insert( + consumer, + consumer.token_to_kv_pool_allocator, + consumer_req_pool, + tokens[:3], + ) + req = self._make_req(consumer_req_pool) + match = consumer.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", tokens)), req=req, cow_mamba=True) + ) + self.assertEqual(match.device_prefix_len, 3) + self.assertEqual(match.host_hit_length, 3) + self.assertIsNotNone(req.kv.mamba_cow_src_index) + self._apply_match_to_req(req, match) + allocator = consumer_req_pool.mamba_allocator + before = allocator.available_size() + + consumer.init_load_back( + InitLoadBackParams( + best_match_node=match.best_match_node, + host_hit_length=match.host_hit_length, + req=req, + ) + ) + # The CoW from the shorter checkpoint would overwrite the loaded state. + loaded_state = next( + transfer.device_indices + for transfer in consumer_linker.queued_loads[req.rid] + if transfer.name == PoolName.MAMBA + ) + self.assertEqual(int(req.kv.mamba_pool_idx), int(loaded_state)) + self.assertIsNone(req.kv.mamba_cow_src_index) + self.assertEqual(allocator.available_size(), before) + + self.assertGreaterEqual(consumer.ready_to_load_host_cache(), 0) + self.assertEqual(consumer.finish_external_linker_loads([req]), []) + consumer.sanity_check() + @unittest.skipUnless(torch.cuda.is_available(), "cache fixtures need CUDA") class TestUnifiedCacheLinkerTreeCorePythonBackend(