From bb1d4cddd953597b4e27a8489367ddcbb99e3762 Mon Sep 17 00:00:00 2001 From: Ke Bao Date: Sun, 18 Oct 2026 23:02:46 +0800 Subject: [PATCH] [Refactor] hicache: Route Mamba host transfers from pool info (#44968) --- .../hybrid_cache/hybrid_pool_assembler.py | 42 ++-- .../kernels/ops/mamba/test_mtp_hicache.py | 2 +- .../mem_cache/test_hybrid_pool_assembler.py | 218 +++++++++++++++++- .../test_unified_hicache_regressions.py | 20 +- 4 files changed, 257 insertions(+), 25 deletions(-) diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py index a715eb9686..9fbece9a17 100644 --- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py +++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py @@ -42,6 +42,7 @@ if TYPE_CHECKING: import torch from sglang.srt.mem_cache.cache_init_params import CacheInitParams + from sglang.srt.mem_cache.device_pool_info import DevicePoolInfo from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache from sglang.srt.server_args import ServerArgs @@ -1182,10 +1183,13 @@ def build_deepseek_v4_hicache_stack( def build_mamba_host_pool( - mamba_pool, *, host_size: float, layout: str, mtp_draft_device_pools=() + mamba_pool, + pool_info: DevicePoolInfo, + *, + host_size: float, + layout: str, + mtp_draft_device_pools=(), ): - (pool_info,) = mamba_pool.get_device_pool_infos() - pool_info.buffer_info.validate(layer_ids=pool_info.layer_ids) draft_buffer_infos = [] for draft_pool in mtp_draft_device_pools: (draft_info,) = draft_pool.get_device_pool_infos() @@ -1211,7 +1215,7 @@ def build_hybrid_mamba_stack( decls: tuple[HostPoolDecl, ...], mamba_pool: Any, full_layer_mapping: dict[int, int], - mamba_layer_mapping: dict[int, int], + start_layer: int, load_cache_event, storage_backend: Optional[str], use_mla: bool, @@ -1225,6 +1229,12 @@ def build_hybrid_mamba_stack( ) -> tuple[HostPoolGroup, HybridCacheController, HostPoolGroupConfig]: """KV plus every pool the hybrid pool declares (e.g. a sparse indexer), then the Mamba state pool, which keeps its own host path.""" + (mamba_info,) = mamba_pool.get_device_pool_infos() + mamba_info.buffer_info.validate(layer_ids=mamba_info.layer_ids) + # Controller IDs are stage-local; Info IDs follow the physical layer axis. + mamba_layer_mapping = { + layer - start_layer: index for index, layer in enumerate(mamba_info.layer_ids) + } transfer_layer_id_max = ( max(full_layer_mapping.keys() | mamba_layer_mapping.keys()) + 1 ) @@ -1270,6 +1280,7 @@ def build_hybrid_mamba_stack( ) mamba_host_pool = build_mamba_host_pool( mamba_pool, + mamba_info, host_size=mamba_host_size, layout=mamba_layout, mtp_draft_device_pools=params.mtp_draft_mamba_pools, @@ -1277,7 +1288,7 @@ def build_hybrid_mamba_stack( mamba_layer_mapping = with_packed_draft_layer_mapping( mamba_layer_mapping, transfer_layer_start=transfer_layer_id_max, - target_device_layer_num=mamba_pool.num_mamba_layers, + target_device_layer_num=len(mamba_info.layer_ids), draft_layer_num=len(params.mtp_draft_mamba_pools), ) entries = _build_pool_entries(config=config, root_host_pool=kv_host_pool) + [ @@ -1343,7 +1354,7 @@ def build_hybrid_mamba_swa_stack( mamba_pool: Any, full_layer_mapping: dict[int, int], swa_layer_mapping: dict[int, int], - mamba_layer_mapping: dict[int, int], + start_layer: int, page_size: int, tp_group, load_cache_event, @@ -1360,6 +1371,12 @@ def build_hybrid_mamba_swa_stack( storage_backend_extra_config: Optional[dict] = None, enable_storage_metrics: bool = False, ) -> tuple[HostPoolGroup, HybridCacheController]: + (mamba_info,) = mamba_pool.get_device_pool_infos() + mamba_info.buffer_info.validate(layer_ids=mamba_info.layer_ids) + # Controller IDs are stage-local; Info IDs follow the physical layer axis. + mamba_layer_mapping = { + layer - start_layer: index for index, layer in enumerate(mamba_info.layer_ids) + } transfer_layer_id_max = ( max( full_layer_mapping.keys() @@ -1398,6 +1415,7 @@ def build_hybrid_mamba_swa_stack( ) mamba_host_pool = build_mamba_host_pool( mamba_pool, + mamba_info, host_size=mamba_host_size, layout=get_memory().hicache_mem_layout, mtp_draft_device_pools=params.mtp_draft_mamba_pools, @@ -1405,7 +1423,7 @@ def build_hybrid_mamba_swa_stack( mamba_layer_mapping = with_packed_draft_layer_mapping( mamba_layer_mapping, transfer_layer_start=transfer_layer_id_max, - target_device_layer_num=mamba_pool.num_mamba_layers, + target_device_layer_num=len(mamba_info.layer_ids), draft_layer_num=len(params.mtp_draft_mamba_pools), ) host_pool_group.add_entry( @@ -1897,15 +1915,12 @@ class _MambaStrategy(StackStrategy): full_layer_mapping = _stage_local_layer_mapping( kvcache.full_attention_layer_id_mapping, kvcache.start_layer ) - mamba_layer_mapping = _stage_local_layer_mapping( - params.req_to_token_pool.mamba_map, kvcache.start_layer - ) host_pool_group, cache_controller, config = build_hybrid_mamba_stack( params=params, decls=kvcache.host_pool_decls(), mamba_pool=params.req_to_token_pool.mamba_pool, full_layer_mapping=full_layer_mapping, - mamba_layer_mapping=mamba_layer_mapping, + start_layer=kvcache.start_layer, load_cache_event=load_cache_event, storage_backend=storage_backend, use_mla=kvcache.use_mla, @@ -2047,9 +2062,6 @@ class _MambaSwaStrategy(StackStrategy): enable_storage_metrics=False, ): full_layer_mapping, swa_layer_mapping = _swa_layer_mappings(kvcache) - mamba_layer_mapping = _stage_local_layer_mapping( - params.req_to_token_pool.mamba_map, kvcache.start_layer - ) host_pool_group, cache_controller = build_hybrid_mamba_swa_stack( params=params, full_kv_pool=kvcache.full_kv_pool, @@ -2057,7 +2069,7 @@ class _MambaSwaStrategy(StackStrategy): mamba_pool=params.req_to_token_pool.mamba_pool, full_layer_mapping=full_layer_mapping, swa_layer_mapping=swa_layer_mapping, - mamba_layer_mapping=mamba_layer_mapping, + start_layer=kvcache.start_layer, page_size=cache.page_size, tp_group=params.tp_cache_group, load_cache_event=load_cache_event, diff --git a/test/registered/kernels/ops/mamba/test_mtp_hicache.py b/test/registered/kernels/ops/mamba/test_mtp_hicache.py index 7866719a7b..46358ccb3b 100644 --- a/test/registered/kernels/ops/mamba/test_mtp_hicache.py +++ b/test/registered/kernels/ops/mamba/test_mtp_hicache.py @@ -161,7 +161,7 @@ def test_hicache_restores_all_mtp_kv_and_conv_state( mamba_pool=target_mamba, full_layer_mapping={layer: i for i, layer in enumerate(full_layers)}, swa_layer_mapping={layer: i for i, layer in enumerate(swa_layers)}, - mamba_layer_mapping={i: i for i in range(target_layers)}, + start_layer=0, page_size=16, tp_group=None, load_cache_event=None, diff --git a/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py b/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py index 70006ca298..e2b0b24008 100644 --- a/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py +++ b/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py @@ -1,6 +1,7 @@ """Unit tests for hybrid HiCache pool assembly.""" import unittest +from functools import partial from queue import Queue from types import SimpleNamespace from unittest.mock import MagicMock, patch @@ -8,7 +9,13 @@ from unittest.mock import MagicMock, patch import msgspec import torch +from sglang.srt.configs.inkling import ( + InklingConvCacheParams, + InklingConvStateShape, + InklingStateDType, +) from sglang.srt.mem_cache.base_prefix_cache import CacheRequestHandle, EvictParams +from sglang.srt.mem_cache.device_pool_info import DevicePoolInfo, MambaStateBufferInfo from sglang.srt.mem_cache.hicache_storage import PoolHitPolicy, PoolName, PoolTransfer from sglang.srt.mem_cache.hybrid_cache import hybrid_pool_assembler from sglang.srt.mem_cache.hybrid_cache.host_pool_config import ( @@ -34,13 +41,18 @@ from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import ( build_host_pool_group, build_hybrid_swa_group, ) -from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool, HybridLinearKVPool +from sglang.srt.mem_cache.memory_pool import ( + DSATokenToKVPool, + HybridLinearKVPool, + MambaPool, +) from sglang.srt.mem_cache.pool_host import HostPoolGroup, PoolEntry from sglang.srt.mem_cache.pool_host import dsa as pool_host_dsa from sglang.srt.mem_cache.pool_host.host_pool_decl import ( HostPoolDecl, make_kv_pool_decl, ) +from sglang.srt.mem_cache.pool_host.mamba import MambaPoolHost from sglang.srt.mem_cache.pool_host.unified import UnifiedPageEnvelopeHostPool from sglang.srt.mem_cache.unified_memory_pool import init_unified_swa_pools from sglang.test.ci.ci_register import register_cpu_ci @@ -169,7 +181,6 @@ class TestHybridStageLayerMappings(CustomTestCase): host_pool_decls=lambda: (), ) req_pool = SimpleNamespace( - mamba_map=global_maps.get("mamba", {}).copy(), mamba_pool=object(), ) params = SimpleNamespace( @@ -195,7 +206,13 @@ class TestHybridStageLayerMappings(CustomTestCase): ) build_stack.assert_called_once() + if "mamba" in local_maps: + self.assertEqual( + build_stack.call_args.kwargs["start_layer"], start_layer + ) for name, mapping in local_maps.items(): + if name == "mamba": + continue self.assertEqual( build_stack.call_args.kwargs[f"{name}_layer_mapping"], mapping, @@ -205,7 +222,189 @@ class TestHybridStageLayerMappings(CustomTestCase): kvcache.full_attention_layer_id_mapping, global_maps["full"] ) self.assertEqual(kvcache.layers_mapping, layers_mapping) - self.assertEqual(req_pool.mamba_map, global_maps.get("mamba", {})) + + +class TestMambaInfoRouting(CustomTestCase): + def test_checkpoint_axis_routes_stage_layers_for_both_stacks(self): + for tri_pool in (False, True): + for start_layer in (0, 8): + for local_ids in ((4, 2, 3), ()): + with self.subTest( + tri_pool=tri_pool, start=start_layer, layers=local_ids + ): + + def make_pool(ids): + return MambaPool( + size=8, + spec_state_size=0, + cache_params=InklingConvCacheParams( + shape=InklingConvStateShape( + conv=[(2,)], temporal=(2,) + ), + layers=list(ids), + dtype=InklingStateDType( + conv=torch.float32, temporal=torch.float32 + ), + ), + mamba_layer_ids=list(ids), + device="cpu", + ) + + mamba_pool = make_pool( + tuple(layer + start_layer for layer in local_ids) + ) + temporal = mamba_pool.mamba_cache.temporal + temporal.copy_( + torch.arange(temporal.numel()).reshape(temporal.shape) + ) + # Buffer-only capacity may exceed the physical checkpoint slots. + mamba_pool.host_capacity_tokens = 16 if start_layer else None + drafts = ( + tuple( + make_pool([start_layer + 5 + depth]) + for depth in range(2) + ) + if local_ids + else () + ) + captured = [] + original_provider = mamba_pool.get_device_pool_infos + + def capture(): + result = original_provider() + captured.append(result[0]) + return result + + provider = MagicMock(side_effect=capture) + kv = SimpleNamespace(layer_num=1, size=8) + allocator = SimpleNamespace(alloc=MagicMock(), free=MagicMock()) + params = SimpleNamespace( + page_size=1, + mtp_draft_device_pools=(), + mtp_draft_mamba_pools=drafts, + token_to_kv_pool_allocator=SimpleNamespace( + swa_attn_allocator=allocator + ), + req_to_token_pool=SimpleNamespace( + mamba_allocator=allocator + ), + tp_cache_group=None, + pp_cache_group=None, + attn_cp_cache_group=None, + attn_tp_cache_group=None, + ) + memory = SimpleNamespace( + hicache_size=0, + hicache_ratio=2, + hicache_mem_layout="page_first_direct", + hicache_write_policy="write_through", + hicache_io_backend="direct", + hicache_host_memory_mode="cache", + enable_unified_memory=False, + ) + with ( + patch.object(mamba_pool, "get_device_pool_infos", provider), + patch.object( + hybrid_pool_assembler, "get_memory", return_value=memory + ), + patch.object( + hybrid_pool_assembler, + "_get_allocator_type", + return_value="default", + ), + patch.object( + hybrid_pool_assembler, + "build_kv_host_pool", + return_value=SimpleNamespace( + layer_num=1, + layout="page_first_direct", + page_size=1, + device="cpu", + size=17, + logical_size=17, + can_use_write_back_jit=False, + ), + ), + patch.object( + hybrid_pool_assembler, + "MambaPoolHost", + partial(MambaPoolHost, pin_memory=False), + ), + patch.object( + hybrid_pool_assembler, "HybridCacheController" + ), + ): + common = dict( + params=params, + mamba_pool=mamba_pool, + full_layer_mapping={0: 0}, + start_layer=start_layer, + load_cache_event=None, + storage_backend=None, + ) + if tri_pool: + group, _ = ( + hybrid_pool_assembler.build_hybrid_mamba_swa_stack( + **common, + full_kv_pool=kv, + swa_kv_pool=kv, + swa_layer_mapping={1: 0}, + page_size=1, + tp_group=None, + ) + ) + else: + group, _, _ = ( + hybrid_pool_assembler.build_hybrid_mamba_stack( + **common, + decls=(make_kv_pool_decl(kv),), + use_mla=False, + ) + ) + entry = group.entry_map[PoolName.MAMBA] + host = entry.host_pool + try: + info = captured[0] + self.assertIs(host.buffer_info, info.buffer_info) + self.assertEqual( + host.size, (16 if start_layer else 8) * 2 + 1 + ) + self.assertEqual( + len(host.mtp_draft_buffer_infos), len(drafts) + ) + for depth, draft in enumerate(drafts): + self.assertIs( + host.mtp_draft_buffer_infos[depth].temporal, + draft.mamba_cache.temporal, + ) + self.assertEqual( + entry.layer_mapper(5 + depth), + len(local_ids) + depth, + ) + self.assertEqual( + host.temporal_device_layers[ + len(local_ids) + depth + ].data_ptr(), + draft.mamba_cache.temporal[0].data_ptr(), + ) + self.assertEqual(provider.call_count, 1) + self.assertEqual( + info.layer_ids, + tuple(layer + start_layer for layer in local_ids), + ) + self.assertIs(entry.device_alloc_fn, allocator.alloc) + self.assertIs(entry.device_free_fn, allocator.free) + for layer in range(-1, 5): + slot = entry.layer_mapper(layer) + if layer not in local_ids: + self.assertIsNone(slot) + else: + torch.testing.assert_close( + host.buffer_info.temporal[slot], + temporal[local_ids.index(layer)], + ) + finally: + host.destroy() class TestDraftSidecarPoolDispatch(CustomTestCase): @@ -748,10 +947,17 @@ class TestHybridMambaDeclaredIndexer(CustomTestCase): kv_pool = _dsa_pool_stub(layer_num=3) kv_pool.skip_topk_layers = [False, True, False] # device layer 1: no buffer - mamba_pool = SimpleNamespace(layer_num=2, num_mamba_layers=2, size=8) + mamba_info = DevicePoolInfo( + pool_name=PoolName.MAMBA, + indices_from_pool=PoolName.MAMBA, + layer_ids=(1, 3), + buffer_info=MambaStateBufferInfo( + temporal=torch.zeros(2, 9, 4), conv=(torch.zeros(2, 9, 4),) + ), + ) + mamba_pool = SimpleNamespace(get_device_pool_infos=lambda: (mamba_info,)) # transfer layers 0,2,4 are DSA (device 0,1,2), 1,3 are Mamba full_mapping = {0: 0, 2: 1, 4: 2} - mamba_mapping = {1: 0, 3: 1} params = SimpleNamespace( page_size=64, mtp_draft_device_pools=(), @@ -824,7 +1030,7 @@ class TestHybridMambaDeclaredIndexer(CustomTestCase): decls=kv_pool.host_pool_decls(), mamba_pool=mamba_pool, full_layer_mapping=full_mapping, - mamba_layer_mapping=mamba_mapping, + start_layer=0, load_cache_event=None, storage_backend=None, use_mla=True, diff --git a/test/registered/unit/mem_cache/test_unified_hicache_regressions.py b/test/registered/unit/mem_cache/test_unified_hicache_regressions.py index 54a8f2c780..09950e3f74 100644 --- a/test/registered/unit/mem_cache/test_unified_hicache_regressions.py +++ b/test/registered/unit/mem_cache/test_unified_hicache_regressions.py @@ -7,11 +7,13 @@ from unittest.mock import MagicMock, Mock, patch import torch from sglang.srt.layers.dcp.layout import maybe_dcp_kernel_indices +from sglang.srt.mem_cache.device_pool_info import DevicePoolInfo, MambaStateBufferInfo from sglang.srt.mem_cache.hicache_storage import PoolName from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import _split_hicache_size from sglang.srt.mem_cache.l2_transfer import L2Transfer, L2TransferEngine from sglang.srt.mem_cache.unified_cache.component_type import ComponentType from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase register_cuda_ci(est_time=15, stage="extra-a", runner_config="1-gpu-small") @@ -219,7 +221,7 @@ class TestDirectBackendTranslation(unittest.TestCase): torch.testing.assert_close(result, torch.tensor([6, 9])) -class TestTriPoolAssembly(unittest.TestCase): +class TestTriPoolAssembly(CustomTestCase): def test_swa_allocation_and_rollback_match_pool_id_ownership(self): from sglang.srt.mem_cache.allocator.unified_sub_pool import ( MultiEndedAllocator, @@ -258,10 +260,22 @@ class TestTriPoolAssembly(unittest.TestCase): params=params, full_kv_pool=object(), swa_kv_pool=object(), - mamba_pool=SimpleNamespace(num_mamba_layers=1), + mamba_pool=SimpleNamespace( + get_device_pool_infos=lambda: ( + DevicePoolInfo( + pool_name=PoolName.MAMBA, + indices_from_pool=PoolName.MAMBA, + layer_ids=(2,), + buffer_info=MambaStateBufferInfo( + temporal=torch.zeros(1, 2, 1), + conv=(torch.zeros(1, 2, 1),), + ), + ), + ) + ), full_layer_mapping={0: 0}, swa_layer_mapping={1: 0}, - mamba_layer_mapping={2: 0}, + start_layer=0, page_size=1, tp_group=None, load_cache_event=None,