[Refactor] hicache: Route Mamba host transfers from pool info (#44968)

This commit is contained in:
Ke Bao
2026-10-18 23:02:46 +08:00
committed by GitHub
parent d4e3c163c9
commit bb1d4cddd9
4 changed files with 257 additions and 25 deletions
@@ -42,6 +42,7 @@ if TYPE_CHECKING:
import torch
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
from sglang.srt.mem_cache.device_pool_info import DevicePoolInfo
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
from sglang.srt.server_args import ServerArgs
@@ -1182,10 +1183,13 @@ def build_deepseek_v4_hicache_stack(
def build_mamba_host_pool(
mamba_pool, *, host_size: float, layout: str, mtp_draft_device_pools=()
mamba_pool,
pool_info: DevicePoolInfo,
*,
host_size: float,
layout: str,
mtp_draft_device_pools=(),
):
(pool_info,) = mamba_pool.get_device_pool_infos()
pool_info.buffer_info.validate(layer_ids=pool_info.layer_ids)
draft_buffer_infos = []
for draft_pool in mtp_draft_device_pools:
(draft_info,) = draft_pool.get_device_pool_infos()
@@ -1211,7 +1215,7 @@ def build_hybrid_mamba_stack(
decls: tuple[HostPoolDecl, ...],
mamba_pool: Any,
full_layer_mapping: dict[int, int],
mamba_layer_mapping: dict[int, int],
start_layer: int,
load_cache_event,
storage_backend: Optional[str],
use_mla: bool,
@@ -1225,6 +1229,12 @@ def build_hybrid_mamba_stack(
) -> tuple[HostPoolGroup, HybridCacheController, HostPoolGroupConfig]:
"""KV plus every pool the hybrid pool declares (e.g. a sparse indexer),
then the Mamba state pool, which keeps its own host path."""
(mamba_info,) = mamba_pool.get_device_pool_infos()
mamba_info.buffer_info.validate(layer_ids=mamba_info.layer_ids)
# Controller IDs are stage-local; Info IDs follow the physical layer axis.
mamba_layer_mapping = {
layer - start_layer: index for index, layer in enumerate(mamba_info.layer_ids)
}
transfer_layer_id_max = (
max(full_layer_mapping.keys() | mamba_layer_mapping.keys()) + 1
)
@@ -1270,6 +1280,7 @@ def build_hybrid_mamba_stack(
)
mamba_host_pool = build_mamba_host_pool(
mamba_pool,
mamba_info,
host_size=mamba_host_size,
layout=mamba_layout,
mtp_draft_device_pools=params.mtp_draft_mamba_pools,
@@ -1277,7 +1288,7 @@ def build_hybrid_mamba_stack(
mamba_layer_mapping = with_packed_draft_layer_mapping(
mamba_layer_mapping,
transfer_layer_start=transfer_layer_id_max,
target_device_layer_num=mamba_pool.num_mamba_layers,
target_device_layer_num=len(mamba_info.layer_ids),
draft_layer_num=len(params.mtp_draft_mamba_pools),
)
entries = _build_pool_entries(config=config, root_host_pool=kv_host_pool) + [
@@ -1343,7 +1354,7 @@ def build_hybrid_mamba_swa_stack(
mamba_pool: Any,
full_layer_mapping: dict[int, int],
swa_layer_mapping: dict[int, int],
mamba_layer_mapping: dict[int, int],
start_layer: int,
page_size: int,
tp_group,
load_cache_event,
@@ -1360,6 +1371,12 @@ def build_hybrid_mamba_swa_stack(
storage_backend_extra_config: Optional[dict] = None,
enable_storage_metrics: bool = False,
) -> tuple[HostPoolGroup, HybridCacheController]:
(mamba_info,) = mamba_pool.get_device_pool_infos()
mamba_info.buffer_info.validate(layer_ids=mamba_info.layer_ids)
# Controller IDs are stage-local; Info IDs follow the physical layer axis.
mamba_layer_mapping = {
layer - start_layer: index for index, layer in enumerate(mamba_info.layer_ids)
}
transfer_layer_id_max = (
max(
full_layer_mapping.keys()
@@ -1398,6 +1415,7 @@ def build_hybrid_mamba_swa_stack(
)
mamba_host_pool = build_mamba_host_pool(
mamba_pool,
mamba_info,
host_size=mamba_host_size,
layout=get_memory().hicache_mem_layout,
mtp_draft_device_pools=params.mtp_draft_mamba_pools,
@@ -1405,7 +1423,7 @@ def build_hybrid_mamba_swa_stack(
mamba_layer_mapping = with_packed_draft_layer_mapping(
mamba_layer_mapping,
transfer_layer_start=transfer_layer_id_max,
target_device_layer_num=mamba_pool.num_mamba_layers,
target_device_layer_num=len(mamba_info.layer_ids),
draft_layer_num=len(params.mtp_draft_mamba_pools),
)
host_pool_group.add_entry(
@@ -1897,15 +1915,12 @@ class _MambaStrategy(StackStrategy):
full_layer_mapping = _stage_local_layer_mapping(
kvcache.full_attention_layer_id_mapping, kvcache.start_layer
)
mamba_layer_mapping = _stage_local_layer_mapping(
params.req_to_token_pool.mamba_map, kvcache.start_layer
)
host_pool_group, cache_controller, config = build_hybrid_mamba_stack(
params=params,
decls=kvcache.host_pool_decls(),
mamba_pool=params.req_to_token_pool.mamba_pool,
full_layer_mapping=full_layer_mapping,
mamba_layer_mapping=mamba_layer_mapping,
start_layer=kvcache.start_layer,
load_cache_event=load_cache_event,
storage_backend=storage_backend,
use_mla=kvcache.use_mla,
@@ -2047,9 +2062,6 @@ class _MambaSwaStrategy(StackStrategy):
enable_storage_metrics=False,
):
full_layer_mapping, swa_layer_mapping = _swa_layer_mappings(kvcache)
mamba_layer_mapping = _stage_local_layer_mapping(
params.req_to_token_pool.mamba_map, kvcache.start_layer
)
host_pool_group, cache_controller = build_hybrid_mamba_swa_stack(
params=params,
full_kv_pool=kvcache.full_kv_pool,
@@ -2057,7 +2069,7 @@ class _MambaSwaStrategy(StackStrategy):
mamba_pool=params.req_to_token_pool.mamba_pool,
full_layer_mapping=full_layer_mapping,
swa_layer_mapping=swa_layer_mapping,
mamba_layer_mapping=mamba_layer_mapping,
start_layer=kvcache.start_layer,
page_size=cache.page_size,
tp_group=params.tp_cache_group,
load_cache_event=load_cache_event,
@@ -161,7 +161,7 @@ def test_hicache_restores_all_mtp_kv_and_conv_state(
mamba_pool=target_mamba,
full_layer_mapping={layer: i for i, layer in enumerate(full_layers)},
swa_layer_mapping={layer: i for i, layer in enumerate(swa_layers)},
mamba_layer_mapping={i: i for i in range(target_layers)},
start_layer=0,
page_size=16,
tp_group=None,
load_cache_event=None,
@@ -1,6 +1,7 @@
"""Unit tests for hybrid HiCache pool assembly."""
import unittest
from functools import partial
from queue import Queue
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
@@ -8,7 +9,13 @@ from unittest.mock import MagicMock, patch
import msgspec
import torch
from sglang.srt.configs.inkling import (
InklingConvCacheParams,
InklingConvStateShape,
InklingStateDType,
)
from sglang.srt.mem_cache.base_prefix_cache import CacheRequestHandle, EvictParams
from sglang.srt.mem_cache.device_pool_info import DevicePoolInfo, MambaStateBufferInfo
from sglang.srt.mem_cache.hicache_storage import PoolHitPolicy, PoolName, PoolTransfer
from sglang.srt.mem_cache.hybrid_cache import hybrid_pool_assembler
from sglang.srt.mem_cache.hybrid_cache.host_pool_config import (
@@ -34,13 +41,18 @@ from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import (
build_host_pool_group,
build_hybrid_swa_group,
)
from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool, HybridLinearKVPool
from sglang.srt.mem_cache.memory_pool import (
DSATokenToKVPool,
HybridLinearKVPool,
MambaPool,
)
from sglang.srt.mem_cache.pool_host import HostPoolGroup, PoolEntry
from sglang.srt.mem_cache.pool_host import dsa as pool_host_dsa
from sglang.srt.mem_cache.pool_host.host_pool_decl import (
HostPoolDecl,
make_kv_pool_decl,
)
from sglang.srt.mem_cache.pool_host.mamba import MambaPoolHost
from sglang.srt.mem_cache.pool_host.unified import UnifiedPageEnvelopeHostPool
from sglang.srt.mem_cache.unified_memory_pool import init_unified_swa_pools
from sglang.test.ci.ci_register import register_cpu_ci
@@ -169,7 +181,6 @@ class TestHybridStageLayerMappings(CustomTestCase):
host_pool_decls=lambda: (),
)
req_pool = SimpleNamespace(
mamba_map=global_maps.get("mamba", {}).copy(),
mamba_pool=object(),
)
params = SimpleNamespace(
@@ -195,7 +206,13 @@ class TestHybridStageLayerMappings(CustomTestCase):
)
build_stack.assert_called_once()
if "mamba" in local_maps:
self.assertEqual(
build_stack.call_args.kwargs["start_layer"], start_layer
)
for name, mapping in local_maps.items():
if name == "mamba":
continue
self.assertEqual(
build_stack.call_args.kwargs[f"{name}_layer_mapping"],
mapping,
@@ -205,7 +222,189 @@ class TestHybridStageLayerMappings(CustomTestCase):
kvcache.full_attention_layer_id_mapping, global_maps["full"]
)
self.assertEqual(kvcache.layers_mapping, layers_mapping)
self.assertEqual(req_pool.mamba_map, global_maps.get("mamba", {}))
class TestMambaInfoRouting(CustomTestCase):
def test_checkpoint_axis_routes_stage_layers_for_both_stacks(self):
for tri_pool in (False, True):
for start_layer in (0, 8):
for local_ids in ((4, 2, 3), ()):
with self.subTest(
tri_pool=tri_pool, start=start_layer, layers=local_ids
):
def make_pool(ids):
return MambaPool(
size=8,
spec_state_size=0,
cache_params=InklingConvCacheParams(
shape=InklingConvStateShape(
conv=[(2,)], temporal=(2,)
),
layers=list(ids),
dtype=InklingStateDType(
conv=torch.float32, temporal=torch.float32
),
),
mamba_layer_ids=list(ids),
device="cpu",
)
mamba_pool = make_pool(
tuple(layer + start_layer for layer in local_ids)
)
temporal = mamba_pool.mamba_cache.temporal
temporal.copy_(
torch.arange(temporal.numel()).reshape(temporal.shape)
)
# Buffer-only capacity may exceed the physical checkpoint slots.
mamba_pool.host_capacity_tokens = 16 if start_layer else None
drafts = (
tuple(
make_pool([start_layer + 5 + depth])
for depth in range(2)
)
if local_ids
else ()
)
captured = []
original_provider = mamba_pool.get_device_pool_infos
def capture():
result = original_provider()
captured.append(result[0])
return result
provider = MagicMock(side_effect=capture)
kv = SimpleNamespace(layer_num=1, size=8)
allocator = SimpleNamespace(alloc=MagicMock(), free=MagicMock())
params = SimpleNamespace(
page_size=1,
mtp_draft_device_pools=(),
mtp_draft_mamba_pools=drafts,
token_to_kv_pool_allocator=SimpleNamespace(
swa_attn_allocator=allocator
),
req_to_token_pool=SimpleNamespace(
mamba_allocator=allocator
),
tp_cache_group=None,
pp_cache_group=None,
attn_cp_cache_group=None,
attn_tp_cache_group=None,
)
memory = SimpleNamespace(
hicache_size=0,
hicache_ratio=2,
hicache_mem_layout="page_first_direct",
hicache_write_policy="write_through",
hicache_io_backend="direct",
hicache_host_memory_mode="cache",
enable_unified_memory=False,
)
with (
patch.object(mamba_pool, "get_device_pool_infos", provider),
patch.object(
hybrid_pool_assembler, "get_memory", return_value=memory
),
patch.object(
hybrid_pool_assembler,
"_get_allocator_type",
return_value="default",
),
patch.object(
hybrid_pool_assembler,
"build_kv_host_pool",
return_value=SimpleNamespace(
layer_num=1,
layout="page_first_direct",
page_size=1,
device="cpu",
size=17,
logical_size=17,
can_use_write_back_jit=False,
),
),
patch.object(
hybrid_pool_assembler,
"MambaPoolHost",
partial(MambaPoolHost, pin_memory=False),
),
patch.object(
hybrid_pool_assembler, "HybridCacheController"
),
):
common = dict(
params=params,
mamba_pool=mamba_pool,
full_layer_mapping={0: 0},
start_layer=start_layer,
load_cache_event=None,
storage_backend=None,
)
if tri_pool:
group, _ = (
hybrid_pool_assembler.build_hybrid_mamba_swa_stack(
**common,
full_kv_pool=kv,
swa_kv_pool=kv,
swa_layer_mapping={1: 0},
page_size=1,
tp_group=None,
)
)
else:
group, _, _ = (
hybrid_pool_assembler.build_hybrid_mamba_stack(
**common,
decls=(make_kv_pool_decl(kv),),
use_mla=False,
)
)
entry = group.entry_map[PoolName.MAMBA]
host = entry.host_pool
try:
info = captured[0]
self.assertIs(host.buffer_info, info.buffer_info)
self.assertEqual(
host.size, (16 if start_layer else 8) * 2 + 1
)
self.assertEqual(
len(host.mtp_draft_buffer_infos), len(drafts)
)
for depth, draft in enumerate(drafts):
self.assertIs(
host.mtp_draft_buffer_infos[depth].temporal,
draft.mamba_cache.temporal,
)
self.assertEqual(
entry.layer_mapper(5 + depth),
len(local_ids) + depth,
)
self.assertEqual(
host.temporal_device_layers[
len(local_ids) + depth
].data_ptr(),
draft.mamba_cache.temporal[0].data_ptr(),
)
self.assertEqual(provider.call_count, 1)
self.assertEqual(
info.layer_ids,
tuple(layer + start_layer for layer in local_ids),
)
self.assertIs(entry.device_alloc_fn, allocator.alloc)
self.assertIs(entry.device_free_fn, allocator.free)
for layer in range(-1, 5):
slot = entry.layer_mapper(layer)
if layer not in local_ids:
self.assertIsNone(slot)
else:
torch.testing.assert_close(
host.buffer_info.temporal[slot],
temporal[local_ids.index(layer)],
)
finally:
host.destroy()
class TestDraftSidecarPoolDispatch(CustomTestCase):
@@ -748,10 +947,17 @@ class TestHybridMambaDeclaredIndexer(CustomTestCase):
kv_pool = _dsa_pool_stub(layer_num=3)
kv_pool.skip_topk_layers = [False, True, False] # device layer 1: no buffer
mamba_pool = SimpleNamespace(layer_num=2, num_mamba_layers=2, size=8)
mamba_info = DevicePoolInfo(
pool_name=PoolName.MAMBA,
indices_from_pool=PoolName.MAMBA,
layer_ids=(1, 3),
buffer_info=MambaStateBufferInfo(
temporal=torch.zeros(2, 9, 4), conv=(torch.zeros(2, 9, 4),)
),
)
mamba_pool = SimpleNamespace(get_device_pool_infos=lambda: (mamba_info,))
# transfer layers 0,2,4 are DSA (device 0,1,2), 1,3 are Mamba
full_mapping = {0: 0, 2: 1, 4: 2}
mamba_mapping = {1: 0, 3: 1}
params = SimpleNamespace(
page_size=64,
mtp_draft_device_pools=(),
@@ -824,7 +1030,7 @@ class TestHybridMambaDeclaredIndexer(CustomTestCase):
decls=kv_pool.host_pool_decls(),
mamba_pool=mamba_pool,
full_layer_mapping=full_mapping,
mamba_layer_mapping=mamba_mapping,
start_layer=0,
load_cache_event=None,
storage_backend=None,
use_mla=True,
@@ -7,11 +7,13 @@ from unittest.mock import MagicMock, Mock, patch
import torch
from sglang.srt.layers.dcp.layout import maybe_dcp_kernel_indices
from sglang.srt.mem_cache.device_pool_info import DevicePoolInfo, MambaStateBufferInfo
from sglang.srt.mem_cache.hicache_storage import PoolName
from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import _split_hicache_size
from sglang.srt.mem_cache.l2_transfer import L2Transfer, L2TransferEngine
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=15, stage="extra-a", runner_config="1-gpu-small")
@@ -219,7 +221,7 @@ class TestDirectBackendTranslation(unittest.TestCase):
torch.testing.assert_close(result, torch.tensor([6, 9]))
class TestTriPoolAssembly(unittest.TestCase):
class TestTriPoolAssembly(CustomTestCase):
def test_swa_allocation_and_rollback_match_pool_id_ownership(self):
from sglang.srt.mem_cache.allocator.unified_sub_pool import (
MultiEndedAllocator,
@@ -258,10 +260,22 @@ class TestTriPoolAssembly(unittest.TestCase):
params=params,
full_kv_pool=object(),
swa_kv_pool=object(),
mamba_pool=SimpleNamespace(num_mamba_layers=1),
mamba_pool=SimpleNamespace(
get_device_pool_infos=lambda: (
DevicePoolInfo(
pool_name=PoolName.MAMBA,
indices_from_pool=PoolName.MAMBA,
layer_ids=(2,),
buffer_info=MambaStateBufferInfo(
temporal=torch.zeros(1, 2, 1),
conv=(torch.zeros(1, 2, 1),),
),
),
)
),
full_layer_mapping={0: 0},
swa_layer_mapping={1: 0},
mamba_layer_mapping={2: 0},
start_layer=0,
page_size=1,
tp_group=None,
load_cache_event=None,