[Refactor] hicache: Route Mamba host transfers from pool info (#44968)
This commit is contained in:
@@ -42,6 +42,7 @@ if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
|
||||
from sglang.srt.mem_cache.device_pool_info import DevicePoolInfo
|
||||
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
|
||||
@@ -1182,10 +1183,13 @@ def build_deepseek_v4_hicache_stack(
|
||||
|
||||
|
||||
def build_mamba_host_pool(
|
||||
mamba_pool, *, host_size: float, layout: str, mtp_draft_device_pools=()
|
||||
mamba_pool,
|
||||
pool_info: DevicePoolInfo,
|
||||
*,
|
||||
host_size: float,
|
||||
layout: str,
|
||||
mtp_draft_device_pools=(),
|
||||
):
|
||||
(pool_info,) = mamba_pool.get_device_pool_infos()
|
||||
pool_info.buffer_info.validate(layer_ids=pool_info.layer_ids)
|
||||
draft_buffer_infos = []
|
||||
for draft_pool in mtp_draft_device_pools:
|
||||
(draft_info,) = draft_pool.get_device_pool_infos()
|
||||
@@ -1211,7 +1215,7 @@ def build_hybrid_mamba_stack(
|
||||
decls: tuple[HostPoolDecl, ...],
|
||||
mamba_pool: Any,
|
||||
full_layer_mapping: dict[int, int],
|
||||
mamba_layer_mapping: dict[int, int],
|
||||
start_layer: int,
|
||||
load_cache_event,
|
||||
storage_backend: Optional[str],
|
||||
use_mla: bool,
|
||||
@@ -1225,6 +1229,12 @@ def build_hybrid_mamba_stack(
|
||||
) -> tuple[HostPoolGroup, HybridCacheController, HostPoolGroupConfig]:
|
||||
"""KV plus every pool the hybrid pool declares (e.g. a sparse indexer),
|
||||
then the Mamba state pool, which keeps its own host path."""
|
||||
(mamba_info,) = mamba_pool.get_device_pool_infos()
|
||||
mamba_info.buffer_info.validate(layer_ids=mamba_info.layer_ids)
|
||||
# Controller IDs are stage-local; Info IDs follow the physical layer axis.
|
||||
mamba_layer_mapping = {
|
||||
layer - start_layer: index for index, layer in enumerate(mamba_info.layer_ids)
|
||||
}
|
||||
transfer_layer_id_max = (
|
||||
max(full_layer_mapping.keys() | mamba_layer_mapping.keys()) + 1
|
||||
)
|
||||
@@ -1270,6 +1280,7 @@ def build_hybrid_mamba_stack(
|
||||
)
|
||||
mamba_host_pool = build_mamba_host_pool(
|
||||
mamba_pool,
|
||||
mamba_info,
|
||||
host_size=mamba_host_size,
|
||||
layout=mamba_layout,
|
||||
mtp_draft_device_pools=params.mtp_draft_mamba_pools,
|
||||
@@ -1277,7 +1288,7 @@ def build_hybrid_mamba_stack(
|
||||
mamba_layer_mapping = with_packed_draft_layer_mapping(
|
||||
mamba_layer_mapping,
|
||||
transfer_layer_start=transfer_layer_id_max,
|
||||
target_device_layer_num=mamba_pool.num_mamba_layers,
|
||||
target_device_layer_num=len(mamba_info.layer_ids),
|
||||
draft_layer_num=len(params.mtp_draft_mamba_pools),
|
||||
)
|
||||
entries = _build_pool_entries(config=config, root_host_pool=kv_host_pool) + [
|
||||
@@ -1343,7 +1354,7 @@ def build_hybrid_mamba_swa_stack(
|
||||
mamba_pool: Any,
|
||||
full_layer_mapping: dict[int, int],
|
||||
swa_layer_mapping: dict[int, int],
|
||||
mamba_layer_mapping: dict[int, int],
|
||||
start_layer: int,
|
||||
page_size: int,
|
||||
tp_group,
|
||||
load_cache_event,
|
||||
@@ -1360,6 +1371,12 @@ def build_hybrid_mamba_swa_stack(
|
||||
storage_backend_extra_config: Optional[dict] = None,
|
||||
enable_storage_metrics: bool = False,
|
||||
) -> tuple[HostPoolGroup, HybridCacheController]:
|
||||
(mamba_info,) = mamba_pool.get_device_pool_infos()
|
||||
mamba_info.buffer_info.validate(layer_ids=mamba_info.layer_ids)
|
||||
# Controller IDs are stage-local; Info IDs follow the physical layer axis.
|
||||
mamba_layer_mapping = {
|
||||
layer - start_layer: index for index, layer in enumerate(mamba_info.layer_ids)
|
||||
}
|
||||
transfer_layer_id_max = (
|
||||
max(
|
||||
full_layer_mapping.keys()
|
||||
@@ -1398,6 +1415,7 @@ def build_hybrid_mamba_swa_stack(
|
||||
)
|
||||
mamba_host_pool = build_mamba_host_pool(
|
||||
mamba_pool,
|
||||
mamba_info,
|
||||
host_size=mamba_host_size,
|
||||
layout=get_memory().hicache_mem_layout,
|
||||
mtp_draft_device_pools=params.mtp_draft_mamba_pools,
|
||||
@@ -1405,7 +1423,7 @@ def build_hybrid_mamba_swa_stack(
|
||||
mamba_layer_mapping = with_packed_draft_layer_mapping(
|
||||
mamba_layer_mapping,
|
||||
transfer_layer_start=transfer_layer_id_max,
|
||||
target_device_layer_num=mamba_pool.num_mamba_layers,
|
||||
target_device_layer_num=len(mamba_info.layer_ids),
|
||||
draft_layer_num=len(params.mtp_draft_mamba_pools),
|
||||
)
|
||||
host_pool_group.add_entry(
|
||||
@@ -1897,15 +1915,12 @@ class _MambaStrategy(StackStrategy):
|
||||
full_layer_mapping = _stage_local_layer_mapping(
|
||||
kvcache.full_attention_layer_id_mapping, kvcache.start_layer
|
||||
)
|
||||
mamba_layer_mapping = _stage_local_layer_mapping(
|
||||
params.req_to_token_pool.mamba_map, kvcache.start_layer
|
||||
)
|
||||
host_pool_group, cache_controller, config = build_hybrid_mamba_stack(
|
||||
params=params,
|
||||
decls=kvcache.host_pool_decls(),
|
||||
mamba_pool=params.req_to_token_pool.mamba_pool,
|
||||
full_layer_mapping=full_layer_mapping,
|
||||
mamba_layer_mapping=mamba_layer_mapping,
|
||||
start_layer=kvcache.start_layer,
|
||||
load_cache_event=load_cache_event,
|
||||
storage_backend=storage_backend,
|
||||
use_mla=kvcache.use_mla,
|
||||
@@ -2047,9 +2062,6 @@ class _MambaSwaStrategy(StackStrategy):
|
||||
enable_storage_metrics=False,
|
||||
):
|
||||
full_layer_mapping, swa_layer_mapping = _swa_layer_mappings(kvcache)
|
||||
mamba_layer_mapping = _stage_local_layer_mapping(
|
||||
params.req_to_token_pool.mamba_map, kvcache.start_layer
|
||||
)
|
||||
host_pool_group, cache_controller = build_hybrid_mamba_swa_stack(
|
||||
params=params,
|
||||
full_kv_pool=kvcache.full_kv_pool,
|
||||
@@ -2057,7 +2069,7 @@ class _MambaSwaStrategy(StackStrategy):
|
||||
mamba_pool=params.req_to_token_pool.mamba_pool,
|
||||
full_layer_mapping=full_layer_mapping,
|
||||
swa_layer_mapping=swa_layer_mapping,
|
||||
mamba_layer_mapping=mamba_layer_mapping,
|
||||
start_layer=kvcache.start_layer,
|
||||
page_size=cache.page_size,
|
||||
tp_group=params.tp_cache_group,
|
||||
load_cache_event=load_cache_event,
|
||||
|
||||
@@ -161,7 +161,7 @@ def test_hicache_restores_all_mtp_kv_and_conv_state(
|
||||
mamba_pool=target_mamba,
|
||||
full_layer_mapping={layer: i for i, layer in enumerate(full_layers)},
|
||||
swa_layer_mapping={layer: i for i, layer in enumerate(swa_layers)},
|
||||
mamba_layer_mapping={i: i for i in range(target_layers)},
|
||||
start_layer=0,
|
||||
page_size=16,
|
||||
tp_group=None,
|
||||
load_cache_event=None,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Unit tests for hybrid HiCache pool assembly."""
|
||||
|
||||
import unittest
|
||||
from functools import partial
|
||||
from queue import Queue
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
@@ -8,7 +9,13 @@ from unittest.mock import MagicMock, patch
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.configs.inkling import (
|
||||
InklingConvCacheParams,
|
||||
InklingConvStateShape,
|
||||
InklingStateDType,
|
||||
)
|
||||
from sglang.srt.mem_cache.base_prefix_cache import CacheRequestHandle, EvictParams
|
||||
from sglang.srt.mem_cache.device_pool_info import DevicePoolInfo, MambaStateBufferInfo
|
||||
from sglang.srt.mem_cache.hicache_storage import PoolHitPolicy, PoolName, PoolTransfer
|
||||
from sglang.srt.mem_cache.hybrid_cache import hybrid_pool_assembler
|
||||
from sglang.srt.mem_cache.hybrid_cache.host_pool_config import (
|
||||
@@ -34,13 +41,18 @@ from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import (
|
||||
build_host_pool_group,
|
||||
build_hybrid_swa_group,
|
||||
)
|
||||
from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool, HybridLinearKVPool
|
||||
from sglang.srt.mem_cache.memory_pool import (
|
||||
DSATokenToKVPool,
|
||||
HybridLinearKVPool,
|
||||
MambaPool,
|
||||
)
|
||||
from sglang.srt.mem_cache.pool_host import HostPoolGroup, PoolEntry
|
||||
from sglang.srt.mem_cache.pool_host import dsa as pool_host_dsa
|
||||
from sglang.srt.mem_cache.pool_host.host_pool_decl import (
|
||||
HostPoolDecl,
|
||||
make_kv_pool_decl,
|
||||
)
|
||||
from sglang.srt.mem_cache.pool_host.mamba import MambaPoolHost
|
||||
from sglang.srt.mem_cache.pool_host.unified import UnifiedPageEnvelopeHostPool
|
||||
from sglang.srt.mem_cache.unified_memory_pool import init_unified_swa_pools
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
@@ -169,7 +181,6 @@ class TestHybridStageLayerMappings(CustomTestCase):
|
||||
host_pool_decls=lambda: (),
|
||||
)
|
||||
req_pool = SimpleNamespace(
|
||||
mamba_map=global_maps.get("mamba", {}).copy(),
|
||||
mamba_pool=object(),
|
||||
)
|
||||
params = SimpleNamespace(
|
||||
@@ -195,7 +206,13 @@ class TestHybridStageLayerMappings(CustomTestCase):
|
||||
)
|
||||
|
||||
build_stack.assert_called_once()
|
||||
if "mamba" in local_maps:
|
||||
self.assertEqual(
|
||||
build_stack.call_args.kwargs["start_layer"], start_layer
|
||||
)
|
||||
for name, mapping in local_maps.items():
|
||||
if name == "mamba":
|
||||
continue
|
||||
self.assertEqual(
|
||||
build_stack.call_args.kwargs[f"{name}_layer_mapping"],
|
||||
mapping,
|
||||
@@ -205,7 +222,189 @@ class TestHybridStageLayerMappings(CustomTestCase):
|
||||
kvcache.full_attention_layer_id_mapping, global_maps["full"]
|
||||
)
|
||||
self.assertEqual(kvcache.layers_mapping, layers_mapping)
|
||||
self.assertEqual(req_pool.mamba_map, global_maps.get("mamba", {}))
|
||||
|
||||
|
||||
class TestMambaInfoRouting(CustomTestCase):
|
||||
def test_checkpoint_axis_routes_stage_layers_for_both_stacks(self):
|
||||
for tri_pool in (False, True):
|
||||
for start_layer in (0, 8):
|
||||
for local_ids in ((4, 2, 3), ()):
|
||||
with self.subTest(
|
||||
tri_pool=tri_pool, start=start_layer, layers=local_ids
|
||||
):
|
||||
|
||||
def make_pool(ids):
|
||||
return MambaPool(
|
||||
size=8,
|
||||
spec_state_size=0,
|
||||
cache_params=InklingConvCacheParams(
|
||||
shape=InklingConvStateShape(
|
||||
conv=[(2,)], temporal=(2,)
|
||||
),
|
||||
layers=list(ids),
|
||||
dtype=InklingStateDType(
|
||||
conv=torch.float32, temporal=torch.float32
|
||||
),
|
||||
),
|
||||
mamba_layer_ids=list(ids),
|
||||
device="cpu",
|
||||
)
|
||||
|
||||
mamba_pool = make_pool(
|
||||
tuple(layer + start_layer for layer in local_ids)
|
||||
)
|
||||
temporal = mamba_pool.mamba_cache.temporal
|
||||
temporal.copy_(
|
||||
torch.arange(temporal.numel()).reshape(temporal.shape)
|
||||
)
|
||||
# Buffer-only capacity may exceed the physical checkpoint slots.
|
||||
mamba_pool.host_capacity_tokens = 16 if start_layer else None
|
||||
drafts = (
|
||||
tuple(
|
||||
make_pool([start_layer + 5 + depth])
|
||||
for depth in range(2)
|
||||
)
|
||||
if local_ids
|
||||
else ()
|
||||
)
|
||||
captured = []
|
||||
original_provider = mamba_pool.get_device_pool_infos
|
||||
|
||||
def capture():
|
||||
result = original_provider()
|
||||
captured.append(result[0])
|
||||
return result
|
||||
|
||||
provider = MagicMock(side_effect=capture)
|
||||
kv = SimpleNamespace(layer_num=1, size=8)
|
||||
allocator = SimpleNamespace(alloc=MagicMock(), free=MagicMock())
|
||||
params = SimpleNamespace(
|
||||
page_size=1,
|
||||
mtp_draft_device_pools=(),
|
||||
mtp_draft_mamba_pools=drafts,
|
||||
token_to_kv_pool_allocator=SimpleNamespace(
|
||||
swa_attn_allocator=allocator
|
||||
),
|
||||
req_to_token_pool=SimpleNamespace(
|
||||
mamba_allocator=allocator
|
||||
),
|
||||
tp_cache_group=None,
|
||||
pp_cache_group=None,
|
||||
attn_cp_cache_group=None,
|
||||
attn_tp_cache_group=None,
|
||||
)
|
||||
memory = SimpleNamespace(
|
||||
hicache_size=0,
|
||||
hicache_ratio=2,
|
||||
hicache_mem_layout="page_first_direct",
|
||||
hicache_write_policy="write_through",
|
||||
hicache_io_backend="direct",
|
||||
hicache_host_memory_mode="cache",
|
||||
enable_unified_memory=False,
|
||||
)
|
||||
with (
|
||||
patch.object(mamba_pool, "get_device_pool_infos", provider),
|
||||
patch.object(
|
||||
hybrid_pool_assembler, "get_memory", return_value=memory
|
||||
),
|
||||
patch.object(
|
||||
hybrid_pool_assembler,
|
||||
"_get_allocator_type",
|
||||
return_value="default",
|
||||
),
|
||||
patch.object(
|
||||
hybrid_pool_assembler,
|
||||
"build_kv_host_pool",
|
||||
return_value=SimpleNamespace(
|
||||
layer_num=1,
|
||||
layout="page_first_direct",
|
||||
page_size=1,
|
||||
device="cpu",
|
||||
size=17,
|
||||
logical_size=17,
|
||||
can_use_write_back_jit=False,
|
||||
),
|
||||
),
|
||||
patch.object(
|
||||
hybrid_pool_assembler,
|
||||
"MambaPoolHost",
|
||||
partial(MambaPoolHost, pin_memory=False),
|
||||
),
|
||||
patch.object(
|
||||
hybrid_pool_assembler, "HybridCacheController"
|
||||
),
|
||||
):
|
||||
common = dict(
|
||||
params=params,
|
||||
mamba_pool=mamba_pool,
|
||||
full_layer_mapping={0: 0},
|
||||
start_layer=start_layer,
|
||||
load_cache_event=None,
|
||||
storage_backend=None,
|
||||
)
|
||||
if tri_pool:
|
||||
group, _ = (
|
||||
hybrid_pool_assembler.build_hybrid_mamba_swa_stack(
|
||||
**common,
|
||||
full_kv_pool=kv,
|
||||
swa_kv_pool=kv,
|
||||
swa_layer_mapping={1: 0},
|
||||
page_size=1,
|
||||
tp_group=None,
|
||||
)
|
||||
)
|
||||
else:
|
||||
group, _, _ = (
|
||||
hybrid_pool_assembler.build_hybrid_mamba_stack(
|
||||
**common,
|
||||
decls=(make_kv_pool_decl(kv),),
|
||||
use_mla=False,
|
||||
)
|
||||
)
|
||||
entry = group.entry_map[PoolName.MAMBA]
|
||||
host = entry.host_pool
|
||||
try:
|
||||
info = captured[0]
|
||||
self.assertIs(host.buffer_info, info.buffer_info)
|
||||
self.assertEqual(
|
||||
host.size, (16 if start_layer else 8) * 2 + 1
|
||||
)
|
||||
self.assertEqual(
|
||||
len(host.mtp_draft_buffer_infos), len(drafts)
|
||||
)
|
||||
for depth, draft in enumerate(drafts):
|
||||
self.assertIs(
|
||||
host.mtp_draft_buffer_infos[depth].temporal,
|
||||
draft.mamba_cache.temporal,
|
||||
)
|
||||
self.assertEqual(
|
||||
entry.layer_mapper(5 + depth),
|
||||
len(local_ids) + depth,
|
||||
)
|
||||
self.assertEqual(
|
||||
host.temporal_device_layers[
|
||||
len(local_ids) + depth
|
||||
].data_ptr(),
|
||||
draft.mamba_cache.temporal[0].data_ptr(),
|
||||
)
|
||||
self.assertEqual(provider.call_count, 1)
|
||||
self.assertEqual(
|
||||
info.layer_ids,
|
||||
tuple(layer + start_layer for layer in local_ids),
|
||||
)
|
||||
self.assertIs(entry.device_alloc_fn, allocator.alloc)
|
||||
self.assertIs(entry.device_free_fn, allocator.free)
|
||||
for layer in range(-1, 5):
|
||||
slot = entry.layer_mapper(layer)
|
||||
if layer not in local_ids:
|
||||
self.assertIsNone(slot)
|
||||
else:
|
||||
torch.testing.assert_close(
|
||||
host.buffer_info.temporal[slot],
|
||||
temporal[local_ids.index(layer)],
|
||||
)
|
||||
finally:
|
||||
host.destroy()
|
||||
|
||||
|
||||
class TestDraftSidecarPoolDispatch(CustomTestCase):
|
||||
@@ -748,10 +947,17 @@ class TestHybridMambaDeclaredIndexer(CustomTestCase):
|
||||
|
||||
kv_pool = _dsa_pool_stub(layer_num=3)
|
||||
kv_pool.skip_topk_layers = [False, True, False] # device layer 1: no buffer
|
||||
mamba_pool = SimpleNamespace(layer_num=2, num_mamba_layers=2, size=8)
|
||||
mamba_info = DevicePoolInfo(
|
||||
pool_name=PoolName.MAMBA,
|
||||
indices_from_pool=PoolName.MAMBA,
|
||||
layer_ids=(1, 3),
|
||||
buffer_info=MambaStateBufferInfo(
|
||||
temporal=torch.zeros(2, 9, 4), conv=(torch.zeros(2, 9, 4),)
|
||||
),
|
||||
)
|
||||
mamba_pool = SimpleNamespace(get_device_pool_infos=lambda: (mamba_info,))
|
||||
# transfer layers 0,2,4 are DSA (device 0,1,2), 1,3 are Mamba
|
||||
full_mapping = {0: 0, 2: 1, 4: 2}
|
||||
mamba_mapping = {1: 0, 3: 1}
|
||||
params = SimpleNamespace(
|
||||
page_size=64,
|
||||
mtp_draft_device_pools=(),
|
||||
@@ -824,7 +1030,7 @@ class TestHybridMambaDeclaredIndexer(CustomTestCase):
|
||||
decls=kv_pool.host_pool_decls(),
|
||||
mamba_pool=mamba_pool,
|
||||
full_layer_mapping=full_mapping,
|
||||
mamba_layer_mapping=mamba_mapping,
|
||||
start_layer=0,
|
||||
load_cache_event=None,
|
||||
storage_backend=None,
|
||||
use_mla=True,
|
||||
|
||||
@@ -7,11 +7,13 @@ from unittest.mock import MagicMock, Mock, patch
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.dcp.layout import maybe_dcp_kernel_indices
|
||||
from sglang.srt.mem_cache.device_pool_info import DevicePoolInfo, MambaStateBufferInfo
|
||||
from sglang.srt.mem_cache.hicache_storage import PoolName
|
||||
from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import _split_hicache_size
|
||||
from sglang.srt.mem_cache.l2_transfer import L2Transfer, L2TransferEngine
|
||||
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(est_time=15, stage="extra-a", runner_config="1-gpu-small")
|
||||
|
||||
@@ -219,7 +221,7 @@ class TestDirectBackendTranslation(unittest.TestCase):
|
||||
torch.testing.assert_close(result, torch.tensor([6, 9]))
|
||||
|
||||
|
||||
class TestTriPoolAssembly(unittest.TestCase):
|
||||
class TestTriPoolAssembly(CustomTestCase):
|
||||
def test_swa_allocation_and_rollback_match_pool_id_ownership(self):
|
||||
from sglang.srt.mem_cache.allocator.unified_sub_pool import (
|
||||
MultiEndedAllocator,
|
||||
@@ -258,10 +260,22 @@ class TestTriPoolAssembly(unittest.TestCase):
|
||||
params=params,
|
||||
full_kv_pool=object(),
|
||||
swa_kv_pool=object(),
|
||||
mamba_pool=SimpleNamespace(num_mamba_layers=1),
|
||||
mamba_pool=SimpleNamespace(
|
||||
get_device_pool_infos=lambda: (
|
||||
DevicePoolInfo(
|
||||
pool_name=PoolName.MAMBA,
|
||||
indices_from_pool=PoolName.MAMBA,
|
||||
layer_ids=(2,),
|
||||
buffer_info=MambaStateBufferInfo(
|
||||
temporal=torch.zeros(1, 2, 1),
|
||||
conv=(torch.zeros(1, 2, 1),),
|
||||
),
|
||||
),
|
||||
)
|
||||
),
|
||||
full_layer_mapping={0: 0},
|
||||
swa_layer_mapping={1: 0},
|
||||
mamba_layer_mapping={2: 0},
|
||||
start_layer=0,
|
||||
page_size=1,
|
||||
tp_group=None,
|
||||
load_cache_event=None,
|
||||
|
||||
Reference in New Issue
Block a user