Files
sglang/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py
T
78b382b1ad Support unified memory decode host pools (#39478)
Co-authored-by: yhzhuang <yhzhuang@fb.com>
Co-authored-by: Lianmin Zheng <lianminzheng@gmail.com>
Co-authored-by: Yonghao Zhuang <yhzhuang@users.noreply.github.com>
Co-authored-by: Cheng Wan <cheng.wan@radixark.ai>
2026-09-24 11:02:15 -07:00

334 lines
12 KiB
Python

"""Unit tests for hybrid HiCache pool assembly."""
import unittest
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import torch
from sglang.srt.mem_cache.base_prefix_cache import EvictParams
from sglang.srt.mem_cache.hybrid_cache import hybrid_pool_assembler
from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import (
_evict_mamba_for_device_alloc,
_evict_swa_for_device_alloc,
_MambaStrategy,
_MambaSwaStrategy,
_require_single_row_dsv4_swa_pages,
_split_hicache_size,
_SwaStrategy,
build_full_draft_pools,
build_hybrid_swa_group,
)
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool
from sglang.srt.mem_cache.pool_host.unified import UnifiedPageEnvelopeHostPool
from sglang.srt.mem_cache.unified_memory_pool import init_unified_swa_pools
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=11, suite="base-a-test-cpu")
class TestDeepSeekV4SWAPageLayout(CustomTestCase):
def test_split_physical_rows_are_rejected_for_hicache_consumers(self):
with self.assertRaisesRegex(ValueError, "direct SWA KV layout"):
_require_single_row_dsv4_swa_pages(
logical_page_size=256,
physical_page_size=64,
consumer="test consumer",
)
def test_matching_page_geometry_is_supported(self):
_require_single_row_dsv4_swa_pages(
logical_page_size=256,
physical_page_size=256,
consumer="test consumer",
)
class _Pool:
def __init__(self, kv_bytes):
self._kv_bytes = kv_bytes
def get_kv_size_bytes(self):
return self._kv_bytes
class TestDeviceAllocEviction(CustomTestCase):
def test_swa_evicts_only_allocation_shortfall(self):
cache = MagicMock()
cache.token_to_kv_pool_allocator.swa_available_size.return_value = 8
_evict_swa_for_device_alloc(cache, required_size=10)
cache.evict_for_alloc.assert_called_once_with(EvictParams(swa_num_tokens=2))
cache.evict.assert_not_called()
def test_mamba_evicts_only_allocation_shortfall(self):
cache = MagicMock()
allocator = cache.req_to_token_pool.mamba_allocator
allocator.schedulable_available_size.return_value = 8
_evict_mamba_for_device_alloc(cache, required_size=10)
cache.evict_for_alloc.assert_called_once_with(EvictParams(mamba_num=2))
cache.evict.assert_not_called()
def test_sufficient_capacity_skips_eviction(self):
cache = MagicMock()
cache.token_to_kv_pool_allocator.swa_available_size.return_value = 10
cache.req_to_token_pool.mamba_allocator.schedulable_available_size.return_value = 10
_evict_swa_for_device_alloc(cache, required_size=10)
_evict_mamba_for_device_alloc(cache, required_size=10)
cache.evict_for_alloc.assert_not_called()
cache.evict.assert_not_called()
class TestSplitHicacheSize(CustomTestCase):
def test_splits_total_budget_by_device_bytes(self):
# scalar and (k, v) tuple return shapes both supported
shares = _split_hicache_size(
100, (_Pool(75 * 10**9), _Pool((15 * 10**9, 10 * 10**9)))
)
self.assertEqual(shares, (75.0, 25.0)) # proportional to device KV bytes
self.assertEqual(sum(shares), 100) # total budget preserved, not doubled
def test_splits_total_budget_by_device_bytes_three_pools(self):
# scalar and (k, v) tuple return shapes both supported
shares = _split_hicache_size(
100, (_Pool(55 * 10**9), _Pool((15 * 10**9, 10 * 10**9)), _Pool(20 * 10**9))
)
self.assertEqual(shares, (55.0, 25.0, 20.0)) # proportional to device KV bytes
self.assertEqual(sum(shares), 100) # total budget preserved, not doubled
class TestHybridStageLayerMappings(CustomTestCase):
def test_strategies_pass_stage_local_maps_without_changing_device_maps(self):
"""Later pipeline stages must transfer every layer before signaling completion."""
cases = [
(
_MambaStrategy,
"build_hybrid_mamba_stack",
{"full": {3: 0}, "mamba": {0: 2, 1: 0, 2: 1}},
),
(
_SwaStrategy,
"build_hybrid_swa_stack",
{"full": {3: 0}, "swa": {0: 2, 1: 0, 2: 1}},
),
(
_MambaSwaStrategy,
"build_hybrid_mamba_swa_stack",
{"full": {3: 0}, "swa": {0: 2, 1: 0, 2: 1}, "mamba": {0: 1, 3: 0}},
),
]
for strategy_cls, builder_name, local_maps in cases:
for start_layer in (0, 4):
with self.subTest(strategy=strategy_cls.__name__, start=start_layer):
global_maps = {
name: {
layer + start_layer: index
for layer, index in mapping.items()
}
for name, mapping in local_maps.items()
}
layers_mapping = {
layer: (index, name == "swa")
for name in ("full", "swa")
for layer, index in global_maps.get(name, {}).items()
}
kvcache = SimpleNamespace(
start_layer=start_layer,
full_attention_layer_id_mapping=global_maps["full"].copy(),
layers_mapping=layers_mapping.copy(),
full_kv_pool=object(),
swa_kv_pool=object(),
use_mla=False,
)
req_pool = SimpleNamespace(
mamba_map=global_maps.get("mamba", {}).copy(),
mamba_pool=object(),
)
params = SimpleNamespace(
req_to_token_pool=req_pool,
tp_cache_group=None,
pp_cache_group=None,
)
with patch.object(
hybrid_pool_assembler,
builder_name,
return_value=(
MagicMock(),
SimpleNamespace(transfer_layer_id_max=4),
),
) as build_stack:
result = strategy_cls().build(
cache=SimpleNamespace(page_size=1),
kvcache=kvcache,
params=params,
server_args=None,
load_cache_event=None,
)
build_stack.assert_called_once()
for name, mapping in local_maps.items():
self.assertEqual(
build_stack.call_args.kwargs[f"{name}_layer_mapping"],
mapping,
)
self.assertEqual(result.cache_controller.transfer_layer_id_max, 4)
self.assertEqual(
kvcache.full_attention_layer_id_mapping, global_maps["full"]
)
self.assertEqual(kvcache.layers_mapping, layers_mapping)
self.assertEqual(req_pool.mamba_map, global_maps.get("mamba", {}))
class TestDraftSidecarPoolDispatch(CustomTestCase):
def test_full_builder_unwraps_empty_hybrid_linear_pool(self):
draft_kv_pool = object.__new__(HybridLinearKVPool)
draft_kv_pool.full_kv_pool = SimpleNamespace(layer_num=0)
specs, entries = build_full_draft_pools(
draft_kv_pool=draft_kv_pool,
tree_cache=None,
)
self.assertEqual(specs, [])
self.assertEqual(entries, [])
def test_full_builder_sizes_sidecar_for_anchor_logical_space(self):
draft_kv_pool = SimpleNamespace(layer_num=1, size=800)
draft_host_pool = SimpleNamespace(layer_num=1)
tree_cache = SimpleNamespace(
cache_controller=SimpleNamespace(
mem_pool_host=SimpleNamespace(size=100, logical_size=800),
page_size=512,
)
)
# The layout comes from the published configuration.
from sglang.srt.runtime_context import publish, reset_context
from sglang.srt.server_args import ServerArgs
server_args = ServerArgs(model_path="dummy", hicache_mem_layout="page_first")
publish(server_args, role="scheduler")
self.addCleanup(reset_context)
with (
patch(
"sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler."
"_build_mha_mla_host_pool",
return_value=draft_host_pool,
) as build_host_pool,
patch(
"sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler."
"_get_allocator_type",
return_value="default",
),
):
specs, entries = build_full_draft_pools(
draft_kv_pool=draft_kv_pool,
tree_cache=tree_cache,
)
self.assertEqual(build_host_pool.call_args.kwargs["host_to_device_ratio"], 1.0)
self.assertEqual(len(specs), 1)
self.assertIs(entries[0].host_pool, draft_host_pool)
_ASSEMBLER = "sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler."
class TestTransferLayerSpan(CustomTestCase):
"""``transfer_layer_id_max`` must span global layer ids, not count the mapped ones.
A hybrid model with an uncached layer type keys its mappings non-contiguously,
and the per-layer transfer loop then never reaches the high layer ids.
"""
def test_pool_entries_span_the_highest_global_layer_id(self):
# Global ids 0/2/4/6 with holes between them, the shape NemotronH's
# cache-ineligible MLP layers produce: 4 mapped layers spanning 7 ids.
full_layer_mapping = {0: 0, 6: 1}
swa_layer_mapping = {2: 0, 4: 1}
with (
patch(_ASSEMBLER + "build_kv_host_pool"),
patch(_ASSEMBLER + "HostPoolGroup"),
patch(_ASSEMBLER + "build_pool_entry") as build_pool_entry,
):
build_hybrid_swa_group(
page_size=64,
full_kv_pool=MagicMock(),
swa_kv_pool=MagicMock(),
full_layer_mapping=full_layer_mapping,
swa_layer_mapping=swa_layer_mapping,
use_mla=False,
)
self.assertEqual(
[
c.kwargs["transfer_layer_id_max"]
for c in build_pool_entry.call_args_list
],
[7, 7],
)
def _build_unified_swa_pool(page_size: int = 4):
return init_unified_swa_pools(
device="cpu",
kv_cache_dtype=torch.float16,
head_num=2,
head_dim=4,
v_head_dim=4,
swa_head_num=2,
swa_head_dim=4,
swa_v_head_dim=4,
page_size=page_size,
start_layer=0,
end_layer=4,
swa_attention_layer_ids=[3],
full_attention_layer_ids=[0, 1, 2],
total_bytes=4096,
enable_memory_saver=False,
need_sort=False,
lazy_compaction=True,
)
def _build_unified_host_pair(bundle):
pool = bundle.token_to_kv_pool
return UnifiedPageEnvelopeHostPool.build_hybrid_swa_pool_pair(
device_pools=(pool.full_kv_pool, pool.swa_kv_pool),
host_to_device_ratio=1.0,
host_size=0,
page_size=pool.page_size,
layout="layer_first",
pin_memory=False,
allocator_type="default",
)
class TestUnifiedPageEnvelopeHostPool(CustomTestCase):
def test_shared_arena_can_reuse_bytes_across_sides(self):
page_size = 4
full_pool, swa_pool = _build_unified_host_pair(
_build_unified_swa_pool(page_size)
)
self.addCleanup(full_pool.destroy)
self.addCleanup(swa_pool.destroy)
full_indices = full_pool.alloc(full_pool.available_size())
self.assertIsNotNone(full_indices)
self.assertIsNotNone(swa_pool.alloc(swa_pool.available_size()))
self.assertIsNone(swa_pool.alloc(page_size))
full_pool.free(full_indices[page_size:])
self.assertIsNotNone(swa_pool.alloc(page_size))
if __name__ == "__main__":
unittest.main()