1025 lines
40 KiB
Python
1025 lines
40 KiB
Python
"""Unit tests for hybrid HiCache pool assembly."""
|
|
|
|
import unittest
|
|
from types import SimpleNamespace
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import msgspec
|
|
import torch
|
|
|
|
from sglang.srt.mem_cache.base_prefix_cache import EvictParams
|
|
from sglang.srt.mem_cache.hicache_storage import PoolName
|
|
from sglang.srt.mem_cache.hybrid_cache import hybrid_pool_assembler
|
|
from sglang.srt.mem_cache.hybrid_cache.host_pool_config import (
|
|
prepare_host_pool_config,
|
|
)
|
|
from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import (
|
|
StackBuildResult,
|
|
_check_declared_pools_present,
|
|
_DsaStrategy,
|
|
_evict_mamba_for_device_alloc,
|
|
_evict_swa_for_device_alloc,
|
|
_MambaStrategy,
|
|
_MambaSwaStrategy,
|
|
_require_single_row_dsv4_swa_pages,
|
|
_split_hicache_size,
|
|
_SwaStrategy,
|
|
build_full_draft_pools,
|
|
build_host_pool_group,
|
|
build_hybrid_swa_group,
|
|
)
|
|
from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool, HybridLinearKVPool
|
|
from sglang.srt.mem_cache.pool_host import dsa as pool_host_dsa
|
|
from sglang.srt.mem_cache.pool_host.host_pool_decl import (
|
|
HostPoolDecl,
|
|
make_kv_pool_decl,
|
|
)
|
|
from sglang.srt.mem_cache.pool_host.unified import UnifiedPageEnvelopeHostPool
|
|
from sglang.srt.mem_cache.unified_memory_pool import init_unified_swa_pools
|
|
from sglang.test.ci.ci_register import register_cpu_ci
|
|
from sglang.test.test_utils import CustomTestCase
|
|
|
|
register_cpu_ci(est_time=11, suite="base-a-test-cpu")
|
|
|
|
|
|
class TestDeepSeekV4SWAPageLayout(CustomTestCase):
|
|
def test_split_physical_rows_are_rejected_for_hicache_consumers(self):
|
|
with self.assertRaisesRegex(ValueError, "direct SWA KV layout"):
|
|
_require_single_row_dsv4_swa_pages(
|
|
logical_page_size=256,
|
|
physical_page_size=64,
|
|
consumer="test consumer",
|
|
)
|
|
|
|
def test_matching_page_geometry_is_supported(self):
|
|
_require_single_row_dsv4_swa_pages(
|
|
logical_page_size=256,
|
|
physical_page_size=256,
|
|
consumer="test consumer",
|
|
)
|
|
|
|
|
|
class _Pool:
|
|
def __init__(self, kv_bytes):
|
|
self._kv_bytes = kv_bytes
|
|
|
|
def get_kv_size_bytes(self):
|
|
return self._kv_bytes
|
|
|
|
|
|
class TestDeviceAllocEviction(CustomTestCase):
|
|
def test_swa_evicts_only_allocation_shortfall(self):
|
|
cache = MagicMock()
|
|
cache.token_to_kv_pool_allocator.swa_available_size.return_value = 8
|
|
|
|
_evict_swa_for_device_alloc(cache, required_size=10)
|
|
|
|
cache.evict_for_alloc.assert_called_once_with(EvictParams(swa_num_tokens=2))
|
|
cache.evict.assert_not_called()
|
|
|
|
def test_mamba_evicts_only_allocation_shortfall(self):
|
|
cache = MagicMock()
|
|
allocator = cache.req_to_token_pool.mamba_allocator
|
|
allocator.schedulable_available_size.return_value = 8
|
|
|
|
_evict_mamba_for_device_alloc(cache, required_size=10)
|
|
|
|
cache.evict_for_alloc.assert_called_once_with(EvictParams(mamba_num=2))
|
|
cache.evict.assert_not_called()
|
|
|
|
def test_sufficient_capacity_skips_eviction(self):
|
|
cache = MagicMock()
|
|
cache.token_to_kv_pool_allocator.swa_available_size.return_value = 10
|
|
cache.req_to_token_pool.mamba_allocator.schedulable_available_size.return_value = 10
|
|
|
|
_evict_swa_for_device_alloc(cache, required_size=10)
|
|
_evict_mamba_for_device_alloc(cache, required_size=10)
|
|
|
|
cache.evict_for_alloc.assert_not_called()
|
|
cache.evict.assert_not_called()
|
|
|
|
|
|
class TestSplitHicacheSize(CustomTestCase):
|
|
def test_splits_total_budget_by_device_bytes(self):
|
|
# scalar and (k, v) tuple return shapes both supported
|
|
shares = _split_hicache_size(
|
|
100, (_Pool(75 * 10**9), _Pool((15 * 10**9, 10 * 10**9)))
|
|
)
|
|
self.assertEqual(shares, (75.0, 25.0)) # proportional to device KV bytes
|
|
self.assertEqual(sum(shares), 100) # total budget preserved, not doubled
|
|
|
|
def test_splits_total_budget_by_device_bytes_three_pools(self):
|
|
# scalar and (k, v) tuple return shapes both supported
|
|
shares = _split_hicache_size(
|
|
100, (_Pool(55 * 10**9), _Pool((15 * 10**9, 10 * 10**9)), _Pool(20 * 10**9))
|
|
)
|
|
self.assertEqual(shares, (55.0, 25.0, 20.0)) # proportional to device KV bytes
|
|
self.assertEqual(sum(shares), 100) # total budget preserved, not doubled
|
|
|
|
|
|
class TestHybridStageLayerMappings(CustomTestCase):
|
|
def test_strategies_pass_stage_local_maps_without_changing_device_maps(self):
|
|
"""Later pipeline stages must transfer every layer before signaling completion."""
|
|
cases = [
|
|
(
|
|
_MambaStrategy,
|
|
"build_hybrid_mamba_stack",
|
|
{"full": {3: 0}, "mamba": {0: 2, 1: 0, 2: 1}},
|
|
),
|
|
(
|
|
_SwaStrategy,
|
|
"build_hybrid_swa_stack",
|
|
{"full": {3: 0}, "swa": {0: 2, 1: 0, 2: 1}},
|
|
),
|
|
(
|
|
_MambaSwaStrategy,
|
|
"build_hybrid_mamba_swa_stack",
|
|
{"full": {3: 0}, "swa": {0: 2, 1: 0, 2: 1}, "mamba": {0: 1, 3: 0}},
|
|
),
|
|
]
|
|
for strategy_cls, builder_name, local_maps in cases:
|
|
for start_layer in (0, 4):
|
|
with self.subTest(strategy=strategy_cls.__name__, start=start_layer):
|
|
global_maps = {
|
|
name: {
|
|
layer + start_layer: index
|
|
for layer, index in mapping.items()
|
|
}
|
|
for name, mapping in local_maps.items()
|
|
}
|
|
layers_mapping = {
|
|
layer: (index, name == "swa")
|
|
for name in ("full", "swa")
|
|
for layer, index in global_maps.get(name, {}).items()
|
|
}
|
|
kvcache = SimpleNamespace(
|
|
start_layer=start_layer,
|
|
full_attention_layer_id_mapping=global_maps["full"].copy(),
|
|
layers_mapping=layers_mapping.copy(),
|
|
full_kv_pool=object(),
|
|
swa_kv_pool=object(),
|
|
use_mla=False,
|
|
host_pool_decls=lambda: (),
|
|
)
|
|
req_pool = SimpleNamespace(
|
|
mamba_map=global_maps.get("mamba", {}).copy(),
|
|
mamba_pool=object(),
|
|
)
|
|
params = SimpleNamespace(
|
|
req_to_token_pool=req_pool,
|
|
tp_cache_group=None,
|
|
pp_cache_group=None,
|
|
)
|
|
with patch.object(
|
|
hybrid_pool_assembler,
|
|
builder_name,
|
|
return_value=(
|
|
MagicMock(),
|
|
SimpleNamespace(transfer_layer_id_max=4),
|
|
),
|
|
) as build_stack:
|
|
result = strategy_cls().build(
|
|
cache=SimpleNamespace(page_size=1),
|
|
kvcache=kvcache,
|
|
params=params,
|
|
server_args=None,
|
|
load_cache_event=None,
|
|
)
|
|
|
|
build_stack.assert_called_once()
|
|
for name, mapping in local_maps.items():
|
|
self.assertEqual(
|
|
build_stack.call_args.kwargs[f"{name}_layer_mapping"],
|
|
mapping,
|
|
)
|
|
self.assertEqual(result.cache_controller.transfer_layer_id_max, 4)
|
|
self.assertEqual(
|
|
kvcache.full_attention_layer_id_mapping, global_maps["full"]
|
|
)
|
|
self.assertEqual(kvcache.layers_mapping, layers_mapping)
|
|
self.assertEqual(req_pool.mamba_map, global_maps.get("mamba", {}))
|
|
|
|
|
|
class TestDraftSidecarPoolDispatch(CustomTestCase):
|
|
def test_full_builder_unwraps_empty_hybrid_linear_pool(self):
|
|
draft_kv_pool = object.__new__(HybridLinearKVPool)
|
|
full = SimpleNamespace(layer_num=0)
|
|
full.host_pool_decls = lambda: (make_kv_pool_decl(full),)
|
|
draft_kv_pool.full_kv_pool = full
|
|
|
|
specs, entries = build_full_draft_pools(
|
|
draft_kv_pool=draft_kv_pool,
|
|
tree_cache=None,
|
|
)
|
|
|
|
self.assertEqual(specs, [])
|
|
self.assertEqual(entries, [])
|
|
|
|
def test_full_builder_sizes_sidecar_for_anchor_logical_space(self):
|
|
draft_kv_pool = SimpleNamespace(layer_num=1, size=800)
|
|
draft_kv_pool.host_pool_decls = lambda: (make_kv_pool_decl(draft_kv_pool),)
|
|
draft_host_pool = SimpleNamespace(layer_num=1)
|
|
tree_cache = SimpleNamespace(
|
|
cache_controller=SimpleNamespace(
|
|
mem_pool_host=SimpleNamespace(size=100, logical_size=800),
|
|
page_size=512,
|
|
)
|
|
)
|
|
# The layout comes from the published configuration.
|
|
from sglang.srt.runtime_context import publish, reset_context
|
|
from sglang.srt.server_args import ServerArgs
|
|
|
|
server_args = ServerArgs(model_path="dummy", hicache_mem_layout="page_first")
|
|
publish(server_args, role="scheduler")
|
|
self.addCleanup(reset_context)
|
|
|
|
with (
|
|
patch(
|
|
"sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler."
|
|
"_build_mha_mla_host_pool",
|
|
return_value=draft_host_pool,
|
|
) as build_host_pool,
|
|
patch(
|
|
"sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler."
|
|
"_get_allocator_type",
|
|
return_value="default",
|
|
),
|
|
):
|
|
specs, entries = build_full_draft_pools(
|
|
draft_kv_pool=draft_kv_pool,
|
|
tree_cache=tree_cache,
|
|
)
|
|
|
|
self.assertEqual(build_host_pool.call_args.kwargs["host_to_device_ratio"], 1.0)
|
|
self.assertEqual(len(specs), 1)
|
|
self.assertIs(entries[0].host_pool, draft_host_pool)
|
|
|
|
def test_full_builder_registers_separate_dsa_draft_indexer(self):
|
|
"""The separate-draft DSA branch must build its indexer mirror from a
|
|
DRAFT_INDEXER decl. A constructor change that skips this call site
|
|
breaks only here, not on the target path."""
|
|
draft_kv_pool = object.__new__(DSATokenToKVPool)
|
|
draft_kv_pool.layer_num = 1
|
|
draft_kv_pool.size = 800
|
|
draft_kv_pool.index_head_dim = 128
|
|
draft_kv_pool.skip_topk_layers = [False]
|
|
draft_kv_pool.index_key_cache = SimpleNamespace(buffer=[object()])
|
|
draft_host_pool = SimpleNamespace(layer_num=1)
|
|
tree_cache = SimpleNamespace(
|
|
cache_controller=SimpleNamespace(
|
|
mem_pool_host=SimpleNamespace(size=100, logical_size=800),
|
|
page_size=512,
|
|
)
|
|
)
|
|
from sglang.srt.runtime_context import publish, reset_context
|
|
from sglang.srt.server_args import ServerArgs
|
|
|
|
publish(
|
|
ServerArgs(model_path="dummy", hicache_mem_layout="page_first"),
|
|
role="scheduler",
|
|
)
|
|
self.addCleanup(reset_context)
|
|
|
|
seen = {}
|
|
|
|
def fake_indexer_host(
|
|
decl, anchor_host, *, allocator_type, packed_draft_device_pools=()
|
|
):
|
|
self.assertIsInstance(decl, HostPoolDecl)
|
|
self.assertIs(decl.device_pool, draft_kv_pool)
|
|
self.assertIs(anchor_host, draft_host_pool)
|
|
self.assertEqual(packed_draft_device_pools, ())
|
|
seen["decl"] = decl
|
|
return SimpleNamespace(layer_num=1)
|
|
|
|
with (
|
|
patch.object(
|
|
hybrid_pool_assembler,
|
|
"_build_mha_mla_host_pool",
|
|
return_value=draft_host_pool,
|
|
),
|
|
patch.object(
|
|
hybrid_pool_assembler, "_get_allocator_type", return_value="default"
|
|
),
|
|
patch.object(pool_host_dsa, "DSAIndexerPoolHost", fake_indexer_host),
|
|
):
|
|
specs, entries = build_full_draft_pools(
|
|
draft_kv_pool=draft_kv_pool,
|
|
tree_cache=tree_cache,
|
|
)
|
|
|
|
self.assertEqual(seen["decl"].pool_name, PoolName.DRAFT_INDEXER)
|
|
self.assertEqual(seen["decl"].indices_from_pool, PoolName.KV)
|
|
self.assertEqual(specs[1], seen["decl"].sidecar_spec())
|
|
self.assertEqual(entries[1].name, PoolName.DRAFT_INDEXER)
|
|
|
|
|
|
def _dsa_pool_stub(*, layer_num: int, size: int = 4096, shard: tuple | None = None):
|
|
"""DSATokenToKVPool shape without CUDA. ``shard=(rank, size)`` marks the pool
|
|
layer-sharded with rank owning a contiguous local layer range."""
|
|
pool = object.__new__(DSATokenToKVPool)
|
|
pool.layer_num = layer_num
|
|
pool.size = size
|
|
pool.start_layer = 0
|
|
pool.end_layer = layer_num - 1
|
|
pool.store_dtype = torch.bfloat16
|
|
pool.kv_lora_rank = 512
|
|
pool.qk_rope_head_dim = 64
|
|
pool.kv_cache_dim = 576
|
|
pool.index_head_dim = 128
|
|
pool.skip_topk_layers = [False] * layer_num
|
|
pool.index_key_cache = SimpleNamespace(buffer=[object()] * layer_num)
|
|
pool.layer_shard_enabled = shard is not None
|
|
if shard is not None:
|
|
from sglang.srt.layers.cp.utils import get_layer_shard_range
|
|
|
|
rank, shard_size = shard
|
|
pool.layer_shard_size = shard_size
|
|
pool._owned_local_layer_range = lambda: get_layer_shard_range(
|
|
rank, shard_size, layer_num
|
|
)
|
|
return pool
|
|
|
|
|
|
def _kv_pool_stub(*, layer_num: int, size: int = 4096):
|
|
"""A plain MLA KV pool without CUDA: declares KV only."""
|
|
from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool
|
|
|
|
pool = object.__new__(MLATokenToKVPool)
|
|
pool.layer_num = layer_num
|
|
pool.size = size
|
|
pool.start_layer = 0
|
|
pool.end_layer = layer_num - 1
|
|
pool.store_dtype = torch.bfloat16
|
|
pool.kv_lora_rank = 512
|
|
pool.qk_rope_head_dim = 64
|
|
pool.kv_cache_dim = 576
|
|
pool.layer_shard_enabled = False
|
|
return pool
|
|
|
|
|
|
def _mirror_shape(host):
|
|
shape = {
|
|
"size": host.size,
|
|
"page_num": host.page_num,
|
|
"layer_num": host.layer_num,
|
|
"size_per_token": host.size_per_token,
|
|
"layout": host.layout,
|
|
}
|
|
if isinstance(host, pool_host_dsa.DSAIndexerPoolHost):
|
|
shape["indexer_page_stride_size"] = host.indexer_page_stride_size
|
|
shape["indexer_layout_dim"] = host.indexer_layout_dim
|
|
return shape
|
|
|
|
|
|
def _entry_shape(group, transfer_layer_num):
|
|
"""Everything the controller reads from a HostPoolGroup, in comparable form."""
|
|
return [
|
|
(
|
|
entry.name,
|
|
entry.is_primary_index_anchor,
|
|
id(entry.device_pool),
|
|
tuple(id(p) for p in entry.packed_draft_device_pools),
|
|
tuple(entry.layer_mapper(i) for i in range(-1, transfer_layer_num + 2)),
|
|
_mirror_shape(entry.host_pool),
|
|
)
|
|
for entry in group.entries
|
|
]
|
|
|
|
|
|
def _target_params(drafts=(), page_size=64):
|
|
return SimpleNamespace(
|
|
page_size=page_size,
|
|
mtp_draft_device_pools=tuple(drafts),
|
|
token_to_kv_pool_allocator=None,
|
|
tp_cache_group=None,
|
|
attn_cp_cache_group=None,
|
|
attn_tp_cache_group=None,
|
|
pp_cache_group=None,
|
|
)
|
|
|
|
|
|
# Mirror geometry for the 4096-token stubs at page 64 and host ratio 2:
|
|
# 8256 host tokens (2 x 4096 plus one reserve page) = 129 pages. The MLA row is
|
|
# kv_cache_dim 576 x bf16 = 1152 B per layer. The DSA index row is 132 B per
|
|
# token, 8448 B per page, per layer.
|
|
_HOST_SIZE, _HOST_PAGES, _KV_ROW, _INDEX_ROW, _INDEX_PAGE = 8256, 129, 1152, 132, 8448
|
|
|
|
|
|
def _kv_shape(layers):
|
|
return {
|
|
"size": _HOST_SIZE,
|
|
"page_num": _HOST_PAGES,
|
|
"layer_num": layers,
|
|
"size_per_token": _KV_ROW * layers,
|
|
"layout": "page_first",
|
|
}
|
|
|
|
|
|
def _indexer_shape(layers):
|
|
return {
|
|
**_kv_shape(layers),
|
|
"size_per_token": _INDEX_ROW * layers,
|
|
"indexer_page_stride_size": _INDEX_PAGE,
|
|
"indexer_layout_dim": _INDEX_PAGE * layers,
|
|
}
|
|
|
|
|
|
class TestDeclaredStackStructure(CustomTestCase):
|
|
"""Everything the controller reads from a declared target stack, pinned
|
|
against the pre-declaration DSA and plain-KV assembly it replaced: entry
|
|
order, anchor, device owners, layer mapper over every transfer layer,
|
|
packed drafts, mirror geometry, controller arguments, sidecars."""
|
|
|
|
def _run(self, *, pool, drafts, full_layer_mapping, page_size=64):
|
|
from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost
|
|
|
|
real_indexer_host = pool_host_dsa.DSAIndexerPoolHost
|
|
|
|
def dummy_kv_host(**kwargs):
|
|
return MLATokenToKVPoolHost(
|
|
kwargs["kv_pool"],
|
|
host_to_device_ratio=2,
|
|
host_size=0,
|
|
page_size=kwargs["page_size"],
|
|
layout="page_first",
|
|
pin_memory=False,
|
|
is_dummy=True,
|
|
override_kv_cache_dim=kwargs["kv_pool"].kv_cache_dim,
|
|
mtp_draft_device_pools=kwargs["mtp_draft_device_pools"],
|
|
)
|
|
|
|
def dummy_indexer_host(
|
|
decl, anchor_host, *, allocator_type, packed_draft_device_pools=()
|
|
):
|
|
return real_indexer_host(
|
|
decl,
|
|
anchor_host,
|
|
packed_draft_device_pools=packed_draft_device_pools,
|
|
allocator_type=allocator_type,
|
|
pin_memory=False,
|
|
is_dummy=True,
|
|
)
|
|
|
|
with (
|
|
patch.object(hybrid_pool_assembler, "build_kv_host_pool", dummy_kv_host),
|
|
patch.object(pool_host_dsa, "DSAIndexerPoolHost", dummy_indexer_host),
|
|
patch.object(
|
|
hybrid_pool_assembler, "_get_allocator_type", return_value="default"
|
|
),
|
|
):
|
|
config = prepare_host_pool_config(
|
|
decls=pool.host_pool_decls(),
|
|
full_layer_mapping=dict(full_layer_mapping),
|
|
transfer_layer_id_max=max(full_layer_mapping) + 1,
|
|
transfer_page_size=page_size,
|
|
packed_draft_device_pools=drafts,
|
|
)
|
|
group = build_host_pool_group(config=config)
|
|
return SimpleNamespace(
|
|
host_pool_group=group,
|
|
sidecars=[
|
|
c.decl.sidecar_spec() for c in config.pools if not c.decl.is_primary
|
|
],
|
|
)
|
|
|
|
def test_dsa_target(self):
|
|
pool = _dsa_pool_stub(layer_num=3)
|
|
stack = self._run(pool=pool, drafts=(), full_layer_mapping={0: 0, 1: 1, 2: 2})
|
|
mapper = (None, 0, 1, 2, None, None)
|
|
self.assertEqual(
|
|
_entry_shape(stack.host_pool_group, 3),
|
|
[
|
|
(PoolName.KV, True, id(pool), (), mapper, _kv_shape(3)),
|
|
(PoolName.INDEXER, False, id(pool), (), mapper, _indexer_shape(3)),
|
|
],
|
|
)
|
|
self.assertEqual(
|
|
stack.sidecars,
|
|
[pool_host_dsa.make_dsa_indexer_pool_decl(pool).sidecar_spec()],
|
|
)
|
|
|
|
def test_group_uses_prepared_page_size(self):
|
|
pool = _dsa_pool_stub(layer_num=1)
|
|
stack = self._run(
|
|
pool=pool, drafts=(), full_layer_mapping={0: 0}, page_size=128
|
|
)
|
|
for entry in stack.host_pool_group.entries:
|
|
self.assertEqual(entry.host_pool.page_size, 128)
|
|
indexer = stack.host_pool_group.get_pool(PoolName.INDEXER)
|
|
self.assertEqual(indexer.indexer_page_stride_size, 128 * _INDEX_ROW)
|
|
|
|
def test_dsa_target_with_packed_draft(self):
|
|
pool = _dsa_pool_stub(layer_num=3)
|
|
draft = _dsa_pool_stub(layer_num=1)
|
|
stack = self._run(
|
|
pool=pool, drafts=(draft,), full_layer_mapping={0: 0, 1: 1, 2: 2}
|
|
)
|
|
# transfer layer 3 is the draft's tail layer, device layer 3 = target_layer_num + depth
|
|
mapper = (None, 0, 1, 2, 3, None, None)
|
|
self.assertEqual(
|
|
_entry_shape(stack.host_pool_group, 4),
|
|
[
|
|
(PoolName.KV, True, id(pool), (id(draft),), mapper, _kv_shape(4)),
|
|
(
|
|
PoolName.INDEXER,
|
|
False,
|
|
id(pool),
|
|
(id(draft),),
|
|
mapper,
|
|
_indexer_shape(4),
|
|
),
|
|
],
|
|
)
|
|
|
|
def test_strategy_keeps_target_controller_range_with_packed_draft(self):
|
|
pool = _dsa_pool_stub(layer_num=3)
|
|
draft = _dsa_pool_stub(layer_num=1)
|
|
stack = self._run(
|
|
pool=pool, drafts=(draft,), full_layer_mapping={0: 0, 1: 1, 2: 2}
|
|
)
|
|
with (
|
|
patch.object(
|
|
hybrid_pool_assembler,
|
|
"build_host_pool_group",
|
|
return_value=stack.host_pool_group,
|
|
) as build,
|
|
patch.object(hybrid_pool_assembler, "HybridCacheController") as controller,
|
|
patch.object(
|
|
hybrid_pool_assembler,
|
|
"get_memory",
|
|
return_value=SimpleNamespace(
|
|
hicache_write_policy="write_through",
|
|
hicache_io_backend="kernel",
|
|
hicache_host_memory_mode="cache",
|
|
),
|
|
),
|
|
):
|
|
result = _DsaStrategy().build(
|
|
cache=None,
|
|
kvcache=pool,
|
|
params=_target_params((draft,)),
|
|
server_args=None,
|
|
load_cache_event=None,
|
|
)
|
|
self.assertEqual(controller.call_args.kwargs["transfer_layer_id_max"], 3)
|
|
self.assertEqual(controller.call_args.args[2], 64)
|
|
for config in build.call_args.kwargs["config"].pools:
|
|
self.assertEqual(config.transfer_layer_id_max, 4)
|
|
self.assertEqual(config.layer_mapping, {0: 0, 1: 1, 2: 2, 3: 3})
|
|
self.assertEqual(config.packed_draft_device_pools, (draft,))
|
|
self.assertIs(result.host_pool_group, stack.host_pool_group)
|
|
self.assertEqual(result.sidecars, stack.sidecars)
|
|
|
|
def test_dsa_target_layer_sharded(self):
|
|
# rank 0 of 2 owns local layers 0 and 1 of 3. The permuted stage mapping
|
|
# is passed through untouched and the host_pools hold only the owned layers.
|
|
pool = _dsa_pool_stub(layer_num=3, shard=(0, 2))
|
|
stack = self._run(pool=pool, drafts=(), full_layer_mapping={0: 1, 1: 2, 2: 0})
|
|
mapper = (None, 1, 2, 0, None, None)
|
|
self.assertEqual(
|
|
_entry_shape(stack.host_pool_group, 3),
|
|
[
|
|
(PoolName.KV, True, id(pool), (), mapper, _kv_shape(2)),
|
|
(PoolName.INDEXER, False, id(pool), (), mapper, _indexer_shape(2)),
|
|
],
|
|
)
|
|
|
|
def test_plain_mla_target(self):
|
|
# A pool that declares KV only builds the anchor-only group the removed
|
|
# build_kv_only_stack built, packed draft included.
|
|
for drafts in (0, 1):
|
|
with self.subTest(drafts=drafts):
|
|
pool = _kv_pool_stub(layer_num=3)
|
|
ds = tuple(_kv_pool_stub(layer_num=1) for _ in range(drafts))
|
|
stack = self._run(
|
|
pool=pool, drafts=ds, full_layer_mapping={0: 0, 1: 1, 2: 2}
|
|
)
|
|
mapper = (None, 0, 1, 2, *([3] if drafts else []), None, None)
|
|
self.assertEqual(
|
|
_entry_shape(stack.host_pool_group, 3 + drafts),
|
|
[
|
|
(
|
|
PoolName.KV,
|
|
True,
|
|
id(pool),
|
|
tuple(id(d) for d in ds),
|
|
mapper,
|
|
_kv_shape(3 + drafts),
|
|
)
|
|
],
|
|
)
|
|
self.assertEqual(stack.sidecars, [])
|
|
|
|
|
|
class TestPackedDraftPairing(CustomTestCase):
|
|
"""Packing appends draft layers to the target host_pools, so a draft must
|
|
declare every target pool with an identical per-layer layout."""
|
|
|
|
def test_draft_without_indexer_is_rejected(self):
|
|
# The pre-declaration filter on ``index_k_with_scale_buffer`` dropped
|
|
# such a draft silently, leaving its state without any host mirror.
|
|
target = _dsa_pool_stub(layer_num=2)
|
|
no_index = _dsa_pool_stub(layer_num=1)
|
|
no_index.index_key_cache = SimpleNamespace(buffer=[])
|
|
with self.assertRaisesRegex(ValueError, "draft counterpart"):
|
|
prepare_host_pool_config(
|
|
decls=target.host_pool_decls(),
|
|
full_layer_mapping={i: i for i in range(target.layer_num)},
|
|
transfer_layer_id_max=target.layer_num,
|
|
transfer_page_size=64,
|
|
packed_draft_device_pools=(no_index,),
|
|
)
|
|
|
|
def test_draft_with_empty_index_layers_is_rejected(self):
|
|
# A 0-row placeholder layer would be packed as a null device pointer.
|
|
target = _dsa_pool_stub(layer_num=2)
|
|
shared = _dsa_pool_stub(layer_num=1)
|
|
shared.skip_topk_layers = [True]
|
|
shared.index_key_cache = SimpleNamespace(buffer=[object()])
|
|
with self.assertRaisesRegex(ValueError, "draft counterpart"):
|
|
prepare_host_pool_config(
|
|
decls=target.host_pool_decls(),
|
|
full_layer_mapping={i: i for i in range(target.layer_num)},
|
|
transfer_layer_id_max=target.layer_num,
|
|
transfer_page_size=64,
|
|
packed_draft_device_pools=(shared,),
|
|
)
|
|
partial = _dsa_pool_stub(layer_num=2)
|
|
partial.skip_topk_layers = [False, True]
|
|
with self.assertRaisesRegex(ValueError, "owns buffers on 1 of 2"):
|
|
prepare_host_pool_config(
|
|
decls=target.host_pool_decls(),
|
|
full_layer_mapping={i: i for i in range(target.layer_num)},
|
|
transfer_layer_id_max=target.layer_num,
|
|
transfer_page_size=64,
|
|
packed_draft_device_pools=(partial,),
|
|
)
|
|
|
|
def test_storage_info_mismatch_is_rejected(self):
|
|
target = _dsa_pool_stub(layer_num=2)
|
|
wide = _dsa_pool_stub(layer_num=1)
|
|
wide.index_head_dim = 256
|
|
with self.assertRaisesRegex(ValueError, "storage"):
|
|
prepare_host_pool_config(
|
|
decls=target.host_pool_decls(),
|
|
full_layer_mapping={i: i for i in range(target.layer_num)},
|
|
transfer_layer_id_max=target.layer_num,
|
|
transfer_page_size=64,
|
|
packed_draft_device_pools=(wide,),
|
|
)
|
|
|
|
def test_pool_with_only_shared_topk_layers_declares_kv_only(self):
|
|
pool = _dsa_pool_stub(layer_num=2)
|
|
pool.skip_topk_layers = [True, True]
|
|
self.assertEqual([d.pool_name for d in pool.host_pool_decls()], [PoolName.KV])
|
|
|
|
|
|
class TestKvHostPoolRow(CustomTestCase):
|
|
"""The KV mirror takes its row width from the MLA device pool (an fp8 DSA
|
|
store is wider than kv_lora_rank + qk_rope_head_dim) and refuses packed
|
|
drafts whose KV rows differ from the target's."""
|
|
|
|
def _build(self, pool, drafts=()):
|
|
seen = {}
|
|
|
|
def fake_host(kv_pool, ratio, size, page_size, layout, **kwargs):
|
|
seen.update(kwargs)
|
|
return SimpleNamespace(layer_num=kv_pool.layer_num)
|
|
|
|
with (
|
|
patch.object(hybrid_pool_assembler, "MLATokenToKVPoolHost", fake_host),
|
|
patch.object(
|
|
hybrid_pool_assembler,
|
|
"get_parallel",
|
|
return_value=SimpleNamespace(dcp_enabled=False),
|
|
),
|
|
patch.object(
|
|
hybrid_pool_assembler,
|
|
"get_memory",
|
|
return_value=SimpleNamespace(
|
|
hicache_ratio=2, hicache_size=0, hicache_mem_layout="page_first"
|
|
),
|
|
),
|
|
patch.object(
|
|
hybrid_pool_assembler, "_get_allocator_type", return_value="default"
|
|
),
|
|
):
|
|
hybrid_pool_assembler.build_kv_host_pool(
|
|
kv_pool=pool,
|
|
page_size=64,
|
|
mtp_draft_device_pools=drafts,
|
|
)
|
|
return seen
|
|
|
|
def test_mla_row_width_comes_from_the_device_pool(self):
|
|
pool = _dsa_pool_stub(layer_num=2)
|
|
pool.kv_cache_dim = 656 # fp8 DSA store: 576 + fp32 scales
|
|
self.assertEqual(self._build(pool)["override_kv_cache_dim"], 656)
|
|
|
|
def test_packed_draft_with_a_different_kv_row_is_rejected(self):
|
|
pool = _dsa_pool_stub(layer_num=2)
|
|
draft = _dsa_pool_stub(layer_num=1)
|
|
draft.store_dtype = torch.float8_e4m3fn
|
|
with self.assertRaisesRegex(ValueError, "KV row"):
|
|
self._build(pool, (draft,))
|
|
self._build(pool, (_dsa_pool_stub(layer_num=1),))
|
|
|
|
|
|
class TestDeclaredPoolPlanning(CustomTestCase):
|
|
"""Sidecar indices resolve from one primary source in HostPoolGroup, so the
|
|
planner must reject self-references and sidecar chains up front."""
|
|
|
|
def _prepare(self, decls):
|
|
return prepare_host_pool_config(
|
|
decls=decls,
|
|
full_layer_mapping={0: 0},
|
|
transfer_layer_id_max=1,
|
|
transfer_page_size=64,
|
|
)
|
|
|
|
def test_rejects_self_referencing_index_source(self):
|
|
import msgspec
|
|
|
|
kv, indexer = _dsa_pool_stub(layer_num=1).host_pool_decls()
|
|
bad = msgspec.structs.replace(indexer, indices_from_pool=PoolName.INDEXER)
|
|
with self.assertRaisesRegex(ValueError, "indices_from_pool"):
|
|
self._prepare((kv, bad))
|
|
|
|
def test_rejects_self_referencing_layout_source(self):
|
|
import msgspec
|
|
|
|
kv, indexer = _dsa_pool_stub(layer_num=1).host_pool_decls()
|
|
bad = msgspec.structs.replace(indexer, layout_source=PoolName.INDEXER)
|
|
with self.assertRaisesRegex(ValueError, "layout_source"):
|
|
self._prepare((kv, bad))
|
|
|
|
def test_rejects_primary_that_is_not_kv(self):
|
|
import msgspec
|
|
|
|
kv, indexer = _dsa_pool_stub(layer_num=1).host_pool_decls()
|
|
swa_primary = msgspec.structs.replace(kv, pool_name=PoolName.SWA)
|
|
follower = msgspec.structs.replace(
|
|
indexer, indices_from_pool=PoolName.SWA, layout_source=PoolName.SWA
|
|
)
|
|
with self.assertRaisesRegex(ValueError, "primary KV pool"):
|
|
self._prepare((swa_primary, follower))
|
|
|
|
def test_accepts_dsa_declaration(self):
|
|
config = self._prepare(_dsa_pool_stub(layer_num=1).host_pool_decls())
|
|
self.assertEqual(
|
|
[c.decl.pool_name for c in config.pools], [PoolName.KV, PoolName.INDEXER]
|
|
)
|
|
|
|
|
|
class TestHostPoolPreflight(CustomTestCase):
|
|
"""Malformed declarations must fail before reserving host memory."""
|
|
|
|
def test_invalid_dependencies_do_not_allocate(self):
|
|
kv, indexer = _dsa_pool_stub(layer_num=2).host_pool_decls()
|
|
cyclic_indexer = msgspec.structs.replace(indexer, layout_source=PoolName.SWA)
|
|
cyclic_swa = msgspec.structs.replace(
|
|
indexer, pool_name=PoolName.SWA, layout_source=PoolName.INDEXER
|
|
)
|
|
cases = [
|
|
((kv, msgspec.structs.replace(indexer, host_pool_builder=None)), "builder"),
|
|
((kv, msgspec.structs.replace(indexer, storage_info=None)), "storage_info"),
|
|
((kv, cyclic_indexer, cyclic_swa), "cyclic"),
|
|
(
|
|
(kv, msgspec.structs.replace(indexer, owned_device_layers=(0, 0))),
|
|
"owned_device_layers",
|
|
),
|
|
]
|
|
for decls, error in cases:
|
|
with (
|
|
self.subTest(error=error),
|
|
patch.object(hybrid_pool_assembler, "build_kv_host_pool") as allocate,
|
|
):
|
|
with self.assertRaisesRegex(ValueError, error):
|
|
config = prepare_host_pool_config(
|
|
decls=decls,
|
|
full_layer_mapping={0: 0, 1: 1},
|
|
transfer_layer_id_max=2,
|
|
transfer_page_size=64,
|
|
)
|
|
build_host_pool_group(config=config)
|
|
allocate.assert_not_called()
|
|
|
|
def test_incompatible_packed_buffers_fail_before_allocation(self):
|
|
target = _dsa_pool_stub(layer_num=2)
|
|
bad_kv = _dsa_pool_stub(layer_num=1)
|
|
bad_kv.store_dtype = torch.float8_e4m3fn
|
|
bad_index = _dsa_pool_stub(layer_num=1)
|
|
# Same 132 bytes per row, different key/scale format.
|
|
bad_index.index_head_dim = 132
|
|
bad_index.quant_block_size = 256
|
|
for draft, error in ((bad_kv, "KV row"), (bad_index, "index key format")):
|
|
with (
|
|
self.subTest(error=error),
|
|
patch.object(hybrid_pool_assembler, "build_kv_host_pool") as allocate,
|
|
):
|
|
with self.assertRaisesRegex(ValueError, error):
|
|
_DsaStrategy().build(
|
|
cache=None,
|
|
kvcache=target,
|
|
params=_target_params((draft,)),
|
|
server_args=None,
|
|
load_cache_event=None,
|
|
)
|
|
allocate.assert_not_called()
|
|
|
|
def test_draft_declarations_are_collected_once(self):
|
|
target = _dsa_pool_stub(layer_num=2)
|
|
draft = _dsa_pool_stub(layer_num=1)
|
|
draft.host_pool_decls = MagicMock(wraps=draft.host_pool_decls)
|
|
decls = target.host_pool_decls()
|
|
target_mapping = {0: 0, 4: 1}
|
|
config = prepare_host_pool_config(
|
|
decls=decls,
|
|
full_layer_mapping=target_mapping,
|
|
transfer_layer_id_max=5,
|
|
transfer_page_size=64,
|
|
packed_draft_device_pools=(draft,),
|
|
)
|
|
self.assertEqual(target_mapping, {0: 0, 4: 1})
|
|
draft.host_pool_decls.assert_called_once()
|
|
self.assertIs(config.pools[1].packed_draft_device_pools[0], draft)
|
|
self.assertEqual(config.pools[1].layer_mapping, {0: 0, 4: 1, 5: 2})
|
|
|
|
def test_packed_draft_rejects_duplicate_pool_names(self):
|
|
target = _dsa_pool_stub(layer_num=2)
|
|
draft = _dsa_pool_stub(layer_num=1)
|
|
declarations = draft.host_pool_decls()
|
|
draft.host_pool_decls = lambda: (*declarations, declarations[1])
|
|
with self.assertRaisesRegex(ValueError, "duplicate"):
|
|
prepare_host_pool_config(
|
|
decls=target.host_pool_decls(),
|
|
full_layer_mapping={i: i for i in range(target.layer_num)},
|
|
transfer_layer_id_max=target.layer_num,
|
|
transfer_page_size=64,
|
|
packed_draft_device_pools=(draft,),
|
|
)
|
|
|
|
def test_transfer_range_is_not_the_number_of_entries(self):
|
|
decls = _dsa_pool_stub(layer_num=2).host_pool_decls()
|
|
with self.assertRaisesRegex(ValueError, "transfer layer 4"):
|
|
prepare_host_pool_config(
|
|
decls=decls,
|
|
full_layer_mapping={0: 0, 4: 1},
|
|
transfer_layer_id_max=2,
|
|
transfer_page_size=64,
|
|
)
|
|
|
|
def test_build_order_resolves_dependencies_before_allocation(self):
|
|
kv, indexer = _dsa_pool_stub(layer_num=1).host_pool_decls()
|
|
config = prepare_host_pool_config(
|
|
decls=(indexer, kv),
|
|
full_layer_mapping={0: 0},
|
|
transfer_layer_id_max=1,
|
|
transfer_page_size=64,
|
|
)
|
|
self.assertEqual(
|
|
[c.decl.pool_name for c in config.pools], [PoolName.KV, PoolName.INDEXER]
|
|
)
|
|
|
|
|
|
class TestDeclaredPoolVerification(CustomTestCase):
|
|
def _result_with(self, *names):
|
|
group = SimpleNamespace(entry_map={n: object() for n in names})
|
|
return StackBuildResult(
|
|
host_pool_group=group, cache_controller=None, component_host_pools={}
|
|
)
|
|
|
|
def test_unmigrated_strategy_logs_missing_indexer(self):
|
|
pool = _dsa_pool_stub(layer_num=2)
|
|
with self.assertLogs(hybrid_pool_assembler.logger, level="ERROR") as logs:
|
|
_check_declared_pools_present(
|
|
kvcache=pool,
|
|
result=self._result_with(PoolName.KV),
|
|
strategy=_SwaStrategy(),
|
|
)
|
|
self.assertIn("indexer", logs.output[0])
|
|
|
|
def test_migrated_strategy_raises_on_missing_indexer(self):
|
|
pool = _dsa_pool_stub(layer_num=2)
|
|
with self.assertRaisesRegex(ValueError, "indexer"):
|
|
_check_declared_pools_present(
|
|
kvcache=pool,
|
|
result=self._result_with(PoolName.KV),
|
|
strategy=_DsaStrategy(),
|
|
)
|
|
|
|
def test_complete_stack_passes_silently(self):
|
|
pool = _dsa_pool_stub(layer_num=2)
|
|
_check_declared_pools_present(
|
|
kvcache=pool,
|
|
result=self._result_with(PoolName.KV, PoolName.INDEXER),
|
|
strategy=_DsaStrategy(),
|
|
)
|
|
|
|
def test_hybrid_pool_declares_through_its_sub_pool(self):
|
|
# HybridLinearKVPool once exposed host_pool_decls as a property, so the
|
|
# polymorphic call every strategy makes raised on hybrid models.
|
|
pool = object.__new__(HybridLinearKVPool)
|
|
pool.full_kv_pool = _dsa_pool_stub(layer_num=2)
|
|
self.assertEqual(
|
|
[(d.pool_name, d.device_pool) for d in pool.host_pool_decls()],
|
|
[(PoolName.KV, pool.full_kv_pool), (PoolName.INDEXER, pool.full_kv_pool)],
|
|
)
|
|
|
|
|
|
_ASSEMBLER = "sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler."
|
|
|
|
|
|
class TestTransferLayerSpan(CustomTestCase):
|
|
"""``transfer_layer_id_max`` must span global layer ids, not count the mapped ones.
|
|
|
|
A hybrid model with an uncached layer type keys its mappings non-contiguously,
|
|
and the per-layer transfer loop then never reaches the high layer ids.
|
|
"""
|
|
|
|
def test_pool_entries_span_the_highest_global_layer_id(self):
|
|
# Global ids 0/2/4/6 with holes between them, the shape NemotronH's
|
|
# cache-ineligible MLP layers produce: 4 mapped layers spanning 7 ids.
|
|
full_layer_mapping = {0: 0, 6: 1}
|
|
swa_layer_mapping = {2: 0, 4: 1}
|
|
|
|
with (
|
|
patch(_ASSEMBLER + "build_kv_host_pool"),
|
|
patch(_ASSEMBLER + "HostPoolGroup"),
|
|
patch(_ASSEMBLER + "build_pool_entry") as build_pool_entry,
|
|
):
|
|
build_hybrid_swa_group(
|
|
page_size=64,
|
|
full_kv_pool=MagicMock(),
|
|
swa_kv_pool=MagicMock(),
|
|
full_layer_mapping=full_layer_mapping,
|
|
swa_layer_mapping=swa_layer_mapping,
|
|
use_mla=False,
|
|
)
|
|
|
|
self.assertEqual(
|
|
[
|
|
c.kwargs["transfer_layer_id_max"]
|
|
for c in build_pool_entry.call_args_list
|
|
],
|
|
[7, 7],
|
|
)
|
|
|
|
|
|
def _build_unified_swa_pool(page_size: int = 4):
|
|
return init_unified_swa_pools(
|
|
device="cpu",
|
|
kv_cache_dtype=torch.float16,
|
|
head_num=2,
|
|
head_dim=4,
|
|
v_head_dim=4,
|
|
swa_head_num=2,
|
|
swa_head_dim=4,
|
|
swa_v_head_dim=4,
|
|
page_size=page_size,
|
|
start_layer=0,
|
|
end_layer=4,
|
|
swa_attention_layer_ids=[3],
|
|
full_attention_layer_ids=[0, 1, 2],
|
|
total_bytes=4096,
|
|
enable_memory_saver=False,
|
|
need_sort=False,
|
|
lazy_compaction=True,
|
|
)
|
|
|
|
|
|
def _build_unified_host_pair(bundle):
|
|
pool = bundle.token_to_kv_pool
|
|
return UnifiedPageEnvelopeHostPool.build_hybrid_swa_pool_pair(
|
|
device_pools=(pool.full_kv_pool, pool.swa_kv_pool),
|
|
host_to_device_ratio=1.0,
|
|
host_size=0,
|
|
page_size=pool.page_size,
|
|
layout="layer_first",
|
|
pin_memory=False,
|
|
allocator_type="default",
|
|
)
|
|
|
|
|
|
class TestUnifiedPageEnvelopeHostPool(CustomTestCase):
|
|
def test_shared_arena_can_reuse_bytes_across_sides(self):
|
|
page_size = 4
|
|
full_pool, swa_pool = _build_unified_host_pair(
|
|
_build_unified_swa_pool(page_size)
|
|
)
|
|
self.addCleanup(full_pool.destroy)
|
|
self.addCleanup(swa_pool.destroy)
|
|
|
|
full_indices = full_pool.alloc(full_pool.available_size())
|
|
self.assertIsNotNone(full_indices)
|
|
self.assertIsNotNone(swa_pool.alloc(swa_pool.available_size()))
|
|
self.assertIsNone(swa_pool.alloc(page_size))
|
|
|
|
full_pool.free(full_indices[page_size:])
|
|
self.assertIsNotNone(swa_pool.alloc(page_size))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|