Co-authored-by: Yutong Wu <yutong.wu@amd.com> Co-authored-by: Zhangheng <hzh0425@apache.org>
829 lines
33 KiB
Python
829 lines
33 KiB
Python
"""Unit tests for external-linker device pool assembly."""
|
|
|
|
import unittest
|
|
from types import SimpleNamespace
|
|
from unittest.mock import Mock, call, patch
|
|
|
|
import torch
|
|
|
|
from sglang.srt.mem_cache.hicache_storage import (
|
|
PoolHitPolicy,
|
|
PoolName,
|
|
PoolTransfer,
|
|
)
|
|
from sglang.srt.mem_cache.hybrid_cache.linker_pool_assembler import (
|
|
DevicePoolEntry,
|
|
DevicePoolGroup,
|
|
_build_deepseek_v4_device_pool_group,
|
|
resolve_hybrid_device_pool_group,
|
|
)
|
|
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
|
|
from sglang.test.ci.ci_register import register_cpu_ci
|
|
from sglang.test.test_utils import CustomTestCase
|
|
|
|
register_cpu_ci(est_time=11, suite="base-a-test-cpu")
|
|
|
|
|
|
class TestDevicePoolEntry(CustomTestCase):
|
|
def test_sparse_multi_component_layer_ranges(self):
|
|
k0 = torch.zeros((8, 3), dtype=torch.uint8)
|
|
k2 = torch.zeros((8, 5), dtype=torch.uint8)
|
|
v0 = torch.zeros((8, 7), dtype=torch.uint8)
|
|
v2 = torch.zeros((8, 11), dtype=torch.uint8)
|
|
pool = DevicePoolEntry(
|
|
name=PoolName.KV,
|
|
indices_from_pool=PoolName.KV,
|
|
device_pool=None,
|
|
components=[[k0, k2], [v0, v2]],
|
|
layer_mapping={0: 0, 2: 1},
|
|
page_size=2,
|
|
rows_are_pages=False,
|
|
packed=False,
|
|
)
|
|
|
|
indices = torch.tensor([0, 1, 4, 5])
|
|
locations = pool.prepare_locations(indices)
|
|
self.assertEqual(locations, [0, 4])
|
|
pointers, sizes = pool.get_page_buffer_meta(indices)
|
|
self.assertEqual(
|
|
pointers,
|
|
[
|
|
buffer[row].data_ptr()
|
|
for row in locations
|
|
for buffer in (k0, k2, v0, v2)
|
|
],
|
|
)
|
|
self.assertEqual(sizes, [6, 10, 14, 22] * 2)
|
|
self.assertIsNone(pool.get_prepared_layer_range_meta(locations, 1))
|
|
|
|
pointers, sizes, offsets = pool.get_prepared_layer_range_meta(locations, 2)
|
|
self.assertEqual(
|
|
pointers,
|
|
[
|
|
[k2[0].data_ptr()],
|
|
[v2[0].data_ptr()],
|
|
[k2[4].data_ptr()],
|
|
[v2[4].data_ptr()],
|
|
],
|
|
)
|
|
self.assertEqual(sizes, [[10], [22], [10], [22]])
|
|
self.assertEqual(offsets, [[6], [14], [6], [14]])
|
|
|
|
def test_rejects_invalid_pages_and_empty_buffers(self):
|
|
with self.assertRaisesRegex(ValueError, "has no storage buffers"):
|
|
DevicePoolEntry(
|
|
name=PoolName.KV,
|
|
indices_from_pool=PoolName.KV,
|
|
device_pool=None,
|
|
components=[],
|
|
layer_mapping={},
|
|
page_size=2,
|
|
rows_are_pages=False,
|
|
)
|
|
|
|
pool = DevicePoolEntry(
|
|
name=PoolName.KV,
|
|
indices_from_pool=PoolName.KV,
|
|
device_pool=None,
|
|
components=[[torch.zeros((8, 3), dtype=torch.uint8)]],
|
|
layer_mapping={0: 0},
|
|
page_size=2,
|
|
rows_are_pages=False,
|
|
)
|
|
for indices, error in (
|
|
(torch.tensor([0]), "multiple of page_size"),
|
|
(torch.tensor([1, 2]), "aligned contiguous pages"),
|
|
(torch.tensor([0, 2]), "aligned contiguous pages"),
|
|
(torch.tensor([8, 9]), "exceeds buffer shapes"),
|
|
):
|
|
with self.subTest(indices=indices.tolist()):
|
|
with self.assertRaisesRegex(ValueError, error):
|
|
pool.prepare_locations(indices)
|
|
|
|
|
|
class TestDevicePoolGroup(CustomTestCase):
|
|
def test_resolve_transfers_expands_physical_pools(self):
|
|
entries = [
|
|
SimpleNamespace(
|
|
name=PoolName.KV,
|
|
indices_from_pool=PoolName.KV,
|
|
translate_indices=lambda indices: indices,
|
|
),
|
|
SimpleNamespace(
|
|
name=PoolName.INDEXER,
|
|
indices_from_pool=PoolName.KV,
|
|
translate_indices=lambda indices: indices + 100,
|
|
),
|
|
]
|
|
group = DevicePoolGroup(entries, num_layers=2, page_size=2)
|
|
transfer = PoolTransfer(
|
|
name=PoolName.KV,
|
|
keys=["a", "b"],
|
|
device_indices=torch.tensor([0, 1, 4, 5]),
|
|
hit_policy=PoolHitPolicy.TRAILING_PAGES,
|
|
)
|
|
|
|
resolved = group.resolve_transfers([transfer])
|
|
|
|
self.assertEqual(
|
|
[item.name for item in resolved], [PoolName.KV, PoolName.INDEXER]
|
|
)
|
|
self.assertEqual(resolved[0].host_indices.tolist(), [0, 1, 4, 5])
|
|
self.assertEqual(resolved[1].host_indices.tolist(), [100, 101, 104, 105])
|
|
self.assertTrue(
|
|
all(item.hit_policy == PoolHitPolicy.ALL_PAGES for item in resolved)
|
|
)
|
|
|
|
def test_partial_side_pool_requires_explicit_opt_in(self):
|
|
entry = SimpleNamespace(
|
|
name=PoolName.SWA,
|
|
indices_from_pool=PoolName.SWA,
|
|
translate_indices=lambda indices: indices + 100,
|
|
)
|
|
group = DevicePoolGroup([entry], num_layers=1, page_size=2)
|
|
transfer = PoolTransfer(
|
|
name=PoolName.SWA,
|
|
keys=["b", "d"],
|
|
device_indices=torch.tensor([20, 21, 24, 25]),
|
|
hit_policy=PoolHitPolicy.TRAILING_PAGES,
|
|
)
|
|
|
|
self.assertEqual(group.resolve_transfers([transfer]), [])
|
|
resolved = group.resolve_transfers(
|
|
[transfer], allow_partial=True, allow_missing_kv=True
|
|
)
|
|
|
|
self.assertEqual(len(resolved), 1)
|
|
self.assertEqual(resolved[0].name, PoolName.SWA)
|
|
self.assertEqual(resolved[0].keys, ["b", "d"])
|
|
self.assertEqual(resolved[0].host_indices.tolist(), [120, 121, 124, 125])
|
|
self.assertEqual(resolved[0].hit_policy, PoolHitPolicy.TRAILING_PAGES)
|
|
|
|
|
|
class TestHybridDevicePoolAssembler(CustomTestCase):
|
|
def test_deepseek_v4_maps_sparse_sidecars(self):
|
|
from sglang.srt.mem_cache.deepseek_v4_memory_pool import (
|
|
DeepSeekV4LayerItem,
|
|
DeepSeekV4TokenToKVPool,
|
|
)
|
|
|
|
def state_pool():
|
|
return SimpleNamespace(
|
|
ring_size=2,
|
|
kv_score_buffer=SimpleNamespace(kv_score=torch.zeros((8, 3))),
|
|
)
|
|
|
|
kvcache = DeepSeekV4TokenToKVPool.__new__(DeepSeekV4TokenToKVPool)
|
|
kvcache._unified_kv = False
|
|
kvcache.start_layer = 1
|
|
kvcache.end_layer = 4
|
|
kvcache.swa_page_size = 2
|
|
kvcache.swa_kv_pool = SimpleNamespace(
|
|
page_size=2,
|
|
kv_buffer=[torch.zeros((8, 3), dtype=torch.uint8) for _ in range(3)],
|
|
)
|
|
kvcache.c4_kv_pool = SimpleNamespace(
|
|
kv_buffer=[torch.zeros((8, 5), dtype=torch.uint8) for _ in range(2)],
|
|
bytes_per_page_padded=5,
|
|
)
|
|
kvcache.c4_indexer_kv_pool = SimpleNamespace(
|
|
index_k_with_scale_buffer=[
|
|
torch.zeros((8, 7), dtype=torch.uint8) for _ in range(2)
|
|
]
|
|
)
|
|
kvcache.c128_kv_pool = SimpleNamespace(
|
|
kv_buffer=[torch.zeros((8, 11), dtype=torch.uint8)],
|
|
bytes_per_page_padded=11,
|
|
)
|
|
kvcache.layer_mapping = [
|
|
DeepSeekV4LayerItem(0, -1),
|
|
DeepSeekV4LayerItem(4, 0),
|
|
DeepSeekV4LayerItem(128, 0),
|
|
DeepSeekV4LayerItem(4, 1),
|
|
]
|
|
kvcache.compress_state_pools = [None, state_pool(), None, state_pool()]
|
|
kvcache.indexer_compress_state_pools = [
|
|
None,
|
|
state_pool(),
|
|
None,
|
|
state_pool(),
|
|
]
|
|
draft_swa_buffers = [
|
|
torch.zeros((8, 13), dtype=torch.uint8),
|
|
torch.zeros((8, 17), dtype=torch.uint8),
|
|
]
|
|
|
|
group = resolve_hybrid_device_pool_group(
|
|
kvcache=kvcache,
|
|
page_size=2,
|
|
params=SimpleNamespace(
|
|
mtp_draft_device_pools=(
|
|
SimpleNamespace(
|
|
swa_kv_pool=SimpleNamespace(kv_buffer=draft_swa_buffers)
|
|
),
|
|
)
|
|
),
|
|
components={ComponentType.FULL, ComponentType.SWA},
|
|
)
|
|
|
|
self.assertEqual(group.num_layers, 3)
|
|
self.assertTrue(group.rank_replicated)
|
|
self.assertEqual(
|
|
set(group.entry_map),
|
|
{
|
|
PoolName.SWA,
|
|
PoolName.DEEPSEEK_V4_C4,
|
|
PoolName.DEEPSEEK_V4_C4_INDEXER,
|
|
PoolName.DEEPSEEK_V4_C128,
|
|
PoolName.DEEPSEEK_V4_C4_STATE,
|
|
PoolName.DEEPSEEK_V4_C4_INDEXER_STATE,
|
|
},
|
|
)
|
|
self.assertEqual(group.sources[PoolName.DEEPSEEK_V4_C4], PoolName.KV)
|
|
self.assertEqual(group.sources[PoolName.DEEPSEEK_V4_C4_STATE], PoolName.SWA)
|
|
|
|
c4_pool = group.entry_map[PoolName.DEEPSEEK_V4_C4]
|
|
pointers, sizes = c4_pool.get_page_buffer_meta(torch.tensor([0, 1]))
|
|
self.assertEqual(len(pointers), 2)
|
|
self.assertEqual(sizes, [5, 5])
|
|
_, sizes, offsets = c4_pool.get_prepared_layer_range_meta([0], 2)
|
|
self.assertEqual(sizes, [[5]])
|
|
self.assertEqual(offsets, [[5]])
|
|
self.assertIsNone(c4_pool.get_prepared_layer_range_meta([0], 1))
|
|
|
|
swa_pool = group.entry_map[PoolName.SWA]
|
|
_, sizes, offsets = swa_pool.get_prepared_layer_range_meta([0], 0)
|
|
self.assertEqual(sizes, [[3, 13]])
|
|
self.assertEqual(offsets, [[0, 9]])
|
|
_, sizes, offsets = swa_pool.get_prepared_layer_range_meta([0], 1)
|
|
self.assertEqual(sizes, [[3, 17]])
|
|
self.assertEqual(offsets, [[3, 22]])
|
|
|
|
def test_deepseek_v41_shared_kv_and_split_indexer_pages(self):
|
|
from sglang.srt.mem_cache.deepseek_v4_memory_pool import (
|
|
DeepSeekV4LayerItem,
|
|
DeepSeekV4TokenToKVPool,
|
|
)
|
|
|
|
def index_pool(layers, rows, width):
|
|
return SimpleNamespace(
|
|
page_size=2,
|
|
index_k_with_scale_buffer=None,
|
|
index_k_payload_buffer=[
|
|
torch.arange(rows * width, dtype=torch.uint8).reshape(rows, width)
|
|
for _ in range(layers)
|
|
],
|
|
index_k_scale_buffer=[
|
|
torch.zeros((rows, 2), dtype=torch.uint8) for _ in range(layers)
|
|
],
|
|
)
|
|
|
|
pool = DeepSeekV4TokenToKVPool.__new__(DeepSeekV4TokenToKVPool)
|
|
pool._unified_kv = False
|
|
pool.start_layer, pool.end_layer = 1, 7
|
|
pool.swa_page_size = 8
|
|
pool.swa_kv_pool = SimpleNamespace(
|
|
page_size=8,
|
|
kv_buffer=[torch.zeros((5, 3), dtype=torch.uint8) for _ in range(6)],
|
|
)
|
|
pool.c4_kv_pool = pool.c128_kv_pool = pool.c4_indexer_kv_pool = None
|
|
pool.sources_by_ratio = {1: [4], 2: [2, 6]}
|
|
pool.kv_pools = {
|
|
1: SimpleNamespace(kv_buffer=[torch.zeros((5, 7), dtype=torch.uint8)]),
|
|
2: SimpleNamespace(
|
|
kv_buffer=[torch.zeros((5, 11), dtype=torch.uint8) for _ in range(2)]
|
|
),
|
|
}
|
|
pool.index_pools = {1: index_pool(1, 23, 4), 2: index_pool(2, 13, 6)}
|
|
pool.layer_mapping = [
|
|
DeepSeekV4LayerItem(ratio, layer)
|
|
for ratio, layer in [
|
|
(0, -1),
|
|
(0, -1),
|
|
(2, 0),
|
|
(2, 0),
|
|
(1, 0),
|
|
(1, 0),
|
|
(2, 1),
|
|
]
|
|
]
|
|
pool.compress_state_pools = [None] * 7
|
|
pool.indexer_compress_state_pools = [None] * 7
|
|
draft = SimpleNamespace(
|
|
swa_kv_pool=SimpleNamespace(
|
|
kv_buffer=[torch.zeros((5, 13), dtype=torch.uint8) for _ in range(2)]
|
|
)
|
|
)
|
|
group = resolve_hybrid_device_pool_group(
|
|
kvcache=pool,
|
|
page_size=8,
|
|
params=SimpleNamespace(mtp_draft_device_pools=(draft,)),
|
|
components={ComponentType.FULL, ComponentType.SWA},
|
|
)
|
|
self.assertEqual(
|
|
set(group.entry_map),
|
|
{
|
|
PoolName.SWA,
|
|
PoolName.DEEPSEEK_V4_C1,
|
|
PoolName.DEEPSEEK_V4_C1_INDEXER,
|
|
PoolName.DEEPSEEK_V4_C1_INDEXER_SCALE,
|
|
PoolName.DEEPSEEK_V4_C2,
|
|
PoolName.DEEPSEEK_V4_C2_INDEXER,
|
|
PoolName.DEEPSEEK_V4_C2_INDEXER_SCALE,
|
|
},
|
|
)
|
|
self.assertEqual(group.entry_map[PoolName.DEEPSEEK_V4_C1].layer_mapping, {3: 0})
|
|
c2 = group.entry_map[PoolName.DEEPSEEK_V4_C2]
|
|
self.assertEqual(c2.layer_mapping, {1: 0, 5: 1})
|
|
self.assertIsNone(c2.get_prepared_layer_range_meta([1], 2))
|
|
self.assertEqual(
|
|
c2.get_prepared_layer_range_meta([1], 5),
|
|
([[pool.kv_pools[2].kv_buffer[1][1].data_ptr()]], [[11]], [[11]]),
|
|
)
|
|
full = PoolTransfer(
|
|
name=PoolName.KV, keys=["page"], device_indices=torch.arange(8, 16)
|
|
)
|
|
swa = PoolTransfer(
|
|
name=PoolName.SWA,
|
|
keys=["tail"],
|
|
device_indices=torch.arange(8, 16),
|
|
hit_policy=PoolHitPolicy.TRAILING_PAGES,
|
|
)
|
|
transfers = {t.name: t for t in group.resolve_transfers([full, swa])}
|
|
self.assertEqual(transfers[PoolName.DEEPSEEK_V4_C2].keys, ["page"])
|
|
self.assertEqual(
|
|
transfers[PoolName.DEEPSEEK_V4_C2].hit_policy, PoolHitPolicy.ALL_PAGES
|
|
)
|
|
self.assertEqual(transfers[PoolName.SWA].keys, ["tail"])
|
|
for ratio, name, width, raw_row in [
|
|
(1, PoolName.DEEPSEEK_V4_C1_INDEXER, 16, 4),
|
|
(2, PoolName.DEEPSEEK_V4_C2_INDEXER, 12, 2),
|
|
]:
|
|
entry = group.entry_map[name]
|
|
pointers, sizes = entry.get_page_buffer_meta(torch.arange(8, 16))
|
|
original = pool.index_pools[ratio].index_k_payload_buffer
|
|
self.assertEqual(pointers, [b[raw_row].data_ptr() for b in original])
|
|
self.assertEqual(sizes, [width] * len(original))
|
|
self.assertTrue(
|
|
torch.equal(
|
|
entry.components[0][0][1],
|
|
original[0][raw_row : raw_row * 2].flatten(),
|
|
)
|
|
)
|
|
self.assertEqual(
|
|
group.entry_map[PoolName.DEEPSEEK_V4_C1_INDEXER_SCALE].get_page_buffer_meta(
|
|
torch.arange(8, 16)
|
|
)[1],
|
|
[8],
|
|
)
|
|
self.assertEqual(
|
|
group.entry_map[PoolName.DEEPSEEK_V4_C2_INDEXER_SCALE].get_page_buffer_meta(
|
|
torch.arange(8, 16)
|
|
)[1],
|
|
[4, 4],
|
|
)
|
|
_, sizes, offsets = group.entry_map[PoolName.SWA].get_prepared_layer_range_meta(
|
|
[1], 0
|
|
)
|
|
self.assertEqual(sizes, [[3, 13]])
|
|
self.assertEqual(offsets, [[0, 18]])
|
|
|
|
def test_deepseek_v41_request_window_drops_swa(self):
|
|
"""Encoder replay leaves no target SWA pool: skip SWA and the draft SWA
|
|
rather than dereference them, and refuse pools that take SWA indices."""
|
|
from sglang.srt.mem_cache.deepseek_v4_memory_pool import (
|
|
DeepSeekV4LayerItem,
|
|
DeepSeekV4TokenToKVPool,
|
|
)
|
|
|
|
def state_pool():
|
|
return SimpleNamespace(
|
|
ring_size=2,
|
|
kv_score_buffer=SimpleNamespace(kv_score=torch.zeros((4, 3))),
|
|
)
|
|
|
|
pool = DeepSeekV4TokenToKVPool.__new__(DeepSeekV4TokenToKVPool)
|
|
pool._unified_kv = False
|
|
pool.start_layer, pool.end_layer = 0, 2
|
|
pool.swa_page_size = 8
|
|
pool.swa_kv_pool = None
|
|
pool.c4_kv_pool = pool.c128_kv_pool = pool.c4_indexer_kv_pool = None
|
|
pool.sources_by_ratio = {1: [1]}
|
|
pool.kv_pools = {
|
|
1: SimpleNamespace(kv_buffer=[torch.zeros((3, 7), dtype=torch.uint8)])
|
|
}
|
|
pool.index_pools = {
|
|
1: SimpleNamespace(
|
|
page_size=2,
|
|
index_k_with_scale_buffer=[torch.zeros((9, 4), dtype=torch.uint8)],
|
|
)
|
|
}
|
|
pool.layer_mapping = [DeepSeekV4LayerItem(0, -1), DeepSeekV4LayerItem(1, 0)]
|
|
pool.compress_state_pools = [None, None]
|
|
pool.indexer_compress_state_pools = [None, None]
|
|
draft = SimpleNamespace(
|
|
swa_kv_pool=SimpleNamespace(
|
|
kv_buffer=[torch.zeros((3, 5), dtype=torch.uint8)]
|
|
)
|
|
)
|
|
params = SimpleNamespace(mtp_draft_device_pools=(draft,))
|
|
group = resolve_hybrid_device_pool_group(
|
|
kvcache=pool,
|
|
page_size=8,
|
|
params=params,
|
|
components={ComponentType.FULL, ComponentType.SWA},
|
|
)
|
|
self.assertEqual(
|
|
set(group.entry_map),
|
|
{PoolName.DEEPSEEK_V4_C1, PoolName.DEEPSEEK_V4_C1_INDEXER},
|
|
)
|
|
indexer = group.entry_map[PoolName.DEEPSEEK_V4_C1_INDEXER]
|
|
_, sizes = indexer.get_page_buffer_meta(torch.arange(8, 16))
|
|
self.assertEqual(sizes, [16])
|
|
|
|
pool.layer_mapping[0] = DeepSeekV4LayerItem(4, 0)
|
|
pool.c4_kv_pool = SimpleNamespace(
|
|
kv_buffer=[torch.zeros((3, 5), dtype=torch.uint8)],
|
|
bytes_per_page_padded=5,
|
|
)
|
|
pool.c4_indexer_kv_pool = SimpleNamespace(
|
|
index_k_with_scale_buffer=[torch.zeros((3, 6), dtype=torch.uint8)]
|
|
)
|
|
pool.compress_state_pools[0] = state_pool()
|
|
pool.indexer_compress_state_pools[0] = state_pool()
|
|
with self.assertRaisesRegex(ValueError, "SWA page indices"):
|
|
resolve_hybrid_device_pool_group(
|
|
kvcache=pool,
|
|
page_size=8,
|
|
params=params,
|
|
components={ComponentType.FULL, ComponentType.SWA},
|
|
)
|
|
|
|
def test_unified_deepseek_v4_uses_only_compressed_pools(self):
|
|
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4LayerItem
|
|
|
|
for split_indexer in (False, True):
|
|
with self.subTest(split_indexer=split_indexer):
|
|
c4 = [torch.zeros((4, width), dtype=torch.uint8) for width in (5, 7)]
|
|
c128 = [torch.zeros((4, 11), dtype=torch.uint8)]
|
|
expected = {
|
|
PoolName.DEEPSEEK_V4_C4: c4,
|
|
PoolName.DEEPSEEK_V4_C128: c128,
|
|
}
|
|
if split_indexer:
|
|
payload = [
|
|
torch.zeros((4, 1, 4, 1, 16), dtype=torch.uint8)
|
|
for _ in range(2)
|
|
]
|
|
scale = [
|
|
torch.zeros((4, 1, 4, 1), dtype=torch.uint8) for _ in range(2)
|
|
]
|
|
indexer = SimpleNamespace(
|
|
index_k_with_scale_buffer=None,
|
|
index_k_payload_buffer=payload,
|
|
index_k_scale_buffer=scale,
|
|
)
|
|
expected[PoolName.DEEPSEEK_V4_C4_INDEXER] = [
|
|
b.flatten(1) for b in payload
|
|
]
|
|
expected[PoolName.DEEPSEEK_V4_C4_INDEXER_SCALE] = [
|
|
b.flatten(1) for b in scale
|
|
]
|
|
else:
|
|
buffers = [
|
|
torch.zeros((4, width), dtype=torch.uint8) for width in (13, 17)
|
|
]
|
|
indexer = SimpleNamespace(index_k_with_scale_buffer=buffers)
|
|
expected[PoolName.DEEPSEEK_V4_C4_INDEXER] = buffers
|
|
regions = {4: (c4, 7), 128: (c128, 11)}
|
|
kvcache = SimpleNamespace(
|
|
_unified_kv=True,
|
|
start_layer=0,
|
|
end_layer=3,
|
|
layer_mapping=[
|
|
DeepSeekV4LayerItem(4, 1),
|
|
DeepSeekV4LayerItem(128, 0),
|
|
DeepSeekV4LayerItem(4, 0),
|
|
],
|
|
# Unified KV has neither paged KV nor an index-addressed SWA pool.
|
|
swa_kv_pool=None,
|
|
c4_kv_pool=None,
|
|
c128_kv_pool=None,
|
|
swa_page_size=3,
|
|
c4_indexer_kv_pool=indexer,
|
|
unified_region_buffers=Mock(side_effect=regions.__getitem__),
|
|
# bf16 unified rows are a single pool: no rope sibling.
|
|
unified_rope_region_buffers=Mock(return_value=None),
|
|
)
|
|
group = _build_deepseek_v4_device_pool_group(kvcache, page_size=2)
|
|
|
|
# expected carries no *_ROPE entry, so this also pins that the
|
|
# bf16 layout ships exactly what it did before.
|
|
self.assertEqual(set(group.entry_map), set(expected))
|
|
self.assertEqual(set(group.sources.values()), {PoolName.KV})
|
|
self.assertTrue(group.rank_replicated)
|
|
for name, buffers in expected.items():
|
|
entry = group.entry_map[name]
|
|
actual = entry.components[0]
|
|
self.assertEqual(len(actual), len(buffers))
|
|
for got, want in zip(actual, buffers):
|
|
self.assertEqual(got.data_ptr(), want.data_ptr())
|
|
self.assertEqual(got.shape, want.shape)
|
|
self.assertEqual(
|
|
kvcache.unified_region_buffers.call_args_list, [call(4), call(128)]
|
|
)
|
|
resolved = group.resolve_transfers(
|
|
[
|
|
PoolTransfer(
|
|
name=PoolName.KV,
|
|
keys=["page-0"],
|
|
device_indices=torch.tensor([0, 1]),
|
|
)
|
|
]
|
|
)
|
|
self.assertEqual({t.name for t in resolved}, set(expected))
|
|
|
|
def test_unified_deepseek_v4_fp8_ships_the_rope_half(self):
|
|
"""fp8 rows are two pools; shipping nope alone leaves stale rope."""
|
|
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4LayerItem
|
|
|
|
c4 = [torch.zeros((4, width), dtype=torch.uint8) for width in (5, 7)]
|
|
c128 = [torch.zeros((4, 11), dtype=torch.uint8)]
|
|
c4_rope = [torch.zeros((4, 3), dtype=torch.uint8) for _ in range(2)]
|
|
c128_rope = [torch.zeros((4, 3), dtype=torch.uint8)]
|
|
indexer = [torch.zeros((4, width), dtype=torch.uint8) for width in (13, 17)]
|
|
regions = {4: (c4, 7), 128: (c128, 11)}
|
|
rope_regions = {4: (c4_rope, 3), 128: (c128_rope, 3)}
|
|
kvcache = SimpleNamespace(
|
|
_unified_kv=True,
|
|
start_layer=0,
|
|
end_layer=3,
|
|
layer_mapping=[
|
|
DeepSeekV4LayerItem(4, 1),
|
|
DeepSeekV4LayerItem(128, 0),
|
|
DeepSeekV4LayerItem(4, 0),
|
|
],
|
|
swa_kv_pool=None,
|
|
c4_kv_pool=None,
|
|
c128_kv_pool=None,
|
|
swa_page_size=3,
|
|
c4_indexer_kv_pool=SimpleNamespace(index_k_with_scale_buffer=indexer),
|
|
unified_region_buffers=Mock(side_effect=regions.__getitem__),
|
|
unified_rope_region_buffers=Mock(side_effect=rope_regions.__getitem__),
|
|
)
|
|
group = _build_deepseek_v4_device_pool_group(kvcache, page_size=2)
|
|
|
|
expected_rope = {
|
|
PoolName.DEEPSEEK_V4_C4_ROPE: c4_rope,
|
|
PoolName.DEEPSEEK_V4_C128_ROPE: c128_rope,
|
|
}
|
|
self.assertTrue(set(expected_rope).issubset(group.entry_map))
|
|
for name, buffers in expected_rope.items():
|
|
actual = group.entry_map[name].components[0]
|
|
self.assertEqual(len(actual), len(buffers))
|
|
for got, want in zip(actual, buffers):
|
|
self.assertEqual(got.data_ptr(), want.data_ptr())
|
|
self.assertEqual(got.shape, want.shape)
|
|
self.assertEqual(
|
|
kvcache.unified_rope_region_buffers.call_args_list, [call(4), call(128)]
|
|
)
|
|
|
|
# One row index addresses both halves: same layer mapping, same source.
|
|
for rope_name, nope_name in (
|
|
(PoolName.DEEPSEEK_V4_C4_ROPE, PoolName.DEEPSEEK_V4_C4),
|
|
(PoolName.DEEPSEEK_V4_C128_ROPE, PoolName.DEEPSEEK_V4_C128),
|
|
):
|
|
self.assertEqual(
|
|
group.entry_map[rope_name].layer_mapping,
|
|
group.entry_map[nope_name].layer_mapping,
|
|
)
|
|
self.assertEqual(group.sources[rope_name], group.sources[nope_name])
|
|
|
|
# One KV transfer must expand to both halves.
|
|
resolved = group.resolve_transfers(
|
|
[
|
|
PoolTransfer(
|
|
name=PoolName.KV,
|
|
keys=["page-0"],
|
|
device_indices=torch.tensor([0, 1]),
|
|
)
|
|
]
|
|
)
|
|
self.assertTrue(set(expected_rope).issubset({t.name for t in resolved}))
|
|
|
|
def test_deepseek_v4_still_rejects_hisparse(self):
|
|
from sglang.srt.mem_cache.deepseek_v4_memory_pool import HiSparseC4DevicePool
|
|
|
|
kvcache = SimpleNamespace(
|
|
c4_kv_pool=HiSparseC4DevicePool.__new__(HiSparseC4DevicePool)
|
|
)
|
|
with self.assertRaisesRegex(ValueError, "does not support HiSparse"):
|
|
_build_deepseek_v4_device_pool_group(kvcache, 2)
|
|
|
|
def test_dsa_uses_hybrid_assembler_strategy(self):
|
|
from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool
|
|
|
|
def dsa_pool(kv_width, index_width):
|
|
pool = DSATokenToKVPool.__new__(DSATokenToKVPool)
|
|
pool.page_size = 2
|
|
pool.layer_num = 1
|
|
pool.kv_buffer = [torch.zeros((8, kv_width), dtype=torch.uint8)]
|
|
pool.index_key_cache = SimpleNamespace(
|
|
buffer=[torch.zeros((4, index_width), dtype=torch.uint8)]
|
|
)
|
|
return pool
|
|
|
|
kvcache = dsa_pool(3, 7)
|
|
kvcache.layer_num = 2
|
|
kvcache.kv_buffer.append(torch.zeros((8, 5), dtype=torch.uint8))
|
|
kvcache.index_key_cache.buffer.append(torch.zeros((4, 11), dtype=torch.uint8))
|
|
draft_pools = (dsa_pool(13, 17), dsa_pool(19, 23))
|
|
|
|
group = resolve_hybrid_device_pool_group(
|
|
kvcache=kvcache,
|
|
page_size=2,
|
|
params=SimpleNamespace(mtp_draft_device_pools=draft_pools),
|
|
components={ComponentType.FULL},
|
|
)
|
|
with self.assertRaisesRegex(ValueError, "draft KV"):
|
|
resolve_hybrid_device_pool_group(
|
|
kvcache=kvcache,
|
|
page_size=2,
|
|
params=SimpleNamespace(
|
|
mtp_draft_device_pools=(), sidecar_draft_device_pools=(object(),)
|
|
),
|
|
components={ComponentType.FULL},
|
|
)
|
|
|
|
self.assertEqual(group.num_layers, 2)
|
|
self.assertTrue(group.rank_replicated)
|
|
self.assertEqual(set(group.entry_map), {PoolName.KV, PoolName.INDEXER})
|
|
self.assertEqual(
|
|
group.sources,
|
|
{
|
|
PoolName.KV: PoolName.KV,
|
|
PoolName.INDEXER: PoolName.KV,
|
|
},
|
|
)
|
|
_, sizes, offsets = group.entry_map[PoolName.KV].get_prepared_layer_range_meta(
|
|
[0], 0
|
|
)
|
|
self.assertEqual(sizes, [[6, 26]])
|
|
self.assertEqual(offsets, [[0, 16]])
|
|
_, sizes, offsets = group.entry_map[
|
|
PoolName.INDEXER
|
|
].get_prepared_layer_range_meta([0], 0)
|
|
self.assertEqual(sizes, [[7, 17]])
|
|
self.assertEqual(offsets, [[0, 18]])
|
|
_, sizes, offsets = group.entry_map[
|
|
PoolName.INDEXER
|
|
].get_prepared_layer_range_meta([0], 1)
|
|
self.assertEqual(sizes, [[11, 23]])
|
|
self.assertEqual(offsets, [[7, 35]])
|
|
|
|
def test_draft_plan_packs_only_nextn_mtp(self):
|
|
from sglang.srt.speculative import base_spec_worker as spec
|
|
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
|
|
|
draft = SimpleNamespace(
|
|
token_to_kv_pool=object(),
|
|
req_to_token_pool=None,
|
|
model_config=SimpleNamespace(
|
|
num_nextn_predict_layers=0,
|
|
hf_config=SimpleNamespace(architectures=["LlamaForCausalLM"]),
|
|
),
|
|
)
|
|
target = SimpleNamespace(
|
|
spec_algorithm=SpeculativeAlgorithm.EAGLE, req_to_token_pool=None
|
|
)
|
|
worker = SimpleNamespace(
|
|
target_worker=SimpleNamespace(model_runner=target),
|
|
_draft_model_runners=lambda: (draft,),
|
|
)
|
|
for linker_enabled, nextn_layers in (
|
|
(False, 0),
|
|
(True, 0),
|
|
(False, 1),
|
|
(True, 1),
|
|
):
|
|
draft.model_config.num_nextn_predict_layers = nextn_layers
|
|
with (
|
|
self.subTest(linker=linker_enabled, nextn=nextn_layers),
|
|
patch.object(
|
|
spec,
|
|
"get_memory",
|
|
return_value=SimpleNamespace(
|
|
enable_hierarchical_cache=not linker_enabled,
|
|
enable_unified_cache_external_linker=linker_enabled,
|
|
),
|
|
),
|
|
):
|
|
plan = spec.BaseSpecWorker._build_hicache_draft_plan(worker)
|
|
self.assertEqual(
|
|
plan.mode,
|
|
spec.HiCacheDraftMode.PACKED
|
|
if nextn_layers
|
|
else spec.HiCacheDraftMode.SIDECAR,
|
|
)
|
|
self.assertEqual(plan.device_pools, (draft.token_to_kv_pool,))
|
|
self.assertEqual(
|
|
target.mtp_draft_device_pools,
|
|
plan.device_pools if nextn_layers else (),
|
|
)
|
|
|
|
def test_mla_mamba_maps_kv_pages_and_state_slots(self):
|
|
from sglang.srt.mem_cache.hybrid_cache import linker_pool_assembler
|
|
from sglang.srt.mem_cache.memory_pool import (
|
|
HybridLinearKVPool,
|
|
MHATokenToKVPool,
|
|
MLATokenToKVPool,
|
|
)
|
|
|
|
kvcache = HybridLinearKVPool.__new__(HybridLinearKVPool)
|
|
kvcache.start_layer = 0
|
|
kvcache.full_attention_layer_id_mapping = {1: 0, 3: 1}
|
|
kvcache.full_kv_pool = MLATokenToKVPool.__new__(MLATokenToKVPool)
|
|
kvcache.full_kv_pool.kv_buffer = [
|
|
torch.zeros((8, 1, 5), dtype=torch.uint8) for _ in range(2)
|
|
]
|
|
mamba_cache = SimpleNamespace(
|
|
temporal=torch.zeros((2, 4, 3), dtype=torch.uint8),
|
|
conv=[torch.zeros((2, 4, 2), dtype=torch.uint8)],
|
|
)
|
|
draft = MHATokenToKVPool.__new__(MHATokenToKVPool)
|
|
draft.page_size, draft.layer_num = 2, 1
|
|
draft.k_scale_buffer = draft.native_k_scale_buffer = None
|
|
draft.k_buffer = [torch.zeros((8, 3), dtype=torch.uint8)]
|
|
draft.v_buffer = [torch.zeros((8, 3), dtype=torch.uint8)]
|
|
params = SimpleNamespace(
|
|
mtp_draft_device_pools=(),
|
|
sidecar_draft_device_pools=(draft,),
|
|
req_to_token_pool=SimpleNamespace(
|
|
mamba_pool=SimpleNamespace(mamba_cache=mamba_cache, num_mamba_layers=2),
|
|
mamba_map={0: 0, 2: 1},
|
|
mamba_ckpt_pool=None,
|
|
short_conv_pool=SimpleNamespace(enabled=False),
|
|
ngram_pool=SimpleNamespace(enabled=False),
|
|
),
|
|
)
|
|
|
|
def resolve(dcp_size=1):
|
|
with (
|
|
patch.object(
|
|
linker_pool_assembler,
|
|
"get_memory",
|
|
return_value=SimpleNamespace(enable_unified_memory=False),
|
|
),
|
|
patch.object(
|
|
linker_pool_assembler,
|
|
"get_parallel",
|
|
return_value=SimpleNamespace(attn_dcp_size=dcp_size),
|
|
),
|
|
):
|
|
return resolve_hybrid_device_pool_group(
|
|
kvcache=kvcache,
|
|
page_size=2,
|
|
params=params,
|
|
components={ComponentType.FULL, ComponentType.MAMBA},
|
|
)
|
|
|
|
group = resolve()
|
|
self.assertEqual(group.num_layers, 4)
|
|
self.assertFalse(group.rank_replicated)
|
|
kv, mamba = group.entry_map[PoolName.KV], group.entry_map[PoolName.MAMBA]
|
|
self.assertEqual(kv.layer_mapping, {1: 0, 3: 1})
|
|
self.assertEqual(mamba.layer_mapping, {0: 0, 2: 1})
|
|
self.assertEqual(kv.prepare_locations(torch.tensor([2, 3])), [2])
|
|
self.assertEqual(mamba.prepare_locations(torch.tensor([3])), [3])
|
|
self.assertEqual(group.sources[PoolName.DRAFT], PoolName.KV)
|
|
self.assertEqual(group.entry_map[PoolName.DRAFT].layer_mapping, {0: (0, 1)})
|
|
|
|
with self.assertRaisesRegex(ValueError, "plain MLA"):
|
|
resolve(dcp_size=2)
|
|
params.mtp_draft_device_pools = (object(),)
|
|
with self.assertRaisesRegex(ValueError, "plain MLA"):
|
|
resolve()
|
|
params.mtp_draft_device_pools = ()
|
|
kvcache.full_kv_pool = MHATokenToKVPool.__new__(MHATokenToKVPool)
|
|
with self.assertRaisesRegex(ValueError, "plain MLA"):
|
|
resolve()
|
|
|
|
def test_unsupported_strategy_fails_with_context(self):
|
|
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
|
|
|
|
kvcache = SWAKVPool.__new__(SWAKVPool)
|
|
with self.assertRaisesRegex(
|
|
ValueError,
|
|
"does not support the direct external linker: _SwaStrategy",
|
|
):
|
|
resolve_hybrid_device_pool_group(
|
|
kvcache=kvcache,
|
|
page_size=2,
|
|
params=SimpleNamespace(),
|
|
components={ComponentType.FULL, ComponentType.SWA},
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|