Files
sglang/test/registered/unit/mem_cache/test_dsv4_compressed_pools.py
T

376 lines
16 KiB
Python

import unittest
from itertools import product
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import torch
from sglang.kernels.ops.attention.dsv4.kv_layout import (
KVLayout,
is_valid_kv_layout_pair,
)
from sglang.srt.mem_cache.allocator.swa import SWATokenToKVPoolAllocator
from sglang.srt.mem_cache.deepseek_v4_memory_pool import (
DeepSeekV4SingleKVPool,
DeepSeekV4TokenToKVPool,
_CompressedPoolConfig,
_num_dsv4_physical_kv_pages,
)
from sglang.srt.runtime_context import get_context
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
class TestDSV4CompressedPools(CustomTestCase):
def test_physical_kv_pages_cover_reserved_logical_page(self):
size = 8192
self.assertEqual(_num_dsv4_physical_kv_pages(size, 256, 256), 33)
self.assertEqual(_num_dsv4_physical_kv_pages(size, 64, 256), 132)
self.assertGreaterEqual(
_num_dsv4_physical_kv_pages(size, 64, 256) * 64,
size + 256,
)
def test_state_buf_item_covers_one_logical_swa_page(self):
pool = DeepSeekV4TokenToKVPool.__new__(DeepSeekV4TokenToKVPool)
pool._unified_kv = False
pool.swa_page_size = 256
pool.compress_state_pools = []
pool.indexer_compress_state_pools = []
for physical_page_size in (256, 64):
with self.subTest(physical_page_size=physical_page_size):
row_bytes = physical_page_size * 4
buf = torch.empty((8, row_bytes), dtype=torch.uint8)
pool.swa_kv_pool = SimpleNamespace(
page_size=physical_page_size, kv_buffer=[buf]
)
data_ptrs, data_lens, item_lens = pool.get_state_buf_infos()
self.assertEqual(data_ptrs, [buf.data_ptr()])
self.assertEqual(data_lens, [buf.nbytes])
self.assertEqual(item_lens, [256 * 4])
def test_state_buf_infos_without_paged_swa(self):
pool = DeepSeekV4TokenToKVPool.__new__(DeepSeekV4TokenToKVPool)
pool.swa_kv_pool = None
pool.compress_state_pools = []
pool.indexer_compress_state_pools = []
self.assertEqual(pool.get_state_buf_infos(), ([], [], []))
def test_pp_mapping_and_pd_buffer_order(self):
for unified, stage_ratios in product(
(False, True), ([4, 0, 128, 4], [128], [0])
):
with self.subTest(unified=unified, stage_ratios=stage_ratios):
pool = DeepSeekV4TokenToKVPool.__new__(DeepSeekV4TokenToKVPool)
pool._unified_kv = unified
pool.uniform_fp8 = False
pool.kv_layout = KVLayout.V4
pool.compressed_kv_layout_option = None
pool.compressed_pool_configs = {
4: _CompressedPoolConfig(
256, 64, torch.bfloat16, indexer_size=1024
),
128: _CompressedPoolConfig(512, 8, torch.float32),
}
pool.indexer_head_dim = 128
pool.page_size = 256
pool.compression_ratios = [128] + stage_ratios + [4]
pool._stage_start = 1
pool._stage_end = 1 + len(stage_ratios)
def kv_factory(**kwargs):
return SimpleNamespace(
kv_buffer=[
torch.empty((3, kwargs["page_size"]), dtype=torch.uint8)
for _ in range(kwargs["layer_num"])
]
)
# Separate payload/scale buffers exercise the FP4 transfer contract.
indexer_buffers = [
torch.empty((3, width), dtype=torch.uint8)
for _ in range(stage_ratios.count(4))
for width in (32, 2)
]
indexer = SimpleNamespace(
contiguous_page_row_buffers=lambda: indexer_buffers
)
with (
patch.object(pool, "_make_kv_pool", side_effect=kv_factory),
patch.object(pool, "_make_indexer_pool", return_value=indexer),
):
pool._init_compressed_pools(
stage_ratios=stage_ratios,
page_size=256,
dtype=torch.float8_e4m3fn,
device="cpu",
enable_memory_saver=False,
enable_hisparse=False,
kv_pool_cls=DeepSeekV4SingleKVPool,
)
pool._init_compressed_layer_mapping()
self.assertIsNone(pool.layer_mapping[0])
self.assertIsNone(pool.layer_mapping[-1])
for local_id, ratio in enumerate(stage_ratios):
item = pool.layer_mapping[local_id + 1]
self.assertEqual(
item.compress_layer_id, stage_ratios[:local_id].count(ratio)
)
self.assertIs(item.compress_kv_pool, pool.kv_pools.get(ratio))
self.assertIs(pool.c4_kv_pool, pool.kv_pools[4])
self.assertIs(pool.c128_kv_pool, pool.kv_pools[128])
self.assertIs(pool.c4_indexer_kv_pool, pool.index_pools[4])
if unified:
buffers = [
torch.empty((9, 8), dtype=torch.uint8) for _ in stage_ratios
]
pool.unified_kv_pool = SimpleNamespace(
swa_pages=2, kv_buffer=buffers
)
def kv_entries(ratio):
return [
(buf.data_ptr() + 16, 56, 256 // ratio * 8)
for buf, r in zip(buffers, stage_ratios)
if r == ratio
]
else:
def kv_entries(ratio):
return [
(b.data_ptr(), b.nbytes, b[0].nbytes)
for b in pool.kv_pools[ratio].kv_buffer
]
indexer_entries = [
(b.data_ptr(), b.nbytes, b[0].nbytes) for b in indexer_buffers
]
expected = kv_entries(4) + indexer_entries + kv_entries(128)
actual = list(zip(*pool.get_contiguous_buf_infos()))
self.assertEqual(actual, expected)
def test_shared_state_factory_preserves_layouts(self):
pool = DeepSeekV4TokenToKVPool.__new__(DeepSeekV4TokenToKVPool)
pool.compressed_pool_configs = {
4: _CompressedPoolConfig(256, 64, torch.bfloat16, indexer_size=1024),
128: _CompressedPoolConfig(512, 8, torch.float32),
}
pool.compression_ratios = [0, 4, 128]
pool._stage_start, pool._stage_end = 0, 3
pool.index_pools = {4: object()}
pool.qk_nope_head_dim, pool.qk_rope_head_dim = 448, 64
pool.indexer_head_dim = 128
pool.device = "cpu"
pool.swa_page_size = 128
pool.online_mtp_max_draft_tokens = 3
for online in (False, True):
with (
self.subTest(online=online),
patch(
"sglang.srt.mem_cache.deepseek_v4_memory_pool.ONLINE_C128", online
),
patch.object(
pool,
"get_ring_size",
side_effect=lambda r: 8 if r == 4 else (1 if online else 128),
),
):
pool._init_paged_compress_states(False)
c4 = pool.compress_state_pools[1].kv_score_buffer.kv_score
indexer = pool.indexer_compress_state_pools[1].kv_score_buffer.kv_score
c128 = pool.compress_state_pools[2].kv_score_buffer.kv_score
self.assertEqual(c4.shape, (76, 2048))
self.assertEqual(indexer.shape, (76, 512))
self.assertEqual(c128.shape, (40, 1536) if online else (256, 1024))
self.assertEqual(c4.dtype, torch.bfloat16)
self.assertEqual(indexer.dtype, torch.bfloat16)
self.assertEqual(c128.dtype, torch.float32)
self.assertNotEqual(c4.data_ptr(), indexer.data_ptr())
self.assertIsNone(pool.compress_state_pools[0])
self.assertIsNone(pool.indexer_compress_state_pools[2])
def test_indexer_access_uses_layer_ratio_and_waits_only_for_reads(self):
pool = DeepSeekV4TokenToKVPool.__new__(DeepSeekV4TokenToKVPool)
pool.kv_pools = {4: None, 128: None}
pool.compression_ratios = [4, 128, 4]
pool._stage_start, pool._stage_end = 0, 3
pool._init_compressed_layer_mapping()
indexer = MagicMock(page_size=32)
pool.index_pools = {4: indexer}
trace = MagicMock()
trace.attach_mock(indexer, "indexer")
with patch.object(pool, "wait_layer_transfer") as wait:
trace.attach_mock(wait, "wait")
pool.get_index_k_fp4_payload_buffer(2)
pool.get_low_ratio_index_k_dequant(2, "slots")
pool.get_low_ratio_index_k_fp4(2, "slots")
pool.set_index_k_fp4(2, "loc", "cache")
self.assertEqual(
trace.mock_calls,
[
unittest.mock.call.wait(2),
unittest.mock.call.indexer.get_index_k_fp4_payload_buffer(1),
unittest.mock.call.wait(2),
unittest.mock.call.indexer.get_index_k_dequant(1, "slots"),
unittest.mock.call.wait(2),
unittest.mock.call.indexer.get_index_k_fp4(1, "slots"),
unittest.mock.call.indexer.set_index_fp4(1, "loc", "cache"),
],
)
self.assertEqual(pool.get_index_k_page_size(4), 32)
with self.assertRaisesRegex(AssertionError, "No indexer pool"):
pool.get_index_k_page_size(128)
HEAD_DIM = 512
ROPE_DIM = 64
PAGE_SIZE = 256
FULL_SIZE = 4 * PAGE_SIZE
class TestV41KVPoolLayouts(CustomTestCase):
"""A V4.1-layout pool hands the attention kernel page-aligned buffers and
picks the compressed layout each ratio asks for."""
def setUp(self):
super().setUp()
override = get_context().override_server_args(page_size=PAGE_SIZE)
override.install()
self.addCleanup(override.restore)
def make_pool(self, ratios, kv_source_layers, kv_layout, compressed=None, **sizes):
return DeepSeekV4TokenToKVPool(
max_num_reqs=16,
swa_size=FULL_SIZE,
c4_size=sizes.get("c4_size", 0),
c128_size=sizes.get("c128_size", 0),
c4_state_pool_size=sizes.get("c4_state_pool_size", 0),
c128_state_pool_size=sizes.get("c128_state_pool_size", 0),
page_size=PAGE_SIZE,
swa_page_size=PAGE_SIZE,
dtype=torch.float8_e4m3fn,
c4_state_dtype=torch.float32,
c128_state_dtype=torch.float32,
qk_nope_head_dim=HEAD_DIM - ROPE_DIM,
qk_rope_head_dim=ROPE_DIM,
indexer_head_dim=128,
layer_num=len(ratios),
device="cpu",
enable_memory_saver=False,
compression_ratios=ratios,
kv_source_layers=kv_source_layers,
full_size=FULL_SIZE,
kv_layout=kv_layout,
compressed_kv_layout=compressed,
)
def assert_kernel_requirements(self, pool, layout):
"""Pages start on the kernel's alignment, and its
(num_pages, page_size, 1, bytes_per_token) view walks one token per row."""
for buf in pool.kv_buffer:
self.assertEqual(buf.stride(0) % layout.page_align, 0)
bpt = layout.bytes_per_token
view = buf[:, : pool.page_size * bpt].view(
buf.shape[0], pool.page_size, 1, bpt
)
self.assertEqual(view.stride(1), bpt)
self.assertEqual(view.stride(0), pool.bytes_per_page_padded)
def test_v41_pool_buffers(self):
for option, expect in ((None, KVLayout.V41_FP4), ("fp8", KVLayout.V41)):
with self.subTest(compressed=option):
pool = self.make_pool([0, 0, 2, 1, 1], [2, 3], KVLayout.V41, option)
self.assert_kernel_requirements(pool.swa_kv_pool, KVLayout.V41)
self.assertEqual(pool.get_swa_key_bytes_per_token(), 528)
for ratio in (1, 2):
layer_id = pool.sources_by_ratio[ratio][0]
self.assertIs(pool.get_extra_key_layout(layer_id), expect)
self.assertEqual(
pool.get_extra_key_bytes_per_token(layer_id),
expect.bytes_per_token,
)
self.assertTrue(is_valid_kv_layout_pair(pool.kv_layout, expect))
self.assert_kernel_requirements(pool.kv_pools[ratio], expect)
# A pool of the fp4 layout cannot be the main cache.
with self.assertRaises(AssertionError):
self.make_pool([0], [], KVLayout.V41_FP4)
def test_v41_pool_with_c4_c128(self):
pool = self.make_pool(
[0, 4, 128],
[],
KVLayout.V41,
c4_size=PAGE_SIZE,
c128_size=PAGE_SIZE,
c4_state_pool_size=16,
c128_state_pool_size=16,
)
for ratio in (4, 128):
self.assertEqual(pool.kv_pools[ratio].page_size, PAGE_SIZE // ratio)
# The 2-token c128 page is the only production page that pads.
self.assertEqual(pool.kv_pools[128].bytes_per_page_padded, 1536)
class TestPagedDSparkWithEncoderReplay(CustomTestCase):
def setUp(self):
super().setUp()
override = get_context().override_server_args(
enable_encoder_swa_bounded_replay=True,
speculative_algorithm="DSPARK",
speculative_num_draft_tokens=6,
speculative_dspark_block_size=5,
page_size=256,
max_running_requests=2,
chunked_prefill_size=256,
)
override.install()
self.addCleanup(override.restore)
def make_pool(self, *, draft):
return DeepSeekV4TokenToKVPool(
max_num_reqs=2,
num_req_slots=3,
swa_size=1024,
c4_size=0,
c128_size=0,
c4_state_pool_size=0,
c128_state_pool_size=0,
page_size=256,
swa_page_size=256,
dtype=torch.float8_e4m3fn,
c4_state_dtype=torch.float32,
c128_state_dtype=torch.bfloat16,
qk_nope_head_dim=448,
qk_rope_head_dim=64,
indexer_head_dim=128,
layer_num=3,
device="cpu",
enable_memory_saver=False,
compression_ratios=[0, 0, 0],
online_mtp_max_draft_tokens=6,
full_size=2048,
is_draft_worker=draft,
)
def test_target_window_and_draft_paged_storage_share_allocator_mapping(self):
target = self.make_pool(draft=False)
draft = self.make_pool(draft=True)
allocator = SWATokenToKVPoolAllocator(
2048, 1024, 256, torch.float8_e4m3fn, "cpu", target, False
)
draft.register_mapping(allocator.full_to_swa_index_mapping)
allocator.full_to_swa_index_mapping[256:512] = torch.arange(768, 1024)
self.assertEqual(
draft.translate_loc_from_full_to_swa(
torch.tensor([256, 300, 511])
).tolist(),
[768, 812, 1023],
)
if __name__ == "__main__":
unittest.main()