diff --git a/python/sglang/srt/arg_groups/deepseek_v4_hook.py b/python/sglang/srt/arg_groups/deepseek_v4_hook.py index a0076764ac..5ad5c455ad 100644 --- a/python/sglang/srt/arg_groups/deepseek_v4_hook.py +++ b/python/sglang/srt/arg_groups/deepseek_v4_hook.py @@ -294,7 +294,6 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None: # ROCm replays hits in a forward of their own, which only the hitting # attention-DP ranks would enter; CUDA folds them into the extend. ("DP attention on ROCm", get_platform().is_hip and attn_dp_enabled_of(cfg)), - ("external cache linker", cfg.enable_unified_cache_external_linker), ("unified memory", cfg.enable_unified_memory), # Decode receives the last SWA window from prefill and never replays. ("PD decode", cfg.disaggregation_mode == "decode"), diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py index d5526623d0..b810d65173 100644 --- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py +++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py @@ -794,14 +794,14 @@ def _dsv4_indexer_regions(kvcache: Any, page_size: int) -> list[_IndexerRegion]: ] -def _dsv4_low_ratio_entries( - kvcache: Any, page_size: int, num_host_pages: int, transfer_layer_id_max: int -): - """Mirror each shared source once, in FULL-page units. Prefixes end on an even - page boundary, so ratio-2's request-scoped ring is rebuilt, not cached.""" +def _dsv4_low_ratio_regions( + kvcache: Any, page_size: int +) -> list[tuple[PoolName, Any, list, dict[int, int]]]: + """(name, device_pool, buffers, layer_mapping) per shared source KV and + indexer buffer group, in FULL-page units.""" import torch - entries = [] + regions = [] for ratio, names in ( ( 1, @@ -832,7 +832,7 @@ def _dsv4_low_ratio_entries( layer_mapping = { source - kvcache.start_layer: index for index, source in enumerate(sources) } - regions = [(names[0], kv_pool, kv_pool.kv_buffer)] + regions.append((names[0], kv_pool, kv_pool.kv_buffer, layer_mapping)) if index_pool.index_k_with_scale_buffer is not None: index_regions = [(names[1], index_pool.index_k_with_scale_buffer)] else: @@ -850,26 +850,36 @@ def _dsv4_low_ratio_entries( .view(torch.uint8) .reshape(full_pages, -1) ) - regions.append((name, index_pool, rows)) - for name, device_pool, buffers in regions: - entries.append( - build_pool_entry( - name=name, - host_pool=DeepSeekV4PagedHostPool( - pool_name=str(name), - device_buffers=buffers, - item_bytes=buffers[0].shape[1] * buffers[0].element_size(), - num_host_pages=num_host_pages, - slot_page_size=page_size, - layout=get_memory().hicache_mem_layout, - allocator_type=_get_allocator_type(), - page_aligned_only=True, - ), - device_pool=device_pool, - layer_mapping=layer_mapping, - transfer_layer_id_max=transfer_layer_id_max, - ) + regions.append((name, index_pool, rows, layer_mapping)) + return regions + + +def _dsv4_low_ratio_entries( + kvcache: Any, page_size: int, num_host_pages: int, transfer_layer_id_max: int +): + """Mirror shared KV; ratio-2 request state is rebuilt at page boundaries.""" + entries = [] + for name, device_pool, buffers, layer_mapping in _dsv4_low_ratio_regions( + kvcache, page_size + ): + entries.append( + build_pool_entry( + name=name, + host_pool=DeepSeekV4PagedHostPool( + pool_name=str(name), + device_buffers=buffers, + item_bytes=buffers[0].shape[1] * buffers[0].element_size(), + num_host_pages=num_host_pages, + slot_page_size=page_size, + layout=get_memory().hicache_mem_layout, + allocator_type=_get_allocator_type(), + page_aligned_only=True, + ), + device_pool=device_pool, + layer_mapping=layer_mapping, + transfer_layer_id_max=transfer_layer_id_max, ) + ) return entries diff --git a/python/sglang/srt/mem_cache/hybrid_cache/linker_pool_assembler.py b/python/sglang/srt/mem_cache/hybrid_cache/linker_pool_assembler.py index 9d663790ab..334f0df9d0 100644 --- a/python/sglang/srt/mem_cache/hybrid_cache/linker_pool_assembler.py +++ b/python/sglang/srt/mem_cache/hybrid_cache/linker_pool_assembler.py @@ -264,6 +264,7 @@ def _build_deepseek_v4_device_pool_group( from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import ( _dsv4_compressed_region_buffers, _dsv4_indexer_regions, + _dsv4_low_ratio_regions, _require_single_row_dsv4_swa_pages, _resolve_deepseek_v4_layer_mappings, ) @@ -273,8 +274,11 @@ def _build_deepseek_v4_device_pool_group( mappings = _resolve_deepseek_v4_layer_mappings(kvcache) is_unified_kv = getattr(kvcache, "_unified_kv", False) + # Encoder SWA replay keeps the target window per request; the draft SWA is + # dropped with it, a known draft accept-length cost. + has_paged_swa = not is_unified_kv and kvcache.swa_kv_pool is not None entries = [] - if not is_unified_kv: + if has_paged_swa: _require_single_row_dsv4_swa_pages( logical_page_size=kvcache.swa_page_size, physical_page_size=kvcache.swa_kv_pool.page_size, @@ -321,31 +325,36 @@ def _build_deepseek_v4_device_pool_group( ) ) - c4_buffers, _ = _dsv4_compressed_region_buffers(kvcache, 4) - c128_buffers, _ = _dsv4_compressed_region_buffers(kvcache, 128) - - add( - PoolName.DEEPSEEK_V4_C4, - PoolName.KV, - kvcache.c4_kv_pool, - c4_buffers, - mappings.c4, - ) - for region in _dsv4_indexer_regions(kvcache, page_size): + if mappings.c4: + c4_buffers, _ = _dsv4_compressed_region_buffers(kvcache, 4) add( - region.name, + PoolName.DEEPSEEK_V4_C4, PoolName.KV, - kvcache.c4_indexer_kv_pool, - region.device_buffers, + kvcache.c4_kv_pool, + c4_buffers, mappings.c4, ) - add( - PoolName.DEEPSEEK_V4_C128, - PoolName.KV, - kvcache.c128_kv_pool, - c128_buffers, - mappings.c128, - ) + for region in _dsv4_indexer_regions(kvcache, page_size): + add( + region.name, + PoolName.KV, + kvcache.c4_indexer_kv_pool, + region.device_buffers, + mappings.c4, + ) + if mappings.c128: + c128_buffers, _ = _dsv4_compressed_region_buffers(kvcache, 128) + add( + PoolName.DEEPSEEK_V4_C128, + PoolName.KV, + kvcache.c128_kv_pool, + c128_buffers, + mappings.c128, + ) + for name, device_pool, buffers, layer_mapping in _dsv4_low_ratio_regions( + kvcache, page_size + ): + add(name, PoolName.KV, device_pool, buffers, layer_mapping) # fp8 two-pool unified_kv: one row index addresses both kv_buffer (fp8 nope) and # kv_buffer_rope (bf16 rope), but unified_region_buffers() above returns the nope @@ -384,6 +393,13 @@ def _build_deepseek_v4_device_pool_group( ), mappings.c4_state, ) + if not has_paged_swa and any( + entry.indices_from_pool == PoolName.SWA for entry in entries + ): + raise ValueError( + "DeepSeek-V4 compress state pools take SWA page indices, which " + "encoder SWA replay does not keep." + ) return DevicePoolGroup( entries, mappings.transfer_layer_id_max, diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_cache_linker.py b/python/sglang/srt/mem_cache/unified_cache/unified_cache_linker.py index 3d31507e5f..64f341e178 100644 --- a/python/sglang/srt/mem_cache/unified_cache/unified_cache_linker.py +++ b/python/sglang/srt/mem_cache/unified_cache/unified_cache_linker.py @@ -30,7 +30,6 @@ from typing import TYPE_CHECKING, NamedTuple, Optional import torch -from sglang.srt.mem_cache.allocator.swa import is_swa_req_ring from sglang.srt.mem_cache.base_prefix_cache import ( DecLockRefParams, MatchResult, @@ -133,6 +132,7 @@ class UnifiedCacheLinker(ABC): """External KV store reached directly from the device pools.""" layer_done_counter: object + pools: dict[PoolName, object] @abstractmethod def lookup(self, rid: str, transfers: list[PoolTransfer]) -> list[int]: @@ -228,9 +228,8 @@ class UnifiedCacheLinkerWrapper: self.cache = cache self.cache_linker = cache_linker swa = cache.components.get(ComponentType.SWA) - self._skip_swa = swa is not None and is_swa_req_ring( - cache.token_to_kv_pool_allocator - ) + # A store without SWA pages tombstones restored tails instead. + self._skip_swa = swa is not None and PoolName.SWA not in cache_linker.pools self._components = tuple( component for component in cache._components_tuple diff --git a/python/sglang/test/mooncake_utils.py b/python/sglang/test/mooncake_utils.py index f8cee645b0..ac1058fb0d 100644 --- a/python/sglang/test/mooncake_utils.py +++ b/python/sglang/test/mooncake_utils.py @@ -2,6 +2,7 @@ import os import signal import socket import subprocess +import tempfile import time import requests @@ -39,6 +40,7 @@ class MooncakeTestServices: self.metadata_process = None self.master_process = None self.store_process = None + self._log_paths: dict[str, str] = {} @staticmethod def _default_device(protocol: str) -> str: @@ -51,26 +53,29 @@ class MooncakeTestServices: def start(self): self.metadata_process = self._launch( + "metadata service", [ "python3", "-m", "mooncake.http_metadata_server", "--port", str(self.metadata_port), - ] + ], ) try: self.master_process = self._launch( + "master service", [ "mooncake_master", "--port", str(self.master_port), "--metrics_port", str(self.master_metrics_port), - ] + ], ) self._wait_for_core_services() self.store_process = self._launch( + "store service", [ "mooncake_client", f"--host={self.local_hostname}", @@ -123,15 +128,19 @@ class MooncakeTestServices: return float(line.split()[1]) raise AssertionError(f"{name} is missing from Mooncake master metrics") - @staticmethod - def _launch(command, env=None): - return subprocess.Popen( - command, - stdout=subprocess.DEVNULL, - stderr=subprocess.STDOUT, - preexec_fn=os.setsid, - env=env, + def _launch(self, name: str, command, env=None): + log = tempfile.NamedTemporaryFile( + prefix=f"mooncake-{name.split()[0]}-", suffix=".log", delete=False ) + self._log_paths[name] = log.name + with log: + return subprocess.Popen( + command, + stdout=log, + stderr=subprocess.STDOUT, + preexec_fn=os.setsid, + env=env, + ) def _wait_for_core_services(self, timeout: int = 30): deadline = time.monotonic() + timeout @@ -164,11 +173,15 @@ class MooncakeTestServices: time.sleep(1) raise TimeoutError("Timed out waiting for Mooncake store") - @staticmethod - def _raise_if_exited(process, name: str): + def _raise_if_exited(self, process, name: str): returncode = process.poll() - if returncode is not None: - raise RuntimeError(f"Mooncake {name} exited with code {returncode}") + if returncode is None: + return + with open(self._log_paths[name], errors="replace") as f: + tail = "".join(f.readlines()[-30:]) + raise RuntimeError( + f"Mooncake {name} exited with code {returncode}; log tail:\n{tail}" + ) @staticmethod def _stop_process_group(process): diff --git a/test/registered/radix_cache/unified_radix_tree/linker/test_unified_cache_linker_kl_dsv41.py b/test/registered/radix_cache/unified_radix_tree/linker/test_unified_cache_linker_kl_dsv41.py new file mode 100644 index 0000000000..f2dad653ea --- /dev/null +++ b/test/registered/radix_cache/unified_radix_tree/linker/test_unified_cache_linker_kl_dsv41.py @@ -0,0 +1,156 @@ +"""DeepSeek-V4.1-Flash UnifiedRadixCache direct-linker load-back KL tests.""" + +import json +import unittest + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kits.unified_radix_cache_kit import UnifiedRadixTreeTestMixin +from sglang.test.kl_multiturn_utils import get_input_ids +from sglang.test.mooncake_utils import MooncakeTestServices +from sglang.test.test_utils import ( + CustomTestCase, + find_available_port, + popen_launch_server, + terminate_and_kill_process_tree, + try_cached_model, +) + +register_cuda_ci(est_time=900, stage="extra-b", runner_config="8-gpu-h200") + +MODEL = "deepseek-ai/DeepSeek-V4.1-Flash" +LAUNCH_TIMEOUT = 3600 +PAGE_SIZE = 256 +KL_THRESHOLD = 0.03 +PREFIX_LEN = 2048 +MAX_NEW_TOKENS = 64 + + +class _DSV41FlashLinkerServer: + """The small device pool evicts every turn, so each case must restore + at least one page through the Mooncake direct linker.""" + + extra_args = [] + + @classmethod + def setUpClass(cls): + cls.model = try_cached_model(MODEL) + cls.base_url = f"http://127.0.0.1:{find_available_port(30000)}" + cls.mooncake = MooncakeTestServices() + cls.mooncake.start() + cls.addClassCleanup(cls.mooncake.stop) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=LAUNCH_TIMEOUT, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--ep-size", + "8", + "--attention-backend", + "dsv4", + "--moe-runner-backend", + "flashinfer_mxfp4", + "--speculative-algorithm", + "DSPARK", + "--speculative-dspark-block-size", + "5", + "--page-size", + str(PAGE_SIZE), + "--chunked-prefill-size", + "4096", + "--mem-fraction-static", + "0.8", + "--max-total-tokens", + "8192", + "--max-running-requests", + "64", + "--cuda-graph-max-bs-decode", + "64", + "--enable-cache-report", + "--enable-unified-cache-external-linker", + "--hicache-storage-backend-extra-config", + json.dumps({"enable_group_semantics": True}), + *cls.extra_args, + ], + env=cls.mooncake.server_env(), + ) + cls.addClassCleanup(terminate_and_kill_process_tree, cls.process) + cls.input_ids = get_input_ids(cls.model, num_samples=18) + + def _record_cache_result(self, result, expected_cached_tokens, label): + meta_info = result["meta_info"] + cached_tokens = int(meta_info["cached_tokens"]) + minimum = max(0, expected_cached_tokens - PAGE_SIZE) + self.assertGreaterEqual( + cached_tokens, + minimum, + f"{label}: expected cached_tokens >= {minimum}, got {cached_tokens}", + ) + details = meta_info.get("cached_tokens_details") or {} + remote_tokens = int(details.get("host", 0)) + self._direct_remote_tokens += remote_tokens + if remote_tokens: + print(f"{label}: Direct load-back confirmed for {remote_tokens} tokens") + + def _run_linker_case(self, test_case): + self._direct_remote_tokens = 0 + test_case() + print(f"Direct load-back total: {self._direct_remote_tokens} tokens") + self.assertGreater( + self._direct_remote_tokens, + 0, + "Expected this case to load KV through the Mooncake Direct Linker", + ) + + +class TestDSV41FlashLinkerKLPagedSWA( + _DSV41FlashLinkerServer, UnifiedRadixTreeTestMixin, CustomTestCase +): + """Target and packed DSpark draft SWA pages restore alongside the shared KV.""" + + page_size = PAGE_SIZE + kl_threshold = KL_THRESHOLD + sampling_temperature = 0 + max_new_tokens = MAX_NEW_TOKENS + prefix_len = PREFIX_LEN + decode_hit_request_batch_size = 3 + decode_hit_inter_batch_delay_s = 0.5 + + @unittest.skip("Linker CI targets Direct load-back KL accuracy") + def test_gsm8k(self): + pass + + @unittest.skip("Linker CI targets Direct load-back KL accuracy") + def test_mmlu(self): + pass + + def prefill_cache_assert(self, result, prefix_len, label): + self._record_cache_result(result, prefix_len, label) + + def decode_cache_assert(self, result, history_len, output_len, label): + self._record_cache_result(result, history_len + output_len, label) + + def test_multiturn_logprobs_match(self): + self._run_linker_case(super().test_multiturn_logprobs_match) + + def test_multiturn_prefill_cache_hit_branching(self): + self._run_linker_case(super().test_multiturn_prefill_cache_hit_branching) + + def test_multiturn_decode_cache_hit_branching(self): + self._run_linker_case(super().test_multiturn_decode_cache_hit_branching) + + +class TestDSV41FlashLinkerKLEncoderReplay(TestDSV41FlashLinkerKLPagedSWA): + """The per-request window is rebuilt on restore instead of stored.""" + + extra_args = [ + "--enable-encoder-swa-bounded-replay", + "--cuda-graph-backend-prefill", + "disabled", + ] + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/mem_cache/test_linker_pool_assembler.py b/test/registered/unit/mem_cache/test_linker_pool_assembler.py index 9a0c6b6378..09a2bedd6b 100644 --- a/test/registered/unit/mem_cache/test_linker_pool_assembler.py +++ b/test/registered/unit/mem_cache/test_linker_pool_assembler.py @@ -259,6 +259,206 @@ class TestHybridDevicePoolAssembler(CustomTestCase): self.assertEqual(sizes, [[3, 17]]) self.assertEqual(offsets, [[3, 22]]) + def test_deepseek_v41_shared_kv_and_split_indexer_pages(self): + from sglang.srt.mem_cache.deepseek_v4_memory_pool import ( + DeepSeekV4LayerItem, + DeepSeekV4TokenToKVPool, + ) + + def index_pool(layers, rows, width): + return SimpleNamespace( + page_size=2, + index_k_with_scale_buffer=None, + index_k_payload_buffer=[ + torch.arange(rows * width, dtype=torch.uint8).reshape(rows, width) + for _ in range(layers) + ], + index_k_scale_buffer=[ + torch.zeros((rows, 2), dtype=torch.uint8) for _ in range(layers) + ], + ) + + pool = DeepSeekV4TokenToKVPool.__new__(DeepSeekV4TokenToKVPool) + pool._unified_kv = False + pool.start_layer, pool.end_layer = 1, 7 + pool.swa_page_size = 8 + pool.swa_kv_pool = SimpleNamespace( + page_size=8, + kv_buffer=[torch.zeros((5, 3), dtype=torch.uint8) for _ in range(6)], + ) + pool.c4_kv_pool = pool.c128_kv_pool = pool.c4_indexer_kv_pool = None + pool.sources_by_ratio = {1: [4], 2: [2, 6]} + pool.kv_pools = { + 1: SimpleNamespace(kv_buffer=[torch.zeros((5, 7), dtype=torch.uint8)]), + 2: SimpleNamespace( + kv_buffer=[torch.zeros((5, 11), dtype=torch.uint8) for _ in range(2)] + ), + } + pool.index_pools = {1: index_pool(1, 23, 4), 2: index_pool(2, 13, 6)} + pool.layer_mapping = [ + DeepSeekV4LayerItem(ratio, layer) + for ratio, layer in [ + (0, -1), + (0, -1), + (2, 0), + (2, 0), + (1, 0), + (1, 0), + (2, 1), + ] + ] + pool.compress_state_pools = [None] * 7 + pool.indexer_compress_state_pools = [None] * 7 + draft = SimpleNamespace( + swa_kv_pool=SimpleNamespace( + kv_buffer=[torch.zeros((5, 13), dtype=torch.uint8) for _ in range(2)] + ) + ) + group = resolve_hybrid_device_pool_group( + kvcache=pool, + page_size=8, + params=SimpleNamespace(mtp_draft_device_pools=(draft,)), + components={ComponentType.FULL, ComponentType.SWA}, + ) + self.assertEqual( + set(group.entry_map), + { + PoolName.SWA, + PoolName.DEEPSEEK_V4_C1, + PoolName.DEEPSEEK_V4_C1_INDEXER, + PoolName.DEEPSEEK_V4_C1_INDEXER_SCALE, + PoolName.DEEPSEEK_V4_C2, + PoolName.DEEPSEEK_V4_C2_INDEXER, + PoolName.DEEPSEEK_V4_C2_INDEXER_SCALE, + }, + ) + self.assertEqual(group.entry_map[PoolName.DEEPSEEK_V4_C1].layer_mapping, {3: 0}) + c2 = group.entry_map[PoolName.DEEPSEEK_V4_C2] + self.assertEqual(c2.layer_mapping, {1: 0, 5: 1}) + self.assertIsNone(c2.get_prepared_layer_range_meta([1], 2)) + self.assertEqual( + c2.get_prepared_layer_range_meta([1], 5), + ([[pool.kv_pools[2].kv_buffer[1][1].data_ptr()]], [[11]], [[11]]), + ) + full = PoolTransfer( + name=PoolName.KV, keys=["page"], device_indices=torch.arange(8, 16) + ) + swa = PoolTransfer( + name=PoolName.SWA, + keys=["tail"], + device_indices=torch.arange(8, 16), + hit_policy=PoolHitPolicy.TRAILING_PAGES, + ) + transfers = {t.name: t for t in group.resolve_transfers([full, swa])} + self.assertEqual(transfers[PoolName.DEEPSEEK_V4_C2].keys, ["page"]) + self.assertEqual( + transfers[PoolName.DEEPSEEK_V4_C2].hit_policy, PoolHitPolicy.ALL_PAGES + ) + self.assertEqual(transfers[PoolName.SWA].keys, ["tail"]) + for ratio, name, width, raw_row in [ + (1, PoolName.DEEPSEEK_V4_C1_INDEXER, 16, 4), + (2, PoolName.DEEPSEEK_V4_C2_INDEXER, 12, 2), + ]: + entry = group.entry_map[name] + pointers, sizes = entry.get_page_buffer_meta(torch.arange(8, 16)) + original = pool.index_pools[ratio].index_k_payload_buffer + self.assertEqual(pointers, [b[raw_row].data_ptr() for b in original]) + self.assertEqual(sizes, [width] * len(original)) + self.assertTrue( + torch.equal( + entry.components[0][0][1], + original[0][raw_row : raw_row * 2].flatten(), + ) + ) + self.assertEqual( + group.entry_map[PoolName.DEEPSEEK_V4_C1_INDEXER_SCALE].get_page_buffer_meta( + torch.arange(8, 16) + )[1], + [8], + ) + self.assertEqual( + group.entry_map[PoolName.DEEPSEEK_V4_C2_INDEXER_SCALE].get_page_buffer_meta( + torch.arange(8, 16) + )[1], + [4, 4], + ) + _, sizes, offsets = group.entry_map[PoolName.SWA].get_prepared_layer_range_meta( + [1], 0 + ) + self.assertEqual(sizes, [[3, 13]]) + self.assertEqual(offsets, [[0, 18]]) + + def test_deepseek_v41_request_window_drops_swa(self): + """Encoder replay leaves no target SWA pool: skip SWA and the draft SWA + rather than dereference them, and refuse pools that take SWA indices.""" + from sglang.srt.mem_cache.deepseek_v4_memory_pool import ( + DeepSeekV4LayerItem, + DeepSeekV4TokenToKVPool, + ) + + def state_pool(): + return SimpleNamespace( + ring_size=2, + kv_score_buffer=SimpleNamespace(kv_score=torch.zeros((4, 3))), + ) + + pool = DeepSeekV4TokenToKVPool.__new__(DeepSeekV4TokenToKVPool) + pool._unified_kv = False + pool.start_layer, pool.end_layer = 0, 2 + pool.swa_page_size = 8 + pool.swa_kv_pool = None + pool.c4_kv_pool = pool.c128_kv_pool = pool.c4_indexer_kv_pool = None + pool.sources_by_ratio = {1: [1]} + pool.kv_pools = { + 1: SimpleNamespace(kv_buffer=[torch.zeros((3, 7), dtype=torch.uint8)]) + } + pool.index_pools = { + 1: SimpleNamespace( + page_size=2, + index_k_with_scale_buffer=[torch.zeros((9, 4), dtype=torch.uint8)], + ) + } + pool.layer_mapping = [DeepSeekV4LayerItem(0, -1), DeepSeekV4LayerItem(1, 0)] + pool.compress_state_pools = [None, None] + pool.indexer_compress_state_pools = [None, None] + draft = SimpleNamespace( + swa_kv_pool=SimpleNamespace( + kv_buffer=[torch.zeros((3, 5), dtype=torch.uint8)] + ) + ) + params = SimpleNamespace(mtp_draft_device_pools=(draft,)) + group = resolve_hybrid_device_pool_group( + kvcache=pool, + page_size=8, + params=params, + components={ComponentType.FULL, ComponentType.SWA}, + ) + self.assertEqual( + set(group.entry_map), + {PoolName.DEEPSEEK_V4_C1, PoolName.DEEPSEEK_V4_C1_INDEXER}, + ) + indexer = group.entry_map[PoolName.DEEPSEEK_V4_C1_INDEXER] + _, sizes = indexer.get_page_buffer_meta(torch.arange(8, 16)) + self.assertEqual(sizes, [16]) + + pool.layer_mapping[0] = DeepSeekV4LayerItem(4, 0) + pool.c4_kv_pool = SimpleNamespace( + kv_buffer=[torch.zeros((3, 5), dtype=torch.uint8)], + bytes_per_page_padded=5, + ) + pool.c4_indexer_kv_pool = SimpleNamespace( + index_k_with_scale_buffer=[torch.zeros((3, 6), dtype=torch.uint8)] + ) + pool.compress_state_pools[0] = state_pool() + pool.indexer_compress_state_pools[0] = state_pool() + with self.assertRaisesRegex(ValueError, "SWA page indices"): + resolve_hybrid_device_pool_group( + kvcache=pool, + page_size=8, + params=params, + components={ComponentType.FULL, ComponentType.SWA}, + ) + def test_unified_deepseek_v4_uses_only_compressed_pools(self): from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4LayerItem diff --git a/test/registered/unit/mem_cache/test_unified_cache_linker.py b/test/registered/unit/mem_cache/test_unified_cache_linker.py index bf75811fe4..69b42bca9d 100644 --- a/test/registered/unit/mem_cache/test_unified_cache_linker.py +++ b/test/registered/unit/mem_cache/test_unified_cache_linker.py @@ -68,6 +68,7 @@ class _FakeLinker(UnifiedCacheLinker): self.completed_offloads = [] self.reset_count = 0 self.closed = False + self.pools = {} def lookup(self, rid, transfers): return list(self.restorable) @@ -201,6 +202,7 @@ class _InMemoryUnifiedCacheLinker(UnifiedCacheLinker): def __init__(self, stored_keys=None): self.layer_done_counter = object() + self.pools = dict.fromkeys(PoolName) self.stored_keys = defaultdict(set) if stored_keys is None else stored_keys self.lookup_calls = [] self.offload_calls = [] @@ -1334,6 +1336,23 @@ def test_cache_without_swa_needs_no_reprefill(): assert cache.swa_reprefill_tail_tokens() == 0 +@pytest.mark.parametrize("swa_stored", [True, False]) +def test_skip_swa_when_linker_store_lacks_swa(swa_stored): + """A store without SWA pages must tombstone restored tails, not mark them hit.""" + linker = _FakeLinker() + if swa_stored: + linker.pools[PoolName.SWA] = object() + swa = object() + cache = _cache_for_wrapper( + components={ComponentType.SWA: swa}, + _components_tuple=(swa,), + tree_components=(ComponentType.FULL, ComponentType.SWA), + ) + wrapper = UnifiedCacheLinkerWrapper(cache, linker) + assert wrapper._skip_swa is (not swa_stored) + assert (swa in wrapper._components) is swa_stored + + @pytest.fixture def full_linker_component(): def build_transfer(phase, node, keys): @@ -1357,9 +1376,7 @@ def full_linker_component(): ) -def test_linker_filters_request_relative_swa_from_lookup( - full_linker_component, -): +def test_linker_filters_unstored_swa_from_lookup(full_linker_component): full = full_linker_component swa = SWAComponent.__new__(SWAComponent) swa.build_external_linker_transfer = MagicMock( @@ -1369,7 +1386,6 @@ def test_linker_filters_request_relative_swa_from_lookup( cache = _cache_for_wrapper( _components_tuple=(full, swa), components={ComponentType.FULL: full, ComponentType.SWA: swa}, - token_to_kv_pool_allocator=_swa_allocator(True), tree_core=SimpleNamespace(enable_external_cache_linker=False, is_eagle=False), page_size=2, _all_reduce_attn_groups=lambda value, op: None, @@ -1398,12 +1414,12 @@ def test_linker_filters_request_relative_swa_from_lookup( swa.build_external_linker_transfer.assert_not_called() -@pytest.mark.parametrize("swa_req_ring", [False, True], ids=["paged", "request-ring"]) +@pytest.mark.parametrize("swa_stored", [True, False], ids=["stored", "unstored"]) @pytest.mark.parametrize( "completion", [True, False, None], ids=["success", "failure", "reset"] ) def test_offload_filters_tree_core_swa_transfers_and_preserves_lifecycle( - swa_req_ring, completion + swa_stored, completion ): node = SimpleNamespace( id=7, external_cache_stored=False, write_through_pending_id=None @@ -1418,7 +1434,6 @@ def test_offload_filters_tree_core_swa_transfers_and_preserves_lifecycle( cache = _cache_for_wrapper( components={ComponentType.SWA: swa}, _components_tuple=(swa,), - token_to_kv_pool_allocator=_swa_allocator(swa_req_ring), tree_core=core, inc_lock_ref=MagicMock( return_value=SimpleNamespace(to_dec_params=lambda: lock_params) @@ -1426,10 +1441,12 @@ def test_offload_filters_tree_core_swa_transfers_and_preserves_lifecycle( dec_lock_ref=MagicMock(), ) backend = _FakeLinker() + if swa_stored: + backend.pools[PoolName.SWA] = object() wrapper = UnifiedCacheLinkerWrapper(cache, backend) wrapper.offload_nodes([node.id, node.id]) - expected = transfers[:1] if swa_req_ring else transfers + expected = transfers if swa_stored else transfers[:1] assert backend.queued_offloads == [expected] assert core.offload_transfers == transfers assert node.write_through_pending_id == node.id @@ -1457,25 +1474,24 @@ def test_offload_filters_tree_core_swa_transfers_and_preserves_lifecycle( @pytest.mark.parametrize( - "swa_req_ring,previous_boundary,expected_boundary", + "swa_stored,previous_boundary,expected_boundary", [ - pytest.param(True, None, 4, id="unified-tombstones"), - pytest.param(True, 8, 8, id="preserve-existing-boundary"), - pytest.param(False, None, 2, id="paged-prepare-boundary"), - pytest.param(None, None, 2, id="other-allocator-prepare-boundary"), + pytest.param(False, None, 4, id="unstored-tombstones"), + pytest.param(False, 8, 8, id="preserve-existing-boundary"), + pytest.param(True, None, 2, id="stored-prepare-boundary"), ], ) @pytest.mark.parametrize("load_refused", [False, True], ids=["queued", "refused"]) def test_linker_load_preserves_swa_boundaries( full_linker_component, - swa_req_ring, + swa_stored, previous_boundary, expected_boundary, load_refused, ): full = full_linker_component swa = SWAComponent.__new__(SWAComponent) - participates = not swa_req_ring + participates = swa_stored def prepare(phase, req, full_transfer, transfer, prefix_len, **kwargs): if phase == ExternalLinkerLoadPhase.PREPARE: @@ -1495,11 +1511,12 @@ def test_linker_load_preserves_swa_boundaries( _components_tuple=(full, swa), page_size=2, components={ComponentType.FULL: full, ComponentType.SWA: swa}, - token_to_kv_pool_allocator=_swa_allocator(swa_req_ring), insert=MagicMock(side_effect=AssertionError("the tail must stay private")), ) backend = _FakeLinker() backend.load = MagicMock(return_value=not load_refused) + if swa_stored: + backend.pools[PoolName.SWA] = object() wrapper = UnifiedCacheLinkerWrapper(cache, backend) wrapper.hit_markers["rid"] = ExternalCacheHitMarker( tail_hashes=["a", "b"],