[Fix] cache: Support DeepSeek-V4.1 external linker pools (#44623)

Co-authored-by: Zhangheng <hzh0425@apache.org>
Co-authored-by: Shuwen Wang <47200617+alphabetc1@users.noreply.github.com>
This commit is contained in:
Khoa Pham
2026-10-19 23:40:25 -07:00
committed by GitHub
co-authored by Zhangheng Shuwen Wang
parent 407bfc94f9
commit 1ec902e00f
8 changed files with 493 additions and 83 deletions
@@ -294,7 +294,6 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
# ROCm replays hits in a forward of their own, which only the hitting
# attention-DP ranks would enter; CUDA folds them into the extend.
("DP attention on ROCm", get_platform().is_hip and attn_dp_enabled_of(cfg)),
("external cache linker", cfg.enable_unified_cache_external_linker),
("unified memory", cfg.enable_unified_memory),
# Decode receives the last SWA window from prefill and never replays.
("PD decode", cfg.disaggregation_mode == "decode"),
@@ -794,14 +794,14 @@ def _dsv4_indexer_regions(kvcache: Any, page_size: int) -> list[_IndexerRegion]:
]
def _dsv4_low_ratio_entries(
kvcache: Any, page_size: int, num_host_pages: int, transfer_layer_id_max: int
):
"""Mirror each shared source once, in FULL-page units. Prefixes end on an even
page boundary, so ratio-2's request-scoped ring is rebuilt, not cached."""
def _dsv4_low_ratio_regions(
kvcache: Any, page_size: int
) -> list[tuple[PoolName, Any, list, dict[int, int]]]:
"""(name, device_pool, buffers, layer_mapping) per shared source KV and
indexer buffer group, in FULL-page units."""
import torch
entries = []
regions = []
for ratio, names in (
(
1,
@@ -832,7 +832,7 @@ def _dsv4_low_ratio_entries(
layer_mapping = {
source - kvcache.start_layer: index for index, source in enumerate(sources)
}
regions = [(names[0], kv_pool, kv_pool.kv_buffer)]
regions.append((names[0], kv_pool, kv_pool.kv_buffer, layer_mapping))
if index_pool.index_k_with_scale_buffer is not None:
index_regions = [(names[1], index_pool.index_k_with_scale_buffer)]
else:
@@ -850,26 +850,36 @@ def _dsv4_low_ratio_entries(
.view(torch.uint8)
.reshape(full_pages, -1)
)
regions.append((name, index_pool, rows))
for name, device_pool, buffers in regions:
entries.append(
build_pool_entry(
name=name,
host_pool=DeepSeekV4PagedHostPool(
pool_name=str(name),
device_buffers=buffers,
item_bytes=buffers[0].shape[1] * buffers[0].element_size(),
num_host_pages=num_host_pages,
slot_page_size=page_size,
layout=get_memory().hicache_mem_layout,
allocator_type=_get_allocator_type(),
page_aligned_only=True,
),
device_pool=device_pool,
layer_mapping=layer_mapping,
transfer_layer_id_max=transfer_layer_id_max,
)
regions.append((name, index_pool, rows, layer_mapping))
return regions
def _dsv4_low_ratio_entries(
kvcache: Any, page_size: int, num_host_pages: int, transfer_layer_id_max: int
):
"""Mirror shared KV; ratio-2 request state is rebuilt at page boundaries."""
entries = []
for name, device_pool, buffers, layer_mapping in _dsv4_low_ratio_regions(
kvcache, page_size
):
entries.append(
build_pool_entry(
name=name,
host_pool=DeepSeekV4PagedHostPool(
pool_name=str(name),
device_buffers=buffers,
item_bytes=buffers[0].shape[1] * buffers[0].element_size(),
num_host_pages=num_host_pages,
slot_page_size=page_size,
layout=get_memory().hicache_mem_layout,
allocator_type=_get_allocator_type(),
page_aligned_only=True,
),
device_pool=device_pool,
layer_mapping=layer_mapping,
transfer_layer_id_max=transfer_layer_id_max,
)
)
return entries
@@ -264,6 +264,7 @@ def _build_deepseek_v4_device_pool_group(
from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import (
_dsv4_compressed_region_buffers,
_dsv4_indexer_regions,
_dsv4_low_ratio_regions,
_require_single_row_dsv4_swa_pages,
_resolve_deepseek_v4_layer_mappings,
)
@@ -273,8 +274,11 @@ def _build_deepseek_v4_device_pool_group(
mappings = _resolve_deepseek_v4_layer_mappings(kvcache)
is_unified_kv = getattr(kvcache, "_unified_kv", False)
# Encoder SWA replay keeps the target window per request; the draft SWA is
# dropped with it, a known draft accept-length cost.
has_paged_swa = not is_unified_kv and kvcache.swa_kv_pool is not None
entries = []
if not is_unified_kv:
if has_paged_swa:
_require_single_row_dsv4_swa_pages(
logical_page_size=kvcache.swa_page_size,
physical_page_size=kvcache.swa_kv_pool.page_size,
@@ -321,31 +325,36 @@ def _build_deepseek_v4_device_pool_group(
)
)
c4_buffers, _ = _dsv4_compressed_region_buffers(kvcache, 4)
c128_buffers, _ = _dsv4_compressed_region_buffers(kvcache, 128)
add(
PoolName.DEEPSEEK_V4_C4,
PoolName.KV,
kvcache.c4_kv_pool,
c4_buffers,
mappings.c4,
)
for region in _dsv4_indexer_regions(kvcache, page_size):
if mappings.c4:
c4_buffers, _ = _dsv4_compressed_region_buffers(kvcache, 4)
add(
region.name,
PoolName.DEEPSEEK_V4_C4,
PoolName.KV,
kvcache.c4_indexer_kv_pool,
region.device_buffers,
kvcache.c4_kv_pool,
c4_buffers,
mappings.c4,
)
add(
PoolName.DEEPSEEK_V4_C128,
PoolName.KV,
kvcache.c128_kv_pool,
c128_buffers,
mappings.c128,
)
for region in _dsv4_indexer_regions(kvcache, page_size):
add(
region.name,
PoolName.KV,
kvcache.c4_indexer_kv_pool,
region.device_buffers,
mappings.c4,
)
if mappings.c128:
c128_buffers, _ = _dsv4_compressed_region_buffers(kvcache, 128)
add(
PoolName.DEEPSEEK_V4_C128,
PoolName.KV,
kvcache.c128_kv_pool,
c128_buffers,
mappings.c128,
)
for name, device_pool, buffers, layer_mapping in _dsv4_low_ratio_regions(
kvcache, page_size
):
add(name, PoolName.KV, device_pool, buffers, layer_mapping)
# fp8 two-pool unified_kv: one row index addresses both kv_buffer (fp8 nope) and
# kv_buffer_rope (bf16 rope), but unified_region_buffers() above returns the nope
@@ -384,6 +393,13 @@ def _build_deepseek_v4_device_pool_group(
),
mappings.c4_state,
)
if not has_paged_swa and any(
entry.indices_from_pool == PoolName.SWA for entry in entries
):
raise ValueError(
"DeepSeek-V4 compress state pools take SWA page indices, which "
"encoder SWA replay does not keep."
)
return DevicePoolGroup(
entries,
mappings.transfer_layer_id_max,
@@ -30,7 +30,6 @@ from typing import TYPE_CHECKING, NamedTuple, Optional
import torch
from sglang.srt.mem_cache.allocator.swa import is_swa_req_ring
from sglang.srt.mem_cache.base_prefix_cache import (
DecLockRefParams,
MatchResult,
@@ -133,6 +132,7 @@ class UnifiedCacheLinker(ABC):
"""External KV store reached directly from the device pools."""
layer_done_counter: object
pools: dict[PoolName, object]
@abstractmethod
def lookup(self, rid: str, transfers: list[PoolTransfer]) -> list[int]:
@@ -228,9 +228,8 @@ class UnifiedCacheLinkerWrapper:
self.cache = cache
self.cache_linker = cache_linker
swa = cache.components.get(ComponentType.SWA)
self._skip_swa = swa is not None and is_swa_req_ring(
cache.token_to_kv_pool_allocator
)
# A store without SWA pages tombstones restored tails instead.
self._skip_swa = swa is not None and PoolName.SWA not in cache_linker.pools
self._components = tuple(
component
for component in cache._components_tuple
+27 -14
View File
@@ -2,6 +2,7 @@ import os
import signal
import socket
import subprocess
import tempfile
import time
import requests
@@ -39,6 +40,7 @@ class MooncakeTestServices:
self.metadata_process = None
self.master_process = None
self.store_process = None
self._log_paths: dict[str, str] = {}
@staticmethod
def _default_device(protocol: str) -> str:
@@ -51,26 +53,29 @@ class MooncakeTestServices:
def start(self):
self.metadata_process = self._launch(
"metadata service",
[
"python3",
"-m",
"mooncake.http_metadata_server",
"--port",
str(self.metadata_port),
]
],
)
try:
self.master_process = self._launch(
"master service",
[
"mooncake_master",
"--port",
str(self.master_port),
"--metrics_port",
str(self.master_metrics_port),
]
],
)
self._wait_for_core_services()
self.store_process = self._launch(
"store service",
[
"mooncake_client",
f"--host={self.local_hostname}",
@@ -123,15 +128,19 @@ class MooncakeTestServices:
return float(line.split()[1])
raise AssertionError(f"{name} is missing from Mooncake master metrics")
@staticmethod
def _launch(command, env=None):
return subprocess.Popen(
command,
stdout=subprocess.DEVNULL,
stderr=subprocess.STDOUT,
preexec_fn=os.setsid,
env=env,
def _launch(self, name: str, command, env=None):
log = tempfile.NamedTemporaryFile(
prefix=f"mooncake-{name.split()[0]}-", suffix=".log", delete=False
)
self._log_paths[name] = log.name
with log:
return subprocess.Popen(
command,
stdout=log,
stderr=subprocess.STDOUT,
preexec_fn=os.setsid,
env=env,
)
def _wait_for_core_services(self, timeout: int = 30):
deadline = time.monotonic() + timeout
@@ -164,11 +173,15 @@ class MooncakeTestServices:
time.sleep(1)
raise TimeoutError("Timed out waiting for Mooncake store")
@staticmethod
def _raise_if_exited(process, name: str):
def _raise_if_exited(self, process, name: str):
returncode = process.poll()
if returncode is not None:
raise RuntimeError(f"Mooncake {name} exited with code {returncode}")
if returncode is None:
return
with open(self._log_paths[name], errors="replace") as f:
tail = "".join(f.readlines()[-30:])
raise RuntimeError(
f"Mooncake {name} exited with code {returncode}; log tail:\n{tail}"
)
@staticmethod
def _stop_process_group(process):