[Fix] cache: Support DeepSeek-V4.1 external linker pools (#44623)
Co-authored-by: Zhangheng <hzh0425@apache.org> Co-authored-by: Shuwen Wang <47200617+alphabetc1@users.noreply.github.com>
This commit is contained in:
co-authored by
Zhangheng
Shuwen Wang
parent
407bfc94f9
commit
1ec902e00f
@@ -294,7 +294,6 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
# ROCm replays hits in a forward of their own, which only the hitting
|
||||
# attention-DP ranks would enter; CUDA folds them into the extend.
|
||||
("DP attention on ROCm", get_platform().is_hip and attn_dp_enabled_of(cfg)),
|
||||
("external cache linker", cfg.enable_unified_cache_external_linker),
|
||||
("unified memory", cfg.enable_unified_memory),
|
||||
# Decode receives the last SWA window from prefill and never replays.
|
||||
("PD decode", cfg.disaggregation_mode == "decode"),
|
||||
|
||||
@@ -794,14 +794,14 @@ def _dsv4_indexer_regions(kvcache: Any, page_size: int) -> list[_IndexerRegion]:
|
||||
]
|
||||
|
||||
|
||||
def _dsv4_low_ratio_entries(
|
||||
kvcache: Any, page_size: int, num_host_pages: int, transfer_layer_id_max: int
|
||||
):
|
||||
"""Mirror each shared source once, in FULL-page units. Prefixes end on an even
|
||||
page boundary, so ratio-2's request-scoped ring is rebuilt, not cached."""
|
||||
def _dsv4_low_ratio_regions(
|
||||
kvcache: Any, page_size: int
|
||||
) -> list[tuple[PoolName, Any, list, dict[int, int]]]:
|
||||
"""(name, device_pool, buffers, layer_mapping) per shared source KV and
|
||||
indexer buffer group, in FULL-page units."""
|
||||
import torch
|
||||
|
||||
entries = []
|
||||
regions = []
|
||||
for ratio, names in (
|
||||
(
|
||||
1,
|
||||
@@ -832,7 +832,7 @@ def _dsv4_low_ratio_entries(
|
||||
layer_mapping = {
|
||||
source - kvcache.start_layer: index for index, source in enumerate(sources)
|
||||
}
|
||||
regions = [(names[0], kv_pool, kv_pool.kv_buffer)]
|
||||
regions.append((names[0], kv_pool, kv_pool.kv_buffer, layer_mapping))
|
||||
if index_pool.index_k_with_scale_buffer is not None:
|
||||
index_regions = [(names[1], index_pool.index_k_with_scale_buffer)]
|
||||
else:
|
||||
@@ -850,26 +850,36 @@ def _dsv4_low_ratio_entries(
|
||||
.view(torch.uint8)
|
||||
.reshape(full_pages, -1)
|
||||
)
|
||||
regions.append((name, index_pool, rows))
|
||||
for name, device_pool, buffers in regions:
|
||||
entries.append(
|
||||
build_pool_entry(
|
||||
name=name,
|
||||
host_pool=DeepSeekV4PagedHostPool(
|
||||
pool_name=str(name),
|
||||
device_buffers=buffers,
|
||||
item_bytes=buffers[0].shape[1] * buffers[0].element_size(),
|
||||
num_host_pages=num_host_pages,
|
||||
slot_page_size=page_size,
|
||||
layout=get_memory().hicache_mem_layout,
|
||||
allocator_type=_get_allocator_type(),
|
||||
page_aligned_only=True,
|
||||
),
|
||||
device_pool=device_pool,
|
||||
layer_mapping=layer_mapping,
|
||||
transfer_layer_id_max=transfer_layer_id_max,
|
||||
)
|
||||
regions.append((name, index_pool, rows, layer_mapping))
|
||||
return regions
|
||||
|
||||
|
||||
def _dsv4_low_ratio_entries(
|
||||
kvcache: Any, page_size: int, num_host_pages: int, transfer_layer_id_max: int
|
||||
):
|
||||
"""Mirror shared KV; ratio-2 request state is rebuilt at page boundaries."""
|
||||
entries = []
|
||||
for name, device_pool, buffers, layer_mapping in _dsv4_low_ratio_regions(
|
||||
kvcache, page_size
|
||||
):
|
||||
entries.append(
|
||||
build_pool_entry(
|
||||
name=name,
|
||||
host_pool=DeepSeekV4PagedHostPool(
|
||||
pool_name=str(name),
|
||||
device_buffers=buffers,
|
||||
item_bytes=buffers[0].shape[1] * buffers[0].element_size(),
|
||||
num_host_pages=num_host_pages,
|
||||
slot_page_size=page_size,
|
||||
layout=get_memory().hicache_mem_layout,
|
||||
allocator_type=_get_allocator_type(),
|
||||
page_aligned_only=True,
|
||||
),
|
||||
device_pool=device_pool,
|
||||
layer_mapping=layer_mapping,
|
||||
transfer_layer_id_max=transfer_layer_id_max,
|
||||
)
|
||||
)
|
||||
return entries
|
||||
|
||||
|
||||
|
||||
@@ -264,6 +264,7 @@ def _build_deepseek_v4_device_pool_group(
|
||||
from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import (
|
||||
_dsv4_compressed_region_buffers,
|
||||
_dsv4_indexer_regions,
|
||||
_dsv4_low_ratio_regions,
|
||||
_require_single_row_dsv4_swa_pages,
|
||||
_resolve_deepseek_v4_layer_mappings,
|
||||
)
|
||||
@@ -273,8 +274,11 @@ def _build_deepseek_v4_device_pool_group(
|
||||
|
||||
mappings = _resolve_deepseek_v4_layer_mappings(kvcache)
|
||||
is_unified_kv = getattr(kvcache, "_unified_kv", False)
|
||||
# Encoder SWA replay keeps the target window per request; the draft SWA is
|
||||
# dropped with it, a known draft accept-length cost.
|
||||
has_paged_swa = not is_unified_kv and kvcache.swa_kv_pool is not None
|
||||
entries = []
|
||||
if not is_unified_kv:
|
||||
if has_paged_swa:
|
||||
_require_single_row_dsv4_swa_pages(
|
||||
logical_page_size=kvcache.swa_page_size,
|
||||
physical_page_size=kvcache.swa_kv_pool.page_size,
|
||||
@@ -321,31 +325,36 @@ def _build_deepseek_v4_device_pool_group(
|
||||
)
|
||||
)
|
||||
|
||||
c4_buffers, _ = _dsv4_compressed_region_buffers(kvcache, 4)
|
||||
c128_buffers, _ = _dsv4_compressed_region_buffers(kvcache, 128)
|
||||
|
||||
add(
|
||||
PoolName.DEEPSEEK_V4_C4,
|
||||
PoolName.KV,
|
||||
kvcache.c4_kv_pool,
|
||||
c4_buffers,
|
||||
mappings.c4,
|
||||
)
|
||||
for region in _dsv4_indexer_regions(kvcache, page_size):
|
||||
if mappings.c4:
|
||||
c4_buffers, _ = _dsv4_compressed_region_buffers(kvcache, 4)
|
||||
add(
|
||||
region.name,
|
||||
PoolName.DEEPSEEK_V4_C4,
|
||||
PoolName.KV,
|
||||
kvcache.c4_indexer_kv_pool,
|
||||
region.device_buffers,
|
||||
kvcache.c4_kv_pool,
|
||||
c4_buffers,
|
||||
mappings.c4,
|
||||
)
|
||||
add(
|
||||
PoolName.DEEPSEEK_V4_C128,
|
||||
PoolName.KV,
|
||||
kvcache.c128_kv_pool,
|
||||
c128_buffers,
|
||||
mappings.c128,
|
||||
)
|
||||
for region in _dsv4_indexer_regions(kvcache, page_size):
|
||||
add(
|
||||
region.name,
|
||||
PoolName.KV,
|
||||
kvcache.c4_indexer_kv_pool,
|
||||
region.device_buffers,
|
||||
mappings.c4,
|
||||
)
|
||||
if mappings.c128:
|
||||
c128_buffers, _ = _dsv4_compressed_region_buffers(kvcache, 128)
|
||||
add(
|
||||
PoolName.DEEPSEEK_V4_C128,
|
||||
PoolName.KV,
|
||||
kvcache.c128_kv_pool,
|
||||
c128_buffers,
|
||||
mappings.c128,
|
||||
)
|
||||
for name, device_pool, buffers, layer_mapping in _dsv4_low_ratio_regions(
|
||||
kvcache, page_size
|
||||
):
|
||||
add(name, PoolName.KV, device_pool, buffers, layer_mapping)
|
||||
|
||||
# fp8 two-pool unified_kv: one row index addresses both kv_buffer (fp8 nope) and
|
||||
# kv_buffer_rope (bf16 rope), but unified_region_buffers() above returns the nope
|
||||
@@ -384,6 +393,13 @@ def _build_deepseek_v4_device_pool_group(
|
||||
),
|
||||
mappings.c4_state,
|
||||
)
|
||||
if not has_paged_swa and any(
|
||||
entry.indices_from_pool == PoolName.SWA for entry in entries
|
||||
):
|
||||
raise ValueError(
|
||||
"DeepSeek-V4 compress state pools take SWA page indices, which "
|
||||
"encoder SWA replay does not keep."
|
||||
)
|
||||
return DevicePoolGroup(
|
||||
entries,
|
||||
mappings.transfer_layer_id_max,
|
||||
|
||||
@@ -30,7 +30,6 @@ from typing import TYPE_CHECKING, NamedTuple, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.mem_cache.allocator.swa import is_swa_req_ring
|
||||
from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
DecLockRefParams,
|
||||
MatchResult,
|
||||
@@ -133,6 +132,7 @@ class UnifiedCacheLinker(ABC):
|
||||
"""External KV store reached directly from the device pools."""
|
||||
|
||||
layer_done_counter: object
|
||||
pools: dict[PoolName, object]
|
||||
|
||||
@abstractmethod
|
||||
def lookup(self, rid: str, transfers: list[PoolTransfer]) -> list[int]:
|
||||
@@ -228,9 +228,8 @@ class UnifiedCacheLinkerWrapper:
|
||||
self.cache = cache
|
||||
self.cache_linker = cache_linker
|
||||
swa = cache.components.get(ComponentType.SWA)
|
||||
self._skip_swa = swa is not None and is_swa_req_ring(
|
||||
cache.token_to_kv_pool_allocator
|
||||
)
|
||||
# A store without SWA pages tombstones restored tails instead.
|
||||
self._skip_swa = swa is not None and PoolName.SWA not in cache_linker.pools
|
||||
self._components = tuple(
|
||||
component
|
||||
for component in cache._components_tuple
|
||||
|
||||
@@ -2,6 +2,7 @@ import os
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
import requests
|
||||
@@ -39,6 +40,7 @@ class MooncakeTestServices:
|
||||
self.metadata_process = None
|
||||
self.master_process = None
|
||||
self.store_process = None
|
||||
self._log_paths: dict[str, str] = {}
|
||||
|
||||
@staticmethod
|
||||
def _default_device(protocol: str) -> str:
|
||||
@@ -51,26 +53,29 @@ class MooncakeTestServices:
|
||||
|
||||
def start(self):
|
||||
self.metadata_process = self._launch(
|
||||
"metadata service",
|
||||
[
|
||||
"python3",
|
||||
"-m",
|
||||
"mooncake.http_metadata_server",
|
||||
"--port",
|
||||
str(self.metadata_port),
|
||||
]
|
||||
],
|
||||
)
|
||||
try:
|
||||
self.master_process = self._launch(
|
||||
"master service",
|
||||
[
|
||||
"mooncake_master",
|
||||
"--port",
|
||||
str(self.master_port),
|
||||
"--metrics_port",
|
||||
str(self.master_metrics_port),
|
||||
]
|
||||
],
|
||||
)
|
||||
self._wait_for_core_services()
|
||||
self.store_process = self._launch(
|
||||
"store service",
|
||||
[
|
||||
"mooncake_client",
|
||||
f"--host={self.local_hostname}",
|
||||
@@ -123,15 +128,19 @@ class MooncakeTestServices:
|
||||
return float(line.split()[1])
|
||||
raise AssertionError(f"{name} is missing from Mooncake master metrics")
|
||||
|
||||
@staticmethod
|
||||
def _launch(command, env=None):
|
||||
return subprocess.Popen(
|
||||
command,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.STDOUT,
|
||||
preexec_fn=os.setsid,
|
||||
env=env,
|
||||
def _launch(self, name: str, command, env=None):
|
||||
log = tempfile.NamedTemporaryFile(
|
||||
prefix=f"mooncake-{name.split()[0]}-", suffix=".log", delete=False
|
||||
)
|
||||
self._log_paths[name] = log.name
|
||||
with log:
|
||||
return subprocess.Popen(
|
||||
command,
|
||||
stdout=log,
|
||||
stderr=subprocess.STDOUT,
|
||||
preexec_fn=os.setsid,
|
||||
env=env,
|
||||
)
|
||||
|
||||
def _wait_for_core_services(self, timeout: int = 30):
|
||||
deadline = time.monotonic() + timeout
|
||||
@@ -164,11 +173,15 @@ class MooncakeTestServices:
|
||||
time.sleep(1)
|
||||
raise TimeoutError("Timed out waiting for Mooncake store")
|
||||
|
||||
@staticmethod
|
||||
def _raise_if_exited(process, name: str):
|
||||
def _raise_if_exited(self, process, name: str):
|
||||
returncode = process.poll()
|
||||
if returncode is not None:
|
||||
raise RuntimeError(f"Mooncake {name} exited with code {returncode}")
|
||||
if returncode is None:
|
||||
return
|
||||
with open(self._log_paths[name], errors="replace") as f:
|
||||
tail = "".join(f.readlines()[-30:])
|
||||
raise RuntimeError(
|
||||
f"Mooncake {name} exited with code {returncode}; log tail:\n{tail}"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _stop_process_group(process):
|
||||
|
||||
Reference in New Issue
Block a user