[Feat] deepseek-v4.1: Run encoder SWA bounded replay on PD prefill workers (#44952)

This commit is contained in:
Liangsheng Yin
2026-10-18 00:07:55 -07:00
committed by GitHub
parent 520df8a491
commit df966f2345
9 changed files with 104 additions and 19 deletions
@@ -297,7 +297,13 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
("DP attention on ROCm", get_platform().is_hip and attn_dp_enabled_of(cfg)),
("external cache linker", cfg.enable_unified_cache_external_linker),
("unified memory", cfg.enable_unified_memory),
("PD disaggregation", cfg.disaggregation_mode != "null"),
# Decode receives the last SWA window from prefill and never replays.
("PD decode", cfg.disaggregation_mode == "decode"),
# On ROCm the draft keeps paged SWA, so it cannot share the window's pages.
(
"PD prefill on ROCm",
cfg.disaggregation_mode == "prefill" and get_platform().is_hip,
),
("mixed prefill/decode", cfg.enable_mixed_chunk),
("LoRA", cfg.enable_lora),
("radix sessions", cfg.enable_session_radix_cache),
+13 -1
View File
@@ -1531,9 +1531,21 @@ class SchedulerDisaggregationPrefillMixin:
]
def _swa_payload():
window_size = self.sliding_window_size
kvcache = self.token_to_kv_pool_allocator.get_kvcache()
request_window = kvcache.request_window
# Under encoder replay the scheduler is not hybrid SWA and keeps no
# window size; the pool holds it.
window_size = (
self.sliding_window_size
if request_window is None
else kvcache.sliding_window
)
window_start = max(req.disagg_decode_prefix_len, seq_len - window_size)
window_start = (window_start // page_size) * page_size
if request_window is not None:
return request_window.page_indices(
int(req.kv.req_pool_idx), window_start, seq_len
)
window_kv_indices_full = self.req_to_token_pool.req_to_token[
req.kv.req_pool_idx, window_start:seq_len
]
+6 -1
View File
@@ -1749,17 +1749,22 @@ def setup_state_kv_args(
raise RuntimeError(
"DSV4 target and draft pools must share the SWA index mapping"
)
# Under encoder replay the target window and the draft ring address SWA
# by request slot; equal rows per slot let them share one page list.
request_window = token_to_kv_pool.request_window
target_geometry = (
token_to_kv_pool.page_size,
token_to_kv_pool.sliding_window,
request_window.capacity if request_window is not None else None,
)
draft_geometry = (
draft_token_to_kv_pool.page_size,
draft_token_to_kv_pool.sliding_window,
draft_token_to_kv_pool.swa_req_ring_size,
)
if target_geometry != draft_geometry:
raise RuntimeError(
"DSV4 target and draft pools must share paged SWA geometry: "
"DSV4 target and draft pools must share SWA geometry: "
f"target={target_geometry}, draft={draft_geometry}"
)
draft_ptrs, draft_lens, draft_item_lens = (
@@ -1,10 +1,15 @@
from __future__ import annotations
import abc
from typing import List, Optional, Tuple
from typing import TYPE_CHECKING, List, Optional, Tuple
import torch
from sglang.srt.mem_cache.memory_pool import KVCache
if TYPE_CHECKING:
from sglang.srt.mem_cache.dsv41_request_window import RequestWindow
class BaseSWAKVPool(KVCache):
"""ABC for SWA-like KV pools.
@@ -18,6 +23,9 @@ class BaseSWAKVPool(KVCache):
# Set when SWA KV is a per-request ring of this many tokens (addressed by
# req_pool_idx) rather than a paged token pool; SWA is then not budgeted per token.
swa_req_ring_size: Optional[int] = None
# Set when the target keeps SWA KV in per-request window rows (DeepSeek-V4.1
# encoder replay) instead of a paged pool; there is no full -> swa mapping.
request_window: Optional[RequestWindow] = None
# Independent draft SWA buffers can consume full-token IDs directly instead
# of the target SWA allocator's physical IDs. HiCache must preserve that
# index ownership even though both buffers implement sliding attention.
@@ -130,9 +130,12 @@ def uses_draft_swa_ring() -> bool:
return get_exec().features.enable_encoder_swa_bounded_replay and not _is_hip
def get_draft_swa_ring_size(sliding_window: int, max_draft_tokens: int) -> int:
# A draft block and a verify commit write past the committed window.
return sliding_window + max_draft_tokens
def get_draft_swa_ring_size(
sliding_window: int, max_draft_tokens: int, page_size: int
) -> int:
# A draft block and a verify commit write past the committed window; whole
# pages match the target request window's rows, so PD sends both as one list.
return ceil_div(sliding_window + max_draft_tokens, page_size) * page_size
def get_swa_ring_size(sliding_window: int, is_speculative: bool = False) -> int:
@@ -1169,7 +1172,7 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
# while attending, so it addresses its ring directly with no workspace.
self.unified_kv_pool = None
self.swa_req_ring_size = get_draft_swa_ring_size(
self.sliding_window, online_mtp_max_draft_tokens or 0
self.sliding_window, online_mtp_max_draft_tokens or 0, swa_page_size
)
self.swa_req_ring_valid_from = torch.zeros(
self.num_req_slots, dtype=torch.int64, device=device
@@ -1451,11 +1454,16 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
data_lens: List[int] = []
item_lens: List[int] = []
if self.swa_kv_pool is not None:
physical_pages_per_logical_page = (
self.swa_page_size // self.swa_kv_pool.page_size
)
for buf in self.swa_kv_pool.kv_buffer:
# Under encoder replay the target's SWA lives in the request window, whose
# pages pair with decode's paged SWA pages (RequestWindow.page_indices).
swa_pool = (
self.request_window.state
if self.request_window is not None
else self.swa_kv_pool
)
if swa_pool is not None:
physical_pages_per_logical_page = self.swa_page_size // swa_pool.page_size
for buf in swa_pool.kv_buffer:
assert buf.ndim == 2, f"expected 2D buffer, got {buf.ndim}D"
data_ptrs.append(buf.data_ptr())
data_lens.append(buf.nbytes)
@@ -213,6 +213,16 @@ class RequestWindow:
size = ((rows + self.page_size - 1) // self.page_size) * self.page_size
self.workspace = self.pool_factory(size, 1)
def row_loc(self, slots, positions):
return slots * self.capacity + positions % self.capacity
def page_indices(self, slot: int, start: int, end: int):
assert start % self.page_size == 0, (start, self.page_size)
positions = torch.arange(start, end, self.page_size)
# capacity is whole pages, so a page's rows keep a paged pool's offsets
# and pair with decode's paged SWA pages one for one.
return (self.row_loc(slot, positions) // self.page_size).to(torch.int32).numpy()
def reset(self, slots):
loc = slots.to(torch.int64)[:, None] * self.capacity + torch.arange(
self.capacity, device=slots.device
@@ -237,7 +247,7 @@ class RequestWindow:
def initialize_dummy_history(self):
layout = self.layout
self.tags.fill_(-1)
loc = layout.history_req * self.capacity + layout.history_pos % self.capacity
loc = self.row_loc(layout.history_req, layout.history_pos)
for buf in self.state.kv_buffer:
buf.zero_()
self.tags[:, loc] = layout.history_pos
@@ -246,7 +256,7 @@ class RequestWindow:
def _history_src(self, layout):
return torch.where(
layout.history_valid,
layout.history_req * self.capacity + layout.history_pos % self.capacity,
self.row_loc(layout.history_req, layout.history_pos),
self.zero_row,
)
@@ -1200,7 +1200,9 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
+ self.page_size
)
draft_ring_size = get_draft_swa_ring_size(
self.sliding_window_size, self.online_c128_mtp_max_draft_tokens
self.sliding_window_size,
self.online_c128_mtp_max_draft_tokens,
self.page_size,
)
self.request_window_bytes = (
(slots * capacity + self.page_size) * layers * (self.kv_bytes + 16)