[Feat] deepseek-v4.1: Run encoder SWA bounded replay on PD prefill workers (#44952)
This commit is contained in:
@@ -297,7 +297,13 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
("DP attention on ROCm", get_platform().is_hip and attn_dp_enabled_of(cfg)),
|
||||
("external cache linker", cfg.enable_unified_cache_external_linker),
|
||||
("unified memory", cfg.enable_unified_memory),
|
||||
("PD disaggregation", cfg.disaggregation_mode != "null"),
|
||||
# Decode receives the last SWA window from prefill and never replays.
|
||||
("PD decode", cfg.disaggregation_mode == "decode"),
|
||||
# On ROCm the draft keeps paged SWA, so it cannot share the window's pages.
|
||||
(
|
||||
"PD prefill on ROCm",
|
||||
cfg.disaggregation_mode == "prefill" and get_platform().is_hip,
|
||||
),
|
||||
("mixed prefill/decode", cfg.enable_mixed_chunk),
|
||||
("LoRA", cfg.enable_lora),
|
||||
("radix sessions", cfg.enable_session_radix_cache),
|
||||
|
||||
@@ -1531,9 +1531,21 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
]
|
||||
|
||||
def _swa_payload():
|
||||
window_size = self.sliding_window_size
|
||||
kvcache = self.token_to_kv_pool_allocator.get_kvcache()
|
||||
request_window = kvcache.request_window
|
||||
# Under encoder replay the scheduler is not hybrid SWA and keeps no
|
||||
# window size; the pool holds it.
|
||||
window_size = (
|
||||
self.sliding_window_size
|
||||
if request_window is None
|
||||
else kvcache.sliding_window
|
||||
)
|
||||
window_start = max(req.disagg_decode_prefix_len, seq_len - window_size)
|
||||
window_start = (window_start // page_size) * page_size
|
||||
if request_window is not None:
|
||||
return request_window.page_indices(
|
||||
int(req.kv.req_pool_idx), window_start, seq_len
|
||||
)
|
||||
window_kv_indices_full = self.req_to_token_pool.req_to_token[
|
||||
req.kv.req_pool_idx, window_start:seq_len
|
||||
]
|
||||
|
||||
@@ -1749,17 +1749,22 @@ def setup_state_kv_args(
|
||||
raise RuntimeError(
|
||||
"DSV4 target and draft pools must share the SWA index mapping"
|
||||
)
|
||||
# Under encoder replay the target window and the draft ring address SWA
|
||||
# by request slot; equal rows per slot let them share one page list.
|
||||
request_window = token_to_kv_pool.request_window
|
||||
target_geometry = (
|
||||
token_to_kv_pool.page_size,
|
||||
token_to_kv_pool.sliding_window,
|
||||
request_window.capacity if request_window is not None else None,
|
||||
)
|
||||
draft_geometry = (
|
||||
draft_token_to_kv_pool.page_size,
|
||||
draft_token_to_kv_pool.sliding_window,
|
||||
draft_token_to_kv_pool.swa_req_ring_size,
|
||||
)
|
||||
if target_geometry != draft_geometry:
|
||||
raise RuntimeError(
|
||||
"DSV4 target and draft pools must share paged SWA geometry: "
|
||||
"DSV4 target and draft pools must share SWA geometry: "
|
||||
f"target={target_geometry}, draft={draft_geometry}"
|
||||
)
|
||||
draft_ptrs, draft_lens, draft_item_lens = (
|
||||
|
||||
@@ -1,10 +1,15 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import abc
|
||||
from typing import List, Optional, Tuple
|
||||
from typing import TYPE_CHECKING, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.mem_cache.memory_pool import KVCache
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.mem_cache.dsv41_request_window import RequestWindow
|
||||
|
||||
|
||||
class BaseSWAKVPool(KVCache):
|
||||
"""ABC for SWA-like KV pools.
|
||||
@@ -18,6 +23,9 @@ class BaseSWAKVPool(KVCache):
|
||||
# Set when SWA KV is a per-request ring of this many tokens (addressed by
|
||||
# req_pool_idx) rather than a paged token pool; SWA is then not budgeted per token.
|
||||
swa_req_ring_size: Optional[int] = None
|
||||
# Set when the target keeps SWA KV in per-request window rows (DeepSeek-V4.1
|
||||
# encoder replay) instead of a paged pool; there is no full -> swa mapping.
|
||||
request_window: Optional[RequestWindow] = None
|
||||
# Independent draft SWA buffers can consume full-token IDs directly instead
|
||||
# of the target SWA allocator's physical IDs. HiCache must preserve that
|
||||
# index ownership even though both buffers implement sliding attention.
|
||||
|
||||
@@ -130,9 +130,12 @@ def uses_draft_swa_ring() -> bool:
|
||||
return get_exec().features.enable_encoder_swa_bounded_replay and not _is_hip
|
||||
|
||||
|
||||
def get_draft_swa_ring_size(sliding_window: int, max_draft_tokens: int) -> int:
|
||||
# A draft block and a verify commit write past the committed window.
|
||||
return sliding_window + max_draft_tokens
|
||||
def get_draft_swa_ring_size(
|
||||
sliding_window: int, max_draft_tokens: int, page_size: int
|
||||
) -> int:
|
||||
# A draft block and a verify commit write past the committed window; whole
|
||||
# pages match the target request window's rows, so PD sends both as one list.
|
||||
return ceil_div(sliding_window + max_draft_tokens, page_size) * page_size
|
||||
|
||||
|
||||
def get_swa_ring_size(sliding_window: int, is_speculative: bool = False) -> int:
|
||||
@@ -1169,7 +1172,7 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
# while attending, so it addresses its ring directly with no workspace.
|
||||
self.unified_kv_pool = None
|
||||
self.swa_req_ring_size = get_draft_swa_ring_size(
|
||||
self.sliding_window, online_mtp_max_draft_tokens or 0
|
||||
self.sliding_window, online_mtp_max_draft_tokens or 0, swa_page_size
|
||||
)
|
||||
self.swa_req_ring_valid_from = torch.zeros(
|
||||
self.num_req_slots, dtype=torch.int64, device=device
|
||||
@@ -1451,11 +1454,16 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
data_lens: List[int] = []
|
||||
item_lens: List[int] = []
|
||||
|
||||
if self.swa_kv_pool is not None:
|
||||
physical_pages_per_logical_page = (
|
||||
self.swa_page_size // self.swa_kv_pool.page_size
|
||||
)
|
||||
for buf in self.swa_kv_pool.kv_buffer:
|
||||
# Under encoder replay the target's SWA lives in the request window, whose
|
||||
# pages pair with decode's paged SWA pages (RequestWindow.page_indices).
|
||||
swa_pool = (
|
||||
self.request_window.state
|
||||
if self.request_window is not None
|
||||
else self.swa_kv_pool
|
||||
)
|
||||
if swa_pool is not None:
|
||||
physical_pages_per_logical_page = self.swa_page_size // swa_pool.page_size
|
||||
for buf in swa_pool.kv_buffer:
|
||||
assert buf.ndim == 2, f"expected 2D buffer, got {buf.ndim}D"
|
||||
data_ptrs.append(buf.data_ptr())
|
||||
data_lens.append(buf.nbytes)
|
||||
|
||||
@@ -213,6 +213,16 @@ class RequestWindow:
|
||||
size = ((rows + self.page_size - 1) // self.page_size) * self.page_size
|
||||
self.workspace = self.pool_factory(size, 1)
|
||||
|
||||
def row_loc(self, slots, positions):
|
||||
return slots * self.capacity + positions % self.capacity
|
||||
|
||||
def page_indices(self, slot: int, start: int, end: int):
|
||||
assert start % self.page_size == 0, (start, self.page_size)
|
||||
positions = torch.arange(start, end, self.page_size)
|
||||
# capacity is whole pages, so a page's rows keep a paged pool's offsets
|
||||
# and pair with decode's paged SWA pages one for one.
|
||||
return (self.row_loc(slot, positions) // self.page_size).to(torch.int32).numpy()
|
||||
|
||||
def reset(self, slots):
|
||||
loc = slots.to(torch.int64)[:, None] * self.capacity + torch.arange(
|
||||
self.capacity, device=slots.device
|
||||
@@ -237,7 +247,7 @@ class RequestWindow:
|
||||
def initialize_dummy_history(self):
|
||||
layout = self.layout
|
||||
self.tags.fill_(-1)
|
||||
loc = layout.history_req * self.capacity + layout.history_pos % self.capacity
|
||||
loc = self.row_loc(layout.history_req, layout.history_pos)
|
||||
for buf in self.state.kv_buffer:
|
||||
buf.zero_()
|
||||
self.tags[:, loc] = layout.history_pos
|
||||
@@ -246,7 +256,7 @@ class RequestWindow:
|
||||
def _history_src(self, layout):
|
||||
return torch.where(
|
||||
layout.history_valid,
|
||||
layout.history_req * self.capacity + layout.history_pos % self.capacity,
|
||||
self.row_loc(layout.history_req, layout.history_pos),
|
||||
self.zero_row,
|
||||
)
|
||||
|
||||
|
||||
@@ -1200,7 +1200,9 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
|
||||
+ self.page_size
|
||||
)
|
||||
draft_ring_size = get_draft_swa_ring_size(
|
||||
self.sliding_window_size, self.online_c128_mtp_max_draft_tokens
|
||||
self.sliding_window_size,
|
||||
self.online_c128_mtp_max_draft_tokens,
|
||||
self.page_size,
|
||||
)
|
||||
self.request_window_bytes = (
|
||||
(slots * capacity + self.page_size) * layers * (self.kv_bytes + 16)
|
||||
|
||||
Reference in New Issue
Block a user