[Fix] deepseek-v4.1: Start the PD decode draft SWA window at the prefill draft valid-from (#45037)

This commit is contained in:
Liangsheng Yin
2026-10-18 15:06:20 -07:00
committed by GitHub
parent da414adda7
commit fe89a7c890
13 changed files with 185 additions and 49 deletions
@@ -1,6 +1,6 @@
from __future__ import annotations
from typing import Tuple
from typing import Optional, Tuple
import msgspec
import torch
@@ -35,12 +35,14 @@ class ComputeDsparkWindowGather:
req_pool_indices_repeated: torch.Tensor,
block_size: int,
swa_window: int,
valid_from: Optional[torch.Tensor] = None,
) -> DsparkWindowGather:
return compute_dspark_window_gather(
seq_lens_casual=seq_lens_casual,
req_pool_indices_repeated=req_pool_indices_repeated,
block_size=block_size,
swa_window=swa_window,
valid_from=valid_from,
)
@classmethod
@@ -51,12 +53,14 @@ class ComputeDsparkWindowGather:
req_pool_indices_repeated: torch.Tensor,
block_size: int,
swa_window: int,
valid_from: Optional[torch.Tensor] = None,
) -> DsparkWindowGather:
return compute_dspark_window_gather_triton(
seq_lens_casual=seq_lens_casual,
req_pool_indices_repeated=req_pool_indices_repeated,
block_size=block_size,
swa_window=swa_window,
valid_from=valid_from,
)
@@ -129,6 +133,7 @@ def compute_dspark_window_gather(
req_pool_indices_repeated: torch.Tensor,
block_size: int,
swa_window: int,
valid_from: Optional[torch.Tensor] = None,
) -> DsparkWindowGather:
seq_lens_casual = seq_lens_casual.to(torch.int32)
num_q = seq_lens_casual.size(0)
@@ -143,6 +148,12 @@ def compute_dspark_window_gather(
prefix_lens = (seq_lens_casual[first_token] - 1).to(torch.int32)
context_lens = torch.clamp(prefix_lens, max=swa_window).to(torch.int32)
req_pool_indices_per_request = req_pool_indices_repeated[first_token]
if valid_from is not None:
# The window keeps its last context_lens positions; none may sit below
# the request's valid_from.
rows_written = prefix_lens - valid_from[req_pool_indices_per_request.long()]
context_lens = torch.minimum(context_lens, rows_written.clamp(min=0))
context_lens = context_lens.to(torch.int32)
offsets = (
prefix_lens.to(torch.int64).unsqueeze(1)
@@ -170,15 +181,22 @@ def _window_gather_kernel(
req_pool_out_ptr,
offsets_ptr,
invalid_ptr,
valid_from_ptr,
block_size,
swa_window,
W_BLOCK: tl.constexpr,
HAS_VALID_FROM: tl.constexpr,
):
i = tl.program_id(0)
ft = i * block_size
prefix = tl.load(seq_lens_casual_ptr + ft).to(tl.int64) - 1
tl.store(context_lens_ptr + i, tl.minimum(prefix, swa_window).to(tl.int32))
tl.store(req_pool_out_ptr + i, tl.load(req_pool_rep_ptr + ft))
req_pool = tl.load(req_pool_rep_ptr + ft)
context_len = tl.minimum(prefix, swa_window)
if HAS_VALID_FROM:
valid_from = tl.load(valid_from_ptr + req_pool.to(tl.int64))
context_len = tl.minimum(context_len, tl.maximum(prefix - valid_from, 0))
tl.store(context_lens_ptr + i, context_len.to(tl.int32))
tl.store(req_pool_out_ptr + i, req_pool)
col = tl.arange(0, W_BLOCK)
cmask = col < swa_window
off = prefix - swa_window + col
@@ -192,6 +210,7 @@ def compute_dspark_window_gather_triton(
req_pool_indices_repeated: torch.Tensor,
block_size: int,
swa_window: int,
valid_from: Optional[torch.Tensor] = None,
) -> DsparkWindowGather:
seq_lens_casual = seq_lens_casual.to(torch.int32).contiguous()
num_q = seq_lens_casual.size(0)
@@ -214,9 +233,11 @@ def compute_dspark_window_gather_triton(
req_pool_out,
offsets,
invalid,
valid_from if valid_from is not None else context_lens,
block_size,
swa_window,
W_BLOCK=W_BLOCK,
HAS_VALID_FROM=valid_from is not None,
)
return DsparkWindowGather(
num_q=num_q,
@@ -30,6 +30,9 @@ class StateType(str, enum.Enum):
SWA_RING = "swa_ring"
# DeepSeek-V4 request-scoped compression state; preserve the legacy wire value.
DSV4_REQUEST_STATE = "c128_state"
# One item per request row (req_pool_idx), sent row to row: the DeepSeek-V4
# DSpark draft's SWA valid-from.
REQUEST_ROW = "request_row"
# A block-scaled KV dtype keeps its per-block scales in buffers parallel to
# K/V, one component per sub-pool so each carries the index payload of the
# KV it describes (whole sequence for full attention, window for SWA).
@@ -1701,6 +1701,9 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
int(decode_req.req.kv.req_pool_idx), seq_len
)
def _request_row_payload():
return np.array([decode_req.req.kv.req_pool_idx], dtype=np.int32)
state_types = self.kv_manager.kv_args.state_types
if StateType.DSV4_REQUEST_STATE in state_types:
clear_request_state = getattr(
@@ -1719,6 +1722,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
StateType.MINIMAX_DENSE_KV: _full_kv_pages_payload,
StateType.SWA_RING: _swa_ring_payload,
StateType.DSV4_REQUEST_STATE: _request_state_payload,
StateType.REQUEST_ROW: _request_row_payload,
StateType.BLOCK_SCALE: _full_kv_pages_payload,
StateType.BLOCK_SCALE_SWA: _swa_payload,
}
@@ -1868,6 +1868,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
StateType.QSA_COMPRESSED,
StateType.SWA_RING,
StateType.DSV4_REQUEST_STATE,
StateType.REQUEST_ROW,
StateType.BLOCK_SCALE,
StateType.BLOCK_SCALE_SWA,
)
@@ -1879,6 +1880,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
StateType.QSA_COMPRESSED,
StateType.SWA_RING,
StateType.DSV4_REQUEST_STATE,
StateType.REQUEST_ROW,
)
def maybe_send_extra(
@@ -3244,6 +3244,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
StateType.QSA_COMPRESSED,
StateType.SWA_RING,
StateType.DSV4_REQUEST_STATE,
StateType.REQUEST_ROW,
):
# QSA state is whole on every rank, so it pairs by state TP.
src_tp_size = (
@@ -1606,6 +1606,9 @@ class SchedulerDisaggregationPrefillMixin:
int(req.kv.req_pool_idx), c128_seq_len
)
def _request_row_payload():
return np.array([req.kv.req_pool_idx], dtype=np.int32)
state_types = (
self.disagg_prefill_bootstrap_queue.kv_manager.kv_args.state_types
)
@@ -1620,6 +1623,7 @@ class SchedulerDisaggregationPrefillMixin:
StateType.MINIMAX_DENSE_KV: _full_kv_pages_payload,
StateType.SWA_RING: _swa_ring_payload,
StateType.DSV4_REQUEST_STATE: _request_state_payload,
StateType.REQUEST_ROW: _request_row_payload,
StateType.BLOCK_SCALE: _full_kv_pages_payload,
StateType.BLOCK_SCALE_SWA: _swa_payload,
}
@@ -1780,6 +1780,13 @@ def setup_state_kv_args(
draft_lens,
draft_item_lens,
)
row_ptrs, row_lens, row_item_lens = (
draft_token_to_kv_pool.get_swa_valid_from_buf_infos()
)
if row_ptrs:
append_state_component(
kv_args, StateType.REQUEST_ROW, row_ptrs, row_lens, row_item_lens
)
if (
StateType.MAMBA not in kv_args.state_types
@@ -4567,7 +4567,7 @@ class DeepseekV4AttnBackend(
return BuildDsparkSwaRingIndices.execute(
seq_lens_casual=seq_lens_casual,
req_pool_indices_repeated=req_pool_indices_repeated,
valid_from=pool.swa_req_ring_valid_from,
valid_from=pool.swa_valid_from,
ring_size=pool.swa_req_ring_size,
block_size=block_size,
swa_window=SWA_WINDOW,
@@ -4578,6 +4578,7 @@ class DeepseekV4AttnBackend(
req_pool_indices_repeated=req_pool_indices_repeated,
block_size=block_size,
swa_window=SWA_WINDOW,
valid_from=pool.swa_valid_from,
)
swa_page_indices, swa_topk_lengths = BuildDsparkSwaPageIndices.execute(
@@ -123,6 +123,16 @@ def get_compress_state_write_pad(compress_ratio: int, ring_size: int) -> int:
return ring_size - window_size + 2 if ring_size > window_size else 0
def is_dspark_spec() -> bool:
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
spec_algorithm = get_spec().speculative_algorithm
return (
spec_algorithm is not None
and SpeculativeAlgorithm.from_string(spec_algorithm).is_dspark()
)
def uses_draft_swa_ring() -> bool:
"""Whether the DSpark draft keeps its SWA KV in a per-request ring, so no SWA
state is tree-owned under encoder replay. The HIP backend still addresses the
@@ -1114,8 +1124,10 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
self.request_window = None
self.full_to_swa_index_mapping = None
# Per req slot, the first position the draft SWA ring holds KV for.
self.swa_req_ring_valid_from: Optional[torch.Tensor] = None
# Per req slot, the first position whose draft SWA rows this request
# wrote: the ring restarts at the cached prefix, and a PD decode receives
# the prefill's mark as request-row state.
self.swa_valid_from: Optional[torch.Tensor] = None
encoder_replay = get_exec().features.enable_encoder_swa_bounded_replay
# Under encoder replay the target keeps SWA in a per-request window. A
# draft without its own ring still reads the target's full-to-SWA mapping,
@@ -1174,9 +1186,6 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
self.swa_req_ring_size = get_draft_swa_ring_size(
self.sliding_window, online_mtp_max_draft_tokens or 0, swa_page_size
)
self.swa_req_ring_valid_from = torch.zeros(
self.num_req_slots, dtype=torch.int64, device=device
)
self.swa_kv_pool = self._make_kv_pool(
size=self.num_req_slots * self.swa_req_ring_size,
page_size=swa_kv_page_size,
@@ -1234,6 +1243,23 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
cls=kv_pool_cls,
)
if (
is_draft_worker
and not _is_hip
and self.swa_kv_pool is not None
and is_dspark_spec()
):
# Kept for the ring and for paged SWA alike, so both PD roles
# register it; it stays 0 unless the ring restarts or PD writes it.
with (
torch.cuda.use_mem_pool(self.custom_mem_pool)
if self.custom_mem_pool
else nullcontext()
):
self.swa_valid_from = torch.zeros(
self.num_req_slots, dtype=torch.int64, device=device
)
if self.request_window is not None:
swa_storage = "request_window"
elif self.swa_req_ring_size is not None:
@@ -1294,10 +1320,17 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
assert self.full_to_swa_index_mapping is not None
return self.full_to_swa_index_mapping[kv_indices]
def reset_swa_req_ring(self, slots: torch.Tensor, valid_from: torch.Tensor) -> None:
"""Restart the draft SWA ring of ``slots`` at ``valid_from``; the rows
before it belong to the slot's previous owner, not this request."""
self.swa_req_ring_valid_from[slots] = valid_from
def set_swa_valid_from(self, slots: torch.Tensor, valid_from: torch.Tensor) -> None:
"""The draft SWA rows of ``slots`` below ``valid_from`` are not this
request's: the slot's previous owner wrote them, or no one did."""
self.swa_valid_from[slots] = valid_from
def get_swa_valid_from_buf_infos(self) -> Tuple[List[int], List[int], List[int]]:
"""The draft SWA valid-from, which PD sends as request-row state."""
if self.swa_valid_from is None:
return [], [], []
mark = self.swa_valid_from
return [mark.data_ptr()], [mark.nbytes], [mark[0].nbytes]
def get_contiguous_buf_infos(self) -> Tuple[List[int], List[int], List[int]]:
data_ptrs: List[int] = []
@@ -713,11 +713,14 @@ class DSparkWorkerV2(BaseSpecWorker):
output_size=num_tokens,
)
draft_pool = self._kv_injector.draft_model_runner.token_to_kv_pool
if getattr(draft_pool, "swa_req_ring_valid_from", None) is not None:
if (
getattr(draft_pool, "swa_valid_from", None) is not None
and batch.encoder_swa_reset is not None
):
# A newly placed request's ring holds nothing below its cached prefix.
placed = [i for i, reset in enumerate(batch.encoder_swa_reset) if reset]
if placed:
draft_pool.reset_swa_req_ring(
draft_pool.set_swa_valid_from(
batch.req_pool_indices[placed],
torch.tensor(
[batch.prefix_lens[i] for i in placed],