[Fix] deepseek-v4.1: Start the PD decode draft SWA window at the prefill draft valid-from (#45037)
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Tuple
|
||||
from typing import Optional, Tuple
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
@@ -35,12 +35,14 @@ class ComputeDsparkWindowGather:
|
||||
req_pool_indices_repeated: torch.Tensor,
|
||||
block_size: int,
|
||||
swa_window: int,
|
||||
valid_from: Optional[torch.Tensor] = None,
|
||||
) -> DsparkWindowGather:
|
||||
return compute_dspark_window_gather(
|
||||
seq_lens_casual=seq_lens_casual,
|
||||
req_pool_indices_repeated=req_pool_indices_repeated,
|
||||
block_size=block_size,
|
||||
swa_window=swa_window,
|
||||
valid_from=valid_from,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@@ -51,12 +53,14 @@ class ComputeDsparkWindowGather:
|
||||
req_pool_indices_repeated: torch.Tensor,
|
||||
block_size: int,
|
||||
swa_window: int,
|
||||
valid_from: Optional[torch.Tensor] = None,
|
||||
) -> DsparkWindowGather:
|
||||
return compute_dspark_window_gather_triton(
|
||||
seq_lens_casual=seq_lens_casual,
|
||||
req_pool_indices_repeated=req_pool_indices_repeated,
|
||||
block_size=block_size,
|
||||
swa_window=swa_window,
|
||||
valid_from=valid_from,
|
||||
)
|
||||
|
||||
|
||||
@@ -129,6 +133,7 @@ def compute_dspark_window_gather(
|
||||
req_pool_indices_repeated: torch.Tensor,
|
||||
block_size: int,
|
||||
swa_window: int,
|
||||
valid_from: Optional[torch.Tensor] = None,
|
||||
) -> DsparkWindowGather:
|
||||
seq_lens_casual = seq_lens_casual.to(torch.int32)
|
||||
num_q = seq_lens_casual.size(0)
|
||||
@@ -143,6 +148,12 @@ def compute_dspark_window_gather(
|
||||
prefix_lens = (seq_lens_casual[first_token] - 1).to(torch.int32)
|
||||
context_lens = torch.clamp(prefix_lens, max=swa_window).to(torch.int32)
|
||||
req_pool_indices_per_request = req_pool_indices_repeated[first_token]
|
||||
if valid_from is not None:
|
||||
# The window keeps its last context_lens positions; none may sit below
|
||||
# the request's valid_from.
|
||||
rows_written = prefix_lens - valid_from[req_pool_indices_per_request.long()]
|
||||
context_lens = torch.minimum(context_lens, rows_written.clamp(min=0))
|
||||
context_lens = context_lens.to(torch.int32)
|
||||
|
||||
offsets = (
|
||||
prefix_lens.to(torch.int64).unsqueeze(1)
|
||||
@@ -170,15 +181,22 @@ def _window_gather_kernel(
|
||||
req_pool_out_ptr,
|
||||
offsets_ptr,
|
||||
invalid_ptr,
|
||||
valid_from_ptr,
|
||||
block_size,
|
||||
swa_window,
|
||||
W_BLOCK: tl.constexpr,
|
||||
HAS_VALID_FROM: tl.constexpr,
|
||||
):
|
||||
i = tl.program_id(0)
|
||||
ft = i * block_size
|
||||
prefix = tl.load(seq_lens_casual_ptr + ft).to(tl.int64) - 1
|
||||
tl.store(context_lens_ptr + i, tl.minimum(prefix, swa_window).to(tl.int32))
|
||||
tl.store(req_pool_out_ptr + i, tl.load(req_pool_rep_ptr + ft))
|
||||
req_pool = tl.load(req_pool_rep_ptr + ft)
|
||||
context_len = tl.minimum(prefix, swa_window)
|
||||
if HAS_VALID_FROM:
|
||||
valid_from = tl.load(valid_from_ptr + req_pool.to(tl.int64))
|
||||
context_len = tl.minimum(context_len, tl.maximum(prefix - valid_from, 0))
|
||||
tl.store(context_lens_ptr + i, context_len.to(tl.int32))
|
||||
tl.store(req_pool_out_ptr + i, req_pool)
|
||||
col = tl.arange(0, W_BLOCK)
|
||||
cmask = col < swa_window
|
||||
off = prefix - swa_window + col
|
||||
@@ -192,6 +210,7 @@ def compute_dspark_window_gather_triton(
|
||||
req_pool_indices_repeated: torch.Tensor,
|
||||
block_size: int,
|
||||
swa_window: int,
|
||||
valid_from: Optional[torch.Tensor] = None,
|
||||
) -> DsparkWindowGather:
|
||||
seq_lens_casual = seq_lens_casual.to(torch.int32).contiguous()
|
||||
num_q = seq_lens_casual.size(0)
|
||||
@@ -214,9 +233,11 @@ def compute_dspark_window_gather_triton(
|
||||
req_pool_out,
|
||||
offsets,
|
||||
invalid,
|
||||
valid_from if valid_from is not None else context_lens,
|
||||
block_size,
|
||||
swa_window,
|
||||
W_BLOCK=W_BLOCK,
|
||||
HAS_VALID_FROM=valid_from is not None,
|
||||
)
|
||||
return DsparkWindowGather(
|
||||
num_q=num_q,
|
||||
|
||||
@@ -30,6 +30,9 @@ class StateType(str, enum.Enum):
|
||||
SWA_RING = "swa_ring"
|
||||
# DeepSeek-V4 request-scoped compression state; preserve the legacy wire value.
|
||||
DSV4_REQUEST_STATE = "c128_state"
|
||||
# One item per request row (req_pool_idx), sent row to row: the DeepSeek-V4
|
||||
# DSpark draft's SWA valid-from.
|
||||
REQUEST_ROW = "request_row"
|
||||
# A block-scaled KV dtype keeps its per-block scales in buffers parallel to
|
||||
# K/V, one component per sub-pool so each carries the index payload of the
|
||||
# KV it describes (whole sequence for full attention, window for SWA).
|
||||
|
||||
@@ -1701,6 +1701,9 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
int(decode_req.req.kv.req_pool_idx), seq_len
|
||||
)
|
||||
|
||||
def _request_row_payload():
|
||||
return np.array([decode_req.req.kv.req_pool_idx], dtype=np.int32)
|
||||
|
||||
state_types = self.kv_manager.kv_args.state_types
|
||||
if StateType.DSV4_REQUEST_STATE in state_types:
|
||||
clear_request_state = getattr(
|
||||
@@ -1719,6 +1722,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
StateType.MINIMAX_DENSE_KV: _full_kv_pages_payload,
|
||||
StateType.SWA_RING: _swa_ring_payload,
|
||||
StateType.DSV4_REQUEST_STATE: _request_state_payload,
|
||||
StateType.REQUEST_ROW: _request_row_payload,
|
||||
StateType.BLOCK_SCALE: _full_kv_pages_payload,
|
||||
StateType.BLOCK_SCALE_SWA: _swa_payload,
|
||||
}
|
||||
|
||||
@@ -1868,6 +1868,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
StateType.QSA_COMPRESSED,
|
||||
StateType.SWA_RING,
|
||||
StateType.DSV4_REQUEST_STATE,
|
||||
StateType.REQUEST_ROW,
|
||||
StateType.BLOCK_SCALE,
|
||||
StateType.BLOCK_SCALE_SWA,
|
||||
)
|
||||
@@ -1879,6 +1880,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
StateType.QSA_COMPRESSED,
|
||||
StateType.SWA_RING,
|
||||
StateType.DSV4_REQUEST_STATE,
|
||||
StateType.REQUEST_ROW,
|
||||
)
|
||||
|
||||
def maybe_send_extra(
|
||||
|
||||
@@ -3244,6 +3244,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
StateType.QSA_COMPRESSED,
|
||||
StateType.SWA_RING,
|
||||
StateType.DSV4_REQUEST_STATE,
|
||||
StateType.REQUEST_ROW,
|
||||
):
|
||||
# QSA state is whole on every rank, so it pairs by state TP.
|
||||
src_tp_size = (
|
||||
|
||||
@@ -1606,6 +1606,9 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
int(req.kv.req_pool_idx), c128_seq_len
|
||||
)
|
||||
|
||||
def _request_row_payload():
|
||||
return np.array([req.kv.req_pool_idx], dtype=np.int32)
|
||||
|
||||
state_types = (
|
||||
self.disagg_prefill_bootstrap_queue.kv_manager.kv_args.state_types
|
||||
)
|
||||
@@ -1620,6 +1623,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
StateType.MINIMAX_DENSE_KV: _full_kv_pages_payload,
|
||||
StateType.SWA_RING: _swa_ring_payload,
|
||||
StateType.DSV4_REQUEST_STATE: _request_state_payload,
|
||||
StateType.REQUEST_ROW: _request_row_payload,
|
||||
StateType.BLOCK_SCALE: _full_kv_pages_payload,
|
||||
StateType.BLOCK_SCALE_SWA: _swa_payload,
|
||||
}
|
||||
|
||||
@@ -1780,6 +1780,13 @@ def setup_state_kv_args(
|
||||
draft_lens,
|
||||
draft_item_lens,
|
||||
)
|
||||
row_ptrs, row_lens, row_item_lens = (
|
||||
draft_token_to_kv_pool.get_swa_valid_from_buf_infos()
|
||||
)
|
||||
if row_ptrs:
|
||||
append_state_component(
|
||||
kv_args, StateType.REQUEST_ROW, row_ptrs, row_lens, row_item_lens
|
||||
)
|
||||
|
||||
if (
|
||||
StateType.MAMBA not in kv_args.state_types
|
||||
|
||||
@@ -4567,7 +4567,7 @@ class DeepseekV4AttnBackend(
|
||||
return BuildDsparkSwaRingIndices.execute(
|
||||
seq_lens_casual=seq_lens_casual,
|
||||
req_pool_indices_repeated=req_pool_indices_repeated,
|
||||
valid_from=pool.swa_req_ring_valid_from,
|
||||
valid_from=pool.swa_valid_from,
|
||||
ring_size=pool.swa_req_ring_size,
|
||||
block_size=block_size,
|
||||
swa_window=SWA_WINDOW,
|
||||
@@ -4578,6 +4578,7 @@ class DeepseekV4AttnBackend(
|
||||
req_pool_indices_repeated=req_pool_indices_repeated,
|
||||
block_size=block_size,
|
||||
swa_window=SWA_WINDOW,
|
||||
valid_from=pool.swa_valid_from,
|
||||
)
|
||||
|
||||
swa_page_indices, swa_topk_lengths = BuildDsparkSwaPageIndices.execute(
|
||||
|
||||
@@ -123,6 +123,16 @@ def get_compress_state_write_pad(compress_ratio: int, ring_size: int) -> int:
|
||||
return ring_size - window_size + 2 if ring_size > window_size else 0
|
||||
|
||||
|
||||
def is_dspark_spec() -> bool:
|
||||
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||
|
||||
spec_algorithm = get_spec().speculative_algorithm
|
||||
return (
|
||||
spec_algorithm is not None
|
||||
and SpeculativeAlgorithm.from_string(spec_algorithm).is_dspark()
|
||||
)
|
||||
|
||||
|
||||
def uses_draft_swa_ring() -> bool:
|
||||
"""Whether the DSpark draft keeps its SWA KV in a per-request ring, so no SWA
|
||||
state is tree-owned under encoder replay. The HIP backend still addresses the
|
||||
@@ -1114,8 +1124,10 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
|
||||
self.request_window = None
|
||||
self.full_to_swa_index_mapping = None
|
||||
# Per req slot, the first position the draft SWA ring holds KV for.
|
||||
self.swa_req_ring_valid_from: Optional[torch.Tensor] = None
|
||||
# Per req slot, the first position whose draft SWA rows this request
|
||||
# wrote: the ring restarts at the cached prefix, and a PD decode receives
|
||||
# the prefill's mark as request-row state.
|
||||
self.swa_valid_from: Optional[torch.Tensor] = None
|
||||
encoder_replay = get_exec().features.enable_encoder_swa_bounded_replay
|
||||
# Under encoder replay the target keeps SWA in a per-request window. A
|
||||
# draft without its own ring still reads the target's full-to-SWA mapping,
|
||||
@@ -1174,9 +1186,6 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
self.swa_req_ring_size = get_draft_swa_ring_size(
|
||||
self.sliding_window, online_mtp_max_draft_tokens or 0, swa_page_size
|
||||
)
|
||||
self.swa_req_ring_valid_from = torch.zeros(
|
||||
self.num_req_slots, dtype=torch.int64, device=device
|
||||
)
|
||||
self.swa_kv_pool = self._make_kv_pool(
|
||||
size=self.num_req_slots * self.swa_req_ring_size,
|
||||
page_size=swa_kv_page_size,
|
||||
@@ -1234,6 +1243,23 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
cls=kv_pool_cls,
|
||||
)
|
||||
|
||||
if (
|
||||
is_draft_worker
|
||||
and not _is_hip
|
||||
and self.swa_kv_pool is not None
|
||||
and is_dspark_spec()
|
||||
):
|
||||
# Kept for the ring and for paged SWA alike, so both PD roles
|
||||
# register it; it stays 0 unless the ring restarts or PD writes it.
|
||||
with (
|
||||
torch.cuda.use_mem_pool(self.custom_mem_pool)
|
||||
if self.custom_mem_pool
|
||||
else nullcontext()
|
||||
):
|
||||
self.swa_valid_from = torch.zeros(
|
||||
self.num_req_slots, dtype=torch.int64, device=device
|
||||
)
|
||||
|
||||
if self.request_window is not None:
|
||||
swa_storage = "request_window"
|
||||
elif self.swa_req_ring_size is not None:
|
||||
@@ -1294,10 +1320,17 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
assert self.full_to_swa_index_mapping is not None
|
||||
return self.full_to_swa_index_mapping[kv_indices]
|
||||
|
||||
def reset_swa_req_ring(self, slots: torch.Tensor, valid_from: torch.Tensor) -> None:
|
||||
"""Restart the draft SWA ring of ``slots`` at ``valid_from``; the rows
|
||||
before it belong to the slot's previous owner, not this request."""
|
||||
self.swa_req_ring_valid_from[slots] = valid_from
|
||||
def set_swa_valid_from(self, slots: torch.Tensor, valid_from: torch.Tensor) -> None:
|
||||
"""The draft SWA rows of ``slots`` below ``valid_from`` are not this
|
||||
request's: the slot's previous owner wrote them, or no one did."""
|
||||
self.swa_valid_from[slots] = valid_from
|
||||
|
||||
def get_swa_valid_from_buf_infos(self) -> Tuple[List[int], List[int], List[int]]:
|
||||
"""The draft SWA valid-from, which PD sends as request-row state."""
|
||||
if self.swa_valid_from is None:
|
||||
return [], [], []
|
||||
mark = self.swa_valid_from
|
||||
return [mark.data_ptr()], [mark.nbytes], [mark[0].nbytes]
|
||||
|
||||
def get_contiguous_buf_infos(self) -> Tuple[List[int], List[int], List[int]]:
|
||||
data_ptrs: List[int] = []
|
||||
|
||||
@@ -713,11 +713,14 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
output_size=num_tokens,
|
||||
)
|
||||
draft_pool = self._kv_injector.draft_model_runner.token_to_kv_pool
|
||||
if getattr(draft_pool, "swa_req_ring_valid_from", None) is not None:
|
||||
if (
|
||||
getattr(draft_pool, "swa_valid_from", None) is not None
|
||||
and batch.encoder_swa_reset is not None
|
||||
):
|
||||
# A newly placed request's ring holds nothing below its cached prefix.
|
||||
placed = [i for i, reset in enumerate(batch.encoder_swa_reset) if reset]
|
||||
if placed:
|
||||
draft_pool.reset_swa_req_ring(
|
||||
draft_pool.set_swa_valid_from(
|
||||
batch.req_pool_indices[placed],
|
||||
torch.tensor(
|
||||
[batch.prefix_lens[i] for i in placed],
|
||||
|
||||
Reference in New Issue
Block a user