[Fix] deepseek-v4.1: Serve prompt-logprob requests under decoder SWA bounded replay (#44850)

Co-authored-by: Pedram Ghazi <pedram.ghazi@amd.com>
Co-authored-by: David Orman <ormandj@corenode.com>
This commit is contained in:
Liangsheng Yin
2026-10-17 01:25:33 -07:00
committed by GitHub
co-authored by Pedram Ghazi David Orman
parent 7a8804edb5
commit a2953d7b7f
12 changed files with 376 additions and 25 deletions
@@ -437,18 +437,44 @@ class BuildCausalSwaPageIndices:
)
def late_layer_tail_lens(
extend_lens_cpu: list[int],
tail_len: int,
whole_extend: Optional[list[bool]] = None,
) -> list[int]:
"""Each request's late-layer tail: its last min(tail_len, extend_len) tokens,
or its whole extend where whole_extend is set."""
if whole_extend is None:
return [min(tail_len, n) for n in extend_lens_cpu]
return [
n if whole else min(tail_len, n)
for n, whole in zip(extend_lens_cpu, whole_extend)
]
def late_layer_tail_layout(
*,
extend_lens_cpu: list[int],
seq_lens_cpu: list[int],
tail_len: int,
device: torch.device,
whole_extend: Optional[list[bool]] = None,
) -> tuple[torch.Tensor, list[int], torch.Tensor]:
"""Tail rows of each prefill extend: its last min(tail_len, extend_len) tokens.
"""Tail rows of each prefill extend: its last min(tail_len, extend_len) tokens,
or its whole extend where whole_extend is set.
Returns (token indices into the extend, per-request tail lengths, per-row
absolute window floor). The floor is seq_len - tail_len, so a tail shorter
than tail_len reads rows the request wrote before this extend."""
tail_lens_cpu = [min(tail_len, n) for n in extend_lens_cpu]
than tail_len reads rows the request wrote before this extend. A whole
extend keeps every row's plain window."""
if whole_extend is not None and any(whole_extend):
return _whole_extend_tail_layout(
extend_lens_cpu=extend_lens_cpu,
seq_lens_cpu=seq_lens_cpu,
tail_len=tail_len,
device=device,
whole_extend=whole_extend,
)
tail_lens_cpu = late_layer_tail_lens(extend_lens_cpu, tail_len)
if len(extend_lens_cpu) == 1:
n, t, s = extend_lens_cpu[0], tail_lens_cpu[0], seq_lens_cpu[0]
floor = torch.full((t,), max(0, s - tail_len), dtype=torch.int32, device=device)
@@ -469,6 +495,32 @@ def late_layer_tail_layout(
return token_indices, tail_lens_cpu, floor
def _whole_extend_tail_layout(
*,
extend_lens_cpu: list[int],
seq_lens_cpu: list[int],
tail_len: int,
device: torch.device,
whole_extend: list[bool],
) -> tuple[torch.Tensor, list[int], torch.Tensor]:
tail_lens_cpu = late_layer_tail_lens(extend_lens_cpu, tail_len, whole_extend)
token_indices, floor, start = [], [], 0
for n, t, s, whole in zip(
extend_lens_cpu, tail_lens_cpu, seq_lens_cpu, whole_extend
):
token_indices.append(torch.arange(start + n - t, start + n))
# A whole extend's rows sit at s - t or later, so its floor never cuts
# their plain window.
row_floor = max(0, s - t - tail_len) if whole else max(0, s - tail_len)
floor.append(torch.full((t,), row_floor, dtype=torch.int32))
start += n
return (
torch.cat(token_indices).to(device, non_blocking=True),
tail_lens_cpu,
torch.cat(floor).to(device, non_blocking=True),
)
def build_causal_swa_page_indices(
*,
req_to_token: torch.Tensor,
@@ -411,6 +411,8 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
(get_platform().is_cuda or get_platform().is_hip)
and cfg.disaggregation_mode == "null"
and not (cfg.enable_prefill_cp or cfg.attn_cp_size != 1 or cfg.dcp_size != 1)
# Returning every hidden state runs every prompt row: nothing to trim.
and not cfg.enable_return_hidden_states
and not any(enabled for _, enabled in incompatible)
)
declare_resolution(
+4 -3
View File
@@ -137,9 +137,10 @@ class ExecFeatures(msgspec.Struct):
Optional[bool],
Arg(
help="Enable DeepSeek-V4.1 decoder SWA bounded replay. Defaults on "
"for supported configurations. Use --no-enable-decoder-swa-bounded-replay "
"to retain full prefill or request prompt logprobs. "
"Not numerically equivalent to full prefill.",
"for supported configurations. Not numerically equivalent to full "
"prefill; requests for prompt logprobs compute every prompt row. Use "
"--no-enable-decoder-swa-bounded-replay to retain full prefill, e.g. "
"for RL rollouts whose logprobs must match the trainer.",
action=argparse.BooleanOptionalAction,
resolvable=True,
),
@@ -42,6 +42,7 @@ from sglang.kernels.ops.attention.dsv4_attn_metadata_kernels import (
BuildPageTablePositions,
ExpandPrefillCausally,
late_layer_tail_layout,
late_layer_tail_lens,
)
from sglang.kernels.ops.speculative.dspark.dspark_attn_metadata import (
BuildBlockSeqLensCausal,
@@ -107,7 +108,12 @@ from sglang.srt.layers.dp_attention import (
)
from sglang.srt.mem_cache.deepseek_v4_compress_state import KVAndScore
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
from sglang.srt.model_executor.forward_batch_info import (
CaptureHiddenMode,
ForwardBatch,
ForwardMode,
get_server_return_hidden_states_mode,
)
from sglang.srt.runtime_context import (
get_exec,
get_parallel,
@@ -129,6 +135,7 @@ if TYPE_CHECKING:
from sgl_kernel.flash_mla import FlashMLASchedMeta
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.managers.schedule_batch import ScheduleBatch
from sglang.srt.model_executor.model_runner import ModelRunner
from sglang.srt.speculative.ragged_verify import RaggedVerifyLayout
@@ -943,6 +950,41 @@ def _tail_rows(
return t[token_indices]
def _late_layer_whole_extend(
*,
extend_lens: List[int],
logprob_start_lens: Optional[List[int]],
full_hidden_states: bool,
) -> Optional[List[bool]]:
"""Requests that run their whole extend with plain windows under decoder SWA
bounded replay: those that read prompt rows (prompt logprobs, or every hidden
state outside DSpark, whose draft reads tail rows only). None when no request
in the batch does."""
if full_hidden_states:
return [True] * len(extend_lens)
if logprob_start_lens is None:
return None
whole = [start < n for start, n in zip(logprob_start_lens, extend_lens)]
return whole if any(whole) else None
def late_layer_whole_extend(
forward_batch: ForwardBatch, *, dspark: bool
) -> Optional[List[bool]]:
"""`_late_layer_whole_extend` of a forward batch. dspark comes from the server:
a capture batch carries DSpark's FULL capture mode without its spec_algorithm."""
return _late_layer_whole_extend(
extend_lens=forward_batch.extend_seq_lens_cpu,
logprob_start_lens=(
forward_batch.extend_logprob_start_lens_cpu
if forward_batch.return_logprob
else None
),
full_hidden_states=forward_batch.capture_hidden_mode == CaptureHiddenMode.FULL
and not dspark,
)
def _pad_rows(t: torch.Tensor, *, num_rows: int) -> torch.Tensor:
assert t.shape[0] <= num_rows, (t.shape[0], num_rows)
if t.shape[0] == num_rows:
@@ -1573,12 +1615,45 @@ class DeepseekV4AttnBackend(
return False
if self.enable_decoder_swa_bounded_replay:
extend_lens_cpu = forward_batch.extend_seq_lens_cpu
if extend_lens_cpu is not None:
tail_rows = sum(min(SWA_WINDOW, int(n)) for n in extend_lens_cpu)
if tail_rows > self._prefill_graph_tail_capacity():
return False
if extend_lens_cpu is not None and not self._prefill_graph_fits_late_tail(
extend_lens_cpu,
late_layer_whole_extend(
forward_batch, dspark=self.model_runner.spec_algorithm.is_dspark()
),
):
return False
return True
def vote_prefill_cuda_graph(self, batch: ScheduleBatch) -> bool:
"""`can_run_prefill_cuda_graph`'s late-layer tail check judged from the
schedule batch, for the DP prefill-graph vote: no rank replays the graph
while another runs the step eagerly."""
if not self.enable_decoder_swa_bounded_replay:
return True
full_hidden_states = (
get_server_return_hidden_states_mode() == CaptureHiddenMode.FULL
or any(
req.return_hidden_states_mode == CaptureHiddenMode.FULL
for req in batch.reqs
)
) and not self.model_runner.spec_algorithm.is_dspark()
whole_extend = _late_layer_whole_extend(
extend_lens=batch.extend_lens,
logprob_start_lens=(
batch.extend_logprob_start_lens if batch.return_logprob else None
),
full_hidden_states=full_hidden_states,
)
return self._prefill_graph_fits_late_tail(batch.extend_lens, whole_extend)
def _prefill_graph_fits_late_tail(
self, extend_lens: List[int], whole_extend: Optional[List[bool]]
) -> bool:
tail_lens = late_layer_tail_lens(
[int(n) for n in extend_lens], SWA_WINDOW, whole_extend
)
return sum(tail_lens) <= self._prefill_graph_tail_capacity()
def _prefill_graph_tail_capacity(self) -> int:
"""Rows the late layers run over inside the prefill CUDA graph: every
request's SWA_WINDOW tail at the request cap, so one capture fits any batch."""
@@ -1634,6 +1709,9 @@ class DeepseekV4AttnBackend(
seq_lens_cpu=seq_lens_cpu.tolist(),
tail_len=SWA_WINDOW,
device=device,
whole_extend=late_layer_whole_extend(
forward_batch, dspark=self.model_runner.spec_algorithm.is_dspark()
),
)
if self.encoder_row_floor is not None:
# Folded hits: a tail row never reads below its request's replay start.
@@ -68,6 +68,7 @@ from sglang.srt.layers.attention.deepseek_v4_backend import (
_as_int_list,
_prefill_graph_max_seq_len,
_tail_rows,
late_layer_whole_extend,
)
from sglang.srt.layers.attention.dsv4.compressor_v2 import (
CompressorBackendMixin,
@@ -2215,7 +2216,8 @@ class DeepseekV4HipRadixBackend(
self, forward_batch: ForwardBatch
) -> DSV4Metadata:
"""Metadata for the layers after the last kv_source layer under decoder SWA bounded
replay: each request's last SWA_WINDOW extend tokens."""
replay: each request's last SWA_WINDOW extend tokens, or its whole extend
when it reads prompt rows (late_layer_whole_extend)."""
extend_lens_cpu = forward_batch.extend_seq_lens_cpu
seq_lens_cpu = forward_batch.seq_lens_cpu
assert extend_lens_cpu is not None and seq_lens_cpu is not None
@@ -2225,6 +2227,7 @@ class DeepseekV4HipRadixBackend(
seq_lens_cpu=seq_lens_cpu.tolist(),
tail_len=SWA_WINDOW,
device=device,
whole_extend=late_layer_whole_extend(forward_batch, dspark=self.is_dspark),
)
contiguous_start = (
extend_lens_cpu[0] - tail_lens_cpu[0] if len(extend_lens_cpu) == 1 else None
@@ -444,6 +444,13 @@ def _local_prefill_cuda_graph_vote(
return True
if not isinstance(prefill_graph_runner, PrefillCudaGraphRunner):
return False
backend_vote = prefill_graph_runner._backend_vote_prefill_cuda_graph
if (
mode in (ForwardMode.EXTEND, ForwardMode.MIXED)
and backend_vote is not None
and not backend_vote(local_batch)
):
return False
return prefill_graph_runner.can_replay_locally(
batch_size=local_batch.batch_size(),
num_tokens=num_tokens,
@@ -300,6 +300,8 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
_use_draft_input_embeds = False
_fa4_prefill = False
_backend_can_run_prefill_cuda_graph = None
# The backend's batch-dependent check, judged from a ScheduleBatch for the DP vote.
_backend_vote_prefill_cuda_graph = None
_captured_attn_metadata_max_bs: Optional[int] = None
dllm_attention = None
@@ -325,6 +327,9 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
if self.dllm_attention is not None
else getattr(model_runner.attn_backend, "can_run_prefill_cuda_graph", None)
)
self._backend_vote_prefill_cuda_graph = getattr(
model_runner.attn_backend, "vote_prefill_cuda_graph", None
)
prefill_attn_backend = getattr(
model_runner.attn_backend, "prefill_backend", model_runner.attn_backend
)
+27 -11
View File
@@ -3694,26 +3694,30 @@ class DeepseekV4Model(nn.Module):
hc_eps=self.hc_eps,
)
def _check_late_layer_tail_readers(self, forward_batch: ForwardBatch) -> None:
# Rows outside the tail are never computed past the last kv_source layer.
def _check_late_layer_tail_readers(
self, forward_batch: ForwardBatch, tail: LateLayerTail
) -> None:
# Rows outside the tail are never computed past the last kv_source layer;
# late_layer_whole_extend gives the requests that read them a whole tail.
extend_lens = forward_batch.extend_seq_lens_cpu
if (
forward_batch.capture_hidden_mode == CaptureHiddenMode.FULL
and self.dspark_layers_to_capture is None
and list(tail.extend_seq_lens_cpu) != list(extend_lens)
):
raise ValueError(
"decoder SWA bounded replay cannot capture hidden states of all "
"prompt tokens"
"decoder SWA bounded replay tail misses captured hidden-state rows"
)
if forward_batch.return_logprob and any(
start < n
for start, n in zip(
start < n - t
for start, n, t in zip(
forward_batch.extend_logprob_start_lens_cpu,
forward_batch.extend_seq_lens_cpu,
extend_lens,
tail.extend_seq_lens_cpu,
)
):
raise ValueError(
"decoder SWA bounded replay cannot return logprobs of prompt tokens; "
"set logprob_start_len to the prompt length"
"decoder SWA bounded replay tail misses prompt-logprob rows"
)
def _forward_layers_hc_pre_from_prev(
@@ -3758,9 +3762,9 @@ class DeepseekV4Model(nn.Module):
self.late_layer_start is not None
and forward_batch.forward_mode.is_extend_without_speculative()
):
self._check_late_layer_tail_readers(forward_batch)
attn_backend = get_attn_backend()
tail = attn_backend.tail_forward_metadata.late_layer_tail
self._check_late_layer_tail_readers(forward_batch, tail)
tail_in_graph = (
tail is not None
and tail.scatter_indices is not None
@@ -4602,7 +4606,19 @@ class DeepseekV4ForCausalLM(nn.Module):
logits_metadata = LogitsMetadata.from_forward_batch(forward_batch)
logits_metadata.extend_seq_lens = tail.extend_seq_lens
logits_metadata.extend_seq_lens_cpu = tail.extend_seq_lens_cpu
logits_metadata.extend_logprob_start_lens_cpu = tail.extend_seq_lens_cpu
logits_metadata.extend_logprob_start_lens_cpu = (
# A prompt-logprob reader's tail is its whole extend, so its start
# stands; every other start is the extend's end.
[
min(start, t)
for start, t in zip(
forward_batch.extend_logprob_start_lens_cpu,
tail.extend_seq_lens_cpu,
)
]
if forward_batch.return_logprob
else tail.extend_seq_lens_cpu
)
output = self.logits_processor(
input_ids,