[Fix] deepseek-v4.1: Serve prompt-logprob requests under decoder SWA bounded replay (#44850)
Co-authored-by: Pedram Ghazi <pedram.ghazi@amd.com> Co-authored-by: David Orman <ormandj@corenode.com>
This commit is contained in:
co-authored by
Pedram Ghazi
David Orman
parent
7a8804edb5
commit
a2953d7b7f
@@ -437,18 +437,44 @@ class BuildCausalSwaPageIndices:
|
||||
)
|
||||
|
||||
|
||||
def late_layer_tail_lens(
|
||||
extend_lens_cpu: list[int],
|
||||
tail_len: int,
|
||||
whole_extend: Optional[list[bool]] = None,
|
||||
) -> list[int]:
|
||||
"""Each request's late-layer tail: its last min(tail_len, extend_len) tokens,
|
||||
or its whole extend where whole_extend is set."""
|
||||
if whole_extend is None:
|
||||
return [min(tail_len, n) for n in extend_lens_cpu]
|
||||
return [
|
||||
n if whole else min(tail_len, n)
|
||||
for n, whole in zip(extend_lens_cpu, whole_extend)
|
||||
]
|
||||
|
||||
|
||||
def late_layer_tail_layout(
|
||||
*,
|
||||
extend_lens_cpu: list[int],
|
||||
seq_lens_cpu: list[int],
|
||||
tail_len: int,
|
||||
device: torch.device,
|
||||
whole_extend: Optional[list[bool]] = None,
|
||||
) -> tuple[torch.Tensor, list[int], torch.Tensor]:
|
||||
"""Tail rows of each prefill extend: its last min(tail_len, extend_len) tokens.
|
||||
"""Tail rows of each prefill extend: its last min(tail_len, extend_len) tokens,
|
||||
or its whole extend where whole_extend is set.
|
||||
Returns (token indices into the extend, per-request tail lengths, per-row
|
||||
absolute window floor). The floor is seq_len - tail_len, so a tail shorter
|
||||
than tail_len reads rows the request wrote before this extend."""
|
||||
tail_lens_cpu = [min(tail_len, n) for n in extend_lens_cpu]
|
||||
than tail_len reads rows the request wrote before this extend. A whole
|
||||
extend keeps every row's plain window."""
|
||||
if whole_extend is not None and any(whole_extend):
|
||||
return _whole_extend_tail_layout(
|
||||
extend_lens_cpu=extend_lens_cpu,
|
||||
seq_lens_cpu=seq_lens_cpu,
|
||||
tail_len=tail_len,
|
||||
device=device,
|
||||
whole_extend=whole_extend,
|
||||
)
|
||||
tail_lens_cpu = late_layer_tail_lens(extend_lens_cpu, tail_len)
|
||||
if len(extend_lens_cpu) == 1:
|
||||
n, t, s = extend_lens_cpu[0], tail_lens_cpu[0], seq_lens_cpu[0]
|
||||
floor = torch.full((t,), max(0, s - tail_len), dtype=torch.int32, device=device)
|
||||
@@ -469,6 +495,32 @@ def late_layer_tail_layout(
|
||||
return token_indices, tail_lens_cpu, floor
|
||||
|
||||
|
||||
def _whole_extend_tail_layout(
|
||||
*,
|
||||
extend_lens_cpu: list[int],
|
||||
seq_lens_cpu: list[int],
|
||||
tail_len: int,
|
||||
device: torch.device,
|
||||
whole_extend: list[bool],
|
||||
) -> tuple[torch.Tensor, list[int], torch.Tensor]:
|
||||
tail_lens_cpu = late_layer_tail_lens(extend_lens_cpu, tail_len, whole_extend)
|
||||
token_indices, floor, start = [], [], 0
|
||||
for n, t, s, whole in zip(
|
||||
extend_lens_cpu, tail_lens_cpu, seq_lens_cpu, whole_extend
|
||||
):
|
||||
token_indices.append(torch.arange(start + n - t, start + n))
|
||||
# A whole extend's rows sit at s - t or later, so its floor never cuts
|
||||
# their plain window.
|
||||
row_floor = max(0, s - t - tail_len) if whole else max(0, s - tail_len)
|
||||
floor.append(torch.full((t,), row_floor, dtype=torch.int32))
|
||||
start += n
|
||||
return (
|
||||
torch.cat(token_indices).to(device, non_blocking=True),
|
||||
tail_lens_cpu,
|
||||
torch.cat(floor).to(device, non_blocking=True),
|
||||
)
|
||||
|
||||
|
||||
def build_causal_swa_page_indices(
|
||||
*,
|
||||
req_to_token: torch.Tensor,
|
||||
|
||||
@@ -411,6 +411,8 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
(get_platform().is_cuda or get_platform().is_hip)
|
||||
and cfg.disaggregation_mode == "null"
|
||||
and not (cfg.enable_prefill_cp or cfg.attn_cp_size != 1 or cfg.dcp_size != 1)
|
||||
# Returning every hidden state runs every prompt row: nothing to trim.
|
||||
and not cfg.enable_return_hidden_states
|
||||
and not any(enabled for _, enabled in incompatible)
|
||||
)
|
||||
declare_resolution(
|
||||
|
||||
@@ -137,9 +137,10 @@ class ExecFeatures(msgspec.Struct):
|
||||
Optional[bool],
|
||||
Arg(
|
||||
help="Enable DeepSeek-V4.1 decoder SWA bounded replay. Defaults on "
|
||||
"for supported configurations. Use --no-enable-decoder-swa-bounded-replay "
|
||||
"to retain full prefill or request prompt logprobs. "
|
||||
"Not numerically equivalent to full prefill.",
|
||||
"for supported configurations. Not numerically equivalent to full "
|
||||
"prefill; requests for prompt logprobs compute every prompt row. Use "
|
||||
"--no-enable-decoder-swa-bounded-replay to retain full prefill, e.g. "
|
||||
"for RL rollouts whose logprobs must match the trainer.",
|
||||
action=argparse.BooleanOptionalAction,
|
||||
resolvable=True,
|
||||
),
|
||||
|
||||
@@ -42,6 +42,7 @@ from sglang.kernels.ops.attention.dsv4_attn_metadata_kernels import (
|
||||
BuildPageTablePositions,
|
||||
ExpandPrefillCausally,
|
||||
late_layer_tail_layout,
|
||||
late_layer_tail_lens,
|
||||
)
|
||||
from sglang.kernels.ops.speculative.dspark.dspark_attn_metadata import (
|
||||
BuildBlockSeqLensCausal,
|
||||
@@ -107,7 +108,12 @@ from sglang.srt.layers.dp_attention import (
|
||||
)
|
||||
from sglang.srt.mem_cache.deepseek_v4_compress_state import KVAndScore
|
||||
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
|
||||
from sglang.srt.model_executor.forward_batch_info import (
|
||||
CaptureHiddenMode,
|
||||
ForwardBatch,
|
||||
ForwardMode,
|
||||
get_server_return_hidden_states_mode,
|
||||
)
|
||||
from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_parallel,
|
||||
@@ -129,6 +135,7 @@ if TYPE_CHECKING:
|
||||
from sgl_kernel.flash_mla import FlashMLASchedMeta
|
||||
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
from sglang.srt.speculative.ragged_verify import RaggedVerifyLayout
|
||||
|
||||
@@ -943,6 +950,41 @@ def _tail_rows(
|
||||
return t[token_indices]
|
||||
|
||||
|
||||
def _late_layer_whole_extend(
|
||||
*,
|
||||
extend_lens: List[int],
|
||||
logprob_start_lens: Optional[List[int]],
|
||||
full_hidden_states: bool,
|
||||
) -> Optional[List[bool]]:
|
||||
"""Requests that run their whole extend with plain windows under decoder SWA
|
||||
bounded replay: those that read prompt rows (prompt logprobs, or every hidden
|
||||
state outside DSpark, whose draft reads tail rows only). None when no request
|
||||
in the batch does."""
|
||||
if full_hidden_states:
|
||||
return [True] * len(extend_lens)
|
||||
if logprob_start_lens is None:
|
||||
return None
|
||||
whole = [start < n for start, n in zip(logprob_start_lens, extend_lens)]
|
||||
return whole if any(whole) else None
|
||||
|
||||
|
||||
def late_layer_whole_extend(
|
||||
forward_batch: ForwardBatch, *, dspark: bool
|
||||
) -> Optional[List[bool]]:
|
||||
"""`_late_layer_whole_extend` of a forward batch. dspark comes from the server:
|
||||
a capture batch carries DSpark's FULL capture mode without its spec_algorithm."""
|
||||
return _late_layer_whole_extend(
|
||||
extend_lens=forward_batch.extend_seq_lens_cpu,
|
||||
logprob_start_lens=(
|
||||
forward_batch.extend_logprob_start_lens_cpu
|
||||
if forward_batch.return_logprob
|
||||
else None
|
||||
),
|
||||
full_hidden_states=forward_batch.capture_hidden_mode == CaptureHiddenMode.FULL
|
||||
and not dspark,
|
||||
)
|
||||
|
||||
|
||||
def _pad_rows(t: torch.Tensor, *, num_rows: int) -> torch.Tensor:
|
||||
assert t.shape[0] <= num_rows, (t.shape[0], num_rows)
|
||||
if t.shape[0] == num_rows:
|
||||
@@ -1573,12 +1615,45 @@ class DeepseekV4AttnBackend(
|
||||
return False
|
||||
if self.enable_decoder_swa_bounded_replay:
|
||||
extend_lens_cpu = forward_batch.extend_seq_lens_cpu
|
||||
if extend_lens_cpu is not None:
|
||||
tail_rows = sum(min(SWA_WINDOW, int(n)) for n in extend_lens_cpu)
|
||||
if tail_rows > self._prefill_graph_tail_capacity():
|
||||
return False
|
||||
if extend_lens_cpu is not None and not self._prefill_graph_fits_late_tail(
|
||||
extend_lens_cpu,
|
||||
late_layer_whole_extend(
|
||||
forward_batch, dspark=self.model_runner.spec_algorithm.is_dspark()
|
||||
),
|
||||
):
|
||||
return False
|
||||
return True
|
||||
|
||||
def vote_prefill_cuda_graph(self, batch: ScheduleBatch) -> bool:
|
||||
"""`can_run_prefill_cuda_graph`'s late-layer tail check judged from the
|
||||
schedule batch, for the DP prefill-graph vote: no rank replays the graph
|
||||
while another runs the step eagerly."""
|
||||
if not self.enable_decoder_swa_bounded_replay:
|
||||
return True
|
||||
full_hidden_states = (
|
||||
get_server_return_hidden_states_mode() == CaptureHiddenMode.FULL
|
||||
or any(
|
||||
req.return_hidden_states_mode == CaptureHiddenMode.FULL
|
||||
for req in batch.reqs
|
||||
)
|
||||
) and not self.model_runner.spec_algorithm.is_dspark()
|
||||
whole_extend = _late_layer_whole_extend(
|
||||
extend_lens=batch.extend_lens,
|
||||
logprob_start_lens=(
|
||||
batch.extend_logprob_start_lens if batch.return_logprob else None
|
||||
),
|
||||
full_hidden_states=full_hidden_states,
|
||||
)
|
||||
return self._prefill_graph_fits_late_tail(batch.extend_lens, whole_extend)
|
||||
|
||||
def _prefill_graph_fits_late_tail(
|
||||
self, extend_lens: List[int], whole_extend: Optional[List[bool]]
|
||||
) -> bool:
|
||||
tail_lens = late_layer_tail_lens(
|
||||
[int(n) for n in extend_lens], SWA_WINDOW, whole_extend
|
||||
)
|
||||
return sum(tail_lens) <= self._prefill_graph_tail_capacity()
|
||||
|
||||
def _prefill_graph_tail_capacity(self) -> int:
|
||||
"""Rows the late layers run over inside the prefill CUDA graph: every
|
||||
request's SWA_WINDOW tail at the request cap, so one capture fits any batch."""
|
||||
@@ -1634,6 +1709,9 @@ class DeepseekV4AttnBackend(
|
||||
seq_lens_cpu=seq_lens_cpu.tolist(),
|
||||
tail_len=SWA_WINDOW,
|
||||
device=device,
|
||||
whole_extend=late_layer_whole_extend(
|
||||
forward_batch, dspark=self.model_runner.spec_algorithm.is_dspark()
|
||||
),
|
||||
)
|
||||
if self.encoder_row_floor is not None:
|
||||
# Folded hits: a tail row never reads below its request's replay start.
|
||||
|
||||
@@ -68,6 +68,7 @@ from sglang.srt.layers.attention.deepseek_v4_backend import (
|
||||
_as_int_list,
|
||||
_prefill_graph_max_seq_len,
|
||||
_tail_rows,
|
||||
late_layer_whole_extend,
|
||||
)
|
||||
from sglang.srt.layers.attention.dsv4.compressor_v2 import (
|
||||
CompressorBackendMixin,
|
||||
@@ -2215,7 +2216,8 @@ class DeepseekV4HipRadixBackend(
|
||||
self, forward_batch: ForwardBatch
|
||||
) -> DSV4Metadata:
|
||||
"""Metadata for the layers after the last kv_source layer under decoder SWA bounded
|
||||
replay: each request's last SWA_WINDOW extend tokens."""
|
||||
replay: each request's last SWA_WINDOW extend tokens, or its whole extend
|
||||
when it reads prompt rows (late_layer_whole_extend)."""
|
||||
extend_lens_cpu = forward_batch.extend_seq_lens_cpu
|
||||
seq_lens_cpu = forward_batch.seq_lens_cpu
|
||||
assert extend_lens_cpu is not None and seq_lens_cpu is not None
|
||||
@@ -2225,6 +2227,7 @@ class DeepseekV4HipRadixBackend(
|
||||
seq_lens_cpu=seq_lens_cpu.tolist(),
|
||||
tail_len=SWA_WINDOW,
|
||||
device=device,
|
||||
whole_extend=late_layer_whole_extend(forward_batch, dspark=self.is_dspark),
|
||||
)
|
||||
contiguous_start = (
|
||||
extend_lens_cpu[0] - tail_lens_cpu[0] if len(extend_lens_cpu) == 1 else None
|
||||
|
||||
@@ -444,6 +444,13 @@ def _local_prefill_cuda_graph_vote(
|
||||
return True
|
||||
if not isinstance(prefill_graph_runner, PrefillCudaGraphRunner):
|
||||
return False
|
||||
backend_vote = prefill_graph_runner._backend_vote_prefill_cuda_graph
|
||||
if (
|
||||
mode in (ForwardMode.EXTEND, ForwardMode.MIXED)
|
||||
and backend_vote is not None
|
||||
and not backend_vote(local_batch)
|
||||
):
|
||||
return False
|
||||
return prefill_graph_runner.can_replay_locally(
|
||||
batch_size=local_batch.batch_size(),
|
||||
num_tokens=num_tokens,
|
||||
|
||||
@@ -300,6 +300,8 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
_use_draft_input_embeds = False
|
||||
_fa4_prefill = False
|
||||
_backend_can_run_prefill_cuda_graph = None
|
||||
# The backend's batch-dependent check, judged from a ScheduleBatch for the DP vote.
|
||||
_backend_vote_prefill_cuda_graph = None
|
||||
_captured_attn_metadata_max_bs: Optional[int] = None
|
||||
dllm_attention = None
|
||||
|
||||
@@ -325,6 +327,9 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
if self.dllm_attention is not None
|
||||
else getattr(model_runner.attn_backend, "can_run_prefill_cuda_graph", None)
|
||||
)
|
||||
self._backend_vote_prefill_cuda_graph = getattr(
|
||||
model_runner.attn_backend, "vote_prefill_cuda_graph", None
|
||||
)
|
||||
prefill_attn_backend = getattr(
|
||||
model_runner.attn_backend, "prefill_backend", model_runner.attn_backend
|
||||
)
|
||||
|
||||
@@ -3694,26 +3694,30 @@ class DeepseekV4Model(nn.Module):
|
||||
hc_eps=self.hc_eps,
|
||||
)
|
||||
|
||||
def _check_late_layer_tail_readers(self, forward_batch: ForwardBatch) -> None:
|
||||
# Rows outside the tail are never computed past the last kv_source layer.
|
||||
def _check_late_layer_tail_readers(
|
||||
self, forward_batch: ForwardBatch, tail: LateLayerTail
|
||||
) -> None:
|
||||
# Rows outside the tail are never computed past the last kv_source layer;
|
||||
# late_layer_whole_extend gives the requests that read them a whole tail.
|
||||
extend_lens = forward_batch.extend_seq_lens_cpu
|
||||
if (
|
||||
forward_batch.capture_hidden_mode == CaptureHiddenMode.FULL
|
||||
and self.dspark_layers_to_capture is None
|
||||
and list(tail.extend_seq_lens_cpu) != list(extend_lens)
|
||||
):
|
||||
raise ValueError(
|
||||
"decoder SWA bounded replay cannot capture hidden states of all "
|
||||
"prompt tokens"
|
||||
"decoder SWA bounded replay tail misses captured hidden-state rows"
|
||||
)
|
||||
if forward_batch.return_logprob and any(
|
||||
start < n
|
||||
for start, n in zip(
|
||||
start < n - t
|
||||
for start, n, t in zip(
|
||||
forward_batch.extend_logprob_start_lens_cpu,
|
||||
forward_batch.extend_seq_lens_cpu,
|
||||
extend_lens,
|
||||
tail.extend_seq_lens_cpu,
|
||||
)
|
||||
):
|
||||
raise ValueError(
|
||||
"decoder SWA bounded replay cannot return logprobs of prompt tokens; "
|
||||
"set logprob_start_len to the prompt length"
|
||||
"decoder SWA bounded replay tail misses prompt-logprob rows"
|
||||
)
|
||||
|
||||
def _forward_layers_hc_pre_from_prev(
|
||||
@@ -3758,9 +3762,9 @@ class DeepseekV4Model(nn.Module):
|
||||
self.late_layer_start is not None
|
||||
and forward_batch.forward_mode.is_extend_without_speculative()
|
||||
):
|
||||
self._check_late_layer_tail_readers(forward_batch)
|
||||
attn_backend = get_attn_backend()
|
||||
tail = attn_backend.tail_forward_metadata.late_layer_tail
|
||||
self._check_late_layer_tail_readers(forward_batch, tail)
|
||||
tail_in_graph = (
|
||||
tail is not None
|
||||
and tail.scatter_indices is not None
|
||||
@@ -4602,7 +4606,19 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
logits_metadata = LogitsMetadata.from_forward_batch(forward_batch)
|
||||
logits_metadata.extend_seq_lens = tail.extend_seq_lens
|
||||
logits_metadata.extend_seq_lens_cpu = tail.extend_seq_lens_cpu
|
||||
logits_metadata.extend_logprob_start_lens_cpu = tail.extend_seq_lens_cpu
|
||||
logits_metadata.extend_logprob_start_lens_cpu = (
|
||||
# A prompt-logprob reader's tail is its whole extend, so its start
|
||||
# stands; every other start is the extend's end.
|
||||
[
|
||||
min(start, t)
|
||||
for start, t in zip(
|
||||
forward_batch.extend_logprob_start_lens_cpu,
|
||||
tail.extend_seq_lens_cpu,
|
||||
)
|
||||
]
|
||||
if forward_batch.return_logprob
|
||||
else tail.extend_seq_lens_cpu
|
||||
)
|
||||
|
||||
output = self.logits_processor(
|
||||
input_ids,
|
||||
|
||||
Reference in New Issue
Block a user