[Fix] deepseek-v4.1: Serve prompt-logprob requests under decoder SWA bounded replay (#44850)
Co-authored-by: Pedram Ghazi <pedram.ghazi@amd.com> Co-authored-by: David Orman <ormandj@corenode.com>
This commit is contained in:
co-authored by
Pedram Ghazi
David Orman
parent
7a8804edb5
commit
a2953d7b7f
@@ -105,7 +105,7 @@ import { agentx } from "/src/snippets/agentx/deepseek-ai/deepseek-v4_1.jsx";
|
||||
|
||||
**Engram.** An additive n-gram hash memory at two layers. Token ids are normalized *before* hashing, so `" The"`, `"the"` and `"THE"` cannot fork into separate rows. Its two fp8 tables are the largest single block of weight in the checkpoint, and by default they load row-sharded across the TP group, which costs an all-reduce per engram layer. `SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1` (**opt-in**) moves them to host memory instead: the freed HBM goes to the KV pool and output is bitwise unchanged — at the cost of host RAM, a longer load, and needing huge-page backing to keep the gather cheap. `SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT` trades the two all-reduces for page size: `shared` (the default) maps a single copy all ranks read and needs no reduce; `per_rank` splits that same table into one shard per rank and keeps the reduce, but is the only one the kernel reliably backs with 2 MiB pages.
|
||||
|
||||
**SWA bounded replay.** Decoder SWA bounded replay runs the early layers over the whole extend and the late layers over only each request’s last 128 tokens, sharing late-layer KV from source layer 20. It defaults on for supported standalone CUDA/HIP configurations without context parallelism. Encoder SWA replay can be combined with decoder replay, subject to encoder-specific validation. Eager and breakable prefill are supported; full prefill graphs keep it disabled. With DP attention, automatic enablement requires attention TP 1 and the `none` or `megamoe` MoE backend on CUDA, or `none` on HIP. Use `--enable-decoder-swa-bounded-replay` to explicitly enable replay, subject to compatibility validation. Use `--no-enable-decoder-swa-bounded-replay` to retain full prefill or request prompt logprobs. Bounded replay is not numerically equivalent to full prefill and does not support prompt logprobs.
|
||||
**SWA bounded replay.** Decoder SWA bounded replay runs the early layers over the whole extend and the late layers over only each request’s last 128 tokens, sharing late-layer KV from source layer 20. It defaults on for supported standalone CUDA/HIP configurations without context parallelism. Encoder SWA replay can be combined with decoder replay, subject to encoder-specific validation. Eager and breakable prefill are supported; full prefill graphs keep it disabled. With DP attention, automatic enablement requires attention TP 1 and the `none` or `megamoe` MoE backend on CUDA, or `none` on HIP. Use `--enable-decoder-swa-bounded-replay` to explicitly enable replay, subject to compatibility validation. Bounded replay is not numerically equivalent to full prefill. A request for prompt logprobs or for every hidden state runs its whole prompt through the late layers with full sliding windows; other requests, including their output-token logprobs, use the bounded computation. Use `--no-enable-decoder-swa-bounded-replay` to retain full prefill, for example for RL rollouts whose logprobs must match the trainer.
|
||||
|
||||
**Kernels.** At decode token counts these kernels are launch-bound rather than bandwidth-bound, so the work is fusion and overlap: the compressor projection, ratio-2 pooling, RoPE with fp4 quantization and indexer packing, paged fp4 indexer scoring with top-k resolved straight to cache addresses, and the mHC statistics and Sinkhorn — with compression, indexing and mHC overlapped against attention and FFN on dedicated streams. All of it is selected automatically; the environment switches it once sat behind were removed, so there is nothing to turn on. One caveat outranks the fusions: **output is not bitwise stable across batch composition today** — two default kernels are shape-guarded to a single token — and `--enable-deterministic-inference` is refused on this backend.
|
||||
|
||||
|
||||
@@ -437,18 +437,44 @@ class BuildCausalSwaPageIndices:
|
||||
)
|
||||
|
||||
|
||||
def late_layer_tail_lens(
|
||||
extend_lens_cpu: list[int],
|
||||
tail_len: int,
|
||||
whole_extend: Optional[list[bool]] = None,
|
||||
) -> list[int]:
|
||||
"""Each request's late-layer tail: its last min(tail_len, extend_len) tokens,
|
||||
or its whole extend where whole_extend is set."""
|
||||
if whole_extend is None:
|
||||
return [min(tail_len, n) for n in extend_lens_cpu]
|
||||
return [
|
||||
n if whole else min(tail_len, n)
|
||||
for n, whole in zip(extend_lens_cpu, whole_extend)
|
||||
]
|
||||
|
||||
|
||||
def late_layer_tail_layout(
|
||||
*,
|
||||
extend_lens_cpu: list[int],
|
||||
seq_lens_cpu: list[int],
|
||||
tail_len: int,
|
||||
device: torch.device,
|
||||
whole_extend: Optional[list[bool]] = None,
|
||||
) -> tuple[torch.Tensor, list[int], torch.Tensor]:
|
||||
"""Tail rows of each prefill extend: its last min(tail_len, extend_len) tokens.
|
||||
"""Tail rows of each prefill extend: its last min(tail_len, extend_len) tokens,
|
||||
or its whole extend where whole_extend is set.
|
||||
Returns (token indices into the extend, per-request tail lengths, per-row
|
||||
absolute window floor). The floor is seq_len - tail_len, so a tail shorter
|
||||
than tail_len reads rows the request wrote before this extend."""
|
||||
tail_lens_cpu = [min(tail_len, n) for n in extend_lens_cpu]
|
||||
than tail_len reads rows the request wrote before this extend. A whole
|
||||
extend keeps every row's plain window."""
|
||||
if whole_extend is not None and any(whole_extend):
|
||||
return _whole_extend_tail_layout(
|
||||
extend_lens_cpu=extend_lens_cpu,
|
||||
seq_lens_cpu=seq_lens_cpu,
|
||||
tail_len=tail_len,
|
||||
device=device,
|
||||
whole_extend=whole_extend,
|
||||
)
|
||||
tail_lens_cpu = late_layer_tail_lens(extend_lens_cpu, tail_len)
|
||||
if len(extend_lens_cpu) == 1:
|
||||
n, t, s = extend_lens_cpu[0], tail_lens_cpu[0], seq_lens_cpu[0]
|
||||
floor = torch.full((t,), max(0, s - tail_len), dtype=torch.int32, device=device)
|
||||
@@ -469,6 +495,32 @@ def late_layer_tail_layout(
|
||||
return token_indices, tail_lens_cpu, floor
|
||||
|
||||
|
||||
def _whole_extend_tail_layout(
|
||||
*,
|
||||
extend_lens_cpu: list[int],
|
||||
seq_lens_cpu: list[int],
|
||||
tail_len: int,
|
||||
device: torch.device,
|
||||
whole_extend: list[bool],
|
||||
) -> tuple[torch.Tensor, list[int], torch.Tensor]:
|
||||
tail_lens_cpu = late_layer_tail_lens(extend_lens_cpu, tail_len, whole_extend)
|
||||
token_indices, floor, start = [], [], 0
|
||||
for n, t, s, whole in zip(
|
||||
extend_lens_cpu, tail_lens_cpu, seq_lens_cpu, whole_extend
|
||||
):
|
||||
token_indices.append(torch.arange(start + n - t, start + n))
|
||||
# A whole extend's rows sit at s - t or later, so its floor never cuts
|
||||
# their plain window.
|
||||
row_floor = max(0, s - t - tail_len) if whole else max(0, s - tail_len)
|
||||
floor.append(torch.full((t,), row_floor, dtype=torch.int32))
|
||||
start += n
|
||||
return (
|
||||
torch.cat(token_indices).to(device, non_blocking=True),
|
||||
tail_lens_cpu,
|
||||
torch.cat(floor).to(device, non_blocking=True),
|
||||
)
|
||||
|
||||
|
||||
def build_causal_swa_page_indices(
|
||||
*,
|
||||
req_to_token: torch.Tensor,
|
||||
|
||||
@@ -411,6 +411,8 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
(get_platform().is_cuda or get_platform().is_hip)
|
||||
and cfg.disaggregation_mode == "null"
|
||||
and not (cfg.enable_prefill_cp or cfg.attn_cp_size != 1 or cfg.dcp_size != 1)
|
||||
# Returning every hidden state runs every prompt row: nothing to trim.
|
||||
and not cfg.enable_return_hidden_states
|
||||
and not any(enabled for _, enabled in incompatible)
|
||||
)
|
||||
declare_resolution(
|
||||
|
||||
@@ -137,9 +137,10 @@ class ExecFeatures(msgspec.Struct):
|
||||
Optional[bool],
|
||||
Arg(
|
||||
help="Enable DeepSeek-V4.1 decoder SWA bounded replay. Defaults on "
|
||||
"for supported configurations. Use --no-enable-decoder-swa-bounded-replay "
|
||||
"to retain full prefill or request prompt logprobs. "
|
||||
"Not numerically equivalent to full prefill.",
|
||||
"for supported configurations. Not numerically equivalent to full "
|
||||
"prefill; requests for prompt logprobs compute every prompt row. Use "
|
||||
"--no-enable-decoder-swa-bounded-replay to retain full prefill, e.g. "
|
||||
"for RL rollouts whose logprobs must match the trainer.",
|
||||
action=argparse.BooleanOptionalAction,
|
||||
resolvable=True,
|
||||
),
|
||||
|
||||
@@ -42,6 +42,7 @@ from sglang.kernels.ops.attention.dsv4_attn_metadata_kernels import (
|
||||
BuildPageTablePositions,
|
||||
ExpandPrefillCausally,
|
||||
late_layer_tail_layout,
|
||||
late_layer_tail_lens,
|
||||
)
|
||||
from sglang.kernels.ops.speculative.dspark.dspark_attn_metadata import (
|
||||
BuildBlockSeqLensCausal,
|
||||
@@ -107,7 +108,12 @@ from sglang.srt.layers.dp_attention import (
|
||||
)
|
||||
from sglang.srt.mem_cache.deepseek_v4_compress_state import KVAndScore
|
||||
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
|
||||
from sglang.srt.model_executor.forward_batch_info import (
|
||||
CaptureHiddenMode,
|
||||
ForwardBatch,
|
||||
ForwardMode,
|
||||
get_server_return_hidden_states_mode,
|
||||
)
|
||||
from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_parallel,
|
||||
@@ -129,6 +135,7 @@ if TYPE_CHECKING:
|
||||
from sgl_kernel.flash_mla import FlashMLASchedMeta
|
||||
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
from sglang.srt.speculative.ragged_verify import RaggedVerifyLayout
|
||||
|
||||
@@ -943,6 +950,41 @@ def _tail_rows(
|
||||
return t[token_indices]
|
||||
|
||||
|
||||
def _late_layer_whole_extend(
|
||||
*,
|
||||
extend_lens: List[int],
|
||||
logprob_start_lens: Optional[List[int]],
|
||||
full_hidden_states: bool,
|
||||
) -> Optional[List[bool]]:
|
||||
"""Requests that run their whole extend with plain windows under decoder SWA
|
||||
bounded replay: those that read prompt rows (prompt logprobs, or every hidden
|
||||
state outside DSpark, whose draft reads tail rows only). None when no request
|
||||
in the batch does."""
|
||||
if full_hidden_states:
|
||||
return [True] * len(extend_lens)
|
||||
if logprob_start_lens is None:
|
||||
return None
|
||||
whole = [start < n for start, n in zip(logprob_start_lens, extend_lens)]
|
||||
return whole if any(whole) else None
|
||||
|
||||
|
||||
def late_layer_whole_extend(
|
||||
forward_batch: ForwardBatch, *, dspark: bool
|
||||
) -> Optional[List[bool]]:
|
||||
"""`_late_layer_whole_extend` of a forward batch. dspark comes from the server:
|
||||
a capture batch carries DSpark's FULL capture mode without its spec_algorithm."""
|
||||
return _late_layer_whole_extend(
|
||||
extend_lens=forward_batch.extend_seq_lens_cpu,
|
||||
logprob_start_lens=(
|
||||
forward_batch.extend_logprob_start_lens_cpu
|
||||
if forward_batch.return_logprob
|
||||
else None
|
||||
),
|
||||
full_hidden_states=forward_batch.capture_hidden_mode == CaptureHiddenMode.FULL
|
||||
and not dspark,
|
||||
)
|
||||
|
||||
|
||||
def _pad_rows(t: torch.Tensor, *, num_rows: int) -> torch.Tensor:
|
||||
assert t.shape[0] <= num_rows, (t.shape[0], num_rows)
|
||||
if t.shape[0] == num_rows:
|
||||
@@ -1573,12 +1615,45 @@ class DeepseekV4AttnBackend(
|
||||
return False
|
||||
if self.enable_decoder_swa_bounded_replay:
|
||||
extend_lens_cpu = forward_batch.extend_seq_lens_cpu
|
||||
if extend_lens_cpu is not None:
|
||||
tail_rows = sum(min(SWA_WINDOW, int(n)) for n in extend_lens_cpu)
|
||||
if tail_rows > self._prefill_graph_tail_capacity():
|
||||
return False
|
||||
if extend_lens_cpu is not None and not self._prefill_graph_fits_late_tail(
|
||||
extend_lens_cpu,
|
||||
late_layer_whole_extend(
|
||||
forward_batch, dspark=self.model_runner.spec_algorithm.is_dspark()
|
||||
),
|
||||
):
|
||||
return False
|
||||
return True
|
||||
|
||||
def vote_prefill_cuda_graph(self, batch: ScheduleBatch) -> bool:
|
||||
"""`can_run_prefill_cuda_graph`'s late-layer tail check judged from the
|
||||
schedule batch, for the DP prefill-graph vote: no rank replays the graph
|
||||
while another runs the step eagerly."""
|
||||
if not self.enable_decoder_swa_bounded_replay:
|
||||
return True
|
||||
full_hidden_states = (
|
||||
get_server_return_hidden_states_mode() == CaptureHiddenMode.FULL
|
||||
or any(
|
||||
req.return_hidden_states_mode == CaptureHiddenMode.FULL
|
||||
for req in batch.reqs
|
||||
)
|
||||
) and not self.model_runner.spec_algorithm.is_dspark()
|
||||
whole_extend = _late_layer_whole_extend(
|
||||
extend_lens=batch.extend_lens,
|
||||
logprob_start_lens=(
|
||||
batch.extend_logprob_start_lens if batch.return_logprob else None
|
||||
),
|
||||
full_hidden_states=full_hidden_states,
|
||||
)
|
||||
return self._prefill_graph_fits_late_tail(batch.extend_lens, whole_extend)
|
||||
|
||||
def _prefill_graph_fits_late_tail(
|
||||
self, extend_lens: List[int], whole_extend: Optional[List[bool]]
|
||||
) -> bool:
|
||||
tail_lens = late_layer_tail_lens(
|
||||
[int(n) for n in extend_lens], SWA_WINDOW, whole_extend
|
||||
)
|
||||
return sum(tail_lens) <= self._prefill_graph_tail_capacity()
|
||||
|
||||
def _prefill_graph_tail_capacity(self) -> int:
|
||||
"""Rows the late layers run over inside the prefill CUDA graph: every
|
||||
request's SWA_WINDOW tail at the request cap, so one capture fits any batch."""
|
||||
@@ -1634,6 +1709,9 @@ class DeepseekV4AttnBackend(
|
||||
seq_lens_cpu=seq_lens_cpu.tolist(),
|
||||
tail_len=SWA_WINDOW,
|
||||
device=device,
|
||||
whole_extend=late_layer_whole_extend(
|
||||
forward_batch, dspark=self.model_runner.spec_algorithm.is_dspark()
|
||||
),
|
||||
)
|
||||
if self.encoder_row_floor is not None:
|
||||
# Folded hits: a tail row never reads below its request's replay start.
|
||||
|
||||
@@ -68,6 +68,7 @@ from sglang.srt.layers.attention.deepseek_v4_backend import (
|
||||
_as_int_list,
|
||||
_prefill_graph_max_seq_len,
|
||||
_tail_rows,
|
||||
late_layer_whole_extend,
|
||||
)
|
||||
from sglang.srt.layers.attention.dsv4.compressor_v2 import (
|
||||
CompressorBackendMixin,
|
||||
@@ -2215,7 +2216,8 @@ class DeepseekV4HipRadixBackend(
|
||||
self, forward_batch: ForwardBatch
|
||||
) -> DSV4Metadata:
|
||||
"""Metadata for the layers after the last kv_source layer under decoder SWA bounded
|
||||
replay: each request's last SWA_WINDOW extend tokens."""
|
||||
replay: each request's last SWA_WINDOW extend tokens, or its whole extend
|
||||
when it reads prompt rows (late_layer_whole_extend)."""
|
||||
extend_lens_cpu = forward_batch.extend_seq_lens_cpu
|
||||
seq_lens_cpu = forward_batch.seq_lens_cpu
|
||||
assert extend_lens_cpu is not None and seq_lens_cpu is not None
|
||||
@@ -2225,6 +2227,7 @@ class DeepseekV4HipRadixBackend(
|
||||
seq_lens_cpu=seq_lens_cpu.tolist(),
|
||||
tail_len=SWA_WINDOW,
|
||||
device=device,
|
||||
whole_extend=late_layer_whole_extend(forward_batch, dspark=self.is_dspark),
|
||||
)
|
||||
contiguous_start = (
|
||||
extend_lens_cpu[0] - tail_lens_cpu[0] if len(extend_lens_cpu) == 1 else None
|
||||
|
||||
@@ -444,6 +444,13 @@ def _local_prefill_cuda_graph_vote(
|
||||
return True
|
||||
if not isinstance(prefill_graph_runner, PrefillCudaGraphRunner):
|
||||
return False
|
||||
backend_vote = prefill_graph_runner._backend_vote_prefill_cuda_graph
|
||||
if (
|
||||
mode in (ForwardMode.EXTEND, ForwardMode.MIXED)
|
||||
and backend_vote is not None
|
||||
and not backend_vote(local_batch)
|
||||
):
|
||||
return False
|
||||
return prefill_graph_runner.can_replay_locally(
|
||||
batch_size=local_batch.batch_size(),
|
||||
num_tokens=num_tokens,
|
||||
|
||||
@@ -300,6 +300,8 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
_use_draft_input_embeds = False
|
||||
_fa4_prefill = False
|
||||
_backend_can_run_prefill_cuda_graph = None
|
||||
# The backend's batch-dependent check, judged from a ScheduleBatch for the DP vote.
|
||||
_backend_vote_prefill_cuda_graph = None
|
||||
_captured_attn_metadata_max_bs: Optional[int] = None
|
||||
dllm_attention = None
|
||||
|
||||
@@ -325,6 +327,9 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
if self.dllm_attention is not None
|
||||
else getattr(model_runner.attn_backend, "can_run_prefill_cuda_graph", None)
|
||||
)
|
||||
self._backend_vote_prefill_cuda_graph = getattr(
|
||||
model_runner.attn_backend, "vote_prefill_cuda_graph", None
|
||||
)
|
||||
prefill_attn_backend = getattr(
|
||||
model_runner.attn_backend, "prefill_backend", model_runner.attn_backend
|
||||
)
|
||||
|
||||
@@ -3694,26 +3694,30 @@ class DeepseekV4Model(nn.Module):
|
||||
hc_eps=self.hc_eps,
|
||||
)
|
||||
|
||||
def _check_late_layer_tail_readers(self, forward_batch: ForwardBatch) -> None:
|
||||
# Rows outside the tail are never computed past the last kv_source layer.
|
||||
def _check_late_layer_tail_readers(
|
||||
self, forward_batch: ForwardBatch, tail: LateLayerTail
|
||||
) -> None:
|
||||
# Rows outside the tail are never computed past the last kv_source layer;
|
||||
# late_layer_whole_extend gives the requests that read them a whole tail.
|
||||
extend_lens = forward_batch.extend_seq_lens_cpu
|
||||
if (
|
||||
forward_batch.capture_hidden_mode == CaptureHiddenMode.FULL
|
||||
and self.dspark_layers_to_capture is None
|
||||
and list(tail.extend_seq_lens_cpu) != list(extend_lens)
|
||||
):
|
||||
raise ValueError(
|
||||
"decoder SWA bounded replay cannot capture hidden states of all "
|
||||
"prompt tokens"
|
||||
"decoder SWA bounded replay tail misses captured hidden-state rows"
|
||||
)
|
||||
if forward_batch.return_logprob and any(
|
||||
start < n
|
||||
for start, n in zip(
|
||||
start < n - t
|
||||
for start, n, t in zip(
|
||||
forward_batch.extend_logprob_start_lens_cpu,
|
||||
forward_batch.extend_seq_lens_cpu,
|
||||
extend_lens,
|
||||
tail.extend_seq_lens_cpu,
|
||||
)
|
||||
):
|
||||
raise ValueError(
|
||||
"decoder SWA bounded replay cannot return logprobs of prompt tokens; "
|
||||
"set logprob_start_len to the prompt length"
|
||||
"decoder SWA bounded replay tail misses prompt-logprob rows"
|
||||
)
|
||||
|
||||
def _forward_layers_hc_pre_from_prev(
|
||||
@@ -3758,9 +3762,9 @@ class DeepseekV4Model(nn.Module):
|
||||
self.late_layer_start is not None
|
||||
and forward_batch.forward_mode.is_extend_without_speculative()
|
||||
):
|
||||
self._check_late_layer_tail_readers(forward_batch)
|
||||
attn_backend = get_attn_backend()
|
||||
tail = attn_backend.tail_forward_metadata.late_layer_tail
|
||||
self._check_late_layer_tail_readers(forward_batch, tail)
|
||||
tail_in_graph = (
|
||||
tail is not None
|
||||
and tail.scatter_indices is not None
|
||||
@@ -4602,7 +4606,19 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
logits_metadata = LogitsMetadata.from_forward_batch(forward_batch)
|
||||
logits_metadata.extend_seq_lens = tail.extend_seq_lens
|
||||
logits_metadata.extend_seq_lens_cpu = tail.extend_seq_lens_cpu
|
||||
logits_metadata.extend_logprob_start_lens_cpu = tail.extend_seq_lens_cpu
|
||||
logits_metadata.extend_logprob_start_lens_cpu = (
|
||||
# A prompt-logprob reader's tail is its whole extend, so its start
|
||||
# stands; every other start is the extend's end.
|
||||
[
|
||||
min(start, t)
|
||||
for start, t in zip(
|
||||
forward_batch.extend_logprob_start_lens_cpu,
|
||||
tail.extend_seq_lens_cpu,
|
||||
)
|
||||
]
|
||||
if forward_batch.return_logprob
|
||||
else tail.extend_seq_lens_cpu
|
||||
)
|
||||
|
||||
output = self.logits_processor(
|
||||
input_ids,
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
"""Decoder SWA bounded replay: requests that read prompt rows (prompt logprobs,
|
||||
every hidden state) run their whole extend through the late layers. CPU only."""
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.attention.dsv4_attn_metadata_kernels import (
|
||||
late_layer_tail_layout,
|
||||
)
|
||||
from sglang.srt.layers.attention import deepseek_v4_backend
|
||||
from sglang.srt.layers.attention.deepseek_v4_backend import (
|
||||
SWA_WINDOW,
|
||||
DeepseekV4AttnBackend,
|
||||
late_layer_whole_extend,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import CaptureHiddenMode
|
||||
from sglang.srt.models.deepseek_v4 import DeepseekV4Model
|
||||
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
|
||||
CPU = torch.device("cpu")
|
||||
|
||||
|
||||
def _batch(extend_lens, starts=None, capture=CaptureHiddenMode.NULL):
|
||||
return SimpleNamespace(
|
||||
extend_seq_lens_cpu=extend_lens,
|
||||
extend_logprob_start_lens_cpu=starts,
|
||||
return_logprob=starts is not None,
|
||||
capture_hidden_mode=capture,
|
||||
)
|
||||
|
||||
|
||||
class TestWholeExtendRequests(CustomTestCase):
|
||||
def test_no_prompt_reader_keeps_the_window_tail(self):
|
||||
# No logprobs and output-only logprobs (start == extend) read no
|
||||
# prompt row.
|
||||
for batch in (
|
||||
_batch([300, 50]),
|
||||
_batch([300, 50], starts=[300, 50]),
|
||||
_batch([300], capture=CaptureHiddenMode.LAST),
|
||||
):
|
||||
self.assertIsNone(late_layer_whole_extend(batch, dspark=False))
|
||||
|
||||
def test_dspark_capture_reads_the_tail_only(self):
|
||||
# DSpark's target prefill, and its graph capture batches, capture FULL
|
||||
# hidden states, but its draft reads tail rows only.
|
||||
batch = _batch([300], capture=CaptureHiddenMode.FULL)
|
||||
self.assertIsNone(late_layer_whole_extend(batch, dspark=True))
|
||||
|
||||
def test_any_prompt_logprob_reads_the_whole_extend(self):
|
||||
# Inside the last window too, and in a prompt shorter than a window:
|
||||
# those rows would otherwise get the window floor.
|
||||
batch = _batch([300, 300, 300, 50], starts=[0, 300, 300 - SWA_WINDOW, 0])
|
||||
whole = late_layer_whole_extend(batch, dspark=False)
|
||||
self.assertEqual(whole, [True, False, True, True])
|
||||
|
||||
def test_full_hidden_capture_reads_every_row(self):
|
||||
batch = _batch([300, 50], capture=CaptureHiddenMode.FULL)
|
||||
self.assertEqual(late_layer_whole_extend(batch, dspark=False), [True, True])
|
||||
|
||||
|
||||
class TestWholeExtendLayout(CustomTestCase):
|
||||
def test_other_requests_match_the_window_layout(self):
|
||||
extend_lens, seq_lens = [200, 300, 50], [300, 600, 50]
|
||||
token_indices, tail_lens, floor = late_layer_tail_layout(
|
||||
extend_lens_cpu=extend_lens,
|
||||
seq_lens_cpu=seq_lens,
|
||||
tail_len=SWA_WINDOW,
|
||||
device=CPU,
|
||||
whole_extend=[False, True, False],
|
||||
)
|
||||
base_indices, base_lens, base_floor = late_layer_tail_layout(
|
||||
extend_lens_cpu=extend_lens,
|
||||
seq_lens_cpu=seq_lens,
|
||||
tail_len=SWA_WINDOW,
|
||||
device=CPU,
|
||||
)
|
||||
self.assertEqual(tail_lens, [128, 300, 50])
|
||||
self.assertEqual(base_lens, [128, 128, 50])
|
||||
# The first and last requests keep the window layout row for row.
|
||||
torch.testing.assert_close(token_indices[:128], base_indices[:128])
|
||||
torch.testing.assert_close(floor[:128], base_floor[:128])
|
||||
torch.testing.assert_close(token_indices[428:], base_indices[256:])
|
||||
torch.testing.assert_close(floor[428:], base_floor[256:])
|
||||
# The whole request: every row of its extend, each with its plain window
|
||||
# (a floor a window below the extend start).
|
||||
torch.testing.assert_close(token_indices[128:428], torch.arange(200, 500))
|
||||
expected = torch.full((300,), 600 - 300 - SWA_WINDOW, dtype=torch.int32)
|
||||
torch.testing.assert_close(floor[128:428], expected)
|
||||
|
||||
def test_short_whole_extend_keeps_its_plain_window(self):
|
||||
# A chunk shorter than a window is floored one window back otherwise.
|
||||
_, tail_lens, floor = late_layer_tail_layout(
|
||||
extend_lens_cpu=[40],
|
||||
seq_lens_cpu=[300],
|
||||
tail_len=SWA_WINDOW,
|
||||
device=CPU,
|
||||
whole_extend=[True],
|
||||
)
|
||||
self.assertEqual(tail_lens, [40])
|
||||
expected = torch.full((40,), 300 - 40 - SWA_WINDOW, dtype=torch.int32)
|
||||
torch.testing.assert_close(floor, expected)
|
||||
|
||||
|
||||
class TestPrefillGraphTailCapacity(CustomTestCase):
|
||||
"""The backend's graph check and the DP vote's view of it agree: no DP rank
|
||||
replays the prefill graph while another runs the step eagerly."""
|
||||
|
||||
def _backend(self, capacity):
|
||||
backend = object.__new__(DeepseekV4AttnBackend)
|
||||
backend.enable_decoder_swa_bounded_replay = True
|
||||
backend._prefill_graph_tail_capacity = lambda: capacity
|
||||
backend.model_runner = SimpleNamespace(spec_algorithm=SpeculativeAlgorithm.NONE)
|
||||
return backend
|
||||
|
||||
def _both(self, extend_lens, capacity, starts=None, full=False):
|
||||
hidden = CaptureHiddenMode.FULL if full else CaptureHiddenMode.NULL
|
||||
forward_batch = _batch(extend_lens, starts, capture=hidden)
|
||||
forward_batch.seq_lens_cpu = torch.tensor([400] * len(extend_lens))
|
||||
schedule_batch = SimpleNamespace(
|
||||
extend_lens=extend_lens,
|
||||
extend_logprob_start_lens=starts,
|
||||
return_logprob=starts is not None,
|
||||
reqs=[SimpleNamespace(return_hidden_states_mode=hidden)] * len(extend_lens),
|
||||
)
|
||||
backend = self._backend(capacity)
|
||||
with (
|
||||
patch.object(
|
||||
deepseek_v4_backend, "_prefill_graph_max_seq_len", return_value=16384
|
||||
),
|
||||
patch.object(
|
||||
deepseek_v4_backend,
|
||||
"get_server_return_hidden_states_mode",
|
||||
return_value=CaptureHiddenMode.NULL,
|
||||
),
|
||||
):
|
||||
can_run = backend.can_run_prefill_cuda_graph(forward_batch)
|
||||
vote = backend.vote_prefill_cuda_graph(schedule_batch)
|
||||
self.assertEqual(can_run, vote)
|
||||
return vote
|
||||
|
||||
def test_whole_extend_counts_against_the_tail_block(self):
|
||||
self.assertTrue(self._both([300, 300], capacity=256))
|
||||
# A prompt-logprob request's tail no longer fits: run the step eagerly.
|
||||
self.assertFalse(self._both([300, 300], 256, starts=[0, 300]))
|
||||
self.assertTrue(self._both([300, 300], 428, starts=[0, 300]))
|
||||
self.assertFalse(self._both([300], 256, full=True))
|
||||
|
||||
|
||||
class TestTailReadersCheck(CustomTestCase):
|
||||
def _check(self, batch, tail_lens, dspark=False):
|
||||
model = SimpleNamespace(dspark_layers_to_capture=[1] if dspark else None)
|
||||
tail = SimpleNamespace(extend_seq_lens_cpu=tail_lens)
|
||||
DeepseekV4Model._check_late_layer_tail_readers(model, batch, tail)
|
||||
|
||||
def test_tail_covering_the_readers_passes(self):
|
||||
self._check(_batch([300, 50], starts=[300, 50]), [128, 50])
|
||||
self._check(_batch([300, 50], starts=[200, 0]), [128, 50])
|
||||
self._check(_batch([300, 50], starts=[0, 0]), [300, 50])
|
||||
self._check(_batch([300], capture=CaptureHiddenMode.FULL), [300])
|
||||
self._check(_batch([300], capture=CaptureHiddenMode.FULL), [128], dspark=True)
|
||||
|
||||
def test_tail_missing_rows_raises(self):
|
||||
with self.assertRaises(ValueError):
|
||||
self._check(_batch([300], starts=[0]), [128])
|
||||
with self.assertRaises(ValueError):
|
||||
self._check(_batch([300], capture=CaptureHiddenMode.FULL), [128])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -466,6 +466,7 @@ class TestPrefillCudaGraphVote(CustomTestCase):
|
||||
runner.max_context_size = None
|
||||
runner.max_seq_len = None
|
||||
runner._qwen_bcg_hc_sidechannel = True
|
||||
runner._backend_vote_prefill_cuda_graph = None
|
||||
runner.can_replay_locally.side_effect = lambda **kwargs: (
|
||||
not kwargs["contains_mm_inputs"]
|
||||
or runner_cls.can_replay_locally(self=runner, **kwargs)
|
||||
@@ -514,6 +515,7 @@ class TestPrefillCudaGraphVote(CustomTestCase):
|
||||
runner.max_context_size = None
|
||||
runner.max_seq_len = None
|
||||
runner._qwen_bcg_hc_sidechannel = False
|
||||
runner._backend_vote_prefill_cuda_graph = None
|
||||
runner._is_full_backend = False
|
||||
runner._captured_attn_metadata_max_bs = 2
|
||||
runner.prefer_eager_mixed_prefill = False
|
||||
|
||||
@@ -25,8 +25,13 @@ from sglang.srt.model_executor.encoder_swa_replay import (
|
||||
drop_folded_rows,
|
||||
folded_replay_rows,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
|
||||
from sglang.srt.model_executor.forward_batch_info import (
|
||||
CaptureHiddenMode,
|
||||
ForwardBatch,
|
||||
ForwardMode,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_context, get_parallel
|
||||
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
@@ -120,6 +125,7 @@ def _backend(fx):
|
||||
backend.low_ratios = ()
|
||||
backend.has_c4 = backend.has_c128 = False
|
||||
backend.trtllm_attn = False
|
||||
backend.model_runner = SimpleNamespace(spec_algorithm=SpeculativeAlgorithm.NONE)
|
||||
backend.cuda_int32_kwargs = {"dtype": torch.int32, "device": "cpu"}
|
||||
backend.token_to_kv_pool = SimpleNamespace(
|
||||
request_window=SimpleNamespace(capacity=2 * SWA_WINDOW)
|
||||
@@ -198,6 +204,8 @@ class TestFoldWithDecoderSwaTail(CustomTestCase):
|
||||
req_pool_indices=fb.req_pool_indices,
|
||||
out_cache_loc=fb.out_cache_loc,
|
||||
positions=fx.folded_positions(),
|
||||
capture_hidden_mode=CaptureHiddenMode.NULL,
|
||||
return_logprob=False,
|
||||
)
|
||||
with (
|
||||
patch.object(DSV4AttnMetadata, "init_compression_metadata"),
|
||||
|
||||
Reference in New Issue
Block a user