From a2953d7b7fe9c7bb705e681e5011d3db80adfac7 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Sat, 17 Oct 2026 01:25:33 -0700 Subject: [PATCH] [Fix] deepseek-v4.1: Serve prompt-logprob requests under decoder SWA bounded replay (#44850) Co-authored-by: Pedram Ghazi Co-authored-by: David Orman --- .../autoregressive/DeepSeek/DeepSeek-V4_1.mdx | 2 +- .../attention/dsv4_attn_metadata_kernels.py | 58 +++++- .../sglang/srt/arg_groups/deepseek_v4_hook.py | 2 + python/sglang/srt/arg_groups/fields/exec_.py | 7 +- .../layers/attention/deepseek_v4_backend.py | 88 ++++++++- .../deepseek_v4_backend_hip_radix.py | 5 +- .../managers/scheduler_components/dp_attn.py | 7 + .../runner/prefill_cuda_graph_runner.py | 5 + python/sglang/srt/models/deepseek_v4.py | 38 ++-- .../test_dsv41_late_layer_tail_readers.py | 177 ++++++++++++++++++ .../scheduler_components/test_dp_attn.py | 2 + .../model_executor/test_encoder_swa_replay.py | 10 +- 12 files changed, 376 insertions(+), 25 deletions(-) create mode 100644 test/registered/unit/layers/test_dsv41_late_layer_tail_readers.py diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1.mdx index 650d63092e..ef885e89b4 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1.mdx @@ -105,7 +105,7 @@ import { agentx } from "/src/snippets/agentx/deepseek-ai/deepseek-v4_1.jsx"; **Engram.** An additive n-gram hash memory at two layers. Token ids are normalized *before* hashing, so `" The"`, `"the"` and `"THE"` cannot fork into separate rows. Its two fp8 tables are the largest single block of weight in the checkpoint, and by default they load row-sharded across the TP group, which costs an all-reduce per engram layer. `SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1` (**opt-in**) moves them to host memory instead: the freed HBM goes to the KV pool and output is bitwise unchanged — at the cost of host RAM, a longer load, and needing huge-page backing to keep the gather cheap. `SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT` trades the two all-reduces for page size: `shared` (the default) maps a single copy all ranks read and needs no reduce; `per_rank` splits that same table into one shard per rank and keeps the reduce, but is the only one the kernel reliably backs with 2 MiB pages. -**SWA bounded replay.** Decoder SWA bounded replay runs the early layers over the whole extend and the late layers over only each request’s last 128 tokens, sharing late-layer KV from source layer 20. It defaults on for supported standalone CUDA/HIP configurations without context parallelism. Encoder SWA replay can be combined with decoder replay, subject to encoder-specific validation. Eager and breakable prefill are supported; full prefill graphs keep it disabled. With DP attention, automatic enablement requires attention TP 1 and the `none` or `megamoe` MoE backend on CUDA, or `none` on HIP. Use `--enable-decoder-swa-bounded-replay` to explicitly enable replay, subject to compatibility validation. Use `--no-enable-decoder-swa-bounded-replay` to retain full prefill or request prompt logprobs. Bounded replay is not numerically equivalent to full prefill and does not support prompt logprobs. +**SWA bounded replay.** Decoder SWA bounded replay runs the early layers over the whole extend and the late layers over only each request’s last 128 tokens, sharing late-layer KV from source layer 20. It defaults on for supported standalone CUDA/HIP configurations without context parallelism. Encoder SWA replay can be combined with decoder replay, subject to encoder-specific validation. Eager and breakable prefill are supported; full prefill graphs keep it disabled. With DP attention, automatic enablement requires attention TP 1 and the `none` or `megamoe` MoE backend on CUDA, or `none` on HIP. Use `--enable-decoder-swa-bounded-replay` to explicitly enable replay, subject to compatibility validation. Bounded replay is not numerically equivalent to full prefill. A request for prompt logprobs or for every hidden state runs its whole prompt through the late layers with full sliding windows; other requests, including their output-token logprobs, use the bounded computation. Use `--no-enable-decoder-swa-bounded-replay` to retain full prefill, for example for RL rollouts whose logprobs must match the trainer. **Kernels.** At decode token counts these kernels are launch-bound rather than bandwidth-bound, so the work is fusion and overlap: the compressor projection, ratio-2 pooling, RoPE with fp4 quantization and indexer packing, paged fp4 indexer scoring with top-k resolved straight to cache addresses, and the mHC statistics and Sinkhorn — with compression, indexing and mHC overlapped against attention and FFN on dedicated streams. All of it is selected automatically; the environment switches it once sat behind were removed, so there is nothing to turn on. One caveat outranks the fusions: **output is not bitwise stable across batch composition today** — two default kernels are shape-guarded to a single token — and `--enable-deterministic-inference` is refused on this backend. diff --git a/python/sglang/kernels/ops/attention/dsv4_attn_metadata_kernels.py b/python/sglang/kernels/ops/attention/dsv4_attn_metadata_kernels.py index d2cbc1aa1e..0f4b589ab9 100644 --- a/python/sglang/kernels/ops/attention/dsv4_attn_metadata_kernels.py +++ b/python/sglang/kernels/ops/attention/dsv4_attn_metadata_kernels.py @@ -437,18 +437,44 @@ class BuildCausalSwaPageIndices: ) +def late_layer_tail_lens( + extend_lens_cpu: list[int], + tail_len: int, + whole_extend: Optional[list[bool]] = None, +) -> list[int]: + """Each request's late-layer tail: its last min(tail_len, extend_len) tokens, + or its whole extend where whole_extend is set.""" + if whole_extend is None: + return [min(tail_len, n) for n in extend_lens_cpu] + return [ + n if whole else min(tail_len, n) + for n, whole in zip(extend_lens_cpu, whole_extend) + ] + + def late_layer_tail_layout( *, extend_lens_cpu: list[int], seq_lens_cpu: list[int], tail_len: int, device: torch.device, + whole_extend: Optional[list[bool]] = None, ) -> tuple[torch.Tensor, list[int], torch.Tensor]: - """Tail rows of each prefill extend: its last min(tail_len, extend_len) tokens. + """Tail rows of each prefill extend: its last min(tail_len, extend_len) tokens, + or its whole extend where whole_extend is set. Returns (token indices into the extend, per-request tail lengths, per-row absolute window floor). The floor is seq_len - tail_len, so a tail shorter - than tail_len reads rows the request wrote before this extend.""" - tail_lens_cpu = [min(tail_len, n) for n in extend_lens_cpu] + than tail_len reads rows the request wrote before this extend. A whole + extend keeps every row's plain window.""" + if whole_extend is not None and any(whole_extend): + return _whole_extend_tail_layout( + extend_lens_cpu=extend_lens_cpu, + seq_lens_cpu=seq_lens_cpu, + tail_len=tail_len, + device=device, + whole_extend=whole_extend, + ) + tail_lens_cpu = late_layer_tail_lens(extend_lens_cpu, tail_len) if len(extend_lens_cpu) == 1: n, t, s = extend_lens_cpu[0], tail_lens_cpu[0], seq_lens_cpu[0] floor = torch.full((t,), max(0, s - tail_len), dtype=torch.int32, device=device) @@ -469,6 +495,32 @@ def late_layer_tail_layout( return token_indices, tail_lens_cpu, floor +def _whole_extend_tail_layout( + *, + extend_lens_cpu: list[int], + seq_lens_cpu: list[int], + tail_len: int, + device: torch.device, + whole_extend: list[bool], +) -> tuple[torch.Tensor, list[int], torch.Tensor]: + tail_lens_cpu = late_layer_tail_lens(extend_lens_cpu, tail_len, whole_extend) + token_indices, floor, start = [], [], 0 + for n, t, s, whole in zip( + extend_lens_cpu, tail_lens_cpu, seq_lens_cpu, whole_extend + ): + token_indices.append(torch.arange(start + n - t, start + n)) + # A whole extend's rows sit at s - t or later, so its floor never cuts + # their plain window. + row_floor = max(0, s - t - tail_len) if whole else max(0, s - tail_len) + floor.append(torch.full((t,), row_floor, dtype=torch.int32)) + start += n + return ( + torch.cat(token_indices).to(device, non_blocking=True), + tail_lens_cpu, + torch.cat(floor).to(device, non_blocking=True), + ) + + def build_causal_swa_page_indices( *, req_to_token: torch.Tensor, diff --git a/python/sglang/srt/arg_groups/deepseek_v4_hook.py b/python/sglang/srt/arg_groups/deepseek_v4_hook.py index c063d053c6..962f9c589a 100644 --- a/python/sglang/srt/arg_groups/deepseek_v4_hook.py +++ b/python/sglang/srt/arg_groups/deepseek_v4_hook.py @@ -411,6 +411,8 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None: (get_platform().is_cuda or get_platform().is_hip) and cfg.disaggregation_mode == "null" and not (cfg.enable_prefill_cp or cfg.attn_cp_size != 1 or cfg.dcp_size != 1) + # Returning every hidden state runs every prompt row: nothing to trim. + and not cfg.enable_return_hidden_states and not any(enabled for _, enabled in incompatible) ) declare_resolution( diff --git a/python/sglang/srt/arg_groups/fields/exec_.py b/python/sglang/srt/arg_groups/fields/exec_.py index 84c2cb5d0c..387fdc077b 100644 --- a/python/sglang/srt/arg_groups/fields/exec_.py +++ b/python/sglang/srt/arg_groups/fields/exec_.py @@ -137,9 +137,10 @@ class ExecFeatures(msgspec.Struct): Optional[bool], Arg( help="Enable DeepSeek-V4.1 decoder SWA bounded replay. Defaults on " - "for supported configurations. Use --no-enable-decoder-swa-bounded-replay " - "to retain full prefill or request prompt logprobs. " - "Not numerically equivalent to full prefill.", + "for supported configurations. Not numerically equivalent to full " + "prefill; requests for prompt logprobs compute every prompt row. Use " + "--no-enable-decoder-swa-bounded-replay to retain full prefill, e.g. " + "for RL rollouts whose logprobs must match the trainer.", action=argparse.BooleanOptionalAction, resolvable=True, ), diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index 15f0d08b57..fda802adaa 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -42,6 +42,7 @@ from sglang.kernels.ops.attention.dsv4_attn_metadata_kernels import ( BuildPageTablePositions, ExpandPrefillCausally, late_layer_tail_layout, + late_layer_tail_lens, ) from sglang.kernels.ops.speculative.dspark.dspark_attn_metadata import ( BuildBlockSeqLensCausal, @@ -107,7 +108,12 @@ from sglang.srt.layers.dp_attention import ( ) from sglang.srt.mem_cache.deepseek_v4_compress_state import KVAndScore from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool -from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode +from sglang.srt.model_executor.forward_batch_info import ( + CaptureHiddenMode, + ForwardBatch, + ForwardMode, + get_server_return_hidden_states_mode, +) from sglang.srt.runtime_context import ( get_exec, get_parallel, @@ -129,6 +135,7 @@ if TYPE_CHECKING: from sgl_kernel.flash_mla import FlashMLASchedMeta from sglang.srt.layers.radix_attention import RadixAttention + from sglang.srt.managers.schedule_batch import ScheduleBatch from sglang.srt.model_executor.model_runner import ModelRunner from sglang.srt.speculative.ragged_verify import RaggedVerifyLayout @@ -943,6 +950,41 @@ def _tail_rows( return t[token_indices] +def _late_layer_whole_extend( + *, + extend_lens: List[int], + logprob_start_lens: Optional[List[int]], + full_hidden_states: bool, +) -> Optional[List[bool]]: + """Requests that run their whole extend with plain windows under decoder SWA + bounded replay: those that read prompt rows (prompt logprobs, or every hidden + state outside DSpark, whose draft reads tail rows only). None when no request + in the batch does.""" + if full_hidden_states: + return [True] * len(extend_lens) + if logprob_start_lens is None: + return None + whole = [start < n for start, n in zip(logprob_start_lens, extend_lens)] + return whole if any(whole) else None + + +def late_layer_whole_extend( + forward_batch: ForwardBatch, *, dspark: bool +) -> Optional[List[bool]]: + """`_late_layer_whole_extend` of a forward batch. dspark comes from the server: + a capture batch carries DSpark's FULL capture mode without its spec_algorithm.""" + return _late_layer_whole_extend( + extend_lens=forward_batch.extend_seq_lens_cpu, + logprob_start_lens=( + forward_batch.extend_logprob_start_lens_cpu + if forward_batch.return_logprob + else None + ), + full_hidden_states=forward_batch.capture_hidden_mode == CaptureHiddenMode.FULL + and not dspark, + ) + + def _pad_rows(t: torch.Tensor, *, num_rows: int) -> torch.Tensor: assert t.shape[0] <= num_rows, (t.shape[0], num_rows) if t.shape[0] == num_rows: @@ -1573,12 +1615,45 @@ class DeepseekV4AttnBackend( return False if self.enable_decoder_swa_bounded_replay: extend_lens_cpu = forward_batch.extend_seq_lens_cpu - if extend_lens_cpu is not None: - tail_rows = sum(min(SWA_WINDOW, int(n)) for n in extend_lens_cpu) - if tail_rows > self._prefill_graph_tail_capacity(): - return False + if extend_lens_cpu is not None and not self._prefill_graph_fits_late_tail( + extend_lens_cpu, + late_layer_whole_extend( + forward_batch, dspark=self.model_runner.spec_algorithm.is_dspark() + ), + ): + return False return True + def vote_prefill_cuda_graph(self, batch: ScheduleBatch) -> bool: + """`can_run_prefill_cuda_graph`'s late-layer tail check judged from the + schedule batch, for the DP prefill-graph vote: no rank replays the graph + while another runs the step eagerly.""" + if not self.enable_decoder_swa_bounded_replay: + return True + full_hidden_states = ( + get_server_return_hidden_states_mode() == CaptureHiddenMode.FULL + or any( + req.return_hidden_states_mode == CaptureHiddenMode.FULL + for req in batch.reqs + ) + ) and not self.model_runner.spec_algorithm.is_dspark() + whole_extend = _late_layer_whole_extend( + extend_lens=batch.extend_lens, + logprob_start_lens=( + batch.extend_logprob_start_lens if batch.return_logprob else None + ), + full_hidden_states=full_hidden_states, + ) + return self._prefill_graph_fits_late_tail(batch.extend_lens, whole_extend) + + def _prefill_graph_fits_late_tail( + self, extend_lens: List[int], whole_extend: Optional[List[bool]] + ) -> bool: + tail_lens = late_layer_tail_lens( + [int(n) for n in extend_lens], SWA_WINDOW, whole_extend + ) + return sum(tail_lens) <= self._prefill_graph_tail_capacity() + def _prefill_graph_tail_capacity(self) -> int: """Rows the late layers run over inside the prefill CUDA graph: every request's SWA_WINDOW tail at the request cap, so one capture fits any batch.""" @@ -1634,6 +1709,9 @@ class DeepseekV4AttnBackend( seq_lens_cpu=seq_lens_cpu.tolist(), tail_len=SWA_WINDOW, device=device, + whole_extend=late_layer_whole_extend( + forward_batch, dspark=self.model_runner.spec_algorithm.is_dspark() + ), ) if self.encoder_row_floor is not None: # Folded hits: a tail row never reads below its request's replay start. diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py index cec3ebf141..980e0ef44d 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py @@ -68,6 +68,7 @@ from sglang.srt.layers.attention.deepseek_v4_backend import ( _as_int_list, _prefill_graph_max_seq_len, _tail_rows, + late_layer_whole_extend, ) from sglang.srt.layers.attention.dsv4.compressor_v2 import ( CompressorBackendMixin, @@ -2215,7 +2216,8 @@ class DeepseekV4HipRadixBackend( self, forward_batch: ForwardBatch ) -> DSV4Metadata: """Metadata for the layers after the last kv_source layer under decoder SWA bounded - replay: each request's last SWA_WINDOW extend tokens.""" + replay: each request's last SWA_WINDOW extend tokens, or its whole extend + when it reads prompt rows (late_layer_whole_extend).""" extend_lens_cpu = forward_batch.extend_seq_lens_cpu seq_lens_cpu = forward_batch.seq_lens_cpu assert extend_lens_cpu is not None and seq_lens_cpu is not None @@ -2225,6 +2227,7 @@ class DeepseekV4HipRadixBackend( seq_lens_cpu=seq_lens_cpu.tolist(), tail_len=SWA_WINDOW, device=device, + whole_extend=late_layer_whole_extend(forward_batch, dspark=self.is_dspark), ) contiguous_start = ( extend_lens_cpu[0] - tail_lens_cpu[0] if len(extend_lens_cpu) == 1 else None diff --git a/python/sglang/srt/managers/scheduler_components/dp_attn.py b/python/sglang/srt/managers/scheduler_components/dp_attn.py index aeb21bbb09..36f0ce00ba 100644 --- a/python/sglang/srt/managers/scheduler_components/dp_attn.py +++ b/python/sglang/srt/managers/scheduler_components/dp_attn.py @@ -444,6 +444,13 @@ def _local_prefill_cuda_graph_vote( return True if not isinstance(prefill_graph_runner, PrefillCudaGraphRunner): return False + backend_vote = prefill_graph_runner._backend_vote_prefill_cuda_graph + if ( + mode in (ForwardMode.EXTEND, ForwardMode.MIXED) + and backend_vote is not None + and not backend_vote(local_batch) + ): + return False return prefill_graph_runner.can_replay_locally( batch_size=local_batch.batch_size(), num_tokens=num_tokens, diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index 119997e5bf..dba75c283e 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -300,6 +300,8 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): _use_draft_input_embeds = False _fa4_prefill = False _backend_can_run_prefill_cuda_graph = None + # The backend's batch-dependent check, judged from a ScheduleBatch for the DP vote. + _backend_vote_prefill_cuda_graph = None _captured_attn_metadata_max_bs: Optional[int] = None dllm_attention = None @@ -325,6 +327,9 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): if self.dllm_attention is not None else getattr(model_runner.attn_backend, "can_run_prefill_cuda_graph", None) ) + self._backend_vote_prefill_cuda_graph = getattr( + model_runner.attn_backend, "vote_prefill_cuda_graph", None + ) prefill_attn_backend = getattr( model_runner.attn_backend, "prefill_backend", model_runner.attn_backend ) diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index 2111eec0ad..b8648f773c 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -3694,26 +3694,30 @@ class DeepseekV4Model(nn.Module): hc_eps=self.hc_eps, ) - def _check_late_layer_tail_readers(self, forward_batch: ForwardBatch) -> None: - # Rows outside the tail are never computed past the last kv_source layer. + def _check_late_layer_tail_readers( + self, forward_batch: ForwardBatch, tail: LateLayerTail + ) -> None: + # Rows outside the tail are never computed past the last kv_source layer; + # late_layer_whole_extend gives the requests that read them a whole tail. + extend_lens = forward_batch.extend_seq_lens_cpu if ( forward_batch.capture_hidden_mode == CaptureHiddenMode.FULL and self.dspark_layers_to_capture is None + and list(tail.extend_seq_lens_cpu) != list(extend_lens) ): raise ValueError( - "decoder SWA bounded replay cannot capture hidden states of all " - "prompt tokens" + "decoder SWA bounded replay tail misses captured hidden-state rows" ) if forward_batch.return_logprob and any( - start < n - for start, n in zip( + start < n - t + for start, n, t in zip( forward_batch.extend_logprob_start_lens_cpu, - forward_batch.extend_seq_lens_cpu, + extend_lens, + tail.extend_seq_lens_cpu, ) ): raise ValueError( - "decoder SWA bounded replay cannot return logprobs of prompt tokens; " - "set logprob_start_len to the prompt length" + "decoder SWA bounded replay tail misses prompt-logprob rows" ) def _forward_layers_hc_pre_from_prev( @@ -3758,9 +3762,9 @@ class DeepseekV4Model(nn.Module): self.late_layer_start is not None and forward_batch.forward_mode.is_extend_without_speculative() ): - self._check_late_layer_tail_readers(forward_batch) attn_backend = get_attn_backend() tail = attn_backend.tail_forward_metadata.late_layer_tail + self._check_late_layer_tail_readers(forward_batch, tail) tail_in_graph = ( tail is not None and tail.scatter_indices is not None @@ -4602,7 +4606,19 @@ class DeepseekV4ForCausalLM(nn.Module): logits_metadata = LogitsMetadata.from_forward_batch(forward_batch) logits_metadata.extend_seq_lens = tail.extend_seq_lens logits_metadata.extend_seq_lens_cpu = tail.extend_seq_lens_cpu - logits_metadata.extend_logprob_start_lens_cpu = tail.extend_seq_lens_cpu + logits_metadata.extend_logprob_start_lens_cpu = ( + # A prompt-logprob reader's tail is its whole extend, so its start + # stands; every other start is the extend's end. + [ + min(start, t) + for start, t in zip( + forward_batch.extend_logprob_start_lens_cpu, + tail.extend_seq_lens_cpu, + ) + ] + if forward_batch.return_logprob + else tail.extend_seq_lens_cpu + ) output = self.logits_processor( input_ids, diff --git a/test/registered/unit/layers/test_dsv41_late_layer_tail_readers.py b/test/registered/unit/layers/test_dsv41_late_layer_tail_readers.py new file mode 100644 index 0000000000..82c96a4fc3 --- /dev/null +++ b/test/registered/unit/layers/test_dsv41_late_layer_tail_readers.py @@ -0,0 +1,177 @@ +"""Decoder SWA bounded replay: requests that read prompt rows (prompt logprobs, +every hidden state) run their whole extend through the late layers. CPU only.""" + +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.kernels.ops.attention.dsv4_attn_metadata_kernels import ( + late_layer_tail_layout, +) +from sglang.srt.layers.attention import deepseek_v4_backend +from sglang.srt.layers.attention.deepseek_v4_backend import ( + SWA_WINDOW, + DeepseekV4AttnBackend, + late_layer_whole_extend, +) +from sglang.srt.model_executor.forward_batch_info import CaptureHiddenMode +from sglang.srt.models.deepseek_v4 import DeepseekV4Model +from sglang.srt.speculative.spec_info import SpeculativeAlgorithm +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +CPU = torch.device("cpu") + + +def _batch(extend_lens, starts=None, capture=CaptureHiddenMode.NULL): + return SimpleNamespace( + extend_seq_lens_cpu=extend_lens, + extend_logprob_start_lens_cpu=starts, + return_logprob=starts is not None, + capture_hidden_mode=capture, + ) + + +class TestWholeExtendRequests(CustomTestCase): + def test_no_prompt_reader_keeps_the_window_tail(self): + # No logprobs and output-only logprobs (start == extend) read no + # prompt row. + for batch in ( + _batch([300, 50]), + _batch([300, 50], starts=[300, 50]), + _batch([300], capture=CaptureHiddenMode.LAST), + ): + self.assertIsNone(late_layer_whole_extend(batch, dspark=False)) + + def test_dspark_capture_reads_the_tail_only(self): + # DSpark's target prefill, and its graph capture batches, capture FULL + # hidden states, but its draft reads tail rows only. + batch = _batch([300], capture=CaptureHiddenMode.FULL) + self.assertIsNone(late_layer_whole_extend(batch, dspark=True)) + + def test_any_prompt_logprob_reads_the_whole_extend(self): + # Inside the last window too, and in a prompt shorter than a window: + # those rows would otherwise get the window floor. + batch = _batch([300, 300, 300, 50], starts=[0, 300, 300 - SWA_WINDOW, 0]) + whole = late_layer_whole_extend(batch, dspark=False) + self.assertEqual(whole, [True, False, True, True]) + + def test_full_hidden_capture_reads_every_row(self): + batch = _batch([300, 50], capture=CaptureHiddenMode.FULL) + self.assertEqual(late_layer_whole_extend(batch, dspark=False), [True, True]) + + +class TestWholeExtendLayout(CustomTestCase): + def test_other_requests_match_the_window_layout(self): + extend_lens, seq_lens = [200, 300, 50], [300, 600, 50] + token_indices, tail_lens, floor = late_layer_tail_layout( + extend_lens_cpu=extend_lens, + seq_lens_cpu=seq_lens, + tail_len=SWA_WINDOW, + device=CPU, + whole_extend=[False, True, False], + ) + base_indices, base_lens, base_floor = late_layer_tail_layout( + extend_lens_cpu=extend_lens, + seq_lens_cpu=seq_lens, + tail_len=SWA_WINDOW, + device=CPU, + ) + self.assertEqual(tail_lens, [128, 300, 50]) + self.assertEqual(base_lens, [128, 128, 50]) + # The first and last requests keep the window layout row for row. + torch.testing.assert_close(token_indices[:128], base_indices[:128]) + torch.testing.assert_close(floor[:128], base_floor[:128]) + torch.testing.assert_close(token_indices[428:], base_indices[256:]) + torch.testing.assert_close(floor[428:], base_floor[256:]) + # The whole request: every row of its extend, each with its plain window + # (a floor a window below the extend start). + torch.testing.assert_close(token_indices[128:428], torch.arange(200, 500)) + expected = torch.full((300,), 600 - 300 - SWA_WINDOW, dtype=torch.int32) + torch.testing.assert_close(floor[128:428], expected) + + def test_short_whole_extend_keeps_its_plain_window(self): + # A chunk shorter than a window is floored one window back otherwise. + _, tail_lens, floor = late_layer_tail_layout( + extend_lens_cpu=[40], + seq_lens_cpu=[300], + tail_len=SWA_WINDOW, + device=CPU, + whole_extend=[True], + ) + self.assertEqual(tail_lens, [40]) + expected = torch.full((40,), 300 - 40 - SWA_WINDOW, dtype=torch.int32) + torch.testing.assert_close(floor, expected) + + +class TestPrefillGraphTailCapacity(CustomTestCase): + """The backend's graph check and the DP vote's view of it agree: no DP rank + replays the prefill graph while another runs the step eagerly.""" + + def _backend(self, capacity): + backend = object.__new__(DeepseekV4AttnBackend) + backend.enable_decoder_swa_bounded_replay = True + backend._prefill_graph_tail_capacity = lambda: capacity + backend.model_runner = SimpleNamespace(spec_algorithm=SpeculativeAlgorithm.NONE) + return backend + + def _both(self, extend_lens, capacity, starts=None, full=False): + hidden = CaptureHiddenMode.FULL if full else CaptureHiddenMode.NULL + forward_batch = _batch(extend_lens, starts, capture=hidden) + forward_batch.seq_lens_cpu = torch.tensor([400] * len(extend_lens)) + schedule_batch = SimpleNamespace( + extend_lens=extend_lens, + extend_logprob_start_lens=starts, + return_logprob=starts is not None, + reqs=[SimpleNamespace(return_hidden_states_mode=hidden)] * len(extend_lens), + ) + backend = self._backend(capacity) + with ( + patch.object( + deepseek_v4_backend, "_prefill_graph_max_seq_len", return_value=16384 + ), + patch.object( + deepseek_v4_backend, + "get_server_return_hidden_states_mode", + return_value=CaptureHiddenMode.NULL, + ), + ): + can_run = backend.can_run_prefill_cuda_graph(forward_batch) + vote = backend.vote_prefill_cuda_graph(schedule_batch) + self.assertEqual(can_run, vote) + return vote + + def test_whole_extend_counts_against_the_tail_block(self): + self.assertTrue(self._both([300, 300], capacity=256)) + # A prompt-logprob request's tail no longer fits: run the step eagerly. + self.assertFalse(self._both([300, 300], 256, starts=[0, 300])) + self.assertTrue(self._both([300, 300], 428, starts=[0, 300])) + self.assertFalse(self._both([300], 256, full=True)) + + +class TestTailReadersCheck(CustomTestCase): + def _check(self, batch, tail_lens, dspark=False): + model = SimpleNamespace(dspark_layers_to_capture=[1] if dspark else None) + tail = SimpleNamespace(extend_seq_lens_cpu=tail_lens) + DeepseekV4Model._check_late_layer_tail_readers(model, batch, tail) + + def test_tail_covering_the_readers_passes(self): + self._check(_batch([300, 50], starts=[300, 50]), [128, 50]) + self._check(_batch([300, 50], starts=[200, 0]), [128, 50]) + self._check(_batch([300, 50], starts=[0, 0]), [300, 50]) + self._check(_batch([300], capture=CaptureHiddenMode.FULL), [300]) + self._check(_batch([300], capture=CaptureHiddenMode.FULL), [128], dspark=True) + + def test_tail_missing_rows_raises(self): + with self.assertRaises(ValueError): + self._check(_batch([300], starts=[0]), [128]) + with self.assertRaises(ValueError): + self._check(_batch([300], capture=CaptureHiddenMode.FULL), [128]) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/managers/scheduler_components/test_dp_attn.py b/test/registered/unit/managers/scheduler_components/test_dp_attn.py index 11ba81d228..c54042397c 100644 --- a/test/registered/unit/managers/scheduler_components/test_dp_attn.py +++ b/test/registered/unit/managers/scheduler_components/test_dp_attn.py @@ -466,6 +466,7 @@ class TestPrefillCudaGraphVote(CustomTestCase): runner.max_context_size = None runner.max_seq_len = None runner._qwen_bcg_hc_sidechannel = True + runner._backend_vote_prefill_cuda_graph = None runner.can_replay_locally.side_effect = lambda **kwargs: ( not kwargs["contains_mm_inputs"] or runner_cls.can_replay_locally(self=runner, **kwargs) @@ -514,6 +515,7 @@ class TestPrefillCudaGraphVote(CustomTestCase): runner.max_context_size = None runner.max_seq_len = None runner._qwen_bcg_hc_sidechannel = False + runner._backend_vote_prefill_cuda_graph = None runner._is_full_backend = False runner._captured_attn_metadata_max_bs = 2 runner.prefer_eager_mixed_prefill = False diff --git a/test/registered/unit/model_executor/test_encoder_swa_replay.py b/test/registered/unit/model_executor/test_encoder_swa_replay.py index 982c698703..bc956ed172 100644 --- a/test/registered/unit/model_executor/test_encoder_swa_replay.py +++ b/test/registered/unit/model_executor/test_encoder_swa_replay.py @@ -25,8 +25,13 @@ from sglang.srt.model_executor.encoder_swa_replay import ( drop_folded_rows, folded_replay_rows, ) -from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode +from sglang.srt.model_executor.forward_batch_info import ( + CaptureHiddenMode, + ForwardBatch, + ForwardMode, +) from sglang.srt.runtime_context import get_context, get_parallel +from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -120,6 +125,7 @@ def _backend(fx): backend.low_ratios = () backend.has_c4 = backend.has_c128 = False backend.trtllm_attn = False + backend.model_runner = SimpleNamespace(spec_algorithm=SpeculativeAlgorithm.NONE) backend.cuda_int32_kwargs = {"dtype": torch.int32, "device": "cpu"} backend.token_to_kv_pool = SimpleNamespace( request_window=SimpleNamespace(capacity=2 * SWA_WINDOW) @@ -198,6 +204,8 @@ class TestFoldWithDecoderSwaTail(CustomTestCase): req_pool_indices=fb.req_pool_indices, out_cache_loc=fb.out_cache_loc, positions=fx.folded_positions(), + capture_hidden_mode=CaptureHiddenMode.NULL, + return_logprob=False, ) with ( patch.object(DSV4AttnMetadata, "init_compression_metadata"),