[Fix] deepseek-v4.1: Serve prompt-logprob requests under decoder SWA bounded replay (#44850)

Co-authored-by: Pedram Ghazi <pedram.ghazi@amd.com>
Co-authored-by: David Orman <ormandj@corenode.com>
This commit is contained in:
Liangsheng Yin
2026-10-17 01:25:33 -07:00
committed by GitHub
co-authored by Pedram Ghazi David Orman
parent 7a8804edb5
commit a2953d7b7f
12 changed files with 376 additions and 25 deletions
@@ -105,7 +105,7 @@ import { agentx } from "/src/snippets/agentx/deepseek-ai/deepseek-v4_1.jsx";
**Engram.** An additive n-gram hash memory at two layers. Token ids are normalized *before* hashing, so `" The"`, `"the"` and `"THE"` cannot fork into separate rows. Its two fp8 tables are the largest single block of weight in the checkpoint, and by default they load row-sharded across the TP group, which costs an all-reduce per engram layer. `SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1` (**opt-in**) moves them to host memory instead: the freed HBM goes to the KV pool and output is bitwise unchanged — at the cost of host RAM, a longer load, and needing huge-page backing to keep the gather cheap. `SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT` trades the two all-reduces for page size: `shared` (the default) maps a single copy all ranks read and needs no reduce; `per_rank` splits that same table into one shard per rank and keeps the reduce, but is the only one the kernel reliably backs with 2 MiB pages.
**SWA bounded replay.** Decoder SWA bounded replay runs the early layers over the whole extend and the late layers over only each request’s last 128 tokens, sharing late-layer KV from source layer 20. It defaults on for supported standalone CUDA/HIP configurations without context parallelism. Encoder SWA replay can be combined with decoder replay, subject to encoder-specific validation. Eager and breakable prefill are supported; full prefill graphs keep it disabled. With DP attention, automatic enablement requires attention TP 1 and the `none` or `megamoe` MoE backend on CUDA, or `none` on HIP. Use `--enable-decoder-swa-bounded-replay` to explicitly enable replay, subject to compatibility validation. Use `--no-enable-decoder-swa-bounded-replay` to retain full prefill or request prompt logprobs. Bounded replay is not numerically equivalent to full prefill and does not support prompt logprobs.
**SWA bounded replay.** Decoder SWA bounded replay runs the early layers over the whole extend and the late layers over only each request’s last 128 tokens, sharing late-layer KV from source layer 20. It defaults on for supported standalone CUDA/HIP configurations without context parallelism. Encoder SWA replay can be combined with decoder replay, subject to encoder-specific validation. Eager and breakable prefill are supported; full prefill graphs keep it disabled. With DP attention, automatic enablement requires attention TP 1 and the `none` or `megamoe` MoE backend on CUDA, or `none` on HIP. Use `--enable-decoder-swa-bounded-replay` to explicitly enable replay, subject to compatibility validation. Bounded replay is not numerically equivalent to full prefill. A request for prompt logprobs or for every hidden state runs its whole prompt through the late layers with full sliding windows; other requests, including their output-token logprobs, use the bounded computation. Use `--no-enable-decoder-swa-bounded-replay` to retain full prefill, for example for RL rollouts whose logprobs must match the trainer.
**Kernels.** At decode token counts these kernels are launch-bound rather than bandwidth-bound, so the work is fusion and overlap: the compressor projection, ratio-2 pooling, RoPE with fp4 quantization and indexer packing, paged fp4 indexer scoring with top-k resolved straight to cache addresses, and the mHC statistics and Sinkhorn — with compression, indexing and mHC overlapped against attention and FFN on dedicated streams. All of it is selected automatically; the environment switches it once sat behind were removed, so there is nothing to turn on. One caveat outranks the fusions: **output is not bitwise stable across batch composition today** — two default kernels are shape-guarded to a single token — and `--enable-deterministic-inference` is refused on this backend.
@@ -437,18 +437,44 @@ class BuildCausalSwaPageIndices:
)
def late_layer_tail_lens(
extend_lens_cpu: list[int],
tail_len: int,
whole_extend: Optional[list[bool]] = None,
) -> list[int]:
"""Each request's late-layer tail: its last min(tail_len, extend_len) tokens,
or its whole extend where whole_extend is set."""
if whole_extend is None:
return [min(tail_len, n) for n in extend_lens_cpu]
return [
n if whole else min(tail_len, n)
for n, whole in zip(extend_lens_cpu, whole_extend)
]
def late_layer_tail_layout(
*,
extend_lens_cpu: list[int],
seq_lens_cpu: list[int],
tail_len: int,
device: torch.device,
whole_extend: Optional[list[bool]] = None,
) -> tuple[torch.Tensor, list[int], torch.Tensor]:
"""Tail rows of each prefill extend: its last min(tail_len, extend_len) tokens.
"""Tail rows of each prefill extend: its last min(tail_len, extend_len) tokens,
or its whole extend where whole_extend is set.
Returns (token indices into the extend, per-request tail lengths, per-row
absolute window floor). The floor is seq_len - tail_len, so a tail shorter
than tail_len reads rows the request wrote before this extend."""
tail_lens_cpu = [min(tail_len, n) for n in extend_lens_cpu]
than tail_len reads rows the request wrote before this extend. A whole
extend keeps every row's plain window."""
if whole_extend is not None and any(whole_extend):
return _whole_extend_tail_layout(
extend_lens_cpu=extend_lens_cpu,
seq_lens_cpu=seq_lens_cpu,
tail_len=tail_len,
device=device,
whole_extend=whole_extend,
)
tail_lens_cpu = late_layer_tail_lens(extend_lens_cpu, tail_len)
if len(extend_lens_cpu) == 1:
n, t, s = extend_lens_cpu[0], tail_lens_cpu[0], seq_lens_cpu[0]
floor = torch.full((t,), max(0, s - tail_len), dtype=torch.int32, device=device)
@@ -469,6 +495,32 @@ def late_layer_tail_layout(
return token_indices, tail_lens_cpu, floor
def _whole_extend_tail_layout(
*,
extend_lens_cpu: list[int],
seq_lens_cpu: list[int],
tail_len: int,
device: torch.device,
whole_extend: list[bool],
) -> tuple[torch.Tensor, list[int], torch.Tensor]:
tail_lens_cpu = late_layer_tail_lens(extend_lens_cpu, tail_len, whole_extend)
token_indices, floor, start = [], [], 0
for n, t, s, whole in zip(
extend_lens_cpu, tail_lens_cpu, seq_lens_cpu, whole_extend
):
token_indices.append(torch.arange(start + n - t, start + n))
# A whole extend's rows sit at s - t or later, so its floor never cuts
# their plain window.
row_floor = max(0, s - t - tail_len) if whole else max(0, s - tail_len)
floor.append(torch.full((t,), row_floor, dtype=torch.int32))
start += n
return (
torch.cat(token_indices).to(device, non_blocking=True),
tail_lens_cpu,
torch.cat(floor).to(device, non_blocking=True),
)
def build_causal_swa_page_indices(
*,
req_to_token: torch.Tensor,
@@ -411,6 +411,8 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
(get_platform().is_cuda or get_platform().is_hip)
and cfg.disaggregation_mode == "null"
and not (cfg.enable_prefill_cp or cfg.attn_cp_size != 1 or cfg.dcp_size != 1)
# Returning every hidden state runs every prompt row: nothing to trim.
and not cfg.enable_return_hidden_states
and not any(enabled for _, enabled in incompatible)
)
declare_resolution(
+4 -3
View File
@@ -137,9 +137,10 @@ class ExecFeatures(msgspec.Struct):
Optional[bool],
Arg(
help="Enable DeepSeek-V4.1 decoder SWA bounded replay. Defaults on "
"for supported configurations. Use --no-enable-decoder-swa-bounded-replay "
"to retain full prefill or request prompt logprobs. "
"Not numerically equivalent to full prefill.",
"for supported configurations. Not numerically equivalent to full "
"prefill; requests for prompt logprobs compute every prompt row. Use "
"--no-enable-decoder-swa-bounded-replay to retain full prefill, e.g. "
"for RL rollouts whose logprobs must match the trainer.",
action=argparse.BooleanOptionalAction,
resolvable=True,
),
@@ -42,6 +42,7 @@ from sglang.kernels.ops.attention.dsv4_attn_metadata_kernels import (
BuildPageTablePositions,
ExpandPrefillCausally,
late_layer_tail_layout,
late_layer_tail_lens,
)
from sglang.kernels.ops.speculative.dspark.dspark_attn_metadata import (
BuildBlockSeqLensCausal,
@@ -107,7 +108,12 @@ from sglang.srt.layers.dp_attention import (
)
from sglang.srt.mem_cache.deepseek_v4_compress_state import KVAndScore
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
from sglang.srt.model_executor.forward_batch_info import (
CaptureHiddenMode,
ForwardBatch,
ForwardMode,
get_server_return_hidden_states_mode,
)
from sglang.srt.runtime_context import (
get_exec,
get_parallel,
@@ -129,6 +135,7 @@ if TYPE_CHECKING:
from sgl_kernel.flash_mla import FlashMLASchedMeta
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.managers.schedule_batch import ScheduleBatch
from sglang.srt.model_executor.model_runner import ModelRunner
from sglang.srt.speculative.ragged_verify import RaggedVerifyLayout
@@ -943,6 +950,41 @@ def _tail_rows(
return t[token_indices]
def _late_layer_whole_extend(
*,
extend_lens: List[int],
logprob_start_lens: Optional[List[int]],
full_hidden_states: bool,
) -> Optional[List[bool]]:
"""Requests that run their whole extend with plain windows under decoder SWA
bounded replay: those that read prompt rows (prompt logprobs, or every hidden
state outside DSpark, whose draft reads tail rows only). None when no request
in the batch does."""
if full_hidden_states:
return [True] * len(extend_lens)
if logprob_start_lens is None:
return None
whole = [start < n for start, n in zip(logprob_start_lens, extend_lens)]
return whole if any(whole) else None
def late_layer_whole_extend(
forward_batch: ForwardBatch, *, dspark: bool
) -> Optional[List[bool]]:
"""`_late_layer_whole_extend` of a forward batch. dspark comes from the server:
a capture batch carries DSpark's FULL capture mode without its spec_algorithm."""
return _late_layer_whole_extend(
extend_lens=forward_batch.extend_seq_lens_cpu,
logprob_start_lens=(
forward_batch.extend_logprob_start_lens_cpu
if forward_batch.return_logprob
else None
),
full_hidden_states=forward_batch.capture_hidden_mode == CaptureHiddenMode.FULL
and not dspark,
)
def _pad_rows(t: torch.Tensor, *, num_rows: int) -> torch.Tensor:
assert t.shape[0] <= num_rows, (t.shape[0], num_rows)
if t.shape[0] == num_rows:
@@ -1573,12 +1615,45 @@ class DeepseekV4AttnBackend(
return False
if self.enable_decoder_swa_bounded_replay:
extend_lens_cpu = forward_batch.extend_seq_lens_cpu
if extend_lens_cpu is not None:
tail_rows = sum(min(SWA_WINDOW, int(n)) for n in extend_lens_cpu)
if tail_rows > self._prefill_graph_tail_capacity():
return False
if extend_lens_cpu is not None and not self._prefill_graph_fits_late_tail(
extend_lens_cpu,
late_layer_whole_extend(
forward_batch, dspark=self.model_runner.spec_algorithm.is_dspark()
),
):
return False
return True
def vote_prefill_cuda_graph(self, batch: ScheduleBatch) -> bool:
"""`can_run_prefill_cuda_graph`'s late-layer tail check judged from the
schedule batch, for the DP prefill-graph vote: no rank replays the graph
while another runs the step eagerly."""
if not self.enable_decoder_swa_bounded_replay:
return True
full_hidden_states = (
get_server_return_hidden_states_mode() == CaptureHiddenMode.FULL
or any(
req.return_hidden_states_mode == CaptureHiddenMode.FULL
for req in batch.reqs
)
) and not self.model_runner.spec_algorithm.is_dspark()
whole_extend = _late_layer_whole_extend(
extend_lens=batch.extend_lens,
logprob_start_lens=(
batch.extend_logprob_start_lens if batch.return_logprob else None
),
full_hidden_states=full_hidden_states,
)
return self._prefill_graph_fits_late_tail(batch.extend_lens, whole_extend)
def _prefill_graph_fits_late_tail(
self, extend_lens: List[int], whole_extend: Optional[List[bool]]
) -> bool:
tail_lens = late_layer_tail_lens(
[int(n) for n in extend_lens], SWA_WINDOW, whole_extend
)
return sum(tail_lens) <= self._prefill_graph_tail_capacity()
def _prefill_graph_tail_capacity(self) -> int:
"""Rows the late layers run over inside the prefill CUDA graph: every
request's SWA_WINDOW tail at the request cap, so one capture fits any batch."""
@@ -1634,6 +1709,9 @@ class DeepseekV4AttnBackend(
seq_lens_cpu=seq_lens_cpu.tolist(),
tail_len=SWA_WINDOW,
device=device,
whole_extend=late_layer_whole_extend(
forward_batch, dspark=self.model_runner.spec_algorithm.is_dspark()
),
)
if self.encoder_row_floor is not None:
# Folded hits: a tail row never reads below its request's replay start.
@@ -68,6 +68,7 @@ from sglang.srt.layers.attention.deepseek_v4_backend import (
_as_int_list,
_prefill_graph_max_seq_len,
_tail_rows,
late_layer_whole_extend,
)
from sglang.srt.layers.attention.dsv4.compressor_v2 import (
CompressorBackendMixin,
@@ -2215,7 +2216,8 @@ class DeepseekV4HipRadixBackend(
self, forward_batch: ForwardBatch
) -> DSV4Metadata:
"""Metadata for the layers after the last kv_source layer under decoder SWA bounded
replay: each request's last SWA_WINDOW extend tokens."""
replay: each request's last SWA_WINDOW extend tokens, or its whole extend
when it reads prompt rows (late_layer_whole_extend)."""
extend_lens_cpu = forward_batch.extend_seq_lens_cpu
seq_lens_cpu = forward_batch.seq_lens_cpu
assert extend_lens_cpu is not None and seq_lens_cpu is not None
@@ -2225,6 +2227,7 @@ class DeepseekV4HipRadixBackend(
seq_lens_cpu=seq_lens_cpu.tolist(),
tail_len=SWA_WINDOW,
device=device,
whole_extend=late_layer_whole_extend(forward_batch, dspark=self.is_dspark),
)
contiguous_start = (
extend_lens_cpu[0] - tail_lens_cpu[0] if len(extend_lens_cpu) == 1 else None
@@ -444,6 +444,13 @@ def _local_prefill_cuda_graph_vote(
return True
if not isinstance(prefill_graph_runner, PrefillCudaGraphRunner):
return False
backend_vote = prefill_graph_runner._backend_vote_prefill_cuda_graph
if (
mode in (ForwardMode.EXTEND, ForwardMode.MIXED)
and backend_vote is not None
and not backend_vote(local_batch)
):
return False
return prefill_graph_runner.can_replay_locally(
batch_size=local_batch.batch_size(),
num_tokens=num_tokens,
@@ -300,6 +300,8 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
_use_draft_input_embeds = False
_fa4_prefill = False
_backend_can_run_prefill_cuda_graph = None
# The backend's batch-dependent check, judged from a ScheduleBatch for the DP vote.
_backend_vote_prefill_cuda_graph = None
_captured_attn_metadata_max_bs: Optional[int] = None
dllm_attention = None
@@ -325,6 +327,9 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
if self.dllm_attention is not None
else getattr(model_runner.attn_backend, "can_run_prefill_cuda_graph", None)
)
self._backend_vote_prefill_cuda_graph = getattr(
model_runner.attn_backend, "vote_prefill_cuda_graph", None
)
prefill_attn_backend = getattr(
model_runner.attn_backend, "prefill_backend", model_runner.attn_backend
)
+27 -11
View File
@@ -3694,26 +3694,30 @@ class DeepseekV4Model(nn.Module):
hc_eps=self.hc_eps,
)
def _check_late_layer_tail_readers(self, forward_batch: ForwardBatch) -> None:
# Rows outside the tail are never computed past the last kv_source layer.
def _check_late_layer_tail_readers(
self, forward_batch: ForwardBatch, tail: LateLayerTail
) -> None:
# Rows outside the tail are never computed past the last kv_source layer;
# late_layer_whole_extend gives the requests that read them a whole tail.
extend_lens = forward_batch.extend_seq_lens_cpu
if (
forward_batch.capture_hidden_mode == CaptureHiddenMode.FULL
and self.dspark_layers_to_capture is None
and list(tail.extend_seq_lens_cpu) != list(extend_lens)
):
raise ValueError(
"decoder SWA bounded replay cannot capture hidden states of all "
"prompt tokens"
"decoder SWA bounded replay tail misses captured hidden-state rows"
)
if forward_batch.return_logprob and any(
start < n
for start, n in zip(
start < n - t
for start, n, t in zip(
forward_batch.extend_logprob_start_lens_cpu,
forward_batch.extend_seq_lens_cpu,
extend_lens,
tail.extend_seq_lens_cpu,
)
):
raise ValueError(
"decoder SWA bounded replay cannot return logprobs of prompt tokens; "
"set logprob_start_len to the prompt length"
"decoder SWA bounded replay tail misses prompt-logprob rows"
)
def _forward_layers_hc_pre_from_prev(
@@ -3758,9 +3762,9 @@ class DeepseekV4Model(nn.Module):
self.late_layer_start is not None
and forward_batch.forward_mode.is_extend_without_speculative()
):
self._check_late_layer_tail_readers(forward_batch)
attn_backend = get_attn_backend()
tail = attn_backend.tail_forward_metadata.late_layer_tail
self._check_late_layer_tail_readers(forward_batch, tail)
tail_in_graph = (
tail is not None
and tail.scatter_indices is not None
@@ -4602,7 +4606,19 @@ class DeepseekV4ForCausalLM(nn.Module):
logits_metadata = LogitsMetadata.from_forward_batch(forward_batch)
logits_metadata.extend_seq_lens = tail.extend_seq_lens
logits_metadata.extend_seq_lens_cpu = tail.extend_seq_lens_cpu
logits_metadata.extend_logprob_start_lens_cpu = tail.extend_seq_lens_cpu
logits_metadata.extend_logprob_start_lens_cpu = (
# A prompt-logprob reader's tail is its whole extend, so its start
# stands; every other start is the extend's end.
[
min(start, t)
for start, t in zip(
forward_batch.extend_logprob_start_lens_cpu,
tail.extend_seq_lens_cpu,
)
]
if forward_batch.return_logprob
else tail.extend_seq_lens_cpu
)
output = self.logits_processor(
input_ids,
@@ -0,0 +1,177 @@
"""Decoder SWA bounded replay: requests that read prompt rows (prompt logprobs,
every hidden state) run their whole extend through the late layers. CPU only."""
import unittest
from types import SimpleNamespace
from unittest.mock import patch
import torch
from sglang.kernels.ops.attention.dsv4_attn_metadata_kernels import (
late_layer_tail_layout,
)
from sglang.srt.layers.attention import deepseek_v4_backend
from sglang.srt.layers.attention.deepseek_v4_backend import (
SWA_WINDOW,
DeepseekV4AttnBackend,
late_layer_whole_extend,
)
from sglang.srt.model_executor.forward_batch_info import CaptureHiddenMode
from sglang.srt.models.deepseek_v4 import DeepseekV4Model
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
CPU = torch.device("cpu")
def _batch(extend_lens, starts=None, capture=CaptureHiddenMode.NULL):
return SimpleNamespace(
extend_seq_lens_cpu=extend_lens,
extend_logprob_start_lens_cpu=starts,
return_logprob=starts is not None,
capture_hidden_mode=capture,
)
class TestWholeExtendRequests(CustomTestCase):
def test_no_prompt_reader_keeps_the_window_tail(self):
# No logprobs and output-only logprobs (start == extend) read no
# prompt row.
for batch in (
_batch([300, 50]),
_batch([300, 50], starts=[300, 50]),
_batch([300], capture=CaptureHiddenMode.LAST),
):
self.assertIsNone(late_layer_whole_extend(batch, dspark=False))
def test_dspark_capture_reads_the_tail_only(self):
# DSpark's target prefill, and its graph capture batches, capture FULL
# hidden states, but its draft reads tail rows only.
batch = _batch([300], capture=CaptureHiddenMode.FULL)
self.assertIsNone(late_layer_whole_extend(batch, dspark=True))
def test_any_prompt_logprob_reads_the_whole_extend(self):
# Inside the last window too, and in a prompt shorter than a window:
# those rows would otherwise get the window floor.
batch = _batch([300, 300, 300, 50], starts=[0, 300, 300 - SWA_WINDOW, 0])
whole = late_layer_whole_extend(batch, dspark=False)
self.assertEqual(whole, [True, False, True, True])
def test_full_hidden_capture_reads_every_row(self):
batch = _batch([300, 50], capture=CaptureHiddenMode.FULL)
self.assertEqual(late_layer_whole_extend(batch, dspark=False), [True, True])
class TestWholeExtendLayout(CustomTestCase):
def test_other_requests_match_the_window_layout(self):
extend_lens, seq_lens = [200, 300, 50], [300, 600, 50]
token_indices, tail_lens, floor = late_layer_tail_layout(
extend_lens_cpu=extend_lens,
seq_lens_cpu=seq_lens,
tail_len=SWA_WINDOW,
device=CPU,
whole_extend=[False, True, False],
)
base_indices, base_lens, base_floor = late_layer_tail_layout(
extend_lens_cpu=extend_lens,
seq_lens_cpu=seq_lens,
tail_len=SWA_WINDOW,
device=CPU,
)
self.assertEqual(tail_lens, [128, 300, 50])
self.assertEqual(base_lens, [128, 128, 50])
# The first and last requests keep the window layout row for row.
torch.testing.assert_close(token_indices[:128], base_indices[:128])
torch.testing.assert_close(floor[:128], base_floor[:128])
torch.testing.assert_close(token_indices[428:], base_indices[256:])
torch.testing.assert_close(floor[428:], base_floor[256:])
# The whole request: every row of its extend, each with its plain window
# (a floor a window below the extend start).
torch.testing.assert_close(token_indices[128:428], torch.arange(200, 500))
expected = torch.full((300,), 600 - 300 - SWA_WINDOW, dtype=torch.int32)
torch.testing.assert_close(floor[128:428], expected)
def test_short_whole_extend_keeps_its_plain_window(self):
# A chunk shorter than a window is floored one window back otherwise.
_, tail_lens, floor = late_layer_tail_layout(
extend_lens_cpu=[40],
seq_lens_cpu=[300],
tail_len=SWA_WINDOW,
device=CPU,
whole_extend=[True],
)
self.assertEqual(tail_lens, [40])
expected = torch.full((40,), 300 - 40 - SWA_WINDOW, dtype=torch.int32)
torch.testing.assert_close(floor, expected)
class TestPrefillGraphTailCapacity(CustomTestCase):
"""The backend's graph check and the DP vote's view of it agree: no DP rank
replays the prefill graph while another runs the step eagerly."""
def _backend(self, capacity):
backend = object.__new__(DeepseekV4AttnBackend)
backend.enable_decoder_swa_bounded_replay = True
backend._prefill_graph_tail_capacity = lambda: capacity
backend.model_runner = SimpleNamespace(spec_algorithm=SpeculativeAlgorithm.NONE)
return backend
def _both(self, extend_lens, capacity, starts=None, full=False):
hidden = CaptureHiddenMode.FULL if full else CaptureHiddenMode.NULL
forward_batch = _batch(extend_lens, starts, capture=hidden)
forward_batch.seq_lens_cpu = torch.tensor([400] * len(extend_lens))
schedule_batch = SimpleNamespace(
extend_lens=extend_lens,
extend_logprob_start_lens=starts,
return_logprob=starts is not None,
reqs=[SimpleNamespace(return_hidden_states_mode=hidden)] * len(extend_lens),
)
backend = self._backend(capacity)
with (
patch.object(
deepseek_v4_backend, "_prefill_graph_max_seq_len", return_value=16384
),
patch.object(
deepseek_v4_backend,
"get_server_return_hidden_states_mode",
return_value=CaptureHiddenMode.NULL,
),
):
can_run = backend.can_run_prefill_cuda_graph(forward_batch)
vote = backend.vote_prefill_cuda_graph(schedule_batch)
self.assertEqual(can_run, vote)
return vote
def test_whole_extend_counts_against_the_tail_block(self):
self.assertTrue(self._both([300, 300], capacity=256))
# A prompt-logprob request's tail no longer fits: run the step eagerly.
self.assertFalse(self._both([300, 300], 256, starts=[0, 300]))
self.assertTrue(self._both([300, 300], 428, starts=[0, 300]))
self.assertFalse(self._both([300], 256, full=True))
class TestTailReadersCheck(CustomTestCase):
def _check(self, batch, tail_lens, dspark=False):
model = SimpleNamespace(dspark_layers_to_capture=[1] if dspark else None)
tail = SimpleNamespace(extend_seq_lens_cpu=tail_lens)
DeepseekV4Model._check_late_layer_tail_readers(model, batch, tail)
def test_tail_covering_the_readers_passes(self):
self._check(_batch([300, 50], starts=[300, 50]), [128, 50])
self._check(_batch([300, 50], starts=[200, 0]), [128, 50])
self._check(_batch([300, 50], starts=[0, 0]), [300, 50])
self._check(_batch([300], capture=CaptureHiddenMode.FULL), [300])
self._check(_batch([300], capture=CaptureHiddenMode.FULL), [128], dspark=True)
def test_tail_missing_rows_raises(self):
with self.assertRaises(ValueError):
self._check(_batch([300], starts=[0]), [128])
with self.assertRaises(ValueError):
self._check(_batch([300], capture=CaptureHiddenMode.FULL), [128])
if __name__ == "__main__":
unittest.main()
@@ -466,6 +466,7 @@ class TestPrefillCudaGraphVote(CustomTestCase):
runner.max_context_size = None
runner.max_seq_len = None
runner._qwen_bcg_hc_sidechannel = True
runner._backend_vote_prefill_cuda_graph = None
runner.can_replay_locally.side_effect = lambda **kwargs: (
not kwargs["contains_mm_inputs"]
or runner_cls.can_replay_locally(self=runner, **kwargs)
@@ -514,6 +515,7 @@ class TestPrefillCudaGraphVote(CustomTestCase):
runner.max_context_size = None
runner.max_seq_len = None
runner._qwen_bcg_hc_sidechannel = False
runner._backend_vote_prefill_cuda_graph = None
runner._is_full_backend = False
runner._captured_attn_metadata_max_bs = 2
runner.prefer_eager_mixed_prefill = False
@@ -25,8 +25,13 @@ from sglang.srt.model_executor.encoder_swa_replay import (
drop_folded_rows,
folded_replay_rows,
)
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
from sglang.srt.model_executor.forward_batch_info import (
CaptureHiddenMode,
ForwardBatch,
ForwardMode,
)
from sglang.srt.runtime_context import get_context, get_parallel
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
@@ -120,6 +125,7 @@ def _backend(fx):
backend.low_ratios = ()
backend.has_c4 = backend.has_c128 = False
backend.trtllm_attn = False
backend.model_runner = SimpleNamespace(spec_algorithm=SpeculativeAlgorithm.NONE)
backend.cuda_int32_kwargs = {"dtype": torch.int32, "device": "cpu"}
backend.token_to_kv_pool = SimpleNamespace(
request_window=SimpleNamespace(capacity=2 * SWA_WINDOW)
@@ -198,6 +204,8 @@ class TestFoldWithDecoderSwaTail(CustomTestCase):
req_pool_indices=fb.req_pool_indices,
out_cache_loc=fb.out_cache_loc,
positions=fx.folded_positions(),
capture_hidden_mode=CaptureHiddenMode.NULL,
return_logprob=False,
)
with (
patch.object(DSV4AttnMetadata, "init_compression_metadata"),