[Perf] prefill-cuda-graph: Make prefill shared-read completion explicit and default-on (#37958)

Co-authored-by: weireweire <20922698+weireweire@users.noreply.github.com>
Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
Co-authored-by: Po-Han Huang <pohanh@nvidia.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: hnyls2002 <lsyincs@gmail.com>
This commit is contained in:
weireweire
2026-10-20 16:12:55 -07:00
committed by GitHub
co-authored by weireweire Po-Han Huang Po-Han Huang Claude Opus 5.5 hnyls2002
parent d5e83ea4f6
commit bf145750a6
15 changed files with 216 additions and 262 deletions
+1 -1
View File
@@ -669,7 +669,7 @@ class Envs:
# read-done fastpath event.
SGLANG_FORCE_COARSE_WAR_BARRIER = EnvBool(False)
# Enable prefill read-done publication after compliant metadata initialization.
SGLANG_ENABLE_PREFILL_WAR_READ_DONE = EnvBool(False)
SGLANG_ENABLE_PREFILL_WAR_READ_DONE = EnvBool(True)
# PP: skip output send/recv when the entire batch consists of non-final chunked prefill requests,
# since process_batch_result_prefill discards next_token_ids for those anyway.
SGLANG_PP_SKIP_PURE_CHUNKED_OUTPUT_COMM = EnvBool(False)
@@ -7,6 +7,8 @@ from typing import TYPE_CHECKING, Iterable, Optional
import torch
from sglang.kernels.kernel_api_logging import debug_kernel_api
from sglang.srt.environ import envs
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.utils.common import is_npu
if TYPE_CHECKING:
@@ -15,7 +17,7 @@ if TYPE_CHECKING:
)
from sglang.srt.layers.attention.verify_mask import VerifyMask
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.speculative.spec_info import SpecInput
@@ -183,15 +185,38 @@ class AttentionBackend(ABC):
return SharedReadEnds.IN_REPLAY
return SharedReadEnds.UNKNOWN
def prepare_prefill_shared_read_snapshot(
self, forward_batch: ForwardBatch, *, num_qo_tokens: int
) -> None:
"""Snapshot late prefill reads before a PRE_REPLAY event is published.
def resolve_prefill_shared_read_ends(
self,
forward_batch: ForwardBatch,
*,
num_qo_tokens: int,
build_snapshot: bool = True,
) -> SharedReadEnds:
"""Return this batch's prefill read end, building a snapshot when asked.
Runners call this only after the actual eager/replay query geometry is
known. Backends that retain scheduler-shared reads into the model
forward keep the default no-op and must not declare PRE_REPLAY.
Spec algorithms with a later draft reader are gated at publish time.
"""
if (
envs.SGLANG_FORCE_COARSE_WAR_BARRIER.get()
or not envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.get()
):
return SharedReadEnds.UNKNOWN
if forward_batch.forward_mode != ForwardMode.EXTEND:
return SharedReadEnds.UNKNOWN
if build_snapshot:
return self._snapshot_prefill_shared_reads(
forward_batch, num_qo_tokens=num_qo_tokens
)
return self.shared_read_ends(forward_batch.forward_mode)
def _snapshot_prefill_shared_reads(
self, forward_batch: ForwardBatch, *, num_qo_tokens: int
) -> SharedReadEnds:
"""Snapshot late shared reads; return PRE_REPLAY only if none remain.
Hook-skipping paths read ``shared_read_ends(EXTEND)``; keep it conservative.
"""
return self.shared_read_ends(forward_batch.forward_mode)
# Chunked-prefix FullCG capture has a second model topology and stable
# prefix buffers. Backends must opt in explicitly so the runner does not
@@ -1056,7 +1056,6 @@ class DSV4Metadata:
# Built at the runner's prefill WAR boundary when the fast path is on,
# otherwise lazily by ``_forward_prefill_sparse``.
sparse_prefill_cache: Optional[SparsePrefillChunkCache] = None
prefill_shared_reads_snapshotted: bool = False
# Set only on the metadata built for the late layers under bounded SWA replay.
late_layer_tail: Optional[LateLayerTail] = None
@@ -1078,7 +1077,6 @@ class DSV4Metadata:
self.c128_compress_metadata, src=other.c128_compress_metadata
)
self.sparse_prefill_cache = None
self.prefill_shared_reads_snapshotted = False
def refresh_for_breakable_cuda_graph_replay_(
self,
@@ -1123,7 +1121,6 @@ class DSV4Metadata:
src=static_metadata.c128_compress_metadata,
)
self.sparse_prefill_cache = None
self.prefill_shared_reads_snapshotted = False
@dataclass
@@ -1203,13 +1200,6 @@ class DeepseekV4AttnBackend(
if self.model_runner.spec_algorithm.is_dspark():
return SharedReadEnds.IN_REPLAY
return SharedReadEnds.POST_REPLAY
metadata = self.forward_metadata
if (
fm == ForwardMode.EXTEND
and isinstance(metadata, DSV4Metadata)
and metadata.prefill_shared_reads_snapshotted
):
return SharedReadEnds.PRE_REPLAY
return super().shared_read_ends(fm)
def __init__(
@@ -2739,47 +2729,46 @@ class DeepseekV4AttnBackend(
def _use_sparse_prefill(
self, forward_batch: ForwardBatch, *, num_qo_tokens: int
) -> bool:
# Shared with the snapshot hook, so no forward builds the cache after read-done.
# The tail stays dense: its window floor lives in swa_page_indices, which
# the chunk cache ignores.
return (
forward_batch.forward_mode.is_extend_without_speculative()
not self.trtllm_attn
and forward_batch.forward_mode.is_extend_without_speculative()
and not is_cp_active(forward_batch)
and not get_platform().is_sm120
and self.forward_metadata.late_layer_tail is None
and self.token_to_kv_pool.request_window is None
and (
num_qo_tokens > _LARGE_INDEXER_QUERY_THRESHOLD
or envs.SGLANG_OPT_FLASHMLA_SPARSE_PREFILL.get()
)
)
def prepare_prefill_shared_read_snapshot(
def _snapshot_prefill_shared_reads(
self, forward_batch: ForwardBatch, *, num_qo_tokens: int
) -> None:
) -> SharedReadEnds:
# Sparse prefill otherwise reads req_to_token/full_to_swa lazily in its
# first layer. DFLASH/DSPARK have no later prefill draft-extend reader;
# CP shards the query layout that this global snapshot assumes.
metadata = self.forward_metadata
# first layer. CP shards the query layout that this global snapshot
# assumes; request-window layouts are not covered by the snapshot.
if self.token_to_kv_pool.request_window is not None:
return
if isinstance(metadata, DSV4Metadata):
metadata.prefill_shared_reads_snapshotted = False
snapshot_shared_prefill_reads = (
envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.get()
and forward_batch.forward_mode == ForwardMode.EXTEND
and self.model_runner.spec_algorithm.is_dflash_family()
and not is_cp_active(forward_batch)
)
if not snapshot_shared_prefill_reads:
return
return SharedReadEnds.UNKNOWN
if is_cp_active(forward_batch):
return SharedReadEnds.UNKNOWN
# The V4.1 low-ratio indexer reads req_to_token and HiSparse translates
# through full_to_hisparse in every layer, after any snapshot.
if self.is_dsv41 or self.hisparse_coordinator is not None:
return SharedReadEnds.UNKNOWN
metadata = self.forward_metadata
assert isinstance(metadata, DSV4Metadata)
# The tail never takes the sparse path, so it carries no chunk cache.
if metadata.late_layer_tail is None and self._use_sparse_prefill(
forward_batch, num_qo_tokens=num_qo_tokens
):
if self._use_sparse_prefill(forward_batch, num_qo_tokens=num_qo_tokens):
metadata.sparse_prefill_cache = self._build_sparse_prefill_chunk_cache(
forward_batch, metadata.core_attn_metadata, num_qo_tokens=num_qo_tokens
)
# Marked for dense prefill too: that path reads only core_attn_metadata,
# which init_forward_metadata already snapshotted.
metadata.prefill_shared_reads_snapshotted = True
# Dense prefill reads only core_attn_metadata, which metadata init has
# already snapshotted, so it reaches the same boundary without a cache.
return SharedReadEnds.PRE_REPLAY
def _build_sparse_prefill_chunk_cache(
self,
@@ -3821,11 +3810,8 @@ class DeepseekV4AttnBackend(
# raw_indices; num_qo_tokens=0 drops the query-count clause, so this errs
# to True. Paged attention under active CP needs the page indices.
return (
self.trtllm_attn
or self.mega_attn
self.mega_attn
or self.tail_forward_metadata is not None
or self.forward_metadata.late_layer_tail is not None
or self.token_to_kv_pool.request_window is not None
or not self._use_sparse_prefill(forward_batch, num_qo_tokens=0)
)
@@ -4065,13 +4051,9 @@ class DeepseekV4AttnBackend(
num_heads=q.shape[-2],
)
# The tail stays dense: its window floor lives in swa_page_indices,
# which the sparse chunk cache ignores.
if (
self._use_sparse_prefill(forward_batch, num_qo_tokens=q.shape[0])
and not use_mega_paged
and self.forward_metadata.late_layer_tail is None
and token_to_kv_pool.request_window is None
):
if use_dsv4_q8kv8_sparse_prefill(self.dsv4_prefill_backend):
return self._forward_prefill_sparse_q8kv8(
@@ -158,6 +158,10 @@ class DotsSWAMLAAttnBackend(AttentionBackend):
return self.backend.verify_mask
def shared_read_ends(self, fm: ForwardMode) -> SharedReadEnds:
# An eager prefill may rebuild metadata from the live tables in its first
# layer when DP padding changes the batch.
if fm.is_extend_without_speculative():
return SharedReadEnds.UNKNOWN
return self.backend.shared_read_ends(fm)
def draft_extend_metadata_captured_in_graph(self) -> bool:
@@ -90,6 +90,14 @@ class HybridAttnBackend(AttentionBackend):
def shared_read_ends(self, fm: ForwardMode) -> SharedReadEnds:
return self._select_backend(fm).shared_read_ends(fm)
def _snapshot_prefill_shared_reads(
self, forward_batch: ForwardBatch, *, num_qo_tokens: int
) -> SharedReadEnds:
backend = self._select_backend(forward_batch.forward_mode)
return backend._snapshot_prefill_shared_reads(
forward_batch, num_qo_tokens=num_qo_tokens
)
def get_fused_kv_write_buffers(self, layer, forward_batch):
return self._select_backend(
forward_batch.forward_mode
@@ -1311,10 +1311,6 @@ class HybridLinearAttnBackend(AttentionBackend):
)
elif not forward_batch.forward_mode.is_draft_extend_v2():
self.linear_attn_backend.init_forward_metadata(forward_batch)
if static_forward_batch is not None:
self.prepare_prefill_shared_read_snapshot(
forward_batch, num_qo_tokens=static_forward_batch.positions.shape[0]
)
def get_indexer_metadata(self, layer_id: int, forward_batch: ForwardBatch):
if layer_id in self.full_attn_layers:
@@ -20,7 +20,7 @@ import inspect
import logging
import time
from dataclasses import dataclass
from typing import Callable, List, Optional, Union
from typing import List, Optional, Union
import torch
import torch.distributed as dist
@@ -465,9 +465,6 @@ class ModelRunner:
# Read-done mailbox: the scheduler's WAR barrier reads it from the runner
# its worker names, and treats None as the coarse whole-forward fence.
self.shared_read_done_event: Optional[torch.cuda.Event] = None
# Scoped by a speculative worker to stage its shared reads before
# the target prefill graph publishes the read-done event.
self.prefill_shared_read_stager: Optional[Callable[[ForwardBatch], bool]] = None
# CPU offload
set_offloader(create_offloader())
@@ -321,13 +321,15 @@ class EagerRunner(BaseRunner):
# e.g. Moss-VL's prefill cross-attention custom mask.
model_runner.model.prepare_forward_batch(forward_batch)
model_runner.attn_backend.init_forward_metadata(forward_batch)
model_runner.attn_backend.prepare_prefill_shared_read_snapshot(
forward_batch,
num_qo_tokens=len(forward_batch.input_ids),
shared_read_ends = (
model_runner.attn_backend.resolve_prefill_shared_read_ends(
forward_batch,
num_qo_tokens=len(forward_batch.input_ids),
)
)
maybe_publish_prefill_shared_read_done(
model_runner,
forward_batch,
shared_read_ends,
torch.get_device_module(model_runner.device),
)
@@ -54,6 +54,7 @@ from sglang.kernels.ops.attention.kv_indices import (
)
from sglang.srt.configs.model_config import is_qwen4_exp
from sglang.srt.distributed.parallel_state import graph_capture
from sglang.srt.layers.attention.base_attn_backend import SharedReadEnds
from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp
from sglang.srt.layers.cp.bcg import (
PrefillCPBCGInput,
@@ -1250,7 +1251,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
forward_batch: ForwardBatch,
static_forward_batch: ForwardBatch,
shape_key: ShapeKey,
) -> None:
) -> SharedReadEnds:
"""Replay-time metadata refresh for the BCG-with-captured-metadata
contract. For opt-in backends, refresh the stashed per-bucket
metadata in place against the current batch; otherwise fall back
@@ -1259,6 +1260,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
real seq_lens / prefix_lens; the captured kernels read the
updated state at replay."""
attn_backend = self.model_runner.attn_backend
build_snapshot = False
if self._is_full_backend:
# Slot-padded shallow view: plan() must see exactly req_slots
# entries (real values in [:bs], sentinels in [bs:req_slots]
@@ -1282,19 +1284,22 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
padded_view.max_seq_len_override = static_forward_batch.max_seq_len_override
padded_view.input_ids = static_forward_batch.input_ids
attn_backend.init_forward_metadata_out_graph(padded_view)
return
if not self.use_captured_attn_metadata:
elif not self.use_captured_attn_metadata:
attn_backend.init_forward_metadata(forward_batch)
attn_backend.prepare_prefill_shared_read_snapshot(
forward_batch, num_qo_tokens=shape_key.size
build_snapshot = True
else:
assert self.attn_metadata_buffers is not None
metadata = self.attn_metadata_buffers[shape_key]
attn_backend.prepare_forward_metadata_for_breakable_cuda_graph_replay(
metadata,
forward_batch,
static_forward_batch=static_forward_batch,
)
return
assert self.attn_metadata_buffers is not None
metadata = self.attn_metadata_buffers[shape_key]
attn_backend.prepare_forward_metadata_for_breakable_cuda_graph_replay(
metadata,
return attn_backend.resolve_prefill_shared_read_ends(
forward_batch,
static_forward_batch=static_forward_batch,
num_qo_tokens=shape_key.size,
build_snapshot=build_snapshot,
)
@staticmethod
@@ -1816,10 +1821,10 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
post_warmup_hook=post_warmup_hook,
)
def load_batch(self, forward_batch: ForwardBatch, **kwargs) -> ForwardBatch:
"""Pad, populate static buffers, and build the static_forward_batch
the model code reads during replay.
"""
def load_batch(
self, forward_batch: ForwardBatch, **kwargs
) -> tuple[ForwardBatch, SharedReadEnds]:
"""Pad, fill static buffers, and prepare replay metadata and its read end."""
num_tokens = len(forward_batch.input_ids)
static_num_tokens = self._pad_to_bucket(num_tokens, self.capture_num_tokens)
if getattr(self, "enable_cp_bcg_capture", False) and is_cp_active(
@@ -2100,11 +2105,11 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
metadata_forward_batch = static_forward_batch
shape_key = self._shape_key(static_num_tokens, forward_batch)
self._prepare_forward_metadata_for_replay(
shared_read_ends = self._prepare_forward_metadata_for_replay(
metadata_forward_batch, static_forward_batch, shape_key
)
return static_forward_batch
return static_forward_batch, shared_read_ends
def _fill_input_embeds_slot(self, args, layer_kwargs, static_num_tokens: int):
"""A text-only batch would otherwise replay the captured input_embeds."""
@@ -2312,17 +2317,19 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
) -> Optional[Union[LogitsProcessorOutput, PPProxyTensors, EmbeddingPoolerOutput]]:
self._validate_capture_hidden_mode(forward_batch)
with self.model_context(), self.backend.replay_session():
static_forward_batch = self.load_batch(forward_batch, **kwargs)
static_forward_batch, shared_read_ends = self.load_batch(
forward_batch, **kwargs
)
static_num_tokens = len(static_forward_batch.input_ids)
raw_num_tokens = self.raw_num_tokens
shape_key = self._shape_key(static_num_tokens, forward_batch)
# The only variants this runner records are chunked-prefix ones.
if shape_key.variant_label is not None:
self._prepare_chunked_prefix_replay(shape_key, forward_batch)
# Replay prep, including the optional chunked-prefix gather above,
# has finished every scheduler-shared read.
# Publish after replay prep, including the chunked-prefix gather
# above, which also reads the shared tables.
maybe_publish_prefill_shared_read_done(
self.model_runner, forward_batch, self.device_module
self.model_runner, shared_read_ends, self.device_module
)
self._wait_for_layer_loads()
@@ -5,9 +5,8 @@ from typing import Optional
import torch
from sglang.srt.environ import envs
from sglang.srt.layers.attention.base_attn_backend import SharedReadEnds
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.runtime_context import get_memory
from sglang.srt.utils import is_cuda
logger = logging.getLogger(__name__)
@@ -24,22 +23,22 @@ def make_external_event(device_module) -> Optional[torch.cuda.Event]:
def maybe_publish_prefill_shared_read_done(
model_runner, forward_batch, device_module
model_runner, shared_read_ends: SharedReadEnds, device_module
) -> None:
"""Publish prefill read-done after compliant metadata initialization."""
if not envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.get():
return
if forward_batch.forward_mode != ForwardMode.EXTEND:
return
# The record lands right after replay prep, so PRE_REPLAY only.
declared = model_runner.attn_backend.shared_read_ends(forward_batch.forward_mode)
if declared is not SharedReadEnds.PRE_REPLAY:
"""Publish prefill read-done at the resolved pre-replay boundary."""
if shared_read_ends is not SharedReadEnds.PRE_REPLAY:
return
if not model_runner.spec_algorithm.supports_prefill_shared_read_done():
# Stage the draft's shared reads before publishing the read-done event.
stage = getattr(model_runner, "prefill_shared_read_stager", None)
if stage is None or not stage(forward_batch):
return
return
# Host and storage offload copy a stashed chunk's KV ordered only behind the
# schedule stream, so they need the whole-forward barrier.
memory = get_memory()
if (
memory.enable_hierarchical_cache
or memory.hicache_storage_backend is not None
or memory.enable_lmcache
):
return
logger.info_once(
"Prefill shared-read-done fastpath active (%s)",
type(model_runner.attn_backend).__name__,
+3 -3
View File
@@ -138,10 +138,10 @@ class SpeculativeAlgorithm(Enum):
def supports_prefill_shared_read_done(self) -> bool:
"""Whether target EXTEND has no later speculative shared-buffer reader.
The backend must still declare a pre-replay read end. Other algorithms
must stage their draft's shared reads before publishing the target event.
The backend must still declare a pre-replay read end.
"""
return self.is_none() or self.is_dflash_family()
# DSPARK's draft KV inject reads the SWA mapping after the target prefill.
return self.is_none() or self.is_dflash()
def supports_mixed_chunk(self) -> bool:
"""Whether mixed chunk prefill may stay enabled with this algorithm.
@@ -511,115 +511,90 @@ class TestDSV4BreakableCudaGraphMetadataContract(CustomTestCase):
DeepseekV4AttnBackend.use_captured_forward_metadata_for_breakable_cuda_graph
)
def test_prefill_snapshot_declares_pre_replay_boundary(self):
from sglang.srt.layers.attention.base_attn_backend import SharedReadEnds
def _make_prefill_backend(self, *, is_dsv41=False, hisparse_coordinator=None):
from sglang.srt.layers.attention.deepseek_v4_backend import (
DeepseekV4AttnBackend,
DSV4Metadata,
)
backend = object.__new__(DeepseekV4AttnBackend)
backend.token_to_kv_pool = SimpleNamespace(request_window=None)
backend.is_dsv41 = is_dsv41
backend.hisparse_coordinator = hisparse_coordinator
backend.forward_metadata = DSV4Metadata(
self._make_core_metadata(0), indexer_metadata=None
)
self.assertIs(
backend.shared_read_ends(ForwardMode.EXTEND),
SharedReadEnds.UNKNOWN,
)
backend._build_sparse_prefill_chunk_cache = mock.Mock(return_value=object())
return backend
backend.forward_metadata.prefill_shared_reads_snapshotted = True
def test_static_extend_read_end_stays_unknown(self):
from sglang.srt.layers.attention.base_attn_backend import SharedReadEnds
# Paths that skip the snapshot hook publish from this declaration.
backend = self._make_prefill_backend()
self.assertIs(
backend.shared_read_ends(ForwardMode.EXTEND),
SharedReadEnds.PRE_REPLAY,
backend.shared_read_ends(ForwardMode.EXTEND), SharedReadEnds.UNKNOWN
)
def test_snapshot_builds_cache_only_for_sparse_prefill(self):
from sglang.srt.environ import envs
from sglang.srt.layers.attention.base_attn_backend import SharedReadEnds
from sglang.srt.layers.attention.deepseek_v4_backend import (
_LARGE_INDEXER_QUERY_THRESHOLD,
DeepseekV4AttnBackend,
DSV4Metadata,
)
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
batch = SimpleNamespace(forward_mode=ForwardMode.EXTEND)
cache = object()
for num_qo_tokens, builds in (
(_LARGE_INDEXER_QUERY_THRESHOLD, False),
(_LARGE_INDEXER_QUERY_THRESHOLD + 1, True),
):
with self.subTest(num_qo_tokens=num_qo_tokens):
backend = object.__new__(DeepseekV4AttnBackend)
backend.model_runner = SimpleNamespace(
spec_algorithm=SpeculativeAlgorithm.DFLASH
)
backend.token_to_kv_pool = SimpleNamespace(request_window=None)
backend.forward_metadata = DSV4Metadata(
self._make_core_metadata(0), indexer_metadata=None
)
backend._build_sparse_prefill_chunk_cache = mock.Mock(
return_value=cache
)
backend = self._make_prefill_backend()
build = backend._build_sparse_prefill_chunk_cache
with (
envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.override(True),
envs.SGLANG_OPT_FLASHMLA_SPARSE_PREFILL.override(False),
mock.patch(
"sglang.srt.layers.attention.deepseek_v4_backend.get_platform",
return_value=SimpleNamespace(is_sm120=False),
),
):
backend.prepare_prefill_shared_read_snapshot(
batch, num_qo_tokens=num_qo_tokens
shared_read_ends = backend.resolve_prefill_shared_read_ends(
batch,
num_qo_tokens=num_qo_tokens,
)
metadata = backend.forward_metadata
if builds:
backend._build_sparse_prefill_chunk_cache.assert_called_once_with(
build.assert_called_once_with(
batch, metadata.core_attn_metadata, num_qo_tokens=num_qo_tokens
)
self.assertIs(metadata.sparse_prefill_cache, cache)
self.assertIs(metadata.sparse_prefill_cache, build.return_value)
else:
backend._build_sparse_prefill_chunk_cache.assert_not_called()
build.assert_not_called()
self.assertIsNone(metadata.sparse_prefill_cache)
# Dense declares the boundary too; it reads only the metadata
# that init_forward_metadata already snapshotted.
self.assertTrue(metadata.prefill_shared_reads_snapshotted)
# Dense reaches the same boundary using metadata init's snapshot.
self.assertIs(shared_read_ends, SharedReadEnds.PRE_REPLAY)
def test_sparse_prefill_snapshot_marks_success_only_after_build(self):
from sglang.srt.environ import envs
def test_prefill_snapshot_skips_layers_reading_live_tables(self):
from sglang.srt.layers.attention.base_attn_backend import SharedReadEnds
from sglang.srt.layers.attention.deepseek_v4_backend import (
DeepseekV4AttnBackend,
DSV4Metadata,
_LARGE_INDEXER_QUERY_THRESHOLD,
)
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
backend = object.__new__(DeepseekV4AttnBackend)
backend.model_runner = SimpleNamespace(
spec_algorithm=SpeculativeAlgorithm.DFLASH
)
backend.token_to_kv_pool = SimpleNamespace(request_window=None)
backend.forward_metadata = DSV4Metadata(
self._make_core_metadata(0), indexer_metadata=None
)
backend.forward_metadata.prefill_shared_reads_snapshotted = True
backend._build_sparse_prefill_chunk_cache = mock.Mock(
side_effect=RuntimeError("snapshot failed")
)
batch = SimpleNamespace(forward_mode=ForwardMode.EXTEND)
for is_dsv41, hisparse_coordinator in ((True, None), (False, object())):
with self.subTest(is_dsv41=is_dsv41, hisparse=hisparse_coordinator):
backend = self._make_prefill_backend(
is_dsv41=is_dsv41, hisparse_coordinator=hisparse_coordinator
)
with (
envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.override(True),
envs.SGLANG_OPT_FLASHMLA_SPARSE_PREFILL.override(True),
mock.patch(
"sglang.srt.layers.attention.deepseek_v4_backend.get_platform",
return_value=SimpleNamespace(is_sm120=False),
),
self.assertRaisesRegex(RuntimeError, "snapshot failed"),
):
backend.prepare_prefill_shared_read_snapshot(batch, num_qo_tokens=12288)
shared_read_ends = backend.resolve_prefill_shared_read_ends(
batch, num_qo_tokens=_LARGE_INDEXER_QUERY_THRESHOLD + 1
)
self.assertFalse(backend.forward_metadata.prefill_shared_reads_snapshotted)
self.assertIs(shared_read_ends, SharedReadEnds.UNKNOWN)
backend._build_sparse_prefill_chunk_cache.assert_not_called()
def test_refresh_replay_metadata_preserves_captured_tensor_storage(self):
capture_metadata = self._make_core_metadata(0)
@@ -692,7 +667,6 @@ class TestDSV4BreakableCudaGraphMetadataContract(CustomTestCase):
self._make_core_metadata(0), indexer_metadata=None
)
capture_metadata.sparse_prefill_cache = object()
capture_metadata.prefill_shared_reads_snapshotted = True
replay_metadata = DSV4Metadata(
self._make_core_metadata(1000), indexer_metadata=None
)
@@ -724,7 +698,6 @@ class TestDSV4BreakableCudaGraphMetadataContract(CustomTestCase):
self.assertTrue(calls[0][2])
self.assertIs(backend.forward_metadata, capture_metadata)
self.assertIsNone(capture_metadata.sparse_prefill_cache)
self.assertFalse(capture_metadata.prefill_shared_reads_snapshotted)
self.assertTrue(
torch.equal(
capture_metadata.core_attn_metadata.seq_lens_casual,
@@ -84,19 +84,23 @@ class TestPrefillCudaGraphPadding(CustomTestCase):
runner = self._make_runner()
runner.use_captured_attn_metadata = False
attn_backend = mock.Mock()
attn_backend.resolve_prefill_shared_read_ends.return_value = mock.sentinel.end
runner.model_runner = SimpleNamespace(attn_backend=attn_backend)
forward_batch = self._make_forward_batch(8)
static_forward_batch = self._make_forward_batch(16)
runner._prepare_forward_metadata_for_replay(
shared_read_ends = runner._prepare_forward_metadata_for_replay(
forward_batch,
static_forward_batch,
shape_key=ShapeKey(size=16),
)
self.assertIs(shared_read_ends, mock.sentinel.end)
attn_backend.init_forward_metadata.assert_called_once_with(forward_batch)
attn_backend.prepare_prefill_shared_read_snapshot.assert_called_once_with(
forward_batch, num_qo_tokens=16
attn_backend.resolve_prefill_shared_read_ends.assert_called_once_with(
forward_batch,
num_qo_tokens=16,
build_snapshot=True,
)
def test_full_replay_pads_request_slot_cpu_mirror(self):
@@ -1,5 +1,4 @@
from types import SimpleNamespace
from unittest.mock import Mock, call, create_autospec
import pytest
@@ -12,6 +11,7 @@ from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.model_executor.runner_utils.shared_read_event import (
maybe_publish_prefill_shared_read_done,
)
from sglang.srt.runtime_context import get_context, get_memory
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
from sglang.srt.speculative.spec_registry import CustomSpecAlgo
from sglang.test.ci.ci_register import register_cpu_ci
@@ -20,6 +20,12 @@ from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=11, suite="base-a-test-cpu")
@pytest.fixture(autouse=True)
def _published_server_args():
with get_context().override_server_args():
yield
class _Event:
def __init__(self):
self.recorded = False
@@ -28,17 +34,19 @@ class _Event:
self.recorded = True
def _model_runner(*, spec_algorithm=SpeculativeAlgorithm.NONE, compliant=True):
declared = SharedReadEnds.PRE_REPLAY if compliant else SharedReadEnds.UNKNOWN
# Spec'd against the real ABC so a rename fails here, not at runtime.
attn_backend = create_autospec(AttentionBackend, instance=True)
attn_backend.shared_read_ends.return_value = declared
return SimpleNamespace(
def _model_runner(
*, spec_algorithm=SpeculativeAlgorithm.NONE, compliant=True, declared=None
):
if declared is None:
declared = SharedReadEnds.PRE_REPLAY if compliant else SharedReadEnds.UNKNOWN
attn_backend = AttentionBackend()
attn_backend.shared_read_ends = lambda _forward_mode: declared
runner = SimpleNamespace(
spec_algorithm=spec_algorithm,
attn_backend=attn_backend,
shared_read_done_event=None,
prefill_shared_read_stager=None,
)
return runner
_DEVICE_MODULE = SimpleNamespace(Event=_Event)
@@ -48,10 +56,16 @@ def _batch(mode=ForwardMode.EXTEND):
return SimpleNamespace(forward_mode=mode)
def test_publishes_recorded_event_when_enabled():
def _prepare_and_publish(runner, batch):
shared_read_ends = runner.attn_backend.resolve_prefill_shared_read_ends(
batch, num_qo_tokens=8
)
maybe_publish_prefill_shared_read_done(runner, shared_read_ends, _DEVICE_MODULE)
def test_publishes_recorded_event_by_default():
runner = _model_runner()
with envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.override(True):
maybe_publish_prefill_shared_read_done(runner, _batch(), _DEVICE_MODULE)
_prepare_and_publish(runner, _batch())
published = runner.shared_read_done_event
assert isinstance(published, _Event) and published.recorded
@@ -59,73 +73,31 @@ def test_publishes_recorded_event_when_enabled():
def test_disabled_when_flag_is_false():
runner = _model_runner()
with envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.override(False):
maybe_publish_prefill_shared_read_done(runner, _batch(), _DEVICE_MODULE)
_prepare_and_publish(runner, _batch())
assert runner.shared_read_done_event is None
@pytest.mark.parametrize(
"algorithm", (SpeculativeAlgorithm.DFLASH, SpeculativeAlgorithm.DSPARK)
"memory",
[
{"enable_hierarchical_cache": True},
{"hicache_storage_backend": "mooncake"},
{"enable_lmcache": True},
],
)
def test_dflash_family_target_prefill_publishes(algorithm):
runner = _model_runner(spec_algorithm=algorithm)
runner.prefill_shared_read_stager = Mock(return_value=False)
def test_kv_offload_keeps_the_coarse_barrier(memory):
runner = _model_runner()
with get_memory().override(**memory):
_prepare_and_publish(runner, _batch())
assert runner.shared_read_done_event is None
def test_dflash_target_prefill_publishes():
runner = _model_runner(spec_algorithm=SpeculativeAlgorithm.DFLASH)
with envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.override(True):
maybe_publish_prefill_shared_read_done(runner, _batch(), _DEVICE_MODULE)
_prepare_and_publish(runner, _batch())
published = runner.shared_read_done_event
assert isinstance(published, _Event) and published.recorded
runner.prefill_shared_read_stager.assert_not_called()
@pytest.mark.parametrize("staged", [False, True])
@pytest.mark.parametrize(
"algorithm",
[
SpeculativeAlgorithm.EAGLE,
SpeculativeAlgorithm.EAGLE3,
SpeculativeAlgorithm.FROZEN_KV_MTP,
SpeculativeAlgorithm.STANDALONE,
SpeculativeAlgorithm.NGRAM,
SpeculativeAlgorithm.UNO,
CustomSpecAlgo("TEST_PREFILL", lambda _: object),
],
)
def test_speculative_prefill_publishes_only_after_staging(algorithm, staged):
runner, batch = _model_runner(spec_algorithm=algorithm), _batch()
calls = Mock()
runner.prefill_shared_read_stager = calls.stage
calls.stage.return_value = staged
calls.Event.side_effect = _Event
with envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.override(True):
maybe_publish_prefill_shared_read_done(
runner, batch, SimpleNamespace(Event=calls.Event)
)
assert calls.mock_calls == [call.stage(batch)] + ([call.Event()] if staged else [])
published = runner.shared_read_done_event
if staged:
assert isinstance(published, _Event) and published.recorded
else:
assert published is None
@pytest.mark.parametrize(
"enabled,mode,compliant",
[
(False, ForwardMode.EXTEND, True),
(True, ForwardMode.TARGET_VERIFY, True),
(True, ForwardMode.MIXED, True),
(True, ForwardMode.DECODE, True),
(True, ForwardMode.EXTEND, False),
],
)
def test_prefill_gates_skip_staging(enabled, mode, compliant):
runner = _model_runner(
spec_algorithm=SpeculativeAlgorithm.EAGLE, compliant=compliant
)
runner.prefill_shared_read_stager = Mock(return_value=True)
with envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.override(enabled):
maybe_publish_prefill_shared_read_done(runner, _batch(mode), _DEVICE_MODULE)
runner.prefill_shared_read_stager.assert_not_called()
assert runner.shared_read_done_event is None
def test_gates_exclude_non_prefill_unsupported_algorithm_and_noncompliant_backend():
@@ -135,7 +107,7 @@ def test_gates_exclude_non_prefill_unsupported_algorithm_and_noncompliant_backen
(_model_runner(), _batch(ForwardMode.TARGET_VERIFY)),
(_model_runner(), _batch(ForwardMode.MIXED)),
(_model_runner(), _batch(ForwardMode.DECODE)),
# The algorithm has a later prefill reader or unverified ownership.
# The algorithm has a later prefill reader.
(_model_runner(spec_algorithm=SpeculativeAlgorithm.EAGLE), _batch()),
(
_model_runner(
@@ -146,7 +118,7 @@ def test_gates_exclude_non_prefill_unsupported_algorithm_and_noncompliant_backen
# Backend has not declared a pre-replay prefill read end.
(_model_runner(compliant=False), _batch()),
):
maybe_publish_prefill_shared_read_done(runner, batch, _DEVICE_MODULE)
_prepare_and_publish(runner, batch)
assert runner.shared_read_done_event is None
@@ -157,18 +129,12 @@ class _TargetOnlyPrefillAlgo(CustomSpecAlgo):
class TestPrefillReadDoneCapability(CustomTestCase):
def test_builtin_allowlist_is_unchanged(self):
allowed = {
SpeculativeAlgorithm.NONE,
SpeculativeAlgorithm.DFLASH,
SpeculativeAlgorithm.DSPARK,
}
allowed = {SpeculativeAlgorithm.NONE, SpeculativeAlgorithm.DFLASH}
for algorithm in SpeculativeAlgorithm:
with self.subTest(algorithm=algorithm):
runner = _model_runner(spec_algorithm=algorithm)
with envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.override(True):
maybe_publish_prefill_shared_read_done(
runner, _batch(), _DEVICE_MODULE
)
_prepare_and_publish(runner, _batch())
self.assertEqual(
runner.shared_read_done_event is not None, algorithm in allowed
)
@@ -182,19 +148,13 @@ class TestPrefillReadDoneCapability(CustomTestCase):
algorithm = algorithm_type(
"TEST_PREFILL", lambda _: object, supports_overlap=True
)
runner, batch = _model_runner(spec_algorithm=algorithm), _batch()
runner.prefill_shared_read_stager = Mock(return_value=False)
runner = _model_runner(spec_algorithm=algorithm)
with envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.override(True):
maybe_publish_prefill_shared_read_done(
runner, batch, _DEVICE_MODULE
)
_prepare_and_publish(runner, _batch())
event = runner.shared_read_done_event
self.assertEqual(event is not None, expected)
if expected:
self.assertTrue(event.recorded)
runner.prefill_shared_read_stager.assert_not_called()
else:
runner.prefill_shared_read_stager.assert_called_once_with(batch)
def test_opt_in_does_not_bypass_mode_backend_or_flag(self):
algorithm = _TargetOnlyPrefillAlgo("TEST_PREFILL", lambda _: object)
@@ -211,12 +171,9 @@ class TestPrefillReadDoneCapability(CustomTestCase):
)
for enabled, mode, declared in cases:
with self.subTest(enabled=enabled, mode=mode, declared=declared):
runner = _model_runner(spec_algorithm=algorithm)
runner.attn_backend.shared_read_ends.return_value = declared
runner = _model_runner(spec_algorithm=algorithm, declared=declared)
with envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.override(enabled):
maybe_publish_prefill_shared_read_done(
runner, _batch(mode), _DEVICE_MODULE
)
_prepare_and_publish(runner, _batch(mode))
self.assertIsNone(runner.shared_read_done_event)
@@ -432,7 +432,7 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase):
)
translator.bind_own_plan(forward_batch)
static_batch = runner.load_batch(forward_batch)
static_batch, _ = runner.load_batch(forward_batch)
self.assertIs(static_batch.mm_input_embeds, mm_input_embeds)