diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index eb2568bb03..4184054347 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -669,7 +669,7 @@ class Envs: # read-done fastpath event. SGLANG_FORCE_COARSE_WAR_BARRIER = EnvBool(False) # Enable prefill read-done publication after compliant metadata initialization. - SGLANG_ENABLE_PREFILL_WAR_READ_DONE = EnvBool(False) + SGLANG_ENABLE_PREFILL_WAR_READ_DONE = EnvBool(True) # PP: skip output send/recv when the entire batch consists of non-final chunked prefill requests, # since process_batch_result_prefill discards next_token_ids for those anyway. SGLANG_PP_SKIP_PURE_CHUNKED_OUTPUT_COMM = EnvBool(False) diff --git a/python/sglang/srt/layers/attention/base_attn_backend.py b/python/sglang/srt/layers/attention/base_attn_backend.py index af3c9bea07..c317997479 100644 --- a/python/sglang/srt/layers/attention/base_attn_backend.py +++ b/python/sglang/srt/layers/attention/base_attn_backend.py @@ -7,6 +7,8 @@ from typing import TYPE_CHECKING, Iterable, Optional import torch from sglang.kernels.kernel_api_logging import debug_kernel_api +from sglang.srt.environ import envs +from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.utils.common import is_npu if TYPE_CHECKING: @@ -15,7 +17,7 @@ if TYPE_CHECKING: ) from sglang.srt.layers.attention.verify_mask import VerifyMask from sglang.srt.layers.radix_attention import RadixAttention - from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode + from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.speculative.spec_info import SpecInput @@ -183,15 +185,38 @@ class AttentionBackend(ABC): return SharedReadEnds.IN_REPLAY return SharedReadEnds.UNKNOWN - def prepare_prefill_shared_read_snapshot( - self, forward_batch: ForwardBatch, *, num_qo_tokens: int - ) -> None: - """Snapshot late prefill reads before a PRE_REPLAY event is published. + def resolve_prefill_shared_read_ends( + self, + forward_batch: ForwardBatch, + *, + num_qo_tokens: int, + build_snapshot: bool = True, + ) -> SharedReadEnds: + """Return this batch's prefill read end, building a snapshot when asked. - Runners call this only after the actual eager/replay query geometry is - known. Backends that retain scheduler-shared reads into the model - forward keep the default no-op and must not declare PRE_REPLAY. + Spec algorithms with a later draft reader are gated at publish time. """ + if ( + envs.SGLANG_FORCE_COARSE_WAR_BARRIER.get() + or not envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.get() + ): + return SharedReadEnds.UNKNOWN + if forward_batch.forward_mode != ForwardMode.EXTEND: + return SharedReadEnds.UNKNOWN + if build_snapshot: + return self._snapshot_prefill_shared_reads( + forward_batch, num_qo_tokens=num_qo_tokens + ) + return self.shared_read_ends(forward_batch.forward_mode) + + def _snapshot_prefill_shared_reads( + self, forward_batch: ForwardBatch, *, num_qo_tokens: int + ) -> SharedReadEnds: + """Snapshot late shared reads; return PRE_REPLAY only if none remain. + + Hook-skipping paths read ``shared_read_ends(EXTEND)``; keep it conservative. + """ + return self.shared_read_ends(forward_batch.forward_mode) # Chunked-prefix FullCG capture has a second model topology and stable # prefix buffers. Backends must opt in explicitly so the runner does not diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index da5596206c..aa2ae0e598 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -1056,7 +1056,6 @@ class DSV4Metadata: # Built at the runner's prefill WAR boundary when the fast path is on, # otherwise lazily by ``_forward_prefill_sparse``. sparse_prefill_cache: Optional[SparsePrefillChunkCache] = None - prefill_shared_reads_snapshotted: bool = False # Set only on the metadata built for the late layers under bounded SWA replay. late_layer_tail: Optional[LateLayerTail] = None @@ -1078,7 +1077,6 @@ class DSV4Metadata: self.c128_compress_metadata, src=other.c128_compress_metadata ) self.sparse_prefill_cache = None - self.prefill_shared_reads_snapshotted = False def refresh_for_breakable_cuda_graph_replay_( self, @@ -1123,7 +1121,6 @@ class DSV4Metadata: src=static_metadata.c128_compress_metadata, ) self.sparse_prefill_cache = None - self.prefill_shared_reads_snapshotted = False @dataclass @@ -1203,13 +1200,6 @@ class DeepseekV4AttnBackend( if self.model_runner.spec_algorithm.is_dspark(): return SharedReadEnds.IN_REPLAY return SharedReadEnds.POST_REPLAY - metadata = self.forward_metadata - if ( - fm == ForwardMode.EXTEND - and isinstance(metadata, DSV4Metadata) - and metadata.prefill_shared_reads_snapshotted - ): - return SharedReadEnds.PRE_REPLAY return super().shared_read_ends(fm) def __init__( @@ -2739,47 +2729,46 @@ class DeepseekV4AttnBackend( def _use_sparse_prefill( self, forward_batch: ForwardBatch, *, num_qo_tokens: int ) -> bool: + # Shared with the snapshot hook, so no forward builds the cache after read-done. + # The tail stays dense: its window floor lives in swa_page_indices, which + # the chunk cache ignores. return ( - forward_batch.forward_mode.is_extend_without_speculative() + not self.trtllm_attn + and forward_batch.forward_mode.is_extend_without_speculative() and not is_cp_active(forward_batch) and not get_platform().is_sm120 + and self.forward_metadata.late_layer_tail is None + and self.token_to_kv_pool.request_window is None and ( num_qo_tokens > _LARGE_INDEXER_QUERY_THRESHOLD or envs.SGLANG_OPT_FLASHMLA_SPARSE_PREFILL.get() ) ) - def prepare_prefill_shared_read_snapshot( + def _snapshot_prefill_shared_reads( self, forward_batch: ForwardBatch, *, num_qo_tokens: int - ) -> None: + ) -> SharedReadEnds: # Sparse prefill otherwise reads req_to_token/full_to_swa lazily in its - # first layer. DFLASH/DSPARK have no later prefill draft-extend reader; - # CP shards the query layout that this global snapshot assumes. - metadata = self.forward_metadata + # first layer. CP shards the query layout that this global snapshot + # assumes; request-window layouts are not covered by the snapshot. if self.token_to_kv_pool.request_window is not None: - return - if isinstance(metadata, DSV4Metadata): - metadata.prefill_shared_reads_snapshotted = False - snapshot_shared_prefill_reads = ( - envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.get() - and forward_batch.forward_mode == ForwardMode.EXTEND - and self.model_runner.spec_algorithm.is_dflash_family() - and not is_cp_active(forward_batch) - ) - if not snapshot_shared_prefill_reads: - return + return SharedReadEnds.UNKNOWN + if is_cp_active(forward_batch): + return SharedReadEnds.UNKNOWN + # The V4.1 low-ratio indexer reads req_to_token and HiSparse translates + # through full_to_hisparse in every layer, after any snapshot. + if self.is_dsv41 or self.hisparse_coordinator is not None: + return SharedReadEnds.UNKNOWN + metadata = self.forward_metadata assert isinstance(metadata, DSV4Metadata) - # The tail never takes the sparse path, so it carries no chunk cache. - if metadata.late_layer_tail is None and self._use_sparse_prefill( - forward_batch, num_qo_tokens=num_qo_tokens - ): + if self._use_sparse_prefill(forward_batch, num_qo_tokens=num_qo_tokens): metadata.sparse_prefill_cache = self._build_sparse_prefill_chunk_cache( forward_batch, metadata.core_attn_metadata, num_qo_tokens=num_qo_tokens ) - # Marked for dense prefill too: that path reads only core_attn_metadata, - # which init_forward_metadata already snapshotted. - metadata.prefill_shared_reads_snapshotted = True + # Dense prefill reads only core_attn_metadata, which metadata init has + # already snapshotted, so it reaches the same boundary without a cache. + return SharedReadEnds.PRE_REPLAY def _build_sparse_prefill_chunk_cache( self, @@ -3821,11 +3810,8 @@ class DeepseekV4AttnBackend( # raw_indices; num_qo_tokens=0 drops the query-count clause, so this errs # to True. Paged attention under active CP needs the page indices. return ( - self.trtllm_attn - or self.mega_attn + self.mega_attn or self.tail_forward_metadata is not None - or self.forward_metadata.late_layer_tail is not None - or self.token_to_kv_pool.request_window is not None or not self._use_sparse_prefill(forward_batch, num_qo_tokens=0) ) @@ -4065,13 +4051,9 @@ class DeepseekV4AttnBackend( num_heads=q.shape[-2], ) - # The tail stays dense: its window floor lives in swa_page_indices, - # which the sparse chunk cache ignores. if ( self._use_sparse_prefill(forward_batch, num_qo_tokens=q.shape[0]) and not use_mega_paged - and self.forward_metadata.late_layer_tail is None - and token_to_kv_pool.request_window is None ): if use_dsv4_q8kv8_sparse_prefill(self.dsv4_prefill_backend): return self._forward_prefill_sparse_q8kv8( diff --git a/python/sglang/srt/layers/attention/dots_hybrid_backend.py b/python/sglang/srt/layers/attention/dots_hybrid_backend.py index cfcff5b4fd..d4995a423f 100644 --- a/python/sglang/srt/layers/attention/dots_hybrid_backend.py +++ b/python/sglang/srt/layers/attention/dots_hybrid_backend.py @@ -158,6 +158,10 @@ class DotsSWAMLAAttnBackend(AttentionBackend): return self.backend.verify_mask def shared_read_ends(self, fm: ForwardMode) -> SharedReadEnds: + # An eager prefill may rebuild metadata from the live tables in its first + # layer when DP padding changes the batch. + if fm.is_extend_without_speculative(): + return SharedReadEnds.UNKNOWN return self.backend.shared_read_ends(fm) def draft_extend_metadata_captured_in_graph(self) -> bool: diff --git a/python/sglang/srt/layers/attention/hybrid_attn_backend.py b/python/sglang/srt/layers/attention/hybrid_attn_backend.py index e3900a0603..38e80df18e 100644 --- a/python/sglang/srt/layers/attention/hybrid_attn_backend.py +++ b/python/sglang/srt/layers/attention/hybrid_attn_backend.py @@ -90,6 +90,14 @@ class HybridAttnBackend(AttentionBackend): def shared_read_ends(self, fm: ForwardMode) -> SharedReadEnds: return self._select_backend(fm).shared_read_ends(fm) + def _snapshot_prefill_shared_reads( + self, forward_batch: ForwardBatch, *, num_qo_tokens: int + ) -> SharedReadEnds: + backend = self._select_backend(forward_batch.forward_mode) + return backend._snapshot_prefill_shared_reads( + forward_batch, num_qo_tokens=num_qo_tokens + ) + def get_fused_kv_write_buffers(self, layer, forward_batch): return self._select_backend( forward_batch.forward_mode diff --git a/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py b/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py index 89121f76f6..0b0b6d412c 100644 --- a/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py +++ b/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py @@ -1311,10 +1311,6 @@ class HybridLinearAttnBackend(AttentionBackend): ) elif not forward_batch.forward_mode.is_draft_extend_v2(): self.linear_attn_backend.init_forward_metadata(forward_batch) - if static_forward_batch is not None: - self.prepare_prefill_shared_read_snapshot( - forward_batch, num_qo_tokens=static_forward_batch.positions.shape[0] - ) def get_indexer_metadata(self, layer_id: int, forward_batch: ForwardBatch): if layer_id in self.full_attn_layers: diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index 506f3c45a4..b3bcaaf6d1 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -20,7 +20,7 @@ import inspect import logging import time from dataclasses import dataclass -from typing import Callable, List, Optional, Union +from typing import List, Optional, Union import torch import torch.distributed as dist @@ -465,9 +465,6 @@ class ModelRunner: # Read-done mailbox: the scheduler's WAR barrier reads it from the runner # its worker names, and treats None as the coarse whole-forward fence. self.shared_read_done_event: Optional[torch.cuda.Event] = None - # Scoped by a speculative worker to stage its shared reads before - # the target prefill graph publishes the read-done event. - self.prefill_shared_read_stager: Optional[Callable[[ForwardBatch], bool]] = None # CPU offload set_offloader(create_offloader()) diff --git a/python/sglang/srt/model_executor/runner/eager_runner.py b/python/sglang/srt/model_executor/runner/eager_runner.py index d77170a3e1..7e0f3b3e14 100644 --- a/python/sglang/srt/model_executor/runner/eager_runner.py +++ b/python/sglang/srt/model_executor/runner/eager_runner.py @@ -321,13 +321,15 @@ class EagerRunner(BaseRunner): # e.g. Moss-VL's prefill cross-attention custom mask. model_runner.model.prepare_forward_batch(forward_batch) model_runner.attn_backend.init_forward_metadata(forward_batch) - model_runner.attn_backend.prepare_prefill_shared_read_snapshot( - forward_batch, - num_qo_tokens=len(forward_batch.input_ids), + shared_read_ends = ( + model_runner.attn_backend.resolve_prefill_shared_read_ends( + forward_batch, + num_qo_tokens=len(forward_batch.input_ids), + ) ) maybe_publish_prefill_shared_read_done( model_runner, - forward_batch, + shared_read_ends, torch.get_device_module(model_runner.device), ) diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index cf291612aa..320bb662fd 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -54,6 +54,7 @@ from sglang.kernels.ops.attention.kv_indices import ( ) from sglang.srt.configs.model_config import is_qwen4_exp from sglang.srt.distributed.parallel_state import graph_capture +from sglang.srt.layers.attention.base_attn_backend import SharedReadEnds from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp from sglang.srt.layers.cp.bcg import ( PrefillCPBCGInput, @@ -1250,7 +1251,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): forward_batch: ForwardBatch, static_forward_batch: ForwardBatch, shape_key: ShapeKey, - ) -> None: + ) -> SharedReadEnds: """Replay-time metadata refresh for the BCG-with-captured-metadata contract. For opt-in backends, refresh the stashed per-bucket metadata in place against the current batch; otherwise fall back @@ -1259,6 +1260,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): real seq_lens / prefix_lens; the captured kernels read the updated state at replay.""" attn_backend = self.model_runner.attn_backend + build_snapshot = False if self._is_full_backend: # Slot-padded shallow view: plan() must see exactly req_slots # entries (real values in [:bs], sentinels in [bs:req_slots] @@ -1282,19 +1284,22 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): padded_view.max_seq_len_override = static_forward_batch.max_seq_len_override padded_view.input_ids = static_forward_batch.input_ids attn_backend.init_forward_metadata_out_graph(padded_view) - return - if not self.use_captured_attn_metadata: + elif not self.use_captured_attn_metadata: attn_backend.init_forward_metadata(forward_batch) - attn_backend.prepare_prefill_shared_read_snapshot( - forward_batch, num_qo_tokens=shape_key.size + build_snapshot = True + else: + assert self.attn_metadata_buffers is not None + metadata = self.attn_metadata_buffers[shape_key] + attn_backend.prepare_forward_metadata_for_breakable_cuda_graph_replay( + metadata, + forward_batch, + static_forward_batch=static_forward_batch, ) - return - assert self.attn_metadata_buffers is not None - metadata = self.attn_metadata_buffers[shape_key] - attn_backend.prepare_forward_metadata_for_breakable_cuda_graph_replay( - metadata, + + return attn_backend.resolve_prefill_shared_read_ends( forward_batch, - static_forward_batch=static_forward_batch, + num_qo_tokens=shape_key.size, + build_snapshot=build_snapshot, ) @staticmethod @@ -1816,10 +1821,10 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): post_warmup_hook=post_warmup_hook, ) - def load_batch(self, forward_batch: ForwardBatch, **kwargs) -> ForwardBatch: - """Pad, populate static buffers, and build the static_forward_batch - the model code reads during replay. - """ + def load_batch( + self, forward_batch: ForwardBatch, **kwargs + ) -> tuple[ForwardBatch, SharedReadEnds]: + """Pad, fill static buffers, and prepare replay metadata and its read end.""" num_tokens = len(forward_batch.input_ids) static_num_tokens = self._pad_to_bucket(num_tokens, self.capture_num_tokens) if getattr(self, "enable_cp_bcg_capture", False) and is_cp_active( @@ -2100,11 +2105,11 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): metadata_forward_batch = static_forward_batch shape_key = self._shape_key(static_num_tokens, forward_batch) - self._prepare_forward_metadata_for_replay( + shared_read_ends = self._prepare_forward_metadata_for_replay( metadata_forward_batch, static_forward_batch, shape_key ) - return static_forward_batch + return static_forward_batch, shared_read_ends def _fill_input_embeds_slot(self, args, layer_kwargs, static_num_tokens: int): """A text-only batch would otherwise replay the captured input_embeds.""" @@ -2312,17 +2317,19 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): ) -> Optional[Union[LogitsProcessorOutput, PPProxyTensors, EmbeddingPoolerOutput]]: self._validate_capture_hidden_mode(forward_batch) with self.model_context(), self.backend.replay_session(): - static_forward_batch = self.load_batch(forward_batch, **kwargs) + static_forward_batch, shared_read_ends = self.load_batch( + forward_batch, **kwargs + ) static_num_tokens = len(static_forward_batch.input_ids) raw_num_tokens = self.raw_num_tokens shape_key = self._shape_key(static_num_tokens, forward_batch) # The only variants this runner records are chunked-prefix ones. if shape_key.variant_label is not None: self._prepare_chunked_prefix_replay(shape_key, forward_batch) - # Replay prep, including the optional chunked-prefix gather above, - # has finished every scheduler-shared read. + # Publish after replay prep, including the chunked-prefix gather + # above, which also reads the shared tables. maybe_publish_prefill_shared_read_done( - self.model_runner, forward_batch, self.device_module + self.model_runner, shared_read_ends, self.device_module ) self._wait_for_layer_loads() diff --git a/python/sglang/srt/model_executor/runner_utils/shared_read_event.py b/python/sglang/srt/model_executor/runner_utils/shared_read_event.py index 8bb7c7c91a..cc77839c14 100644 --- a/python/sglang/srt/model_executor/runner_utils/shared_read_event.py +++ b/python/sglang/srt/model_executor/runner_utils/shared_read_event.py @@ -5,9 +5,8 @@ from typing import Optional import torch -from sglang.srt.environ import envs from sglang.srt.layers.attention.base_attn_backend import SharedReadEnds -from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.srt.runtime_context import get_memory from sglang.srt.utils import is_cuda logger = logging.getLogger(__name__) @@ -24,22 +23,22 @@ def make_external_event(device_module) -> Optional[torch.cuda.Event]: def maybe_publish_prefill_shared_read_done( - model_runner, forward_batch, device_module + model_runner, shared_read_ends: SharedReadEnds, device_module ) -> None: - """Publish prefill read-done after compliant metadata initialization.""" - if not envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.get(): - return - if forward_batch.forward_mode != ForwardMode.EXTEND: - return - # The record lands right after replay prep, so PRE_REPLAY only. - declared = model_runner.attn_backend.shared_read_ends(forward_batch.forward_mode) - if declared is not SharedReadEnds.PRE_REPLAY: + """Publish prefill read-done at the resolved pre-replay boundary.""" + if shared_read_ends is not SharedReadEnds.PRE_REPLAY: return if not model_runner.spec_algorithm.supports_prefill_shared_read_done(): - # Stage the draft's shared reads before publishing the read-done event. - stage = getattr(model_runner, "prefill_shared_read_stager", None) - if stage is None or not stage(forward_batch): - return + return + # Host and storage offload copy a stashed chunk's KV ordered only behind the + # schedule stream, so they need the whole-forward barrier. + memory = get_memory() + if ( + memory.enable_hierarchical_cache + or memory.hicache_storage_backend is not None + or memory.enable_lmcache + ): + return logger.info_once( "Prefill shared-read-done fastpath active (%s)", type(model_runner.attn_backend).__name__, diff --git a/python/sglang/srt/speculative/spec_info.py b/python/sglang/srt/speculative/spec_info.py index df5f983b41..60890b9675 100644 --- a/python/sglang/srt/speculative/spec_info.py +++ b/python/sglang/srt/speculative/spec_info.py @@ -138,10 +138,10 @@ class SpeculativeAlgorithm(Enum): def supports_prefill_shared_read_done(self) -> bool: """Whether target EXTEND has no later speculative shared-buffer reader. - The backend must still declare a pre-replay read end. Other algorithms - must stage their draft's shared reads before publishing the target event. + The backend must still declare a pre-replay read end. """ - return self.is_none() or self.is_dflash_family() + # DSPARK's draft KV inject reads the SWA mapping after the target prefill. + return self.is_none() or self.is_dflash() def supports_mixed_chunk(self) -> bool: """Whether mixed chunk prefill may stay enabled with this algorithm. diff --git a/test/registered/attention/unittests/dsv4/test_deepseek_v4.py b/test/registered/attention/unittests/dsv4/test_deepseek_v4.py index 7d31fe3d6a..188f47176a 100644 --- a/test/registered/attention/unittests/dsv4/test_deepseek_v4.py +++ b/test/registered/attention/unittests/dsv4/test_deepseek_v4.py @@ -511,115 +511,90 @@ class TestDSV4BreakableCudaGraphMetadataContract(CustomTestCase): DeepseekV4AttnBackend.use_captured_forward_metadata_for_breakable_cuda_graph ) - def test_prefill_snapshot_declares_pre_replay_boundary(self): - from sglang.srt.layers.attention.base_attn_backend import SharedReadEnds + def _make_prefill_backend(self, *, is_dsv41=False, hisparse_coordinator=None): from sglang.srt.layers.attention.deepseek_v4_backend import ( DeepseekV4AttnBackend, DSV4Metadata, ) backend = object.__new__(DeepseekV4AttnBackend) + backend.token_to_kv_pool = SimpleNamespace(request_window=None) + backend.is_dsv41 = is_dsv41 + backend.hisparse_coordinator = hisparse_coordinator backend.forward_metadata = DSV4Metadata( self._make_core_metadata(0), indexer_metadata=None ) - self.assertIs( - backend.shared_read_ends(ForwardMode.EXTEND), - SharedReadEnds.UNKNOWN, - ) + backend._build_sparse_prefill_chunk_cache = mock.Mock(return_value=object()) + return backend - backend.forward_metadata.prefill_shared_reads_snapshotted = True + def test_static_extend_read_end_stays_unknown(self): + from sglang.srt.layers.attention.base_attn_backend import SharedReadEnds + + # Paths that skip the snapshot hook publish from this declaration. + backend = self._make_prefill_backend() self.assertIs( - backend.shared_read_ends(ForwardMode.EXTEND), - SharedReadEnds.PRE_REPLAY, + backend.shared_read_ends(ForwardMode.EXTEND), SharedReadEnds.UNKNOWN ) def test_snapshot_builds_cache_only_for_sparse_prefill(self): from sglang.srt.environ import envs + from sglang.srt.layers.attention.base_attn_backend import SharedReadEnds from sglang.srt.layers.attention.deepseek_v4_backend import ( _LARGE_INDEXER_QUERY_THRESHOLD, - DeepseekV4AttnBackend, - DSV4Metadata, ) - from sglang.srt.speculative.spec_info import SpeculativeAlgorithm batch = SimpleNamespace(forward_mode=ForwardMode.EXTEND) - cache = object() for num_qo_tokens, builds in ( (_LARGE_INDEXER_QUERY_THRESHOLD, False), (_LARGE_INDEXER_QUERY_THRESHOLD + 1, True), ): with self.subTest(num_qo_tokens=num_qo_tokens): - backend = object.__new__(DeepseekV4AttnBackend) - backend.model_runner = SimpleNamespace( - spec_algorithm=SpeculativeAlgorithm.DFLASH - ) - backend.token_to_kv_pool = SimpleNamespace(request_window=None) - backend.forward_metadata = DSV4Metadata( - self._make_core_metadata(0), indexer_metadata=None - ) - backend._build_sparse_prefill_chunk_cache = mock.Mock( - return_value=cache - ) + backend = self._make_prefill_backend() + build = backend._build_sparse_prefill_chunk_cache with ( - envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.override(True), envs.SGLANG_OPT_FLASHMLA_SPARSE_PREFILL.override(False), mock.patch( "sglang.srt.layers.attention.deepseek_v4_backend.get_platform", return_value=SimpleNamespace(is_sm120=False), ), ): - backend.prepare_prefill_shared_read_snapshot( - batch, num_qo_tokens=num_qo_tokens + shared_read_ends = backend.resolve_prefill_shared_read_ends( + batch, + num_qo_tokens=num_qo_tokens, ) metadata = backend.forward_metadata if builds: - backend._build_sparse_prefill_chunk_cache.assert_called_once_with( + build.assert_called_once_with( batch, metadata.core_attn_metadata, num_qo_tokens=num_qo_tokens ) - self.assertIs(metadata.sparse_prefill_cache, cache) + self.assertIs(metadata.sparse_prefill_cache, build.return_value) else: - backend._build_sparse_prefill_chunk_cache.assert_not_called() + build.assert_not_called() self.assertIsNone(metadata.sparse_prefill_cache) - # Dense declares the boundary too; it reads only the metadata - # that init_forward_metadata already snapshotted. - self.assertTrue(metadata.prefill_shared_reads_snapshotted) + # Dense reaches the same boundary using metadata init's snapshot. + self.assertIs(shared_read_ends, SharedReadEnds.PRE_REPLAY) - def test_sparse_prefill_snapshot_marks_success_only_after_build(self): - from sglang.srt.environ import envs + def test_prefill_snapshot_skips_layers_reading_live_tables(self): + from sglang.srt.layers.attention.base_attn_backend import SharedReadEnds from sglang.srt.layers.attention.deepseek_v4_backend import ( - DeepseekV4AttnBackend, - DSV4Metadata, + _LARGE_INDEXER_QUERY_THRESHOLD, ) - from sglang.srt.speculative.spec_info import SpeculativeAlgorithm - backend = object.__new__(DeepseekV4AttnBackend) - backend.model_runner = SimpleNamespace( - spec_algorithm=SpeculativeAlgorithm.DFLASH - ) - backend.token_to_kv_pool = SimpleNamespace(request_window=None) - backend.forward_metadata = DSV4Metadata( - self._make_core_metadata(0), indexer_metadata=None - ) - backend.forward_metadata.prefill_shared_reads_snapshotted = True - backend._build_sparse_prefill_chunk_cache = mock.Mock( - side_effect=RuntimeError("snapshot failed") - ) batch = SimpleNamespace(forward_mode=ForwardMode.EXTEND) + for is_dsv41, hisparse_coordinator in ((True, None), (False, object())): + with self.subTest(is_dsv41=is_dsv41, hisparse=hisparse_coordinator): + backend = self._make_prefill_backend( + is_dsv41=is_dsv41, hisparse_coordinator=hisparse_coordinator + ) - with ( - envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.override(True), - envs.SGLANG_OPT_FLASHMLA_SPARSE_PREFILL.override(True), - mock.patch( - "sglang.srt.layers.attention.deepseek_v4_backend.get_platform", - return_value=SimpleNamespace(is_sm120=False), - ), - self.assertRaisesRegex(RuntimeError, "snapshot failed"), - ): - backend.prepare_prefill_shared_read_snapshot(batch, num_qo_tokens=12288) + shared_read_ends = backend.resolve_prefill_shared_read_ends( + batch, num_qo_tokens=_LARGE_INDEXER_QUERY_THRESHOLD + 1 + ) - self.assertFalse(backend.forward_metadata.prefill_shared_reads_snapshotted) + self.assertIs(shared_read_ends, SharedReadEnds.UNKNOWN) + backend._build_sparse_prefill_chunk_cache.assert_not_called() def test_refresh_replay_metadata_preserves_captured_tensor_storage(self): capture_metadata = self._make_core_metadata(0) @@ -692,7 +667,6 @@ class TestDSV4BreakableCudaGraphMetadataContract(CustomTestCase): self._make_core_metadata(0), indexer_metadata=None ) capture_metadata.sparse_prefill_cache = object() - capture_metadata.prefill_shared_reads_snapshotted = True replay_metadata = DSV4Metadata( self._make_core_metadata(1000), indexer_metadata=None ) @@ -724,7 +698,6 @@ class TestDSV4BreakableCudaGraphMetadataContract(CustomTestCase): self.assertTrue(calls[0][2]) self.assertIs(backend.forward_metadata, capture_metadata) self.assertIsNone(capture_metadata.sparse_prefill_cache) - self.assertFalse(capture_metadata.prefill_shared_reads_snapshotted) self.assertTrue( torch.equal( capture_metadata.core_attn_metadata.seq_lens_casual, diff --git a/test/registered/unit/model_executor/runner/test_prefill_cuda_graph_padding.py b/test/registered/unit/model_executor/runner/test_prefill_cuda_graph_padding.py index 427ecfed5a..20dbcf5c1b 100644 --- a/test/registered/unit/model_executor/runner/test_prefill_cuda_graph_padding.py +++ b/test/registered/unit/model_executor/runner/test_prefill_cuda_graph_padding.py @@ -84,19 +84,23 @@ class TestPrefillCudaGraphPadding(CustomTestCase): runner = self._make_runner() runner.use_captured_attn_metadata = False attn_backend = mock.Mock() + attn_backend.resolve_prefill_shared_read_ends.return_value = mock.sentinel.end runner.model_runner = SimpleNamespace(attn_backend=attn_backend) forward_batch = self._make_forward_batch(8) static_forward_batch = self._make_forward_batch(16) - runner._prepare_forward_metadata_for_replay( + shared_read_ends = runner._prepare_forward_metadata_for_replay( forward_batch, static_forward_batch, shape_key=ShapeKey(size=16), ) + self.assertIs(shared_read_ends, mock.sentinel.end) attn_backend.init_forward_metadata.assert_called_once_with(forward_batch) - attn_backend.prepare_prefill_shared_read_snapshot.assert_called_once_with( - forward_batch, num_qo_tokens=16 + attn_backend.resolve_prefill_shared_read_ends.assert_called_once_with( + forward_batch, + num_qo_tokens=16, + build_snapshot=True, ) def test_full_replay_pads_request_slot_cpu_mirror(self): diff --git a/test/registered/unit/model_executor/runner/test_prefill_shared_read_done.py b/test/registered/unit/model_executor/runner/test_prefill_shared_read_done.py index abfc0ddb56..7a1dc0ea00 100644 --- a/test/registered/unit/model_executor/runner/test_prefill_shared_read_done.py +++ b/test/registered/unit/model_executor/runner/test_prefill_shared_read_done.py @@ -1,5 +1,4 @@ from types import SimpleNamespace -from unittest.mock import Mock, call, create_autospec import pytest @@ -12,6 +11,7 @@ from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.model_executor.runner_utils.shared_read_event import ( maybe_publish_prefill_shared_read_done, ) +from sglang.srt.runtime_context import get_context, get_memory from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.srt.speculative.spec_registry import CustomSpecAlgo from sglang.test.ci.ci_register import register_cpu_ci @@ -20,6 +20,12 @@ from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=11, suite="base-a-test-cpu") +@pytest.fixture(autouse=True) +def _published_server_args(): + with get_context().override_server_args(): + yield + + class _Event: def __init__(self): self.recorded = False @@ -28,17 +34,19 @@ class _Event: self.recorded = True -def _model_runner(*, spec_algorithm=SpeculativeAlgorithm.NONE, compliant=True): - declared = SharedReadEnds.PRE_REPLAY if compliant else SharedReadEnds.UNKNOWN - # Spec'd against the real ABC so a rename fails here, not at runtime. - attn_backend = create_autospec(AttentionBackend, instance=True) - attn_backend.shared_read_ends.return_value = declared - return SimpleNamespace( +def _model_runner( + *, spec_algorithm=SpeculativeAlgorithm.NONE, compliant=True, declared=None +): + if declared is None: + declared = SharedReadEnds.PRE_REPLAY if compliant else SharedReadEnds.UNKNOWN + attn_backend = AttentionBackend() + attn_backend.shared_read_ends = lambda _forward_mode: declared + runner = SimpleNamespace( spec_algorithm=spec_algorithm, attn_backend=attn_backend, shared_read_done_event=None, - prefill_shared_read_stager=None, ) + return runner _DEVICE_MODULE = SimpleNamespace(Event=_Event) @@ -48,10 +56,16 @@ def _batch(mode=ForwardMode.EXTEND): return SimpleNamespace(forward_mode=mode) -def test_publishes_recorded_event_when_enabled(): +def _prepare_and_publish(runner, batch): + shared_read_ends = runner.attn_backend.resolve_prefill_shared_read_ends( + batch, num_qo_tokens=8 + ) + maybe_publish_prefill_shared_read_done(runner, shared_read_ends, _DEVICE_MODULE) + + +def test_publishes_recorded_event_by_default(): runner = _model_runner() - with envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.override(True): - maybe_publish_prefill_shared_read_done(runner, _batch(), _DEVICE_MODULE) + _prepare_and_publish(runner, _batch()) published = runner.shared_read_done_event assert isinstance(published, _Event) and published.recorded @@ -59,73 +73,31 @@ def test_publishes_recorded_event_when_enabled(): def test_disabled_when_flag_is_false(): runner = _model_runner() with envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.override(False): - maybe_publish_prefill_shared_read_done(runner, _batch(), _DEVICE_MODULE) + _prepare_and_publish(runner, _batch()) assert runner.shared_read_done_event is None @pytest.mark.parametrize( - "algorithm", (SpeculativeAlgorithm.DFLASH, SpeculativeAlgorithm.DSPARK) + "memory", + [ + {"enable_hierarchical_cache": True}, + {"hicache_storage_backend": "mooncake"}, + {"enable_lmcache": True}, + ], ) -def test_dflash_family_target_prefill_publishes(algorithm): - runner = _model_runner(spec_algorithm=algorithm) - runner.prefill_shared_read_stager = Mock(return_value=False) +def test_kv_offload_keeps_the_coarse_barrier(memory): + runner = _model_runner() + with get_memory().override(**memory): + _prepare_and_publish(runner, _batch()) + assert runner.shared_read_done_event is None + + +def test_dflash_target_prefill_publishes(): + runner = _model_runner(spec_algorithm=SpeculativeAlgorithm.DFLASH) with envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.override(True): - maybe_publish_prefill_shared_read_done(runner, _batch(), _DEVICE_MODULE) + _prepare_and_publish(runner, _batch()) published = runner.shared_read_done_event assert isinstance(published, _Event) and published.recorded - runner.prefill_shared_read_stager.assert_not_called() - - -@pytest.mark.parametrize("staged", [False, True]) -@pytest.mark.parametrize( - "algorithm", - [ - SpeculativeAlgorithm.EAGLE, - SpeculativeAlgorithm.EAGLE3, - SpeculativeAlgorithm.FROZEN_KV_MTP, - SpeculativeAlgorithm.STANDALONE, - SpeculativeAlgorithm.NGRAM, - SpeculativeAlgorithm.UNO, - CustomSpecAlgo("TEST_PREFILL", lambda _: object), - ], -) -def test_speculative_prefill_publishes_only_after_staging(algorithm, staged): - runner, batch = _model_runner(spec_algorithm=algorithm), _batch() - calls = Mock() - runner.prefill_shared_read_stager = calls.stage - calls.stage.return_value = staged - calls.Event.side_effect = _Event - with envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.override(True): - maybe_publish_prefill_shared_read_done( - runner, batch, SimpleNamespace(Event=calls.Event) - ) - assert calls.mock_calls == [call.stage(batch)] + ([call.Event()] if staged else []) - published = runner.shared_read_done_event - if staged: - assert isinstance(published, _Event) and published.recorded - else: - assert published is None - - -@pytest.mark.parametrize( - "enabled,mode,compliant", - [ - (False, ForwardMode.EXTEND, True), - (True, ForwardMode.TARGET_VERIFY, True), - (True, ForwardMode.MIXED, True), - (True, ForwardMode.DECODE, True), - (True, ForwardMode.EXTEND, False), - ], -) -def test_prefill_gates_skip_staging(enabled, mode, compliant): - runner = _model_runner( - spec_algorithm=SpeculativeAlgorithm.EAGLE, compliant=compliant - ) - runner.prefill_shared_read_stager = Mock(return_value=True) - with envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.override(enabled): - maybe_publish_prefill_shared_read_done(runner, _batch(mode), _DEVICE_MODULE) - runner.prefill_shared_read_stager.assert_not_called() - assert runner.shared_read_done_event is None def test_gates_exclude_non_prefill_unsupported_algorithm_and_noncompliant_backend(): @@ -135,7 +107,7 @@ def test_gates_exclude_non_prefill_unsupported_algorithm_and_noncompliant_backen (_model_runner(), _batch(ForwardMode.TARGET_VERIFY)), (_model_runner(), _batch(ForwardMode.MIXED)), (_model_runner(), _batch(ForwardMode.DECODE)), - # The algorithm has a later prefill reader or unverified ownership. + # The algorithm has a later prefill reader. (_model_runner(spec_algorithm=SpeculativeAlgorithm.EAGLE), _batch()), ( _model_runner( @@ -146,7 +118,7 @@ def test_gates_exclude_non_prefill_unsupported_algorithm_and_noncompliant_backen # Backend has not declared a pre-replay prefill read end. (_model_runner(compliant=False), _batch()), ): - maybe_publish_prefill_shared_read_done(runner, batch, _DEVICE_MODULE) + _prepare_and_publish(runner, batch) assert runner.shared_read_done_event is None @@ -157,18 +129,12 @@ class _TargetOnlyPrefillAlgo(CustomSpecAlgo): class TestPrefillReadDoneCapability(CustomTestCase): def test_builtin_allowlist_is_unchanged(self): - allowed = { - SpeculativeAlgorithm.NONE, - SpeculativeAlgorithm.DFLASH, - SpeculativeAlgorithm.DSPARK, - } + allowed = {SpeculativeAlgorithm.NONE, SpeculativeAlgorithm.DFLASH} for algorithm in SpeculativeAlgorithm: with self.subTest(algorithm=algorithm): runner = _model_runner(spec_algorithm=algorithm) with envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.override(True): - maybe_publish_prefill_shared_read_done( - runner, _batch(), _DEVICE_MODULE - ) + _prepare_and_publish(runner, _batch()) self.assertEqual( runner.shared_read_done_event is not None, algorithm in allowed ) @@ -182,19 +148,13 @@ class TestPrefillReadDoneCapability(CustomTestCase): algorithm = algorithm_type( "TEST_PREFILL", lambda _: object, supports_overlap=True ) - runner, batch = _model_runner(spec_algorithm=algorithm), _batch() - runner.prefill_shared_read_stager = Mock(return_value=False) + runner = _model_runner(spec_algorithm=algorithm) with envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.override(True): - maybe_publish_prefill_shared_read_done( - runner, batch, _DEVICE_MODULE - ) + _prepare_and_publish(runner, _batch()) event = runner.shared_read_done_event self.assertEqual(event is not None, expected) if expected: self.assertTrue(event.recorded) - runner.prefill_shared_read_stager.assert_not_called() - else: - runner.prefill_shared_read_stager.assert_called_once_with(batch) def test_opt_in_does_not_bypass_mode_backend_or_flag(self): algorithm = _TargetOnlyPrefillAlgo("TEST_PREFILL", lambda _: object) @@ -211,12 +171,9 @@ class TestPrefillReadDoneCapability(CustomTestCase): ) for enabled, mode, declared in cases: with self.subTest(enabled=enabled, mode=mode, declared=declared): - runner = _model_runner(spec_algorithm=algorithm) - runner.attn_backend.shared_read_ends.return_value = declared + runner = _model_runner(spec_algorithm=algorithm, declared=declared) with envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.override(enabled): - maybe_publish_prefill_shared_read_done( - runner, _batch(mode), _DEVICE_MODULE - ) + _prepare_and_publish(runner, _batch(mode)) self.assertIsNone(runner.shared_read_done_event) diff --git a/test/registered/unit/model_executor/test_prefill_cuda_graph_runner.py b/test/registered/unit/model_executor/test_prefill_cuda_graph_runner.py index 834e57baa9..c3ea9030ec 100644 --- a/test/registered/unit/model_executor/test_prefill_cuda_graph_runner.py +++ b/test/registered/unit/model_executor/test_prefill_cuda_graph_runner.py @@ -432,7 +432,7 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase): ) translator.bind_own_plan(forward_batch) - static_batch = runner.load_batch(forward_batch) + static_batch, _ = runner.load_batch(forward_batch) self.assertIs(static_batch.mm_input_embeds, mm_input_embeds)