[Perf] prefill-cuda-graph: Make prefill shared-read completion explicit and default-on (#37958)

Co-authored-by: weireweire <20922698+weireweire@users.noreply.github.com>
Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
Co-authored-by: Po-Han Huang <pohanh@nvidia.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: hnyls2002 <lsyincs@gmail.com>
This commit is contained in:
weireweire
2026-10-20 16:12:55 -07:00
committed by GitHub
co-authored by weireweire Po-Han Huang Po-Han Huang Claude Opus 5.5 hnyls2002
parent d5e83ea4f6
commit bf145750a6
15 changed files with 216 additions and 262 deletions
+1 -1
View File
@@ -669,7 +669,7 @@ class Envs:
# read-done fastpath event.
SGLANG_FORCE_COARSE_WAR_BARRIER = EnvBool(False)
# Enable prefill read-done publication after compliant metadata initialization.
SGLANG_ENABLE_PREFILL_WAR_READ_DONE = EnvBool(False)
SGLANG_ENABLE_PREFILL_WAR_READ_DONE = EnvBool(True)
# PP: skip output send/recv when the entire batch consists of non-final chunked prefill requests,
# since process_batch_result_prefill discards next_token_ids for those anyway.
SGLANG_PP_SKIP_PURE_CHUNKED_OUTPUT_COMM = EnvBool(False)
@@ -7,6 +7,8 @@ from typing import TYPE_CHECKING, Iterable, Optional
import torch
from sglang.kernels.kernel_api_logging import debug_kernel_api
from sglang.srt.environ import envs
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.utils.common import is_npu
if TYPE_CHECKING:
@@ -15,7 +17,7 @@ if TYPE_CHECKING:
)
from sglang.srt.layers.attention.verify_mask import VerifyMask
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.speculative.spec_info import SpecInput
@@ -183,15 +185,38 @@ class AttentionBackend(ABC):
return SharedReadEnds.IN_REPLAY
return SharedReadEnds.UNKNOWN
def prepare_prefill_shared_read_snapshot(
self, forward_batch: ForwardBatch, *, num_qo_tokens: int
) -> None:
"""Snapshot late prefill reads before a PRE_REPLAY event is published.
def resolve_prefill_shared_read_ends(
self,
forward_batch: ForwardBatch,
*,
num_qo_tokens: int,
build_snapshot: bool = True,
) -> SharedReadEnds:
"""Return this batch's prefill read end, building a snapshot when asked.
Runners call this only after the actual eager/replay query geometry is
known. Backends that retain scheduler-shared reads into the model
forward keep the default no-op and must not declare PRE_REPLAY.
Spec algorithms with a later draft reader are gated at publish time.
"""
if (
envs.SGLANG_FORCE_COARSE_WAR_BARRIER.get()
or not envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.get()
):
return SharedReadEnds.UNKNOWN
if forward_batch.forward_mode != ForwardMode.EXTEND:
return SharedReadEnds.UNKNOWN
if build_snapshot:
return self._snapshot_prefill_shared_reads(
forward_batch, num_qo_tokens=num_qo_tokens
)
return self.shared_read_ends(forward_batch.forward_mode)
def _snapshot_prefill_shared_reads(
self, forward_batch: ForwardBatch, *, num_qo_tokens: int
) -> SharedReadEnds:
"""Snapshot late shared reads; return PRE_REPLAY only if none remain.
Hook-skipping paths read ``shared_read_ends(EXTEND)``; keep it conservative.
"""
return self.shared_read_ends(forward_batch.forward_mode)
# Chunked-prefix FullCG capture has a second model topology and stable
# prefix buffers. Backends must opt in explicitly so the runner does not
@@ -1056,7 +1056,6 @@ class DSV4Metadata:
# Built at the runner's prefill WAR boundary when the fast path is on,
# otherwise lazily by ``_forward_prefill_sparse``.
sparse_prefill_cache: Optional[SparsePrefillChunkCache] = None
prefill_shared_reads_snapshotted: bool = False
# Set only on the metadata built for the late layers under bounded SWA replay.
late_layer_tail: Optional[LateLayerTail] = None
@@ -1078,7 +1077,6 @@ class DSV4Metadata:
self.c128_compress_metadata, src=other.c128_compress_metadata
)
self.sparse_prefill_cache = None
self.prefill_shared_reads_snapshotted = False
def refresh_for_breakable_cuda_graph_replay_(
self,
@@ -1123,7 +1121,6 @@ class DSV4Metadata:
src=static_metadata.c128_compress_metadata,
)
self.sparse_prefill_cache = None
self.prefill_shared_reads_snapshotted = False
@dataclass
@@ -1203,13 +1200,6 @@ class DeepseekV4AttnBackend(
if self.model_runner.spec_algorithm.is_dspark():
return SharedReadEnds.IN_REPLAY
return SharedReadEnds.POST_REPLAY
metadata = self.forward_metadata
if (
fm == ForwardMode.EXTEND
and isinstance(metadata, DSV4Metadata)
and metadata.prefill_shared_reads_snapshotted
):
return SharedReadEnds.PRE_REPLAY
return super().shared_read_ends(fm)
def __init__(
@@ -2739,47 +2729,46 @@ class DeepseekV4AttnBackend(
def _use_sparse_prefill(
self, forward_batch: ForwardBatch, *, num_qo_tokens: int
) -> bool:
# Shared with the snapshot hook, so no forward builds the cache after read-done.
# The tail stays dense: its window floor lives in swa_page_indices, which
# the chunk cache ignores.
return (
forward_batch.forward_mode.is_extend_without_speculative()
not self.trtllm_attn
and forward_batch.forward_mode.is_extend_without_speculative()
and not is_cp_active(forward_batch)
and not get_platform().is_sm120
and self.forward_metadata.late_layer_tail is None
and self.token_to_kv_pool.request_window is None
and (
num_qo_tokens > _LARGE_INDEXER_QUERY_THRESHOLD
or envs.SGLANG_OPT_FLASHMLA_SPARSE_PREFILL.get()
)
)
def prepare_prefill_shared_read_snapshot(
def _snapshot_prefill_shared_reads(
self, forward_batch: ForwardBatch, *, num_qo_tokens: int
) -> None:
) -> SharedReadEnds:
# Sparse prefill otherwise reads req_to_token/full_to_swa lazily in its
# first layer. DFLASH/DSPARK have no later prefill draft-extend reader;
# CP shards the query layout that this global snapshot assumes.
metadata = self.forward_metadata
# first layer. CP shards the query layout that this global snapshot
# assumes; request-window layouts are not covered by the snapshot.
if self.token_to_kv_pool.request_window is not None:
return
if isinstance(metadata, DSV4Metadata):
metadata.prefill_shared_reads_snapshotted = False
snapshot_shared_prefill_reads = (
envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.get()
and forward_batch.forward_mode == ForwardMode.EXTEND
and self.model_runner.spec_algorithm.is_dflash_family()
and not is_cp_active(forward_batch)
)
if not snapshot_shared_prefill_reads:
return
return SharedReadEnds.UNKNOWN
if is_cp_active(forward_batch):
return SharedReadEnds.UNKNOWN
# The V4.1 low-ratio indexer reads req_to_token and HiSparse translates
# through full_to_hisparse in every layer, after any snapshot.
if self.is_dsv41 or self.hisparse_coordinator is not None:
return SharedReadEnds.UNKNOWN
metadata = self.forward_metadata
assert isinstance(metadata, DSV4Metadata)
# The tail never takes the sparse path, so it carries no chunk cache.
if metadata.late_layer_tail is None and self._use_sparse_prefill(
forward_batch, num_qo_tokens=num_qo_tokens
):
if self._use_sparse_prefill(forward_batch, num_qo_tokens=num_qo_tokens):
metadata.sparse_prefill_cache = self._build_sparse_prefill_chunk_cache(
forward_batch, metadata.core_attn_metadata, num_qo_tokens=num_qo_tokens
)
# Marked for dense prefill too: that path reads only core_attn_metadata,
# which init_forward_metadata already snapshotted.
metadata.prefill_shared_reads_snapshotted = True
# Dense prefill reads only core_attn_metadata, which metadata init has
# already snapshotted, so it reaches the same boundary without a cache.
return SharedReadEnds.PRE_REPLAY
def _build_sparse_prefill_chunk_cache(
self,
@@ -3821,11 +3810,8 @@ class DeepseekV4AttnBackend(
# raw_indices; num_qo_tokens=0 drops the query-count clause, so this errs
# to True. Paged attention under active CP needs the page indices.
return (
self.trtllm_attn
or self.mega_attn
self.mega_attn
or self.tail_forward_metadata is not None
or self.forward_metadata.late_layer_tail is not None
or self.token_to_kv_pool.request_window is not None
or not self._use_sparse_prefill(forward_batch, num_qo_tokens=0)
)
@@ -4065,13 +4051,9 @@ class DeepseekV4AttnBackend(
num_heads=q.shape[-2],
)
# The tail stays dense: its window floor lives in swa_page_indices,
# which the sparse chunk cache ignores.
if (
self._use_sparse_prefill(forward_batch, num_qo_tokens=q.shape[0])
and not use_mega_paged
and self.forward_metadata.late_layer_tail is None
and token_to_kv_pool.request_window is None
):
if use_dsv4_q8kv8_sparse_prefill(self.dsv4_prefill_backend):
return self._forward_prefill_sparse_q8kv8(
@@ -158,6 +158,10 @@ class DotsSWAMLAAttnBackend(AttentionBackend):
return self.backend.verify_mask
def shared_read_ends(self, fm: ForwardMode) -> SharedReadEnds:
# An eager prefill may rebuild metadata from the live tables in its first
# layer when DP padding changes the batch.
if fm.is_extend_without_speculative():
return SharedReadEnds.UNKNOWN
return self.backend.shared_read_ends(fm)
def draft_extend_metadata_captured_in_graph(self) -> bool:
@@ -90,6 +90,14 @@ class HybridAttnBackend(AttentionBackend):
def shared_read_ends(self, fm: ForwardMode) -> SharedReadEnds:
return self._select_backend(fm).shared_read_ends(fm)
def _snapshot_prefill_shared_reads(
self, forward_batch: ForwardBatch, *, num_qo_tokens: int
) -> SharedReadEnds:
backend = self._select_backend(forward_batch.forward_mode)
return backend._snapshot_prefill_shared_reads(
forward_batch, num_qo_tokens=num_qo_tokens
)
def get_fused_kv_write_buffers(self, layer, forward_batch):
return self._select_backend(
forward_batch.forward_mode
@@ -1311,10 +1311,6 @@ class HybridLinearAttnBackend(AttentionBackend):
)
elif not forward_batch.forward_mode.is_draft_extend_v2():
self.linear_attn_backend.init_forward_metadata(forward_batch)
if static_forward_batch is not None:
self.prepare_prefill_shared_read_snapshot(
forward_batch, num_qo_tokens=static_forward_batch.positions.shape[0]
)
def get_indexer_metadata(self, layer_id: int, forward_batch: ForwardBatch):
if layer_id in self.full_attn_layers:
@@ -20,7 +20,7 @@ import inspect
import logging
import time
from dataclasses import dataclass
from typing import Callable, List, Optional, Union
from typing import List, Optional, Union
import torch
import torch.distributed as dist
@@ -465,9 +465,6 @@ class ModelRunner:
# Read-done mailbox: the scheduler's WAR barrier reads it from the runner
# its worker names, and treats None as the coarse whole-forward fence.
self.shared_read_done_event: Optional[torch.cuda.Event] = None
# Scoped by a speculative worker to stage its shared reads before
# the target prefill graph publishes the read-done event.
self.prefill_shared_read_stager: Optional[Callable[[ForwardBatch], bool]] = None
# CPU offload
set_offloader(create_offloader())
@@ -321,13 +321,15 @@ class EagerRunner(BaseRunner):
# e.g. Moss-VL's prefill cross-attention custom mask.
model_runner.model.prepare_forward_batch(forward_batch)
model_runner.attn_backend.init_forward_metadata(forward_batch)
model_runner.attn_backend.prepare_prefill_shared_read_snapshot(
forward_batch,
num_qo_tokens=len(forward_batch.input_ids),
shared_read_ends = (
model_runner.attn_backend.resolve_prefill_shared_read_ends(
forward_batch,
num_qo_tokens=len(forward_batch.input_ids),
)
)
maybe_publish_prefill_shared_read_done(
model_runner,
forward_batch,
shared_read_ends,
torch.get_device_module(model_runner.device),
)
@@ -54,6 +54,7 @@ from sglang.kernels.ops.attention.kv_indices import (
)
from sglang.srt.configs.model_config import is_qwen4_exp
from sglang.srt.distributed.parallel_state import graph_capture
from sglang.srt.layers.attention.base_attn_backend import SharedReadEnds
from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp
from sglang.srt.layers.cp.bcg import (
PrefillCPBCGInput,
@@ -1250,7 +1251,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
forward_batch: ForwardBatch,
static_forward_batch: ForwardBatch,
shape_key: ShapeKey,
) -> None:
) -> SharedReadEnds:
"""Replay-time metadata refresh for the BCG-with-captured-metadata
contract. For opt-in backends, refresh the stashed per-bucket
metadata in place against the current batch; otherwise fall back
@@ -1259,6 +1260,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
real seq_lens / prefix_lens; the captured kernels read the
updated state at replay."""
attn_backend = self.model_runner.attn_backend
build_snapshot = False
if self._is_full_backend:
# Slot-padded shallow view: plan() must see exactly req_slots
# entries (real values in [:bs], sentinels in [bs:req_slots]
@@ -1282,19 +1284,22 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
padded_view.max_seq_len_override = static_forward_batch.max_seq_len_override
padded_view.input_ids = static_forward_batch.input_ids
attn_backend.init_forward_metadata_out_graph(padded_view)
return
if not self.use_captured_attn_metadata:
elif not self.use_captured_attn_metadata:
attn_backend.init_forward_metadata(forward_batch)
attn_backend.prepare_prefill_shared_read_snapshot(
forward_batch, num_qo_tokens=shape_key.size
build_snapshot = True
else:
assert self.attn_metadata_buffers is not None
metadata = self.attn_metadata_buffers[shape_key]
attn_backend.prepare_forward_metadata_for_breakable_cuda_graph_replay(
metadata,
forward_batch,
static_forward_batch=static_forward_batch,
)
return
assert self.attn_metadata_buffers is not None
metadata = self.attn_metadata_buffers[shape_key]
attn_backend.prepare_forward_metadata_for_breakable_cuda_graph_replay(
metadata,
return attn_backend.resolve_prefill_shared_read_ends(
forward_batch,
static_forward_batch=static_forward_batch,
num_qo_tokens=shape_key.size,
build_snapshot=build_snapshot,
)
@staticmethod
@@ -1816,10 +1821,10 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
post_warmup_hook=post_warmup_hook,
)
def load_batch(self, forward_batch: ForwardBatch, **kwargs) -> ForwardBatch:
"""Pad, populate static buffers, and build the static_forward_batch
the model code reads during replay.
"""
def load_batch(
self, forward_batch: ForwardBatch, **kwargs
) -> tuple[ForwardBatch, SharedReadEnds]:
"""Pad, fill static buffers, and prepare replay metadata and its read end."""
num_tokens = len(forward_batch.input_ids)
static_num_tokens = self._pad_to_bucket(num_tokens, self.capture_num_tokens)
if getattr(self, "enable_cp_bcg_capture", False) and is_cp_active(
@@ -2100,11 +2105,11 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
metadata_forward_batch = static_forward_batch
shape_key = self._shape_key(static_num_tokens, forward_batch)
self._prepare_forward_metadata_for_replay(
shared_read_ends = self._prepare_forward_metadata_for_replay(
metadata_forward_batch, static_forward_batch, shape_key
)
return static_forward_batch
return static_forward_batch, shared_read_ends
def _fill_input_embeds_slot(self, args, layer_kwargs, static_num_tokens: int):
"""A text-only batch would otherwise replay the captured input_embeds."""
@@ -2312,17 +2317,19 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
) -> Optional[Union[LogitsProcessorOutput, PPProxyTensors, EmbeddingPoolerOutput]]:
self._validate_capture_hidden_mode(forward_batch)
with self.model_context(), self.backend.replay_session():
static_forward_batch = self.load_batch(forward_batch, **kwargs)
static_forward_batch, shared_read_ends = self.load_batch(
forward_batch, **kwargs
)
static_num_tokens = len(static_forward_batch.input_ids)
raw_num_tokens = self.raw_num_tokens
shape_key = self._shape_key(static_num_tokens, forward_batch)
# The only variants this runner records are chunked-prefix ones.
if shape_key.variant_label is not None:
self._prepare_chunked_prefix_replay(shape_key, forward_batch)
# Replay prep, including the optional chunked-prefix gather above,
# has finished every scheduler-shared read.
# Publish after replay prep, including the chunked-prefix gather
# above, which also reads the shared tables.
maybe_publish_prefill_shared_read_done(
self.model_runner, forward_batch, self.device_module
self.model_runner, shared_read_ends, self.device_module
)
self._wait_for_layer_loads()
@@ -5,9 +5,8 @@ from typing import Optional
import torch
from sglang.srt.environ import envs
from sglang.srt.layers.attention.base_attn_backend import SharedReadEnds
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.runtime_context import get_memory
from sglang.srt.utils import is_cuda
logger = logging.getLogger(__name__)
@@ -24,22 +23,22 @@ def make_external_event(device_module) -> Optional[torch.cuda.Event]:
def maybe_publish_prefill_shared_read_done(
model_runner, forward_batch, device_module
model_runner, shared_read_ends: SharedReadEnds, device_module
) -> None:
"""Publish prefill read-done after compliant metadata initialization."""
if not envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.get():
return
if forward_batch.forward_mode != ForwardMode.EXTEND:
return
# The record lands right after replay prep, so PRE_REPLAY only.
declared = model_runner.attn_backend.shared_read_ends(forward_batch.forward_mode)
if declared is not SharedReadEnds.PRE_REPLAY:
"""Publish prefill read-done at the resolved pre-replay boundary."""
if shared_read_ends is not SharedReadEnds.PRE_REPLAY:
return
if not model_runner.spec_algorithm.supports_prefill_shared_read_done():
# Stage the draft's shared reads before publishing the read-done event.
stage = getattr(model_runner, "prefill_shared_read_stager", None)
if stage is None or not stage(forward_batch):
return
return
# Host and storage offload copy a stashed chunk's KV ordered only behind the
# schedule stream, so they need the whole-forward barrier.
memory = get_memory()
if (
memory.enable_hierarchical_cache
or memory.hicache_storage_backend is not None
or memory.enable_lmcache
):
return
logger.info_once(
"Prefill shared-read-done fastpath active (%s)",
type(model_runner.attn_backend).__name__,
+3 -3
View File
@@ -138,10 +138,10 @@ class SpeculativeAlgorithm(Enum):
def supports_prefill_shared_read_done(self) -> bool:
"""Whether target EXTEND has no later speculative shared-buffer reader.
The backend must still declare a pre-replay read end. Other algorithms
must stage their draft's shared reads before publishing the target event.
The backend must still declare a pre-replay read end.
"""
return self.is_none() or self.is_dflash_family()
# DSPARK's draft KV inject reads the SWA mapping after the target prefill.
return self.is_none() or self.is_dflash()
def supports_mixed_chunk(self) -> bool:
"""Whether mixed chunk prefill may stay enabled with this algorithm.