[Perf] prefill-cuda-graph: Make prefill shared-read completion explicit and default-on (#37958)
Co-authored-by: weireweire <20922698+weireweire@users.noreply.github.com> Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com> Co-authored-by: Po-Han Huang <pohanh@nvidia.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: hnyls2002 <lsyincs@gmail.com>
This commit is contained in:
co-authored by
weireweire
Po-Han Huang
Po-Han Huang
Claude Opus 5.5
hnyls2002
parent
d5e83ea4f6
commit
bf145750a6
@@ -669,7 +669,7 @@ class Envs:
|
||||
# read-done fastpath event.
|
||||
SGLANG_FORCE_COARSE_WAR_BARRIER = EnvBool(False)
|
||||
# Enable prefill read-done publication after compliant metadata initialization.
|
||||
SGLANG_ENABLE_PREFILL_WAR_READ_DONE = EnvBool(False)
|
||||
SGLANG_ENABLE_PREFILL_WAR_READ_DONE = EnvBool(True)
|
||||
# PP: skip output send/recv when the entire batch consists of non-final chunked prefill requests,
|
||||
# since process_batch_result_prefill discards next_token_ids for those anyway.
|
||||
SGLANG_PP_SKIP_PURE_CHUNKED_OUTPUT_COMM = EnvBool(False)
|
||||
|
||||
@@ -7,6 +7,8 @@ from typing import TYPE_CHECKING, Iterable, Optional
|
||||
import torch
|
||||
|
||||
from sglang.kernels.kernel_api_logging import debug_kernel_api
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.utils.common import is_npu
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -15,7 +17,7 @@ if TYPE_CHECKING:
|
||||
)
|
||||
from sglang.srt.layers.attention.verify_mask import VerifyMask
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.speculative.spec_info import SpecInput
|
||||
|
||||
|
||||
@@ -183,15 +185,38 @@ class AttentionBackend(ABC):
|
||||
return SharedReadEnds.IN_REPLAY
|
||||
return SharedReadEnds.UNKNOWN
|
||||
|
||||
def prepare_prefill_shared_read_snapshot(
|
||||
self, forward_batch: ForwardBatch, *, num_qo_tokens: int
|
||||
) -> None:
|
||||
"""Snapshot late prefill reads before a PRE_REPLAY event is published.
|
||||
def resolve_prefill_shared_read_ends(
|
||||
self,
|
||||
forward_batch: ForwardBatch,
|
||||
*,
|
||||
num_qo_tokens: int,
|
||||
build_snapshot: bool = True,
|
||||
) -> SharedReadEnds:
|
||||
"""Return this batch's prefill read end, building a snapshot when asked.
|
||||
|
||||
Runners call this only after the actual eager/replay query geometry is
|
||||
known. Backends that retain scheduler-shared reads into the model
|
||||
forward keep the default no-op and must not declare PRE_REPLAY.
|
||||
Spec algorithms with a later draft reader are gated at publish time.
|
||||
"""
|
||||
if (
|
||||
envs.SGLANG_FORCE_COARSE_WAR_BARRIER.get()
|
||||
or not envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.get()
|
||||
):
|
||||
return SharedReadEnds.UNKNOWN
|
||||
if forward_batch.forward_mode != ForwardMode.EXTEND:
|
||||
return SharedReadEnds.UNKNOWN
|
||||
if build_snapshot:
|
||||
return self._snapshot_prefill_shared_reads(
|
||||
forward_batch, num_qo_tokens=num_qo_tokens
|
||||
)
|
||||
return self.shared_read_ends(forward_batch.forward_mode)
|
||||
|
||||
def _snapshot_prefill_shared_reads(
|
||||
self, forward_batch: ForwardBatch, *, num_qo_tokens: int
|
||||
) -> SharedReadEnds:
|
||||
"""Snapshot late shared reads; return PRE_REPLAY only if none remain.
|
||||
|
||||
Hook-skipping paths read ``shared_read_ends(EXTEND)``; keep it conservative.
|
||||
"""
|
||||
return self.shared_read_ends(forward_batch.forward_mode)
|
||||
|
||||
# Chunked-prefix FullCG capture has a second model topology and stable
|
||||
# prefix buffers. Backends must opt in explicitly so the runner does not
|
||||
|
||||
@@ -1056,7 +1056,6 @@ class DSV4Metadata:
|
||||
# Built at the runner's prefill WAR boundary when the fast path is on,
|
||||
# otherwise lazily by ``_forward_prefill_sparse``.
|
||||
sparse_prefill_cache: Optional[SparsePrefillChunkCache] = None
|
||||
prefill_shared_reads_snapshotted: bool = False
|
||||
|
||||
# Set only on the metadata built for the late layers under bounded SWA replay.
|
||||
late_layer_tail: Optional[LateLayerTail] = None
|
||||
@@ -1078,7 +1077,6 @@ class DSV4Metadata:
|
||||
self.c128_compress_metadata, src=other.c128_compress_metadata
|
||||
)
|
||||
self.sparse_prefill_cache = None
|
||||
self.prefill_shared_reads_snapshotted = False
|
||||
|
||||
def refresh_for_breakable_cuda_graph_replay_(
|
||||
self,
|
||||
@@ -1123,7 +1121,6 @@ class DSV4Metadata:
|
||||
src=static_metadata.c128_compress_metadata,
|
||||
)
|
||||
self.sparse_prefill_cache = None
|
||||
self.prefill_shared_reads_snapshotted = False
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -1203,13 +1200,6 @@ class DeepseekV4AttnBackend(
|
||||
if self.model_runner.spec_algorithm.is_dspark():
|
||||
return SharedReadEnds.IN_REPLAY
|
||||
return SharedReadEnds.POST_REPLAY
|
||||
metadata = self.forward_metadata
|
||||
if (
|
||||
fm == ForwardMode.EXTEND
|
||||
and isinstance(metadata, DSV4Metadata)
|
||||
and metadata.prefill_shared_reads_snapshotted
|
||||
):
|
||||
return SharedReadEnds.PRE_REPLAY
|
||||
return super().shared_read_ends(fm)
|
||||
|
||||
def __init__(
|
||||
@@ -2739,47 +2729,46 @@ class DeepseekV4AttnBackend(
|
||||
def _use_sparse_prefill(
|
||||
self, forward_batch: ForwardBatch, *, num_qo_tokens: int
|
||||
) -> bool:
|
||||
# Shared with the snapshot hook, so no forward builds the cache after read-done.
|
||||
# The tail stays dense: its window floor lives in swa_page_indices, which
|
||||
# the chunk cache ignores.
|
||||
return (
|
||||
forward_batch.forward_mode.is_extend_without_speculative()
|
||||
not self.trtllm_attn
|
||||
and forward_batch.forward_mode.is_extend_without_speculative()
|
||||
and not is_cp_active(forward_batch)
|
||||
and not get_platform().is_sm120
|
||||
and self.forward_metadata.late_layer_tail is None
|
||||
and self.token_to_kv_pool.request_window is None
|
||||
and (
|
||||
num_qo_tokens > _LARGE_INDEXER_QUERY_THRESHOLD
|
||||
or envs.SGLANG_OPT_FLASHMLA_SPARSE_PREFILL.get()
|
||||
)
|
||||
)
|
||||
|
||||
def prepare_prefill_shared_read_snapshot(
|
||||
def _snapshot_prefill_shared_reads(
|
||||
self, forward_batch: ForwardBatch, *, num_qo_tokens: int
|
||||
) -> None:
|
||||
) -> SharedReadEnds:
|
||||
# Sparse prefill otherwise reads req_to_token/full_to_swa lazily in its
|
||||
# first layer. DFLASH/DSPARK have no later prefill draft-extend reader;
|
||||
# CP shards the query layout that this global snapshot assumes.
|
||||
metadata = self.forward_metadata
|
||||
# first layer. CP shards the query layout that this global snapshot
|
||||
# assumes; request-window layouts are not covered by the snapshot.
|
||||
if self.token_to_kv_pool.request_window is not None:
|
||||
return
|
||||
if isinstance(metadata, DSV4Metadata):
|
||||
metadata.prefill_shared_reads_snapshotted = False
|
||||
snapshot_shared_prefill_reads = (
|
||||
envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.get()
|
||||
and forward_batch.forward_mode == ForwardMode.EXTEND
|
||||
and self.model_runner.spec_algorithm.is_dflash_family()
|
||||
and not is_cp_active(forward_batch)
|
||||
)
|
||||
if not snapshot_shared_prefill_reads:
|
||||
return
|
||||
return SharedReadEnds.UNKNOWN
|
||||
if is_cp_active(forward_batch):
|
||||
return SharedReadEnds.UNKNOWN
|
||||
# The V4.1 low-ratio indexer reads req_to_token and HiSparse translates
|
||||
# through full_to_hisparse in every layer, after any snapshot.
|
||||
if self.is_dsv41 or self.hisparse_coordinator is not None:
|
||||
return SharedReadEnds.UNKNOWN
|
||||
|
||||
metadata = self.forward_metadata
|
||||
assert isinstance(metadata, DSV4Metadata)
|
||||
# The tail never takes the sparse path, so it carries no chunk cache.
|
||||
if metadata.late_layer_tail is None and self._use_sparse_prefill(
|
||||
forward_batch, num_qo_tokens=num_qo_tokens
|
||||
):
|
||||
if self._use_sparse_prefill(forward_batch, num_qo_tokens=num_qo_tokens):
|
||||
metadata.sparse_prefill_cache = self._build_sparse_prefill_chunk_cache(
|
||||
forward_batch, metadata.core_attn_metadata, num_qo_tokens=num_qo_tokens
|
||||
)
|
||||
# Marked for dense prefill too: that path reads only core_attn_metadata,
|
||||
# which init_forward_metadata already snapshotted.
|
||||
metadata.prefill_shared_reads_snapshotted = True
|
||||
# Dense prefill reads only core_attn_metadata, which metadata init has
|
||||
# already snapshotted, so it reaches the same boundary without a cache.
|
||||
return SharedReadEnds.PRE_REPLAY
|
||||
|
||||
def _build_sparse_prefill_chunk_cache(
|
||||
self,
|
||||
@@ -3821,11 +3810,8 @@ class DeepseekV4AttnBackend(
|
||||
# raw_indices; num_qo_tokens=0 drops the query-count clause, so this errs
|
||||
# to True. Paged attention under active CP needs the page indices.
|
||||
return (
|
||||
self.trtllm_attn
|
||||
or self.mega_attn
|
||||
self.mega_attn
|
||||
or self.tail_forward_metadata is not None
|
||||
or self.forward_metadata.late_layer_tail is not None
|
||||
or self.token_to_kv_pool.request_window is not None
|
||||
or not self._use_sparse_prefill(forward_batch, num_qo_tokens=0)
|
||||
)
|
||||
|
||||
@@ -4065,13 +4051,9 @@ class DeepseekV4AttnBackend(
|
||||
num_heads=q.shape[-2],
|
||||
)
|
||||
|
||||
# The tail stays dense: its window floor lives in swa_page_indices,
|
||||
# which the sparse chunk cache ignores.
|
||||
if (
|
||||
self._use_sparse_prefill(forward_batch, num_qo_tokens=q.shape[0])
|
||||
and not use_mega_paged
|
||||
and self.forward_metadata.late_layer_tail is None
|
||||
and token_to_kv_pool.request_window is None
|
||||
):
|
||||
if use_dsv4_q8kv8_sparse_prefill(self.dsv4_prefill_backend):
|
||||
return self._forward_prefill_sparse_q8kv8(
|
||||
|
||||
@@ -158,6 +158,10 @@ class DotsSWAMLAAttnBackend(AttentionBackend):
|
||||
return self.backend.verify_mask
|
||||
|
||||
def shared_read_ends(self, fm: ForwardMode) -> SharedReadEnds:
|
||||
# An eager prefill may rebuild metadata from the live tables in its first
|
||||
# layer when DP padding changes the batch.
|
||||
if fm.is_extend_without_speculative():
|
||||
return SharedReadEnds.UNKNOWN
|
||||
return self.backend.shared_read_ends(fm)
|
||||
|
||||
def draft_extend_metadata_captured_in_graph(self) -> bool:
|
||||
|
||||
@@ -90,6 +90,14 @@ class HybridAttnBackend(AttentionBackend):
|
||||
def shared_read_ends(self, fm: ForwardMode) -> SharedReadEnds:
|
||||
return self._select_backend(fm).shared_read_ends(fm)
|
||||
|
||||
def _snapshot_prefill_shared_reads(
|
||||
self, forward_batch: ForwardBatch, *, num_qo_tokens: int
|
||||
) -> SharedReadEnds:
|
||||
backend = self._select_backend(forward_batch.forward_mode)
|
||||
return backend._snapshot_prefill_shared_reads(
|
||||
forward_batch, num_qo_tokens=num_qo_tokens
|
||||
)
|
||||
|
||||
def get_fused_kv_write_buffers(self, layer, forward_batch):
|
||||
return self._select_backend(
|
||||
forward_batch.forward_mode
|
||||
|
||||
@@ -1311,10 +1311,6 @@ class HybridLinearAttnBackend(AttentionBackend):
|
||||
)
|
||||
elif not forward_batch.forward_mode.is_draft_extend_v2():
|
||||
self.linear_attn_backend.init_forward_metadata(forward_batch)
|
||||
if static_forward_batch is not None:
|
||||
self.prepare_prefill_shared_read_snapshot(
|
||||
forward_batch, num_qo_tokens=static_forward_batch.positions.shape[0]
|
||||
)
|
||||
|
||||
def get_indexer_metadata(self, layer_id: int, forward_batch: ForwardBatch):
|
||||
if layer_id in self.full_attn_layers:
|
||||
|
||||
@@ -20,7 +20,7 @@ import inspect
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, List, Optional, Union
|
||||
from typing import List, Optional, Union
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
@@ -465,9 +465,6 @@ class ModelRunner:
|
||||
# Read-done mailbox: the scheduler's WAR barrier reads it from the runner
|
||||
# its worker names, and treats None as the coarse whole-forward fence.
|
||||
self.shared_read_done_event: Optional[torch.cuda.Event] = None
|
||||
# Scoped by a speculative worker to stage its shared reads before
|
||||
# the target prefill graph publishes the read-done event.
|
||||
self.prefill_shared_read_stager: Optional[Callable[[ForwardBatch], bool]] = None
|
||||
|
||||
# CPU offload
|
||||
set_offloader(create_offloader())
|
||||
|
||||
@@ -321,13 +321,15 @@ class EagerRunner(BaseRunner):
|
||||
# e.g. Moss-VL's prefill cross-attention custom mask.
|
||||
model_runner.model.prepare_forward_batch(forward_batch)
|
||||
model_runner.attn_backend.init_forward_metadata(forward_batch)
|
||||
model_runner.attn_backend.prepare_prefill_shared_read_snapshot(
|
||||
forward_batch,
|
||||
num_qo_tokens=len(forward_batch.input_ids),
|
||||
shared_read_ends = (
|
||||
model_runner.attn_backend.resolve_prefill_shared_read_ends(
|
||||
forward_batch,
|
||||
num_qo_tokens=len(forward_batch.input_ids),
|
||||
)
|
||||
)
|
||||
maybe_publish_prefill_shared_read_done(
|
||||
model_runner,
|
||||
forward_batch,
|
||||
shared_read_ends,
|
||||
torch.get_device_module(model_runner.device),
|
||||
)
|
||||
|
||||
|
||||
@@ -54,6 +54,7 @@ from sglang.kernels.ops.attention.kv_indices import (
|
||||
)
|
||||
from sglang.srt.configs.model_config import is_qwen4_exp
|
||||
from sglang.srt.distributed.parallel_state import graph_capture
|
||||
from sglang.srt.layers.attention.base_attn_backend import SharedReadEnds
|
||||
from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp
|
||||
from sglang.srt.layers.cp.bcg import (
|
||||
PrefillCPBCGInput,
|
||||
@@ -1250,7 +1251,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
forward_batch: ForwardBatch,
|
||||
static_forward_batch: ForwardBatch,
|
||||
shape_key: ShapeKey,
|
||||
) -> None:
|
||||
) -> SharedReadEnds:
|
||||
"""Replay-time metadata refresh for the BCG-with-captured-metadata
|
||||
contract. For opt-in backends, refresh the stashed per-bucket
|
||||
metadata in place against the current batch; otherwise fall back
|
||||
@@ -1259,6 +1260,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
real seq_lens / prefix_lens; the captured kernels read the
|
||||
updated state at replay."""
|
||||
attn_backend = self.model_runner.attn_backend
|
||||
build_snapshot = False
|
||||
if self._is_full_backend:
|
||||
# Slot-padded shallow view: plan() must see exactly req_slots
|
||||
# entries (real values in [:bs], sentinels in [bs:req_slots]
|
||||
@@ -1282,19 +1284,22 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
padded_view.max_seq_len_override = static_forward_batch.max_seq_len_override
|
||||
padded_view.input_ids = static_forward_batch.input_ids
|
||||
attn_backend.init_forward_metadata_out_graph(padded_view)
|
||||
return
|
||||
if not self.use_captured_attn_metadata:
|
||||
elif not self.use_captured_attn_metadata:
|
||||
attn_backend.init_forward_metadata(forward_batch)
|
||||
attn_backend.prepare_prefill_shared_read_snapshot(
|
||||
forward_batch, num_qo_tokens=shape_key.size
|
||||
build_snapshot = True
|
||||
else:
|
||||
assert self.attn_metadata_buffers is not None
|
||||
metadata = self.attn_metadata_buffers[shape_key]
|
||||
attn_backend.prepare_forward_metadata_for_breakable_cuda_graph_replay(
|
||||
metadata,
|
||||
forward_batch,
|
||||
static_forward_batch=static_forward_batch,
|
||||
)
|
||||
return
|
||||
assert self.attn_metadata_buffers is not None
|
||||
metadata = self.attn_metadata_buffers[shape_key]
|
||||
attn_backend.prepare_forward_metadata_for_breakable_cuda_graph_replay(
|
||||
metadata,
|
||||
|
||||
return attn_backend.resolve_prefill_shared_read_ends(
|
||||
forward_batch,
|
||||
static_forward_batch=static_forward_batch,
|
||||
num_qo_tokens=shape_key.size,
|
||||
build_snapshot=build_snapshot,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -1816,10 +1821,10 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
post_warmup_hook=post_warmup_hook,
|
||||
)
|
||||
|
||||
def load_batch(self, forward_batch: ForwardBatch, **kwargs) -> ForwardBatch:
|
||||
"""Pad, populate static buffers, and build the static_forward_batch
|
||||
the model code reads during replay.
|
||||
"""
|
||||
def load_batch(
|
||||
self, forward_batch: ForwardBatch, **kwargs
|
||||
) -> tuple[ForwardBatch, SharedReadEnds]:
|
||||
"""Pad, fill static buffers, and prepare replay metadata and its read end."""
|
||||
num_tokens = len(forward_batch.input_ids)
|
||||
static_num_tokens = self._pad_to_bucket(num_tokens, self.capture_num_tokens)
|
||||
if getattr(self, "enable_cp_bcg_capture", False) and is_cp_active(
|
||||
@@ -2100,11 +2105,11 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
metadata_forward_batch = static_forward_batch
|
||||
|
||||
shape_key = self._shape_key(static_num_tokens, forward_batch)
|
||||
self._prepare_forward_metadata_for_replay(
|
||||
shared_read_ends = self._prepare_forward_metadata_for_replay(
|
||||
metadata_forward_batch, static_forward_batch, shape_key
|
||||
)
|
||||
|
||||
return static_forward_batch
|
||||
return static_forward_batch, shared_read_ends
|
||||
|
||||
def _fill_input_embeds_slot(self, args, layer_kwargs, static_num_tokens: int):
|
||||
"""A text-only batch would otherwise replay the captured input_embeds."""
|
||||
@@ -2312,17 +2317,19 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
) -> Optional[Union[LogitsProcessorOutput, PPProxyTensors, EmbeddingPoolerOutput]]:
|
||||
self._validate_capture_hidden_mode(forward_batch)
|
||||
with self.model_context(), self.backend.replay_session():
|
||||
static_forward_batch = self.load_batch(forward_batch, **kwargs)
|
||||
static_forward_batch, shared_read_ends = self.load_batch(
|
||||
forward_batch, **kwargs
|
||||
)
|
||||
static_num_tokens = len(static_forward_batch.input_ids)
|
||||
raw_num_tokens = self.raw_num_tokens
|
||||
shape_key = self._shape_key(static_num_tokens, forward_batch)
|
||||
# The only variants this runner records are chunked-prefix ones.
|
||||
if shape_key.variant_label is not None:
|
||||
self._prepare_chunked_prefix_replay(shape_key, forward_batch)
|
||||
# Replay prep, including the optional chunked-prefix gather above,
|
||||
# has finished every scheduler-shared read.
|
||||
# Publish after replay prep, including the chunked-prefix gather
|
||||
# above, which also reads the shared tables.
|
||||
maybe_publish_prefill_shared_read_done(
|
||||
self.model_runner, forward_batch, self.device_module
|
||||
self.model_runner, shared_read_ends, self.device_module
|
||||
)
|
||||
self._wait_for_layer_loads()
|
||||
|
||||
|
||||
@@ -5,9 +5,8 @@ from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.attention.base_attn_backend import SharedReadEnds
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.runtime_context import get_memory
|
||||
from sglang.srt.utils import is_cuda
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -24,22 +23,22 @@ def make_external_event(device_module) -> Optional[torch.cuda.Event]:
|
||||
|
||||
|
||||
def maybe_publish_prefill_shared_read_done(
|
||||
model_runner, forward_batch, device_module
|
||||
model_runner, shared_read_ends: SharedReadEnds, device_module
|
||||
) -> None:
|
||||
"""Publish prefill read-done after compliant metadata initialization."""
|
||||
if not envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.get():
|
||||
return
|
||||
if forward_batch.forward_mode != ForwardMode.EXTEND:
|
||||
return
|
||||
# The record lands right after replay prep, so PRE_REPLAY only.
|
||||
declared = model_runner.attn_backend.shared_read_ends(forward_batch.forward_mode)
|
||||
if declared is not SharedReadEnds.PRE_REPLAY:
|
||||
"""Publish prefill read-done at the resolved pre-replay boundary."""
|
||||
if shared_read_ends is not SharedReadEnds.PRE_REPLAY:
|
||||
return
|
||||
if not model_runner.spec_algorithm.supports_prefill_shared_read_done():
|
||||
# Stage the draft's shared reads before publishing the read-done event.
|
||||
stage = getattr(model_runner, "prefill_shared_read_stager", None)
|
||||
if stage is None or not stage(forward_batch):
|
||||
return
|
||||
return
|
||||
# Host and storage offload copy a stashed chunk's KV ordered only behind the
|
||||
# schedule stream, so they need the whole-forward barrier.
|
||||
memory = get_memory()
|
||||
if (
|
||||
memory.enable_hierarchical_cache
|
||||
or memory.hicache_storage_backend is not None
|
||||
or memory.enable_lmcache
|
||||
):
|
||||
return
|
||||
logger.info_once(
|
||||
"Prefill shared-read-done fastpath active (%s)",
|
||||
type(model_runner.attn_backend).__name__,
|
||||
|
||||
@@ -138,10 +138,10 @@ class SpeculativeAlgorithm(Enum):
|
||||
def supports_prefill_shared_read_done(self) -> bool:
|
||||
"""Whether target EXTEND has no later speculative shared-buffer reader.
|
||||
|
||||
The backend must still declare a pre-replay read end. Other algorithms
|
||||
must stage their draft's shared reads before publishing the target event.
|
||||
The backend must still declare a pre-replay read end.
|
||||
"""
|
||||
return self.is_none() or self.is_dflash_family()
|
||||
# DSPARK's draft KV inject reads the SWA mapping after the target prefill.
|
||||
return self.is_none() or self.is_dflash()
|
||||
|
||||
def supports_mixed_chunk(self) -> bool:
|
||||
"""Whether mixed chunk prefill may stay enabled with this algorithm.
|
||||
|
||||
Reference in New Issue
Block a user