[Perf] prefill-cuda-graph: Make prefill shared-read completion explicit and default-on (#37958)
Co-authored-by: weireweire <20922698+weireweire@users.noreply.github.com> Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com> Co-authored-by: Po-Han Huang <pohanh@nvidia.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: hnyls2002 <lsyincs@gmail.com>
This commit is contained in:
co-authored by
weireweire
Po-Han Huang
Po-Han Huang
Claude Opus 5.5
hnyls2002
parent
d5e83ea4f6
commit
bf145750a6
@@ -669,7 +669,7 @@ class Envs:
|
||||
# read-done fastpath event.
|
||||
SGLANG_FORCE_COARSE_WAR_BARRIER = EnvBool(False)
|
||||
# Enable prefill read-done publication after compliant metadata initialization.
|
||||
SGLANG_ENABLE_PREFILL_WAR_READ_DONE = EnvBool(False)
|
||||
SGLANG_ENABLE_PREFILL_WAR_READ_DONE = EnvBool(True)
|
||||
# PP: skip output send/recv when the entire batch consists of non-final chunked prefill requests,
|
||||
# since process_batch_result_prefill discards next_token_ids for those anyway.
|
||||
SGLANG_PP_SKIP_PURE_CHUNKED_OUTPUT_COMM = EnvBool(False)
|
||||
|
||||
@@ -7,6 +7,8 @@ from typing import TYPE_CHECKING, Iterable, Optional
|
||||
import torch
|
||||
|
||||
from sglang.kernels.kernel_api_logging import debug_kernel_api
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.utils.common import is_npu
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -15,7 +17,7 @@ if TYPE_CHECKING:
|
||||
)
|
||||
from sglang.srt.layers.attention.verify_mask import VerifyMask
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.speculative.spec_info import SpecInput
|
||||
|
||||
|
||||
@@ -183,15 +185,38 @@ class AttentionBackend(ABC):
|
||||
return SharedReadEnds.IN_REPLAY
|
||||
return SharedReadEnds.UNKNOWN
|
||||
|
||||
def prepare_prefill_shared_read_snapshot(
|
||||
self, forward_batch: ForwardBatch, *, num_qo_tokens: int
|
||||
) -> None:
|
||||
"""Snapshot late prefill reads before a PRE_REPLAY event is published.
|
||||
def resolve_prefill_shared_read_ends(
|
||||
self,
|
||||
forward_batch: ForwardBatch,
|
||||
*,
|
||||
num_qo_tokens: int,
|
||||
build_snapshot: bool = True,
|
||||
) -> SharedReadEnds:
|
||||
"""Return this batch's prefill read end, building a snapshot when asked.
|
||||
|
||||
Runners call this only after the actual eager/replay query geometry is
|
||||
known. Backends that retain scheduler-shared reads into the model
|
||||
forward keep the default no-op and must not declare PRE_REPLAY.
|
||||
Spec algorithms with a later draft reader are gated at publish time.
|
||||
"""
|
||||
if (
|
||||
envs.SGLANG_FORCE_COARSE_WAR_BARRIER.get()
|
||||
or not envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.get()
|
||||
):
|
||||
return SharedReadEnds.UNKNOWN
|
||||
if forward_batch.forward_mode != ForwardMode.EXTEND:
|
||||
return SharedReadEnds.UNKNOWN
|
||||
if build_snapshot:
|
||||
return self._snapshot_prefill_shared_reads(
|
||||
forward_batch, num_qo_tokens=num_qo_tokens
|
||||
)
|
||||
return self.shared_read_ends(forward_batch.forward_mode)
|
||||
|
||||
def _snapshot_prefill_shared_reads(
|
||||
self, forward_batch: ForwardBatch, *, num_qo_tokens: int
|
||||
) -> SharedReadEnds:
|
||||
"""Snapshot late shared reads; return PRE_REPLAY only if none remain.
|
||||
|
||||
Hook-skipping paths read ``shared_read_ends(EXTEND)``; keep it conservative.
|
||||
"""
|
||||
return self.shared_read_ends(forward_batch.forward_mode)
|
||||
|
||||
# Chunked-prefix FullCG capture has a second model topology and stable
|
||||
# prefix buffers. Backends must opt in explicitly so the runner does not
|
||||
|
||||
@@ -1056,7 +1056,6 @@ class DSV4Metadata:
|
||||
# Built at the runner's prefill WAR boundary when the fast path is on,
|
||||
# otherwise lazily by ``_forward_prefill_sparse``.
|
||||
sparse_prefill_cache: Optional[SparsePrefillChunkCache] = None
|
||||
prefill_shared_reads_snapshotted: bool = False
|
||||
|
||||
# Set only on the metadata built for the late layers under bounded SWA replay.
|
||||
late_layer_tail: Optional[LateLayerTail] = None
|
||||
@@ -1078,7 +1077,6 @@ class DSV4Metadata:
|
||||
self.c128_compress_metadata, src=other.c128_compress_metadata
|
||||
)
|
||||
self.sparse_prefill_cache = None
|
||||
self.prefill_shared_reads_snapshotted = False
|
||||
|
||||
def refresh_for_breakable_cuda_graph_replay_(
|
||||
self,
|
||||
@@ -1123,7 +1121,6 @@ class DSV4Metadata:
|
||||
src=static_metadata.c128_compress_metadata,
|
||||
)
|
||||
self.sparse_prefill_cache = None
|
||||
self.prefill_shared_reads_snapshotted = False
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -1203,13 +1200,6 @@ class DeepseekV4AttnBackend(
|
||||
if self.model_runner.spec_algorithm.is_dspark():
|
||||
return SharedReadEnds.IN_REPLAY
|
||||
return SharedReadEnds.POST_REPLAY
|
||||
metadata = self.forward_metadata
|
||||
if (
|
||||
fm == ForwardMode.EXTEND
|
||||
and isinstance(metadata, DSV4Metadata)
|
||||
and metadata.prefill_shared_reads_snapshotted
|
||||
):
|
||||
return SharedReadEnds.PRE_REPLAY
|
||||
return super().shared_read_ends(fm)
|
||||
|
||||
def __init__(
|
||||
@@ -2739,47 +2729,46 @@ class DeepseekV4AttnBackend(
|
||||
def _use_sparse_prefill(
|
||||
self, forward_batch: ForwardBatch, *, num_qo_tokens: int
|
||||
) -> bool:
|
||||
# Shared with the snapshot hook, so no forward builds the cache after read-done.
|
||||
# The tail stays dense: its window floor lives in swa_page_indices, which
|
||||
# the chunk cache ignores.
|
||||
return (
|
||||
forward_batch.forward_mode.is_extend_without_speculative()
|
||||
not self.trtllm_attn
|
||||
and forward_batch.forward_mode.is_extend_without_speculative()
|
||||
and not is_cp_active(forward_batch)
|
||||
and not get_platform().is_sm120
|
||||
and self.forward_metadata.late_layer_tail is None
|
||||
and self.token_to_kv_pool.request_window is None
|
||||
and (
|
||||
num_qo_tokens > _LARGE_INDEXER_QUERY_THRESHOLD
|
||||
or envs.SGLANG_OPT_FLASHMLA_SPARSE_PREFILL.get()
|
||||
)
|
||||
)
|
||||
|
||||
def prepare_prefill_shared_read_snapshot(
|
||||
def _snapshot_prefill_shared_reads(
|
||||
self, forward_batch: ForwardBatch, *, num_qo_tokens: int
|
||||
) -> None:
|
||||
) -> SharedReadEnds:
|
||||
# Sparse prefill otherwise reads req_to_token/full_to_swa lazily in its
|
||||
# first layer. DFLASH/DSPARK have no later prefill draft-extend reader;
|
||||
# CP shards the query layout that this global snapshot assumes.
|
||||
metadata = self.forward_metadata
|
||||
# first layer. CP shards the query layout that this global snapshot
|
||||
# assumes; request-window layouts are not covered by the snapshot.
|
||||
if self.token_to_kv_pool.request_window is not None:
|
||||
return
|
||||
if isinstance(metadata, DSV4Metadata):
|
||||
metadata.prefill_shared_reads_snapshotted = False
|
||||
snapshot_shared_prefill_reads = (
|
||||
envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.get()
|
||||
and forward_batch.forward_mode == ForwardMode.EXTEND
|
||||
and self.model_runner.spec_algorithm.is_dflash_family()
|
||||
and not is_cp_active(forward_batch)
|
||||
)
|
||||
if not snapshot_shared_prefill_reads:
|
||||
return
|
||||
return SharedReadEnds.UNKNOWN
|
||||
if is_cp_active(forward_batch):
|
||||
return SharedReadEnds.UNKNOWN
|
||||
# The V4.1 low-ratio indexer reads req_to_token and HiSparse translates
|
||||
# through full_to_hisparse in every layer, after any snapshot.
|
||||
if self.is_dsv41 or self.hisparse_coordinator is not None:
|
||||
return SharedReadEnds.UNKNOWN
|
||||
|
||||
metadata = self.forward_metadata
|
||||
assert isinstance(metadata, DSV4Metadata)
|
||||
# The tail never takes the sparse path, so it carries no chunk cache.
|
||||
if metadata.late_layer_tail is None and self._use_sparse_prefill(
|
||||
forward_batch, num_qo_tokens=num_qo_tokens
|
||||
):
|
||||
if self._use_sparse_prefill(forward_batch, num_qo_tokens=num_qo_tokens):
|
||||
metadata.sparse_prefill_cache = self._build_sparse_prefill_chunk_cache(
|
||||
forward_batch, metadata.core_attn_metadata, num_qo_tokens=num_qo_tokens
|
||||
)
|
||||
# Marked for dense prefill too: that path reads only core_attn_metadata,
|
||||
# which init_forward_metadata already snapshotted.
|
||||
metadata.prefill_shared_reads_snapshotted = True
|
||||
# Dense prefill reads only core_attn_metadata, which metadata init has
|
||||
# already snapshotted, so it reaches the same boundary without a cache.
|
||||
return SharedReadEnds.PRE_REPLAY
|
||||
|
||||
def _build_sparse_prefill_chunk_cache(
|
||||
self,
|
||||
@@ -3821,11 +3810,8 @@ class DeepseekV4AttnBackend(
|
||||
# raw_indices; num_qo_tokens=0 drops the query-count clause, so this errs
|
||||
# to True. Paged attention under active CP needs the page indices.
|
||||
return (
|
||||
self.trtllm_attn
|
||||
or self.mega_attn
|
||||
self.mega_attn
|
||||
or self.tail_forward_metadata is not None
|
||||
or self.forward_metadata.late_layer_tail is not None
|
||||
or self.token_to_kv_pool.request_window is not None
|
||||
or not self._use_sparse_prefill(forward_batch, num_qo_tokens=0)
|
||||
)
|
||||
|
||||
@@ -4065,13 +4051,9 @@ class DeepseekV4AttnBackend(
|
||||
num_heads=q.shape[-2],
|
||||
)
|
||||
|
||||
# The tail stays dense: its window floor lives in swa_page_indices,
|
||||
# which the sparse chunk cache ignores.
|
||||
if (
|
||||
self._use_sparse_prefill(forward_batch, num_qo_tokens=q.shape[0])
|
||||
and not use_mega_paged
|
||||
and self.forward_metadata.late_layer_tail is None
|
||||
and token_to_kv_pool.request_window is None
|
||||
):
|
||||
if use_dsv4_q8kv8_sparse_prefill(self.dsv4_prefill_backend):
|
||||
return self._forward_prefill_sparse_q8kv8(
|
||||
|
||||
@@ -158,6 +158,10 @@ class DotsSWAMLAAttnBackend(AttentionBackend):
|
||||
return self.backend.verify_mask
|
||||
|
||||
def shared_read_ends(self, fm: ForwardMode) -> SharedReadEnds:
|
||||
# An eager prefill may rebuild metadata from the live tables in its first
|
||||
# layer when DP padding changes the batch.
|
||||
if fm.is_extend_without_speculative():
|
||||
return SharedReadEnds.UNKNOWN
|
||||
return self.backend.shared_read_ends(fm)
|
||||
|
||||
def draft_extend_metadata_captured_in_graph(self) -> bool:
|
||||
|
||||
@@ -90,6 +90,14 @@ class HybridAttnBackend(AttentionBackend):
|
||||
def shared_read_ends(self, fm: ForwardMode) -> SharedReadEnds:
|
||||
return self._select_backend(fm).shared_read_ends(fm)
|
||||
|
||||
def _snapshot_prefill_shared_reads(
|
||||
self, forward_batch: ForwardBatch, *, num_qo_tokens: int
|
||||
) -> SharedReadEnds:
|
||||
backend = self._select_backend(forward_batch.forward_mode)
|
||||
return backend._snapshot_prefill_shared_reads(
|
||||
forward_batch, num_qo_tokens=num_qo_tokens
|
||||
)
|
||||
|
||||
def get_fused_kv_write_buffers(self, layer, forward_batch):
|
||||
return self._select_backend(
|
||||
forward_batch.forward_mode
|
||||
|
||||
@@ -1311,10 +1311,6 @@ class HybridLinearAttnBackend(AttentionBackend):
|
||||
)
|
||||
elif not forward_batch.forward_mode.is_draft_extend_v2():
|
||||
self.linear_attn_backend.init_forward_metadata(forward_batch)
|
||||
if static_forward_batch is not None:
|
||||
self.prepare_prefill_shared_read_snapshot(
|
||||
forward_batch, num_qo_tokens=static_forward_batch.positions.shape[0]
|
||||
)
|
||||
|
||||
def get_indexer_metadata(self, layer_id: int, forward_batch: ForwardBatch):
|
||||
if layer_id in self.full_attn_layers:
|
||||
|
||||
@@ -20,7 +20,7 @@ import inspect
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, List, Optional, Union
|
||||
from typing import List, Optional, Union
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
@@ -465,9 +465,6 @@ class ModelRunner:
|
||||
# Read-done mailbox: the scheduler's WAR barrier reads it from the runner
|
||||
# its worker names, and treats None as the coarse whole-forward fence.
|
||||
self.shared_read_done_event: Optional[torch.cuda.Event] = None
|
||||
# Scoped by a speculative worker to stage its shared reads before
|
||||
# the target prefill graph publishes the read-done event.
|
||||
self.prefill_shared_read_stager: Optional[Callable[[ForwardBatch], bool]] = None
|
||||
|
||||
# CPU offload
|
||||
set_offloader(create_offloader())
|
||||
|
||||
@@ -321,13 +321,15 @@ class EagerRunner(BaseRunner):
|
||||
# e.g. Moss-VL's prefill cross-attention custom mask.
|
||||
model_runner.model.prepare_forward_batch(forward_batch)
|
||||
model_runner.attn_backend.init_forward_metadata(forward_batch)
|
||||
model_runner.attn_backend.prepare_prefill_shared_read_snapshot(
|
||||
forward_batch,
|
||||
num_qo_tokens=len(forward_batch.input_ids),
|
||||
shared_read_ends = (
|
||||
model_runner.attn_backend.resolve_prefill_shared_read_ends(
|
||||
forward_batch,
|
||||
num_qo_tokens=len(forward_batch.input_ids),
|
||||
)
|
||||
)
|
||||
maybe_publish_prefill_shared_read_done(
|
||||
model_runner,
|
||||
forward_batch,
|
||||
shared_read_ends,
|
||||
torch.get_device_module(model_runner.device),
|
||||
)
|
||||
|
||||
|
||||
@@ -54,6 +54,7 @@ from sglang.kernels.ops.attention.kv_indices import (
|
||||
)
|
||||
from sglang.srt.configs.model_config import is_qwen4_exp
|
||||
from sglang.srt.distributed.parallel_state import graph_capture
|
||||
from sglang.srt.layers.attention.base_attn_backend import SharedReadEnds
|
||||
from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp
|
||||
from sglang.srt.layers.cp.bcg import (
|
||||
PrefillCPBCGInput,
|
||||
@@ -1250,7 +1251,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
forward_batch: ForwardBatch,
|
||||
static_forward_batch: ForwardBatch,
|
||||
shape_key: ShapeKey,
|
||||
) -> None:
|
||||
) -> SharedReadEnds:
|
||||
"""Replay-time metadata refresh for the BCG-with-captured-metadata
|
||||
contract. For opt-in backends, refresh the stashed per-bucket
|
||||
metadata in place against the current batch; otherwise fall back
|
||||
@@ -1259,6 +1260,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
real seq_lens / prefix_lens; the captured kernels read the
|
||||
updated state at replay."""
|
||||
attn_backend = self.model_runner.attn_backend
|
||||
build_snapshot = False
|
||||
if self._is_full_backend:
|
||||
# Slot-padded shallow view: plan() must see exactly req_slots
|
||||
# entries (real values in [:bs], sentinels in [bs:req_slots]
|
||||
@@ -1282,19 +1284,22 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
padded_view.max_seq_len_override = static_forward_batch.max_seq_len_override
|
||||
padded_view.input_ids = static_forward_batch.input_ids
|
||||
attn_backend.init_forward_metadata_out_graph(padded_view)
|
||||
return
|
||||
if not self.use_captured_attn_metadata:
|
||||
elif not self.use_captured_attn_metadata:
|
||||
attn_backend.init_forward_metadata(forward_batch)
|
||||
attn_backend.prepare_prefill_shared_read_snapshot(
|
||||
forward_batch, num_qo_tokens=shape_key.size
|
||||
build_snapshot = True
|
||||
else:
|
||||
assert self.attn_metadata_buffers is not None
|
||||
metadata = self.attn_metadata_buffers[shape_key]
|
||||
attn_backend.prepare_forward_metadata_for_breakable_cuda_graph_replay(
|
||||
metadata,
|
||||
forward_batch,
|
||||
static_forward_batch=static_forward_batch,
|
||||
)
|
||||
return
|
||||
assert self.attn_metadata_buffers is not None
|
||||
metadata = self.attn_metadata_buffers[shape_key]
|
||||
attn_backend.prepare_forward_metadata_for_breakable_cuda_graph_replay(
|
||||
metadata,
|
||||
|
||||
return attn_backend.resolve_prefill_shared_read_ends(
|
||||
forward_batch,
|
||||
static_forward_batch=static_forward_batch,
|
||||
num_qo_tokens=shape_key.size,
|
||||
build_snapshot=build_snapshot,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -1816,10 +1821,10 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
post_warmup_hook=post_warmup_hook,
|
||||
)
|
||||
|
||||
def load_batch(self, forward_batch: ForwardBatch, **kwargs) -> ForwardBatch:
|
||||
"""Pad, populate static buffers, and build the static_forward_batch
|
||||
the model code reads during replay.
|
||||
"""
|
||||
def load_batch(
|
||||
self, forward_batch: ForwardBatch, **kwargs
|
||||
) -> tuple[ForwardBatch, SharedReadEnds]:
|
||||
"""Pad, fill static buffers, and prepare replay metadata and its read end."""
|
||||
num_tokens = len(forward_batch.input_ids)
|
||||
static_num_tokens = self._pad_to_bucket(num_tokens, self.capture_num_tokens)
|
||||
if getattr(self, "enable_cp_bcg_capture", False) and is_cp_active(
|
||||
@@ -2100,11 +2105,11 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
metadata_forward_batch = static_forward_batch
|
||||
|
||||
shape_key = self._shape_key(static_num_tokens, forward_batch)
|
||||
self._prepare_forward_metadata_for_replay(
|
||||
shared_read_ends = self._prepare_forward_metadata_for_replay(
|
||||
metadata_forward_batch, static_forward_batch, shape_key
|
||||
)
|
||||
|
||||
return static_forward_batch
|
||||
return static_forward_batch, shared_read_ends
|
||||
|
||||
def _fill_input_embeds_slot(self, args, layer_kwargs, static_num_tokens: int):
|
||||
"""A text-only batch would otherwise replay the captured input_embeds."""
|
||||
@@ -2312,17 +2317,19 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
) -> Optional[Union[LogitsProcessorOutput, PPProxyTensors, EmbeddingPoolerOutput]]:
|
||||
self._validate_capture_hidden_mode(forward_batch)
|
||||
with self.model_context(), self.backend.replay_session():
|
||||
static_forward_batch = self.load_batch(forward_batch, **kwargs)
|
||||
static_forward_batch, shared_read_ends = self.load_batch(
|
||||
forward_batch, **kwargs
|
||||
)
|
||||
static_num_tokens = len(static_forward_batch.input_ids)
|
||||
raw_num_tokens = self.raw_num_tokens
|
||||
shape_key = self._shape_key(static_num_tokens, forward_batch)
|
||||
# The only variants this runner records are chunked-prefix ones.
|
||||
if shape_key.variant_label is not None:
|
||||
self._prepare_chunked_prefix_replay(shape_key, forward_batch)
|
||||
# Replay prep, including the optional chunked-prefix gather above,
|
||||
# has finished every scheduler-shared read.
|
||||
# Publish after replay prep, including the chunked-prefix gather
|
||||
# above, which also reads the shared tables.
|
||||
maybe_publish_prefill_shared_read_done(
|
||||
self.model_runner, forward_batch, self.device_module
|
||||
self.model_runner, shared_read_ends, self.device_module
|
||||
)
|
||||
self._wait_for_layer_loads()
|
||||
|
||||
|
||||
@@ -5,9 +5,8 @@ from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.attention.base_attn_backend import SharedReadEnds
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.runtime_context import get_memory
|
||||
from sglang.srt.utils import is_cuda
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -24,22 +23,22 @@ def make_external_event(device_module) -> Optional[torch.cuda.Event]:
|
||||
|
||||
|
||||
def maybe_publish_prefill_shared_read_done(
|
||||
model_runner, forward_batch, device_module
|
||||
model_runner, shared_read_ends: SharedReadEnds, device_module
|
||||
) -> None:
|
||||
"""Publish prefill read-done after compliant metadata initialization."""
|
||||
if not envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.get():
|
||||
return
|
||||
if forward_batch.forward_mode != ForwardMode.EXTEND:
|
||||
return
|
||||
# The record lands right after replay prep, so PRE_REPLAY only.
|
||||
declared = model_runner.attn_backend.shared_read_ends(forward_batch.forward_mode)
|
||||
if declared is not SharedReadEnds.PRE_REPLAY:
|
||||
"""Publish prefill read-done at the resolved pre-replay boundary."""
|
||||
if shared_read_ends is not SharedReadEnds.PRE_REPLAY:
|
||||
return
|
||||
if not model_runner.spec_algorithm.supports_prefill_shared_read_done():
|
||||
# Stage the draft's shared reads before publishing the read-done event.
|
||||
stage = getattr(model_runner, "prefill_shared_read_stager", None)
|
||||
if stage is None or not stage(forward_batch):
|
||||
return
|
||||
return
|
||||
# Host and storage offload copy a stashed chunk's KV ordered only behind the
|
||||
# schedule stream, so they need the whole-forward barrier.
|
||||
memory = get_memory()
|
||||
if (
|
||||
memory.enable_hierarchical_cache
|
||||
or memory.hicache_storage_backend is not None
|
||||
or memory.enable_lmcache
|
||||
):
|
||||
return
|
||||
logger.info_once(
|
||||
"Prefill shared-read-done fastpath active (%s)",
|
||||
type(model_runner.attn_backend).__name__,
|
||||
|
||||
@@ -138,10 +138,10 @@ class SpeculativeAlgorithm(Enum):
|
||||
def supports_prefill_shared_read_done(self) -> bool:
|
||||
"""Whether target EXTEND has no later speculative shared-buffer reader.
|
||||
|
||||
The backend must still declare a pre-replay read end. Other algorithms
|
||||
must stage their draft's shared reads before publishing the target event.
|
||||
The backend must still declare a pre-replay read end.
|
||||
"""
|
||||
return self.is_none() or self.is_dflash_family()
|
||||
# DSPARK's draft KV inject reads the SWA mapping after the target prefill.
|
||||
return self.is_none() or self.is_dflash()
|
||||
|
||||
def supports_mixed_chunk(self) -> bool:
|
||||
"""Whether mixed chunk prefill may stay enabled with this algorithm.
|
||||
|
||||
@@ -511,115 +511,90 @@ class TestDSV4BreakableCudaGraphMetadataContract(CustomTestCase):
|
||||
DeepseekV4AttnBackend.use_captured_forward_metadata_for_breakable_cuda_graph
|
||||
)
|
||||
|
||||
def test_prefill_snapshot_declares_pre_replay_boundary(self):
|
||||
from sglang.srt.layers.attention.base_attn_backend import SharedReadEnds
|
||||
def _make_prefill_backend(self, *, is_dsv41=False, hisparse_coordinator=None):
|
||||
from sglang.srt.layers.attention.deepseek_v4_backend import (
|
||||
DeepseekV4AttnBackend,
|
||||
DSV4Metadata,
|
||||
)
|
||||
|
||||
backend = object.__new__(DeepseekV4AttnBackend)
|
||||
backend.token_to_kv_pool = SimpleNamespace(request_window=None)
|
||||
backend.is_dsv41 = is_dsv41
|
||||
backend.hisparse_coordinator = hisparse_coordinator
|
||||
backend.forward_metadata = DSV4Metadata(
|
||||
self._make_core_metadata(0), indexer_metadata=None
|
||||
)
|
||||
self.assertIs(
|
||||
backend.shared_read_ends(ForwardMode.EXTEND),
|
||||
SharedReadEnds.UNKNOWN,
|
||||
)
|
||||
backend._build_sparse_prefill_chunk_cache = mock.Mock(return_value=object())
|
||||
return backend
|
||||
|
||||
backend.forward_metadata.prefill_shared_reads_snapshotted = True
|
||||
def test_static_extend_read_end_stays_unknown(self):
|
||||
from sglang.srt.layers.attention.base_attn_backend import SharedReadEnds
|
||||
|
||||
# Paths that skip the snapshot hook publish from this declaration.
|
||||
backend = self._make_prefill_backend()
|
||||
self.assertIs(
|
||||
backend.shared_read_ends(ForwardMode.EXTEND),
|
||||
SharedReadEnds.PRE_REPLAY,
|
||||
backend.shared_read_ends(ForwardMode.EXTEND), SharedReadEnds.UNKNOWN
|
||||
)
|
||||
|
||||
def test_snapshot_builds_cache_only_for_sparse_prefill(self):
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.attention.base_attn_backend import SharedReadEnds
|
||||
from sglang.srt.layers.attention.deepseek_v4_backend import (
|
||||
_LARGE_INDEXER_QUERY_THRESHOLD,
|
||||
DeepseekV4AttnBackend,
|
||||
DSV4Metadata,
|
||||
)
|
||||
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||
|
||||
batch = SimpleNamespace(forward_mode=ForwardMode.EXTEND)
|
||||
cache = object()
|
||||
for num_qo_tokens, builds in (
|
||||
(_LARGE_INDEXER_QUERY_THRESHOLD, False),
|
||||
(_LARGE_INDEXER_QUERY_THRESHOLD + 1, True),
|
||||
):
|
||||
with self.subTest(num_qo_tokens=num_qo_tokens):
|
||||
backend = object.__new__(DeepseekV4AttnBackend)
|
||||
backend.model_runner = SimpleNamespace(
|
||||
spec_algorithm=SpeculativeAlgorithm.DFLASH
|
||||
)
|
||||
backend.token_to_kv_pool = SimpleNamespace(request_window=None)
|
||||
backend.forward_metadata = DSV4Metadata(
|
||||
self._make_core_metadata(0), indexer_metadata=None
|
||||
)
|
||||
backend._build_sparse_prefill_chunk_cache = mock.Mock(
|
||||
return_value=cache
|
||||
)
|
||||
backend = self._make_prefill_backend()
|
||||
build = backend._build_sparse_prefill_chunk_cache
|
||||
|
||||
with (
|
||||
envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.override(True),
|
||||
envs.SGLANG_OPT_FLASHMLA_SPARSE_PREFILL.override(False),
|
||||
mock.patch(
|
||||
"sglang.srt.layers.attention.deepseek_v4_backend.get_platform",
|
||||
return_value=SimpleNamespace(is_sm120=False),
|
||||
),
|
||||
):
|
||||
backend.prepare_prefill_shared_read_snapshot(
|
||||
batch, num_qo_tokens=num_qo_tokens
|
||||
shared_read_ends = backend.resolve_prefill_shared_read_ends(
|
||||
batch,
|
||||
num_qo_tokens=num_qo_tokens,
|
||||
)
|
||||
|
||||
metadata = backend.forward_metadata
|
||||
if builds:
|
||||
backend._build_sparse_prefill_chunk_cache.assert_called_once_with(
|
||||
build.assert_called_once_with(
|
||||
batch, metadata.core_attn_metadata, num_qo_tokens=num_qo_tokens
|
||||
)
|
||||
self.assertIs(metadata.sparse_prefill_cache, cache)
|
||||
self.assertIs(metadata.sparse_prefill_cache, build.return_value)
|
||||
else:
|
||||
backend._build_sparse_prefill_chunk_cache.assert_not_called()
|
||||
build.assert_not_called()
|
||||
self.assertIsNone(metadata.sparse_prefill_cache)
|
||||
# Dense declares the boundary too; it reads only the metadata
|
||||
# that init_forward_metadata already snapshotted.
|
||||
self.assertTrue(metadata.prefill_shared_reads_snapshotted)
|
||||
# Dense reaches the same boundary using metadata init's snapshot.
|
||||
self.assertIs(shared_read_ends, SharedReadEnds.PRE_REPLAY)
|
||||
|
||||
def test_sparse_prefill_snapshot_marks_success_only_after_build(self):
|
||||
from sglang.srt.environ import envs
|
||||
def test_prefill_snapshot_skips_layers_reading_live_tables(self):
|
||||
from sglang.srt.layers.attention.base_attn_backend import SharedReadEnds
|
||||
from sglang.srt.layers.attention.deepseek_v4_backend import (
|
||||
DeepseekV4AttnBackend,
|
||||
DSV4Metadata,
|
||||
_LARGE_INDEXER_QUERY_THRESHOLD,
|
||||
)
|
||||
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||
|
||||
backend = object.__new__(DeepseekV4AttnBackend)
|
||||
backend.model_runner = SimpleNamespace(
|
||||
spec_algorithm=SpeculativeAlgorithm.DFLASH
|
||||
)
|
||||
backend.token_to_kv_pool = SimpleNamespace(request_window=None)
|
||||
backend.forward_metadata = DSV4Metadata(
|
||||
self._make_core_metadata(0), indexer_metadata=None
|
||||
)
|
||||
backend.forward_metadata.prefill_shared_reads_snapshotted = True
|
||||
backend._build_sparse_prefill_chunk_cache = mock.Mock(
|
||||
side_effect=RuntimeError("snapshot failed")
|
||||
)
|
||||
batch = SimpleNamespace(forward_mode=ForwardMode.EXTEND)
|
||||
for is_dsv41, hisparse_coordinator in ((True, None), (False, object())):
|
||||
with self.subTest(is_dsv41=is_dsv41, hisparse=hisparse_coordinator):
|
||||
backend = self._make_prefill_backend(
|
||||
is_dsv41=is_dsv41, hisparse_coordinator=hisparse_coordinator
|
||||
)
|
||||
|
||||
with (
|
||||
envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.override(True),
|
||||
envs.SGLANG_OPT_FLASHMLA_SPARSE_PREFILL.override(True),
|
||||
mock.patch(
|
||||
"sglang.srt.layers.attention.deepseek_v4_backend.get_platform",
|
||||
return_value=SimpleNamespace(is_sm120=False),
|
||||
),
|
||||
self.assertRaisesRegex(RuntimeError, "snapshot failed"),
|
||||
):
|
||||
backend.prepare_prefill_shared_read_snapshot(batch, num_qo_tokens=12288)
|
||||
shared_read_ends = backend.resolve_prefill_shared_read_ends(
|
||||
batch, num_qo_tokens=_LARGE_INDEXER_QUERY_THRESHOLD + 1
|
||||
)
|
||||
|
||||
self.assertFalse(backend.forward_metadata.prefill_shared_reads_snapshotted)
|
||||
self.assertIs(shared_read_ends, SharedReadEnds.UNKNOWN)
|
||||
backend._build_sparse_prefill_chunk_cache.assert_not_called()
|
||||
|
||||
def test_refresh_replay_metadata_preserves_captured_tensor_storage(self):
|
||||
capture_metadata = self._make_core_metadata(0)
|
||||
@@ -692,7 +667,6 @@ class TestDSV4BreakableCudaGraphMetadataContract(CustomTestCase):
|
||||
self._make_core_metadata(0), indexer_metadata=None
|
||||
)
|
||||
capture_metadata.sparse_prefill_cache = object()
|
||||
capture_metadata.prefill_shared_reads_snapshotted = True
|
||||
replay_metadata = DSV4Metadata(
|
||||
self._make_core_metadata(1000), indexer_metadata=None
|
||||
)
|
||||
@@ -724,7 +698,6 @@ class TestDSV4BreakableCudaGraphMetadataContract(CustomTestCase):
|
||||
self.assertTrue(calls[0][2])
|
||||
self.assertIs(backend.forward_metadata, capture_metadata)
|
||||
self.assertIsNone(capture_metadata.sparse_prefill_cache)
|
||||
self.assertFalse(capture_metadata.prefill_shared_reads_snapshotted)
|
||||
self.assertTrue(
|
||||
torch.equal(
|
||||
capture_metadata.core_attn_metadata.seq_lens_casual,
|
||||
|
||||
@@ -84,19 +84,23 @@ class TestPrefillCudaGraphPadding(CustomTestCase):
|
||||
runner = self._make_runner()
|
||||
runner.use_captured_attn_metadata = False
|
||||
attn_backend = mock.Mock()
|
||||
attn_backend.resolve_prefill_shared_read_ends.return_value = mock.sentinel.end
|
||||
runner.model_runner = SimpleNamespace(attn_backend=attn_backend)
|
||||
forward_batch = self._make_forward_batch(8)
|
||||
static_forward_batch = self._make_forward_batch(16)
|
||||
|
||||
runner._prepare_forward_metadata_for_replay(
|
||||
shared_read_ends = runner._prepare_forward_metadata_for_replay(
|
||||
forward_batch,
|
||||
static_forward_batch,
|
||||
shape_key=ShapeKey(size=16),
|
||||
)
|
||||
|
||||
self.assertIs(shared_read_ends, mock.sentinel.end)
|
||||
attn_backend.init_forward_metadata.assert_called_once_with(forward_batch)
|
||||
attn_backend.prepare_prefill_shared_read_snapshot.assert_called_once_with(
|
||||
forward_batch, num_qo_tokens=16
|
||||
attn_backend.resolve_prefill_shared_read_ends.assert_called_once_with(
|
||||
forward_batch,
|
||||
num_qo_tokens=16,
|
||||
build_snapshot=True,
|
||||
)
|
||||
|
||||
def test_full_replay_pads_request_slot_cpu_mirror(self):
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock, call, create_autospec
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -12,6 +11,7 @@ from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.model_executor.runner_utils.shared_read_event import (
|
||||
maybe_publish_prefill_shared_read_done,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_context, get_memory
|
||||
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||
from sglang.srt.speculative.spec_registry import CustomSpecAlgo
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
@@ -20,6 +20,12 @@ from sglang.test.test_utils import CustomTestCase
|
||||
register_cpu_ci(est_time=11, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _published_server_args():
|
||||
with get_context().override_server_args():
|
||||
yield
|
||||
|
||||
|
||||
class _Event:
|
||||
def __init__(self):
|
||||
self.recorded = False
|
||||
@@ -28,17 +34,19 @@ class _Event:
|
||||
self.recorded = True
|
||||
|
||||
|
||||
def _model_runner(*, spec_algorithm=SpeculativeAlgorithm.NONE, compliant=True):
|
||||
declared = SharedReadEnds.PRE_REPLAY if compliant else SharedReadEnds.UNKNOWN
|
||||
# Spec'd against the real ABC so a rename fails here, not at runtime.
|
||||
attn_backend = create_autospec(AttentionBackend, instance=True)
|
||||
attn_backend.shared_read_ends.return_value = declared
|
||||
return SimpleNamespace(
|
||||
def _model_runner(
|
||||
*, spec_algorithm=SpeculativeAlgorithm.NONE, compliant=True, declared=None
|
||||
):
|
||||
if declared is None:
|
||||
declared = SharedReadEnds.PRE_REPLAY if compliant else SharedReadEnds.UNKNOWN
|
||||
attn_backend = AttentionBackend()
|
||||
attn_backend.shared_read_ends = lambda _forward_mode: declared
|
||||
runner = SimpleNamespace(
|
||||
spec_algorithm=spec_algorithm,
|
||||
attn_backend=attn_backend,
|
||||
shared_read_done_event=None,
|
||||
prefill_shared_read_stager=None,
|
||||
)
|
||||
return runner
|
||||
|
||||
|
||||
_DEVICE_MODULE = SimpleNamespace(Event=_Event)
|
||||
@@ -48,10 +56,16 @@ def _batch(mode=ForwardMode.EXTEND):
|
||||
return SimpleNamespace(forward_mode=mode)
|
||||
|
||||
|
||||
def test_publishes_recorded_event_when_enabled():
|
||||
def _prepare_and_publish(runner, batch):
|
||||
shared_read_ends = runner.attn_backend.resolve_prefill_shared_read_ends(
|
||||
batch, num_qo_tokens=8
|
||||
)
|
||||
maybe_publish_prefill_shared_read_done(runner, shared_read_ends, _DEVICE_MODULE)
|
||||
|
||||
|
||||
def test_publishes_recorded_event_by_default():
|
||||
runner = _model_runner()
|
||||
with envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.override(True):
|
||||
maybe_publish_prefill_shared_read_done(runner, _batch(), _DEVICE_MODULE)
|
||||
_prepare_and_publish(runner, _batch())
|
||||
published = runner.shared_read_done_event
|
||||
assert isinstance(published, _Event) and published.recorded
|
||||
|
||||
@@ -59,73 +73,31 @@ def test_publishes_recorded_event_when_enabled():
|
||||
def test_disabled_when_flag_is_false():
|
||||
runner = _model_runner()
|
||||
with envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.override(False):
|
||||
maybe_publish_prefill_shared_read_done(runner, _batch(), _DEVICE_MODULE)
|
||||
_prepare_and_publish(runner, _batch())
|
||||
assert runner.shared_read_done_event is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"algorithm", (SpeculativeAlgorithm.DFLASH, SpeculativeAlgorithm.DSPARK)
|
||||
"memory",
|
||||
[
|
||||
{"enable_hierarchical_cache": True},
|
||||
{"hicache_storage_backend": "mooncake"},
|
||||
{"enable_lmcache": True},
|
||||
],
|
||||
)
|
||||
def test_dflash_family_target_prefill_publishes(algorithm):
|
||||
runner = _model_runner(spec_algorithm=algorithm)
|
||||
runner.prefill_shared_read_stager = Mock(return_value=False)
|
||||
def test_kv_offload_keeps_the_coarse_barrier(memory):
|
||||
runner = _model_runner()
|
||||
with get_memory().override(**memory):
|
||||
_prepare_and_publish(runner, _batch())
|
||||
assert runner.shared_read_done_event is None
|
||||
|
||||
|
||||
def test_dflash_target_prefill_publishes():
|
||||
runner = _model_runner(spec_algorithm=SpeculativeAlgorithm.DFLASH)
|
||||
with envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.override(True):
|
||||
maybe_publish_prefill_shared_read_done(runner, _batch(), _DEVICE_MODULE)
|
||||
_prepare_and_publish(runner, _batch())
|
||||
published = runner.shared_read_done_event
|
||||
assert isinstance(published, _Event) and published.recorded
|
||||
runner.prefill_shared_read_stager.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("staged", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"algorithm",
|
||||
[
|
||||
SpeculativeAlgorithm.EAGLE,
|
||||
SpeculativeAlgorithm.EAGLE3,
|
||||
SpeculativeAlgorithm.FROZEN_KV_MTP,
|
||||
SpeculativeAlgorithm.STANDALONE,
|
||||
SpeculativeAlgorithm.NGRAM,
|
||||
SpeculativeAlgorithm.UNO,
|
||||
CustomSpecAlgo("TEST_PREFILL", lambda _: object),
|
||||
],
|
||||
)
|
||||
def test_speculative_prefill_publishes_only_after_staging(algorithm, staged):
|
||||
runner, batch = _model_runner(spec_algorithm=algorithm), _batch()
|
||||
calls = Mock()
|
||||
runner.prefill_shared_read_stager = calls.stage
|
||||
calls.stage.return_value = staged
|
||||
calls.Event.side_effect = _Event
|
||||
with envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.override(True):
|
||||
maybe_publish_prefill_shared_read_done(
|
||||
runner, batch, SimpleNamespace(Event=calls.Event)
|
||||
)
|
||||
assert calls.mock_calls == [call.stage(batch)] + ([call.Event()] if staged else [])
|
||||
published = runner.shared_read_done_event
|
||||
if staged:
|
||||
assert isinstance(published, _Event) and published.recorded
|
||||
else:
|
||||
assert published is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"enabled,mode,compliant",
|
||||
[
|
||||
(False, ForwardMode.EXTEND, True),
|
||||
(True, ForwardMode.TARGET_VERIFY, True),
|
||||
(True, ForwardMode.MIXED, True),
|
||||
(True, ForwardMode.DECODE, True),
|
||||
(True, ForwardMode.EXTEND, False),
|
||||
],
|
||||
)
|
||||
def test_prefill_gates_skip_staging(enabled, mode, compliant):
|
||||
runner = _model_runner(
|
||||
spec_algorithm=SpeculativeAlgorithm.EAGLE, compliant=compliant
|
||||
)
|
||||
runner.prefill_shared_read_stager = Mock(return_value=True)
|
||||
with envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.override(enabled):
|
||||
maybe_publish_prefill_shared_read_done(runner, _batch(mode), _DEVICE_MODULE)
|
||||
runner.prefill_shared_read_stager.assert_not_called()
|
||||
assert runner.shared_read_done_event is None
|
||||
|
||||
|
||||
def test_gates_exclude_non_prefill_unsupported_algorithm_and_noncompliant_backend():
|
||||
@@ -135,7 +107,7 @@ def test_gates_exclude_non_prefill_unsupported_algorithm_and_noncompliant_backen
|
||||
(_model_runner(), _batch(ForwardMode.TARGET_VERIFY)),
|
||||
(_model_runner(), _batch(ForwardMode.MIXED)),
|
||||
(_model_runner(), _batch(ForwardMode.DECODE)),
|
||||
# The algorithm has a later prefill reader or unverified ownership.
|
||||
# The algorithm has a later prefill reader.
|
||||
(_model_runner(spec_algorithm=SpeculativeAlgorithm.EAGLE), _batch()),
|
||||
(
|
||||
_model_runner(
|
||||
@@ -146,7 +118,7 @@ def test_gates_exclude_non_prefill_unsupported_algorithm_and_noncompliant_backen
|
||||
# Backend has not declared a pre-replay prefill read end.
|
||||
(_model_runner(compliant=False), _batch()),
|
||||
):
|
||||
maybe_publish_prefill_shared_read_done(runner, batch, _DEVICE_MODULE)
|
||||
_prepare_and_publish(runner, batch)
|
||||
assert runner.shared_read_done_event is None
|
||||
|
||||
|
||||
@@ -157,18 +129,12 @@ class _TargetOnlyPrefillAlgo(CustomSpecAlgo):
|
||||
|
||||
class TestPrefillReadDoneCapability(CustomTestCase):
|
||||
def test_builtin_allowlist_is_unchanged(self):
|
||||
allowed = {
|
||||
SpeculativeAlgorithm.NONE,
|
||||
SpeculativeAlgorithm.DFLASH,
|
||||
SpeculativeAlgorithm.DSPARK,
|
||||
}
|
||||
allowed = {SpeculativeAlgorithm.NONE, SpeculativeAlgorithm.DFLASH}
|
||||
for algorithm in SpeculativeAlgorithm:
|
||||
with self.subTest(algorithm=algorithm):
|
||||
runner = _model_runner(spec_algorithm=algorithm)
|
||||
with envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.override(True):
|
||||
maybe_publish_prefill_shared_read_done(
|
||||
runner, _batch(), _DEVICE_MODULE
|
||||
)
|
||||
_prepare_and_publish(runner, _batch())
|
||||
self.assertEqual(
|
||||
runner.shared_read_done_event is not None, algorithm in allowed
|
||||
)
|
||||
@@ -182,19 +148,13 @@ class TestPrefillReadDoneCapability(CustomTestCase):
|
||||
algorithm = algorithm_type(
|
||||
"TEST_PREFILL", lambda _: object, supports_overlap=True
|
||||
)
|
||||
runner, batch = _model_runner(spec_algorithm=algorithm), _batch()
|
||||
runner.prefill_shared_read_stager = Mock(return_value=False)
|
||||
runner = _model_runner(spec_algorithm=algorithm)
|
||||
with envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.override(True):
|
||||
maybe_publish_prefill_shared_read_done(
|
||||
runner, batch, _DEVICE_MODULE
|
||||
)
|
||||
_prepare_and_publish(runner, _batch())
|
||||
event = runner.shared_read_done_event
|
||||
self.assertEqual(event is not None, expected)
|
||||
if expected:
|
||||
self.assertTrue(event.recorded)
|
||||
runner.prefill_shared_read_stager.assert_not_called()
|
||||
else:
|
||||
runner.prefill_shared_read_stager.assert_called_once_with(batch)
|
||||
|
||||
def test_opt_in_does_not_bypass_mode_backend_or_flag(self):
|
||||
algorithm = _TargetOnlyPrefillAlgo("TEST_PREFILL", lambda _: object)
|
||||
@@ -211,12 +171,9 @@ class TestPrefillReadDoneCapability(CustomTestCase):
|
||||
)
|
||||
for enabled, mode, declared in cases:
|
||||
with self.subTest(enabled=enabled, mode=mode, declared=declared):
|
||||
runner = _model_runner(spec_algorithm=algorithm)
|
||||
runner.attn_backend.shared_read_ends.return_value = declared
|
||||
runner = _model_runner(spec_algorithm=algorithm, declared=declared)
|
||||
with envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.override(enabled):
|
||||
maybe_publish_prefill_shared_read_done(
|
||||
runner, _batch(mode), _DEVICE_MODULE
|
||||
)
|
||||
_prepare_and_publish(runner, _batch(mode))
|
||||
self.assertIsNone(runner.shared_read_done_event)
|
||||
|
||||
|
||||
|
||||
@@ -432,7 +432,7 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase):
|
||||
)
|
||||
translator.bind_own_plan(forward_batch)
|
||||
|
||||
static_batch = runner.load_batch(forward_batch)
|
||||
static_batch, _ = runner.load_batch(forward_batch)
|
||||
|
||||
self.assertIs(static_batch.mm_input_embeds, mm_input_embeds)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user