revert: #42035 continuous prefill input polling (CP rank collective mismatch deadlock)
build-sglang-image / build (push) Failing after 39m1s

compose 3p1d regression on 1f6ca2cf6: all 3 P nodes wedged with split-brain
CP ranks (one blocked in _sync_hicache_ready_counts all_reduce, partner in
request broadcast). Root cause: #42035 wraps get_next_disagg_prefill_batch_to_run
(which contains the hicache attn_cp_group all_reduce) in rank-local guards
(len(result_queue)<2, _deferred_input_requests, skip_batch_creation); any
cross-rank divergence in batch-None decisions misaligns collective pairing
permanently. Continuous polling requires attn_cp_size==1 anyway, so cp2
deployments get zero benefit. Reverts 93d610c1fd + test adaptations
6ba34507f1/bde1fcbf51/1f6ca2cf68 + deploy bump 236686454c.
Keeps #42194 (9ef65b51c2) and checkpoint_disagg_prefill shim (c4f3e3be33).
Evidence: D:\B300\investigations\pd-deadlock-20261009.md
This commit is contained in:
2026-10-09 16:39:06 +08:00
parent 21f7ce37dd
commit 7578cdd218
14 changed files with 80 additions and 1838 deletions
+32 -151
View File
@@ -21,7 +21,6 @@ from __future__ import annotations
import hashlib
import logging
import time
from array import array
from collections import deque
from http import HTTPStatus
@@ -104,12 +103,6 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
# TP=1 yield between polls of a pending result. An initial heuristic rather
# than a tuned value: long enough that the bootstrap, transfer and cache
# threads reliably win the GIL even when the input socket is empty, and far
# below the duration of any prefill forward, so it adds no visible latency.
PREFILL_INPUT_POLL_INTERVAL_S = 0.0001
_is_npu = is_npu()
@@ -615,12 +608,9 @@ class SchedulerDisaggregationPrefillMixin:
self: Scheduler,
running_batch: ScheduleBatch,
last_batch: Optional[ScheduleBatch],
should_retry_storage_prefetch: bool = True,
) -> NextBatchPlan:
self.process_pending_chunked_abort()
self._process_hicache_events(
should_retry_storage_prefetch=should_retry_storage_prefetch
)
self._process_hicache_events()
# HACK (byronhsu): reset the batch_is_full flag because we never enter update_running_batch which resets it
# Otherwise, it hangs under high concurrency
@@ -628,13 +618,7 @@ class SchedulerDisaggregationPrefillMixin:
self.resolve_waiting_queue_bootstrap()
# Skip repeated chunk processing while polling a pending batch.
if last_batch is None or not last_batch.prefill_chunk_processed:
self.process_prefill_chunk(
last_batch=last_batch, running_batch=running_batch
)
if last_batch is not None:
last_batch.prefill_chunk_processed = True
self.process_prefill_chunk(last_batch=last_batch, running_batch=running_batch)
prefill_plan = self.get_new_batch_prefill(running_batch)
batch = prefill_plan.batch_to_run
@@ -685,120 +669,46 @@ class SchedulerDisaggregationPrefillMixin:
# Update last_batch
self.last_batch = batch
def _is_continuous_input_polling_enabled(self: Scheduler) -> bool:
"""Check support once on entry to the prefill overlap loop."""
if not envs.SGLANG_ENABLE_DISAGG_PREFILL_CONTINUOUS_INPUT_POLLING.get():
return False
parallel = get_parallel()
enabled = (
parallel.attn_dp_size == 1
and parallel.attn_cp_size == 1
and self.spec_algorithm.is_none()
and self.is_generation
and self.dllm_config is None
)
if not enabled:
logger.warning(
"Continuous prefill input polling requires attention DP=CP=1 and "
"non-speculative autoregressive generation; using the regular overlap loop."
)
return enabled
def _yield_gil_if_needed(self: Scheduler) -> None:
if self.tp_size == 1:
# No collective parks this thread at TP=1. A bounded sleep lets
# bootstrap, transfer and cache workers reacquire the GIL.
time.sleep(PREFILL_INPUT_POLL_INTERVAL_S)
@torch.no_grad()
def event_loop_overlap_disagg_prefill(self: Scheduler) -> None:
"""Run intake while pending copies finish, keeping at most two forwards.
A deferred pause stops preparation until the current result wait finishes.
Shutdown stops new launches; a wait finishes through the blocking handler.
After a failed preparation, only new input or bootstrap arrivals retry
during the wait; other readiness is checked after a result finishes.
"""
self.enable_continuous_input_polling = (
self._is_continuous_input_polling_enabled()
)
self.result_queue = deque()
batch_result_completion_status = torch.empty(1, dtype=torch.int32, device="cpu")
# True when intake repeats while the oldest batch result is unfinished.
waiting_for_batch_result = False
# Avoid retrying failed batch preparation after every copy poll.
skip_batch_creation = False
# Keep the newer forward's sampling context across unfinished-result passes.
batch = None
batch_result = None
while True:
if self.gracefully_exit:
break
# Stop dispatching at pause requests while waiting for a batch result.
received_inputs = self.ingest_requests(
stop_at_pause=waiting_for_batch_result
)
# Receive requests
self.ingest_requests()
if self._engine_paused:
self._record_scheduler_state_for_paused_engine()
self._yield_gil_if_needed()
continue
bootstrapped_reqs = self.disagg_prefill_bootstrap_queue.pop_bootstrapped()
self.waiting_queue.extend(bootstrapped_reqs)
# Resume batch creation on completion, new/ready requests, or an idle step.
if not waiting_for_batch_result or received_inputs or bootstrapped_reqs:
skip_batch_creation = False
self.waiting_queue.extend(
self.disagg_prefill_bootstrap_queue.pop_bootstrapped()
)
# Keep at most two batches in flight.
if (
len(self.result_queue) < 2
and not self.gracefully_exit
and not self._deferred_input_requests
and not skip_batch_creation
):
# Get the next batch to run
plan = self.get_next_disagg_prefill_batch_to_run(
running_batch=self.running_batch,
last_batch=self.last_batch,
# Storage retry deadlines count scheduling steps, not copy polls.
should_retry_storage_prefetch=not waiting_for_batch_result,
)
self.running_batch = plan.running_batch
batch = plan.batch_to_run
batch = self.ngram_embedding_manager.prepare_for_forward(
batch, chunked_req=self.chunked_req
)
self.cur_batch_for_debug = batch
# Skip batch creation next round to avoid repeated preparation failures.
skip_batch_creation = batch is None
# Get the next batch to run
plan = self.get_next_disagg_prefill_batch_to_run(
running_batch=self.running_batch, last_batch=self.last_batch
)
self.running_batch = plan.running_batch
batch = plan.batch_to_run
batch = self.ngram_embedding_manager.prepare_for_forward(
batch, chunked_req=self.chunked_req
)
self.cur_batch_for_debug = batch
# Launch the current batch
if batch:
if self.enable_staging:
self.maybe_prefetch_staging_for_batch(batch)
batch_result = self.run_batch(batch)
self._apply_war_barrier()
self.result_queue.append((batch.copy(), batch_result))
else:
batch_result = None
# Launch the current batch
if batch:
if self.enable_staging:
self.maybe_prefetch_staging_for_batch(batch)
batch_result = self.run_batch(batch)
self._apply_war_barrier()
self.result_queue.append((batch.copy(), batch_result))
else:
batch_result = None
self._sched_idled = True
# Process the last batch
if self.last_batch:
if self.enable_continuous_input_polling:
# Resume intake until the oldest result is ready on all TP ranks.
oldest_batch_result = self.result_queue[0][1]
if (
not self.gracefully_exit
and not self.is_disagg_prefill_batch_result_ready(
oldest_batch_result, batch_result_completion_status
)
):
waiting_for_batch_result = True
self._yield_gil_if_needed()
continue
# Process the oldest batch result.
oldest_batch, oldest_batch_result = self.result_queue.popleft()
self.process_batch_result(oldest_batch, oldest_batch_result)
tmp_batch, tmp_result = self.result_queue.popleft()
self.process_batch_result(tmp_batch, tmp_result)
elif batch is None:
# When the server is idle, do self-check and re-init some states
self.on_idle()
@@ -809,36 +719,8 @@ class SchedulerDisaggregationPrefillMixin:
# It depends on the result of the last batch (e.g., grammar), so we run it after the last batch is processed.
self.launch_batch_sample_if_needed(batch_result, batch)
# Update last_batch and scheduler status
self._sched_idled = batch is None
# Update last_batch
self.last_batch = batch
waiting_for_batch_result = False
def is_disagg_prefill_batch_result_ready(
self: Scheduler,
batch_result: GenerationBatchResult,
batch_result_completion_status: torch.Tensor,
) -> bool:
# 0: sampling/copy has not been submitted, so polling cannot make progress.
state = 0
if (
batch_result.copy_done is not None
and batch_result.delay_sample_func is None
):
# 1: copy unfinished; 2: copy finished locally.
state = 2 if batch_result.copy_done.query() else 1
if self.tp_size > 1:
batch_result_completion_status.fill_(state)
torch.distributed.all_reduce(
batch_result_completion_status,
op=torch.distributed.ReduceOp.MIN,
group=self.tp_cpu_group,
)
state = batch_result_completion_status.item()
if state == 0:
# Every rank must submit delayed sampling before entering this wait.
raise RuntimeError("Prefill result copy must be submitted before polling")
return state == 2
def process_batch_result_disagg_prefill(
self: Scheduler,
@@ -1335,8 +1217,7 @@ class SchedulerDisaggregationPrefillMixin:
elif self.has_bootstrapped_waiting_req():
# optimistic request yields to waiting requests
self.chunked_req = None
# Retract may drain every result, leaving no handler to requeue.
if not self.enable_overlap or req.inflight_middle_chunks == 0:
if not self.enable_overlap:
self.optimistic_release_and_requeue(req)
# else: still bootstrapping, keep computing without sending
elif self.enable_overlap:
-2
View File
@@ -658,8 +658,6 @@ class Envs:
# do not fence the next forward through the scheduler stream.
SGLANG_PP_COMM_OVERLAP = EnvBool(False)
SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH = EnvBool(False)
# Opt-in: keep receiving prefill requests while forward results are pending.
SGLANG_ENABLE_DISAGG_PREFILL_CONTINUOUS_INPUT_POLLING = EnvBool(False)
# ===================================================================
# Radix and sparse KV caches
@@ -2335,9 +2335,6 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
chunked_req_next_prompt_token: Optional[int] = None
contains_last_prefill_chunk: bool = True
# Tracks process_prefill_chunk() on the original scheduler batch only.
prefill_chunk_processed: bool = False
# For DP attention
inner_idle_batch: Optional[ScheduleBatch] = None
# Decode requests carried alongside a chunked-prefill batch
+15 -59
View File
@@ -1270,7 +1270,6 @@ class Scheduler(
self.hisparse_coordinator.set_decode_producer_stream(self.forward_stream)
def init_running_status(self):
self.tp_size = get_parallel().tp_size
# Set by a runtime PD role switch to break out of the current event loop.
self._event_loop_should_restart = False
# Guards against concurrent/re-entrant PD role switches.
@@ -1287,10 +1286,6 @@ class Scheduler(
self.cur_batch_for_debug: Optional[ScheduleBatch] = None
# The last forward batch
self.last_batch: Optional[ScheduleBatch] = None
self.result_queue: deque[
tuple[ScheduleBatch, GenerationBatchResult | EmbeddingBatchResult]
] = deque()
self.enable_continuous_input_polling = False
self.forward_ct = 0
self.return_health_check_ipcs: Deque[Optional[str]] = deque()
self.flush_wrapper = SchedulerFlushWrapper(
@@ -1302,7 +1297,6 @@ class Scheduler(
self.session_controller = SessionController(self.tree_cache)
self.forward_sleep_time = None
self._engine_paused = False
self._deferred_input_requests: List = []
def init_chunked_prefill(self):
self.chunked_prefill_size = get_schedule().chunked_prefill_size
@@ -2073,38 +2067,20 @@ class Scheduler(
for prev_batch, prev_result in self.result_queue:
self.batch_result_processor.advance_grammar_fsm(prev_result, prev_batch)
def ingest_requests(self, stop_at_pause: bool = False) -> List:
"""Receive and broadcast input, then dispatch queued requests in order.
With stop_at_pause, retain the first pause and all following inputs
until the caller reaches a scheduling boundary. Receiving and broadcasting
continue each pass; all ranks retain the same suffix.
def ingest_requests(self) -> List:
"""Receive, broadcast and dispatch this iteration's external input.
The one place a new per-iteration input source belongs; the return
value exists for the pipeline stages that relay requests onward.
"""
local_reqs = []
# Avoid re-enqueuing timeout aborts while dispatch is stopped at a pause.
if (
not self._deferred_input_requests
and get_parallel().pp_rank == 0
get_parallel().pp_rank == 0
and get_parallel().attn_tp_rank == 0
and get_parallel().attn_cp_rank == 0
):
local_reqs = self._poll_timeout_aborts()
self._deferred_input_requests.extend(
self.request_receiver.recv_requests(local_reqs=local_reqs)
)
num_requests_to_process = len(self._deferred_input_requests)
if stop_at_pause:
for index, req in enumerate(self._deferred_input_requests):
if isinstance(req, PauseGenerationReqInput):
num_requests_to_process = index
break
recv_reqs = self._deferred_input_requests[:num_requests_to_process]
del self._deferred_input_requests[:num_requests_to_process]
recv_reqs = self.request_receiver.recv_requests(local_reqs=local_reqs)
if recv_reqs:
self.metrics_reporter.record_scheduler_active()
self.process_input_requests(recv_reqs)
@@ -3415,18 +3391,16 @@ class Scheduler(
if (timeout_s := envs.SGLANG_REQ_RUNNING_TIMEOUT.get()) > 0:
deadline = time.perf_counter() - timeout_s
inflight_batches = self._collect_inflight_batches()
if get_parallel().pp_size == 1:
inflight_batches = [self.running_batch, self.last_batch]
else:
inflight_batches = [*self.running_mbs, *self.mbs]
seen_rids = set()
for batch in inflight_batches:
if batch is None:
continue
for req in batch.reqs:
# Polling must not re-emit an abort awaiting result processing.
if (
req.rid in seen_rids
or req.finished()
or req.to_finish is not None
):
if req.rid in seen_rids or req.finished():
continue
seen_rids.add(req.rid)
if 0 < req.time_stats.forward_entry_time < deadline:
@@ -3636,9 +3610,7 @@ class Scheduler(
# todo hisparse, maybe other info to contain for the new batch
return batch
def _process_hicache_events(
self, should_retry_storage_prefetch: bool = True
) -> None:
def _process_hicache_events(self) -> None:
# The HiCache drain is TP-wide consensus; run it before rank-local
# decisions (_should_defer_prefill) or ranks enter different collectives.
if (
@@ -3647,7 +3619,7 @@ class Scheduler(
or self.enable_unified_cache_external_linker
):
self.tree_cache.check_hicache_events()
if self.enable_hicache_storage and should_retry_storage_prefetch:
if self.enable_hicache_storage:
self._process_storage_prefetch_retries()
@scheduler_stage_method(SCHEDULER_STAGE_GET_NEXT_BATCH)
@@ -5461,22 +5433,13 @@ class Scheduler(
)
def collect_inflight_reqs(self) -> Set[Req]:
return {
req
for batch in self._collect_inflight_batches()
if batch is not None
for req in batch.reqs
}
def _collect_inflight_batches(self) -> list[ScheduleBatch | None]:
if get_parallel().pp_size == 1:
inflight_batches = [self.running_batch, self.last_batch]
else:
inflight_batches = [*self.running_mbs, *self.mbs]
if self.enable_continuous_input_polling and self.result_queue:
# Polling can run intake before the newest queued batch becomes last_batch.
inflight_batches.extend(batch for batch, _ in self.result_queue)
return inflight_batches
return {
req for batch in inflight_batches if batch is not None for req in batch.reqs
}
def abort_request(self, recv_req: AbortReq):
if (chunked_req := self.chunked_req) is not None:
@@ -5651,12 +5614,7 @@ class Scheduler(
return
if self.enable_overlap and self.last_batch:
if (
self.disaggregation_mode == DisaggregationMode.PREFILL
and (req := self.chunked_req) is not None
):
# Retract skips the chunk step that sets this result's KV send boundary.
req.tmp_end_idx = min(req.extend_range.end, len(req.origin_input_ids))
# Process the results of the last batch
tmp_batch, tmp_result = self.result_queue.popleft()
self.process_batch_result(tmp_batch, tmp_result)
@@ -5990,8 +5948,6 @@ def dispatch_event_loop(scheduler: Scheduler):
def _dispatch_event_loop_once(scheduler: Scheduler):
# A PD role switch can select a different loop on the same scheduler.
scheduler.enable_continuous_input_polling = False
disaggregation_mode: DisaggregationMode = scheduler.disaggregation_mode
if disaggregation_mode == DisaggregationMode.NULL:
if scheduler.enable_pdmux: