revert: #42035 continuous prefill input polling (CP rank collective mismatch deadlock)
build-sglang-image / build (push) Failing after 39m1s
build-sglang-image / build (push) Failing after 39m1s
compose 3p1d regression on1f6ca2cf6: all 3 P nodes wedged with split-brain CP ranks (one blocked in _sync_hicache_ready_counts all_reduce, partner in request broadcast). Root cause: #42035 wraps get_next_disagg_prefill_batch_to_run (which contains the hicache attn_cp_group all_reduce) in rank-local guards (len(result_queue)<2, _deferred_input_requests, skip_batch_creation); any cross-rank divergence in batch-None decisions misaligns collective pairing permanently. Continuous polling requires attn_cp_size==1 anyway, so cp2 deployments get zero benefit. Reverts93d610c1fd+ test adaptations 6ba34507f1/bde1fcbf51/1f6ca2cf68 + deploy bump236686454c. Keeps #42194 (9ef65b51c2) and checkpoint_disagg_prefill shim (c4f3e3be33). Evidence: D:\B300\investigations\pd-deadlock-20261009.md
This commit is contained in:
@@ -21,7 +21,6 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import time
|
||||
from array import array
|
||||
from collections import deque
|
||||
from http import HTTPStatus
|
||||
@@ -104,12 +103,6 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# TP=1 yield between polls of a pending result. An initial heuristic rather
|
||||
# than a tuned value: long enough that the bootstrap, transfer and cache
|
||||
# threads reliably win the GIL even when the input socket is empty, and far
|
||||
# below the duration of any prefill forward, so it adds no visible latency.
|
||||
PREFILL_INPUT_POLL_INTERVAL_S = 0.0001
|
||||
|
||||
_is_npu = is_npu()
|
||||
|
||||
|
||||
@@ -615,12 +608,9 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
self: Scheduler,
|
||||
running_batch: ScheduleBatch,
|
||||
last_batch: Optional[ScheduleBatch],
|
||||
should_retry_storage_prefetch: bool = True,
|
||||
) -> NextBatchPlan:
|
||||
self.process_pending_chunked_abort()
|
||||
self._process_hicache_events(
|
||||
should_retry_storage_prefetch=should_retry_storage_prefetch
|
||||
)
|
||||
self._process_hicache_events()
|
||||
|
||||
# HACK (byronhsu): reset the batch_is_full flag because we never enter update_running_batch which resets it
|
||||
# Otherwise, it hangs under high concurrency
|
||||
@@ -628,13 +618,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
|
||||
self.resolve_waiting_queue_bootstrap()
|
||||
|
||||
# Skip repeated chunk processing while polling a pending batch.
|
||||
if last_batch is None or not last_batch.prefill_chunk_processed:
|
||||
self.process_prefill_chunk(
|
||||
last_batch=last_batch, running_batch=running_batch
|
||||
)
|
||||
if last_batch is not None:
|
||||
last_batch.prefill_chunk_processed = True
|
||||
self.process_prefill_chunk(last_batch=last_batch, running_batch=running_batch)
|
||||
|
||||
prefill_plan = self.get_new_batch_prefill(running_batch)
|
||||
batch = prefill_plan.batch_to_run
|
||||
@@ -685,120 +669,46 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
# Update last_batch
|
||||
self.last_batch = batch
|
||||
|
||||
def _is_continuous_input_polling_enabled(self: Scheduler) -> bool:
|
||||
"""Check support once on entry to the prefill overlap loop."""
|
||||
if not envs.SGLANG_ENABLE_DISAGG_PREFILL_CONTINUOUS_INPUT_POLLING.get():
|
||||
return False
|
||||
parallel = get_parallel()
|
||||
enabled = (
|
||||
parallel.attn_dp_size == 1
|
||||
and parallel.attn_cp_size == 1
|
||||
and self.spec_algorithm.is_none()
|
||||
and self.is_generation
|
||||
and self.dllm_config is None
|
||||
)
|
||||
if not enabled:
|
||||
logger.warning(
|
||||
"Continuous prefill input polling requires attention DP=CP=1 and "
|
||||
"non-speculative autoregressive generation; using the regular overlap loop."
|
||||
)
|
||||
return enabled
|
||||
|
||||
def _yield_gil_if_needed(self: Scheduler) -> None:
|
||||
if self.tp_size == 1:
|
||||
# No collective parks this thread at TP=1. A bounded sleep lets
|
||||
# bootstrap, transfer and cache workers reacquire the GIL.
|
||||
time.sleep(PREFILL_INPUT_POLL_INTERVAL_S)
|
||||
|
||||
@torch.no_grad()
|
||||
def event_loop_overlap_disagg_prefill(self: Scheduler) -> None:
|
||||
"""Run intake while pending copies finish, keeping at most two forwards.
|
||||
|
||||
A deferred pause stops preparation until the current result wait finishes.
|
||||
Shutdown stops new launches; a wait finishes through the blocking handler.
|
||||
After a failed preparation, only new input or bootstrap arrivals retry
|
||||
during the wait; other readiness is checked after a result finishes.
|
||||
"""
|
||||
self.enable_continuous_input_polling = (
|
||||
self._is_continuous_input_polling_enabled()
|
||||
)
|
||||
self.result_queue = deque()
|
||||
batch_result_completion_status = torch.empty(1, dtype=torch.int32, device="cpu")
|
||||
# True when intake repeats while the oldest batch result is unfinished.
|
||||
waiting_for_batch_result = False
|
||||
# Avoid retrying failed batch preparation after every copy poll.
|
||||
skip_batch_creation = False
|
||||
# Keep the newer forward's sampling context across unfinished-result passes.
|
||||
batch = None
|
||||
batch_result = None
|
||||
|
||||
while True:
|
||||
if self.gracefully_exit:
|
||||
break
|
||||
|
||||
# Stop dispatching at pause requests while waiting for a batch result.
|
||||
received_inputs = self.ingest_requests(
|
||||
stop_at_pause=waiting_for_batch_result
|
||||
)
|
||||
# Receive requests
|
||||
self.ingest_requests()
|
||||
if self._engine_paused:
|
||||
self._record_scheduler_state_for_paused_engine()
|
||||
self._yield_gil_if_needed()
|
||||
continue
|
||||
bootstrapped_reqs = self.disagg_prefill_bootstrap_queue.pop_bootstrapped()
|
||||
self.waiting_queue.extend(bootstrapped_reqs)
|
||||
# Resume batch creation on completion, new/ready requests, or an idle step.
|
||||
if not waiting_for_batch_result or received_inputs or bootstrapped_reqs:
|
||||
skip_batch_creation = False
|
||||
self.waiting_queue.extend(
|
||||
self.disagg_prefill_bootstrap_queue.pop_bootstrapped()
|
||||
)
|
||||
|
||||
# Keep at most two batches in flight.
|
||||
if (
|
||||
len(self.result_queue) < 2
|
||||
and not self.gracefully_exit
|
||||
and not self._deferred_input_requests
|
||||
and not skip_batch_creation
|
||||
):
|
||||
# Get the next batch to run
|
||||
plan = self.get_next_disagg_prefill_batch_to_run(
|
||||
running_batch=self.running_batch,
|
||||
last_batch=self.last_batch,
|
||||
# Storage retry deadlines count scheduling steps, not copy polls.
|
||||
should_retry_storage_prefetch=not waiting_for_batch_result,
|
||||
)
|
||||
self.running_batch = plan.running_batch
|
||||
batch = plan.batch_to_run
|
||||
batch = self.ngram_embedding_manager.prepare_for_forward(
|
||||
batch, chunked_req=self.chunked_req
|
||||
)
|
||||
self.cur_batch_for_debug = batch
|
||||
# Skip batch creation next round to avoid repeated preparation failures.
|
||||
skip_batch_creation = batch is None
|
||||
# Get the next batch to run
|
||||
plan = self.get_next_disagg_prefill_batch_to_run(
|
||||
running_batch=self.running_batch, last_batch=self.last_batch
|
||||
)
|
||||
self.running_batch = plan.running_batch
|
||||
batch = plan.batch_to_run
|
||||
batch = self.ngram_embedding_manager.prepare_for_forward(
|
||||
batch, chunked_req=self.chunked_req
|
||||
)
|
||||
self.cur_batch_for_debug = batch
|
||||
|
||||
# Launch the current batch
|
||||
if batch:
|
||||
if self.enable_staging:
|
||||
self.maybe_prefetch_staging_for_batch(batch)
|
||||
batch_result = self.run_batch(batch)
|
||||
self._apply_war_barrier()
|
||||
self.result_queue.append((batch.copy(), batch_result))
|
||||
else:
|
||||
batch_result = None
|
||||
# Launch the current batch
|
||||
if batch:
|
||||
if self.enable_staging:
|
||||
self.maybe_prefetch_staging_for_batch(batch)
|
||||
batch_result = self.run_batch(batch)
|
||||
self._apply_war_barrier()
|
||||
self.result_queue.append((batch.copy(), batch_result))
|
||||
else:
|
||||
batch_result = None
|
||||
self._sched_idled = True
|
||||
|
||||
# Process the last batch
|
||||
if self.last_batch:
|
||||
if self.enable_continuous_input_polling:
|
||||
# Resume intake until the oldest result is ready on all TP ranks.
|
||||
oldest_batch_result = self.result_queue[0][1]
|
||||
if (
|
||||
not self.gracefully_exit
|
||||
and not self.is_disagg_prefill_batch_result_ready(
|
||||
oldest_batch_result, batch_result_completion_status
|
||||
)
|
||||
):
|
||||
waiting_for_batch_result = True
|
||||
self._yield_gil_if_needed()
|
||||
continue
|
||||
# Process the oldest batch result.
|
||||
oldest_batch, oldest_batch_result = self.result_queue.popleft()
|
||||
self.process_batch_result(oldest_batch, oldest_batch_result)
|
||||
tmp_batch, tmp_result = self.result_queue.popleft()
|
||||
self.process_batch_result(tmp_batch, tmp_result)
|
||||
elif batch is None:
|
||||
# When the server is idle, do self-check and re-init some states
|
||||
self.on_idle()
|
||||
@@ -809,36 +719,8 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
# It depends on the result of the last batch (e.g., grammar), so we run it after the last batch is processed.
|
||||
self.launch_batch_sample_if_needed(batch_result, batch)
|
||||
|
||||
# Update last_batch and scheduler status
|
||||
self._sched_idled = batch is None
|
||||
# Update last_batch
|
||||
self.last_batch = batch
|
||||
waiting_for_batch_result = False
|
||||
|
||||
def is_disagg_prefill_batch_result_ready(
|
||||
self: Scheduler,
|
||||
batch_result: GenerationBatchResult,
|
||||
batch_result_completion_status: torch.Tensor,
|
||||
) -> bool:
|
||||
# 0: sampling/copy has not been submitted, so polling cannot make progress.
|
||||
state = 0
|
||||
if (
|
||||
batch_result.copy_done is not None
|
||||
and batch_result.delay_sample_func is None
|
||||
):
|
||||
# 1: copy unfinished; 2: copy finished locally.
|
||||
state = 2 if batch_result.copy_done.query() else 1
|
||||
if self.tp_size > 1:
|
||||
batch_result_completion_status.fill_(state)
|
||||
torch.distributed.all_reduce(
|
||||
batch_result_completion_status,
|
||||
op=torch.distributed.ReduceOp.MIN,
|
||||
group=self.tp_cpu_group,
|
||||
)
|
||||
state = batch_result_completion_status.item()
|
||||
if state == 0:
|
||||
# Every rank must submit delayed sampling before entering this wait.
|
||||
raise RuntimeError("Prefill result copy must be submitted before polling")
|
||||
return state == 2
|
||||
|
||||
def process_batch_result_disagg_prefill(
|
||||
self: Scheduler,
|
||||
@@ -1335,8 +1217,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
elif self.has_bootstrapped_waiting_req():
|
||||
# optimistic request yields to waiting requests
|
||||
self.chunked_req = None
|
||||
# Retract may drain every result, leaving no handler to requeue.
|
||||
if not self.enable_overlap or req.inflight_middle_chunks == 0:
|
||||
if not self.enable_overlap:
|
||||
self.optimistic_release_and_requeue(req)
|
||||
# else: still bootstrapping, keep computing without sending
|
||||
elif self.enable_overlap:
|
||||
|
||||
@@ -658,8 +658,6 @@ class Envs:
|
||||
# do not fence the next forward through the scheduler stream.
|
||||
SGLANG_PP_COMM_OVERLAP = EnvBool(False)
|
||||
SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH = EnvBool(False)
|
||||
# Opt-in: keep receiving prefill requests while forward results are pending.
|
||||
SGLANG_ENABLE_DISAGG_PREFILL_CONTINUOUS_INPUT_POLLING = EnvBool(False)
|
||||
|
||||
# ===================================================================
|
||||
# Radix and sparse KV caches
|
||||
|
||||
@@ -2335,9 +2335,6 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
chunked_req_next_prompt_token: Optional[int] = None
|
||||
contains_last_prefill_chunk: bool = True
|
||||
|
||||
# Tracks process_prefill_chunk() on the original scheduler batch only.
|
||||
prefill_chunk_processed: bool = False
|
||||
|
||||
# For DP attention
|
||||
inner_idle_batch: Optional[ScheduleBatch] = None
|
||||
# Decode requests carried alongside a chunked-prefill batch
|
||||
|
||||
@@ -1270,7 +1270,6 @@ class Scheduler(
|
||||
self.hisparse_coordinator.set_decode_producer_stream(self.forward_stream)
|
||||
|
||||
def init_running_status(self):
|
||||
self.tp_size = get_parallel().tp_size
|
||||
# Set by a runtime PD role switch to break out of the current event loop.
|
||||
self._event_loop_should_restart = False
|
||||
# Guards against concurrent/re-entrant PD role switches.
|
||||
@@ -1287,10 +1286,6 @@ class Scheduler(
|
||||
self.cur_batch_for_debug: Optional[ScheduleBatch] = None
|
||||
# The last forward batch
|
||||
self.last_batch: Optional[ScheduleBatch] = None
|
||||
self.result_queue: deque[
|
||||
tuple[ScheduleBatch, GenerationBatchResult | EmbeddingBatchResult]
|
||||
] = deque()
|
||||
self.enable_continuous_input_polling = False
|
||||
self.forward_ct = 0
|
||||
self.return_health_check_ipcs: Deque[Optional[str]] = deque()
|
||||
self.flush_wrapper = SchedulerFlushWrapper(
|
||||
@@ -1302,7 +1297,6 @@ class Scheduler(
|
||||
self.session_controller = SessionController(self.tree_cache)
|
||||
self.forward_sleep_time = None
|
||||
self._engine_paused = False
|
||||
self._deferred_input_requests: List = []
|
||||
|
||||
def init_chunked_prefill(self):
|
||||
self.chunked_prefill_size = get_schedule().chunked_prefill_size
|
||||
@@ -2073,38 +2067,20 @@ class Scheduler(
|
||||
for prev_batch, prev_result in self.result_queue:
|
||||
self.batch_result_processor.advance_grammar_fsm(prev_result, prev_batch)
|
||||
|
||||
def ingest_requests(self, stop_at_pause: bool = False) -> List:
|
||||
"""Receive and broadcast input, then dispatch queued requests in order.
|
||||
|
||||
With stop_at_pause, retain the first pause and all following inputs
|
||||
until the caller reaches a scheduling boundary. Receiving and broadcasting
|
||||
continue each pass; all ranks retain the same suffix.
|
||||
def ingest_requests(self) -> List:
|
||||
"""Receive, broadcast and dispatch this iteration's external input.
|
||||
|
||||
The one place a new per-iteration input source belongs; the return
|
||||
value exists for the pipeline stages that relay requests onward.
|
||||
"""
|
||||
local_reqs = []
|
||||
# Avoid re-enqueuing timeout aborts while dispatch is stopped at a pause.
|
||||
if (
|
||||
not self._deferred_input_requests
|
||||
and get_parallel().pp_rank == 0
|
||||
get_parallel().pp_rank == 0
|
||||
and get_parallel().attn_tp_rank == 0
|
||||
and get_parallel().attn_cp_rank == 0
|
||||
):
|
||||
local_reqs = self._poll_timeout_aborts()
|
||||
self._deferred_input_requests.extend(
|
||||
self.request_receiver.recv_requests(local_reqs=local_reqs)
|
||||
)
|
||||
|
||||
num_requests_to_process = len(self._deferred_input_requests)
|
||||
if stop_at_pause:
|
||||
for index, req in enumerate(self._deferred_input_requests):
|
||||
if isinstance(req, PauseGenerationReqInput):
|
||||
num_requests_to_process = index
|
||||
break
|
||||
recv_reqs = self._deferred_input_requests[:num_requests_to_process]
|
||||
del self._deferred_input_requests[:num_requests_to_process]
|
||||
|
||||
recv_reqs = self.request_receiver.recv_requests(local_reqs=local_reqs)
|
||||
if recv_reqs:
|
||||
self.metrics_reporter.record_scheduler_active()
|
||||
self.process_input_requests(recv_reqs)
|
||||
@@ -3415,18 +3391,16 @@ class Scheduler(
|
||||
|
||||
if (timeout_s := envs.SGLANG_REQ_RUNNING_TIMEOUT.get()) > 0:
|
||||
deadline = time.perf_counter() - timeout_s
|
||||
inflight_batches = self._collect_inflight_batches()
|
||||
if get_parallel().pp_size == 1:
|
||||
inflight_batches = [self.running_batch, self.last_batch]
|
||||
else:
|
||||
inflight_batches = [*self.running_mbs, *self.mbs]
|
||||
seen_rids = set()
|
||||
for batch in inflight_batches:
|
||||
if batch is None:
|
||||
continue
|
||||
for req in batch.reqs:
|
||||
# Polling must not re-emit an abort awaiting result processing.
|
||||
if (
|
||||
req.rid in seen_rids
|
||||
or req.finished()
|
||||
or req.to_finish is not None
|
||||
):
|
||||
if req.rid in seen_rids or req.finished():
|
||||
continue
|
||||
seen_rids.add(req.rid)
|
||||
if 0 < req.time_stats.forward_entry_time < deadline:
|
||||
@@ -3636,9 +3610,7 @@ class Scheduler(
|
||||
# todo hisparse, maybe other info to contain for the new batch
|
||||
return batch
|
||||
|
||||
def _process_hicache_events(
|
||||
self, should_retry_storage_prefetch: bool = True
|
||||
) -> None:
|
||||
def _process_hicache_events(self) -> None:
|
||||
# The HiCache drain is TP-wide consensus; run it before rank-local
|
||||
# decisions (_should_defer_prefill) or ranks enter different collectives.
|
||||
if (
|
||||
@@ -3647,7 +3619,7 @@ class Scheduler(
|
||||
or self.enable_unified_cache_external_linker
|
||||
):
|
||||
self.tree_cache.check_hicache_events()
|
||||
if self.enable_hicache_storage and should_retry_storage_prefetch:
|
||||
if self.enable_hicache_storage:
|
||||
self._process_storage_prefetch_retries()
|
||||
|
||||
@scheduler_stage_method(SCHEDULER_STAGE_GET_NEXT_BATCH)
|
||||
@@ -5461,22 +5433,13 @@ class Scheduler(
|
||||
)
|
||||
|
||||
def collect_inflight_reqs(self) -> Set[Req]:
|
||||
return {
|
||||
req
|
||||
for batch in self._collect_inflight_batches()
|
||||
if batch is not None
|
||||
for req in batch.reqs
|
||||
}
|
||||
|
||||
def _collect_inflight_batches(self) -> list[ScheduleBatch | None]:
|
||||
if get_parallel().pp_size == 1:
|
||||
inflight_batches = [self.running_batch, self.last_batch]
|
||||
else:
|
||||
inflight_batches = [*self.running_mbs, *self.mbs]
|
||||
if self.enable_continuous_input_polling and self.result_queue:
|
||||
# Polling can run intake before the newest queued batch becomes last_batch.
|
||||
inflight_batches.extend(batch for batch, _ in self.result_queue)
|
||||
return inflight_batches
|
||||
return {
|
||||
req for batch in inflight_batches if batch is not None for req in batch.reqs
|
||||
}
|
||||
|
||||
def abort_request(self, recv_req: AbortReq):
|
||||
if (chunked_req := self.chunked_req) is not None:
|
||||
@@ -5651,12 +5614,7 @@ class Scheduler(
|
||||
return
|
||||
|
||||
if self.enable_overlap and self.last_batch:
|
||||
if (
|
||||
self.disaggregation_mode == DisaggregationMode.PREFILL
|
||||
and (req := self.chunked_req) is not None
|
||||
):
|
||||
# Retract skips the chunk step that sets this result's KV send boundary.
|
||||
req.tmp_end_idx = min(req.extend_range.end, len(req.origin_input_ids))
|
||||
# Process the results of the last batch
|
||||
tmp_batch, tmp_result = self.result_queue.popleft()
|
||||
self.process_batch_result(tmp_batch, tmp_result)
|
||||
|
||||
@@ -5990,8 +5948,6 @@ def dispatch_event_loop(scheduler: Scheduler):
|
||||
|
||||
|
||||
def _dispatch_event_loop_once(scheduler: Scheduler):
|
||||
# A PD role switch can select a different loop on the same scheduler.
|
||||
scheduler.enable_continuous_input_polling = False
|
||||
disaggregation_mode: DisaggregationMode = scheduler.disaggregation_mode
|
||||
if disaggregation_mode == DisaggregationMode.NULL:
|
||||
if scheduler.enable_pdmux:
|
||||
|
||||
Reference in New Issue
Block a user