diff --git a/deploy/b300-ds41/dockerserve-3p1d-otlp.yml b/deploy/b300-ds41/dockerserve-3p1d-otlp.yml index af45a7d554..056228c81b 100644 --- a/deploy/b300-ds41/dockerserve-3p1d-otlp.yml +++ b/deploy/b300-ds41/dockerserve-3p1d-otlp.yml @@ -10,7 +10,7 @@ # 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002 # bootstrap: P1=8998 P2=8999 P3=9000 D=9001 x-sglang-common: &sglang-common - image: ymkymx/sglang:dsv41-pd-visioncp-1f6ca2cf6-local-20261009-0333 + image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831 shm_size: "32gb" ipc: host pid: host diff --git a/deploy/b300-ds41/dockerserve-b4-dp.yml b/deploy/b300-ds41/dockerserve-b4-dp.yml index d641f953fb..78d1027fd3 100644 --- a/deploy/b300-ds41/dockerserve-b4-dp.yml +++ b/deploy/b300-ds41/dockerserve-b4-dp.yml @@ -1,6 +1,6 @@ services: sglang: - image: ymkymx/sglang:dsv41-pd-visioncp-1f6ca2cf6-local-20261009-0333 + image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831 container_name: ds41-sglang-b4 shm_size: "32gb" ipc: host diff --git a/deploy/b300-ds41/dockerserve-cpdspark-nr.yml b/deploy/b300-ds41/dockerserve-cpdspark-nr.yml index 49f3830659..7e709e61ae 100644 --- a/deploy/b300-ds41/dockerserve-cpdspark-nr.yml +++ b/deploy/b300-ds41/dockerserve-cpdspark-nr.yml @@ -1,6 +1,6 @@ services: sglang: - image: ymkymx/sglang:dsv41-pd-visioncp-1f6ca2cf6-local-20261009-0333 + image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831 container_name: ds41-cpdspark-nr shm_size: "32gb" ipc: host diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index e4c8b08258..c794b0ff89 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -21,7 +21,6 @@ from __future__ import annotations import hashlib import logging -import time from array import array from collections import deque from http import HTTPStatus @@ -104,12 +103,6 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -# TP=1 yield between polls of a pending result. An initial heuristic rather -# than a tuned value: long enough that the bootstrap, transfer and cache -# threads reliably win the GIL even when the input socket is empty, and far -# below the duration of any prefill forward, so it adds no visible latency. -PREFILL_INPUT_POLL_INTERVAL_S = 0.0001 - _is_npu = is_npu() @@ -615,12 +608,9 @@ class SchedulerDisaggregationPrefillMixin: self: Scheduler, running_batch: ScheduleBatch, last_batch: Optional[ScheduleBatch], - should_retry_storage_prefetch: bool = True, ) -> NextBatchPlan: self.process_pending_chunked_abort() - self._process_hicache_events( - should_retry_storage_prefetch=should_retry_storage_prefetch - ) + self._process_hicache_events() # HACK (byronhsu): reset the batch_is_full flag because we never enter update_running_batch which resets it # Otherwise, it hangs under high concurrency @@ -628,13 +618,7 @@ class SchedulerDisaggregationPrefillMixin: self.resolve_waiting_queue_bootstrap() - # Skip repeated chunk processing while polling a pending batch. - if last_batch is None or not last_batch.prefill_chunk_processed: - self.process_prefill_chunk( - last_batch=last_batch, running_batch=running_batch - ) - if last_batch is not None: - last_batch.prefill_chunk_processed = True + self.process_prefill_chunk(last_batch=last_batch, running_batch=running_batch) prefill_plan = self.get_new_batch_prefill(running_batch) batch = prefill_plan.batch_to_run @@ -685,120 +669,46 @@ class SchedulerDisaggregationPrefillMixin: # Update last_batch self.last_batch = batch - def _is_continuous_input_polling_enabled(self: Scheduler) -> bool: - """Check support once on entry to the prefill overlap loop.""" - if not envs.SGLANG_ENABLE_DISAGG_PREFILL_CONTINUOUS_INPUT_POLLING.get(): - return False - parallel = get_parallel() - enabled = ( - parallel.attn_dp_size == 1 - and parallel.attn_cp_size == 1 - and self.spec_algorithm.is_none() - and self.is_generation - and self.dllm_config is None - ) - if not enabled: - logger.warning( - "Continuous prefill input polling requires attention DP=CP=1 and " - "non-speculative autoregressive generation; using the regular overlap loop." - ) - return enabled - - def _yield_gil_if_needed(self: Scheduler) -> None: - if self.tp_size == 1: - # No collective parks this thread at TP=1. A bounded sleep lets - # bootstrap, transfer and cache workers reacquire the GIL. - time.sleep(PREFILL_INPUT_POLL_INTERVAL_S) - @torch.no_grad() def event_loop_overlap_disagg_prefill(self: Scheduler) -> None: - """Run intake while pending copies finish, keeping at most two forwards. - - A deferred pause stops preparation until the current result wait finishes. - Shutdown stops new launches; a wait finishes through the blocking handler. - After a failed preparation, only new input or bootstrap arrivals retry - during the wait; other readiness is checked after a result finishes. - """ - self.enable_continuous_input_polling = ( - self._is_continuous_input_polling_enabled() - ) self.result_queue = deque() - batch_result_completion_status = torch.empty(1, dtype=torch.int32, device="cpu") - # True when intake repeats while the oldest batch result is unfinished. - waiting_for_batch_result = False - # Avoid retrying failed batch preparation after every copy poll. - skip_batch_creation = False - # Keep the newer forward's sampling context across unfinished-result passes. - batch = None - batch_result = None while True: - if self.gracefully_exit: - break - - # Stop dispatching at pause requests while waiting for a batch result. - received_inputs = self.ingest_requests( - stop_at_pause=waiting_for_batch_result - ) + # Receive requests + self.ingest_requests() if self._engine_paused: self._record_scheduler_state_for_paused_engine() - self._yield_gil_if_needed() continue - bootstrapped_reqs = self.disagg_prefill_bootstrap_queue.pop_bootstrapped() - self.waiting_queue.extend(bootstrapped_reqs) - # Resume batch creation on completion, new/ready requests, or an idle step. - if not waiting_for_batch_result or received_inputs or bootstrapped_reqs: - skip_batch_creation = False + self.waiting_queue.extend( + self.disagg_prefill_bootstrap_queue.pop_bootstrapped() + ) - # Keep at most two batches in flight. - if ( - len(self.result_queue) < 2 - and not self.gracefully_exit - and not self._deferred_input_requests - and not skip_batch_creation - ): - # Get the next batch to run - plan = self.get_next_disagg_prefill_batch_to_run( - running_batch=self.running_batch, - last_batch=self.last_batch, - # Storage retry deadlines count scheduling steps, not copy polls. - should_retry_storage_prefetch=not waiting_for_batch_result, - ) - self.running_batch = plan.running_batch - batch = plan.batch_to_run - batch = self.ngram_embedding_manager.prepare_for_forward( - batch, chunked_req=self.chunked_req - ) - self.cur_batch_for_debug = batch - # Skip batch creation next round to avoid repeated preparation failures. - skip_batch_creation = batch is None + # Get the next batch to run + plan = self.get_next_disagg_prefill_batch_to_run( + running_batch=self.running_batch, last_batch=self.last_batch + ) + self.running_batch = plan.running_batch + batch = plan.batch_to_run + batch = self.ngram_embedding_manager.prepare_for_forward( + batch, chunked_req=self.chunked_req + ) + self.cur_batch_for_debug = batch - # Launch the current batch - if batch: - if self.enable_staging: - self.maybe_prefetch_staging_for_batch(batch) - batch_result = self.run_batch(batch) - self._apply_war_barrier() - self.result_queue.append((batch.copy(), batch_result)) - else: - batch_result = None + # Launch the current batch + if batch: + if self.enable_staging: + self.maybe_prefetch_staging_for_batch(batch) + batch_result = self.run_batch(batch) + self._apply_war_barrier() + self.result_queue.append((batch.copy(), batch_result)) + else: + batch_result = None + self._sched_idled = True + # Process the last batch if self.last_batch: - if self.enable_continuous_input_polling: - # Resume intake until the oldest result is ready on all TP ranks. - oldest_batch_result = self.result_queue[0][1] - if ( - not self.gracefully_exit - and not self.is_disagg_prefill_batch_result_ready( - oldest_batch_result, batch_result_completion_status - ) - ): - waiting_for_batch_result = True - self._yield_gil_if_needed() - continue - # Process the oldest batch result. - oldest_batch, oldest_batch_result = self.result_queue.popleft() - self.process_batch_result(oldest_batch, oldest_batch_result) + tmp_batch, tmp_result = self.result_queue.popleft() + self.process_batch_result(tmp_batch, tmp_result) elif batch is None: # When the server is idle, do self-check and re-init some states self.on_idle() @@ -809,36 +719,8 @@ class SchedulerDisaggregationPrefillMixin: # It depends on the result of the last batch (e.g., grammar), so we run it after the last batch is processed. self.launch_batch_sample_if_needed(batch_result, batch) - # Update last_batch and scheduler status - self._sched_idled = batch is None + # Update last_batch self.last_batch = batch - waiting_for_batch_result = False - - def is_disagg_prefill_batch_result_ready( - self: Scheduler, - batch_result: GenerationBatchResult, - batch_result_completion_status: torch.Tensor, - ) -> bool: - # 0: sampling/copy has not been submitted, so polling cannot make progress. - state = 0 - if ( - batch_result.copy_done is not None - and batch_result.delay_sample_func is None - ): - # 1: copy unfinished; 2: copy finished locally. - state = 2 if batch_result.copy_done.query() else 1 - if self.tp_size > 1: - batch_result_completion_status.fill_(state) - torch.distributed.all_reduce( - batch_result_completion_status, - op=torch.distributed.ReduceOp.MIN, - group=self.tp_cpu_group, - ) - state = batch_result_completion_status.item() - if state == 0: - # Every rank must submit delayed sampling before entering this wait. - raise RuntimeError("Prefill result copy must be submitted before polling") - return state == 2 def process_batch_result_disagg_prefill( self: Scheduler, @@ -1335,8 +1217,7 @@ class SchedulerDisaggregationPrefillMixin: elif self.has_bootstrapped_waiting_req(): # optimistic request yields to waiting requests self.chunked_req = None - # Retract may drain every result, leaving no handler to requeue. - if not self.enable_overlap or req.inflight_middle_chunks == 0: + if not self.enable_overlap: self.optimistic_release_and_requeue(req) # else: still bootstrapping, keep computing without sending elif self.enable_overlap: diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index def1728255..b497b0c3ce 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -658,8 +658,6 @@ class Envs: # do not fence the next forward through the scheduler stream. SGLANG_PP_COMM_OVERLAP = EnvBool(False) SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH = EnvBool(False) - # Opt-in: keep receiving prefill requests while forward results are pending. - SGLANG_ENABLE_DISAGG_PREFILL_CONTINUOUS_INPUT_POLLING = EnvBool(False) # =================================================================== # Radix and sparse KV caches diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index 59a0aac630..cda14197fd 100755 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -2335,9 +2335,6 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): chunked_req_next_prompt_token: Optional[int] = None contains_last_prefill_chunk: bool = True - # Tracks process_prefill_chunk() on the original scheduler batch only. - prefill_chunk_processed: bool = False - # For DP attention inner_idle_batch: Optional[ScheduleBatch] = None # Decode requests carried alongside a chunked-prefill batch diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 5ab3016556..47e0e160d6 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -1270,7 +1270,6 @@ class Scheduler( self.hisparse_coordinator.set_decode_producer_stream(self.forward_stream) def init_running_status(self): - self.tp_size = get_parallel().tp_size # Set by a runtime PD role switch to break out of the current event loop. self._event_loop_should_restart = False # Guards against concurrent/re-entrant PD role switches. @@ -1287,10 +1286,6 @@ class Scheduler( self.cur_batch_for_debug: Optional[ScheduleBatch] = None # The last forward batch self.last_batch: Optional[ScheduleBatch] = None - self.result_queue: deque[ - tuple[ScheduleBatch, GenerationBatchResult | EmbeddingBatchResult] - ] = deque() - self.enable_continuous_input_polling = False self.forward_ct = 0 self.return_health_check_ipcs: Deque[Optional[str]] = deque() self.flush_wrapper = SchedulerFlushWrapper( @@ -1302,7 +1297,6 @@ class Scheduler( self.session_controller = SessionController(self.tree_cache) self.forward_sleep_time = None self._engine_paused = False - self._deferred_input_requests: List = [] def init_chunked_prefill(self): self.chunked_prefill_size = get_schedule().chunked_prefill_size @@ -2073,38 +2067,20 @@ class Scheduler( for prev_batch, prev_result in self.result_queue: self.batch_result_processor.advance_grammar_fsm(prev_result, prev_batch) - def ingest_requests(self, stop_at_pause: bool = False) -> List: - """Receive and broadcast input, then dispatch queued requests in order. - - With stop_at_pause, retain the first pause and all following inputs - until the caller reaches a scheduling boundary. Receiving and broadcasting - continue each pass; all ranks retain the same suffix. + def ingest_requests(self) -> List: + """Receive, broadcast and dispatch this iteration's external input. The one place a new per-iteration input source belongs; the return value exists for the pipeline stages that relay requests onward. """ local_reqs = [] - # Avoid re-enqueuing timeout aborts while dispatch is stopped at a pause. if ( - not self._deferred_input_requests - and get_parallel().pp_rank == 0 + get_parallel().pp_rank == 0 and get_parallel().attn_tp_rank == 0 and get_parallel().attn_cp_rank == 0 ): local_reqs = self._poll_timeout_aborts() - self._deferred_input_requests.extend( - self.request_receiver.recv_requests(local_reqs=local_reqs) - ) - - num_requests_to_process = len(self._deferred_input_requests) - if stop_at_pause: - for index, req in enumerate(self._deferred_input_requests): - if isinstance(req, PauseGenerationReqInput): - num_requests_to_process = index - break - recv_reqs = self._deferred_input_requests[:num_requests_to_process] - del self._deferred_input_requests[:num_requests_to_process] - + recv_reqs = self.request_receiver.recv_requests(local_reqs=local_reqs) if recv_reqs: self.metrics_reporter.record_scheduler_active() self.process_input_requests(recv_reqs) @@ -3415,18 +3391,16 @@ class Scheduler( if (timeout_s := envs.SGLANG_REQ_RUNNING_TIMEOUT.get()) > 0: deadline = time.perf_counter() - timeout_s - inflight_batches = self._collect_inflight_batches() + if get_parallel().pp_size == 1: + inflight_batches = [self.running_batch, self.last_batch] + else: + inflight_batches = [*self.running_mbs, *self.mbs] seen_rids = set() for batch in inflight_batches: if batch is None: continue for req in batch.reqs: - # Polling must not re-emit an abort awaiting result processing. - if ( - req.rid in seen_rids - or req.finished() - or req.to_finish is not None - ): + if req.rid in seen_rids or req.finished(): continue seen_rids.add(req.rid) if 0 < req.time_stats.forward_entry_time < deadline: @@ -3636,9 +3610,7 @@ class Scheduler( # todo hisparse, maybe other info to contain for the new batch return batch - def _process_hicache_events( - self, should_retry_storage_prefetch: bool = True - ) -> None: + def _process_hicache_events(self) -> None: # The HiCache drain is TP-wide consensus; run it before rank-local # decisions (_should_defer_prefill) or ranks enter different collectives. if ( @@ -3647,7 +3619,7 @@ class Scheduler( or self.enable_unified_cache_external_linker ): self.tree_cache.check_hicache_events() - if self.enable_hicache_storage and should_retry_storage_prefetch: + if self.enable_hicache_storage: self._process_storage_prefetch_retries() @scheduler_stage_method(SCHEDULER_STAGE_GET_NEXT_BATCH) @@ -5461,22 +5433,13 @@ class Scheduler( ) def collect_inflight_reqs(self) -> Set[Req]: - return { - req - for batch in self._collect_inflight_batches() - if batch is not None - for req in batch.reqs - } - - def _collect_inflight_batches(self) -> list[ScheduleBatch | None]: if get_parallel().pp_size == 1: inflight_batches = [self.running_batch, self.last_batch] else: inflight_batches = [*self.running_mbs, *self.mbs] - if self.enable_continuous_input_polling and self.result_queue: - # Polling can run intake before the newest queued batch becomes last_batch. - inflight_batches.extend(batch for batch, _ in self.result_queue) - return inflight_batches + return { + req for batch in inflight_batches if batch is not None for req in batch.reqs + } def abort_request(self, recv_req: AbortReq): if (chunked_req := self.chunked_req) is not None: @@ -5651,12 +5614,7 @@ class Scheduler( return if self.enable_overlap and self.last_batch: - if ( - self.disaggregation_mode == DisaggregationMode.PREFILL - and (req := self.chunked_req) is not None - ): - # Retract skips the chunk step that sets this result's KV send boundary. - req.tmp_end_idx = min(req.extend_range.end, len(req.origin_input_ids)) + # Process the results of the last batch tmp_batch, tmp_result = self.result_queue.popleft() self.process_batch_result(tmp_batch, tmp_result) @@ -5990,8 +5948,6 @@ def dispatch_event_loop(scheduler: Scheduler): def _dispatch_event_loop_once(scheduler: Scheduler): - # A PD role switch can select a different loop on the same scheduler. - scheduler.enable_continuous_input_polling = False disaggregation_mode: DisaggregationMode = scheduler.disaggregation_mode if disaggregation_mode == DisaggregationMode.NULL: if scheduler.enable_pdmux: diff --git a/test/registered/unit/disaggregation/test_prefill_result_polling.py b/test/registered/unit/disaggregation/test_prefill_result_polling.py deleted file mode 100644 index 04a5daa3ac..0000000000 --- a/test/registered/unit/disaggregation/test_prefill_result_polling.py +++ /dev/null @@ -1,1426 +0,0 @@ -"""CPU coverage for continuous input polling and coordinated prefill completion.""" - -import multiprocessing -import tempfile -import unittest -from array import array -from collections import Counter, deque -from concurrent.futures import Future -from contextlib import contextmanager, nullcontext -from datetime import timedelta -from pathlib import Path -from types import SimpleNamespace -from unittest.mock import Mock, call, patch - -import torch -import torch.distributed as dist - -from sglang.srt.constrained.grammar_manager import GrammarManager -from sglang.srt.disaggregation.base import KVPoll -from sglang.srt.disaggregation.prefill import SchedulerDisaggregationPrefillMixin -from sglang.srt.disaggregation.utils import ( - FAKE_BOOTSTRAP_HOST, - DisaggregationMode, - ReqToMetadataIdxAllocator, -) -from sglang.srt.environ import envs -from sglang.srt.managers.io_struct import ( - ContinueGenerationReqInput, - PauseGenerationReqInput, - ShutdownReq, - TokenizedGenerateReqInput, -) -from sglang.srt.managers.schedule_batch import NextBatchPlan, Req, ScheduleBatch -from sglang.srt.managers.schedule_policy import SchedulePolicy -from sglang.srt.managers.scheduler import Scheduler, _dispatch_event_loop_once -from sglang.srt.managers.utils import GenerationBatchResult -from sglang.srt.mem_cache.radix_cache import RadixCache -from sglang.srt.mem_cache.storage_prefetch import StoragePrefetchRetries -from sglang.srt.model_executor.forward_batch_info import ForwardMode -from sglang.srt.runtime_context import get_parallel -from sglang.srt.sampling.sampling_params import SamplingParams -from sglang.srt.speculative.spec_info import SpeculativeAlgorithm -from sglang.test.ci.ci_register import register_cpu_ci -from sglang.test.test_utils import CustomTestCase, published_topology - -register_cpu_ci(est_time=20, suite="base-a-test-cpu") - - -class _StopLoop(Exception): - pass - - -class _Event: - def __init__(self, ready=False, recorded=True): - self.ready = ready - self.recorded = recorded - self.queries = 0 - self.synchronizations = 0 - self.blocking_synchronizations = 0 - self.records = 0 - - def record(self): - self.recorded = True - self.records += 1 - - def query(self): - assert self.recorded, "Polled a copy before submission" - self.queries += 1 - return self.ready - - def synchronize(self): - assert self.recorded, "Waited for a copy before submission" - self.synchronizations += 1 - if not self.ready: - self.blocking_synchronizations += 1 - self.ready = True - - -def _batch(forward_iter, reqs=None): - if reqs is None: - reqs = [Req(str(forward_iter), "", [1, 2, 3, 4], SamplingParams())] - return ScheduleBatch( - reqs=reqs, - forward_iter=forward_iter, - forward_mode=ForwardMode.EXTEND, - spec_algorithm=SpeculativeAlgorithm.NONE, - ) - - -def _result(event=None, delayed=False): - result = GenerationBatchResult( - next_token_ids=torch.tensor([11]), - copy_done=event if event is not None else _Event(), - ) - if delayed: - result.delay_sample_func = lambda: result - return result - - -def _input(batch): - return TokenizedGenerateReqInput( - rid=str(batch.forward_iter), - input_text="", - input_ids=array("q", [1, 2, 3, 4]), - input_embeds=None, - mm_inputs=None, - token_type_ids=None, - sampling_params=SamplingParams(), - return_logprob=False, - logprob_start_len=-1, - top_logprobs_num=0, - token_ids_logprob=None, - stream=False, - ) - - -class _Scheduler(SchedulerDisaggregationPrefillMixin): - """Run real scheduling and result handling with controlled intake and GPU events.""" - - launch_batch_sample_if_needed = Scheduler.launch_batch_sample_if_needed - pause_generation = Scheduler.pause_generation - continue_generation = Scheduler.continue_generation - - def __init__( - self, - arrivals, - results, - ready_at, - pause_at=None, - resume_at=None, - pause_mode="in_place", - ): - self.arrivals = arrivals - self.batches = {b.forward_iter: b for b in arrivals if b is not None} - self.results = results - self.copy_events = {name: result.copy_done for name, result in results.items()} - self.ready_at = ready_at - self.ready_batches = deque() - self.request_receiver = SimpleNamespace(recv_requests=self.recv_requests) - self._deferred_input_requests = [] - self._incoming_requests = [] - self.pause_at = pause_at - self.resume_at = resume_at - self.pause_mode = pause_mode - self.paused_states = [] - self.iteration = -1 - self.actions = [] - self.admission_attempts = [] - self.chunk_processes = Counter() - self.result_processes = Counter() - self.depths = [] - self.scheduler_stage_metrics = None - self.disaggregation_mode = DisaggregationMode.PREFILL - self.spec_algorithm = SpeculativeAlgorithm.NONE - self.is_generation = True - self.dllm_config = None - self.tp_cpu_group = None - self.tp_size = 1 - self.attn_cp_cpu_group = None - self.attn_tp_cpu_group = None - self.last_batch = None - self.result_queue = deque() - self.chunked_req = None - self.decode_offload_manager = None - self.running_batch = ScheduleBatch(reqs=[]) - self.waiting_queue = [] - self.cur_batch_for_debug = None - self._engine_paused = False - self.gracefully_exit = False - self.enable_staging = False - self.enable_overlap = True - self.disagg_prefill_bootstrap_queue = SimpleNamespace( - pop_bootstrapped=lambda: [] - ) - self.ngram_embedding_manager = SimpleNamespace( - prepare_for_forward=lambda batch, chunked_req: batch - ) - self.forward_stream_ctx = nullcontext() - self.copy_stream_ctx = nullcontext() - self.forward_stream = Mock(spec=["wait_stream"]) - self.copy_stream = Mock(spec=["wait_stream"]) - self.schedule_stream = object() - self._relay_forward_payload = Mock() - self.tree_cache = Mock(spec=["flush_pending_backups", "finish"]) - self.batch_result_processor = Mock( - spec=["snapshot_auxiliary_output_starts", "move_logprobs_to_cpu"] - ) - self.metrics_reporter = Mock( - spec=[ - "report_prefill_stats", - "log_batch_result_stats", - "update_device_timer", - "record_scheduler_active", - ] - ) - self.metrics_reporter.current_scheduler_metrics_enabled = False - self.kv_events_publisher = Mock(spec=["publish_kv_events"]) - self.publish_load_snapshot = Mock() - self.load_publisher = Mock(spec=["publish_load_stat"]) - self.load_inquirer = SimpleNamespace(get_loads=Mock()) - self.enable_fpm = False - self._record_step_counters = Mock() - self._maybe_clear_mm_inputs = Mock() - self.maybe_send_health_check_signal = Mock() - self.disagg_prefill_inflight_queue = [] - self.send_kv_chunk = Mock() - self.process_pending_chunked_abort = Mock() - self.dp_attn_adapter = SimpleNamespace( - maybe_prepare_mlp_sync_batch=lambda batch: batch - ) - - def _process_hicache_events(self, should_retry_storage_prefetch=True): - pass - - def _poll_timeout_aborts(self): - return [] - - def ingest_requests(self, stop_at_pause=False): - # GPU completion and incoming traffic advance even when intake is held. - self.iteration += 1 - if self.iteration == len(self.arrivals): - raise _StopLoop - self.depths.append(len(self.result_queue)) - for name, iteration in self.ready_at.items(): - event = self.copy_events[name] - event.ready = event.ready or self.iteration >= iteration - self._incoming_requests.extend(self._inputs_for_step()) - return Scheduler.ingest_requests(self, stop_at_pause=stop_at_pause) - - def recv_requests(self, local_reqs=None): - self.actions.append((self.iteration, "receive", None)) - if self.tp_cpu_group is not None: - # A completion decision must never reorder the next intake collective. - tick = torch.tensor([self.iteration]) - dist.broadcast(tick, src=0, group=self.tp_cpu_group) - assert tick.item() == self.iteration - inputs = self._incoming_requests - self._incoming_requests = [] - return inputs + (local_reqs or []) - - def _inputs_for_step(self): - inputs = [] - if self.iteration == self.pause_at: - inputs.append(PauseGenerationReqInput(mode=self.pause_mode)) - if self.iteration == self.resume_at: - inputs.append(ContinueGenerationReqInput(torch_empty_cache=False)) - if (batch := self.arrivals[self.iteration]) is not None: - inputs.append(_input(batch)) - return inputs - - def process_input_requests(self, recv_reqs): - for req in recv_reqs: - name = ( - req.rid - if isinstance(req, TokenizedGenerateReqInput) - else type(req).__name__ - ) - self.actions.append((self.iteration, "dispatch", name)) - if isinstance(req, TokenizedGenerateReqInput): - self.ready_batches.append(self.batches[int(req.rid)]) - elif isinstance(req, PauseGenerationReqInput): - assert len(self.result_queue) <= 1, "Pause bypassed a result wait" - self.pause_generation(req) - elif isinstance(req, ContinueGenerationReqInput): - self.continue_generation(req) - elif isinstance(req, ShutdownReq): - Scheduler.handle_shutdown(self, req) - else: - raise AssertionError(f"Unexpected input: {req}") - - def _record_scheduler_state_for_paused_engine(self): - self.actions.append((self.iteration, "pause", None)) - self.paused_states.append((self.last_batch, tuple(self.result_queue))) - - def process_prefill_chunk(self, last_batch, running_batch): - if last_batch is not None: - self.chunk_processes[last_batch.forward_iter] += 1 - SchedulerDisaggregationPrefillMixin.process_prefill_chunk( - self, last_batch=last_batch, running_batch=running_batch - ) - - def get_new_batch_prefill(self, running_batch): - self.admission_attempts.append(self.iteration) - batch = self.ready_batches.popleft() if self.ready_batches else None - if batch is not None: - self.actions.append((self.iteration, "prepare_batch", batch.forward_iter)) - self.waiting_queue = [ - req for req in self.waiting_queue if req not in batch.reqs - ] - return NextBatchPlan(batch_to_run=batch, running_batch=running_batch) - - def run_batch(self, batch): - assert len(self.result_queue) < 2, "Overwrote a live forward's keep-alive slot" - self.actions.append((self.iteration, "launch", batch.forward_iter)) - result = self.results[batch.forward_iter] - if result.delay_sample_func is not None: - - def sample(): - self.actions.append((self.iteration, "sample", batch.forward_iter)) - return result - - result.delay_sample_func = sample - return result - - def _apply_war_barrier(self): - pass - - def process_batch_result(self, batch, result): - assert all(pending is not result for _, pending in self.result_queue) - self.result_processes[batch.forward_iter] += 1 - self.actions.append((self.iteration, "process_result", batch.forward_iter)) - Scheduler.process_batch_result(self, batch, result) - event = self.copy_events[batch.forward_iter] - assert event.ready and event.recorded - assert result.delay_sample_func is None - self.actions.append((self.iteration, "result", batch.forward_iter)) - - def process_disagg_prefill_inflight_queue(self): - self.actions.append((self.iteration, "inflight", None)) - - def on_idle(self): - self.actions.append((self.iteration, "idle", None)) - - -def _run_loop(scheduler, enabled=True, **topology): - with ( - published_topology(**topology), - envs.SGLANG_ENABLE_DISAGG_PREFILL_CONTINUOUS_INPUT_POLLING.override(enabled), - patch("sglang.srt.disaggregation.prefill.maybe_cache_unfinished_req"), - ): - scheduler.tp_size = get_parallel().tp_size - try: - _dispatch_event_loop_once(scheduler) - except _StopLoop: - return - if scheduler.gracefully_exit: - return - raise AssertionError("Scheduler exited before all arrivals were polled") - - -@contextmanager -def _single_rank_gloo_group(): - dist.init_process_group("gloo", store=dist.HashStore(), rank=0, world_size=1) - try: - yield - finally: - dist.destroy_process_group() - - -def _rank_consensus_worker(rank, rendezvous): - dist.init_process_group( - "gloo", - init_method=f"file://{rendezvous}", - rank=rank, - world_size=2, - timeout=timedelta(seconds=20), - ) - try: - scheduler = _Scheduler( - [_batch(1), None, _batch(2)] + [None] * 6, - {name: _result() for name in (1, 2)}, - {1: (2, 5)[rank], 2: (7, 6)[rank]}, - ) - scheduler.tp_cpu_group = dist.group.WORLD - _run_loop(scheduler, tp_size=2, ranks={"world_rank": rank}) - finished = [a for a in scheduler.actions if a[1] == "result"] - assert finished == [(5, "result", 1), (7, "result", 2)], finished - assert max(scheduler.depths) == 2 - for event in scheduler.copy_events.values(): - assert event.synchronizations == 1 - assert event.blocking_synchronizations == 0 - - for depth in (1, 2): - for mode in ("in_place", "retract"): - arrivals = [ - _batch(1), - _batch(2) if depth == 2 else None, - None, - _batch(3), - ] - names = (1, 2, 3) if depth == 2 else (1, 3) - scheduler = _Scheduler( - arrivals + [None] * 8, - { - name: _result(_Event(recorded=False), delayed=True) - for name in names - }, - { - 1: (3, 5)[rank], - 3: (10, 9)[rank], - **({2: (8, 7)[rank]} if depth == 2 else {}), - }, - pause_at=2, - resume_at=7, - pause_mode=mode, - ) - scheduler.tp_cpu_group = dist.group.WORLD - _run_loop(scheduler, tp_size=2, ranks={"world_rank": rank}) - assert (5, "result", 1) in scheduler.actions - assert (6, "dispatch", "PauseGenerationReqInput") in scheduler.actions - assert (6, "dispatch", "3") in scheduler.actions - assert (7, "launch", 3) in scheduler.actions - assert not set(range(2, 7)).intersection(scheduler.admission_attempts) - assert all( - (step, "receive", None) in scheduler.actions for step in range(3, 7) - ) - assert scheduler.result_processes == {name: 1 for name in names} - assert [a[2] for a in scheduler.actions if a[1] == "result"] == list( - names - ) - assert not scheduler._deferred_input_requests - for name, event in scheduler.copy_events.items(): - assert event.synchronizations == 1 - assert event.blocking_synchronizations == int( - name == 2 and mode == "retract" - ) - assert event.records == 1 - - # An unsubmitted copy on either rank is a lifecycle error on every rank. - with published_topology(tp_size=2, ranks={"world_rank": rank}): - batch_result_completion_status = torch.empty(1, dtype=torch.int32) - for missing_event in (True, False): - result = _result() - if rank == 0: - if missing_event: - result.copy_done = None - else: - result.delay_sample_func = lambda: None - result.copy_done.recorded = False - scheduler.result_queue = deque([(_batch(99), result)]) - try: - scheduler.is_disagg_prefill_batch_result_ready( - result, batch_result_completion_status - ) - except RuntimeError as error: - assert "must be submitted" in str(error) - else: - raise AssertionError("An unsubmitted copy was polled") - finally: - dist.destroy_process_group() - - -class TestPrefillResultPolling(CustomTestCase): - def test_intake_retains_pause_suffix_while_continuing_to_receive(self): - for stop_at_pause in (False, True): - for prefix_size in (0, 1): - with self.subTest(stop_at_pause=stop_at_pause, prefix_size=prefix_size): - prefix = [object()] * prefix_size - suffix = [ - PauseGenerationReqInput(mode="retract"), - object(), - ShutdownReq(), - ] - newer = [object() for _ in range(4)] - receiver = Mock( - side_effect=[prefix + suffix, *[[req] for req in newer]] - ) - scheduler = SimpleNamespace( - _deferred_input_requests=[], - _poll_timeout_aborts=Mock(return_value=[]), - request_receiver=SimpleNamespace(recv_requests=receiver), - metrics_reporter=Mock(), - process_input_requests=Mock(), - ) - with published_topology(): - received = [ - Scheduler.ingest_requests(scheduler, stop_at_pause=stop) - for stop in (stop_at_pause,) * 3 + (False, False) - ] - self.assertEqual( - received, - [prefix, [], [], suffix + newer[:3], newer[3:]] - if stop_at_pause - else [prefix + suffix, *[[req] for req in newer]], - ) - self.assertEqual(receiver.call_count, 5) - # A held pause must not accumulate repeated timeout aborts. - self.assertEqual( - scheduler._poll_timeout_aborts.call_count, - 2 if stop_at_pause else 5, - ) - self.assertFalse(scheduler._deferred_input_requests) - dispatched = [ - req - for args in scheduler.process_input_requests.call_args_list - for req in args.args[0] - ] - self.assertEqual(dispatched, prefix + suffix + newer) - - def test_deferred_pause_stops_preparation_with_a_free_forward_slot(self): - class ReadyAfterPauseScheduler(_Scheduler): - def get_new_batch_prefill(self, running_batch): - if self.iteration == 1: - self.admission_attempts.append(self.iteration) - return NextBatchPlan(batch_to_run=None, running_batch=running_batch) - return super().get_new_batch_prefill(running_batch) - - scheduler = ReadyAfterPauseScheduler( - [_batch(1), _batch(2), None, _batch(3)] + [None] * 8, - {name: _result() for name in (1, 2, 3)}, - {1: 4, 2: 8, 3: 10}, - pause_at=2, - resume_at=6, - ) - _run_loop(scheduler) - - self.assertIn((4, "result", 1), scheduler.actions) - self.assertIn((5, "dispatch", "PauseGenerationReqInput"), scheduler.actions) - self.assertIn((5, "dispatch", "3"), scheduler.actions) - self.assertIn((6, "launch", 2), scheduler.actions) - for iteration in range(2, 6): - self.assertNotIn(iteration, scheduler.admission_attempts) - for iteration in range(3, 6): - self.assertIn((iteration, "receive", None), scheduler.actions) - self.assertEqual(scheduler.result_processes, {1: 1, 2: 1, 3: 1}) - self.assertFalse(scheduler._deferred_input_requests) - - @unittest.skipUnless(dist.is_gloo_available(), "requires Gloo") - def test_empty_polls_and_arrival_bursts_preserve_storage_retry_pacing(self): - class CapacityBlockedScheduler(_Scheduler): - _process_hicache_events = Scheduler._process_hicache_events - _process_storage_prefetch_retries = ( - Scheduler._process_storage_prefetch_retries - ) - _retry_storage_prefetch = Scheduler._retry_storage_prefetch - - def _prefetch_kvcache(self, req, storage_hit_end): - self.reissues.append(self.iteration) - self.tree_cache.storage_prefetch_retries.poll_miss(req.rid) - - def get_new_batch_prefill(self, running_batch): - if self.iteration > 0 and not self.result_processes[1]: - self.admission_attempts.append(self.iteration) - self.policy.calc_priority(self.waiting_queue, running_batch) - return NextBatchPlan(batch_to_run=None, running_batch=running_batch) - return super().get_new_batch_prefill(running_batch) - - for arrivals_during_wait in (False, True): - with self.subTest(arrivals_during_wait=arrivals_during_wait): - arrivals = [_batch(1)] + [None] * 13 - if arrivals_during_wait: - arrivals[2:12] = [_batch(i) for i in range(2, 12)] - scheduler = CapacityBlockedScheduler(arrivals, {1: _result()}, {1: 13}) - scheduler.enable_hierarchical_cache = True - scheduler.enable_hicache_storage = True - scheduler.reissues = [] - retries = StoragePrefetchRetries() - radix_cache = RadixCache.create_simulated() - matched_at = [] - - def match_prefix(params): - matched_at.append(scheduler.iteration) - return radix_cache.match_prefix(params) - - scheduler.tree_cache = SimpleNamespace( - check_hicache_events=Mock(), - flush_pending_backups=Mock(), - storage_prefetch_retries=retries, - supports_fast_match_prefix=lambda: True, - swa_reprefill_tail_tokens=lambda: 0, - match_prefix=match_prefix, - ) - scheduler.policy = SchedulePolicy( - "fcfs", scheduler.tree_cache, False, False, False - ) - scheduler.ngram_embedding_manager.prepare_for_forward = Mock( - side_effect=lambda batch, chunked_req: batch - ) - head, paced = (_batch(i).reqs[0] for i in (90, 91)) - for req in (head, paced): - req.origin_input_ids = array("q", req.origin_input_ids) - req.pending_bootstrap = False - req.disagg_kv_sender = SimpleNamespace( - poll=lambda: KVPoll.WaitingForInput - ) - scheduler.waiting_queue = [head, paced] - retries.poll_miss(paced.rid) - - with _single_rank_gloo_group(): - _run_loop(scheduler, hicache_storage_prefetch_retry_poll_interval=2) - - attempts = list(range(12)) if arrivals_during_wait else [0, 1] - self.assertEqual(scheduler.admission_attempts, attempts) - self.assertEqual( - matched_at, [step for step in attempts[1:] for _ in range(2)] - ) - self.assertEqual( - scheduler.ngram_embedding_manager.prepare_for_forward.call_count, - len(attempts), - ) - # Empty polls still receive inputs and query completion after a miss. - self.assertEqual(scheduler.copy_events[1].queries, 13) - self.assertEqual( - [ - step - for step, action, _ in scheduler.actions - if action == "receive" - ], - list(range(14)), - ) - self.assertEqual(scheduler.reissues, []) - self.assertEqual(paced.storage_prefetch_retry_attempts, 0) - self.assertIn((13, "result", 1), scheduler.actions) - # The next outer admission still gets the retry due at step three. - self.assertEqual( - retries.pop_ready([head, paced], 2, 8), [(paced, None)] - ) - - def test_deferred_readiness_retries_after_completion_or_while_idle(self): - class ReadinessBlockedScheduler(_Scheduler): - _process_hicache_events = Scheduler._process_hicache_events - - def process_input_requests(self, inputs): - super().process_input_requests(inputs) - if self.readiness_source == "grammar" and any( - isinstance(req, TokenizedGenerateReqInput) and req.rid == "2" - for req in inputs - ): - req = self.ready_batches.pop().reqs[0] - req.grammar = self.grammar_future - req.grammar_key = ("regex", "[ab]") - self.grammar_manager.grammar_queue.append(req) - - def _inputs_for_step(self): - if self.readiness_source == "grammar" and self.iteration == 3: - self.grammar_future.set_result(self.compiled_grammar) - return super()._inputs_for_step() - - def get_new_batch_prefill(self, running_batch): - if self.readiness_source == "grammar": - if self.grammar_manager.has_waiting_grammars(): - ready = self.grammar_manager.get_ready_grammar_requests() - self.waiting_queue.extend(ready) - if ready: - self.ready_batches.append(self.batches[2]) - elif ( - self.ready_batches - and self.iteration > 0 - and not self.prefetch_ready - ): - self.admission_attempts.append(self.iteration) - return NextBatchPlan(batch_to_run=None, running_batch=running_batch) - return super().get_new_batch_prefill(running_batch) - - for readiness_source in ("cache", "grammar"): - for pending_forward, enabled in ( - (False, False), - (False, True), - (True, False), - (True, True), - ): - with self.subTest( - readiness_source=readiness_source, - pending_forward=pending_forward, - continuous_input_polling=enabled, - ): - names = (1, 2) if pending_forward else (2,) - scheduler = ReadinessBlockedScheduler( - [_batch(1) if pending_forward else None, _batch(2)] - + [None] * 8, - {name: _result() for name in names}, - {2: 7, **({1: 5} if pending_forward else {})}, - ) - scheduler.readiness_source = readiness_source - scheduler.prefetch_ready = False - scheduler.enable_hierarchical_cache = readiness_source == "cache" - scheduler.enable_unified_cache_external_linker = False - scheduler.enable_lmcache = False - scheduler.enable_hicache_storage = False - - def drain_cache_events(): - # Readiness is visible only after the real cache-drain path. - if scheduler.iteration >= 3: - scheduler.prefetch_ready = True - - scheduler.tree_cache = SimpleNamespace( - check_hicache_events=drain_cache_events, - flush_pending_backups=Mock(), - finish=Mock(), - ) - if readiness_source == "grammar": - scheduler.grammar_future = Future() - scheduler.compiled_grammar = Mock( - spec=["copy", "accept_token", "finished"] - ) - with published_topology(skip_tokenizer_init=True): - scheduler.grammar_manager = GrammarManager( - SimpleNamespace( - server_args=SimpleNamespace(), - dp_tp_cpu_group=None, - dp_tp_group=SimpleNamespace( - world_size=1, first_rank=0, is_first_rank=True - ), - pp_group=None, - ) - ) - scheduler.grammar_manager.grammar_backend = Mock( - spec=["set_cache"] - ) - _run_loop(scheduler, enabled=enabled) - - # Long-tail readiness waits for completion, but an empty pipeline - # retries normally and must not retain a failed-creation marker. - deferred = pending_forward and enabled - self.assertIn( - (6 if deferred else 3, "launch", 2), scheduler.actions - ) - if deferred: - self.assertEqual( - [step for step in scheduler.admission_attempts if step < 6], - [0, 1], - ) - else: - self.assertEqual(scheduler.admission_attempts[:4], [0, 1, 2, 3]) - if pending_forward: - self.assertIn( - (5 if enabled else 1, "result", 1), scheduler.actions - ) - self.assertEqual( - scheduler.result_processes, {name: 1 for name in names} - ) - if readiness_source == "grammar": - self.assertIs( - scheduler.batches[2].reqs[0].grammar, - scheduler.compiled_grammar, - ) - self.assertFalse( - scheduler.grammar_manager.has_waiting_grammars() - ) - - @unittest.skipUnless(dist.is_gloo_available(), "requires Gloo") - def test_bootstrap_completion_admits_without_new_input(self): - second = _batch(2) - req = second.reqs[0] - req.pending_bootstrap = False - req.disagg_kv_sender = SimpleNamespace(poll=lambda: KVPoll.WaitingForInput) - scheduler = _Scheduler( - [_batch(1)] + [None] * 7, - {1: _result(), 2: _result()}, - {1: 5, 2: 6}, - ) - - def pop_bootstrapped(): - if scheduler.iteration == 3: - scheduler.ready_batches.append(second) - return [req] - return [] - - scheduler.disagg_prefill_bootstrap_queue.pop_bootstrapped = pop_bootstrapped - with _single_rank_gloo_group(): - _run_loop(scheduler) - - self.assertIn((3, "launch", 2), scheduler.actions) - self.assertIn((5, "result", 1), scheduler.actions) - self.assertEqual(scheduler.result_processes, {1: 1, 2: 1}) - - @unittest.skipUnless(dist.is_gloo_available(), "requires Gloo") - def test_transfer_retirement_waits_for_batch_result_before_freeing_metadata(self): - second = _batch(2) - - class MetadataBlockedScheduler(_Scheduler): - process_disagg_prefill_inflight_queue = SchedulerDisaggregationPrefillMixin.process_disagg_prefill_inflight_queue - - def run_batch(self, batch): - result = super().run_batch(batch) - if batch.forward_iter == 1: - self.ready_batches.append(second) - self.waiting_queue.extend(second.reqs) - return result - - def get_new_batch_prefill(self, running_batch): - if self.iteration > 0 and second in self.ready_batches: - index = self.req_to_metadata_buffer_idx_allocator.alloc() - if index is None: - self.admission_attempts.append(self.iteration) - return NextBatchPlan( - batch_to_run=None, running_batch=running_batch - ) - second.reqs[0].metadata_buffer_index = index - return super().get_new_batch_prefill(running_batch) - - first = _batch(1) - scheduler = MetadataBlockedScheduler( - [first] + [None] * 7, - {1: _result(), 2: _result()}, - {1: 5, 2: 6}, - ) - allocator = ReqToMetadataIdxAllocator(2) - scheduler.req_to_metadata_buffer_idx_allocator = allocator - scheduler.output_streamer = SimpleNamespace(stream_output=Mock()) - prior = _batch(0).reqs[0] - prior.metadata_buffer_index = allocator.alloc() - prior.pending_bootstrap = False - prior.bootstrap_host = FAKE_BOOTSTRAP_HOST - prior.disagg_kv_sender = SimpleNamespace( - poll=lambda: ( - KVPoll.Success if scheduler.iteration >= 3 else KVPoll.Transferring - ), - clear=Mock(), - ) - scheduler.disagg_prefill_inflight_queue = [prior] - first.reqs[0].metadata_buffer_index = allocator.alloc() - for req in (first.reqs[0], second.reqs[0]): - req.pending_bootstrap = False - req.disagg_kv_sender = SimpleNamespace(poll=lambda: KVPoll.Transferring) - with ( - _single_rank_gloo_group(), - patch("sglang.srt.disaggregation.prefill.release_kv_cache") as release_kv, - ): - _run_loop(scheduler) - - release_kv.assert_called_once_with(prior, scheduler.tree_cache) - self.assertEqual(prior.metadata_buffer_index, -1) - self.assertEqual(second.reqs[0].metadata_buffer_index, 0) - self.assertIn((6, "launch", 2), scheduler.actions) - self.assertIn((5, "result", 1), scheduler.actions) - self.assertEqual(scheduler.admission_attempts[:3], [0, 1, 6]) - self.assertEqual(scheduler.copy_events[1].blocking_synchronizations, 0) - self.assertEqual(scheduler.result_processes, {1: 1, 2: 1}) - - def test_input_polling_continues_at_both_queue_depths(self): - scheduler = _Scheduler( - [_batch(1), None, _batch(2), _batch(3)] + [None] * 6, - {name: _result() for name in (1, 2, 3)}, - {1: 6, 2: 4, 3: 8}, - ) - _run_loop(scheduler) - - finished = [a for a in scheduler.actions if a[1] == "result"] - self.assertEqual( - finished, - [ - (6, "result", 1), - (7, "result", 2), - (8, "result", 3), - ], - ) - self.assertIn((2, "launch", 2), scheduler.actions) - self.assertIn((7, "launch", 3), scheduler.actions) - self.assertEqual( - [a for a in scheduler.actions if a[1] == "prepare_batch"], - [(0, "prepare_batch", 1), (2, "prepare_batch", 2), (7, "prepare_batch", 3)], - ) - self.assertEqual(max(scheduler.depths), 2) - self.assertEqual(scheduler.chunk_processes, {1: 1, 2: 1, 3: 1}) - self.assertEqual(scheduler.result_processes, {1: 1, 2: 1, 3: 1}) - self.assertEqual(scheduler.load_publisher.publish_load_stat.call_count, 3) - self.assertEqual(scheduler.tree_cache.flush_pending_backups.call_count, 3) - self.assertEqual(scheduler._record_step_counters.call_count, 3) - for iteration in range(1, 8): - self.assertIn((iteration, "receive", None), scheduler.actions) - self.assertNotIn((iteration, "idle", None), scheduler.actions) - self.assertEqual( - [a for a in scheduler.actions if a[1] == "process_result"], - [ - (6, "process_result", 1), - (7, "process_result", 2), - (8, "process_result", 3), - ], - ) - self.assertFalse(scheduler.result_queue) - self.assertIsNone(scheduler.last_batch) - self.assertEqual( - [req.rid for req in scheduler.disagg_prefill_inflight_queue], - ["1", "2", "3"], - ) - self.assertEqual( - [list(req.output_ids) for req in scheduler.disagg_prefill_inflight_queue], - [[11], [11], [11]], - ) - for event in scheduler.copy_events.values(): - self.assertEqual(event.synchronizations, 1) - self.assertEqual(event.blocking_synchronizations, 0) - - def test_waiting_and_paused_tp1_passes_allow_worker_progress(self): - scheduler = _Scheduler( - [_batch(1)] + [None] * 9, - {1: _result()}, - {1: 5}, - pause_at=3, - resume_at=8, - ) - parked_passes = [] - - def park(interval_s): - self.assertGreater(interval_s, 0) - self.assertLessEqual(interval_s, 0.001) - parked_passes.append(scheduler.iteration) - - with patch("sglang.srt.disaggregation.prefill.time.sleep", side_effect=park): - _run_loop(scheduler) - - # Copy waits 1..4 and paused passes 6..7 must let workers reacquire the GIL. - self.assertEqual(parked_passes, [1, 2, 3, 4, 6, 7]) - - def test_new_batches_process_each_forward_once(self): - for overlap, enabled in ((False, False), (True, False), (True, True)): - with self.subTest(overlap=overlap, continuous_input_polling=enabled): - first, second = _batch(1), _batch(2) - results = {i: _result(_Event(ready=True)) for i in (1, 2)} - scheduler = _Scheduler( - [first, None, None, second, None, None], results, ready_at={} - ) - scheduler.enable_overlap = overlap - _run_loop(scheduler, enabled=enabled) - - self.assertEqual(scheduler.chunk_processes, {1: 1, 2: 1}) - self.assertEqual(scheduler.result_processes, {1: 1, 2: 1}) - self.assertEqual( - scheduler.load_publisher.publish_load_stat.call_count, 2 - ) - first_done = 1 if overlap else 0 - self.assertEqual( - [a for a in scheduler.actions if a[1] == "result"], - [ - (first_done, "result", 1), - (first_done + 3, "result", 2), - ], - ) - self.assertFalse(scheduler.result_queue) - - def test_delayed_sample_is_submitted_after_predecessor_before_query(self): - for enabled in (False, True): - with self.subTest(enabled=enabled): - scheduler = _Scheduler( - [_batch(1), _batch(2)] + [None] * 5, - { - 1: _result(_Event(recorded=False), delayed=True), - 2: _result(_Event(recorded=False), delayed=True), - }, - {1: 4, 2: 5}, - ) - _run_loop(scheduler, enabled=enabled) - first_done, second_done = (4, 5) if enabled else (1, 2) - self.assertIn((0, "sample", 1), scheduler.actions) - self.assertLess( - scheduler.actions.index((first_done, "result", 1)), - scheduler.actions.index((first_done, "sample", 2)), - ) - self.assertIn((second_done, "result", 2), scheduler.actions) - for name, result in scheduler.results.items(): - event = scheduler.copy_events[name] - self.assertEqual(event.records, 1) - self.assertIsNone(result.delay_sample_func) - self.assertEqual(event.synchronizations, 1) - self.assertEqual(event.blocking_synchronizations, int(not enabled)) - - def test_in_place_pause_accepts_inputs_and_preserves_pending_results(self): - for third_at in (3, 4): - with self.subTest(third_at=third_at): - first, second, third = _batch(1), _batch(2), _batch(3) - arrivals = [first, second] + [None] * 8 - arrivals[third_at] = third - scheduler = _Scheduler( - arrivals, - {name: _result() for name in (1, 2, 3)}, - {1: 3, 2: 6, 3: 8}, - pause_at=3, - resume_at=5, - ) - _run_loop(scheduler) - - self.assertEqual(len(scheduler.paused_states), 1) - for last_batch, pending in scheduler.paused_states: - self.assertIs(last_batch, second) - self.assertEqual([b.forward_iter for b, _ in pending], [2]) - self.assertEqual( - [a for a in scheduler.actions if a[1] == "dispatch"], - [ - (0, "dispatch", "1"), - (1, "dispatch", "2"), - (4, "dispatch", "PauseGenerationReqInput"), - (4, "dispatch", "3"), - (5, "dispatch", "ContinueGenerationReqInput"), - ], - ) - self.assertEqual( - [a for a in scheduler.actions if a[1] == "result"], - [(3, "result", 1), (6, "result", 2), (8, "result", 3)], - ) - self.assertIn((5, "launch", 3), scheduler.actions) - self.assertEqual(scheduler.result_processes, {1: 1, 2: 1, 3: 1}) - for event in scheduler.copy_events.values(): - self.assertEqual(event.synchronizations, 1) - self.assertEqual(event.blocking_synchronizations, 0) - - def test_retract_during_copy_wait_finishes_each_result_once(self): - for enabled, second_at in ((False, None), (True, None), (True, 1), (True, 2)): - with self.subTest(enabled=enabled, second_at=second_at): - pending_count = 1 if second_at is None else 2 - pause_at = 3 if enabled else 1 - results = { - name: _result(_Event(recorded=False), delayed=True) - for name in range(1, pending_count + 1) - } - results[3] = _result() - arrivals = [None] * 12 - arrivals[0] = _batch(1) - if second_at is not None: - arrivals[second_at] = _batch(2) - arrivals[5] = _batch(3) - scheduler = _Scheduler( - arrivals, - results, - { - 1: 4, - **({2: 20} if pending_count == 2 else {}), - 3: 10, - }, - pause_at=pause_at, - resume_at=7, - pause_mode="retract", - ) - _run_loop(scheduler, enabled=enabled) - - self.assertEqual( - [a for a in scheduler.actions if a[1] == "result"], - [(4 if enabled else 1, "result", 1)] - + ([(5, "result", 2)] if pending_count == 2 else []) - + [(10 if enabled else 8, "result", 3)], - ) - self.assertEqual( - scheduler.result_processes, {name: 1 for name in results} - ) - self.assertEqual( - scheduler.chunk_processes, - # Pause finishes results; resume owns the next chunk preparation. - {1: 1, 3: 1} if enabled else {3: 1}, - ) - self.assertEqual( - [ - list(req.output_ids) - for req in scheduler.disagg_prefill_inflight_queue - ], - [[11]] * len(results), - ) - for name, event in scheduler.copy_events.items(): - self.assertEqual(event.synchronizations, 1) - self.assertEqual( - event.blocking_synchronizations, int(name == 2 or not enabled) - ) - if name != 3: - self.assertEqual(event.records, 1) - self.assertIsNone(results[name].delay_sample_func) - self.assertFalse(scheduler.result_queue) - self.assertIsNone(scheduler.last_batch) - - def test_shutdown_during_intake_stops_new_batches_in_both_modes(self): - class StoppingScheduler(_Scheduler): - def _inputs_for_step(self): - inputs = super()._inputs_for_step() - if self.iteration == shutdown_at: - inputs.insert(0, ShutdownReq()) - return inputs - - for enabled in (False, True): - for shutdown_at in (0, 1, 2): - with self.subTest(enabled=enabled, shutdown_at=shutdown_at): - scheduler = StoppingScheduler( - [_batch(1), _batch(2), _batch(3), None], - { - name: _result( - _Event(ready=True, recorded=False), delayed=True - ) - for name in (1, 2, 3) - }, - {}, - ) - _run_loop(scheduler, enabled=enabled) - - self.assertTrue(scheduler.gracefully_exit) - self.assertEqual( - scheduler.admission_attempts, list(range(shutdown_at)) - ) - self.assertEqual( - [a[2] for a in scheduler.actions if a[1] == "launch"], - list(range(1, shutdown_at + 1)), - ) - self.assertEqual( - scheduler.result_processes, - {name: 1 for name in range(1, shutdown_at + 1)}, - ) - self.assertFalse(scheduler.result_queue) - - def test_shutdown_during_copy_wait_finishes_oldest_result_before_exiting(self): - class StoppingScheduler(_Scheduler): - def _inputs_for_step(self): - inputs = super()._inputs_for_step() - if self.iteration == 2: - inputs.insert(0, ShutdownReq()) - return inputs - - for pending_count, pause_on_shutdown in ( - (1, False), - (1, True), - (2, False), - (2, True), - ): - with self.subTest( - pending_count=pending_count, pause_on_shutdown=pause_on_shutdown - ): - second = _batch(2) if pending_count == 2 else None - scheduler = StoppingScheduler( - [_batch(1), second, _batch(3), None, None], - {name: _result() for name in (1, 2, 3)}, - {name: 20 for name in (1, 2, 3)}, - pause_at=2 if pause_on_shutdown else None, - ) - _run_loop(scheduler) - - self.assertTrue(scheduler.gracefully_exit) - self.assertEqual( - [a for a in scheduler.actions if a[1] == "launch"], - [ - (name - 1, "launch", name) - for name in range(1, pending_count + 1) - ], - ) - self.assertEqual(scheduler.result_processes, {1: 1}) - self.assertEqual(scheduler.copy_events[1].synchronizations, 1) - self.assertEqual(scheduler.copy_events[1].blocking_synchronizations, 1) - self.assertIn((2, "dispatch", "ShutdownReq"), scheduler.actions) - self.assertEqual( - [batch.forward_iter for batch, _ in scheduler.result_queue], - [2] if pending_count == 2 else [], - ) - self.assertEqual( - [req.rid for req in scheduler.disagg_prefill_inflight_queue], ["1"] - ) - self.assertEqual( - list(scheduler.disagg_prefill_inflight_queue[0].output_ids), [11] - ) - - def test_shutdown_behind_deferred_pause_keeps_input_order(self): - class StoppingScheduler(_Scheduler): - def _inputs_for_step(self): - inputs = super()._inputs_for_step() - if self.iteration == 2: - inputs.append(ShutdownReq()) - return inputs - - for depth in (1, 2): - for mode in ("in_place", "retract"): - with self.subTest(depth=depth, mode=mode): - second = _batch(2) if depth == 2 else None - scheduler = StoppingScheduler( - [_batch(1), second, _batch(3)] + [None] * 5, - {name: _result() for name in (1, 2, 3)}, - {1: 4, 2: 20, 3: 20}, - pause_at=2, - pause_mode=mode, - ) - _run_loop(scheduler) - - self.assertTrue(scheduler.gracefully_exit) - self.assertEqual( - [a for a in scheduler.actions if a[1] == "dispatch"][-3:], - [ - (5, "dispatch", "PauseGenerationReqInput"), - (5, "dispatch", "3"), - (5, "dispatch", "ShutdownReq"), - ], - ) - self.assertIn((4, "result", 1), scheduler.actions) - self.assertEqual( - [a[2] for a in scheduler.actions if a[1] == "launch"], - list(range(1, depth + 1)), - ) - self.assertEqual( - scheduler.result_processes, - {1: 1, 2: 1} if depth == 2 and mode == "retract" else {1: 1}, - ) - self.assertEqual( - [batch.forward_iter for batch, _ in scheduler.result_queue], - [2] if depth == 2 and mode == "in_place" else [], - ) - self.assertFalse(scheduler._deferred_input_requests) - - def test_only_disabled_and_unsupported_modes_keep_blocking_results(self): - for mode in ( - "disabled", - "independent_dp", - "attention_dp", - "cp", - "speculation", - "embedding", - "dllm", - ): - with self.subTest(mode=mode): - scheduler = _Scheduler([_batch(1), None, None], {1: _result()}, {1: 2}) - topology = {} - if mode == "independent_dp": - topology = {"dp_size": 2} - elif mode == "attention_dp": - topology = {"dp_size": 2, "tp_size": 2, "enable_dp_attention": True} - elif mode == "cp": - topology = {"attn_cp_size": 2, "tp_size": 2} - elif mode == "speculation": - scheduler.spec_algorithm = SpeculativeAlgorithm.EAGLE - elif mode == "embedding": - scheduler.is_generation = False - elif mode == "dllm": - scheduler.dllm_config = object() - _run_loop(scheduler, enabled=mode != "disabled", **topology) - blocking = mode != "independent_dp" - self.assertIn((1 if blocking else 2, "result", 1), scheduler.actions) - self.assertEqual(scheduler.copy_events[1].queries == 0, blocking) - self.assertEqual( - scheduler.copy_events[1].blocking_synchronizations, int(blocking) - ) - - def test_dispatch_clears_polling_state_outside_prefill_overlap(self): - scheduler = _Scheduler([], {}, {}) - for mode, pp_size, overlap, loop in ( - (DisaggregationMode.PREFILL, 2, True, "event_loop_pp_disagg_prefill"), - (DisaggregationMode.PREFILL, 1, False, "event_loop_normal_disagg_prefill"), - (DisaggregationMode.DECODE, 1, True, "event_loop_overlap_disagg_decode"), - ): - with self.subTest(mode=mode, pp_size=pp_size, overlap=overlap): - scheduler.disaggregation_mode = mode - scheduler.enable_overlap = overlap - scheduler.enable_continuous_input_polling = True - with ( - published_topology(pp_size=pp_size), - patch.object(scheduler, loop, create=True) as run_loop, - patch.object( - scheduler, "_is_continuous_input_polling_enabled" - ) as check_support, - ): - _dispatch_event_loop_once(scheduler) - run_loop.assert_called_once_with() - check_support.assert_not_called() - self.assertFalse(scheduler.enable_continuous_input_polling) - - def test_chunk_handoffs_across_polling_and_retract_pause(self): - for enabled, pause_at, blocking_ends in ( - (True, None, ()), - (False, 1, (4,)), - (True, 1, (4,)), - (True, 2, (8,)), - (True, 4, (12,)), - ): - with self.subTest(enabled=enabled, pause_at=pause_at): - req = Req("chunked", "", list(range(12)), SamplingParams()) - req.metadata_buffer_index = 1 - req.pending_bootstrap = False - batches = [_batch(end, [req]) for end in (4, 8, 12)] - results = { - b.forward_iter: GenerationBatchResult( - next_token_ids=torch.tensor([11]), copy_done=_Event() - ) - for b in batches - } - checkpoint_ends = [] - - class ChunkScheduler(_Scheduler): - def run_batch(self, batch): - end = int(batch.forward_iter) - req.extend_range = SimpleNamespace(end=end) - self.chunked_req = req if end < 12 else None - batch.chunked_req = self.chunked_req - if self.chunked_req is not None: - req.inflight_middle_chunks += 1 - return super().run_batch(batch) - - def checkpoint_disagg_prefill(self, req, **kwargs): - checkpoint_ends.append(req.extend_range.end) - super().checkpoint_disagg_prefill(req, **kwargs) - - if pause_at == 1: - arrivals = [batches[0], None, None, batches[1], batches[2]] - ready_at = {4: 20, 8: 7, 12: 8} - elif pause_at == 2: - arrivals = [batches[0], batches[1], None, None, batches[2]] - ready_at = {4: 3, 8: 20, 12: 8} - elif pause_at == 4: - # The second pending result is the request's final chunk. - arrivals = [batches[0], batches[1], None, batches[2], None] - ready_at = {4: 1, 8: 5, 12: 20} - else: - arrivals = [batches[0], None, batches[1], None, batches[2]] - ready_at = {4: 4, 8: 6, 12: 8} - scheduler = ChunkScheduler( - arrivals + [None] * 5, - results, - ready_at, - pause_at=pause_at, - resume_at=pause_at + 2 if pause_at is not None else None, - pause_mode="retract", - ) - _run_loop(scheduler, enabled=enabled) - - self.assertEqual( - scheduler.send_kv_chunk.call_args_list, - [ - call(req, last_chunk=False, end_idx=4), - call(req, last_chunk=False, end_idx=8), - call(req, last_chunk=True), - ], - ) - # Pause must not take a cache snapshot that resume repeats. - self.assertEqual(checkpoint_ends, [4, 8, 12]) - self.assertEqual(req.inflight_middle_chunks, 0) - self.assertEqual(list(req.output_ids), [11]) - self.assertEqual(scheduler.disagg_prefill_inflight_queue, [req]) - self.assertEqual(scheduler.result_processes, {4: 1, 8: 1, 12: 1}) - for end, event in scheduler.copy_events.items(): - self.assertEqual(event.synchronizations, 1) - self.assertEqual( - event.blocking_synchronizations, - int(not enabled or end in blocking_ends), - ) - - @unittest.skipUnless(dist.is_gloo_available(), "requires Gloo") - def test_retract_resume_requeues_optimistic_chunk_after_result_finishes(self): - for depth, yield_before_pause in ((1, False), (2, False), (2, True)): - with self.subTest(depth=depth, yield_before_pause=yield_before_pause): - req = Req("optimistic", "", list(range(12)), SamplingParams()) - req.metadata_buffer_index = 1 - req.pending_bootstrap = True - req.kv.req_pool_idx = 1 - req.kv.kv_allocated_len = 8 - req.disagg_kv_sender = SimpleNamespace( - poll=lambda: KVPoll.Bootstrapping - ) - chunk_count = 1 if yield_before_pause else depth - chunks = [_batch(end, [req]) for end in (4, 8)[:chunk_count]] - other = _batch(100) - other.reqs[0].disagg_kv_sender = SimpleNamespace( - poll=lambda: KVPoll.WaitingForInput - ) - - class OptimisticScheduler(_Scheduler): - _release_aborted_request = Scheduler._release_aborted_request - - def run_batch(self, batch): - if req in batch.reqs: - req.extend_range = SimpleNamespace(end=batch.forward_iter) - self.chunked_req = batch.chunked_req = req - req.inflight_middle_chunks += 1 - return super().run_batch(batch) - - def process_input_requests(self, inputs): - super().process_input_requests(inputs) - if any( - isinstance(item, TokenizedGenerateReqInput) - and item.rid == "100" - for item in inputs - ): - self.waiting_queue.extend(other.reqs) - - pause_at = 2 if yield_before_pause else (1 if depth == 1 else 3) - ready_at = {**{batch.forward_iter: 20 for batch in chunks}, 100: 7} - if depth == 2: - ready_at[4] = pause_at + 1 - scheduler = OptimisticScheduler( - chunks + [other] + [None] * 8, - {batch.forward_iter: _result() for batch in [*chunks, other]}, - ready_at, - pause_at=pause_at, - resume_at=pause_at + 2, - pause_mode="retract", - ) - scheduler.disagg_prefill_pending_chunk_rids = set() - scheduler.processed_tokens_counter = 0 - scheduler.metrics_reporter.enable_metrics = False - release_steps = [] - - def release(req, tree_cache, is_insert): - self.assertEqual(req.inflight_middle_chunks, 0) - release_steps.append(scheduler.iteration) - req.kv.req_pool_idx = None - req.kv.mark_kv_released() - - with ( - _single_rank_gloo_group(), - patch( - "sglang.srt.disaggregation.prefill.release_kv_cache", - side_effect=release, - ) as release_kv, - ): - _run_loop(scheduler, optimistic_prefill_attempts=2) - - release_kv.assert_called_once_with( - req, scheduler.tree_cache, is_insert=False - ) - self.assertEqual( - release_steps, [3 if depth == 1 or yield_before_pause else 5] - ) - self.assertEqual(scheduler.waiting_queue, [req]) - self.assertEqual(req.prefill_attempt_count, 1) - self.assertEqual(req.inflight_middle_chunks, 0) - self.assertEqual(list(req.output_ids), []) - self.assertIsNone(scheduler.chunked_req) - self.assertEqual( - scheduler.result_processes, - {b.forward_iter: 1 for b in [*chunks, other]}, - ) - scheduler.send_kv_chunk.assert_called_once_with( - other.reqs[0], last_chunk=True - ) - - @unittest.skipUnless(dist.is_gloo_available(), "requires Gloo") - def test_ranks_agree_on_fifo_completion_and_copy_submission(self): - context = multiprocessing.get_context("spawn") - with tempfile.TemporaryDirectory() as directory: - rendezvous = str(Path(directory) / "rendezvous") - processes = [ - context.Process(target=_rank_consensus_worker, args=(rank, rendezvous)) - for rank in range(2) - ] - try: - for process in processes: - process.start() - for process in processes: - process.join(timeout=60) - self.assertEqual([process.exitcode for process in processes], [0, 0]) - finally: - for process in processes: - if process.is_alive(): - process.terminate() - process.join(timeout=5) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/unit/managers/test_auxiliary_output.py b/test/registered/unit/managers/test_auxiliary_output.py index fec69a41ac..4265ea61ee 100644 --- a/test/registered/unit/managers/test_auxiliary_output.py +++ b/test/registered/unit/managers/test_auxiliary_output.py @@ -595,21 +595,19 @@ def test_disaggregated_prefill_consumes_auxiliary_output_after_commit(): spec_algorithm=SimpleNamespace(is_eagle=lambda: False), tree_cache=object(), disagg_prefill_inflight_queue=[], - checkpoint_disagg_prefill=Mock(), send_kv_chunk=Mock(), metrics_reporter=SimpleNamespace(report_prefill_stats=Mock()), maybe_send_health_check_signal=Mock(), ) - SchedulerDisaggregationPrefillMixin.process_batch_result_disagg_prefill( - scheduler, - batch, - result, - ) + with patch("sglang.srt.disaggregation.prefill.maybe_cache_unfinished_req"): + SchedulerDisaggregationPrefillMixin.process_batch_result_disagg_prefill( + scheduler, + batch, + result, + ) assert req.output_ids == [7] - scheduler.checkpoint_disagg_prefill.assert_called_once_with(req) - copy_done.synchronize.assert_called_once_with() snapshot_auxiliary_output_starts.assert_called_once_with(batch, result) processor.consume_auxiliary_output.assert_called_once_with( batch, diff --git a/test/registered/unit/managers/test_disagg_idle_step_counters.py b/test/registered/unit/managers/test_disagg_idle_step_counters.py index f216286ea7..90a769d534 100644 --- a/test/registered/unit/managers/test_disagg_idle_step_counters.py +++ b/test/registered/unit/managers/test_disagg_idle_step_counters.py @@ -11,16 +11,13 @@ from unittest.mock import Mock, patch from parameterized import parameterized -from sglang.srt.disaggregation.utils import DisaggregationMode from sglang.srt.distributed import parallel_state -from sglang.srt.environ import envs from sglang.srt.managers import scheduler as scheduler_module -from sglang.srt.managers.schedule_batch import Req, ScheduleBatch +from sglang.srt.managers.schedule_batch import ScheduleBatch from sglang.srt.managers.scheduler import Scheduler from sglang.srt.managers.utils import GenerationBatchResult from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.runtime_context import get_parallel -from sglang.srt.sampling.sampling_params import SamplingParams from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import ( @@ -81,11 +78,6 @@ class TestSchedulerIdleStepCounters(CustomTestCase): Scheduler.event_loop_overlap_disagg_prefill, ForwardMode.EXTEND, ), - ( - "prefill_polling", - Scheduler.event_loop_overlap_disagg_prefill, - ForwardMode.EXTEND, - ), ( "decode_normal", Scheduler.event_loop_normal_disagg_decode, @@ -105,45 +97,8 @@ class TestSchedulerIdleStepCounters(CustomTestCase): with self.subTest(idle_iterations=idle_iterations): batches = self.make_batches(mode) schedule = batches[:2] + [None] * idle_iterations + batches[2:] + [None] - poll_pending_copy = name == "prefill_polling" - if poll_pending_copy and idle_iterations: - # Keep work unavailable through the extra unfinished-copy pass. - schedule.insert(2, None) scheduler = self.make_scheduler(schedule) - if poll_pending_copy: - # Each pending copy adds one intake pass before completing. - scheduler.request_receiver.recv_requests.side_effect = [[]] * ( - len(schedule) + len(batches) - ) + [StopIteration] - with ( - envs.SGLANG_ENABLE_DISAGG_PREFILL_CONTINUOUS_INPUT_POLLING.override( - poll_pending_copy - ) - ): - self.run_and_check( - scheduler, - event_loop, - mode, - idle_iterations > 0, - poll_pending_copy, - ) - - def test_polling_admission_miss_during_copy_wait_preserves_busy_interval(self): - batches = self.make_batches(ForwardMode.EXTEND) - # New input after a miss can launch while the second copy is unfinished. - schedule = batches[:2] + [None] + batches[2:] + [None] - scheduler = self.make_scheduler(schedule) - inputs = [[]] * (len(schedule) + len(batches)) - inputs[4] = [object()] - scheduler.request_receiver.recv_requests.side_effect = inputs + [StopIteration] - with envs.SGLANG_ENABLE_DISAGG_PREFILL_CONTINUOUS_INPUT_POLLING.override(True): - self.run_and_check( - scheduler, - Scheduler.event_loop_overlap_disagg_prefill, - ForwardMode.EXTEND, - after_idle=False, - poll_pending_copy=True, - ) + self.run_and_check(scheduler, event_loop, mode, idle_iterations > 0) @parameterized.expand( [ @@ -308,23 +263,6 @@ class TestSchedulerIdleStepCounters(CustomTestCase): 2 if chained else 0, ) - def test_mlx_pending_job_preserves_inflight_request_inventory(self): - module = load_mlx_scheduler_module() - scheduler = self.make_scheduler([]) - scheduler.disaggregation_mode = DisaggregationMode.NULL - req = Req("mlx-pending", "", [1], SamplingParams()) - batch = ScheduleBatch(reqs=[req], forward_mode=ForwardMode.DECODE) - scheduler.last_batch = batch - pending = module.MlxPendingJob( - launch=None, - batch_copy=batch.copy(), - schedule_batch=batch, - reqs=batch.reqs[:], - ) - scheduler.result_queue = deque([pending]) - - self.assertEqual(scheduler.collect_inflight_reqs(), {req}) - def test_split_prefill_is_charged_once_from_its_first_chunk(self): # A split prefill spans several run_batch calls but yields one result, # so only its first chunk may set the burst's start and its idle flag. @@ -383,26 +321,16 @@ class TestSchedulerIdleStepCounters(CustomTestCase): for launch_ts in LAUNCH_TIMESTAMPS ] - def run_and_check( - self, scheduler, event_loop, mode, after_idle, poll_pending_copy=False - ): + def run_and_check(self, scheduler, event_loop, mode, after_idle): observed_idle_flags = [] observed_iters = [] - copy_events = [] def run_batch(batch, pp_proxy_tensors=None): # Exercise the real timestamp, iteration, and flag handoff. self.launch_batch( scheduler, batch, batch.launch_ts, pp_proxy_tensors=pp_proxy_tensors ) - copy_done = Mock( - query=Mock( - return_value=True, - side_effect=[False, True] if poll_pending_copy else None, - ) - ) - copy_events.append(copy_done) - return GenerationBatchResult(copy_done=copy_done) + return GenerationBatchResult() def process_batch_result(batch, result): observed_idle_flags.append(batch.after_idle_gap) @@ -422,8 +350,6 @@ class TestSchedulerIdleStepCounters(CustomTestCase): ): event_loop(scheduler) - if poll_pending_copy: - self.assertTrue(all(event.query.call_count > 1 for event in copy_events)) self.assertEqual(observed_idle_flags, [False, False, after_idle, False]) self.assertEqual(scheduler.forward_ct, 4) self.assertEqual(observed_iters, [1, 2, 3, 4]) @@ -477,21 +403,13 @@ class TestSchedulerIdleStepCounters(CustomTestCase): def make_scheduler(self, schedule): scheduler = Scheduler.__new__(Scheduler) - scheduler.tp_size = get_parallel().tp_size - scheduler.gracefully_exit = False scheduler._engine_paused = False - scheduler._deferred_input_requests = [] scheduler._sched_idled = False scheduler._prev_step = None scheduler._prev_prefill_end_ts = None scheduler.forward_ct = 0 scheduler.processed_tokens_counter = 0 scheduler.spec_algorithm = SpeculativeAlgorithm.NONE - scheduler.enable_overlap = True - scheduler.enable_continuous_input_polling = False - scheduler.is_generation = True - scheduler.dllm_config = None - scheduler.disaggregation_mode = DisaggregationMode.PREFILL scheduler._poll_timeout_aborts = Mock(return_value=[]) scheduler.scheduler_stage_metrics = None scheduler.metrics_reporter = SimpleNamespace(record_scheduler_active=Mock()) @@ -505,7 +423,6 @@ class TestSchedulerIdleStepCounters(CustomTestCase): scheduler.last_batch = None scheduler.chunked_req = None scheduler.waiting_queue = [] - scheduler.disagg_prefill_inflight_queue = [] scheduler.enable_staging = False scheduler.request_receiver = SimpleNamespace( recv_requests=Mock(side_effect=[[]] * len(schedule) + [StopIteration]) @@ -521,13 +438,13 @@ class TestSchedulerIdleStepCounters(CustomTestCase): scheduler.ngram_embedding_manager = SimpleNamespace( prepare_for_forward=lambda batch, chunked_req: batch ) - scheduled_batches = iter(schedule) - # Intake ends the test; extra admission attempts simply find no work. next_plan = Mock( - side_effect=lambda *args, **kwargs: SimpleNamespace( - running_batch=scheduler.running_batch, - batch_to_run=next(scheduled_batches, None), - ) + side_effect=[ + SimpleNamespace( + running_batch=scheduler.running_batch, batch_to_run=batch + ) + for batch in schedule + ] ) scheduler.get_next_disagg_prefill_batch_to_run = next_plan scheduler.get_next_disagg_decode_batch_to_run = next_plan diff --git a/test/registered/unit/managers/test_scheduler_chunked_abort_race.py b/test/registered/unit/managers/test_scheduler_chunked_abort_race.py index afef550365..b848b69d9b 100644 --- a/test/registered/unit/managers/test_scheduler_chunked_abort_race.py +++ b/test/registered/unit/managers/test_scheduler_chunked_abort_race.py @@ -1,7 +1,6 @@ """Tests for deferred chunked-prefill aborts.""" import unittest -from collections import deque from types import SimpleNamespace from unittest.mock import Mock @@ -36,8 +35,6 @@ class _FakeReq: def _make_scheduler(pending_req, *, chunked_req, running_reqs) -> Scheduler: sched = Scheduler.__new__(Scheduler) - sched.enable_continuous_input_polling = False - sched.result_queue = deque() sched.chunked_req = chunked_req sched._pending_chunked_abort_req = pending_req sched.waiting_queue = [] diff --git a/test/registered/unit/managers/test_scheduler_pause_generation.py b/test/registered/unit/managers/test_scheduler_pause_generation.py index 424a65b080..716c59e578 100644 --- a/test/registered/unit/managers/test_scheduler_pause_generation.py +++ b/test/registered/unit/managers/test_scheduler_pause_generation.py @@ -405,12 +405,9 @@ class TestSchedulerPauseGeneration(unittest.TestCase): ) scheduler.result_queue = deque([(MagicMock(), MagicMock())]) event_log: List[str] = [] - - def process_result(batch, result): - self.assertFalse(scheduler.result_queue) - event_log.append("drain") - - scheduler.process_batch_result = MagicMock(side_effect=process_result) + scheduler.process_batch_result = MagicMock( + side_effect=lambda *args, **kwargs: event_log.append("drain") + ) scheduler._add_request_to_queue = MagicMock( side_effect=lambda req: event_log.append( "requeue-released" if req.is_retracted else "requeue-unreleased" @@ -451,27 +448,19 @@ class TestSchedulerPauseGeneration(unittest.TestCase): self.assertIs(scheduler.chunked_req, chunked_req) def test_retract_drains_overlap_queue(self): - """Single-result retract must preserve the existing chunk-preparation state.""" - for mode in ( - DisaggregationMode.NULL, - DisaggregationMode.PREFILL, - DisaggregationMode.DECODE, - ): - with self.subTest(mode=mode): - scheduler = self._new_scheduler() - scheduler.enable_overlap = True - scheduler.disaggregation_mode = mode - batch = ScheduleBatch(reqs=[], forward_mode=ForwardMode.EXTEND) - result = SimpleNamespace(delay_sample_func=None) - scheduler.last_batch = batch - scheduler.result_queue = deque([(batch, result)]) - scheduler.process_batch_result = MagicMock() + """retract with overlap enabled should drain the result_queue.""" + scheduler = self._new_scheduler() + scheduler.enable_overlap = True + mock_batch = MagicMock() + mock_batch.forward_mode.is_extend.return_value = False + scheduler.last_batch = mock_batch + scheduler.result_queue = deque([(MagicMock(), MagicMock())]) + scheduler.process_batch_result = MagicMock() - scheduler.pause_generation(PauseGenerationReqInput(mode="retract")) + scheduler.pause_generation(PauseGenerationReqInput(mode="retract")) - scheduler.process_batch_result.assert_called_once_with(batch, result) - self.assertFalse(scheduler.result_queue) - self.assertFalse(batch.prefill_chunk_processed) + scheduler.process_batch_result.assert_called_once() + self.assertEqual(len(scheduler.result_queue), 0) def test_pd_decode_retract_requeues_for_rebootstrap(self): """PD decode retract should rebootstrap instead of resuming stale CPU KV.""" diff --git a/test/registered/unit/managers/test_scheduler_timeouts.py b/test/registered/unit/managers/test_scheduler_timeouts.py index 57f83fec6d..a6c152e32e 100644 --- a/test/registered/unit/managers/test_scheduler_timeouts.py +++ b/test/registered/unit/managers/test_scheduler_timeouts.py @@ -7,13 +7,10 @@ reaching the client is covered by scheduler/test_scheduler_control.py. import time import unittest -from collections import deque from types import SimpleNamespace from unittest.mock import MagicMock, patch -from sglang.srt.disaggregation.utils import DisaggregationMode from sglang.srt.environ import envs -from sglang.srt.runtime_context import get_parallel from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import ( CustomTestCase, @@ -24,7 +21,6 @@ from sglang.test.test_utils import ( maybe_stub_sgl_kernel() -from sglang.srt.managers.schedule_batch import FINISH_ABORT from sglang.srt.managers.scheduler import Scheduler from sglang.srt.mem_cache.base_prefix_cache import ( CacheRequestHandle, @@ -65,8 +61,6 @@ def _batch(reqs): def _scheduler(waiting_queue, running_reqs=(), last_batch_reqs=()): s = Scheduler.__new__(Scheduler) - s.enable_continuous_input_polling = False - s.result_queue = deque() s.waiting_queue = waiting_queue s.enable_hierarchical_cache = False s.enable_hicache_storage = False @@ -162,61 +156,6 @@ class TestRunningTimeout(CustomTestCase): aborts = s._poll_timeout_aborts() self.assertEqual([a.rid for a in aborts], ["stale"]) - def test_pending_abort_is_not_emitted_again_while_forward_is_running(self): - stale = _req("stale", forward_entry=time.perf_counter() - 10) - fresh = _req("fresh", forward_entry=time.perf_counter()) - s = _scheduler([], last_batch_reqs=[fresh]) - s.enable_continuous_input_polling = True - s.result_queue = deque([(_batch([stale]), None), (s.last_batch, None)]) - s.disaggregation_mode = DisaggregationMode.PREFILL - s.chunked_req = None - s.mm_receiver = None - s.dllm_config = None - s.grammar_manager = MagicMock() - s.disagg_prefill_bootstrap_queue = SimpleNamespace(queue=[]) - s.disagg_prefill_inflight_queue = [] - with envs.SGLANG_REQ_RUNNING_TIMEOUT.override(1.0): - aborts = s._poll_timeout_aborts() - self.assertEqual([abort.rid for abort in aborts], ["stale"]) - # The broadcast abort marks to_finish before the GPU result arrives. - s.abort_request(aborts[0]) - self.assertIsInstance(stale.to_finish, FINISH_ABORT) - self.assertIsNone(fresh.to_finish) - self.assertFalse(stale.finished()) - for _ in range(3): - self.assertEqual(s._poll_timeout_aborts(), []) - - def test_queued_result_remains_visible_after_a_new_forward_launches(self): - now = time.perf_counter() - older = _req("older", forward_entry=now - 10) - newer = _req("newer", forward_entry=now - 10) - for pp_size in (1, 2): - with ( - self.subTest(pp_size=pp_size), - get_parallel().override(pp_size=pp_size), - ): - # Polling re-enters intake before last_batch advances to newer. - last_req = older if pp_size == 1 else newer - s = _scheduler([], last_batch_reqs=[last_req]) - s.enable_continuous_input_polling = pp_size == 1 - s.running_mbs = [s.running_batch] - s.mbs = [s.last_batch, _batch([older])] - if pp_size == 1: - s.result_queue = deque( - [(_batch([older]), None), (_batch([newer]), None)] - ) - - with envs.SGLANG_REQ_RUNNING_TIMEOUT.override(1.0): - aborts = s._poll_timeout_aborts() - # Abort and weight-version updates share this request inventory. - self.assertEqual(s.collect_inflight_reqs(), {older, newer}) - - self.assertEqual( - [a.rid for a in aborts], - ["older", "newer"] if pp_size == 1 else ["newer", "older"], - ) - self.assertEqual(len(s.result_queue), 2 if pp_size == 1 else 0) - def test_unset_forward_entry_time_is_never_emitted(self): s = _scheduler([], running_reqs=[_req("unstamped", forward_entry=0.0)]) with envs.SGLANG_REQ_RUNNING_TIMEOUT.override(1e-9): diff --git a/test/registered/unit/utils/test_weight_versions.py b/test/registered/unit/utils/test_weight_versions.py index b15302fcd1..7c2508b4b7 100644 --- a/test/registered/unit/utils/test_weight_versions.py +++ b/test/registered/unit/utils/test_weight_versions.py @@ -1,6 +1,5 @@ import random import unittest -from collections import deque from types import SimpleNamespace from unittest.mock import patch @@ -296,7 +295,6 @@ class _ContextStub: class _SchedulerStub: - _collect_inflight_batches = Scheduler._collect_inflight_batches collect_inflight_reqs = Scheduler.collect_inflight_reqs def __init__( @@ -309,8 +307,6 @@ class _SchedulerStub: hisparse=None, ): self.serving = _ServingStub(version) - self.enable_continuous_input_polling = False - self.result_queue = deque() self.running_batch = SimpleNamespace(reqs=running) self.last_batch = last_batch self.waiting_queue = waiting