[PD] Keep ingesting requests while a prefill forward result is pending (#42035)
build-sglang-image / build (push) Successful in 26m6s
build-sglang-image / build (push) Successful in 26m6s
Co-authored-by: merrymercy <merrymercy@users.noreply.github.com>
Co-authored-by: Jialin <Jialin@users.noreply.github.com>
(cherry picked from commit 6acf2f4736)
This commit is contained in:
@@ -21,6 +21,7 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import time
|
||||
from array import array
|
||||
from collections import deque
|
||||
from http import HTTPStatus
|
||||
@@ -103,6 +104,12 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# TP=1 yield between polls of a pending result. An initial heuristic rather
|
||||
# than a tuned value: long enough that the bootstrap, transfer and cache
|
||||
# threads reliably win the GIL even when the input socket is empty, and far
|
||||
# below the duration of any prefill forward, so it adds no visible latency.
|
||||
PREFILL_INPUT_POLL_INTERVAL_S = 0.0001
|
||||
|
||||
_is_npu = is_npu()
|
||||
|
||||
|
||||
@@ -608,9 +615,12 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
self: Scheduler,
|
||||
running_batch: ScheduleBatch,
|
||||
last_batch: Optional[ScheduleBatch],
|
||||
should_retry_storage_prefetch: bool = True,
|
||||
) -> NextBatchPlan:
|
||||
self.process_pending_chunked_abort()
|
||||
self._process_hicache_events()
|
||||
self._process_hicache_events(
|
||||
should_retry_storage_prefetch=should_retry_storage_prefetch
|
||||
)
|
||||
|
||||
# HACK (byronhsu): reset the batch_is_full flag because we never enter update_running_batch which resets it
|
||||
# Otherwise, it hangs under high concurrency
|
||||
@@ -618,7 +628,13 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
|
||||
self.resolve_waiting_queue_bootstrap()
|
||||
|
||||
self.process_prefill_chunk(last_batch=last_batch, running_batch=running_batch)
|
||||
# Skip repeated chunk processing while polling a pending batch.
|
||||
if last_batch is None or not last_batch.prefill_chunk_processed:
|
||||
self.process_prefill_chunk(
|
||||
last_batch=last_batch, running_batch=running_batch
|
||||
)
|
||||
if last_batch is not None:
|
||||
last_batch.prefill_chunk_processed = True
|
||||
|
||||
prefill_plan = self.get_new_batch_prefill(running_batch)
|
||||
batch = prefill_plan.batch_to_run
|
||||
@@ -669,46 +685,120 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
# Update last_batch
|
||||
self.last_batch = batch
|
||||
|
||||
def _is_continuous_input_polling_enabled(self: Scheduler) -> bool:
|
||||
"""Check support once on entry to the prefill overlap loop."""
|
||||
if not envs.SGLANG_ENABLE_DISAGG_PREFILL_CONTINUOUS_INPUT_POLLING.get():
|
||||
return False
|
||||
parallel = get_parallel()
|
||||
enabled = (
|
||||
parallel.attn_dp_size == 1
|
||||
and parallel.attn_cp_size == 1
|
||||
and self.spec_algorithm.is_none()
|
||||
and self.is_generation
|
||||
and self.dllm_config is None
|
||||
)
|
||||
if not enabled:
|
||||
logger.warning(
|
||||
"Continuous prefill input polling requires attention DP=CP=1 and "
|
||||
"non-speculative autoregressive generation; using the regular overlap loop."
|
||||
)
|
||||
return enabled
|
||||
|
||||
def _yield_gil_if_needed(self: Scheduler) -> None:
|
||||
if self.tp_size == 1:
|
||||
# No collective parks this thread at TP=1. A bounded sleep lets
|
||||
# bootstrap, transfer and cache workers reacquire the GIL.
|
||||
time.sleep(PREFILL_INPUT_POLL_INTERVAL_S)
|
||||
|
||||
@torch.no_grad()
|
||||
def event_loop_overlap_disagg_prefill(self: Scheduler) -> None:
|
||||
"""Run intake while pending copies finish, keeping at most two forwards.
|
||||
|
||||
A deferred pause stops preparation until the current result wait finishes.
|
||||
Shutdown stops new launches; a wait finishes through the blocking handler.
|
||||
After a failed preparation, only new input or bootstrap arrivals retry
|
||||
during the wait; other readiness is checked after a result finishes.
|
||||
"""
|
||||
self.enable_continuous_input_polling = (
|
||||
self._is_continuous_input_polling_enabled()
|
||||
)
|
||||
self.result_queue = deque()
|
||||
batch_result_completion_status = torch.empty(1, dtype=torch.int32, device="cpu")
|
||||
# True when intake repeats while the oldest batch result is unfinished.
|
||||
waiting_for_batch_result = False
|
||||
# Avoid retrying failed batch preparation after every copy poll.
|
||||
skip_batch_creation = False
|
||||
# Keep the newer forward's sampling context across unfinished-result passes.
|
||||
batch = None
|
||||
batch_result = None
|
||||
|
||||
while True:
|
||||
# Receive requests
|
||||
self.ingest_requests()
|
||||
if self.gracefully_exit:
|
||||
break
|
||||
|
||||
# Stop dispatching at pause requests while waiting for a batch result.
|
||||
received_inputs = self.ingest_requests(
|
||||
stop_at_pause=waiting_for_batch_result
|
||||
)
|
||||
if self._engine_paused:
|
||||
self._record_scheduler_state_for_paused_engine()
|
||||
self._yield_gil_if_needed()
|
||||
continue
|
||||
self.waiting_queue.extend(
|
||||
self.disagg_prefill_bootstrap_queue.pop_bootstrapped()
|
||||
)
|
||||
bootstrapped_reqs = self.disagg_prefill_bootstrap_queue.pop_bootstrapped()
|
||||
self.waiting_queue.extend(bootstrapped_reqs)
|
||||
# Resume batch creation on completion, new/ready requests, or an idle step.
|
||||
if not waiting_for_batch_result or received_inputs or bootstrapped_reqs:
|
||||
skip_batch_creation = False
|
||||
|
||||
# Get the next batch to run
|
||||
plan = self.get_next_disagg_prefill_batch_to_run(
|
||||
running_batch=self.running_batch, last_batch=self.last_batch
|
||||
)
|
||||
self.running_batch = plan.running_batch
|
||||
batch = plan.batch_to_run
|
||||
batch = self.ngram_embedding_manager.prepare_for_forward(
|
||||
batch, chunked_req=self.chunked_req
|
||||
)
|
||||
self.cur_batch_for_debug = batch
|
||||
# Keep at most two batches in flight.
|
||||
if (
|
||||
len(self.result_queue) < 2
|
||||
and not self.gracefully_exit
|
||||
and not self._deferred_input_requests
|
||||
and not skip_batch_creation
|
||||
):
|
||||
# Get the next batch to run
|
||||
plan = self.get_next_disagg_prefill_batch_to_run(
|
||||
running_batch=self.running_batch,
|
||||
last_batch=self.last_batch,
|
||||
# Storage retry deadlines count scheduling steps, not copy polls.
|
||||
should_retry_storage_prefetch=not waiting_for_batch_result,
|
||||
)
|
||||
self.running_batch = plan.running_batch
|
||||
batch = plan.batch_to_run
|
||||
batch = self.ngram_embedding_manager.prepare_for_forward(
|
||||
batch, chunked_req=self.chunked_req
|
||||
)
|
||||
self.cur_batch_for_debug = batch
|
||||
# Skip batch creation next round to avoid repeated preparation failures.
|
||||
skip_batch_creation = batch is None
|
||||
|
||||
# Launch the current batch
|
||||
if batch:
|
||||
if self.enable_staging:
|
||||
self.maybe_prefetch_staging_for_batch(batch)
|
||||
batch_result = self.run_batch(batch)
|
||||
self._apply_war_barrier()
|
||||
self.result_queue.append((batch.copy(), batch_result))
|
||||
else:
|
||||
batch_result = None
|
||||
self._sched_idled = True
|
||||
# Launch the current batch
|
||||
if batch:
|
||||
if self.enable_staging:
|
||||
self.maybe_prefetch_staging_for_batch(batch)
|
||||
batch_result = self.run_batch(batch)
|
||||
self._apply_war_barrier()
|
||||
self.result_queue.append((batch.copy(), batch_result))
|
||||
else:
|
||||
batch_result = None
|
||||
|
||||
# Process the last batch
|
||||
if self.last_batch:
|
||||
tmp_batch, tmp_result = self.result_queue.popleft()
|
||||
self.process_batch_result(tmp_batch, tmp_result)
|
||||
if self.enable_continuous_input_polling:
|
||||
# Resume intake until the oldest result is ready on all TP ranks.
|
||||
oldest_batch_result = self.result_queue[0][1]
|
||||
if (
|
||||
not self.gracefully_exit
|
||||
and not self.is_disagg_prefill_batch_result_ready(
|
||||
oldest_batch_result, batch_result_completion_status
|
||||
)
|
||||
):
|
||||
waiting_for_batch_result = True
|
||||
self._yield_gil_if_needed()
|
||||
continue
|
||||
# Process the oldest batch result.
|
||||
oldest_batch, oldest_batch_result = self.result_queue.popleft()
|
||||
self.process_batch_result(oldest_batch, oldest_batch_result)
|
||||
elif batch is None:
|
||||
# When the server is idle, do self-check and re-init some states
|
||||
self.on_idle()
|
||||
@@ -719,8 +809,36 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
# It depends on the result of the last batch (e.g., grammar), so we run it after the last batch is processed.
|
||||
self.launch_batch_sample_if_needed(batch_result, batch)
|
||||
|
||||
# Update last_batch
|
||||
# Update last_batch and scheduler status
|
||||
self._sched_idled = batch is None
|
||||
self.last_batch = batch
|
||||
waiting_for_batch_result = False
|
||||
|
||||
def is_disagg_prefill_batch_result_ready(
|
||||
self: Scheduler,
|
||||
batch_result: GenerationBatchResult,
|
||||
batch_result_completion_status: torch.Tensor,
|
||||
) -> bool:
|
||||
# 0: sampling/copy has not been submitted, so polling cannot make progress.
|
||||
state = 0
|
||||
if (
|
||||
batch_result.copy_done is not None
|
||||
and batch_result.delay_sample_func is None
|
||||
):
|
||||
# 1: copy unfinished; 2: copy finished locally.
|
||||
state = 2 if batch_result.copy_done.query() else 1
|
||||
if self.tp_size > 1:
|
||||
batch_result_completion_status.fill_(state)
|
||||
torch.distributed.all_reduce(
|
||||
batch_result_completion_status,
|
||||
op=torch.distributed.ReduceOp.MIN,
|
||||
group=self.tp_cpu_group,
|
||||
)
|
||||
state = batch_result_completion_status.item()
|
||||
if state == 0:
|
||||
# Every rank must submit delayed sampling before entering this wait.
|
||||
raise RuntimeError("Prefill result copy must be submitted before polling")
|
||||
return state == 2
|
||||
|
||||
def process_batch_result_disagg_prefill(
|
||||
self: Scheduler,
|
||||
@@ -1217,7 +1335,8 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
elif self.has_bootstrapped_waiting_req():
|
||||
# optimistic request yields to waiting requests
|
||||
self.chunked_req = None
|
||||
if not self.enable_overlap:
|
||||
# Retract may drain every result, leaving no handler to requeue.
|
||||
if not self.enable_overlap or req.inflight_middle_chunks == 0:
|
||||
self.optimistic_release_and_requeue(req)
|
||||
# else: still bootstrapping, keep computing without sending
|
||||
elif self.enable_overlap:
|
||||
|
||||
@@ -658,6 +658,8 @@ class Envs:
|
||||
# do not fence the next forward through the scheduler stream.
|
||||
SGLANG_PP_COMM_OVERLAP = EnvBool(False)
|
||||
SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH = EnvBool(False)
|
||||
# Opt-in: keep receiving prefill requests while forward results are pending.
|
||||
SGLANG_ENABLE_DISAGG_PREFILL_CONTINUOUS_INPUT_POLLING = EnvBool(False)
|
||||
|
||||
# ===================================================================
|
||||
# Radix and sparse KV caches
|
||||
|
||||
@@ -2335,6 +2335,9 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
chunked_req_next_prompt_token: Optional[int] = None
|
||||
contains_last_prefill_chunk: bool = True
|
||||
|
||||
# Tracks process_prefill_chunk() on the original scheduler batch only.
|
||||
prefill_chunk_processed: bool = False
|
||||
|
||||
# For DP attention
|
||||
inner_idle_batch: Optional[ScheduleBatch] = None
|
||||
# Decode requests carried alongside a chunked-prefill batch
|
||||
|
||||
@@ -1270,6 +1270,7 @@ class Scheduler(
|
||||
self.hisparse_coordinator.set_decode_producer_stream(self.forward_stream)
|
||||
|
||||
def init_running_status(self):
|
||||
self.tp_size = get_parallel().tp_size
|
||||
# Set by a runtime PD role switch to break out of the current event loop.
|
||||
self._event_loop_should_restart = False
|
||||
# Guards against concurrent/re-entrant PD role switches.
|
||||
@@ -1286,6 +1287,10 @@ class Scheduler(
|
||||
self.cur_batch_for_debug: Optional[ScheduleBatch] = None
|
||||
# The last forward batch
|
||||
self.last_batch: Optional[ScheduleBatch] = None
|
||||
self.result_queue: deque[
|
||||
tuple[ScheduleBatch, GenerationBatchResult | EmbeddingBatchResult]
|
||||
] = deque()
|
||||
self.enable_continuous_input_polling = False
|
||||
self.forward_ct = 0
|
||||
self.return_health_check_ipcs: Deque[Optional[str]] = deque()
|
||||
self.flush_wrapper = SchedulerFlushWrapper(
|
||||
@@ -1297,6 +1302,7 @@ class Scheduler(
|
||||
self.session_controller = SessionController(self.tree_cache)
|
||||
self.forward_sleep_time = None
|
||||
self._engine_paused = False
|
||||
self._deferred_input_requests: List = []
|
||||
|
||||
def init_chunked_prefill(self):
|
||||
self.chunked_prefill_size = get_schedule().chunked_prefill_size
|
||||
@@ -2067,20 +2073,38 @@ class Scheduler(
|
||||
for prev_batch, prev_result in self.result_queue:
|
||||
self.batch_result_processor.advance_grammar_fsm(prev_result, prev_batch)
|
||||
|
||||
def ingest_requests(self) -> List:
|
||||
"""Receive, broadcast and dispatch this iteration's external input.
|
||||
def ingest_requests(self, stop_at_pause: bool = False) -> List:
|
||||
"""Receive and broadcast input, then dispatch queued requests in order.
|
||||
|
||||
With stop_at_pause, retain the first pause and all following inputs
|
||||
until the caller reaches a scheduling boundary. Receiving and broadcasting
|
||||
continue each pass; all ranks retain the same suffix.
|
||||
|
||||
The one place a new per-iteration input source belongs; the return
|
||||
value exists for the pipeline stages that relay requests onward.
|
||||
"""
|
||||
local_reqs = []
|
||||
# Avoid re-enqueuing timeout aborts while dispatch is stopped at a pause.
|
||||
if (
|
||||
get_parallel().pp_rank == 0
|
||||
not self._deferred_input_requests
|
||||
and get_parallel().pp_rank == 0
|
||||
and get_parallel().attn_tp_rank == 0
|
||||
and get_parallel().attn_cp_rank == 0
|
||||
):
|
||||
local_reqs = self._poll_timeout_aborts()
|
||||
recv_reqs = self.request_receiver.recv_requests(local_reqs=local_reqs)
|
||||
self._deferred_input_requests.extend(
|
||||
self.request_receiver.recv_requests(local_reqs=local_reqs)
|
||||
)
|
||||
|
||||
num_requests_to_process = len(self._deferred_input_requests)
|
||||
if stop_at_pause:
|
||||
for index, req in enumerate(self._deferred_input_requests):
|
||||
if isinstance(req, PauseGenerationReqInput):
|
||||
num_requests_to_process = index
|
||||
break
|
||||
recv_reqs = self._deferred_input_requests[:num_requests_to_process]
|
||||
del self._deferred_input_requests[:num_requests_to_process]
|
||||
|
||||
if recv_reqs:
|
||||
self.metrics_reporter.record_scheduler_active()
|
||||
self.process_input_requests(recv_reqs)
|
||||
@@ -3391,16 +3415,18 @@ class Scheduler(
|
||||
|
||||
if (timeout_s := envs.SGLANG_REQ_RUNNING_TIMEOUT.get()) > 0:
|
||||
deadline = time.perf_counter() - timeout_s
|
||||
if get_parallel().pp_size == 1:
|
||||
inflight_batches = [self.running_batch, self.last_batch]
|
||||
else:
|
||||
inflight_batches = [*self.running_mbs, *self.mbs]
|
||||
inflight_batches = self._collect_inflight_batches()
|
||||
seen_rids = set()
|
||||
for batch in inflight_batches:
|
||||
if batch is None:
|
||||
continue
|
||||
for req in batch.reqs:
|
||||
if req.rid in seen_rids or req.finished():
|
||||
# Polling must not re-emit an abort awaiting result processing.
|
||||
if (
|
||||
req.rid in seen_rids
|
||||
or req.finished()
|
||||
or req.to_finish is not None
|
||||
):
|
||||
continue
|
||||
seen_rids.add(req.rid)
|
||||
if 0 < req.time_stats.forward_entry_time < deadline:
|
||||
@@ -3610,7 +3636,9 @@ class Scheduler(
|
||||
# todo hisparse, maybe other info to contain for the new batch
|
||||
return batch
|
||||
|
||||
def _process_hicache_events(self) -> None:
|
||||
def _process_hicache_events(
|
||||
self, should_retry_storage_prefetch: bool = True
|
||||
) -> None:
|
||||
# The HiCache drain is TP-wide consensus; run it before rank-local
|
||||
# decisions (_should_defer_prefill) or ranks enter different collectives.
|
||||
if (
|
||||
@@ -3619,7 +3647,7 @@ class Scheduler(
|
||||
or self.enable_unified_cache_external_linker
|
||||
):
|
||||
self.tree_cache.check_hicache_events()
|
||||
if self.enable_hicache_storage:
|
||||
if self.enable_hicache_storage and should_retry_storage_prefetch:
|
||||
self._process_storage_prefetch_retries()
|
||||
|
||||
@scheduler_stage_method(SCHEDULER_STAGE_GET_NEXT_BATCH)
|
||||
@@ -5433,13 +5461,22 @@ class Scheduler(
|
||||
)
|
||||
|
||||
def collect_inflight_reqs(self) -> Set[Req]:
|
||||
return {
|
||||
req
|
||||
for batch in self._collect_inflight_batches()
|
||||
if batch is not None
|
||||
for req in batch.reqs
|
||||
}
|
||||
|
||||
def _collect_inflight_batches(self) -> list[ScheduleBatch | None]:
|
||||
if get_parallel().pp_size == 1:
|
||||
inflight_batches = [self.running_batch, self.last_batch]
|
||||
else:
|
||||
inflight_batches = [*self.running_mbs, *self.mbs]
|
||||
return {
|
||||
req for batch in inflight_batches if batch is not None for req in batch.reqs
|
||||
}
|
||||
if self.enable_continuous_input_polling and self.result_queue:
|
||||
# Polling can run intake before the newest queued batch becomes last_batch.
|
||||
inflight_batches.extend(batch for batch, _ in self.result_queue)
|
||||
return inflight_batches
|
||||
|
||||
def abort_request(self, recv_req: AbortReq):
|
||||
if (chunked_req := self.chunked_req) is not None:
|
||||
@@ -5614,7 +5651,12 @@ class Scheduler(
|
||||
return
|
||||
|
||||
if self.enable_overlap and self.last_batch:
|
||||
# Process the results of the last batch
|
||||
if (
|
||||
self.disaggregation_mode == DisaggregationMode.PREFILL
|
||||
and (req := self.chunked_req) is not None
|
||||
):
|
||||
# Retract skips the chunk step that sets this result's KV send boundary.
|
||||
req.tmp_end_idx = min(req.extend_range.end, len(req.origin_input_ids))
|
||||
tmp_batch, tmp_result = self.result_queue.popleft()
|
||||
self.process_batch_result(tmp_batch, tmp_result)
|
||||
|
||||
@@ -5948,6 +5990,8 @@ def dispatch_event_loop(scheduler: Scheduler):
|
||||
|
||||
|
||||
def _dispatch_event_loop_once(scheduler: Scheduler):
|
||||
# A PD role switch can select a different loop on the same scheduler.
|
||||
scheduler.enable_continuous_input_polling = False
|
||||
disaggregation_mode: DisaggregationMode = scheduler.disaggregation_mode
|
||||
if disaggregation_mode == DisaggregationMode.NULL:
|
||||
if scheduler.enable_pdmux:
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -11,13 +11,16 @@ from unittest.mock import Mock, patch
|
||||
|
||||
from parameterized import parameterized
|
||||
|
||||
from sglang.srt.disaggregation.utils import DisaggregationMode
|
||||
from sglang.srt.distributed import parallel_state
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.managers import scheduler as scheduler_module
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.managers.schedule_batch import Req, ScheduleBatch
|
||||
from sglang.srt.managers.scheduler import Scheduler
|
||||
from sglang.srt.managers.utils import GenerationBatchResult
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.sampling.sampling_params import SamplingParams
|
||||
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import (
|
||||
@@ -78,6 +81,11 @@ class TestSchedulerIdleStepCounters(CustomTestCase):
|
||||
Scheduler.event_loop_overlap_disagg_prefill,
|
||||
ForwardMode.EXTEND,
|
||||
),
|
||||
(
|
||||
"prefill_polling",
|
||||
Scheduler.event_loop_overlap_disagg_prefill,
|
||||
ForwardMode.EXTEND,
|
||||
),
|
||||
(
|
||||
"decode_normal",
|
||||
Scheduler.event_loop_normal_disagg_decode,
|
||||
@@ -97,8 +105,45 @@ class TestSchedulerIdleStepCounters(CustomTestCase):
|
||||
with self.subTest(idle_iterations=idle_iterations):
|
||||
batches = self.make_batches(mode)
|
||||
schedule = batches[:2] + [None] * idle_iterations + batches[2:] + [None]
|
||||
poll_pending_copy = name == "prefill_polling"
|
||||
if poll_pending_copy and idle_iterations:
|
||||
# Keep work unavailable through the extra unfinished-copy pass.
|
||||
schedule.insert(2, None)
|
||||
scheduler = self.make_scheduler(schedule)
|
||||
self.run_and_check(scheduler, event_loop, mode, idle_iterations > 0)
|
||||
if poll_pending_copy:
|
||||
# Each pending copy adds one intake pass before completing.
|
||||
scheduler.request_receiver.recv_requests.side_effect = [[]] * (
|
||||
len(schedule) + len(batches)
|
||||
) + [StopIteration]
|
||||
with (
|
||||
envs.SGLANG_ENABLE_DISAGG_PREFILL_CONTINUOUS_INPUT_POLLING.override(
|
||||
poll_pending_copy
|
||||
)
|
||||
):
|
||||
self.run_and_check(
|
||||
scheduler,
|
||||
event_loop,
|
||||
mode,
|
||||
idle_iterations > 0,
|
||||
poll_pending_copy,
|
||||
)
|
||||
|
||||
def test_polling_admission_miss_during_copy_wait_preserves_busy_interval(self):
|
||||
batches = self.make_batches(ForwardMode.EXTEND)
|
||||
# New input after a miss can launch while the second copy is unfinished.
|
||||
schedule = batches[:2] + [None] + batches[2:] + [None]
|
||||
scheduler = self.make_scheduler(schedule)
|
||||
inputs = [[]] * (len(schedule) + len(batches))
|
||||
inputs[4] = [object()]
|
||||
scheduler.request_receiver.recv_requests.side_effect = inputs + [StopIteration]
|
||||
with envs.SGLANG_ENABLE_DISAGG_PREFILL_CONTINUOUS_INPUT_POLLING.override(True):
|
||||
self.run_and_check(
|
||||
scheduler,
|
||||
Scheduler.event_loop_overlap_disagg_prefill,
|
||||
ForwardMode.EXTEND,
|
||||
after_idle=False,
|
||||
poll_pending_copy=True,
|
||||
)
|
||||
|
||||
@parameterized.expand(
|
||||
[
|
||||
@@ -263,6 +308,23 @@ class TestSchedulerIdleStepCounters(CustomTestCase):
|
||||
2 if chained else 0,
|
||||
)
|
||||
|
||||
def test_mlx_pending_job_preserves_inflight_request_inventory(self):
|
||||
module = load_mlx_scheduler_module()
|
||||
scheduler = self.make_scheduler([])
|
||||
scheduler.disaggregation_mode = DisaggregationMode.NULL
|
||||
req = Req("mlx-pending", "", [1], SamplingParams())
|
||||
batch = ScheduleBatch(reqs=[req], forward_mode=ForwardMode.DECODE)
|
||||
scheduler.last_batch = batch
|
||||
pending = module.MlxPendingJob(
|
||||
launch=None,
|
||||
batch_copy=batch.copy(),
|
||||
schedule_batch=batch,
|
||||
reqs=batch.reqs[:],
|
||||
)
|
||||
scheduler.result_queue = deque([pending])
|
||||
|
||||
self.assertEqual(scheduler.collect_inflight_reqs(), {req})
|
||||
|
||||
def test_split_prefill_is_charged_once_from_its_first_chunk(self):
|
||||
# A split prefill spans several run_batch calls but yields one result,
|
||||
# so only its first chunk may set the burst's start and its idle flag.
|
||||
@@ -321,16 +383,26 @@ class TestSchedulerIdleStepCounters(CustomTestCase):
|
||||
for launch_ts in LAUNCH_TIMESTAMPS
|
||||
]
|
||||
|
||||
def run_and_check(self, scheduler, event_loop, mode, after_idle):
|
||||
def run_and_check(
|
||||
self, scheduler, event_loop, mode, after_idle, poll_pending_copy=False
|
||||
):
|
||||
observed_idle_flags = []
|
||||
observed_iters = []
|
||||
copy_events = []
|
||||
|
||||
def run_batch(batch, pp_proxy_tensors=None):
|
||||
# Exercise the real timestamp, iteration, and flag handoff.
|
||||
self.launch_batch(
|
||||
scheduler, batch, batch.launch_ts, pp_proxy_tensors=pp_proxy_tensors
|
||||
)
|
||||
return GenerationBatchResult()
|
||||
copy_done = Mock(
|
||||
query=Mock(
|
||||
return_value=True,
|
||||
side_effect=[False, True] if poll_pending_copy else None,
|
||||
)
|
||||
)
|
||||
copy_events.append(copy_done)
|
||||
return GenerationBatchResult(copy_done=copy_done)
|
||||
|
||||
def process_batch_result(batch, result):
|
||||
observed_idle_flags.append(batch.after_idle_gap)
|
||||
@@ -350,6 +422,8 @@ class TestSchedulerIdleStepCounters(CustomTestCase):
|
||||
):
|
||||
event_loop(scheduler)
|
||||
|
||||
if poll_pending_copy:
|
||||
self.assertTrue(all(event.query.call_count > 1 for event in copy_events))
|
||||
self.assertEqual(observed_idle_flags, [False, False, after_idle, False])
|
||||
self.assertEqual(scheduler.forward_ct, 4)
|
||||
self.assertEqual(observed_iters, [1, 2, 3, 4])
|
||||
@@ -403,13 +477,21 @@ class TestSchedulerIdleStepCounters(CustomTestCase):
|
||||
|
||||
def make_scheduler(self, schedule):
|
||||
scheduler = Scheduler.__new__(Scheduler)
|
||||
scheduler.tp_size = get_parallel().tp_size
|
||||
scheduler.gracefully_exit = False
|
||||
scheduler._engine_paused = False
|
||||
scheduler._deferred_input_requests = []
|
||||
scheduler._sched_idled = False
|
||||
scheduler._prev_step = None
|
||||
scheduler._prev_prefill_end_ts = None
|
||||
scheduler.forward_ct = 0
|
||||
scheduler.processed_tokens_counter = 0
|
||||
scheduler.spec_algorithm = SpeculativeAlgorithm.NONE
|
||||
scheduler.enable_overlap = True
|
||||
scheduler.enable_continuous_input_polling = False
|
||||
scheduler.is_generation = True
|
||||
scheduler.dllm_config = None
|
||||
scheduler.disaggregation_mode = DisaggregationMode.PREFILL
|
||||
scheduler._poll_timeout_aborts = Mock(return_value=[])
|
||||
scheduler.scheduler_stage_metrics = None
|
||||
scheduler.metrics_reporter = SimpleNamespace(record_scheduler_active=Mock())
|
||||
@@ -423,6 +505,7 @@ class TestSchedulerIdleStepCounters(CustomTestCase):
|
||||
scheduler.last_batch = None
|
||||
scheduler.chunked_req = None
|
||||
scheduler.waiting_queue = []
|
||||
scheduler.disagg_prefill_inflight_queue = []
|
||||
scheduler.enable_staging = False
|
||||
scheduler.request_receiver = SimpleNamespace(
|
||||
recv_requests=Mock(side_effect=[[]] * len(schedule) + [StopIteration])
|
||||
@@ -438,13 +521,13 @@ class TestSchedulerIdleStepCounters(CustomTestCase):
|
||||
scheduler.ngram_embedding_manager = SimpleNamespace(
|
||||
prepare_for_forward=lambda batch, chunked_req: batch
|
||||
)
|
||||
scheduled_batches = iter(schedule)
|
||||
# Intake ends the test; extra admission attempts simply find no work.
|
||||
next_plan = Mock(
|
||||
side_effect=[
|
||||
SimpleNamespace(
|
||||
running_batch=scheduler.running_batch, batch_to_run=batch
|
||||
)
|
||||
for batch in schedule
|
||||
]
|
||||
side_effect=lambda *args, **kwargs: SimpleNamespace(
|
||||
running_batch=scheduler.running_batch,
|
||||
batch_to_run=next(scheduled_batches, None),
|
||||
)
|
||||
)
|
||||
scheduler.get_next_disagg_prefill_batch_to_run = next_plan
|
||||
scheduler.get_next_disagg_decode_batch_to_run = next_plan
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Tests for deferred chunked-prefill aborts."""
|
||||
|
||||
import unittest
|
||||
from collections import deque
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
@@ -35,6 +36,8 @@ class _FakeReq:
|
||||
|
||||
def _make_scheduler(pending_req, *, chunked_req, running_reqs) -> Scheduler:
|
||||
sched = Scheduler.__new__(Scheduler)
|
||||
sched.enable_continuous_input_polling = False
|
||||
sched.result_queue = deque()
|
||||
sched.chunked_req = chunked_req
|
||||
sched._pending_chunked_abort_req = pending_req
|
||||
sched.waiting_queue = []
|
||||
|
||||
@@ -405,9 +405,12 @@ class TestSchedulerPauseGeneration(unittest.TestCase):
|
||||
)
|
||||
scheduler.result_queue = deque([(MagicMock(), MagicMock())])
|
||||
event_log: List[str] = []
|
||||
scheduler.process_batch_result = MagicMock(
|
||||
side_effect=lambda *args, **kwargs: event_log.append("drain")
|
||||
)
|
||||
|
||||
def process_result(batch, result):
|
||||
self.assertFalse(scheduler.result_queue)
|
||||
event_log.append("drain")
|
||||
|
||||
scheduler.process_batch_result = MagicMock(side_effect=process_result)
|
||||
scheduler._add_request_to_queue = MagicMock(
|
||||
side_effect=lambda req: event_log.append(
|
||||
"requeue-released" if req.is_retracted else "requeue-unreleased"
|
||||
@@ -448,19 +451,27 @@ class TestSchedulerPauseGeneration(unittest.TestCase):
|
||||
self.assertIs(scheduler.chunked_req, chunked_req)
|
||||
|
||||
def test_retract_drains_overlap_queue(self):
|
||||
"""retract with overlap enabled should drain the result_queue."""
|
||||
scheduler = self._new_scheduler()
|
||||
scheduler.enable_overlap = True
|
||||
mock_batch = MagicMock()
|
||||
mock_batch.forward_mode.is_extend.return_value = False
|
||||
scheduler.last_batch = mock_batch
|
||||
scheduler.result_queue = deque([(MagicMock(), MagicMock())])
|
||||
scheduler.process_batch_result = MagicMock()
|
||||
"""Single-result retract must preserve the existing chunk-preparation state."""
|
||||
for mode in (
|
||||
DisaggregationMode.NULL,
|
||||
DisaggregationMode.PREFILL,
|
||||
DisaggregationMode.DECODE,
|
||||
):
|
||||
with self.subTest(mode=mode):
|
||||
scheduler = self._new_scheduler()
|
||||
scheduler.enable_overlap = True
|
||||
scheduler.disaggregation_mode = mode
|
||||
batch = ScheduleBatch(reqs=[], forward_mode=ForwardMode.EXTEND)
|
||||
result = SimpleNamespace(delay_sample_func=None)
|
||||
scheduler.last_batch = batch
|
||||
scheduler.result_queue = deque([(batch, result)])
|
||||
scheduler.process_batch_result = MagicMock()
|
||||
|
||||
scheduler.pause_generation(PauseGenerationReqInput(mode="retract"))
|
||||
scheduler.pause_generation(PauseGenerationReqInput(mode="retract"))
|
||||
|
||||
scheduler.process_batch_result.assert_called_once()
|
||||
self.assertEqual(len(scheduler.result_queue), 0)
|
||||
scheduler.process_batch_result.assert_called_once_with(batch, result)
|
||||
self.assertFalse(scheduler.result_queue)
|
||||
self.assertFalse(batch.prefill_chunk_processed)
|
||||
|
||||
def test_pd_decode_retract_requeues_for_rebootstrap(self):
|
||||
"""PD decode retract should rebootstrap instead of resuming stale CPU KV."""
|
||||
|
||||
@@ -7,10 +7,13 @@ reaching the client is covered by scheduler/test_scheduler_control.py.
|
||||
|
||||
import time
|
||||
import unittest
|
||||
from collections import deque
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from sglang.srt.disaggregation.utils import DisaggregationMode
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import (
|
||||
CustomTestCase,
|
||||
@@ -21,6 +24,7 @@ from sglang.test.test_utils import (
|
||||
|
||||
maybe_stub_sgl_kernel()
|
||||
|
||||
from sglang.srt.managers.schedule_batch import FINISH_ABORT
|
||||
from sglang.srt.managers.scheduler import Scheduler
|
||||
from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
CacheRequestHandle,
|
||||
@@ -61,6 +65,8 @@ def _batch(reqs):
|
||||
|
||||
def _scheduler(waiting_queue, running_reqs=(), last_batch_reqs=()):
|
||||
s = Scheduler.__new__(Scheduler)
|
||||
s.enable_continuous_input_polling = False
|
||||
s.result_queue = deque()
|
||||
s.waiting_queue = waiting_queue
|
||||
s.enable_hierarchical_cache = False
|
||||
s.enable_hicache_storage = False
|
||||
@@ -156,6 +162,61 @@ class TestRunningTimeout(CustomTestCase):
|
||||
aborts = s._poll_timeout_aborts()
|
||||
self.assertEqual([a.rid for a in aborts], ["stale"])
|
||||
|
||||
def test_pending_abort_is_not_emitted_again_while_forward_is_running(self):
|
||||
stale = _req("stale", forward_entry=time.perf_counter() - 10)
|
||||
fresh = _req("fresh", forward_entry=time.perf_counter())
|
||||
s = _scheduler([], last_batch_reqs=[fresh])
|
||||
s.enable_continuous_input_polling = True
|
||||
s.result_queue = deque([(_batch([stale]), None), (s.last_batch, None)])
|
||||
s.disaggregation_mode = DisaggregationMode.PREFILL
|
||||
s.chunked_req = None
|
||||
s.mm_receiver = None
|
||||
s.dllm_config = None
|
||||
s.grammar_manager = MagicMock()
|
||||
s.disagg_prefill_bootstrap_queue = SimpleNamespace(queue=[])
|
||||
s.disagg_prefill_inflight_queue = []
|
||||
with envs.SGLANG_REQ_RUNNING_TIMEOUT.override(1.0):
|
||||
aborts = s._poll_timeout_aborts()
|
||||
self.assertEqual([abort.rid for abort in aborts], ["stale"])
|
||||
# The broadcast abort marks to_finish before the GPU result arrives.
|
||||
s.abort_request(aborts[0])
|
||||
self.assertIsInstance(stale.to_finish, FINISH_ABORT)
|
||||
self.assertIsNone(fresh.to_finish)
|
||||
self.assertFalse(stale.finished())
|
||||
for _ in range(3):
|
||||
self.assertEqual(s._poll_timeout_aborts(), [])
|
||||
|
||||
def test_queued_result_remains_visible_after_a_new_forward_launches(self):
|
||||
now = time.perf_counter()
|
||||
older = _req("older", forward_entry=now - 10)
|
||||
newer = _req("newer", forward_entry=now - 10)
|
||||
for pp_size in (1, 2):
|
||||
with (
|
||||
self.subTest(pp_size=pp_size),
|
||||
get_parallel().override(pp_size=pp_size),
|
||||
):
|
||||
# Polling re-enters intake before last_batch advances to newer.
|
||||
last_req = older if pp_size == 1 else newer
|
||||
s = _scheduler([], last_batch_reqs=[last_req])
|
||||
s.enable_continuous_input_polling = pp_size == 1
|
||||
s.running_mbs = [s.running_batch]
|
||||
s.mbs = [s.last_batch, _batch([older])]
|
||||
if pp_size == 1:
|
||||
s.result_queue = deque(
|
||||
[(_batch([older]), None), (_batch([newer]), None)]
|
||||
)
|
||||
|
||||
with envs.SGLANG_REQ_RUNNING_TIMEOUT.override(1.0):
|
||||
aborts = s._poll_timeout_aborts()
|
||||
# Abort and weight-version updates share this request inventory.
|
||||
self.assertEqual(s.collect_inflight_reqs(), {older, newer})
|
||||
|
||||
self.assertEqual(
|
||||
[a.rid for a in aborts],
|
||||
["older", "newer"] if pp_size == 1 else ["newer", "older"],
|
||||
)
|
||||
self.assertEqual(len(s.result_queue), 2 if pp_size == 1 else 0)
|
||||
|
||||
def test_unset_forward_entry_time_is_never_emitted(self):
|
||||
s = _scheduler([], running_reqs=[_req("unstamped", forward_entry=0.0)])
|
||||
with envs.SGLANG_REQ_RUNNING_TIMEOUT.override(1e-9):
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import random
|
||||
import unittest
|
||||
from collections import deque
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
@@ -295,6 +296,7 @@ class _ContextStub:
|
||||
|
||||
|
||||
class _SchedulerStub:
|
||||
_collect_inflight_batches = Scheduler._collect_inflight_batches
|
||||
collect_inflight_reqs = Scheduler.collect_inflight_reqs
|
||||
|
||||
def __init__(
|
||||
@@ -307,6 +309,8 @@ class _SchedulerStub:
|
||||
hisparse=None,
|
||||
):
|
||||
self.serving = _ServingStub(version)
|
||||
self.enable_continuous_input_polling = False
|
||||
self.result_queue = deque()
|
||||
self.running_batch = SimpleNamespace(reqs=running)
|
||||
self.last_batch = last_batch
|
||||
self.waiting_queue = waiting
|
||||
|
||||
Reference in New Issue
Block a user