[PD] Keep ingesting requests while a prefill forward result is pending (#42035)
build-sglang-image / build (push) Successful in 26m6s

Co-authored-by: merrymercy <merrymercy@users.noreply.github.com>
Co-authored-by: Jialin <Jialin@users.noreply.github.com>
(cherry picked from commit 6acf2f4736)
This commit is contained in:
metamergebot
2026-10-09 10:56:14 +08:00
committed by minke.yu
parent 4d0efcc921
commit 93d610c1fd
10 changed files with 1827 additions and 71 deletions
+151 -32
View File
@@ -21,6 +21,7 @@ from __future__ import annotations
import hashlib
import logging
import time
from array import array
from collections import deque
from http import HTTPStatus
@@ -103,6 +104,12 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
# TP=1 yield between polls of a pending result. An initial heuristic rather
# than a tuned value: long enough that the bootstrap, transfer and cache
# threads reliably win the GIL even when the input socket is empty, and far
# below the duration of any prefill forward, so it adds no visible latency.
PREFILL_INPUT_POLL_INTERVAL_S = 0.0001
_is_npu = is_npu()
@@ -608,9 +615,12 @@ class SchedulerDisaggregationPrefillMixin:
self: Scheduler,
running_batch: ScheduleBatch,
last_batch: Optional[ScheduleBatch],
should_retry_storage_prefetch: bool = True,
) -> NextBatchPlan:
self.process_pending_chunked_abort()
self._process_hicache_events()
self._process_hicache_events(
should_retry_storage_prefetch=should_retry_storage_prefetch
)
# HACK (byronhsu): reset the batch_is_full flag because we never enter update_running_batch which resets it
# Otherwise, it hangs under high concurrency
@@ -618,7 +628,13 @@ class SchedulerDisaggregationPrefillMixin:
self.resolve_waiting_queue_bootstrap()
self.process_prefill_chunk(last_batch=last_batch, running_batch=running_batch)
# Skip repeated chunk processing while polling a pending batch.
if last_batch is None or not last_batch.prefill_chunk_processed:
self.process_prefill_chunk(
last_batch=last_batch, running_batch=running_batch
)
if last_batch is not None:
last_batch.prefill_chunk_processed = True
prefill_plan = self.get_new_batch_prefill(running_batch)
batch = prefill_plan.batch_to_run
@@ -669,46 +685,120 @@ class SchedulerDisaggregationPrefillMixin:
# Update last_batch
self.last_batch = batch
def _is_continuous_input_polling_enabled(self: Scheduler) -> bool:
"""Check support once on entry to the prefill overlap loop."""
if not envs.SGLANG_ENABLE_DISAGG_PREFILL_CONTINUOUS_INPUT_POLLING.get():
return False
parallel = get_parallel()
enabled = (
parallel.attn_dp_size == 1
and parallel.attn_cp_size == 1
and self.spec_algorithm.is_none()
and self.is_generation
and self.dllm_config is None
)
if not enabled:
logger.warning(
"Continuous prefill input polling requires attention DP=CP=1 and "
"non-speculative autoregressive generation; using the regular overlap loop."
)
return enabled
def _yield_gil_if_needed(self: Scheduler) -> None:
if self.tp_size == 1:
# No collective parks this thread at TP=1. A bounded sleep lets
# bootstrap, transfer and cache workers reacquire the GIL.
time.sleep(PREFILL_INPUT_POLL_INTERVAL_S)
@torch.no_grad()
def event_loop_overlap_disagg_prefill(self: Scheduler) -> None:
"""Run intake while pending copies finish, keeping at most two forwards.
A deferred pause stops preparation until the current result wait finishes.
Shutdown stops new launches; a wait finishes through the blocking handler.
After a failed preparation, only new input or bootstrap arrivals retry
during the wait; other readiness is checked after a result finishes.
"""
self.enable_continuous_input_polling = (
self._is_continuous_input_polling_enabled()
)
self.result_queue = deque()
batch_result_completion_status = torch.empty(1, dtype=torch.int32, device="cpu")
# True when intake repeats while the oldest batch result is unfinished.
waiting_for_batch_result = False
# Avoid retrying failed batch preparation after every copy poll.
skip_batch_creation = False
# Keep the newer forward's sampling context across unfinished-result passes.
batch = None
batch_result = None
while True:
# Receive requests
self.ingest_requests()
if self.gracefully_exit:
break
# Stop dispatching at pause requests while waiting for a batch result.
received_inputs = self.ingest_requests(
stop_at_pause=waiting_for_batch_result
)
if self._engine_paused:
self._record_scheduler_state_for_paused_engine()
self._yield_gil_if_needed()
continue
self.waiting_queue.extend(
self.disagg_prefill_bootstrap_queue.pop_bootstrapped()
)
bootstrapped_reqs = self.disagg_prefill_bootstrap_queue.pop_bootstrapped()
self.waiting_queue.extend(bootstrapped_reqs)
# Resume batch creation on completion, new/ready requests, or an idle step.
if not waiting_for_batch_result or received_inputs or bootstrapped_reqs:
skip_batch_creation = False
# Get the next batch to run
plan = self.get_next_disagg_prefill_batch_to_run(
running_batch=self.running_batch, last_batch=self.last_batch
)
self.running_batch = plan.running_batch
batch = plan.batch_to_run
batch = self.ngram_embedding_manager.prepare_for_forward(
batch, chunked_req=self.chunked_req
)
self.cur_batch_for_debug = batch
# Keep at most two batches in flight.
if (
len(self.result_queue) < 2
and not self.gracefully_exit
and not self._deferred_input_requests
and not skip_batch_creation
):
# Get the next batch to run
plan = self.get_next_disagg_prefill_batch_to_run(
running_batch=self.running_batch,
last_batch=self.last_batch,
# Storage retry deadlines count scheduling steps, not copy polls.
should_retry_storage_prefetch=not waiting_for_batch_result,
)
self.running_batch = plan.running_batch
batch = plan.batch_to_run
batch = self.ngram_embedding_manager.prepare_for_forward(
batch, chunked_req=self.chunked_req
)
self.cur_batch_for_debug = batch
# Skip batch creation next round to avoid repeated preparation failures.
skip_batch_creation = batch is None
# Launch the current batch
if batch:
if self.enable_staging:
self.maybe_prefetch_staging_for_batch(batch)
batch_result = self.run_batch(batch)
self._apply_war_barrier()
self.result_queue.append((batch.copy(), batch_result))
else:
batch_result = None
self._sched_idled = True
# Launch the current batch
if batch:
if self.enable_staging:
self.maybe_prefetch_staging_for_batch(batch)
batch_result = self.run_batch(batch)
self._apply_war_barrier()
self.result_queue.append((batch.copy(), batch_result))
else:
batch_result = None
# Process the last batch
if self.last_batch:
tmp_batch, tmp_result = self.result_queue.popleft()
self.process_batch_result(tmp_batch, tmp_result)
if self.enable_continuous_input_polling:
# Resume intake until the oldest result is ready on all TP ranks.
oldest_batch_result = self.result_queue[0][1]
if (
not self.gracefully_exit
and not self.is_disagg_prefill_batch_result_ready(
oldest_batch_result, batch_result_completion_status
)
):
waiting_for_batch_result = True
self._yield_gil_if_needed()
continue
# Process the oldest batch result.
oldest_batch, oldest_batch_result = self.result_queue.popleft()
self.process_batch_result(oldest_batch, oldest_batch_result)
elif batch is None:
# When the server is idle, do self-check and re-init some states
self.on_idle()
@@ -719,8 +809,36 @@ class SchedulerDisaggregationPrefillMixin:
# It depends on the result of the last batch (e.g., grammar), so we run it after the last batch is processed.
self.launch_batch_sample_if_needed(batch_result, batch)
# Update last_batch
# Update last_batch and scheduler status
self._sched_idled = batch is None
self.last_batch = batch
waiting_for_batch_result = False
def is_disagg_prefill_batch_result_ready(
self: Scheduler,
batch_result: GenerationBatchResult,
batch_result_completion_status: torch.Tensor,
) -> bool:
# 0: sampling/copy has not been submitted, so polling cannot make progress.
state = 0
if (
batch_result.copy_done is not None
and batch_result.delay_sample_func is None
):
# 1: copy unfinished; 2: copy finished locally.
state = 2 if batch_result.copy_done.query() else 1
if self.tp_size > 1:
batch_result_completion_status.fill_(state)
torch.distributed.all_reduce(
batch_result_completion_status,
op=torch.distributed.ReduceOp.MIN,
group=self.tp_cpu_group,
)
state = batch_result_completion_status.item()
if state == 0:
# Every rank must submit delayed sampling before entering this wait.
raise RuntimeError("Prefill result copy must be submitted before polling")
return state == 2
def process_batch_result_disagg_prefill(
self: Scheduler,
@@ -1217,7 +1335,8 @@ class SchedulerDisaggregationPrefillMixin:
elif self.has_bootstrapped_waiting_req():
# optimistic request yields to waiting requests
self.chunked_req = None
if not self.enable_overlap:
# Retract may drain every result, leaving no handler to requeue.
if not self.enable_overlap or req.inflight_middle_chunks == 0:
self.optimistic_release_and_requeue(req)
# else: still bootstrapping, keep computing without sending
elif self.enable_overlap:
+2
View File
@@ -658,6 +658,8 @@ class Envs:
# do not fence the next forward through the scheduler stream.
SGLANG_PP_COMM_OVERLAP = EnvBool(False)
SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH = EnvBool(False)
# Opt-in: keep receiving prefill requests while forward results are pending.
SGLANG_ENABLE_DISAGG_PREFILL_CONTINUOUS_INPUT_POLLING = EnvBool(False)
# ===================================================================
# Radix and sparse KV caches
@@ -2335,6 +2335,9 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
chunked_req_next_prompt_token: Optional[int] = None
contains_last_prefill_chunk: bool = True
# Tracks process_prefill_chunk() on the original scheduler batch only.
prefill_chunk_processed: bool = False
# For DP attention
inner_idle_batch: Optional[ScheduleBatch] = None
# Decode requests carried alongside a chunked-prefill batch
+59 -15
View File
@@ -1270,6 +1270,7 @@ class Scheduler(
self.hisparse_coordinator.set_decode_producer_stream(self.forward_stream)
def init_running_status(self):
self.tp_size = get_parallel().tp_size
# Set by a runtime PD role switch to break out of the current event loop.
self._event_loop_should_restart = False
# Guards against concurrent/re-entrant PD role switches.
@@ -1286,6 +1287,10 @@ class Scheduler(
self.cur_batch_for_debug: Optional[ScheduleBatch] = None
# The last forward batch
self.last_batch: Optional[ScheduleBatch] = None
self.result_queue: deque[
tuple[ScheduleBatch, GenerationBatchResult | EmbeddingBatchResult]
] = deque()
self.enable_continuous_input_polling = False
self.forward_ct = 0
self.return_health_check_ipcs: Deque[Optional[str]] = deque()
self.flush_wrapper = SchedulerFlushWrapper(
@@ -1297,6 +1302,7 @@ class Scheduler(
self.session_controller = SessionController(self.tree_cache)
self.forward_sleep_time = None
self._engine_paused = False
self._deferred_input_requests: List = []
def init_chunked_prefill(self):
self.chunked_prefill_size = get_schedule().chunked_prefill_size
@@ -2067,20 +2073,38 @@ class Scheduler(
for prev_batch, prev_result in self.result_queue:
self.batch_result_processor.advance_grammar_fsm(prev_result, prev_batch)
def ingest_requests(self) -> List:
"""Receive, broadcast and dispatch this iteration's external input.
def ingest_requests(self, stop_at_pause: bool = False) -> List:
"""Receive and broadcast input, then dispatch queued requests in order.
With stop_at_pause, retain the first pause and all following inputs
until the caller reaches a scheduling boundary. Receiving and broadcasting
continue each pass; all ranks retain the same suffix.
The one place a new per-iteration input source belongs; the return
value exists for the pipeline stages that relay requests onward.
"""
local_reqs = []
# Avoid re-enqueuing timeout aborts while dispatch is stopped at a pause.
if (
get_parallel().pp_rank == 0
not self._deferred_input_requests
and get_parallel().pp_rank == 0
and get_parallel().attn_tp_rank == 0
and get_parallel().attn_cp_rank == 0
):
local_reqs = self._poll_timeout_aborts()
recv_reqs = self.request_receiver.recv_requests(local_reqs=local_reqs)
self._deferred_input_requests.extend(
self.request_receiver.recv_requests(local_reqs=local_reqs)
)
num_requests_to_process = len(self._deferred_input_requests)
if stop_at_pause:
for index, req in enumerate(self._deferred_input_requests):
if isinstance(req, PauseGenerationReqInput):
num_requests_to_process = index
break
recv_reqs = self._deferred_input_requests[:num_requests_to_process]
del self._deferred_input_requests[:num_requests_to_process]
if recv_reqs:
self.metrics_reporter.record_scheduler_active()
self.process_input_requests(recv_reqs)
@@ -3391,16 +3415,18 @@ class Scheduler(
if (timeout_s := envs.SGLANG_REQ_RUNNING_TIMEOUT.get()) > 0:
deadline = time.perf_counter() - timeout_s
if get_parallel().pp_size == 1:
inflight_batches = [self.running_batch, self.last_batch]
else:
inflight_batches = [*self.running_mbs, *self.mbs]
inflight_batches = self._collect_inflight_batches()
seen_rids = set()
for batch in inflight_batches:
if batch is None:
continue
for req in batch.reqs:
if req.rid in seen_rids or req.finished():
# Polling must not re-emit an abort awaiting result processing.
if (
req.rid in seen_rids
or req.finished()
or req.to_finish is not None
):
continue
seen_rids.add(req.rid)
if 0 < req.time_stats.forward_entry_time < deadline:
@@ -3610,7 +3636,9 @@ class Scheduler(
# todo hisparse, maybe other info to contain for the new batch
return batch
def _process_hicache_events(self) -> None:
def _process_hicache_events(
self, should_retry_storage_prefetch: bool = True
) -> None:
# The HiCache drain is TP-wide consensus; run it before rank-local
# decisions (_should_defer_prefill) or ranks enter different collectives.
if (
@@ -3619,7 +3647,7 @@ class Scheduler(
or self.enable_unified_cache_external_linker
):
self.tree_cache.check_hicache_events()
if self.enable_hicache_storage:
if self.enable_hicache_storage and should_retry_storage_prefetch:
self._process_storage_prefetch_retries()
@scheduler_stage_method(SCHEDULER_STAGE_GET_NEXT_BATCH)
@@ -5433,13 +5461,22 @@ class Scheduler(
)
def collect_inflight_reqs(self) -> Set[Req]:
return {
req
for batch in self._collect_inflight_batches()
if batch is not None
for req in batch.reqs
}
def _collect_inflight_batches(self) -> list[ScheduleBatch | None]:
if get_parallel().pp_size == 1:
inflight_batches = [self.running_batch, self.last_batch]
else:
inflight_batches = [*self.running_mbs, *self.mbs]
return {
req for batch in inflight_batches if batch is not None for req in batch.reqs
}
if self.enable_continuous_input_polling and self.result_queue:
# Polling can run intake before the newest queued batch becomes last_batch.
inflight_batches.extend(batch for batch, _ in self.result_queue)
return inflight_batches
def abort_request(self, recv_req: AbortReq):
if (chunked_req := self.chunked_req) is not None:
@@ -5614,7 +5651,12 @@ class Scheduler(
return
if self.enable_overlap and self.last_batch:
# Process the results of the last batch
if (
self.disaggregation_mode == DisaggregationMode.PREFILL
and (req := self.chunked_req) is not None
):
# Retract skips the chunk step that sets this result's KV send boundary.
req.tmp_end_idx = min(req.extend_range.end, len(req.origin_input_ids))
tmp_batch, tmp_result = self.result_queue.popleft()
self.process_batch_result(tmp_batch, tmp_result)
@@ -5948,6 +5990,8 @@ def dispatch_event_loop(scheduler: Scheduler):
def _dispatch_event_loop_once(scheduler: Scheduler):
# A PD role switch can select a different loop on the same scheduler.
scheduler.enable_continuous_input_polling = False
disaggregation_mode: DisaggregationMode = scheduler.disaggregation_mode
if disaggregation_mode == DisaggregationMode.NULL:
if scheduler.enable_pdmux:
File diff suppressed because it is too large Load Diff
@@ -11,13 +11,16 @@ from unittest.mock import Mock, patch
from parameterized import parameterized
from sglang.srt.disaggregation.utils import DisaggregationMode
from sglang.srt.distributed import parallel_state
from sglang.srt.environ import envs
from sglang.srt.managers import scheduler as scheduler_module
from sglang.srt.managers.schedule_batch import ScheduleBatch
from sglang.srt.managers.schedule_batch import Req, ScheduleBatch
from sglang.srt.managers.scheduler import Scheduler
from sglang.srt.managers.utils import GenerationBatchResult
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.runtime_context import get_parallel
from sglang.srt.sampling.sampling_params import SamplingParams
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import (
@@ -78,6 +81,11 @@ class TestSchedulerIdleStepCounters(CustomTestCase):
Scheduler.event_loop_overlap_disagg_prefill,
ForwardMode.EXTEND,
),
(
"prefill_polling",
Scheduler.event_loop_overlap_disagg_prefill,
ForwardMode.EXTEND,
),
(
"decode_normal",
Scheduler.event_loop_normal_disagg_decode,
@@ -97,8 +105,45 @@ class TestSchedulerIdleStepCounters(CustomTestCase):
with self.subTest(idle_iterations=idle_iterations):
batches = self.make_batches(mode)
schedule = batches[:2] + [None] * idle_iterations + batches[2:] + [None]
poll_pending_copy = name == "prefill_polling"
if poll_pending_copy and idle_iterations:
# Keep work unavailable through the extra unfinished-copy pass.
schedule.insert(2, None)
scheduler = self.make_scheduler(schedule)
self.run_and_check(scheduler, event_loop, mode, idle_iterations > 0)
if poll_pending_copy:
# Each pending copy adds one intake pass before completing.
scheduler.request_receiver.recv_requests.side_effect = [[]] * (
len(schedule) + len(batches)
) + [StopIteration]
with (
envs.SGLANG_ENABLE_DISAGG_PREFILL_CONTINUOUS_INPUT_POLLING.override(
poll_pending_copy
)
):
self.run_and_check(
scheduler,
event_loop,
mode,
idle_iterations > 0,
poll_pending_copy,
)
def test_polling_admission_miss_during_copy_wait_preserves_busy_interval(self):
batches = self.make_batches(ForwardMode.EXTEND)
# New input after a miss can launch while the second copy is unfinished.
schedule = batches[:2] + [None] + batches[2:] + [None]
scheduler = self.make_scheduler(schedule)
inputs = [[]] * (len(schedule) + len(batches))
inputs[4] = [object()]
scheduler.request_receiver.recv_requests.side_effect = inputs + [StopIteration]
with envs.SGLANG_ENABLE_DISAGG_PREFILL_CONTINUOUS_INPUT_POLLING.override(True):
self.run_and_check(
scheduler,
Scheduler.event_loop_overlap_disagg_prefill,
ForwardMode.EXTEND,
after_idle=False,
poll_pending_copy=True,
)
@parameterized.expand(
[
@@ -263,6 +308,23 @@ class TestSchedulerIdleStepCounters(CustomTestCase):
2 if chained else 0,
)
def test_mlx_pending_job_preserves_inflight_request_inventory(self):
module = load_mlx_scheduler_module()
scheduler = self.make_scheduler([])
scheduler.disaggregation_mode = DisaggregationMode.NULL
req = Req("mlx-pending", "", [1], SamplingParams())
batch = ScheduleBatch(reqs=[req], forward_mode=ForwardMode.DECODE)
scheduler.last_batch = batch
pending = module.MlxPendingJob(
launch=None,
batch_copy=batch.copy(),
schedule_batch=batch,
reqs=batch.reqs[:],
)
scheduler.result_queue = deque([pending])
self.assertEqual(scheduler.collect_inflight_reqs(), {req})
def test_split_prefill_is_charged_once_from_its_first_chunk(self):
# A split prefill spans several run_batch calls but yields one result,
# so only its first chunk may set the burst's start and its idle flag.
@@ -321,16 +383,26 @@ class TestSchedulerIdleStepCounters(CustomTestCase):
for launch_ts in LAUNCH_TIMESTAMPS
]
def run_and_check(self, scheduler, event_loop, mode, after_idle):
def run_and_check(
self, scheduler, event_loop, mode, after_idle, poll_pending_copy=False
):
observed_idle_flags = []
observed_iters = []
copy_events = []
def run_batch(batch, pp_proxy_tensors=None):
# Exercise the real timestamp, iteration, and flag handoff.
self.launch_batch(
scheduler, batch, batch.launch_ts, pp_proxy_tensors=pp_proxy_tensors
)
return GenerationBatchResult()
copy_done = Mock(
query=Mock(
return_value=True,
side_effect=[False, True] if poll_pending_copy else None,
)
)
copy_events.append(copy_done)
return GenerationBatchResult(copy_done=copy_done)
def process_batch_result(batch, result):
observed_idle_flags.append(batch.after_idle_gap)
@@ -350,6 +422,8 @@ class TestSchedulerIdleStepCounters(CustomTestCase):
):
event_loop(scheduler)
if poll_pending_copy:
self.assertTrue(all(event.query.call_count > 1 for event in copy_events))
self.assertEqual(observed_idle_flags, [False, False, after_idle, False])
self.assertEqual(scheduler.forward_ct, 4)
self.assertEqual(observed_iters, [1, 2, 3, 4])
@@ -403,13 +477,21 @@ class TestSchedulerIdleStepCounters(CustomTestCase):
def make_scheduler(self, schedule):
scheduler = Scheduler.__new__(Scheduler)
scheduler.tp_size = get_parallel().tp_size
scheduler.gracefully_exit = False
scheduler._engine_paused = False
scheduler._deferred_input_requests = []
scheduler._sched_idled = False
scheduler._prev_step = None
scheduler._prev_prefill_end_ts = None
scheduler.forward_ct = 0
scheduler.processed_tokens_counter = 0
scheduler.spec_algorithm = SpeculativeAlgorithm.NONE
scheduler.enable_overlap = True
scheduler.enable_continuous_input_polling = False
scheduler.is_generation = True
scheduler.dllm_config = None
scheduler.disaggregation_mode = DisaggregationMode.PREFILL
scheduler._poll_timeout_aborts = Mock(return_value=[])
scheduler.scheduler_stage_metrics = None
scheduler.metrics_reporter = SimpleNamespace(record_scheduler_active=Mock())
@@ -423,6 +505,7 @@ class TestSchedulerIdleStepCounters(CustomTestCase):
scheduler.last_batch = None
scheduler.chunked_req = None
scheduler.waiting_queue = []
scheduler.disagg_prefill_inflight_queue = []
scheduler.enable_staging = False
scheduler.request_receiver = SimpleNamespace(
recv_requests=Mock(side_effect=[[]] * len(schedule) + [StopIteration])
@@ -438,13 +521,13 @@ class TestSchedulerIdleStepCounters(CustomTestCase):
scheduler.ngram_embedding_manager = SimpleNamespace(
prepare_for_forward=lambda batch, chunked_req: batch
)
scheduled_batches = iter(schedule)
# Intake ends the test; extra admission attempts simply find no work.
next_plan = Mock(
side_effect=[
SimpleNamespace(
running_batch=scheduler.running_batch, batch_to_run=batch
)
for batch in schedule
]
side_effect=lambda *args, **kwargs: SimpleNamespace(
running_batch=scheduler.running_batch,
batch_to_run=next(scheduled_batches, None),
)
)
scheduler.get_next_disagg_prefill_batch_to_run = next_plan
scheduler.get_next_disagg_decode_batch_to_run = next_plan
@@ -1,6 +1,7 @@
"""Tests for deferred chunked-prefill aborts."""
import unittest
from collections import deque
from types import SimpleNamespace
from unittest.mock import Mock
@@ -35,6 +36,8 @@ class _FakeReq:
def _make_scheduler(pending_req, *, chunked_req, running_reqs) -> Scheduler:
sched = Scheduler.__new__(Scheduler)
sched.enable_continuous_input_polling = False
sched.result_queue = deque()
sched.chunked_req = chunked_req
sched._pending_chunked_abort_req = pending_req
sched.waiting_queue = []
@@ -405,9 +405,12 @@ class TestSchedulerPauseGeneration(unittest.TestCase):
)
scheduler.result_queue = deque([(MagicMock(), MagicMock())])
event_log: List[str] = []
scheduler.process_batch_result = MagicMock(
side_effect=lambda *args, **kwargs: event_log.append("drain")
)
def process_result(batch, result):
self.assertFalse(scheduler.result_queue)
event_log.append("drain")
scheduler.process_batch_result = MagicMock(side_effect=process_result)
scheduler._add_request_to_queue = MagicMock(
side_effect=lambda req: event_log.append(
"requeue-released" if req.is_retracted else "requeue-unreleased"
@@ -448,19 +451,27 @@ class TestSchedulerPauseGeneration(unittest.TestCase):
self.assertIs(scheduler.chunked_req, chunked_req)
def test_retract_drains_overlap_queue(self):
"""retract with overlap enabled should drain the result_queue."""
scheduler = self._new_scheduler()
scheduler.enable_overlap = True
mock_batch = MagicMock()
mock_batch.forward_mode.is_extend.return_value = False
scheduler.last_batch = mock_batch
scheduler.result_queue = deque([(MagicMock(), MagicMock())])
scheduler.process_batch_result = MagicMock()
"""Single-result retract must preserve the existing chunk-preparation state."""
for mode in (
DisaggregationMode.NULL,
DisaggregationMode.PREFILL,
DisaggregationMode.DECODE,
):
with self.subTest(mode=mode):
scheduler = self._new_scheduler()
scheduler.enable_overlap = True
scheduler.disaggregation_mode = mode
batch = ScheduleBatch(reqs=[], forward_mode=ForwardMode.EXTEND)
result = SimpleNamespace(delay_sample_func=None)
scheduler.last_batch = batch
scheduler.result_queue = deque([(batch, result)])
scheduler.process_batch_result = MagicMock()
scheduler.pause_generation(PauseGenerationReqInput(mode="retract"))
scheduler.pause_generation(PauseGenerationReqInput(mode="retract"))
scheduler.process_batch_result.assert_called_once()
self.assertEqual(len(scheduler.result_queue), 0)
scheduler.process_batch_result.assert_called_once_with(batch, result)
self.assertFalse(scheduler.result_queue)
self.assertFalse(batch.prefill_chunk_processed)
def test_pd_decode_retract_requeues_for_rebootstrap(self):
"""PD decode retract should rebootstrap instead of resuming stale CPU KV."""
@@ -7,10 +7,13 @@ reaching the client is covered by scheduler/test_scheduler_control.py.
import time
import unittest
from collections import deque
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
from sglang.srt.disaggregation.utils import DisaggregationMode
from sglang.srt.environ import envs
from sglang.srt.runtime_context import get_parallel
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import (
CustomTestCase,
@@ -21,6 +24,7 @@ from sglang.test.test_utils import (
maybe_stub_sgl_kernel()
from sglang.srt.managers.schedule_batch import FINISH_ABORT
from sglang.srt.managers.scheduler import Scheduler
from sglang.srt.mem_cache.base_prefix_cache import (
CacheRequestHandle,
@@ -61,6 +65,8 @@ def _batch(reqs):
def _scheduler(waiting_queue, running_reqs=(), last_batch_reqs=()):
s = Scheduler.__new__(Scheduler)
s.enable_continuous_input_polling = False
s.result_queue = deque()
s.waiting_queue = waiting_queue
s.enable_hierarchical_cache = False
s.enable_hicache_storage = False
@@ -156,6 +162,61 @@ class TestRunningTimeout(CustomTestCase):
aborts = s._poll_timeout_aborts()
self.assertEqual([a.rid for a in aborts], ["stale"])
def test_pending_abort_is_not_emitted_again_while_forward_is_running(self):
stale = _req("stale", forward_entry=time.perf_counter() - 10)
fresh = _req("fresh", forward_entry=time.perf_counter())
s = _scheduler([], last_batch_reqs=[fresh])
s.enable_continuous_input_polling = True
s.result_queue = deque([(_batch([stale]), None), (s.last_batch, None)])
s.disaggregation_mode = DisaggregationMode.PREFILL
s.chunked_req = None
s.mm_receiver = None
s.dllm_config = None
s.grammar_manager = MagicMock()
s.disagg_prefill_bootstrap_queue = SimpleNamespace(queue=[])
s.disagg_prefill_inflight_queue = []
with envs.SGLANG_REQ_RUNNING_TIMEOUT.override(1.0):
aborts = s._poll_timeout_aborts()
self.assertEqual([abort.rid for abort in aborts], ["stale"])
# The broadcast abort marks to_finish before the GPU result arrives.
s.abort_request(aborts[0])
self.assertIsInstance(stale.to_finish, FINISH_ABORT)
self.assertIsNone(fresh.to_finish)
self.assertFalse(stale.finished())
for _ in range(3):
self.assertEqual(s._poll_timeout_aborts(), [])
def test_queued_result_remains_visible_after_a_new_forward_launches(self):
now = time.perf_counter()
older = _req("older", forward_entry=now - 10)
newer = _req("newer", forward_entry=now - 10)
for pp_size in (1, 2):
with (
self.subTest(pp_size=pp_size),
get_parallel().override(pp_size=pp_size),
):
# Polling re-enters intake before last_batch advances to newer.
last_req = older if pp_size == 1 else newer
s = _scheduler([], last_batch_reqs=[last_req])
s.enable_continuous_input_polling = pp_size == 1
s.running_mbs = [s.running_batch]
s.mbs = [s.last_batch, _batch([older])]
if pp_size == 1:
s.result_queue = deque(
[(_batch([older]), None), (_batch([newer]), None)]
)
with envs.SGLANG_REQ_RUNNING_TIMEOUT.override(1.0):
aborts = s._poll_timeout_aborts()
# Abort and weight-version updates share this request inventory.
self.assertEqual(s.collect_inflight_reqs(), {older, newer})
self.assertEqual(
[a.rid for a in aborts],
["older", "newer"] if pp_size == 1 else ["newer", "older"],
)
self.assertEqual(len(s.result_queue), 2 if pp_size == 1 else 0)
def test_unset_forward_entry_time_is_never_emitted(self):
s = _scheduler([], running_reqs=[_req("unstamped", forward_entry=0.0)])
with envs.SGLANG_REQ_RUNNING_TIMEOUT.override(1e-9):
@@ -1,5 +1,6 @@
import random
import unittest
from collections import deque
from types import SimpleNamespace
from unittest.mock import patch
@@ -295,6 +296,7 @@ class _ContextStub:
class _SchedulerStub:
_collect_inflight_batches = Scheduler._collect_inflight_batches
collect_inflight_reqs = Scheduler.collect_inflight_reqs
def __init__(
@@ -307,6 +309,8 @@ class _SchedulerStub:
hisparse=None,
):
self.serving = _ServingStub(version)
self.enable_continuous_input_polling = False
self.result_queue = deque()
self.running_batch = SimpleNamespace(reqs=running)
self.last_batch = last_batch
self.waiting_queue = waiting