From 448759b2358eb38e2a244fbe7d5a293a69dd314c Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Sat, 10 Oct 2026 22:46:55 -0700 Subject: [PATCH] [Scheduler] Requeue preempted requests when admission admits nothing (#43298) --- python/sglang/srt/managers/scheduler.py | 4 + .../scheduler/test_priority_scheduling.py | 134 ++++++++++-------- 2 files changed, 78 insertions(+), 60 deletions(-) diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 81230b84c3..e8d7ed755e 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -4103,6 +4103,10 @@ class Scheduler( # Update waiting queue can_run_list: List[Req] = adder.can_run_list if len(can_run_list) == 0: + # Preemption already released these requests; requeue them even + # though the request they made room for was not admitted. + for req in adder.preempt_list: + self._add_request_to_queue(req) return None, running_batch can_run_set = set(can_run_list) diff --git a/test/registered/scheduler/test_priority_scheduling.py b/test/registered/scheduler/test_priority_scheduling.py index bbd2b13a82..a6370c767f 100644 --- a/test/registered/scheduler/test_priority_scheduling.py +++ b/test/registered/scheduler/test_priority_scheduling.py @@ -57,19 +57,14 @@ class TestPriorityScheduling(CustomTestCase): def test_priority_scheduling_request_ordering_validation(self): """Verify pending requests are ordered by priority and received timestamp.""" - responses = asyncio.run( - send_concurrent_generate_requests_with_custom_params( - self.base_url, - [ - { - "priority": 0, - "sampling_params": {"max_new_tokens": 10000}, - }, # starts being processed first - {"priority": 1}, # third - {"priority": 1}, # fourth - {"priority": 2}, # second - ], - ) + responses, finish_times = _send_in_waves( + self.base_url, + [ + [_HOLDER | {"priority": 0}], # starts being processed first + [{"priority": 1}], # third + [{"priority": 1}], # fourth + [{"priority": 2}], # second + ], ) expected_status_and_error_messages = [ @@ -79,31 +74,22 @@ class TestPriorityScheduling(CustomTestCase): (200, None), ] - e2e_latencies = [] - _verify_genereate_responses( - responses, expected_status_and_error_messages, e2e_latencies - ) - assert e2e_latencies[0] < e2e_latencies[3] < e2e_latencies[1] < e2e_latencies[2] + _verify_genereate_responses(responses, expected_status_and_error_messages, []) + assert finish_times[0] < finish_times[3] < finish_times[1] < finish_times[2] def test_priority_scheduling_existing_requests_abortion_validation(self): """Verify lower priority requests are aborted when incoming requests have higher priority""" - responses = asyncio.run( - send_concurrent_generate_requests_with_custom_params( - self.base_url, - [ - { - "priority": 1, - "sampling_params": {"max_new_tokens": 10000}, - }, # starts being processed first and holds the running queue capacity - {"priority": 2}, # aborted by request 5 - {"priority": 3}, # aborted by request 6 - {"priority": 4}, # aborted by request 7 - {"priority": 5}, # fourth - {"priority": 6}, # third - {"priority": 7}, # second - ], - ) + responses, finish_times = _send_in_waves( + self.base_url, + [ + # starts being processed first and holds the running queue capacity + [_HOLDER | {"priority": 1}], + # fill the waiting queue; each is aborted by a later request + [{"priority": 2}, {"priority": 3}, {"priority": 4}], + # finish fourth, third, second + [{"priority": 5}, {"priority": 6}, {"priority": 7}], + ], ) expected_status_and_error_messages = [ @@ -116,31 +102,22 @@ class TestPriorityScheduling(CustomTestCase): (200, None), ] - e2e_latencies = [] - _verify_genereate_responses( - responses, expected_status_and_error_messages, e2e_latencies - ) - assert e2e_latencies[0] < e2e_latencies[6] < e2e_latencies[5] < e2e_latencies[4] + _verify_genereate_responses(responses, expected_status_and_error_messages, []) + assert finish_times[0] < finish_times[6] < finish_times[5] < finish_times[4] def test_priority_scheduling_incoming_request_rejection_validation(self): """Verify incoming requests are rejected when existing requests have higher priority""" - responses = asyncio.run( - send_concurrent_generate_requests_with_custom_params( - self.base_url, - [ - { - "priority": 7, - "sampling_params": {"max_new_tokens": 10000}, - }, # starts being processed first and holds the running queue capacity - {"priority": 6}, # second - {"priority": 5}, # third - {"priority": 4}, # fourth - {"priority": 3}, # rejected - {"priority": 2}, # rejected - {"priority": 1}, # rejected - ], - ) + responses, finish_times = _send_in_waves( + self.base_url, + [ + # starts being processed first and holds the running queue capacity + [_HOLDER | {"priority": 7}], + # fill the waiting queue; finish second, third, fourth + [{"priority": 6}, {"priority": 5}, {"priority": 4}], + # rejected: the queue is full of higher-priority requests + [{"priority": 3}, {"priority": 2}, {"priority": 1}], + ], ) expected_status_and_error_messages = [ @@ -153,11 +130,8 @@ class TestPriorityScheduling(CustomTestCase): (503, "The request queue is full."), ] - e2e_latencies = [] - _verify_genereate_responses( - responses, expected_status_and_error_messages, e2e_latencies - ) - assert e2e_latencies[0] < e2e_latencies[1] < e2e_latencies[2] < e2e_latencies[3] + _verify_genereate_responses(responses, expected_status_and_error_messages, []) + assert finish_times[0] < finish_times[1] < finish_times[2] < finish_times[3] def test_priority_scheduling_preemption_meeting_threshold_validation(self): """Verify running requests are preempted by requests with priorities meeting the preemption threshold""" @@ -378,6 +352,46 @@ class TestPrioritySchedulingMultipleRunningRequests(CustomTestCase): # assert e2e_latencies[2] < e2e_latencies[3] < e2e_latencies[1] < e2e_latencies[0] +# Holds the only running slot while later waves arrive: ignore_eos keeps it +# running for its full length. +_HOLDER = {"sampling_params": {"max_new_tokens": 1000, "ignore_eos": True}} + + +def _send_in_waves( + base_url: str, waves: List[List[dict]], gap_s: float = 0.3 +) -> Tuple[List[Tuple[int, Any]], List[Optional[float]]]: + """Send each wave concurrently, gap_s after the previous one. + + asyncio.gather gives no arrival-order guarantee, so requests whose + relative arrival matters go in separate waves. Returns the responses in + order and each request's finish time (send offset + server e2e latency), + or None for a failed request. + """ + + async def _run(): + tasks, offsets = [], [] + for i, wave in enumerate(waves): + if i: + await asyncio.sleep(gap_s) + tasks.append( + asyncio.create_task( + send_concurrent_generate_requests_with_custom_params(base_url, wave) + ) + ) + offsets += [i * gap_s] * len(wave) + responses = [] + for task in tasks: + responses += await task + return responses, offsets + + responses, offsets = asyncio.run(_run()) + finish_times = [ + offset + body["meta_info"]["e2e_latency"] if status == 200 else None + for (status, body), offset in zip(responses, offsets) + ] + return responses, finish_times + + def _verify_genereate_responses( responses: Tuple[int, Any, float], expected_code_and_error_message: Tuple[int, Any],