diff --git a/python/sglang/srt/managers/prefill_delayer.py b/python/sglang/srt/managers/prefill_delayer.py index 5c7e968e0..bae1f65bc 100644 --- a/python/sglang/srt/managers/prefill_delayer.py +++ b/python/sglang/srt/managers/prefill_delayer.py @@ -95,6 +95,7 @@ class PrefillDelayer: if self._max_delay_ms is None: self._max_delay_ms = 5000.0 self._queue_trigger_enabled = self._queue_min_ratio is not None + self._prefill_max_requests = server_args.prefill_max_requests logger.info( f"PrefillDelayer initialized with " f"max_delay_passes={self._max_delay_passes} " @@ -250,9 +251,14 @@ class PrefillDelayer: # and fragment prefill into many tiny batches. queue_condition = False if self._queue_trigger_enabled and global_running_batch_max > 0: + queue_capacity = ( + self._prefill_max_requests + if self._prefill_max_requests is not None + else global_max_prefill_bs_max + ) queue_min_effective = min( int(global_running_batch_max * self._queue_min_ratio), - global_max_prefill_bs_max, + queue_capacity, ) queue_condition = ( queue_min_effective > 0 diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 9cd4d653f..c5b5c84ab 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -3356,9 +3356,9 @@ class ServerArgs: ( "Opt-in to the adaptive queue-based delay trigger (independent of the " "slot-based one). Delays prefill until the waiting queue reaches " - "min(running_req * ratio, max_prefill_bs) so small fragments batch " - "into a larger prefill. Unset (default) keeps the original slot-only " - "behavior. Typical: 0.1 ~ 0.5." + "min(running_req * ratio, prefill_max_requests), falling back to the " + "observed max_prefill_bs when no request limit is set. Unset (default) " + "keeps the original slot-only behavior. Typical: 0.1 ~ 0.5." ), NS("schedule"), ] = None diff --git a/test/registered/scheduler/test_prefill_delayer.py b/test/registered/scheduler/test_prefill_delayer.py index de0b97d07..08add3fb8 100644 --- a/test/registered/scheduler/test_prefill_delayer.py +++ b/test/registered/scheduler/test_prefill_delayer.py @@ -67,6 +67,7 @@ class NegotiateTestCase: # to exercise the legacy slot-only code paths. queue_min_ratio: Optional[float] = None max_delay_ms: Optional[float] = None + prefill_max_requests: Optional[int] = None # Expected accumulated wait surfaced on the final (release) outcome. When # set, asserts the wait histograms would observe this value instead of 0. expected_wait_forward_passes: Optional[int] = None @@ -90,6 +91,7 @@ def _run_negotiate_test(rank, test_cases): disable_overlap_schedule=False, prefill_delayer_queue_min_ratio=case.queue_min_ratio, prefill_delayer_max_delay_ms=case.max_delay_ms, + prefill_max_requests=case.prefill_max_requests, ), max_delay_passes=case.max_delay_passes, token_usage_low_watermark=case.token_usage_low_watermark, @@ -291,6 +293,34 @@ _NEGOTIATE_TEST_CASES = [ expected_allow=False, expected_reason="delay", ), + NegotiateTestCase( + name="queue_trigger_uses_admission_capacity", + max_delay_passes=100, + token_usage_low_watermark=0.8, + queue_min_ratio=0.02, + max_delay_ms=5000, + prefill_max_requests=128, + calls=[ + NegotiateCall( + prefillable=[True, True, True, True], + token_usage=[0.9, 0.9, 0.9, 0.9], + running_batch=[500, 500, 500, 500], + max_prefill_bs=[1, 1, 1, 1], + waiting_queue_len=[1, 1, 1, 1], + max_running_requests=1024, + ), + NegotiateCall( + prefillable=[True, True, True, True], + token_usage=[0.9, 0.9, 0.9, 0.9], + running_batch=[500, 500, 500, 500], + max_prefill_bs=[1, 1, 1, 1], + waiting_queue_len=[1, 1, 1, 1], + max_running_requests=1024, + ), + ], + expected_allow=False, + expected_reason="delay", + ), # Waiting queue at or above queue_min: queue trigger must not fire. NegotiateTestCase( name="queue_trigger_above_threshold",