[Scheduler] Cap prefill-delayer queue target by admission capacity (#35191)

python/sglang/srt/managers/prefill_delayer.py

@@ -95,6 +95,7 @@ class PrefillDelayer:
         if self._max_delay_ms is None:
             self._max_delay_ms = 5000.0
         self._queue_trigger_enabled = self._queue_min_ratio is not None
+        self._prefill_max_requests = server_args.prefill_max_requests
         logger.info(
             f"PrefillDelayer initialized with "
             f"max_delay_passes={self._max_delay_passes} "
@@ -250,9 +251,14 @@ class PrefillDelayer:
             # and fragment prefill into many tiny batches.
             queue_condition = False
             if self._queue_trigger_enabled and global_running_batch_max > 0:
+                queue_capacity = (
+                    self._prefill_max_requests
+                    if self._prefill_max_requests is not None
+                    else global_max_prefill_bs_max
+                )
                 queue_min_effective = min(
                     int(global_running_batch_max * self._queue_min_ratio),
-                    global_max_prefill_bs_max,
+                    queue_capacity,
                 )
                 queue_condition = (
                     queue_min_effective > 0

python/sglang/srt/server_args.py

@@ -3356,9 +3356,9 @@ class ServerArgs:
         (
             "Opt-in to the adaptive queue-based delay trigger (independent of the "
             "slot-based one). Delays prefill until the waiting queue reaches "
-            "min(running_req * ratio, max_prefill_bs) so small fragments batch "
+            "min(running_req * ratio, prefill_max_requests), falling back to the "
-            "into a larger prefill. Unset (default) keeps the original slot-only "
+            "observed max_prefill_bs when no request limit is set. Unset (default) "
-            "behavior. Typical: 0.1 ~ 0.5."
+            "keeps the original slot-only behavior. Typical: 0.1 ~ 0.5."
         ),
         NS("schedule"),
     ] = None

test/registered/scheduler/test_prefill_delayer.py

@@ -67,6 +67,7 @@ class NegotiateTestCase:
     # to exercise the legacy slot-only code paths.
     queue_min_ratio: Optional[float] = None
     max_delay_ms: Optional[float] = None
+    prefill_max_requests: Optional[int] = None
     # Expected accumulated wait surfaced on the final (release) outcome. When
     # set, asserts the wait histograms would observe this value instead of 0.
     expected_wait_forward_passes: Optional[int] = None
@@ -90,6 +91,7 @@ def _run_negotiate_test(rank, test_cases):
                 disable_overlap_schedule=False,
                 prefill_delayer_queue_min_ratio=case.queue_min_ratio,
                 prefill_delayer_max_delay_ms=case.max_delay_ms,
+                prefill_max_requests=case.prefill_max_requests,
             ),
             max_delay_passes=case.max_delay_passes,
             token_usage_low_watermark=case.token_usage_low_watermark,
@@ -291,6 +293,34 @@ _NEGOTIATE_TEST_CASES = [
         expected_allow=False,
         expected_reason="delay",
     ),
+    NegotiateTestCase(
+        name="queue_trigger_uses_admission_capacity",
+        max_delay_passes=100,
+        token_usage_low_watermark=0.8,
+        queue_min_ratio=0.02,
+        max_delay_ms=5000,
+        prefill_max_requests=128,
+        calls=[
+            NegotiateCall(
+                prefillable=[True, True, True, True],
+                token_usage=[0.9, 0.9, 0.9, 0.9],
+                running_batch=[500, 500, 500, 500],
+                max_prefill_bs=[1, 1, 1, 1],
+                waiting_queue_len=[1, 1, 1, 1],
+                max_running_requests=1024,
+            ),
+            NegotiateCall(
+                prefillable=[True, True, True, True],
+                token_usage=[0.9, 0.9, 0.9, 0.9],
+                running_batch=[500, 500, 500, 500],
+                max_prefill_bs=[1, 1, 1, 1],
+                waiting_queue_len=[1, 1, 1, 1],
+                max_running_requests=1024,
+            ),
+        ],
+        expected_allow=False,
+        expected_reason="delay",
+    ),
     # Waiting queue at or above queue_min: queue trigger must not fire.
     NegotiateTestCase(
         name="queue_trigger_above_threshold",