diff --git a/python/sglang/srt/managers/min_free_slots_delayer.py b/python/sglang/srt/managers/min_free_slots_delayer.py index 9c0020939..9e31e2107 100644 --- a/python/sglang/srt/managers/min_free_slots_delayer.py +++ b/python/sglang/srt/managers/min_free_slots_delayer.py @@ -8,21 +8,18 @@ def resolve_min_free_slots( ) -> Optional[int]: """Resolve the min-free-slots threshold (None = disabled). - A user value (>1) is capped to the DFlash formula so the trigger never - delays more aggressively than the legacy heuristic. When unset, DFlash - workloads fall back to the formula (preserving the always-on behavior); - other workloads stay disabled. Also disabled when max_running_requests < 8. + An explicit user value always wins, capped by max_running_requests + (<= 1 disables). When unset, DFlash workloads fall back to the legacy + formula (preserving the always-on behavior, disabled when + max_running_requests < 8); other workloads stay disabled. """ max_running_requests = max(0, int(max_running_requests)) - formula = min(4, max(2, (max_running_requests + 5) // 6)) - if user_value is None: - user_value = formula if is_dflash_family else None - - if user_value is None or user_value <= 1: - return None - if max_running_requests < 8: - return None - return min(user_value, formula) + if user_value is not None: + threshold = min(user_value, max_running_requests) + return threshold if threshold > 1 else None + if is_dflash_family and max_running_requests >= 8: + return min(4, max(2, (max_running_requests + 5) // 6)) + return None class MinFreeSlotsDelayer: diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index f9f324ee3..278d7e2a1 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -3234,10 +3234,11 @@ class ServerArgs: "Hold new prefills until at least N running-request slots have freed " "up, so they are admitted in one batch instead of one at a time. " "Useful when each admission is disproportionately expensive, e.g. " - "speculative decoding with a separate draft prefill pass. Capped to " - "the DFlash formula (disabled when max-running-requests < 8; " - "min(4, max(2, (max-run + 5) // 6))). DFlash workloads auto-enable " - "this with the formula when unset; other workloads stay disabled." + "speculative decoding with a separate draft prefill pass. An " + "explicit value always wins, capped by max-running-requests " + "(1 disables). When unset, DFlash workloads auto-enable the " + "formula; other workloads stay disabled. Not supported with " + "pipeline parallelism." ), NS("schedule"), ] = None @@ -8722,6 +8723,11 @@ class ServerArgs: assert ( self.disable_overlap_schedule and self.speculative_algorithm is None ), "Pipeline parallelism is not compatible with overlap schedule, speculative decoding" + assert self.min_free_slots_delay is None, ( + "--min-free-slots-delay is not supported with pipeline " + "parallelism: allocatable slots per microbatch are bounded by " + "pp-max-micro-batch-size, so the threshold may never be reached" + ) assert not ( self.dp_size > 1 and self.nnodes != 1 and not self.enable_dp_attention diff --git a/test/registered/scheduler/test_min_free_slots_delayer.py b/test/registered/scheduler/test_min_free_slots_delayer.py index 0fdcb6153..44f2f8c49 100644 --- a/test/registered/scheduler/test_min_free_slots_delayer.py +++ b/test/registered/scheduler/test_min_free_slots_delayer.py @@ -10,19 +10,15 @@ register_cpu_ci(est_time=5, suite="base-a-test-cpu") class TestResolveMinFreeSlots(unittest.TestCase): - """Unit tests for resolve_min_free_slots threshold resolution.""" def test_unset_non_dflash_disables(self): - # Unset + not DFlash -> trigger stays disabled. self.assertIsNone(resolve_min_free_slots(None, 512, is_dflash_family=False)) def test_unset_dflash_auto_enables(self): - # Unset + DFlash -> falls back to the legacy formula (full mapping). self.assertEqual(resolve_min_free_slots(None, 512, is_dflash_family=True), 4) self.assertEqual(resolve_min_free_slots(None, 8, is_dflash_family=True), 2) def test_unset_dflash_small_cluster_disables(self): - # DFlash auto-default still respects the < 8 guard. self.assertIsNone(resolve_min_free_slots(None, 7, is_dflash_family=True)) self.assertIsNone(resolve_min_free_slots(None, 0, is_dflash_family=True)) @@ -31,27 +27,29 @@ class TestResolveMinFreeSlots(unittest.TestCase): self.assertIsNone(resolve_min_free_slots(1, 512)) self.assertIsNone(resolve_min_free_slots(0, 512)) - def test_small_cluster_disables(self): - # max_running_requests < 8 disables, matching DFlash. - self.assertIsNone(resolve_min_free_slots(4, 7)) + def test_explicit_value_survives_small_cluster(self): + # The < 8 guard belongs to the DFlash auto-default, not explicit values. + self.assertEqual(resolve_min_free_slots(4, 7), 4) + self.assertEqual(resolve_min_free_slots(4, 7, is_dflash_family=True), 4) - def test_caps_to_formula(self): - # Capped down so it never delays more aggressively than DFlash. - self.assertEqual(resolve_min_free_slots(10, 512), 4) - self.assertEqual(resolve_min_free_slots(10, 8), 2) # (8 + 5) // 6 = 2 - - def test_respects_smaller_user_value(self): - # Below the formula cap is taken as-is. - self.assertEqual(resolve_min_free_slots(3, 512), 3) + def test_non_dflash_uses_explicit_value(self): self.assertEqual(resolve_min_free_slots(2, 8), 2) + self.assertEqual(resolve_min_free_slots(3, 512), 3) + self.assertEqual(resolve_min_free_slots(8, 512), 8) + self.assertEqual(resolve_min_free_slots(16, 512), 16) + + def test_explicit_value_is_capped_to_max_running_requests(self): + self.assertEqual(resolve_min_free_slots(16, 8), 8) def test_user_value_overrides_dflash_default(self): - # An explicit user value wins over the DFlash auto-default. self.assertEqual(resolve_min_free_slots(3, 512, is_dflash_family=True), 3) + self.assertEqual(resolve_min_free_slots(16, 512, is_dflash_family=True), 16) + + def test_explicit_one_disables_dflash_default(self): + self.assertIsNone(resolve_min_free_slots(1, 512, is_dflash_family=True)) class TestMinFreeSlotsDelayer(unittest.TestCase): - """Unit tests for the per-rank local should_delay decision.""" def test_delays_below_threshold(self): delayer = MinFreeSlotsDelayer(min_free_slots=4)