From 9569a3656155a34dc763826b1608f247d3f66832 Mon Sep 17 00:00:00 2001 From: Shangming Cai Date: Thu, 22 Oct 2026 19:30:12 +1300 Subject: [PATCH] [Feat] scheduler: Gate --prefill-decode-interval on running batch size (#44001) --- .../advanced_features/server_arguments.mdx | 12 ++++ .../server_arguments_undocumented.txt | 1 - python/sglang/srt/arg_groups/field_order.py | 1 + .../sglang/srt/arg_groups/fields/schedule.py | 4 ++ .../sglang/srt/arg_groups/validation_hook.py | 2 + python/sglang/srt/managers/scheduler.py | 13 +++- .../managers/test_scheduler_hicache_events.py | 7 +- .../test_scheduler_prefill_decode_interval.py | 66 +++++++++++++++++++ .../unit/server_args/test_server_args.py | 8 +++ 9 files changed, 110 insertions(+), 4 deletions(-) create mode 100644 test/registered/unit/managers/test_scheduler_prefill_decode_interval.py diff --git a/docs/docs/advanced_features/server_arguments.mdx b/docs/docs/advanced_features/server_arguments.mdx index e202df203e..6ef4f93807 100644 --- a/docs/docs/advanced_features/server_arguments.mdx +++ b/docs/docs/advanced_features/server_arguments.mdx @@ -471,6 +471,18 @@ Please consult the documentation below and [server_args.py](https://github.com/s The maximum number of tokens in a chunk for the chunked prefill. Setting this to -1 means disabling chunked prefill. `None` Type: int + + + `--prefill-decode-interval` + The number of decode rounds to run after a prefill batch before scheduling the next prefill. By default, this is disabled except for profiled Qwen3-VL serving configurations on Hopper. In data-parallel attention mode, the interval is synchronized across all DP ranks. Set to 0 to disable. + `None` + Type: int + + + `--prefill-decode-interval-min-bs` + Apply `--prefill-decode-interval` only while the running batch holds at least this many requests. Below it, prefill is admitted every round so the batch fills quickly. In data-parallel attention mode, each rank compares its own running batch. Set to 0 to always apply the interval. + `0` + Type: int `--prefill-max-requests` diff --git a/docs/docs/advanced_features/server_arguments_undocumented.txt b/docs/docs/advanced_features/server_arguments_undocumented.txt index c2f7910b05..48052e59e1 100644 --- a/docs/docs/advanced_features/server_arguments_undocumented.txt +++ b/docs/docs/advanced_features/server_arguments_undocumented.txt @@ -63,7 +63,6 @@ --mm-preprocess-cache-size-mb --mm-processor-worker-num --otlp-service-name ---prefill-decode-interval --prefill-kv-cache-dequant-dtype --qsa-indexer-dtype --radix-eviction-policy-config diff --git a/python/sglang/srt/arg_groups/field_order.py b/python/sglang/srt/arg_groups/field_order.py index 9df5aa674f..041a9607a3 100644 --- a/python/sglang/srt/arg_groups/field_order.py +++ b/python/sglang/srt/arg_groups/field_order.py @@ -547,5 +547,6 @@ POSITIONAL_FIELD_ORDER = ( "enable_nvidia_megakernel", "enable_megamoe_for_layernorm_sp", "enable_http_worker_reuseport", + "prefill_decode_interval_min_bs", ) # fmt: on diff --git a/python/sglang/srt/arg_groups/fields/schedule.py b/python/sglang/srt/arg_groups/fields/schedule.py index d84e927be9..66cac646b0 100644 --- a/python/sglang/srt/arg_groups/fields/schedule.py +++ b/python/sglang/srt/arg_groups/fields/schedule.py @@ -60,6 +60,10 @@ class Schedule(msgspec.Struct): resolvable=True, ), ] = None + prefill_decode_interval_min_bs: A[ + int, + "Apply --prefill-decode-interval only while the running batch holds at least this many requests. Below it, prefill is admitted every round so the batch fills quickly. In data-parallel attention mode, each rank compares its own running batch. Set to 0 to always apply the interval.", + ] = 0 enable_dynamic_chunking: A[ bool, "Enable dynamic chunk size adjustment for pipeline parallelism. When enabled, chunk sizes are dynamically calculated based on fitted function to maintain consistent execution time across chunks.", diff --git a/python/sglang/srt/arg_groups/validation_hook.py b/python/sglang/srt/arg_groups/validation_hook.py index 1c929ef180..84ebb8254b 100644 --- a/python/sglang/srt/arg_groups/validation_hook.py +++ b/python/sglang/srt/arg_groups/validation_hook.py @@ -657,6 +657,8 @@ def validate_prefill_decode_interval(server_args: Any): cfg = resolving_view(server_args) if cfg.prefill_decode_interval is not None and cfg.prefill_decode_interval < 0: raise ValueError("--prefill-decode-interval must be non-negative.") + if cfg.prefill_decode_interval_min_bs < 0: + raise ValueError("--prefill-decode-interval-min-bs must be non-negative.") def default_unset_prefill_decode_interval(server_args: Any): diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index e1f2a0dac1..1831160646 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -1335,6 +1335,9 @@ class Scheduler( self.chunked_prefill_size = get_schedule().chunked_prefill_size self.prefill_decode_interval = get_schedule().prefill_decode_interval or 0 self._prefill_decode_interval_remaining = 0 + self.prefill_decode_interval_min_bs = ( + get_schedule().prefill_decode_interval_min_bs + ) uses_transformers_backend = ( get_resolved_model_impl(self.model_config) == ModelImpl.TRANSFORMERS ) @@ -1377,10 +1380,16 @@ class Scheduler( if sizer.profile_and_fit(): self.dynamic_chunk_sizer = sizer - def _should_defer_prefill(self) -> bool: + def _should_defer_prefill(self, running_batch: ScheduleBatch) -> bool: if self._prefill_decode_interval_remaining == 0: return False + # A small running batch admits prefill every round so it can grow; + # the interval only protects decode once the batch is large enough. + if running_batch.batch_size() < self.prefill_decode_interval_min_bs: + self._prefill_decode_interval_remaining = 0 + return False + self._prefill_decode_interval_remaining -= 1 return True @@ -3905,7 +3914,7 @@ class Scheduler( if self.dllm_config is not None: new_batch = self.get_new_batch_dllm(running_batch) - elif self._should_defer_prefill(): + elif self._should_defer_prefill(running_batch): new_batch = None else: prefill_plan = self.get_new_batch_prefill(running_batch) diff --git a/test/registered/unit/managers/test_scheduler_hicache_events.py b/test/registered/unit/managers/test_scheduler_hicache_events.py index ed1167cc0a..c9dc056c85 100644 --- a/test/registered/unit/managers/test_scheduler_hicache_events.py +++ b/test/registered/unit/managers/test_scheduler_hicache_events.py @@ -85,7 +85,12 @@ class TestSchedulerHiCacheEvents(unittest.TestCase): self.assertIsNone(plan.batch_to_run) s.get_new_batch_prefill.assert_not_called() self.assertEqual( - self.calls.mock_calls, [call.drain(), call.retry(), call.defer()] + self.calls.mock_calls, + [ + call.drain(), + call.retry(), + call.defer(self.running_batch), + ], ) def test_pp_prefill_drains_before_admission(self): diff --git a/test/registered/unit/managers/test_scheduler_prefill_decode_interval.py b/test/registered/unit/managers/test_scheduler_prefill_decode_interval.py new file mode 100644 index 0000000000..d7a86e2130 --- /dev/null +++ b/test/registered/unit/managers/test_scheduler_prefill_decode_interval.py @@ -0,0 +1,66 @@ +"""Tests for gating the prefill/decode interval on running batch size.""" + +import unittest +from types import SimpleNamespace + +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import maybe_stub_sgl_kernel + +maybe_stub_sgl_kernel() + +from sglang.srt.managers.scheduler import Scheduler + +register_cpu_ci(est_time=11, suite="base-a-test-cpu") + + +def _make_scheduler(*, interval: int, min_bs: int) -> Scheduler: + scheduler = Scheduler.__new__(Scheduler) + scheduler.prefill_decode_interval = interval + scheduler._prefill_decode_interval_remaining = 0 + scheduler.prefill_decode_interval_min_bs = min_bs + scheduler.require_mlp_sync = False + return scheduler + + +def _arm(scheduler: Scheduler) -> None: + scheduler._arm_prefill_decode_interval( + SimpleNamespace( + forward_mode=SimpleNamespace(is_extend=lambda: True), + is_extend_in_batch=False, + ) + ) + + +def _make_running_batch(batch_size: int): + return SimpleNamespace(batch_size=lambda: batch_size) + + +class TestPrefillDecodeIntervalMinBs(unittest.TestCase): + def test_small_running_batch_admits_prefill_and_clears_interval(self): + scheduler = _make_scheduler(interval=4, min_bs=8) + _arm(scheduler) + + self.assertFalse(scheduler._should_defer_prefill(_make_running_batch(7))) + self.assertEqual(scheduler._prefill_decode_interval_remaining, 0) + # The cleared interval does not resume once the batch grows. + self.assertFalse(scheduler._should_defer_prefill(_make_running_batch(8))) + + def test_large_running_batch_defers_prefill(self): + scheduler = _make_scheduler(interval=2, min_bs=8) + _arm(scheduler) + + self.assertTrue(scheduler._should_defer_prefill(_make_running_batch(8))) + self.assertTrue(scheduler._should_defer_prefill(_make_running_batch(9))) + self.assertFalse(scheduler._should_defer_prefill(_make_running_batch(9))) + + def test_running_batch_shrinking_below_threshold_ends_interval(self): + scheduler = _make_scheduler(interval=4, min_bs=8) + _arm(scheduler) + + self.assertTrue(scheduler._should_defer_prefill(_make_running_batch(8))) + self.assertFalse(scheduler._should_defer_prefill(_make_running_batch(5))) + self.assertEqual(scheduler._prefill_decode_interval_remaining, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py index 6ecb9ebd04..a8a9975a85 100644 --- a/test/registered/unit/server_args/test_server_args.py +++ b/test/registered/unit/server_args/test_server_args.py @@ -494,6 +494,14 @@ class TestPrepareServerArgs(CustomTestCase): ): ServerArgs(model_path="dummy", prefill_decode_interval=-1).resolve_once() + with self.assertRaisesRegex( + ValueError, + "--prefill-decode-interval-min-bs must be non-negative", + ): + ServerArgs( + model_path="dummy", prefill_decode_interval_min_bs=-1 + ).resolve_once() + def test_sampling_mask_max_tokens(self): self.assertEqual(ServerArgs(model_path="dummy").sampling_mask_max_tokens, 4096) self.assertEqual(