diff --git a/docs/docs/advanced_features/server_arguments.mdx b/docs/docs/advanced_features/server_arguments.mdx
index e202df203e..6ef4f93807 100644
--- a/docs/docs/advanced_features/server_arguments.mdx
+++ b/docs/docs/advanced_features/server_arguments.mdx
@@ -471,6 +471,18 @@ Please consult the documentation below and [server_args.py](https://github.com/s
The maximum number of tokens in a chunk for the chunked prefill. Setting this to -1 means disabling chunked prefill. |
`None` |
Type: int |
+
+
+ | `--prefill-decode-interval` |
+ The number of decode rounds to run after a prefill batch before scheduling the next prefill. By default, this is disabled except for profiled Qwen3-VL serving configurations on Hopper. In data-parallel attention mode, the interval is synchronized across all DP ranks. Set to 0 to disable. |
+ `None` |
+ Type: int |
+
+
+ | `--prefill-decode-interval-min-bs` |
+ Apply `--prefill-decode-interval` only while the running batch holds at least this many requests. Below it, prefill is admitted every round so the batch fills quickly. In data-parallel attention mode, each rank compares its own running batch. Set to 0 to always apply the interval. |
+ `0` |
+ Type: int |
| `--prefill-max-requests` |
diff --git a/docs/docs/advanced_features/server_arguments_undocumented.txt b/docs/docs/advanced_features/server_arguments_undocumented.txt
index c2f7910b05..48052e59e1 100644
--- a/docs/docs/advanced_features/server_arguments_undocumented.txt
+++ b/docs/docs/advanced_features/server_arguments_undocumented.txt
@@ -63,7 +63,6 @@
--mm-preprocess-cache-size-mb
--mm-processor-worker-num
--otlp-service-name
---prefill-decode-interval
--prefill-kv-cache-dequant-dtype
--qsa-indexer-dtype
--radix-eviction-policy-config
diff --git a/python/sglang/srt/arg_groups/field_order.py b/python/sglang/srt/arg_groups/field_order.py
index 9df5aa674f..041a9607a3 100644
--- a/python/sglang/srt/arg_groups/field_order.py
+++ b/python/sglang/srt/arg_groups/field_order.py
@@ -547,5 +547,6 @@ POSITIONAL_FIELD_ORDER = (
"enable_nvidia_megakernel",
"enable_megamoe_for_layernorm_sp",
"enable_http_worker_reuseport",
+ "prefill_decode_interval_min_bs",
)
# fmt: on
diff --git a/python/sglang/srt/arg_groups/fields/schedule.py b/python/sglang/srt/arg_groups/fields/schedule.py
index d84e927be9..66cac646b0 100644
--- a/python/sglang/srt/arg_groups/fields/schedule.py
+++ b/python/sglang/srt/arg_groups/fields/schedule.py
@@ -60,6 +60,10 @@ class Schedule(msgspec.Struct):
resolvable=True,
),
] = None
+ prefill_decode_interval_min_bs: A[
+ int,
+ "Apply --prefill-decode-interval only while the running batch holds at least this many requests. Below it, prefill is admitted every round so the batch fills quickly. In data-parallel attention mode, each rank compares its own running batch. Set to 0 to always apply the interval.",
+ ] = 0
enable_dynamic_chunking: A[
bool,
"Enable dynamic chunk size adjustment for pipeline parallelism. When enabled, chunk sizes are dynamically calculated based on fitted function to maintain consistent execution time across chunks.",
diff --git a/python/sglang/srt/arg_groups/validation_hook.py b/python/sglang/srt/arg_groups/validation_hook.py
index 1c929ef180..84ebb8254b 100644
--- a/python/sglang/srt/arg_groups/validation_hook.py
+++ b/python/sglang/srt/arg_groups/validation_hook.py
@@ -657,6 +657,8 @@ def validate_prefill_decode_interval(server_args: Any):
cfg = resolving_view(server_args)
if cfg.prefill_decode_interval is not None and cfg.prefill_decode_interval < 0:
raise ValueError("--prefill-decode-interval must be non-negative.")
+ if cfg.prefill_decode_interval_min_bs < 0:
+ raise ValueError("--prefill-decode-interval-min-bs must be non-negative.")
def default_unset_prefill_decode_interval(server_args: Any):
diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py
index e1f2a0dac1..1831160646 100644
--- a/python/sglang/srt/managers/scheduler.py
+++ b/python/sglang/srt/managers/scheduler.py
@@ -1335,6 +1335,9 @@ class Scheduler(
self.chunked_prefill_size = get_schedule().chunked_prefill_size
self.prefill_decode_interval = get_schedule().prefill_decode_interval or 0
self._prefill_decode_interval_remaining = 0
+ self.prefill_decode_interval_min_bs = (
+ get_schedule().prefill_decode_interval_min_bs
+ )
uses_transformers_backend = (
get_resolved_model_impl(self.model_config) == ModelImpl.TRANSFORMERS
)
@@ -1377,10 +1380,16 @@ class Scheduler(
if sizer.profile_and_fit():
self.dynamic_chunk_sizer = sizer
- def _should_defer_prefill(self) -> bool:
+ def _should_defer_prefill(self, running_batch: ScheduleBatch) -> bool:
if self._prefill_decode_interval_remaining == 0:
return False
+ # A small running batch admits prefill every round so it can grow;
+ # the interval only protects decode once the batch is large enough.
+ if running_batch.batch_size() < self.prefill_decode_interval_min_bs:
+ self._prefill_decode_interval_remaining = 0
+ return False
+
self._prefill_decode_interval_remaining -= 1
return True
@@ -3905,7 +3914,7 @@ class Scheduler(
if self.dllm_config is not None:
new_batch = self.get_new_batch_dllm(running_batch)
- elif self._should_defer_prefill():
+ elif self._should_defer_prefill(running_batch):
new_batch = None
else:
prefill_plan = self.get_new_batch_prefill(running_batch)
diff --git a/test/registered/unit/managers/test_scheduler_hicache_events.py b/test/registered/unit/managers/test_scheduler_hicache_events.py
index ed1167cc0a..c9dc056c85 100644
--- a/test/registered/unit/managers/test_scheduler_hicache_events.py
+++ b/test/registered/unit/managers/test_scheduler_hicache_events.py
@@ -85,7 +85,12 @@ class TestSchedulerHiCacheEvents(unittest.TestCase):
self.assertIsNone(plan.batch_to_run)
s.get_new_batch_prefill.assert_not_called()
self.assertEqual(
- self.calls.mock_calls, [call.drain(), call.retry(), call.defer()]
+ self.calls.mock_calls,
+ [
+ call.drain(),
+ call.retry(),
+ call.defer(self.running_batch),
+ ],
)
def test_pp_prefill_drains_before_admission(self):
diff --git a/test/registered/unit/managers/test_scheduler_prefill_decode_interval.py b/test/registered/unit/managers/test_scheduler_prefill_decode_interval.py
new file mode 100644
index 0000000000..d7a86e2130
--- /dev/null
+++ b/test/registered/unit/managers/test_scheduler_prefill_decode_interval.py
@@ -0,0 +1,66 @@
+"""Tests for gating the prefill/decode interval on running batch size."""
+
+import unittest
+from types import SimpleNamespace
+
+from sglang.test.ci.ci_register import register_cpu_ci
+from sglang.test.test_utils import maybe_stub_sgl_kernel
+
+maybe_stub_sgl_kernel()
+
+from sglang.srt.managers.scheduler import Scheduler
+
+register_cpu_ci(est_time=11, suite="base-a-test-cpu")
+
+
+def _make_scheduler(*, interval: int, min_bs: int) -> Scheduler:
+ scheduler = Scheduler.__new__(Scheduler)
+ scheduler.prefill_decode_interval = interval
+ scheduler._prefill_decode_interval_remaining = 0
+ scheduler.prefill_decode_interval_min_bs = min_bs
+ scheduler.require_mlp_sync = False
+ return scheduler
+
+
+def _arm(scheduler: Scheduler) -> None:
+ scheduler._arm_prefill_decode_interval(
+ SimpleNamespace(
+ forward_mode=SimpleNamespace(is_extend=lambda: True),
+ is_extend_in_batch=False,
+ )
+ )
+
+
+def _make_running_batch(batch_size: int):
+ return SimpleNamespace(batch_size=lambda: batch_size)
+
+
+class TestPrefillDecodeIntervalMinBs(unittest.TestCase):
+ def test_small_running_batch_admits_prefill_and_clears_interval(self):
+ scheduler = _make_scheduler(interval=4, min_bs=8)
+ _arm(scheduler)
+
+ self.assertFalse(scheduler._should_defer_prefill(_make_running_batch(7)))
+ self.assertEqual(scheduler._prefill_decode_interval_remaining, 0)
+ # The cleared interval does not resume once the batch grows.
+ self.assertFalse(scheduler._should_defer_prefill(_make_running_batch(8)))
+
+ def test_large_running_batch_defers_prefill(self):
+ scheduler = _make_scheduler(interval=2, min_bs=8)
+ _arm(scheduler)
+
+ self.assertTrue(scheduler._should_defer_prefill(_make_running_batch(8)))
+ self.assertTrue(scheduler._should_defer_prefill(_make_running_batch(9)))
+ self.assertFalse(scheduler._should_defer_prefill(_make_running_batch(9)))
+
+ def test_running_batch_shrinking_below_threshold_ends_interval(self):
+ scheduler = _make_scheduler(interval=4, min_bs=8)
+ _arm(scheduler)
+
+ self.assertTrue(scheduler._should_defer_prefill(_make_running_batch(8)))
+ self.assertFalse(scheduler._should_defer_prefill(_make_running_batch(5)))
+ self.assertEqual(scheduler._prefill_decode_interval_remaining, 0)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py
index 6ecb9ebd04..a8a9975a85 100644
--- a/test/registered/unit/server_args/test_server_args.py
+++ b/test/registered/unit/server_args/test_server_args.py
@@ -494,6 +494,14 @@ class TestPrepareServerArgs(CustomTestCase):
):
ServerArgs(model_path="dummy", prefill_decode_interval=-1).resolve_once()
+ with self.assertRaisesRegex(
+ ValueError,
+ "--prefill-decode-interval-min-bs must be non-negative",
+ ):
+ ServerArgs(
+ model_path="dummy", prefill_decode_interval_min_bs=-1
+ ).resolve_once()
+
def test_sampling_mask_max_tokens(self):
self.assertEqual(ServerArgs(model_path="dummy").sampling_mask_max_tokens, 4096)
self.assertEqual(