[Feat] scheduler: Gate --prefill-decode-interval on running batch size (#44001)

This commit is contained in:
Shangming Cai
2026-10-22 14:30:12 +08:00
committed by GitHub
parent ce66b3ab59
commit 9569a36561
9 changed files with 110 additions and 4 deletions
@@ -547,5 +547,6 @@ POSITIONAL_FIELD_ORDER = (
"enable_nvidia_megakernel",
"enable_megamoe_for_layernorm_sp",
"enable_http_worker_reuseport",
"prefill_decode_interval_min_bs",
)
# fmt: on
@@ -60,6 +60,10 @@ class Schedule(msgspec.Struct):
resolvable=True,
),
] = None
prefill_decode_interval_min_bs: A[
int,
"Apply --prefill-decode-interval only while the running batch holds at least this many requests. Below it, prefill is admitted every round so the batch fills quickly. In data-parallel attention mode, each rank compares its own running batch. Set to 0 to always apply the interval.",
] = 0
enable_dynamic_chunking: A[
bool,
"Enable dynamic chunk size adjustment for pipeline parallelism. When enabled, chunk sizes are dynamically calculated based on fitted function to maintain consistent execution time across chunks.",
@@ -657,6 +657,8 @@ def validate_prefill_decode_interval(server_args: Any):
cfg = resolving_view(server_args)
if cfg.prefill_decode_interval is not None and cfg.prefill_decode_interval < 0:
raise ValueError("--prefill-decode-interval must be non-negative.")
if cfg.prefill_decode_interval_min_bs < 0:
raise ValueError("--prefill-decode-interval-min-bs must be non-negative.")
def default_unset_prefill_decode_interval(server_args: Any):
+11 -2
View File
@@ -1335,6 +1335,9 @@ class Scheduler(
self.chunked_prefill_size = get_schedule().chunked_prefill_size
self.prefill_decode_interval = get_schedule().prefill_decode_interval or 0
self._prefill_decode_interval_remaining = 0
self.prefill_decode_interval_min_bs = (
get_schedule().prefill_decode_interval_min_bs
)
uses_transformers_backend = (
get_resolved_model_impl(self.model_config) == ModelImpl.TRANSFORMERS
)
@@ -1377,10 +1380,16 @@ class Scheduler(
if sizer.profile_and_fit():
self.dynamic_chunk_sizer = sizer
def _should_defer_prefill(self) -> bool:
def _should_defer_prefill(self, running_batch: ScheduleBatch) -> bool:
if self._prefill_decode_interval_remaining == 0:
return False
# A small running batch admits prefill every round so it can grow;
# the interval only protects decode once the batch is large enough.
if running_batch.batch_size() < self.prefill_decode_interval_min_bs:
self._prefill_decode_interval_remaining = 0
return False
self._prefill_decode_interval_remaining -= 1
return True
@@ -3905,7 +3914,7 @@ class Scheduler(
if self.dllm_config is not None:
new_batch = self.get_new_batch_dllm(running_batch)
elif self._should_defer_prefill():
elif self._should_defer_prefill(running_batch):
new_batch = None
else:
prefill_plan = self.get_new_batch_prefill(running_batch)