[Feat] scheduler: Gate --prefill-decode-interval on running batch size (#44001)
This commit is contained in:
@@ -547,5 +547,6 @@ POSITIONAL_FIELD_ORDER = (
|
||||
"enable_nvidia_megakernel",
|
||||
"enable_megamoe_for_layernorm_sp",
|
||||
"enable_http_worker_reuseport",
|
||||
"prefill_decode_interval_min_bs",
|
||||
)
|
||||
# fmt: on
|
||||
|
||||
@@ -60,6 +60,10 @@ class Schedule(msgspec.Struct):
|
||||
resolvable=True,
|
||||
),
|
||||
] = None
|
||||
prefill_decode_interval_min_bs: A[
|
||||
int,
|
||||
"Apply --prefill-decode-interval only while the running batch holds at least this many requests. Below it, prefill is admitted every round so the batch fills quickly. In data-parallel attention mode, each rank compares its own running batch. Set to 0 to always apply the interval.",
|
||||
] = 0
|
||||
enable_dynamic_chunking: A[
|
||||
bool,
|
||||
"Enable dynamic chunk size adjustment for pipeline parallelism. When enabled, chunk sizes are dynamically calculated based on fitted function to maintain consistent execution time across chunks.",
|
||||
|
||||
@@ -657,6 +657,8 @@ def validate_prefill_decode_interval(server_args: Any):
|
||||
cfg = resolving_view(server_args)
|
||||
if cfg.prefill_decode_interval is not None and cfg.prefill_decode_interval < 0:
|
||||
raise ValueError("--prefill-decode-interval must be non-negative.")
|
||||
if cfg.prefill_decode_interval_min_bs < 0:
|
||||
raise ValueError("--prefill-decode-interval-min-bs must be non-negative.")
|
||||
|
||||
|
||||
def default_unset_prefill_decode_interval(server_args: Any):
|
||||
|
||||
@@ -1335,6 +1335,9 @@ class Scheduler(
|
||||
self.chunked_prefill_size = get_schedule().chunked_prefill_size
|
||||
self.prefill_decode_interval = get_schedule().prefill_decode_interval or 0
|
||||
self._prefill_decode_interval_remaining = 0
|
||||
self.prefill_decode_interval_min_bs = (
|
||||
get_schedule().prefill_decode_interval_min_bs
|
||||
)
|
||||
uses_transformers_backend = (
|
||||
get_resolved_model_impl(self.model_config) == ModelImpl.TRANSFORMERS
|
||||
)
|
||||
@@ -1377,10 +1380,16 @@ class Scheduler(
|
||||
if sizer.profile_and_fit():
|
||||
self.dynamic_chunk_sizer = sizer
|
||||
|
||||
def _should_defer_prefill(self) -> bool:
|
||||
def _should_defer_prefill(self, running_batch: ScheduleBatch) -> bool:
|
||||
if self._prefill_decode_interval_remaining == 0:
|
||||
return False
|
||||
|
||||
# A small running batch admits prefill every round so it can grow;
|
||||
# the interval only protects decode once the batch is large enough.
|
||||
if running_batch.batch_size() < self.prefill_decode_interval_min_bs:
|
||||
self._prefill_decode_interval_remaining = 0
|
||||
return False
|
||||
|
||||
self._prefill_decode_interval_remaining -= 1
|
||||
return True
|
||||
|
||||
@@ -3905,7 +3914,7 @@ class Scheduler(
|
||||
|
||||
if self.dllm_config is not None:
|
||||
new_batch = self.get_new_batch_dllm(running_batch)
|
||||
elif self._should_defer_prefill():
|
||||
elif self._should_defer_prefill(running_batch):
|
||||
new_batch = None
|
||||
else:
|
||||
prefill_plan = self.get_new_batch_prefill(running_batch)
|
||||
|
||||
Reference in New Issue
Block a user