[Scheduler] Hold back decode rows on a mixed step when they do not fit the KV pool (#43290)
This commit is contained in:
@@ -4184,7 +4184,10 @@ class Scheduler(
|
||||
running_batch.filter_batch(reqs_to_exclude=finishing_reqs)
|
||||
if finishing_reqs:
|
||||
running_batch.batch_is_full = False
|
||||
if not running_batch.is_empty():
|
||||
# The prefill admitted above may leave no KV room for the decode
|
||||
# rows. They then sit out this step and decode on the next decode
|
||||
# step, which fits them; a mixed step does not retract them.
|
||||
if not running_batch.is_empty() and running_batch.check_decode_mem():
|
||||
running_batch.prepare_for_decode()
|
||||
new_batch.mix_with_running(running_batch)
|
||||
new_batch.decoding_reqs = running_batch.reqs
|
||||
|
||||
Reference in New Issue
Block a user