[Scheduler] Hold back decode rows on a mixed step when they do not fit the KV pool (#43290)

This commit is contained in:
Liangsheng Yin
2026-10-11 00:18:27 -07:00
committed by GitHub
parent 89e24e7a20
commit dab108b9f3
2 changed files with 89 additions and 2 deletions
+4 -1
View File
@@ -4184,7 +4184,10 @@ class Scheduler(
running_batch.filter_batch(reqs_to_exclude=finishing_reqs)
if finishing_reqs:
running_batch.batch_is_full = False
if not running_batch.is_empty():
# The prefill admitted above may leave no KV room for the decode
# rows. They then sit out this step and decode on the next decode
# step, which fits them; a mixed step does not retract them.
if not running_batch.is_empty() and running_batch.check_decode_mem():
running_batch.prepare_for_decode()
new_batch.mix_with_running(running_batch)
new_batch.decoding_reqs = running_batch.reqs