[PD] Reclaim parked optimistic prefills when request slots are exhausted (#43442)

This commit is contained in:
cctry
2026-10-09 20:31:24 -07:00
committed by GitHub
parent 7d44269907
commit 20948cdcc6
2 changed files with 103 additions and 29 deletions
@@ -1148,6 +1148,24 @@ class SchedulerDisaggregationPrefillMixin:
if self.handle_pending_bootstrap(req, poll):
self.send_kv_chunk(req, last_chunk=True)
undone_reqs.append(req)
elif (
poll == KVPoll.Bootstrapping
and get_disagg().disaggregation_decode_allocation_policy
!= "prefill_complete"
and self.req_to_token_pool.available_size() == 0
and (
self.waiting_queue
or (
self.disagg_prefill_bootstrap_queue.queue
and self.req_to_metadata_buffer_idx_allocator.available_size()
== 0
)
)
):
# Decode may have admitted different requests. Yield one
# slot so they can prefill, including peers still waiting
# for a metadata buffer in the bootstrap queue.
self.optimistic_release_and_requeue(req)
elif poll != KVPoll.Failed:
undone_reqs.append(req)
continue
@@ -1694,6 +1712,9 @@ class SchedulerDisaggregationPrefillMixin:
)
# Reset it so the next real bootstrap done can be recorded.
req.time_stats.bootstrap_done_time = 0.0
maybe_release_metadata_buffer(
req, self.req_to_metadata_buffer_idx_allocator
)
self.disagg_prefill_bootstrap_queue.queue.append(req)
else:
req.prefill_attempt_count += 1