[PD] Reclaim parked optimistic prefills when request slots are exhausted (#43442)
This commit is contained in:
@@ -1148,6 +1148,24 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
if self.handle_pending_bootstrap(req, poll):
|
||||
self.send_kv_chunk(req, last_chunk=True)
|
||||
undone_reqs.append(req)
|
||||
elif (
|
||||
poll == KVPoll.Bootstrapping
|
||||
and get_disagg().disaggregation_decode_allocation_policy
|
||||
!= "prefill_complete"
|
||||
and self.req_to_token_pool.available_size() == 0
|
||||
and (
|
||||
self.waiting_queue
|
||||
or (
|
||||
self.disagg_prefill_bootstrap_queue.queue
|
||||
and self.req_to_metadata_buffer_idx_allocator.available_size()
|
||||
== 0
|
||||
)
|
||||
)
|
||||
):
|
||||
# Decode may have admitted different requests. Yield one
|
||||
# slot so they can prefill, including peers still waiting
|
||||
# for a metadata buffer in the bootstrap queue.
|
||||
self.optimistic_release_and_requeue(req)
|
||||
elif poll != KVPoll.Failed:
|
||||
undone_reqs.append(req)
|
||||
continue
|
||||
@@ -1694,6 +1712,9 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
)
|
||||
# Reset it so the next real bootstrap done can be recorded.
|
||||
req.time_stats.bootstrap_done_time = 0.0
|
||||
maybe_release_metadata_buffer(
|
||||
req, self.req_to_metadata_buffer_idx_allocator
|
||||
)
|
||||
self.disagg_prefill_bootstrap_queue.queue.append(req)
|
||||
else:
|
||||
req.prefill_attempt_count += 1
|
||||
|
||||
Reference in New Issue
Block a user