[PD] Reclaim parked optimistic prefills when request slots are exhausted (#43442)

This commit is contained in:
cctry
2026-10-09 20:31:24 -07:00
committed by GitHub
parent 7d44269907
commit 20948cdcc6
2 changed files with 103 additions and 29 deletions
@@ -1148,6 +1148,24 @@ class SchedulerDisaggregationPrefillMixin:
if self.handle_pending_bootstrap(req, poll):
self.send_kv_chunk(req, last_chunk=True)
undone_reqs.append(req)
elif (
poll == KVPoll.Bootstrapping
and get_disagg().disaggregation_decode_allocation_policy
!= "prefill_complete"
and self.req_to_token_pool.available_size() == 0
and (
self.waiting_queue
or (
self.disagg_prefill_bootstrap_queue.queue
and self.req_to_metadata_buffer_idx_allocator.available_size()
== 0
)
)
):
# Decode may have admitted different requests. Yield one
# slot so they can prefill, including peers still waiting
# for a metadata buffer in the bootstrap queue.
self.optimistic_release_and_requeue(req)
elif poll != KVPoll.Failed:
undone_reqs.append(req)
continue
@@ -1694,6 +1712,9 @@ class SchedulerDisaggregationPrefillMixin:
)
# Reset it so the next real bootstrap done can be recorded.
req.time_stats.bootstrap_done_time = 0.0
maybe_release_metadata_buffer(
req, self.req_to_metadata_buffer_idx_allocator
)
self.disagg_prefill_bootstrap_queue.queue.append(req)
else:
req.prefill_attempt_count += 1
@@ -22,7 +22,10 @@ from sglang.srt.disaggregation.mooncake.conn import (
MooncakeKVSender,
)
from sglang.srt.disaggregation.prefill import SchedulerDisaggregationPrefillMixin
from sglang.srt.managers.schedule_batch import Req
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
from sglang.srt.runtime_context import get_context, publish, reset_context
from sglang.srt.sampling.sampling_params import SamplingParams
from sglang.srt.server_args import ServerArgs
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
@@ -475,7 +478,7 @@ class TestAllocationIntegration(CustomTestCase):
queue.req_to_token_pool.available_size.assert_called_once()
self.assertEqual(queue.queue, [entry])
def test_exhausted_optimistic_attempt_remains_compute_eligible(self):
def test_exhausted_optimistic_attempt_follows_allocation_policy(self):
override = get_context().override_server_args(
optimistic_prefill_attempts=1,
disaggregation_decode_allocation_policy="prefill_complete",
@@ -492,6 +495,7 @@ class TestAllocationIntegration(CustomTestCase):
_compute_max_prefix_len=Mock(return_value=31),
storage_prefetch_retry_attempts=3,
storage_prefetch_last_match_len=8,
metadata_buffer_index=0,
reset_for_retract=Mock(),
advance_cache_request_handle=Mock(),
time_stats=MagicMock(),
@@ -504,6 +508,7 @@ class TestAllocationIntegration(CustomTestCase):
disagg_prefill_bootstrap_queue=SimpleNamespace(queue=[]),
metrics_reporter=SimpleNamespace(enable_metrics=False),
processed_tokens_counter=0,
req_to_metadata_buffer_idx_allocator=Mock(),
)
with (
patch("sglang.srt.disaggregation.prefill.checkpoint_kv_cache"),
@@ -512,13 +517,28 @@ class TestAllocationIntegration(CustomTestCase):
SchedulerDisaggregationPrefillMixin.optimistic_release_and_requeue(
scheduler, req
)
self.assertEqual(scheduler.waiting_queue, [req])
self.assertEqual(scheduler.disagg_prefill_bootstrap_queue.queue, [])
self.assertEqual(req.prefill_attempt_count, 2)
self.assertEqual(list(req.output_ids), [])
self.assertTrue(req.pending_bootstrap)
self.assertEqual(req.storage_prefetch_retry_attempts, 0)
self.assertEqual(req.storage_prefetch_last_match_len, 16)
self.assertEqual(scheduler.waiting_queue, [req])
self.assertEqual(scheduler.disagg_prefill_bootstrap_queue.queue, [])
self.assertEqual(req.prefill_attempt_count, 2)
self.assertEqual(list(req.output_ids), [])
self.assertTrue(req.pending_bootstrap)
self.assertEqual(req.storage_prefetch_retry_attempts, 0)
self.assertEqual(req.storage_prefetch_last_match_len, 16)
scheduler.req_to_metadata_buffer_idx_allocator.free.assert_not_called()
with get_context().override_server_args(
disaggregation_decode_allocation_policy="early"
):
scheduler.waiting_queue.clear()
SchedulerDisaggregationPrefillMixin.optimistic_release_and_requeue(
scheduler, req
)
self.assertEqual(scheduler.waiting_queue, [])
self.assertEqual(scheduler.disagg_prefill_bootstrap_queue.queue, [req])
self.assertEqual(req.metadata_buffer_index, -1)
scheduler.req_to_metadata_buffer_idx_allocator.free.assert_called_once_with(
0
)
def test_chunked_prefill_does_not_yield_for_decode_admission(self):
for policy in ("early", "prefill_complete"):
@@ -562,6 +582,25 @@ class TestAllocationIntegration(CustomTestCase):
finally:
override.restore()
def inflight_scheduler(self, reqs):
scheduler = SchedulerDisaggregationPrefillMixin()
scheduler.tree_cache = Mock()
scheduler.scheduler_stage_metrics = None
scheduler.disagg_prefill_inflight_queue = list(reqs)
scheduler.attn_cp_cpu_group = scheduler.attn_tp_cpu_group = object()
scheduler.output_streamer = Mock()
scheduler.send_kv_chunk = Mock()
self.enterContext(
patch(
"sglang.srt.disaggregation.prefill.poll_and_all_reduce_attn_cp_tp_group",
side_effect=lambda senders, *groups: [s.poll() for s in senders],
)
)
self.enterContext(
patch("sglang.srt.disaggregation.prefill.checkpoint_kv_cache")
)
return scheduler
def test_completed_prefill_parks_until_normal_pointer_handshake(self):
sender, receiver = self.sender(), self.receiver()
self.flush()
@@ -577,13 +616,9 @@ class TestAllocationIntegration(CustomTestCase):
prefill_attempt_count=1,
skip_radix_cache_insert=False,
)
scheduler = SchedulerDisaggregationPrefillMixin()
scheduler.tree_cache = Mock()
scheduler.scheduler_stage_metrics = None
scheduler.disagg_prefill_inflight_queue = [req]
scheduler.attn_cp_cpu_group = scheduler.attn_tp_cpu_group = object()
scheduler.output_streamer = Mock()
scheduler.send_kv_chunk = Mock()
scheduler = self.inflight_scheduler([req])
scheduler.req_to_token_pool = SimpleNamespace(available_size=lambda: 0)
scheduler.waiting_queue = [Mock()]
def finalize(r):
r.pending_bootstrap = False
@@ -592,21 +627,39 @@ class TestAllocationIntegration(CustomTestCase):
scheduler.disagg_prefill_bootstrap_queue = SimpleNamespace(
finalize_bootstrap=Mock(side_effect=finalize)
)
with (
patch(
"sglang.srt.disaggregation.prefill.poll_and_all_reduce_attn_cp_tp_group",
side_effect=lambda senders, *groups: [s.poll() for s in senders],
),
patch("sglang.srt.disaggregation.prefill.checkpoint_kv_cache"),
self.assertEqual(scheduler.process_disagg_prefill_inflight_queue(), [])
self.assertEqual(scheduler.disagg_prefill_inflight_queue, [req])
scheduler.send_kv_chunk.assert_not_called()
# This is the existing metadata-received transition, not readiness.
self.prefill.update_status(1, KVPoll.WaitingForInput)
self.assertEqual(scheduler.process_disagg_prefill_inflight_queue(), [])
scheduler.send_kv_chunk.assert_called_once_with(req, last_chunk=True)
self.assertFalse(req.pending_bootstrap)
def test_parked_prefill_reclaims_only_one_slot(self):
reqs = [Req(str(i), "", [1], SamplingParams()) for i in range(2)]
for i, req in enumerate(reqs):
req.pending_bootstrap = True
req.disagg_kv_sender = self.sender(room=i + 1)
scheduler = self.inflight_scheduler(reqs)
pool = scheduler.req_to_token_pool = ReqToTokenPool(2, 2, "cpu", False)
pool.alloc(reqs)
scheduler.waiting_queue = []
scheduler.disagg_prefill_bootstrap_queue = SimpleNamespace(queue=[])
scheduler.optimistic_release_and_requeue = Mock(side_effect=pool.free)
with get_context().override_server_args(
disaggregation_decode_allocation_policy="early"
):
self.assertEqual(scheduler.process_disagg_prefill_inflight_queue(), [])
self.assertEqual(scheduler.disagg_prefill_inflight_queue, [req])
scheduler.send_kv_chunk.assert_not_called()
# This is the existing metadata-received transition, not readiness.
self.prefill.update_status(1, KVPoll.WaitingForInput)
self.assertEqual(scheduler.process_disagg_prefill_inflight_queue(), [])
scheduler.send_kv_chunk.assert_called_once_with(req, last_chunk=True)
self.assertFalse(req.pending_bootstrap)
scheduler.process_disagg_prefill_inflight_queue()
self.assertEqual(scheduler.disagg_prefill_inflight_queue, reqs)
scheduler.waiting_queue.append(object())
scheduler.process_disagg_prefill_inflight_queue()
self.assertEqual(scheduler.disagg_prefill_inflight_queue, reqs[1:])
self.assertEqual(pool.available_size(), 1)
scheduler.process_disagg_prefill_inflight_queue()
scheduler.optimistic_release_and_requeue.assert_called_once_with(reqs[0])
def test_clear_before_allocation_notifies_prefill(self):
sender, receiver = self.sender(), self.receiver()