diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index 556dcc5798..7ff3603cd7 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -1148,6 +1148,24 @@ class SchedulerDisaggregationPrefillMixin: if self.handle_pending_bootstrap(req, poll): self.send_kv_chunk(req, last_chunk=True) undone_reqs.append(req) + elif ( + poll == KVPoll.Bootstrapping + and get_disagg().disaggregation_decode_allocation_policy + != "prefill_complete" + and self.req_to_token_pool.available_size() == 0 + and ( + self.waiting_queue + or ( + self.disagg_prefill_bootstrap_queue.queue + and self.req_to_metadata_buffer_idx_allocator.available_size() + == 0 + ) + ) + ): + # Decode may have admitted different requests. Yield one + # slot so they can prefill, including peers still waiting + # for a metadata buffer in the bootstrap queue. + self.optimistic_release_and_requeue(req) elif poll != KVPoll.Failed: undone_reqs.append(req) continue @@ -1694,6 +1712,9 @@ class SchedulerDisaggregationPrefillMixin: ) # Reset it so the next real bootstrap done can be recorded. req.time_stats.bootstrap_done_time = 0.0 + maybe_release_metadata_buffer( + req, self.req_to_metadata_buffer_idx_allocator + ) self.disagg_prefill_bootstrap_queue.queue.append(req) else: req.prefill_attempt_count += 1 diff --git a/test/registered/unit/disaggregation/test_prefill_complete_integration.py b/test/registered/unit/disaggregation/test_prefill_complete_integration.py index b99d03e906..b778f441ea 100644 --- a/test/registered/unit/disaggregation/test_prefill_complete_integration.py +++ b/test/registered/unit/disaggregation/test_prefill_complete_integration.py @@ -22,7 +22,10 @@ from sglang.srt.disaggregation.mooncake.conn import ( MooncakeKVSender, ) from sglang.srt.disaggregation.prefill import SchedulerDisaggregationPrefillMixin +from sglang.srt.managers.schedule_batch import Req +from sglang.srt.mem_cache.memory_pool import ReqToTokenPool from sglang.srt.runtime_context import get_context, publish, reset_context +from sglang.srt.sampling.sampling_params import SamplingParams from sglang.srt.server_args import ServerArgs from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -475,7 +478,7 @@ class TestAllocationIntegration(CustomTestCase): queue.req_to_token_pool.available_size.assert_called_once() self.assertEqual(queue.queue, [entry]) - def test_exhausted_optimistic_attempt_remains_compute_eligible(self): + def test_exhausted_optimistic_attempt_follows_allocation_policy(self): override = get_context().override_server_args( optimistic_prefill_attempts=1, disaggregation_decode_allocation_policy="prefill_complete", @@ -492,6 +495,7 @@ class TestAllocationIntegration(CustomTestCase): _compute_max_prefix_len=Mock(return_value=31), storage_prefetch_retry_attempts=3, storage_prefetch_last_match_len=8, + metadata_buffer_index=0, reset_for_retract=Mock(), advance_cache_request_handle=Mock(), time_stats=MagicMock(), @@ -504,6 +508,7 @@ class TestAllocationIntegration(CustomTestCase): disagg_prefill_bootstrap_queue=SimpleNamespace(queue=[]), metrics_reporter=SimpleNamespace(enable_metrics=False), processed_tokens_counter=0, + req_to_metadata_buffer_idx_allocator=Mock(), ) with ( patch("sglang.srt.disaggregation.prefill.checkpoint_kv_cache"), @@ -512,13 +517,28 @@ class TestAllocationIntegration(CustomTestCase): SchedulerDisaggregationPrefillMixin.optimistic_release_and_requeue( scheduler, req ) - self.assertEqual(scheduler.waiting_queue, [req]) - self.assertEqual(scheduler.disagg_prefill_bootstrap_queue.queue, []) - self.assertEqual(req.prefill_attempt_count, 2) - self.assertEqual(list(req.output_ids), []) - self.assertTrue(req.pending_bootstrap) - self.assertEqual(req.storage_prefetch_retry_attempts, 0) - self.assertEqual(req.storage_prefetch_last_match_len, 16) + self.assertEqual(scheduler.waiting_queue, [req]) + self.assertEqual(scheduler.disagg_prefill_bootstrap_queue.queue, []) + self.assertEqual(req.prefill_attempt_count, 2) + self.assertEqual(list(req.output_ids), []) + self.assertTrue(req.pending_bootstrap) + self.assertEqual(req.storage_prefetch_retry_attempts, 0) + self.assertEqual(req.storage_prefetch_last_match_len, 16) + scheduler.req_to_metadata_buffer_idx_allocator.free.assert_not_called() + + with get_context().override_server_args( + disaggregation_decode_allocation_policy="early" + ): + scheduler.waiting_queue.clear() + SchedulerDisaggregationPrefillMixin.optimistic_release_and_requeue( + scheduler, req + ) + self.assertEqual(scheduler.waiting_queue, []) + self.assertEqual(scheduler.disagg_prefill_bootstrap_queue.queue, [req]) + self.assertEqual(req.metadata_buffer_index, -1) + scheduler.req_to_metadata_buffer_idx_allocator.free.assert_called_once_with( + 0 + ) def test_chunked_prefill_does_not_yield_for_decode_admission(self): for policy in ("early", "prefill_complete"): @@ -562,6 +582,25 @@ class TestAllocationIntegration(CustomTestCase): finally: override.restore() + def inflight_scheduler(self, reqs): + scheduler = SchedulerDisaggregationPrefillMixin() + scheduler.tree_cache = Mock() + scheduler.scheduler_stage_metrics = None + scheduler.disagg_prefill_inflight_queue = list(reqs) + scheduler.attn_cp_cpu_group = scheduler.attn_tp_cpu_group = object() + scheduler.output_streamer = Mock() + scheduler.send_kv_chunk = Mock() + self.enterContext( + patch( + "sglang.srt.disaggregation.prefill.poll_and_all_reduce_attn_cp_tp_group", + side_effect=lambda senders, *groups: [s.poll() for s in senders], + ) + ) + self.enterContext( + patch("sglang.srt.disaggregation.prefill.checkpoint_kv_cache") + ) + return scheduler + def test_completed_prefill_parks_until_normal_pointer_handshake(self): sender, receiver = self.sender(), self.receiver() self.flush() @@ -577,13 +616,9 @@ class TestAllocationIntegration(CustomTestCase): prefill_attempt_count=1, skip_radix_cache_insert=False, ) - scheduler = SchedulerDisaggregationPrefillMixin() - scheduler.tree_cache = Mock() - scheduler.scheduler_stage_metrics = None - scheduler.disagg_prefill_inflight_queue = [req] - scheduler.attn_cp_cpu_group = scheduler.attn_tp_cpu_group = object() - scheduler.output_streamer = Mock() - scheduler.send_kv_chunk = Mock() + scheduler = self.inflight_scheduler([req]) + scheduler.req_to_token_pool = SimpleNamespace(available_size=lambda: 0) + scheduler.waiting_queue = [Mock()] def finalize(r): r.pending_bootstrap = False @@ -592,21 +627,39 @@ class TestAllocationIntegration(CustomTestCase): scheduler.disagg_prefill_bootstrap_queue = SimpleNamespace( finalize_bootstrap=Mock(side_effect=finalize) ) - with ( - patch( - "sglang.srt.disaggregation.prefill.poll_and_all_reduce_attn_cp_tp_group", - side_effect=lambda senders, *groups: [s.poll() for s in senders], - ), - patch("sglang.srt.disaggregation.prefill.checkpoint_kv_cache"), + self.assertEqual(scheduler.process_disagg_prefill_inflight_queue(), []) + self.assertEqual(scheduler.disagg_prefill_inflight_queue, [req]) + scheduler.send_kv_chunk.assert_not_called() + # This is the existing metadata-received transition, not readiness. + self.prefill.update_status(1, KVPoll.WaitingForInput) + self.assertEqual(scheduler.process_disagg_prefill_inflight_queue(), []) + scheduler.send_kv_chunk.assert_called_once_with(req, last_chunk=True) + self.assertFalse(req.pending_bootstrap) + + def test_parked_prefill_reclaims_only_one_slot(self): + reqs = [Req(str(i), "", [1], SamplingParams()) for i in range(2)] + for i, req in enumerate(reqs): + req.pending_bootstrap = True + req.disagg_kv_sender = self.sender(room=i + 1) + scheduler = self.inflight_scheduler(reqs) + pool = scheduler.req_to_token_pool = ReqToTokenPool(2, 2, "cpu", False) + pool.alloc(reqs) + scheduler.waiting_queue = [] + scheduler.disagg_prefill_bootstrap_queue = SimpleNamespace(queue=[]) + scheduler.optimistic_release_and_requeue = Mock(side_effect=pool.free) + with get_context().override_server_args( + disaggregation_decode_allocation_policy="early" ): - self.assertEqual(scheduler.process_disagg_prefill_inflight_queue(), []) - self.assertEqual(scheduler.disagg_prefill_inflight_queue, [req]) - scheduler.send_kv_chunk.assert_not_called() - # This is the existing metadata-received transition, not readiness. - self.prefill.update_status(1, KVPoll.WaitingForInput) - self.assertEqual(scheduler.process_disagg_prefill_inflight_queue(), []) - scheduler.send_kv_chunk.assert_called_once_with(req, last_chunk=True) - self.assertFalse(req.pending_bootstrap) + scheduler.process_disagg_prefill_inflight_queue() + self.assertEqual(scheduler.disagg_prefill_inflight_queue, reqs) + + scheduler.waiting_queue.append(object()) + scheduler.process_disagg_prefill_inflight_queue() + self.assertEqual(scheduler.disagg_prefill_inflight_queue, reqs[1:]) + self.assertEqual(pool.available_size(), 1) + + scheduler.process_disagg_prefill_inflight_queue() + scheduler.optimistic_release_and_requeue.assert_called_once_with(reqs[0]) def test_clear_before_allocation_notifies_prefill(self): sender, receiver = self.sender(), self.receiver()