[PD] Retract transferred decode requests on a retract pause (#43757)

This commit is contained in:
Liangsheng Yin
2026-10-11 21:29:39 -07:00
committed by GitHub
parent 0d0be8fc98
commit a1001bcf3a
3 changed files with 62 additions and 1 deletions
+14
View File
@@ -5156,6 +5156,10 @@ class Scheduler(
# Waiting queues: waiting + bootstrapping + preallocation + kv transfer (decode)
if not ignore_waiting:
idle &= len(self.waiting_queue) == 0
else:
# A PD-decode request waiting with its transferred KV still owns
# its request row; a flush would free it.
idle &= not any(r.kv.holds_kv for r in self.waiting_queue)
if (
for_health_check
@@ -5618,6 +5622,16 @@ class Scheduler(
):
retract_reqs.append(self.chunked_req)
if self.disaggregation_mode == DisaggregationMode.DECODE:
# Transferred requests wait for a slot holding their row and KV;
# retract them too, so their KV is rebuilt under the new weights.
staged = [r for r in self.waiting_queue if r.kv.holds_kv]
if staged:
self.waiting_queue = [
r for r in self.waiting_queue if not r.kv.holds_kv
]
retract_reqs += staged
self.last_batch = None
self.cur_batch_for_debug = None
@@ -13,9 +13,11 @@ from sglang.srt.managers.scheduler import Scheduler
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
def _make_req(is_retracted: bool) -> MagicMock:
def _make_req(is_retracted: bool, holds_kv: bool = False) -> MagicMock:
req = MagicMock()
req.is_retracted = is_retracted
# Retracted and never-scheduled requests hold no request row.
req.kv.holds_kv = holds_kv
return req
@@ -85,6 +87,18 @@ class TestSchedulerFlushCacheAfterRetract(unittest.TestCase):
self.assertTrue(scheduler.flush_cache())
scheduler.tree_cache.reset.assert_called_once()
def test_paused_flush_refuses_waiting_req_holding_kv(self):
"""A PD-decode request whose KV has arrived waits holding its request
row; a paused flush would free KV it still owns."""
scheduler = self._new_scheduler()
scheduler.waiting_queue = [
_make_req(is_retracted=False),
_make_req(is_retracted=False, holds_kv=True),
]
self.assertFalse(scheduler.flush_cache())
scheduler.tree_cache.reset.assert_not_called()
def test_is_fully_idle_default_ignores_nothing(self):
"""Regression guard: every other is_fully_idle() caller passes no args,
so a non-empty waiting_queue must still report not-idle by default."""
@@ -647,6 +647,39 @@ class TestSchedulerPauseGeneration(CustomTestCase):
mock_retract_all.assert_called_once()
self.assertEqual(mock_retract_all.call_args.kwargs["offload_kv"], False)
def test_pd_decode_retract_includes_transferred_waiting_reqs(self):
"""A transferred request waiting for a running slot holds a request row
and KV; retract must rebuild it too, so a paused flush cannot free it."""
scheduler = self._new_scheduler()
scheduler.disaggregation_mode = DisaggregationMode.DECODE
scheduler.disagg_decode_prealloc_queue = MagicMock()
def _req(rid, req_pool_idx):
return SimpleNamespace(
rid=rid,
finished=lambda: False,
output_ids=[7],
time_stats=MagicMock(),
kv=SimpleNamespace(holds_kv=req_pool_idx is not None),
)
running = _req("running", 0)
staged = _req("staged", 1)
unstaged = _req("unstaged", None)
scheduler.running_batch.reqs = [running]
scheduler.waiting_queue = [staged, unstaged]
with patch("sglang.srt.managers.scheduler.retract_all") as mock_retract_all:
scheduler.pause_generation(PauseGenerationReqInput(mode="retract"))
self.assertEqual(mock_retract_all.call_args.kwargs["reqs"], [running, staged])
self.assertEqual(scheduler.waiting_queue, [unstaged])
held = [
c.args[0]
for c in scheduler.disagg_decode_prealloc_queue.hold_rebootstrap.call_args_list
]
self.assertEqual(held, [running, staged])
def test_pd_decode_retract_completes_offload_before_free(self):
"""PD decode retract must not free KV an offload copy is still reading, and
must not leave offload acks that keep the paused engine from going idle."""