From b91137ab988788d7587a329c0d35c679a7afc551 Mon Sep 17 00:00:00 2001 From: "minke.yu" Date: Thu, 24 Sep 2026 12:01:57 +0800 Subject: [PATCH] scheduler: CP-symmetric idle check for health-check admission Health-check admit/skip used is_fully_idle(), which includes rank-local hicache drain queues; ranks diverge right after activity, so one rank dispatched the health-check generate while others piggyback-skipped, deadlocking CP (hicache drain all_reduce vs CP request broadcast). Seen on cp2/cp4 + hicache L3 after router health checks. Recovered from b300 /data/ymk/sglang working copy (uncommitted WIP). --- python/sglang/srt/managers/scheduler.py | 35 ++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 6c99d9e5c..47e0e160d 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -2097,9 +2097,13 @@ class Scheduler( vmm_errors = self._materialize_cuda_vmm_inputs(recv_req) # Skip health check when server is busy — ongoing requests already carry health info. - if is_health_check_generate_req(recv_req) and not self.is_fully_idle( - for_health_check=True - ): + # NOTE: the admit/skip decision must be identical on every CP/TP rank. + # is_fully_idle() includes rank-local hicache drain queues, which diverge + # across ranks right after activity; a divergent decision lets one rank + # dispatch the health-check generate while others piggyback-skip, breaking + # collective ordering (deadlock: one rank blocks in the hicache drain + # all_reduce while another waits in the CP request broadcast). + if is_health_check_generate_req(recv_req) and not self.is_sched_idle_cp_symmetric(): self.return_health_check_ipcs.append( getattr(recv_req, "http_worker_ipc", None) ) @@ -4982,6 +4986,31 @@ class Scheduler( else: self.metrics_reporter.record_scheduler_active() + def is_sched_idle_cp_symmetric(self) -> bool: + """Idle check using only state that is identical across CP/TP ranks. + + Request/batch/queue state is collectively maintained (requests arrive + via broadcast, batches are collectively scheduled), so every rank + computes the same result. Rank-local hicache drain and disagg transfer + queues are deliberately excluded: those are exactly the terms that + diverge across ranks and caused the CP health-check deadlock + (hicache drain all_reduce vs CP request broadcast cross-collective + wait, seen on cp2/cp4 + hicache L3 right after router health checks). + + Used only for health-check admission; all other idle logic keeps using + is_fully_idle(). + """ + return ( + self.running_batch.is_empty() + and self.chunked_req is None + and not self.dllm_manager.any_staging_reqs() + and (self.last_batch is None or self.last_batch.is_empty()) + and (not self.enable_overlap or len(self.result_queue) == 0) + and self._pp_microbatches_drained() + and len(self.waiting_queue) == 0 + and len(self.grammar_manager.grammar_queue) == 0 + ) + def is_fully_idle(self, for_health_check=False) -> bool: # Health check piggybacks on running requests in process_output. # Only running_batch + waiting_queue guarantee active GPU processing;