[PD] Honor gracefully_exit in disaggregation event loops and keep non-zero-rank launchers alive on SIGTERM (#40793)
Co-authored-by: metamergebot <324680979+metamergebot@users.noreply.github.com> Co-authored-by: Shiyan Deng <dsy842974287@meta.com>
This commit is contained in:
co-authored by
metamergebot
Shiyan Deng
parent
37ebcac3c4
commit
7325b38412
@@ -2865,6 +2865,9 @@ class SchedulerDisaggregationDecodeMixin:
|
||||
"""A normal scheduler loop for decode worker in disaggregation mode."""
|
||||
|
||||
while True:
|
||||
if self.gracefully_exit:
|
||||
break
|
||||
|
||||
# Pending rooms from the prior cycle can overlap request intake and
|
||||
# the tail of the in-flight decode graph.
|
||||
if not self._engine_paused:
|
||||
@@ -2909,6 +2912,9 @@ class SchedulerDisaggregationDecodeMixin:
|
||||
self.process_batch_result(tmp_batch, tmp_result)
|
||||
|
||||
while True:
|
||||
if self.gracefully_exit:
|
||||
break
|
||||
|
||||
# Pending rooms from the prior cycle can overlap request intake and
|
||||
# the tail of the in-flight decode graph.
|
||||
if not self._engine_paused:
|
||||
|
||||
@@ -666,6 +666,9 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
def event_loop_normal_disagg_prefill(self: Scheduler) -> None:
|
||||
"""A normal scheduler loop for prefill worker in disaggregation mode."""
|
||||
while True:
|
||||
if self.gracefully_exit:
|
||||
break
|
||||
|
||||
# Receive requests
|
||||
self.ingest_requests()
|
||||
if self._engine_paused:
|
||||
@@ -706,6 +709,9 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
self.result_queue = deque()
|
||||
|
||||
while True:
|
||||
if self.gracefully_exit:
|
||||
break
|
||||
|
||||
# Receive requests
|
||||
self.ingest_requests()
|
||||
if self._engine_paused:
|
||||
|
||||
@@ -1184,6 +1184,20 @@ class Engine(EngineScoreMixin, EngineBase):
|
||||
weight_cache_daemon_procs,
|
||||
)
|
||||
|
||||
# Non-zero ranks cannot drain on their own: rank 0 stops every TP rank
|
||||
# via the ShutdownReq broadcast, and the orchestrator's kill timeout is
|
||||
# the backstop.
|
||||
if threading.current_thread() is threading.main_thread():
|
||||
|
||||
def sigterm_handler(signum, frame):
|
||||
logger.warning(
|
||||
f"SIGTERM received on node_rank {get_parallel().node_rank}; "
|
||||
"waiting for the rank-0 ShutdownReq broadcast to stop "
|
||||
"the schedulers."
|
||||
)
|
||||
|
||||
signal.signal(signal.SIGTERM, sigterm_handler)
|
||||
|
||||
# A node-local Rust listener owns the health endpoints when present.
|
||||
rust_server_owns_base_port = (
|
||||
envs.SGLANG_RUST_SERVER.get() and node_hosts_rust_server()
|
||||
|
||||
@@ -2862,6 +2862,10 @@ def launch_server(
|
||||
run_detokenizer_process_func=run_detokenizer_process_func,
|
||||
)
|
||||
|
||||
if get_parallel().node_rank >= 1:
|
||||
# _launch_subprocesses already blocked until the schedulers exited.
|
||||
return
|
||||
|
||||
if envs.SGLANG_RUST_SERVER.get():
|
||||
# The Rust server serves api-server, tokenizer, and detokenizer, so the
|
||||
# main process has no Python HTTP server / tokenizer manager to run.
|
||||
|
||||
Reference in New Issue
Block a user