Fail-fast on PD subprocess exit and scheduler exception (#26298)

This commit is contained in:
Liangsheng Yin
2026-05-25 16:42:50 -07:00
committed by GitHub
parent e7b12fe6fa
commit 8805f4cf16
4 changed files with 57 additions and 0 deletions
+7
View File
@@ -3841,6 +3841,13 @@ def run_scheduler_process(
traceback = get_exception_traceback()
logger.error(f"Scheduler hit an exception: {traceback}")
parent_process.send_signal(signal.SIGQUIT)
# Opt-in: SIGKILL the pgroup so sibling ranks don't spew thousands
# of NCCL/TCPStore tracebacks before they finally die.
if envs.SGLANG_KILLPG_ON_SCHEDULER_EXCEPTION.get():
try:
os.killpg(os.getpgrp(), signal.SIGKILL)
except Exception:
pass
finally:
if scheduler is not None:
# FPM has a background ZMQ publisher thread that needs explicit