Add subprocess liveness monitor to detect scheduler crashes (#18582)

Co-authored-by: 继优 <jiyou.ljy@alibaba-inc.com> Co-authored-by: shuwenn <47200617+alphabetc1@users.noreply.github.com>
2026-03-29 00:09:13 -07:00
parent 35f5a0ff35
commit 22e4733ab9
7 changed files with 300 additions and 29 deletions
@@ -0,0 +1,137 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for SubprocessWatchdog in watchdog.py"""
+
+import multiprocessing as mp
+import os
+import signal
+import threading
+import time
+import unittest.mock
+
+from sglang.srt.utils.watchdog import SubprocessWatchdog
+from sglang.test.ci.ci_register import register_cpu_ci
+from sglang.test.test_utils import CustomTestCase
+
+register_cpu_ci(est_time=10, suite="stage-a-cpu-only")
+
+
+def healthy_worker():
+    time.sleep(10)
+
+
+def crashing_worker():
+    os._exit(1)
+
+
+def slow_crash_worker(delay: float = 0.5):
+    time.sleep(delay)
+    os._exit(42)
+
+
+class TestSubprocessWatchdog(CustomTestCase):
+    def setUp(self):
+        self.sigquit_triggered = threading.Event()
+        self._procs = []
+        self._monitor = None
+
+        original_kill = os.kill
+
+        def mock_kill(pid, sig):
+            if sig == signal.SIGQUIT:
+                self.sigquit_triggered.set()
+            else:
+                original_kill(pid, sig)
+
+        self._patcher = unittest.mock.patch("os.kill", side_effect=mock_kill)
+        self._patcher.start()
+
+    def tearDown(self):
+        if self._monitor is not None:
+            self._monitor.stop()
+        self._patcher.stop()
+        for p in self._procs:
+            if p.is_alive():
+                p.terminate()
+                p.join(timeout=1)
+
+    def _spawn(self, target, args=()):
+        proc = mp.Process(target=target, args=args)
+        proc.start()
+        self._procs.append(proc)
+        return proc
+
+    def _watch(self, procs, names=None, interval=0.1):
+        if not isinstance(procs, list):
+            procs = [procs]
+        self._monitor = SubprocessWatchdog(
+            processes=procs,
+            process_names=names,
+            interval=interval,
+        )
+        self._monitor.start()
+        return self._monitor
+
+    def test_healthy_processes_no_sigquit(self):
+        proc = self._spawn(healthy_worker)
+        self._watch(proc)
+        time.sleep(0.5)
+        self.assertFalse(self.sigquit_triggered.is_set())
+
+    def test_crashed_process_triggers_sigquit(self):
+        proc = self._spawn(slow_crash_worker, args=(0.2,))
+        self._watch(proc)
+        self.assertTrue(
+            self.sigquit_triggered.wait(timeout=2.0),
+            "SIGQUIT was not triggered within timeout",
+        )
+
+    def test_immediate_crash_detection(self):
+        proc = self._spawn(crashing_worker)
+        self._watch(proc, interval=0.05)
+        self.assertTrue(
+            self.sigquit_triggered.wait(timeout=1.0),
+            "Immediate crash was not detected",
+        )
+
+    def test_multiple_processes_one_crashes(self):
+        healthy = self._spawn(healthy_worker)
+        crashing = self._spawn(slow_crash_worker, args=(0.2,))
+        self._watch([healthy, crashing], names=["healthy", "crashing"])
+        self.assertTrue(
+            self.sigquit_triggered.wait(timeout=2.0),
+            "Crash was not detected when one of multiple processes crashed",
+        )
+
+    def test_empty_processes_list(self):
+        self._watch([], interval=0.1)
+        time.sleep(0.3)
+        self.assertFalse(self.sigquit_triggered.is_set())
+
+    def test_normal_exit_no_sigquit(self):
+        proc = self._spawn(lambda: None)
+        proc.join(timeout=2)
+        self._watch(proc)
+        time.sleep(0.3)
+        self.assertFalse(
+            self.sigquit_triggered.is_set(),
+            "SIGQUIT should not be triggered for normal exit (exitcode=0)",
+        )
+
+
+if __name__ == "__main__":
+    mp.set_start_method("spawn", force=True)
+    import unittest
+
+    unittest.main()