[diffusion] fix: wait for multimodal process teardown (#40765)
Signed-off-by: Xucheng Zhou <aden1350@outlook.com> Co-authored-by: Mick <mickjagger19@icloud.com> Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
co-authored by
Mick
Mick Qian
parent
4f320fb85a
commit
74308cf1d6
@@ -30,8 +30,18 @@ def kill_itself_when_parent_died() -> None:
|
||||
os.kill(os.getpid(), signal.SIGKILL)
|
||||
|
||||
|
||||
def kill_process_tree(parent_pid, include_parent: bool = True, skip_pid: int = None):
|
||||
"""Kill the process and all its child processes."""
|
||||
def kill_process_tree(
|
||||
parent_pid,
|
||||
include_parent: bool = True,
|
||||
skip_pid: int = None,
|
||||
wait_timeout: float | None = 10,
|
||||
):
|
||||
"""Kill a process tree and wait for its resources to be released.
|
||||
|
||||
SIGKILL is asynchronous. Waiting prevents a subsequent server test from
|
||||
racing the old process for ports or accelerator resources. Pass ``None``
|
||||
only in a context where blocking is not acceptable.
|
||||
"""
|
||||
# Remove sigchld handler to avoid spammy logs.
|
||||
if threading.current_thread() is threading.main_thread():
|
||||
signal.signal(signal.SIGCHLD, signal.SIG_DFL)
|
||||
@@ -46,11 +56,13 @@ def kill_process_tree(parent_pid, include_parent: bool = True, skip_pid: int = N
|
||||
return
|
||||
|
||||
children = itself.children(recursive=True)
|
||||
killed = []
|
||||
for child in children:
|
||||
if child.pid == skip_pid:
|
||||
continue
|
||||
try:
|
||||
child.kill()
|
||||
killed.append(child)
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
|
||||
@@ -65,5 +77,14 @@ def kill_process_tree(parent_pid, include_parent: bool = True, skip_pid: int = N
|
||||
# Sometime processes cannot be killed with SIGKILL (e.g, PID=1 launched by kubernetes),
|
||||
# so we send an additional signal to kill them.
|
||||
itself.send_signal(signal.SIGQUIT)
|
||||
killed.append(itself)
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
|
||||
if wait_timeout is not None and killed:
|
||||
_, alive = psutil.wait_procs(killed, timeout=wait_timeout)
|
||||
if alive:
|
||||
raise RuntimeError(
|
||||
f"kill_process_tree: {len(alive)} process(es) did not exit "
|
||||
f"within {wait_timeout}s; pids={[process.pid for process in alive]}"
|
||||
)
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import textwrap
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
import psutil
|
||||
|
||||
PROCESS_PATH = Path(__file__).parents[2] / "runtime" / "utils" / "process.py"
|
||||
SPEC = importlib.util.spec_from_file_location("multimodal_process", PROCESS_PATH)
|
||||
assert SPEC is not None and SPEC.loader is not None
|
||||
PROCESS = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(PROCESS)
|
||||
kill_process_tree = PROCESS.kill_process_tree
|
||||
|
||||
|
||||
@unittest.skipUnless(os.name == "posix", "requires POSIX process semantics")
|
||||
class TestKillProcessTree(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.children = []
|
||||
|
||||
def tearDown(self):
|
||||
for process in self.children:
|
||||
# let the tree root reap its children before terminating the root
|
||||
kill_process_tree(process.pid, include_parent=False)
|
||||
kill_process_tree(process.pid)
|
||||
process.wait(timeout=10)
|
||||
|
||||
def spawn_child(self):
|
||||
process = subprocess.Popen(
|
||||
[sys.executable, "-c", "import time; time.sleep(600)"]
|
||||
)
|
||||
self.children.append(process)
|
||||
return process.pid
|
||||
|
||||
def spawn_tree(self):
|
||||
script = textwrap.dedent(
|
||||
"""
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
child = subprocess.Popen(
|
||||
[sys.executable, "-c", "import time; time.sleep(600)"]
|
||||
)
|
||||
print(child.pid, flush=True)
|
||||
child.wait()
|
||||
time.sleep(600)
|
||||
"""
|
||||
)
|
||||
process = subprocess.Popen(
|
||||
[sys.executable, "-c", script], stdout=subprocess.PIPE, text=True
|
||||
)
|
||||
self.children.append(process)
|
||||
with process.stdout:
|
||||
child_pid = int(process.stdout.readline())
|
||||
return process.pid, child_pid
|
||||
|
||||
def test_waits_for_killed_children(self):
|
||||
parent_pid, child_pid = self.spawn_tree()
|
||||
sibling_pid = self.spawn_child()
|
||||
|
||||
kill_process_tree(parent_pid, include_parent=False)
|
||||
|
||||
self.assertFalse(psutil.pid_exists(child_pid))
|
||||
self.assertTrue(psutil.pid_exists(parent_pid))
|
||||
self.assertTrue(psutil.pid_exists(sibling_pid))
|
||||
|
||||
def test_skip_pid_keeps_child_alive(self):
|
||||
parent_pid, child_pid = self.spawn_tree()
|
||||
sibling_pid = self.spawn_child()
|
||||
|
||||
kill_process_tree(parent_pid, include_parent=False, skip_pid=child_pid)
|
||||
|
||||
self.assertTrue(psutil.pid_exists(child_pid))
|
||||
self.assertTrue(psutil.pid_exists(parent_pid))
|
||||
self.assertTrue(psutil.pid_exists(sibling_pid))
|
||||
|
||||
def test_waits_for_included_parent(self):
|
||||
child_pid = self.spawn_child()
|
||||
|
||||
kill_process_tree(child_pid)
|
||||
|
||||
self.assertFalse(psutil.pid_exists(child_pid))
|
||||
|
||||
def test_missing_parent_is_noop(self):
|
||||
kill_process_tree(0x7FFFFFFF)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user