[diffusion] fix: wait for multimodal process teardown (#40765)

Signed-off-by: Xucheng Zhou <aden1350@outlook.com>
Co-authored-by: Mick <mickjagger19@icloud.com>
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
adenzhou1350
2026-10-08 14:36:17 +08:00
committed by GitHub
co-authored by Mick Mick Qian
parent 4f320fb85a
commit 74308cf1d6
2 changed files with 118 additions and 2 deletions
@@ -30,8 +30,18 @@ def kill_itself_when_parent_died() -> None:
os.kill(os.getpid(), signal.SIGKILL)
def kill_process_tree(parent_pid, include_parent: bool = True, skip_pid: int = None):
"""Kill the process and all its child processes."""
def kill_process_tree(
parent_pid,
include_parent: bool = True,
skip_pid: int = None,
wait_timeout: float | None = 10,
):
"""Kill a process tree and wait for its resources to be released.
SIGKILL is asynchronous. Waiting prevents a subsequent server test from
racing the old process for ports or accelerator resources. Pass ``None``
only in a context where blocking is not acceptable.
"""
# Remove sigchld handler to avoid spammy logs.
if threading.current_thread() is threading.main_thread():
signal.signal(signal.SIGCHLD, signal.SIG_DFL)
@@ -46,11 +56,13 @@ def kill_process_tree(parent_pid, include_parent: bool = True, skip_pid: int = N
return
children = itself.children(recursive=True)
killed = []
for child in children:
if child.pid == skip_pid:
continue
try:
child.kill()
killed.append(child)
except psutil.NoSuchProcess:
pass
@@ -65,5 +77,14 @@ def kill_process_tree(parent_pid, include_parent: bool = True, skip_pid: int = N
# Sometime processes cannot be killed with SIGKILL (e.g, PID=1 launched by kubernetes),
# so we send an additional signal to kill them.
itself.send_signal(signal.SIGQUIT)
killed.append(itself)
except psutil.NoSuchProcess:
pass
if wait_timeout is not None and killed:
_, alive = psutil.wait_procs(killed, timeout=wait_timeout)
if alive:
raise RuntimeError(
f"kill_process_tree: {len(alive)} process(es) did not exit "
f"within {wait_timeout}s; pids={[process.pid for process in alive]}"
)
@@ -0,0 +1,95 @@
# SPDX-License-Identifier: Apache-2.0
import importlib.util
import os
import subprocess
import sys
import textwrap
import unittest
from pathlib import Path
import psutil
PROCESS_PATH = Path(__file__).parents[2] / "runtime" / "utils" / "process.py"
SPEC = importlib.util.spec_from_file_location("multimodal_process", PROCESS_PATH)
assert SPEC is not None and SPEC.loader is not None
PROCESS = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(PROCESS)
kill_process_tree = PROCESS.kill_process_tree
@unittest.skipUnless(os.name == "posix", "requires POSIX process semantics")
class TestKillProcessTree(unittest.TestCase):
def setUp(self):
self.children = []
def tearDown(self):
for process in self.children:
# let the tree root reap its children before terminating the root
kill_process_tree(process.pid, include_parent=False)
kill_process_tree(process.pid)
process.wait(timeout=10)
def spawn_child(self):
process = subprocess.Popen(
[sys.executable, "-c", "import time; time.sleep(600)"]
)
self.children.append(process)
return process.pid
def spawn_tree(self):
script = textwrap.dedent(
"""
import subprocess
import sys
import time
child = subprocess.Popen(
[sys.executable, "-c", "import time; time.sleep(600)"]
)
print(child.pid, flush=True)
child.wait()
time.sleep(600)
"""
)
process = subprocess.Popen(
[sys.executable, "-c", script], stdout=subprocess.PIPE, text=True
)
self.children.append(process)
with process.stdout:
child_pid = int(process.stdout.readline())
return process.pid, child_pid
def test_waits_for_killed_children(self):
parent_pid, child_pid = self.spawn_tree()
sibling_pid = self.spawn_child()
kill_process_tree(parent_pid, include_parent=False)
self.assertFalse(psutil.pid_exists(child_pid))
self.assertTrue(psutil.pid_exists(parent_pid))
self.assertTrue(psutil.pid_exists(sibling_pid))
def test_skip_pid_keeps_child_alive(self):
parent_pid, child_pid = self.spawn_tree()
sibling_pid = self.spawn_child()
kill_process_tree(parent_pid, include_parent=False, skip_pid=child_pid)
self.assertTrue(psutil.pid_exists(child_pid))
self.assertTrue(psutil.pid_exists(parent_pid))
self.assertTrue(psutil.pid_exists(sibling_pid))
def test_waits_for_included_parent(self):
child_pid = self.spawn_child()
kill_process_tree(child_pid)
self.assertFalse(psutil.pid_exists(child_pid))
def test_missing_parent_is_noop(self):
kill_process_tree(0x7FFFFFFF)
if __name__ == "__main__":
unittest.main()