From 74308cf1d65edf27238dc0dac15d3d62ecaab67e Mon Sep 17 00:00:00 2001 From: adenzhou1350 Date: Thu, 8 Oct 2026 14:36:17 +0800 Subject: [PATCH] [diffusion] fix: wait for multimodal process teardown (#40765) Signed-off-by: Xucheng Zhou Co-authored-by: Mick Co-authored-by: Mick Qian --- .../multimodal_gen/runtime/utils/process.py | 25 ++++- .../multimodal_gen/test/unit/test_process.py | 95 +++++++++++++++++++ 2 files changed, 118 insertions(+), 2 deletions(-) create mode 100644 python/sglang/multimodal_gen/test/unit/test_process.py diff --git a/python/sglang/multimodal_gen/runtime/utils/process.py b/python/sglang/multimodal_gen/runtime/utils/process.py index 90c318949a..8ec8ba02ff 100644 --- a/python/sglang/multimodal_gen/runtime/utils/process.py +++ b/python/sglang/multimodal_gen/runtime/utils/process.py @@ -30,8 +30,18 @@ def kill_itself_when_parent_died() -> None: os.kill(os.getpid(), signal.SIGKILL) -def kill_process_tree(parent_pid, include_parent: bool = True, skip_pid: int = None): - """Kill the process and all its child processes.""" +def kill_process_tree( + parent_pid, + include_parent: bool = True, + skip_pid: int = None, + wait_timeout: float | None = 10, +): + """Kill a process tree and wait for its resources to be released. + + SIGKILL is asynchronous. Waiting prevents a subsequent server test from + racing the old process for ports or accelerator resources. Pass ``None`` + only in a context where blocking is not acceptable. + """ # Remove sigchld handler to avoid spammy logs. if threading.current_thread() is threading.main_thread(): signal.signal(signal.SIGCHLD, signal.SIG_DFL) @@ -46,11 +56,13 @@ def kill_process_tree(parent_pid, include_parent: bool = True, skip_pid: int = N return children = itself.children(recursive=True) + killed = [] for child in children: if child.pid == skip_pid: continue try: child.kill() + killed.append(child) except psutil.NoSuchProcess: pass @@ -65,5 +77,14 @@ def kill_process_tree(parent_pid, include_parent: bool = True, skip_pid: int = N # Sometime processes cannot be killed with SIGKILL (e.g, PID=1 launched by kubernetes), # so we send an additional signal to kill them. itself.send_signal(signal.SIGQUIT) + killed.append(itself) except psutil.NoSuchProcess: pass + + if wait_timeout is not None and killed: + _, alive = psutil.wait_procs(killed, timeout=wait_timeout) + if alive: + raise RuntimeError( + f"kill_process_tree: {len(alive)} process(es) did not exit " + f"within {wait_timeout}s; pids={[process.pid for process in alive]}" + ) diff --git a/python/sglang/multimodal_gen/test/unit/test_process.py b/python/sglang/multimodal_gen/test/unit/test_process.py new file mode 100644 index 0000000000..5e2b1224b3 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_process.py @@ -0,0 +1,95 @@ +# SPDX-License-Identifier: Apache-2.0 + +import importlib.util +import os +import subprocess +import sys +import textwrap +import unittest +from pathlib import Path + +import psutil + +PROCESS_PATH = Path(__file__).parents[2] / "runtime" / "utils" / "process.py" +SPEC = importlib.util.spec_from_file_location("multimodal_process", PROCESS_PATH) +assert SPEC is not None and SPEC.loader is not None +PROCESS = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(PROCESS) +kill_process_tree = PROCESS.kill_process_tree + + +@unittest.skipUnless(os.name == "posix", "requires POSIX process semantics") +class TestKillProcessTree(unittest.TestCase): + def setUp(self): + self.children = [] + + def tearDown(self): + for process in self.children: + # let the tree root reap its children before terminating the root + kill_process_tree(process.pid, include_parent=False) + kill_process_tree(process.pid) + process.wait(timeout=10) + + def spawn_child(self): + process = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(600)"] + ) + self.children.append(process) + return process.pid + + def spawn_tree(self): + script = textwrap.dedent( + """ + import subprocess + import sys + import time + + child = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(600)"] + ) + print(child.pid, flush=True) + child.wait() + time.sleep(600) + """ + ) + process = subprocess.Popen( + [sys.executable, "-c", script], stdout=subprocess.PIPE, text=True + ) + self.children.append(process) + with process.stdout: + child_pid = int(process.stdout.readline()) + return process.pid, child_pid + + def test_waits_for_killed_children(self): + parent_pid, child_pid = self.spawn_tree() + sibling_pid = self.spawn_child() + + kill_process_tree(parent_pid, include_parent=False) + + self.assertFalse(psutil.pid_exists(child_pid)) + self.assertTrue(psutil.pid_exists(parent_pid)) + self.assertTrue(psutil.pid_exists(sibling_pid)) + + def test_skip_pid_keeps_child_alive(self): + parent_pid, child_pid = self.spawn_tree() + sibling_pid = self.spawn_child() + + kill_process_tree(parent_pid, include_parent=False, skip_pid=child_pid) + + self.assertTrue(psutil.pid_exists(child_pid)) + self.assertTrue(psutil.pid_exists(parent_pid)) + self.assertTrue(psutil.pid_exists(sibling_pid)) + + def test_waits_for_included_parent(self): + child_pid = self.spawn_child() + + kill_process_tree(child_pid) + + self.assertFalse(psutil.pid_exists(child_pid)) + + def test_missing_parent_is_noop(self): + kill_process_tree(0x7FFFFFFF) + + +if __name__ == "__main__": + unittest.main()