From 78d36f5f62d514f2fe4712bb21c547db66063955 Mon Sep 17 00:00:00 2001 From: Khoa Pham Date: Thu, 27 Aug 2026 02:34:05 -0700 Subject: [PATCH] fix: kill_process_tree waits for the reap by default (#36589) --- python/sglang/lang/backend/runtime_endpoint.py | 4 +++- python/sglang/srt/managers/tokenizer_manager.py | 2 +- python/sglang/srt/utils/common.py | 13 ++++++++----- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/python/sglang/lang/backend/runtime_endpoint.py b/python/sglang/lang/backend/runtime_endpoint.py index e0e3a4c82..db61e431f 100644 --- a/python/sglang/lang/backend/runtime_endpoint.py +++ b/python/sglang/lang/backend/runtime_endpoint.py @@ -443,7 +443,9 @@ class Runtime: from sglang.srt.utils import kill_process_tree if self.pid is not None: - kill_process_tree(self.pid) + # Note(kpham-sgl): __del__ routes here, so the reap wait has to stay + # off -- blocking inside GC stalls whichever thread is allocating. + kill_process_tree(self.pid, wait_timeout=None) self.pid = None def start_profile(self): diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index 1e6b03f91..194f25703 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -3166,7 +3166,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): deadline = time.monotonic() + 15 while time.monotonic() < deadline and collect_scheduler_processes(): time.sleep(0.1) - kill_process_tree(os.getpid(), include_parent=True) + kill_process_tree(os.getpid(), include_parent=False, wait_timeout=60) sys.exit(0) def force_exit_handler(self): diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index 8f826768e..1f6cdfef7 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -2219,14 +2219,17 @@ def kill_process_tree( parent_pid, include_parent: bool = True, skip_pid: int = None, - wait_timeout: Optional[float] = None, + wait_timeout: Optional[float] = 60, ): """Kill the process and all its child processes. `wait_timeout` (seconds) blocks until every killed process is reaped and - raises `RuntimeError` on timeout; `None` is fire-and-forget. The - `parent_pid == os.getpid()` branch calls `sys.exit(0)` and cannot wait - for itself -- use `include_parent=False` if child reap must finish first. + raises `RuntimeError` on timeout. SIGKILL only queues the teardown, so + returning without waiting leaves the GPU context, the pinned host memory + and the ports held for seconds; pass `None` only where blocking is + unacceptable, such as a `__del__`. The `parent_pid == os.getpid()` branch + calls `sys.exit(0)` and cannot wait for itself -- use + `include_parent=False` if child reap must finish first. """ logger.info( f"kill_process_tree called: parent_pid={parent_pid}, " @@ -2239,10 +2242,10 @@ def kill_process_tree( try: itself = psutil.Process(parent_pid) + children = itself.children(recursive=True) except psutil.NoSuchProcess: return - children = itself.children(recursive=True) killed = [] for child in children: if child.pid == skip_pid: