Trigger scheduler diagnostics on health failure (#26757)

python/sglang/srt/environ.py

@@ -630,6 +630,11 @@ class Envs:
     # Health Check
     SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION = EnvBool(True)
 
+    # Crash diagnostics
+    SGLANG_PYSPY_DUMP_BEFORE_CRASH = EnvBool(True)
+    SGLANG_CUDA_COREDUMP_BEFORE_CRASH = EnvBool(True)
+    SGLANG_CUDA_COREDUMP_BEFORE_CRASH_WAIT_SECS = EnvFloat(60.0)
+
     # Encoder gRPC
     SGLANG_ENCODER_GRPC_TIMEOUT_SECS = EnvInt(60)
     # Encoder receiver selection: http|grpc (used by EPD paths).

python/sglang/srt/managers/tokenizer_manager.py

@@ -26,6 +26,7 @@ import signal
 import socket
 import sys
 import threading
+import time
 from array import array
 from collections import deque
 from contextlib import nullcontext
@@ -113,6 +114,11 @@ from sglang.srt.utils import (
     kill_process_tree,
 )
 from sglang.srt.utils.aio_rwlock import RWLock
+from sglang.srt.utils.cudacore_pyspy_dump_utils import (
+    collect_scheduler_processes,
+    pyspy_dump_schedulers,
+    trigger_cuda_user_coredump,
+)
 from sglang.srt.utils.hf_transformers_utils import (
     get_processor,
     get_tokenizer,
@@ -554,7 +560,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
         if isinstance(obj, GenerateReqInput) and obj.routed_dp_rank is not None:
             dp_size = self.server_args.dp_size
             if dp_size <= 1 and obj.routed_dp_rank == 0:
-                logger.warning(
+                logger.debug(
                     f"routed_dp_rank={obj.routed_dp_rank} is ignored because dp_size={dp_size}"
                 )
             elif obj.routed_dp_rank < 0 or obj.routed_dp_rank >= dp_size:
@@ -2404,7 +2410,10 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
     def dump_requests_before_crash(
         self, hostname: str = os.getenv("HOSTNAME", socket.gethostname())
     ):
-        if not self.crash_dump_folder:
+        should_dump_pyspy = envs.SGLANG_PYSPY_DUMP_BEFORE_CRASH.get()
+        should_dump_cuda_coredump = envs.SGLANG_CUDA_COREDUMP_BEFORE_CRASH.get()
+        should_dump_diagnostics = should_dump_pyspy or should_dump_cuda_coredump
+        if not self.crash_dump_folder and not should_dump_diagnostics:
             return
 
         if self.crash_dump_performed:
@@ -2415,69 +2424,96 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
         else:
             self.crash_dump_performed = True
 
-        logger.error(f"Dumping requests before crash. {self.crash_dump_folder=}")
+        # Dump requests info
+        if self.crash_dump_folder:
+            logger.error(f"Dumping requests before crash. {self.crash_dump_folder=}")
 
-        # Add finished requests from crash_dump_request_list
+            # Add finished requests from crash_dump_request_list
-        data_to_dump = []
+            data_to_dump = []
-        if self.crash_dump_request_list:
+            if self.crash_dump_request_list:
-            data_to_dump.extend(self.crash_dump_request_list)
+                data_to_dump.extend(self.crash_dump_request_list)
 
-        # Add unfinished requests from rid_to_state
+            # Add unfinished requests from rid_to_state
-        unfinished_requests = []
+            unfinished_requests = []
-        for rid, state in self.rid_to_state.items():
+            for rid, state in self.rid_to_state.items():
-            if not state.finished:
+                if not state.finished:
-                state.time_stats.set_finished_time()
+                    state.time_stats.set_finished_time()
-                unfinished_requests.append(
+                    unfinished_requests.append(
-                    (
-                        state.obj,
                         (
-                            state.out_list[-1]
+                            state.obj,
-                            if state.out_list
+                            (
-                            else state.get_crash_dump_output()
+                                state.out_list[-1]
-                        ),
+                                if state.out_list
-                        convert_time_to_realtime(state.time_stats.created_time),
+                                else state.get_crash_dump_output()
-                        convert_time_to_realtime(state.time_stats.finished_time),
+                            ),
+                            convert_time_to_realtime(state.time_stats.created_time),
+                            convert_time_to_realtime(state.time_stats.finished_time),
+                        )
                     )
+            if unfinished_requests:
+                data_to_dump.extend(unfinished_requests)
+
+            if data_to_dump:
+                # Create a file
+                filename = os.path.join(
+                    self.crash_dump_folder,
+                    hostname,
+                    f'crash_dump_{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}.pkl',
                 )
-        if unfinished_requests:
+                os.makedirs(os.path.dirname(filename), exist_ok=True)
-            data_to_dump.extend(unfinished_requests)
 
-        if not data_to_dump:
+                # Write the data to the file
-            return
+                data_to_dump_with_server_args = {
-
+                    "server_args": self.server_args,
-        # Create a file
+                    "requests": data_to_dump,
-        filename = os.path.join(
+                    "launch_command": " ".join(sys.argv),
-            self.crash_dump_folder,
+                }
-            hostname,
+                with open(filename, "wb") as f:
-            f'crash_dump_{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}.pkl',
+                    try:
-        )
+                        pickle.dump(data_to_dump_with_server_args, f)
-        os.makedirs(os.path.dirname(filename), exist_ok=True)
+                    except Exception as e:
-
+                        # When the server is launched with --trust-remote-code,
-        # Write the data to the file
+                        # server_args sometimes fails to pickle. Retry without
-        data_to_dump_with_server_args = {
+                        # server_args so the request data still gets persisted.
-            "server_args": self.server_args,  # Include server_args in the dump
+                        logger.error(
-            "requests": data_to_dump,
+                            f"Failed to pickle dump with server_args: {e!r}; "
-            "launch_command": " ".join(sys.argv),
+                            "retrying without server_args"
-        }
+                        )
-        with open(filename, "wb") as f:
+                        f.seek(0)
-            try:
+                        f.truncate()
-                pickle.dump(data_to_dump_with_server_args, f)
+                        data_to_dump_with_server_args["server_args"] = None
-            except Exception as e:
+                        pickle.dump(data_to_dump_with_server_args, f)
-                # When the server is launched with --trust-remote-code,
-                # server_args sometimes fails to pickle. Retry without
-                # server_args so the request data still gets persisted.
                 logger.error(
-                    f"Failed to pickle dump with server_args: {e!r}; "
+                    f"Dumped {len(self.crash_dump_request_list)} finished and {len(unfinished_requests)} unfinished requests before crash to {filename}"
-                    "retrying without server_args"
+                )
+
+        # Dump pyspy and cuda coredump
+        if should_dump_diagnostics:
+            logger.info(
+                "Sleeping 5 seconds before crash diagnostics to let GPU activity settle."
+            )
+            time.sleep(5)
+
+            scheduler_procs = collect_scheduler_processes()
+            if scheduler_procs:
+                if should_dump_pyspy:
+                    pyspy_dump_schedulers(scheduler_only=True)
+
+                if should_dump_cuda_coredump:
+                    trigger_cuda_user_coredump(scheduler_only=True)
+                    cuda_coredump_wait_secs = (
+                        envs.SGLANG_CUDA_COREDUMP_BEFORE_CRASH_WAIT_SECS.get()
+                    )
+                    if cuda_coredump_wait_secs > 0:
+                        logger.info(
+                            "Waiting %.1f seconds for CUDA coredumps before exiting.",
+                            cuda_coredump_wait_secs,
+                        )
+                        time.sleep(cuda_coredump_wait_secs)
+            else:
+                logger.error(
+                    "No live scheduler processes found; skipping py-spy and CUDA coredump."
                 )
-                f.seek(0)
-                f.truncate()
-                data_to_dump_with_server_args["server_args"] = None
-                pickle.dump(data_to_dump_with_server_args, f)
-        logger.error(
-            f"Dumped {len(self.crash_dump_request_list)} finished and {len(unfinished_requests)} unfinished requests before crash to {filename}"
-        )
-        return filename
 
     async def sigterm_watchdog(self):
         while not self.gracefully_exit:

python/sglang/srt/server_args.py

@@ -24,6 +24,7 @@ import json
 import logging
 import os
 import random
+import socket
 import tempfile
 import uuid
 from functools import cached_property
@@ -1034,6 +1035,9 @@ class ServerArgs:
         # Handle diffusion LLM inference.
         self._handle_dllm_inference()
 
+        # Handle crash dump environment variables (must run before CUDA init).
+        self._handle_crash_dump_env()
+
         # Handle debug utilities.
         self._handle_debug_utils()
 
@@ -4383,6 +4387,34 @@ class ServerArgs:
                     self.preferred_sampling_params
                 )
 
+    def _handle_crash_dump_env(self):
+        if not self.crash_dump_folder:
+            return
+        _CUDA_COREDUMP_DEFAULTS = {
+            "CUDA_ENABLE_COREDUMP_ON_EXCEPTION": "1",
+            "CUDA_ENABLE_USER_TRIGGERED_COREDUMP": "1",
+            "CUDA_COREDUMP_SHOW_PROGRESS": "1",
+            "CUDA_COREDUMP_GENERATION_FLAGS": (
+                "skip_nonrelocated_elf_images,skip_global_memory,"
+                "skip_shared_memory,skip_local_memory,skip_constbank_memory"
+            ),
+            "CUDA_COREDUMP_FILE": f"{self.crash_dump_folder}/%h/core.cuda.%t.%p",
+            "CUDA_COREDUMP_PIPE": "/tmp/corepipe.cuda.%h.%p",
+        }
+        for key, value in _CUDA_COREDUMP_DEFAULTS.items():
+            if key not in os.environ:
+                os.environ[key] = value
+                logger.info("Auto-set %s=%s (from --crash-dump-folder)", key, value)
+
+                if key == "CUDA_COREDUMP_FILE":
+                    # cuda curedump cannot write to a folder that does not exist,
+                    # so we have to create the folder first.
+                    hostname = socket.gethostname()
+                    os.makedirs(
+                        os.path.join(self.crash_dump_folder, hostname),
+                        exist_ok=True,
+                    )
+
     def _handle_debug_utils(self):
         if is_in_ci() and self.soft_watchdog_timeout is None:
             logger.info("Set soft_watchdog_timeout since in CI")

python/sglang/srt/utils/common.py

@@ -2515,22 +2515,6 @@ def human_readable_int(value: str) -> int:
         )
 
 
-def pyspy_dump_schedulers():
-    """py-spy dump on all scheduler in a local node."""
-    pid = psutil.Process().pid
-    for attempt, native_flag in enumerate(["--native", ""]):
-        try:
-            cmd = f"py-spy dump {native_flag} --pid {pid}".strip()
-            result = subprocess.run(
-                cmd, shell=True, capture_output=True, text=True, check=True
-            )
-            logger.error(f"Pyspy dump for PID {pid} ({cmd}):\n{result.stdout}")
-            return
-        except subprocess.CalledProcessError as e:
-            logger.error(f"Pyspy failed ({cmd}). Error: {e.stderr}")
-    logger.error(f"All pyspy dump attempts failed for PID {pid}.")
-
-
 def kill_itself_when_parent_died():
     if sys.platform == "linux":
         # sigkill this process when parent worker manager dies

python/sglang/srt/utils/cudacore_pyspy_dump_utils.py

@@ -0,0 +1,144 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""CUDA core dump and py-spy dump utilities."""
+
+from __future__ import annotations
+
+import logging
+import os
+import platform
+import subprocess
+import time
+from errno import ENXIO
+from pathlib import Path
+from typing import List
+
+import psutil
+
+logger = logging.getLogger(__name__)
+
+
+def _resolve_cuda_coredump_pipe_path(proc: psutil.Process) -> Path:
+    pipe_template = os.environ.get("CUDA_COREDUMP_PIPE")
+    if pipe_template is None:
+        pipe_path = f"corepipe.cuda.{platform.node()}.{proc.pid}"
+    else:
+        pipe_path = (
+            pipe_template.replace("%h", platform.node())
+            .replace("%p", str(proc.pid))
+            .replace("%t", str(int(time.time())))
+        )
+
+    path = Path(pipe_path)
+    if path.is_absolute():
+        return path
+
+    try:
+        return Path(proc.cwd()) / path
+    except (psutil.Error, OSError):
+        return Path.cwd() / path
+
+
+def _is_sglang_scheduler_process(proc: psutil.Process) -> bool:
+    try:
+        proc_title = " ".join(proc.cmdline())
+    except (psutil.Error, OSError):
+        return False
+    return proc_title.startswith("sglang::scheduler")
+
+
+def collect_scheduler_processes() -> List[psutil.Process]:
+    current = psutil.Process()
+    return [
+        proc
+        for proc in current.children(recursive=True)
+        if _is_sglang_scheduler_process(proc)
+    ]
+
+
+def pyspy_dump_schedulers(scheduler_only=False):
+    """py-spy dump on all scheduler in a local node."""
+    if scheduler_only:
+        procs = collect_scheduler_processes()
+        if not procs:
+            logger.error("No sglang scheduler processes found for py-spy dump.")
+            return
+        pids = [proc.pid for proc in procs]
+    else:
+        pids = [psutil.Process().pid]
+    for pid in pids:
+        for attempt, native_flag in enumerate(["--native", ""]):
+            try:
+                cmd = f"py-spy dump {native_flag} --pid {pid}".strip()
+                result = subprocess.run(
+                    cmd, shell=True, capture_output=True, text=True, check=True
+                )
+                logger.error(f"Pyspy dump for PID {pid} ({cmd}):\n{result.stdout}")
+                break
+            except subprocess.CalledProcessError as e:
+                logger.error(f"Pyspy failed ({cmd}). Error: {e.stderr}")
+                if attempt == 1:
+                    logger.error(f"All pyspy dump attempts failed for PID {pid}.")
+
+
+def trigger_cuda_user_coredump(scheduler_only=False):
+    """Trigger CUDA user-induced GPU core dumps by writing to coredump pipes."""
+    if os.environ.get("CUDA_ENABLE_USER_TRIGGERED_COREDUMP") != "1":
+        logger.error(
+            "CUDA user-triggered coredump is not enabled. Set "
+            "CUDA_ENABLE_USER_TRIGGERED_COREDUMP=1 before CUDA initialization."
+        )
+
+    if scheduler_only:
+        procs = collect_scheduler_processes()
+        if not procs:
+            logger.error("No sglang scheduler processes found for CUDA coredump.")
+            return
+    else:
+        procs = [psutil.Process()]
+
+    for proc in procs:
+        pipe_path = _resolve_cuda_coredump_pipe_path(proc)
+        try:
+            fd = os.open(pipe_path, os.O_WRONLY | os.O_NONBLOCK)
+            try:
+                os.write(fd, b"1")
+            finally:
+                os.close(fd)
+            logger.error(
+                "Triggered CUDA user coredump for PID %s via %s",
+                proc.pid,
+                pipe_path,
+            )
+        except FileNotFoundError:
+            logger.error(
+                "CUDA coredump pipe not found for PID %s: %s. Ensure "
+                "CUDA_ENABLE_USER_TRIGGERED_COREDUMP=1 was set before this "
+                "process initialized CUDA.",
+                proc.pid,
+                pipe_path,
+            )
+        except OSError as e:
+            if e.errno == ENXIO:
+                logger.error(
+                    "CUDA coredump pipe has no reader for PID %s: %s",
+                    proc.pid,
+                    pipe_path,
+                )
+            else:
+                logger.exception(
+                    "Failed to trigger CUDA user coredump for PID %s via %s",
+                    proc.pid,
+                    pipe_path,
+                )

python/sglang/srt/utils/watchdog.py

@@ -12,7 +12,7 @@ from typing import Callable, List, Optional
 
 import psutil
 
-from sglang.srt.utils.common import pyspy_dump_schedulers
+from sglang.srt.utils.cudacore_pyspy_dump_utils import pyspy_dump_schedulers
 
 logger = logging.getLogger(__name__)