diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 815490dc9..82ec4ab7d 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -630,6 +630,11 @@ class Envs: # Health Check SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION = EnvBool(True) + # Crash diagnostics + SGLANG_PYSPY_DUMP_BEFORE_CRASH = EnvBool(True) + SGLANG_CUDA_COREDUMP_BEFORE_CRASH = EnvBool(True) + SGLANG_CUDA_COREDUMP_BEFORE_CRASH_WAIT_SECS = EnvFloat(60.0) + # Encoder gRPC SGLANG_ENCODER_GRPC_TIMEOUT_SECS = EnvInt(60) # Encoder receiver selection: http|grpc (used by EPD paths). diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index 2168f053c..580f0725f 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -26,6 +26,7 @@ import signal import socket import sys import threading +import time from array import array from collections import deque from contextlib import nullcontext @@ -113,6 +114,11 @@ from sglang.srt.utils import ( kill_process_tree, ) from sglang.srt.utils.aio_rwlock import RWLock +from sglang.srt.utils.cudacore_pyspy_dump_utils import ( + collect_scheduler_processes, + pyspy_dump_schedulers, + trigger_cuda_user_coredump, +) from sglang.srt.utils.hf_transformers_utils import ( get_processor, get_tokenizer, @@ -554,7 +560,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): if isinstance(obj, GenerateReqInput) and obj.routed_dp_rank is not None: dp_size = self.server_args.dp_size if dp_size <= 1 and obj.routed_dp_rank == 0: - logger.warning( + logger.debug( f"routed_dp_rank={obj.routed_dp_rank} is ignored because dp_size={dp_size}" ) elif obj.routed_dp_rank < 0 or obj.routed_dp_rank >= dp_size: @@ -2404,7 +2410,10 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): def dump_requests_before_crash( self, hostname: str = os.getenv("HOSTNAME", socket.gethostname()) ): - if not self.crash_dump_folder: + should_dump_pyspy = envs.SGLANG_PYSPY_DUMP_BEFORE_CRASH.get() + should_dump_cuda_coredump = envs.SGLANG_CUDA_COREDUMP_BEFORE_CRASH.get() + should_dump_diagnostics = should_dump_pyspy or should_dump_cuda_coredump + if not self.crash_dump_folder and not should_dump_diagnostics: return if self.crash_dump_performed: @@ -2415,69 +2424,96 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): else: self.crash_dump_performed = True - logger.error(f"Dumping requests before crash. {self.crash_dump_folder=}") + # Dump requests info + if self.crash_dump_folder: + logger.error(f"Dumping requests before crash. {self.crash_dump_folder=}") - # Add finished requests from crash_dump_request_list - data_to_dump = [] - if self.crash_dump_request_list: - data_to_dump.extend(self.crash_dump_request_list) + # Add finished requests from crash_dump_request_list + data_to_dump = [] + if self.crash_dump_request_list: + data_to_dump.extend(self.crash_dump_request_list) - # Add unfinished requests from rid_to_state - unfinished_requests = [] - for rid, state in self.rid_to_state.items(): - if not state.finished: - state.time_stats.set_finished_time() - unfinished_requests.append( - ( - state.obj, + # Add unfinished requests from rid_to_state + unfinished_requests = [] + for rid, state in self.rid_to_state.items(): + if not state.finished: + state.time_stats.set_finished_time() + unfinished_requests.append( ( - state.out_list[-1] - if state.out_list - else state.get_crash_dump_output() - ), - convert_time_to_realtime(state.time_stats.created_time), - convert_time_to_realtime(state.time_stats.finished_time), + state.obj, + ( + state.out_list[-1] + if state.out_list + else state.get_crash_dump_output() + ), + convert_time_to_realtime(state.time_stats.created_time), + convert_time_to_realtime(state.time_stats.finished_time), + ) ) + if unfinished_requests: + data_to_dump.extend(unfinished_requests) + + if data_to_dump: + # Create a file + filename = os.path.join( + self.crash_dump_folder, + hostname, + f'crash_dump_{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}.pkl', ) - if unfinished_requests: - data_to_dump.extend(unfinished_requests) + os.makedirs(os.path.dirname(filename), exist_ok=True) - if not data_to_dump: - return - - # Create a file - filename = os.path.join( - self.crash_dump_folder, - hostname, - f'crash_dump_{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}.pkl', - ) - os.makedirs(os.path.dirname(filename), exist_ok=True) - - # Write the data to the file - data_to_dump_with_server_args = { - "server_args": self.server_args, # Include server_args in the dump - "requests": data_to_dump, - "launch_command": " ".join(sys.argv), - } - with open(filename, "wb") as f: - try: - pickle.dump(data_to_dump_with_server_args, f) - except Exception as e: - # When the server is launched with --trust-remote-code, - # server_args sometimes fails to pickle. Retry without - # server_args so the request data still gets persisted. + # Write the data to the file + data_to_dump_with_server_args = { + "server_args": self.server_args, + "requests": data_to_dump, + "launch_command": " ".join(sys.argv), + } + with open(filename, "wb") as f: + try: + pickle.dump(data_to_dump_with_server_args, f) + except Exception as e: + # When the server is launched with --trust-remote-code, + # server_args sometimes fails to pickle. Retry without + # server_args so the request data still gets persisted. + logger.error( + f"Failed to pickle dump with server_args: {e!r}; " + "retrying without server_args" + ) + f.seek(0) + f.truncate() + data_to_dump_with_server_args["server_args"] = None + pickle.dump(data_to_dump_with_server_args, f) logger.error( - f"Failed to pickle dump with server_args: {e!r}; " - "retrying without server_args" + f"Dumped {len(self.crash_dump_request_list)} finished and {len(unfinished_requests)} unfinished requests before crash to {filename}" + ) + + # Dump pyspy and cuda coredump + if should_dump_diagnostics: + logger.info( + "Sleeping 5 seconds before crash diagnostics to let GPU activity settle." + ) + time.sleep(5) + + scheduler_procs = collect_scheduler_processes() + if scheduler_procs: + if should_dump_pyspy: + pyspy_dump_schedulers(scheduler_only=True) + + if should_dump_cuda_coredump: + trigger_cuda_user_coredump(scheduler_only=True) + cuda_coredump_wait_secs = ( + envs.SGLANG_CUDA_COREDUMP_BEFORE_CRASH_WAIT_SECS.get() + ) + if cuda_coredump_wait_secs > 0: + logger.info( + "Waiting %.1f seconds for CUDA coredumps before exiting.", + cuda_coredump_wait_secs, + ) + time.sleep(cuda_coredump_wait_secs) + else: + logger.error( + "No live scheduler processes found; skipping py-spy and CUDA coredump." ) - f.seek(0) - f.truncate() - data_to_dump_with_server_args["server_args"] = None - pickle.dump(data_to_dump_with_server_args, f) - logger.error( - f"Dumped {len(self.crash_dump_request_list)} finished and {len(unfinished_requests)} unfinished requests before crash to {filename}" - ) - return filename async def sigterm_watchdog(self): while not self.gracefully_exit: diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index f27612cc2..010a4792e 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -24,6 +24,7 @@ import json import logging import os import random +import socket import tempfile import uuid from functools import cached_property @@ -1034,6 +1035,9 @@ class ServerArgs: # Handle diffusion LLM inference. self._handle_dllm_inference() + # Handle crash dump environment variables (must run before CUDA init). + self._handle_crash_dump_env() + # Handle debug utilities. self._handle_debug_utils() @@ -4383,6 +4387,34 @@ class ServerArgs: self.preferred_sampling_params ) + def _handle_crash_dump_env(self): + if not self.crash_dump_folder: + return + _CUDA_COREDUMP_DEFAULTS = { + "CUDA_ENABLE_COREDUMP_ON_EXCEPTION": "1", + "CUDA_ENABLE_USER_TRIGGERED_COREDUMP": "1", + "CUDA_COREDUMP_SHOW_PROGRESS": "1", + "CUDA_COREDUMP_GENERATION_FLAGS": ( + "skip_nonrelocated_elf_images,skip_global_memory," + "skip_shared_memory,skip_local_memory,skip_constbank_memory" + ), + "CUDA_COREDUMP_FILE": f"{self.crash_dump_folder}/%h/core.cuda.%t.%p", + "CUDA_COREDUMP_PIPE": "/tmp/corepipe.cuda.%h.%p", + } + for key, value in _CUDA_COREDUMP_DEFAULTS.items(): + if key not in os.environ: + os.environ[key] = value + logger.info("Auto-set %s=%s (from --crash-dump-folder)", key, value) + + if key == "CUDA_COREDUMP_FILE": + # cuda curedump cannot write to a folder that does not exist, + # so we have to create the folder first. + hostname = socket.gethostname() + os.makedirs( + os.path.join(self.crash_dump_folder, hostname), + exist_ok=True, + ) + def _handle_debug_utils(self): if is_in_ci() and self.soft_watchdog_timeout is None: logger.info("Set soft_watchdog_timeout since in CI") diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index 5e6fea055..fad3e7764 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -2515,22 +2515,6 @@ def human_readable_int(value: str) -> int: ) -def pyspy_dump_schedulers(): - """py-spy dump on all scheduler in a local node.""" - pid = psutil.Process().pid - for attempt, native_flag in enumerate(["--native", ""]): - try: - cmd = f"py-spy dump {native_flag} --pid {pid}".strip() - result = subprocess.run( - cmd, shell=True, capture_output=True, text=True, check=True - ) - logger.error(f"Pyspy dump for PID {pid} ({cmd}):\n{result.stdout}") - return - except subprocess.CalledProcessError as e: - logger.error(f"Pyspy failed ({cmd}). Error: {e.stderr}") - logger.error(f"All pyspy dump attempts failed for PID {pid}.") - - def kill_itself_when_parent_died(): if sys.platform == "linux": # sigkill this process when parent worker manager dies diff --git a/python/sglang/srt/utils/cudacore_pyspy_dump_utils.py b/python/sglang/srt/utils/cudacore_pyspy_dump_utils.py new file mode 100644 index 000000000..ea31a8166 --- /dev/null +++ b/python/sglang/srt/utils/cudacore_pyspy_dump_utils.py @@ -0,0 +1,144 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""CUDA core dump and py-spy dump utilities.""" + +from __future__ import annotations + +import logging +import os +import platform +import subprocess +import time +from errno import ENXIO +from pathlib import Path +from typing import List + +import psutil + +logger = logging.getLogger(__name__) + + +def _resolve_cuda_coredump_pipe_path(proc: psutil.Process) -> Path: + pipe_template = os.environ.get("CUDA_COREDUMP_PIPE") + if pipe_template is None: + pipe_path = f"corepipe.cuda.{platform.node()}.{proc.pid}" + else: + pipe_path = ( + pipe_template.replace("%h", platform.node()) + .replace("%p", str(proc.pid)) + .replace("%t", str(int(time.time()))) + ) + + path = Path(pipe_path) + if path.is_absolute(): + return path + + try: + return Path(proc.cwd()) / path + except (psutil.Error, OSError): + return Path.cwd() / path + + +def _is_sglang_scheduler_process(proc: psutil.Process) -> bool: + try: + proc_title = " ".join(proc.cmdline()) + except (psutil.Error, OSError): + return False + return proc_title.startswith("sglang::scheduler") + + +def collect_scheduler_processes() -> List[psutil.Process]: + current = psutil.Process() + return [ + proc + for proc in current.children(recursive=True) + if _is_sglang_scheduler_process(proc) + ] + + +def pyspy_dump_schedulers(scheduler_only=False): + """py-spy dump on all scheduler in a local node.""" + if scheduler_only: + procs = collect_scheduler_processes() + if not procs: + logger.error("No sglang scheduler processes found for py-spy dump.") + return + pids = [proc.pid for proc in procs] + else: + pids = [psutil.Process().pid] + for pid in pids: + for attempt, native_flag in enumerate(["--native", ""]): + try: + cmd = f"py-spy dump {native_flag} --pid {pid}".strip() + result = subprocess.run( + cmd, shell=True, capture_output=True, text=True, check=True + ) + logger.error(f"Pyspy dump for PID {pid} ({cmd}):\n{result.stdout}") + break + except subprocess.CalledProcessError as e: + logger.error(f"Pyspy failed ({cmd}). Error: {e.stderr}") + if attempt == 1: + logger.error(f"All pyspy dump attempts failed for PID {pid}.") + + +def trigger_cuda_user_coredump(scheduler_only=False): + """Trigger CUDA user-induced GPU core dumps by writing to coredump pipes.""" + if os.environ.get("CUDA_ENABLE_USER_TRIGGERED_COREDUMP") != "1": + logger.error( + "CUDA user-triggered coredump is not enabled. Set " + "CUDA_ENABLE_USER_TRIGGERED_COREDUMP=1 before CUDA initialization." + ) + + if scheduler_only: + procs = collect_scheduler_processes() + if not procs: + logger.error("No sglang scheduler processes found for CUDA coredump.") + return + else: + procs = [psutil.Process()] + + for proc in procs: + pipe_path = _resolve_cuda_coredump_pipe_path(proc) + try: + fd = os.open(pipe_path, os.O_WRONLY | os.O_NONBLOCK) + try: + os.write(fd, b"1") + finally: + os.close(fd) + logger.error( + "Triggered CUDA user coredump for PID %s via %s", + proc.pid, + pipe_path, + ) + except FileNotFoundError: + logger.error( + "CUDA coredump pipe not found for PID %s: %s. Ensure " + "CUDA_ENABLE_USER_TRIGGERED_COREDUMP=1 was set before this " + "process initialized CUDA.", + proc.pid, + pipe_path, + ) + except OSError as e: + if e.errno == ENXIO: + logger.error( + "CUDA coredump pipe has no reader for PID %s: %s", + proc.pid, + pipe_path, + ) + else: + logger.exception( + "Failed to trigger CUDA user coredump for PID %s via %s", + proc.pid, + pipe_path, + ) diff --git a/python/sglang/srt/utils/watchdog.py b/python/sglang/srt/utils/watchdog.py index 092ff887f..7c774e758 100644 --- a/python/sglang/srt/utils/watchdog.py +++ b/python/sglang/srt/utils/watchdog.py @@ -12,7 +12,7 @@ from typing import Callable, List, Optional import psutil -from sglang.srt.utils.common import pyspy_dump_schedulers +from sglang.srt.utils.cudacore_pyspy_dump_utils import pyspy_dump_schedulers logger = logging.getLogger(__name__)