minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Trigger scheduler diagnostics on health failure (#26757)

Browse Source
This commit is contained in:
Yinghai Lu
2026-06-03 17:19:55 -07:00
committed by GitHub
parent d097cd2212
commit 7bb5c96685
6 changed files with 275 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/environ.py
+5
View File
@@ -630,6 +630,11 @@ class Envs:
# Health Check
SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION = EnvBool(True)
# Crash diagnostics
SGLANG_PYSPY_DUMP_BEFORE_CRASH = EnvBool(True)
SGLANG_CUDA_COREDUMP_BEFORE_CRASH = EnvBool(True)
SGLANG_CUDA_COREDUMP_BEFORE_CRASH_WAIT_SECS = EnvFloat(60.0)
# Encoder gRPC
SGLANG_ENCODER_GRPC_TIMEOUT_SECS = EnvInt(60)
# Encoder receiver selection: http|grpc (used by EPD paths).
python/sglang/srt/managers/tokenizer_manager.py
+93 -57
View File
@@ -26,6 +26,7 @@ import signal
import socket
import sys
import threading
import time
from array import array
from collections import deque
from contextlib import nullcontext
@@ -113,6 +114,11 @@ from sglang.srt.utils import (
kill_process_tree,
)
from sglang.srt.utils.aio_rwlock import RWLock
from sglang.srt.utils.cudacore_pyspy_dump_utils import (
collect_scheduler_processes,
pyspy_dump_schedulers,
trigger_cuda_user_coredump,
)
from sglang.srt.utils.hf_transformers_utils import (
get_processor,
get_tokenizer,
@@ -554,7 +560,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
if isinstance(obj, GenerateReqInput) and obj.routed_dp_rank is not None:
dp_size = self.server_args.dp_size
if dp_size <= 1 and obj.routed_dp_rank == 0:
logger.warning(
logger.debug(
f"routed_dp_rank={obj.routed_dp_rank} is ignored because dp_size={dp_size}"
)
elif obj.routed_dp_rank < 0 or obj.routed_dp_rank >= dp_size:
@@ -2404,7 +2410,10 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
def dump_requests_before_crash(
self, hostname: str = os.getenv("HOSTNAME", socket.gethostname())
):
if not self.crash_dump_folder:
should_dump_pyspy = envs.SGLANG_PYSPY_DUMP_BEFORE_CRASH.get()
should_dump_cuda_coredump = envs.SGLANG_CUDA_COREDUMP_BEFORE_CRASH.get()
should_dump_diagnostics = should_dump_pyspy or should_dump_cuda_coredump
if not self.crash_dump_folder and not should_dump_diagnostics:
return
if self.crash_dump_performed:
@@ -2415,69 +2424,96 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
else:
self.crash_dump_performed = True
logger.error(f"Dumping requests before crash. {self.crash_dump_folder=}")
# Dump requests info
if self.crash_dump_folder:
logger.error(f"Dumping requests before crash. {self.crash_dump_folder=}")
# Add finished requests from crash_dump_request_list
data_to_dump = []
if self.crash_dump_request_list:
data_to_dump.extend(self.crash_dump_request_list)
# Add finished requests from crash_dump_request_list
data_to_dump = []
if self.crash_dump_request_list:
data_to_dump.extend(self.crash_dump_request_list)
# Add unfinished requests from rid_to_state
unfinished_requests = []
for rid, state in self.rid_to_state.items():
if not state.finished:
state.time_stats.set_finished_time()
unfinished_requests.append(
(
state.obj,
# Add unfinished requests from rid_to_state
unfinished_requests = []
for rid, state in self.rid_to_state.items():
if not state.finished:
state.time_stats.set_finished_time()
unfinished_requests.append(
(
state.out_list[-1]
if state.out_list
else state.get_crash_dump_output()
),
convert_time_to_realtime(state.time_stats.created_time),
convert_time_to_realtime(state.time_stats.finished_time),
state.obj,
(
state.out_list[-1]
if state.out_list
else state.get_crash_dump_output()
),
convert_time_to_realtime(state.time_stats.created_time),
convert_time_to_realtime(state.time_stats.finished_time),
)
)
if unfinished_requests:
data_to_dump.extend(unfinished_requests)
if data_to_dump:
# Create a file
filename = os.path.join(
self.crash_dump_folder,
hostname,
f'crash_dump_{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}.pkl',
)
if unfinished_requests:
data_to_dump.extend(unfinished_requests)
os.makedirs(os.path.dirname(filename), exist_ok=True)
if not data_to_dump:
return
# Create a file
filename = os.path.join(
self.crash_dump_folder,
hostname,
f'crash_dump_{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}.pkl',
)
os.makedirs(os.path.dirname(filename), exist_ok=True)
# Write the data to the file
data_to_dump_with_server_args = {
"server_args": self.server_args, # Include server_args in the dump
"requests": data_to_dump,
"launch_command": " ".join(sys.argv),
}
with open(filename, "wb") as f:
try:
pickle.dump(data_to_dump_with_server_args, f)
except Exception as e:
# When the server is launched with --trust-remote-code,
# server_args sometimes fails to pickle. Retry without
# server_args so the request data still gets persisted.
# Write the data to the file
data_to_dump_with_server_args = {
"server_args": self.server_args,
"requests": data_to_dump,
"launch_command": " ".join(sys.argv),
}
with open(filename, "wb") as f:
try:
pickle.dump(data_to_dump_with_server_args, f)
except Exception as e:
# When the server is launched with --trust-remote-code,
# server_args sometimes fails to pickle. Retry without
# server_args so the request data still gets persisted.
logger.error(
f"Failed to pickle dump with server_args: {e!r}; "
"retrying without server_args"
)
f.seek(0)
f.truncate()
data_to_dump_with_server_args["server_args"] = None
pickle.dump(data_to_dump_with_server_args, f)
logger.error(
f"Failed to pickle dump with server_args: {e!r}; "
"retrying without server_args"
f"Dumped {len(self.crash_dump_request_list)} finished and {len(unfinished_requests)} unfinished requests before crash to {filename}"
)
# Dump pyspy and cuda coredump
if should_dump_diagnostics:
logger.info(
"Sleeping 5 seconds before crash diagnostics to let GPU activity settle."
)
time.sleep(5)
scheduler_procs = collect_scheduler_processes()
if scheduler_procs:
if should_dump_pyspy:
pyspy_dump_schedulers(scheduler_only=True)
if should_dump_cuda_coredump:
trigger_cuda_user_coredump(scheduler_only=True)
cuda_coredump_wait_secs = (
envs.SGLANG_CUDA_COREDUMP_BEFORE_CRASH_WAIT_SECS.get()
)
if cuda_coredump_wait_secs > 0:
logger.info(
"Waiting %.1f seconds for CUDA coredumps before exiting.",
cuda_coredump_wait_secs,
)
time.sleep(cuda_coredump_wait_secs)
else:
logger.error(
"No live scheduler processes found; skipping py-spy and CUDA coredump."
)
f.seek(0)
f.truncate()
data_to_dump_with_server_args["server_args"] = None
pickle.dump(data_to_dump_with_server_args, f)
logger.error(
f"Dumped {len(self.crash_dump_request_list)} finished and {len(unfinished_requests)} unfinished requests before crash to {filename}"
)
return filename
async def sigterm_watchdog(self):
while not self.gracefully_exit:
python/sglang/srt/server_args.py
+32
View File
@@ -24,6 +24,7 @@ import json
import logging
import os
import random
import socket
import tempfile
import uuid
from functools import cached_property
@@ -1034,6 +1035,9 @@ class ServerArgs:
# Handle diffusion LLM inference.
self._handle_dllm_inference()
# Handle crash dump environment variables (must run before CUDA init).
self._handle_crash_dump_env()
# Handle debug utilities.
self._handle_debug_utils()
@@ -4383,6 +4387,34 @@ class ServerArgs:
self.preferred_sampling_params
)
def _handle_crash_dump_env(self):
if not self.crash_dump_folder:
return
_CUDA_COREDUMP_DEFAULTS = {
"CUDA_ENABLE_COREDUMP_ON_EXCEPTION": "1",
"CUDA_ENABLE_USER_TRIGGERED_COREDUMP": "1",
"CUDA_COREDUMP_SHOW_PROGRESS": "1",
"CUDA_COREDUMP_GENERATION_FLAGS": (
"skip_nonrelocated_elf_images,skip_global_memory,"
"skip_shared_memory,skip_local_memory,skip_constbank_memory"
),
"CUDA_COREDUMP_FILE": f"{self.crash_dump_folder}/%h/core.cuda.%t.%p",
"CUDA_COREDUMP_PIPE": "/tmp/corepipe.cuda.%h.%p",
}
for key, value in _CUDA_COREDUMP_DEFAULTS.items():
if key not in os.environ:
os.environ[key] = value
logger.info("Auto-set %s=%s (from --crash-dump-folder)", key, value)
if key == "CUDA_COREDUMP_FILE":
# cuda curedump cannot write to a folder that does not exist,
# so we have to create the folder first.
hostname = socket.gethostname()
os.makedirs(
os.path.join(self.crash_dump_folder, hostname),
exist_ok=True,
)
def _handle_debug_utils(self):
if is_in_ci() and self.soft_watchdog_timeout is None:
logger.info("Set soft_watchdog_timeout since in CI")
python/sglang/srt/utils/common.py
-16
View File
@@ -2515,22 +2515,6 @@ def human_readable_int(value: str) -> int:
)
def pyspy_dump_schedulers():
"""py-spy dump on all scheduler in a local node."""
pid = psutil.Process().pid
for attempt, native_flag in enumerate(["--native", ""]):
try:
cmd = f"py-spy dump {native_flag} --pid {pid}".strip()
result = subprocess.run(
cmd, shell=True, capture_output=True, text=True, check=True
)
logger.error(f"Pyspy dump for PID {pid} ({cmd}):\n{result.stdout}")
return
except subprocess.CalledProcessError as e:
logger.error(f"Pyspy failed ({cmd}). Error: {e.stderr}")
logger.error(f"All pyspy dump attempts failed for PID {pid}.")
def kill_itself_when_parent_died():
if sys.platform == "linux":
# sigkill this process when parent worker manager dies
python/sglang/srt/utils/cudacore_pyspy_dump_utils.py
+144
View File
@@ -0,0 +1,144 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""CUDA core dump and py-spy dump utilities."""
from __future__ import annotations
import logging
import os
import platform
import subprocess
import time
from errno import ENXIO
from pathlib import Path
from typing import List
import psutil
logger = logging.getLogger(__name__)
def _resolve_cuda_coredump_pipe_path(proc: psutil.Process) -> Path:
pipe_template = os.environ.get("CUDA_COREDUMP_PIPE")
if pipe_template is None:
pipe_path = f"corepipe.cuda.{platform.node()}.{proc.pid}"
else:
pipe_path = (
pipe_template.replace("%h", platform.node())
.replace("%p", str(proc.pid))
.replace("%t", str(int(time.time())))
)
path = Path(pipe_path)
if path.is_absolute():
return path
try:
return Path(proc.cwd()) / path
except (psutil.Error, OSError):
return Path.cwd() / path
def _is_sglang_scheduler_process(proc: psutil.Process) -> bool:
try:
proc_title = " ".join(proc.cmdline())
except (psutil.Error, OSError):
return False
return proc_title.startswith("sglang::scheduler")
def collect_scheduler_processes() -> List[psutil.Process]:
current = psutil.Process()
return [
proc
for proc in current.children(recursive=True)
if _is_sglang_scheduler_process(proc)
]
def pyspy_dump_schedulers(scheduler_only=False):
"""py-spy dump on all scheduler in a local node."""
if scheduler_only:
procs = collect_scheduler_processes()
if not procs:
logger.error("No sglang scheduler processes found for py-spy dump.")
return
pids = [proc.pid for proc in procs]
else:
pids = [psutil.Process().pid]
for pid in pids:
for attempt, native_flag in enumerate(["--native", ""]):
try:
cmd = f"py-spy dump {native_flag} --pid {pid}".strip()
result = subprocess.run(
cmd, shell=True, capture_output=True, text=True, check=True
)
logger.error(f"Pyspy dump for PID {pid} ({cmd}):\n{result.stdout}")
break
except subprocess.CalledProcessError as e:
logger.error(f"Pyspy failed ({cmd}). Error: {e.stderr}")
if attempt == 1:
logger.error(f"All pyspy dump attempts failed for PID {pid}.")
def trigger_cuda_user_coredump(scheduler_only=False):
"""Trigger CUDA user-induced GPU core dumps by writing to coredump pipes."""
if os.environ.get("CUDA_ENABLE_USER_TRIGGERED_COREDUMP") != "1":
logger.error(
"CUDA user-triggered coredump is not enabled. Set "
"CUDA_ENABLE_USER_TRIGGERED_COREDUMP=1 before CUDA initialization."
)
if scheduler_only:
procs = collect_scheduler_processes()
if not procs:
logger.error("No sglang scheduler processes found for CUDA coredump.")
return
else:
procs = [psutil.Process()]
for proc in procs:
pipe_path = _resolve_cuda_coredump_pipe_path(proc)
try:
fd = os.open(pipe_path, os.O_WRONLY | os.O_NONBLOCK)
try:
os.write(fd, b"1")
finally:
os.close(fd)
logger.error(
"Triggered CUDA user coredump for PID %s via %s",
proc.pid,
pipe_path,
)
except FileNotFoundError:
logger.error(
"CUDA coredump pipe not found for PID %s: %s. Ensure "
"CUDA_ENABLE_USER_TRIGGERED_COREDUMP=1 was set before this "
"process initialized CUDA.",
proc.pid,
pipe_path,
)
except OSError as e:
if e.errno == ENXIO:
logger.error(
"CUDA coredump pipe has no reader for PID %s: %s",
proc.pid,
pipe_path,
)
else:
logger.exception(
"Failed to trigger CUDA user coredump for PID %s via %s",
proc.pid,
pipe_path,
)
python/sglang/srt/utils/watchdog.py
+1 -1
View File
@@ -12,7 +12,7 @@ from typing import Callable, List, Optional
import psutil
from sglang.srt.utils.common import pyspy_dump_schedulers
from sglang.srt.utils.cudacore_pyspy_dump_utils import pyspy_dump_schedulers
logger = logging.getLogger(__name__)