From 6b6963f42646e1e8e19cd5b3f2c6e8c0636f1b87 Mon Sep 17 00:00:00 2001 From: YC Yen-Ching Tseng Date: Mon, 11 May 2026 18:47:09 +0800 Subject: [PATCH] [AMD] Update scripts/ci/amd/ensure_vram_clear.sh (#24586) --- .codespellrc | 2 +- scripts/ci/amd/ensure_vram_clear.sh | 272 ++++++++++++++++++++++------ 2 files changed, 220 insertions(+), 54 deletions(-) diff --git a/.codespellrc b/.codespellrc index b95d08495..ae46cf6ec 100644 --- a/.codespellrc +++ b/.codespellrc @@ -1,3 +1,3 @@ [codespell] -ignore-words-list = ans, als, hel, boostrap, childs, te, vas, hsa, ment, cann, thi, makro, wil, rouge, PRIS, ather, MIS, medias, allready, inout, nd, fo, visibles, nothink +ignore-words-list = ans, als, hel, boostrap, childs, te, vas, hsa, ment, cann, thi, makro, wil, rouge, PRIS, ather, MIS, medias, allready, inout, nd, fo, visibles, nothink, renderD skip = *.json, *.jsonl, *.patch, *.txt, *.lock diff --git a/scripts/ci/amd/ensure_vram_clear.sh b/scripts/ci/amd/ensure_vram_clear.sh index 0dd720960..1754bb7ab 100755 --- a/scripts/ci/amd/ensure_vram_clear.sh +++ b/scripts/ci/amd/ensure_vram_clear.sh @@ -4,15 +4,162 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/check_vram_clear.sh" +# Stop and remove every container that holds any /dev/kfd or /dev/dri device. +# Some failing CI runs leave behind containers other than `ci_sglang` (e.g. +# from previous AMD jobs that were force-killed mid-run); those still hold +# VRAM via KFD even though the host pgrep finds nothing. +stop_all_gpu_containers() { + if ! command -v docker >/dev/null 2>&1; then + return 0 + fi + + local all_ids + all_ids=$(docker ps -aq 2>/dev/null || true) + if [ -z "$all_ids" ]; then + echo "No docker containers found on host" + return 0 + fi + + local gpu_ids="" + local cid + for cid in $all_ids; do + # A container is "GPU-attached" if its inspect output mentions any + # GPU device or NVIDIA/ROCm GPU capability. Inspecting the raw JSON + # (instead of a specific field) survives docker version differences. + if docker inspect "$cid" 2>/dev/null \ + | grep -qE '"PathOnHost":"/dev/(kfd|dri)|"Capabilities":\[\["gpu"\]\]'; then + gpu_ids+=" $cid" + fi + done + + gpu_ids=$(echo "$gpu_ids" | tr ' ' '\n' | grep -E '^[a-f0-9]+$' || true) + if [ -z "$gpu_ids" ]; then + echo "No GPU-attached docker containers found on host" + return 0 + fi + + echo "Found GPU-attached containers, stopping them:" + for cid in $gpu_ids; do + docker ps -a --filter "id=$cid" --format ' {{.ID}} {{.Image}} {{.Status}} {{.Names}}' 2>/dev/null || true + done + echo "$gpu_ids" | xargs -r docker stop --time 5 2>/dev/null || true + echo "$gpu_ids" | xargs -r docker rm -f 2>/dev/null || true +} + +# Find and kill any host process that holds an open handle to /dev/kfd or +# /dev/dri/renderD*. This is far more reliable than `rocm-smi --showpids`, +# which only sees processes that registered a HSA queue (zombies and +# processes that crashed mid-init are invisible to it). +kill_processes_holding_gpu_devices() { + local signal=${1:-TERM} + local pids="" + + # If neither tool is present we silently degrade to a no-op, which used + # to look identical in the log to "no holders found" and made the + # script's failures very confusing. Emit a loud warning so the runner + # owner knows why GPU device cleanup isn't happening. + if ! command -v fuser >/dev/null 2>&1 && ! command -v lsof >/dev/null 2>&1; then + echo "WARNING: neither fuser nor lsof installed on the host;" \ + "cannot detect processes holding /dev/kfd or /dev/dri/renderD*." + echo " Install psmisc (for fuser) or lsof on the runner host" \ + "to enable device-fd-based cleanup." + return 0 + fi + + if command -v fuser >/dev/null 2>&1; then + # `fuser` prints PIDs to stdout, names to stderr; collect everything + # that has any handle on KFD or render nodes. + pids+=" $(fuser /dev/kfd 2>/dev/null || true)" + for dev in /dev/dri/renderD*; do + [ -e "$dev" ] || continue + pids+=" $(fuser "$dev" 2>/dev/null || true)" + done + fi + + if command -v lsof >/dev/null 2>&1; then + pids+=" $(lsof -t /dev/kfd 2>/dev/null || true)" + for dev in /dev/dri/renderD*; do + [ -e "$dev" ] || continue + pids+=" $(lsof -t "$dev" 2>/dev/null || true)" + done + fi + + pids=$(echo "$pids" | tr ' ' '\n' | grep -E '^[0-9]+$' | sort -u || true) + if [ -z "$pids" ]; then + return 0 + fi + + local self_pid=$$ + echo "Processes holding /dev/kfd or /dev/dri/renderD*:" + for pid in $pids; do + # Skip our own PID and any of our ancestors so we don't suicide. + if [ "$pid" = "$self_pid" ]; then + continue + fi + local cmd + cmd=$(ps -p "$pid" -o pid,ppid,stat,cmd --no-headers 2>/dev/null || true) + if [ -z "$cmd" ]; then + continue + fi + echo " $cmd" + # If it's a zombie, kill the parent instead — kill -9 on a zombie is + # a no-op, the only way to reap it is to make its parent reap it. + local stat + stat=$(ps -p "$pid" -o stat= 2>/dev/null | tr -d ' ') + if [[ "$stat" == Z* ]]; then + local ppid + ppid=$(ps -p "$pid" -o ppid= 2>/dev/null | tr -d ' ') + if [ -n "$ppid" ] && [ "$ppid" != "1" ] && [ "$ppid" != "$self_pid" ]; then + echo " -> $pid is a zombie, sending SIG$signal to parent $ppid" + kill "-$signal" "$ppid" 2>/dev/null || true + fi + continue + fi + kill "-$signal" "$pid" 2>/dev/null || true + done +} + +# Print rich diagnostics that explain *why* VRAM is still allocated when no +# obvious owner exists. Helpful when zombies / other namespaces hold memory. +dump_gpu_diagnostics() { + echo "=== GPU device file holders (fuser) ===" + if command -v fuser >/dev/null 2>&1; then + fuser -v /dev/kfd 2>&1 || true + for dev in /dev/dri/renderD* /dev/dri/card*; do + [ -e "$dev" ] || continue + fuser -v "$dev" 2>&1 || true + done + else + echo "fuser not installed" + fi + + echo "=== GPU device file holders (lsof) ===" + if command -v lsof >/dev/null 2>&1; then + lsof /dev/kfd 2>/dev/null || true + lsof /dev/dri/renderD* 2>/dev/null || true + else + echo "lsof not installed" + fi + + echo "=== Zombie processes on host ===" + ps -eo pid,ppid,stat,etime,cmd 2>/dev/null | awk 'NR==1 || $3 ~ /^Z/ {print}' + + echo "=== Docker containers on host ===" + if command -v docker >/dev/null 2>&1; then + docker ps -a --format 'table {{.ID}}\t{{.Image}}\t{{.Status}}\t{{.Names}}' 2>/dev/null || true + fi + + echo "=== rocm-smi --showpids ===" + timeout 30 rocm-smi --showpids 2>&1 || echo "rocm-smi --showpids timed out" + + echo "=== rocm-smi --showmemuse ===" + timeout 30 rocm-smi --showmemuse 2>&1 || echo "rocm-smi --showmemuse timed out" +} + ensure_vram_clear() { local max_retries=3 local retry_count=0 - # Stop and remove any existing ci_sglang container - echo "Stopping any existing ci_sglang container..." - docker stop ci_sglang || true - docker rm ci_sglang || true - # Log host information for debugging echo "=== Host Information ===" echo "Hostname: $(hostname)" @@ -22,76 +169,95 @@ ensure_vram_clear() { echo "========================" echo "Running in ROCm mode" + # Always stop the well-known CI container first (best-effort). + echo "Stopping any existing ci_sglang container..." + docker stop ci_sglang 2>/dev/null || true + docker rm -f ci_sglang 2>/dev/null || true + # Show initial GPU status echo "=== Initial GPU Memory Status ===" rocm-smi --showmemuse echo "==================================" + # Fast path: if the runner is already clean, skip the cleanup loop + # entirely so healthy jobs don't pay the ~35s/attempt cleanup cost. + if check_vram_clear; then + echo "✓ VRAM is already clear; skipping cleanup." + return 0 + fi + while [ $retry_count -lt $max_retries ]; do echo "=== Cleanup Attempt $((retry_count + 1))/$max_retries ===" - # Clean SGLang processes + # Step 1: kill SGLang-named processes on the host (cheap, fast). + # NOTE: host pgrep cannot see PIDs inside a container's PID + # namespace, so in CI this almost never matches anything; the + # heavy lifting is done by step 2 below. Kept as a fast early + # cleanup for the rare case where something runs on the host. echo "Killing SGLang processes..." - pgrep -f 'sglang::|sglang\.launch_server|sglang\.bench|sglang\.data_parallel|sglang\.srt' | xargs -r kill -9 || true + pgrep -f 'sglang::|sglang\.launch_server|sglang\.bench|sglang\.data_parallel|sglang\.srt' \ + | xargs -r kill -9 2>/dev/null || true - if [ $retry_count -gt 0 ]; then - echo "Performing aggressive cleanup..." - # Kill all processes using KFD - rocm-smi --showpids 2>/dev/null | grep 'PID:' | awk '{print $2}' | xargs -r kill -9 2>/dev/null || true - # Wait a bit for cleanup to take effect - echo "Waiting 30 seconds for VRAM to clear..." - sleep 30 - fi + # Step 2: aggressive cleanup. Run on EVERY attempt — the previous + # version skipped this on attempt 1, which made attempt 1 a near + # no-op for the most common failure mode (a leftover container + # holding VRAM, invisible to host pgrep). + echo "Performing aggressive cleanup..." - # Check VRAM + # 2a. Stop ALL GPU-attached containers, not just ci_sglang. A + # leftover container from a previous job will keep VRAM held even + # though `pgrep` on the host shows nothing. + stop_all_gpu_containers + + # 2b. SIGTERM anything that has /dev/kfd or /dev/dri/renderD* open. + # `lsof`/`fuser` see processes that `rocm-smi --showpids` misses + # (notably zombies and processes outside our PID namespace). + echo "Sending SIGTERM to processes holding GPU device files..." + kill_processes_holding_gpu_devices TERM + sleep 5 + + # 2c. SIGKILL anything still holding GPU device files. + echo "Sending SIGKILL to remaining holders..." + kill_processes_holding_gpu_devices KILL + + # 2d. Best-effort: also kill anything `rocm-smi --showpids` reports. + # Handles both the legacy "PID: " line format and the modern + # tabular format (`\t\t\t...`); the previous + # `grep 'PID:'` matched nothing on ROCm 5+ tabular output. + rocm-smi --showpids 2>/dev/null \ + | awk '/^PID:[[:space:]]*[0-9]+/ {print $2} /^[0-9]+/ {print $1}' \ + | xargs -r kill -9 2>/dev/null || true + + echo "Waiting 30 seconds for VRAM to clear..." + sleep 30 + + # Step 3: re-check. echo "Checking VRAM status..." if check_vram_clear; then echo "✓ VRAM cleanup successful after $((retry_count + 1)) attempts" return 0 else echo "✗ VRAM still not clear after attempt $((retry_count + 1))" + # Step 4: dump diagnostics on every failed attempt so the next + # attempt's logs already explain WHY cleanup didn't work. + # Without this we'd only see what's holding the GPU at the very + # end, which makes triage much harder. + echo "--- Diagnostics for failed attempt $((retry_count + 1)) ---" + dump_gpu_diagnostics + echo "--- End of diagnostics for attempt $((retry_count + 1)) ---" retry_count=$((retry_count + 1)) fi done - # Failed after all retries + # Failed after all retries — diagnostics for the last cleanup attempt + # were already dumped above; just print the actionable hint. echo "=== FAILED: VRAM cleanup unsuccessful after $max_retries attempts ===" - echo "Final GPU status:" - timeout 30 rocm-smi --showmemuse || echo "rocm-smi timed out" - echo "Processes using GPU:" - rocm-smi --showpids 2>/dev/null | grep -q 'PID:' || echo "No processes found using /dev/kfd" - - # Print detailed information about suspicious processes - echo "=== Detailed Process Information ===" - if command -v rocm-smi >/dev/null 2>&1; then - # For AMD GPUs, get processes from rocm-smi --showpids - kfd_pids=$(rocm-smi --showpids 2>/dev/null | grep 'PID:' | awk '{print $2}' | sort -u) - if [ -n "$kfd_pids" ]; then - echo "Processes accessing /dev/kfd (AMD GPU device):" - for pid in $kfd_pids; do - if ps -p $pid -o pid,ppid,cmd --no-headers 2>/dev/null; then - echo " └─ Command line: $(ps -p $pid -o cmd --no-headers 2>/dev/null | head -1)" - else - echo " └─ PID $pid: Process not found or already terminated" - fi - done - else - echo "No processes found accessing /dev/kfd" - fi - fi - - # Check for any remaining sglang-related processes - echo "Checking for any remaining sglang-related processes:" - sglang_procs=$(pgrep -f 'sglang::|sglang\.launch_server|sglang\.bench|sglang\.data_parallel|sglang\.srt' 2>/dev/null) - if [ -n "$sglang_procs" ]; then - echo "Found sglang processes still running:" - for pid in $sglang_procs; do - ps -p $pid -o pid,ppid,cmd --no-headers 2>/dev/null || echo "PID $pid not found" - done - else - echo "No sglang-related processes found." - fi - + echo "(See diagnostics above for the final attempt.)" + echo "==================================================================" + echo "Hint: if no host process / container holds the GPU but VRAM is" + echo "still allocated, this is almost certainly a zombie KFD context" + echo "(see ROCm/aiter#2061). The node will need to be rebooted before" + echo "subsequent jobs can succeed." echo "==================================================================" return 1 }