[CI] Isolate CUDA coredump dir per run to fix tracker mis-attribution (#27344)

This commit is contained in:
Liangsheng Yin
2026-06-05 21:31:22 -04:00
committed by GitHub
parent 3030119ef7
commit 58a05d3dd2
3 changed files with 40 additions and 5 deletions
@@ -29,7 +29,23 @@ runs:
id: check
shell: bash
run: |
dir="${SGLANG_CUDA_COREDUMP_DIR:-/tmp/sglang_cuda_coredumps}"
# Mirror get_dump_dir() in python/sglang/srt/debug_utils/cuda_coredump.py:
# explicit override > per-job RUNNER_TEMP > /tmp default, then a
# per-(run, attempt) subdir so dumps are never mis-attributed across CI
# jobs that share a self-hosted runner's filesystem.
if [ -n "$SGLANG_CUDA_COREDUMP_DIR" ]; then
base="$SGLANG_CUDA_COREDUMP_DIR"
elif [ -n "$RUNNER_TEMP" ]; then
base="$RUNNER_TEMP/sglang_cuda_coredumps"
else
base="/tmp/sglang_cuda_coredumps"
fi
if [ -n "$GITHUB_RUN_ID" ]; then
dir="$base/${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT:-1}"
else
dir="$base"
fi
echo "dump_dir=$dir" >> "$GITHUB_OUTPUT"
if [ -d "$dir" ] && [ -n "$(ls -A "$dir" 2>/dev/null)" ]; then
echo "has_dumps=true" >> "$GITHUB_OUTPUT"
else
@@ -41,7 +57,7 @@ runs:
uses: actions/upload-artifact@v4
with:
name: cuda-coredumps-${{ github.job }}${{ inputs.artifact-suffix && format('-{0}', inputs.artifact-suffix) }}
path: ${{ env.SGLANG_CUDA_COREDUMP_DIR || '/tmp/sglang_cuda_coredumps' }}/
path: ${{ steps.check.outputs.dump_dir }}/
retention-days: ${{ inputs.retention-days }}
- name: Signal coredump to tracker issue
@@ -82,4 +98,4 @@ runs:
- name: Cleanup CUDA coredumps
shell: bash
run: rm -rf "${{ env.SGLANG_CUDA_COREDUMP_DIR || '/tmp/sglang_cuda_coredumps' }}"
run: rm -rf "${{ steps.check.outputs.dump_dir }}"
+18 -1
View File
@@ -29,7 +29,24 @@ def is_enabled() -> bool:
def get_dump_dir() -> str:
return envs.SGLANG_CUDA_COREDUMP_DIR.get()
# Resolve the base dir the same way as the uploader
# (.github/actions/upload-cuda-coredumps/action.yml) so they agree; an empty
# SGLANG_CUDA_COREDUMP_DIR counts as unset, like the action's `[ -n ... ]`.
explicit = envs.SGLANG_CUDA_COREDUMP_DIR.get()
runner_temp = os.getenv("RUNNER_TEMP")
if explicit:
base = explicit
elif runner_temp:
base = os.path.join(runner_temp, "sglang_cuda_coredumps")
else:
base = "/tmp/sglang_cuda_coredumps"
# Isolate dumps per (run, attempt): on a shared self-hosted runner a leftover
# dump from one job must not be picked up and mis-attributed by a later one.
run_id = os.getenv("GITHUB_RUN_ID")
if run_id:
attempt = os.getenv("GITHUB_RUN_ATTEMPT", "1")
return os.path.join(base, f"{run_id}-{attempt}")
return base
def _inject_env():
+3 -1
View File
@@ -221,7 +221,9 @@ class Envs:
SGLANG_IS_IN_CI = EnvBool(False)
SGLANG_IS_IN_CI_AMD = EnvBool(False)
SGLANG_CUDA_COREDUMP = EnvBool(False)
SGLANG_CUDA_COREDUMP_DIR = EnvStr("/tmp/sglang_cuda_coredumps")
# None = unset, letting get_dump_dir() resolve the base (RUNNER_TEMP in CI,
# else /tmp); see debug_utils/cuda_coredump.py.
SGLANG_CUDA_COREDUMP_DIR = EnvStr(None)
SGLANG_TEST_MAX_RETRY = EnvInt(None)
# Constrained Decoding (Grammar)