[CI] Isolate CUDA coredump dir per run to fix tracker mis-attribution (#27344)
This commit is contained in:
@@ -29,7 +29,23 @@ runs:
|
||||
id: check
|
||||
shell: bash
|
||||
run: |
|
||||
dir="${SGLANG_CUDA_COREDUMP_DIR:-/tmp/sglang_cuda_coredumps}"
|
||||
# Mirror get_dump_dir() in python/sglang/srt/debug_utils/cuda_coredump.py:
|
||||
# explicit override > per-job RUNNER_TEMP > /tmp default, then a
|
||||
# per-(run, attempt) subdir so dumps are never mis-attributed across CI
|
||||
# jobs that share a self-hosted runner's filesystem.
|
||||
if [ -n "$SGLANG_CUDA_COREDUMP_DIR" ]; then
|
||||
base="$SGLANG_CUDA_COREDUMP_DIR"
|
||||
elif [ -n "$RUNNER_TEMP" ]; then
|
||||
base="$RUNNER_TEMP/sglang_cuda_coredumps"
|
||||
else
|
||||
base="/tmp/sglang_cuda_coredumps"
|
||||
fi
|
||||
if [ -n "$GITHUB_RUN_ID" ]; then
|
||||
dir="$base/${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT:-1}"
|
||||
else
|
||||
dir="$base"
|
||||
fi
|
||||
echo "dump_dir=$dir" >> "$GITHUB_OUTPUT"
|
||||
if [ -d "$dir" ] && [ -n "$(ls -A "$dir" 2>/dev/null)" ]; then
|
||||
echo "has_dumps=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
@@ -41,7 +57,7 @@ runs:
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: cuda-coredumps-${{ github.job }}${{ inputs.artifact-suffix && format('-{0}', inputs.artifact-suffix) }}
|
||||
path: ${{ env.SGLANG_CUDA_COREDUMP_DIR || '/tmp/sglang_cuda_coredumps' }}/
|
||||
path: ${{ steps.check.outputs.dump_dir }}/
|
||||
retention-days: ${{ inputs.retention-days }}
|
||||
|
||||
- name: Signal coredump to tracker issue
|
||||
@@ -82,4 +98,4 @@ runs:
|
||||
|
||||
- name: Cleanup CUDA coredumps
|
||||
shell: bash
|
||||
run: rm -rf "${{ env.SGLANG_CUDA_COREDUMP_DIR || '/tmp/sglang_cuda_coredumps' }}"
|
||||
run: rm -rf "${{ steps.check.outputs.dump_dir }}"
|
||||
|
||||
@@ -29,7 +29,24 @@ def is_enabled() -> bool:
|
||||
|
||||
|
||||
def get_dump_dir() -> str:
|
||||
return envs.SGLANG_CUDA_COREDUMP_DIR.get()
|
||||
# Resolve the base dir the same way as the uploader
|
||||
# (.github/actions/upload-cuda-coredumps/action.yml) so they agree; an empty
|
||||
# SGLANG_CUDA_COREDUMP_DIR counts as unset, like the action's `[ -n ... ]`.
|
||||
explicit = envs.SGLANG_CUDA_COREDUMP_DIR.get()
|
||||
runner_temp = os.getenv("RUNNER_TEMP")
|
||||
if explicit:
|
||||
base = explicit
|
||||
elif runner_temp:
|
||||
base = os.path.join(runner_temp, "sglang_cuda_coredumps")
|
||||
else:
|
||||
base = "/tmp/sglang_cuda_coredumps"
|
||||
# Isolate dumps per (run, attempt): on a shared self-hosted runner a leftover
|
||||
# dump from one job must not be picked up and mis-attributed by a later one.
|
||||
run_id = os.getenv("GITHUB_RUN_ID")
|
||||
if run_id:
|
||||
attempt = os.getenv("GITHUB_RUN_ATTEMPT", "1")
|
||||
return os.path.join(base, f"{run_id}-{attempt}")
|
||||
return base
|
||||
|
||||
|
||||
def _inject_env():
|
||||
|
||||
@@ -221,7 +221,9 @@ class Envs:
|
||||
SGLANG_IS_IN_CI = EnvBool(False)
|
||||
SGLANG_IS_IN_CI_AMD = EnvBool(False)
|
||||
SGLANG_CUDA_COREDUMP = EnvBool(False)
|
||||
SGLANG_CUDA_COREDUMP_DIR = EnvStr("/tmp/sglang_cuda_coredumps")
|
||||
# None = unset, letting get_dump_dir() resolve the base (RUNNER_TEMP in CI,
|
||||
# else /tmp); see debug_utils/cuda_coredump.py.
|
||||
SGLANG_CUDA_COREDUMP_DIR = EnvStr(None)
|
||||
SGLANG_TEST_MAX_RETRY = EnvInt(None)
|
||||
|
||||
# Constrained Decoding (Grammar)
|
||||
|
||||
Reference in New Issue
Block a user