diff --git a/.github/actions/upload-cuda-coredumps/action.yml b/.github/actions/upload-cuda-coredumps/action.yml index c741df174..88c1d22d8 100644 --- a/.github/actions/upload-cuda-coredumps/action.yml +++ b/.github/actions/upload-cuda-coredumps/action.yml @@ -29,7 +29,23 @@ runs: id: check shell: bash run: | - dir="${SGLANG_CUDA_COREDUMP_DIR:-/tmp/sglang_cuda_coredumps}" + # Mirror get_dump_dir() in python/sglang/srt/debug_utils/cuda_coredump.py: + # explicit override > per-job RUNNER_TEMP > /tmp default, then a + # per-(run, attempt) subdir so dumps are never mis-attributed across CI + # jobs that share a self-hosted runner's filesystem. + if [ -n "$SGLANG_CUDA_COREDUMP_DIR" ]; then + base="$SGLANG_CUDA_COREDUMP_DIR" + elif [ -n "$RUNNER_TEMP" ]; then + base="$RUNNER_TEMP/sglang_cuda_coredumps" + else + base="/tmp/sglang_cuda_coredumps" + fi + if [ -n "$GITHUB_RUN_ID" ]; then + dir="$base/${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT:-1}" + else + dir="$base" + fi + echo "dump_dir=$dir" >> "$GITHUB_OUTPUT" if [ -d "$dir" ] && [ -n "$(ls -A "$dir" 2>/dev/null)" ]; then echo "has_dumps=true" >> "$GITHUB_OUTPUT" else @@ -41,7 +57,7 @@ runs: uses: actions/upload-artifact@v4 with: name: cuda-coredumps-${{ github.job }}${{ inputs.artifact-suffix && format('-{0}', inputs.artifact-suffix) }} - path: ${{ env.SGLANG_CUDA_COREDUMP_DIR || '/tmp/sglang_cuda_coredumps' }}/ + path: ${{ steps.check.outputs.dump_dir }}/ retention-days: ${{ inputs.retention-days }} - name: Signal coredump to tracker issue @@ -82,4 +98,4 @@ runs: - name: Cleanup CUDA coredumps shell: bash - run: rm -rf "${{ env.SGLANG_CUDA_COREDUMP_DIR || '/tmp/sglang_cuda_coredumps' }}" + run: rm -rf "${{ steps.check.outputs.dump_dir }}" diff --git a/python/sglang/srt/debug_utils/cuda_coredump.py b/python/sglang/srt/debug_utils/cuda_coredump.py index 1507467dd..fa6daa44d 100644 --- a/python/sglang/srt/debug_utils/cuda_coredump.py +++ b/python/sglang/srt/debug_utils/cuda_coredump.py @@ -29,7 +29,24 @@ def is_enabled() -> bool: def get_dump_dir() -> str: - return envs.SGLANG_CUDA_COREDUMP_DIR.get() + # Resolve the base dir the same way as the uploader + # (.github/actions/upload-cuda-coredumps/action.yml) so they agree; an empty + # SGLANG_CUDA_COREDUMP_DIR counts as unset, like the action's `[ -n ... ]`. + explicit = envs.SGLANG_CUDA_COREDUMP_DIR.get() + runner_temp = os.getenv("RUNNER_TEMP") + if explicit: + base = explicit + elif runner_temp: + base = os.path.join(runner_temp, "sglang_cuda_coredumps") + else: + base = "/tmp/sglang_cuda_coredumps" + # Isolate dumps per (run, attempt): on a shared self-hosted runner a leftover + # dump from one job must not be picked up and mis-attributed by a later one. + run_id = os.getenv("GITHUB_RUN_ID") + if run_id: + attempt = os.getenv("GITHUB_RUN_ATTEMPT", "1") + return os.path.join(base, f"{run_id}-{attempt}") + return base def _inject_env(): diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index dc1e063aa..8eaa4168f 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -221,7 +221,9 @@ class Envs: SGLANG_IS_IN_CI = EnvBool(False) SGLANG_IS_IN_CI_AMD = EnvBool(False) SGLANG_CUDA_COREDUMP = EnvBool(False) - SGLANG_CUDA_COREDUMP_DIR = EnvStr("/tmp/sglang_cuda_coredumps") + # None = unset, letting get_dump_dir() resolve the base (RUNNER_TEMP in CI, + # else /tmp); see debug_utils/cuda_coredump.py. + SGLANG_CUDA_COREDUMP_DIR = EnvStr(None) SGLANG_TEST_MAX_RETRY = EnvInt(None) # Constrained Decoding (Grammar)