feat: add cuda core dump CI warpper (#18909)

.github/actions/upload-cuda-coredumps/action.yml

@@ -0,0 +1,27 @@
+name: Upload CUDA Coredumps
+description: Upload CUDA coredump files as artifacts and clean up the directory.
+
+inputs:
+  artifact-suffix:
+    description: Suffix appended to the artifact name (e.g. matrix partition id)
+    required: false
+    default: ""
+  retention-days:
+    description: Number of days to retain the artifact
+    required: false
+    default: "7"
+
+runs:
+  using: composite
+  steps:
+    - name: Upload CUDA coredumps
+      uses: actions/upload-artifact@v4
+      with:
+        name: cuda-coredumps-${{ github.job }}${{ inputs.artifact-suffix && format('-{0}', inputs.artifact-suffix) }}
+        path: ${{ env.SGLANG_CUDA_COREDUMP_DIR || '/tmp/sglang_cuda_coredumps' }}/
+        retention-days: ${{ inputs.retention-days }}
+        if-no-files-found: ignore
+
+    - name: Cleanup CUDA coredumps
+      shell: bash
+      run: rm -rf "${{ env.SGLANG_CUDA_COREDUMP_DIR || '/tmp/sglang_cuda_coredumps' }}"

.github/workflows/nightly-test-nvidia.yml

@@ -44,6 +44,7 @@ concurrency:
 
 env:
   SGLANG_IS_IN_CI: true
+  SGLANG_CUDA_COREDUMP: "1"
   HF_HUB_DOWNLOAD_TIMEOUT: 300
   HF_HUB_ETAG_TIMEOUT: 300
 
@@ -68,6 +69,9 @@ jobs:
           cd test
           python3 run_suite.py --hw cuda --suite nightly-1-gpu --nightly --continue-on-error
 
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+
   # General tests - 4 GPU H100
   nightly-test-general-4-gpu-h100:
     if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-4-gpu-h100')
@@ -88,6 +92,9 @@ jobs:
           cd test
           python3 run_suite.py --hw cuda --suite nightly-4-gpu --nightly --continue-on-error
 
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+
   # General tests - 8 GPU H200
   nightly-test-general-8-gpu-h200:
     if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-h200')
@@ -167,6 +174,11 @@ jobs:
           retention-days: 5
           if-no-files-found: ignore
 
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+        with:
+          artifact-suffix: ${{ matrix.partition }}
+
   # General tests - 8 GPU H20
   nightly-test-general-8-gpu-h20:
     if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-h20')
@@ -191,6 +203,9 @@ jobs:
           cd test
           python3 run_suite.py --hw cuda --suite nightly-8-gpu-h20 --nightly --continue-on-error
 
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+
   # General tests - 8 GPU B200
   nightly-test-general-8-gpu-b200:
     if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-b200')
@@ -259,6 +274,11 @@ jobs:
           retention-days: 5
           if-no-files-found: ignore
 
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+        with:
+          artifact-suffix: ${{ matrix.partition }}
+
   # Text model accuracy tests
   nightly-test-text-accuracy-2-gpu-runner:
     if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-text-accuracy-2-gpu-runner')
@@ -279,6 +299,9 @@ jobs:
           cd test
           python3 run_suite.py --hw cuda --suite nightly-eval-text-2-gpu --nightly --continue-on-error --timeout-per-file 4500
 
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+
   # Text model performance tests
   nightly-test-text-perf-2-gpu-runner:
     if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-text-perf-2-gpu-runner')
@@ -312,6 +335,9 @@ jobs:
         run: |
           python3 scripts/ci/utils/publish_traces.py --traces-dir test/performance_profiles_text_models
 
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+
   # VLM accuracy tests
   nightly-test-vlm-accuracy-2-gpu-runner:
     if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-vlm-accuracy-2-gpu-runner')
@@ -332,6 +358,9 @@ jobs:
           cd test
           python3 run_suite.py --hw cuda --suite nightly-eval-vlm-2-gpu --nightly --continue-on-error --timeout-per-file 9000
 
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+
   # VLM performance tests
   nightly-test-vlm-perf-2-gpu-runner:
     if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-vlm-perf-2-gpu-runner')
@@ -365,6 +394,9 @@ jobs:
         run: |
           python3 scripts/ci/utils/publish_traces.py --traces-dir test/performance_profiles_vlms
 
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+
   # diffusion performance tests
   nightly-test-multimodal-server-1-gpu:
     if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-multimodal-server-1-gpu')
@@ -417,6 +449,10 @@ jobs:
           retention-days: 90
           if-no-files-found: ignore
 
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+        with:
+          artifact-suffix: ${{ matrix.part }}
 
   nightly-test-multimodal-server-2-gpu:
     if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-multimodal-server-2-gpu')
@@ -469,6 +505,11 @@ jobs:
           retention-days: 90
           if-no-files-found: ignore
 
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+        with:
+          artifact-suffix: ${{ matrix.part }}
+
   # B200 Performance tests - 4 GPU
   nightly-test-perf-4-gpu-b200:
     if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-4-gpu-b200')
@@ -489,6 +530,9 @@ jobs:
           cd test
           python3 run_suite.py --hw cuda --suite nightly-4-gpu-b200 --nightly --continue-on-error --timeout-per-file 12000
 
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+
   # Specialized B200 tests - 8 GPU, for specific backends and configs
   nightly-test-specialized-8-gpu-b200:
     if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-8-gpu-b200')
@@ -513,6 +557,9 @@ jobs:
           cd test
           python3 run_suite.py --hw cuda --suite nightly-8-gpu-b200 --nightly --continue-on-error --timeout-per-file 2400
 
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+
   # Consolidate performance metrics from all jobs
   consolidate-metrics:
     if: github.repository == 'sgl-project/sglang' && always()

.github/workflows/pr-test.yml

@@ -63,6 +63,7 @@ concurrency:
 
 env:
   SGLANG_IS_IN_CI: true
+  SGLANG_CUDA_COREDUMP: "1"
   SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true
 
 permissions:
@@ -874,6 +875,9 @@ jobs:
           # temporarily put backend-independent cpu tests here
           python3 run_suite.py --hw cpu --suite default $CONTINUE_ON_ERROR_FLAG
 
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+
   stage-a-cpu-only:
     needs: [check-changes, call-gate]
     if: |
@@ -976,6 +980,11 @@ jobs:
           fi
           python3 run_suite.py --hw cuda --suite stage-b-test-small-1-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 8 $CONTINUE_ON_ERROR_FLAG
 
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+        with:
+          artifact-suffix: ${{ matrix.partition }}
+
   # Runs on H100 (80GB, SM90) - tests that don't pass on 5090 (FA3, FP8, high VRAM, etc.)
   stage-b-test-large-1-gpu:
     needs: [check-changes, call-gate, wait-for-stage-a, sgl-kernel-build-wheels]
@@ -1027,6 +1036,11 @@ jobs:
           fi
           python3 run_suite.py --hw cuda --suite stage-b-test-large-1-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 14 --timeout-per-file 1800 $CONTINUE_ON_ERROR_FLAG
 
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+        with:
+          artifact-suffix: ${{ matrix.partition }}
+
   stage-b-test-large-2-gpu:
     needs: [check-changes, call-gate, wait-for-stage-a, sgl-kernel-build-wheels]
     if: |
@@ -1079,6 +1093,11 @@ jobs:
           fi
           python3 run_suite.py --hw cuda --suite stage-b-test-large-2-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 4 $CONTINUE_ON_ERROR_FLAG
 
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+        with:
+          artifact-suffix: ${{ matrix.partition }}
+
   stage-b-test-4-gpu-b200:
     needs: [check-changes, call-gate, wait-for-stage-a, sgl-kernel-build-wheels]
     if: |
@@ -1132,6 +1151,9 @@ jobs:
         run: |
           IS_BLACKWELL=1 python3 -m pytest -q python/sglang/jit_kernel/tests/test_flash_attention_4.py
 
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+
   multimodal-gen-test-1-gpu:
     needs: [check-changes, call-gate, sgl-kernel-build-wheels]
     if: |
@@ -1184,6 +1206,10 @@ jobs:
             --total-partitions 2 \
             $CONTINUE_ON_ERROR_FLAG
 
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+        with:
+          artifact-suffix: ${{ matrix.part }}
 
   multimodal-gen-test-2-gpu:
     needs: [check-changes, call-gate, sgl-kernel-build-wheels]
@@ -1238,6 +1264,11 @@ jobs:
             --total-partitions 2 \
             $CONTINUE_ON_ERROR_FLAG
 
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+        with:
+          artifact-suffix: ${{ matrix.part }}
+
   stage-c-test-4-gpu-h100:
     needs: [check-changes, call-gate, wait-for-stage-b]
     if: |
@@ -1287,6 +1318,11 @@ jobs:
           fi
           python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-h100 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 $CONTINUE_ON_ERROR_FLAG
 
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+        with:
+          artifact-suffix: ${{ matrix.part }}
+
   stage-c-test-8-gpu-h200:
     needs: [check-changes, call-gate, wait-for-stage-b]
     if: |
@@ -1342,6 +1378,11 @@ jobs:
           fi
           python3 run_suite.py --hw cuda --suite stage-c-test-8-gpu-h200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 4 $CONTINUE_ON_ERROR_FLAG
 
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+        with:
+          artifact-suffix: ${{ matrix.part }}
+
   stage-c-test-8-gpu-h20:
     needs: [check-changes, call-gate, wait-for-stage-b]
     if: |
@@ -1392,6 +1433,11 @@ jobs:
           fi
           python3 run_suite.py --hw cuda --suite stage-c-test-8-gpu-h20 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 $CONTINUE_ON_ERROR_FLAG
 
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+        with:
+          artifact-suffix: ${{ matrix.part }}
+
   stage-c-test-deepep-4-gpu:
     needs: [check-changes, call-gate, wait-for-stage-b]
     if: |
@@ -1437,6 +1483,9 @@ jobs:
           fi
           python3 run_suite.py --hw cuda --suite stage-c-test-deepep-4-gpu $CONTINUE_ON_ERROR_FLAG
 
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+
   stage-c-test-deepep-8-gpu-h200:
     needs: [check-changes, call-gate, wait-for-stage-b]
     if: |
@@ -1482,6 +1531,9 @@ jobs:
           fi
           python3 run_suite.py --hw cuda --suite stage-c-test-deepep-8-gpu-h200 $CONTINUE_ON_ERROR_FLAG
 
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+
   stage-c-test-4-gpu-b200:
     needs: [check-changes, call-gate, wait-for-stage-b]
     if: |
@@ -1532,6 +1584,11 @@ jobs:
           fi
           IS_BLACKWELL=1 python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-b200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 1800 $CONTINUE_ON_ERROR_FLAG
 
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+        with:
+          artifact-suffix: ${{ matrix.part }}
+
   stage-c-test-4-gpu-gb200:
     needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels-arm]
     if: |
@@ -1579,6 +1636,9 @@ jobs:
           fi
           python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-gb200 --timeout-per-file 3600 $CONTINUE_ON_ERROR_FLAG
 
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+
   pr-test-finish:
     needs:
       [

python/sglang/srt/debug_utils/cuda_coredump.py

@@ -0,0 +1,93 @@
+"""CUDA coredump helpers.
+
+When SGLANG_CUDA_COREDUMP=1, this module injects CUDA coredump environment
+variables into the current process so that GPU exceptions (e.g. illegal
+memory access) produce lightweight coredump files for post-mortem analysis
+with cuda-gdb.
+
+The injection happens at module import time via _inject_env() on a
+best-effort basis.  If any CUDA_* variable is already present in the
+environment (e.g. set by the user in the shell), injection is skipped for
+that variable and a warning is logged.  For strict guarantees, set the
+CUDA_* env vars in the shell before launching Python.
+"""
+
+import glob
+import logging
+import os
+
+from sglang.srt.environ import envs
+
+logger = logging.getLogger(__name__)
+
+_CUDA_COREDUMP_FLAGS = (
+    "skip_nonrelocated_elf_images,skip_global_memory,"
+    "skip_shared_memory,skip_local_memory,skip_constbank_memory"
+)
+
+
+def is_enabled() -> bool:
+    return envs.SGLANG_CUDA_COREDUMP.get()
+
+
+def get_dump_dir() -> str:
+    return envs.SGLANG_CUDA_COREDUMP_DIR.get()
+
+
+def _inject_env():
+    """Inject CUDA coredump environment variables into the current process.
+    If a CUDA_* variable is already present, skip it and log a warning."""
+    dump_dir = get_dump_dir()
+    os.makedirs(dump_dir, exist_ok=True)
+
+    env_vars = {
+        "CUDA_ENABLE_COREDUMP_ON_EXCEPTION": "1",
+        "CUDA_COREDUMP_SHOW_PROGRESS": "1",
+        "CUDA_COREDUMP_GENERATION_FLAGS": _CUDA_COREDUMP_FLAGS,
+        "CUDA_COREDUMP_FILE": f"{dump_dir}/cuda_coredump_%h.%p.%t",
+    }
+    for key, value in env_vars.items():
+        if key in os.environ:
+            logger.warning(
+                "CUDA coredump env var %s is already set to '%s', "
+                "skipping injection of '%s'.",
+                key,
+                os.environ[key],
+                value,
+            )
+        else:
+            os.environ[key] = value
+
+
+def cleanup_dump_dir():
+    """Remove stale coredump files from the dump directory."""
+    dump_dir = get_dump_dir()
+    for f in glob.glob(os.path.join(dump_dir, "cuda_coredump_*")):
+        os.remove(f)
+
+
+def report():
+    """Log any CUDA coredump files found after a test failure."""
+    dump_dir = get_dump_dir()
+    coredump_files = glob.glob(os.path.join(dump_dir, "cuda_coredump_*"))
+    if not coredump_files:
+        return
+
+    logger.info(f"\n{'='*60}")
+    logger.info(f"CUDA coredump(s) detected ({len(coredump_files)} file(s)):")
+    for f in coredump_files:
+        size_mb = os.path.getsize(f) / (1024 * 1024)
+        logger.info(f"  {f} ({size_mb:.1f} MB)")
+    logger.info("Use cuda-gdb to analyze: cuda-gdb -c <coredump_file>")
+
+    run_id = os.environ.get("GITHUB_RUN_ID")
+    if run_id:
+        repo = os.environ.get("GITHUB_REPOSITORY", "sgl-project/sglang")
+        logger.info(f"Download from CI: gh run download {run_id} --repo {repo}")
+
+    logger.info(f"{'='*60}\n")
+
+
+# Auto-inject CUDA coredump env vars at import time.
+if is_enabled():
+    _inject_env()

python/sglang/srt/environ.py

@@ -176,6 +176,8 @@ class Envs:
     # SGLang CI
     SGLANG_IS_IN_CI = EnvBool(False)
     SGLANG_IS_IN_CI_AMD = EnvBool(False)
+    SGLANG_CUDA_COREDUMP = EnvBool(False)
+    SGLANG_CUDA_COREDUMP_DIR = EnvStr("/tmp/sglang_cuda_coredumps")
     SGLANG_TEST_MAX_RETRY = EnvInt(None)
 
     # Constrained Decoding (Grammar)
@@ -575,6 +577,11 @@ _warn_deprecated_env_to_cli_flag(
     "Please use '--prefill-delayer-token-usage-low-watermark' instead.",
 )
 
+# Import cuda_coredump to trigger auto-injection of CUDA env vars
+# when SGLANG_CUDA_COREDUMP=1. Best-effort; for strict guarantees,
+# set CUDA_* env vars in the shell before launching Python.
+import sglang.srt.debug_utils.cuda_coredump  # noqa: F401, E402
+
 
 def example_with_exit_stack():
     # Use this style of context manager in unit test

python/sglang/test/ci/ci_utils.py

@@ -7,6 +7,7 @@ import time
 from dataclasses import dataclass
 from typing import Callable, List, Optional, Union
 
+from sglang.srt.debug_utils import cuda_coredump
 from sglang.srt.utils.common import kill_process_tree
 from sglang.test.ci.ci_register import CIRegistry
 
@@ -130,6 +131,10 @@ def run_unittest_files(
         max_attempts: Maximum number of attempts per file including initial run (default: 2).
         retry_wait_seconds: Seconds to wait between retries (default: 60).
     """
+    coredump_enabled = cuda_coredump.is_enabled()
+    if coredump_enabled:
+        cuda_coredump.cleanup_dump_dir()
+
     tic = time.perf_counter()
     success = True
     passed_tests = []
@@ -161,7 +166,6 @@ def run_unittest_files(
                     ["python3", full_path],
                     stdout=subprocess.PIPE,
                     stderr=subprocess.STDOUT,
-                    env=os.environ,
                     text=True,
                     errors="ignore",  # Ignore non-UTF-8 bytes to prevent UnicodeDecodeError
                 )
@@ -172,7 +176,7 @@ def run_unittest_files(
                 process.wait()
             else:
                 process = subprocess.Popen(
-                    ["python3", full_path], stdout=None, stderr=None, env=os.environ
+                    ["python3", full_path], stdout=None, stderr=None
                 )
                 process.wait()
 
@@ -258,6 +262,9 @@ def run_unittest_files(
 
     elapsed_total = time.perf_counter() - tic
 
+    if coredump_enabled and not success:
+        cuda_coredump.report()
+
     if success:
         logger.info(f"Success. Time elapsed: {elapsed_total:.2f}s")
     else:

test/registered/debug_utils/test_cuda_coredump_smoke.py

@@ -0,0 +1,29 @@
+"""Smoke test: intentionally trigger a CUDA illegal memory access
+to verify the coredump collection pipeline works end-to-end.
+
+Manual use:  python3 test/registered/debug_utils/test_cuda_coredump_smoke.py
+"""
+
+import unittest
+
+import torch
+
+from sglang.test.ci.ci_register import register_cuda_ci
+
+register_cuda_ci(
+    est_time=10,
+    suite="stage-a-test-1",
+    disabled="Manual only: triggers intentional CUDA crash for coredump verification",
+)
+
+
+class TestCudaCoredumpSmoke(unittest.TestCase):
+    def test_trigger_illegal_memory_access(self):
+        x = torch.zeros(10, device="cuda")
+        y = torch.arange(10, device="cuda")
+        x[y * y] = 1
+        torch.cuda.synchronize()
+
+
+if __name__ == "__main__":
+    unittest.main()