diff --git a/.github/workflows/release-whl-deepgemm.yml b/.github/workflows/release-whl-deepgemm.yml index ed24da92d..a300e8a23 100644 --- a/.github/workflows/release-whl-deepgemm.yml +++ b/.github/workflows/release-whl-deepgemm.yml @@ -100,9 +100,11 @@ jobs: - arch_label: sm90 runner: 8-gpu-h200 wheel_arch: x86_64 + timeout: 120 - arch_label: sm100 runner: 8-gpu-b200 wheel_arch: x86_64 + timeout: 240 # The sm120 tests require a runner with RTX 6000. It causes OOM on 5090 runners. # - arch_label: sm120 # runner: 1-gpu-5090 @@ -110,8 +112,9 @@ jobs: - arch_label: sm100-aarch64 runner: 4-gpu-gb300 wheel_arch: aarch64 + timeout: 240 runs-on: ${{ matrix.runner }} - timeout-minutes: 120 + timeout-minutes: ${{ matrix.timeout }} steps: - uses: actions/checkout@v4 @@ -142,6 +145,36 @@ jobs: python3 -m pip install dist/*.whl python3 -c "import deep_gemm; print('deep_gemm:', deep_gemm.__file__)" + - name: Install Compute Sanitizer 2025.4.1 + env: + WHEEL_ARCH: ${{ matrix.wheel_arch }} + run: | + set -euo pipefail + # CUDA 13.1.1's standalone sanitizer fixes Python tensor retention in + # host backtraces. Keep the wheel and runtime on CUDA 13.0. + case "${WHEEL_ARCH}" in + x86_64) + sanitizer_arch=linux-x86_64 + sanitizer_sha256=b9637777a31cd0f0ffe1e965523e8b0d382019b8496855fd688c886988583cf4 + ;; + aarch64) + sanitizer_arch=linux-sbsa + sanitizer_sha256=1dc68ccb146032bcdc9d932569c865d68e3966ecf136940ca38fd7d5abfda4ac + ;; + *) echo "Unsupported sanitizer architecture: ${WHEEL_ARCH}" >&2; exit 1 ;; + esac + archive="cuda_sanitizer_api-${sanitizer_arch}-13.1.118-archive" + install_dir="${RUNNER_TEMP}/deepgemm-compute-sanitizer" + mkdir -p "${install_dir}" + curl -fsSL --retry 3 \ + "https://developer.download.nvidia.com/compute/cuda/redist/cuda_sanitizer_api/${sanitizer_arch}/${archive}.tar.xz" \ + -o "${install_dir}/${archive}.tar.xz" + echo "${sanitizer_sha256} ${install_dir}/${archive}.tar.xz" | sha256sum -c - + tar -xJf "${install_dir}/${archive}.tar.xz" -C "${install_dir}" + sanitizer="${install_dir}/${archive}/bin/compute-sanitizer" + "${sanitizer}" --version + echo "COMPUTE_SANITIZER=${sanitizer}" >> "${GITHUB_ENV}" + - name: Run DeepGEMM test suite run: | chmod +x "${{ github.workspace }}/DeepGEMM/sgl_deep_gemm/run_tests.sh"