[CI] Fix DeepGEMM sanitizer setup and Blackwell test timeouts (#39163)

This commit is contained in:
Baizhou Zhang
2026-09-11 20:54:52 -07:00
committed by GitHub
parent ff1ce11348
commit 9c365e97b1
+34 -1
View File
@@ -100,9 +100,11 @@ jobs:
- arch_label: sm90
runner: 8-gpu-h200
wheel_arch: x86_64
timeout: 120
- arch_label: sm100
runner: 8-gpu-b200
wheel_arch: x86_64
timeout: 240
# The sm120 tests require a runner with RTX 6000. It causes OOM on 5090 runners.
# - arch_label: sm120
# runner: 1-gpu-5090
@@ -110,8 +112,9 @@ jobs:
- arch_label: sm100-aarch64
runner: 4-gpu-gb300
wheel_arch: aarch64
timeout: 240
runs-on: ${{ matrix.runner }}
timeout-minutes: 120
timeout-minutes: ${{ matrix.timeout }}
steps:
- uses: actions/checkout@v4
@@ -142,6 +145,36 @@ jobs:
python3 -m pip install dist/*.whl
python3 -c "import deep_gemm; print('deep_gemm:', deep_gemm.__file__)"
- name: Install Compute Sanitizer 2025.4.1
env:
WHEEL_ARCH: ${{ matrix.wheel_arch }}
run: |
set -euo pipefail
# CUDA 13.1.1's standalone sanitizer fixes Python tensor retention in
# host backtraces. Keep the wheel and runtime on CUDA 13.0.
case "${WHEEL_ARCH}" in
x86_64)
sanitizer_arch=linux-x86_64
sanitizer_sha256=b9637777a31cd0f0ffe1e965523e8b0d382019b8496855fd688c886988583cf4
;;
aarch64)
sanitizer_arch=linux-sbsa
sanitizer_sha256=1dc68ccb146032bcdc9d932569c865d68e3966ecf136940ca38fd7d5abfda4ac
;;
*) echo "Unsupported sanitizer architecture: ${WHEEL_ARCH}" >&2; exit 1 ;;
esac
archive="cuda_sanitizer_api-${sanitizer_arch}-13.1.118-archive"
install_dir="${RUNNER_TEMP}/deepgemm-compute-sanitizer"
mkdir -p "${install_dir}"
curl -fsSL --retry 3 \
"https://developer.download.nvidia.com/compute/cuda/redist/cuda_sanitizer_api/${sanitizer_arch}/${archive}.tar.xz" \
-o "${install_dir}/${archive}.tar.xz"
echo "${sanitizer_sha256} ${install_dir}/${archive}.tar.xz" | sha256sum -c -
tar -xJf "${install_dir}/${archive}.tar.xz" -C "${install_dir}"
sanitizer="${install_dir}/${archive}/bin/compute-sanitizer"
"${sanitizer}" --version
echo "COMPUTE_SANITIZER=${sanitizer}" >> "${GITHUB_ENV}"
- name: Run DeepGEMM test suite
run: |
chmod +x "${{ github.workspace }}/DeepGEMM/sgl_deep_gemm/run_tests.sh"