[CI] Fix DeepGEMM sanitizer setup and Blackwell test timeouts (#39163)
This commit is contained in:
@@ -100,9 +100,11 @@ jobs:
|
||||
- arch_label: sm90
|
||||
runner: 8-gpu-h200
|
||||
wheel_arch: x86_64
|
||||
timeout: 120
|
||||
- arch_label: sm100
|
||||
runner: 8-gpu-b200
|
||||
wheel_arch: x86_64
|
||||
timeout: 240
|
||||
# The sm120 tests require a runner with RTX 6000. It causes OOM on 5090 runners.
|
||||
# - arch_label: sm120
|
||||
# runner: 1-gpu-5090
|
||||
@@ -110,8 +112,9 @@ jobs:
|
||||
- arch_label: sm100-aarch64
|
||||
runner: 4-gpu-gb300
|
||||
wheel_arch: aarch64
|
||||
timeout: 240
|
||||
runs-on: ${{ matrix.runner }}
|
||||
timeout-minutes: 120
|
||||
timeout-minutes: ${{ matrix.timeout }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
@@ -142,6 +145,36 @@ jobs:
|
||||
python3 -m pip install dist/*.whl
|
||||
python3 -c "import deep_gemm; print('deep_gemm:', deep_gemm.__file__)"
|
||||
|
||||
- name: Install Compute Sanitizer 2025.4.1
|
||||
env:
|
||||
WHEEL_ARCH: ${{ matrix.wheel_arch }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# CUDA 13.1.1's standalone sanitizer fixes Python tensor retention in
|
||||
# host backtraces. Keep the wheel and runtime on CUDA 13.0.
|
||||
case "${WHEEL_ARCH}" in
|
||||
x86_64)
|
||||
sanitizer_arch=linux-x86_64
|
||||
sanitizer_sha256=b9637777a31cd0f0ffe1e965523e8b0d382019b8496855fd688c886988583cf4
|
||||
;;
|
||||
aarch64)
|
||||
sanitizer_arch=linux-sbsa
|
||||
sanitizer_sha256=1dc68ccb146032bcdc9d932569c865d68e3966ecf136940ca38fd7d5abfda4ac
|
||||
;;
|
||||
*) echo "Unsupported sanitizer architecture: ${WHEEL_ARCH}" >&2; exit 1 ;;
|
||||
esac
|
||||
archive="cuda_sanitizer_api-${sanitizer_arch}-13.1.118-archive"
|
||||
install_dir="${RUNNER_TEMP}/deepgemm-compute-sanitizer"
|
||||
mkdir -p "${install_dir}"
|
||||
curl -fsSL --retry 3 \
|
||||
"https://developer.download.nvidia.com/compute/cuda/redist/cuda_sanitizer_api/${sanitizer_arch}/${archive}.tar.xz" \
|
||||
-o "${install_dir}/${archive}.tar.xz"
|
||||
echo "${sanitizer_sha256} ${install_dir}/${archive}.tar.xz" | sha256sum -c -
|
||||
tar -xJf "${install_dir}/${archive}.tar.xz" -C "${install_dir}"
|
||||
sanitizer="${install_dir}/${archive}/bin/compute-sanitizer"
|
||||
"${sanitizer}" --version
|
||||
echo "COMPUTE_SANITIZER=${sanitizer}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Run DeepGEMM test suite
|
||||
run: |
|
||||
chmod +x "${{ github.workspace }}/DeepGEMM/sgl_deep_gemm/run_tests.sh"
|
||||
|
||||
Reference in New Issue
Block a user