[CI] Fix DeepGEMM sanitizer setup and Blackwell test timeouts (#39163)
This commit is contained in:
@@ -100,9 +100,11 @@ jobs:
|
|||||||
- arch_label: sm90
|
- arch_label: sm90
|
||||||
runner: 8-gpu-h200
|
runner: 8-gpu-h200
|
||||||
wheel_arch: x86_64
|
wheel_arch: x86_64
|
||||||
|
timeout: 120
|
||||||
- arch_label: sm100
|
- arch_label: sm100
|
||||||
runner: 8-gpu-b200
|
runner: 8-gpu-b200
|
||||||
wheel_arch: x86_64
|
wheel_arch: x86_64
|
||||||
|
timeout: 240
|
||||||
# The sm120 tests require a runner with RTX 6000. It causes OOM on 5090 runners.
|
# The sm120 tests require a runner with RTX 6000. It causes OOM on 5090 runners.
|
||||||
# - arch_label: sm120
|
# - arch_label: sm120
|
||||||
# runner: 1-gpu-5090
|
# runner: 1-gpu-5090
|
||||||
@@ -110,8 +112,9 @@ jobs:
|
|||||||
- arch_label: sm100-aarch64
|
- arch_label: sm100-aarch64
|
||||||
runner: 4-gpu-gb300
|
runner: 4-gpu-gb300
|
||||||
wheel_arch: aarch64
|
wheel_arch: aarch64
|
||||||
|
timeout: 240
|
||||||
runs-on: ${{ matrix.runner }}
|
runs-on: ${{ matrix.runner }}
|
||||||
timeout-minutes: 120
|
timeout-minutes: ${{ matrix.timeout }}
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
@@ -142,6 +145,36 @@ jobs:
|
|||||||
python3 -m pip install dist/*.whl
|
python3 -m pip install dist/*.whl
|
||||||
python3 -c "import deep_gemm; print('deep_gemm:', deep_gemm.__file__)"
|
python3 -c "import deep_gemm; print('deep_gemm:', deep_gemm.__file__)"
|
||||||
|
|
||||||
|
- name: Install Compute Sanitizer 2025.4.1
|
||||||
|
env:
|
||||||
|
WHEEL_ARCH: ${{ matrix.wheel_arch }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
# CUDA 13.1.1's standalone sanitizer fixes Python tensor retention in
|
||||||
|
# host backtraces. Keep the wheel and runtime on CUDA 13.0.
|
||||||
|
case "${WHEEL_ARCH}" in
|
||||||
|
x86_64)
|
||||||
|
sanitizer_arch=linux-x86_64
|
||||||
|
sanitizer_sha256=b9637777a31cd0f0ffe1e965523e8b0d382019b8496855fd688c886988583cf4
|
||||||
|
;;
|
||||||
|
aarch64)
|
||||||
|
sanitizer_arch=linux-sbsa
|
||||||
|
sanitizer_sha256=1dc68ccb146032bcdc9d932569c865d68e3966ecf136940ca38fd7d5abfda4ac
|
||||||
|
;;
|
||||||
|
*) echo "Unsupported sanitizer architecture: ${WHEEL_ARCH}" >&2; exit 1 ;;
|
||||||
|
esac
|
||||||
|
archive="cuda_sanitizer_api-${sanitizer_arch}-13.1.118-archive"
|
||||||
|
install_dir="${RUNNER_TEMP}/deepgemm-compute-sanitizer"
|
||||||
|
mkdir -p "${install_dir}"
|
||||||
|
curl -fsSL --retry 3 \
|
||||||
|
"https://developer.download.nvidia.com/compute/cuda/redist/cuda_sanitizer_api/${sanitizer_arch}/${archive}.tar.xz" \
|
||||||
|
-o "${install_dir}/${archive}.tar.xz"
|
||||||
|
echo "${sanitizer_sha256} ${install_dir}/${archive}.tar.xz" | sha256sum -c -
|
||||||
|
tar -xJf "${install_dir}/${archive}.tar.xz" -C "${install_dir}"
|
||||||
|
sanitizer="${install_dir}/${archive}/bin/compute-sanitizer"
|
||||||
|
"${sanitizer}" --version
|
||||||
|
echo "COMPUTE_SANITIZER=${sanitizer}" >> "${GITHUB_ENV}"
|
||||||
|
|
||||||
- name: Run DeepGEMM test suite
|
- name: Run DeepGEMM test suite
|
||||||
run: |
|
run: |
|
||||||
chmod +x "${{ github.workspace }}/DeepGEMM/sgl_deep_gemm/run_tests.sh"
|
chmod +x "${{ github.workspace }}/DeepGEMM/sgl_deep_gemm/run_tests.sh"
|
||||||
|
|||||||
Reference in New Issue
Block a user