From b441317aa43084d9c43709ef00ec692b8f3ff20e Mon Sep 17 00:00:00 2001 From: Baizhou Zhang Date: Mon, 13 Apr 2026 14:39:24 -0700 Subject: [PATCH] Revert "Upgrade CI default CUDA version from 12.9 to 13.0" (#22727) --- .github/workflows/pr-test.yml | 31 ++++++++++--------- python/pyproject.toml | 8 ++--- .../cuda/ci_download_flashinfer_jit_cache.sh | 2 +- scripts/ci/cuda/ci_install_deepep.sh | 9 ++++-- scripts/ci/cuda/ci_install_dependency.sh | 16 +++------- 5 files changed, 33 insertions(+), 33 deletions(-) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 7c50a46d5..8feb364bd 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -429,7 +429,10 @@ jobs: matrix: include: - python-version: "3.10" - cuda-version: "13.0" + cuda-version: "12.9" + # Add back when CUDA 13.0 is supported on CI + # - python-version: "3.10" + # cuda-version: "13.0" name: Build Wheel steps: - name: Cleanup @@ -477,7 +480,7 @@ jobs: matrix: include: - python-version: "3.10" - cuda-version: "13.0" + cuda-version: "12.9" name: Build Wheel Arm steps: - name: Cleanup @@ -584,7 +587,7 @@ jobs: with: path: sgl-kernel/dist/ merge-multiple: true - pattern: wheel-python3.10-cuda13.0 + pattern: wheel-python3.10-cuda12.9 - name: Install dependencies timeout-minutes: 20 @@ -691,7 +694,7 @@ jobs: with: path: sgl-kernel/dist/ merge-multiple: true - pattern: wheel-python3.10-cuda13.0 + pattern: wheel-python3.10-cuda12.9 - name: Install dependencies timeout-minutes: 20 @@ -749,7 +752,7 @@ jobs: with: path: sgl-kernel/dist/ merge-multiple: true - pattern: wheel-python3.10-cuda13.0 + pattern: wheel-python3.10-cuda12.9 - name: Install dependencies timeout-minutes: 20 @@ -804,7 +807,7 @@ jobs: with: path: sgl-kernel/dist/ merge-multiple: true - pattern: wheel-python3.10-cuda13.0 + pattern: wheel-python3.10-cuda12.9 - name: Install dependencies timeout-minutes: 20 @@ -857,7 +860,7 @@ jobs: with: path: sgl-kernel/dist/ merge-multiple: true - pattern: wheel-python3.10-cuda13.0 + pattern: wheel-python3.10-cuda12.9 - name: Install dependencies timeout-minutes: 20 @@ -946,7 +949,7 @@ jobs: with: path: sgl-kernel/dist/ merge-multiple: true - pattern: wheel-python3.10-cuda13.0 + pattern: wheel-python3.10-cuda12.9 - name: Install dependencies timeout-minutes: 20 @@ -1000,7 +1003,7 @@ jobs: with: path: sgl-kernel/dist/ merge-multiple: true - pattern: wheel-python3.10-cuda13.0 + pattern: wheel-python3.10-cuda12.9 - name: Install dependencies timeout-minutes: 20 @@ -1070,7 +1073,7 @@ jobs: with: path: sgl-kernel/dist/ merge-multiple: true - pattern: wheel-python3.10-cuda13.0 + pattern: wheel-python3.10-cuda12.9 - name: Install dependencies timeout-minutes: 20 @@ -1120,7 +1123,7 @@ jobs: with: path: sgl-kernel/dist/ merge-multiple: true - pattern: wheel-python3.10-cuda13.0 + pattern: wheel-python3.10-cuda12.9 - name: Install dependencies timeout-minutes: 20 @@ -1180,7 +1183,7 @@ jobs: with: path: sgl-kernel/dist/ merge-multiple: true - pattern: wheel-python3.10-cuda13.0 + pattern: wheel-python3.10-cuda12.9 - name: Install dependencies timeout-minutes: 20 @@ -1246,7 +1249,7 @@ jobs: with: path: sgl-kernel/dist/ merge-multiple: true - pattern: wheel-python3.10-cuda13.0 + pattern: wheel-python3.10-cuda12.9 - name: Install dependencies timeout-minutes: 20 @@ -1300,7 +1303,7 @@ jobs: # with: # path: sgl-kernel/dist/ # merge-multiple: true - # pattern: wheel-python3.10-cuda13.0-aarch64 + # pattern: wheel-python3.10-cuda12.9-aarch64 # # - name: Install dependencies # timeout-minutes: 20 diff --git a/python/pyproject.toml b/python/pyproject.toml index 80ba96cc6..491958878 100755 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -22,7 +22,7 @@ dependencies = [ "blobfile==3.0.0", "build", "compressed-tensors", - "cuda-python>=13.0", + "cuda-python==12.9", "decord2 ; sys_platform == 'linux' and (platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'armv7l')", "datasets", "einops", @@ -86,14 +86,14 @@ url = "https://pypi.org/simple" default = true [[tool.uv.index]] -name = "torch-cu130" -url = "https://download.pytorch.org/whl/cu130" +name = "torch-cu129" +url = "https://download.pytorch.org/whl/cu129" explicit = true [tool.uv.sources] torch = [ { index = "pypi", marker = "platform_machine == 'x86_64'"}, - { index = "torch-cu130", marker = "platform_machine == 'aarch64'"}, + { index = "torch-cu129", marker = "platform_machine == 'aarch64'"}, ] [project.optional-dependencies] diff --git a/scripts/ci/cuda/ci_download_flashinfer_jit_cache.sh b/scripts/ci/cuda/ci_download_flashinfer_jit_cache.sh index 6358b5caf..ab41dcef2 100755 --- a/scripts/ci/cuda/ci_download_flashinfer_jit_cache.sh +++ b/scripts/ci/cuda/ci_download_flashinfer_jit_cache.sh @@ -5,7 +5,7 @@ # Required environment (caller must export or set): # UNINSTALL_JIT_CACHE — literal true/false (skip download when false) # FLASHINFER_PYTHON_REQUIRED — e.g. from python/pyproject.toml (flashinfer_python) -# CU_VERSION — e.g. cu130 +# CU_VERSION — e.g. cu129 # PIP_CMD — e.g. "pip" or "uv pip" # PIP_INSTALL_SUFFIX — extra pip args for this runner set -euxo pipefail diff --git a/scripts/ci/cuda/ci_install_deepep.sh b/scripts/ci/cuda/ci_install_deepep.sh index b6e188751..bb4185799 100755 --- a/scripts/ci/cuda/ci_install_deepep.sh +++ b/scripts/ci/cuda/ci_install_deepep.sh @@ -100,8 +100,13 @@ if [ "$GRACE_BLACKWELL" = "1" ]; then if [ "$CUDA_VERSION" = "12.8" ]; then CHOSEN_TORCH_CUDA_ARCH_LIST='10.0' elif awk -v ver="$CUDA_VERSION" 'BEGIN {exit !(ver > 12.8)}'; then - # CUDA > 12.8 supports sm_103 (Blackwell) - CHOSEN_TORCH_CUDA_ARCH_LIST='10.0;10.3' + # With cuda > 12.8, the compiler supports 10.3, so we should use + # CHOSEN_TORCH_CUDA_ARCH_LIST='10.0;10.3' + # + # However, our CI machine has a weird setup and nvidia-smi reports wrong CUDA version in the container. + # The container is actually cuda 12.8, but nvidia-smi reports 13.0, leading to compilation errors. so we + # drop 10.3. + CHOSEN_TORCH_CUDA_ARCH_LIST='10.0' else echo "Unsupported CUDA version for Grace Blackwell: $CUDA_VERSION" && exit 1 fi && \ diff --git a/scripts/ci/cuda/ci_install_dependency.sh b/scripts/ci/cuda/ci_install_dependency.sh index 9ac79327c..7735977ef 100755 --- a/scripts/ci/cuda/ci_install_dependency.sh +++ b/scripts/ci/cuda/ci_install_dependency.sh @@ -23,7 +23,7 @@ set -euxo pipefail # Configuration & timing # ------------------------------------------------------------------------------ # Set up environment variables -CU_VERSION="cu130" +CU_VERSION="cu129" # Nvidia package versions we override (torch pins older versions). # Used both as pip constraints during install and for post-install verification. @@ -244,15 +244,7 @@ if [ "${CUSTOM_BUILD_SGL_KERNEL:-}" = "true" ] && [ -d "sgl-kernel/dist" ]; then else WHEEL_ARCH="x86_64" fi - # Wheel may have +cuXYZ suffix (e.g. sglang_kernel-0.4.0+cu130-...) depending on CUDA version - KERNEL_WHL=$(ls sgl-kernel/dist/sglang_kernel-${SGL_KERNEL_VERSION_FROM_KERNEL}*-cp310-abi3-manylinux2014_${WHEEL_ARCH}.whl 2>/dev/null | head -1) - if [ -z "$KERNEL_WHL" ]; then - echo "ERROR: No matching sgl-kernel wheel found in sgl-kernel/dist/ for version ${SGL_KERNEL_VERSION_FROM_KERNEL} arch ${WHEEL_ARCH}" - ls -alh sgl-kernel/dist/ - exit 1 - fi - echo "Installing sgl-kernel wheel: $KERNEL_WHL" - $PIP_CMD install "$KERNEL_WHL" --force-reinstall $PIP_INSTALL_SUFFIX + $PIP_CMD install sgl-kernel/dist/sglang_kernel-${SGL_KERNEL_VERSION_FROM_KERNEL}-cp310-abi3-manylinux2014_${WHEEL_ARCH}.whl --force-reinstall $PIP_INSTALL_SUFFIX elif [ "${CUSTOM_BUILD_SGL_KERNEL:-}" = "true" ] && [ ! -d "sgl-kernel/dist" ]; then # CUSTOM_BUILD_SGL_KERNEL was set but artifacts not available (e.g., stage rerun without wheel build) # Fail instead of falling back to PyPI - we need to test the built kernel, not PyPI version @@ -326,10 +318,10 @@ mark_step_done "Install extra dependency" # Fix other dependencies # ------------------------------------------------------------------------------ # Fix CUDA version mismatch between torch and torchaudio. -# PyPI's torch bundles a specific CUDA version but torchaudio from pytorch.org/cu130 may use a different one. +# PyPI's torch 2.9.1 bundles cu128 but torchaudio from pytorch.org/cu129 uses cu129. # This mismatch causes torchaudio's C extension to fail loading, producing: # "partially initialized module 'torchaudio' has no attribute 'lib'" -# We cannot replace torch with a different CUDA version (breaks sgl_kernel ABI), so instead we reinstall +# We cannot replace torch with cu129 (breaks sgl_kernel ABI), so instead we reinstall # torchaudio/torchvision from an index matching torch's CUDA version. TORCH_CUDA_VER=$(python3 -c "import torch; v=torch.version.cuda; parts=v.split('.'); print(f'cu{parts[0]}{parts[1]}')") echo "Detected torch CUDA version: ${TORCH_CUDA_VER}"