From 3f4fbc165d83e17ce3732471465121a326d9db90 Mon Sep 17 00:00:00 2001 From: Alison Shao <54658187+alisonshao@users.noreply.github.com> Date: Sun, 12 Apr 2026 23:48:40 -0500 Subject: [PATCH] Upgrade CI default CUDA version from 12.9 to 13.0 (#21441) --- .github/workflows/pr-test.yml | 31 +++++++++---------- python/pyproject.toml | 8 ++--- .../cuda/ci_download_flashinfer_jit_cache.sh | 2 +- scripts/ci/cuda/ci_install_deepep.sh | 9 ++---- scripts/ci/cuda/ci_install_dependency.sh | 16 +++++++--- 5 files changed, 33 insertions(+), 33 deletions(-) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 8feb364bd..7c50a46d5 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -429,10 +429,7 @@ jobs: matrix: include: - python-version: "3.10" - cuda-version: "12.9" - # Add back when CUDA 13.0 is supported on CI - # - python-version: "3.10" - # cuda-version: "13.0" + cuda-version: "13.0" name: Build Wheel steps: - name: Cleanup @@ -480,7 +477,7 @@ jobs: matrix: include: - python-version: "3.10" - cuda-version: "12.9" + cuda-version: "13.0" name: Build Wheel Arm steps: - name: Cleanup @@ -587,7 +584,7 @@ jobs: with: path: sgl-kernel/dist/ merge-multiple: true - pattern: wheel-python3.10-cuda12.9 + pattern: wheel-python3.10-cuda13.0 - name: Install dependencies timeout-minutes: 20 @@ -694,7 +691,7 @@ jobs: with: path: sgl-kernel/dist/ merge-multiple: true - pattern: wheel-python3.10-cuda12.9 + pattern: wheel-python3.10-cuda13.0 - name: Install dependencies timeout-minutes: 20 @@ -752,7 +749,7 @@ jobs: with: path: sgl-kernel/dist/ merge-multiple: true - pattern: wheel-python3.10-cuda12.9 + pattern: wheel-python3.10-cuda13.0 - name: Install dependencies timeout-minutes: 20 @@ -807,7 +804,7 @@ jobs: with: path: sgl-kernel/dist/ merge-multiple: true - pattern: wheel-python3.10-cuda12.9 + pattern: wheel-python3.10-cuda13.0 - name: Install dependencies timeout-minutes: 20 @@ -860,7 +857,7 @@ jobs: with: path: sgl-kernel/dist/ merge-multiple: true - pattern: wheel-python3.10-cuda12.9 + pattern: wheel-python3.10-cuda13.0 - name: Install dependencies timeout-minutes: 20 @@ -949,7 +946,7 @@ jobs: with: path: sgl-kernel/dist/ merge-multiple: true - pattern: wheel-python3.10-cuda12.9 + pattern: wheel-python3.10-cuda13.0 - name: Install dependencies timeout-minutes: 20 @@ -1003,7 +1000,7 @@ jobs: with: path: sgl-kernel/dist/ merge-multiple: true - pattern: wheel-python3.10-cuda12.9 + pattern: wheel-python3.10-cuda13.0 - name: Install dependencies timeout-minutes: 20 @@ -1073,7 +1070,7 @@ jobs: with: path: sgl-kernel/dist/ merge-multiple: true - pattern: wheel-python3.10-cuda12.9 + pattern: wheel-python3.10-cuda13.0 - name: Install dependencies timeout-minutes: 20 @@ -1123,7 +1120,7 @@ jobs: with: path: sgl-kernel/dist/ merge-multiple: true - pattern: wheel-python3.10-cuda12.9 + pattern: wheel-python3.10-cuda13.0 - name: Install dependencies timeout-minutes: 20 @@ -1183,7 +1180,7 @@ jobs: with: path: sgl-kernel/dist/ merge-multiple: true - pattern: wheel-python3.10-cuda12.9 + pattern: wheel-python3.10-cuda13.0 - name: Install dependencies timeout-minutes: 20 @@ -1249,7 +1246,7 @@ jobs: with: path: sgl-kernel/dist/ merge-multiple: true - pattern: wheel-python3.10-cuda12.9 + pattern: wheel-python3.10-cuda13.0 - name: Install dependencies timeout-minutes: 20 @@ -1303,7 +1300,7 @@ jobs: # with: # path: sgl-kernel/dist/ # merge-multiple: true - # pattern: wheel-python3.10-cuda12.9-aarch64 + # pattern: wheel-python3.10-cuda13.0-aarch64 # # - name: Install dependencies # timeout-minutes: 20 diff --git a/python/pyproject.toml b/python/pyproject.toml index 538ad4908..7eeb44f59 100755 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -22,7 +22,7 @@ dependencies = [ "blobfile==3.0.0", "build", "compressed-tensors", - "cuda-python==12.9", + "cuda-python>=13.0", "decord2 ; sys_platform == 'linux' and (platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'armv7l')", "datasets", "einops", @@ -86,14 +86,14 @@ url = "https://pypi.org/simple" default = true [[tool.uv.index]] -name = "torch-cu129" -url = "https://download.pytorch.org/whl/cu129" +name = "torch-cu130" +url = "https://download.pytorch.org/whl/cu130" explicit = true [tool.uv.sources] torch = [ { index = "pypi", marker = "platform_machine == 'x86_64'"}, - { index = "torch-cu129", marker = "platform_machine == 'aarch64'"}, + { index = "torch-cu130", marker = "platform_machine == 'aarch64'"}, ] [project.optional-dependencies] diff --git a/scripts/ci/cuda/ci_download_flashinfer_jit_cache.sh b/scripts/ci/cuda/ci_download_flashinfer_jit_cache.sh index ab41dcef2..6358b5caf 100755 --- a/scripts/ci/cuda/ci_download_flashinfer_jit_cache.sh +++ b/scripts/ci/cuda/ci_download_flashinfer_jit_cache.sh @@ -5,7 +5,7 @@ # Required environment (caller must export or set): # UNINSTALL_JIT_CACHE — literal true/false (skip download when false) # FLASHINFER_PYTHON_REQUIRED — e.g. from python/pyproject.toml (flashinfer_python) -# CU_VERSION — e.g. cu129 +# CU_VERSION — e.g. cu130 # PIP_CMD — e.g. "pip" or "uv pip" # PIP_INSTALL_SUFFIX — extra pip args for this runner set -euxo pipefail diff --git a/scripts/ci/cuda/ci_install_deepep.sh b/scripts/ci/cuda/ci_install_deepep.sh index bb4185799..b6e188751 100755 --- a/scripts/ci/cuda/ci_install_deepep.sh +++ b/scripts/ci/cuda/ci_install_deepep.sh @@ -100,13 +100,8 @@ if [ "$GRACE_BLACKWELL" = "1" ]; then if [ "$CUDA_VERSION" = "12.8" ]; then CHOSEN_TORCH_CUDA_ARCH_LIST='10.0' elif awk -v ver="$CUDA_VERSION" 'BEGIN {exit !(ver > 12.8)}'; then - # With cuda > 12.8, the compiler supports 10.3, so we should use - # CHOSEN_TORCH_CUDA_ARCH_LIST='10.0;10.3' - # - # However, our CI machine has a weird setup and nvidia-smi reports wrong CUDA version in the container. - # The container is actually cuda 12.8, but nvidia-smi reports 13.0, leading to compilation errors. so we - # drop 10.3. - CHOSEN_TORCH_CUDA_ARCH_LIST='10.0' + # CUDA > 12.8 supports sm_103 (Blackwell) + CHOSEN_TORCH_CUDA_ARCH_LIST='10.0;10.3' else echo "Unsupported CUDA version for Grace Blackwell: $CUDA_VERSION" && exit 1 fi && \ diff --git a/scripts/ci/cuda/ci_install_dependency.sh b/scripts/ci/cuda/ci_install_dependency.sh index 7735977ef..9ac79327c 100755 --- a/scripts/ci/cuda/ci_install_dependency.sh +++ b/scripts/ci/cuda/ci_install_dependency.sh @@ -23,7 +23,7 @@ set -euxo pipefail # Configuration & timing # ------------------------------------------------------------------------------ # Set up environment variables -CU_VERSION="cu129" +CU_VERSION="cu130" # Nvidia package versions we override (torch pins older versions). # Used both as pip constraints during install and for post-install verification. @@ -244,7 +244,15 @@ if [ "${CUSTOM_BUILD_SGL_KERNEL:-}" = "true" ] && [ -d "sgl-kernel/dist" ]; then else WHEEL_ARCH="x86_64" fi - $PIP_CMD install sgl-kernel/dist/sglang_kernel-${SGL_KERNEL_VERSION_FROM_KERNEL}-cp310-abi3-manylinux2014_${WHEEL_ARCH}.whl --force-reinstall $PIP_INSTALL_SUFFIX + # Wheel may have +cuXYZ suffix (e.g. sglang_kernel-0.4.0+cu130-...) depending on CUDA version + KERNEL_WHL=$(ls sgl-kernel/dist/sglang_kernel-${SGL_KERNEL_VERSION_FROM_KERNEL}*-cp310-abi3-manylinux2014_${WHEEL_ARCH}.whl 2>/dev/null | head -1) + if [ -z "$KERNEL_WHL" ]; then + echo "ERROR: No matching sgl-kernel wheel found in sgl-kernel/dist/ for version ${SGL_KERNEL_VERSION_FROM_KERNEL} arch ${WHEEL_ARCH}" + ls -alh sgl-kernel/dist/ + exit 1 + fi + echo "Installing sgl-kernel wheel: $KERNEL_WHL" + $PIP_CMD install "$KERNEL_WHL" --force-reinstall $PIP_INSTALL_SUFFIX elif [ "${CUSTOM_BUILD_SGL_KERNEL:-}" = "true" ] && [ ! -d "sgl-kernel/dist" ]; then # CUSTOM_BUILD_SGL_KERNEL was set but artifacts not available (e.g., stage rerun without wheel build) # Fail instead of falling back to PyPI - we need to test the built kernel, not PyPI version @@ -318,10 +326,10 @@ mark_step_done "Install extra dependency" # Fix other dependencies # ------------------------------------------------------------------------------ # Fix CUDA version mismatch between torch and torchaudio. -# PyPI's torch 2.9.1 bundles cu128 but torchaudio from pytorch.org/cu129 uses cu129. +# PyPI's torch bundles a specific CUDA version but torchaudio from pytorch.org/cu130 may use a different one. # This mismatch causes torchaudio's C extension to fail loading, producing: # "partially initialized module 'torchaudio' has no attribute 'lib'" -# We cannot replace torch with cu129 (breaks sgl_kernel ABI), so instead we reinstall +# We cannot replace torch with a different CUDA version (breaks sgl_kernel ABI), so instead we reinstall # torchaudio/torchvision from an index matching torch's CUDA version. TORCH_CUDA_VER=$(python3 -c "import torch; v=torch.version.cuda; parts=v.split('.'); print(f'cu{parts[0]}{parts[1]}')") echo "Detected torch CUDA version: ${TORCH_CUDA_VER}"