[AMD][CI] Publish ROCm 10 release images and kernel wheel (#38763)

.github/workflows/release-docker-amd.yml

@@ -15,6 +15,7 @@ on:
         default: 'all'
         options:
           - 'all'
+          - rocm10
           - rocm724
           - rocm720
           - rocm700
@@ -25,11 +26,11 @@ jobs:
     runs-on: amd-docker-scale
     environment: 'prod'
     strategy:
-      # Six flavors publish independently: a failure in one must not cancel the
+      # Eight flavors publish independently: a failure in one must not cancel
-      # others mid-push and leave the release tag with a partial image set.
+      # the others mid-push and leave the release tag with a partial image set.
       fail-fast: false
       matrix:
-        rocm_version: ${{ fromJson((github.event_name == 'workflow_dispatch' && inputs.rocm_version != 'all' && inputs.rocm_version != '') && format('["{0}"]', inputs.rocm_version) || '["rocm700", "rocm720", "rocm724"]') }}
+        rocm_version: ${{ fromJson((github.event_name == 'workflow_dispatch' && inputs.rocm_version != 'all' && inputs.rocm_version != '') && format('["{0}"]', inputs.rocm_version) || '["rocm700", "rocm720", "rocm724", "rocm10"]') }}
         gpu_arch: ['gfx942', 'gfx950']
         build_type: ['all']
     steps:
@@ -89,6 +90,19 @@ jobs:
               echo "Unsupported gfx arch"
               exit 1
             fi
+          elif [ "${{ matrix.rocm_version }}" = "rocm10" ]; then
+            # The Dockerfile stages carry the full SDK version (gfx942-rocm1000)
+            # while the published tag keeps the rocm10 flavor name that CI and
+            # the nightly images already use.
+            gpu_arch_suffix="-rocm1000"
+            if [ "${{ matrix.gpu_arch }}" = "gfx942" ]; then
+              rocm_tag="rocm10-mi30x"
+            elif [ "${{ matrix.gpu_arch }}" = "gfx950" ]; then
+              rocm_tag="rocm10-mi35x"
+            else
+              echo "Unsupported gfx arch"
+              exit 1
+            fi
           else
             echo "Unsupported rocm version"
             exit 1

.github/workflows/release-whl-kernel.yml

@@ -19,6 +19,7 @@ on:
           - 'cu130'
           - 'rocm700'
           - 'rocm720'
+          - 'rocm1000'
           - 'musa43'
       tag_name:
         description: "Version number, must be in the form of vX.Y.Z (e.g. v0.4.0)"
@@ -273,12 +274,20 @@ jobs:
   build-rocm-matrix:
     if: |
       github.repository == 'sgl-project/sglang' &&
-      (github.event_name == 'push' || github.event.inputs.target == 'all' || github.event.inputs.target == 'rocm700' || github.event.inputs.target == 'rocm720')
+      (github.event_name == 'push' || github.event.inputs.target == 'all' || github.event.inputs.target == 'rocm700' || github.event.inputs.target == 'rocm720' || github.event.inputs.target == 'rocm1000')
     runs-on: amd-docker-scale
     strategy:
       matrix:
         python-version: ["3.10"]
-        rocm-version: ["700", "720"]
+        # ROCm 10.0.0 builds inside a Python 3.12 container; the wheel is still
+        # tagged cp310-abi3 (wheel.py-api in pyproject.toml), so it stays under
+        # the same python-version leg as the 7.x wheels.
+        #
+        # A dispatch that names one flavor builds only that flavor. Every extra
+        # leg is another image build on the shared AMD node, and its release job
+        # appends the same wheel to the index a second time. `inputs` is empty
+        # on push, which falls through to the full set.
+        rocm-version: ${{ fromJson(inputs.target == 'rocm700' && '["700"]' || inputs.target == 'rocm720' && '["720"]' || inputs.target == 'rocm1000' && '["1000"]' || '["700", "720", "1000"]') }}
     steps:
       # Self-hosted build nodes retain the workspace across jobs. Prior builds
       # leave root-owned artifacts under python/sglang/kernels/aot/build/ that actions/checkout
@@ -314,6 +323,8 @@ jobs:
 
   release-rocm700:
     needs: build-rocm-matrix
+    # Only publish the flavors this run built; see build-rocm-matrix.
+    if: ${{ github.event_name == 'push' || inputs.target == 'all' || inputs.target == 'rocm700' }}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -365,6 +376,7 @@ jobs:
 
   release-rocm720:
     needs: build-rocm-matrix
+    if: ${{ github.event_name == 'push' || inputs.target == 'all' || inputs.target == 'rocm720' }}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -414,6 +426,58 @@ jobs:
           git commit -m "update whl index"
           git push
 
+  release-rocm1000:
+    needs: build-rocm-matrix
+    if: ${{ github.event_name == 'push' || inputs.target == 'all' || inputs.target == 'rocm1000' }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.pr_number && format('refs/pull/{0}/head', inputs.pr_number) || '' }}
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: python/sglang/kernels/aot/dist/
+          merge-multiple: true
+          pattern: wheel-*-rocm1000
+
+      - name: Set tag name
+        id: set_tag_name
+        run: |
+          if [ -z "${{ inputs.tag_name }}" ]; then
+            TAG_NAME="v$(cat python/sglang/kernels/aot/python/sgl_kernel/version.py | cut -d'"' -f2)"
+            echo "tag_name=$TAG_NAME" >> $GITHUB_OUTPUT
+          else
+            echo "tag_name=${{ inputs.tag_name }}" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: ${{ steps.set_tag_name.outputs.tag_name }}
+          repository: sgl-project/whl
+          token: ${{ secrets.GH_PAT_FOR_WHL_RELEASE }}
+          files: |
+            python/sglang/kernels/aot/dist/*
+
+      - name: Clone wheel index
+        run: git clone https://oauth2:${WHL_TOKEN}@github.com/sgl-project/whl.git sgl-whl
+        env:
+          WHL_TOKEN: ${{ secrets.GH_PAT_FOR_WHL_RELEASE }}
+
+      - name: Update wheel index
+        run: python3 scripts/update_kernel_whl_index.py --rocm 1000
+
+      - name: Push wheel index
+        run: |
+          cd sgl-whl
+          git config --local user.name "sglang-bot"
+          git config --local user.email "sglangbot@gmail.com"
+          git add -A
+          git commit -m "update whl index"
+          git push
+
   build-musa43:
     if: |
       github.repository == 'sgl-project/sglang' &&

3rdparty/amd/wheel/sgl-kernel/build_rocm.sh

@@ -7,14 +7,57 @@ if [[ "${ROCM_VERSION}" == "700" ]]; then
   IMAGE="lmsysorg/sglang:v0.5.8.post1-rocm700-mi35x"
 elif [[ "${ROCM_VERSION}" == "720" ]]; then
   IMAGE="rocm/pytorch:rocm7.2_ubuntu22.04_py3.10_pytorch_release_2.9.1"
+elif [[ "${ROCM_VERSION}" == "1000" ]]; then
+  # Ubuntu 24.04 / Python 3.12 / torch 2.11 matches the stack the released
+  # ROCm 10 images carry, and this image is built for every device arch, so one
+  # wheel can still cover both gfx942 and gfx950.
+  IMAGE="rocm/pytorch:rocm10.0_ubuntu24.04_py3.12_pytorch_release_2.11.0"
 else
-  echo "ERROR: Unsupported ROCM_VERSION='${ROCM_VERSION}'. Only '700' and '720' are supported." >&2
+  echo "ERROR: Unsupported ROCM_VERSION='${ROCM_VERSION}'. Only '700', '720' and '1000' are supported." >&2
   exit 1
 fi
 
 PYTHON_ROOT_PATH="/opt/venv/bin"
 AMDGPU_TARGET="gfx942;gfx950"
 
+# ROCm 10.0.0 is distributed as pip packages that unpack into site-packages, so
+# the image has neither the /opt/rocm tree CMakeLists_rocm.txt looks for
+# hip-lang under nor, being a runtime image, a devel tree to compile against.
+# These are the same fixups docker/rocm.Dockerfile's rocm1000-base stage makes
+# for the released images.
+ROCM_SETUP=""
+if [[ "${ROCM_VERSION}" == "1000" ]]; then
+  ROCM_SETUP=$(cat <<'ROCM1000_SETUP'
+  set -e
+  apt-get update && apt-get install -y --no-install-recommends build-essential ca-certificates wget
+  rocm_sdk_version=$(pip show rocm-sdk-core | awk '/^Version:/ {print $2}')
+  test -n "${rocm_sdk_version}"
+  pip install --no-cache-dir --index-url https://stable.repo.amd.com/rocm/whl-next/ "rocm-sdk-devel==${rocm_sdk_version}"
+  rocm-sdk init
+  export ROCM_HOME=$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')/_rocm_sdk_devel
+  export ROCM_PATH="${ROCM_HOME}"
+  export CPATH="${ROCM_HOME}/include"
+  export LIBRARY_PATH="${ROCM_HOME}/lib"
+  export LD_LIBRARY_PATH="${ROCM_HOME}/lib"
+  export PATH="${ROCM_HOME}/llvm/bin:${ROCM_HOME}/bin:${PATH}"
+  ln -sfn "${ROCM_HOME}" /opt/rocm
+  # The SDK's hsakmtTargets.cmake hardcodes /usr/lib64/libc.so from its own
+  # build host; Ubuntu keeps libc in /lib/x86_64-linux-gnu.
+  mkdir -p /usr/lib64 && ln -sf /lib/x86_64-linux-gnu/libc.so /usr/lib64/libc.so
+ROCM1000_SETUP
+)
+fi
+
+# ROCm 10 has no /opt/rocm-<version> directory for rename_wheels_rocm.sh to read
+# the local version tag from, so name it here instead. Keeping it in step with
+# the `--rocm` argument release-whl-kernel.yml passes to
+# scripts/update_kernel_whl_index.py is what puts the wheel in the index it
+# gets published under.
+WHEEL_ROCM_VERSION=""
+if [[ "${ROCM_VERSION}" == "1000" ]]; then
+  WHEEL_ROCM_VERSION="1000"
+fi
+
 # Pull and run the latest image
 echo "Pulling Docker image: ${IMAGE}"
 docker pull "${IMAGE}"
@@ -31,6 +74,7 @@ docker run --rm \
   elif [[ "${ROCM_VERSION}" == "720" ]]; then
     ${PYTHON_ROOT_PATH}/pip install https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torch-2.9.1%2Brocm7.2.0.lw.git7e1940d4-cp310-cp310-linux_x86_64.whl https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/triton-3.5.1%2Brocm7.2.0.gita272dfa8-cp310-cp310-linux_x86_64.whl https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torchaudio-2.9.0%2Brocm7.2.0.gite3c6ee2b-cp310-cp310-linux_x86_64.whl https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torchvision-0.24.0%2Brocm7.2.0.gitb919bd0c-cp310-cp310-linux_x86_64.whl
   fi
+${ROCM_SETUP}
   # Install CMake (version >= 3.26) - Robust Installation
   export CMAKE_VERSION_MAJOR=3.31
   export CMAKE_VERSION_MINOR=1
@@ -46,5 +90,5 @@ docker run --rm \
   rm -rf CMakeLists.txt && mv CMakeLists_rocm.txt CMakeLists.txt && \
   ${PYTHON_ROOT_PATH}/python rocm_hipify.py && \
   ${PYTHON_ROOT_PATH}/python -m uv build --wheel -Cbuild-dir=build . --color=always --no-build-isolation && \
-  ./rename_wheels_rocm.sh
+  ./rename_wheels_rocm.sh ${WHEEL_ROCM_VERSION}
 "

3rdparty/amd/wheel/sgl-kernel/rename_wheels_rocm.sh

@@ -3,6 +3,11 @@ set -ex
 
 WHEEL_DIR="dist"
 
+# ROCm version for the wheel's local version tag (e.g. 720 -> +rocm720). It is
+# read from the /opt/rocm-<version> tree when the caller does not name one; the
+# pip-installed ROCm 10 SDK has no such tree, so that flavor passes it in.
+ROCM_WHEEL_VERSION="${1:-}"
+
 wheel_files=($WHEEL_DIR/*.whl)
 for wheel in "${wheel_files[@]}"; do
     intermediate_wheel="${wheel/linux/manylinux2014}"
@@ -17,7 +22,10 @@ for wheel in "${wheel_files[@]}"; do
     fi
 
     # Detect ROCm version and add appropriate suffix
-    ver_abrv=$(realpath /opt/rocm-* | sed -e 's/.*-//' -e 's/\.//g')
+    ver_abrv="${ROCM_WHEEL_VERSION}"
+    if [[ -z "$ver_abrv" ]]; then
+        ver_abrv=$(realpath /opt/rocm-* | sed -e 's/.*-//' -e 's/\.//g')
+    fi
     new_wheel=${intermediate_wheel/-cp${cp_version}/+rocm${ver_abrv}-cp${cp_version}}
 
     if [[ "$wheel" != "$new_wheel" ]]; then