[AMD] Add ROCm 10 (gfx942 / gfx950) release images (#36434)

.github/workflows/nightly-test-amd-rocm720.yml

@@ -11,12 +11,13 @@ on:
   workflow_dispatch:
     inputs:
       rocm_version:
-        description: 'ROCm image version ("all" runs every flavor, as the nightly schedule does)'
+        description: 'ROCm image version ("all" runs the scheduled ROCm 7.2 matrix; select rocm10 explicitly)'
         required: false
         type: choice
         default: 'all'
         options:
           - 'all'
+          - rocm10
           - rocm724
           - rocm720
       aiter_ref:
@@ -110,7 +111,7 @@ on:
   workflow_call:
     inputs:
       rocm_version:
-        description: 'ROCm image version ("all" runs every flavor)'
+        description: 'ROCm image version ("all" runs the scheduled ROCm 7.2 matrix; pass rocm10 explicitly)'
         required: false
         type: string
         # A single flavor, unlike the schedule and the dispatch form: a caller
@@ -444,7 +445,7 @@ jobs:
 
       - name: Install dependencies
         run: bash scripts/ci/amd/amd_ci_install_dependency.sh
-      - name: Performance Test ROCm 7.2 (2-GPU Text Models)
+      - name: Performance Test (2-GPU Text Models)
         timeout-minutes: 120
         run: |
           > github_summary.md  # Clear summary file
@@ -1008,7 +1009,7 @@ jobs:
           echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
           exit ${TEST_EXIT_CODE:-0}
 
-      - name: Performance Test MI35x ROCm 7.2 (8-GPU DeepSeek-V3.2 Basic)
+      - name: Performance Test MI35x (8-GPU DeepSeek-V3.2 Basic)
         timeout-minutes: 150
         continue-on-error: true
         run: |

.github/workflows/pr-test-amd-extra.yml

@@ -42,6 +42,7 @@ on:
         type: choice
         default: rocm724
         options:
+          - rocm10
           - rocm724
           - rocm720
           - rocm700

.github/workflows/pr-test-amd-rocm720.yml

@@ -22,6 +22,7 @@ on:
         type: choice
         default: rocm724
         options:
+          - rocm10
           - rocm724
           - rocm720
       target_stage_select:

.github/workflows/release-docker-amd-rocm10.yml

@@ -0,0 +1,118 @@
+name: Release Docker Images Nightly ROCm 10 (AMD)
+on:
+  workflow_dispatch:
+    inputs:
+      job_select:
+        description: 'Select which release job to run'
+        required: false
+        type: choice
+        default: 'all'
+        options:
+          - 'all'
+          - publish
+      gpu_arch:
+        description: 'Select which ROCm 10 GPU arch to build'
+        required: false
+        type: choice
+        default: 'all'
+        options:
+          - 'all'
+          - gfx942-rocm1000
+          - gfx950-rocm1000
+  schedule:
+    - cron: '0 12 * * *'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.number || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  publish:
+    if: github.repository == 'sgl-project/sglang' && (github.event_name != 'workflow_dispatch' || inputs.job_select == 'all' || inputs.job_select == 'publish')
+    runs-on: amd-docker-scale
+    environment: 'prod'
+    strategy:
+      fail-fast: false
+      matrix:
+        gpu_arch: ${{ fromJson((github.event_name == 'workflow_dispatch' && inputs.gpu_arch != 'all' && inputs.gpu_arch != '') && format('["{0}"]', inputs.gpu_arch) || '["gfx942-rocm1000", "gfx950-rocm1000"]') }}
+        build_type: ['all']
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Set date
+        run: echo "DATE=$(date +%Y%m%d)" >> "$GITHUB_ENV"
+
+      - name: Get version from latest tag
+        id: version
+        run: |
+          VERSION=$(python3 scripts/release/get_version_tag.py --tag-only | sed 's/^v//')
+
+          if [ -z "$VERSION" ]; then
+            echo "::error::Could not determine version from git tags"
+            exit 1
+          fi
+
+          COMMIT_HASH=$(git rev-parse --short HEAD)
+          PRETEND_VERSION="${VERSION}.dev${DATE}+g${COMMIT_HASH}"
+
+          echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
+          echo "pretend_version=${PRETEND_VERSION}" >> "$GITHUB_OUTPUT"
+          echo "Detected version: ${VERSION}"
+          echo "Pretend version for pip: ${PRETEND_VERSION}"
+
+      - name: Login to Docker Hub (AMD)
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_AMD_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_AMD_TOKEN }}
+
+      - name: Build and push to rocm/sgl-dev
+        env:
+          BUILD_TYPE: ${{ matrix.build_type }}
+          GPU_ARCH: ${{ matrix.gpu_arch }}
+          PRETEND_VERSION: ${{ steps.version.outputs.pretend_version }}
+          VERSION: ${{ steps.version.outputs.version }}
+        run: |
+          case "$GPU_ARCH" in
+            gfx942-rocm1000) rocm_tag="rocm10-mi30x" ;;
+            gfx950-rocm1000) rocm_tag="rocm10-mi35x" ;;
+            *)
+              echo "Unsupported GPU arch: $GPU_ARCH"
+              exit 1
+              ;;
+          esac
+
+          image_tag="v${VERSION}-${rocm_tag}-${DATE}"
+          echo "IMAGE_TAG=${image_tag}" >> "$GITHUB_ENV"
+
+          docker build . \
+            -f docker/rocm.Dockerfile \
+            --build-arg "SGL_BRANCH=${GITHUB_SHA}" \
+            --build-arg "BUILD_TYPE=${BUILD_TYPE}" \
+            --build-arg "GPU_ARCH=${GPU_ARCH}" \
+            --build-arg ENABLE_MORI=1 \
+            --build-arg ENABLE_NIXL=1 \
+            --build-arg "SETUPTOOLS_SCM_PRETEND_VERSION=${PRETEND_VERSION}" \
+            --build-arg UBUNTU_MIRROR=https://archive.ubuntu.com \
+            -t "rocm/sgl-dev:${image_tag}" \
+            --no-cache
+          docker push "rocm/sgl-dev:${image_tag}"
+
+      - name: Login to Docker Hub (lmsys)
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Push to lmsysorg/sglang-rocm
+        run: |
+          docker tag "rocm/sgl-dev:${IMAGE_TAG}" "lmsysorg/sglang-rocm:${IMAGE_TAG}"
+          docker push "lmsysorg/sglang-rocm:${IMAGE_TAG}"

docker/rocm.Dockerfile

@@ -5,11 +5,16 @@
 #   docker build --build-arg SGL_BRANCH=v0.5.10.post1 --build-arg GPU_ARCH=gfx950 -t v0.5.10.post1-rocm700-mi35x -f rocm.Dockerfile .
 #   docker build --build-arg SGL_BRANCH=v0.5.10.post1 --build-arg GPU_ARCH=gfx950-rocm720 -t v0.5.10.post1-rocm720-mi35x -f rocm.Dockerfile .
 #   docker build --build-arg SGL_BRANCH=v0.5.10.post1 --build-arg GPU_ARCH=gfx950-rocm724 -t v0.5.10.post1-rocm724-mi35x -f rocm.Dockerfile .
+#   docker build --build-arg SGL_BRANCH=v0.5.10.post1 --build-arg GPU_ARCH=gfx942-rocm1000 -t v0.5.10.post1-rocm1000-mi30x -f rocm.Dockerfile .
+#   docker build --build-arg SGL_BRANCH=v0.5.10.post1 --build-arg GPU_ARCH=gfx950-rocm1000 -t v0.5.10.post1-rocm1000-mi35x -f rocm.Dockerfile .
 #
 # Flavor notes:
 #   GPU_ARCH=*-rocm724 is built on a Python 3.12 base and upgrades the stack to
 #   torch 2.11 (+torchvision 0.26 / torchaudio 2.11) and Triton 3.7.
 #   The ROCm 7.2.0 flavors remain on Python 3.10 and torch 2.9.1.
+#   GPU_ARCH=*-rocm1000 is Python 3.12 + torch 2.11, and takes the
+#   whole ROCm stack from AMD's stable wheel channel rather than an apt
+#   ROCm base image; see the rocm1000-base stage for what that changes.
 
 # Usage (to build SGLang ROCm + Mori docker image):
 # remove --build-arg NIC_BACKEND=ainic since new MoRI JIT will do NIC auto detection on target
@@ -34,6 +39,11 @@ ARG BASE_IMAGE_942_ROCM724="rocm/pytorch:rocm7.2.4_ubuntu24.04_py3.12_pytorch_re
 ARG BASE_IMAGE_950="rocm/sgl-dev:rocm7-vllm-20250904"
 ARG BASE_IMAGE_950_ROCM720="rocm/pytorch:rocm7.2_ubuntu22.04_py3.10_pytorch_release_2.9.1"
 ARG BASE_IMAGE_950_ROCM724="rocm/pytorch:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0"
+# The ROCm 10.0.0 flavors default to the rocm1000-base stage below rather
+# than a published image; point these at one to build on a prebuilt base.
+ARG BASE_IMAGE_942_ROCM1000="rocm1000-base"
+ARG BASE_IMAGE_950_ROCM1000="rocm1000-base"
+ARG BASE_IMAGE_ROCM1000="ubuntu:24.04"
 
 # This is necessary for scope purpose
 ARG GPU_ARCH=gfx950
@@ -126,6 +136,167 @@ RUN mkdir -p /etc/sglang/constraints && : > /etc/sglang/constraints/torch-rocm.t
 # IPC behavior.
 ENV HSA_ENABLE_IPC_MODE_LEGACY=1
 
+# ===============================
+# Shared ROCm 10.0.0 base for gfx942 and gfx950. Assemble the stack from
+# AMD's stable wheels on a plain Ubuntu base so each output image carries only
+# its own GPU device payload.
+# The SDK lands in site-packages instead of /opt/rocm, which the rest of this
+# Dockerfile and AITER both assume, hence the path fixups below.
+#
+# This is deliberately AMD's stable channel, not a prerelease or nightly.
+# Every ROCm/PyTorch artifact below is pinned to the 10.0.0 release.
+#
+# Python 3.12 (the Ubuntu 24.04 default) rather than 3.13/3.14: st_attn==0.0.7,
+# vsa==0.0.4, petit_kernel==0.0.2 and wave-lang==3.8.2 publish wheels only up to
+# cp313 and no sdist, so pip has no candidate at all for srt_hip on 3.14.
+FROM $BASE_IMAGE_ROCM1000 AS rocm1000-base
+
+# Redeclare the global selector inside this stage so each matrix build installs
+# only the device payload for its target image (gfx942 or gfx950).
+ARG GPU_ARCH
+
+# ROCM_TRITON_VERSION rather than TRITON_VERSION: the final stage declares a
+# TRITON_VERSION of its own for the ROCm 7.2 wheel, and a --build-arg would
+# otherwise land on both.
+ARG ROCM_SDK_VERSION="10.0.0"
+ARG ROCM_TORCH_VERSION="2.11.0"
+ARG ROCM_TORCHVISION_VERSION="0.26.0"
+ARG ROCM_TORCHAUDIO_VERSION="2.11.0"
+ARG ROCM_TRITON_VERSION="3.8.0+git4cff872c"
+ARG ROCM_INDEX_URL="https://stable.repo.amd.com/rocm/whl-next/"
+# Keep device targets data-driven: adding a new image should require one list
+# entry here, not another pairwise OTHER_ROCM_DEVICE_ARCH mapping.
+ARG ROCM_DEVICE_ARCH_LIST="gfx942 gfx950 gfx1250"
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        build-essential \
+        ca-certificates \
+        curl \
+        git \
+        gnupg \
+        libstdc++-12-dev \
+        python-is-python3 \
+        python3 \
+        python3-dev \
+        python3-pip \
+        python3.12-venv \
+        wget \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV VIRTUAL_ENV=/opt/venv
+RUN python3 -m venv "$VIRTUAL_ENV"
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+RUN python3 -m pip install --no-cache-dir -U pip setuptools setuptools_scm wheel
+
+# The two release jobs invoke separate Docker builds. Derive the device target
+# from GPU_ARCH so the MI300 image carries only gfx942 wheels and the MI350
+# image carries only gfx950 wheels. Keeping the packages as explicit specs also
+# makes the intended per-image device payload explicit to the resolver.
+RUN set -eux; \
+    ROCM_DEVICE_ARCH="${GPU_ARCH%%-*}"; \
+    case " ${ROCM_DEVICE_ARCH_LIST} " in \
+      *" ${ROCM_DEVICE_ARCH} "*) ;; \
+      *) echo "Unsupported ROCm 10.0.0 GPU_ARCH=${GPU_ARCH}"; exit 1 ;; \
+    esac; \
+    python3 -m pip install --no-cache-dir \
+        --index-url ${ROCM_INDEX_URL} \
+        "rocm-sdk-core==${ROCM_SDK_VERSION}" \
+        "rocm-sdk-libraries==${ROCM_SDK_VERSION}" \
+        "rocm-sdk-devel==${ROCM_SDK_VERSION}" \
+        "rocm-sdk-device-${ROCM_DEVICE_ARCH}==${ROCM_SDK_VERSION}" \
+        "torch==${ROCM_TORCH_VERSION}+rocm${ROCM_SDK_VERSION}" \
+        "torchvision==${ROCM_TORCHVISION_VERSION}+rocm${ROCM_SDK_VERSION}" \
+        "torchaudio==${ROCM_TORCHAUDIO_VERSION}+rocm${ROCM_SDK_VERSION}" \
+        "amd-torch-device-${ROCM_DEVICE_ARCH}==${ROCM_TORCH_VERSION}+rocm${ROCM_SDK_VERSION}" \
+        "amd-torchvision-device-${ROCM_DEVICE_ARCH}==${ROCM_TORCHVISION_VERSION}+rocm${ROCM_SDK_VERSION}" \
+        "triton==${ROCM_TRITON_VERSION}.rocm${ROCM_SDK_VERSION}"; \
+    for package in \
+        "rocm-sdk-device-${ROCM_DEVICE_ARCH}" \
+        "amd-torch-device-${ROCM_DEVICE_ARCH}" \
+        "amd-torchvision-device-${ROCM_DEVICE_ARCH}"; do \
+      if ! python3 -m pip show "${package}" >/dev/null 2>&1; then \
+        echo "Missing target ROCm device package: ${package}"; \
+        exit 1; \
+      fi; \
+    done; \
+    for candidate_arch in ${ROCM_DEVICE_ARCH_LIST}; do \
+      [ "${candidate_arch}" = "${ROCM_DEVICE_ARCH}" ] && continue; \
+      for package in \
+          "rocm-sdk-device-${candidate_arch}" \
+          "amd-torch-device-${candidate_arch}" \
+          "amd-torchvision-device-${candidate_arch}"; do \
+        if python3 -m pip show "${package}" >/dev/null 2>&1; then \
+          echo "Unexpected non-target ROCm device package: ${package}"; \
+          exit 1; \
+        fi; \
+      done; \
+    done
+
+RUN rocm-sdk init && rocm-sdk targets
+
+# rocm-sdk init expands a devel tree that carries its own copy of libamd_smi,
+# byte-identical to the one in _rocm_sdk_core that HIP loads through its RPATH.
+# Since ROCM_HOME below puts the devel tree on LD_LIBRARY_PATH, the amdsmi
+# python package binds that second copy while torch already holds the first,
+# and two independent copies in one process each keep their own global state:
+# whichever initialises second enumerates no devices. torch asks amdsmi for the
+# device count before HIP, so `torch.cuda.device_count()` comes back 0 on a
+# machine where hipGetDeviceCount() says 1. Collapse the duplicate so both land
+# on the same library. Idempotent when the SDK already ships a symlink here.
+RUN set -eux; \
+    SP="$VIRTUAL_ENV/lib/python3.12/site-packages"; \
+    CORE=$(ls "$SP"/_rocm_sdk_core/lib/libamd_smi.so.* 2>/dev/null | head -1); \
+    DEVEL="$SP/_rocm_sdk_devel/lib/libamd_smi.so"; \
+    if [ -n "${CORE}" ] && [ -e "${DEVEL}" ] && [ ! -L "${DEVEL}" ]; then \
+      ln -sf "${CORE}" "${DEVEL}"; \
+      echo "linked ${DEVEL} -> ${CORE}"; \
+    fi
+
+ENV ROCM_HOME=$VIRTUAL_ENV/lib/python3.12/site-packages/_rocm_sdk_devel
+ENV ROCM_PATH=$ROCM_HOME
+ENV CPATH=$ROCM_HOME/include
+ENV LIBRARY_PATH=$ROCM_HOME/lib
+ENV LD_LIBRARY_PATH=$ROCM_HOME/lib
+RUN echo 'export PATH=$ROCM_HOME/llvm/bin:$ROCM_HOME/bin:$PATH' >> /etc/bash.bashrc
+
+# The SDK's hsakmtTargets.cmake hardcodes /usr/lib64/libc.so from its own build
+# host; Ubuntu keeps libc in /lib/x86_64-linux-gnu, so cmake would otherwise
+# fail with "ninja: error: /usr/lib64/libc.so missing and no known rule to make it".
+RUN mkdir -p /usr/lib64 && ln -sf /lib/x86_64-linux-gnu/libc.so /usr/lib64/libc.so
+
+# ROCm lives in site-packages here, but AITER shells out to
+# /opt/rocm/llvm/bin/amdgpu-arch at runtime to pick DEFAULT_GPU_ARCH, and the
+# rest of this Dockerfile (TileLang, UCX, amd_smi) refers to /opt/rocm throughout.
+RUN ln -s ${ROCM_HOME} /opt/rocm
+
+# ===============================
+# Base image 942 with ROCm 10.0.0 and args (Python 3.12 + torch 2.11)
+# BUILD_TRITON=0 keeps the Triton installed above, which is the build AMD ships
+# with this SDK; the BUILD_TRITON=1 path installs a ROCm 7.2 wheel instead.
+FROM $BASE_IMAGE_942_ROCM1000 AS gfx942-rocm1000
+ENV BUILD_VLLM="0"
+ENV BUILD_TRITON="0"
+ENV BUILD_LLVM="0"
+ENV BUILD_AITER_ALL="1"
+ENV BUILD_MOONCAKE="1"
+ENV AITER_COMMIT_DEFAULT="c16d44b93a528b2a4bfd6d8d3409116d465872a9"
+# Same reasoning as the rocm724 stages: keep pip from resolving the image's
+# ROCm torch away to a PyPI CUDA build. Populated after the stack is in place.
+ENV PIP_CONSTRAINT="/etc/sglang/constraints/torch-rocm.txt"
+RUN mkdir -p /etc/sglang/constraints && : > /etc/sglang/constraints/torch-rocm.txt
+
+# ===============================
+# Base image 950 with ROCm 10.0.0 and args (Python 3.12 + torch 2.11)
+FROM $BASE_IMAGE_950_ROCM1000 AS gfx950-rocm1000
+ENV BUILD_VLLM="0"
+ENV BUILD_TRITON="0"
+ENV BUILD_LLVM="0"
+ENV BUILD_AITER_ALL="1"
+ENV BUILD_MOONCAKE="1"
+ENV AITER_COMMIT_DEFAULT="c16d44b93a528b2a4bfd6d8d3409116d465872a9"
+ENV PIP_CONSTRAINT="/etc/sglang/constraints/torch-rocm.txt"
+RUN mkdir -p /etc/sglang/constraints && : > /etc/sglang/constraints/torch-rocm.txt
+
 # Local source stage: with BRANCH_TYPE=local the build context is copied here and
 # used instead of git clone (mirrors docker/Dockerfile's local_src stage).
 FROM scratch AS local_src
@@ -241,9 +412,15 @@ RUN if [ -n "$UBUNTU_MIRROR" ]; then \
 # ubuntu24.04 base installs the `rocm` apt metapackage and does not; noble's
 # distro table has MI300 (74A*) but no MI355X (75A3), so gfx950-rocm724 would
 # otherwise report "AMD Radeon Graphics" and miss every name-keyed config.
+# The ROCm 10.0.0 flavors need nothing here: their libdrm comes from the pip SDK, which
+# links the ids table into libdrm_amdgpu.so itself (the .so carries the MI300X /
+# MI325X / MI355X names and never opens share/libdrm/amdgpu.ids).
 # See https://github.com/ROCm/ROCm/issues/5992
 RUN set -eux; \
     case "${GPU_ARCH}" in \
+      *rocm1000*) \
+        echo "ROCm 10.0.0 (GPU_ARCH=${GPU_ARCH}): pip SDK libdrm has the ids table built in, skipping"; \
+        ;; \
       *rocm724*) \
         echo "ROCm 7.2.4 (GPU_ARCH=${GPU_ARCH}): installing libdrm-amdgpu from graphics/7.2.4 noble"; \
         curl -fsSL --retry 5 --retry-delay 3 --retry-all-errors https://repo.radeon.com/rocm/rocm.gpg.key \
@@ -283,16 +460,17 @@ RUN python -m pip install --upgrade pip && pip install setuptools_scm
 RUN apt-get purge -y sccache; python -m pip uninstall -y sccache; rm -f "$(which sccache)"
 
 # Install AMD SMI Python package from ROCm distribution.
-# The ROCm 7.2 base image (rocm/pytorch) does not pre-install this package.
+# Neither the ROCm 7.2 base image (rocm/pytorch) nor the pip-installed ROCm 10.0.0
+# SDK pre-installs this package.
 RUN set -eux; \
     case "${GPU_ARCH}" in \
-      *rocm720*|*rocm724*) \
+      *rocm720*|*rocm724*|*rocm1000*) \
-        echo "ROCm 7.2 flavor detected from GPU_ARCH=${GPU_ARCH}"; \
+        echo "ROCm 7.2 / 10.0.0 flavor detected from GPU_ARCH=${GPU_ARCH}"; \
         cd /opt/rocm/share/amd_smi \
         && python3 -m pip install --no-cache-dir . \
         ;; \
       *) \
-        echo "Not rocm720 (GPU_ARCH=${GPU_ARCH}), skip amdsmi installation"; \
+        echo "Not rocm720/rocm724/rocm1000 (GPU_ARCH=${GPU_ARCH}), skip amdsmi installation"; \
         ;; \
     esac
 
@@ -311,11 +489,12 @@ RUN case "${GPU_ARCH}" in \
         ;; \
     esac
 
-# Populate the PIP_CONSTRAINT file, which only the rocm724 stages define, so that
+# Populate the PIP_CONSTRAINT file, which only the explicitly upgraded torch
-# resolving AITER and SGLang dependencies cannot replace the torch stack above.
+# stages define, so resolving AITER and SGLang cannot replace the torch stack.
-# Triton is left out: the BUILD_TRITON step installs it later.
+# Triton is left out: on rocm724 the BUILD_TRITON step installs it later, and on
+# rocm1000 it came from the ROCm SDK alongside torch.
 RUN case "${GPU_ARCH}" in \
-      *-rocm724) \
+      *-rocm724|*-rocm1000) \
         python3 -m pip freeze \
           | grep -E '^(torch|torchvision|torchaudio)(==| @ )' \
           > /etc/sglang/constraints/torch-rocm.txt \
@@ -467,11 +646,12 @@ RUN if [ "$BRANCH_TYPE" = "local" ]; then \
     && AMDGPU_TARGET=$GPU_ARCH_LIST python setup_rocm.py install \
     && cd ../../../.. \
     && rm -rf python/pyproject.toml && mv python/pyproject_other.toml python/pyproject.toml \
-    # srt_hip pins compressed-tensors==0.15.0, which requires torch<2.11 and so
+    # srt_hip pins compressed-tensors==0.15.0, which requires torch<2.11. Use
-    # cannot be satisfied on the ROCm 7.2.4 torch 2.11 stack. The *_rocm724 extras
+    # the torch-2.11 ROCm 7.2.4 extra for both upgraded stacks; all other
-    # carry a 0.16.0 pin instead; all other flavors keep the extras they used before.
+    # flavors keep the extras they used before.
     && case "${GPU_ARCH}" in \
          *-rocm724) srt_extras="srt_hip_rocm724,diffusion_hip"; all_extras="all_hip_rocm724" ;; \
+         *-rocm1000) srt_extras="srt_hip_rocm724,diffusion_hip"; all_extras="all_hip_rocm724" ;; \
          *) srt_extras="srt_hip,diffusion_hip"; all_extras="all_hip" ;; \
        esac \
     && if [ "$BUILD_TYPE" = "srt" ]; then \
@@ -482,14 +662,17 @@ RUN if [ "$BRANCH_TYPE" = "local" ]; then \
 
 RUN python -m pip cache purge
 
-RUN if [ "${GPU_ARCH##*-}" = "rocm724" ]; then \
+RUN case "${GPU_ARCH##*-}" in \
-      python3 -m pip check \
+      rocm724) expected_torch="2.11."; expected_audio="2.11."; expected_vision="0.26." ;; \
-      && python3 -c "import torch, torchaudio, torchvision, triton; expected={'torch':'2.11.','torchaudio':'2.11.','torchvision':'0.26.'}; actual={'torch':torch.__version__,'torchaudio':torchaudio.__version__,'torchvision':torchvision.__version__,'triton':triton.__version__}; assert torch.version.hip, actual; assert all(actual[name].startswith(version) for name, version in expected.items()), actual; print('Validated ROCm stack:', actual, 'HIP', torch.version.hip)" \
+      rocm1000) expected_torch="2.11."; expected_audio="2.11."; expected_vision="0.26." ;; \
-      && if pip list --format=freeze | grep -Eq '^nvidia-.*-cu[0-9]+'; then \
+      *) exit 0 ;; \
-           echo "ERROR: NVIDIA CUDA runtime packages were installed into the ROCm image"; \
+    esac \
-           exit 1; \
+    && python3 -m pip check \
-         fi; \
+    && python3 -c "import torch, torchaudio, torchvision, triton; expected={'torch':'${expected_torch}','torchaudio':'${expected_audio}','torchvision':'${expected_vision}'}; actual={'torch':torch.__version__,'torchaudio':torchaudio.__version__,'torchvision':torchvision.__version__,'triton':triton.__version__}; assert torch.version.hip, actual; assert all(actual[name].startswith(version) for name, version in expected.items()), actual; print('Validated ROCm stack:', actual, 'HIP', torch.version.hip)" \
-    fi
+    && if pip list --format=freeze | grep -Eq '^nvidia-.*-cu[0-9]+'; then \
+         echo "ERROR: NVIDIA CUDA runtime packages were installed into the ROCm image"; \
+         exit 1; \
+       fi
 
 # Copy config files to support MI300X in virtualized environments (MI300X_VF).  Symlinks will not be created in image build.
 RUN find /sgl-workspace/sglang/python/sglang/srt/layers/quantization/configs/ \
@@ -681,6 +864,23 @@ RUN /bin/bash -lc 'set -euo pipefail; \
   cd /sgl-workspace/mori; \
   git checkout "${MORI_COMMIT}"; \
   git submodule update --init --recursive; \
+  # The pip ROCm SDK vendors NUMA and libdrm under lib/rocm_sysdeps, which is on
+  # none of the three search paths the MORI build needs: hsakmt-config.cmake
+  # calls find_dependency(NUMA), rocm_smi.h reaches for <libdrm/drm.h>, and
+  # mori_application links -ldrm/-ldrm_amdgpu. The SDK's own libraries find
+  # these through an $ORIGIN/rocm_sysdeps/lib RPATH that MORI does not inherit,
+  # hence the ldconfig entry; every soname in there is librocm_sysdeps_*-prefixed,
+  # so it shadows nothing system-wide. Scope this explicitly to ROCm 10 so the
+  # ROCm 7.2 and 7.2.4 MORI build paths remain byte-for-byte equivalent here.
+  ROCM_SYSDEPS="${ROCM_HOME:-/opt/rocm}/lib/rocm_sysdeps"; \
+  if [ "${GPU_ARCH##*-}" = "rocm1000" ] && [ -d "${ROCM_SYSDEPS}" ]; then \
+    export CMAKE_PREFIX_PATH="${ROCM_SYSDEPS}${CMAKE_PREFIX_PATH:+:${CMAKE_PREFIX_PATH}}"; \
+    export CPATH="${ROCM_SYSDEPS}/include${CPATH:+:${CPATH}}"; \
+    export LIBRARY_PATH="${ROCM_SYSDEPS}/lib${LIBRARY_PATH:+:${LIBRARY_PATH}}"; \
+    echo "${ROCM_SYSDEPS}/lib" > /etc/ld.so.conf.d/rocm-sysdeps.conf; \
+    ldconfig; \
+    echo "[MORI] rocm_sysdeps prefix: ${ROCM_SYSDEPS}"; \
+  fi; \
   python3 setup.py develop; \
   python3 -c "import os, torch; print(os.path.join(os.path.dirname(torch.__file__), \"lib\"))" > /etc/ld.so.conf.d/torch.conf; \
   ldconfig; \
@@ -701,12 +901,13 @@ RUN /bin/bash -lc 'set -euo pipefail; \
       build-essential autoconf automake libtool pkg-config git \
       libibverbs-dev librdmacm-dev rdma-core && rm -rf /var/lib/apt/lists/*; \
   # Mooncake's dependencies.sh apt-installs Ubuntu's libabsl-dev (20220623 on
-  # the noble base used by rocm724). NIXL's meson then finds absl_base but no
+  # the noble base used by rocm724 and rocm1000). NIXL's meson then finds
-  # absl_log and refuses to fall back to its bundled Abseil -- "that would
+  # absl_base but no absl_log and refuses to fall back to its bundled Abseil --
-  # result in a mix of Abseil versions at runtime" -- so nixl fails at metadata
+  # "that would result in a mix of Abseil versions at runtime" -- so nixl fails
-  # generation. Drop just the -dev package (headers and pkg-config files); the
+  # at metadata generation. Drop just the -dev package (headers and pkg-config
-  # runtime library that already-built components link against stays in place.
+  # files); the runtime library that already-built components link against
-  case "${GPU_ARCH}" in *-rocm724) apt-get remove -y libabsl-dev ;; esac; \
+  # stays in place.
+  case "${GPU_ARCH}" in *-rocm724|*-rocm1000) apt-get remove -y libabsl-dev ;; esac; \
   pip install --no-cache-dir meson ninja pybind11 meson-python patchelf pyyaml; \
   git clone --depth=1 -b "${UCX_BRANCH}" "${UCX_REPO}" /sgl-workspace/ucx; \
   cd /sgl-workspace/ucx && ./autogen.sh && mkdir build && cd build && \

scripts/ci/amd/amd_ci_install_dependency.sh

@@ -50,11 +50,12 @@ fi
 IMAGE_TORCH_VERSION=$(docker exec ci_sglang python3 -c 'import torch; print(torch.__version__)')
 IMAGE_HIP_VERSION=$(docker exec ci_sglang python3 -c 'import torch; print(torch.version.hip or "")')
 IMAGE_GPU_ARCH=$(docker exec ci_sglang printenv GPU_ARCH 2>/dev/null || true)
-if [[ "${IMAGE_GPU_ARCH}" =~ ^(gfx942|gfx950)(-rocm720|-rocm724)?$ ]]; then
+if [[ "${IMAGE_GPU_ARCH}" =~ ^(gfx942|gfx950)(-rocm720|-rocm724|-rocm1000)?$ ]]; then
     echo "[CI-IMAGE] Image GPU_ARCH=${IMAGE_GPU_ARCH}"
     case "${IMAGE_GPU_ARCH}" in
         *-rocm724) IMAGE_BASE_ARG_SUFFIX="_ROCM724"; IMAGE_STAGE_SUFFIX="-rocm724" ;;
         *-rocm720) IMAGE_BASE_ARG_SUFFIX="_ROCM720"; IMAGE_STAGE_SUFFIX="-rocm720" ;;
+        *-rocm1000) IMAGE_BASE_ARG_SUFFIX="_ROCM1000"; IMAGE_STAGE_SUFFIX="-rocm1000" ;;
         *)         IMAGE_BASE_ARG_SUFFIX=""; IMAGE_STAGE_SUFFIX="" ;;
     esac
     IMAGE_GFX="${IMAGE_GPU_ARCH%-*}"
@@ -82,9 +83,9 @@ fi
 unset IMAGE_GPU_ARCH
 
 # Install the required dependencies in CI.
-# ROCm 7.2.4 images ship torch 2.11, which srt_hip cannot satisfy (it pins
+# Select the dependency extra that matches each image's torch stack. Plain
-# compressed-tensors 0.15.0, requiring torch<2.11). Select the rocm724 extras.
+# srt_hip pins compressed-tensors below the version required by torch 2.11.
-if [[ "${IMAGE_STAGE_SUFFIX}" == "-rocm724" ]]; then
+if [[ "${IMAGE_STAGE_SUFFIX}" == "-rocm724" || "${IMAGE_STAGE_SUFFIX}" == "-rocm1000" ]]; then
   EXTRAS="${EXTRAS/dev_hip/dev_hip_rocm724}"
 fi
 echo "Image torch ${IMAGE_TORCH_VERSION}, HIP ${IMAGE_HIP_VERSION}; installing python extras: [${EXTRAS}]"
@@ -265,9 +266,10 @@ if docker exec ci_sglang test -d /sgl-workspace/mori; then
   fi
 
   echo "[MORI] Reinstalling MORI ${MORI_COMMIT} (MORI_GPU_ARCHS=${MORI_GPU_ARCHS})"
-  # Only the rocm724 (noble) base is missing libgrpc++-dev; 7.0 and 7.2.0 built
+  # Only the rocm724 and rocm1000 (noble) bases attempt to install
-  # MORI without it for months before this step existed, so skip the apt round
+  # libgrpc++-dev; 7.0 and 7.2.0 built MORI without it for months before this
-  # trip there. Where it does run, neither step may be fatal: apt-get update
+  # step existed, so skip the apt round trip there. Where it does run, neither
+  # step may be fatal: apt-get update
   # exits 100 for a single unreachable index while still keeping every index it
   # did fetch, which under set -e is enough to take out the dependency install
   # on every AMD runner at once. Six external apt hosts are in play, so the
@@ -282,10 +284,20 @@ if docker exec ci_sglang test -d /sgl-workspace/mori; then
     cd /sgl-workspace/mori
     git checkout '${MORI_COMMIT}'
     git submodule update --init --recursive
-    if [ '${IMAGE_STAGE_SUFFIX}' = '-rocm724' ]; then
+    if [ '${IMAGE_STAGE_SUFFIX}' = '-rocm724' ] || [ '${IMAGE_STAGE_SUFFIX}' = '-rocm1000' ]; then
       apt-get update || echo '[MORI] apt-get update reported errors; continuing with the indexes it did fetch'
       apt-get install -y --no-install-recommends libgrpc++-dev || echo '[MORI] libgrpc++-dev unavailable; building MORI without it'
     fi
+    # The pip ROCm SDK vendors NUMA and libdrm under rocm_sysdeps, outside the
+    # default compiler, CMake, and linker search paths used by MORI.
+    ROCM_SYSDEPS="\${ROCM_HOME:-/opt/rocm}/lib/rocm_sysdeps"
+    if [ '${IMAGE_STAGE_SUFFIX}' = '-rocm1000' ] && [ -d "\${ROCM_SYSDEPS}" ]; then
+      export CMAKE_PREFIX_PATH="\${ROCM_SYSDEPS}\${CMAKE_PREFIX_PATH:+:\${CMAKE_PREFIX_PATH}}"
+      export CPATH="\${ROCM_SYSDEPS}/include\${CPATH:+:\${CPATH}}"
+      export LIBRARY_PATH="\${ROCM_SYSDEPS}/lib\${LIBRARY_PATH:+:\${LIBRARY_PATH}}"
+      echo "\${ROCM_SYSDEPS}/lib" > /etc/ld.so.conf.d/rocm-sysdeps.conf
+      ldconfig
+    fi
     python3 setup.py develop
     python3 -c 'import os, torch; print(os.path.join(os.path.dirname(torch.__file__), \"lib\"))' > /etc/ld.so.conf.d/torch.conf
     ldconfig
@@ -398,7 +410,7 @@ if [[ "${NEED_REBUILD}" == "true" ]]; then
     "
 
     # Re-apply the Dockerfile torch.Stream patch after re-clone (ROCm/aiter#4817).
-    if [[ "${IMAGE_STAGE_SUFFIX}" == "-rocm724" ]]; then
+    if [[ "${IMAGE_STAGE_SUFFIX}" == "-rocm724" || "${IMAGE_STAGE_SUFFIX}" == "-rocm1000" ]]; then
         docker exec -i ci_sglang python3 - <<'PY'
 from pathlib import Path
 p = Path("/sgl-workspace/aiter/csrc/cpp_itfs/torch_utils.py")