From 9579bff86085f886cf6d1ec69349017d0caeced4 Mon Sep 17 00:00:00 2001 From: Bingxu Chen Date: Fri, 28 Aug 2026 23:03:39 +0800 Subject: [PATCH] [AMD] Add ROCm 10 (gfx942 / gfx950) release images (#36434) --- .../workflows/nightly-test-amd-rocm720.yml | 9 +- .github/workflows/pr-test-amd-extra.yml | 1 + .github/workflows/pr-test-amd-rocm720.yml | 1 + .../workflows/release-docker-amd-rocm10.yml | 118 ++++++++ docker/rocm.Dockerfile | 251 ++++++++++++++++-- scripts/ci/amd/amd_ci_install_dependency.sh | 30 ++- 6 files changed, 372 insertions(+), 38 deletions(-) create mode 100644 .github/workflows/release-docker-amd-rocm10.yml diff --git a/.github/workflows/nightly-test-amd-rocm720.yml b/.github/workflows/nightly-test-amd-rocm720.yml index fe0b20b6a..e7081ebc6 100644 --- a/.github/workflows/nightly-test-amd-rocm720.yml +++ b/.github/workflows/nightly-test-amd-rocm720.yml @@ -11,12 +11,13 @@ on: workflow_dispatch: inputs: rocm_version: - description: 'ROCm image version ("all" runs every flavor, as the nightly schedule does)' + description: 'ROCm image version ("all" runs the scheduled ROCm 7.2 matrix; select rocm10 explicitly)' required: false type: choice default: 'all' options: - 'all' + - rocm10 - rocm724 - rocm720 aiter_ref: @@ -110,7 +111,7 @@ on: workflow_call: inputs: rocm_version: - description: 'ROCm image version ("all" runs every flavor)' + description: 'ROCm image version ("all" runs the scheduled ROCm 7.2 matrix; pass rocm10 explicitly)' required: false type: string # A single flavor, unlike the schedule and the dispatch form: a caller @@ -444,7 +445,7 @@ jobs: - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh - - name: Performance Test ROCm 7.2 (2-GPU Text Models) + - name: Performance Test (2-GPU Text Models) timeout-minutes: 120 run: | > github_summary.md # Clear summary file @@ -1008,7 +1009,7 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - - name: Performance Test MI35x ROCm 7.2 (8-GPU DeepSeek-V3.2 Basic) + - name: Performance Test MI35x (8-GPU DeepSeek-V3.2 Basic) timeout-minutes: 150 continue-on-error: true run: | diff --git a/.github/workflows/pr-test-amd-extra.yml b/.github/workflows/pr-test-amd-extra.yml index a99d27d08..931aa5f0a 100644 --- a/.github/workflows/pr-test-amd-extra.yml +++ b/.github/workflows/pr-test-amd-extra.yml @@ -42,6 +42,7 @@ on: type: choice default: rocm724 options: + - rocm10 - rocm724 - rocm720 - rocm700 diff --git a/.github/workflows/pr-test-amd-rocm720.yml b/.github/workflows/pr-test-amd-rocm720.yml index 59225ee64..367b9170f 100644 --- a/.github/workflows/pr-test-amd-rocm720.yml +++ b/.github/workflows/pr-test-amd-rocm720.yml @@ -22,6 +22,7 @@ on: type: choice default: rocm724 options: + - rocm10 - rocm724 - rocm720 target_stage_select: diff --git a/.github/workflows/release-docker-amd-rocm10.yml b/.github/workflows/release-docker-amd-rocm10.yml new file mode 100644 index 000000000..5dc1f99b6 --- /dev/null +++ b/.github/workflows/release-docker-amd-rocm10.yml @@ -0,0 +1,118 @@ +name: Release Docker Images Nightly ROCm 10 (AMD) +on: + workflow_dispatch: + inputs: + job_select: + description: 'Select which release job to run' + required: false + type: choice + default: 'all' + options: + - 'all' + - publish + gpu_arch: + description: 'Select which ROCm 10 GPU arch to build' + required: false + type: choice + default: 'all' + options: + - 'all' + - gfx942-rocm1000 + - gfx950-rocm1000 + schedule: + - cron: '0 12 * * *' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.number || github.sha }} + cancel-in-progress: true + +jobs: + publish: + if: github.repository == 'sgl-project/sglang' && (github.event_name != 'workflow_dispatch' || inputs.job_select == 'all' || inputs.job_select == 'publish') + runs-on: amd-docker-scale + environment: 'prod' + strategy: + fail-fast: false + matrix: + gpu_arch: ${{ fromJson((github.event_name == 'workflow_dispatch' && inputs.gpu_arch != 'all' && inputs.gpu_arch != '') && format('["{0}"]', inputs.gpu_arch) || '["gfx942-rocm1000", "gfx950-rocm1000"]') }} + build_type: ['all'] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Set date + run: echo "DATE=$(date +%Y%m%d)" >> "$GITHUB_ENV" + + - name: Get version from latest tag + id: version + run: | + VERSION=$(python3 scripts/release/get_version_tag.py --tag-only | sed 's/^v//') + + if [ -z "$VERSION" ]; then + echo "::error::Could not determine version from git tags" + exit 1 + fi + + COMMIT_HASH=$(git rev-parse --short HEAD) + PRETEND_VERSION="${VERSION}.dev${DATE}+g${COMMIT_HASH}" + + echo "version=${VERSION}" >> "$GITHUB_OUTPUT" + echo "pretend_version=${PRETEND_VERSION}" >> "$GITHUB_OUTPUT" + echo "Detected version: ${VERSION}" + echo "Pretend version for pip: ${PRETEND_VERSION}" + + - name: Login to Docker Hub (AMD) + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_AMD_USERNAME }} + password: ${{ secrets.DOCKERHUB_AMD_TOKEN }} + + - name: Build and push to rocm/sgl-dev + env: + BUILD_TYPE: ${{ matrix.build_type }} + GPU_ARCH: ${{ matrix.gpu_arch }} + PRETEND_VERSION: ${{ steps.version.outputs.pretend_version }} + VERSION: ${{ steps.version.outputs.version }} + run: | + case "$GPU_ARCH" in + gfx942-rocm1000) rocm_tag="rocm10-mi30x" ;; + gfx950-rocm1000) rocm_tag="rocm10-mi35x" ;; + *) + echo "Unsupported GPU arch: $GPU_ARCH" + exit 1 + ;; + esac + + image_tag="v${VERSION}-${rocm_tag}-${DATE}" + echo "IMAGE_TAG=${image_tag}" >> "$GITHUB_ENV" + + docker build . \ + -f docker/rocm.Dockerfile \ + --build-arg "SGL_BRANCH=${GITHUB_SHA}" \ + --build-arg "BUILD_TYPE=${BUILD_TYPE}" \ + --build-arg "GPU_ARCH=${GPU_ARCH}" \ + --build-arg ENABLE_MORI=1 \ + --build-arg ENABLE_NIXL=1 \ + --build-arg "SETUPTOOLS_SCM_PRETEND_VERSION=${PRETEND_VERSION}" \ + --build-arg UBUNTU_MIRROR=https://archive.ubuntu.com \ + -t "rocm/sgl-dev:${image_tag}" \ + --no-cache + docker push "rocm/sgl-dev:${image_tag}" + + - name: Login to Docker Hub (lmsys) + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Push to lmsysorg/sglang-rocm + run: | + docker tag "rocm/sgl-dev:${IMAGE_TAG}" "lmsysorg/sglang-rocm:${IMAGE_TAG}" + docker push "lmsysorg/sglang-rocm:${IMAGE_TAG}" diff --git a/docker/rocm.Dockerfile b/docker/rocm.Dockerfile index bf94941bb..a580d1077 100644 --- a/docker/rocm.Dockerfile +++ b/docker/rocm.Dockerfile @@ -5,11 +5,16 @@ # docker build --build-arg SGL_BRANCH=v0.5.10.post1 --build-arg GPU_ARCH=gfx950 -t v0.5.10.post1-rocm700-mi35x -f rocm.Dockerfile . # docker build --build-arg SGL_BRANCH=v0.5.10.post1 --build-arg GPU_ARCH=gfx950-rocm720 -t v0.5.10.post1-rocm720-mi35x -f rocm.Dockerfile . # docker build --build-arg SGL_BRANCH=v0.5.10.post1 --build-arg GPU_ARCH=gfx950-rocm724 -t v0.5.10.post1-rocm724-mi35x -f rocm.Dockerfile . +# docker build --build-arg SGL_BRANCH=v0.5.10.post1 --build-arg GPU_ARCH=gfx942-rocm1000 -t v0.5.10.post1-rocm1000-mi30x -f rocm.Dockerfile . +# docker build --build-arg SGL_BRANCH=v0.5.10.post1 --build-arg GPU_ARCH=gfx950-rocm1000 -t v0.5.10.post1-rocm1000-mi35x -f rocm.Dockerfile . # # Flavor notes: # GPU_ARCH=*-rocm724 is built on a Python 3.12 base and upgrades the stack to # torch 2.11 (+torchvision 0.26 / torchaudio 2.11) and Triton 3.7. # The ROCm 7.2.0 flavors remain on Python 3.10 and torch 2.9.1. +# GPU_ARCH=*-rocm1000 is Python 3.12 + torch 2.11, and takes the +# whole ROCm stack from AMD's stable wheel channel rather than an apt +# ROCm base image; see the rocm1000-base stage for what that changes. # Usage (to build SGLang ROCm + Mori docker image): # remove --build-arg NIC_BACKEND=ainic since new MoRI JIT will do NIC auto detection on target @@ -34,6 +39,11 @@ ARG BASE_IMAGE_942_ROCM724="rocm/pytorch:rocm7.2.4_ubuntu24.04_py3.12_pytorch_re ARG BASE_IMAGE_950="rocm/sgl-dev:rocm7-vllm-20250904" ARG BASE_IMAGE_950_ROCM720="rocm/pytorch:rocm7.2_ubuntu22.04_py3.10_pytorch_release_2.9.1" ARG BASE_IMAGE_950_ROCM724="rocm/pytorch:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0" +# The ROCm 10.0.0 flavors default to the rocm1000-base stage below rather +# than a published image; point these at one to build on a prebuilt base. +ARG BASE_IMAGE_942_ROCM1000="rocm1000-base" +ARG BASE_IMAGE_950_ROCM1000="rocm1000-base" +ARG BASE_IMAGE_ROCM1000="ubuntu:24.04" # This is necessary for scope purpose ARG GPU_ARCH=gfx950 @@ -126,6 +136,167 @@ RUN mkdir -p /etc/sglang/constraints && : > /etc/sglang/constraints/torch-rocm.t # IPC behavior. ENV HSA_ENABLE_IPC_MODE_LEGACY=1 +# =============================== +# Shared ROCm 10.0.0 base for gfx942 and gfx950. Assemble the stack from +# AMD's stable wheels on a plain Ubuntu base so each output image carries only +# its own GPU device payload. +# The SDK lands in site-packages instead of /opt/rocm, which the rest of this +# Dockerfile and AITER both assume, hence the path fixups below. +# +# This is deliberately AMD's stable channel, not a prerelease or nightly. +# Every ROCm/PyTorch artifact below is pinned to the 10.0.0 release. +# +# Python 3.12 (the Ubuntu 24.04 default) rather than 3.13/3.14: st_attn==0.0.7, +# vsa==0.0.4, petit_kernel==0.0.2 and wave-lang==3.8.2 publish wheels only up to +# cp313 and no sdist, so pip has no candidate at all for srt_hip on 3.14. +FROM $BASE_IMAGE_ROCM1000 AS rocm1000-base + +# Redeclare the global selector inside this stage so each matrix build installs +# only the device payload for its target image (gfx942 or gfx950). +ARG GPU_ARCH + +# ROCM_TRITON_VERSION rather than TRITON_VERSION: the final stage declares a +# TRITON_VERSION of its own for the ROCm 7.2 wheel, and a --build-arg would +# otherwise land on both. +ARG ROCM_SDK_VERSION="10.0.0" +ARG ROCM_TORCH_VERSION="2.11.0" +ARG ROCM_TORCHVISION_VERSION="0.26.0" +ARG ROCM_TORCHAUDIO_VERSION="2.11.0" +ARG ROCM_TRITON_VERSION="3.8.0+git4cff872c" +ARG ROCM_INDEX_URL="https://stable.repo.amd.com/rocm/whl-next/" +# Keep device targets data-driven: adding a new image should require one list +# entry here, not another pairwise OTHER_ROCM_DEVICE_ARCH mapping. +ARG ROCM_DEVICE_ARCH_LIST="gfx942 gfx950 gfx1250" + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + curl \ + git \ + gnupg \ + libstdc++-12-dev \ + python-is-python3 \ + python3 \ + python3-dev \ + python3-pip \ + python3.12-venv \ + wget \ + && rm -rf /var/lib/apt/lists/* + +ENV VIRTUAL_ENV=/opt/venv +RUN python3 -m venv "$VIRTUAL_ENV" +ENV PATH="$VIRTUAL_ENV/bin:$PATH" +RUN python3 -m pip install --no-cache-dir -U pip setuptools setuptools_scm wheel + +# The two release jobs invoke separate Docker builds. Derive the device target +# from GPU_ARCH so the MI300 image carries only gfx942 wheels and the MI350 +# image carries only gfx950 wheels. Keeping the packages as explicit specs also +# makes the intended per-image device payload explicit to the resolver. +RUN set -eux; \ + ROCM_DEVICE_ARCH="${GPU_ARCH%%-*}"; \ + case " ${ROCM_DEVICE_ARCH_LIST} " in \ + *" ${ROCM_DEVICE_ARCH} "*) ;; \ + *) echo "Unsupported ROCm 10.0.0 GPU_ARCH=${GPU_ARCH}"; exit 1 ;; \ + esac; \ + python3 -m pip install --no-cache-dir \ + --index-url ${ROCM_INDEX_URL} \ + "rocm-sdk-core==${ROCM_SDK_VERSION}" \ + "rocm-sdk-libraries==${ROCM_SDK_VERSION}" \ + "rocm-sdk-devel==${ROCM_SDK_VERSION}" \ + "rocm-sdk-device-${ROCM_DEVICE_ARCH}==${ROCM_SDK_VERSION}" \ + "torch==${ROCM_TORCH_VERSION}+rocm${ROCM_SDK_VERSION}" \ + "torchvision==${ROCM_TORCHVISION_VERSION}+rocm${ROCM_SDK_VERSION}" \ + "torchaudio==${ROCM_TORCHAUDIO_VERSION}+rocm${ROCM_SDK_VERSION}" \ + "amd-torch-device-${ROCM_DEVICE_ARCH}==${ROCM_TORCH_VERSION}+rocm${ROCM_SDK_VERSION}" \ + "amd-torchvision-device-${ROCM_DEVICE_ARCH}==${ROCM_TORCHVISION_VERSION}+rocm${ROCM_SDK_VERSION}" \ + "triton==${ROCM_TRITON_VERSION}.rocm${ROCM_SDK_VERSION}"; \ + for package in \ + "rocm-sdk-device-${ROCM_DEVICE_ARCH}" \ + "amd-torch-device-${ROCM_DEVICE_ARCH}" \ + "amd-torchvision-device-${ROCM_DEVICE_ARCH}"; do \ + if ! python3 -m pip show "${package}" >/dev/null 2>&1; then \ + echo "Missing target ROCm device package: ${package}"; \ + exit 1; \ + fi; \ + done; \ + for candidate_arch in ${ROCM_DEVICE_ARCH_LIST}; do \ + [ "${candidate_arch}" = "${ROCM_DEVICE_ARCH}" ] && continue; \ + for package in \ + "rocm-sdk-device-${candidate_arch}" \ + "amd-torch-device-${candidate_arch}" \ + "amd-torchvision-device-${candidate_arch}"; do \ + if python3 -m pip show "${package}" >/dev/null 2>&1; then \ + echo "Unexpected non-target ROCm device package: ${package}"; \ + exit 1; \ + fi; \ + done; \ + done + +RUN rocm-sdk init && rocm-sdk targets + +# rocm-sdk init expands a devel tree that carries its own copy of libamd_smi, +# byte-identical to the one in _rocm_sdk_core that HIP loads through its RPATH. +# Since ROCM_HOME below puts the devel tree on LD_LIBRARY_PATH, the amdsmi +# python package binds that second copy while torch already holds the first, +# and two independent copies in one process each keep their own global state: +# whichever initialises second enumerates no devices. torch asks amdsmi for the +# device count before HIP, so `torch.cuda.device_count()` comes back 0 on a +# machine where hipGetDeviceCount() says 1. Collapse the duplicate so both land +# on the same library. Idempotent when the SDK already ships a symlink here. +RUN set -eux; \ + SP="$VIRTUAL_ENV/lib/python3.12/site-packages"; \ + CORE=$(ls "$SP"/_rocm_sdk_core/lib/libamd_smi.so.* 2>/dev/null | head -1); \ + DEVEL="$SP/_rocm_sdk_devel/lib/libamd_smi.so"; \ + if [ -n "${CORE}" ] && [ -e "${DEVEL}" ] && [ ! -L "${DEVEL}" ]; then \ + ln -sf "${CORE}" "${DEVEL}"; \ + echo "linked ${DEVEL} -> ${CORE}"; \ + fi + +ENV ROCM_HOME=$VIRTUAL_ENV/lib/python3.12/site-packages/_rocm_sdk_devel +ENV ROCM_PATH=$ROCM_HOME +ENV CPATH=$ROCM_HOME/include +ENV LIBRARY_PATH=$ROCM_HOME/lib +ENV LD_LIBRARY_PATH=$ROCM_HOME/lib +RUN echo 'export PATH=$ROCM_HOME/llvm/bin:$ROCM_HOME/bin:$PATH' >> /etc/bash.bashrc + +# The SDK's hsakmtTargets.cmake hardcodes /usr/lib64/libc.so from its own build +# host; Ubuntu keeps libc in /lib/x86_64-linux-gnu, so cmake would otherwise +# fail with "ninja: error: /usr/lib64/libc.so missing and no known rule to make it". +RUN mkdir -p /usr/lib64 && ln -sf /lib/x86_64-linux-gnu/libc.so /usr/lib64/libc.so + +# ROCm lives in site-packages here, but AITER shells out to +# /opt/rocm/llvm/bin/amdgpu-arch at runtime to pick DEFAULT_GPU_ARCH, and the +# rest of this Dockerfile (TileLang, UCX, amd_smi) refers to /opt/rocm throughout. +RUN ln -s ${ROCM_HOME} /opt/rocm + +# =============================== +# Base image 942 with ROCm 10.0.0 and args (Python 3.12 + torch 2.11) +# BUILD_TRITON=0 keeps the Triton installed above, which is the build AMD ships +# with this SDK; the BUILD_TRITON=1 path installs a ROCm 7.2 wheel instead. +FROM $BASE_IMAGE_942_ROCM1000 AS gfx942-rocm1000 +ENV BUILD_VLLM="0" +ENV BUILD_TRITON="0" +ENV BUILD_LLVM="0" +ENV BUILD_AITER_ALL="1" +ENV BUILD_MOONCAKE="1" +ENV AITER_COMMIT_DEFAULT="c16d44b93a528b2a4bfd6d8d3409116d465872a9" +# Same reasoning as the rocm724 stages: keep pip from resolving the image's +# ROCm torch away to a PyPI CUDA build. Populated after the stack is in place. +ENV PIP_CONSTRAINT="/etc/sglang/constraints/torch-rocm.txt" +RUN mkdir -p /etc/sglang/constraints && : > /etc/sglang/constraints/torch-rocm.txt + +# =============================== +# Base image 950 with ROCm 10.0.0 and args (Python 3.12 + torch 2.11) +FROM $BASE_IMAGE_950_ROCM1000 AS gfx950-rocm1000 +ENV BUILD_VLLM="0" +ENV BUILD_TRITON="0" +ENV BUILD_LLVM="0" +ENV BUILD_AITER_ALL="1" +ENV BUILD_MOONCAKE="1" +ENV AITER_COMMIT_DEFAULT="c16d44b93a528b2a4bfd6d8d3409116d465872a9" +ENV PIP_CONSTRAINT="/etc/sglang/constraints/torch-rocm.txt" +RUN mkdir -p /etc/sglang/constraints && : > /etc/sglang/constraints/torch-rocm.txt + # Local source stage: with BRANCH_TYPE=local the build context is copied here and # used instead of git clone (mirrors docker/Dockerfile's local_src stage). FROM scratch AS local_src @@ -241,9 +412,15 @@ RUN if [ -n "$UBUNTU_MIRROR" ]; then \ # ubuntu24.04 base installs the `rocm` apt metapackage and does not; noble's # distro table has MI300 (74A*) but no MI355X (75A3), so gfx950-rocm724 would # otherwise report "AMD Radeon Graphics" and miss every name-keyed config. +# The ROCm 10.0.0 flavors need nothing here: their libdrm comes from the pip SDK, which +# links the ids table into libdrm_amdgpu.so itself (the .so carries the MI300X / +# MI325X / MI355X names and never opens share/libdrm/amdgpu.ids). # See https://github.com/ROCm/ROCm/issues/5992 RUN set -eux; \ case "${GPU_ARCH}" in \ + *rocm1000*) \ + echo "ROCm 10.0.0 (GPU_ARCH=${GPU_ARCH}): pip SDK libdrm has the ids table built in, skipping"; \ + ;; \ *rocm724*) \ echo "ROCm 7.2.4 (GPU_ARCH=${GPU_ARCH}): installing libdrm-amdgpu from graphics/7.2.4 noble"; \ curl -fsSL --retry 5 --retry-delay 3 --retry-all-errors https://repo.radeon.com/rocm/rocm.gpg.key \ @@ -283,16 +460,17 @@ RUN python -m pip install --upgrade pip && pip install setuptools_scm RUN apt-get purge -y sccache; python -m pip uninstall -y sccache; rm -f "$(which sccache)" # Install AMD SMI Python package from ROCm distribution. -# The ROCm 7.2 base image (rocm/pytorch) does not pre-install this package. +# Neither the ROCm 7.2 base image (rocm/pytorch) nor the pip-installed ROCm 10.0.0 +# SDK pre-installs this package. RUN set -eux; \ case "${GPU_ARCH}" in \ - *rocm720*|*rocm724*) \ - echo "ROCm 7.2 flavor detected from GPU_ARCH=${GPU_ARCH}"; \ + *rocm720*|*rocm724*|*rocm1000*) \ + echo "ROCm 7.2 / 10.0.0 flavor detected from GPU_ARCH=${GPU_ARCH}"; \ cd /opt/rocm/share/amd_smi \ && python3 -m pip install --no-cache-dir . \ ;; \ *) \ - echo "Not rocm720 (GPU_ARCH=${GPU_ARCH}), skip amdsmi installation"; \ + echo "Not rocm720/rocm724/rocm1000 (GPU_ARCH=${GPU_ARCH}), skip amdsmi installation"; \ ;; \ esac @@ -311,11 +489,12 @@ RUN case "${GPU_ARCH}" in \ ;; \ esac -# Populate the PIP_CONSTRAINT file, which only the rocm724 stages define, so that -# resolving AITER and SGLang dependencies cannot replace the torch stack above. -# Triton is left out: the BUILD_TRITON step installs it later. +# Populate the PIP_CONSTRAINT file, which only the explicitly upgraded torch +# stages define, so resolving AITER and SGLang cannot replace the torch stack. +# Triton is left out: on rocm724 the BUILD_TRITON step installs it later, and on +# rocm1000 it came from the ROCm SDK alongside torch. RUN case "${GPU_ARCH}" in \ - *-rocm724) \ + *-rocm724|*-rocm1000) \ python3 -m pip freeze \ | grep -E '^(torch|torchvision|torchaudio)(==| @ )' \ > /etc/sglang/constraints/torch-rocm.txt \ @@ -467,11 +646,12 @@ RUN if [ "$BRANCH_TYPE" = "local" ]; then \ && AMDGPU_TARGET=$GPU_ARCH_LIST python setup_rocm.py install \ && cd ../../../.. \ && rm -rf python/pyproject.toml && mv python/pyproject_other.toml python/pyproject.toml \ - # srt_hip pins compressed-tensors==0.15.0, which requires torch<2.11 and so - # cannot be satisfied on the ROCm 7.2.4 torch 2.11 stack. The *_rocm724 extras - # carry a 0.16.0 pin instead; all other flavors keep the extras they used before. + # srt_hip pins compressed-tensors==0.15.0, which requires torch<2.11. Use + # the torch-2.11 ROCm 7.2.4 extra for both upgraded stacks; all other + # flavors keep the extras they used before. && case "${GPU_ARCH}" in \ *-rocm724) srt_extras="srt_hip_rocm724,diffusion_hip"; all_extras="all_hip_rocm724" ;; \ + *-rocm1000) srt_extras="srt_hip_rocm724,diffusion_hip"; all_extras="all_hip_rocm724" ;; \ *) srt_extras="srt_hip,diffusion_hip"; all_extras="all_hip" ;; \ esac \ && if [ "$BUILD_TYPE" = "srt" ]; then \ @@ -482,14 +662,17 @@ RUN if [ "$BRANCH_TYPE" = "local" ]; then \ RUN python -m pip cache purge -RUN if [ "${GPU_ARCH##*-}" = "rocm724" ]; then \ - python3 -m pip check \ - && python3 -c "import torch, torchaudio, torchvision, triton; expected={'torch':'2.11.','torchaudio':'2.11.','torchvision':'0.26.'}; actual={'torch':torch.__version__,'torchaudio':torchaudio.__version__,'torchvision':torchvision.__version__,'triton':triton.__version__}; assert torch.version.hip, actual; assert all(actual[name].startswith(version) for name, version in expected.items()), actual; print('Validated ROCm stack:', actual, 'HIP', torch.version.hip)" \ - && if pip list --format=freeze | grep -Eq '^nvidia-.*-cu[0-9]+'; then \ - echo "ERROR: NVIDIA CUDA runtime packages were installed into the ROCm image"; \ - exit 1; \ - fi; \ - fi +RUN case "${GPU_ARCH##*-}" in \ + rocm724) expected_torch="2.11."; expected_audio="2.11."; expected_vision="0.26." ;; \ + rocm1000) expected_torch="2.11."; expected_audio="2.11."; expected_vision="0.26." ;; \ + *) exit 0 ;; \ + esac \ + && python3 -m pip check \ + && python3 -c "import torch, torchaudio, torchvision, triton; expected={'torch':'${expected_torch}','torchaudio':'${expected_audio}','torchvision':'${expected_vision}'}; actual={'torch':torch.__version__,'torchaudio':torchaudio.__version__,'torchvision':torchvision.__version__,'triton':triton.__version__}; assert torch.version.hip, actual; assert all(actual[name].startswith(version) for name, version in expected.items()), actual; print('Validated ROCm stack:', actual, 'HIP', torch.version.hip)" \ + && if pip list --format=freeze | grep -Eq '^nvidia-.*-cu[0-9]+'; then \ + echo "ERROR: NVIDIA CUDA runtime packages were installed into the ROCm image"; \ + exit 1; \ + fi # Copy config files to support MI300X in virtualized environments (MI300X_VF). Symlinks will not be created in image build. RUN find /sgl-workspace/sglang/python/sglang/srt/layers/quantization/configs/ \ @@ -681,6 +864,23 @@ RUN /bin/bash -lc 'set -euo pipefail; \ cd /sgl-workspace/mori; \ git checkout "${MORI_COMMIT}"; \ git submodule update --init --recursive; \ + # The pip ROCm SDK vendors NUMA and libdrm under lib/rocm_sysdeps, which is on + # none of the three search paths the MORI build needs: hsakmt-config.cmake + # calls find_dependency(NUMA), rocm_smi.h reaches for , and + # mori_application links -ldrm/-ldrm_amdgpu. The SDK's own libraries find + # these through an $ORIGIN/rocm_sysdeps/lib RPATH that MORI does not inherit, + # hence the ldconfig entry; every soname in there is librocm_sysdeps_*-prefixed, + # so it shadows nothing system-wide. Scope this explicitly to ROCm 10 so the + # ROCm 7.2 and 7.2.4 MORI build paths remain byte-for-byte equivalent here. + ROCM_SYSDEPS="${ROCM_HOME:-/opt/rocm}/lib/rocm_sysdeps"; \ + if [ "${GPU_ARCH##*-}" = "rocm1000" ] && [ -d "${ROCM_SYSDEPS}" ]; then \ + export CMAKE_PREFIX_PATH="${ROCM_SYSDEPS}${CMAKE_PREFIX_PATH:+:${CMAKE_PREFIX_PATH}}"; \ + export CPATH="${ROCM_SYSDEPS}/include${CPATH:+:${CPATH}}"; \ + export LIBRARY_PATH="${ROCM_SYSDEPS}/lib${LIBRARY_PATH:+:${LIBRARY_PATH}}"; \ + echo "${ROCM_SYSDEPS}/lib" > /etc/ld.so.conf.d/rocm-sysdeps.conf; \ + ldconfig; \ + echo "[MORI] rocm_sysdeps prefix: ${ROCM_SYSDEPS}"; \ + fi; \ python3 setup.py develop; \ python3 -c "import os, torch; print(os.path.join(os.path.dirname(torch.__file__), \"lib\"))" > /etc/ld.so.conf.d/torch.conf; \ ldconfig; \ @@ -701,12 +901,13 @@ RUN /bin/bash -lc 'set -euo pipefail; \ build-essential autoconf automake libtool pkg-config git \ libibverbs-dev librdmacm-dev rdma-core && rm -rf /var/lib/apt/lists/*; \ # Mooncake's dependencies.sh apt-installs Ubuntu's libabsl-dev (20220623 on - # the noble base used by rocm724). NIXL's meson then finds absl_base but no - # absl_log and refuses to fall back to its bundled Abseil -- "that would - # result in a mix of Abseil versions at runtime" -- so nixl fails at metadata - # generation. Drop just the -dev package (headers and pkg-config files); the - # runtime library that already-built components link against stays in place. - case "${GPU_ARCH}" in *-rocm724) apt-get remove -y libabsl-dev ;; esac; \ + # the noble base used by rocm724 and rocm1000). NIXL's meson then finds + # absl_base but no absl_log and refuses to fall back to its bundled Abseil -- + # "that would result in a mix of Abseil versions at runtime" -- so nixl fails + # at metadata generation. Drop just the -dev package (headers and pkg-config + # files); the runtime library that already-built components link against + # stays in place. + case "${GPU_ARCH}" in *-rocm724|*-rocm1000) apt-get remove -y libabsl-dev ;; esac; \ pip install --no-cache-dir meson ninja pybind11 meson-python patchelf pyyaml; \ git clone --depth=1 -b "${UCX_BRANCH}" "${UCX_REPO}" /sgl-workspace/ucx; \ cd /sgl-workspace/ucx && ./autogen.sh && mkdir build && cd build && \ diff --git a/scripts/ci/amd/amd_ci_install_dependency.sh b/scripts/ci/amd/amd_ci_install_dependency.sh index 93e6d4e41..7db997e27 100755 --- a/scripts/ci/amd/amd_ci_install_dependency.sh +++ b/scripts/ci/amd/amd_ci_install_dependency.sh @@ -50,11 +50,12 @@ fi IMAGE_TORCH_VERSION=$(docker exec ci_sglang python3 -c 'import torch; print(torch.__version__)') IMAGE_HIP_VERSION=$(docker exec ci_sglang python3 -c 'import torch; print(torch.version.hip or "")') IMAGE_GPU_ARCH=$(docker exec ci_sglang printenv GPU_ARCH 2>/dev/null || true) -if [[ "${IMAGE_GPU_ARCH}" =~ ^(gfx942|gfx950)(-rocm720|-rocm724)?$ ]]; then +if [[ "${IMAGE_GPU_ARCH}" =~ ^(gfx942|gfx950)(-rocm720|-rocm724|-rocm1000)?$ ]]; then echo "[CI-IMAGE] Image GPU_ARCH=${IMAGE_GPU_ARCH}" case "${IMAGE_GPU_ARCH}" in *-rocm724) IMAGE_BASE_ARG_SUFFIX="_ROCM724"; IMAGE_STAGE_SUFFIX="-rocm724" ;; *-rocm720) IMAGE_BASE_ARG_SUFFIX="_ROCM720"; IMAGE_STAGE_SUFFIX="-rocm720" ;; + *-rocm1000) IMAGE_BASE_ARG_SUFFIX="_ROCM1000"; IMAGE_STAGE_SUFFIX="-rocm1000" ;; *) IMAGE_BASE_ARG_SUFFIX=""; IMAGE_STAGE_SUFFIX="" ;; esac IMAGE_GFX="${IMAGE_GPU_ARCH%-*}" @@ -82,9 +83,9 @@ fi unset IMAGE_GPU_ARCH # Install the required dependencies in CI. -# ROCm 7.2.4 images ship torch 2.11, which srt_hip cannot satisfy (it pins -# compressed-tensors 0.15.0, requiring torch<2.11). Select the rocm724 extras. -if [[ "${IMAGE_STAGE_SUFFIX}" == "-rocm724" ]]; then +# Select the dependency extra that matches each image's torch stack. Plain +# srt_hip pins compressed-tensors below the version required by torch 2.11. +if [[ "${IMAGE_STAGE_SUFFIX}" == "-rocm724" || "${IMAGE_STAGE_SUFFIX}" == "-rocm1000" ]]; then EXTRAS="${EXTRAS/dev_hip/dev_hip_rocm724}" fi echo "Image torch ${IMAGE_TORCH_VERSION}, HIP ${IMAGE_HIP_VERSION}; installing python extras: [${EXTRAS}]" @@ -265,9 +266,10 @@ if docker exec ci_sglang test -d /sgl-workspace/mori; then fi echo "[MORI] Reinstalling MORI ${MORI_COMMIT} (MORI_GPU_ARCHS=${MORI_GPU_ARCHS})" - # Only the rocm724 (noble) base is missing libgrpc++-dev; 7.0 and 7.2.0 built - # MORI without it for months before this step existed, so skip the apt round - # trip there. Where it does run, neither step may be fatal: apt-get update + # Only the rocm724 and rocm1000 (noble) bases attempt to install + # libgrpc++-dev; 7.0 and 7.2.0 built MORI without it for months before this + # step existed, so skip the apt round trip there. Where it does run, neither + # step may be fatal: apt-get update # exits 100 for a single unreachable index while still keeping every index it # did fetch, which under set -e is enough to take out the dependency install # on every AMD runner at once. Six external apt hosts are in play, so the @@ -282,10 +284,20 @@ if docker exec ci_sglang test -d /sgl-workspace/mori; then cd /sgl-workspace/mori git checkout '${MORI_COMMIT}' git submodule update --init --recursive - if [ '${IMAGE_STAGE_SUFFIX}' = '-rocm724' ]; then + if [ '${IMAGE_STAGE_SUFFIX}' = '-rocm724' ] || [ '${IMAGE_STAGE_SUFFIX}' = '-rocm1000' ]; then apt-get update || echo '[MORI] apt-get update reported errors; continuing with the indexes it did fetch' apt-get install -y --no-install-recommends libgrpc++-dev || echo '[MORI] libgrpc++-dev unavailable; building MORI without it' fi + # The pip ROCm SDK vendors NUMA and libdrm under rocm_sysdeps, outside the + # default compiler, CMake, and linker search paths used by MORI. + ROCM_SYSDEPS="\${ROCM_HOME:-/opt/rocm}/lib/rocm_sysdeps" + if [ '${IMAGE_STAGE_SUFFIX}' = '-rocm1000' ] && [ -d "\${ROCM_SYSDEPS}" ]; then + export CMAKE_PREFIX_PATH="\${ROCM_SYSDEPS}\${CMAKE_PREFIX_PATH:+:\${CMAKE_PREFIX_PATH}}" + export CPATH="\${ROCM_SYSDEPS}/include\${CPATH:+:\${CPATH}}" + export LIBRARY_PATH="\${ROCM_SYSDEPS}/lib\${LIBRARY_PATH:+:\${LIBRARY_PATH}}" + echo "\${ROCM_SYSDEPS}/lib" > /etc/ld.so.conf.d/rocm-sysdeps.conf + ldconfig + fi python3 setup.py develop python3 -c 'import os, torch; print(os.path.join(os.path.dirname(torch.__file__), \"lib\"))' > /etc/ld.so.conf.d/torch.conf ldconfig @@ -398,7 +410,7 @@ if [[ "${NEED_REBUILD}" == "true" ]]; then " # Re-apply the Dockerfile torch.Stream patch after re-clone (ROCm/aiter#4817). - if [[ "${IMAGE_STAGE_SUFFIX}" == "-rocm724" ]]; then + if [[ "${IMAGE_STAGE_SUFFIX}" == "-rocm724" || "${IMAGE_STAGE_SUFFIX}" == "-rocm1000" ]]; then docker exec -i ci_sglang python3 - <<'PY' from pathlib import Path p = Path("/sgl-workspace/aiter/csrc/cpp_itfs/torch_utils.py")