From f399997d2f8573a4d32e964b6f230c94cb8a450e Mon Sep 17 00:00:00 2001 From: YC Yen-Ching Tseng Date: Fri, 17 Apr 2026 19:49:26 +0800 Subject: [PATCH] [AMD] mirror nightly images to local registry and prefer LAN pulls (#23073) Co-authored-by: bingxche --- .../workflows/release-docker-amd-nightly.yml | 178 ++++++++++++------ .../release-docker-amd-rocm720-nightly.yml | 62 ++++++ scripts/ci/amd/amd_ci_start_container.sh | 82 ++++++-- .../ci/amd/amd_ci_start_container_disagg.sh | 76 ++++++-- 4 files changed, 317 insertions(+), 81 deletions(-) diff --git a/.github/workflows/release-docker-amd-nightly.yml b/.github/workflows/release-docker-amd-nightly.yml index 6463de799..616f2b9fe 100644 --- a/.github/workflows/release-docker-amd-nightly.yml +++ b/.github/workflows/release-docker-amd-nightly.yml @@ -82,6 +82,22 @@ jobs: docker build . -f docker/rocm.Dockerfile --build-arg SGL_BRANCH=${{ github.ref_name }} --build-arg BUILD_TYPE=${{ matrix.build_type }} --build-arg GPU_ARCH=${{ matrix.gpu_arch }} --build-arg ENABLE_MORI=1 --build-arg NIC_BACKEND=ainic --build-arg SETUPTOOLS_SCM_PRETEND_VERSION=${pretend_version} -t rocm/sgl-dev:${tag}-${{ env.DATE }} --no-cache docker push rocm/sgl-dev:${tag}-${{ env.DATE }} + # Persist the tag right after rocm/sgl-dev push succeeds so the local + # registry mirror can run even if a later step in this job (lmsys push) + # fails. By default this step only runs when the previous step succeeded, + # so the artifact only exists when an image actually landed on Docker Hub. + - name: Save published image tag + run: | + mkdir -p image-tag + echo "${{ env.IMAGE_TAG }}" > "image-tag/${{ matrix.gpu_arch }}.txt" + + - name: Upload image tag artifact + uses: actions/upload-artifact@v4 + with: + name: image-tag-${{ matrix.gpu_arch }} + path: image-tag/${{ matrix.gpu_arch }}.txt + retention-days: 1 + - name: Login to Docker Hub (lmsys) uses: docker/login-action@v2 with: @@ -93,90 +109,136 @@ jobs: docker tag rocm/sgl-dev:${{ env.IMAGE_TAG }} lmsysorg/sglang-rocm:${{ env.IMAGE_TAG }} docker push lmsysorg/sglang-rocm:${{ env.IMAGE_TAG }} - # Temporarily disable docker cache seeding until performant storage is in place - cache: - if: false - # if: always() && github.repository == 'sgl-project/sglang' - runs-on: linux-mi300-gpu-1 + # Mirror the freshly published rocm/sgl-dev image to the in-network Docker + # registry so AMD CI runners can pull without hitting Docker Hub rate limits. + # The tag is read verbatim from the publish job's artifact so this job uses + # exactly the same tag that publish pushed (only the registry prefix differs). + # `!cancelled()` lets us still mirror successful matrix legs when other legs + # of publish failed; legs without an artifact will fail at download and be + # the only ones marked red. + push_local_registry: + if: ${{ !cancelled() && github.repository == 'sgl-project/sglang' }} + runs-on: linux-mi300-1gpu-sglang environment: 'prod' needs: publish strategy: fail-fast: false matrix: - gpu_arch: ['gfx942'] - build_type: ['all'] + gpu_arch: ['gfx942', 'gfx950'] steps: - - name: Checkout repository - uses: actions/checkout@v4 + - name: Download image tag artifact + uses: actions/download-artifact@v4 with: - fetch-depth: 0 # Required for git describe to find tags + name: image-tag-${{ matrix.gpu_arch }} - - name: "Set Date" + - name: Read image tag run: | - echo "DATE=$(date +%Y%m%d)" >> $GITHUB_ENV - - - name: Get version from latest tag - id: version - run: | - # Get the latest version tag sorted by version number (e.g., v0.5.7 -> 0.5.7) - VERSION=$(git tag -l 'v[0-9]*' --sort=-v:refname | head -1 | sed 's/^v//') - - if [ -z "$VERSION" ]; then - echo "::error::Could not determine version from git tags" + image_tag=$(tr -d '[:space:]' < "${{ matrix.gpu_arch }}.txt") + if [ -z "${image_tag}" ]; then + echo "::error::Image tag artifact is empty" exit 1 fi + echo "IMAGE_TAG=${image_tag}" >> $GITHUB_ENV + echo "Resolved IMAGE_TAG=${image_tag}" - echo "version=${VERSION}" >> $GITHUB_OUTPUT - echo "Detected version: ${VERSION}" - - - name: Login to Docker Hub + - name: Login to Docker Hub (AMD) uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_AMD_USERNAME }} password: ${{ secrets.DOCKERHUB_AMD_TOKEN }} - - name: Pull and Save Docker Image to Cache + - name: Mirror rocm/sgl-dev to local registry run: | - set -euxo pipefail + src="rocm/sgl-dev:${{ env.IMAGE_TAG }}" + dst="172.29.8.23:5000/rocm/sgl-dev:${{ env.IMAGE_TAG }}" + docker pull "${src}" + docker tag "${src}" "${dst}" + docker push "${dst}" - version=${{ steps.version.outputs.version }} - echo "Version: ${version}" + # Temporarily disable docker cache seeding until performant storage is in place + # cache: + # if: false + # # if: always() && github.repository == 'sgl-project/sglang' + # runs-on: linux-mi300-gpu-1 + # environment: 'prod' + # needs: publish + # strategy: + # fail-fast: false + # matrix: + # gpu_arch: ['gfx942'] + # build_type: ['all'] + # steps: + # - name: Checkout repository + # uses: actions/checkout@v4 + # with: + # fetch-depth: 0 # Required for git describe to find tags - if [ "${{ matrix.gpu_arch }}" = "gfx942" ]; then - rocm_tag="rocm700-mi30x" - else - echo "Unsupported gfx arch" - exit 1 - fi + # - name: "Set Date" + # run: | + # echo "DATE=$(date +%Y%m%d)" >> $GITHUB_ENV - tag=v${version}-${rocm_tag} + # - name: Get version from latest tag + # id: version + # run: | + # # Get the latest version tag sorted by version number (e.g., v0.5.7 -> 0.5.7) + # VERSION=$(git tag -l 'v[0-9]*' --sort=-v:refname | head -1 | sed 's/^v//') - if [ "${{ matrix.build_type }}" = "all" ]; then - tag_suffix="" - else - echo "Unsupported build type" - exit 1 - fi + # if [ -z "$VERSION" ]; then + # echo "::error::Could not determine version from git tags" + # exit 1 + # fi - image="rocm/sgl-dev:${tag}-${{ env.DATE }}${tag_suffix}" + # echo "version=${VERSION}" >> $GITHUB_OUTPUT + # echo "Detected version: ${VERSION}" - # Determine target cache file name based on ROCm variant - if [[ "${rocm_tag}" == rocm700* ]]; then - final_path="/home/runner/sgl-data/docker/image-700.tar" - else - echo "Unexpected ROCm tag: ${rocm_tag}" - exit 1 - fi + # - name: Login to Docker Hub + # uses: docker/login-action@v2 + # with: + # username: ${{ secrets.DOCKERHUB_AMD_USERNAME }} + # password: ${{ secrets.DOCKERHUB_AMD_TOKEN }} - tmp_path="${final_path}.tmp" + # - name: Pull and Save Docker Image to Cache + # run: | + # set -euxo pipefail - echo "Pulling image: ${image}" - docker pull "${image}" + # version=${{ steps.version.outputs.version }} + # echo "Version: ${version}" - echo "Saving to temp file: ${tmp_path}" - docker save "${image}" -o "${tmp_path}" + # if [ "${{ matrix.gpu_arch }}" = "gfx942" ]; then + # rocm_tag="rocm700-mi30x" + # else + # echo "Unsupported gfx arch" + # exit 1 + # fi - echo "Moving to final path: ${final_path}" - mv -f "${tmp_path}" "${final_path}" + # tag=v${version}-${rocm_tag} - echo "Cache populated successfully at ${final_path}" + # if [ "${{ matrix.build_type }}" = "all" ]; then + # tag_suffix="" + # else + # echo "Unsupported build type" + # exit 1 + # fi + + # image="rocm/sgl-dev:${tag}-${{ env.DATE }}${tag_suffix}" + + # # Determine target cache file name based on ROCm variant + # if [[ "${rocm_tag}" == rocm700* ]]; then + # final_path="/home/runner/sgl-data/docker/image-700.tar" + # else + # echo "Unexpected ROCm tag: ${rocm_tag}" + # exit 1 + # fi + + # tmp_path="${final_path}.tmp" + + # echo "Pulling image: ${image}" + # docker pull "${image}" + + # echo "Saving to temp file: ${tmp_path}" + # docker save "${image}" -o "${tmp_path}" + + # echo "Moving to final path: ${final_path}" + # mv -f "${tmp_path}" "${final_path}" + + # echo "Cache populated successfully at ${final_path}" diff --git a/.github/workflows/release-docker-amd-rocm720-nightly.yml b/.github/workflows/release-docker-amd-rocm720-nightly.yml index b30628673..8686550e1 100644 --- a/.github/workflows/release-docker-amd-rocm720-nightly.yml +++ b/.github/workflows/release-docker-amd-rocm720-nightly.yml @@ -82,6 +82,22 @@ jobs: docker build . -f docker/rocm.Dockerfile --build-arg SGL_BRANCH=${{ github.ref_name }} --build-arg BUILD_TYPE=${{ matrix.build_type }} --build-arg GPU_ARCH=${{ matrix.gpu_arch }} --build-arg ENABLE_MORI=1 --build-arg NIC_BACKEND=ainic --build-arg SETUPTOOLS_SCM_PRETEND_VERSION=${pretend_version} -t rocm/sgl-dev:${tag}-${{ env.DATE }} --no-cache docker push rocm/sgl-dev:${tag}-${{ env.DATE }} + # Persist the tag right after rocm/sgl-dev push succeeds so the local + # registry mirror can run even if a later step in this job (lmsys push) + # fails. By default this step only runs when the previous step succeeded, + # so the artifact only exists when an image actually landed on Docker Hub. + - name: Save published image tag + run: | + mkdir -p image-tag + echo "${{ env.IMAGE_TAG }}" > "image-tag/${{ matrix.gpu_arch }}.txt" + + - name: Upload image tag artifact + uses: actions/upload-artifact@v4 + with: + name: image-tag-${{ matrix.gpu_arch }} + path: image-tag/${{ matrix.gpu_arch }}.txt + retention-days: 1 + - name: Login to Docker Hub (lmsys) uses: docker/login-action@v2 with: @@ -92,3 +108,49 @@ jobs: run: | docker tag rocm/sgl-dev:${{ env.IMAGE_TAG }} lmsysorg/sglang-rocm:${{ env.IMAGE_TAG }} docker push lmsysorg/sglang-rocm:${{ env.IMAGE_TAG }} + + # Mirror the freshly published rocm/sgl-dev image to the in-network Docker + # registry so AMD CI runners can pull without hitting Docker Hub rate limits. + # The tag is read verbatim from the publish job's artifact so this job uses + # exactly the same tag that publish pushed (only the registry prefix differs). + # `!cancelled()` lets us still mirror successful matrix legs when other legs + # of publish failed; legs without an artifact will fail at download and be + # the only ones marked red. + push_local_registry: + if: ${{ !cancelled() && github.repository == 'sgl-project/sglang' }} + runs-on: linux-mi300-1gpu-sglang + environment: 'prod' + needs: publish + strategy: + fail-fast: false + matrix: + gpu_arch: ['gfx942-rocm720', 'gfx950-rocm720'] + steps: + - name: Download image tag artifact + uses: actions/download-artifact@v4 + with: + name: image-tag-${{ matrix.gpu_arch }} + + - name: Read image tag + run: | + image_tag=$(tr -d '[:space:]' < "${{ matrix.gpu_arch }}.txt") + if [ -z "${image_tag}" ]; then + echo "::error::Image tag artifact is empty" + exit 1 + fi + echo "IMAGE_TAG=${image_tag}" >> $GITHUB_ENV + echo "Resolved IMAGE_TAG=${image_tag}" + + - name: Login to Docker Hub (AMD) + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_AMD_USERNAME }} + password: ${{ secrets.DOCKERHUB_AMD_TOKEN }} + + - name: Mirror rocm/sgl-dev to local registry + run: | + src="rocm/sgl-dev:${{ env.IMAGE_TAG }}" + dst="172.29.8.23:5000/rocm/sgl-dev:${{ env.IMAGE_TAG }}" + docker pull "${src}" + docker tag "${src}" "${dst}" + docker push "${dst}" diff --git a/scripts/ci/amd/amd_ci_start_container.sh b/scripts/ci/amd/amd_ci_start_container.sh index 327da9348..9cc733884 100755 --- a/scripts/ci/amd/amd_ci_start_container.sh +++ b/scripts/ci/amd/amd_ci_start_container.sh @@ -23,6 +23,7 @@ fi ROCM_VERSION="rocm700" DEFAULT_MI30X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi30x" DEFAULT_MI35X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi35x" +LOCAL_DOCKER_REGISTRY="172.29.8.23:5000" # Parse command line arguments MI30X_BASE_TAG="${DEFAULT_MI30X_BASE_TAG}" @@ -96,11 +97,45 @@ else DEVICE_FLAG="--device /dev/dri" fi +# Retry a command with exponential backoff. Usage: retry_with_backoff +retry_with_backoff() { + local max_attempts=$1; shift + local attempt=1 + local wait_secs=30 + # Add jitter (0-30s) so concurrent jobs don't all retry at the same instant + local jitter=$(( RANDOM % 30 )) + while true; do + if "$@"; then + return 0 + fi + if (( attempt >= max_attempts )); then + echo "Error: '$*' failed after ${max_attempts} attempts" >&2 + return 1 + fi + local sleep_time=$(( wait_secs + jitter )) + echo "Attempt ${attempt}/${max_attempts} failed. Retrying in ${sleep_time}s…" >&2 + sleep "${sleep_time}" + (( attempt++ )) + (( wait_secs = wait_secs * 2 > 300 ? 300 : wait_secs * 2 )) + jitter=$(( RANDOM % 30 )) + done +} + +# Authenticate to Docker Hub to avoid anonymous pull rate limits. +# Credentials are optional; when absent we fall back to unauthenticated pulls. +if [[ -n "${DOCKERHUB_AMD_USERNAME:-}" && -n "${DOCKERHUB_AMD_TOKEN:-}" ]]; then + echo "Logging in to Docker Hub…" + if retry_with_backoff 6 sh -c 'echo "${DOCKERHUB_AMD_TOKEN}" | docker login -u "${DOCKERHUB_AMD_USERNAME}" --password-stdin >/dev/null 2>&1'; then + echo "Docker Hub login successful" + else + echo "Warning: Docker Hub login failed after retries; continuing with unauthenticated pulls" >&2 + fi +fi # Find the latest image find_latest_image() { local gpu_arch=$1 - local base_tag days_back image_tag + local base_tag days_back image_tag image_id remote_tags case "${gpu_arch}" in mi30x) base_tag="${MI30X_BASE_TAG}" ;; @@ -108,19 +143,29 @@ find_latest_image() { *) echo "Error: unsupported GPU architecture '${gpu_arch}'" >&2; return 1 ;; esac - # First, check local cache + # First, check local cache on the runner. for days_back in {0..6}; do image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)" - local local_image="rocm/sgl-dev:${image_tag}" - image_id=$(docker images -q "${local_image}") + image_id=$(docker images -q "rocm/sgl-dev:${image_tag}") if [[ -n "$image_id" ]]; then - echo "Found cached image locally: ${local_image}" >&2 - echo "${local_image}" - return 0 + echo "Found cached image locally: rocm/sgl-dev:${image_tag}" >&2 + echo "rocm/sgl-dev:${image_tag}" + return 0 fi done - # If not found locally, fall back to pulling from public registry + # Then try the local registry. + for days_back in {0..6}; do + image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)" + echo "Checking for image: ${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}" >&2 + if docker manifest inspect --insecure "${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}" >/dev/null 2>&1; then + echo "Found available image: ${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}" >&2 + echo "${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}" + return 0 + fi + done + + # Finally, try the public registry. for days_back in {0..6}; do image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)" echo "Checking for image: rocm/sgl-dev:${image_tag}" >&2 @@ -135,7 +180,7 @@ find_latest_image() { echo "Exact version not found. Searching remote registry for any ${ROCM_VERSION}-${gpu_arch} image…" >&2 for days_back in {0..6}; do local target_date=$(date -d "${days_back} days ago" +%Y%m%d) - local remote_tags=$(curl -s "https://registry.hub.docker.com/v2/repositories/rocm/sgl-dev/tags?page_size=100&name=${ROCM_VERSION}-${gpu_arch}-${target_date}" 2>/dev/null | grep -o '"name":"[^"]*"' | cut -d'"' -f4 | head -n 1) + remote_tags=$(curl -s "https://registry.hub.docker.com/v2/repositories/rocm/sgl-dev/tags?page_size=100&name=${ROCM_VERSION}-${gpu_arch}-${target_date}" 2>/dev/null | grep -o '"name":"[^"]*"' | cut -d'"' -f4 | head -n 1 || true) if [[ -n "$remote_tags" ]]; then echo "Found available image: rocm/sgl-dev:${remote_tags}" >&2 echo "rocm/sgl-dev:${remote_tags}" @@ -181,7 +226,11 @@ if [[ -n "${CUSTOM_IMAGE}" ]]; then # Use explicitly provided custom image IMAGE="${CUSTOM_IMAGE}" echo "Using custom image: ${IMAGE}" - docker pull "${IMAGE}" + if [[ "${IMAGE}" == "${LOCAL_DOCKER_REGISTRY}/"* ]]; then + docker pull "${IMAGE}" + else + retry_with_backoff 6 docker pull "${IMAGE}" + fi elif [[ -n "${BUILD_FROM_DOCKERFILE}" ]]; then # Build image from Dockerfile if [[ -z "${GPU_ARCH_BUILD}" ]]; then @@ -212,10 +261,19 @@ else # Find the latest pre-built image IMAGE=$(find_latest_image "${GPU_ARCH}") echo "Pulling Docker image: ${IMAGE}" - docker pull "${IMAGE}" + if [[ "${IMAGE}" == "${LOCAL_DOCKER_REGISTRY}/"* ]]; then + # Local registry is on-LAN; no need to retry. + docker pull "${IMAGE}" + docker tag "${IMAGE}" "${IMAGE#${LOCAL_DOCKER_REGISTRY}/}" + IMAGE="${IMAGE#${LOCAL_DOCKER_REGISTRY}/}" + else + # Public registry pulls can hit rate limits; retry with backoff. + retry_with_backoff 6 docker pull "${IMAGE}" + fi fi -CACHE_HOST=/home/runner/sgl-data +# CACHE_HOST=/home/runner/sgl-data +CACHE_HOST=/home/runner/sglang-data if [[ -d "$CACHE_HOST" ]]; then CACHE_VOLUME="-v $CACHE_HOST:/sgl-data" else diff --git a/scripts/ci/amd/amd_ci_start_container_disagg.sh b/scripts/ci/amd/amd_ci_start_container_disagg.sh index 6ae28f82b..852114482 100755 --- a/scripts/ci/amd/amd_ci_start_container_disagg.sh +++ b/scripts/ci/amd/amd_ci_start_container_disagg.sh @@ -23,6 +23,7 @@ fi ROCM_VERSION="rocm700" DEFAULT_MI30X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi30x" DEFAULT_MI35X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi35x" +LOCAL_DOCKER_REGISTRY="172.29.8.23:5000" # Parse command line arguments MI30X_BASE_TAG="${DEFAULT_MI30X_BASE_TAG}" @@ -83,11 +84,45 @@ else DEVICE_FLAG="--device /dev/dri" fi +# Retry a command with exponential backoff. Usage: retry_with_backoff +retry_with_backoff() { + local max_attempts=$1; shift + local attempt=1 + local wait_secs=30 + # Add jitter (0-30s) so concurrent jobs don't all retry at the same instant + local jitter=$(( RANDOM % 30 )) + while true; do + if "$@"; then + return 0 + fi + if (( attempt >= max_attempts )); then + echo "Error: '$*' failed after ${max_attempts} attempts" >&2 + return 1 + fi + local sleep_time=$(( wait_secs + jitter )) + echo "Attempt ${attempt}/${max_attempts} failed. Retrying in ${sleep_time}s…" >&2 + sleep "${sleep_time}" + (( attempt++ )) + (( wait_secs = wait_secs * 2 > 300 ? 300 : wait_secs * 2 )) + jitter=$(( RANDOM % 30 )) + done +} + +# Authenticate to Docker Hub to avoid anonymous pull rate limits. +# Credentials are optional; when absent we fall back to unauthenticated pulls. +if [[ -n "${DOCKERHUB_AMD_USERNAME:-}" && -n "${DOCKERHUB_AMD_TOKEN:-}" ]]; then + echo "Logging in to Docker Hub…" + if retry_with_backoff 6 sh -c 'echo "${DOCKERHUB_AMD_TOKEN}" | docker login -u "${DOCKERHUB_AMD_USERNAME}" --password-stdin >/dev/null 2>&1'; then + echo "Docker Hub login successful" + else + echo "Warning: Docker Hub login failed after retries; continuing with unauthenticated pulls" >&2 + fi +fi # Find the latest image find_latest_image() { local gpu_arch=$1 - local base_tag days_back image_tag + local base_tag days_back image_tag image_id remote_tags case "${gpu_arch}" in mi30x) base_tag="${MI30X_BASE_TAG}" ;; @@ -95,19 +130,29 @@ find_latest_image() { *) echo "Error: unsupported GPU architecture '${gpu_arch}'" >&2; return 1 ;; esac - # First, check local cache + # First, check local cache on the runner. for days_back in {0..6}; do image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)" - local local_image="rocm/sgl-dev:${image_tag}" - image_id=$(docker images -q "${local_image}") + image_id=$(docker images -q "rocm/sgl-dev:${image_tag}") if [[ -n "$image_id" ]]; then - echo "Found cached image locally: ${local_image}" >&2 - echo "${local_image}" - return 0 + echo "Found cached image locally: rocm/sgl-dev:${image_tag}" >&2 + echo "rocm/sgl-dev:${image_tag}" + return 0 fi done - # If not found locally, fall back to pulling from public registry + # Then try the local registry. + for days_back in {0..6}; do + image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)" + echo "Checking for image: ${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}" >&2 + if docker manifest inspect --insecure "${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}" >/dev/null 2>&1; then + echo "Found available image: ${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}" >&2 + echo "${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}" + return 0 + fi + done + + # Finally, try the public registry. for days_back in {0..6}; do image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)" echo "Checking for image: rocm/sgl-dev:${image_tag}" >&2 @@ -122,7 +167,7 @@ find_latest_image() { echo "Exact version not found. Searching remote registry for any ${ROCM_VERSION}-${gpu_arch} image…" >&2 for days_back in {0..6}; do local target_date=$(date -d "${days_back} days ago" +%Y%m%d) - local remote_tags=$(curl -s "https://registry.hub.docker.com/v2/repositories/rocm/sgl-dev/tags?page_size=100&name=${ROCM_VERSION}-${gpu_arch}-${target_date}" 2>/dev/null | grep -o '"name":"[^"]*"' | cut -d'"' -f4 | head -n 1) + remote_tags=$(curl -s "https://registry.hub.docker.com/v2/repositories/rocm/sgl-dev/tags?page_size=100&name=${ROCM_VERSION}-${gpu_arch}-${target_date}" 2>/dev/null | grep -o '"name":"[^"]*"' | cut -d'"' -f4 | head -n 1 || true) if [[ -n "$remote_tags" ]]; then echo "Found available image: rocm/sgl-dev:${remote_tags}" >&2 echo "rocm/sgl-dev:${remote_tags}" @@ -166,9 +211,18 @@ find_latest_image() { # Pull and run the latest image IMAGE=$(find_latest_image "${GPU_ARCH}") echo "Pulling Docker image: ${IMAGE}" -docker pull "${IMAGE}" +if [[ "${IMAGE}" == "${LOCAL_DOCKER_REGISTRY}/"* ]]; then + # Local registry is on-LAN; no need to retry. + docker pull "${IMAGE}" + docker tag "${IMAGE}" "${IMAGE#${LOCAL_DOCKER_REGISTRY}/}" + IMAGE="${IMAGE#${LOCAL_DOCKER_REGISTRY}/}" +else + # Public registry pulls can hit rate limits; retry with backoff. + retry_with_backoff 6 docker pull "${IMAGE}" +fi -CACHE_HOST=/home/runner/sgl-data +# CACHE_HOST=/home/runner/sgl-data +CACHE_HOST=/home/runner/sglang-data if [[ -d "$CACHE_HOST" ]]; then CACHE_VOLUME="-v $CACHE_HOST:/sgl-data" else