[AMD] mirror nightly images to local registry and prefer LAN pulls (#23073)

Co-authored-by: bingxche <bingxche@amd.com>

.github/workflows/release-docker-amd-nightly.yml

@@ -82,6 +82,22 @@ jobs:
           docker build . -f docker/rocm.Dockerfile --build-arg SGL_BRANCH=${{ github.ref_name }} --build-arg BUILD_TYPE=${{ matrix.build_type }} --build-arg GPU_ARCH=${{ matrix.gpu_arch }} --build-arg ENABLE_MORI=1 --build-arg NIC_BACKEND=ainic --build-arg SETUPTOOLS_SCM_PRETEND_VERSION=${pretend_version} -t rocm/sgl-dev:${tag}-${{ env.DATE }} --no-cache
           docker push rocm/sgl-dev:${tag}-${{ env.DATE }}
 
+      # Persist the tag right after rocm/sgl-dev push succeeds so the local
+      # registry mirror can run even if a later step in this job (lmsys push)
+      # fails. By default this step only runs when the previous step succeeded,
+      # so the artifact only exists when an image actually landed on Docker Hub.
+      - name: Save published image tag
+        run: |
+          mkdir -p image-tag
+          echo "${{ env.IMAGE_TAG }}" > "image-tag/${{ matrix.gpu_arch }}.txt"
+
+      - name: Upload image tag artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: image-tag-${{ matrix.gpu_arch }}
+          path: image-tag/${{ matrix.gpu_arch }}.txt
+          retention-days: 1
+
       - name: Login to Docker Hub (lmsys)
         uses: docker/login-action@v2
         with:
@@ -93,90 +109,136 @@ jobs:
           docker tag rocm/sgl-dev:${{ env.IMAGE_TAG }} lmsysorg/sglang-rocm:${{ env.IMAGE_TAG }}
           docker push lmsysorg/sglang-rocm:${{ env.IMAGE_TAG }}
 
-  # Temporarily disable docker cache seeding until performant storage is in place
+  # Mirror the freshly published rocm/sgl-dev image to the in-network Docker
-  cache:
+  # registry so AMD CI runners can pull without hitting Docker Hub rate limits.
-    if: false
+  # The tag is read verbatim from the publish job's artifact so this job uses
-    # if: always() && github.repository == 'sgl-project/sglang'
+  # exactly the same tag that publish pushed (only the registry prefix differs).
-    runs-on: linux-mi300-gpu-1
+  # `!cancelled()` lets us still mirror successful matrix legs when other legs
+  # of publish failed; legs without an artifact will fail at download and be
+  # the only ones marked red.
+  push_local_registry:
+    if: ${{ !cancelled() && github.repository == 'sgl-project/sglang' }}
+    runs-on: linux-mi300-1gpu-sglang
     environment: 'prod'
     needs: publish
     strategy:
       fail-fast: false
       matrix:
-        gpu_arch: ['gfx942']
+        gpu_arch: ['gfx942', 'gfx950']
-        build_type: ['all']
     steps:
-      - name: Checkout repository
+      - name: Download image tag artifact
-        uses: actions/checkout@v4
+        uses: actions/download-artifact@v4
         with:
-          fetch-depth: 0  # Required for git describe to find tags
+          name: image-tag-${{ matrix.gpu_arch }}
 
-      - name: "Set Date"
+      - name: Read image tag
         run: |
-          echo "DATE=$(date +%Y%m%d)" >> $GITHUB_ENV
+          image_tag=$(tr -d '[:space:]' < "${{ matrix.gpu_arch }}.txt")
-
+          if [ -z "${image_tag}" ]; then
-      - name: Get version from latest tag
+            echo "::error::Image tag artifact is empty"
-        id: version
-        run: |
-          # Get the latest version tag sorted by version number (e.g., v0.5.7 -> 0.5.7)
-          VERSION=$(git tag -l 'v[0-9]*' --sort=-v:refname | head -1 | sed 's/^v//')
-
-          if [ -z "$VERSION" ]; then
-            echo "::error::Could not determine version from git tags"
             exit 1
           fi
+          echo "IMAGE_TAG=${image_tag}" >> $GITHUB_ENV
+          echo "Resolved IMAGE_TAG=${image_tag}"
 
-          echo "version=${VERSION}" >> $GITHUB_OUTPUT
+      - name: Login to Docker Hub (AMD)
-          echo "Detected version: ${VERSION}"
-
-      - name: Login to Docker Hub
         uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_AMD_USERNAME }}
           password: ${{ secrets.DOCKERHUB_AMD_TOKEN }}
 
-      - name: Pull and Save Docker Image to Cache
+      - name: Mirror rocm/sgl-dev to local registry
         run: |
-          set -euxo pipefail
+          src="rocm/sgl-dev:${{ env.IMAGE_TAG }}"
+          dst="172.29.8.23:5000/rocm/sgl-dev:${{ env.IMAGE_TAG }}"
+          docker pull "${src}"
+          docker tag "${src}" "${dst}"
+          docker push "${dst}"
 
-          version=${{ steps.version.outputs.version }}
+  # Temporarily disable docker cache seeding until performant storage is in place
-          echo "Version: ${version}"
+  # cache:
+  #   if: false
+  #   # if: always() && github.repository == 'sgl-project/sglang'
+  #   runs-on: linux-mi300-gpu-1
+  #   environment: 'prod'
+  #   needs: publish
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       gpu_arch: ['gfx942']
+  #       build_type: ['all']
+  #   steps:
+  #     - name: Checkout repository
+  #       uses: actions/checkout@v4
+  #       with:
+  #         fetch-depth: 0  # Required for git describe to find tags
 
-          if [ "${{ matrix.gpu_arch }}" = "gfx942" ]; then
+  #     - name: "Set Date"
-            rocm_tag="rocm700-mi30x"
+  #       run: |
-          else
+  #         echo "DATE=$(date +%Y%m%d)" >> $GITHUB_ENV
-            echo "Unsupported gfx arch"
-            exit 1
-          fi
 
-          tag=v${version}-${rocm_tag}
+  #     - name: Get version from latest tag
+  #       id: version
+  #       run: |
+  #         # Get the latest version tag sorted by version number (e.g., v0.5.7 -> 0.5.7)
+  #         VERSION=$(git tag -l 'v[0-9]*' --sort=-v:refname | head -1 | sed 's/^v//')
 
-          if [ "${{ matrix.build_type }}" = "all" ]; then
+  #         if [ -z "$VERSION" ]; then
-            tag_suffix=""
+  #           echo "::error::Could not determine version from git tags"
-          else
+  #           exit 1
-            echo "Unsupported build type"
+  #         fi
-            exit 1
-          fi
 
-          image="rocm/sgl-dev:${tag}-${{ env.DATE }}${tag_suffix}"
+  #         echo "version=${VERSION}" >> $GITHUB_OUTPUT
+  #         echo "Detected version: ${VERSION}"
 
-          # Determine target cache file name based on ROCm variant
+  #     - name: Login to Docker Hub
-          if [[ "${rocm_tag}" == rocm700* ]]; then
+  #       uses: docker/login-action@v2
-            final_path="/home/runner/sgl-data/docker/image-700.tar"
+  #       with:
-          else
+  #         username: ${{ secrets.DOCKERHUB_AMD_USERNAME }}
-            echo "Unexpected ROCm tag: ${rocm_tag}"
+  #         password: ${{ secrets.DOCKERHUB_AMD_TOKEN }}
-            exit 1
-          fi
 
-          tmp_path="${final_path}.tmp"
+  #     - name: Pull and Save Docker Image to Cache
+  #       run: |
+  #         set -euxo pipefail
 
-          echo "Pulling image: ${image}"
+  #         version=${{ steps.version.outputs.version }}
-          docker pull "${image}"
+  #         echo "Version: ${version}"
 
-          echo "Saving to temp file: ${tmp_path}"
+  #         if [ "${{ matrix.gpu_arch }}" = "gfx942" ]; then
-          docker save "${image}" -o "${tmp_path}"
+  #           rocm_tag="rocm700-mi30x"
+  #         else
+  #           echo "Unsupported gfx arch"
+  #           exit 1
+  #         fi
 
-          echo "Moving to final path: ${final_path}"
+  #         tag=v${version}-${rocm_tag}
-          mv -f "${tmp_path}" "${final_path}"
 
-          echo "Cache populated successfully at ${final_path}"
+  #         if [ "${{ matrix.build_type }}" = "all" ]; then
+  #           tag_suffix=""
+  #         else
+  #           echo "Unsupported build type"
+  #           exit 1
+  #         fi
+
+  #         image="rocm/sgl-dev:${tag}-${{ env.DATE }}${tag_suffix}"
+
+  #         # Determine target cache file name based on ROCm variant
+  #         if [[ "${rocm_tag}" == rocm700* ]]; then
+  #           final_path="/home/runner/sgl-data/docker/image-700.tar"
+  #         else
+  #           echo "Unexpected ROCm tag: ${rocm_tag}"
+  #           exit 1
+  #         fi
+
+  #         tmp_path="${final_path}.tmp"
+
+  #         echo "Pulling image: ${image}"
+  #         docker pull "${image}"
+
+  #         echo "Saving to temp file: ${tmp_path}"
+  #         docker save "${image}" -o "${tmp_path}"
+
+  #         echo "Moving to final path: ${final_path}"
+  #         mv -f "${tmp_path}" "${final_path}"
+
+  #         echo "Cache populated successfully at ${final_path}"

.github/workflows/release-docker-amd-rocm720-nightly.yml

@@ -82,6 +82,22 @@ jobs:
           docker build . -f docker/rocm.Dockerfile --build-arg SGL_BRANCH=${{ github.ref_name }} --build-arg BUILD_TYPE=${{ matrix.build_type }} --build-arg GPU_ARCH=${{ matrix.gpu_arch }} --build-arg ENABLE_MORI=1 --build-arg NIC_BACKEND=ainic --build-arg SETUPTOOLS_SCM_PRETEND_VERSION=${pretend_version} -t rocm/sgl-dev:${tag}-${{ env.DATE }} --no-cache
           docker push rocm/sgl-dev:${tag}-${{ env.DATE }}
 
+      # Persist the tag right after rocm/sgl-dev push succeeds so the local
+      # registry mirror can run even if a later step in this job (lmsys push)
+      # fails. By default this step only runs when the previous step succeeded,
+      # so the artifact only exists when an image actually landed on Docker Hub.
+      - name: Save published image tag
+        run: |
+          mkdir -p image-tag
+          echo "${{ env.IMAGE_TAG }}" > "image-tag/${{ matrix.gpu_arch }}.txt"
+
+      - name: Upload image tag artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: image-tag-${{ matrix.gpu_arch }}
+          path: image-tag/${{ matrix.gpu_arch }}.txt
+          retention-days: 1
+
       - name: Login to Docker Hub (lmsys)
         uses: docker/login-action@v2
         with:
@@ -92,3 +108,49 @@ jobs:
         run: |
           docker tag rocm/sgl-dev:${{ env.IMAGE_TAG }} lmsysorg/sglang-rocm:${{ env.IMAGE_TAG }}
           docker push lmsysorg/sglang-rocm:${{ env.IMAGE_TAG }}
+
+  # Mirror the freshly published rocm/sgl-dev image to the in-network Docker
+  # registry so AMD CI runners can pull without hitting Docker Hub rate limits.
+  # The tag is read verbatim from the publish job's artifact so this job uses
+  # exactly the same tag that publish pushed (only the registry prefix differs).
+  # `!cancelled()` lets us still mirror successful matrix legs when other legs
+  # of publish failed; legs without an artifact will fail at download and be
+  # the only ones marked red.
+  push_local_registry:
+    if: ${{ !cancelled() && github.repository == 'sgl-project/sglang' }}
+    runs-on: linux-mi300-1gpu-sglang
+    environment: 'prod'
+    needs: publish
+    strategy:
+      fail-fast: false
+      matrix:
+        gpu_arch: ['gfx942-rocm720', 'gfx950-rocm720']
+    steps:
+      - name: Download image tag artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: image-tag-${{ matrix.gpu_arch }}
+
+      - name: Read image tag
+        run: |
+          image_tag=$(tr -d '[:space:]' < "${{ matrix.gpu_arch }}.txt")
+          if [ -z "${image_tag}" ]; then
+            echo "::error::Image tag artifact is empty"
+            exit 1
+          fi
+          echo "IMAGE_TAG=${image_tag}" >> $GITHUB_ENV
+          echo "Resolved IMAGE_TAG=${image_tag}"
+
+      - name: Login to Docker Hub (AMD)
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_AMD_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_AMD_TOKEN }}
+
+      - name: Mirror rocm/sgl-dev to local registry
+        run: |
+          src="rocm/sgl-dev:${{ env.IMAGE_TAG }}"
+          dst="172.29.8.23:5000/rocm/sgl-dev:${{ env.IMAGE_TAG }}"
+          docker pull "${src}"
+          docker tag "${src}" "${dst}"
+          docker push "${dst}"

scripts/ci/amd/amd_ci_start_container.sh

@@ -23,6 +23,7 @@ fi
 ROCM_VERSION="rocm700"
 DEFAULT_MI30X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi30x"
 DEFAULT_MI35X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi35x"
+LOCAL_DOCKER_REGISTRY="172.29.8.23:5000"
 
 # Parse command line arguments
 MI30X_BASE_TAG="${DEFAULT_MI30X_BASE_TAG}"
@@ -96,11 +97,45 @@ else
   DEVICE_FLAG="--device /dev/dri"
 fi
 
+# Retry a command with exponential backoff. Usage: retry_with_backoff <max_attempts> <cmd...>
+retry_with_backoff() {
+  local max_attempts=$1; shift
+  local attempt=1
+  local wait_secs=30
+  # Add jitter (0-30s) so concurrent jobs don't all retry at the same instant
+  local jitter=$(( RANDOM % 30 ))
+  while true; do
+    if "$@"; then
+      return 0
+    fi
+    if (( attempt >= max_attempts )); then
+      echo "Error: '$*' failed after ${max_attempts} attempts" >&2
+      return 1
+    fi
+    local sleep_time=$(( wait_secs + jitter ))
+    echo "Attempt ${attempt}/${max_attempts} failed. Retrying in ${sleep_time}s…" >&2
+    sleep "${sleep_time}"
+    (( attempt++ ))
+    (( wait_secs = wait_secs * 2 > 300 ? 300 : wait_secs * 2 ))
+    jitter=$(( RANDOM % 30 ))
+  done
+}
+
+# Authenticate to Docker Hub to avoid anonymous pull rate limits.
+# Credentials are optional; when absent we fall back to unauthenticated pulls.
+if [[ -n "${DOCKERHUB_AMD_USERNAME:-}" && -n "${DOCKERHUB_AMD_TOKEN:-}" ]]; then
+  echo "Logging in to Docker Hub…"
+  if retry_with_backoff 6 sh -c 'echo "${DOCKERHUB_AMD_TOKEN}" | docker login -u "${DOCKERHUB_AMD_USERNAME}" --password-stdin >/dev/null 2>&1'; then
+    echo "Docker Hub login successful"
+  else
+    echo "Warning: Docker Hub login failed after retries; continuing with unauthenticated pulls" >&2
+  fi
+fi
 
 # Find the latest image
 find_latest_image() {
   local gpu_arch=$1
-  local base_tag days_back image_tag
+  local base_tag days_back image_tag image_id remote_tags
 
   case "${gpu_arch}" in
       mi30x) base_tag="${MI30X_BASE_TAG}" ;;
@@ -108,19 +143,29 @@ find_latest_image() {
       *)     echo "Error: unsupported GPU architecture '${gpu_arch}'" >&2; return 1 ;;
   esac
 
-  # First, check local cache
+  # First, check local cache on the runner.
   for days_back in {0..6}; do
     image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)"
-    local local_image="rocm/sgl-dev:${image_tag}"
+    image_id=$(docker images -q "rocm/sgl-dev:${image_tag}")
-    image_id=$(docker images -q "${local_image}")
     if [[ -n "$image_id" ]]; then
-        echo "Found cached image locally: ${local_image}" >&2
+      echo "Found cached image locally: rocm/sgl-dev:${image_tag}" >&2
-        echo "${local_image}"
+      echo "rocm/sgl-dev:${image_tag}"
-        return 0
+      return 0
     fi
   done
 
-  # If not found locally, fall back to pulling from public registry
+  # Then try the local registry.
+  for days_back in {0..6}; do
+    image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)"
+    echo "Checking for image: ${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}" >&2
+    if docker manifest inspect --insecure "${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}" >/dev/null 2>&1; then
+      echo "Found available image: ${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}" >&2
+      echo "${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}"
+      return 0
+    fi
+  done
+
+  # Finally, try the public registry.
   for days_back in {0..6}; do
     image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)"
     echo "Checking for image: rocm/sgl-dev:${image_tag}" >&2
@@ -135,7 +180,7 @@ find_latest_image() {
   echo "Exact version not found. Searching remote registry for any ${ROCM_VERSION}-${gpu_arch} image…" >&2
   for days_back in {0..6}; do
     local target_date=$(date -d "${days_back} days ago" +%Y%m%d)
-    local remote_tags=$(curl -s "https://registry.hub.docker.com/v2/repositories/rocm/sgl-dev/tags?page_size=100&name=${ROCM_VERSION}-${gpu_arch}-${target_date}" 2>/dev/null | grep -o '"name":"[^"]*"' | cut -d'"' -f4 | head -n 1)
+    remote_tags=$(curl -s "https://registry.hub.docker.com/v2/repositories/rocm/sgl-dev/tags?page_size=100&name=${ROCM_VERSION}-${gpu_arch}-${target_date}" 2>/dev/null | grep -o '"name":"[^"]*"' | cut -d'"' -f4 | head -n 1 || true)
     if [[ -n "$remote_tags" ]]; then
       echo "Found available image: rocm/sgl-dev:${remote_tags}" >&2
       echo "rocm/sgl-dev:${remote_tags}"
@@ -181,7 +226,11 @@ if [[ -n "${CUSTOM_IMAGE}" ]]; then
   # Use explicitly provided custom image
   IMAGE="${CUSTOM_IMAGE}"
   echo "Using custom image: ${IMAGE}"
-  docker pull "${IMAGE}"
+  if [[ "${IMAGE}" == "${LOCAL_DOCKER_REGISTRY}/"* ]]; then
+    docker pull "${IMAGE}"
+  else
+    retry_with_backoff 6 docker pull "${IMAGE}"
+  fi
 elif [[ -n "${BUILD_FROM_DOCKERFILE}" ]]; then
   # Build image from Dockerfile
   if [[ -z "${GPU_ARCH_BUILD}" ]]; then
@@ -212,10 +261,19 @@ else
   # Find the latest pre-built image
   IMAGE=$(find_latest_image "${GPU_ARCH}")
   echo "Pulling Docker image: ${IMAGE}"
-  docker pull "${IMAGE}"
+  if [[ "${IMAGE}" == "${LOCAL_DOCKER_REGISTRY}/"* ]]; then
+    # Local registry is on-LAN; no need to retry.
+    docker pull "${IMAGE}"
+    docker tag "${IMAGE}" "${IMAGE#${LOCAL_DOCKER_REGISTRY}/}"
+    IMAGE="${IMAGE#${LOCAL_DOCKER_REGISTRY}/}"
+  else
+    # Public registry pulls can hit rate limits; retry with backoff.
+    retry_with_backoff 6 docker pull "${IMAGE}"
+  fi
 fi
 
-CACHE_HOST=/home/runner/sgl-data
+# CACHE_HOST=/home/runner/sgl-data
+CACHE_HOST=/home/runner/sglang-data
 if [[ -d "$CACHE_HOST" ]]; then
     CACHE_VOLUME="-v $CACHE_HOST:/sgl-data"
 else

scripts/ci/amd/amd_ci_start_container_disagg.sh

@@ -23,6 +23,7 @@ fi
 ROCM_VERSION="rocm700"
 DEFAULT_MI30X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi30x"
 DEFAULT_MI35X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi35x"
+LOCAL_DOCKER_REGISTRY="172.29.8.23:5000"
 
 # Parse command line arguments
 MI30X_BASE_TAG="${DEFAULT_MI30X_BASE_TAG}"
@@ -83,11 +84,45 @@ else
   DEVICE_FLAG="--device /dev/dri"
 fi
 
+# Retry a command with exponential backoff. Usage: retry_with_backoff <max_attempts> <cmd...>
+retry_with_backoff() {
+  local max_attempts=$1; shift
+  local attempt=1
+  local wait_secs=30
+  # Add jitter (0-30s) so concurrent jobs don't all retry at the same instant
+  local jitter=$(( RANDOM % 30 ))
+  while true; do
+    if "$@"; then
+      return 0
+    fi
+    if (( attempt >= max_attempts )); then
+      echo "Error: '$*' failed after ${max_attempts} attempts" >&2
+      return 1
+    fi
+    local sleep_time=$(( wait_secs + jitter ))
+    echo "Attempt ${attempt}/${max_attempts} failed. Retrying in ${sleep_time}s…" >&2
+    sleep "${sleep_time}"
+    (( attempt++ ))
+    (( wait_secs = wait_secs * 2 > 300 ? 300 : wait_secs * 2 ))
+    jitter=$(( RANDOM % 30 ))
+  done
+}
+
+# Authenticate to Docker Hub to avoid anonymous pull rate limits.
+# Credentials are optional; when absent we fall back to unauthenticated pulls.
+if [[ -n "${DOCKERHUB_AMD_USERNAME:-}" && -n "${DOCKERHUB_AMD_TOKEN:-}" ]]; then
+  echo "Logging in to Docker Hub…"
+  if retry_with_backoff 6 sh -c 'echo "${DOCKERHUB_AMD_TOKEN}" | docker login -u "${DOCKERHUB_AMD_USERNAME}" --password-stdin >/dev/null 2>&1'; then
+    echo "Docker Hub login successful"
+  else
+    echo "Warning: Docker Hub login failed after retries; continuing with unauthenticated pulls" >&2
+  fi
+fi
 
 # Find the latest image
 find_latest_image() {
   local gpu_arch=$1
-  local base_tag days_back image_tag
+  local base_tag days_back image_tag image_id remote_tags
 
   case "${gpu_arch}" in
       mi30x) base_tag="${MI30X_BASE_TAG}" ;;
@@ -95,19 +130,29 @@ find_latest_image() {
       *)     echo "Error: unsupported GPU architecture '${gpu_arch}'" >&2; return 1 ;;
   esac
 
-  # First, check local cache
+  # First, check local cache on the runner.
   for days_back in {0..6}; do
     image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)"
-    local local_image="rocm/sgl-dev:${image_tag}"
+    image_id=$(docker images -q "rocm/sgl-dev:${image_tag}")
-    image_id=$(docker images -q "${local_image}")
     if [[ -n "$image_id" ]]; then
-        echo "Found cached image locally: ${local_image}" >&2
+      echo "Found cached image locally: rocm/sgl-dev:${image_tag}" >&2
-        echo "${local_image}"
+      echo "rocm/sgl-dev:${image_tag}"
-        return 0
+      return 0
     fi
   done
 
-  # If not found locally, fall back to pulling from public registry
+  # Then try the local registry.
+  for days_back in {0..6}; do
+    image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)"
+    echo "Checking for image: ${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}" >&2
+    if docker manifest inspect --insecure "${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}" >/dev/null 2>&1; then
+      echo "Found available image: ${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}" >&2
+      echo "${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}"
+      return 0
+    fi
+  done
+
+  # Finally, try the public registry.
   for days_back in {0..6}; do
     image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)"
     echo "Checking for image: rocm/sgl-dev:${image_tag}" >&2
@@ -122,7 +167,7 @@ find_latest_image() {
   echo "Exact version not found. Searching remote registry for any ${ROCM_VERSION}-${gpu_arch} image…" >&2
   for days_back in {0..6}; do
     local target_date=$(date -d "${days_back} days ago" +%Y%m%d)
-    local remote_tags=$(curl -s "https://registry.hub.docker.com/v2/repositories/rocm/sgl-dev/tags?page_size=100&name=${ROCM_VERSION}-${gpu_arch}-${target_date}" 2>/dev/null | grep -o '"name":"[^"]*"' | cut -d'"' -f4 | head -n 1)
+    remote_tags=$(curl -s "https://registry.hub.docker.com/v2/repositories/rocm/sgl-dev/tags?page_size=100&name=${ROCM_VERSION}-${gpu_arch}-${target_date}" 2>/dev/null | grep -o '"name":"[^"]*"' | cut -d'"' -f4 | head -n 1 || true)
     if [[ -n "$remote_tags" ]]; then
       echo "Found available image: rocm/sgl-dev:${remote_tags}" >&2
       echo "rocm/sgl-dev:${remote_tags}"
@@ -166,9 +211,18 @@ find_latest_image() {
 # Pull and run the latest image
 IMAGE=$(find_latest_image "${GPU_ARCH}")
 echo "Pulling Docker image: ${IMAGE}"
-docker pull "${IMAGE}"
+if [[ "${IMAGE}" == "${LOCAL_DOCKER_REGISTRY}/"* ]]; then
+  # Local registry is on-LAN; no need to retry.
+  docker pull "${IMAGE}"
+  docker tag "${IMAGE}" "${IMAGE#${LOCAL_DOCKER_REGISTRY}/}"
+  IMAGE="${IMAGE#${LOCAL_DOCKER_REGISTRY}/}"
+else
+  # Public registry pulls can hit rate limits; retry with backoff.
+  retry_with_backoff 6 docker pull "${IMAGE}"
+fi
 
-CACHE_HOST=/home/runner/sgl-data
+# CACHE_HOST=/home/runner/sgl-data
+CACHE_HOST=/home/runner/sglang-data
 if [[ -d "$CACHE_HOST" ]]; then
     CACHE_VOLUME="-v $CACHE_HOST:/sgl-data"
 else